jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Breaking changes (#5191)

Browse Source 
Co-authored-by: Bjerg <onbjerg@users.noreply.github.com>
Co-authored-by: Roman Krasiuk <rokrassyuk@gmail.com>
Co-authored-by: joshieDo <ranriver@protonmail.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Oliver Nordbjerg <hi@notbjerg.me>
Co-authored-by: Thomas Coratger <thomas.coratger@gmail.com>
This commit is contained in:
Alexey Shekhirin
2024-02-29 12:37:28 +00:00
committed by GitHub
parent 025fa5f038
commit 6b5b6f7a40
252 changed files with 10154 additions and 6327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/storage/codecs/derive/src/compact/generator.rs
View File

@ -52,7 +52,7 @@ pub fn generate_from_to(ident: &Ident, fields: &FieldList, is_zstd: bool) -> Tok
/// Generates code to implement the `Compact` trait method `to_compact`.
fn generate_from_compact(fields: &FieldList, ident: &Ident, is_zstd: bool) -> TokenStream2 {
let mut lines = vec![];
let mut known_types = vec!["B256", "Address", "Bloom", "Vec", "TxHash"];
let mut known_types = vec!["B256", "Address", "Bloom", "Vec", "TxHash", "BlockHash"];
// Only types without `Bytes` should be added here. It's currently manually added, since
// it's hard to figure out with derive_macro which types have Bytes fields.

7
crates/storage/codecs/derive/src/compact/mod.rs
View File

@ -143,7 +143,7 @@ fn should_use_alt_impl(ftype: &String, segment: &syn::PathSegment) -> bool {
if let (Some(path), 1) =
(arg_path.path.segments.first(), arg_path.path.segments.len())
{
if ["B256", "Address", "Address", "Bloom", "TxHash"]
if ["B256", "Address", "Address", "Bloom", "TxHash", "BlockHash"]
.contains(&path.ident.to_string().as_str())
{
return true
@ -164,11 +164,6 @@ pub fn get_bit_size(ftype: &str) -> u8 {
"u64" | "BlockNumber" | "TxNumber" | "ChainId" | "NumTransactions" => 4,
"u128" => 5,
"U256" => 6,
#[cfg(not(feature = "optimism"))]
"TxValue" => 5, // u128 for ethereum chains assuming high order bits are not used
#[cfg(feature = "optimism")]
// for fuzz/prop testing and chains that may require full 256 bits
"TxValue" => 6,
_ => 0,
}
}

8
crates/storage/db/benches/criterion.rs
View File

@ -23,12 +23,12 @@ pub fn db(c: &mut Criterion) {
group.warm_up_time(std::time::Duration::from_millis(200));
measure_table_db::<CanonicalHeaders>(&mut group);
measure_table_db::<HeaderTD>(&mut group);
measure_table_db::<HeaderTerminalDifficulties>(&mut group);
measure_table_db::<HeaderNumbers>(&mut group);
measure_table_db::<Headers>(&mut group);
measure_table_db::<BlockBodyIndices>(&mut group);
measure_table_db::<BlockOmmers>(&mut group);
measure_table_db::<TxHashNumber>(&mut group);
measure_table_db::<TransactionHashNumbers>(&mut group);
measure_table_db::<Transactions>(&mut group);
measure_dupsort_db::<PlainStorageState>(&mut group);
measure_table_db::<PlainAccountState>(&mut group);
@ -40,12 +40,12 @@ pub fn serialization(c: &mut Criterion) {
group.warm_up_time(std::time::Duration::from_millis(200));
measure_table_serialization::<CanonicalHeaders>(&mut group);
measure_table_serialization::<HeaderTD>(&mut group);
measure_table_serialization::<HeaderTerminalDifficulties>(&mut group);
measure_table_serialization::<HeaderNumbers>(&mut group);
measure_table_serialization::<Headers>(&mut group);
measure_table_serialization::<BlockBodyIndices>(&mut group);
measure_table_serialization::<BlockOmmers>(&mut group);
measure_table_serialization::<TxHashNumber>(&mut group);
measure_table_serialization::<TransactionHashNumbers>(&mut group);
measure_table_serialization::<Transactions>(&mut group);
measure_table_serialization::<PlainStorageState>(&mut group);
measure_table_serialization::<PlainAccountState>(&mut group);

4
crates/storage/db/benches/hash_keys.rs
View File

@ -9,7 +9,7 @@ use proptest::{
strategy::{Strategy, ValueTree},
test_runner::TestRunner,
};
use reth_db::{cursor::DbCursorRW, TxHashNumber};
use reth_db::{cursor::DbCursorRW, TransactionHashNumbers};
use std::collections::HashSet;
criterion_group! {
@ -34,7 +34,7 @@ pub fn hash_keys(c: &mut Criterion) {
group.sample_size(10);
for size in [10_000, 100_000, 1_000_000] {
measure_table_insertion::<TxHashNumber>(&mut group, size);
measure_table_insertion::<TransactionHashNumbers>(&mut group, size);
}
}

4
crates/storage/db/benches/iai.rs
View File

@ -79,12 +79,12 @@ macro_rules! impl_iai {
impl_iai!(
CanonicalHeaders,
HeaderTD,
HeaderTerminalDifficulties,
HeaderNumbers,
Headers,
BlockBodyIndices,
BlockOmmers,
TxHashNumber,
TransactionHashNumbers,
Transactions,
PlainStorageState,
PlainAccountState

42
crates/storage/db/src/implementation/mdbx/mod.rs
View File

@ -232,7 +232,10 @@ impl DatabaseEnv {
}
};
inner_env.set_max_dbs(Tables::ALL.len());
// Note: We set max dbs to 256 here to allow for custom tables. This needs to be set on
// environment creation.
debug_assert!(Tables::ALL.len() <= 256, "number of tables exceed max dbs");
inner_env.set_max_dbs(256);
inner_env.set_geometry(Geometry {
// Maximum database size of 4 terabytes
size: Some(0..(4 * TERABYTE)),
@ -385,10 +388,12 @@ mod tests {
abstraction::table::{Encode, Table},
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, ReverseWalker, Walker},
models::{AccountBeforeTx, ShardedKey},
tables::{AccountHistory, CanonicalHeaders, Headers, PlainAccountState, PlainStorageState},
tables::{
AccountsHistory, CanonicalHeaders, Headers, PlainAccountState, PlainStorageState,
},
test_utils::*,
transaction::{DbTx, DbTxMut},
AccountChangeSet,
AccountChangeSets,
};
use reth_interfaces::db::{DatabaseWriteError, DatabaseWriteOperation};
use reth_libmdbx::Error;
@ -544,24 +549,24 @@ mod tests {
let address2 = Address::with_last_byte(2);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
tx.put::<AccountChangeSet>(0, AccountBeforeTx { address: address0, info: None })
tx.put::<AccountChangeSets>(0, AccountBeforeTx { address: address0, info: None })
.expect(ERROR_PUT);
tx.put::<AccountChangeSet>(0, AccountBeforeTx { address: address1, info: None })
tx.put::<AccountChangeSets>(0, AccountBeforeTx { address: address1, info: None })
.expect(ERROR_PUT);
tx.put::<AccountChangeSet>(0, AccountBeforeTx { address: address2, info: None })
tx.put::<AccountChangeSets>(0, AccountBeforeTx { address: address2, info: None })
.expect(ERROR_PUT);
tx.put::<AccountChangeSet>(1, AccountBeforeTx { address: address0, info: None })
tx.put::<AccountChangeSets>(1, AccountBeforeTx { address: address0, info: None })
.expect(ERROR_PUT);
tx.put::<AccountChangeSet>(1, AccountBeforeTx { address: address1, info: None })
tx.put::<AccountChangeSets>(1, AccountBeforeTx { address: address1, info: None })
.expect(ERROR_PUT);
tx.put::<AccountChangeSet>(1, AccountBeforeTx { address: address2, info: None })
tx.put::<AccountChangeSets>(1, AccountBeforeTx { address: address2, info: None })
.expect(ERROR_PUT);
tx.put::<AccountChangeSet>(2, AccountBeforeTx { address: address0, info: None }) // <- should not be returned by the walker
tx.put::<AccountChangeSets>(2, AccountBeforeTx { address: address0, info: None }) // <- should not be returned by the walker
.expect(ERROR_PUT);
tx.commit().expect(ERROR_COMMIT);
let tx = db.tx().expect(ERROR_INIT_TX);
let mut cursor = tx.cursor_read::<AccountChangeSet>().unwrap();
let mut cursor = tx.cursor_read::<AccountChangeSets>().unwrap();
let entries = cursor.walk_range(..).unwrap().collect::<Result<Vec<_>, _>>().unwrap();
assert_eq!(entries.len(), 7);
@ -958,7 +963,7 @@ mod tests {
let transition_id = 2;
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let mut cursor = tx.cursor_write::<AccountChangeSet>().unwrap();
let mut cursor = tx.cursor_write::<AccountChangeSets>().unwrap();
vec![0, 1, 3, 4, 5]
.into_iter()
.try_for_each(|val| {
@ -973,7 +978,7 @@ mod tests {
// APPEND DUP & APPEND
let subkey_to_append = 2;
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let mut cursor = tx.cursor_write::<AccountChangeSet>().unwrap();
let mut cursor = tx.cursor_write::<AccountChangeSets>().unwrap();
assert_eq!(
cursor.append_dup(
transition_id,
@ -982,7 +987,7 @@ mod tests {
Err(DatabaseWriteError {
info: Error::KeyMismatch.into(),
operation: DatabaseWriteOperation::CursorAppendDup,
table_name: AccountChangeSet::NAME,
table_name: AccountChangeSets::NAME,
key: transition_id.encode().into(),
}
.into())
@ -995,7 +1000,7 @@ mod tests {
Err(DatabaseWriteError {
info: Error::KeyMismatch.into(),
operation: DatabaseWriteOperation::CursorAppend,
table_name: AccountChangeSet::NAME,
table_name: AccountChangeSets::NAME,
key: (transition_id - 1).encode().into(),
}
.into())
@ -1184,13 +1189,14 @@ mod tests {
let key = ShardedKey::new(real_key, i * 100);
let list: IntegerList = vec![i * 100u64].into();
db.update(|tx| tx.put::<AccountHistory>(key.clone(), list.clone()).expect("")).unwrap();
db.update(|tx| tx.put::<AccountsHistory>(key.clone(), list.clone()).expect(""))
.unwrap();
}
// Seek value with non existing key.
{
let tx = db.tx().expect(ERROR_INIT_TX);
let mut cursor = tx.cursor_read::<AccountHistory>().unwrap();
let mut cursor = tx.cursor_read::<AccountsHistory>().unwrap();
// It will seek the one greater or equal to the query. Since we have `Address | 100`,
// `Address | 200` in the database and we're querying `Address | 150` it will return us
@ -1208,7 +1214,7 @@ mod tests {
// Seek greatest index
{
let tx = db.tx().expect(ERROR_INIT_TX);
let mut cursor = tx.cursor_read::<AccountHistory>().unwrap();
let mut cursor = tx.cursor_read::<AccountsHistory>().unwrap();
// It will seek the MAX value of transition index and try to use prev to get first
// biggers.

15
crates/storage/db/src/lib.rs
View File

@ -67,7 +67,7 @@ pub mod abstraction;
mod implementation;
mod metrics;
pub mod snapshot;
pub mod static_file;
pub mod tables;
mod utils;
pub mod version;
@ -98,7 +98,7 @@ pub fn init_db<P: AsRef<Path>>(path: P, args: DatabaseArguments) -> eyre::Result
let rpath = path.as_ref();
if is_database_empty(rpath) {
std::fs::create_dir_all(rpath)
reth_primitives::fs::create_dir_all(rpath)
.wrap_err_with(|| format!("Could not create database directory {}", rpath.display()))?;
create_db_version_file(rpath)?;
} else {
@ -163,6 +163,8 @@ pub mod test_utils {
pub const ERROR_DB_OPEN: &str = "Not able to open the database file.";
/// Error during database creation
pub const ERROR_DB_CREATION: &str = "Not able to create the database file.";
/// Error during database creation
pub const ERROR_STATIC_FILES_CREATION: &str = "Not able to create the static file path.";
/// Error during table creation
pub const ERROR_TABLE_CREATION: &str = "Not able to create tables in the database.";
/// Error during tempdir creation
@ -225,6 +227,15 @@ pub mod test_utils {
}
}
/// Create static_files path for testing
pub fn create_test_static_files_dir() -> PathBuf {
let path = tempdir_path();
let emsg = format!("{}: {:?}", ERROR_STATIC_FILES_CREATION, path);
reth_primitives::fs::create_dir_all(path.clone()).expect(&emsg);
path
}
/// Get a temporary directory path to use for the database
pub fn tempdir_path() -> PathBuf {
let builder = tempfile::Builder::new().prefix("reth-test-").rand_bytes(8).tempdir();

28
crates/storage/db/src/snapshot/masks.rs
View File

@ -1,28 +0,0 @@
use super::{ReceiptMask, TransactionMask};
use crate::{
add_snapshot_mask,
snapshot::mask::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask},
table::Table,
CanonicalHeaders, HeaderTD, Receipts, Transactions,
};
use reth_primitives::{BlockHash, Header};
// HEADER MASKS
add_snapshot_mask!(HeaderMask, Header, 0b001);
add_snapshot_mask!(HeaderMask, <HeaderTD as Table>::Value, 0b010);
add_snapshot_mask!(HeaderMask, BlockHash, 0b100);
add_snapshot_mask!(HeaderMask, Header, BlockHash, 0b101);
add_snapshot_mask!(
HeaderMask,
<HeaderTD as Table>::Value,
<CanonicalHeaders as Table>::Value,
0b110
);
// RECEIPT MASKS
add_snapshot_mask!(ReceiptMask, <Receipts as Table>::Value, 0b1);
// TRANSACTION MASKS
add_snapshot_mask!(TransactionMask, <Transactions as Table>::Value, 0b1);

76
crates/storage/db/src/snapshot/mod.rs
View File

@ -1,76 +0,0 @@
//! reth's snapshot database table import and access
mod generation;
use std::{
collections::{hash_map::Entry, HashMap},
ops::RangeInclusive,
path::Path,
};
pub use generation::*;
mod cursor;
pub use cursor::SnapshotCursor;
mod mask;
pub use mask::*;
use reth_nippy_jar::{NippyJar, NippyJarError};
use reth_primitives::{snapshot::SegmentHeader, BlockNumber, SnapshotSegment, TxNumber};
mod masks;
/// Alias type for a map of [`SnapshotSegment`] and sorted lists of existing snapshot ranges.
type SortedSnapshots =
HashMap<SnapshotSegment, Vec<(RangeInclusive<BlockNumber>, RangeInclusive<TxNumber>)>>;
/// Given the snapshots directory path, it returns a list over the existing snapshots organized by
/// [`SnapshotSegment`]. Each segment has a sorted list of block ranges and transaction ranges.
pub fn iter_snapshots(path: impl AsRef<Path>) -> Result<SortedSnapshots, NippyJarError> {
let mut static_files = SortedSnapshots::default();
let entries = reth_primitives::fs::read_dir(path.as_ref())
.map_err(|err| NippyJarError::Custom(err.to_string()))?
.filter_map(Result::ok)
.collect::<Vec<_>>();
for entry in entries {
if entry.metadata().map_or(false, |metadata| metadata.is_file()) {
if let Some((segment, block_range, tx_range)) =
SnapshotSegment::parse_filename(&entry.file_name())
{
let ranges = (block_range, tx_range);
match static_files.entry(segment) {
Entry::Occupied(mut entry) => {
entry.get_mut().push(ranges);
}
Entry::Vacant(entry) => {
entry.insert(vec![ranges]);
}
}
}
}
}
for (segment, range_list) in static_files.iter_mut() {
// Sort by block end range.
range_list.sort_by(|a, b| a.0.end().cmp(b.0.end()));
if let Some((block_range, tx_range)) = range_list.pop() {
// The highest height static file filename might not be indicative of its actual
// block_range, so we need to read its actual configuration.
let jar = NippyJar::<SegmentHeader>::load(
&path.as_ref().join(segment.filename(&block_range, &tx_range)),
)?;
if &tx_range != jar.user_header().tx_range() {
// TODO(joshie): rename
}
range_list.push((
jar.user_header().block_range().clone(),
jar.user_header().tx_range().clone(),
));
}
}
Ok(static_files)
}

34
crates/storage/db/src/snapshot/cursor.rs → crates/storage/db/src/static_file/cursor.rs
View File

@ -3,23 +3,23 @@ use crate::table::Decompress;
use derive_more::{Deref, DerefMut};
use reth_interfaces::provider::ProviderResult;
use reth_nippy_jar::{DataReader, NippyJar, NippyJarCursor};
use reth_primitives::{snapshot::SegmentHeader, B256};
use reth_primitives::{static_file::SegmentHeader, B256};
use std::sync::Arc;
/// Cursor of a snapshot segment.
/// Cursor of a static file segment.
#[derive(Debug, Deref, DerefMut)]
pub struct SnapshotCursor<'a>(NippyJarCursor<'a, SegmentHeader>);
pub struct StaticFileCursor<'a>(NippyJarCursor<'a, SegmentHeader>);
impl<'a> SnapshotCursor<'a> {
/// Returns a new [`SnapshotCursor`].
impl<'a> StaticFileCursor<'a> {
/// Returns a new [`StaticFileCursor`].
pub fn new(jar: &'a NippyJar<SegmentHeader>, reader: Arc<DataReader>) -> ProviderResult<Self> {
Ok(Self(NippyJarCursor::with_reader(jar, reader)?))
}
/// Returns the current `BlockNumber` or `TxNumber` of the cursor depending on the kind of
/// snapshot segment.
pub fn number(&self) -> u64 {
self.row_index() + self.jar().user_header().start()
/// static file segment.
pub fn number(&self) -> Option<u64> {
self.jar().user_header().start().map(|start| self.row_index() + start)
}
/// Gets a row of values.
@ -28,15 +28,21 @@ impl<'a> SnapshotCursor<'a> {
key_or_num: KeyOrNumber<'_>,
mask: usize,
) -> ProviderResult<Option<Vec<&'_ [u8]>>> {
if self.jar().rows() == 0 {
return Ok(None)
}
let row = match key_or_num {
KeyOrNumber::Key(k) => self.row_by_key_with_cols(k, mask),
KeyOrNumber::Number(n) => {
let offset = self.jar().user_header().start();
if offset > n {
return Ok(None)
KeyOrNumber::Number(n) => match self.jar().user_header().start() {
Some(offset) => {
if offset > n {
return Ok(None)
}
self.row_by_number_with_cols((n - offset) as usize, mask)
}
self.row_by_number_with_cols((n - offset) as usize, mask)
}
None => Ok(None),
},
}?;
Ok(row)

36
crates/storage/db/src/snapshot/generation.rs → crates/storage/db/src/static_file/generation.rs
View File

@ -10,16 +10,16 @@ use reth_nippy_jar::{ColumnResult, NippyJar, NippyJarHeader, PHFKey};
use reth_tracing::tracing::*;
use std::{error::Error as StdError, ops::RangeInclusive};
/// Macro that generates snapshot creation functions that take an arbitratry number of [`Table`] and
/// creates a [`NippyJar`] file out of their [`Table::Value`]. Each list of [`Table::Value`] from a
/// table is a column of values.
/// Macro that generates static file creation functions that take an arbitratry number of [`Table`]
/// and creates a [`NippyJar`] file out of their [`Table::Value`]. Each list of [`Table::Value`]
/// from a table is a column of values.
///
/// Has membership filter set and compression dictionary support.
macro_rules! generate_snapshot_func {
macro_rules! generate_static_file_func {
($(($($tbl:ident),+)),+ $(,)? ) => {
$(
paste::item! {
/// Creates a snapshot from specified tables. Each table's `Value` iterator represents a column.
/// Creates a static file from specified tables. Each table's `Value` iterator represents a column.
///
/// **Ensure the range contains the same number of rows.**
///
@ -29,9 +29,9 @@ macro_rules! generate_snapshot_func {
/// * `keys`: Iterator of keys (eg. `TxHash` or `BlockHash`) with length equal to `row_count` and ordered by future column insertion from `range`.
/// * `dict_compression_set`: Sets of column data for compression dictionaries. Max size is 2GB. Row count is independent.
/// * `row_count`: Total rows to add to `NippyJar`. Must match row count in `range`.
/// * `nippy_jar`: Snapshot object responsible for file generation.
/// * `nippy_jar`: Static File object responsible for file generation.
#[allow(non_snake_case)]
pub fn [<create_snapshot$(_ $tbl)+>]<
pub fn [<create_static_file$(_ $tbl)+>]<
$($tbl: Table<Key=K>,)+
K,
H: NippyJarHeader
@ -43,27 +43,27 @@ macro_rules! generate_snapshot_func {
dict_compression_set: Option<Vec<impl Iterator<Item = Vec<u8>>>>,
keys: Option<impl Iterator<Item = ColumnResult<impl PHFKey>>>,
row_count: usize,
nippy_jar: &mut NippyJar<H>
mut nippy_jar: NippyJar<H>
) -> ProviderResult<()>
where K: Key + Copy
{
let additional = additional.unwrap_or_default();
debug!(target: "reth::snapshot", ?range, "Creating snapshot {:?} and {} more columns.", vec![$($tbl::NAME,)+], additional.len());
debug!(target: "reth::static_file", ?range, "Creating static file {:?} and {} more columns.", vec![$($tbl::NAME,)+], additional.len());
let range: RangeInclusive<RawKey<K>> = RawKey::new(*range.start())..=RawKey::new(*range.end());
// Create PHF and Filter if required
if let Some(keys) = keys {
debug!(target: "reth::snapshot", "Calculating Filter, PHF and offset index list");
debug!(target: "reth::static_file", "Calculating Filter, PHF and offset index list");
nippy_jar.prepare_index(keys, row_count)?;
debug!(target: "reth::snapshot", "Filter, PHF and offset index list calculated.");
debug!(target: "reth::static_file", "Filter, PHF and offset index list calculated.");
}
// Create compression dictionaries if required
if let Some(data_sets) = dict_compression_set {
debug!(target: "reth::snapshot", "Creating compression dictionaries.");
debug!(target: "reth::static_file", "Creating compression dictionaries.");
nippy_jar.prepare_compression(data_sets)?;
debug!(target: "reth::snapshot", "Compression dictionaries created.");
debug!(target: "reth::static_file", "Compression dictionaries created.");
}
// Creates the cursors for the columns
@ -80,17 +80,17 @@ macro_rules! generate_snapshot_func {
)+
// Create the snapshot from the data
// Create the static file from the data
let col_iterators: Vec<Box<dyn Iterator<Item = Result<Vec<u8>,_>>>> = vec![
$(Box::new([< $tbl _iter>]),)+
];
debug!(target: "reth::snapshot", jar=?nippy_jar, "Generating snapshot file.");
debug!(target: "reth::static_file", jar=?nippy_jar, "Generating static file.");
nippy_jar.freeze(col_iterators.into_iter().chain(additional).collect(), row_count as u64)?;
let nippy_jar = nippy_jar.freeze(col_iterators.into_iter().chain(additional).collect(), row_count as u64)?;
debug!(target: "reth::snapshot", jar=?nippy_jar, "Snapshot file generated.");
debug!(target: "reth::static_file", jar=?nippy_jar, "Static file generated.");
Ok(())
}
@ -99,4 +99,4 @@ macro_rules! generate_snapshot_func {
};
}
generate_snapshot_func!((T1), (T1, T2), (T1, T2, T3), (T1, T2, T3, T4), (T1, T2, T3, T4, T5),);
generate_static_file_func!((T1), (T1, T2), (T1, T2, T3), (T1, T2, T3, T4), (T1, T2, T3, T4, T5),);

24
crates/storage/db/src/snapshot/mask.rs → crates/storage/db/src/static_file/mask.rs
View File

@ -4,14 +4,14 @@ use crate::table::Decompress;
///
/// #### Explanation:
///
/// A `NippyJar` snapshot row can contain multiple column values. To specify the column values
/// A `NippyJar` static file row can contain multiple column values. To specify the column values
/// to be read, a mask is utilized.
///
/// For example, a snapshot with three columns, if the first and last columns are queried, the mask
/// `0b101` would be passed. To select only the second column, the mask `0b010` would be used.
/// For example, a static file with three columns, if the first and last columns are queried, the
/// mask `0b101` would be passed. To select only the second column, the mask `0b010` would be used.
///
/// Since each snapshot has its own column distribution, different wrapper types are necessary. For
/// instance, `B256` might be the third column in the `Header` segment, while being the second
/// Since each static file has its own column distribution, different wrapper types are necessary.
/// For instance, `B256` might be the third column in the `Header` segment, while being the second
/// column in another segment. Hence, `Mask<B256>` would only be applicable to one of these
/// scenarios.
///
@ -24,7 +24,7 @@ macro_rules! add_segments {
($($segment:tt),+) => {
paste::paste! {
$(
#[doc = concat!("Mask for ", stringify!($segment), " snapshot segment. See [`Mask`] for more.")]
#[doc = concat!("Mask for ", stringify!($segment), " static file segment. See [`Mask`] for more.")]
#[derive(Debug)]
pub struct [<$segment Mask>]<FIRST, SECOND = (), THIRD = ()>(Mask<FIRST, SECOND, THIRD>);
)+
@ -37,7 +37,7 @@ add_segments!(Header, Receipt, Transaction);
pub trait ColumnSelectorOne {
/// First desired column value
type FIRST: Decompress;
/// Mask to obtain desired values, should correspond to the order of columns in a snapshot.
/// Mask to obtain desired values, should correspond to the order of columns in a static_file.
const MASK: usize;
}
@ -47,7 +47,7 @@ pub trait ColumnSelectorTwo {
type FIRST: Decompress;
/// Second desired column value
type SECOND: Decompress;
/// Mask to obtain desired values, should correspond to the order of columns in a snapshot.
/// Mask to obtain desired values, should correspond to the order of columns in a static_file.
const MASK: usize;
}
@ -59,13 +59,13 @@ pub trait ColumnSelectorThree {
type SECOND: Decompress;
/// Third desired column value
type THIRD: Decompress;
/// Mask to obtain desired values, should correspond to the order of columns in a snapshot.
/// Mask to obtain desired values, should correspond to the order of columns in a static_file.
const MASK: usize;
}
#[macro_export]
/// Add mask to select `N` column values from a specific snapshot segment row.
macro_rules! add_snapshot_mask {
/// Add mask to select `N` column values from a specific static file segment row.
macro_rules! add_static_file_mask {
($mask_struct:tt, $type1:ty, $mask:expr) => {
impl ColumnSelectorOne for $mask_struct<$type1> {
type FIRST = $type1;
@ -80,7 +80,7 @@ macro_rules! add_snapshot_mask {
}
};
($mask_struct:tt, $type1:ty, $type2:ty, $type3:ty, $mask:expr) => {
impl ColumnSelectorTwo for $mask_struct<$type1, $type2, $type3> {
impl ColumnSelectorThree for $mask_struct<$type1, $type2, $type3> {
type FIRST = $type1;
type SECOND = $type2;
type THIRD = $type3;

21
crates/storage/db/src/static_file/masks.rs Normal file
View File

@ -0,0 +1,21 @@
use super::{ReceiptMask, TransactionMask};
use crate::{
add_static_file_mask,
static_file::mask::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask},
table::Table,
HeaderTerminalDifficulties, Receipts, Transactions,
};
use reth_primitives::{BlockHash, Header};
// HEADER MASKS
add_static_file_mask!(HeaderMask, Header, 0b001);
add_static_file_mask!(HeaderMask, <HeaderTerminalDifficulties as Table>::Value, 0b010);
add_static_file_mask!(HeaderMask, BlockHash, 0b100);
add_static_file_mask!(HeaderMask, Header, BlockHash, 0b101);
add_static_file_mask!(HeaderMask, <HeaderTerminalDifficulties as Table>::Value, BlockHash, 0b110);
// RECEIPT MASKS
add_static_file_mask!(ReceiptMask, <Receipts as Table>::Value, 0b1);
// TRANSACTION MASKS
add_static_file_mask!(TransactionMask, <Transactions as Table>::Value, 0b1);

76
crates/storage/db/src/static_file/mod.rs Normal file
View File

@ -0,0 +1,76 @@
//! reth's static file database table import and access
mod generation;
use std::{
collections::{hash_map::Entry, HashMap},
path::Path,
};
pub use generation::*;
mod cursor;
pub use cursor::StaticFileCursor;
mod mask;
pub use mask::*;
use reth_nippy_jar::{NippyJar, NippyJarError};
use reth_primitives::{
static_file::{SegmentHeader, SegmentRangeInclusive},
StaticFileSegment,
};
mod masks;
/// Alias type for a map of [`StaticFileSegment`] and sorted lists of existing static file ranges.
type SortedStaticFiles =
HashMap<StaticFileSegment, Vec<(SegmentRangeInclusive, Option<SegmentRangeInclusive>)>>;
/// Given the static_files directory path, it returns a list over the existing static_files
/// organized by [`StaticFileSegment`]. Each segment has a sorted list of block ranges and
/// transaction ranges as presented in the file configuration.
pub fn iter_static_files(path: impl AsRef<Path>) -> Result<SortedStaticFiles, NippyJarError> {
let path = path.as_ref();
if !path.exists() {
reth_primitives::fs::create_dir_all(path)
.map_err(|err| NippyJarError::Custom(err.to_string()))?;
}
let mut static_files = SortedStaticFiles::default();
let entries = reth_primitives::fs::read_dir(path)
.map_err(|err| NippyJarError::Custom(err.to_string()))?
.filter_map(Result::ok)
.collect::<Vec<_>>();
for entry in entries {
if entry.metadata().map_or(false, |metadata| metadata.is_file()) {
if let Some((segment, _)) =
StaticFileSegment::parse_filename(&entry.file_name().to_string_lossy())
{
let jar = NippyJar::<SegmentHeader>::load(&entry.path())?;
let (block_range, tx_range) = (
jar.user_header().block_range().copied(),
jar.user_header().tx_range().copied(),
);
if let Some(block_range) = block_range {
match static_files.entry(segment) {
Entry::Occupied(mut entry) => {
entry.get_mut().push((block_range, tx_range));
}
Entry::Vacant(entry) => {
entry.insert(vec![(block_range, tx_range)]);
}
}
}
}
}
}
for (_, range_list) in static_files.iter_mut() {
// Sort by block end range.
range_list.sort_by(|a, b| a.0.end().cmp(&b.0.end()));
}
Ok(static_files)
}

1
crates/storage/db/src/tables/codecs/compact.rs
View File

@ -29,6 +29,7 @@ macro_rules! impl_compression_for_compact {
}
impl_compression_for_compact!(
SealedHeader,
Header,
Account,
Log,

24
crates/storage/db/src/tables/mod.rs
View File

@ -246,7 +246,7 @@ tables! {
table CanonicalHeaders<Key = BlockNumber, Value = HeaderHash>;
/// Stores the total difficulty from a block header.
table HeaderTD<Key = BlockNumber, Value = CompactU256>;
table HeaderTerminalDifficulties<Key = BlockNumber, Value = CompactU256>;
/// Stores the block number corresponding to a header.
table HeaderNumbers<Key = BlockHash, Value = BlockNumber>;
@ -269,12 +269,12 @@ tables! {
table Transactions<Key = TxNumber, Value = TransactionSignedNoHash>;
/// Stores the mapping of the transaction hash to the transaction number.
table TxHashNumber<Key = TxHash, Value = TxNumber>;
table TransactionHashNumbers<Key = TxHash, Value = TxNumber>;
/// Stores the mapping of transaction number to the blocks number.
///
/// The key is the highest transaction ID in the block.
table TransactionBlock<Key = TxNumber, Value = BlockNumber>;
table TransactionBlocks<Key = TxNumber, Value = BlockNumber>;
/// Canonical only Stores transaction receipts.
table Receipts<Key = TxNumber, Value = Receipt>;
@ -309,7 +309,7 @@ tables! {
/// * If there were no shard we would get `None` entry or entry of different storage key.
///
/// Code example can be found in `reth_provider::HistoricalStateProviderRef`
table AccountHistory<Key = ShardedKey<Address>, Value = BlockNumberList>;
table AccountsHistory<Key = ShardedKey<Address>, Value = BlockNumberList>;
/// Stores pointers to block number changeset with changes for each storage key.
///
@ -329,29 +329,29 @@ tables! {
/// * If there were no shard we would get `None` entry or entry of different storage key.
///
/// Code example can be found in `reth_provider::HistoricalStateProviderRef`
table StorageHistory<Key = StorageShardedKey, Value = BlockNumberList>;
table StoragesHistory<Key = StorageShardedKey, Value = BlockNumberList>;
/// Stores the state of an account before a certain transaction changed it.
/// Change on state can be: account is created, selfdestructed, touched while empty
/// or changed balance,nonce.
table AccountChangeSet<Key = BlockNumber, Value = AccountBeforeTx, SubKey = Address>;
table AccountChangeSets<Key = BlockNumber, Value = AccountBeforeTx, SubKey = Address>;
/// Stores the state of a storage key before a certain transaction changed it.
/// If [`StorageEntry::value`] is zero, this means storage was not existing
/// and needs to be removed.
table StorageChangeSet<Key = BlockNumberAddress, Value = StorageEntry, SubKey = B256>;
table StorageChangeSets<Key = BlockNumberAddress, Value = StorageEntry, SubKey = B256>;
/// Stores the current state of an [`Account`] indexed with `keccak256Address`
/// This table is in preparation for merkelization and calculation of state root.
/// We are saving whole account data as it is needed for partial update when
/// part of storage is changed. Benefit for merkelization is that hashed addresses are sorted.
table HashedAccount<Key = B256, Value = Account>;
table HashedAccounts<Key = B256, Value = Account>;
/// Stores the current storage values indexed with `keccak256Address` and
/// hash of storage key `keccak256key`.
/// This table is in preparation for merkelization and calculation of state root.
/// Benefit for merklization is that hashed addresses/keys are sorted.
table HashedStorage<Key = B256, Value = StorageEntry, SubKey = B256>;
table HashedStorages<Key = B256, Value = StorageEntry, SubKey = B256>;
/// Stores the current state's Merkle Patricia Tree.
table AccountsTrie<Key = StoredNibbles, Value = StoredBranchNode>;
@ -362,13 +362,13 @@ tables! {
/// Stores the transaction sender for each canonical transaction.
/// It is needed to speed up execution stage and allows fetching signer without doing
/// transaction signed recovery
table TxSenders<Key = TxNumber, Value = Address>;
table TransactionSenders<Key = TxNumber, Value = Address>;
/// Stores the highest synced block number and stage-specific checkpoint of each stage.
table SyncStage<Key = StageId, Value = StageCheckpoint>;
table StageCheckpoints<Key = StageId, Value = StageCheckpoint>;
/// Stores arbitrary data to keep track of a stage first-sync progress.
table SyncStageProgress<Key = StageId, Value = Vec<u8>>;
table StageCheckpointProgresses<Key = StageId, Value = Vec<u8>>;
/// Stores the highest pruned block number and prune mode of each prune segment.
table PruneCheckpoints<Key = PruneSegment, Value = PruneCheckpoint>;

4
crates/storage/db/src/tables/models/accounts.rs
View File

@ -11,7 +11,7 @@ use reth_codecs::{derive_arbitrary, Compact};
use reth_primitives::{Account, Address, BlockNumber, Buf};
use serde::{Deserialize, Serialize};
/// Account as it is saved inside [`AccountChangeSet`][crate::tables::AccountChangeSet].
/// Account as it is saved inside [`AccountChangeSets`][crate::tables::AccountChangeSets].
///
/// [`Address`] is the subkey.
#[derive_arbitrary(compact)]
@ -57,7 +57,7 @@ impl Compact for AccountBeforeTx {
}
/// [`BlockNumber`] concatenated with [`Address`]. Used as the key for
/// [`StorageChangeSet`](crate::tables::StorageChangeSet)
/// [`StorageChangeSets`](crate::tables::StorageChangeSets)
///
/// Since it's used as a key, it isn't compressed when encoding it.
#[derive(

12
crates/storage/db/src/tables/raw.rs
View File

@ -53,6 +53,12 @@ impl<K: Key> RawKey<K> {
Self { key: K::encode(key).into(), _phantom: std::marker::PhantomData }
}
/// Creates a raw key from an existing `Vec`. Useful when we already have the encoded
/// key.
pub fn from_vec(vec: Vec<u8>) -> Self {
Self { key: vec, _phantom: std::marker::PhantomData }
}
/// Returns the decoded value.
pub fn key(&self) -> Result<K, DatabaseError> {
K::decode(&self.key)
@ -112,6 +118,12 @@ impl<V: Value> RawValue<V> {
Self { value: V::compress(value).into(), _phantom: std::marker::PhantomData }
}
/// Creates a raw value from an existing `Vec`. Useful when we already have the encoded
/// value.
pub fn from_vec(vec: Vec<u8>) -> Self {
Self { value: vec, _phantom: std::marker::PhantomData }
}
/// Returns the decompressed value.
pub fn value(&self) -> Result<V, DatabaseError> {
V::decompress(&self.value)

4
crates/storage/db/src/version.rs
View File

@ -8,8 +8,8 @@ use std::{
/// The name of the file that contains the version of the database.
pub const DB_VERSION_FILE_NAME: &str = "database.version";
/// The version of the database stored in the [DB_VERSION_FILE_NAME] file in the same directory as
/// database. Example: `1`.
pub const DB_VERSION: u64 = 1;
/// database.
pub const DB_VERSION: u64 = 2;
/// Error when checking a database version using [check_db_version_file]
#[derive(thiserror::Error, Debug)]

6
crates/storage/libmdbx-rs/Cargo.toml
View File

@ -17,13 +17,13 @@ name = "reth_libmdbx"
[dependencies]
bitflags.workspace = true
byteorder = "1"
derive_more = "0.99"
derive_more.workspace = true
indexmap = "2"
libc = "0.2"
parking_lot.workspace = true
thiserror.workspace = true
dashmap = { version = "5.5.3", features = ["inline"], optional = true }
tracing = { workspace = true, optional = true }
tracing.workspace = true
ffi = { package = "reth-mdbx-sys", path = "./mdbx-sys" }
@ -33,7 +33,7 @@ libffi = "3.2.0"
[features]
default = []
return-borrowed = []
read-tx-timeouts = ["dashmap", "dashmap/inline", "tracing"]
read-tx-timeouts = ["dashmap", "dashmap/inline"]
[dev-dependencies]
pprof = { workspace = true, features = ["flamegraph", "frame-pointer", "criterion"] }

14
crates/storage/libmdbx-rs/src/environment.rs
View File

@ -20,6 +20,7 @@ use std::{
thread::sleep,
time::Duration,
};
use tracing::warn;
/// The default maximum duration of a read transaction.
#[cfg(feature = "read-tx-timeouts")]
@ -96,6 +97,7 @@ impl Environment {
/// Create a read-write transaction for use with the environment. This method will block while
/// there are any other read-write transactions open on the environment.
pub fn begin_rw_txn(&self) -> Result<Transaction<RW>> {
let mut warned = false;
let txn = loop {
let (tx, rx) = sync_channel(0);
self.txn_manager().send_message(TxnManagerMessage::Begin {
@ -105,6 +107,10 @@ impl Environment {
});
let res = rx.recv().unwrap();
if let Err(Error::Busy) = &res {
if !warned {
warned = true;
warn!(target: "libmdbx", "Process stalled, awaiting read-write transaction lock.");
}
sleep(Duration::from_millis(250));
continue
}
@ -937,7 +943,8 @@ mod tests {
.open(tempdir.path())
.unwrap();
// Insert some data in the database, so the read transaction can lock on the snapshot of it
// Insert some data in the database, so the read transaction can lock on the static file of
// it
{
let tx = env.begin_rw_txn().unwrap();
let db = tx.open_db(None).unwrap();
@ -950,7 +957,8 @@ mod tests {
// Create a read transaction
let _tx_ro = env.begin_ro_txn().unwrap();
// Change previously inserted data, so the read transaction would use the previous snapshot
// Change previously inserted data, so the read transaction would use the previous static
// file
{
let tx = env.begin_rw_txn().unwrap();
let db = tx.open_db(None).unwrap();
@ -961,7 +969,7 @@ mod tests {
}
// Insert more data in the database, so we hit the DB size limit error, and MDBX tries to
// kick long-lived readers and delete their snapshots
// kick long-lived readers and delete their static_files
{
let tx = env.begin_rw_txn().unwrap();
let db = tx.open_db(None).unwrap();

2
crates/storage/libmdbx-rs/src/flags.rs
View File

@ -25,7 +25,7 @@ pub enum SyncMode {
///
/// [SyncMode::UtterlyNoSync] the [SyncMode::SafeNoSync] flag disable similarly flush system
/// buffers to disk when committing a transaction. But there is a huge difference in how
/// are recycled the MVCC snapshots corresponding to previous "steady" transactions (see
/// are recycled the MVCC static_files corresponding to previous "steady" transactions (see
/// below).
///
/// With [crate::EnvironmentKind::WriteMap] the [SyncMode::SafeNoSync] instructs MDBX to use

4
crates/storage/nippy-jar/Cargo.toml
View File

@ -15,6 +15,8 @@ workspace = true
name = "reth_nippy_jar"
[dependencies]
# reth
reth-primitives.workspace = true
# filter
ph = "0.8.0"
@ -33,7 +35,7 @@ serde = { version = "1.0", features = ["derive"] }
tracing = "0.1.0"
anyhow = "1.0"
thiserror.workspace = true
derive_more = "0.99"
derive_more.workspace = true
[dev-dependencies]
rand = { version = "0.8", features = ["small_rng"] }

2
crates/storage/nippy-jar/src/error.rs
View File

@ -7,6 +7,8 @@ pub enum NippyJarError {
Internal(#[from] Box<dyn std::error::Error + Send + Sync>),
#[error(transparent)]
Disconnect(#[from] std::io::Error),
#[error(transparent)]
FileSystem(#[from] reth_primitives::fs::FsPathError),
#[error("{0}")]
Custom(String),
#[error(transparent)]

195
crates/storage/nippy-jar/src/lib.rs
View File

@ -206,6 +206,16 @@ impl<H: NippyJarHeader> NippyJar<H> {
&self.user_header
}
/// Gets total columns in jar.
pub fn columns(&self) -> usize {
self.columns
}
/// Gets total rows in jar.
pub fn rows(&self) -> usize {
self.rows
}
/// Returns the size of inclusion filter
pub fn filter_size(&self) -> usize {
self.size()
@ -232,7 +242,9 @@ impl<H: NippyJarHeader> NippyJar<H> {
/// **The user must ensure the header type matches the one used during the jar's creation.**
pub fn load(path: &Path) -> Result<Self, NippyJarError> {
// Read [`Self`] located at the data file.
let config_file = File::open(path.with_extension(CONFIG_FILE_EXTENSION))?;
let config_path = path.with_extension(CONFIG_FILE_EXTENSION);
let config_file = File::open(&config_path)
.map_err(|err| reth_primitives::fs::FsPathError::open(err, config_path))?;
let mut obj: Self = bincode::deserialize_from(&config_file)?;
obj.path = path.to_path_buf();
@ -269,6 +281,21 @@ impl<H: NippyJarHeader> NippyJar<H> {
self.path.with_extension(CONFIG_FILE_EXTENSION)
}
/// Deletes from disk this [`NippyJar`] alongside every satellite file.
pub fn delete(self) -> Result<(), NippyJarError> {
// TODO(joshie): ensure consistency on unexpected shutdown
for path in
[self.data_path().into(), self.index_path(), self.offsets_path(), self.config_path()]
{
if path.exists() {
reth_primitives::fs::remove_file(path)?;
}
}
Ok(())
}
/// Returns a [`DataReader`] of the data and offset file
pub fn open_data_reader(&self) -> Result<DataReader, NippyJarError> {
DataReader::new(self.data_path())
@ -338,14 +365,17 @@ impl<H: NippyJarHeader> NippyJar<H> {
/// Writes all data and configuration to a file and the offset index to another.
pub fn freeze(
&mut self,
mut self,
columns: Vec<impl IntoIterator<Item = ColumnResult<Vec<u8>>>>,
total_rows: u64,
) -> Result<(), NippyJarError> {
) -> Result<Self, NippyJarError> {
self.check_before_freeze(&columns)?;
debug!(target: "nippy-jar", path=?self.data_path(), "Opening data file.");
// Write phf, filter and offset index to file
self.freeze_filters()?;
// Creates the writer, data and offsets file
let mut writer = NippyJarWriter::new(self)?;
@ -355,12 +385,9 @@ impl<H: NippyJarHeader> NippyJar<H> {
// Flushes configuration and offsets to disk
writer.commit()?;
// Write phf, filter and offset index to file
self.freeze_filters()?;
debug!(target: "nippy-jar", ?writer, "Finished writing data.");
debug!(target: "nippy-jar", jar=?self, "Finished writing data.");
Ok(())
Ok(writer.into_jar())
}
/// Freezes [`PerfectHashingFunction`], [`InclusionFilter`] and the offset index to file.
@ -428,9 +455,9 @@ impl<H: NippyJarHeader> PerfectHashingFunction for NippyJar<H> {
}
}
/// Manages the reading of snapshot data using memory-mapped files.
/// Manages the reading of static file data using memory-mapped files.
///
/// Holds file and mmap descriptors of the data and offsets files of a snapshot.
/// Holds file and mmap descriptors of the data and offsets files of a static_file.
#[derive(Debug)]
pub struct DataReader {
/// Data file descriptor. Needs to be kept alive as long as `data_mmap` handle.
@ -558,15 +585,21 @@ mod tests {
let num_rows = col1.len() as u64;
let file_path = tempfile::NamedTempFile::new().unwrap();
let mut nippy = NippyJar::new_without_header(num_columns, file_path.path());
assert!(matches!(NippyJar::set_keys(&mut nippy, &col1), Err(NippyJarError::PHFMissing)));
let check_phf = |nippy: &mut NippyJar<_>| {
let create_nippy = || -> NippyJar<()> {
let mut nippy = NippyJar::new_without_header(num_columns, file_path.path());
assert!(matches!(
NippyJar::get_index(nippy, &col1[0]),
NippyJar::set_keys(&mut nippy, &col1),
Err(NippyJarError::PHFMissing)
));
nippy
};
let check_phf = |mut nippy: NippyJar<_>| {
assert!(matches!(
NippyJar::get_index(&nippy, &col1[0]),
Err(NippyJarError::PHFMissingKeys)
));
assert!(NippyJar::set_keys(nippy, &col1).is_ok());
assert!(NippyJar::set_keys(&mut nippy, &col1).is_ok());
let collect_indexes = |nippy: &NippyJar<_>| -> Vec<u64> {
col1.iter()
@ -575,12 +608,12 @@ mod tests {
};
// Ensure all indexes are unique
let indexes = collect_indexes(nippy);
let indexes = collect_indexes(&nippy);
assert_eq!(indexes.iter().collect::<HashSet<_>>().len(), indexes.len());
// Ensure reproducibility
assert!(NippyJar::set_keys(nippy, &col1).is_ok());
assert_eq!(indexes, collect_indexes(nippy));
assert!(NippyJar::set_keys(&mut nippy, &col1).is_ok());
assert_eq!(indexes, collect_indexes(&nippy));
// Ensure that loaded phf provides the same function outputs
nippy.prepare_index(clone_with_result(&col1), col1.len()).unwrap();
@ -593,12 +626,10 @@ mod tests {
};
// fmph bytes size for 100 values of 32 bytes: 54
nippy = nippy.with_fmph();
check_phf(&mut nippy);
check_phf(create_nippy().with_fmph());
// fmph bytes size for 100 values of 32 bytes: 46
nippy = nippy.with_gofmph();
check_phf(&mut nippy);
check_phf(create_nippy().with_gofmph());
}
#[test]
@ -631,7 +662,9 @@ mod tests {
assert!(InclusionFilter::add(&mut nippy, &col1[2]).is_ok());
assert!(InclusionFilter::add(&mut nippy, &col1[3]).is_ok());
nippy.freeze(vec![clone_with_result(&col1), clone_with_result(&col2)], num_rows).unwrap();
let nippy = nippy
.freeze(vec![clone_with_result(&col1), clone_with_result(&col2)], num_rows)
.unwrap();
let mut loaded_nippy = NippyJar::load_without_header(file_path.path()).unwrap();
loaded_nippy.load_filters().unwrap();
@ -675,6 +708,10 @@ mod tests {
Err(NippyJarError::CompressorNotReady)
));
let mut nippy =
NippyJar::new_without_header(num_columns, file_path.path()).with_zstd(true, 5000);
assert!(nippy.compressor().is_some());
nippy.prepare_compression(vec![col1.clone(), col2.clone()]).unwrap();
if let Some(Compressors::Zstd(zstd)) = &nippy.compressor() {
@ -684,7 +721,9 @@ mod tests {
));
}
nippy.freeze(vec![clone_with_result(&col1), clone_with_result(&col2)], num_rows).unwrap();
let nippy = nippy
.freeze(vec![clone_with_result(&col1), clone_with_result(&col2)], num_rows)
.unwrap();
let mut loaded_nippy = NippyJar::load_without_header(file_path.path()).unwrap();
loaded_nippy.load_filters().unwrap();
@ -724,10 +763,12 @@ mod tests {
let nippy = NippyJar::new_without_header(num_columns, file_path.path());
assert!(nippy.compressor().is_none());
let mut nippy = NippyJar::new_without_header(num_columns, file_path.path()).with_lz4();
let nippy = NippyJar::new_without_header(num_columns, file_path.path()).with_lz4();
assert!(nippy.compressor().is_some());
nippy.freeze(vec![clone_with_result(&col1), clone_with_result(&col2)], num_rows).unwrap();
let nippy = nippy
.freeze(vec![clone_with_result(&col1), clone_with_result(&col2)], num_rows)
.unwrap();
let mut loaded_nippy = NippyJar::load_without_header(file_path.path()).unwrap();
loaded_nippy.load_filters().unwrap();
@ -760,11 +801,13 @@ mod tests {
let nippy = NippyJar::new_without_header(num_columns, file_path.path());
assert!(nippy.compressor().is_none());
let mut nippy =
let nippy =
NippyJar::new_without_header(num_columns, file_path.path()).with_zstd(false, 5000);
assert!(nippy.compressor().is_some());
nippy.freeze(vec![clone_with_result(&col1), clone_with_result(&col2)], num_rows).unwrap();
let nippy = nippy
.freeze(vec![clone_with_result(&col1), clone_with_result(&col2)], num_rows)
.unwrap();
let mut loaded_nippy = NippyJar::load_without_header(file_path.path()).unwrap();
loaded_nippy.load_filters().unwrap();
@ -903,7 +946,7 @@ mod tests {
let mut data = col1.iter().zip(col2.iter()).enumerate().collect::<Vec<_>>();
data.shuffle(&mut rand::thread_rng());
// Imagine `Blocks` snapshot file has two columns: `Block | StoredWithdrawals`
// Imagine `Blocks` static file has two columns: `Block | StoredWithdrawals`
const BLOCKS_FULL_MASK: usize = 0b11;
// Read both columns
@ -1047,7 +1090,7 @@ mod tests {
col1: &[Vec<u8>],
col2: &[Vec<u8>],
) {
let mut nippy = NippyJar::load_without_header(file_path).unwrap();
let nippy = NippyJar::load_without_header(file_path).unwrap();
// Set the baseline that should be unwinded to
let initial_rows = nippy.rows;
@ -1059,7 +1102,7 @@ mod tests {
assert!(initial_offset_size > 0);
// Appends a third row
let mut writer = NippyJarWriter::new(&mut nippy).unwrap();
let mut writer = NippyJarWriter::new(nippy).unwrap();
writer.append_column(Some(Ok(&col1[2]))).unwrap();
writer.append_column(Some(Ok(&col2[2]))).unwrap();
@ -1073,7 +1116,7 @@ mod tests {
// Simulate an unexpected shutdown of the writer, before it can finish commit()
drop(writer);
let mut nippy = NippyJar::load_without_header(file_path).unwrap();
let nippy = NippyJar::load_without_header(file_path).unwrap();
assert_eq!(initial_rows, nippy.rows);
// Data was written successfuly
@ -1090,21 +1133,20 @@ mod tests {
// Writer will execute a consistency check and verify first that the offset list on disk
// doesn't match the nippy.rows, and prune it. Then, it will prune the data file
// accordingly as well.
let _writer = NippyJarWriter::new(&mut nippy).unwrap();
assert_eq!(initial_rows, nippy.rows);
let writer = NippyJarWriter::new(nippy).unwrap();
assert_eq!(initial_rows, writer.rows());
assert_eq!(
initial_offset_size,
File::open(nippy.offsets_path()).unwrap().metadata().unwrap().len() as usize
File::open(writer.offsets_path()).unwrap().metadata().unwrap().len() as usize
);
assert_eq!(
initial_data_size,
File::open(nippy.data_path()).unwrap().metadata().unwrap().len() as usize
File::open(writer.data_path()).unwrap().metadata().unwrap().len() as usize
);
assert_eq!(initial_rows, nippy.rows);
}
fn test_append_consistency_no_commit(file_path: &Path, col1: &[Vec<u8>], col2: &[Vec<u8>]) {
let mut nippy = NippyJar::load_without_header(file_path).unwrap();
let nippy = NippyJar::load_without_header(file_path).unwrap();
// Set the baseline that should be unwinded to
let initial_rows = nippy.rows;
@ -1117,14 +1159,14 @@ mod tests {
// Appends a third row, so we have an offset list in memory, which is not flushed to disk,
// while the data has been.
let mut writer = NippyJarWriter::new(&mut nippy).unwrap();
let mut writer = NippyJarWriter::new(nippy).unwrap();
writer.append_column(Some(Ok(&col1[2]))).unwrap();
writer.append_column(Some(Ok(&col2[2]))).unwrap();
// Simulate an unexpected shutdown of the writer, before it can call commit()
drop(writer);
let mut nippy = NippyJar::load_without_header(file_path).unwrap();
let nippy = NippyJar::load_without_header(file_path).unwrap();
assert_eq!(initial_rows, nippy.rows);
// Data was written successfuly
@ -1140,13 +1182,12 @@ mod tests {
// Writer will execute a consistency check and verify that the data file has more data than
// it should, and resets it to the last offset of the list (on disk here)
let _writer = NippyJarWriter::new(&mut nippy).unwrap();
assert_eq!(initial_rows, nippy.rows);
let writer = NippyJarWriter::new(nippy).unwrap();
assert_eq!(initial_rows, writer.rows());
assert_eq!(
initial_data_size,
File::open(nippy.data_path()).unwrap().metadata().unwrap().len() as usize
File::open(writer.data_path()).unwrap().metadata().unwrap().len() as usize
);
assert_eq!(initial_rows, nippy.rows);
}
fn append_two_rows(num_columns: usize, file_path: &Path, col1: &[Vec<u8>], col2: &[Vec<u8>]) {
@ -1157,7 +1198,7 @@ mod tests {
assert_eq!(nippy.max_row_size, 0);
assert_eq!(nippy.rows, 0);
let mut writer = NippyJarWriter::new(&mut nippy).unwrap();
let mut writer = NippyJarWriter::new(nippy).unwrap();
assert_eq!(writer.column(), 0);
writer.append_column(Some(Ok(&col1[0]))).unwrap();
@ -1173,26 +1214,26 @@ mod tests {
let expected_data_file_size = *writer.offsets().last().unwrap();
writer.commit().unwrap();
assert_eq!(nippy.max_row_size, col1[0].len() + col2[0].len());
assert_eq!(nippy.rows, 1);
assert_eq!(writer.max_row_size(), col1[0].len() + col2[0].len());
assert_eq!(writer.rows(), 1);
assert_eq!(
File::open(nippy.offsets_path()).unwrap().metadata().unwrap().len(),
File::open(writer.offsets_path()).unwrap().metadata().unwrap().len(),
1 + num_columns as u64 * 8 + 8
);
assert_eq!(
File::open(nippy.data_path()).unwrap().metadata().unwrap().len(),
File::open(writer.data_path()).unwrap().metadata().unwrap().len(),
expected_data_file_size
);
}
// Load and add 1 row
{
let mut nippy = NippyJar::load_without_header(file_path).unwrap();
let nippy = NippyJar::load_without_header(file_path).unwrap();
// Check if it was committed successfuly
assert_eq!(nippy.max_row_size, col1[0].len() + col2[0].len());
assert_eq!(nippy.rows, 1);
let mut writer = NippyJarWriter::new(&mut nippy).unwrap();
let mut writer = NippyJarWriter::new(nippy).unwrap();
assert_eq!(writer.column(), 0);
writer.append_column(Some(Ok(&col1[1]))).unwrap();
@ -1208,22 +1249,22 @@ mod tests {
let expected_data_file_size = *writer.offsets().last().unwrap();
writer.commit().unwrap();
assert_eq!(nippy.max_row_size, col1[0].len() + col2[0].len());
assert_eq!(nippy.rows, 2);
assert_eq!(writer.max_row_size(), col1[0].len() + col2[0].len());
assert_eq!(writer.rows(), 2);
assert_eq!(
File::open(nippy.offsets_path()).unwrap().metadata().unwrap().len(),
1 + nippy.rows as u64 * num_columns as u64 * 8 + 8
File::open(writer.offsets_path()).unwrap().metadata().unwrap().len(),
1 + writer.rows() as u64 * num_columns as u64 * 8 + 8
);
assert_eq!(
File::open(nippy.data_path()).unwrap().metadata().unwrap().len(),
File::open(writer.data_path()).unwrap().metadata().unwrap().len(),
expected_data_file_size
);
}
}
fn prune_rows(num_columns: usize, file_path: &Path, col1: &[Vec<u8>], col2: &[Vec<u8>]) {
let mut nippy = NippyJar::load_without_header(file_path).unwrap();
let mut writer = NippyJarWriter::new(&mut nippy).unwrap();
let nippy = NippyJar::load_without_header(file_path).unwrap();
let mut writer = NippyJarWriter::new(nippy).unwrap();
// Appends a third row, so we have an offset list in memory, which is not flushed to disk
writer.append_column(Some(Ok(&col1[2]))).unwrap();
@ -1231,32 +1272,38 @@ mod tests {
// This should prune from the on-memory offset list and ondisk offset list
writer.prune_rows(2).unwrap();
assert_eq!(nippy.rows, 1);
assert_eq!(writer.rows(), 1);
assert_eq!(
File::open(nippy.offsets_path()).unwrap().metadata().unwrap().len(),
1 + nippy.rows as u64 * num_columns as u64 * 8 + 8
File::open(writer.offsets_path()).unwrap().metadata().unwrap().len(),
1 + writer.rows() as u64 * num_columns as u64 * 8 + 8
);
let expected_data_size = col1[0].len() + col2[0].len();
assert_eq!(
File::open(nippy.data_path()).unwrap().metadata().unwrap().len() as usize,
File::open(writer.data_path()).unwrap().metadata().unwrap().len() as usize,
expected_data_size
);
let data_reader = nippy.open_data_reader().unwrap();
// there are only two valid offsets. so index 2 actually represents the expected file
// data size.
assert_eq!(data_reader.offset(2), expected_data_size as u64);
let nippy = NippyJar::load_without_header(file_path).unwrap();
{
let data_reader = nippy.open_data_reader().unwrap();
// there are only two valid offsets. so index 2 actually represents the expected file
// data size.
assert_eq!(data_reader.offset(2), expected_data_size as u64);
}
// This should prune from the ondisk offset list and clear the jar.
let mut writer = NippyJarWriter::new(&mut nippy).unwrap();
let mut writer = NippyJarWriter::new(nippy).unwrap();
writer.prune_rows(1).unwrap();
assert_eq!(nippy.rows, 0);
assert_eq!(nippy.max_row_size, 0);
assert_eq!(File::open(nippy.data_path()).unwrap().metadata().unwrap().len() as usize, 0);
assert_eq!(writer.rows(), 0);
assert_eq!(writer.max_row_size(), 0);
assert_eq!(File::open(writer.data_path()).unwrap().metadata().unwrap().len() as usize, 0);
// Only the byte that indicates how many bytes per offset should be left
assert_eq!(File::open(nippy.offsets_path()).unwrap().metadata().unwrap().len() as usize, 1);
assert_eq!(
File::open(writer.offsets_path()).unwrap().metadata().unwrap().len() as usize,
1
);
}
fn simulate_interrupted_prune(
@ -1265,7 +1312,7 @@ mod tests {
num_rows: u64,
missing_offsets: u64,
) {
let mut nippy = NippyJar::load_without_header(file_path).unwrap();
let nippy = NippyJar::load_without_header(file_path).unwrap();
let reader = nippy.open_data_reader().unwrap();
let offsets_file =
OpenOptions::new().read(true).write(true).open(nippy.offsets_path()).unwrap();
@ -1284,6 +1331,6 @@ mod tests {
data_file.set_len(data_len - 32 * missing_offsets).unwrap();
// runs the consistency check.
let _ = NippyJarWriter::new(&mut nippy).unwrap();
let _ = NippyJarWriter::new(nippy).unwrap();
}
}

162
crates/storage/nippy-jar/src/writer.rs
View File

@ -1,9 +1,8 @@
use crate::{compression::Compression, ColumnResult, NippyJar, NippyJarError, NippyJarHeader};
use std::{
cmp::Ordering,
fmt,
fs::{File, OpenOptions},
io::{Read, Seek, SeekFrom, Write},
io::{BufWriter, Read, Seek, SeekFrom, Write},
path::Path,
};
@ -23,14 +22,15 @@ const OFFSET_SIZE_BYTES: u64 = 8;
///
/// ## Data file layout
/// The data file is represented just as a sequence of bytes of data without any delimiters
pub struct NippyJarWriter<'a, H> {
/// Reference to the associated [`NippyJar`], containing all necessary configurations for data
#[derive(Debug)]
pub struct NippyJarWriter<H: NippyJarHeader = ()> {
/// Associated [`NippyJar`], containing all necessary configurations for data
/// handling.
jar: &'a mut NippyJar<H>,
jar: NippyJar<H>,
/// File handle to where the data is stored.
data_file: File,
data_file: BufWriter<File>,
/// File handle to where the offsets are stored.
offsets_file: File,
offsets_file: BufWriter<File>,
/// Temporary buffer to reuse when compressing data.
tmp_buf: Vec<u8>,
/// Used to find the maximum uncompressed size of a row in a jar.
@ -41,21 +41,19 @@ pub struct NippyJarWriter<'a, H> {
column: usize,
}
impl<H> fmt::Debug for NippyJarWriter<'_, H> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("NippyJarWriter").finish_non_exhaustive()
}
}
impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
pub fn new(jar: &'a mut NippyJar<H>) -> Result<Self, NippyJarError> {
impl<H: NippyJarHeader> NippyJarWriter<H> {
/// Creates a [`NippyJarWriter`] from [`NippyJar`].
pub fn new(mut jar: NippyJar<H>) -> Result<Self, NippyJarError> {
let (data_file, offsets_file, is_created) =
Self::create_or_open_files(jar.data_path(), &jar.offsets_path())?;
// Makes sure we don't have dangling data and offset files
jar.freeze_config()?;
let mut writer = Self {
jar,
data_file,
offsets_file,
data_file: BufWriter::new(data_file),
offsets_file: BufWriter::new(offsets_file),
tmp_buf: Vec::with_capacity(1_000_000),
uncompressed_row_size: 0,
offsets: Vec::with_capacity(1_000_000),
@ -66,35 +64,56 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
// changes if necessary.
if !is_created {
writer.check_consistency_and_heal()?;
writer.commit()?;
}
Ok(writer)
}
/// Returns a reference to `H` of [`NippyJar`]
pub fn user_header(&self) -> &H {
&self.jar.user_header
}
/// Returns a mutable reference to `H` of [`NippyJar`]
pub fn user_header_mut(&mut self) -> &mut H {
&mut self.jar.user_header
}
/// Gets total writter rows in jar.
pub fn rows(&self) -> usize {
self.jar.rows()
}
/// Consumes the writer and returns the associated [`NippyJar`].
pub fn into_jar(self) -> NippyJar<H> {
self.jar
}
fn create_or_open_files(
data: &Path,
offsets: &Path,
) -> Result<(File, File, bool), NippyJarError> {
let is_created = !data.exists() || !offsets.exists();
let mut data_file = if !data.exists() {
File::create(data)?
} else {
OpenOptions::new().read(true).write(true).open(data)?
};
if !data.exists() {
// File::create is write-only (no reading possible)
File::create(data)?;
}
let mut data_file = OpenOptions::new().read(true).write(true).open(data)?;
data_file.seek(SeekFrom::End(0))?;
let mut offsets_file = if !offsets.exists() {
let mut offsets = File::create(offsets)?;
if !offsets.exists() {
// File::create is write-only (no reading possible)
File::create(offsets)?;
}
// First byte of the offset file is the size of one offset in bytes
offsets.write_all(&[OFFSET_SIZE_BYTES as u8])?;
offsets.sync_all()?;
let mut offsets_file = OpenOptions::new().read(true).write(true).open(offsets)?;
offsets
} else {
OpenOptions::new().read(true).write(true).open(offsets)?
};
// First byte of the offset file is the size of one offset in bytes
offsets_file.write_all(&[OFFSET_SIZE_BYTES as u8])?;
offsets_file.sync_all()?;
offsets_file.seek(SeekFrom::End(0))?;
Ok((data_file, offsets_file, is_created))
@ -118,7 +137,7 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
let expected_offsets_file_size = 1 + // first byte is the size of one offset
OFFSET_SIZE_BYTES * self.jar.rows as u64 * self.jar.columns as u64 + // `offset size * num rows * num columns`
OFFSET_SIZE_BYTES; // expected size of the data file
let actual_offsets_file_size = self.offsets_file.metadata()?.len();
let actual_offsets_file_size = self.offsets_file.get_ref().metadata()?.len();
// Offsets configuration wasn't properly committed
match expected_offsets_file_size.cmp(&actual_offsets_file_size) {
@ -126,7 +145,7 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
// Happened during an appending job
// TODO: ideally we could truncate until the last offset of the last column of the
// last row inserted
self.offsets_file.set_len(expected_offsets_file_size)?;
self.offsets_file.get_mut().set_len(expected_offsets_file_size)?;
}
Ordering::Greater => {
// Happened during a pruning job
@ -145,14 +164,14 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
// last offset should match the data_file_len
let last_offset = reader.reverse_offset(0)?;
let data_file_len = self.data_file.metadata()?.len();
let data_file_len = self.data_file.get_ref().metadata()?.len();
// Offset list wasn't properly committed
match last_offset.cmp(&data_file_len) {
Ordering::Less => {
// Happened during an appending job, so we need to truncate the data, since there's
// no way to recover it.
self.data_file.set_len(last_offset)?;
self.data_file.get_mut().set_len(last_offset)?;
}
Ordering::Greater => {
// Happened during a pruning job, so we need to reverse iterate offsets until we
@ -160,12 +179,13 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
for index in 0..reader.offsets_count()? {
let offset = reader.reverse_offset(index + 1)?;
if offset == data_file_len {
self.offsets_file.set_len(
self.offsets_file
.metadata()?
.len()
.saturating_sub(OFFSET_SIZE_BYTES * (index as u64 + 1)),
)?;
let new_len = self
.offsets_file
.get_ref()
.metadata()?
.len()
.saturating_sub(OFFSET_SIZE_BYTES * (index as u64 + 1));
self.offsets_file.get_mut().set_len(new_len)?;
drop(reader);
@ -229,11 +249,11 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
self.offsets.push(self.data_file.stream_position()?);
}
self.write_column(value.as_ref())?;
let written = self.write_column(value.as_ref())?;
// Last offset represents the size of the data file if no more data is to be
// appended. Otherwise, represents the offset of the next data item.
self.offsets.push(self.data_file.stream_position()?);
self.offsets.push(self.offsets.last().expect("qed") + written as u64);
}
None => {
return Err(NippyJarError::UnexpectedMissingValue(
@ -248,15 +268,17 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
}
/// Writes column to data file. If it's the last column of the row, call `finalize_row()`
fn write_column(&mut self, value: &[u8]) -> Result<(), NippyJarError> {
fn write_column(&mut self, value: &[u8]) -> Result<usize, NippyJarError> {
self.uncompressed_row_size += value.len();
if let Some(compression) = &self.jar.compressor {
let len = if let Some(compression) = &self.jar.compressor {
let before = self.tmp_buf.len();
let len = compression.compress_to(value, &mut self.tmp_buf)?;
self.data_file.write_all(&self.tmp_buf[before..before + len])?;
len
} else {
self.data_file.write_all(value)?;
}
value.len()
};
self.column += 1;
@ -264,11 +286,14 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
self.finalize_row();
}
Ok(())
Ok(len)
}
/// Prunes rows from data and offsets file and updates its configuration on disk
pub fn prune_rows(&mut self, num_rows: usize) -> Result<(), NippyJarError> {
self.offsets_file.flush()?;
self.data_file.flush()?;
// Each column of a row is one offset
let num_offsets = num_rows * self.jar.columns;
@ -283,13 +308,13 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
self.offsets.truncate(self.offsets.len() - offsets_prune_count);
// Truncate the data file to the new length
self.data_file.set_len(new_len)?;
self.data_file.get_mut().set_len(new_len)?;
}
// Prune from on-disk offset list if there are still rows left to prune
if remaining_to_prune > 0 {
// Get the current length of the on-disk offset file
let length = self.offsets_file.metadata()?.len();
let length = self.offsets_file.get_ref().metadata()?.len();
// Handle non-empty offset file
if length > 1 {
@ -308,8 +333,8 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
// If all rows are to be pruned
if new_num_offsets <= 1 {
// <= 1 because the one offset would actually be the expected file data size
self.offsets_file.set_len(1)?;
self.data_file.set_len(0)?;
self.offsets_file.get_mut().set_len(1)?;
self.data_file.get_mut().set_len(0)?;
} else {
// Calculate the new length for the on-disk offset list
let new_len = 1 + new_num_offsets * OFFSET_SIZE_BYTES;
@ -318,20 +343,20 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
.seek(SeekFrom::Start(new_len.saturating_sub(OFFSET_SIZE_BYTES)))?;
// Read the last offset value
let mut last_offset = [0u8; OFFSET_SIZE_BYTES as usize];
self.offsets_file.read_exact(&mut last_offset)?;
self.offsets_file.get_ref().read_exact(&mut last_offset)?;
let last_offset = u64::from_le_bytes(last_offset);
// Update the lengths of both the offsets and data files
self.offsets_file.set_len(new_len)?;
self.data_file.set_len(last_offset)?;
self.offsets_file.get_mut().set_len(new_len)?;
self.data_file.get_mut().set_len(last_offset)?;
}
} else {
return Err(NippyJarError::InvalidPruning(0, remaining_to_prune as u64))
}
}
self.offsets_file.sync_all()?;
self.data_file.sync_all()?;
self.offsets_file.get_ref().sync_all()?;
self.data_file.get_ref().sync_all()?;
self.offsets_file.seek(SeekFrom::End(0))?;
self.data_file.seek(SeekFrom::End(0))?;
@ -358,7 +383,8 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
/// Commits configuration and offsets to disk. It drains the internal offset list.
pub fn commit(&mut self) -> Result<(), NippyJarError> {
self.data_file.sync_all()?;
self.data_file.flush()?;
self.data_file.get_ref().sync_all()?;
self.commit_offsets()?;
@ -374,11 +400,11 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
// `append_column()` works alongside commit. So we need to skip it.
let mut last_offset_ondisk = None;
if self.offsets_file.metadata()?.len() > 1 {
if self.offsets_file.get_ref().metadata()?.len() > 1 {
self.offsets_file.seek(SeekFrom::End(-(OFFSET_SIZE_BYTES as i64)))?;
let mut buf = [0u8; OFFSET_SIZE_BYTES as usize];
self.offsets_file.read_exact(&mut buf)?;
self.offsets_file.get_ref().read_exact(&mut buf)?;
last_offset_ondisk = Some(u64::from_le_bytes(buf));
}
@ -393,11 +419,17 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
}
self.offsets_file.write_all(&offset.to_le_bytes())?;
}
self.offsets_file.sync_all()?;
self.offsets_file.flush()?;
self.offsets_file.get_ref().sync_all()?;
Ok(())
}
#[cfg(test)]
pub fn max_row_size(&self) -> usize {
self.jar.max_row_size
}
#[cfg(test)]
pub fn column(&self) -> usize {
self.column
@ -412,4 +444,14 @@ impl<'a, H: NippyJarHeader> NippyJarWriter<'a, H> {
pub fn offsets_mut(&mut self) -> &mut Vec<u64> {
&mut self.offsets
}
#[cfg(test)]
pub fn offsets_path(&self) -> std::path::PathBuf {
self.jar.offsets_path()
}
#[cfg(test)]
pub fn data_path(&self) -> &Path {
self.jar.data_path()
}
}

1
crates/storage/provider/Cargo.toml
View File

@ -18,6 +18,7 @@ reth-interfaces.workspace = true
reth-db.workspace = true
reth-trie.workspace = true
reth-nippy-jar.workspace = true
reth-codecs.workspace = true
reth-node-api.workspace = true
revm.workspace = true

71
crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs
View File

@ -1,15 +1,15 @@
use crate::{StateChanges, StateReverts};
use crate::{providers::StaticFileProviderRWRefMut, StateChanges, StateReverts};
use reth_db::{
cursor::{DbCursorRO, DbCursorRW},
tables,
transaction::{DbTx, DbTxMut},
};
use reth_interfaces::db::DatabaseError;
use reth_interfaces::provider::{ProviderError, ProviderResult};
use reth_primitives::{
logs_bloom,
revm::compat::{into_reth_acc, into_revm_acc},
Account, Address, BlockNumber, Bloom, Bytecode, Log, Receipt, Receipts, StorageEntry, B256,
U256,
Account, Address, BlockNumber, Bloom, Bytecode, Log, Receipt, Receipts, StaticFileSegment,
StorageEntry, B256, U256,
};
use reth_trie::HashedPostState;
use revm::{
@ -285,15 +285,21 @@ impl BundleStateWithReceipts {
std::mem::swap(&mut self.bundle, &mut other)
}
/// Write the [BundleStateWithReceipts] to the database.
/// Write the [BundleStateWithReceipts] to database and receipts to either database or static
/// files if `static_file_producer` is `Some`. It should be none if there is any kind of
/// pruning/filtering over the receipts.
///
/// `is_value_known` should be set to `Not` if the [BundleStateWithReceipts] has some of its
/// state detached, This would make some original values not known.
pub fn write_to_db<TX: DbTxMut + DbTx>(
/// `omit_changed_check` should be set to true of bundle has some of it data
/// detached, This would make some original values not known.
pub fn write_to_storage<TX>(
self,
tx: &TX,
mut static_file_producer: Option<StaticFileProviderRWRefMut<'_>>,
is_value_known: OriginalValuesKnown,
) -> Result<(), DatabaseError> {
) -> ProviderResult<()>
where
TX: DbTxMut + DbTx,
{
let (plain_state, reverts) = self.bundle.into_plain_state_and_reverts(is_value_known);
StateReverts(reverts).write_to_db(tx, self.first_block)?;
@ -303,15 +309,22 @@ impl BundleStateWithReceipts {
let mut receipts_cursor = tx.cursor_write::<tables::Receipts>()?;
for (idx, receipts) in self.receipts.into_iter().enumerate() {
if !receipts.is_empty() {
let block_number = self.first_block + idx as u64;
let (_, body_indices) =
bodies_cursor.seek_exact(block_number)?.unwrap_or_else(|| {
let last_available = bodies_cursor.last().ok().flatten().map(|(number, _)| number);
panic!("body indices for block {block_number} must exist. last available block number: {last_available:?}");
});
let block_number = self.first_block + idx as u64;
let first_tx_index = bodies_cursor
.seek_exact(block_number)?
.map(|(_, indices)| indices.first_tx_num())
.ok_or_else(|| ProviderError::BlockBodyIndicesNotFound(block_number))?;
let first_tx_index = body_indices.first_tx_num();
if let Some(static_file_producer) = &mut static_file_producer {
// Increment block on static file header.
static_file_producer.increment_block(StaticFileSegment::Receipts)?;
for (tx_idx, receipt) in receipts.into_iter().enumerate() {
let receipt = receipt
.expect("receipt should not be filtered when saving to static files.");
static_file_producer.append_receipt(first_tx_index + tx_idx as u64, receipt)?;
}
} else if !receipts.is_empty() {
for (tx_idx, receipt) in receipts.into_iter().enumerate() {
if let Some(receipt) = receipt {
receipts_cursor.append(first_tx_index + tx_idx as u64, receipt)?;
@ -426,7 +439,7 @@ mod tests {
// Check change set
let mut changeset_cursor = provider
.tx_ref()
.cursor_dup_read::<tables::AccountChangeSet>()
.cursor_dup_read::<tables::AccountChangeSets>()
.expect("Could not open changeset cursor");
assert_eq!(
changeset_cursor.seek_exact(1).expect("Could not read account change set"),
@ -549,7 +562,7 @@ mod tests {
state.merge_transitions(BundleRetention::Reverts);
BundleStateWithReceipts::new(state.take_bundle(), Receipts::new(), 1)
.write_to_db(provider.tx_ref(), OriginalValuesKnown::Yes)
.write_to_storage(provider.tx_ref(), None, OriginalValuesKnown::Yes)
.expect("Could not write bundle state to DB");
// Check plain storage state
@ -594,7 +607,7 @@ mod tests {
// Check change set
let mut changeset_cursor = provider
.tx_ref()
.cursor_dup_read::<tables::StorageChangeSet>()
.cursor_dup_read::<tables::StorageChangeSets>()
.expect("Could not open storage changeset cursor");
assert_eq!(
changeset_cursor.seek_exact(BlockNumberAddress((1, address_a))).unwrap(),
@ -647,7 +660,7 @@ mod tests {
state.merge_transitions(BundleRetention::Reverts);
BundleStateWithReceipts::new(state.take_bundle(), Receipts::new(), 2)
.write_to_db(provider.tx_ref(), OriginalValuesKnown::Yes)
.write_to_storage(provider.tx_ref(), None, OriginalValuesKnown::Yes)
.expect("Could not write bundle state to DB");
assert_eq!(
@ -711,7 +724,7 @@ mod tests {
)]));
init_state.merge_transitions(BundleRetention::Reverts);
BundleStateWithReceipts::new(init_state.take_bundle(), Receipts::new(), 0)
.write_to_db(provider.tx_ref(), OriginalValuesKnown::Yes)
.write_to_storage(provider.tx_ref(), None, OriginalValuesKnown::Yes)
.expect("Could not write init bundle state to DB");
let mut state = State::builder().with_bundle_update().build();
@ -856,12 +869,12 @@ mod tests {
let bundle = state.take_bundle();
BundleStateWithReceipts::new(bundle, Receipts::new(), 1)
.write_to_db(provider.tx_ref(), OriginalValuesKnown::Yes)
.write_to_storage(provider.tx_ref(), None, OriginalValuesKnown::Yes)
.expect("Could not write bundle state to DB");
let mut storage_changeset_cursor = provider
.tx_ref()
.cursor_dup_read::<tables::StorageChangeSet>()
.cursor_dup_read::<tables::StorageChangeSets>()
.expect("Could not open plain storage state cursor");
let mut storage_changes = storage_changeset_cursor.walk_range(..).unwrap();
@ -1019,7 +1032,7 @@ mod tests {
)]));
init_state.merge_transitions(BundleRetention::Reverts);
BundleStateWithReceipts::new(init_state.take_bundle(), Receipts::new(), 0)
.write_to_db(provider.tx_ref(), OriginalValuesKnown::Yes)
.write_to_storage(provider.tx_ref(), None, OriginalValuesKnown::Yes)
.expect("Could not write init bundle state to DB");
let mut state = State::builder().with_bundle_update().build();
@ -1064,12 +1077,12 @@ mod tests {
// Commit block #1 changes to the database.
state.merge_transitions(BundleRetention::Reverts);
BundleStateWithReceipts::new(state.take_bundle(), Receipts::new(), 1)
.write_to_db(provider.tx_ref(), OriginalValuesKnown::Yes)
.write_to_storage(provider.tx_ref(), None, OriginalValuesKnown::Yes)
.expect("Could not write bundle state to DB");
let mut storage_changeset_cursor = provider
.tx_ref()
.cursor_dup_read::<tables::StorageChangeSet>()
.cursor_dup_read::<tables::StorageChangeSets>()
.expect("Could not open plain storage state cursor");
let range = BlockNumberAddress::range(1..=1);
let mut storage_changes = storage_changeset_cursor.walk_range(range).unwrap();
@ -1138,9 +1151,9 @@ mod tests {
db.update(|tx| {
for (address, (account, storage)) in prestate.iter() {
let hashed_address = keccak256(address);
tx.put::<tables::HashedAccount>(hashed_address, *account).unwrap();
tx.put::<tables::HashedAccounts>(hashed_address, *account).unwrap();
for (slot, value) in storage {
tx.put::<tables::HashedStorage>(
tx.put::<tables::HashedStorages>(
hashed_address,
StorageEntry { key: keccak256(slot), value: *value },
)

12
crates/storage/provider/src/bundle_state/hashed_state_changes.rs
View File

@ -17,7 +17,7 @@ impl HashedStateChanges {
pub fn write_to_db<TX: DbTxMut + DbTx>(self, tx: &TX) -> Result<(), DatabaseError> {
// Write hashed account updates.
let sorted_accounts = self.0.accounts.into_iter().sorted_unstable_by_key(|(key, _)| *key);
let mut hashed_accounts_cursor = tx.cursor_write::<tables::HashedAccount>()?;
let mut hashed_accounts_cursor = tx.cursor_write::<tables::HashedAccounts>()?;
for (hashed_address, account) in sorted_accounts {
if let Some(account) = account {
hashed_accounts_cursor.upsert(hashed_address, account)?;
@ -28,7 +28,7 @@ impl HashedStateChanges {
// Write hashed storage changes.
let sorted_storages = self.0.storages.into_iter().sorted_by_key(|(key, _)| *key);
let mut hashed_storage_cursor = tx.cursor_dup_write::<tables::HashedStorage>()?;
let mut hashed_storage_cursor = tx.cursor_dup_write::<tables::HashedStorages>()?;
for (hashed_address, storage) in sorted_storages {
if storage.wiped && hashed_storage_cursor.seek_exact(hashed_address)?.is_some() {
hashed_storage_cursor.delete_current_duplicates()?;
@ -74,9 +74,9 @@ mod tests {
{
let provider_rw = provider_factory.provider_rw().unwrap();
let mut accounts_cursor =
provider_rw.tx_ref().cursor_write::<tables::HashedAccount>().unwrap();
provider_rw.tx_ref().cursor_write::<tables::HashedAccounts>().unwrap();
let mut storage_cursor =
provider_rw.tx_ref().cursor_write::<tables::HashedStorage>().unwrap();
provider_rw.tx_ref().cursor_write::<tables::HashedStorages>().unwrap();
for address in addresses {
let hashed_address = keccak256(address);
@ -100,13 +100,13 @@ mod tests {
let provider = provider_factory.provider().unwrap();
assert_eq!(
provider.tx_ref().get::<tables::HashedAccount>(destroyed_address_hashed),
provider.tx_ref().get::<tables::HashedAccounts>(destroyed_address_hashed),
Ok(None)
);
assert_eq!(
provider
.tx_ref()
.cursor_read::<tables::HashedStorage>()
.cursor_read::<tables::HashedStorages>()
.unwrap()
.seek_by_key_subkey(destroyed_address_hashed, hashed_slot),
Ok(None)

4
crates/storage/provider/src/bundle_state/state_reverts.rs
View File

@ -32,7 +32,7 @@ impl StateReverts {
// Write storage changes
tracing::trace!(target: "provider::reverts", "Writing storage changes");
let mut storages_cursor = tx.cursor_dup_write::<tables::PlainStorageState>()?;
let mut storage_changeset_cursor = tx.cursor_dup_write::<tables::StorageChangeSet>()?;
let mut storage_changeset_cursor = tx.cursor_dup_write::<tables::StorageChangeSets>()?;
for (block_index, mut storage_changes) in self.0.storage.into_iter().enumerate() {
let block_number = first_block + block_index as BlockNumber;
@ -73,7 +73,7 @@ impl StateReverts {
// Write account changes
tracing::trace!(target: "provider::reverts", "Writing account changes");
let mut account_changeset_cursor = tx.cursor_dup_write::<tables::AccountChangeSet>()?;
let mut account_changeset_cursor = tx.cursor_dup_write::<tables::AccountChangeSets>()?;
for (block_index, mut account_block_reverts) in self.0.accounts.into_iter().enumerate() {
let block_number = first_block + block_index as BlockNumber;
// Sort accounts by address.

16
crates/storage/provider/src/providers/database/metrics.rs
View File

@ -50,14 +50,14 @@ pub(crate) enum Action {
InsertCanonicalHeaders,
InsertHeaders,
InsertHeaderNumbers,
InsertHeaderTD,
InsertHeaderTerminalDifficulties,
InsertBlockOmmers,
InsertTxSenders,
InsertTransactionSenders,
InsertTransactions,
InsertTxHashNumbers,
InsertTransactionHashNumbers,
InsertBlockWithdrawals,
InsertBlockBodyIndices,
InsertTransactionBlock,
InsertTransactionBlocks,
GetNextTxNum,
GetParentTD,
@ -77,14 +77,14 @@ impl Action {
Action::InsertCanonicalHeaders => "insert canonical headers",
Action::InsertHeaders => "insert headers",
Action::InsertHeaderNumbers => "insert header numbers",
Action::InsertHeaderTD => "insert header TD",
Action::InsertHeaderTerminalDifficulties => "insert header TD",
Action::InsertBlockOmmers => "insert block ommers",
Action::InsertTxSenders => "insert tx senders",
Action::InsertTransactionSenders => "insert tx senders",
Action::InsertTransactions => "insert transactions",
Action::InsertTxHashNumbers => "insert tx hash numbers",
Action::InsertTransactionHashNumbers => "insert transaction hash numbers",
Action::InsertBlockWithdrawals => "insert block withdrawals",
Action::InsertBlockBodyIndices => "insert block body indices",
Action::InsertTransactionBlock => "insert transaction block",
Action::InsertTransactionBlocks => "insert transaction blocks",
Action::GetNextTxNum => "get next tx num",
Action::GetParentTD => "get parent TD",
}

251
crates/storage/provider/src/providers/database/mod.rs
View File

@ -1,8 +1,9 @@
use crate::{
providers::{
state::{historical::HistoricalStateProvider, latest::LatestStateProvider},
SnapshotProvider,
StaticFileProvider,
},
to_range,
traits::{BlockSource, ReceiptProvider},
BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, EvmEnvProvider,
HeaderProvider, HeaderSyncGap, HeaderSyncGapProvider, HeaderSyncMode, ProviderError,
@ -13,12 +14,11 @@ use reth_db::{database::Database, init_db, models::StoredBlockBodyIndices, Datab
use reth_interfaces::{provider::ProviderResult, RethError, RethResult};
use reth_node_api::ConfigureEvmEnv;
use reth_primitives::{
snapshot::HighestSnapshots,
stage::{StageCheckpoint, StageId},
Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders, ChainInfo,
ChainSpec, Header, PruneCheckpoint, PruneSegment, Receipt, SealedBlock, SealedBlockWithSenders,
SealedHeader, TransactionMeta, TransactionSigned, TransactionSignedNoHash, TxHash, TxNumber,
Withdrawal, Withdrawals, B256, U256,
SealedHeader, StaticFileSegment, TransactionMeta, TransactionSigned, TransactionSignedNoHash,
TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256,
};
use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg};
use std::{
@ -26,7 +26,6 @@ use std::{
path::{Path, PathBuf},
sync::Arc,
};
use tokio::sync::watch;
use tracing::trace;
mod metrics;
@ -38,49 +37,51 @@ use reth_db::mdbx::DatabaseArguments;
/// A common provider that fetches data from a database.
///
/// This provider implements most provider or provider factory traits.
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct ProviderFactory<DB> {
/// Database
db: DB,
/// Chain spec
chain_spec: Arc<ChainSpec>,
/// Snapshot Provider
snapshot_provider: Option<Arc<SnapshotProvider>>,
}
impl<DB: Clone> Clone for ProviderFactory<DB> {
fn clone(&self) -> Self {
Self {
db: self.db.clone(),
chain_spec: Arc::clone(&self.chain_spec),
snapshot_provider: self.snapshot_provider.clone(),
}
}
/// Static File Provider
static_file_provider: StaticFileProvider,
}
impl<DB> ProviderFactory<DB> {
/// Create new database provider factory.
pub fn new(db: DB, chain_spec: Arc<ChainSpec>) -> Self {
Self { db, chain_spec, snapshot_provider: None }
pub fn new(
db: DB,
chain_spec: Arc<ChainSpec>,
static_files_path: PathBuf,
) -> RethResult<ProviderFactory<DB>> {
Ok(Self {
db,
chain_spec,
static_file_provider: StaticFileProvider::new(static_files_path)?,
})
}
/// Database provider that comes with a shared snapshot provider.
pub fn with_snapshots(
mut self,
snapshots_path: PathBuf,
highest_snapshot_tracker: watch::Receiver<Option<HighestSnapshots>>,
) -> ProviderResult<Self> {
self.snapshot_provider = Some(Arc::new(
SnapshotProvider::new(snapshots_path)?
.with_highest_tracker(Some(highest_snapshot_tracker)),
));
Ok(self)
/// Enables metrics on the static file provider.
pub fn with_static_files_metrics(mut self) -> Self {
self.static_file_provider = self.static_file_provider.with_metrics();
self
}
/// Returns reference to the underlying database.
pub fn db_ref(&self) -> &DB {
&self.db
}
/// Returns static file provider
pub fn static_file_provider(&self) -> StaticFileProvider {
self.static_file_provider.clone()
}
#[cfg(any(test, feature = "test-utils"))]
/// Consumes Self and returns DB
pub fn into_db(self) -> DB {
self.db
}
}
impl ProviderFactory<DatabaseEnv> {
@ -90,11 +91,12 @@ impl ProviderFactory<DatabaseEnv> {
path: P,
chain_spec: Arc<ChainSpec>,
args: DatabaseArguments,
static_files_path: PathBuf,
) -> RethResult<Self> {
Ok(ProviderFactory::<DatabaseEnv> {
db: init_db(path, args).map_err(|e| RethError::Custom(e.to_string()))?,
chain_spec,
snapshot_provider: None,
static_file_provider: StaticFileProvider::new(static_files_path)?,
})
}
}
@ -105,13 +107,11 @@ impl<DB: Database> ProviderFactory<DB> {
/// [`BlockHashReader`]. This may fail if the inner read database transaction fails to open.
#[track_caller]
pub fn provider(&self) -> ProviderResult<DatabaseProviderRO<DB>> {
let mut provider = DatabaseProvider::new(self.db.tx()?, self.chain_spec.clone());
if let Some(snapshot_provider) = &self.snapshot_provider {
provider = provider.with_snapshot_provider(snapshot_provider.clone());
}
Ok(provider)
Ok(DatabaseProvider::new(
self.db.tx()?,
self.chain_spec.clone(),
self.static_file_provider.clone(),
))
}
/// Returns a provider with a created `DbTxMut` inside, which allows fetching and updating
@ -120,20 +120,18 @@ impl<DB: Database> ProviderFactory<DB> {
/// open.
#[track_caller]
pub fn provider_rw(&self) -> ProviderResult<DatabaseProviderRW<DB>> {
let mut provider = DatabaseProvider::new_rw(self.db.tx_mut()?, self.chain_spec.clone());
if let Some(snapshot_provider) = &self.snapshot_provider {
provider = provider.with_snapshot_provider(snapshot_provider.clone());
}
Ok(DatabaseProviderRW(provider))
Ok(DatabaseProviderRW(DatabaseProvider::new_rw(
self.db.tx_mut()?,
self.chain_spec.clone(),
self.static_file_provider.clone(),
)))
}
/// Storage provider for latest block
#[track_caller]
pub fn latest(&self) -> ProviderResult<StateProviderBox> {
trace!(target: "providers::db", "Returning latest state provider");
Ok(Box::new(LatestStateProvider::new(self.db.tx()?)))
Ok(Box::new(LatestStateProvider::new(self.db.tx()?, self.static_file_provider())))
}
/// Storage provider for state at that given block
@ -145,7 +143,10 @@ impl<DB: Database> ProviderFactory<DB> {
if block_number == provider.best_block_number().unwrap_or_default() &&
block_number == provider.last_block_number().unwrap_or_default()
{
return Ok(Box::new(LatestStateProvider::new(provider.into_tx())))
return Ok(Box::new(LatestStateProvider::new(
provider.into_tx(),
self.static_file_provider(),
)))
}
// +1 as the changeset that we want is the one that was applied after this block.
@ -156,7 +157,11 @@ impl<DB: Database> ProviderFactory<DB> {
let storage_history_prune_checkpoint =
provider.get_prune_checkpoint(PruneSegment::StorageHistory)?;
let mut state_provider = HistoricalStateProvider::new(provider.into_tx(), block_number);
let mut state_provider = HistoricalStateProvider::new(
provider.into_tx(),
block_number,
self.static_file_provider(),
);
// If we pruned account or storage history, we can't return state on every historical block.
// Instead, we should cap it at the latest prune checkpoint for corresponding prune segment.
@ -219,7 +224,12 @@ impl<DB: Database> HeaderProvider for ProviderFactory<DB> {
}
fn header_by_number(&self, num: BlockNumber) -> ProviderResult<Option<Header>> {
self.provider()?.header_by_number(num)
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
num,
|static_file| static_file.header_by_number(num),
|| self.provider()?.header_by_number(num),
)
}
fn header_td(&self, hash: &BlockHash) -> ProviderResult<Option<U256>> {
@ -227,22 +237,44 @@ impl<DB: Database> HeaderProvider for ProviderFactory<DB> {
}
fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult<Option<U256>> {
self.provider()?.header_td_by_number(number)
if let Some(td) = self.chain_spec.final_paris_total_difficulty(number) {
// if this block is higher than the final paris(merge) block, return the final paris
// difficulty
return Ok(Some(td))
}
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
number,
|static_file| static_file.header_td_by_number(number),
|| self.provider()?.header_td_by_number(number),
)
}
fn headers_range(&self, range: impl RangeBounds<BlockNumber>) -> ProviderResult<Vec<Header>> {
self.provider()?.headers_range(range)
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Headers,
to_range(range),
|static_file, range, _| static_file.headers_range(range),
|range, _| self.provider()?.headers_range(range),
|_| true,
)
}
fn sealed_header(&self, number: BlockNumber) -> ProviderResult<Option<SealedHeader>> {
self.provider()?.sealed_header(number)
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
number,
|static_file| static_file.sealed_header(number),
|| self.provider()?.sealed_header(number),
)
}
fn sealed_headers_range(
&self,
range: impl RangeBounds<BlockNumber>,
) -> ProviderResult<Vec<SealedHeader>> {
self.provider()?.sealed_headers_range(range)
self.sealed_headers_while(range, |_| true)
}
fn sealed_headers_while(
@ -250,13 +282,24 @@ impl<DB: Database> HeaderProvider for ProviderFactory<DB> {
range: impl RangeBounds<BlockNumber>,
predicate: impl FnMut(&SealedHeader) -> bool,
) -> ProviderResult<Vec<SealedHeader>> {
self.provider()?.sealed_headers_while(range, predicate)
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Headers,
to_range(range),
|static_file, range, predicate| static_file.sealed_headers_while(range, predicate),
|range, predicate| self.provider()?.sealed_headers_while(range, predicate),
predicate,
)
}
}
impl<DB: Database> BlockHashReader for ProviderFactory<DB> {
fn block_hash(&self, number: u64) -> ProviderResult<Option<B256>> {
self.provider()?.block_hash(number)
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
number,
|static_file| static_file.block_hash(number),
|| self.provider()?.block_hash(number),
)
}
fn canonical_hashes_range(
@ -264,7 +307,13 @@ impl<DB: Database> BlockHashReader for ProviderFactory<DB> {
start: BlockNumber,
end: BlockNumber,
) -> ProviderResult<Vec<B256>> {
self.provider()?.canonical_hashes_range(start, end)
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Headers,
start..end,
|static_file, range, _| static_file.canonical_hashes_range(range.start, range.end),
|range, _| self.provider()?.canonical_hashes_range(range.start, range.end),
|_| true,
)
}
}
@ -337,14 +386,24 @@ impl<DB: Database> TransactionsProvider for ProviderFactory<DB> {
}
fn transaction_by_id(&self, id: TxNumber) -> ProviderResult<Option<TransactionSigned>> {
self.provider()?.transaction_by_id(id)
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Transactions,
id,
|static_file| static_file.transaction_by_id(id),
|| self.provider()?.transaction_by_id(id),
)
}
fn transaction_by_id_no_hash(
&self,
id: TxNumber,
) -> ProviderResult<Option<TransactionSignedNoHash>> {
self.provider()?.transaction_by_id_no_hash(id)
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Transactions,
id,
|static_file| static_file.transaction_by_id_no_hash(id),
|| self.provider()?.transaction_by_id_no_hash(id),
)
}
fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult<Option<TransactionSigned>> {
@ -397,7 +456,12 @@ impl<DB: Database> TransactionsProvider for ProviderFactory<DB> {
impl<DB: Database> ReceiptProvider for ProviderFactory<DB> {
fn receipt(&self, id: TxNumber) -> ProviderResult<Option<Receipt>> {
self.provider()?.receipt(id)
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Receipts,
id,
|static_file| static_file.receipt(id),
|| self.provider()?.receipt(id),
)
}
fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult<Option<Receipt>> {
@ -412,7 +476,13 @@ impl<DB: Database> ReceiptProvider for ProviderFactory<DB> {
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Receipt>> {
self.provider()?.receipts_by_tx_range(range)
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Receipts,
to_range(range),
|static_file, range, _| static_file.receipts_by_tx_range(range),
|range, _| self.provider()?.receipts_by_tx_range(range),
|_| true,
)
}
}
@ -530,13 +600,16 @@ impl<DB: Database> PruneCheckpointReader for ProviderFactory<DB> {
mod tests {
use super::ProviderFactory;
use crate::{
test_utils::create_test_provider_factory, BlockHashReader, BlockNumReader, BlockWriter,
HeaderSyncGapProvider, HeaderSyncMode, TransactionsProvider,
providers::StaticFileWriter, test_utils::create_test_provider_factory, BlockHashReader,
BlockNumReader, BlockWriter, HeaderSyncGapProvider, HeaderSyncMode, TransactionsProvider,
};
use alloy_rlp::Decodable;
use assert_matches::assert_matches;
use rand::Rng;
use reth_db::{tables, test_utils::ERROR_TEMPDIR, transaction::DbTxMut};
use reth_db::{
tables,
test_utils::{create_test_static_files_dir, ERROR_TEMPDIR},
};
use reth_interfaces::{
provider::ProviderError,
test_utils::{
@ -546,7 +619,8 @@ mod tests {
RethError,
};
use reth_primitives::{
hex_literal::hex, ChainSpecBuilder, PruneMode, PruneModes, SealedBlock, TxNumber, B256,
hex_literal::hex, ChainSpecBuilder, PruneMode, PruneModes, SealedBlock, StaticFileSegment,
TxNumber, B256, U256,
};
use std::{ops::RangeInclusive, sync::Arc};
use tokio::sync::watch;
@ -584,6 +658,7 @@ mod tests {
tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(),
Arc::new(chain_spec),
Default::default(),
create_test_static_files_dir(),
)
.unwrap();
@ -648,7 +723,7 @@ mod tests {
Ok(_)
);
let senders = provider.get_or_take::<tables::TxSenders, true>(range.clone());
let senders = provider.get_or_take::<tables::TransactionSenders, true>(range.clone());
assert_eq!(
senders,
Ok(range
@ -687,8 +762,6 @@ mod tests {
// Genesis
let checkpoint = 0;
let head = random_header(&mut rng, 0, None);
let gap_fill = random_header(&mut rng, 1, Some(head.hash()));
let gap_tip = random_header(&mut rng, 2, Some(gap_fill.hash()));
// Empty database
assert_matches!(
@ -698,46 +771,14 @@ mod tests {
);
// Checkpoint and no gap
provider
.tx_ref()
.put::<tables::CanonicalHeaders>(head.number, head.hash())
.expect("failed to write canonical");
provider
.tx_ref()
.put::<tables::Headers>(head.number, head.clone().unseal())
.expect("failed to write header");
let mut static_file_writer =
provider.static_file_provider().latest_writer(StaticFileSegment::Headers).unwrap();
static_file_writer.append_header(head.header().clone(), U256::ZERO, head.hash()).unwrap();
static_file_writer.commit().unwrap();
drop(static_file_writer);
let gap = provider.sync_gap(mode.clone(), checkpoint).unwrap();
assert_eq!(gap.local_head, head);
assert_eq!(gap.target.tip(), consensus_tip.into());
// Checkpoint and gap
provider
.tx_ref()
.put::<tables::CanonicalHeaders>(gap_tip.number, gap_tip.hash())
.expect("failed to write canonical");
provider
.tx_ref()
.put::<tables::Headers>(gap_tip.number, gap_tip.clone().unseal())
.expect("failed to write header");
let gap = provider.sync_gap(mode.clone(), checkpoint).unwrap();
assert_eq!(gap.local_head, head);
assert_eq!(gap.target.tip(), gap_tip.parent_hash.into());
// Checkpoint and gap closed
provider
.tx_ref()
.put::<tables::CanonicalHeaders>(gap_fill.number, gap_fill.hash())
.expect("failed to write canonical");
provider
.tx_ref()
.put::<tables::Headers>(gap_fill.number, gap_fill.clone().unseal())
.expect("failed to write header");
assert_matches!(
provider.sync_gap(mode, checkpoint),
Err(RethError::Provider(ProviderError::InconsistentHeaderGap))
);
}
}

447
crates/storage/provider/src/providers/database/provider.rs
View File

@ -1,6 +1,6 @@
use crate::{
bundle_state::{BundleStateInit, BundleStateWithReceipts, HashedStateChanges, RevertsInit},
providers::{database::metrics, SnapshotProvider},
providers::{database::metrics, static_file::StaticFileWriter, StaticFileProvider},
to_range,
traits::{
AccountExtReader, BlockSource, ChangeSetReader, ReceiptProvider, StageCheckpointWriter,
@ -8,7 +8,7 @@ use crate::{
AccountReader, BlockExecutionWriter, BlockHashReader, BlockNumReader, BlockReader, BlockWriter,
Chain, EvmEnvProvider, HashingWriter, HeaderProvider, HeaderSyncGap, HeaderSyncGapProvider,
HeaderSyncMode, HistoryWriter, OriginalValuesKnown, ProviderError, PruneCheckpointReader,
PruneCheckpointWriter, StageCheckpointReader, StorageReader, TransactionVariant,
PruneCheckpointWriter, StageCheckpointReader, StatsReader, StorageReader, TransactionVariant,
TransactionsProvider, TransactionsProviderExt, WithdrawalsProvider,
};
use itertools::{izip, Itertools};
@ -28,7 +28,7 @@ use reth_db::{
use reth_interfaces::{
p2p::headers::downloader::SyncTarget,
provider::{ProviderResult, RootMismatch},
RethError, RethResult,
RethResult,
};
use reth_node_api::ConfigureEvmEnv;
use reth_primitives::{
@ -38,7 +38,7 @@ use reth_primitives::{
trie::Nibbles,
Account, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders,
ChainInfo, ChainSpec, GotExpected, Hardfork, Head, Header, PruneCheckpoint, PruneModes,
PruneSegment, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, SnapshotSegment,
PruneSegment, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, StaticFileSegment,
StorageEntry, TransactionMeta, TransactionSigned, TransactionSignedEcRecovered,
TransactionSignedNoHash, TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256,
};
@ -49,6 +49,7 @@ use reth_trie::{
};
use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg, SpecId};
use std::{
cmp::Ordering,
collections::{hash_map, BTreeMap, BTreeSet, HashMap, HashSet},
fmt::Debug,
ops::{Bound, Deref, DerefMut, Range, RangeBounds, RangeInclusive},
@ -82,7 +83,7 @@ impl<DB: Database> DerefMut for DatabaseProviderRW<DB> {
}
impl<DB: Database> DatabaseProviderRW<DB> {
/// Commit database transaction
/// Commit database transaction and static file if it exists.
pub fn commit(self) -> ProviderResult<bool> {
self.0.commit()
}
@ -101,15 +102,25 @@ pub struct DatabaseProvider<TX> {
tx: TX,
/// Chain spec
chain_spec: Arc<ChainSpec>,
/// Snapshot provider
#[allow(dead_code)]
snapshot_provider: Option<Arc<SnapshotProvider>>,
/// Static File provider
static_file_provider: StaticFileProvider,
}
impl<TX> DatabaseProvider<TX> {
/// Returns a static file provider
pub fn static_file_provider(&self) -> &StaticFileProvider {
&self.static_file_provider
}
}
impl<TX: DbTxMut> DatabaseProvider<TX> {
/// Creates a provider with an inner read-write transaction.
pub fn new_rw(tx: TX, chain_spec: Arc<ChainSpec>) -> Self {
Self { tx, chain_spec, snapshot_provider: None }
pub fn new_rw(
tx: TX,
chain_spec: Arc<ChainSpec>,
static_file_provider: StaticFileProvider,
) -> Self {
Self { tx, chain_spec, static_file_provider }
}
}
@ -153,6 +164,29 @@ impl<TX: DbTx> DatabaseProvider<TX> {
}
}
impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
#[cfg(any(test, feature = "test-utils"))]
/// Inserts an historical block. Used for setting up test environments
pub fn insert_historical_block(
&self,
block: SealedBlockWithSenders,
prune_modes: Option<&PruneModes>,
) -> ProviderResult<StoredBlockBodyIndices> {
let ttd = if block.number == 0 {
block.difficulty
} else {
let parent_block_number = block.number - 1;
let parent_ttd = self.header_td_by_number(parent_block_number)?.unwrap_or_default();
parent_ttd + block.difficulty
};
let mut writer = self.static_file_provider.latest_writer(StaticFileSegment::Headers)?;
writer.append_header(block.header.as_ref().clone(), ttd, block.hash())?;
self.insert_block(block, prune_modes)
}
}
/// For a given key, unwind all history shards that are below the given block number.
///
/// S - Sharded key subtype.
@ -170,7 +204,7 @@ fn unwind_history_shards<S, T, C>(
start_key: T::Key,
block_number: BlockNumber,
mut shard_belongs_to_key: impl FnMut(&T::Key) -> bool,
) -> ProviderResult<Vec<usize>>
) -> ProviderResult<Vec<u64>>
where
T: Table<Value = BlockNumberList>,
T::Key: AsRef<ShardedKey<S>>,
@ -186,15 +220,15 @@ where
// Check the first item.
// If it is greater or eq to the block number, delete it.
let first = list.iter(0).next().expect("List can't be empty");
if first >= block_number as usize {
let first = list.iter().next().expect("List can't be empty");
if first >= block_number {
item = cursor.prev()?;
continue
} else if block_number <= sharded_key.as_ref().highest_block_number {
// Filter out all elements greater than block number.
return Ok(list.iter(0).take_while(|i| *i < block_number as usize).collect::<Vec<_>>())
return Ok(list.iter().take_while(|i| *i < block_number).collect::<Vec<_>>())
} else {
return Ok(list.iter(0).collect::<Vec<_>>())
return Ok(list.iter().collect::<Vec<_>>())
}
}
@ -203,14 +237,12 @@ where
impl<TX: DbTx> DatabaseProvider<TX> {
/// Creates a provider with an inner read-only transaction.
pub fn new(tx: TX, chain_spec: Arc<ChainSpec>) -> Self {
Self { tx, chain_spec, snapshot_provider: None }
}
/// Creates a new [`Self`] with access to a [`SnapshotProvider`].
pub fn with_snapshot_provider(mut self, snapshot_provider: Arc<SnapshotProvider>) -> Self {
self.snapshot_provider = Some(snapshot_provider);
self
pub fn new(
tx: TX,
chain_spec: Arc<ChainSpec>,
static_file_provider: StaticFileProvider,
) -> Self {
Self { tx, chain_spec, static_file_provider }
}
/// Consume `DbTx` or `DbTxMut`.
@ -250,96 +282,6 @@ impl<TX: DbTx> DatabaseProvider<TX> {
self
}
/// Gets data within a specified range, potentially spanning different snapshots and database.
///
/// # Arguments
/// * `segment` - The segment of the snapshot to query.
/// * `block_range` - The range of data to fetch.
/// * `fetch_from_snapshot` - A function to fetch data from the snapshot.
/// * `fetch_from_database` - A function to fetch data from the database.
/// * `predicate` - A function used to evaluate each item in the fetched data. Fetching is
/// terminated when this function returns false, thereby filtering the data based on the
/// provided condition.
fn get_range_with_snapshot<T, P, FS, FD>(
&self,
segment: SnapshotSegment,
mut block_or_tx_range: Range<u64>,
fetch_from_snapshot: FS,
mut fetch_from_database: FD,
mut predicate: P,
) -> ProviderResult<Vec<T>>
where
FS: Fn(&SnapshotProvider, Range<u64>, &mut P) -> ProviderResult<Vec<T>>,
FD: FnMut(Range<u64>, P) -> ProviderResult<Vec<T>>,
P: FnMut(&T) -> bool,
{
let mut data = Vec::new();
if let Some(snapshot_provider) = &self.snapshot_provider {
// If there is, check the maximum block or transaction number of the segment.
if let Some(snapshot_upper_bound) = match segment {
SnapshotSegment::Headers => snapshot_provider.get_highest_snapshot_block(segment),
SnapshotSegment::Transactions | SnapshotSegment::Receipts => {
snapshot_provider.get_highest_snapshot_tx(segment)
}
} {
if block_or_tx_range.start <= snapshot_upper_bound {
let end = block_or_tx_range.end.min(snapshot_upper_bound + 1);
data.extend(fetch_from_snapshot(
snapshot_provider,
block_or_tx_range.start..end,
&mut predicate,
)?);
block_or_tx_range.start = end;
}
}
}
if block_or_tx_range.end > block_or_tx_range.start {
data.extend(fetch_from_database(block_or_tx_range, predicate)?)
}
Ok(data)
}
/// Retrieves data from the database or snapshot, wherever it's available.
///
/// # Arguments
/// * `segment` - The segment of the snapshot to check against.
/// * `index_key` - Requested index key, usually a block or transaction number.
/// * `fetch_from_snapshot` - A closure that defines how to fetch the data from the snapshot
/// provider.
/// * `fetch_from_database` - A closure that defines how to fetch the data from the database
/// when the snapshot doesn't contain the required data or is not available.
fn get_with_snapshot<T, FS, FD>(
&self,
segment: SnapshotSegment,
number: u64,
fetch_from_snapshot: FS,
fetch_from_database: FD,
) -> ProviderResult<Option<T>>
where
FS: Fn(&SnapshotProvider) -> ProviderResult<Option<T>>,
FD: Fn() -> ProviderResult<Option<T>>,
{
if let Some(provider) = &self.snapshot_provider {
// If there is, check the maximum block or transaction number of the segment.
let snapshot_upper_bound = match segment {
SnapshotSegment::Headers => provider.get_highest_snapshot_block(segment),
SnapshotSegment::Transactions | SnapshotSegment::Receipts => {
provider.get_highest_snapshot_tx(segment)
}
};
if snapshot_upper_bound
.map_or(false, |snapshot_upper_bound| snapshot_upper_bound >= number)
{
return fetch_from_snapshot(provider)
}
}
fetch_from_database()
}
fn transactions_by_tx_range_with_cursor<C>(
&self,
range: impl RangeBounds<TxNumber>,
@ -348,10 +290,10 @@ impl<TX: DbTx> DatabaseProvider<TX> {
where
C: DbCursorRO<tables::Transactions>,
{
self.get_range_with_snapshot(
SnapshotSegment::Transactions,
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Transactions,
to_range(range),
|snapshot, range, _| snapshot.transactions_by_tx_range(range),
|static_file, range, _| static_file.transactions_by_tx_range(range),
|range, _| self.cursor_collect(cursor, range),
|_| true,
)
@ -375,9 +317,9 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
///
/// 1. Iterate over the [BlockBodyIndices][tables::BlockBodyIndices] table to get all
/// the transaction ids.
/// 2. Iterate over the [StorageChangeSet][tables::StorageChangeSet] table
/// and the [AccountChangeSet][tables::AccountChangeSet] tables in reverse order to reconstruct
/// the changesets.
/// 2. Iterate over the [StorageChangeSets][tables::StorageChangeSets] table
/// and the [AccountChangeSets][tables::AccountChangeSets] tables in reverse order to
/// reconstruct the changesets.
/// - In order to have both the old and new values in the changesets, we also access the
/// plain state tables.
/// 3. While iterating over the changeset tables, if we encounter a new account or storage slot,
@ -411,8 +353,8 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
let storage_range = BlockNumberAddress::range(range.clone());
let storage_changeset =
self.get_or_take::<tables::StorageChangeSet, UNWIND>(storage_range)?;
let account_changeset = self.get_or_take::<tables::AccountChangeSet, UNWIND>(range)?;
self.get_or_take::<tables::StorageChangeSets, UNWIND>(storage_range)?;
let account_changeset = self.get_or_take::<tables::AccountChangeSets, UNWIND>(range)?;
// iterate previous value and get plain state value to create changeset
// Double option around Account represent if Account state is know (first option) and
@ -592,8 +534,9 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
.map(|(id, tx)| (id, tx.into()))
.collect::<Vec<(u64, TransactionSigned)>>();
let mut senders =
self.get_or_take::<tables::TxSenders, TAKE>(first_transaction..=last_transaction)?;
let mut senders = self.get_or_take::<tables::TransactionSenders, TAKE>(
first_transaction..=last_transaction,
)?;
// Recover senders manually if not found in db
// NOTE: Transactions are always guaranteed to be in the database whereas
@ -655,18 +598,18 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
}
if TAKE {
// Remove TxHashNumber
let mut tx_hash_cursor = self.tx.cursor_write::<tables::TxHashNumber>()?;
// Remove TransactionHashNumbers
let mut tx_hash_cursor = self.tx.cursor_write::<tables::TransactionHashNumbers>()?;
for (_, tx) in transactions.iter() {
if tx_hash_cursor.seek_exact(tx.hash())?.is_some() {
tx_hash_cursor.delete_current()?;
}
}
// Remove TransactionBlock index if there are transaction present
// Remove TransactionBlocks index if there are transaction present
if !transactions.is_empty() {
let tx_id_range = transactions.first().unwrap().0..=transactions.last().unwrap().0;
self.get_or_take::<tables::TransactionBlock, TAKE>(tx_id_range)?;
self.get_or_take::<tables::TransactionBlocks, TAKE>(tx_id_range)?;
}
}
@ -723,8 +666,8 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
let block_tx = self.get_take_block_transaction_range::<TAKE>(range.clone())?;
if TAKE {
// rm HeaderTD
self.get_or_take::<tables::HeaderTD, TAKE>(range)?;
// rm HeaderTerminalDifficulties
self.get_or_take::<tables::HeaderTerminalDifficulties, TAKE>(range)?;
// rm HeaderNumbers
let mut header_number_cursor = self.tx.cursor_write::<tables::HeaderNumbers>()?;
for (_, hash) in block_header_hashes.iter() {
@ -914,7 +857,7 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
if let Some((shard_key, list)) = shard {
// delete old shard so new one can be inserted.
self.tx.delete::<T>(shard_key, None)?;
let list = list.iter(0).map(|i| i as u64).collect::<Vec<_>>();
let list = list.iter().collect::<Vec<_>>();
return Ok(list)
}
Ok(Vec::new())
@ -943,13 +886,13 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
let chunks = indices
.chunks(sharded_key::NUM_OF_INDICES_IN_SHARD)
.into_iter()
.map(|chunks| chunks.map(|i| *i as usize).collect::<Vec<usize>>())
.collect::<Vec<_>>();
.map(|chunks| chunks.copied().collect())
.collect::<Vec<Vec<_>>>();
let mut chunks = chunks.into_iter().peekable();
while let Some(list) = chunks.next() {
let highest_block_number = if chunks.peek().is_some() {
*list.last().expect("`chunks` does not return empty list") as u64
*list.last().expect("`chunks` does not return empty list")
} else {
// Insert last list with u64::MAX
u64::MAX
@ -976,7 +919,7 @@ impl<TX: DbTx> AccountExtReader for DatabaseProvider<TX> {
range: impl RangeBounds<BlockNumber>,
) -> ProviderResult<BTreeSet<Address>> {
self.tx
.cursor_read::<tables::AccountChangeSet>()?
.cursor_read::<tables::AccountChangeSets>()?
.walk_range(range)?
.map(|entry| {
entry.map(|(_, account_before)| account_before.address).map_err(Into::into)
@ -999,7 +942,7 @@ impl<TX: DbTx> AccountExtReader for DatabaseProvider<TX> {
&self,
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<BTreeMap<Address, Vec<u64>>> {
let mut changeset_cursor = self.tx.cursor_read::<tables::AccountChangeSet>()?;
let mut changeset_cursor = self.tx.cursor_read::<tables::AccountChangeSets>()?;
let account_transitions = changeset_cursor.walk_range(range)?.try_fold(
BTreeMap::new(),
@ -1021,7 +964,7 @@ impl<TX: DbTx> ChangeSetReader for DatabaseProvider<TX> {
) -> ProviderResult<Vec<AccountBeforeTx>> {
let range = block_number..=block_number;
self.tx
.cursor_read::<tables::AccountChangeSet>()?
.cursor_read::<tables::AccountChangeSets>()?
.walk_range(range)?
.map(|result| -> ProviderResult<_> {
let (_, account_before) = result?;
@ -1037,45 +980,38 @@ impl<TX: DbTx> HeaderSyncGapProvider for DatabaseProvider<TX> {
mode: HeaderSyncMode,
highest_uninterrupted_block: BlockNumber,
) -> RethResult<HeaderSyncGap> {
// Create a cursor over canonical header hashes
let mut cursor = self.tx.cursor_read::<tables::CanonicalHeaders>()?;
let mut header_cursor = self.tx.cursor_read::<tables::Headers>()?;
let static_file_provider = self.static_file_provider();
// Get head hash and reposition the cursor
let (head_num, head_hash) = cursor
.seek_exact(highest_uninterrupted_block)?
// Make sure Headers static file is at the same height. If it's further, this
// input execution was interrupted previously and we need to unwind the static file.
let next_static_file_block_num = static_file_provider
.get_highest_static_file_block(StaticFileSegment::Headers)
.map(|id| id + 1)
.unwrap_or_default();
let next_block = highest_uninterrupted_block + 1;
match next_static_file_block_num.cmp(&next_block) {
// The node shutdown between an executed static file commit and before the database
// commit, so we need to unwind the static files.
Ordering::Greater => {
let mut static_file_producer =
static_file_provider.latest_writer(StaticFileSegment::Headers)?;
static_file_producer.prune_headers(next_static_file_block_num - next_block)?
}
Ordering::Less => {
// There's either missing or corrupted files.
return Err(ProviderError::HeaderNotFound(next_static_file_block_num.into()).into())
}
Ordering::Equal => {}
}
let local_head = static_file_provider
.sealed_header(highest_uninterrupted_block)?
.ok_or_else(|| ProviderError::HeaderNotFound(highest_uninterrupted_block.into()))?;
// Construct head
let (_, head) = header_cursor
.seek_exact(head_num)?
.ok_or_else(|| ProviderError::HeaderNotFound(head_num.into()))?;
let local_head = head.seal(head_hash);
// Look up the next header
let next_header = cursor
.next()?
.map(|(next_num, next_hash)| -> Result<SealedHeader, RethError> {
let (_, next) = header_cursor
.seek_exact(next_num)?
.ok_or_else(|| ProviderError::HeaderNotFound(next_num.into()))?;
Ok(next.seal(next_hash))
})
.transpose()?;
// Decide the tip or error out on invalid input.
// If the next element found in the cursor is not the "expected" next block per our current
// checkpoint, then there is a gap in the database and we should start downloading in
// reverse from there. Else, it should use whatever the forkchoice state reports.
let target = match next_header {
Some(header) if highest_uninterrupted_block + 1 != header.number => {
SyncTarget::Gap(header)
}
None => match mode {
HeaderSyncMode::Tip(rx) => SyncTarget::Tip(*rx.borrow()),
HeaderSyncMode::Continuous => SyncTarget::TipNum(head_num + 1),
},
_ => return Err(ProviderError::InconsistentHeaderGap.into()),
let target = match mode {
HeaderSyncMode::Tip(rx) => SyncTarget::Tip(*rx.borrow()),
HeaderSyncMode::Continuous => SyncTarget::TipNum(highest_uninterrupted_block + 1),
};
Ok(HeaderSyncGap { local_head, target })
@ -1092,10 +1028,10 @@ impl<TX: DbTx> HeaderProvider for DatabaseProvider<TX> {
}
fn header_by_number(&self, num: BlockNumber) -> ProviderResult<Option<Header>> {
self.get_with_snapshot(
SnapshotSegment::Headers,
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
num,
|snapshot| snapshot.header_by_number(num),
|static_file| static_file.header_by_number(num),
|| Ok(self.tx.get::<tables::Headers>(num)?),
)
}
@ -1115,29 +1051,29 @@ impl<TX: DbTx> HeaderProvider for DatabaseProvider<TX> {
return Ok(Some(td))
}
self.get_with_snapshot(
SnapshotSegment::Headers,
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
number,
|snapshot| snapshot.header_td_by_number(number),
|| Ok(self.tx.get::<tables::HeaderTD>(number)?.map(|td| td.0)),
|static_file| static_file.header_td_by_number(number),
|| Ok(self.tx.get::<tables::HeaderTerminalDifficulties>(number)?.map(|td| td.0)),
)
}
fn headers_range(&self, range: impl RangeBounds<BlockNumber>) -> ProviderResult<Vec<Header>> {
self.get_range_with_snapshot(
SnapshotSegment::Headers,
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Headers,
to_range(range),
|snapshot, range, _| snapshot.headers_range(range),
|static_file, range, _| static_file.headers_range(range),
|range, _| self.cursor_read_collect::<tables::Headers>(range).map_err(Into::into),
|_| true,
)
}
fn sealed_header(&self, number: BlockNumber) -> ProviderResult<Option<SealedHeader>> {
self.get_with_snapshot(
SnapshotSegment::Headers,
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
number,
|snapshot| snapshot.sealed_header(number),
|static_file| static_file.sealed_header(number),
|| {
if let Some(header) = self.header_by_number(number)? {
let hash = self
@ -1156,10 +1092,10 @@ impl<TX: DbTx> HeaderProvider for DatabaseProvider<TX> {
range: impl RangeBounds<BlockNumber>,
predicate: impl FnMut(&SealedHeader) -> bool,
) -> ProviderResult<Vec<SealedHeader>> {
self.get_range_with_snapshot(
SnapshotSegment::Headers,
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Headers,
to_range(range),
|snapshot, range, predicate| snapshot.sealed_headers_while(range, predicate),
|static_file, range, predicate| static_file.sealed_headers_while(range, predicate),
|range, mut predicate| {
let mut headers = vec![];
for entry in self.tx.cursor_read::<tables::Headers>()?.walk_range(range)? {
@ -1182,10 +1118,10 @@ impl<TX: DbTx> HeaderProvider for DatabaseProvider<TX> {
impl<TX: DbTx> BlockHashReader for DatabaseProvider<TX> {
fn block_hash(&self, number: u64) -> ProviderResult<Option<B256>> {
self.get_with_snapshot(
SnapshotSegment::Headers,
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
number,
|snapshot| snapshot.block_hash(number),
|static_file| static_file.block_hash(number),
|| Ok(self.tx.get::<tables::CanonicalHeaders>(number)?),
)
}
@ -1195,10 +1131,10 @@ impl<TX: DbTx> BlockHashReader for DatabaseProvider<TX> {
start: BlockNumber,
end: BlockNumber,
) -> ProviderResult<Vec<B256>> {
self.get_range_with_snapshot(
SnapshotSegment::Headers,
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Headers,
start..end,
|snapshot, range, _| snapshot.canonical_hashes_range(range.start, range.end),
|static_file, range, _| static_file.canonical_hashes_range(range.start, range.end),
|range, _| {
self.cursor_read_collect::<tables::CanonicalHeaders>(range).map_err(Into::into)
},
@ -1418,10 +1354,10 @@ impl<TX: DbTx> TransactionsProviderExt for DatabaseProvider<TX> {
&self,
tx_range: Range<TxNumber>,
) -> ProviderResult<Vec<(TxHash, TxNumber)>> {
self.get_range_with_snapshot(
SnapshotSegment::Transactions,
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Transactions,
tx_range,
|snapshot, range, _| snapshot.transaction_hashes_by_range(range),
|static_file, range, _| static_file.transaction_hashes_by_range(range),
|tx_range, _| {
let mut tx_cursor = self.tx.cursor_read::<tables::Transactions>()?;
let tx_range_size = tx_range.clone().count();
@ -1482,14 +1418,14 @@ impl<TX: DbTx> TransactionsProviderExt for DatabaseProvider<TX> {
impl<TX: DbTx> TransactionsProvider for DatabaseProvider<TX> {
fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult<Option<TxNumber>> {
Ok(self.tx.get::<tables::TxHashNumber>(tx_hash)?)
Ok(self.tx.get::<tables::TransactionHashNumbers>(tx_hash)?)
}
fn transaction_by_id(&self, id: TxNumber) -> ProviderResult<Option<TransactionSigned>> {
self.get_with_snapshot(
SnapshotSegment::Transactions,
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Transactions,
id,
|snapshot| snapshot.transaction_by_id(id),
|static_file| static_file.transaction_by_id(id),
|| Ok(self.tx.get::<tables::Transactions>(id)?.map(Into::into)),
)
}
@ -1498,10 +1434,10 @@ impl<TX: DbTx> TransactionsProvider for DatabaseProvider<TX> {
&self,
id: TxNumber,
) -> ProviderResult<Option<TransactionSignedNoHash>> {
self.get_with_snapshot(
SnapshotSegment::Transactions,
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Transactions,
id,
|snapshot| snapshot.transaction_by_id_no_hash(id),
|static_file| static_file.transaction_by_id_no_hash(id),
|| Ok(self.tx.get::<tables::Transactions>(id)?),
)
}
@ -1523,7 +1459,7 @@ impl<TX: DbTx> TransactionsProvider for DatabaseProvider<TX> {
&self,
tx_hash: TxHash,
) -> ProviderResult<Option<(TransactionSigned, TransactionMeta)>> {
let mut transaction_cursor = self.tx.cursor_read::<tables::TransactionBlock>()?;
let mut transaction_cursor = self.tx.cursor_read::<tables::TransactionBlocks>()?;
if let Some(transaction_id) = self.transaction_id(tx_hash)? {
if let Some(tx) = self.transaction_by_id_no_hash(transaction_id)? {
let transaction = TransactionSigned {
@ -1563,7 +1499,7 @@ impl<TX: DbTx> TransactionsProvider for DatabaseProvider<TX> {
}
fn transaction_block(&self, id: TxNumber) -> ProviderResult<Option<BlockNumber>> {
let mut cursor = self.tx.cursor_read::<tables::TransactionBlock>()?;
let mut cursor = self.tx.cursor_read::<tables::TransactionBlocks>()?;
Ok(cursor.seek(id)?.map(|(_, bn)| bn))
}
@ -1629,20 +1565,20 @@ impl<TX: DbTx> TransactionsProvider for DatabaseProvider<TX> {
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Address>> {
self.cursor_read_collect::<tables::TxSenders>(range).map_err(Into::into)
self.cursor_read_collect::<tables::TransactionSenders>(range).map_err(Into::into)
}
fn transaction_sender(&self, id: TxNumber) -> ProviderResult<Option<Address>> {
Ok(self.tx.get::<tables::TxSenders>(id)?)
Ok(self.tx.get::<tables::TransactionSenders>(id)?)
}
}
impl<TX: DbTx> ReceiptProvider for DatabaseProvider<TX> {
fn receipt(&self, id: TxNumber) -> ProviderResult<Option<Receipt>> {
self.get_with_snapshot(
SnapshotSegment::Receipts,
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Receipts,
id,
|snapshot| snapshot.receipt(id),
|static_file| static_file.receipt(id),
|| Ok(self.tx.get::<tables::Receipts>(id)?),
)
}
@ -1673,10 +1609,10 @@ impl<TX: DbTx> ReceiptProvider for DatabaseProvider<TX> {
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Receipt>> {
self.get_range_with_snapshot(
SnapshotSegment::Receipts,
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Receipts,
to_range(range),
|snapshot, range, _| snapshot.receipts_by_tx_range(range),
|static_file, range, _| static_file.receipts_by_tx_range(range),
|range, _| self.cursor_read_collect::<tables::Receipts>(range).map_err(Into::into),
|_| true,
)
@ -1818,12 +1754,12 @@ impl<TX: DbTx> EvmEnvProvider for DatabaseProvider<TX> {
impl<TX: DbTx> StageCheckpointReader for DatabaseProvider<TX> {
fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult<Option<StageCheckpoint>> {
Ok(self.tx.get::<tables::SyncStage>(id.to_string())?)
Ok(self.tx.get::<tables::StageCheckpoints>(id.to_string())?)
}
/// Get stage checkpoint progress.
fn get_stage_checkpoint_progress(&self, id: StageId) -> ProviderResult<Option<Vec<u8>>> {
Ok(self.tx.get::<tables::SyncStageProgress>(id.to_string())?)
Ok(self.tx.get::<tables::StageCheckpointProgresses>(id.to_string())?)
}
}
@ -1834,7 +1770,7 @@ impl<TX: DbTxMut> StageCheckpointWriter for DatabaseProvider<TX> {
id: StageId,
checkpoint: StageCheckpoint,
) -> ProviderResult<()> {
Ok(self.tx.put::<tables::SyncStage>(id.to_string(), checkpoint)?)
Ok(self.tx.put::<tables::StageCheckpoints>(id.to_string(), checkpoint)?)
}
/// Save stage checkpoint progress.
@ -1843,7 +1779,7 @@ impl<TX: DbTxMut> StageCheckpointWriter for DatabaseProvider<TX> {
id: StageId,
checkpoint: Vec<u8>,
) -> ProviderResult<()> {
Ok(self.tx.put::<tables::SyncStageProgress>(id.to_string(), checkpoint)?)
Ok(self.tx.put::<tables::StageCheckpointProgresses>(id.to_string(), checkpoint)?)
}
fn update_pipeline_stages(
@ -1852,7 +1788,7 @@ impl<TX: DbTxMut> StageCheckpointWriter for DatabaseProvider<TX> {
drop_stage_checkpoint: bool,
) -> ProviderResult<()> {
// iterate over all existing stages in the table and update its progress.
let mut cursor = self.tx.cursor_write::<tables::SyncStage>()?;
let mut cursor = self.tx.cursor_write::<tables::StageCheckpoints>()?;
for stage_id in StageId::ALL {
let (_, checkpoint) = cursor.seek_exact(stage_id.to_string())?.unwrap_or_default();
cursor.upsert(
@ -1897,7 +1833,7 @@ impl<TX: DbTx> StorageReader for DatabaseProvider<TX> {
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<BTreeMap<Address, BTreeSet<B256>>> {
self.tx
.cursor_read::<tables::StorageChangeSet>()?
.cursor_read::<tables::StorageChangeSets>()?
.walk_range(BlockNumberAddress::range(range))?
// fold all storages and save its old state so we can remove it from HashedStorage
// it is needed as it is dup table.
@ -1912,7 +1848,7 @@ impl<TX: DbTx> StorageReader for DatabaseProvider<TX> {
&self,
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<BTreeMap<(Address, B256), Vec<u64>>> {
let mut changeset_cursor = self.tx.cursor_read::<tables::StorageChangeSet>()?;
let mut changeset_cursor = self.tx.cursor_read::<tables::StorageChangeSets>()?;
let storage_changeset_lists =
changeset_cursor.walk_range(BlockNumberAddress::range(range))?.try_fold(
@ -1941,7 +1877,7 @@ impl<TX: DbTxMut + DbTx> HashingWriter for DatabaseProvider<TX> {
// changes are applied in the correct order.
let hashed_accounts = self
.tx
.cursor_read::<tables::AccountChangeSet>()?
.cursor_read::<tables::AccountChangeSets>()?
.walk_range(range)?
.map(|entry| entry.map(|(_, e)| (keccak256(e.address), e.info)))
.collect::<Result<Vec<_>, _>>()?
@ -1950,7 +1886,7 @@ impl<TX: DbTxMut + DbTx> HashingWriter for DatabaseProvider<TX> {
.collect::<BTreeMap<_, _>>();
// Apply values to HashedState, and remove the account if it's None.
let mut hashed_accounts_cursor = self.tx.cursor_write::<tables::HashedAccount>()?;
let mut hashed_accounts_cursor = self.tx.cursor_write::<tables::HashedAccounts>()?;
for (hashed_address, account) in &hashed_accounts {
if let Some(account) = account {
hashed_accounts_cursor.upsert(*hashed_address, *account)?;
@ -1966,7 +1902,7 @@ impl<TX: DbTxMut + DbTx> HashingWriter for DatabaseProvider<TX> {
&self,
accounts: impl IntoIterator<Item = (Address, Option<Account>)>,
) -> ProviderResult<BTreeMap<B256, Option<Account>>> {
let mut hashed_accounts_cursor = self.tx.cursor_write::<tables::HashedAccount>()?;
let mut hashed_accounts_cursor = self.tx.cursor_write::<tables::HashedAccounts>()?;
let hashed_accounts =
accounts.into_iter().map(|(ad, ac)| (keccak256(ad), ac)).collect::<BTreeMap<_, _>>();
for (hashed_address, account) in &hashed_accounts {
@ -1984,7 +1920,7 @@ impl<TX: DbTxMut + DbTx> HashingWriter for DatabaseProvider<TX> {
range: Range<BlockNumberAddress>,
) -> ProviderResult<HashMap<B256, BTreeSet<B256>>> {
// Aggregate all block changesets and make list of accounts that have been changed.
let mut changesets = self.tx.cursor_read::<tables::StorageChangeSet>()?;
let mut changesets = self.tx.cursor_read::<tables::StorageChangeSets>()?;
let mut hashed_storages = changesets
.walk_range(range)?
.map(|entry| {
@ -1997,7 +1933,7 @@ impl<TX: DbTxMut + DbTx> HashingWriter for DatabaseProvider<TX> {
// Apply values to HashedState, and remove the account if it's None.
let mut hashed_storage_keys: HashMap<B256, BTreeSet<B256>> = HashMap::new();
let mut hashed_storage = self.tx.cursor_dup_write::<tables::HashedStorage>()?;
let mut hashed_storage = self.tx.cursor_dup_write::<tables::HashedStorages>()?;
for (hashed_address, key, value) in hashed_storages.into_iter().rev() {
hashed_storage_keys.entry(hashed_address).or_default().insert(key);
@ -2036,7 +1972,7 @@ impl<TX: DbTxMut + DbTx> HashingWriter for DatabaseProvider<TX> {
(*hashed_address, BTreeSet::from_iter(entries.keys().copied()))
}));
let mut hashed_storage_cursor = self.tx.cursor_dup_write::<tables::HashedStorage>()?;
let mut hashed_storage_cursor = self.tx.cursor_dup_write::<tables::HashedStorages>()?;
// Hash the address and key and apply them to HashedStorage (if Storage is None
// just remove it);
hashed_storages.into_iter().try_for_each(|(hashed_address, storage)| {
@ -2157,7 +2093,7 @@ impl<TX: DbTxMut + DbTx> HistoryWriter for DatabaseProvider<TX> {
&self,
storage_transitions: BTreeMap<(Address, B256), Vec<u64>>,
) -> ProviderResult<()> {
self.append_history_index::<_, tables::StorageHistory>(
self.append_history_index::<_, tables::StoragesHistory>(
storage_transitions,
|(address, storage_key), highest_block_number| {
StorageShardedKey::new(address, storage_key, highest_block_number)
@ -2169,7 +2105,10 @@ impl<TX: DbTxMut + DbTx> HistoryWriter for DatabaseProvider<TX> {
&self,
account_transitions: BTreeMap<Address, Vec<u64>>,
) -> ProviderResult<()> {
self.append_history_index::<_, tables::AccountHistory>(account_transitions, ShardedKey::new)
self.append_history_index::<_, tables::AccountsHistory>(
account_transitions,
ShardedKey::new,
)
}
fn unwind_storage_history_indices(
@ -2178,7 +2117,7 @@ impl<TX: DbTxMut + DbTx> HistoryWriter for DatabaseProvider<TX> {
) -> ProviderResult<usize> {
let mut storage_changesets = self
.tx
.cursor_read::<tables::StorageChangeSet>()?
.cursor_read::<tables::StorageChangeSets>()?
.walk_range(range)?
.map(|entry| {
entry.map(|(BlockNumberAddress((bn, address)), storage)| (address, storage.key, bn))
@ -2186,9 +2125,9 @@ impl<TX: DbTxMut + DbTx> HistoryWriter for DatabaseProvider<TX> {
.collect::<Result<Vec<_>, _>>()?;
storage_changesets.sort_by_key(|(address, key, _)| (*address, *key));
let mut cursor = self.tx.cursor_write::<tables::StorageHistory>()?;
let mut cursor = self.tx.cursor_write::<tables::StoragesHistory>()?;
for &(address, storage_key, rem_index) in &storage_changesets {
let partial_shard = unwind_history_shards::<_, tables::StorageHistory, _>(
let partial_shard = unwind_history_shards::<_, tables::StoragesHistory, _>(
&mut cursor,
StorageShardedKey::last(address, storage_key),
rem_index,
@ -2218,16 +2157,16 @@ impl<TX: DbTxMut + DbTx> HistoryWriter for DatabaseProvider<TX> {
) -> ProviderResult<usize> {
let mut last_indices = self
.tx
.cursor_read::<tables::AccountChangeSet>()?
.cursor_read::<tables::AccountChangeSets>()?
.walk_range(range)?
.map(|entry| entry.map(|(index, account)| (account.address, index)))
.collect::<Result<Vec<_>, _>>()?;
last_indices.sort_by_key(|(a, _)| *a);
// Unwind the account history index.
let mut cursor = self.tx.cursor_write::<tables::AccountHistory>()?;
let mut cursor = self.tx.cursor_write::<tables::AccountsHistory>()?;
for &(address, rem_index) in &last_indices {
let partial_shard = unwind_history_shards::<_, tables::AccountHistory, _>(
let partial_shard = unwind_history_shards::<_, tables::AccountsHistory, _>(
&mut cursor,
ShardedKey::last(address),
rem_index,
@ -2375,8 +2314,8 @@ impl<TX: DbTxMut + DbTx> BlockWriter for DatabaseProvider<TX> {
parent_ttd + block.difficulty
};
self.tx.put::<tables::HeaderTD>(block_number, ttd.into())?;
durations_recorder.record_relative(metrics::Action::InsertHeaderTD);
self.tx.put::<tables::HeaderTerminalDifficulties>(block_number, ttd.into())?;
durations_recorder.record_relative(metrics::Action::InsertHeaderTerminalDifficulties);
// insert body ommers data
if !block.ommers.is_empty() {
@ -2389,7 +2328,7 @@ impl<TX: DbTxMut + DbTx> BlockWriter for DatabaseProvider<TX> {
let mut next_tx_num = self
.tx
.cursor_read::<tables::Transactions>()?
.cursor_read::<tables::TransactionBlocks>()?
.last()?
.map(|(n, _)| n + 1)
.unwrap_or_default();
@ -2412,7 +2351,7 @@ impl<TX: DbTxMut + DbTx> BlockWriter for DatabaseProvider<TX> {
.is_none()
{
let start = Instant::now();
self.tx.put::<tables::TxSenders>(next_tx_num, *sender)?;
self.tx.put::<tables::TransactionSenders>(next_tx_num, *sender)?;
tx_senders_elapsed += start.elapsed();
}
@ -2437,16 +2376,19 @@ impl<TX: DbTxMut + DbTx> BlockWriter for DatabaseProvider<TX> {
.is_none()
{
let start = Instant::now();
self.tx.put::<tables::TxHashNumber>(hash, next_tx_num)?;
self.tx.put::<tables::TransactionHashNumbers>(hash, next_tx_num)?;
tx_hash_numbers_elapsed += start.elapsed();
}
next_tx_num += 1;
}
durations_recorder.record_duration(metrics::Action::InsertTxSenders, tx_senders_elapsed);
durations_recorder
.record_duration(metrics::Action::InsertTransactionSenders, tx_senders_elapsed);
durations_recorder
.record_duration(metrics::Action::InsertTransactions, transactions_elapsed);
durations_recorder
.record_duration(metrics::Action::InsertTxHashNumbers, tx_hash_numbers_elapsed);
durations_recorder.record_duration(
metrics::Action::InsertTransactionHashNumbers,
tx_hash_numbers_elapsed,
);
if let Some(withdrawals) = block.block.withdrawals {
if !withdrawals.is_empty() {
@ -2463,8 +2405,8 @@ impl<TX: DbTxMut + DbTx> BlockWriter for DatabaseProvider<TX> {
durations_recorder.record_relative(metrics::Action::InsertBlockBodyIndices);
if !block_indices.is_empty() {
self.tx.put::<tables::TransactionBlock>(block_indices.last_tx_num(), block_number)?;
durations_recorder.record_relative(metrics::Action::InsertTransactionBlock);
self.tx.put::<tables::TransactionBlocks>(block_indices.last_tx_num(), block_number)?;
durations_recorder.record_relative(metrics::Action::InsertTransactionBlocks);
}
debug!(
@ -2505,7 +2447,7 @@ impl<TX: DbTxMut + DbTx> BlockWriter for DatabaseProvider<TX> {
// Write state and changesets to the database.
// Must be written after blocks because of the receipt lookup.
state.write_to_db(self.tx_ref(), OriginalValuesKnown::No)?;
state.write_to_storage(self.tx_ref(), None, OriginalValuesKnown::No)?;
durations_recorder.record_relative(metrics::Action::InsertState);
// insert hashes and intermediate merkle nodes
@ -2547,6 +2489,19 @@ impl<TX: DbTxMut> PruneCheckpointWriter for DatabaseProvider<TX> {
}
}
impl<TX: DbTx> StatsReader for DatabaseProvider<TX> {
fn count_entries<T: Table>(&self) -> ProviderResult<usize> {
let db_entries = self.tx.entries::<T>()?;
let static_file_entries = match self.static_file_provider.count_entries::<T>() {
Ok(entries) => entries,
Err(ProviderError::UnsupportedProvider) => 0,
Err(err) => return Err(err),
};
Ok(db_entries + static_file_entries)
}
}
fn range_size_hint(range: &impl RangeBounds<TxNumber>) -> Option<usize> {
let start = match range.start_bound().cloned() {
Bound::Included(start) => start,

79
crates/storage/provider/src/providers/mod.rs
View File

@ -39,8 +39,11 @@ pub use state::{
mod bundle_state_provider;
mod chain_info;
mod database;
mod snapshot;
pub use snapshot::{SnapshotJarProvider, SnapshotProvider};
mod static_file;
pub use static_file::{
StaticFileJarProvider, StaticFileProvider, StaticFileProviderRW, StaticFileProviderRWRefMut,
StaticFileWriter,
};
mod state;
use crate::{providers::chain_info::ChainInfoTracker, traits::BlockSource};
pub use bundle_state_provider::BundleStateProvider;
@ -131,34 +134,34 @@ where
Tree: Send + Sync,
{
fn header(&self, block_hash: &BlockHash) -> ProviderResult<Option<Header>> {
self.database.provider()?.header(block_hash)
self.database.header(block_hash)
}
fn header_by_number(&self, num: BlockNumber) -> ProviderResult<Option<Header>> {
self.database.provider()?.header_by_number(num)
self.database.header_by_number(num)
}
fn header_td(&self, hash: &BlockHash) -> ProviderResult<Option<U256>> {
self.database.provider()?.header_td(hash)
self.database.header_td(hash)
}
fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult<Option<U256>> {
self.database.provider()?.header_td_by_number(number)
self.database.header_td_by_number(number)
}
fn headers_range(&self, range: impl RangeBounds<BlockNumber>) -> ProviderResult<Vec<Header>> {
self.database.provider()?.headers_range(range)
self.database.headers_range(range)
}
fn sealed_header(&self, number: BlockNumber) -> ProviderResult<Option<SealedHeader>> {
self.database.provider()?.sealed_header(number)
self.database.sealed_header(number)
}
fn sealed_headers_range(
&self,
range: impl RangeBounds<BlockNumber>,
) -> ProviderResult<Vec<SealedHeader>> {
self.database.provider()?.sealed_headers_range(range)
self.database.sealed_headers_range(range)
}
fn sealed_headers_while(
@ -166,7 +169,7 @@ where
range: impl RangeBounds<BlockNumber>,
predicate: impl FnMut(&SealedHeader) -> bool,
) -> ProviderResult<Vec<SealedHeader>> {
self.database.provider()?.sealed_headers_while(range, predicate)
self.database.sealed_headers_while(range, predicate)
}
}
@ -176,7 +179,7 @@ where
Tree: Send + Sync,
{
fn block_hash(&self, number: u64) -> ProviderResult<Option<B256>> {
self.database.provider()?.block_hash(number)
self.database.block_hash(number)
}
fn canonical_hashes_range(
@ -184,7 +187,7 @@ where
start: BlockNumber,
end: BlockNumber,
) -> ProviderResult<Vec<B256>> {
self.database.provider()?.canonical_hashes_range(start, end)
self.database.canonical_hashes_range(start, end)
}
}
@ -202,11 +205,11 @@ where
}
fn last_block_number(&self) -> ProviderResult<BlockNumber> {
self.database.provider()?.last_block_number()
self.database.last_block_number()
}
fn block_number(&self, hash: B256) -> ProviderResult<Option<BlockNumber>> {
self.database.provider()?.block_number(hash)
self.database.block_number(hash)
}
}
@ -237,7 +240,7 @@ where
let block = match source {
BlockSource::Any => {
// check database first
let mut block = self.database.provider()?.block_by_hash(hash)?;
let mut block = self.database.block_by_hash(hash)?;
if block.is_none() {
// Note: it's fine to return the unsealed block because the caller already has
// the hash
@ -246,7 +249,7 @@ where
block
}
BlockSource::Pending => self.tree.block_by_hash(hash).map(|block| block.unseal()),
BlockSource::Database => self.database.provider()?.block_by_hash(hash)?,
BlockSource::Database => self.database.block_by_hash(hash)?,
};
Ok(block)
@ -255,7 +258,7 @@ where
fn block(&self, id: BlockHashOrNumber) -> ProviderResult<Option<Block>> {
match id {
BlockHashOrNumber::Hash(hash) => self.find_block_by_hash(hash, BlockSource::Any),
BlockHashOrNumber::Number(num) => self.database.provider()?.block_by_number(num),
BlockHashOrNumber::Number(num) => self.database.block_by_number(num),
}
}
@ -272,14 +275,14 @@ where
}
fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult<Option<Vec<Header>>> {
self.database.provider()?.ommers(id)
self.database.ommers(id)
}
fn block_body_indices(
&self,
number: BlockNumber,
) -> ProviderResult<Option<StoredBlockBodyIndices>> {
self.database.provider()?.block_body_indices(number)
self.database.block_body_indices(number)
}
/// Returns the block with senders with matching number or hash from database.
@ -293,11 +296,11 @@ where
id: BlockHashOrNumber,
transaction_kind: TransactionVariant,
) -> ProviderResult<Option<BlockWithSenders>> {
self.database.provider()?.block_with_senders(id, transaction_kind)
self.database.block_with_senders(id, transaction_kind)
}
fn block_range(&self, range: RangeInclusive<BlockNumber>) -> ProviderResult<Vec<Block>> {
self.database.provider()?.block_range(range)
self.database.block_range(range)
}
}
@ -307,65 +310,65 @@ where
Tree: BlockchainTreeViewer + Send + Sync,
{
fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult<Option<TxNumber>> {
self.database.provider()?.transaction_id(tx_hash)
self.database.transaction_id(tx_hash)
}
fn transaction_by_id(&self, id: TxNumber) -> ProviderResult<Option<TransactionSigned>> {
self.database.provider()?.transaction_by_id(id)
self.database.transaction_by_id(id)
}
fn transaction_by_id_no_hash(
&self,
id: TxNumber,
) -> ProviderResult<Option<TransactionSignedNoHash>> {
self.database.provider()?.transaction_by_id_no_hash(id)
self.database.transaction_by_id_no_hash(id)
}
fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult<Option<TransactionSigned>> {
self.database.provider()?.transaction_by_hash(hash)
self.database.transaction_by_hash(hash)
}
fn transaction_by_hash_with_meta(
&self,
tx_hash: TxHash,
) -> ProviderResult<Option<(TransactionSigned, TransactionMeta)>> {
self.database.provider()?.transaction_by_hash_with_meta(tx_hash)
self.database.transaction_by_hash_with_meta(tx_hash)
}
fn transaction_block(&self, id: TxNumber) -> ProviderResult<Option<BlockNumber>> {
self.database.provider()?.transaction_block(id)
self.database.transaction_block(id)
}
fn transactions_by_block(
&self,
id: BlockHashOrNumber,
) -> ProviderResult<Option<Vec<TransactionSigned>>> {
self.database.provider()?.transactions_by_block(id)
self.database.transactions_by_block(id)
}
fn transactions_by_block_range(
&self,
range: impl RangeBounds<BlockNumber>,
) -> ProviderResult<Vec<Vec<TransactionSigned>>> {
self.database.provider()?.transactions_by_block_range(range)
self.database.transactions_by_block_range(range)
}
fn transactions_by_tx_range(
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<TransactionSignedNoHash>> {
self.database.provider()?.transactions_by_tx_range(range)
self.database.transactions_by_tx_range(range)
}
fn senders_by_tx_range(
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Address>> {
self.database.provider()?.senders_by_tx_range(range)
self.database.senders_by_tx_range(range)
}
fn transaction_sender(&self, id: TxNumber) -> ProviderResult<Option<Address>> {
self.database.provider()?.transaction_sender(id)
self.database.transaction_sender(id)
}
}
@ -375,22 +378,22 @@ where
Tree: Send + Sync,
{
fn receipt(&self, id: TxNumber) -> ProviderResult<Option<Receipt>> {
self.database.provider()?.receipt(id)
self.database.receipt(id)
}
fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult<Option<Receipt>> {
self.database.provider()?.receipt_by_hash(hash)
self.database.receipt_by_hash(hash)
}
fn receipts_by_block(&self, block: BlockHashOrNumber) -> ProviderResult<Option<Vec<Receipt>>> {
self.database.provider()?.receipts_by_block(block)
self.database.receipts_by_block(block)
}
fn receipts_by_tx_range(
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Receipt>> {
self.database.provider()?.receipts_by_tx_range(range)
self.database.receipts_by_tx_range(range)
}
}
impl<DB, Tree> ReceiptProviderIdExt for BlockchainProvider<DB, Tree>
@ -431,11 +434,11 @@ where
id: BlockHashOrNumber,
timestamp: u64,
) -> ProviderResult<Option<Withdrawals>> {
self.database.provider()?.withdrawals_by_block(id, timestamp)
self.database.withdrawals_by_block(id, timestamp)
}
fn latest_withdrawal(&self) -> ProviderResult<Option<Withdrawal>> {
self.database.provider()?.latest_withdrawal()
self.database.latest_withdrawal()
}
}

685
crates/storage/provider/src/providers/snapshot/manager.rs
View File

@ -1,685 +0,0 @@
use super::{LoadedJar, SnapshotJarProvider};
use crate::{
to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, HeaderProvider,
ReceiptProvider, TransactionVariant, TransactionsProvider, TransactionsProviderExt,
WithdrawalsProvider,
};
use dashmap::DashMap;
use parking_lot::RwLock;
use reth_db::{
codecs::CompactU256,
models::StoredBlockBodyIndices,
snapshot::{iter_snapshots, HeaderMask, ReceiptMask, SnapshotCursor, TransactionMask},
};
use reth_interfaces::provider::{ProviderError, ProviderResult};
use reth_nippy_jar::NippyJar;
use reth_primitives::{
snapshot::HighestSnapshots, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber,
BlockWithSenders, ChainInfo, Header, Receipt, SealedBlock, SealedBlockWithSenders,
SealedHeader, SnapshotSegment, TransactionMeta, TransactionSigned, TransactionSignedNoHash,
TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256,
};
use std::{
collections::{hash_map::Entry, BTreeMap, HashMap},
ops::{Range, RangeBounds, RangeInclusive},
path::{Path, PathBuf},
};
use tokio::sync::watch;
/// Alias type for a map that can be queried for transaction/block ranges from a block/transaction
/// segment respectively. It uses `BlockNumber` to represent the block end of a snapshot range or
/// `TxNumber` to represent the transaction end of a snapshot range.
///
/// Can be in one of the two formats:
/// - `HashMap<SnapshotSegment, BTreeMap<BlockNumber, RangeInclusive<TxNumber>>>`
/// - `HashMap<SnapshotSegment, BTreeMap<TxNumber, RangeInclusive<BlockNumber>>>`
type SegmentRanges = HashMap<SnapshotSegment, BTreeMap<u64, RangeInclusive<u64>>>;
/// [`SnapshotProvider`] manages all existing [`SnapshotJarProvider`].
#[derive(Debug, Default)]
pub struct SnapshotProvider {
/// Maintains a map which allows for concurrent access to different `NippyJars`, over different
/// segments and ranges.
map: DashMap<(BlockNumber, SnapshotSegment), LoadedJar>,
/// Available snapshot transaction ranges on disk indexed by max blocks.
snapshots_block_index: RwLock<SegmentRanges>,
/// Available snapshot block ranges on disk indexed by max transactions.
snapshots_tx_index: RwLock<SegmentRanges>,
/// Tracks the highest snapshot of every segment.
highest_tracker: Option<watch::Receiver<Option<HighestSnapshots>>>,
/// Directory where snapshots are located
path: PathBuf,
/// Whether [`SnapshotJarProvider`] loads filters into memory. If not, `by_hash` queries won't
/// be able to be queried directly.
load_filters: bool,
}
impl SnapshotProvider {
/// Creates a new [`SnapshotProvider`].
pub fn new(path: impl AsRef<Path>) -> ProviderResult<Self> {
let provider = Self {
map: Default::default(),
snapshots_block_index: Default::default(),
snapshots_tx_index: Default::default(),
highest_tracker: None,
path: path.as_ref().to_path_buf(),
load_filters: false,
};
provider.update_index()?;
Ok(provider)
}
/// Loads filters into memory when creating a [`SnapshotJarProvider`].
pub fn with_filters(mut self) -> Self {
self.load_filters = true;
self
}
/// Adds a highest snapshot tracker to the provider
pub fn with_highest_tracker(
mut self,
highest_tracker: Option<watch::Receiver<Option<HighestSnapshots>>>,
) -> Self {
self.highest_tracker = highest_tracker;
self
}
/// Gets the [`SnapshotJarProvider`] of the requested segment and block.
pub fn get_segment_provider_from_block(
&self,
segment: SnapshotSegment,
block: BlockNumber,
path: Option<&Path>,
) -> ProviderResult<SnapshotJarProvider<'_>> {
self.get_segment_provider(
segment,
|| self.get_segment_ranges_from_block(segment, block),
path,
)?
.ok_or_else(|| ProviderError::MissingSnapshotBlock(segment, block))
}
/// Gets the [`SnapshotJarProvider`] of the requested segment and transaction.
pub fn get_segment_provider_from_transaction(
&self,
segment: SnapshotSegment,
tx: TxNumber,
path: Option<&Path>,
) -> ProviderResult<SnapshotJarProvider<'_>> {
self.get_segment_provider(
segment,
|| self.get_segment_ranges_from_transaction(segment, tx),
path,
)?
.ok_or_else(|| ProviderError::MissingSnapshotTx(segment, tx))
}
/// Gets the [`SnapshotJarProvider`] of the requested segment and block or transaction.
pub fn get_segment_provider(
&self,
segment: SnapshotSegment,
fn_ranges: impl Fn() -> Option<(RangeInclusive<BlockNumber>, RangeInclusive<TxNumber>)>,
path: Option<&Path>,
) -> ProviderResult<Option<SnapshotJarProvider<'_>>> {
// If we have a path, then get the block range and transaction range from its name.
// Otherwise, check `self.available_snapshots`
let snapshot_ranges = match path {
Some(path) => {
SnapshotSegment::parse_filename(path.file_name().ok_or_else(|| {
ProviderError::MissingSnapshotPath(segment, path.to_path_buf())
})?)
.and_then(|(parsed_segment, block_range, tx_range)| {
if parsed_segment == segment {
return Some((block_range, tx_range))
}
None
})
}
None => fn_ranges(),
};
// Return cached `LoadedJar` or insert it for the first time, and then, return it.
if let Some((block_range, tx_range)) = snapshot_ranges {
return Ok(Some(self.get_or_create_jar_provider(segment, &block_range, &tx_range)?))
}
Ok(None)
}
/// Given a segment, block range and transaction range it returns a cached
/// [`SnapshotJarProvider`]. TODO: we should check the size and pop N if there's too many.
fn get_or_create_jar_provider(
&self,
segment: SnapshotSegment,
block_range: &RangeInclusive<u64>,
tx_range: &RangeInclusive<u64>,
) -> ProviderResult<SnapshotJarProvider<'_>> {
let key = (*block_range.end(), segment);
let entry = match self.map.entry(key) {
dashmap::mapref::entry::Entry::Occupied(entry) => entry.into_ref(),
dashmap::mapref::entry::Entry::Vacant(entry) => {
let path = self.path.join(segment.filename(block_range, tx_range));
let mut jar = NippyJar::load(&path)?;
if self.load_filters {
jar.load_filters()?;
}
let loaded_jar = LoadedJar::new(jar)?;
entry.insert(loaded_jar)
}
};
Ok(entry.downgrade().into())
}
/// Gets a snapshot segment's block range and transaction range from the provider inner block
/// index.
fn get_segment_ranges_from_block(
&self,
segment: SnapshotSegment,
block: u64,
) -> Option<(RangeInclusive<BlockNumber>, RangeInclusive<TxNumber>)> {
let snapshots = self.snapshots_block_index.read();
let segment_snapshots = snapshots.get(&segment)?;
// It's more probable that the request comes from a newer block height, so we iterate
// the snapshots in reverse.
let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable();
while let Some((block_end, tx_range)) = snapshots_rev_iter.next() {
if block > *block_end {
// request block is higher than highest snapshot block
return None
}
// `unwrap_or(0) is safe here as it sets block_start to 0 if the iterator is empty,
// indicating the lowest height snapshot has been reached.
let block_start =
snapshots_rev_iter.peek().map(|(block_end, _)| *block_end + 1).unwrap_or(0);
if block_start <= block {
return Some((block_start..=*block_end, tx_range.clone()))
}
}
None
}
/// Gets a snapshot segment's block range and transaction range from the provider inner
/// transaction index.
fn get_segment_ranges_from_transaction(
&self,
segment: SnapshotSegment,
tx: u64,
) -> Option<(RangeInclusive<BlockNumber>, RangeInclusive<TxNumber>)> {
let snapshots = self.snapshots_tx_index.read();
let segment_snapshots = snapshots.get(&segment)?;
// It's more probable that the request comes from a newer tx height, so we iterate
// the snapshots in reverse.
let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable();
while let Some((tx_end, block_range)) = snapshots_rev_iter.next() {
if tx > *tx_end {
// request tx is higher than highest snapshot tx
return None
}
let tx_start = snapshots_rev_iter.peek().map(|(tx_end, _)| *tx_end + 1).unwrap_or(0);
if tx_start <= tx {
return Some((block_range.clone(), tx_start..=*tx_end))
}
}
None
}
/// Updates the inner transaction and block index
pub fn update_index(&self) -> ProviderResult<()> {
let mut block_index = self.snapshots_block_index.write();
let mut tx_index = self.snapshots_tx_index.write();
for (segment, ranges) in iter_snapshots(&self.path)? {
for (block_range, tx_range) in ranges {
let block_end = *block_range.end();
let tx_end = *tx_range.end();
match tx_index.entry(segment) {
Entry::Occupied(mut index) => {
index.get_mut().insert(tx_end, block_range);
}
Entry::Vacant(index) => {
index.insert(BTreeMap::from([(tx_end, block_range)]));
}
};
match block_index.entry(segment) {
Entry::Occupied(mut index) => {
index.get_mut().insert(block_end, tx_range);
}
Entry::Vacant(index) => {
index.insert(BTreeMap::from([(block_end, tx_range)]));
}
};
}
}
Ok(())
}
/// Gets the highest snapshot block if it exists for a snapshot segment.
pub fn get_highest_snapshot_block(&self, segment: SnapshotSegment) -> Option<BlockNumber> {
self.snapshots_block_index
.read()
.get(&segment)
.and_then(|index| index.last_key_value().map(|(last_block, _)| *last_block))
}
/// Gets the highest snapshotted transaction.
pub fn get_highest_snapshot_tx(&self, segment: SnapshotSegment) -> Option<TxNumber> {
self.snapshots_tx_index
.read()
.get(&segment)
.and_then(|index| index.last_key_value().map(|(last_tx, _)| *last_tx))
}
/// Iterates through segment snapshots in reverse order, executing a function until it returns
/// some object. Useful for finding objects by [`TxHash`] or [`BlockHash`].
pub fn find_snapshot<T>(
&self,
segment: SnapshotSegment,
func: impl Fn(SnapshotJarProvider<'_>) -> ProviderResult<Option<T>>,
) -> ProviderResult<Option<T>> {
let snapshots = self.snapshots_block_index.read();
if let Some(segment_snapshots) = snapshots.get(&segment) {
// It's more probable that the request comes from a newer block height, so we iterate
// the snapshots in reverse.
let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable();
while let Some((block_end, tx_range)) = snapshots_rev_iter.next() {
// `unwrap_or(0) is safe here as it sets block_start to 0 if the iterator
// is empty, indicating the lowest height snapshot has been reached.
let block_start =
snapshots_rev_iter.peek().map(|(block_end, _)| *block_end + 1).unwrap_or(0);
if let Some(res) = func(self.get_or_create_jar_provider(
segment,
&(block_start..=*block_end),
tx_range,
)?)? {
return Ok(Some(res))
}
}
}
Ok(None)
}
/// Fetches data within a specified range across multiple snapshot files.
///
/// This function iteratively retrieves data using `get_fn` for each item in the given range.
/// It continues fetching until the end of the range is reached or the provided `predicate`
/// returns false.
pub fn fetch_range<T, F, P>(
&self,
segment: SnapshotSegment,
range: Range<u64>,
get_fn: F,
mut predicate: P,
) -> ProviderResult<Vec<T>>
where
F: Fn(&mut SnapshotCursor<'_>, u64) -> ProviderResult<Option<T>>,
P: FnMut(&T) -> bool,
{
let get_provider = |start: u64| match segment {
SnapshotSegment::Headers => self.get_segment_provider_from_block(segment, start, None),
SnapshotSegment::Transactions | SnapshotSegment::Receipts => {
self.get_segment_provider_from_transaction(segment, start, None)
}
};
let mut result = Vec::with_capacity((range.end - range.start).min(100) as usize);
let mut provider = get_provider(range.start)?;
let mut cursor = provider.cursor()?;
// advances number in range
'outer: for number in range {
// advances snapshot files if `get_fn` returns None
'inner: loop {
match get_fn(&mut cursor, number)? {
Some(res) => {
if !predicate(&res) {
break 'outer
}
result.push(res);
break 'inner
}
None => {
provider = get_provider(number)?;
cursor = provider.cursor()?;
}
}
}
}
Ok(result)
}
}
impl HeaderProvider for SnapshotProvider {
fn header(&self, block_hash: &BlockHash) -> ProviderResult<Option<Header>> {
self.find_snapshot(SnapshotSegment::Headers, |jar_provider| {
Ok(jar_provider
.cursor()?
.get_two::<HeaderMask<Header, BlockHash>>(block_hash.into())?
.and_then(|(header, hash)| {
if &hash == block_hash {
return Some(header)
}
None
}))
})
}
fn header_by_number(&self, num: BlockNumber) -> ProviderResult<Option<Header>> {
self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)?
.header_by_number(num)
}
fn header_td(&self, block_hash: &BlockHash) -> ProviderResult<Option<U256>> {
self.find_snapshot(SnapshotSegment::Headers, |jar_provider| {
Ok(jar_provider
.cursor()?
.get_two::<HeaderMask<CompactU256, BlockHash>>(block_hash.into())?
.and_then(|(td, hash)| (&hash == block_hash).then_some(td.0)))
})
}
fn header_td_by_number(&self, num: BlockNumber) -> ProviderResult<Option<U256>> {
self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)?
.header_td_by_number(num)
}
fn headers_range(&self, range: impl RangeBounds<BlockNumber>) -> ProviderResult<Vec<Header>> {
self.fetch_range(
SnapshotSegment::Headers,
to_range(range),
|cursor, number| cursor.get_one::<HeaderMask<Header>>(number.into()),
|_| true,
)
}
fn sealed_header(&self, num: BlockNumber) -> ProviderResult<Option<SealedHeader>> {
self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)?
.sealed_header(num)
}
fn sealed_headers_while(
&self,
range: impl RangeBounds<BlockNumber>,
predicate: impl FnMut(&SealedHeader) -> bool,
) -> ProviderResult<Vec<SealedHeader>> {
self.fetch_range(
SnapshotSegment::Headers,
to_range(range),
|cursor, number| {
Ok(cursor
.get_two::<HeaderMask<Header, BlockHash>>(number.into())?
.map(|(header, hash)| header.seal(hash)))
},
predicate,
)
}
}
impl BlockHashReader for SnapshotProvider {
fn block_hash(&self, num: u64) -> ProviderResult<Option<B256>> {
self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)?.block_hash(num)
}
fn canonical_hashes_range(
&self,
start: BlockNumber,
end: BlockNumber,
) -> ProviderResult<Vec<B256>> {
self.fetch_range(
SnapshotSegment::Headers,
start..end,
|cursor, number| cursor.get_one::<HeaderMask<BlockHash>>(number.into()),
|_| true,
)
}
}
impl ReceiptProvider for SnapshotProvider {
fn receipt(&self, num: TxNumber) -> ProviderResult<Option<Receipt>> {
self.get_segment_provider_from_transaction(SnapshotSegment::Receipts, num, None)?
.receipt(num)
}
fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult<Option<Receipt>> {
if let Some(num) = self.transaction_id(hash)? {
return self.receipt(num)
}
Ok(None)
}
fn receipts_by_block(&self, _block: BlockHashOrNumber) -> ProviderResult<Option<Vec<Receipt>>> {
unreachable!()
}
fn receipts_by_tx_range(
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Receipt>> {
self.fetch_range(
SnapshotSegment::Receipts,
to_range(range),
|cursor, number| cursor.get_one::<ReceiptMask<Receipt>>(number.into()),
|_| true,
)
}
}
impl TransactionsProviderExt for SnapshotProvider {
fn transaction_hashes_by_range(
&self,
tx_range: Range<TxNumber>,
) -> ProviderResult<Vec<(TxHash, TxNumber)>> {
self.fetch_range(
SnapshotSegment::Transactions,
tx_range,
|cursor, number| {
let tx =
cursor.get_one::<TransactionMask<TransactionSignedNoHash>>(number.into())?;
Ok(tx.map(|tx| (tx.hash(), cursor.number())))
},
|_| true,
)
}
}
impl TransactionsProvider for SnapshotProvider {
fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult<Option<TxNumber>> {
self.find_snapshot(SnapshotSegment::Transactions, |jar_provider| {
let mut cursor = jar_provider.cursor()?;
if cursor
.get_one::<TransactionMask<TransactionSignedNoHash>>((&tx_hash).into())?
.and_then(|tx| (tx.hash() == tx_hash).then_some(tx))
.is_some()
{
Ok(Some(cursor.number()))
} else {
Ok(None)
}
})
}
fn transaction_by_id(&self, num: TxNumber) -> ProviderResult<Option<TransactionSigned>> {
self.get_segment_provider_from_transaction(SnapshotSegment::Transactions, num, None)?
.transaction_by_id(num)
}
fn transaction_by_id_no_hash(
&self,
num: TxNumber,
) -> ProviderResult<Option<TransactionSignedNoHash>> {
self.get_segment_provider_from_transaction(SnapshotSegment::Transactions, num, None)?
.transaction_by_id_no_hash(num)
}
fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult<Option<TransactionSigned>> {
self.find_snapshot(SnapshotSegment::Transactions, |jar_provider| {
Ok(jar_provider
.cursor()?
.get_one::<TransactionMask<TransactionSignedNoHash>>((&hash).into())?
.map(|tx| tx.with_hash())
.and_then(|tx| (tx.hash_ref() == &hash).then_some(tx)))
})
}
fn transaction_by_hash_with_meta(
&self,
_hash: TxHash,
) -> ProviderResult<Option<(TransactionSigned, TransactionMeta)>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn transaction_block(&self, _id: TxNumber) -> ProviderResult<Option<BlockNumber>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn transactions_by_block(
&self,
_block_id: BlockHashOrNumber,
) -> ProviderResult<Option<Vec<TransactionSigned>>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn transactions_by_block_range(
&self,
_range: impl RangeBounds<BlockNumber>,
) -> ProviderResult<Vec<Vec<TransactionSigned>>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn senders_by_tx_range(
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Address>> {
let txes = self.transactions_by_tx_range(range)?;
TransactionSignedNoHash::recover_signers(&txes, txes.len())
.ok_or(ProviderError::SenderRecoveryError)
}
fn transactions_by_tx_range(
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<reth_primitives::TransactionSignedNoHash>> {
self.fetch_range(
SnapshotSegment::Transactions,
to_range(range),
|cursor, number| {
cursor.get_one::<TransactionMask<TransactionSignedNoHash>>(number.into())
},
|_| true,
)
}
fn transaction_sender(&self, id: TxNumber) -> ProviderResult<Option<Address>> {
Ok(self.transaction_by_id_no_hash(id)?.and_then(|tx| tx.recover_signer()))
}
}
/* Cannot be successfully implemented but must exist for trait requirements */
impl BlockNumReader for SnapshotProvider {
fn chain_info(&self) -> ProviderResult<ChainInfo> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn best_block_number(&self) -> ProviderResult<BlockNumber> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn last_block_number(&self) -> ProviderResult<BlockNumber> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn block_number(&self, _hash: B256) -> ProviderResult<Option<BlockNumber>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
}
impl BlockReader for SnapshotProvider {
fn find_block_by_hash(
&self,
_hash: B256,
_source: BlockSource,
) -> ProviderResult<Option<Block>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn block(&self, _id: BlockHashOrNumber) -> ProviderResult<Option<Block>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn pending_block(&self) -> ProviderResult<Option<SealedBlock>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn pending_block_with_senders(&self) -> ProviderResult<Option<SealedBlockWithSenders>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn pending_block_and_receipts(&self) -> ProviderResult<Option<(SealedBlock, Vec<Receipt>)>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn ommers(&self, _id: BlockHashOrNumber) -> ProviderResult<Option<Vec<Header>>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn block_body_indices(&self, _num: u64) -> ProviderResult<Option<StoredBlockBodyIndices>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn block_with_senders(
&self,
_id: BlockHashOrNumber,
_transaction_kind: TransactionVariant,
) -> ProviderResult<Option<BlockWithSenders>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn block_range(&self, _range: RangeInclusive<BlockNumber>) -> ProviderResult<Vec<Block>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
}
impl WithdrawalsProvider for SnapshotProvider {
fn withdrawals_by_block(
&self,
_id: BlockHashOrNumber,
_timestamp: u64,
) -> ProviderResult<Option<Withdrawals>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
fn latest_withdrawal(&self) -> ProviderResult<Option<Withdrawal>> {
// Required data not present in snapshots
Err(ProviderError::UnsupportedProvider)
}
}

289
crates/storage/provider/src/providers/state/historical.rs
View File

@ -1,6 +1,7 @@
use crate::{
providers::state::macros::delegate_provider_impls, AccountReader, BlockHashReader,
BundleStateWithReceipts, ProviderError, StateProvider, StateRootProvider,
providers::{state::macros::delegate_provider_impls, StaticFileProvider},
AccountReader, BlockHashReader, BundleStateWithReceipts, ProviderError, StateProvider,
StateRootProvider,
};
use reth_db::{
cursor::{DbCursorRO, DbDupCursorRO},
@ -13,9 +14,10 @@ use reth_db::{
use reth_interfaces::provider::ProviderResult;
use reth_primitives::{
constants::EPOCH_SLOTS, trie::AccountProof, Account, Address, BlockNumber, Bytecode,
StorageKey, StorageValue, B256,
StaticFileSegment, StorageKey, StorageValue, B256,
};
use reth_trie::{updates::TrieUpdates, HashedPostState};
use std::fmt::Debug;
/// State provider for a given block number which takes a tx reference.
///
@ -23,11 +25,11 @@ use reth_trie::{updates::TrieUpdates, HashedPostState};
/// It means that all changes made in the provided block number are not included.
///
/// Historical state provider reads the following tables:
/// - [tables::AccountHistory]
/// - [tables::AccountsHistory]
/// - [tables::Bytecodes]
/// - [tables::StorageHistory]
/// - [tables::AccountChangeSet]
/// - [tables::StorageChangeSet]
/// - [tables::StoragesHistory]
/// - [tables::AccountChangeSets]
/// - [tables::StorageChangeSets]
#[derive(Debug)]
pub struct HistoricalStateProviderRef<'b, TX: DbTx> {
/// Transaction
@ -36,6 +38,8 @@ pub struct HistoricalStateProviderRef<'b, TX: DbTx> {
block_number: BlockNumber,
/// Lowest blocks at which different parts of the state are available.
lowest_available_blocks: LowestAvailableBlocks,
/// Static File provider
static_file_provider: StaticFileProvider,
}
#[derive(Debug, Eq, PartialEq)]
@ -48,8 +52,12 @@ pub enum HistoryInfo {
impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> {
/// Create new StateProvider for historical block number
pub fn new(tx: &'b TX, block_number: BlockNumber) -> Self {
Self { tx, block_number, lowest_available_blocks: Default::default() }
pub fn new(
tx: &'b TX,
block_number: BlockNumber,
static_file_provider: StaticFileProvider,
) -> Self {
Self { tx, block_number, lowest_available_blocks: Default::default(), static_file_provider }
}
/// Create new StateProvider for historical block number and lowest block numbers at which
@ -58,11 +66,12 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> {
tx: &'b TX,
block_number: BlockNumber,
lowest_available_blocks: LowestAvailableBlocks,
static_file_provider: StaticFileProvider,
) -> Self {
Self { tx, block_number, lowest_available_blocks }
Self { tx, block_number, lowest_available_blocks, static_file_provider }
}
/// Lookup an account in the AccountHistory table
/// Lookup an account in the AccountsHistory table
pub fn account_history_lookup(&self, address: Address) -> ProviderResult<HistoryInfo> {
if !self.lowest_available_blocks.is_account_history_available(self.block_number) {
return Err(ProviderError::StateAtBlockPruned(self.block_number))
@ -70,14 +79,14 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> {
// history key to search IntegerList of block number changesets.
let history_key = ShardedKey::new(address, self.block_number);
self.history_info::<tables::AccountHistory, _>(
self.history_info::<tables::AccountsHistory, _>(
history_key,
|key| key.key == address,
self.lowest_available_blocks.account_history_block_number,
)
}
/// Lookup a storage key in the StorageHistory table
/// Lookup a storage key in the StoragesHistory table
pub fn storage_history_lookup(
&self,
address: Address,
@ -89,7 +98,7 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> {
// history key to search IntegerList of block number changesets.
let history_key = StorageShardedKey::new(address, storage_key, self.block_number);
self.history_info::<tables::StorageHistory, _>(
self.history_info::<tables::StoragesHistory, _>(
history_key,
|key| key.address == address && key.sharded_key.key == storage_key,
self.lowest_available_blocks.storage_history_block_number,
@ -104,10 +113,14 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> {
return Err(ProviderError::StateAtBlockPruned(self.block_number))
}
let (tip, _) = self
let tip = self
.tx
.cursor_read::<tables::CanonicalHeaders>()?
.last()?
.map(|(tip, _)| tip)
.or_else(|| {
self.static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers)
})
.ok_or(ProviderError::BestBlockNotFound)?;
if tip.saturating_sub(self.block_number) > EPOCH_SLOTS {
@ -136,10 +149,16 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> {
// index, the first chunk for the next key will be returned so we filter out chunks that
// have a different key.
if let Some(chunk) = cursor.seek(key)?.filter(|(key, _)| key_filter(key)).map(|x| x.1 .0) {
let chunk = chunk.enable_rank();
// Get the rank of the first entry before or equal to our block.
let mut rank = chunk.rank(self.block_number);
// Get the rank of the first entry after our block.
let rank = chunk.rank(self.block_number as usize);
// Adjust the rank, so that we have the rank of the first entry strictly before our
// block (not equal to it).
if rank.checked_sub(1).and_then(|rank| chunk.select(rank)) == Some(self.block_number) {
rank -= 1
};
let block_number = chunk.select(rank);
// If our block is before the first entry in the index chunk and this first entry
// doesn't equal to our block, it might be before the first write ever. To check, we
@ -148,20 +167,21 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> {
// short-circuit) and when it passes we save a full seek into the changeset/plain state
// table.
if rank == 0 &&
chunk.select(rank) as u64 != self.block_number &&
block_number != Some(self.block_number) &&
!cursor.prev()?.is_some_and(|(key, _)| key_filter(&key))
{
if lowest_available_block_number.is_some() {
if let (Some(_), Some(block_number)) = (lowest_available_block_number, block_number)
{
// The key may have been written, but due to pruning we may not have changesets
// and history, so we need to make a changeset lookup.
Ok(HistoryInfo::InChangeset(chunk.select(rank) as u64))
Ok(HistoryInfo::InChangeset(block_number))
} else {
// The key is written to, but only after our block.
Ok(HistoryInfo::NotYetWritten)
}
} else if rank < chunk.len() {
} else if let Some(block_number) = block_number {
// The chunk contains an entry for a write after our block, return it.
Ok(HistoryInfo::InChangeset(chunk.select(rank) as u64))
Ok(HistoryInfo::InChangeset(block_number))
} else {
// The chunk does not contain an entry for a write after our block. This can only
// happen if this is the last chunk and so we need to look in the plain state.
@ -185,7 +205,7 @@ impl<'b, TX: DbTx> AccountReader for HistoricalStateProviderRef<'b, TX> {
HistoryInfo::NotYetWritten => Ok(None),
HistoryInfo::InChangeset(changeset_block_number) => Ok(self
.tx
.cursor_dup_read::<tables::AccountChangeSet>()?
.cursor_dup_read::<tables::AccountChangeSets>()?
.seek_by_key_subkey(changeset_block_number, address)?
.filter(|acc| acc.address == address)
.ok_or(ProviderError::AccountChangesetNotFound {
@ -203,7 +223,12 @@ impl<'b, TX: DbTx> AccountReader for HistoricalStateProviderRef<'b, TX> {
impl<'b, TX: DbTx> BlockHashReader for HistoricalStateProviderRef<'b, TX> {
/// Get block hash by number.
fn block_hash(&self, number: u64) -> ProviderResult<Option<B256>> {
self.tx.get::<tables::CanonicalHeaders>(number).map_err(Into::into)
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
number,
|static_file| static_file.block_hash(number),
|| Ok(self.tx.get::<tables::CanonicalHeaders>(number)?),
)
}
fn canonical_hashes_range(
@ -211,16 +236,23 @@ impl<'b, TX: DbTx> BlockHashReader for HistoricalStateProviderRef<'b, TX> {
start: BlockNumber,
end: BlockNumber,
) -> ProviderResult<Vec<B256>> {
let range = start..end;
self.tx
.cursor_read::<tables::CanonicalHeaders>()
.map(|mut cursor| {
cursor
.walk_range(range)?
.map(|result| result.map(|(_, hash)| hash).map_err(Into::into))
.collect::<ProviderResult<Vec<_>>>()
})?
.map_err(Into::into)
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Headers,
start..end,
|static_file, range, _| static_file.canonical_hashes_range(range.start, range.end),
|range, _| {
self.tx
.cursor_read::<tables::CanonicalHeaders>()
.map(|mut cursor| {
cursor
.walk_range(range)?
.map(|result| result.map(|(_, hash)| hash).map_err(Into::into))
.collect::<ProviderResult<Vec<_>>>()
})?
.map_err(Into::into)
},
|_| true,
)
}
}
@ -254,7 +286,7 @@ impl<'b, TX: DbTx> StateProvider for HistoricalStateProviderRef<'b, TX> {
HistoryInfo::NotYetWritten => Ok(None),
HistoryInfo::InChangeset(changeset_block_number) => Ok(Some(
self.tx
.cursor_dup_read::<tables::StorageChangeSet>()?
.cursor_dup_read::<tables::StorageChangeSets>()?
.seek_by_key_subkey((changeset_block_number, address).into(), storage_key)?
.filter(|entry| entry.key == storage_key)
.ok_or_else(|| ProviderError::StorageChangesetNotFound {
@ -295,12 +327,18 @@ pub struct HistoricalStateProvider<TX: DbTx> {
block_number: BlockNumber,
/// Lowest blocks at which different parts of the state are available.
lowest_available_blocks: LowestAvailableBlocks,
/// Static File provider
static_file_provider: StaticFileProvider,
}
impl<TX: DbTx> HistoricalStateProvider<TX> {
/// Create new StateProvider for historical block number
pub fn new(tx: TX, block_number: BlockNumber) -> Self {
Self { tx, block_number, lowest_available_blocks: Default::default() }
pub fn new(
tx: TX,
block_number: BlockNumber,
static_file_provider: StaticFileProvider,
) -> Self {
Self { tx, block_number, lowest_available_blocks: Default::default(), static_file_provider }
}
/// Set the lowest block number at which the account history is available.
@ -328,6 +366,7 @@ impl<TX: DbTx> HistoricalStateProvider<TX> {
&self.tx,
self.block_number,
self.lowest_available_blocks,
self.static_file_provider.clone(),
)
}
}
@ -367,13 +406,12 @@ impl LowestAvailableBlocks {
mod tests {
use crate::{
providers::state::historical::{HistoryInfo, LowestAvailableBlocks},
test_utils::create_test_provider_factory,
AccountReader, HistoricalStateProvider, HistoricalStateProviderRef, StateProvider,
};
use reth_db::{
database::Database,
models::{storage_sharded_key::StorageShardedKey, AccountBeforeTx, ShardedKey},
tables,
test_utils::create_test_rw_db,
transaction::{DbTx, DbTxMut},
BlockNumberList,
};
@ -392,20 +430,21 @@ mod tests {
#[test]
fn history_provider_get_account() {
let db = create_test_rw_db();
let tx = db.tx_mut().unwrap();
let factory = create_test_provider_factory();
let tx = factory.provider_rw().unwrap().into_tx();
let static_file_provider = factory.static_file_provider();
tx.put::<tables::AccountHistory>(
tx.put::<tables::AccountsHistory>(
ShardedKey { key: ADDRESS, highest_block_number: 7 },
BlockNumberList::new([1, 3, 7]).unwrap(),
)
.unwrap();
tx.put::<tables::AccountHistory>(
tx.put::<tables::AccountsHistory>(
ShardedKey { key: ADDRESS, highest_block_number: u64::MAX },
BlockNumberList::new([10, 15]).unwrap(),
)
.unwrap();
tx.put::<tables::AccountHistory>(
tx.put::<tables::AccountsHistory>(
ShardedKey { key: HIGHER_ADDRESS, highest_block_number: u64::MAX },
BlockNumberList::new([4]).unwrap(),
)
@ -420,29 +459,29 @@ mod tests {
let higher_acc_plain = Account { nonce: 4, balance: U256::ZERO, bytecode_hash: None };
// setup
tx.put::<tables::AccountChangeSet>(1, AccountBeforeTx { address: ADDRESS, info: None })
tx.put::<tables::AccountChangeSets>(1, AccountBeforeTx { address: ADDRESS, info: None })
.unwrap();
tx.put::<tables::AccountChangeSet>(
tx.put::<tables::AccountChangeSets>(
3,
AccountBeforeTx { address: ADDRESS, info: Some(acc_at3) },
)
.unwrap();
tx.put::<tables::AccountChangeSet>(
tx.put::<tables::AccountChangeSets>(
4,
AccountBeforeTx { address: HIGHER_ADDRESS, info: None },
)
.unwrap();
tx.put::<tables::AccountChangeSet>(
tx.put::<tables::AccountChangeSets>(
7,
AccountBeforeTx { address: ADDRESS, info: Some(acc_at7) },
)
.unwrap();
tx.put::<tables::AccountChangeSet>(
tx.put::<tables::AccountChangeSets>(
10,
AccountBeforeTx { address: ADDRESS, info: Some(acc_at10) },
)
.unwrap();
tx.put::<tables::AccountChangeSet>(
tx.put::<tables::AccountChangeSets>(
15,
AccountBeforeTx { address: ADDRESS, info: Some(acc_at15) },
)
@ -453,56 +492,75 @@ mod tests {
tx.put::<tables::PlainAccountState>(HIGHER_ADDRESS, higher_acc_plain).unwrap();
tx.commit().unwrap();
let tx = db.tx().unwrap();
let tx = factory.provider().unwrap().into_tx();
// run
assert_eq!(HistoricalStateProviderRef::new(&tx, 1).basic_account(ADDRESS), Ok(None));
assert_eq!(
HistoricalStateProviderRef::new(&tx, 2).basic_account(ADDRESS),
HistoricalStateProviderRef::new(&tx, 1, static_file_provider.clone())
.basic_account(ADDRESS)
.clone(),
Ok(None)
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 2, static_file_provider.clone())
.basic_account(ADDRESS),
Ok(Some(acc_at3))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 3).basic_account(ADDRESS),
HistoricalStateProviderRef::new(&tx, 3, static_file_provider.clone())
.basic_account(ADDRESS),
Ok(Some(acc_at3))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 4).basic_account(ADDRESS),
HistoricalStateProviderRef::new(&tx, 4, static_file_provider.clone())
.basic_account(ADDRESS),
Ok(Some(acc_at7))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 7).basic_account(ADDRESS),
HistoricalStateProviderRef::new(&tx, 7, static_file_provider.clone())
.basic_account(ADDRESS),
Ok(Some(acc_at7))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 9).basic_account(ADDRESS),
HistoricalStateProviderRef::new(&tx, 9, static_file_provider.clone())
.basic_account(ADDRESS),
Ok(Some(acc_at10))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 10).basic_account(ADDRESS),
HistoricalStateProviderRef::new(&tx, 10, static_file_provider.clone())
.basic_account(ADDRESS),
Ok(Some(acc_at10))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 11).basic_account(ADDRESS),
HistoricalStateProviderRef::new(&tx, 11, static_file_provider.clone())
.basic_account(ADDRESS),
Ok(Some(acc_at15))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 16).basic_account(ADDRESS),
HistoricalStateProviderRef::new(&tx, 16, static_file_provider.clone())
.basic_account(ADDRESS),
Ok(Some(acc_plain))
);
assert_eq!(HistoricalStateProviderRef::new(&tx, 1).basic_account(HIGHER_ADDRESS), Ok(None));
assert_eq!(
HistoricalStateProviderRef::new(&tx, 1000).basic_account(HIGHER_ADDRESS),
HistoricalStateProviderRef::new(&tx, 1, static_file_provider.clone())
.basic_account(HIGHER_ADDRESS),
Ok(None)
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 1000, static_file_provider.clone())
.basic_account(HIGHER_ADDRESS),
Ok(Some(higher_acc_plain))
);
}
#[test]
fn history_provider_get_storage() {
let db = create_test_rw_db();
let tx = db.tx_mut().unwrap();
let factory = create_test_provider_factory();
let tx = factory.provider_rw().unwrap().into_tx();
let static_file_provider = factory.static_file_provider();
tx.put::<tables::StorageHistory>(
tx.put::<tables::StoragesHistory>(
StorageShardedKey {
address: ADDRESS,
sharded_key: ShardedKey { key: STORAGE, highest_block_number: 7 },
@ -510,7 +568,7 @@ mod tests {
BlockNumberList::new([3, 7]).unwrap(),
)
.unwrap();
tx.put::<tables::StorageHistory>(
tx.put::<tables::StoragesHistory>(
StorageShardedKey {
address: ADDRESS,
sharded_key: ShardedKey { key: STORAGE, highest_block_number: u64::MAX },
@ -518,7 +576,7 @@ mod tests {
BlockNumberList::new([10, 15]).unwrap(),
)
.unwrap();
tx.put::<tables::StorageHistory>(
tx.put::<tables::StoragesHistory>(
StorageShardedKey {
address: HIGHER_ADDRESS,
sharded_key: ShardedKey { key: STORAGE, highest_block_number: u64::MAX },
@ -536,63 +594,77 @@ mod tests {
let entry_at3 = StorageEntry { key: STORAGE, value: U256::from(0) };
// setup
tx.put::<tables::StorageChangeSet>((3, ADDRESS).into(), entry_at3).unwrap();
tx.put::<tables::StorageChangeSet>((4, HIGHER_ADDRESS).into(), higher_entry_at4).unwrap();
tx.put::<tables::StorageChangeSet>((7, ADDRESS).into(), entry_at7).unwrap();
tx.put::<tables::StorageChangeSet>((10, ADDRESS).into(), entry_at10).unwrap();
tx.put::<tables::StorageChangeSet>((15, ADDRESS).into(), entry_at15).unwrap();
tx.put::<tables::StorageChangeSets>((3, ADDRESS).into(), entry_at3).unwrap();
tx.put::<tables::StorageChangeSets>((4, HIGHER_ADDRESS).into(), higher_entry_at4).unwrap();
tx.put::<tables::StorageChangeSets>((7, ADDRESS).into(), entry_at7).unwrap();
tx.put::<tables::StorageChangeSets>((10, ADDRESS).into(), entry_at10).unwrap();
tx.put::<tables::StorageChangeSets>((15, ADDRESS).into(), entry_at15).unwrap();
// setup plain state
tx.put::<tables::PlainStorageState>(ADDRESS, entry_plain).unwrap();
tx.put::<tables::PlainStorageState>(HIGHER_ADDRESS, higher_entry_plain).unwrap();
tx.commit().unwrap();
let tx = db.tx().unwrap();
let tx = factory.provider().unwrap().into_tx();
// run
assert_eq!(HistoricalStateProviderRef::new(&tx, 0).storage(ADDRESS, STORAGE), Ok(None));
assert_eq!(
HistoricalStateProviderRef::new(&tx, 3).storage(ADDRESS, STORAGE),
Ok(Some(U256::ZERO))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 4).storage(ADDRESS, STORAGE),
Ok(Some(entry_at7.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 7).storage(ADDRESS, STORAGE),
Ok(Some(entry_at7.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 9).storage(ADDRESS, STORAGE),
Ok(Some(entry_at10.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 10).storage(ADDRESS, STORAGE),
Ok(Some(entry_at10.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 11).storage(ADDRESS, STORAGE),
Ok(Some(entry_at15.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 16).storage(ADDRESS, STORAGE),
Ok(Some(entry_plain.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 1).storage(HIGHER_ADDRESS, STORAGE),
HistoricalStateProviderRef::new(&tx, 0, static_file_provider.clone())
.storage(ADDRESS, STORAGE),
Ok(None)
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 1000).storage(HIGHER_ADDRESS, STORAGE),
HistoricalStateProviderRef::new(&tx, 3, static_file_provider.clone())
.storage(ADDRESS, STORAGE),
Ok(Some(U256::ZERO))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 4, static_file_provider.clone())
.storage(ADDRESS, STORAGE),
Ok(Some(entry_at7.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 7, static_file_provider.clone())
.storage(ADDRESS, STORAGE),
Ok(Some(entry_at7.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 9, static_file_provider.clone())
.storage(ADDRESS, STORAGE),
Ok(Some(entry_at10.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 10, static_file_provider.clone())
.storage(ADDRESS, STORAGE),
Ok(Some(entry_at10.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 11, static_file_provider.clone())
.storage(ADDRESS, STORAGE),
Ok(Some(entry_at15.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 16, static_file_provider.clone())
.storage(ADDRESS, STORAGE),
Ok(Some(entry_plain.value))
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 1, static_file_provider.clone())
.storage(HIGHER_ADDRESS, STORAGE),
Ok(None)
);
assert_eq!(
HistoricalStateProviderRef::new(&tx, 1000, static_file_provider)
.storage(HIGHER_ADDRESS, STORAGE),
Ok(Some(higher_entry_plain.value))
);
}
#[test]
fn history_provider_unavailable() {
let db = create_test_rw_db();
let tx = db.tx().unwrap();
let factory = create_test_provider_factory();
let tx = factory.provider_rw().unwrap().into_tx();
let static_file_provider = factory.static_file_provider();
// provider block_number < lowest available block number,
// i.e. state at provider block is pruned
@ -603,6 +675,7 @@ mod tests {
account_history_block_number: Some(3),
storage_history_block_number: Some(3),
},
static_file_provider.clone(),
);
assert_eq!(
provider.account_history_lookup(ADDRESS),
@ -622,6 +695,7 @@ mod tests {
account_history_block_number: Some(2),
storage_history_block_number: Some(2),
},
static_file_provider.clone(),
);
assert_eq!(provider.account_history_lookup(ADDRESS), Ok(HistoryInfo::MaybeInPlainState));
assert_eq!(
@ -638,6 +712,7 @@ mod tests {
account_history_block_number: Some(1),
storage_history_block_number: Some(1),
},
static_file_provider.clone(),
);
assert_eq!(provider.account_history_lookup(ADDRESS), Ok(HistoryInfo::MaybeInPlainState));
assert_eq!(

55
crates/storage/provider/src/providers/state/latest.rs
View File

@ -1,6 +1,6 @@
use crate::{
providers::state::macros::delegate_provider_impls, AccountReader, BlockHashReader,
BundleStateWithReceipts, StateProvider, StateRootProvider,
providers::{state::macros::delegate_provider_impls, StaticFileProvider},
AccountReader, BlockHashReader, BundleStateWithReceipts, StateProvider, StateRootProvider,
};
use reth_db::{
cursor::{DbCursorRO, DbDupCursorRO},
@ -9,7 +9,8 @@ use reth_db::{
};
use reth_interfaces::provider::{ProviderError, ProviderResult};
use reth_primitives::{
trie::AccountProof, Account, Address, BlockNumber, Bytecode, StorageKey, StorageValue, B256,
trie::AccountProof, Account, Address, BlockNumber, Bytecode, StaticFileSegment, StorageKey,
StorageValue, B256,
};
use reth_trie::{proof::Proof, updates::TrieUpdates};
@ -18,12 +19,14 @@ use reth_trie::{proof::Proof, updates::TrieUpdates};
pub struct LatestStateProviderRef<'b, TX: DbTx> {
/// database transaction
db: &'b TX,
/// Static File provider
static_file_provider: StaticFileProvider,
}
impl<'b, TX: DbTx> LatestStateProviderRef<'b, TX> {
/// Create new state provider
pub fn new(db: &'b TX) -> Self {
Self { db }
pub fn new(db: &'b TX, static_file_provider: StaticFileProvider) -> Self {
Self { db, static_file_provider }
}
}
@ -37,7 +40,12 @@ impl<'b, TX: DbTx> AccountReader for LatestStateProviderRef<'b, TX> {
impl<'b, TX: DbTx> BlockHashReader for LatestStateProviderRef<'b, TX> {
/// Get block hash by number.
fn block_hash(&self, number: u64) -> ProviderResult<Option<B256>> {
self.db.get::<tables::CanonicalHeaders>(number).map_err(Into::into)
self.static_file_provider.get_with_static_file_or_database(
StaticFileSegment::Headers,
number,
|static_file| static_file.block_hash(number),
|| Ok(self.db.get::<tables::CanonicalHeaders>(number)?),
)
}
fn canonical_hashes_range(
@ -45,16 +53,23 @@ impl<'b, TX: DbTx> BlockHashReader for LatestStateProviderRef<'b, TX> {
start: BlockNumber,
end: BlockNumber,
) -> ProviderResult<Vec<B256>> {
let range = start..end;
self.db
.cursor_read::<tables::CanonicalHeaders>()
.map(|mut cursor| {
cursor
.walk_range(range)?
.map(|result| result.map(|(_, hash)| hash).map_err(Into::into))
.collect::<ProviderResult<Vec<_>>>()
})?
.map_err(Into::into)
self.static_file_provider.get_range_with_static_file_or_database(
StaticFileSegment::Headers,
start..end,
|static_file, range, _| static_file.canonical_hashes_range(range.start, range.end),
|range, _| {
self.db
.cursor_read::<tables::CanonicalHeaders>()
.map(|mut cursor| {
cursor
.walk_range(range)?
.map(|result| result.map(|(_, hash)| hash).map_err(Into::into))
.collect::<ProviderResult<Vec<_>>>()
})?
.map_err(Into::into)
},
|_| true,
)
}
}
@ -110,18 +125,20 @@ impl<'b, TX: DbTx> StateProvider for LatestStateProviderRef<'b, TX> {
pub struct LatestStateProvider<TX: DbTx> {
/// database transaction
db: TX,
/// Static File provider
static_file_provider: StaticFileProvider,
}
impl<TX: DbTx> LatestStateProvider<TX> {
/// Create new state provider
pub fn new(db: TX) -> Self {
Self { db }
pub fn new(db: TX, static_file_provider: StaticFileProvider) -> Self {
Self { db, static_file_provider }
}
/// Returns a new provider that takes the `TX` as reference
#[inline(always)]
fn as_ref(&self) -> LatestStateProviderRef<'_, TX> {
LatestStateProviderRef::new(&self.db)
LatestStateProviderRef::new(&self.db, self.static_file_provider.clone())
}
}

79
crates/storage/provider/src/providers/snapshot/jar.rs → crates/storage/provider/src/providers/static_file/jar.rs
View File

@ -1,58 +1,81 @@
use super::LoadedJarRef;
use super::{
metrics::{StaticFileProviderMetrics, StaticFileProviderOperation},
LoadedJarRef,
};
use crate::{
to_range, BlockHashReader, BlockNumReader, HeaderProvider, ReceiptProvider,
TransactionsProvider,
};
use reth_db::{
codecs::CompactU256,
snapshot::{HeaderMask, ReceiptMask, SnapshotCursor, TransactionMask},
static_file::{HeaderMask, ReceiptMask, StaticFileCursor, TransactionMask},
};
use reth_interfaces::provider::{ProviderError, ProviderResult};
use reth_primitives::{
Address, BlockHash, BlockHashOrNumber, BlockNumber, ChainInfo, Header, Receipt, SealedHeader,
TransactionMeta, TransactionSigned, TransactionSignedNoHash, TxHash, TxNumber, B256, U256,
};
use std::ops::{Deref, RangeBounds};
use std::{
ops::{Deref, RangeBounds},
sync::Arc,
};
/// Provider over a specific `NippyJar` and range.
#[derive(Debug)]
pub struct SnapshotJarProvider<'a> {
/// Main snapshot segment
pub struct StaticFileJarProvider<'a> {
/// Main static file segment
jar: LoadedJarRef<'a>,
/// Another kind of snapshot segment to help query data from the main one.
/// Another kind of static file segment to help query data from the main one.
auxiliar_jar: Option<Box<Self>>,
metrics: Option<Arc<StaticFileProviderMetrics>>,
}
impl<'a> Deref for SnapshotJarProvider<'a> {
impl<'a> Deref for StaticFileJarProvider<'a> {
type Target = LoadedJarRef<'a>;
fn deref(&self) -> &Self::Target {
&self.jar
}
}
impl<'a> From<LoadedJarRef<'a>> for SnapshotJarProvider<'a> {
impl<'a> From<LoadedJarRef<'a>> for StaticFileJarProvider<'a> {
fn from(value: LoadedJarRef<'a>) -> Self {
SnapshotJarProvider { jar: value, auxiliar_jar: None }
StaticFileJarProvider { jar: value, auxiliar_jar: None, metrics: None }
}
}
impl<'a> SnapshotJarProvider<'a> {
impl<'a> StaticFileJarProvider<'a> {
/// Provides a cursor for more granular data access.
pub fn cursor<'b>(&'b self) -> ProviderResult<SnapshotCursor<'a>>
pub fn cursor<'b>(&'b self) -> ProviderResult<StaticFileCursor<'a>>
where
'b: 'a,
{
SnapshotCursor::new(self.value(), self.mmap_handle())
let result = StaticFileCursor::new(self.value(), self.mmap_handle())?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
self.segment(),
StaticFileProviderOperation::InitCursor,
None,
);
}
Ok(result)
}
/// Adds a new auxiliar snapshot to help query data from the main one
pub fn with_auxiliar(mut self, auxiliar_jar: SnapshotJarProvider<'a>) -> Self {
/// Adds a new auxiliar static file to help query data from the main one
pub fn with_auxiliar(mut self, auxiliar_jar: StaticFileJarProvider<'a>) -> Self {
self.auxiliar_jar = Some(Box::new(auxiliar_jar));
self
}
/// Enables metrics on the provider.
pub fn with_metrics(mut self, metrics: Arc<StaticFileProviderMetrics>) -> Self {
self.metrics = Some(metrics);
self
}
}
impl<'a> HeaderProvider for SnapshotJarProvider<'a> {
impl<'a> HeaderProvider for StaticFileJarProvider<'a> {
fn header(&self, block_hash: &BlockHash) -> ProviderResult<Option<Header>> {
Ok(self
.cursor()?
@ -124,7 +147,7 @@ impl<'a> HeaderProvider for SnapshotJarProvider<'a> {
}
}
impl<'a> BlockHashReader for SnapshotJarProvider<'a> {
impl<'a> BlockHashReader for StaticFileJarProvider<'a> {
fn block_hash(&self, number: u64) -> ProviderResult<Option<B256>> {
self.cursor()?.get_one::<HeaderMask<BlockHash>>(number.into())
}
@ -146,7 +169,7 @@ impl<'a> BlockHashReader for SnapshotJarProvider<'a> {
}
}
impl<'a> BlockNumReader for SnapshotJarProvider<'a> {
impl<'a> BlockNumReader for StaticFileJarProvider<'a> {
fn chain_info(&self) -> ProviderResult<ChainInfo> {
// Information on live database
Err(ProviderError::UnsupportedProvider)
@ -167,17 +190,17 @@ impl<'a> BlockNumReader for SnapshotJarProvider<'a> {
Ok(cursor
.get_one::<HeaderMask<BlockHash>>((&hash).into())?
.and_then(|res| (res == hash).then(|| cursor.number())))
.and_then(|res| (res == hash).then(|| cursor.number()).flatten()))
}
}
impl<'a> TransactionsProvider for SnapshotJarProvider<'a> {
impl<'a> TransactionsProvider for StaticFileJarProvider<'a> {
fn transaction_id(&self, hash: TxHash) -> ProviderResult<Option<TxNumber>> {
let mut cursor = self.cursor()?;
Ok(cursor
.get_one::<TransactionMask<TransactionSignedNoHash>>((&hash).into())?
.and_then(|res| (res.hash() == hash).then(|| cursor.number())))
.and_then(|res| (res.hash() == hash).then(|| cursor.number()).flatten()))
}
fn transaction_by_id(&self, num: TxNumber) -> ProviderResult<Option<TransactionSigned>> {
@ -205,12 +228,12 @@ impl<'a> TransactionsProvider for SnapshotJarProvider<'a> {
&self,
_hash: TxHash,
) -> ProviderResult<Option<(TransactionSigned, TransactionMeta)>> {
// Information required on indexing table [`tables::TransactionBlock`]
// Information required on indexing table [`tables::TransactionBlocks`]
Err(ProviderError::UnsupportedProvider)
}
fn transaction_block(&self, _id: TxNumber) -> ProviderResult<Option<BlockNumber>> {
// Information on indexing table [`tables::TransactionBlock`]
// Information on indexing table [`tables::TransactionBlocks`]
Err(ProviderError::UnsupportedProvider)
}
@ -218,7 +241,7 @@ impl<'a> TransactionsProvider for SnapshotJarProvider<'a> {
&self,
_block_id: BlockHashOrNumber,
) -> ProviderResult<Option<Vec<TransactionSigned>>> {
// Related to indexing tables. Live database should get the tx_range and call snapshot
// Related to indexing tables. Live database should get the tx_range and call static file
// provider with `transactions_by_tx_range` instead.
Err(ProviderError::UnsupportedProvider)
}
@ -227,7 +250,7 @@ impl<'a> TransactionsProvider for SnapshotJarProvider<'a> {
&self,
_range: impl RangeBounds<BlockNumber>,
) -> ProviderResult<Vec<Vec<TransactionSigned>>> {
// Related to indexing tables. Live database should get the tx_range and call snapshot
// Related to indexing tables. Live database should get the tx_range and call static file
// provider with `transactions_by_tx_range` instead.
Err(ProviderError::UnsupportedProvider)
}
@ -267,14 +290,14 @@ impl<'a> TransactionsProvider for SnapshotJarProvider<'a> {
}
}
impl<'a> ReceiptProvider for SnapshotJarProvider<'a> {
impl<'a> ReceiptProvider for StaticFileJarProvider<'a> {
fn receipt(&self, num: TxNumber) -> ProviderResult<Option<Receipt>> {
self.cursor()?.get_one::<ReceiptMask<Receipt>>(num.into())
}
fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult<Option<Receipt>> {
if let Some(tx_snapshot) = &self.auxiliar_jar {
if let Some(num) = tx_snapshot.transaction_id(hash)? {
if let Some(tx_static_file) = &self.auxiliar_jar {
if let Some(num) = tx_static_file.transaction_id(hash)? {
return self.receipt(num)
}
}
@ -282,7 +305,7 @@ impl<'a> ReceiptProvider for SnapshotJarProvider<'a> {
}
fn receipts_by_block(&self, _block: BlockHashOrNumber) -> ProviderResult<Option<Vec<Receipt>>> {
// Related to indexing tables. Snapshot should get the tx_range and call snapshot
// Related to indexing tables. StaticFile should get the tx_range and call static file
// provider with `receipt()` instead for each
Err(ProviderError::UnsupportedProvider)
}

1110
crates/storage/provider/src/providers/static_file/manager.rs Normal file
View File

File diff suppressed because it is too large Load Diff

90
crates/storage/provider/src/providers/static_file/metrics.rs Normal file
View File

@ -0,0 +1,90 @@
use std::{collections::HashMap, time::Duration};
use itertools::Itertools;
use metrics::{Counter, Histogram};
use reth_metrics::Metrics;
use reth_primitives::StaticFileSegment;
use strum::{EnumIter, IntoEnumIterator};
/// Metrics for the static file provider.
#[derive(Debug)]
pub struct StaticFileProviderMetrics {
segment_operations: HashMap<
(StaticFileSegment, StaticFileProviderOperation),
StaticFileProviderOperationMetrics,
>,
}
impl Default for StaticFileProviderMetrics {
fn default() -> Self {
Self {
segment_operations: StaticFileSegment::iter()
.cartesian_product(StaticFileProviderOperation::iter())
.map(|(segment, operation)| {
(
(segment, operation),
StaticFileProviderOperationMetrics::new_with_labels(&[
("segment", segment.as_str()),
("operation", operation.as_str()),
]),
)
})
.collect(),
}
}
}
impl StaticFileProviderMetrics {
pub(crate) fn record_segment_operation(
&self,
segment: StaticFileSegment,
operation: StaticFileProviderOperation,
duration: Option<Duration>,
) {
self.segment_operations
.get(&(segment, operation))
.expect("segment operation metrics should exist")
.calls_total
.increment(1);
if let Some(duration) = duration {
self.segment_operations
.get(&(segment, operation))
.expect("segment operation metrics should exist")
.write_duration_seconds
.record(duration.as_secs_f64());
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter)]
pub(crate) enum StaticFileProviderOperation {
InitCursor,
OpenWriter,
Append,
Prune,
IncrementBlock,
CommitWriter,
}
impl StaticFileProviderOperation {
const fn as_str(&self) -> &'static str {
match self {
Self::InitCursor => "init-cursor",
Self::OpenWriter => "open-writer",
Self::Append => "append",
Self::Prune => "prune",
Self::IncrementBlock => "increment-block",
Self::CommitWriter => "commit-writer",
}
}
}
#[derive(Metrics)]
#[metrics(scope = "static_files.jar_provider")]
pub(crate) struct StaticFileProviderOperationMetrics {
/// Total number of static file jar provider operations made.
calls_total: Counter,
/// The time it took to execute the static file jar provider operation that writes data.
write_duration_seconds: Histogram,
}

63
crates/storage/provider/src/providers/snapshot/mod.rs → crates/storage/provider/src/providers/static_file/mod.rs
View File

@ -1,18 +1,25 @@
mod manager;
pub use manager::SnapshotProvider;
pub use manager::{StaticFileProvider, StaticFileWriter};
mod jar;
pub use jar::SnapshotJarProvider;
pub use jar::StaticFileJarProvider;
mod writer;
pub use writer::{StaticFileProviderRW, StaticFileProviderRWRefMut};
mod metrics;
use reth_interfaces::provider::ProviderResult;
use reth_nippy_jar::NippyJar;
use reth_primitives::{snapshot::SegmentHeader, SnapshotSegment};
use reth_primitives::{static_file::SegmentHeader, StaticFileSegment};
use std::{ops::Deref, sync::Arc};
/// Alias type for each specific `NippyJar`.
type LoadedJarRef<'a> = dashmap::mapref::one::Ref<'a, (u64, SnapshotSegment), LoadedJar>;
const BLOCKS_PER_STATIC_FILE: u64 = 500_000;
/// Helper type to reuse an associated snapshot mmap handle on created cursors.
/// Alias type for each specific `NippyJar`.
type LoadedJarRef<'a> = dashmap::mapref::one::Ref<'a, (u64, StaticFileSegment), LoadedJar>;
/// Helper type to reuse an associated static file mmap handle on created cursors.
#[derive(Debug)]
pub struct LoadedJar {
jar: NippyJar<SegmentHeader>,
@ -29,6 +36,10 @@ impl LoadedJar {
fn mmap_handle(&self) -> Arc<reth_nippy_jar::DataReader> {
self.mmap_handle.clone()
}
fn segment(&self) -> StaticFileSegment {
self.jar.user_header().segment()
}
}
impl Deref for LoadedJar {
@ -45,25 +56,31 @@ mod tests {
use rand::seq::SliceRandom;
use reth_db::{
cursor::DbCursorRO,
snapshot::create_snapshot_T1_T2_T3,
static_file::create_static_file_T1_T2_T3,
transaction::{DbTx, DbTxMut},
CanonicalHeaders, HeaderNumbers, HeaderTD, Headers, RawTable,
CanonicalHeaders, HeaderNumbers, HeaderTerminalDifficulties, Headers, RawTable,
};
use reth_interfaces::test_utils::generators::{self, random_header_range};
use reth_primitives::{BlockNumber, B256, U256};
use reth_primitives::{static_file::find_fixed_range, BlockNumber, B256, U256};
#[test]
fn test_snap() {
// Ranges
let row_count = 100u64;
let range = 0..=(row_count - 1);
let segment_header =
SegmentHeader::new(range.clone(), range.clone(), SnapshotSegment::Headers);
let segment_header = SegmentHeader::new(
range.clone().into(),
Some(range.clone().into()),
Some(range.clone().into()),
StaticFileSegment::Headers,
);
// Data sources
let factory = create_test_provider_factory();
let snap_path = tempfile::tempdir().unwrap();
let snap_file = snap_path.path().join(SnapshotSegment::Headers.filename(&range, &range));
let static_files_path = tempfile::tempdir().unwrap();
let static_file = static_files_path
.path()
.join(StaticFileSegment::Headers.filename(&find_fixed_range(*range.end())));
// Setup data
let mut headers = random_header_range(
@ -81,17 +98,17 @@ mod tests {
tx.put::<CanonicalHeaders>(header.number, hash).unwrap();
tx.put::<Headers>(header.number, header.clone().unseal()).unwrap();
tx.put::<HeaderTD>(header.number, td.into()).unwrap();
tx.put::<HeaderTerminalDifficulties>(header.number, td.into()).unwrap();
tx.put::<HeaderNumbers>(hash, header.number).unwrap();
}
provider_rw.commit().unwrap();
// Create Snapshot
// Create StaticFile
{
let with_compression = true;
let with_filter = true;
let mut nippy_jar = NippyJar::new(3, snap_file.as_path(), segment_header);
let mut nippy_jar = NippyJar::new(3, static_file.as_path(), segment_header);
if with_compression {
nippy_jar = nippy_jar.with_zstd(false, 0);
@ -115,24 +132,22 @@ mod tests {
.unwrap()
.map(|row| row.map(|(_key, value)| value.into_value()).map_err(|e| e.into()));
create_snapshot_T1_T2_T3::<
create_static_file_T1_T2_T3::<
Headers,
HeaderTD,
HeaderTerminalDifficulties,
CanonicalHeaders,
BlockNumber,
SegmentHeader,
>(
tx, range, None, none_vec, Some(hashes), row_count as usize, &mut nippy_jar
)
>(tx, range, None, none_vec, Some(hashes), row_count as usize, nippy_jar)
.unwrap();
}
// Use providers to query Header data and compare if it matches
{
let db_provider = factory.provider().unwrap();
let manager = SnapshotProvider::new(snap_path.path()).unwrap().with_filters();
let manager = StaticFileProvider::new(static_files_path.path()).unwrap().with_filters();
let jar_provider = manager
.get_segment_provider_from_block(SnapshotSegment::Headers, 0, Some(&snap_file))
.get_segment_provider_from_block(StaticFileSegment::Headers, 0, Some(&static_file))
.unwrap();
assert!(!headers.is_empty());
@ -148,7 +163,7 @@ mod tests {
assert_eq!(header, db_provider.header(&header_hash).unwrap().unwrap());
assert_eq!(header, jar_provider.header(&header_hash).unwrap().unwrap());
// Compare HeaderTD
// Compare HeaderTerminalDifficulties
assert_eq!(
db_provider.header_td(&header_hash).unwrap().unwrap(),
jar_provider.header_td(&header_hash).unwrap().unwrap()

488
crates/storage/provider/src/providers/static_file/writer.rs Normal file
View File

@ -0,0 +1,488 @@
use crate::providers::static_file::metrics::StaticFileProviderOperation;
use super::{
manager::StaticFileProviderInner, metrics::StaticFileProviderMetrics, StaticFileProvider,
};
use dashmap::mapref::one::RefMut;
use reth_codecs::Compact;
use reth_db::codecs::CompactU256;
use reth_interfaces::provider::{ProviderError, ProviderResult};
use reth_nippy_jar::{NippyJar, NippyJarError, NippyJarWriter};
use reth_primitives::{
static_file::{find_fixed_range, SegmentHeader, SegmentRangeInclusive},
BlockHash, BlockNumber, Header, Receipt, StaticFileSegment, TransactionSignedNoHash, TxNumber,
U256,
};
use std::{
path::{Path, PathBuf},
sync::{Arc, Weak},
time::Instant,
};
use tracing::debug;
/// Mutable reference to a dashmap element of [`StaticFileProviderRW`].
pub type StaticFileProviderRWRefMut<'a> = RefMut<'a, StaticFileSegment, StaticFileProviderRW>;
#[derive(Debug)]
/// Extends `StaticFileProvider` with writing capabilities
pub struct StaticFileProviderRW {
/// Reference back to the provider. We need [Weak] here because [StaticFileProviderRW] is
/// stored in a [dashmap::DashMap] inside the parent [StaticFileProvider].which is an [Arc].
/// If we were to use an [Arc] here, we would create a reference cycle.
reader: Weak<StaticFileProviderInner>,
writer: NippyJarWriter<SegmentHeader>,
data_path: PathBuf,
buf: Vec<u8>,
metrics: Option<Arc<StaticFileProviderMetrics>>,
}
impl StaticFileProviderRW {
/// Creates a new [`StaticFileProviderRW`] for a [`StaticFileSegment`].
pub fn new(
segment: StaticFileSegment,
block: BlockNumber,
reader: Weak<StaticFileProviderInner>,
metrics: Option<Arc<StaticFileProviderMetrics>>,
) -> ProviderResult<Self> {
let (writer, data_path) = Self::open(segment, block, reader.clone(), metrics.clone())?;
Ok(Self { writer, data_path, buf: Vec::with_capacity(100), reader, metrics })
}
fn open(
segment: StaticFileSegment,
block: u64,
reader: Weak<StaticFileProviderInner>,
metrics: Option<Arc<StaticFileProviderMetrics>>,
) -> ProviderResult<(NippyJarWriter<SegmentHeader>, PathBuf)> {
let start = Instant::now();
let static_file_provider = Self::upgrade_provider_to_strong_reference(&reader);
let block_range = find_fixed_range(block);
let (jar, path) = match static_file_provider.get_segment_provider_from_block(
segment,
block_range.start(),
None,
) {
Ok(provider) => (NippyJar::load(provider.data_path())?, provider.data_path().into()),
Err(ProviderError::MissingStaticFileBlock(_, _)) => {
let path = static_file_provider.directory().join(segment.filename(&block_range));
(create_jar(segment, &path, block_range), path)
}
Err(err) => return Err(err),
};
let result = match NippyJarWriter::new(jar) {
Ok(writer) => Ok((writer, path)),
Err(NippyJarError::FrozenJar) => {
// This static file has been frozen, so we should
Err(ProviderError::FinalizedStaticFile(segment, block))
}
Err(e) => Err(e.into()),
}?;
if let Some(metrics) = &metrics {
metrics.record_segment_operation(
segment,
StaticFileProviderOperation::OpenWriter,
Some(start.elapsed()),
);
}
Ok(result)
}
/// Commits configuration changes to disk and updates the reader index with the new changes.
pub fn commit(&mut self) -> ProviderResult<()> {
let start = Instant::now();
// Commits offsets and new user_header to disk
self.writer.commit()?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
self.writer.user_header().segment(),
StaticFileProviderOperation::CommitWriter,
Some(start.elapsed()),
);
}
debug!(
target: "provider::static_file",
segment = ?self.writer.user_header().segment(),
path = ?self.data_path,
duration = ?start.elapsed(),
"Commit"
);
self.update_index()?;
Ok(())
}
/// Updates the `self.reader` internal index.
fn update_index(&self) -> ProviderResult<()> {
// We find the maximum block of the segment by checking this writer's last block.
//
// However if there's no block range (because there's no data), we try to calculate it by
// substracting 1 from the expected block start, resulting on the last block of the
// previous file.
//
// If that expected block start is 0, then it means that there's no actual block data, and
// there's no block data in static files.
let segment_max_block = match self.writer.user_header().block_range() {
Some(block_range) => Some(block_range.end()),
None => {
if self.writer.user_header().expected_block_start() > 0 {
Some(self.writer.user_header().expected_block_start() - 1)
} else {
None
}
}
};
self.reader().update_index(self.writer.user_header().segment(), segment_max_block)
}
/// Allows to increment the [`SegmentHeader`] end block. It will commit the current static file,
/// and create the next one if we are past the end range.
///
/// Returns the current [`BlockNumber`] as seen in the static file.
pub fn increment_block(&mut self, segment: StaticFileSegment) -> ProviderResult<BlockNumber> {
let start = Instant::now();
if let Some(last_block) = self.writer.user_header().block_end() {
// We have finished the previous static file and must freeze it
if last_block == self.writer.user_header().expected_block_end() {
// Commits offsets and new user_header to disk
self.commit()?;
// Opens the new static file
let (writer, data_path) =
Self::open(segment, last_block + 1, self.reader.clone(), self.metrics.clone())?;
self.writer = writer;
self.data_path = data_path;
*self.writer.user_header_mut() =
SegmentHeader::new(find_fixed_range(last_block + 1), None, None, segment);
}
}
let block = self.writer.user_header_mut().increment_block();
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
segment,
StaticFileProviderOperation::IncrementBlock,
Some(start.elapsed()),
);
}
Ok(block)
}
/// Truncates a number of rows from disk. It deletes and loads an older static file if block
/// goes beyond the start of the current block range.
///
/// **last_block** should be passed only with transaction based segments.
///
/// # Note
/// Commits to the configuration file at the end.
fn truncate(
&mut self,
segment: StaticFileSegment,
mut num_rows: u64,
last_block: Option<u64>,
) -> ProviderResult<()> {
while num_rows > 0 {
let len = match segment {
StaticFileSegment::Headers => {
self.writer.user_header().block_len().unwrap_or_default()
}
StaticFileSegment::Transactions | StaticFileSegment::Receipts => {
self.writer.user_header().tx_len().unwrap_or_default()
}
};
if num_rows >= len {
// If there's more rows to delete than this static file contains, then just
// delete the whole file and go to the next static file
let previous_snap = self.data_path.clone();
let block_start = self.writer.user_header().expected_block_start();
if block_start != 0 {
let (writer, data_path) = Self::open(
segment,
self.writer.user_header().expected_block_start() - 1,
self.reader.clone(),
self.metrics.clone(),
)?;
self.writer = writer;
self.data_path = data_path;
NippyJar::<SegmentHeader>::load(&previous_snap)?.delete()?;
} else {
// Update `SegmentHeader`
self.writer.user_header_mut().prune(len);
self.writer.prune_rows(len as usize)?;
break
}
num_rows -= len;
} else {
// Update `SegmentHeader`
self.writer.user_header_mut().prune(num_rows);
// Truncate data
self.writer.prune_rows(num_rows as usize)?;
num_rows = 0;
}
}
// Only Transactions and Receipts
if let Some(last_block) = last_block {
let header = self.writer.user_header_mut();
header.set_block_range(header.expected_block_start(), last_block);
}
// Commits new changes to disk.
self.commit()?;
Ok(())
}
/// Appends column to static file.
fn append_column<T: Compact>(&mut self, column: T) -> ProviderResult<()> {
self.buf.clear();
column.to_compact(&mut self.buf);
self.writer.append_column(Some(Ok(&self.buf)))?;
Ok(())
}
/// Appends to tx number-based static file.
///
/// Returns the current [`TxNumber`] as seen in the static file.
fn append_with_tx_number<V: Compact>(
&mut self,
segment: StaticFileSegment,
tx_num: TxNumber,
value: V,
) -> ProviderResult<TxNumber> {
debug_assert!(self.writer.user_header().segment() == segment);
if self.writer.user_header().tx_range().is_none() {
self.writer.user_header_mut().set_tx_range(tx_num, tx_num);
} else {
self.writer.user_header_mut().increment_tx();
}
self.append_column(value)?;
Ok(self.writer.user_header().tx_end().expect("qed"))
}
/// Appends header to static file.
///
/// It **CALLS** `increment_block()` since the number of headers is equal to the number of
/// blocks.
///
/// Returns the current [`BlockNumber`] as seen in the static file.
pub fn append_header(
&mut self,
header: Header,
terminal_difficulty: U256,
hash: BlockHash,
) -> ProviderResult<BlockNumber> {
let start = Instant::now();
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::Headers);
let block_number = self.increment_block(StaticFileSegment::Headers)?;
self.append_column(header)?;
self.append_column(CompactU256::from(terminal_difficulty))?;
self.append_column(hash)?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::Headers,
StaticFileProviderOperation::Append,
Some(start.elapsed()),
);
}
Ok(block_number)
}
/// Appends transaction to static file.
///
/// It **DOES NOT CALL** `increment_block()`, it should be handled elsewhere. There might be
/// empty blocks and this function wouldn't be called.
///
/// Returns the current [`TxNumber`] as seen in the static file.
pub fn append_transaction(
&mut self,
tx_num: TxNumber,
tx: TransactionSignedNoHash,
) -> ProviderResult<TxNumber> {
let start = Instant::now();
let result = self.append_with_tx_number(StaticFileSegment::Transactions, tx_num, tx)?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::Transactions,
StaticFileProviderOperation::Append,
Some(start.elapsed()),
);
}
Ok(result)
}
/// Appends receipt to static file.
///
/// It **DOES NOT** call `increment_block()`, it should be handled elsewhere. There might be
/// empty blocks and this function wouldn't be called.
///
/// Returns the current [`TxNumber`] as seen in the static file.
pub fn append_receipt(
&mut self,
tx_num: TxNumber,
receipt: Receipt,
) -> ProviderResult<TxNumber> {
let start = Instant::now();
let result = self.append_with_tx_number(StaticFileSegment::Receipts, tx_num, receipt)?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::Receipts,
StaticFileProviderOperation::Append,
Some(start.elapsed()),
);
}
Ok(result)
}
/// Removes the last `number` of transactions from static files.
///
/// # Note
/// Commits to the configuration file at the end.
pub fn prune_transactions(
&mut self,
number: u64,
last_block: BlockNumber,
) -> ProviderResult<()> {
let start = Instant::now();
let segment = StaticFileSegment::Transactions;
debug_assert!(self.writer.user_header().segment() == segment);
self.truncate(segment, number, Some(last_block))?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::Transactions,
StaticFileProviderOperation::Prune,
Some(start.elapsed()),
);
}
Ok(())
}
/// Prunes `to_delete` number of receipts from static_files.
///
/// # Note
/// Commits to the configuration file at the end.
pub fn prune_receipts(
&mut self,
to_delete: u64,
last_block: BlockNumber,
) -> ProviderResult<()> {
let start = Instant::now();
let segment = StaticFileSegment::Receipts;
debug_assert!(self.writer.user_header().segment() == segment);
self.truncate(segment, to_delete, Some(last_block))?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::Receipts,
StaticFileProviderOperation::Prune,
Some(start.elapsed()),
);
}
Ok(())
}
/// Prunes `to_delete` number of headers from static_files.
///
/// # Note
/// Commits to the configuration file at the end.
pub fn prune_headers(&mut self, to_delete: u64) -> ProviderResult<()> {
let start = Instant::now();
let segment = StaticFileSegment::Headers;
debug_assert!(self.writer.user_header().segment() == segment);
self.truncate(segment, to_delete, None)?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::Headers,
StaticFileProviderOperation::Prune,
Some(start.elapsed()),
);
}
Ok(())
}
fn reader(&self) -> StaticFileProvider {
Self::upgrade_provider_to_strong_reference(&self.reader)
}
/// Upgrades a weak reference of [`StaticFileProviderInner`] to a strong reference
/// [`StaticFileProvider`].
///
/// # Panics
///
/// Panics if the parent [`StaticFileProvider`] is fully dropped while the child writer is still
/// active. In reality, it's impossible to detach the [`StaticFileProviderRW`] from the
/// [`StaticFileProvider`].
fn upgrade_provider_to_strong_reference(
provider: &Weak<StaticFileProviderInner>,
) -> StaticFileProvider {
provider.upgrade().map(StaticFileProvider).expect("StaticFileProvider is dropped")
}
#[cfg(any(test, feature = "test-utils"))]
/// Helper function to override block range for testing.
pub fn set_block_range(&mut self, block_range: std::ops::RangeInclusive<BlockNumber>) {
self.writer.user_header_mut().set_block_range(*block_range.start(), *block_range.end())
}
}
fn create_jar(
segment: StaticFileSegment,
path: &Path,
expected_block_range: SegmentRangeInclusive,
) -> NippyJar<SegmentHeader> {
let mut jar = NippyJar::new(
segment.columns(),
path,
SegmentHeader::new(expected_block_range, None, None, segment),
);
// Transaction and Receipt already have the compression scheme used natively in its encoding.
// (zstd-dictionary)
if segment.is_headers() {
jar = jar.with_lz4();
}
jar
}

25
crates/storage/provider/src/test_utils/blocks.rs
View File

@ -26,7 +26,10 @@ pub fn assert_genesis_block<DB: Database>(provider: &DatabaseProviderRW<DB>, g:
assert_eq!(tx.table::<tables::HeaderNumbers>().unwrap(), vec![(h, n)]);
assert_eq!(tx.table::<tables::CanonicalHeaders>().unwrap(), vec![(n, h)]);
assert_eq!(tx.table::<tables::HeaderTD>().unwrap(), vec![(n, g.difficulty.into())]);
assert_eq!(
tx.table::<tables::HeaderTerminalDifficulties>().unwrap(),
vec![(n, g.difficulty.into())]
);
assert_eq!(
tx.table::<tables::BlockBodyIndices>().unwrap(),
vec![(0, StoredBlockBodyIndices::default())]
@ -34,23 +37,23 @@ pub fn assert_genesis_block<DB: Database>(provider: &DatabaseProviderRW<DB>, g:
assert_eq!(tx.table::<tables::BlockOmmers>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::BlockWithdrawals>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::Transactions>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::TransactionBlock>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::TxHashNumber>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::TransactionBlocks>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::TransactionHashNumbers>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::Receipts>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::PlainAccountState>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::PlainStorageState>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::AccountHistory>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::StorageHistory>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::AccountsHistory>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::StoragesHistory>().unwrap(), vec![]);
// TODO check after this gets done: https://github.com/paradigmxyz/reth/issues/1588
// Bytecodes are not reverted assert_eq!(tx.table::<tables::Bytecodes>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::AccountChangeSet>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::StorageChangeSet>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::HashedAccount>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::HashedStorage>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::AccountChangeSets>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::StorageChangeSets>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::HashedAccounts>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::HashedStorages>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::AccountsTrie>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::StoragesTrie>().unwrap(), vec![]);
assert_eq!(tx.table::<tables::TxSenders>().unwrap(), vec![]);
// SyncStage is not updated in tests
assert_eq!(tx.table::<tables::TransactionSenders>().unwrap(), vec![]);
// StageCheckpoints is not updated in tests
}
const BLOCK_RLP: [u8; 610] = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0");

5
crates/storage/provider/src/test_utils/mod.rs
View File

@ -1,6 +1,6 @@
use crate::ProviderFactory;
use reth_db::{
test_utils::{create_test_rw_db, TempDatabase},
test_utils::{create_test_rw_db, create_test_static_files_dir, TempDatabase},
DatabaseEnv,
};
use reth_primitives::{ChainSpec, MAINNET};
@ -27,5 +27,6 @@ pub fn create_test_provider_factory_with_chain_spec(
chain_spec: Arc<ChainSpec>,
) -> ProviderFactory<Arc<TempDatabase<DatabaseEnv>>> {
let db = create_test_rw_db();
ProviderFactory::new(db, chain_spec)
ProviderFactory::new(db, chain_spec, create_test_static_files_dir())
.expect("create provider factory with static_files")
}

2
crates/storage/provider/src/traits/hashing.rs
View File

@ -20,7 +20,7 @@ pub trait HashingWriter: Send + Sync {
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<BTreeMap<B256, Option<Account>>>;
/// Inserts all accounts into [reth_db::tables::AccountHistory] table.
/// Inserts all accounts into [reth_db::tables::AccountsHistory] table.
///
/// # Returns
///

3
crates/storage/provider/src/traits/mod.rs
View File

@ -71,3 +71,6 @@ pub use prune_checkpoint::{PruneCheckpointReader, PruneCheckpointWriter};
mod database_provider;
pub use database_provider::DatabaseProviderFactory;
mod stats;
pub use stats::StatsReader;

10
crates/storage/provider/src/traits/stats.rs Normal file
View File

@ -0,0 +1,10 @@
use reth_db::table::Table;
use reth_interfaces::provider::ProviderResult;
/// The trait for fetching provider statistics.
#[auto_impl::auto_impl(&, Arc)]
pub trait StatsReader: Send + Sync {
/// Fetch the number of entries in the corresponding [Table]. Depending on the provider, it may
/// route to different data sources other than [Table].
fn count_entries<T: Table>(&self) -> ProviderResult<usize>;
}