jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add reth db snapshot <TYPE> command (#4889)

Browse Source
This commit is contained in:
joshieDo
2023-10-06 17:33:56 +01:00
committed by GitHub
parent 529635f8d4
commit 9c8eca6a49
19 changed files with 954 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/storage/db/Cargo.toml
View File

@ -15,6 +15,7 @@ reth-interfaces.workspace = true
reth-codecs = { path = "../codecs" }
reth-libmdbx = { path = "../libmdbx-rs", optional = true, features = ["return-borrowed"] }
reth-nippy-jar = { path = "../nippy-jar" }
reth-tracing = { path = "../../tracing" }
# codecs
serde = { workspace = true, default-features = false }

19
crates/storage/db/src/snapshot.rs
View File

@ -8,6 +8,7 @@ use crate::{
};
use reth_interfaces::RethResult;
use reth_nippy_jar::{ColumnResult, NippyJar, PHFKey};
use reth_tracing::tracing::*;
use std::{error::Error as StdError, ops::RangeInclusive};
/// Macro that generates snapshot creation functions that take an arbitratry number of [`Table`] and
@ -25,6 +26,7 @@ macro_rules! generate_snapshot_func {
///
/// * `tx`: Database transaction.
/// * `range`: Data range for columns in tables.
/// * `additional`: Additional columns which can't be straight straightforwardly walked on.
/// * `keys`: Iterator of keys (eg. `TxHash` or `BlockHash`) with length equal to `row_count` and ordered by future column insertion from `range`.
/// * `dict_compression_set`: Sets of column data for compression dictionaries. Max size is 2GB. Row count is independent.
/// * `row_count`: Total rows to add to `NippyJar`. Must match row count in `range`.
@ -37,6 +39,7 @@ macro_rules! generate_snapshot_func {
(
tx: &impl DbTx<'tx>,
range: RangeInclusive<K>,
additional: Option<Vec<Box<dyn Iterator<Item = Result<Vec<u8>, Box<dyn StdError + Send + Sync>>>>>>,
dict_compression_set: Option<Vec<impl Iterator<Item = Vec<u8>>>>,
keys: Option<impl Iterator<Item = ColumnResult<impl PHFKey>>>,
row_count: usize,
@ -44,16 +47,23 @@ macro_rules! generate_snapshot_func {
) -> RethResult<()>
where K: Key + Copy
{
let additional = additional.unwrap_or_default();
debug!(target: "reth::snapshot", ?range, "Creating snapshot {:?} and {} more columns.", vec![$($tbl::NAME,)+], additional.len());
let range: RangeInclusive<RawKey<K>> = RawKey::new(*range.start())..=RawKey::new(*range.end());
// Create PHF and Filter if required
if let Some(keys) = keys {
debug!(target: "reth::snapshot", "Calculating Filter, PHF and offset index list");
nippy_jar.prepare_index(keys, row_count)?;
debug!(target: "reth::snapshot", "Filter, PHF and offset index list calculated.");
}
// Create compression dictionaries if required
if let Some(data_sets) = dict_compression_set {
debug!(target: "reth::snapshot", "Creating compression dictionaries.");
nippy_jar.prepare_compression(data_sets)?;
debug!(target: "reth::snapshot", "Compression dictionaries created.");
}
// Creates the cursors for the columns
@ -75,7 +85,12 @@ macro_rules! generate_snapshot_func {
$(Box::new([< $tbl _iter>]),)+
];
nippy_jar.freeze(col_iterators, row_count as u64)?;
debug!(target: "reth::snapshot", jar=?nippy_jar, "Generating snapshot file.");
nippy_jar.freeze(col_iterators.into_iter().chain(additional).collect(), row_count as u64)?;
debug!(target: "reth::snapshot", jar=?nippy_jar, "Snapshot file generated.");
Ok(())
}
@ -84,4 +99,4 @@ macro_rules! generate_snapshot_func {
};
}
generate_snapshot_func!((T1), (T1, T2), (T1, T2, T3), (T1, T2, T3, T4),);
generate_snapshot_func!((T1), (T1, T2), (T1, T2, T3), (T1, T2, T3, T4), (T1, T2, T3, T4, T5),);