jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(prune): timeout (#6958)

Browse Source 
Co-authored-by: Alexey Shekhirin <a.shekhirin@gmail.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
This commit is contained in:
Emilia Hane
2024-04-03 12:28:49 +02:00
committed by GitHub
parent 68727699db
commit f71d9c0003
23 changed files with 899 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
crates/storage/provider/src/providers/database/provider.rs
View File

@ -15,7 +15,7 @@ use crate::{
use itertools::{izip, Itertools};
use reth_db::{
common::KeyValue,
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO},
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, RangeWalker},
database::Database,
models::{
sharded_key, storage_sharded_key::StorageShardedKey, AccountBeforeTx, BlockNumberAddress,
@ -38,10 +38,10 @@ use reth_primitives::{
stage::{StageCheckpoint, StageId},
trie::Nibbles,
Account, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders,
ChainInfo, ChainSpec, GotExpected, Head, Header, PruneCheckpoint, PruneModes, PruneSegment,
Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, StaticFileSegment, StorageEntry,
TransactionMeta, TransactionSigned, TransactionSignedEcRecovered, TransactionSignedNoHash,
TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256,
ChainInfo, ChainSpec, GotExpected, Head, Header, PruneCheckpoint, PruneLimiter, PruneModes,
PruneSegment, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, StaticFileSegment,
StorageEntry, TransactionMeta, TransactionSigned, TransactionSignedEcRecovered,
TransactionSignedNoHash, TxHash, TxNumber, Withdrawal, Withdrawals, B256, U256,
};
use reth_trie::{
prefix_set::{PrefixSet, PrefixSetMut, TriePrefixSets},
@ -850,30 +850,38 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
pub fn prune_table_with_iterator<T: Table>(
&self,
keys: impl IntoIterator<Item = T::Key>,
limit: usize,
limiter: &mut PruneLimiter,
mut delete_callback: impl FnMut(TableRow<T>),
) -> Result<(usize, bool), DatabaseError> {
let mut cursor = self.tx.cursor_write::<T>()?;
let mut deleted = 0;
let mut keys = keys.into_iter();
if limit != 0 {
for key in &mut keys {
let row = cursor.seek_exact(key.clone())?;
if let Some(row) = row {
cursor.delete_current()?;
deleted += 1;
delete_callback(row);
}
let mut deleted_entries = 0;
if deleted == limit {
break
}
for key in &mut keys {
if limiter.is_limit_reached() {
debug!(
target: "providers::db",
?limiter,
deleted_entries_limit = %limiter.is_deleted_entries_limit_reached(),
time_limit = %limiter.is_time_limit_reached(),
table = %T::NAME,
"Pruning limit reached"
);
break
}
let row = cursor.seek_exact(key)?;
if let Some(row) = row {
cursor.delete_current()?;
limiter.increment_deleted_entries_count();
deleted_entries += 1;
delete_callback(row);
}
}
Ok((deleted, keys.next().is_none()))
let done = keys.next().is_none();
Ok((deleted_entries, done))
}
/// Prune the table for the specified key range.
@ -882,29 +890,72 @@ impl<TX: DbTxMut + DbTx> DatabaseProvider<TX> {
pub fn prune_table_with_range<T: Table>(
&self,
keys: impl RangeBounds<T::Key> + Clone + Debug,
limit: usize,
limiter: &mut PruneLimiter,
mut skip_filter: impl FnMut(&TableRow<T>) -> bool,
mut delete_callback: impl FnMut(TableRow<T>),
) -> Result<(usize, bool), DatabaseError> {
let mut cursor = self.tx.cursor_write::<T>()?;
let mut walker = cursor.walk_range(keys)?;
let mut deleted = 0;
if limit != 0 {
while let Some(row) = walker.next().transpose()? {
if !skip_filter(&row) {
walker.delete_current()?;
deleted += 1;
delete_callback(row);
}
let mut deleted_entries = 0;
if deleted == limit {
break
}
let done = loop {
// check for time out must be done in this scope since it's not done in
// `prune_table_with_range_step`
if limiter.is_limit_reached() {
debug!(
target: "providers::db",
?limiter,
deleted_entries_limit = %limiter.is_deleted_entries_limit_reached(),
time_limit = %limiter.is_time_limit_reached(),
table = %T::NAME,
"Pruning limit reached"
);
break false
}
let done = self.prune_table_with_range_step(
&mut walker,
limiter,
&mut skip_filter,
&mut delete_callback,
)?;
if done {
break true
} else {
deleted_entries += 1;
}
};
Ok((deleted_entries, done))
}
/// Steps once with the given walker and prunes the entry in the table.
///
/// Returns `true` if the walker is finished, `false` if it may have more data to prune.
///
/// CAUTION: Pruner limits are not checked. This allows for a clean exit of a prune run that's
/// pruning different tables concurrently, by letting them step to the same height before
/// timing out.
pub fn prune_table_with_range_step<T: Table>(
&self,
walker: &mut RangeWalker<'_, T, <TX as DbTxMut>::CursorMut<T>>,
limiter: &mut PruneLimiter,
skip_filter: &mut impl FnMut(&TableRow<T>) -> bool,
delete_callback: &mut impl FnMut(TableRow<T>),
) -> Result<bool, DatabaseError> {
let Some(res) = walker.next() else { return Ok(true) };
let row = res?;
if !skip_filter(&row) {
walker.delete_current()?;
limiter.increment_deleted_entries_count();
delete_callback(row);
}
Ok((deleted, walker.next().transpose()?.is_none()))
Ok(false)
}
/// Load shard and remove it. If list is empty, last shard was full or