jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add blocks_per_file field to StaticFileProvider (#11043)

Browse Source
This commit is contained in:
joshieDo
2024-09-19 18:36:23 +01:00
committed by GitHub
parent 9f23443029
commit 69c8ddb330
8 changed files with 63 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
crates/storage/provider/src/providers/static_file/manager.rs
View File

@ -1,7 +1,6 @@
use super::{
metrics::StaticFileProviderMetrics, writer::StaticFileWriters, LoadedJar,
StaticFileJarProvider, StaticFileProviderRW, StaticFileProviderRWRefMut,
BLOCKS_PER_STATIC_FILE,
};
use crate::{
to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, HeaderProvider,
@ -27,7 +26,10 @@ use reth_db_api::{
};
use reth_nippy_jar::{NippyJar, NippyJarChecker, CONFIG_FILE_EXTENSION};
use reth_primitives::{
static_file::{find_fixed_range, HighestStaticFiles, SegmentHeader, SegmentRangeInclusive},
static_file::{
find_fixed_range, HighestStaticFiles, SegmentHeader, SegmentRangeInclusive,
DEFAULT_BLOCKS_PER_STATIC_FILE,
},
Block, BlockWithSenders, Header, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader,
StaticFileSegment, TransactionMeta, TransactionSigned, TransactionSignedNoHash, Withdrawal,
Withdrawals,
@ -72,7 +74,7 @@ impl StaticFileAccess {
}
/// [`StaticFileProvider`] manages all existing [`StaticFileJarProvider`].
#[derive(Debug, Default, Clone)]
#[derive(Debug, Clone)]
pub struct StaticFileProvider(pub(crate) Arc<StaticFileProviderInner>);
impl StaticFileProvider {
@ -194,7 +196,7 @@ impl Deref for StaticFileProvider {
}
/// [`StaticFileProviderInner`] manages all existing [`StaticFileJarProvider`].
#[derive(Debug, Default)]
#[derive(Debug)]
pub struct StaticFileProviderInner {
/// Maintains a map which allows for concurrent access to different `NippyJars`, over different
/// segments and ranges.
@ -210,6 +212,8 @@ pub struct StaticFileProviderInner {
metrics: Option<Arc<StaticFileProviderMetrics>>,
/// Access rights of the provider.
access: StaticFileAccess,
/// Number of blocks per file.
blocks_per_file: u64,
/// Write lock for when access is [`StaticFileAccess::RW`].
_lock_file: Option<StorageLock>,
}
@ -231,6 +235,7 @@ impl StaticFileProviderInner {
path: path.as_ref().to_path_buf(),
metrics: None,
access,
blocks_per_file: DEFAULT_BLOCKS_PER_STATIC_FILE,
_lock_file,
};
@ -240,9 +245,24 @@ impl StaticFileProviderInner {
pub const fn is_read_only(&self) -> bool {
self.access.is_read_only()
}
/// Each static file has a fixed number of blocks. This gives out the range where the requested
/// block is positioned.
pub const fn find_fixed_range(&self, block: BlockNumber) -> SegmentRangeInclusive {
find_fixed_range(block, self.blocks_per_file)
}
}
impl StaticFileProvider {
/// Set a custom number of blocks per file.
#[cfg(any(test, feature = "test-utils"))]
pub fn with_custom_blocks_per_file(self, blocks_per_file: u64) -> Self {
let mut provider =
Arc::try_unwrap(self.0).expect("should be called when initializing only");
provider.blocks_per_file = blocks_per_file;
Self(Arc::new(provider))
}
/// Enables metrics on the [`StaticFileProvider`].
pub fn with_metrics(self) -> Self {
let mut provider =
@ -262,7 +282,7 @@ impl StaticFileProvider {
let mut size = 0;
for (block_range, _) in &ranges {
let fixed_block_range = find_fixed_range(block_range.start());
let fixed_block_range = self.find_fixed_range(block_range.start());
let jar_provider = self
.get_segment_provider(segment, || Some(fixed_block_range), None)?
.ok_or(ProviderError::MissingStaticFileBlock(segment, block_range.start()))?;
@ -369,14 +389,11 @@ impl StaticFileProvider {
self.map.remove(&(fixed_block_range_end, segment));
}
/// Given a segment and block range it deletes the jar and all files associated with it.
/// Given a segment and block, it deletes the jar and all files from the respective block range.
///
/// CAUTION: destructive. Deletes files on disk.
pub fn delete_jar(
&self,
segment: StaticFileSegment,
fixed_block_range: SegmentRangeInclusive,
) -> ProviderResult<()> {
pub fn delete_jar(&self, segment: StaticFileSegment, block: BlockNumber) -> ProviderResult<()> {
let fixed_block_range = self.find_fixed_range(block);
let key = (fixed_block_range.end(), segment);
let jar = if let Some((_, jar)) = self.map.remove(&key) {
jar.jar
@ -435,7 +452,7 @@ impl StaticFileProvider {
.read()
.get(&segment)
.filter(|max| **max >= block)
.map(|_| find_fixed_range(block))
.map(|_| self.find_fixed_range(block))
}
/// Gets a static file segment's fixed block range from the provider inner
@ -459,7 +476,7 @@ impl StaticFileProvider {
}
let tx_start = static_files_rev_iter.peek().map(|(tx_end, _)| *tx_end + 1).unwrap_or(0);
if tx_start <= tx {
return Some(find_fixed_range(block_range.end()))
return Some(self.find_fixed_range(block_range.end()))
}
}
None
@ -483,7 +500,7 @@ impl StaticFileProvider {
Some(segment_max_block) => {
// Update the max block for the segment
max_block.insert(segment, segment_max_block);
let fixed_range = find_fixed_range(segment_max_block);
let fixed_range = self.find_fixed_range(segment_max_block);
let jar = NippyJar::<SegmentHeader>::load(
&self.path.join(segment.filename(&fixed_range)),
@ -744,7 +761,7 @@ impl StaticFileProvider {
pub fn check_segment_consistency(&self, segment: StaticFileSegment) -> ProviderResult<()> {
if let Some(latest_block) = self.get_highest_static_file_block(segment) {
let file_path =
self.directory().join(segment.filename(&find_fixed_range(latest_block)));
self.directory().join(segment.filename(&self.find_fixed_range(latest_block)));
let jar = NippyJar::<SegmentHeader>::load(&file_path)
.map_err(|e| ProviderError::NippyJar(e.to_string()))?;
@ -894,14 +911,14 @@ impl StaticFileProvider {
func: impl Fn(StaticFileJarProvider<'_>) -> ProviderResult<Option<T>>,
) -> ProviderResult<Option<T>> {
if let Some(highest_block) = self.get_highest_static_file_block(segment) {
let mut range = find_fixed_range(highest_block);
let mut range = self.find_fixed_range(highest_block);
while range.end() > 0 {
if let Some(res) = func(self.get_or_create_jar_provider(segment, &range)?)? {
return Ok(Some(res))
}
range = SegmentRangeInclusive::new(
range.start().saturating_sub(BLOCKS_PER_STATIC_FILE),
range.end().saturating_sub(BLOCKS_PER_STATIC_FILE),
range.start().saturating_sub(self.blocks_per_file),
range.end().saturating_sub(self.blocks_per_file),
);
}
}

11
crates/storage/provider/src/providers/static_file/mod.rs
View File

@ -14,8 +14,6 @@ use reth_primitives::{static_file::SegmentHeader, StaticFileSegment};
use reth_storage_errors::provider::{ProviderError, ProviderResult};
use std::{ops::Deref, sync::Arc};
const BLOCKS_PER_STATIC_FILE: u64 = 500_000;
/// Alias type for each specific `NippyJar`.
type LoadedJarRef<'a> = dashmap::mapref::one::Ref<'a, (u64, StaticFileSegment), LoadedJar>;
@ -62,7 +60,7 @@ mod tests {
use rand::seq::SliceRandom;
use reth_db::{CanonicalHeaders, HeaderNumbers, HeaderTerminalDifficulties, Headers};
use reth_db_api::transaction::DbTxMut;
use reth_primitives::static_file::find_fixed_range;
use reth_primitives::static_file::{find_fixed_range, DEFAULT_BLOCKS_PER_STATIC_FILE};
use reth_testing_utils::generators::{self, random_header_range};
#[test]
@ -74,9 +72,10 @@ mod tests {
// Data sources
let factory = create_test_provider_factory();
let static_files_path = tempfile::tempdir().unwrap();
let static_file = static_files_path
.path()
.join(StaticFileSegment::Headers.filename(&find_fixed_range(*range.end())));
let static_file = static_files_path.path().join(
StaticFileSegment::Headers
.filename(&find_fixed_range(*range.end(), DEFAULT_BLOCKS_PER_STATIC_FILE)),
);
// Setup data
let mut headers = random_header_range(

12
crates/storage/provider/src/providers/static_file/writer.rs
View File

@ -8,7 +8,7 @@ use reth_codecs::Compact;
use reth_db_api::models::CompactU256;
use reth_nippy_jar::{NippyJar, NippyJarError, NippyJarWriter};
use reth_primitives::{
static_file::{find_fixed_range, SegmentHeader, SegmentRangeInclusive},
static_file::{SegmentHeader, SegmentRangeInclusive},
Header, Receipt, StaticFileSegment, TransactionSignedNoHash,
};
use reth_storage_errors::provider::{ProviderError, ProviderResult};
@ -139,7 +139,7 @@ impl StaticFileProviderRW {
let static_file_provider = Self::upgrade_provider_to_strong_reference(&reader);
let block_range = find_fixed_range(block);
let block_range = static_file_provider.find_fixed_range(block);
let (jar, path) = match static_file_provider.get_segment_provider_from_block(
segment,
block_range.start(),
@ -328,8 +328,12 @@ impl StaticFileProviderRW {
self.writer = writer;
self.data_path = data_path;
*self.writer.user_header_mut() =
SegmentHeader::new(find_fixed_range(last_block + 1), None, None, segment);
*self.writer.user_header_mut() = SegmentHeader::new(
self.reader().find_fixed_range(last_block + 1),
None,
None,
segment,
);
}
}

3
crates/storage/provider/src/test_utils/noop.rs
View File

@ -1,6 +1,7 @@
use std::{
collections::{HashMap, HashSet},
ops::{RangeBounds, RangeInclusive},
path::PathBuf,
sync::Arc,
};
@ -554,7 +555,7 @@ impl PruneCheckpointReader for NoopProvider {
impl StaticFileProviderFactory for NoopProvider {
fn static_file_provider(&self) -> StaticFileProvider {
StaticFileProvider::default()
StaticFileProvider::read_only(PathBuf::default(), false).unwrap()
}
}