jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 19:09:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename downloaders (#1108)

Browse Source 
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
This commit is contained in:
Aurélien
2023-02-01 01:44:29 +01:00
committed by GitHub
parent ae771d291d
commit 0149bde8fb
12 changed files with 85 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
crates/net/downloaders/src/bodies/concurrent.rs → crates/net/downloaders/src/bodies/bodies.rs
View File

@ -43,7 +43,7 @@ pub const BODIES_DOWNLOADER_SCOPE: &str = "downloaders.bodies";
/// All blocks in a batch are fetched at the same time.
#[must_use = "Stream does nothing unless polled"]
#[derive(Debug)]
pub struct ConcurrentDownloader<B: BodiesClient, DB> {
pub struct BodiesDownloader<B: BodiesClient, DB> {
/// The bodies client
client: Arc<B>,
/// The consensus client
@ -73,7 +73,7 @@ pub struct ConcurrentDownloader<B: BodiesClient, DB> {
metrics: DownloaderMetrics,
}
impl<B, DB> ConcurrentDownloader<B, DB>
impl<B, DB> BodiesDownloader<B, DB>
where
B: BodiesClient + 'static,
DB: Database,
@ -241,7 +241,7 @@ where
}
}
impl<B, DB> BodyDownloader for ConcurrentDownloader<B, DB>
impl<B, DB> BodyDownloader for BodiesDownloader<B, DB>
where
B: BodiesClient + 'static,
DB: Database,
@ -331,7 +331,7 @@ where
}
}
impl<B, DB> Stream for ConcurrentDownloader<B, DB>
impl<B, DB> Stream for BodiesDownloader<B, DB>
where
B: BodiesClient + 'static,
DB: Database,
@ -452,8 +452,8 @@ impl Ord for OrderedBodiesResponse {
}
}
/// Builder for [ConcurrentDownloader].
pub struct ConcurrentDownloaderBuilder {
/// Builder for [BodiesDownloader].
pub struct BodiesDownloaderBuilder {
/// The batch size of non-empty blocks per one request
request_limit: u64,
/// The maximum number of block bodies returned at once from the stream
@ -464,7 +464,7 @@ pub struct ConcurrentDownloaderBuilder {
concurrent_requests_range: RangeInclusive<usize>,
}
impl Default for ConcurrentDownloaderBuilder {
impl Default for BodiesDownloaderBuilder {
fn default() -> Self {
Self {
request_limit: 200,
@ -475,7 +475,7 @@ impl Default for ConcurrentDownloaderBuilder {
}
}
impl ConcurrentDownloaderBuilder {
impl BodiesDownloaderBuilder {
/// Set request batch size on the downloader.
pub fn with_request_limit(mut self, request_limit: u64) -> Self {
self.request_limit = request_limit;
@ -509,7 +509,7 @@ impl ConcurrentDownloaderBuilder {
client: Arc<B>,
consensus: Arc<dyn Consensus>,
db: Arc<DB>,
) -> ConcurrentDownloader<B, DB>
) -> BodiesDownloader<B, DB>
where
B: BodiesClient + 'static,
DB: Database,
@ -522,7 +522,7 @@ impl ConcurrentDownloaderBuilder {
} = self;
let metrics = DownloaderMetrics::new(BODIES_DOWNLOADER_SCOPE);
let in_progress_queue = BodiesRequestQueue::new(metrics.clone());
ConcurrentDownloader {
BodiesDownloader {
client,
consensus,
db,
@ -566,7 +566,7 @@ mod tests {
let client = Arc::new(
TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true),
);
let mut downloader = ConcurrentDownloaderBuilder::default().build(
let mut downloader = BodiesDownloaderBuilder::default().build(
client.clone(),
Arc::new(TestConsensus::default()),
db,
@ -595,7 +595,7 @@ mod tests {
let client = Arc::new(
TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true),
);
let mut downloader = ConcurrentDownloaderBuilder::default()
let mut downloader = BodiesDownloaderBuilder::default()
.with_stream_batch_size(stream_batch_size)
.with_request_limit(request_limit)
.build(client.clone(), Arc::new(TestConsensus::default()), db);
@ -631,9 +631,11 @@ mod tests {
insert_headers(&db, &headers);
let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone()));
let mut downloader = ConcurrentDownloaderBuilder::default()
.with_stream_batch_size(100)
.build(client.clone(), Arc::new(TestConsensus::default()), db);
let mut downloader = BodiesDownloaderBuilder::default().with_stream_batch_size(100).build(
client.clone(),
Arc::new(TestConsensus::default()),
db,
);
// Set and download the first range
downloader.set_download_range(0..100).expect("failed to set download range");

3
crates/net/downloaders/src/bodies/mod.rs
View File

@ -1,5 +1,6 @@
/// A naive concurrent downloader.
pub mod concurrent;
#[allow(clippy::module_inception)]
pub mod bodies;
/// TODO:
pub mod task;

10
crates/net/downloaders/src/bodies/task.rs
View File

@ -44,13 +44,13 @@ impl TaskDownloader {
///
/// ```
/// use std::sync::Arc;
/// use reth_downloaders::bodies::concurrent::ConcurrentDownloaderBuilder;
/// use reth_downloaders::bodies::bodies::BodiesDownloaderBuilder;
/// use reth_downloaders::bodies::task::TaskDownloader;
/// use reth_interfaces::consensus::Consensus;
/// use reth_interfaces::p2p::bodies::client::BodiesClient;
/// use reth_db::database::Database;
/// fn t<B: BodiesClient + 'static, DB: Database + 'static>(client: Arc<B>, consensus:Arc<dyn Consensus>, db: Arc<DB>) {
/// let downloader = ConcurrentDownloaderBuilder::default().build(
/// let downloader = BodiesDownloaderBuilder::default().build(
/// client,
/// consensus,
/// db
@ -133,7 +133,7 @@ mod tests {
use super::*;
use crate::{
bodies::{
concurrent::ConcurrentDownloaderBuilder,
bodies::BodiesDownloaderBuilder,
test_utils::{insert_headers, zip_blocks},
},
test_utils::{generate_bodies, TestBodiesClient},
@ -155,7 +155,7 @@ mod tests {
let client = Arc::new(
TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true),
);
let downloader = ConcurrentDownloaderBuilder::default().build(
let downloader = BodiesDownloaderBuilder::default().build(
client.clone(),
Arc::new(TestConsensus::default()),
db,
@ -184,7 +184,7 @@ mod tests {
let client = Arc::new(
TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true),
);
let downloader = ConcurrentDownloaderBuilder::default().build(
let downloader = BodiesDownloaderBuilder::default().build(
client.clone(),
Arc::new(TestConsensus::default()),
db,

2
crates/net/downloaders/src/headers/mod.rs
View File

@ -1,5 +1,5 @@
/// A Linear downloader implementation.
pub mod linear;
pub mod reverse_headers;
/// A downloader implementation that spawns a downloader to a task
pub mod task;

55
crates/net/downloaders/src/headers/linear.rs → crates/net/downloaders/src/headers/reverse_headers.rs
View File

@ -44,7 +44,7 @@ pub const HEADERS_DOWNLOADER_SCOPE: &str = "downloaders.headers";
/// the batches of headers that this downloader yields will start at the chain tip and move towards
/// the local head: falling block numbers.
#[must_use = "Stream does nothing unless polled"]
pub struct LinearDownloader<H: HeadersClient> {
pub struct ReverseHeadersDownloader<H: HeadersClient> {
/// Consensus client used to validate headers
consensus: Arc<dyn Consensus>,
/// Client used to download headers.
@ -86,15 +86,15 @@ pub struct LinearDownloader<H: HeadersClient> {
metrics: DownloaderMetrics,
}
// === impl LinearDownloader ===
// === impl ReverseHeadersDownloader ===
impl<H> LinearDownloader<H>
impl<H> ReverseHeadersDownloader<H>
where
H: HeadersClient + 'static,
{
/// Convenience method to create a [LinearDownloadBuilder] without importing it
pub fn builder() -> LinearDownloadBuilder {
LinearDownloadBuilder::default()
/// Convenience method to create a [ReverseHeadersDownloaderBuilder] without importing it
pub fn builder() -> ReverseHeadersDownloaderBuilder {
ReverseHeadersDownloaderBuilder::default()
}
/// Returns the block number the local node is at.
@ -501,7 +501,7 @@ where
}
}
impl<H> HeaderDownloader for LinearDownloader<H>
impl<H> HeaderDownloader for ReverseHeadersDownloader<H>
where
H: HeadersClient + 'static,
{
@ -577,7 +577,7 @@ where
}
}
impl<H> Stream for LinearDownloader<H>
impl<H> Stream for ReverseHeadersDownloader<H>
where
H: HeadersClient + 'static,
{
@ -793,10 +793,10 @@ impl SyncTargetBlock {
}
}
/// The builder for [LinearDownloader] with
/// The builder for [ReverseHeadersDownloader] with
/// some default settings
#[derive(Debug)]
pub struct LinearDownloadBuilder {
pub struct ReverseHeadersDownloaderBuilder {
/// The batch size per one request
request_limit: u64,
/// Batch size for headers
@ -809,7 +809,7 @@ pub struct LinearDownloadBuilder {
max_buffered_responses: usize,
}
impl Default for LinearDownloadBuilder {
impl Default for ReverseHeadersDownloaderBuilder {
fn default() -> Self {
Self {
request_limit: 1_000,
@ -821,7 +821,7 @@ impl Default for LinearDownloadBuilder {
}
}
impl LinearDownloadBuilder {
impl ReverseHeadersDownloaderBuilder {
/// Set the request batch size.
///
/// This determines the `limit` for a [GetHeaders](reth_eth_wire::GetBlockHeaders) requests, the
@ -833,8 +833,9 @@ impl LinearDownloadBuilder {
/// Set the stream batch size
///
/// This determines the number of headers the [LinearDownloader] will yield on `Stream::next`.
/// This will be the amount of headers the headers stage will commit at a time.
/// This determines the number of headers the [ReverseHeadersDownloader] will yield on
/// `Stream::next`. This will be the amount of headers the headers stage will commit at a
/// time.
pub fn stream_batch_size(mut self, size: usize) -> Self {
self.stream_batch_size = size;
self
@ -842,8 +843,8 @@ impl LinearDownloadBuilder {
/// Set the min amount of concurrent requests.
///
/// If there's capacity the [LinearDownloader] will keep at least this many requests active at a
/// time.
/// If there's capacity the [ReverseHeadersDownloader] will keep at least this many requests
/// active at a time.
pub fn min_concurrent_requests(mut self, min_concurrent_requests: usize) -> Self {
self.min_concurrent_requests = min_concurrent_requests;
self
@ -861,16 +862,20 @@ impl LinearDownloadBuilder {
///
/// This essentially determines how much memory the downloader can use for buffering responses
/// that arrive out of order. The total number of buffered headers is `request_limit *
/// max_buffered_responses`. If the [LinearDownloader]'s buffered responses exceeds this
/// max_buffered_responses`. If the [ReverseHeadersDownloader]'s buffered responses exceeds this
/// threshold it waits until there's capacity again before sending new requests.
pub fn max_buffered_responses(mut self, max_buffered_responses: usize) -> Self {
self.max_buffered_responses = max_buffered_responses;
self
}
/// Build [LinearDownloader] with provided consensus
/// Build [ReverseHeadersDownloader] with provided consensus
/// and header client implementations
pub fn build<H>(self, consensus: Arc<dyn Consensus>, client: Arc<H>) -> LinearDownloader<H>
pub fn build<H>(
self,
consensus: Arc<dyn Consensus>,
client: Arc<H>,
) -> ReverseHeadersDownloader<H>
where
H: HeadersClient + 'static,
{
@ -881,7 +886,7 @@ impl LinearDownloadBuilder {
max_concurrent_requests,
max_buffered_responses,
} = self;
LinearDownloader {
ReverseHeadersDownloader {
consensus,
client,
local_head: None,
@ -940,7 +945,7 @@ mod tests {
let genesis = SealedHeader::default();
let mut downloader = LinearDownloadBuilder::default()
let mut downloader = ReverseHeadersDownloaderBuilder::default()
.build(Arc::new(TestConsensus::default()), Arc::clone(&client));
downloader.update_local_head(genesis);
downloader.update_sync_target(SyncTarget::Tip(H256::random()));
@ -968,7 +973,7 @@ mod tests {
let header = SealedHeader::default();
let mut downloader = LinearDownloadBuilder::default()
let mut downloader = ReverseHeadersDownloaderBuilder::default()
.build(Arc::new(TestConsensus::default()), Arc::clone(&client));
downloader.update_local_head(header.clone());
downloader.update_sync_target(SyncTarget::Tip(H256::random()));
@ -1008,7 +1013,7 @@ mod tests {
let batch_size = 99;
let start = 1000;
let mut downloader = LinearDownloadBuilder::default()
let mut downloader = ReverseHeadersDownloaderBuilder::default()
.request_limit(batch_size)
.build(Arc::new(TestConsensus::default()), Arc::clone(&client));
downloader.update_local_head(genesis);
@ -1057,7 +1062,7 @@ mod tests {
let p1 = child_header(&p2);
let p0 = child_header(&p1);
let mut downloader = LinearDownloadBuilder::default()
let mut downloader = ReverseHeadersDownloaderBuilder::default()
.stream_batch_size(3)
.request_limit(3)
.build(Arc::new(TestConsensus::default()), Arc::clone(&client));
@ -1089,7 +1094,7 @@ mod tests {
let p0 = child_header(&p1);
let client = Arc::new(TestHeadersClient::default());
let mut downloader = LinearDownloadBuilder::default()
let mut downloader = ReverseHeadersDownloaderBuilder::default()
.stream_batch_size(1)
.request_limit(1)
.build(Arc::new(TestConsensus::default()), Arc::clone(&client));

10
crates/net/downloaders/src/headers/task.rs
View File

@ -40,12 +40,12 @@ impl TaskDownloader {
///
/// ```
/// # use std::sync::Arc;
/// # use reth_downloaders::headers::linear::LinearDownloader;
/// # use reth_downloaders::headers::reverse_headers::ReverseHeadersDownloader;
/// # use reth_downloaders::headers::task::TaskDownloader;
/// # use reth_interfaces::consensus::Consensus;
/// # use reth_interfaces::p2p::headers::client::HeadersClient;
/// # fn t<H: HeadersClient + 'static>(consensus:Arc<dyn Consensus>, client: Arc<H>) {
/// let downloader = LinearDownloader::<H>::builder().build(
/// let downloader = ReverseHeadersDownloader::<H>::builder().build(
/// consensus,
/// client,
/// );
@ -153,7 +153,9 @@ enum DownloaderUpdates {
#[cfg(test)]
mod tests {
use super::*;
use crate::headers::{linear::LinearDownloadBuilder, test_utils::child_header};
use crate::headers::{
reverse_headers::ReverseHeadersDownloaderBuilder, test_utils::child_header,
};
use reth_interfaces::test_utils::{TestConsensus, TestHeadersClient};
use std::sync::Arc;
@ -167,7 +169,7 @@ mod tests {
let p0 = child_header(&p1);
let client = Arc::new(TestHeadersClient::default());
let downloader = LinearDownloadBuilder::default()
let downloader = ReverseHeadersDownloaderBuilder::default()
.stream_batch_size(1)
.request_limit(1)
.build(Arc::new(TestConsensus::default()), Arc::clone(&client));

1
crates/net/downloaders/src/lib.rs
View File

@ -4,6 +4,7 @@
no_crate_inject,
attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
))]
#![allow(clippy::result_large_err)]
//! Implements the downloader algorithms.