feat: add reth init and reth import (#877)

Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>

crates/net/downloaders/Cargo.toml

@@ -32,6 +32,7 @@ reth-rlp = { path = "../../rlp", optional = true }
 tokio-util = { version = "0.7", features = ["codec"], optional = true }
 bytes = { version = "1", optional = true }
 tempfile = { version = "3.3", optional = true }
+itertools = { version = "0.10", optional = true }
 
 [dev-dependencies]
 reth-db = { path = "../../storage/db", features = ["test-utils"] }
@@ -42,10 +43,11 @@ assert_matches = "1.5.0"
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
 tokio-util = { version = "0.7", features = ["codec"] }
 reth-rlp = { path = "../../rlp" }
+itertools = "0.10"
 bytes = "1"
 
 thiserror = "1"
 tempfile = "3.3"
 
 [features]
-test-utils = ["dep:reth-rlp", "dep:thiserror", "dep:tokio-util", "dep:tempfile", "dep:bytes"]
+test-utils = ["dep:reth-rlp", "dep:thiserror", "dep:tokio-util", "dep:tempfile", "dep:bytes", "dep:itertools"]

crates/net/downloaders/src/bodies/bodies.rs

@@ -1,6 +1,5 @@
-use crate::metrics::DownloaderMetrics;
-
 use super::queue::BodiesRequestQueue;
+use crate::{bodies::task::TaskDownloader, metrics::DownloaderMetrics};
 use futures::Stream;
 use futures_util::StreamExt;
 use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx};
@@ -186,7 +185,7 @@ where
     fn is_terminated(&self) -> bool {
         // There is nothing to request if the range is empty
         let nothing_to_request = self.download_range.is_empty() ||
-            // or all blocks have already been requested. 
+            // or all blocks have already been requested.
             self.in_progress_queue
                 .last_requested_block_number
                 .map(|last| last + 1 == self.download_range.end)
@@ -241,6 +240,19 @@ where
     }
 }
 
+impl<B, DB> BodiesDownloader<B, DB>
+where
+    B: BodiesClient + 'static,
+    DB: Database,
+    Self: BodyDownloader + 'static,
+{
+    /// Convert the downloader into a [`TaskDownloader`](super::task::TaskDownloader) by spawning
+    /// it.
+    pub fn as_task(self) -> TaskDownloader {
+        TaskDownloader::spawn(self)
+    }
+}
+
 impl<B, DB> BodyDownloader for BodiesDownloader<B, DB>
 where
     B: BodiesClient + 'static,

crates/net/downloaders/src/headers/reverse_headers.rs

@@ -1,5 +1,6 @@
 //! A headers downloader that can handle multiple requests concurrently.
 
+use super::task::TaskDownloader;
 use crate::metrics::DownloaderMetrics;
 use futures::{stream::Stream, FutureExt};
 use futures_util::{stream::FuturesUnordered, StreamExt};
@@ -357,6 +358,8 @@ where
                 // validate the response
                 let highest = &headers[0];
 
+                trace!(target: "downloaders::headers", requested_block_number, highest=?highest.number, "Validating non-empty headers response");
+
                 if highest.number != requested_block_number {
                     return Err(HeadersResponseError {
                         request,
@@ -462,6 +465,7 @@ where
 
     /// Starts a request future
     fn submit_request(&mut self, request: HeadersRequest, priority: Priority) {
+        trace!(target: "downloaders::headers", ?request, "Submitting headers request");
         self.in_progress_queue.push(self.request_fut(request, priority));
     }
 
@@ -499,6 +503,18 @@ where
     }
 }
 
+impl<H> ReverseHeadersDownloader<H>
+where
+    H: HeadersClient,
+    Self: HeaderDownloader + 'static,
+{
+    /// Convert the downloader into a [`TaskDownloader`](super::task::TaskDownloader) by spawning
+    /// it.
+    pub fn as_task(self) -> TaskDownloader {
+        TaskDownloader::spawn(self)
+    }
+}
+
 impl<H> HeaderDownloader for ReverseHeadersDownloader<H>
 where
     H: HeadersClient + 'static,

crates/net/downloaders/src/test_utils/file_client.rs

@@ -1,4 +1,5 @@
 use super::file_codec::BlockFileCodec;
+use itertools::Either;
 use reth_eth_wire::{BlockBody, RawBlockBody};
 use reth_interfaces::{
     p2p::{
@@ -30,7 +31,7 @@ use tokio::{
 };
 use tokio_stream::StreamExt;
 use tokio_util::codec::FramedRead;
-use tracing::warn;
+use tracing::{trace, warn};
 
 /// Front-end API for fetching chain data from a file.
 ///
@@ -94,26 +95,29 @@ impl FileClient {
         // use with_capacity to make sure the internal buffer contains the entire file
         let mut stream = FramedRead::with_capacity(&reader[..], BlockFileCodec, file_len as usize);
 
-        let mut block_num = 0;
         while let Some(block_res) = stream.next().await {
             let block = block_res?;
             let block_hash = block.header.hash_slow();
 
             // add to the internal maps
-            headers.insert(block_num, block.header.clone());
-            hash_to_number.insert(block_hash, block_num);
+            headers.insert(block.header.number, block.header.clone());
+            hash_to_number.insert(block_hash, block.header.number);
             bodies.insert(
                 block_hash,
                 BlockBody { transactions: block.transactions, ommers: block.ommers },
             );
-
-            // update block num
-            block_num += 1;
         }
 
+        trace!(blocks = headers.len(), "Initialized file client");
+
         Ok(Self { headers, hash_to_number, bodies, is_syncing: Arc::new(Default::default()) })
     }
 
+    /// Get the tip hash of the chain.
+    pub fn tip(&self) -> Option<H256> {
+        self.headers.get(&(self.headers.len() as u64 - 1)).map(|h| h.hash_slow())
+    }
+
     /// Use the provided bodies as the file client's block body buffer.
     pub(crate) fn with_bodies(mut self, bodies: HashMap<BlockHash, BlockBody>) -> Self {
         self.bodies = bodies;
@@ -140,24 +144,39 @@ impl HeadersClient for FileClient {
     ) -> Self::Output {
         // this just searches the buffer, and fails if it can't find the header
         let mut headers = Vec::new();
+        trace!(target : "downloaders::file", request=?request, "Getting headers");
 
         let start_num = match request.start {
             BlockHashOrNumber::Hash(hash) => match self.hash_to_number.get(&hash) {
                 Some(num) => *num,
-                None => return Box::pin(async move { Err(RequestError::BadResponse) }),
+                None => {
+                    warn!(%hash, "Could not find starting block number for requested header hash");
+                    return Box::pin(async move { Err(RequestError::BadResponse) })
+                }
             },
             BlockHashOrNumber::Number(num) => num,
         };
 
-        let range = match request.direction {
-            HeadersDirection::Rising => start_num..=start_num + 1 - request.limit,
-            HeadersDirection::Falling => start_num + 1 - request.limit..=start_num,
+        let range = if request.limit == 1 {
+            Either::Left(start_num..start_num + 1)
+        } else {
+            match request.direction {
+                HeadersDirection::Rising => Either::Left(start_num..start_num + request.limit),
+                HeadersDirection::Falling => {
+                    Either::Right((start_num - request.limit + 1..=start_num).rev())
+                }
+            }
         };
 
+        trace!(target : "downloaders::file", range=?range, "Getting headers with range");
+
         for block_number in range {
             match self.headers.get(&block_number).cloned() {
                 Some(header) => headers.push(header),
-                None => return Box::pin(async move { Err(RequestError::BadResponse) }),
+                None => {
+                    warn!(number=%block_number, "Could not find header");
+                    return Box::pin(async move { Err(RequestError::BadResponse) })
+                }
             }
         }