feat: add directory paths to Snapshotter and SnapshotProvider (#5283)

Co-authored-by: Alexey Shekhirin <a.shekhirin@gmail.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

crates/snapshot/Cargo.toml

@@ -30,7 +30,7 @@ reth-db = { workspace = true, features = ["test-utils"] }
 reth-stages = { workspace = true, features = ["test-utils"] }
 
 # misc
-
+tempfile.workspace = true
 assert_matches.workspace = true
 
 [features]

crates/snapshot/src/segments/mod.rs

@@ -19,7 +19,7 @@ use reth_primitives::{
     BlockNumber, SnapshotSegment,
 };
 use reth_provider::{DatabaseProviderRO, TransactionsProviderExt};
-use std::ops::RangeInclusive;
+use std::{ops::RangeInclusive, path::Path};
 
 pub(crate) type Rows<const COLUMNS: usize> = [Vec<Vec<u8>>; COLUMNS];
 
@@ -61,7 +61,7 @@ pub(crate) fn prepare_jar<DB: Database, const COLUMNS: usize>(
     let tx_range = provider.transaction_range_by_block_range(block_range.clone())?;
     let mut nippy_jar = NippyJar::new(
         COLUMNS,
-        &segment.filename_with_configuration(filters, compression, &block_range),
+        Path::new(segment.filename(&block_range).as_str()),
         SegmentHeader::new(block_range, tx_range, segment),
     );
 

crates/snapshot/src/snapshotter.rs

@@ -3,9 +3,11 @@
 use crate::SnapshotterError;
 use reth_db::database::Database;
 use reth_interfaces::{RethError, RethResult};
-use reth_primitives::{snapshot::HighestSnapshots, BlockNumber, ChainSpec, TxNumber};
+use reth_primitives::{
+    snapshot::HighestSnapshots, BlockNumber, ChainSpec, SnapshotSegment, TxNumber,
+};
 use reth_provider::{BlockReader, DatabaseProviderRO, ProviderFactory};
-use std::{collections::HashMap, ops::RangeInclusive, sync::Arc};
+use std::{collections::HashMap, ops::RangeInclusive, path::PathBuf, sync::Arc};
 use tokio::sync::watch;
 use tracing::warn;
 
@@ -20,6 +22,8 @@ pub type SnapshotterWithResult<DB> = (Snapshotter<DB>, SnapshotterResult);
 pub struct Snapshotter<DB> {
     /// Provider factory
     provider_factory: ProviderFactory<DB>,
+    /// Directory where snapshots are located
+    snapshots_path: PathBuf,
     /// Highest snapshotted block numbers for each segment
     highest_snapshots: HighestSnapshots,
     /// Channel sender to notify other components of the new highest snapshots
@@ -79,11 +83,22 @@ impl SnapshotTargets {
 
 impl<DB: Database> Snapshotter<DB> {
     /// Creates a new [Snapshotter].
-    pub fn new(db: DB, chain_spec: Arc<ChainSpec>, block_interval: u64) -> Self {
+    pub fn new(
+        db: DB,
+        snapshots_path: PathBuf,
+        chain_spec: Arc<ChainSpec>,
+        block_interval: u64,
+    ) -> RethResult<Self> {
+        // Create directory for snapshots if it doesn't exist.
+        if !snapshots_path.exists() {
+            reth_primitives::fs::create_dir_all(&snapshots_path)?;
+        }
+
         let (highest_snapshots_notifier, highest_snapshots_tracker) = watch::channel(None);
 
-        let snapshotter = Self {
+        let mut snapshotter = Self {
             provider_factory: ProviderFactory::new(db, chain_spec),
+            snapshots_path,
             // TODO(alexey): fill from on-disk snapshot data
             highest_snapshots: HighestSnapshots::default(),
             highest_snapshots_notifier,
@@ -91,9 +106,9 @@ impl<DB: Database> Snapshotter<DB> {
             block_interval,
         };
 
-        snapshotter.update_highest_snapshots_tracker();
+        snapshotter.update_highest_snapshots_tracker()?;
 
-        snapshotter
+        Ok(snapshotter)
     }
 
     #[cfg(test)]
@@ -109,10 +124,37 @@ impl<DB: Database> Snapshotter<DB> {
         }
     }
 
-    fn update_highest_snapshots_tracker(&self) {
+    /// Looks into the snapshot directory to find the highest snapshotted block of each segment, and
+    /// notifies every tracker.
+    fn update_highest_snapshots_tracker(&mut self) -> RethResult<()> {
+        // It walks over the directory and parses the snapshot filenames extracting
+        // `SnapshotSegment` and their inclusive range. It then takes the maximum block
+        // number for each specific segment.
+        for (segment, range) in reth_primitives::fs::read_dir(&self.snapshots_path)?
+            .filter_map(Result::ok)
+            .filter_map(|entry| {
+                if let Ok(true) = entry.metadata().map(|metadata| metadata.is_file()) {
+                    return SnapshotSegment::parse_filename(&entry.file_name().to_string_lossy())
+                }
+                None
+            })
+        {
+            let max_segment_block = match segment {
+                SnapshotSegment::Headers => &mut self.highest_snapshots.headers,
+                SnapshotSegment::Transactions => &mut self.highest_snapshots.transactions,
+                SnapshotSegment::Receipts => &mut self.highest_snapshots.receipts,
+            };
+
+            if max_segment_block.map_or(true, |block| block < *range.end()) {
+                *max_segment_block = Some(*range.end());
+            }
+        }
+
         let _ = self.highest_snapshots_notifier.send(Some(self.highest_snapshots)).map_err(|_| {
             warn!(target: "snapshot", "Highest snapshots channel closed");
         });
+
+        Ok(())
     }
 
     /// Returns a new [`HighestSnapshotsTracker`].
@@ -127,7 +169,7 @@ impl<DB: Database> Snapshotter<DB> {
 
         // TODO(alexey): snapshot logic
 
-        self.update_highest_snapshots_tracker();
+        self.update_highest_snapshots_tracker()?;
 
         Ok(targets)
     }
@@ -246,8 +288,10 @@ mod tests {
     #[test]
     fn new() {
         let tx = TestTransaction::default();
-
-        let snapshotter = Snapshotter::new(tx.inner_raw(), MAINNET.clone(), 2);
+        let snapshots_dir = tempfile::TempDir::new().unwrap();
+        let snapshotter =
+            Snapshotter::new(tx.inner_raw(), snapshots_dir.into_path(), MAINNET.clone(), 2)
+                .unwrap();
 
         assert_eq!(
             *snapshotter.highest_snapshot_receiver().borrow(),
@@ -258,12 +302,15 @@ mod tests {
     #[test]
     fn get_snapshot_targets() {
         let tx = TestTransaction::default();
+        let snapshots_dir = tempfile::TempDir::new().unwrap();
         let mut rng = generators::rng();
 
         let blocks = random_block_range(&mut rng, 0..=3, B256::ZERO, 2..3);
         tx.insert_blocks(blocks.iter(), None).expect("insert blocks");
 
-        let mut snapshotter = Snapshotter::new(tx.inner_raw(), MAINNET.clone(), 2);
+        let mut snapshotter =
+            Snapshotter::new(tx.inner_raw(), snapshots_dir.into_path(), MAINNET.clone(), 2)
+                .unwrap();
 
         // Snapshot targets has data per part up to the passed finalized block number,
         // respecting the block interval