[Feature]: Add Opstack superchain registry support for genesis files (#14260)

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

.github/assets/check_wasm.sh

@@ -44,6 +44,7 @@ exclude_crates=(
   reth-optimism-node
   reth-optimism-payload-builder
   reth-optimism-rpc
+  reth-optimism-chain-registry
   reth-rpc
   reth-rpc-api
   reth-rpc-api-testing-util

Cargo.lock

@@ -8353,6 +8353,19 @@ dependencies = [
  "reth-storage-api",
 ]
 
+[[package]]
+name = "reth-optimism-chain-registry"
+version = "1.2.0"
+dependencies = [
+ "eyre",
+ "reqwest",
+ "reth-fs-util",
+ "serde_json",
+ "tempfile",
+ "tracing",
+ "zstd",
+]
+
 [[package]]
 name = "reth-optimism-chainspec"
 version = "1.2.0"

Cargo.toml

@@ -73,6 +73,7 @@ members = [
     "crates/optimism/evm/",
     "crates/optimism/hardforks/",
     "crates/optimism/node/",
+    "crates/optimism/chain-registry/",
     "crates/optimism/payload/",
     "crates/optimism/primitives/",
     "crates/optimism/reth/",
@@ -377,6 +378,7 @@ reth-optimism-node = { path = "crates/optimism/node" }
 reth-node-types = { path = "crates/node/types" }
 reth-op = { path = "crates/optimism/reth" }
 reth-optimism-chainspec = { path = "crates/optimism/chainspec" }
+reth-optimism-chain-resitry = { path = "crates/optimism/chain-registry" }
 reth-optimism-cli = { path = "crates/optimism/cli" }
 reth-optimism-consensus = { path = "crates/optimism/consensus" }
 reth-optimism-forks = { path = "crates/optimism/hardforks", default-features = false }
@@ -490,6 +492,7 @@ cfg-if = "1.0"
 clap = "4"
 dashmap = "6.0"
 derive_more = { version = "1", default-features = false, features = ["full"] }
+dirs-next = "2.0.0"
 dyn-clone = "1.0.17"
 eyre = "0.6"
 fdlimit = "0.3.0"

crates/optimism/chain-registry/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "reth-optimism-chain-registry"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+exclude.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+reth-fs-util.workspace = true
+
+# misc
+serde_json = { workspace = true, features = ["std"] }
+zstd.workspace = true
+eyre.workspace = true
+
+# tracing
+tracing.workspace = true
+
+# async
+reqwest = { workspace = true, features = ["blocking", "rustls-tls"] }
+
+[dev-dependencies]
+tempfile.workspace = true

crates/optimism/chain-registry/src/client.rs

@@ -0,0 +1,124 @@
+//! Directory Manager downloads and manages files from the op-superchain-registry
+
+use eyre::Context;
+use reth_fs_util as fs;
+use reth_fs_util::Result;
+use serde_json::Value;
+use std::path::{Path, PathBuf};
+use tracing::{debug, trace};
+use zstd::{dict::DecoderDictionary, stream::read::Decoder};
+
+/// Directory manager that handles caching and downloading of genesis files
+#[derive(Debug)]
+pub struct SuperChainRegistryManager {
+    base_path: PathBuf,
+}
+
+impl SuperChainRegistryManager {
+    const DICT_URL: &'static str = "https://raw.githubusercontent.com/ethereum-optimism/superchain-registry/main/superchain/extra/dictionary";
+    const GENESIS_BASE_URL: &'static str = "https://raw.githubusercontent.com/ethereum-optimism/superchain-registry/main/superchain/extra/genesis";
+
+    /// Create a new registry manager with the given base path
+    pub fn new(base_path: impl Into<PathBuf>) -> Result<Self> {
+        let base_path = base_path.into();
+        fs::create_dir_all(&base_path)?;
+        Ok(Self { base_path })
+    }
+
+    /// Get the path to the dictionary file
+    pub fn dictionary_path(&self) -> PathBuf {
+        self.base_path.join("dictionary")
+    }
+
+    /// Get the path to a genesis file for the given network (`mainnet`, `base`).
+    pub fn genesis_path(&self, network_type: &str, network: &str) -> PathBuf {
+        self.base_path.join(network_type).join(format!("{}.json.zst", network))
+    }
+
+    /// Read file from the given path
+    fn read_file(&self, path: &Path) -> Result<Vec<u8>> {
+        fs::read(path)
+    }
+
+    /// Save data to the given path
+    fn save_file(&self, path: &Path, data: &[u8]) -> Result<()> {
+        if let Some(parent) = path.parent() {
+            fs::create_dir_all(parent)?;
+        }
+        fs::write(path, data)
+    }
+
+    /// Download a file from the given URL
+    fn download_file(&self, url: &str, path: &Path) -> eyre::Result<Vec<u8>> {
+        if path.exists() {
+            debug!(target: "reth::cli", path = ?path.display() ,"Reading from cache");
+            return Ok(self.read_file(path)?);
+        }
+
+        trace!(target: "reth::cli", url = ?url ,"Downloading from URL");
+        let response = reqwest::blocking::get(url).context("Failed to download file")?;
+
+        if !response.status().is_success() {
+            eyre::bail!("Failed to download: Status {}", response.status());
+        }
+
+        let bytes = response.bytes()?.to_vec();
+        self.save_file(path, &bytes)?;
+
+        Ok(bytes)
+    }
+
+    /// Download and update the dictionary
+    fn update_dictionary(&self) -> eyre::Result<Vec<u8>> {
+        let path = self.dictionary_path();
+        self.download_file(Self::DICT_URL, &path)
+    }
+
+    /// Get genesis data for a network, downloading it if necessary
+    pub fn get_genesis(&self, network_type: &str, network: &str) -> eyre::Result<Value> {
+        let dict_bytes = self.update_dictionary()?;
+        trace!(target: "reth::cli", bytes = ?dict_bytes.len(),"Got dictionary");
+
+        let dictionary = DecoderDictionary::copy(&dict_bytes);
+
+        let url = format!("{}/{}/{}.json.zst", Self::GENESIS_BASE_URL, network_type, network);
+        let path = self.genesis_path(network_type, network);
+
+        let compressed_bytes = self.download_file(&url, &path)?;
+        trace!(target: "reth::cli", bytes = ?compressed_bytes.len(),"Got genesis file");
+
+        let decoder = Decoder::with_prepared_dictionary(&compressed_bytes[..], &dictionary)
+            .context("Failed to create decoder with dictionary")?;
+
+        let json: Value = serde_json::from_reader(decoder)
+            .with_context(|| format!("Failed to parse JSON: {path:?}"))?;
+
+        Ok(json)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use eyre::Result;
+
+    #[test]
+    fn test_directory_manager() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        // Create a temporary directory for testing
+        let manager = SuperChainRegistryManager::new(dir.path())?;
+
+        assert!(!manager.genesis_path("mainnet", "base").exists());
+        // Test downloading genesis data
+        let json_data = manager.get_genesis("mainnet", "base")?;
+        assert!(json_data.is_object(), "Parsed JSON should be an object");
+
+        assert!(manager.genesis_path("mainnet", "base").exists());
+
+        // Test using cached data
+        let cached_json_data = manager.get_genesis("mainnet", "base")?;
+        assert!(cached_json_data.is_object(), "Cached JSON should be an object");
+
+        Ok(())
+    }
+}

crates/optimism/chain-registry/src/lib.rs

@@ -0,0 +1,15 @@
+//! Utilities for interacting the the optimism superchain registry
+
+#![doc(
+    html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
+    html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
+    issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
+)]
+#![cfg_attr(not(test), warn(unused_crate_dependencies))]
+#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
+
+//! Downloads and maintains config for different chains which
+//! are part of the op superchain
+mod client;
+
+pub use client::*;