feat: exex manager (#7340)

Co-authored-by: Alexey Shekhirin <a.shekhirin@gmail.com>

crates/node-builder/Cargo.toml

@@ -36,7 +36,6 @@ reth-prune.workspace = true
 reth-stages.workspace = true
 reth-config.workspace = true
 
-
 ## async
 futures.workspace = true
 tokio = { workspace = true, features = [

crates/node-builder/src/builder.rs

@@ -14,7 +14,7 @@ use crate::{
     Node, NodeHandle,
 };
 use eyre::Context;
-use futures::{future::Either, stream, stream_select, Future, StreamExt};
+use futures::{future, future::Either, stream, stream_select, Future, StreamExt};
 use rayon::ThreadPoolBuilder;
 use reth_beacon_consensus::{
     hooks::{EngineHooks, PruneHook, StaticFileHook},
@@ -28,7 +28,7 @@ use reth_db::{
     test_utils::{create_test_rw_db, TempDatabase},
     DatabaseEnv,
 };
-use reth_exex::ExExContext;
+use reth_exex::{ExExContext, ExExHandle, ExExManager};
 use reth_interfaces::p2p::either::EitherDownloader;
 use reth_network::{NetworkBuilder, NetworkConfig, NetworkEvents, NetworkHandle};
 use reth_node_api::{FullNodeTypes, FullNodeTypesAdapter, NodeTypes};
@@ -44,7 +44,9 @@ use reth_node_core::{
     utils::write_peers_to_file,
 };
 use reth_primitives::{constants::eip4844::MAINNET_KZG_TRUSTED_SETUP, format_ether, ChainSpec};
-use reth_provider::{providers::BlockchainProvider, ChainSpecProvider, ProviderFactory};
+use reth_provider::{
+    providers::BlockchainProvider, CanonStateSubscriptions, ChainSpecProvider, ProviderFactory,
+};
 use reth_prune::PrunerBuilder;
 use reth_revm::EvmProcessorFactory;
 use reth_rpc_engine_api::EngineApi;
@@ -434,7 +436,11 @@ where
     }
 
     /// Installs an ExEx (Execution Extension) in the node.
-    pub fn install_exex<F, R, E>(mut self, exex: F) -> Self
+    ///
+    /// # Note
+    ///
+    /// The ExEx ID must be unique.
+    pub fn install_exex<F, R, E>(mut self, exex_id: impl Into<String>, exex: F) -> Self
     where
         F: Fn(
                 ExExContext<
@@ -449,7 +455,7 @@ where
         R: Future<Output = eyre::Result<E>> + Send,
         E: Future<Output = eyre::Result<()>> + Send,
     {
-        self.state.exexs.push(Box::new(exex));
+        self.state.exexs.push((exex_id.into(), Box::new(exex)));
         self
     }
 
@@ -561,8 +567,6 @@ where
         let NodeComponents { transaction_pool, network, payload_builder } =
             components_builder.build_components(&ctx).await?;
 
-        // TODO(alexey): launch ExExs and consume their events
-
         let BuilderContext {
             provider: blockchain_db,
             executor,
@@ -585,6 +589,69 @@ where
         debug!(target: "reth::cli", "calling on_component_initialized hook");
         on_component_initialized.on_event(node_components.clone())?;
 
+        // spawn exexs
+        let mut exex_handles = Vec::with_capacity(self.state.exexs.len());
+        let mut exexs = Vec::with_capacity(self.state.exexs.len());
+        for (id, exex) in self.state.exexs {
+            // create a new exex handle
+            let (handle, events, notifications) = ExExHandle::new(id.clone());
+            exex_handles.push(handle);
+
+            // create the launch context for the exex
+            let context = ExExContext {
+                head,
+                provider: blockchain_db.clone(),
+                task_executor: executor.clone(),
+                data_dir: data_dir.clone(),
+                config: config.clone(),
+                reth_config: reth_config.clone(),
+                events,
+                notifications,
+            };
+
+            let executor = executor.clone();
+            exexs.push(async move {
+                debug!(target: "reth::cli", id, "spawning exex");
+                let span = reth_tracing::tracing::info_span!("exex", id);
+                let _enter = span.enter();
+
+                // init the exex
+                let exex = exex.launch(context).await.unwrap();
+
+                // spawn it as a crit task
+                executor.spawn_critical("exex", async move {
+                    info!(target: "reth::cli", id, "ExEx started");
+                    exex.await.unwrap_or_else(|_| panic!("exex {} crashed", id))
+                });
+            });
+        }
+
+        future::join_all(exexs).await;
+
+        // spawn exex manager
+        if !exex_handles.is_empty() {
+            debug!(target: "reth::cli", "spawning exex manager");
+            // todo(onbjerg): rm magic number
+            let exex_manager = ExExManager::new(exex_handles, 1024);
+            let mut exex_manager_handle = exex_manager.handle();
+            executor.spawn_critical("exex manager", async move {
+                exex_manager.await.expect("exex manager crashed");
+            });
+
+            // send notifications from the blockchain tree to exex manager
+            let mut canon_state_notifications = blockchain_tree.subscribe_to_canonical_state();
+            executor.spawn_critical("exex manager blockchain tree notifications", async move {
+                while let Ok(notification) = canon_state_notifications.recv().await {
+                    exex_manager_handle
+                        .send_async(notification)
+                        .await
+                        .expect("blockchain tree notification could not be sent to exex manager");
+                }
+            });
+
+            info!(target: "reth::cli", "ExEx Manager started");
+        }
+
         // create pipeline
         let network_client = network.fetch_client().await?;
         let (consensus_engine_tx, mut consensus_engine_rx) = unbounded_channel();
@@ -1070,7 +1137,7 @@ where
     }
 
     /// Installs an ExEx (Execution Extension) in the node.
-    pub fn install_exex<F, R, E>(mut self, exex: F) -> Self
+    pub fn install_exex<F, R, E>(mut self, exex_id: impl Into<String>, exex: F) -> Self
     where
         F: Fn(
                 ExExContext<
@@ -1085,7 +1152,7 @@ where
         R: Future<Output = eyre::Result<E>> + Send,
         E: Future<Output = eyre::Result<()>> + Send,
     {
-        self.builder.state.exexs.push(Box::new(exex));
+        self.builder.state.exexs.push((exex_id.into(), Box::new(exex)));
         self
     }
 
@@ -1301,7 +1368,7 @@ pub struct ComponentsState<Types, Components, FullNode: FullNodeComponents> {
     /// Additional RPC hooks.
     rpc: RpcHooks<FullNode>,
     /// The ExExs (execution extensions) of the node.
-    exexs: Vec<Box<dyn BoxedLaunchExEx<FullNode>>>,
+    exexs: Vec<(String, Box<dyn BoxedLaunchExEx<FullNode>>)>,
 }
 
 impl<Types, Components, FullNode: FullNodeComponents> std::fmt::Debug

crates/node-builder/src/exex.rs

@@ -1,33 +1,4 @@
-#![allow(dead_code)]
-// todo: expand this (examples, assumptions, invariants)
-//! Execution extensions (ExEx).
-//!
-//! An execution extension is a task that derives its state from Reth's state.
-//!
-//! Some examples of state such state derives are rollups, bridges, and indexers.
-//!
-//! An ExEx is a [`Future`] resolving to a `Result<()>` that is run indefinitely alongside Reth.
-//!
-//! ExEx's are initialized using an async closure that resolves to the ExEx; this closure gets
-//! passed an [`ExExContext`] where it is possible to spawn additional tasks and modify Reth.
-//!
-//! Most ExEx's will want to derive their state from the [`CanonStateNotification`] channel given in
-//! [`ExExContext`]. A new notification is emitted whenever blocks are executed in live and
-//! historical sync.
-//!
-//! # Pruning
-//!
-//! ExEx's **SHOULD** emit an `ExExEvent::FinishedHeight` event to signify what blocks have been
-//! processed. This event is used by Reth to determine what state can be pruned.
-//!
-//! An ExEx will not receive notifications for blocks less than the block emitted in the event. To
-//! clarify: if the ExEx emits `ExExEvent::FinishedHeight(0)` it will receive notifications for any
-//! `block_number >= 0`.
-//!
-//! [`Future`]: std::future::Future
-//! [`ExExContext`]: reth_exex::ExExContext
-//! [`CanonStateNotification`]: reth_provider::CanonStateNotification
-
+//! Types for launching execution extensions (ExEx).
 use crate::FullNodeTypes;
 use futures::{future::BoxFuture, FutureExt};
 use reth_exex::ExExContext;