feat: gracefully shutdown prometheus server (#7728)

Co-authored-by: Oliver Nordbjerg <hi@notbjerg.me>
Co-authored-by: Oliver Nordbjerg <onbjerg@users.noreply.github.com>

crates/node-builder/src/builder.rs

@@ -514,6 +514,7 @@ where
                 prometheus_handle,
                 database.clone(),
                 provider_factory.static_file_provider(),
+                executor.clone(),
             )
             .await?;
 

crates/node-core/src/metrics/prometheus_exporter.rs

@@ -12,6 +12,7 @@ use metrics_util::layers::{PrefixLayer, Stack};
 use reth_db::database_metrics::DatabaseMetrics;
 use reth_metrics::metrics::Unit;
 use reth_provider::providers::StaticFileProvider;
+use reth_tasks::TaskExecutor;
 use std::{convert::Infallible, net::SocketAddr, sync::Arc};
 
 pub(crate) trait Hook: Fn() + Send + Sync {}
@@ -39,13 +40,19 @@ pub(crate) async fn serve_with_hooks<F: Hook + 'static>(
     listen_addr: SocketAddr,
     handle: PrometheusHandle,
     hooks: impl IntoIterator<Item = F>,
+    task_executor: TaskExecutor,
 ) -> eyre::Result<()> {
     let hooks: Vec<_> = hooks.into_iter().collect();
 
     // Start endpoint
-    start_endpoint(listen_addr, handle, Arc::new(move || hooks.iter().for_each(|hook| hook())))
-        .await
-        .wrap_err("Could not start Prometheus endpoint")?;
+    start_endpoint(
+        listen_addr,
+        handle,
+        Arc::new(move || hooks.iter().for_each(|hook| hook())),
+        task_executor,
+    )
+    .await
+    .wrap_err("Could not start Prometheus endpoint")?;
 
     Ok(())
 }
@@ -55,6 +62,7 @@ async fn start_endpoint<F: Hook + 'static>(
     listen_addr: SocketAddr,
     handle: PrometheusHandle,
     hook: Arc<F>,
+    task_executor: TaskExecutor,
 ) -> eyre::Result<()> {
     let make_svc = make_service_fn(move |_| {
         let handle = handle.clone();
@@ -67,10 +75,20 @@ async fn start_endpoint<F: Hook + 'static>(
             }))
         }
     });
+
     let server =
         Server::try_bind(&listen_addr).wrap_err("Could not bind to address")?.serve(make_svc);
 
-    tokio::spawn(async move { server.await.expect("Metrics endpoint crashed") });
+    task_executor.spawn_with_graceful_shutdown_signal(move |signal| async move {
+        if let Err(error) = server
+            .with_graceful_shutdown(async move {
+                let _ = signal.await;
+            })
+            .await
+        {
+            tracing::error!(%error, "metrics endpoint crashed")
+        }
+    });
 
     Ok(())
 }
@@ -82,6 +100,7 @@ pub async fn serve<Metrics>(
     db: Metrics,
     static_file_provider: StaticFileProvider,
     process: metrics_process::Collector,
+    task_executor: TaskExecutor,
 ) -> eyre::Result<()>
 where
     Metrics: DatabaseMetrics + 'static + Send + Sync,
@@ -102,7 +121,7 @@ where
         Box::new(collect_memory_stats),
         Box::new(collect_io_stats),
     ];
-    serve_with_hooks(listen_addr, handle, hooks).await?;
+    serve_with_hooks(listen_addr, handle, hooks, task_executor).await?;
 
     // We describe the metrics after the recorder is installed, otherwise this information is not
     // registered

crates/node-core/src/node_config.rs

@@ -365,6 +365,7 @@ impl NodeConfig {
         prometheus_handle: PrometheusHandle,
         db: Metrics,
         static_file_provider: StaticFileProvider,
+        task_executor: TaskExecutor,
     ) -> eyre::Result<()>
     where
         Metrics: DatabaseMetrics + 'static + Send + Sync,
@@ -377,6 +378,7 @@ impl NodeConfig {
                 db,
                 static_file_provider,
                 metrics_process::Collector::default(),
+                task_executor,
             )
             .await?;
         }