From ea44c1d63f61f5a86dbd023bb13c8de160acf0c5 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 1 Mar 2024 16:36:50 +0000 Subject: [PATCH] feat(node-core, storage): static files segment metrics (#6908) --- bin/reth/src/commands/db/stats.rs | 14 +- bin/reth/src/commands/stage/run.rs | 1 + crates/node-builder/src/builder.rs | 15 +- .../src/metrics/prometheus_exporter.rs | 16 + crates/node-core/src/node_config.rs | 9 +- crates/primitives/src/fs.rs | 20 + .../src/providers/static_file/manager.rs | 39 + .../src/providers/static_file/metrics.rs | 39 +- etc/grafana/dashboards/overview.json | 879 ++++++++++++++---- 9 files changed, 819 insertions(+), 213 deletions(-) diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs index 09ad3a61b..37228356d 100644 --- a/bin/reth/src/commands/db/stats.rs +++ b/bin/reth/src/commands/db/stats.rs @@ -8,7 +8,6 @@ use reth_db::{database::Database, mdbx, static_file::iter_static_files, Database use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::static_file::{find_fixed_range, SegmentRangeInclusive}; use reth_provider::providers::StaticFileProvider; -use std::fs::File; #[derive(Parser, Debug)] /// The arguments for the `reth db stats` command @@ -174,20 +173,17 @@ impl Command { let columns = jar_provider.columns(); let rows = jar_provider.rows(); - let data_size = File::open(jar_provider.data_path()) - .and_then(|file| file.metadata()) + + let data_size = reth_primitives::fs::metadata(jar_provider.data_path()) .map(|metadata| metadata.len()) .unwrap_or_default(); - let index_size = File::open(jar_provider.index_path()) - .and_then(|file| file.metadata()) + let index_size = reth_primitives::fs::metadata(jar_provider.index_path()) .map(|metadata| metadata.len()) .unwrap_or_default(); - let offsets_size = File::open(jar_provider.offsets_path()) - .and_then(|file| file.metadata()) + let offsets_size = reth_primitives::fs::metadata(jar_provider.offsets_path()) .map(|metadata| metadata.len()) .unwrap_or_default(); - let config_size = File::open(jar_provider.config_path()) - .and_then(|file| file.metadata()) + let config_size = reth_primitives::fs::metadata(jar_provider.config_path()) .map(|metadata| metadata.len()) .unwrap_or_default(); diff --git a/bin/reth/src/commands/stage/run.rs b/bin/reth/src/commands/stage/run.rs index e0016e646..23f749164 100644 --- a/bin/reth/src/commands/stage/run.rs +++ b/bin/reth/src/commands/stage/run.rs @@ -144,6 +144,7 @@ impl Command { listen_addr, prometheus_exporter::install_recorder()?, Arc::clone(&db), + factory.static_file_provider(), metrics_process::Collector::default(), ) .await?; diff --git a/crates/node-builder/src/builder.rs b/crates/node-builder/src/builder.rs index 9b43e52f2..e1fab7761 100644 --- a/crates/node-builder/src/builder.rs +++ b/crates/node-builder/src/builder.rs @@ -395,17 +395,22 @@ where // Does not do anything on windows. fdlimit::raise_fd_limit()?; - let prometheus_handle = config.install_prometheus_recorder()?; - config.start_metrics_endpoint(prometheus_handle, database.clone()).await?; - - info!(target: "reth::cli", "Database opened"); - let provider_factory = ProviderFactory::new( database.clone(), Arc::clone(&config.chain), data_dir.static_files_path(), )? .with_static_files_metrics(); + info!(target: "reth::cli", "Database opened"); + + let prometheus_handle = config.install_prometheus_recorder()?; + config + .start_metrics_endpoint( + prometheus_handle, + database.clone(), + provider_factory.static_file_provider(), + ) + .await?; debug!(target: "reth::cli", chain=%config.chain.chain, genesis=?config.chain.genesis_hash(), "Initializing genesis"); diff --git a/crates/node-core/src/metrics/prometheus_exporter.rs b/crates/node-core/src/metrics/prometheus_exporter.rs index 4bfcddf0c..952c02fd8 100644 --- a/crates/node-core/src/metrics/prometheus_exporter.rs +++ b/crates/node-core/src/metrics/prometheus_exporter.rs @@ -11,6 +11,7 @@ use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; use metrics_util::layers::{PrefixLayer, Stack}; use reth_db::database_metrics::DatabaseMetrics; use reth_metrics::metrics::Unit; +use reth_provider::providers::StaticFileProvider; use std::{convert::Infallible, net::SocketAddr, sync::Arc}; pub(crate) trait Hook: Fn() + Send + Sync {} @@ -79,17 +80,24 @@ pub async fn serve( listen_addr: SocketAddr, handle: PrometheusHandle, db: Metrics, + static_file_provider: StaticFileProvider, process: metrics_process::Collector, ) -> eyre::Result<()> where Metrics: DatabaseMetrics + 'static + Send + Sync, { let db_metrics_hook = move || db.report_metrics(); + let static_file_metrics_hook = move || { + let _ = static_file_provider.report_metrics().map_err( + |error| tracing::error!(%error, "Failed to report static file provider metrics"), + ); + }; // Clone `process` to move it into the hook and use the original `process` for describe below. let cloned_process = process.clone(); let hooks: Vec>> = vec![ Box::new(db_metrics_hook), + Box::new(static_file_metrics_hook), Box::new(move || cloned_process.collect()), Box::new(collect_memory_stats), Box::new(collect_io_stats), @@ -106,6 +114,14 @@ where "db.timed_out_not_aborted_transactions", "Number of timed out transactions that were not aborted by the user yet" ); + + describe_gauge!("static_files.segment_size", Unit::Bytes, "The size of a static file segment"); + describe_gauge!("static_files.segment_files", "The number of files for a static file segment"); + describe_gauge!( + "static_files.segment_entries", + "The number of entries for a static file segment" + ); + process.describe(); describe_memory_stats(); describe_io_stats(); diff --git a/crates/node-core/src/node_config.rs b/crates/node-core/src/node_config.rs index 7eee5252f..95ebc044d 100644 --- a/crates/node-core/src/node_config.rs +++ b/crates/node-core/src/node_config.rs @@ -47,9 +47,10 @@ use reth_primitives::{ BlockHashOrNumber, BlockNumber, ChainSpec, Head, SealedHeader, TxHash, B256, MAINNET, }; use reth_provider::{ - providers::BlockchainProvider, BlockHashReader, BlockNumReader, BlockReader, - BlockchainTreePendingStateProvider, CanonStateSubscriptions, HeaderProvider, HeaderSyncMode, - ProviderFactory, StageCheckpointReader, + providers::{BlockchainProvider, StaticFileProvider}, + BlockHashReader, BlockNumReader, BlockReader, BlockchainTreePendingStateProvider, + CanonStateSubscriptions, HeaderProvider, HeaderSyncMode, ProviderFactory, + StageCheckpointReader, }; use reth_revm::EvmProcessorFactory; use reth_stages::{ @@ -599,6 +600,7 @@ impl NodeConfig { &self, prometheus_handle: PrometheusHandle, db: Metrics, + static_file_provider: StaticFileProvider, ) -> eyre::Result<()> where Metrics: DatabaseMetrics + 'static + Send + Sync, @@ -609,6 +611,7 @@ impl NodeConfig { listen_addr, prometheus_handle, db, + static_file_provider, metrics_process::Collector::default(), ) .await?; diff --git a/crates/primitives/src/fs.rs b/crates/primitives/src/fs.rs index f9dac8fb5..1bcb908db 100644 --- a/crates/primitives/src/fs.rs +++ b/crates/primitives/src/fs.rs @@ -119,6 +119,15 @@ pub enum FsPathError { /// The path related to the operation. path: PathBuf, }, + + /// Error variant for failed file metadata operation with additional path context. + #[error("failed to get metadata for {path:?}: {source}")] + Metadata { + /// The source `io::Error`. + source: io::Error, + /// The path related to the operation. + path: PathBuf, + }, } impl FsPathError { @@ -171,6 +180,11 @@ impl FsPathError { pub fn rename(source: io::Error, from: impl Into, to: impl Into) -> Self { FsPathError::Rename { source, from: from.into(), to: to.into() } } + + /// Returns the complementary error variant for [`std::fs::File::metadata`]. + pub fn metadata(source: io::Error, path: impl Into) -> Self { + FsPathError::Metadata { source, path: path.into() } + } } type Result = std::result::Result; @@ -225,3 +239,9 @@ pub fn rename(from: impl AsRef, to: impl AsRef) -> Result<()> { let to = to.as_ref(); fs::rename(from, to).map_err(|err| FsPathError::rename(err, from, to)) } + +/// Wrapper for `std::fs::metadata` +pub fn metadata(path: impl AsRef) -> Result { + let path = path.as_ref(); + fs::metadata(path).map_err(|err| FsPathError::metadata(err, path)) +} diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 1d5a36bd5..5a27a2a98 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -114,6 +114,45 @@ impl StaticFileProvider { Self(Arc::new(provider)) } + /// Reports metrics for the static files. + pub fn report_metrics(&self) -> ProviderResult<()> { + let Some(metrics) = &self.metrics else { return Ok(()) }; + + let static_files = iter_static_files(&self.path)?; + for (segment, ranges) in static_files { + let mut entries = 0; + let mut size = 0; + + for (block_range, _) in &ranges { + let fixed_block_range = find_fixed_range(block_range.start()); + let jar_provider = self + .get_segment_provider(segment, || Some(fixed_block_range), None)? + .ok_or(ProviderError::MissingStaticFileBlock(segment, block_range.start()))?; + + entries += jar_provider.rows(); + + let data_size = reth_primitives::fs::metadata(jar_provider.data_path()) + .map(|metadata| metadata.len()) + .unwrap_or_default(); + let index_size = reth_primitives::fs::metadata(jar_provider.index_path()) + .map(|metadata| metadata.len()) + .unwrap_or_default(); + let offsets_size = reth_primitives::fs::metadata(jar_provider.offsets_path()) + .map(|metadata| metadata.len()) + .unwrap_or_default(); + let config_size = reth_primitives::fs::metadata(jar_provider.config_path()) + .map(|metadata| metadata.len()) + .unwrap_or_default(); + + size += data_size + index_size + offsets_size + config_size; + } + + metrics.record_segment(segment, size, ranges.len(), entries); + } + + Ok(()) + } + /// Gets the [`StaticFileJarProvider`] of the requested segment and block. pub fn get_segment_provider_from_block( &self, diff --git a/crates/storage/provider/src/providers/static_file/metrics.rs b/crates/storage/provider/src/providers/static_file/metrics.rs index 497620b64..f1a4204a7 100644 --- a/crates/storage/provider/src/providers/static_file/metrics.rs +++ b/crates/storage/provider/src/providers/static_file/metrics.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, time::Duration}; use itertools::Itertools; -use metrics::{Counter, Histogram}; +use metrics::{Counter, Gauge, Histogram}; use reth_metrics::Metrics; use reth_primitives::StaticFileSegment; use strum::{EnumIter, IntoEnumIterator}; @@ -9,6 +9,7 @@ use strum::{EnumIter, IntoEnumIterator}; /// Metrics for the static file provider. #[derive(Debug)] pub struct StaticFileProviderMetrics { + segments: HashMap, segment_operations: HashMap< (StaticFileSegment, StaticFileProviderOperation), StaticFileProviderOperationMetrics, @@ -18,6 +19,14 @@ pub struct StaticFileProviderMetrics { impl Default for StaticFileProviderMetrics { fn default() -> Self { Self { + segments: StaticFileSegment::iter() + .map(|segment| { + ( + segment, + StaticFileSegmentMetrics::new_with_labels(&[("segment", segment.as_str())]), + ) + }) + .collect(), segment_operations: StaticFileSegment::iter() .cartesian_product(StaticFileProviderOperation::iter()) .map(|(segment, operation)| { @@ -35,6 +44,22 @@ impl Default for StaticFileProviderMetrics { } impl StaticFileProviderMetrics { + pub(crate) fn record_segment( + &self, + segment: StaticFileSegment, + size: u64, + files: usize, + entries: usize, + ) { + self.segments.get(&segment).expect("segment metrics should exist").size.set(size as f64); + self.segments.get(&segment).expect("segment metrics should exist").files.set(files as f64); + self.segments + .get(&segment) + .expect("segment metrics should exist") + .entries + .set(entries as f64); + } + pub(crate) fn record_segment_operation( &self, segment: StaticFileSegment, @@ -80,6 +105,18 @@ impl StaticFileProviderOperation { } } +/// Metrics for a specific static file segment. +#[derive(Metrics)] +#[metrics(scope = "static_files.segment")] +pub(crate) struct StaticFileSegmentMetrics { + /// The size of a static file segment + size: Gauge, + /// The number of files for a static file segment + files: Gauge, + /// The number of entries for a static file segment + entries: Gauge, +} + #[derive(Metrics)] #[metrics(scope = "static_files.jar_provider")] pub(crate) struct StaticFileProviderOperationMetrics { diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 6470e01b1..6a8c1b693 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -158,9 +158,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -225,9 +223,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -292,9 +288,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -359,9 +353,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -426,9 +418,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -493,9 +483,7 @@ "justifyMode": "auto", "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -571,9 +559,7 @@ "options": { "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -638,9 +624,7 @@ "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["last"], "fields": "", "values": false }, @@ -1500,22 +1484,16 @@ }, "id": 48, "options": { - "displayLabels": [ - "name" - ], + "displayLabels": ["name"], "legend": { "displayMode": "table", "placement": "right", "showLegend": true, - "values": [ - "value" - ] + "values": ["value"] }, "pieType": "pie", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1672,15 +1650,11 @@ "displayMode": "table", "placement": "right", "showLegend": true, - "values": [ - "value" - ] + "values": ["value"] }, "pieType": "pie", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1840,9 +1814,7 @@ "footer": { "countRows": false, "fields": "", - "reducer": [ - "sum" - ], + "reducer": ["sum"], "show": false }, "showHeader": true @@ -2093,6 +2065,572 @@ "x": 0, "y": 69 }, + "id": 203, + "panels": [], + "title": "Static Files", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The size of segments in the static files", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 70 + }, + "id": 202, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": ["value"] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_static_files_segment_size{instance=~\"$instance\"}", + "interval": "", + "legendFormat": "{{segment}}", + "range": true, + "refId": "A" + } + ], + "title": "Segments size", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Entries" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "segment" + }, + "properties": [ + { + "id": "displayName", + "value": "Segment" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 70 + }, + "id": 204, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "reth_static_files_segment_entries{instance=~\"$instance\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Entries per segment", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Files" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "segment" + }, + "properties": [ + { + "id": "displayName", + "value": "Segment" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 70 + }, + "id": 205, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "reth_static_files_segment_files{instance=~\"$instance\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Files per segment", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The size of the static files over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 206, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (job) ( reth_static_files_segment_size{instance=~\"$instance\"} )", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Static Files growth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The maximum time the static files operation which commits a writer took.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 207, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(max_over_time(reth_static_files_jar_provider_write_duration_seconds{instance=~\"$instance\", operation=\"commit-writer\", quantile=\"1\"}[$__interval]) > 0) by (segment)", + "legendFormat": "{{segment}}", + "range": true, + "refId": "A" + } + ], + "title": "Max writer commit time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, "id": 46, "panels": [], "repeat": "instance", @@ -2160,7 +2698,7 @@ "h": 8, "w": 24, "x": 0, - "y": 70 + "y": 87 }, "id": 56, "options": { @@ -2233,7 +2771,7 @@ "h": 1, "w": 24, "x": 0, - "y": 78 + "y": 95 }, "id": 6, "panels": [], @@ -2289,8 +2827,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2305,7 +2842,7 @@ "h": 8, "w": 8, "x": 0, - "y": 79 + "y": 96 }, "id": 18, "options": { @@ -2383,8 +2920,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2399,7 +2935,7 @@ "h": 8, "w": 8, "x": 8, - "y": 79 + "y": 96 }, "id": 16, "options": { @@ -2502,8 +3038,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2519,7 +3054,7 @@ "h": 8, "w": 8, "x": 16, - "y": 79 + "y": 96 }, "id": 8, "options": { @@ -2528,9 +3063,7 @@ "displayMode": "table", "placement": "right", "showLegend": true, - "values": [ - "value" - ] + "values": ["value"] }, "tooltip": { "mode": "multi", @@ -2602,7 +3135,7 @@ "h": 8, "w": 8, "x": 0, - "y": 87 + "y": 104 }, "id": 54, "options": { @@ -2613,9 +3146,7 @@ }, "pieType": "pie", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -2807,8 +3338,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2823,7 +3353,7 @@ "h": 8, "w": 14, "x": 8, - "y": 87 + "y": 104 }, "id": 103, "options": { @@ -2860,7 +3390,7 @@ "h": 1, "w": 24, "x": 0, - "y": 95 + "y": 112 }, "id": 24, "panels": [], @@ -2915,8 +3445,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2956,7 +3485,7 @@ "h": 8, "w": 12, "x": 0, - "y": 96 + "y": 113 }, "id": 26, "options": { @@ -3071,8 +3600,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3088,7 +3616,7 @@ "h": 8, "w": 12, "x": 12, - "y": 96 + "y": 113 }, "id": 33, "options": { @@ -3190,8 +3718,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3206,7 +3733,7 @@ "h": 8, "w": 12, "x": 0, - "y": 104 + "y": 121 }, "id": 36, "options": { @@ -3255,7 +3782,7 @@ "h": 1, "w": 24, "x": 0, - "y": 112 + "y": 129 }, "id": 32, "panels": [], @@ -3311,8 +3838,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3361,7 +3887,7 @@ "h": 8, "w": 12, "x": 0, - "y": 113 + "y": 130 }, "id": 30, "options": { @@ -3512,8 +4038,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3525,7 +4050,7 @@ "h": 8, "w": 12, "x": 12, - "y": 113 + "y": 130 }, "id": 28, "options": { @@ -3627,8 +4152,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3643,7 +4167,7 @@ "h": 8, "w": 12, "x": 0, - "y": 121 + "y": 138 }, "id": 35, "options": { @@ -3733,8 +4257,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3767,7 +4290,7 @@ "h": 8, "w": 12, "x": 12, - "y": 121 + "y": 138 }, "id": 73, "options": { @@ -3858,8 +4381,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3892,7 +4414,7 @@ "h": 8, "w": 12, "x": 0, - "y": 129 + "y": 146 }, "id": 102, "options": { @@ -3955,7 +4477,7 @@ "h": 1, "w": 24, "x": 0, - "y": 137 + "y": 154 }, "id": 89, "panels": [], @@ -4011,8 +4533,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4028,7 +4549,7 @@ "h": 8, "w": 12, "x": 0, - "y": 138 + "y": 155 }, "id": 91, "options": { @@ -4141,8 +4662,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4157,7 +4677,7 @@ "h": 8, "w": 12, "x": 12, - "y": 138 + "y": 155 }, "id": 92, "options": { @@ -4270,8 +4790,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4286,7 +4805,7 @@ "h": 8, "w": 12, "x": 0, - "y": 146 + "y": 163 }, "id": 104, "options": { @@ -4365,8 +4884,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4381,7 +4899,7 @@ "h": 8, "w": 12, "x": 12, - "y": 146 + "y": 163 }, "id": 94, "options": { @@ -4459,8 +4977,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4475,7 +4992,7 @@ "h": 8, "w": 12, "x": 0, - "y": 154 + "y": 171 }, "id": 199, "options": { @@ -4585,8 +5102,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4615,7 +5131,7 @@ "h": 8, "w": 12, "x": 12, - "y": 154 + "y": 171 }, "id": 93, "options": { @@ -4729,8 +5245,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4771,7 +5286,7 @@ "h": 8, "w": 12, "x": 0, - "y": 162 + "y": 179 }, "id": 95, "options": { @@ -4875,8 +5390,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4891,7 +5405,7 @@ "h": 8, "w": 12, "x": 12, - "y": 162 + "y": 179 }, "id": 115, "options": { @@ -5025,8 +5539,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5042,7 +5555,7 @@ "h": 8, "w": 12, "x": 0, - "y": 170 + "y": 187 }, "id": 200, "options": { @@ -5212,8 +5725,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5230,9 +5742,7 @@ "id": "byNames", "options": { "mode": "exclude", - "names": [ - "Transactions Manager Future" - ], + "names": ["Transactions Manager Future"], "prefix": "All except:", "readOnly": true } @@ -5254,7 +5764,7 @@ "h": 8, "w": 12, "x": 12, - "y": 170 + "y": 187 }, "id": 201, "options": { @@ -5446,7 +5956,7 @@ "h": 1, "w": 24, "x": 0, - "y": 178 + "y": 195 }, "id": 79, "panels": [], @@ -5502,8 +6012,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5518,7 +6027,7 @@ "h": 8, "w": 12, "x": 0, - "y": 179 + "y": 196 }, "id": 74, "options": { @@ -5597,8 +6106,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5613,7 +6121,7 @@ "h": 8, "w": 12, "x": 12, - "y": 179 + "y": 196 }, "id": 80, "options": { @@ -5692,8 +6200,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5708,7 +6215,7 @@ "h": 8, "w": 12, "x": 0, - "y": 187 + "y": 204 }, "id": 81, "options": { @@ -5786,8 +6293,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5803,7 +6309,7 @@ "h": 8, "w": 12, "x": 12, - "y": 187 + "y": 204 }, "id": 114, "options": { @@ -5882,8 +6388,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5899,7 +6404,7 @@ "h": 8, "w": 12, "x": 0, - "y": 195 + "y": 212 }, "id": 158, "options": { @@ -6004,8 +6509,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6021,7 +6525,7 @@ "h": 8, "w": 12, "x": 12, - "y": 195 + "y": 212 }, "id": 190, "options": { @@ -6059,7 +6563,7 @@ "h": 1, "w": 24, "x": 0, - "y": 203 + "y": 220 }, "id": 87, "panels": [], @@ -6115,8 +6619,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6131,7 +6634,7 @@ "h": 8, "w": 12, "x": 0, - "y": 204 + "y": 221 }, "id": 83, "options": { @@ -6209,8 +6712,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6225,7 +6727,7 @@ "h": 8, "w": 12, "x": 12, - "y": 204 + "y": 221 }, "id": 84, "options": { @@ -6315,8 +6817,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6331,7 +6832,7 @@ "h": 8, "w": 12, "x": 0, - "y": 212 + "y": 229 }, "id": 85, "options": { @@ -6368,7 +6869,7 @@ "h": 1, "w": 24, "x": 0, - "y": 220 + "y": 237 }, "id": 68, "panels": [], @@ -6424,8 +6925,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6440,7 +6940,7 @@ "h": 8, "w": 12, "x": 0, - "y": 221 + "y": 238 }, "id": 60, "options": { @@ -6518,8 +7018,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6534,7 +7033,7 @@ "h": 8, "w": 12, "x": 12, - "y": 221 + "y": 238 }, "id": 62, "options": { @@ -6612,8 +7111,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6628,7 +7126,7 @@ "h": 8, "w": 12, "x": 0, - "y": 229 + "y": 246 }, "id": 64, "options": { @@ -6665,7 +7163,7 @@ "h": 1, "w": 24, "x": 0, - "y": 237 + "y": 254 }, "id": 97, "panels": [], @@ -6718,8 +7216,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6735,7 +7232,7 @@ "h": 8, "w": 12, "x": 0, - "y": 238 + "y": 255 }, "id": 98, "options": { @@ -6879,8 +7376,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6896,7 +7392,7 @@ "h": 8, "w": 12, "x": 12, - "y": 238 + "y": 255 }, "id": 101, "options": { @@ -6975,8 +7471,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6992,7 +7487,7 @@ "h": 8, "w": 12, "x": 0, - "y": 246 + "y": 263 }, "id": 99, "options": { @@ -7071,8 +7566,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7088,7 +7582,7 @@ "h": 8, "w": 12, "x": 12, - "y": 246 + "y": 263 }, "id": 100, "options": { @@ -7126,7 +7620,7 @@ "h": 1, "w": 24, "x": 0, - "y": 254 + "y": 271 }, "id": 105, "panels": [], @@ -7180,8 +7674,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7197,7 +7690,7 @@ "h": 8, "w": 12, "x": 0, - "y": 255 + "y": 272 }, "id": 106, "options": { @@ -7276,8 +7769,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7293,7 +7785,7 @@ "h": 8, "w": 12, "x": 12, - "y": 255 + "y": 272 }, "id": 107, "options": { @@ -7331,7 +7823,7 @@ "h": 1, "w": 24, "x": 0, - "y": 263 + "y": 280 }, "id": 108, "panels": [], @@ -7385,8 +7877,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7427,7 +7918,7 @@ "h": 8, "w": 12, "x": 0, - "y": 264 + "y": 281 }, "id": 109, "options": { @@ -7489,7 +7980,7 @@ "h": 8, "w": 12, "x": 12, - "y": 264 + "y": 281 }, "id": 111, "maxDataPoints": 25, @@ -7599,8 +8090,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7616,7 +8106,7 @@ "h": 8, "w": 12, "x": 0, - "y": 272 + "y": 289 }, "id": 120, "options": { @@ -7674,7 +8164,7 @@ "h": 8, "w": 12, "x": 12, - "y": 272 + "y": 289 }, "id": 112, "maxDataPoints": 25, @@ -7784,8 +8274,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7825,7 +8314,7 @@ "h": 8, "w": 12, "x": 0, - "y": 280 + "y": 297 }, "id": 198, "options": { @@ -8034,4 +8523,4 @@ "uid": "2k8BXz24x", "version": 4, "weekStart": "" -} \ No newline at end of file +}