From a9fa281816e02c1ba936c65b290b4d20fcf83a6b Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 27 Oct 2023 18:47:41 +0100 Subject: [PATCH] feat(storage): database/transaction/cursor metrics (#5149) --- bin/reth/src/node/mod.rs | 45 +- bin/reth/src/prometheus_exporter.rs | 44 +- bin/reth/src/stage/run.rs | 3 +- crates/storage/db/src/abstraction/mock.rs | 2 +- .../storage/db/src/abstraction/transaction.rs | 4 +- .../db/src/implementation/mdbx/cursor.rs | 135 +++-- .../storage/db/src/implementation/mdbx/mod.rs | 23 +- .../storage/db/src/implementation/mdbx/tx.rs | 186 +++++-- crates/storage/db/src/lib.rs | 1 + crates/storage/db/src/metrics.rs | 168 ++++++ crates/storage/libmdbx-rs/src/transaction.rs | 7 +- etc/grafana/dashboards/overview.json | 517 +++++++++++++++--- 12 files changed, 954 insertions(+), 181 deletions(-) create mode 100644 crates/storage/db/src/metrics.rs diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index dcb7b7d11..f06de042b 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -25,6 +25,7 @@ use clap::{value_parser, Parser}; use eyre::Context; use fdlimit::raise_fd_limit; use futures::{future::Either, pin_mut, stream, stream_select, StreamExt}; +use metrics_exporter_prometheus::PrometheusHandle; use reth_auto_seal_consensus::{AutoSealBuilder, AutoSealConsensus, MiningMode}; use reth_beacon_consensus::{ hooks::{EngineHooks, PruneHook}, @@ -72,7 +73,6 @@ use reth_stages::{ IndexAccountHistoryStage, IndexStorageHistoryStage, MerkleStage, SenderRecoveryStage, StorageHashingStage, TotalDifficultyStage, TransactionLookupStage, }, - MetricEventsSender, MetricsListener, }; use reth_tasks::TaskExecutor; use reth_transaction_pool::{ @@ -247,12 +247,14 @@ impl NodeCommand { // always store reth.toml in the data dir, not the chain specific data dir info!(target: "reth::cli", path = ?config_path, "Configuration loaded"); + let prometheus_handle = self.install_prometheus_recorder()?; + let db_path = data_dir.db_path(); info!(target: "reth::cli", path = ?db_path, "Opening database"); - let db = Arc::new(init_db(&db_path, self.db.log_level)?); + let db = Arc::new(init_db(&db_path, self.db.log_level)?.with_metrics()); info!(target: "reth::cli", "Database opened"); - self.start_metrics_endpoint(Arc::clone(&db)).await?; + self.start_metrics_endpoint(prometheus_handle, Arc::clone(&db)).await?; debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); @@ -269,10 +271,10 @@ impl NodeCommand { self.init_trusted_nodes(&mut config); - debug!(target: "reth::cli", "Spawning metrics listener task"); - let (metrics_tx, metrics_rx) = unbounded_channel(); - let metrics_listener = MetricsListener::new(metrics_rx); - ctx.task_executor.spawn_critical("metrics listener task", metrics_listener); + debug!(target: "reth::cli", "Spawning stages metrics listener task"); + let (sync_metrics_tx, sync_metrics_rx) = unbounded_channel(); + let sync_metrics_listener = reth_stages::MetricsListener::new(sync_metrics_rx); + ctx.task_executor.spawn_critical("stages metrics listener task", sync_metrics_listener); let prune_config = self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); @@ -289,7 +291,7 @@ impl NodeCommand { BlockchainTreeConfig::default(), prune_config.clone().map(|config| config.segments), )? - .with_sync_metrics_tx(metrics_tx.clone()); + .with_sync_metrics_tx(sync_metrics_tx.clone()); let canon_state_notification_sender = tree.canon_state_notification_sender(); let blockchain_tree = ShareableBlockchainTree::new(tree); debug!(target: "reth::cli", "configured blockchain tree"); @@ -409,7 +411,7 @@ impl NodeCommand { Arc::clone(&consensus), db.clone(), &ctx.task_executor, - metrics_tx, + sync_metrics_tx, prune_config.clone(), max_block, ) @@ -429,7 +431,7 @@ impl NodeCommand { Arc::clone(&consensus), db.clone(), &ctx.task_executor, - metrics_tx, + sync_metrics_tx, prune_config.clone(), max_block, ) @@ -565,7 +567,7 @@ impl NodeCommand { consensus: Arc, db: DB, task_executor: &TaskExecutor, - metrics_tx: MetricEventsSender, + metrics_tx: reth_stages::MetricEventsSender, prune_config: Option, max_block: Option, ) -> eyre::Result> @@ -628,11 +630,24 @@ impl NodeCommand { } } - async fn start_metrics_endpoint(&self, db: Arc) -> eyre::Result<()> { + fn install_prometheus_recorder(&self) -> eyre::Result { + prometheus_exporter::install_recorder() + } + + async fn start_metrics_endpoint( + &self, + prometheus_handle: PrometheusHandle, + db: Arc, + ) -> eyre::Result<()> { if let Some(listen_addr) = self.metrics { info!(target: "reth::cli", addr = %listen_addr, "Starting metrics endpoint"); - prometheus_exporter::initialize(listen_addr, db, metrics_process::Collector::default()) - .await?; + prometheus_exporter::serve( + listen_addr, + prometheus_handle, + db, + metrics_process::Collector::default(), + ) + .await?; } Ok(()) @@ -790,7 +805,7 @@ impl NodeCommand { consensus: Arc, max_block: Option, continuous: bool, - metrics_tx: MetricEventsSender, + metrics_tx: reth_stages::MetricEventsSender, prune_config: Option, ) -> eyre::Result> where diff --git a/bin/reth/src/prometheus_exporter.rs b/bin/reth/src/prometheus_exporter.rs index bb612d538..4fa622bff 100644 --- a/bin/reth/src/prometheus_exporter.rs +++ b/bin/reth/src/prometheus_exporter.rs @@ -15,30 +15,36 @@ use tracing::error; pub(crate) trait Hook: Fn() + Send + Sync {} impl Hook for T {} -/// Installs Prometheus as the metrics recorder and serves it over HTTP with hooks. -/// -/// The hooks are called every time the metrics are requested at the given endpoint, and can be used -/// to record values for pull-style metrics, i.e. metrics that are not automatically updated. -pub(crate) async fn initialize_with_hooks( - listen_addr: SocketAddr, - hooks: impl IntoIterator, -) -> eyre::Result<()> { +/// Installs Prometheus as the metrics recorder. +pub(crate) fn install_recorder() -> eyre::Result { let recorder = PrometheusBuilder::new().build_recorder(); let handle = recorder.handle(); - let hooks: Vec<_> = hooks.into_iter().collect(); - - // Start endpoint - start_endpoint(listen_addr, handle, Arc::new(move || hooks.iter().for_each(|hook| hook()))) - .await - .wrap_err("Could not start Prometheus endpoint")?; - // Build metrics stack Stack::new(recorder) .push(PrefixLayer::new("reth")) .install() .wrap_err("Couldn't set metrics recorder.")?; + Ok(handle) +} + +/// Serves Prometheus metrics over HTTP with hooks. +/// +/// The hooks are called every time the metrics are requested at the given endpoint, and can be used +/// to record values for pull-style metrics, i.e. metrics that are not automatically updated. +pub(crate) async fn serve_with_hooks( + listen_addr: SocketAddr, + handle: PrometheusHandle, + hooks: impl IntoIterator, +) -> eyre::Result<()> { + let hooks: Vec<_> = hooks.into_iter().collect(); + + // Start endpoint + start_endpoint(listen_addr, handle, Arc::new(move || hooks.iter().for_each(|hook| hook()))) + .await + .wrap_err("Could not start Prometheus endpoint")?; + Ok(()) } @@ -67,10 +73,10 @@ async fn start_endpoint( Ok(()) } -/// Installs Prometheus as the metrics recorder and serves it over HTTP with database and process -/// metrics. -pub(crate) async fn initialize( +/// Serves Prometheus metrics over HTTP with database and process metrics. +pub(crate) async fn serve( listen_addr: SocketAddr, + handle: PrometheusHandle, db: Arc, process: metrics_process::Collector, ) -> eyre::Result<()> { @@ -119,7 +125,7 @@ pub(crate) async fn initialize( Box::new(move || cloned_process.collect()), Box::new(collect_memory_stats), ]; - initialize_with_hooks(listen_addr, hooks).await?; + serve_with_hooks(listen_addr, handle, hooks).await?; // We describe the metrics after the recorder is installed, otherwise this information is not // registered diff --git a/bin/reth/src/stage/run.rs b/bin/reth/src/stage/run.rs index d08c8835e..53ae2eec8 100644 --- a/bin/reth/src/stage/run.rs +++ b/bin/reth/src/stage/run.rs @@ -131,8 +131,9 @@ impl Command { if let Some(listen_addr) = self.metrics { info!(target: "reth::cli", "Starting metrics endpoint at {}", listen_addr); - prometheus_exporter::initialize( + prometheus_exporter::serve( listen_addr, + prometheus_exporter::install_recorder()?, Arc::clone(&db), metrics_process::Collector::default(), ) diff --git a/crates/storage/db/src/abstraction/mock.rs b/crates/storage/db/src/abstraction/mock.rs index bac7e061f..737797008 100644 --- a/crates/storage/db/src/abstraction/mock.rs +++ b/crates/storage/db/src/abstraction/mock.rs @@ -63,7 +63,7 @@ impl DbTx for TxMock { Ok(true) } - fn drop(self) {} + fn abort(self) {} fn cursor_read(&self) -> Result<>::Cursor, DatabaseError> { Ok(CursorMock { _cursor: 0 }) diff --git a/crates/storage/db/src/abstraction/transaction.rs b/crates/storage/db/src/abstraction/transaction.rs index 798b1d276..bbbd775d7 100644 --- a/crates/storage/db/src/abstraction/transaction.rs +++ b/crates/storage/db/src/abstraction/transaction.rs @@ -39,8 +39,8 @@ pub trait DbTx: for<'a> DbTxGAT<'a> { /// Commit for read only transaction will consume and free transaction and allows /// freeing of memory pages fn commit(self) -> Result; - /// Drops transaction - fn drop(self); + /// Aborts transaction + fn abort(self); /// Iterate over read only values in table. fn cursor_read(&self) -> Result<>::Cursor, DatabaseError>; /// Iterate over read only values in dup sorted table. diff --git a/crates/storage/db/src/implementation/mdbx/cursor.rs b/crates/storage/db/src/implementation/mdbx/cursor.rs index 936069ca8..e8a6f1e33 100644 --- a/crates/storage/db/src/implementation/mdbx/cursor.rs +++ b/crates/storage/db/src/implementation/mdbx/cursor.rs @@ -1,7 +1,7 @@ //! Cursor wrapper for libmdbx-sys. use reth_interfaces::db::DatabaseWriteOperation; -use std::{borrow::Cow, collections::Bound, ops::RangeBounds}; +use std::{borrow::Cow, collections::Bound, marker::PhantomData, ops::RangeBounds}; use crate::{ common::{PairResult, ValueOnlyResult}, @@ -9,6 +9,7 @@ use crate::{ DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, RangeWalker, ReverseWalker, Walker, }, + metrics::{Operation, OperationMetrics}, table::{Compress, DupSort, Encode, Table}, tables::utils::*, DatabaseError, @@ -24,13 +25,38 @@ pub type CursorRW<'tx, T> = Cursor<'tx, RW, T>; #[derive(Debug)] pub struct Cursor<'tx, K: TransactionKind, T: Table> { /// Inner `libmdbx` cursor. - pub inner: reth_libmdbx::Cursor<'tx, K>, - /// Table name as is inside the database. - pub table: &'static str, - /// Phantom data to enforce encoding/decoding. - pub _dbi: std::marker::PhantomData, + pub(crate) inner: reth_libmdbx::Cursor<'tx, K>, /// Cache buffer that receives compressed values. - pub buf: Vec, + buf: Vec, + /// Whether to record metrics or not. + with_metrics: bool, + /// Phantom data to enforce encoding/decoding. + _dbi: PhantomData, +} + +impl<'tx, K: TransactionKind, T: Table> Cursor<'tx, K, T> { + pub(crate) fn new_with_metrics( + inner: reth_libmdbx::Cursor<'tx, K>, + with_metrics: bool, + ) -> Self { + Self { inner, buf: Vec::new(), with_metrics, _dbi: PhantomData } + } + + /// If `self.with_metrics == true`, record a metric with the provided operation and value size. + /// + /// Otherwise, just execute the closure. + fn execute_with_operation_metric( + &mut self, + operation: Operation, + value_size: Option, + f: impl FnOnce(&mut Self) -> R, + ) -> R { + if self.with_metrics { + OperationMetrics::record(T::NAME, operation, value_size, || f(self)) + } else { + f(self) + } + } } /// Takes `(key, value)` from the database and decodes it appropriately. @@ -43,14 +69,14 @@ macro_rules! decode { /// Some types don't support compression (eg. B256), and we don't want to be copying them to the /// allocated buffer when we can just use their reference. -macro_rules! compress_or_ref { +macro_rules! compress_to_buf_or_ref { ($self:expr, $value:expr) => { if let Some(value) = $value.uncompressable_ref() { - value + Some(value) } else { $self.buf.truncate(0); $value.compress_to_buf(&mut $self.buf); - $self.buf.as_ref() + None } }; } @@ -229,61 +255,92 @@ impl DbCursorRW for Cursor<'_, RW, T> { /// found, before calling `upsert`. fn upsert(&mut self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { let key = key.encode(); - // Default `WriteFlags` is UPSERT - self.inner.put(key.as_ref(), compress_or_ref!(self, value), WriteFlags::UPSERT).map_err( - |e| DatabaseError::Write { - code: e.into(), - operation: DatabaseWriteOperation::CursorUpsert, - table_name: T::NAME, - key: Box::from(key.as_ref()), + let value = compress_to_buf_or_ref!(self, value); + self.execute_with_operation_metric( + Operation::CursorUpsert, + Some(value.unwrap_or(&self.buf).len()), + |this| { + this.inner + .put(key.as_ref(), value.unwrap_or(&this.buf), WriteFlags::UPSERT) + .map_err(|e| DatabaseError::Write { + code: e.into(), + operation: DatabaseWriteOperation::CursorUpsert, + table_name: T::NAME, + key: Box::from(key.as_ref()), + }) }, ) } fn insert(&mut self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { let key = key.encode(); - self.inner - .put(key.as_ref(), compress_or_ref!(self, value), WriteFlags::NO_OVERWRITE) - .map_err(|e| DatabaseError::Write { - code: e.into(), - operation: DatabaseWriteOperation::CursorInsert, - table_name: T::NAME, - key: Box::from(key.as_ref()), - }) + let value = compress_to_buf_or_ref!(self, value); + self.execute_with_operation_metric( + Operation::CursorInsert, + Some(value.unwrap_or(&self.buf).len()), + |this| { + this.inner + .put(key.as_ref(), value.unwrap_or(&this.buf), WriteFlags::NO_OVERWRITE) + .map_err(|e| DatabaseError::Write { + code: e.into(), + operation: DatabaseWriteOperation::CursorInsert, + table_name: T::NAME, + key: Box::from(key.as_ref()), + }) + }, + ) } /// Appends the data to the end of the table. Consequently, the append operation /// will fail if the inserted key is less than the last table key fn append(&mut self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { let key = key.encode(); - self.inner.put(key.as_ref(), compress_or_ref!(self, value), WriteFlags::APPEND).map_err( - |e| DatabaseError::Write { - code: e.into(), - operation: DatabaseWriteOperation::CursorAppend, - table_name: T::NAME, - key: Box::from(key.as_ref()), + let value = compress_to_buf_or_ref!(self, value); + self.execute_with_operation_metric( + Operation::CursorAppend, + Some(value.unwrap_or(&self.buf).len()), + |this| { + this.inner + .put(key.as_ref(), value.unwrap_or(&this.buf), WriteFlags::APPEND) + .map_err(|e| DatabaseError::Write { + code: e.into(), + operation: DatabaseWriteOperation::CursorAppend, + table_name: T::NAME, + key: Box::from(key.as_ref()), + }) }, ) } fn delete_current(&mut self) -> Result<(), DatabaseError> { - self.inner.del(WriteFlags::CURRENT).map_err(|e| DatabaseError::Delete(e.into())) + self.execute_with_operation_metric(Operation::CursorDeleteCurrent, None, |this| { + this.inner.del(WriteFlags::CURRENT).map_err(|e| DatabaseError::Delete(e.into())) + }) } } impl DbDupCursorRW for Cursor<'_, RW, T> { fn delete_current_duplicates(&mut self) -> Result<(), DatabaseError> { - self.inner.del(WriteFlags::NO_DUP_DATA).map_err(|e| DatabaseError::Delete(e.into())) + self.execute_with_operation_metric(Operation::CursorDeleteCurrentDuplicates, None, |this| { + this.inner.del(WriteFlags::NO_DUP_DATA).map_err(|e| DatabaseError::Delete(e.into())) + }) } fn append_dup(&mut self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { let key = key.encode(); - self.inner.put(key.as_ref(), compress_or_ref!(self, value), WriteFlags::APPEND_DUP).map_err( - |e| DatabaseError::Write { - code: e.into(), - operation: DatabaseWriteOperation::CursorAppendDup, - table_name: T::NAME, - key: Box::from(key.as_ref()), + let value = compress_to_buf_or_ref!(self, value); + self.execute_with_operation_metric( + Operation::CursorAppendDup, + Some(value.unwrap_or(&self.buf).len()), + |this| { + this.inner + .put(key.as_ref(), value.unwrap_or(&this.buf), WriteFlags::APPEND_DUP) + .map_err(|e| DatabaseError::Write { + code: e.into(), + operation: DatabaseWriteOperation::CursorAppendDup, + table_name: T::NAME, + key: Box::from(key.as_ref()), + }) }, ) } diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 7c0f6051f..2fac746cf 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -37,6 +37,8 @@ pub enum EnvKind { pub struct Env { /// Libmdbx-sys environment. pub inner: Environment, + /// Whether to record metrics or not. + with_metrics: bool, } impl<'a, E: EnvironmentKind> DatabaseGAT<'a> for Env { @@ -46,11 +48,17 @@ impl<'a, E: EnvironmentKind> DatabaseGAT<'a> for Env { impl Database for Env { fn tx(&self) -> Result<>::TX, DatabaseError> { - Ok(Tx::new(self.inner.begin_ro_txn().map_err(|e| DatabaseError::InitTx(e.into()))?)) + Ok(Tx::new_with_metrics( + self.inner.begin_ro_txn().map_err(|e| DatabaseError::InitTx(e.into()))?, + self.with_metrics, + )) } fn tx_mut(&self) -> Result<>::TXMut, DatabaseError> { - Ok(Tx::new(self.inner.begin_rw_txn().map_err(|e| DatabaseError::InitTx(e.into()))?)) + Ok(Tx::new_with_metrics( + self.inner.begin_rw_txn().map_err(|e| DatabaseError::InitTx(e.into()))?, + self.with_metrics, + )) } } @@ -116,11 +124,20 @@ impl Env { } } - let env = Env { inner: inner_env.open(path).map_err(|e| DatabaseError::Open(e.into()))? }; + let env = Env { + inner: inner_env.open(path).map_err(|e| DatabaseError::Open(e.into()))?, + with_metrics: false, + }; Ok(env) } + /// Enables metrics on the database. + pub fn with_metrics(mut self) -> Self { + self.with_metrics = true; + self + } + /// Creates all the defined tables, if necessary. pub fn create_tables(&self) -> Result<(), DatabaseError> { let tx = self.inner.begin_rw_txn().map_err(|e| DatabaseError::InitTx(e.into()))?; diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index 2bf8450ca..276cd594d 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -2,6 +2,9 @@ use super::cursor::Cursor; use crate::{ + metrics::{ + Operation, OperationMetrics, TransactionMetrics, TransactionMode, TransactionOutcome, + }, table::{Compress, DupSort, Encode, Table, TableImporter}, tables::{utils::decode_one, Tables, NUM_TABLES}, transaction::{DbTx, DbTxGAT, DbTxMut, DbTxMutGAT}, @@ -10,7 +13,6 @@ use crate::{ use parking_lot::RwLock; use reth_interfaces::db::DatabaseWriteOperation; use reth_libmdbx::{ffi::DBI, EnvironmentKind, Transaction, TransactionKind, WriteFlags, RW}; -use reth_metrics::metrics::histogram; use std::{marker::PhantomData, str::FromStr, sync::Arc, time::Instant}; /// Wrapper for the libmdbx transaction. @@ -18,8 +20,13 @@ use std::{marker::PhantomData, str::FromStr, sync::Arc, time::Instant}; pub struct Tx<'a, K: TransactionKind, E: EnvironmentKind> { /// Libmdbx-sys transaction. pub inner: Transaction<'a, K, E>, - /// Database table handle cache - pub db_handles: Arc; NUM_TABLES]>>, + /// Database table handle cache. + pub(crate) db_handles: Arc; NUM_TABLES]>>, + /// Handler for metrics with its own [Drop] implementation for cases when the transaction isn't + /// closed by [Tx::commit] or [Tx::abort], but we still need to report it in the metrics. + /// + /// If [Some], then metrics are reported. + metrics_handler: Option>, } impl<'env, K: TransactionKind, E: EnvironmentKind> Tx<'env, K, E> { @@ -28,12 +35,30 @@ impl<'env, K: TransactionKind, E: EnvironmentKind> Tx<'env, K, E> { where 'a: 'env, { - Self { inner, db_handles: Default::default() } + Self { inner, db_handles: Default::default(), metrics_handler: None } + } + + /// Creates new `Tx` object with a `RO` or `RW` transaction and optionally enables metrics. + pub fn new_with_metrics<'a>(inner: Transaction<'a, K, E>, with_metrics: bool) -> Self + where + 'a: 'env, + { + let metrics_handler = with_metrics.then(|| { + let handler = MetricsHandler:: { + txn_id: inner.id(), + start: Instant::now(), + close_recorded: false, + _marker: PhantomData, + }; + TransactionMetrics::record_open(handler.transaction_mode()); + handler + }); + Self { inner, db_handles: Default::default(), metrics_handler } } /// Gets this transaction ID. pub fn id(&self) -> u64 { - self.inner.id() + self.metrics_handler.as_ref().map_or_else(|| self.inner.id(), |handler| handler.txn_id) } /// Gets a table database handle if it exists, otherwise creates it. @@ -57,15 +82,94 @@ impl<'env, K: TransactionKind, E: EnvironmentKind> Tx<'env, K, E> { /// Create db Cursor pub fn new_cursor(&self) -> Result, DatabaseError> { - Ok(Cursor { - inner: self - .inner - .cursor_with_dbi(self.get_dbi::()?) - .map_err(|e| DatabaseError::InitCursor(e.into()))?, - table: T::NAME, - _dbi: PhantomData, - buf: vec![], - }) + let inner = self + .inner + .cursor_with_dbi(self.get_dbi::()?) + .map_err(|e| DatabaseError::InitCursor(e.into()))?; + + Ok(Cursor::new_with_metrics(inner, self.metrics_handler.is_some())) + } + + /// If `self.metrics_handler == Some(_)`, measure the time it takes to execute the closure and + /// record a metric with the provided transaction outcome. + /// + /// Otherwise, just execute the closure. + fn execute_with_close_transaction_metric( + mut self, + outcome: TransactionOutcome, + f: impl FnOnce(Self) -> R, + ) -> R { + if let Some(mut metrics_handler) = self.metrics_handler.take() { + metrics_handler.close_recorded = true; + + let start = Instant::now(); + let result = f(self); + let close_duration = start.elapsed(); + let open_duration = metrics_handler.start.elapsed(); + + TransactionMetrics::record_close( + metrics_handler.transaction_mode(), + outcome, + open_duration, + Some(close_duration), + ); + + result + } else { + f(self) + } + } + + /// If `self.metrics_handler == Some(_)`, measure the time it takes to execute the closure and + /// record a metric with the provided operation. + /// + /// Otherwise, just execute the closure. + fn execute_with_operation_metric( + &self, + operation: Operation, + value_size: Option, + f: impl FnOnce(&Transaction<'_, K, E>) -> R, + ) -> R { + if self.metrics_handler.is_some() { + OperationMetrics::record(T::NAME, operation, value_size, || f(&self.inner)) + } else { + f(&self.inner) + } + } +} + +#[derive(Debug)] +struct MetricsHandler { + /// Cached internal transaction ID provided by libmdbx. + txn_id: u64, + /// The time when transaction has started. + start: Instant, + /// If true, the metric about transaction closing has already been recorded and we don't need + /// to do anything on [Drop::drop]. + close_recorded: bool, + _marker: PhantomData, +} + +impl MetricsHandler { + const fn transaction_mode(&self) -> TransactionMode { + if K::IS_READ_ONLY { + TransactionMode::ReadOnly + } else { + TransactionMode::ReadWrite + } + } +} + +impl Drop for MetricsHandler { + fn drop(&mut self) { + if !self.close_recorded { + TransactionMetrics::record_close( + self.transaction_mode(), + TransactionOutcome::Drop, + self.start.elapsed(), + None, + ); + } } } @@ -83,22 +187,24 @@ impl TableImporter for Tx<'_, RW, E> {} impl DbTx for Tx<'_, K, E> { fn get(&self, key: T::Key) -> Result::Value>, DatabaseError> { - self.inner - .get(self.get_dbi::()?, key.encode().as_ref()) - .map_err(|e| DatabaseError::Read(e.into()))? - .map(decode_one::) - .transpose() + self.execute_with_operation_metric::(Operation::Get, None, |tx| { + tx.get(self.get_dbi::()?, key.encode().as_ref()) + .map_err(|e| DatabaseError::Read(e.into()))? + .map(decode_one::) + .transpose() + }) } fn commit(self) -> Result { - let start = Instant::now(); - let result = self.inner.commit().map_err(|e| DatabaseError::Commit(e.into())); - histogram!("tx.commit", start.elapsed()); - result + self.execute_with_close_transaction_metric(TransactionOutcome::Commit, |this| { + this.inner.commit().map_err(|e| DatabaseError::Commit(e.into())) + }) } - fn drop(self) { - drop(self.inner) + fn abort(self) { + self.execute_with_close_transaction_metric(TransactionOutcome::Abort, |this| { + drop(this.inner) + }) } // Iterate over read only values in database. @@ -126,14 +232,21 @@ impl DbTx for Tx<'_, K, E> { impl DbTxMut for Tx<'_, RW, E> { fn put(&self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { let key = key.encode(); - self.inner - .put(self.get_dbi::()?, key.as_ref(), &value.compress(), WriteFlags::UPSERT) - .map_err(|e| DatabaseError::Write { - code: e.into(), - operation: DatabaseWriteOperation::Put, - table_name: T::NAME, - key: Box::from(key.as_ref()), - }) + let value = value.compress(); + self.execute_with_operation_metric::( + Operation::Put, + Some(value.as_ref().len()), + |tx| { + tx.put(self.get_dbi::()?, key.as_ref(), value, WriteFlags::UPSERT).map_err(|e| { + DatabaseError::Write { + code: e.into(), + operation: DatabaseWriteOperation::Put, + table_name: T::NAME, + key: Box::from(key.as_ref()), + } + }) + }, + ) } fn delete( @@ -148,9 +261,10 @@ impl DbTxMut for Tx<'_, RW, E> { data = Some(value.as_ref()); }; - self.inner - .del(self.get_dbi::()?, key.encode(), data) - .map_err(|e| DatabaseError::Delete(e.into())) + self.execute_with_operation_metric::(Operation::Delete, None, |tx| { + tx.del(self.get_dbi::()?, key.encode(), data) + .map_err(|e| DatabaseError::Delete(e.into())) + }) } fn clear(&self) -> Result<(), DatabaseError> { diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs index a3ce47568..a1511fb09 100644 --- a/crates/storage/db/src/lib.rs +++ b/crates/storage/db/src/lib.rs @@ -68,6 +68,7 @@ pub mod abstraction; mod implementation; +mod metrics; pub mod snapshot; pub mod tables; mod utils; diff --git a/crates/storage/db/src/metrics.rs b/crates/storage/db/src/metrics.rs new file mode 100644 index 000000000..fff6cecbd --- /dev/null +++ b/crates/storage/db/src/metrics.rs @@ -0,0 +1,168 @@ +use metrics::{Gauge, Histogram}; +use reth_metrics::{metrics::Counter, Metrics}; +use std::time::{Duration, Instant}; + +const LARGE_VALUE_THRESHOLD_BYTES: usize = 4096; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[allow(missing_docs)] +pub(crate) enum TransactionMode { + ReadOnly, + ReadWrite, +} + +impl TransactionMode { + pub(crate) const fn as_str(&self) -> &'static str { + match self { + TransactionMode::ReadOnly => "read-only", + TransactionMode::ReadWrite => "read-write", + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[allow(missing_docs)] +pub(crate) enum TransactionOutcome { + Commit, + Abort, + Drop, +} + +impl TransactionOutcome { + pub(crate) const fn as_str(&self) -> &'static str { + match self { + TransactionOutcome::Commit => "commit", + TransactionOutcome::Abort => "abort", + TransactionOutcome::Drop => "drop", + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +#[allow(missing_docs)] +pub(crate) enum Operation { + Get, + Put, + Delete, + CursorUpsert, + CursorInsert, + CursorAppend, + CursorAppendDup, + CursorDeleteCurrent, + CursorDeleteCurrentDuplicates, +} + +impl Operation { + pub(crate) const fn as_str(&self) -> &'static str { + match self { + Operation::Get => "get", + Operation::Put => "put", + Operation::Delete => "delete", + Operation::CursorUpsert => "cursor-upsert", + Operation::CursorInsert => "cursor-insert", + Operation::CursorAppend => "cursor-append", + Operation::CursorAppendDup => "cursor-append-dup", + Operation::CursorDeleteCurrent => "cursor-delete-current", + Operation::CursorDeleteCurrentDuplicates => "cursor-delete-current-duplicates", + } + } +} + +enum Labels { + Table, + TransactionMode, + TransactionOutcome, + Operation, +} + +impl Labels { + pub(crate) fn as_str(&self) -> &'static str { + match self { + Labels::Table => "table", + Labels::TransactionMode => "mode", + Labels::TransactionOutcome => "outcome", + Labels::Operation => "operation", + } + } +} + +#[derive(Metrics, Clone)] +#[metrics(scope = "database.transaction")] +pub(crate) struct TransactionMetrics { + /// Total number of currently open database transactions + open_total: Gauge, + /// The time a database transaction has been open + open_duration_seconds: Histogram, + /// The time it took to close a database transaction + close_duration_seconds: Histogram, +} + +impl TransactionMetrics { + /// Record transaction opening. + pub(crate) fn record_open(mode: TransactionMode) { + let metrics = Self::new_with_labels(&[(Labels::TransactionMode.as_str(), mode.as_str())]); + metrics.open_total.increment(1.0); + } + + /// Record transaction closing with the duration it was open and the duration it took to close + /// it. + pub(crate) fn record_close( + mode: TransactionMode, + outcome: TransactionOutcome, + open_duration: Duration, + close_duration: Option, + ) { + let metrics = Self::new_with_labels(&[(Labels::TransactionMode.as_str(), mode.as_str())]); + metrics.open_total.decrement(1.0); + + let metrics = Self::new_with_labels(&[ + (Labels::TransactionMode.as_str(), mode.as_str()), + (Labels::TransactionOutcome.as_str(), outcome.as_str()), + ]); + metrics.open_duration_seconds.record(open_duration); + + if let Some(close_duration) = close_duration { + metrics.close_duration_seconds.record(close_duration) + } + } +} + +#[derive(Metrics, Clone)] +#[metrics(scope = "database.operation")] +pub(crate) struct OperationMetrics { + /// Total number of database operations made + calls_total: Counter, + /// The time it took to execute a database operation (put/upsert/insert/append/append_dup) with + /// value larger than [LARGE_VALUE_THRESHOLD_BYTES] bytes. + large_value_duration_seconds: Histogram, +} + +impl OperationMetrics { + /// Record operation metric. + /// + /// The duration it took to execute the closure is recorded only if the provided `value_size` is + /// larger than [LARGE_VALUE_THRESHOLD_BYTES]. + pub(crate) fn record( + table: &'static str, + operation: Operation, + value_size: Option, + f: impl FnOnce() -> T, + ) -> T { + let metrics = Self::new_with_labels(&[ + (Labels::Table.as_str(), table), + (Labels::Operation.as_str(), operation.as_str()), + ]); + metrics.calls_total.increment(1); + + // Record duration only for large values to prevent the performance hit of clock syscall + // on small operations + if value_size.map_or(false, |size| size > LARGE_VALUE_THRESHOLD_BYTES) { + let start = Instant::now(); + let result = f(); + metrics.large_value_duration_seconds.record(start.elapsed()); + result + } else { + f() + } + } +} diff --git a/crates/storage/libmdbx-rs/src/transaction.rs b/crates/storage/libmdbx-rs/src/transaction.rs index bb6b42486..61cf48c87 100644 --- a/crates/storage/libmdbx-rs/src/transaction.rs +++ b/crates/storage/libmdbx-rs/src/transaction.rs @@ -23,12 +23,15 @@ mod private { impl Sealed for RW {} } -pub trait TransactionKind: private::Sealed + Debug + 'static { +pub trait TransactionKind: private::Sealed + Send + Sync + Debug + 'static { #[doc(hidden)] const ONLY_CLEAN: bool; #[doc(hidden)] const OPEN_FLAGS: MDBX_txn_flags_t; + + #[doc(hidden)] + const IS_READ_ONLY: bool; } #[derive(Debug)] @@ -42,10 +45,12 @@ pub struct RW; impl TransactionKind for RO { const ONLY_CLEAN: bool = true; const OPEN_FLAGS: MDBX_txn_flags_t = MDBX_TXN_RDONLY; + const IS_READ_ONLY: bool = true; } impl TransactionKind for RW { const ONLY_CLEAN: bool = false; const OPEN_FLAGS: MDBX_txn_flags_t = MDBX_TXN_READWRITE; + const IS_READ_ONLY: bool = false; } /// An MDBX transaction. diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index d83cd3aa3..b31b03cfa 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -497,7 +497,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -523,7 +522,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -539,7 +538,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "rate(reth_tx_commit_sum{instance=~\"$instance\"}[$__rate_interval]) / rate(reth_tx_commit_count{instance=~\"$instance\"}[$__rate_interval])", + "expr": "avg(rate(reth_database_transaction_close_duration_seconds_sum{instance=~\"$instance\", outcome=\"commit\"}[$__rate_interval]) / rate(reth_database_transaction_close_duration_seconds_count{instance=~\"$instance\", outcome=\"commit\"}[$__rate_interval]) >= 0)", "format": "time_series", "instant": false, "legendFormat": "Commit time", @@ -628,7 +627,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(reth_tx_commit{instance=~\"$instance\"}[$__interval])) by (quantile)", + "expr": "avg(avg_over_time(reth_database_transaction_close_duration_seconds{instance=~\"$instance\", outcome=\"commit\"}[$__interval])) by (quantile)", "format": "time_series", "instant": false, "legendFormat": "{{quantile}}", @@ -639,6 +638,394 @@ "title": "Commit time heatmap", "type": "heatmap" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The average time a database transaction was open.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "last" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(reth_database_transaction_open_duration_seconds_sum{instance=~\"$instance\", outcome!=\"\"}[$__rate_interval]) / rate(reth_database_transaction_open_duration_seconds_count{instance=~\"$instance\", outcome!=\"\"}[$__rate_interval])) by (outcome, mode)", + "format": "time_series", + "instant": false, + "legendFormat": "{{mode}}, {{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Average transaction open time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The maximum time the database transaction was open.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(max_over_time(reth_database_transaction_open_duration_seconds{instance=~\"$instance\", outcome!=\"\", quantile=\"1\"}[$__interval])) by (outcome, mode)", + "format": "time_series", + "instant": false, + "legendFormat": "{{mode}}, {{outcome}}", + "range": true, + "refId": "A" + } + ], + "title": "Max transaction open time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(reth_database_transaction_open_total{instance=~\"$instance\"}) by (mode)", + "format": "time_series", + "instant": false, + "legendFormat": "{{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of open transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The maximum time the database transaction operation which inserts a large value took.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 118, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(max_over_time(reth_database_operation_large_value_duration_seconds{instance=~\"$instance\", quantile=\"1\"}[$__interval]) > 0) by (table)", + "format": "time_series", + "instant": false, + "legendFormat": "{{table}}", + "range": true, + "refId": "A" + } + ], + "title": "Max insertion operation time", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -666,7 +1053,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 42 }, "id": 48, "options": { @@ -776,7 +1163,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 42 }, "id": 52, "options": { @@ -834,7 +1221,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 50 }, "id": 50, "options": { @@ -1002,7 +1389,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 50 }, "id": 58, "options": { @@ -1101,7 +1488,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 58 }, "id": 113, "options": { @@ -1138,7 +1525,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 66 }, "id": 46, "panels": [], @@ -1207,7 +1594,7 @@ "h": 8, "w": 24, "x": 0, - "y": 51 + "y": 67 }, "id": 56, "options": { @@ -1280,7 +1667,7 @@ "h": 1, "w": 24, "x": 0, - "y": 59 + "y": 75 }, "id": 6, "panels": [], @@ -1352,7 +1739,7 @@ "h": 8, "w": 8, "x": 0, - "y": 60 + "y": 76 }, "id": 18, "options": { @@ -1446,7 +1833,7 @@ "h": 8, "w": 8, "x": 8, - "y": 60 + "y": 76 }, "id": 16, "options": { @@ -1566,7 +1953,7 @@ "h": 8, "w": 8, "x": 16, - "y": 60 + "y": 76 }, "id": 8, "options": { @@ -1649,7 +2036,7 @@ "h": 8, "w": 8, "x": 0, - "y": 68 + "y": 84 }, "id": 54, "options": { @@ -1870,7 +2257,7 @@ "h": 8, "w": 14, "x": 8, - "y": 68 + "y": 84 }, "id": 103, "options": { @@ -1907,7 +2294,7 @@ "h": 1, "w": 24, "x": 0, - "y": 76 + "y": 92 }, "id": 24, "panels": [], @@ -2003,7 +2390,7 @@ "h": 8, "w": 12, "x": 0, - "y": 77 + "y": 93 }, "id": 26, "options": { @@ -2135,7 +2522,7 @@ "h": 8, "w": 12, "x": 12, - "y": 77 + "y": 93 }, "id": 33, "options": { @@ -2253,7 +2640,7 @@ "h": 8, "w": 12, "x": 0, - "y": 85 + "y": 101 }, "id": 36, "options": { @@ -2302,7 +2689,7 @@ "h": 1, "w": 24, "x": 0, - "y": 93 + "y": 109 }, "id": 32, "panels": [], @@ -2358,7 +2745,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2407,7 +2795,7 @@ "h": 8, "w": 12, "x": 0, - "y": 94 + "y": 110 }, "id": 30, "options": { @@ -2558,7 +2946,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -2570,7 +2959,7 @@ "h": 8, "w": 12, "x": 12, - "y": 94 + "y": 110 }, "id": 28, "options": { @@ -2687,7 +3076,7 @@ "h": 8, "w": 12, "x": 0, - "y": 102 + "y": 118 }, "id": 35, "options": { @@ -2810,7 +3199,7 @@ "h": 8, "w": 12, "x": 12, - "y": 102 + "y": 118 }, "id": 73, "options": { @@ -2860,7 +3249,7 @@ "h": 1, "w": 24, "x": 0, - "y": 110 + "y": 126 }, "id": 89, "panels": [], @@ -2932,7 +3321,7 @@ "h": 8, "w": 12, "x": 0, - "y": 111 + "y": 127 }, "id": 91, "options": { @@ -3049,7 +3438,7 @@ "h": 8, "w": 12, "x": 12, - "y": 111 + "y": 127 }, "id": 92, "options": { @@ -3184,7 +3573,7 @@ "h": 8, "w": 12, "x": 0, - "y": 119 + "y": 135 }, "id": 102, "options": { @@ -3303,7 +3692,7 @@ "h": 8, "w": 12, "x": 12, - "y": 119 + "y": 135 }, "id": 94, "options": { @@ -3397,7 +3786,7 @@ "h": 8, "w": 12, "x": 0, - "y": 127 + "y": 143 }, "id": 104, "options": { @@ -3521,7 +3910,7 @@ "h": 8, "w": 12, "x": 12, - "y": 127 + "y": 143 }, "id": 93, "options": { @@ -3664,7 +4053,7 @@ "h": 8, "w": 12, "x": 0, - "y": 135 + "y": 151 }, "id": 95, "options": { @@ -3783,7 +4172,7 @@ "h": 8, "w": 12, "x": 12, - "y": 135 + "y": 151 }, "id": 115, "options": { @@ -3876,7 +4265,7 @@ "h": 1, "w": 24, "x": 0, - "y": 143 + "y": 159 }, "id": 79, "panels": [], @@ -3947,7 +4336,7 @@ "h": 8, "w": 12, "x": 0, - "y": 144 + "y": 160 }, "id": 74, "options": { @@ -4041,7 +4430,7 @@ "h": 8, "w": 12, "x": 12, - "y": 144 + "y": 160 }, "id": 80, "options": { @@ -4135,7 +4524,7 @@ "h": 8, "w": 12, "x": 0, - "y": 152 + "y": 168 }, "id": 81, "options": { @@ -4229,7 +4618,7 @@ "h": 8, "w": 12, "x": 12, - "y": 152 + "y": 168 }, "id": 114, "options": { @@ -4267,7 +4656,7 @@ "h": 1, "w": 24, "x": 0, - "y": 160 + "y": 176 }, "id": 87, "panels": [], @@ -4338,7 +4727,7 @@ "h": 8, "w": 12, "x": 0, - "y": 161 + "y": 177 }, "id": 83, "options": { @@ -4431,7 +4820,7 @@ "h": 8, "w": 12, "x": 12, - "y": 161 + "y": 177 }, "id": 84, "options": { @@ -4536,7 +4925,7 @@ "h": 8, "w": 12, "x": 0, - "y": 169 + "y": 185 }, "id": 85, "options": { @@ -4573,7 +4962,7 @@ "h": 1, "w": 24, "x": 0, - "y": 177 + "y": 193 }, "id": 68, "panels": [], @@ -4644,7 +5033,7 @@ "h": 8, "w": 12, "x": 0, - "y": 178 + "y": 194 }, "id": 60, "options": { @@ -4737,7 +5126,7 @@ "h": 8, "w": 12, "x": 12, - "y": 178 + "y": 194 }, "id": 62, "options": { @@ -4830,7 +5219,7 @@ "h": 8, "w": 12, "x": 0, - "y": 186 + "y": 202 }, "id": 64, "options": { @@ -4867,7 +5256,7 @@ "h": 1, "w": 24, "x": 0, - "y": 194 + "y": 210 }, "id": 97, "panels": [], @@ -4936,7 +5325,7 @@ "h": 8, "w": 12, "x": 0, - "y": 195 + "y": 211 }, "id": 98, "options": { @@ -5096,7 +5485,7 @@ "h": 8, "w": 12, "x": 12, - "y": 195 + "y": 211 }, "id": 101, "options": { @@ -5191,7 +5580,7 @@ "h": 8, "w": 12, "x": 0, - "y": 203 + "y": 219 }, "id": 99, "options": { @@ -5286,7 +5675,7 @@ "h": 8, "w": 12, "x": 12, - "y": 203 + "y": 219 }, "id": 100, "options": { @@ -5324,7 +5713,7 @@ "h": 1, "w": 24, "x": 0, - "y": 211 + "y": 227 }, "id": 105, "panels": [], @@ -5394,7 +5783,7 @@ "h": 8, "w": 12, "x": 0, - "y": 212 + "y": 228 }, "id": 106, "options": { @@ -5489,7 +5878,7 @@ "h": 8, "w": 12, "x": 12, - "y": 212 + "y": 228 }, "id": 107, "options": { @@ -5527,7 +5916,7 @@ "h": 1, "w": 24, "x": 0, - "y": 220 + "y": 236 }, "id": 108, "panels": [], @@ -5550,7 +5939,7 @@ "h": 8, "w": 12, "x": 0, - "y": 221 + "y": 237 }, "hiddenSeries": false, "id": 109, @@ -5638,7 +6027,7 @@ "h": 8, "w": 12, "x": 12, - "y": 221 + "y": 237 }, "hiddenSeries": false, "id": 110, @@ -5735,7 +6124,7 @@ "h": 8, "w": 12, "x": 0, - "y": 229 + "y": 245 }, "id": 111, "maxDataPoints": 25, @@ -5824,7 +6213,7 @@ "h": 8, "w": 12, "x": 12, - "y": 229 + "y": 245 }, "id": 112, "maxDataPoints": 25, @@ -5928,6 +6317,6 @@ "timezone": "", "title": "reth", "uid": "2k8BXz24x", - "version": 10, + "version": 11, "weekStart": "" } \ No newline at end of file