From 0e166f0f326b86491c0b23a8cc483e8a224e9731 Mon Sep 17 00:00:00 2001
From: Seva Zhidkov <zhidkovseva@gmail.com>
Date: Tue, 13 Feb 2024 01:59:52 +0000
Subject: [PATCH] perf(db): pre-populate metric handles hashmap on env init
 (#6573)

---
 Cargo.lock                       |  1 -
 crates/storage/db/Cargo.toml     |  1 -
 crates/storage/db/src/metrics.rs | 44 ++++++++++++++++++++------------
 3 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d32a703db..7819b709e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5946,7 +5946,6 @@ dependencies = [
  "assert_matches",
  "bytes",
  "criterion",
- "dashmap",
  "derive_more",
  "eyre",
  "iai",
diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml
index b133db21c..896d70921 100644
--- a/crates/storage/db/Cargo.toml
+++ b/crates/storage/db/Cargo.toml
@@ -38,7 +38,6 @@ parking_lot.workspace = true
 derive_more.workspace = true
 eyre.workspace = true
 paste = "1.0"
-dashmap = "5.5.3"
 rustc-hash = "1.1.0"
 
 # arbitrary utils
diff --git a/crates/storage/db/src/metrics.rs b/crates/storage/db/src/metrics.rs
index 928085a6a..4e3939ef2 100644
--- a/crates/storage/db/src/metrics.rs
+++ b/crates/storage/db/src/metrics.rs
@@ -1,14 +1,14 @@
 use crate::Tables;
-use dashmap::DashMap;
 use metrics::{Gauge, Histogram};
 use reth_libmdbx::CommitLatency;
 use reth_metrics::{metrics::Counter, Metrics};
 use rustc_hash::FxHasher;
 use std::{
+    collections::HashMap,
     hash::BuildHasherDefault,
     time::{Duration, Instant},
 };
-use strum::EnumCount;
+use strum::{EnumCount, EnumIter, IntoEnumIterator};
 
 const LARGE_VALUE_THRESHOLD_BYTES: usize = 4096;
 
@@ -17,20 +17,33 @@ const LARGE_VALUE_THRESHOLD_BYTES: usize = 4096;
 #[derive(Debug)]
 pub struct DatabaseEnvMetrics {
     /// Caches OperationMetrics handles for each table and operation tuple.
-    operations: DashMap<(Tables, Operation), OperationMetrics, BuildHasherDefault<FxHasher>>,
+    operations: HashMap<(Tables, Operation), OperationMetrics, BuildHasherDefault<FxHasher>>,
 }
 
 impl DatabaseEnvMetrics {
     pub(crate) fn new() -> Self {
-        Self {
-            operations: DashMap::with_capacity_and_hasher(
-                Tables::COUNT * Operation::COUNT,
-                BuildHasherDefault::<FxHasher>::default(),
-            ),
+        // Pre-populate the map with all possible table and operation combinations
+        // to avoid runtime locks on the map when recording metrics.
+        let mut operations = HashMap::with_capacity_and_hasher(
+            Tables::COUNT * Operation::COUNT,
+            BuildHasherDefault::<FxHasher>::default(),
+        );
+        for table in Tables::ALL {
+            for operation in Operation::iter() {
+                operations.insert(
+                    (*table, operation),
+                    OperationMetrics::new_with_labels(&[
+                        (Labels::Table.as_str(), table.name()),
+                        (Labels::Operation.as_str(), operation.as_str()),
+                    ]),
+                );
+            }
         }
+        Self { operations }
     }
 
-    /// Record a metric for database operation executed in `f`. Panics if the table name is unknown.
+    /// Record a metric for database operation executed in `f`.
+    /// Panics if a metric recorder is not found for the given table and operation.
     pub(crate) fn record_operation<R>(
         &self,
         table: Tables,
@@ -38,13 +51,10 @@ impl DatabaseEnvMetrics {
         value_size: Option<usize>,
         f: impl FnOnce() -> R,
     ) -> R {
-        let handle = self.operations.entry((table, operation)).or_insert_with(|| {
-            OperationMetrics::new_with_labels(&[
-                (Labels::Table.as_str(), table.name()),
-                (Labels::Operation.as_str(), operation.as_str()),
-            ])
-        });
-        handle.record(value_size, f)
+        self.operations
+            .get(&(table, operation))
+            .expect("operation & table metric handle not found")
+            .record(value_size, f)
     }
 }
 
@@ -94,7 +104,7 @@ impl TransactionOutcome {
 }
 
 /// Types of operations conducted on the database: get, put, delete, and various cursor operations.
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, EnumCount)]
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, EnumCount, EnumIter)]
 pub(crate) enum Operation {
     /// Database get operation.
     Get,