jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 19:09:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(bin): process & jemalloc metrics (#3435)

Browse Source
This commit is contained in:
Alexey Shekhirin
2023-06-28 19:53:52 +01:00
committed by GitHub
parent 6b8b478c6f
commit 6e2fa845d8
5 changed files with 228 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
Cargo.lock generated
View File

@ -387,6 +387,26 @@ dependencies = [
"serde",
]
[[package]]
name = "bindgen"
version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"
dependencies = [
"bitflags 1.3.2",
"cexpr",
"clang-sys",
"lazy_static",
"lazycell",
"peeking_take_while",
"proc-macro2 1.0.60",
"quote 1.0.28",
"regex",
"rustc-hash",
"shlex",
"syn 1.0.109",
]
[[package]]
name = "bindgen"
version = "0.65.1"
@ -3157,6 +3177,17 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
[[package]]
name = "jemalloc-ctl"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1891c671f3db85d8ea8525dd43ab147f9977041911d24a03e5a36187a7bfde9"
dependencies = [
"jemalloc-sys",
"libc",
"paste",
]
[[package]]
name = "jemalloc-sys"
version = "0.5.3+5.3.0-patched"
@ -3431,6 +3462,17 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
[[package]]
name = "libproc"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b18cbf29f8ff3542ba22bdce9ac610fcb75d74bb4e2b306b2a2762242025b4f"
dependencies = [
"bindgen 0.64.0",
"errno 0.2.8",
"libc",
]
[[package]]
name = "lifetimed-bytes"
version = "0.1.0"
@ -3537,6 +3579,15 @@ dependencies = [
"libc",
]
[[package]]
name = "mach2"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8"
dependencies = [
"libc",
]
[[package]]
name = "match_cfg"
version = "0.1.0"
@ -3623,6 +3674,21 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "metrics-process"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99eab79be9f7c18565e889d6eaed6f1ebdafb2b6a88aef446d2fee5e7796ed10"
dependencies = [
"libproc",
"mach2",
"metrics",
"once_cell",
"procfs",
"rlimit",
"windows",
]
[[package]]
name = "metrics-util"
version = "0.14.0"
@ -4478,6 +4544,19 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "procfs"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f"
dependencies = [
"bitflags 1.3.2",
"byteorder",
"hex",
"lazy_static",
"rustix 0.36.11",
]
[[package]]
name = "proptest"
version = "1.1.0"
@ -4851,8 +4930,10 @@ dependencies = [
"human_bytes",
"humantime",
"hyper",
"jemalloc-ctl",
"jemallocator",
"metrics-exporter-prometheus",
"metrics-process",
"metrics-util",
"pin-project",
"pretty_assertions",
@ -5264,7 +5345,7 @@ dependencies = [
name = "reth-mdbx-sys"
version = "0.1.0-alpha.1"
dependencies = [
"bindgen",
"bindgen 0.65.1",
"cc",
"libc",
]
@ -5910,6 +5991,15 @@ dependencies = [
"digest 0.10.6",
]
[[package]]
name = "rlimit"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8a29d87a652dc4d43c586328706bb5cdff211f3f39a530f240b53f7221dab8e"
dependencies = [
"libc",
]
[[package]]
name = "rlp"
version = "0.5.2"
@ -7819,6 +7909,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
dependencies = [
"windows-targets 0.48.0",
]
[[package]]
name = "windows-sys"
version = "0.42.0"

4
bin/reth/Cargo.toml
View File

@ -38,6 +38,7 @@ reth-basic-payload-builder = { path = "../../crates/payload/basic" }
reth-discv4 = { path = "../../crates/net/discv4" }
reth-metrics = { workspace = true }
jemallocator = { version = "0.5.0", optional = true }
jemalloc-ctl = { version = "0.5.0", optional = true }
# crypto
secp256k1 = { workspace = true, features = ["global-context", "rand-std", "recovery"] }
@ -57,6 +58,7 @@ toml = { version = "0.7", features = ["display"] }
# metrics
metrics-exporter-prometheus = "0.11.0"
metrics-util = "0.14.0"
metrics-process = "1.0.9"
# test vectors generation
proptest = "1.0"
@ -86,7 +88,7 @@ pretty_assertions = "1.3.0"
humantime = "2.1.0"
[features]
jemalloc = ["dep:jemallocator"]
jemalloc = ["dep:jemallocator", "dep:jemalloc-ctl"]
jemalloc-prof = ["jemalloc", "jemallocator?/profiling"]
min-error-logs = ["tracing/release_max_level_error"]
min-warn-logs = ["tracing/release_max_level_warn"]

4
bin/reth/src/node/mod.rs
View File

@ -481,8 +481,8 @@ impl Command {
async fn start_metrics_endpoint(&self, db: Arc<Env<WriteMap>>) -> eyre::Result<()> {
if let Some(listen_addr) = self.metrics {
info!(target: "reth::cli", addr = %listen_addr, "Starting metrics endpoint");
prometheus_exporter::initialize_with_db_metrics(listen_addr, db).await?;
prometheus_exporter::initialize(listen_addr, db, metrics_process::Collector::default())
.await?;
}
Ok(())

127
bin/reth/src/prometheus_exporter.rs
View File

@ -14,19 +14,24 @@ use reth_db::{
use reth_metrics::metrics::{self, absolute_counter, describe_counter, Unit};
use std::{convert::Infallible, net::SocketAddr, sync::Arc};
/// Installs Prometheus as the metrics recorder and serves it over HTTP with a hook.
pub(crate) trait Hook: Fn() + Send + Sync {}
impl<T: Fn() + Send + Sync> Hook for T {}
/// Installs Prometheus as the metrics recorder and serves it over HTTP with hooks.
///
/// The hook is called every time the metrics are requested at the given endpoint, and can be used
/// The hooks are called every time the metrics are requested at the given endpoint, and can be used
/// to record values for pull-style metrics, i.e. metrics that are not automatically updated.
pub(crate) async fn initialize_with_hook<F: Fn() + Send + Sync + 'static>(
pub(crate) async fn initialize_with_hooks<F: Hook + 'static>(
listen_addr: SocketAddr,
hook: F,
hooks: impl IntoIterator<Item = F>,
) -> eyre::Result<()> {
let recorder = PrometheusBuilder::new().build_recorder();
let handle = recorder.handle();
let hooks: Vec<_> = hooks.into_iter().collect();
// Start endpoint
start_endpoint(listen_addr, handle, Arc::new(hook))
start_endpoint(listen_addr, handle, Arc::new(move || hooks.iter().for_each(|hook| hook())))
.await
.wrap_err("Could not start Prometheus endpoint")?;
@ -40,7 +45,7 @@ pub(crate) async fn initialize_with_hook<F: Fn() + Send + Sync + 'static>(
}
/// Starts an endpoint at the given address to serve Prometheus metrics.
async fn start_endpoint<F: Fn() + Send + Sync + 'static>(
async fn start_endpoint<F: Hook + 'static>(
listen_addr: SocketAddr,
handle: PrometheusHandle,
hook: Arc<F>,
@ -64,14 +69,16 @@ async fn start_endpoint<F: Fn() + Send + Sync + 'static>(
Ok(())
}
/// Installs Prometheus as the metrics recorder and serves it over HTTP with database metrics.
pub(crate) async fn initialize_with_db_metrics(
/// Installs Prometheus as the metrics recorder and serves it over HTTP with database and process
/// metrics.
pub(crate) async fn initialize(
listen_addr: SocketAddr,
db: Arc<Env<WriteMap>>,
process: metrics_process::Collector,
) -> eyre::Result<()> {
let db_stats = move || {
// TODO: A generic stats abstraction for other DB types to deduplicate this and `reth db
// stats`
// stats`
let _ = db.view(|tx| {
for table in tables::Tables::ALL.iter().map(|table| table.name()) {
let table_db =
@ -99,12 +106,112 @@ pub(crate) async fn initialize_with_db_metrics(
});
};
initialize_with_hook(listen_addr, db_stats).await?;
// Clone `process` to move it into the hook and use the original `process` for describe below.
let cloned_process = process.clone();
let hooks: Vec<Box<dyn Hook<Output = ()>>> = vec![
Box::new(db_stats),
Box::new(move || cloned_process.collect()),
Box::new(collect_memory_stats),
];
initialize_with_hooks(listen_addr, hooks).await?;
// We describe the metrics after the recorder is installed, otherwise this information is not
// registered
describe_counter!("db.table_size", Unit::Bytes, "The size of a database table (in bytes)");
describe_counter!("db.table_pages", "The number of database pages for a table");
process.describe();
describe_memory_stats();
Ok(())
}
#[cfg(feature = "jemalloc")]
fn collect_memory_stats() {
use jemalloc_ctl::{epoch, stats};
use reth_metrics::metrics::gauge;
use tracing::error;
if epoch::advance().map_err(|error| error!(?error, "Failed to advance jemalloc epoch")).is_err()
{
return
}
if let Ok(value) = stats::active::read()
.map_err(|error| error!(?error, "Failed to read jemalloc.stats.active"))
{
gauge!("jemalloc.active", value as f64);
}
if let Ok(value) = stats::allocated::read()
.map_err(|error| error!(?error, "Failed to read jemalloc.stats.allocated"))
{
gauge!("jemalloc.allocated", value as f64);
}
if let Ok(value) = stats::mapped::read()
.map_err(|error| error!(?error, "Failed to read jemalloc.stats.mapped"))
{
gauge!("jemalloc.mapped", value as f64);
}
if let Ok(value) = stats::metadata::read()
.map_err(|error| error!(?error, "Failed to read jemalloc.stats.metadata"))
{
gauge!("jemalloc.metadata", value as f64);
}
if let Ok(value) = stats::resident::read()
.map_err(|error| error!(?error, "Failed to read jemalloc.stats.resident"))
{
gauge!("jemalloc.resident", value as f64);
}
if let Ok(value) = stats::retained::read()
.map_err(|error| error!(?error, "Failed to read jemalloc.stats.retained"))
{
gauge!("jemalloc.retained", value as f64);
}
}
#[cfg(feature = "jemalloc")]
fn describe_memory_stats() {
use reth_metrics::metrics::describe_gauge;
describe_gauge!(
"jemalloc.active",
Unit::Bytes,
"Total number of bytes in active pages allocated by the application"
);
describe_gauge!(
"jemalloc.allocated",
Unit::Bytes,
"Total number of bytes allocated by the application"
);
describe_gauge!(
"jemalloc.mapped",
Unit::Bytes,
"Total number of bytes in active extents mapped by the allocator"
);
describe_gauge!(
"jemalloc.metadata",
Unit::Bytes,
"Total number of bytes dedicated to jemalloc metadata"
);
describe_gauge!(
"jemalloc.resident",
Unit::Bytes,
"Total number of bytes in physically resident data pages mapped by the allocator"
);
describe_gauge!(
"jemalloc.retained",
Unit::Bytes,
"Total number of bytes in virtual memory mappings that were retained rather than \
being returned to the operating system via e.g. munmap(2)"
);
}
#[cfg(not(feature = "jemalloc"))]
fn collect_memory_stats() {}
#[cfg(not(feature = "jemalloc"))]
fn describe_memory_stats() {}

7
bin/reth/src/stage/run.rs
View File

@ -127,7 +127,12 @@ impl Command {
if let Some(listen_addr) = self.metrics {
info!(target: "reth::cli", "Starting metrics endpoint at {}", listen_addr);
prometheus_exporter::initialize_with_db_metrics(listen_addr, Arc::clone(&db)).await?;
prometheus_exporter::initialize(
listen_addr,
Arc::clone(&db),
metrics_process::Collector::default(),
)
.await?;
}
let batch_size = self.batch_size.unwrap_or(self.to - self.from + 1);