From d570cf3e8ddd406b4073fe9a2e55ed38c758c5da Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Thu, 9 Oct 2025 08:13:45 +0000 Subject: [PATCH 1/5] fix: Create directory before migration --- src/node/migrate.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/node/migrate.rs b/src/node/migrate.rs index 650e098a6..3223ae9fa 100644 --- a/src/node/migrate.rs +++ b/src/node/migrate.rs @@ -130,7 +130,14 @@ impl<'a, N: HlNodeType> MigratorMdbx<'a, N> { fn migrate_mdbx_inner(&self) -> eyre::Result<()> { // There shouldn't be many headers in mdbx, but using file for safety info!("Old database detected, migrating mdbx..."); - let tmp_path = self.0.conversion_tmp_dir().join("headers.rmp"); + let conversion_tmp = self.0.conversion_tmp_dir(); + let tmp_path = conversion_tmp.join("headers.rmp"); + + if conversion_tmp.exists() { + std::fs::remove_dir_all(&conversion_tmp)?; + } + std::fs::create_dir_all(&conversion_tmp)?; + let count = self.export_old_headers(&tmp_path)?; self.import_new_headers(tmp_path, count)?; Ok(()) From 8a179a6d9ea2bfe582783ca74df83fe33cf56ac5 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Thu, 9 Oct 2025 08:13:53 +0000 Subject: [PATCH 2/5] perf: Use smaller chunks --- src/node/migrate.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/node/migrate.rs b/src/node/migrate.rs index 3223ae9fa..497eba31f 100644 --- a/src/node/migrate.rs +++ b/src/node/migrate.rs @@ -327,8 +327,8 @@ fn migrate_single_static_file( ) -> Result<(), eyre::Error> { info!("Migrating block range {}...", block_range); - // block_ranges into chunks of 100000 blocks - const CHUNK_SIZE: u64 = 100000; + // block_ranges into chunks of 10000 blocks + const CHUNK_SIZE: u64 = 10000; for chunk in (0..=block_range.end()).step_by(CHUNK_SIZE as usize) { let end = std::cmp::min(chunk + CHUNK_SIZE - 1, block_range.end()); let block_range = chunk..=end; From 5ba12a4850e14b9ab7dcb2bc9830706ddc2546d5 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Thu, 9 Oct 2025 08:20:22 +0000 Subject: [PATCH 3/5] perf: adjust chunk size, do not hold tx too long --- src/node/migrate.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/node/migrate.rs b/src/node/migrate.rs index 497eba31f..caa0b5d7f 100644 --- a/src/node/migrate.rs +++ b/src/node/migrate.rs @@ -251,7 +251,6 @@ impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> { let mut all_static_files = iter_static_files(&old_path)?; let all_static_files = all_static_files.remove(&StaticFileSegment::Headers).unwrap_or_default(); - let provider = self.0.provider_factory.provider()?; let mut first = true; @@ -271,6 +270,7 @@ impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> { let sf_provider = self.0.sf_provider(); let sf_tmp_provider = StaticFileProvider::::read_write(&conversion_tmp)?; + let provider = self.0.provider_factory.provider()?; let block_range_for_filename = sf_provider.find_fixed_range(block_range.start()); migrate_single_static_file(&sf_tmp_provider, &sf_provider, &provider, block_range)?; @@ -327,8 +327,8 @@ fn migrate_single_static_file( ) -> Result<(), eyre::Error> { info!("Migrating block range {}...", block_range); - // block_ranges into chunks of 10000 blocks - const CHUNK_SIZE: u64 = 10000; + // block_ranges into chunks of 50000 blocks + const CHUNK_SIZE: u64 = 50000; for chunk in (0..=block_range.end()).step_by(CHUNK_SIZE as usize) { let end = std::cmp::min(chunk + CHUNK_SIZE - 1, block_range.end()); let block_range = chunk..=end; From ca8c374116fdbc12e868bfec71e239b21fbb0059 Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Thu, 9 Oct 2025 08:49:29 +0000 Subject: [PATCH 4/5] feat: Mark migrator as experimental --- src/node/migrate.rs | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/node/migrate.rs b/src/node/migrate.rs index caa0b5d7f..8d49259b1 100644 --- a/src/node/migrate.rs +++ b/src/node/migrate.rs @@ -1,5 +1,5 @@ use alloy_consensus::Header; -use alloy_primitives::{b256, hex::ToHexExt, BlockHash, Bytes, B256, U256}; +use alloy_primitives::{B256, BlockHash, Bytes, U256, b256, hex::ToHexExt}; use reth::{ api::NodeTypesWithDBAdapter, args::{DatabaseArgs, DatadirArgs}, @@ -7,11 +7,12 @@ use reth::{ }; use reth_chainspec::EthChainSpec; use reth_db::{ - mdbx::{tx::Tx, RO}, + DatabaseEnv, + mdbx::{RO, tx::Tx}, models::CompactU256, static_file::iter_static_files, table::Decompress, - tables, DatabaseEnv, + tables, }; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW}, @@ -20,15 +21,15 @@ use reth_db_api::{ use reth_errors::ProviderResult; use reth_ethereum_primitives::EthereumReceipt; use reth_provider::{ - providers::{NodeTypesForProvider, StaticFileProvider}, - static_file::SegmentRangeInclusive, DatabaseProvider, ProviderFactory, ReceiptProvider, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, + providers::{NodeTypesForProvider, StaticFileProvider}, + static_file::SegmentRangeInclusive, }; use std::{fs::File, io::Write, path::PathBuf, sync::Arc}; use tracing::{info, warn}; -use crate::{chainspec::HlChainSpec, HlHeader, HlPrimitives}; +use crate::{HlHeader, HlPrimitives, chainspec::HlChainSpec}; pub(crate) trait HlNodeType: NodeTypesForProvider @@ -123,6 +124,8 @@ impl<'a, N: HlNodeType> MigratorMdbx<'a, N> { return Ok(false); } + check_if_migration_enabled()?; + self.migrate_mdbx_inner()?; Ok(true) } @@ -179,6 +182,18 @@ impl<'a, N: HlNodeType> MigratorMdbx<'a, N> { } } +fn check_if_migration_enabled() -> Result<(), eyre::Error> { + if !std::env::var("EXPERIMENTAL_MIGRATE_DB").is_ok() { + let err_msg = concat!( + "Detected an old database format but experimental database migration is currently disabled. ", + "To enable migration, set EXPERIMENTAL_MIGRATE_DB=1, or alternatively, resync your node (safest option)." + ); + warn!("{}", err_msg); + return Err(eyre::eyre!("{}", err_msg)); + } + Ok(()) +} + struct MigrateStaticFiles<'a, N: HlNodeType>(&'a Migrator); impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> { @@ -255,8 +270,8 @@ impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> { let mut first = true; for (block_range, _tx_ranges) in all_static_files { - let migration_needed = self.using_old_header(block_range.start())? || - self.using_old_header(block_range.end())?; + let migration_needed = self.using_old_header(block_range.start())? + || self.using_old_header(block_range.end())?; if !migration_needed { // Create a placeholder symlink self.create_placeholder(block_range)?; @@ -264,6 +279,8 @@ impl<'a, N: HlNodeType> MigrateStaticFiles<'a, N> { } if first { + check_if_migration_enabled()?; + info!("Old database detected, migrating static files..."); first = false; } From f83326059fdcc0102082e175cee38f320a6e311f Mon Sep 17 00:00:00 2001 From: sprites0 <199826320+sprites0@users.noreply.github.com> Date: Thu, 9 Oct 2025 08:55:40 +0000 Subject: [PATCH 5/5] chore: clippy --- src/node/migrate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/migrate.rs b/src/node/migrate.rs index 8d49259b1..8ed0919fc 100644 --- a/src/node/migrate.rs +++ b/src/node/migrate.rs @@ -183,7 +183,7 @@ impl<'a, N: HlNodeType> MigratorMdbx<'a, N> { } fn check_if_migration_enabled() -> Result<(), eyre::Error> { - if !std::env::var("EXPERIMENTAL_MIGRATE_DB").is_ok() { + if std::env::var("EXPERIMENTAL_MIGRATE_DB").is_err() { let err_msg = concat!( "Detected an old database format but experimental database migration is currently disabled. ", "To enable migration, set EXPERIMENTAL_MIGRATE_DB=1, or alternatively, resync your node (safest option)."