jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(op): add empty receipts for genesis if first block is one (#9769)

Browse Source 
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
This commit is contained in:
Emilia Hane
2024-07-26 19:07:25 +02:00
committed by GitHub
parent 87564bd17e
commit 0a4b717d1b
5 changed files with 172 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -7907,6 +7907,7 @@ dependencies = [
"reth-consensus",
"reth-db",
"reth-db-api",
"reth-db-common",
"reth-downloaders",
"reth-errors",
"reth-evm-optimism",
@ -7924,6 +7925,7 @@ dependencies = [
"reth-static-file-types",
"serde_json",
"shellexpand",
"tempfile",
"tokio",
"tokio-util",
"tracing",

18
crates/net/downloaders/src/receipt_file_client.rs
View File

@ -289,11 +289,14 @@ mod test {
}
}
pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_1: &[u8] = &hex!("f901a4f901a1800183031843f90197f89b948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef863a00109fc6f55cf40689f02fbaad7af7fe7bbac8a3d2186600afc7d3e10cac6027ba00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2da000000000000000000000000000000000000000000000000000000000618d8837f89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68ba000000000000000000000000000000000000000000000000000000000d0e3ebf0a00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2d80f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234fa000000000000000000000000000000000000000000000007edc6ca0bb683480008001");
/// No receipts for genesis block
const MOCK_RECEIPT_BLOCK_NO_TRANSACTIONS: &[u8] = &hex!("c0");
pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_2: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d0ea0e40a00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000e5e7492282fd1e3bfac337a0beccd29b15b7b24080f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007eda7867e0c7d480008002");
const MOCK_RECEIPT_ENCODED_BLOCK_1: &[u8] = &hex!("f901a4f901a1800183031843f90197f89b948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef863a00109fc6f55cf40689f02fbaad7af7fe7bbac8a3d2186600afc7d3e10cac6027ba00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2da000000000000000000000000000000000000000000000000000000000618d8837f89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68ba000000000000000000000000000000000000000000000000000000000d0e3ebf0a00000000000000000000000000000000000000000000000000000000000014218a000000000000000000000000070b17c0fe982ab4a7ac17a4c25485643151a1f2d80f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234fa000000000000000000000000000000000000000000000007edc6ca0bb683480008001");
pub(crate) const MOCK_RECEIPT_ENCODED_BLOCK_3: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d101e54ba00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000fa011d8d6c26f13abe2cefed38226e401b2b8a9980f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007ed8842f06277480008003");
const MOCK_RECEIPT_ENCODED_BLOCK_2: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d0ea0e40a00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000e5e7492282fd1e3bfac337a0beccd29b15b7b24080f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007eda7867e0c7d480008002");
const MOCK_RECEIPT_ENCODED_BLOCK_3: &[u8] = &hex!("f90106f9010380018301c60df8faf89c948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef884a092e98423f8adac6e64d0608e519fd1cefb861498385c6dee70d58fc926ddc68da000000000000000000000000000000000000000000000000000000000d101e54ba00000000000000000000000000000000000000000000000000000000000014218a0000000000000000000000000fa011d8d6c26f13abe2cefed38226e401b2b8a9980f85a948ce8c13d816fe6daf12d6fd9e4952e1fc88850aef842a0fe25c73e3b9089fac37d55c4c7efcba6f04af04cebd2fc4d6d7dbb07e1e5234ea000000000000000000000000000000000000000000000007ed8842f06277480008003");
fn mock_receipt_1() -> MockReceipt {
let receipt = receipt_block_1();
@ -331,7 +334,7 @@ mod test {
}
}
pub(crate) fn receipt_block_1() -> ReceiptWithBlockNumber {
fn receipt_block_1() -> ReceiptWithBlockNumber {
let log_1 = Log {
address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")),
data: LogData::new(
@ -404,7 +407,7 @@ mod test {
ReceiptWithBlockNumber { receipt, number: 1 }
}
pub(crate) fn receipt_block_2() -> ReceiptWithBlockNumber {
fn receipt_block_2() -> ReceiptWithBlockNumber {
let log_1 = Log {
address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")),
data: LogData::new(
@ -456,7 +459,7 @@ mod test {
ReceiptWithBlockNumber { receipt, number: 2 }
}
pub(crate) fn receipt_block_3() -> ReceiptWithBlockNumber {
fn receipt_block_3() -> ReceiptWithBlockNumber {
let log_1 = Log {
address: Address::from(hex!("8ce8c13d816fe6daf12d6fd9e4952e1fc88850ae")),
data: LogData::new(
@ -560,9 +563,6 @@ mod test {
assert_eq!(receipt_block_3(), third_decoded_receipt);
}
/// No receipts for genesis block
const MOCK_RECEIPT_BLOCK_NO_TRANSACTIONS: &[u8] = &hex!("c0");
#[tokio::test]
async fn receipt_file_client_ovm_codec() {
init_test_tracing();

6
crates/optimism/cli/Cargo.toml
View File

@ -60,8 +60,14 @@ tokio-util = { workspace = true, features = ["codec"] }
tracing.workspace = true
eyre.workspace = true
[dev-dependencies]
tempfile.workspace = true
reth-stages = { workspace = true, features = ["test-utils"] }
reth-db-common.workspace = true
[features]
optimism = [
"reth-primitives/optimism",
"reth-evm-optimism/optimism",
"reth-provider/optimism",
]

215
crates/optimism/cli/src/commands/import_receipts.rs
View File

@ -16,8 +16,8 @@ use reth_node_core::version::SHORT_VERSION;
use reth_optimism_primitives::bedrock_import::is_dup_tx;
use reth_primitives::Receipts;
use reth_provider::{
writer::StorageWriter, OriginalValuesKnown, ProviderFactory, StageCheckpointReader,
StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader,
writer::StorageWriter, DatabaseProviderFactory, OriginalValuesKnown, ProviderFactory,
StageCheckpointReader, StateWriter, StaticFileProviderFactory, StaticFileWriter, StatsReader,
};
use reth_stages::StageId;
use reth_static_file_types::StaticFileSegment;
@ -75,35 +75,29 @@ impl ImportReceiptsOpCommand {
}
}
/// Imports receipts to static files. Takes a filter callback as parameter, that returns the total
/// number of filtered out receipts.
///
/// Caution! Filter callback must replace completely filtered out receipts for a block, with empty
/// vectors, rather than `vec!(None)`. This is since the code for writing to static files, expects
/// indices in the [`Receipts`] list, to map to sequential block numbers.
/// Imports receipts to static files from file in chunks. See [`import_receipts_from_reader`].
pub async fn import_receipts_from_file<DB, P, F>(
provider_factory: ProviderFactory<DB>,
path: P,
chunk_len: Option<u64>,
mut filter: F,
filter: F,
) -> eyre::Result<()>
where
DB: Database,
P: AsRef<Path>,
F: FnMut(u64, &mut Receipts) -> usize,
{
let provider = provider_factory.provider_rw()?;
let static_file_provider = provider_factory.static_file_provider();
let total_imported_txns = static_file_provider
let total_imported_txns = provider_factory
.static_file_provider()
.count_entries::<tables::Transactions>()
.expect("transaction static files must exist before importing receipts");
let highest_block_transactions = static_file_provider
let highest_block_transactions = provider_factory
.static_file_provider()
.get_highest_static_file_block(StaticFileSegment::Transactions)
.expect("transaction static files must exist before importing receipts");
for stage in StageId::ALL {
let checkpoint = provider.get_stage_checkpoint(stage)?;
let checkpoint = provider_factory.database_provider_ro()?.get_stage_checkpoint(stage)?;
trace!(target: "reth::cli",
?stage,
?checkpoint,
@ -111,59 +105,20 @@ where
);
}
let mut total_decoded_receipts = 0;
let mut total_filtered_out_dup_txns = 0;
// open file
let mut reader = ChunkedFileReader::new(path, chunk_len).await?;
let reader = ChunkedFileReader::new(&path, chunk_len).await?;
while let Some(file_client) =
reader.next_chunk::<ReceiptFileClient<HackReceiptFileCodec>>().await?
{
// create a new file client from chunk read from file
let ReceiptFileClient {
mut receipts,
first_block,
total_receipts: total_receipts_chunk,
..
} = file_client;
// mark these as decoded
total_decoded_receipts += total_receipts_chunk;
total_filtered_out_dup_txns += filter(first_block, &mut receipts);
info!(target: "reth::cli",
first_receipts_block=?first_block,
total_receipts_chunk,
"Importing receipt file chunk"
);
// We're reusing receipt writing code internal to
// `StorageWriter::append_receipts_from_blocks`, so we just use a default empty
// `BundleState`.
let execution_outcome =
ExecutionOutcome::new(Default::default(), receipts, first_block, Default::default());
let static_file_producer =
static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?;
// finally, write the receipts
let mut storage_writer = StorageWriter::new(Some(&provider), Some(static_file_producer));
storage_writer.write_to_storage(execution_outcome, OriginalValuesKnown::Yes)?;
}
provider.commit()?;
// as static files works in file ranges, internally it will be committing when creating the
// next file range already, so we only need to call explicitly at the end.
static_file_provider.commit()?;
// import receipts
let ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns } =
import_receipts_from_reader(&provider_factory, reader, filter).await?;
if total_decoded_receipts == 0 {
error!(target: "reth::cli", "No receipts were imported, ensure the receipt file is valid and not empty");
return Ok(())
}
let total_imported_receipts = static_file_provider
let total_imported_receipts = provider_factory
.static_file_provider()
.count_entries::<tables::Receipts>()
.expect("static files must exist after ensuring we decoded more than zero");
@ -184,7 +139,8 @@ where
);
}
let highest_block_receipts = static_file_provider
let highest_block_receipts = provider_factory
.static_file_provider()
.get_highest_static_file_block(StaticFileSegment::Receipts)
.expect("static files must exist after ensuring we decoded more than zero");
@ -205,3 +161,140 @@ where
Ok(())
}
/// Imports receipts to static files. Takes a filter callback as parameter, that returns the total
/// number of filtered out receipts.
///
/// Caution! Filter callback must replace completely filtered out receipts for a block, with empty
/// vectors, rather than `vec!(None)`. This is since the code for writing to static files, expects
/// indices in the [`Receipts`] list, to map to sequential block numbers.
pub async fn import_receipts_from_reader<DB, F>(
provider_factory: &ProviderFactory<DB>,
mut reader: ChunkedFileReader,
mut filter: F,
) -> eyre::Result<ImportReceiptsResult>
where
DB: Database,
F: FnMut(u64, &mut Receipts) -> usize,
{
let mut total_decoded_receipts = 0;
let mut total_filtered_out_dup_txns = 0;
let provider = provider_factory.provider_rw()?;
let static_file_provider = provider_factory.static_file_provider();
while let Some(file_client) =
reader.next_chunk::<ReceiptFileClient<HackReceiptFileCodec>>().await?
{
// create a new file client from chunk read from file
let ReceiptFileClient {
mut receipts,
mut first_block,
total_receipts: total_receipts_chunk,
..
} = file_client;
// mark these as decoded
total_decoded_receipts += total_receipts_chunk;
total_filtered_out_dup_txns += filter(first_block, &mut receipts);
info!(target: "reth::cli",
first_receipts_block=?first_block,
total_receipts_chunk,
"Importing receipt file chunk"
);
// It is possible for the first receipt returned by the file client to be the genesis
// block. In this case, we just prepend empty receipts to the current list of receipts.
// When initially writing to static files, the provider expects the first block to be block
// one. So, if the first block returned by the file client is the genesis block, we remove
// those receipts.
if first_block == 0 {
// remove the first empty receipts
let genesis_receipts = receipts.remove(0);
debug_assert!(genesis_receipts.is_empty());
// this ensures the execution outcome and static file producer start at block 1
first_block = 1;
// we don't count this as decoded so the partial import check later does not error if
// this branch is executed
total_decoded_receipts -= 1; // safe because chunk will be `None` if empty
}
// We're reusing receipt writing code internal to
// `StorageWriter::append_receipts_from_blocks`, so we just use a default empty
// `BundleState`.
let execution_outcome =
ExecutionOutcome::new(Default::default(), receipts, first_block, Default::default());
let static_file_producer =
static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)?;
// finally, write the receipts
let mut storage_writer = StorageWriter::new(Some(&provider), Some(static_file_producer));
storage_writer.write_to_storage(execution_outcome, OriginalValuesKnown::Yes)?;
}
provider.commit()?;
// as static files works in file ranges, internally it will be committing when creating the
// next file range already, so we only need to call explicitly at the end.
static_file_provider.commit()?;
Ok(ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns })
}
/// Result of importing receipts in chunks.
#[derive(Debug)]
pub struct ImportReceiptsResult {
total_decoded_receipts: usize,
total_filtered_out_dup_txns: usize,
}
#[cfg(test)]
mod test {
use reth_db_common::init::init_genesis;
use reth_primitives::hex;
use reth_stages::test_utils::TestStageDB;
use tempfile::tempfile;
use tokio::{
fs::File,
io::{AsyncSeekExt, AsyncWriteExt, SeekFrom},
};
use crate::file_codec_ovm_receipt::test::{
HACK_RECEIPT_ENCODED_BLOCK_1, HACK_RECEIPT_ENCODED_BLOCK_2, HACK_RECEIPT_ENCODED_BLOCK_3,
};
use super::*;
/// No receipts for genesis block
const EMPTY_RECEIPTS_GENESIS_BLOCK: &[u8] = &hex!("c0");
#[ignore]
#[tokio::test]
async fn filter_out_genesis_block_receipts() {
let mut f: File = tempfile().unwrap().into();
f.write_all(EMPTY_RECEIPTS_GENESIS_BLOCK).await.unwrap();
f.write_all(HACK_RECEIPT_ENCODED_BLOCK_1).await.unwrap();
f.write_all(HACK_RECEIPT_ENCODED_BLOCK_2).await.unwrap();
f.write_all(HACK_RECEIPT_ENCODED_BLOCK_3).await.unwrap();
f.flush().await.unwrap();
f.seek(SeekFrom::Start(0)).await.unwrap();
let reader =
ChunkedFileReader::from_file(f, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE).await.unwrap();
let db = TestStageDB::default();
init_genesis(db.factory.clone()).unwrap();
// todo: where does import command init receipts ? probably somewhere in pipeline
let ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns } =
import_receipts_from_reader(&TestStageDB::default().factory, reader, |_, _| 0)
.await
.unwrap();
assert_eq!(total_decoded_receipts, 3);
assert_eq!(total_filtered_out_dup_txns, 0);
}
}

2
crates/optimism/cli/src/file_codec_ovm_receipt.rs
View File

@ -93,7 +93,7 @@ impl TryFrom<HackReceipt> for ReceiptWithBlockNumber {
}
#[cfg(test)]
pub(super) mod test {
pub(crate) mod test {
use reth_primitives::{alloy_primitives::LogData, hex};
use super::*;