feat: continuous download (#1744)

This commit is contained in:
Dan Cline
2023-03-16 22:33:11 -04:00
committed by GitHub
parent 241ec32abf
commit 1711d801af
6 changed files with 404 additions and 83 deletions

View File

@ -37,7 +37,7 @@ use reth_network::{
error::NetworkError, FetchClient, NetworkConfig, NetworkHandle, NetworkManager,
};
use reth_network_api::NetworkInfo;
use reth_primitives::{BlockHashOrNumber, ChainSpec, Head, H256};
use reth_primitives::{BlockHashOrNumber, ChainSpec, Head, SealedHeader, H256};
use reth_provider::{BlockProvider, HeaderProvider, ShareableDatabase};
use reth_rpc_engine_api::{EngineApi, EngineApiHandle};
use reth_staged_sync::{
@ -50,7 +50,7 @@ use reth_staged_sync::{
};
use reth_stages::{
prelude::*,
stages::{ExecutionStage, SenderRecoveryStage, TotalDifficultyStage, FINISH},
stages::{ExecutionStage, HeaderStage, SenderRecoveryStage, TotalDifficultyStage, FINISH},
};
use reth_tasks::TaskExecutor;
use std::{
@ -106,6 +106,12 @@ pub struct Command {
#[clap(flatten)]
network: NetworkArgs,
/// Prompt the downloader to download blocks one at a time.
///
/// NOTE: This is for testing purposes only.
#[arg(long = "debug.continuous", help_heading = "Debug")]
continuous: bool,
/// Set the chain tip manually for testing purposes.
///
/// NOTE: This is a temporary flag
@ -173,6 +179,10 @@ impl Command {
.await?;
info!(target: "reth::cli", "Started RPC server");
if self.continuous {
info!(target: "reth::cli", "Continuous sync mode enabled");
}
let engine_api_handle =
self.init_engine_api(Arc::clone(&db), forkchoice_state_tx, &ctx.task_executor);
info!(target: "reth::cli", "Engine API handler initialized");
@ -259,6 +269,7 @@ impl Command {
network.clone(),
consensus,
max_block,
self.continuous,
)
.await?;
@ -391,17 +402,38 @@ impl Command {
fetch_client: FetchClient,
tip: H256,
) -> Result<u64, reth_interfaces::Error> {
if let Some(number) = db.view(|tx| tx.get::<tables::HeaderNumbers>(tip))?? {
info!(target: "reth::cli", ?tip, number, "Successfully looked up tip block number in the database");
return Ok(number)
Ok(self.fetch_tip(db, fetch_client, BlockHashOrNumber::Hash(tip)).await?.number)
}
/// Attempt to look up the block with the given number and return the header.
///
/// NOTE: The download is attempted with infinite retries.
async fn fetch_tip(
&self,
db: Arc<Env<WriteMap>>,
fetch_client: FetchClient,
tip: BlockHashOrNumber,
) -> Result<SealedHeader, reth_interfaces::Error> {
let tip_num = match tip {
BlockHashOrNumber::Hash(hash) => {
info!(target: "reth::cli", ?hash, "Fetching tip block from the network.");
db.view(|tx| tx.get::<tables::HeaderNumbers>(hash))??.unwrap()
}
BlockHashOrNumber::Number(number) => number,
};
// try to look up the header in the database
if let Some(header) = db.view(|tx| tx.get::<tables::Headers>(tip_num))?? {
info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database");
return Ok(header.seal_slow())
}
info!(target: "reth::cli", ?tip, "Fetching tip block number from the network.");
info!(target: "reth::cli", ?tip, "Fetching tip block from the network.");
loop {
match get_single_header(fetch_client.clone(), BlockHashOrNumber::Hash(tip)).await {
match get_single_header(fetch_client.clone(), tip).await {
Ok(tip_header) => {
info!(target: "reth::cli", ?tip, number = tip_header.number, "Successfully fetched tip block number");
return Ok(tip_header.number)
info!(target: "reth::cli", ?tip, "Successfully fetched tip");
return Ok(tip_header)
}
Err(error) => {
error!(target: "reth::cli", %error, "Failed to fetch the tip. Retrying...");
@ -429,6 +461,7 @@ impl Command {
.build(ShareableDatabase::new(db, self.chain.clone()))
}
#[allow(clippy::too_many_arguments)]
async fn build_pipeline<H, B, U>(
&self,
config: &Config,
@ -437,6 +470,7 @@ impl Command {
updater: U,
consensus: &Arc<dyn Consensus>,
max_block: Option<u64>,
continuous: bool,
) -> eyre::Result<Pipeline<Env<WriteMap>, U>>
where
H: HeaderDownloader + 'static,
@ -453,24 +487,43 @@ impl Command {
}
let factory = reth_executor::Factory::new(self.chain.clone());
let default_stages = if continuous {
let continuous_headers =
HeaderStage::new(header_downloader, consensus.clone()).continuous();
let online_builder = OnlineStages::builder_with_headers(
continuous_headers,
consensus.clone(),
body_downloader,
);
DefaultStages::<H, B, U, reth_executor::Factory>::add_offline_stages(
online_builder,
updater.clone(),
factory.clone(),
)
} else {
DefaultStages::new(
consensus.clone(),
header_downloader,
body_downloader,
updater.clone(),
factory.clone(),
)
.builder()
};
let pipeline = builder
.with_sync_state_updater(updater.clone())
.with_sync_state_updater(updater)
.add_stages(
DefaultStages::new(
consensus.clone(),
header_downloader,
body_downloader,
updater,
factory.clone(),
)
.set(
TotalDifficultyStage::new(consensus.clone())
.with_commit_threshold(stage_conf.total_difficulty.commit_threshold),
)
.set(SenderRecoveryStage {
commit_threshold: stage_conf.sender_recovery.commit_threshold,
})
.set(ExecutionStage::new(factory, stage_conf.execution.commit_threshold)),
default_stages
.set(
TotalDifficultyStage::new(consensus.clone())
.with_commit_threshold(stage_conf.total_difficulty.commit_threshold),
)
.set(SenderRecoveryStage {
commit_threshold: stage_conf.sender_recovery.commit_threshold,
})
.set(ExecutionStage::new(factory, stage_conf.execution.commit_threshold)),
)
.build();

View File

@ -148,6 +148,14 @@ pub enum DownloadError {
/// The hash of the expected tip
expected: H256,
},
/// Received a tip with an invalid tip number
#[error("Received invalid tip number: {received:?}. Expected {expected:?}.")]
InvalidTipNumber {
/// The block number of the received tip
received: u64,
/// The block number of the expected tip
expected: u64,
},
/// Received a response to a request with unexpected start block
#[error("Headers response starts at unexpected block: {received:?}. Expected {expected:?}.")]
HeadersResponseStartBlockMismatch {

View File

@ -3,7 +3,7 @@ use crate::{
p2p::error::{DownloadError, DownloadResult},
};
use futures::Stream;
use reth_primitives::{SealedHeader, H256};
use reth_primitives::{BlockHashOrNumber, SealedHeader, H256};
/// A downloader capable of fetching and yielding block headers.
///
@ -48,6 +48,8 @@ pub enum SyncTarget {
/// The benefit of this variant is, that this already provides the block number of the highest
/// missing block.
Gap(SealedHeader),
/// This represents a tip by block number
TipNum(u64),
}
// === impl SyncTarget ===
@ -57,10 +59,11 @@ impl SyncTarget {
///
/// This returns the hash if the target is [SyncTarget::Tip] or the `parent_hash` of the given
/// header in [SyncTarget::Gap]
pub fn tip(&self) -> H256 {
pub fn tip(&self) -> BlockHashOrNumber {
match self {
SyncTarget::Tip(tip) => *tip,
SyncTarget::Gap(gap) => gap.parent_hash,
SyncTarget::Tip(tip) => (*tip).into(),
SyncTarget::Gap(gap) => gap.parent_hash.into(),
SyncTarget::TipNum(num) => (*num).into(),
}
}
}

View File

@ -16,7 +16,9 @@ use reth_interfaces::{
priority::Priority,
},
};
use reth_primitives::{BlockNumber, Header, HeadersDirection, PeerId, SealedHeader, H256};
use reth_primitives::{
BlockHashOrNumber, BlockNumber, Header, HeadersDirection, PeerId, SealedHeader, H256,
};
use reth_tasks::{TaskSpawner, TokioTaskExecutor};
use std::{
cmp::{Ordering, Reverse},
@ -116,14 +118,14 @@ where
self.local_head.as_ref().expect("is initialized").number
}
/// Returns the existing sync target hash.
/// Returns the existing sync target.
///
/// # Panics
///
/// If the sync target has never been set.
#[inline]
fn existing_sync_target_hash(&self) -> H256 {
self.sync_target.as_ref().expect("is initialized").hash
fn existing_sync_target(&self) -> SyncTargetBlock {
self.sync_target.as_ref().expect("is initialized").clone()
}
/// Max requests to handle at the same time
@ -198,7 +200,7 @@ where
headers: Vec<Header>,
peer_id: PeerId,
) -> Result<(), HeadersResponseError> {
let sync_target_hash = self.existing_sync_target_hash();
let sync_target = self.existing_sync_target();
let mut validated = Vec::with_capacity(headers.len());
let sealed_headers = headers.into_par_iter().map(|h| h.seal_slow()).collect::<Vec<_>>();
@ -211,15 +213,45 @@ where
trace!(target: "downloaders::headers", ?error ,"Failed to validate header");
return Err(HeadersResponseError { request, peer_id: Some(peer_id), error })
}
} else if parent.hash() != sync_target_hash {
return Err(HeadersResponseError {
request,
peer_id: Some(peer_id),
error: DownloadError::InvalidTip {
received: parent.hash(),
expected: sync_target_hash,
},
})
} else {
match sync_target {
SyncTargetBlock::Hash(hash) => {
if parent.hash() != hash {
return Err(HeadersResponseError {
request,
peer_id: Some(peer_id),
error: DownloadError::InvalidTip {
received: parent.hash(),
expected: hash,
},
})
}
}
SyncTargetBlock::Number(number) => {
if parent.number != number {
return Err(HeadersResponseError {
request,
peer_id: Some(peer_id),
error: DownloadError::InvalidTipNumber {
received: parent.number,
expected: number,
},
})
}
}
SyncTargetBlock::HashAndNumber { hash, .. } => {
if parent.hash() != hash {
return Err(HeadersResponseError {
request,
peer_id: Some(peer_id),
error: DownloadError::InvalidTip {
received: parent.hash(),
expected: hash,
},
})
}
}
}
}
validated.push(parent);
@ -246,7 +278,7 @@ where
fn on_block_number_update(&mut self, target_block_number: u64, next_block: u64) {
// Update the trackers
if let Some(old_target) =
self.sync_target.as_mut().and_then(|t| t.number.replace(target_block_number))
self.sync_target.as_mut().and_then(|t| t.replace_number(target_block_number))
{
if target_block_number > old_target {
// the new target is higher than the old target we need to update the
@ -277,7 +309,7 @@ where
&mut self,
response: HeadersRequestOutcome,
) -> Result<(), HeadersResponseError> {
let sync_target_hash = self.existing_sync_target_hash();
let sync_target = self.existing_sync_target();
let HeadersRequestOutcome { request, outcome } = response;
match outcome {
Ok(res) => {
@ -299,15 +331,43 @@ where
let target = headers.remove(0).seal_slow();
if target.hash() != sync_target_hash {
return Err(HeadersResponseError {
request,
peer_id: Some(peer_id),
error: DownloadError::InvalidTip {
received: target.hash(),
expected: sync_target_hash,
},
})
match sync_target {
SyncTargetBlock::Hash(hash) => {
if target.hash() != hash {
return Err(HeadersResponseError {
request,
peer_id: Some(peer_id),
error: DownloadError::InvalidTip {
received: target.hash(),
expected: hash,
},
})
}
}
SyncTargetBlock::Number(number) => {
if target.number != number {
return Err(HeadersResponseError {
request,
peer_id: Some(peer_id),
error: DownloadError::InvalidTipNumber {
received: target.number,
expected: number,
},
})
}
}
SyncTargetBlock::HashAndNumber { hash, .. } => {
if target.hash() != hash {
return Err(HeadersResponseError {
request,
peer_id: Some(peer_id),
error: DownloadError::InvalidTip {
received: target.hash(),
expected: hash,
},
})
}
}
}
trace!(target: "downloaders::headers", head=?self.local_block_number(), hash=?target.hash(), number=%target.number, "Received sync target");
@ -463,8 +523,8 @@ where
}
/// Returns the request for the `sync_target` header.
fn get_sync_target_request(&self, start: H256) -> HeadersRequest {
HeadersRequest { start: start.into(), limit: 1, direction: HeadersDirection::Falling }
fn get_sync_target_request(&self, start: BlockHashOrNumber) -> HeadersRequest {
HeadersRequest { start, limit: 1, direction: HeadersDirection::Falling }
}
/// Starts a request future
@ -552,7 +612,7 @@ where
/// If the given target is different from the current target, we need to update the sync target
fn update_sync_target(&mut self, target: SyncTarget) {
let current_tip = self.sync_target.as_ref().map(|t| t.hash);
let current_tip = self.sync_target.as_ref().and_then(|t| t.hash());
match target {
SyncTarget::Tip(tip) => {
if Some(tip) != current_tip {
@ -574,8 +634,9 @@ where
trace!(target: "downloaders::headers", new=?target, "Request new sync target");
self.metrics.out_of_order_requests.increment(1);
self.sync_target = Some(new_sync_target);
self.sync_target_request =
Some(self.request_fut(self.get_sync_target_request(tip), Priority::High));
self.sync_target_request = Some(
self.request_fut(self.get_sync_target_request(tip.into()), Priority::High),
);
}
}
SyncTarget::Gap(existing) => {
@ -591,15 +652,23 @@ where
// Update the sync target hash
self.sync_target = match self.sync_target.take() {
Some(mut sync_target) => {
sync_target.hash = target;
Some(sync_target)
}
Some(sync_target) => Some(sync_target.with_hash(target)),
None => Some(SyncTargetBlock::from_hash(target)),
};
self.on_block_number_update(parent_block_number, parent_block_number);
}
}
SyncTarget::TipNum(num) => {
let current_tip_num = self.sync_target.as_ref().and_then(|t| t.number());
if Some(num) != current_tip_num {
trace!(target: "downloaders::headers", %num, "Updating sync target based on num");
// just update the sync target
self.sync_target = Some(SyncTargetBlock::from_number(num));
self.sync_target_request = Some(
self.request_fut(self.get_sync_target_request(num.into()), Priority::High),
);
}
}
}
}
@ -823,24 +892,90 @@ impl HeadersResponseError {
}
/// The block to which we want to close the gap: (local head...sync target]
#[derive(Debug, Default)]
struct SyncTargetBlock {
/// This tracks the sync target block, so this could be either a block number or hash.
#[derive(Clone, Debug)]
pub enum SyncTargetBlock {
/// Block hash of the targeted block
hash: H256,
/// This is an `Option` because we don't know the block number at first
number: Option<u64>,
Hash(H256),
/// Block number of the targeted block
Number(u64),
/// Both the block hash and number of the targeted block
HashAndNumber {
/// Block hash of the targeted block
hash: H256,
/// Block number of the targeted block
number: u64,
},
}
impl SyncTargetBlock {
/// Create new instance from hash.
fn from_hash(hash: H256) -> Self {
Self { hash, number: None }
Self::Hash(hash)
}
/// Create new instance from number.
fn from_number(num: u64) -> Self {
Self::Number(num)
}
/// Set the hash for the sync target.
fn with_hash(self, hash: H256) -> Self {
match self {
Self::Hash(_) => Self::Hash(hash),
Self::Number(number) => Self::HashAndNumber { hash, number },
Self::HashAndNumber { number, .. } => Self::HashAndNumber { hash, number },
}
}
/// Set a number on the instance.
fn with_number(mut self, number: u64) -> Self {
self.number = Some(number);
self
fn with_number(self, number: u64) -> Self {
match self {
Self::Hash(hash) => Self::HashAndNumber { hash, number },
Self::Number(_) => Self::Number(number),
Self::HashAndNumber { hash, .. } => Self::HashAndNumber { hash, number },
}
}
/// Replace the target block number, and return the old block number, if it was set.
///
/// If the target block is a hash, this be converted into a `HashAndNumber`, but return `None`.
/// The semantics should be equivalent to that of `Option::replace`.
fn replace_number(&mut self, number: u64) -> Option<u64> {
match self {
Self::Hash(hash) => {
*self = Self::HashAndNumber { hash: *hash, number };
None
}
Self::Number(old_number) => {
let res = Some(*old_number);
*self = Self::Number(number);
res
}
Self::HashAndNumber { number: old_number, hash } => {
let res = Some(*old_number);
*self = Self::HashAndNumber { hash: *hash, number };
res
}
}
}
/// Return the hash of the target block, if it is set.
fn hash(&self) -> Option<H256> {
match self {
Self::Hash(hash) => Some(*hash),
Self::Number(_) => None,
Self::HashAndNumber { hash, .. } => Some(*hash),
}
}
/// Return the block number of the sync target, if it is set.
fn number(&self) -> Option<u64> {
match self {
Self::Hash(_) => None,
Self::Number(number) => Some(*number),
Self::HashAndNumber { number, .. } => Some(*number),
}
}
}
@ -989,6 +1124,65 @@ mod tests {
use reth_interfaces::test_utils::{TestConsensus, TestHeadersClient};
use reth_primitives::SealedHeader;
/// Tests that `replace_number` works the same way as Option::replace
#[test]
fn test_replace_number_semantics() {
struct Fixture {
// input fields (both SyncTargetBlock and Option<u64>)
sync_target_block: SyncTargetBlock,
sync_target_option: Option<u64>,
// option to replace
replace_number: u64,
// expected method result
expected_result: Option<u64>,
// output state
new_number: u64,
}
let fixtures = vec![
Fixture {
sync_target_block: SyncTargetBlock::Hash(H256::random()),
// Hash maps to None here, all other variants map to Some
sync_target_option: None,
replace_number: 1,
expected_result: None,
new_number: 1,
},
Fixture {
sync_target_block: SyncTargetBlock::Number(1),
sync_target_option: Some(1),
replace_number: 2,
expected_result: Some(1),
new_number: 2,
},
Fixture {
sync_target_block: SyncTargetBlock::HashAndNumber {
hash: H256::random(),
number: 1,
},
sync_target_option: Some(1),
replace_number: 2,
expected_result: Some(1),
new_number: 2,
},
];
for fixture in fixtures {
let mut sync_target_block = fixture.sync_target_block;
let result = sync_target_block.replace_number(fixture.replace_number);
assert_eq!(result, fixture.expected_result);
assert_eq!(sync_target_block.number(), Some(fixture.new_number));
let mut sync_target_option = fixture.sync_target_option;
let option_result = sync_target_option.replace(fixture.replace_number);
assert_eq!(option_result, fixture.expected_result);
assert_eq!(sync_target_option, Some(fixture.new_number));
}
}
/// Tests that request calc works
#[test]
fn test_sync_target_update() {
@ -1013,7 +1207,7 @@ mod tests {
assert!(downloader.sync_target_request.is_none());
assert_matches!(
downloader.sync_target,
Some(target) => target.number.is_some()
Some(target) => target.number().is_some()
);
}

View File

@ -94,6 +94,23 @@ impl<H, B, S, EF> DefaultStages<H, B, S, EF> {
}
}
impl<H, B, S, EF> DefaultStages<H, B, S, EF>
where
S: StatusUpdater + 'static,
EF: ExecutorFactory,
{
/// Appends the default offline stages and default finish stage to the given builder.
pub fn add_offline_stages<DB: Database>(
default_offline: StageSetBuilder<DB>,
status_updater: S,
executor_factory: EF,
) -> StageSetBuilder<DB> {
default_offline
.add_set(OfflineStages::new(executor_factory))
.add_stage(FinishStage::new(status_updater))
}
}
impl<DB, H, B, S, EF> StageSet<DB> for DefaultStages<H, B, S, EF>
where
DB: Database,
@ -103,10 +120,7 @@ where
EF: ExecutorFactory,
{
fn builder(self) -> StageSetBuilder<DB> {
self.online
.builder()
.add_set(OfflineStages::new(self.executor_factory))
.add_stage(FinishStage::new(self.status_updater))
Self::add_offline_stages(self.online.builder(), self.status_updater, self.executor_factory)
}
}
@ -131,6 +145,36 @@ impl<H, B> OnlineStages<H, B> {
}
}
impl<H, B> OnlineStages<H, B>
where
H: HeaderDownloader + 'static,
B: BodyDownloader + 'static,
{
/// Create a new builder using the given headers stage.
pub fn builder_with_headers<DB: Database>(
headers: HeaderStage<H>,
consensus: Arc<dyn Consensus>,
body_downloader: B,
) -> StageSetBuilder<DB> {
StageSetBuilder::default()
.add_stage(headers)
.add_stage(TotalDifficultyStage::new(consensus.clone()))
.add_stage(BodyStage { downloader: body_downloader, consensus })
}
/// Create a new builder using the given bodies stage.
pub fn builder_with_bodies<DB: Database>(
bodies: BodyStage<B>,
consensus: Arc<dyn Consensus>,
header_downloader: H,
) -> StageSetBuilder<DB> {
StageSetBuilder::default()
.add_stage(HeaderStage::new(header_downloader, consensus.clone()))
.add_stage(TotalDifficultyStage::new(consensus.clone()))
.add_stage(bodies)
}
}
impl<DB, H, B> StageSet<DB> for OnlineStages<H, B>
where
DB: Database,

View File

@ -11,7 +11,7 @@ use reth_interfaces::{
p2p::headers::downloader::{HeaderDownloader, SyncTarget},
provider::ProviderError,
};
use reth_primitives::{BlockNumber, SealedHeader};
use reth_primitives::{BlockHashOrNumber, BlockNumber, SealedHeader};
use reth_provider::Transaction;
use std::sync::Arc;
use tracing::*;
@ -38,6 +38,8 @@ pub struct HeaderStage<D: HeaderDownloader> {
downloader: D,
/// Consensus client implementation
consensus: Arc<dyn Consensus>,
/// Whether or not the stage should download continuously, or wait for the fork choice state
continuous: bool,
}
// === impl HeaderStage ===
@ -48,7 +50,7 @@ where
{
/// Create a new header stage
pub fn new(downloader: D, consensus: Arc<dyn Consensus>) -> Self {
Self { downloader, consensus }
Self { downloader, consensus, continuous: false }
}
fn is_stage_done<DB: Database>(
@ -64,6 +66,12 @@ where
Ok(header_cursor.next()?.map(|(next_num, _)| head_num + 1 == next_num).unwrap_or_default())
}
/// Set the stage to download continuously
pub fn continuous(mut self) -> Self {
self.continuous = true;
self
}
/// Get the head and tip of the range we need to sync
///
/// See also [SyncTarget]
@ -104,7 +112,14 @@ where
// reverse from there. Else, it should use whatever the forkchoice state reports.
let target = match next_header {
Some(header) if stage_progress + 1 != header.number => SyncTarget::Gap(header),
None => SyncTarget::Tip(self.next_fork_choice_state().await.head_block_hash),
None => {
if self.continuous {
tracing::trace!(target: "sync::stages::headers", ?head_num, "No next header found, using continuous sync strategy");
SyncTarget::TipNum(head_num + 1)
} else {
SyncTarget::Tip(self.next_fork_choice_state().await.head_block_hash)
}
}
_ => return Err(StageError::StageProgress(stage_progress)),
};
@ -250,7 +265,10 @@ impl SyncGap {
/// Returns `true` if the gap from the head to the target was closed
#[inline]
pub fn is_closed(&self) -> bool {
self.local_head.hash() == self.target.tip()
match self.target.tip() {
BlockHashOrNumber::Hash(hash) => self.local_head.hash() == hash,
BlockHashOrNumber::Number(num) => self.local_head.number == num,
}
}
}
@ -326,6 +344,7 @@ mod tests {
HeaderStage {
consensus: self.consensus.clone(),
downloader: (*self.downloader_factory)(),
continuous: false,
}
}
}
@ -510,7 +529,7 @@ mod tests {
let gap = stage.get_sync_gap(&tx, stage_progress).await.unwrap();
assert_eq!(gap.local_head, head);
assert_eq!(gap.target.tip(), consensus_tip);
assert_eq!(gap.target.tip(), consensus_tip.into());
// Checkpoint and gap
tx.put::<tables::CanonicalHeaders>(gap_tip.number, gap_tip.hash())
@ -520,7 +539,7 @@ mod tests {
let gap = stage.get_sync_gap(&tx, stage_progress).await.unwrap();
assert_eq!(gap.local_head, head);
assert_eq!(gap.target.tip(), gap_tip.parent_hash);
assert_eq!(gap.target.tip(), gap_tip.parent_hash.into());
// Checkpoint and gap closed
tx.put::<tables::CanonicalHeaders>(gap_fill.number, gap_fill.hash())