metrics(download): out of order requests metric (#1185)

crates/net/downloaders/src/bodies/bodies.rs

@@ -308,6 +308,8 @@ where
                         )?
                         .ok_or(DownloadError::MissingHeader { block_number: *range.start() })?;
 
+                    self.metrics.out_of_order_requests.increment(1);
+
                     // Dispatch contiguous request.
                     self.in_progress_queue.push_new_request(
                         Arc::clone(&self.client),

crates/net/downloaders/src/headers/reverse_headers.rs

@@ -540,6 +540,7 @@ where
                     }
 
                     trace!(target: "downloaders::headers", new=?target, "Request new sync target");
+                    self.metrics.out_of_order_requests.increment(1);
                     self.sync_target = Some(new_sync_target);
                     self.sync_target_request =
                         Some(self.request_fut(self.get_sync_target_request(tip), Priority::High));

crates/net/downloaders/src/metrics.rs

@@ -22,6 +22,15 @@ pub struct DownloaderMetrics {
     pub total_flushed: Counter,
     /// Number of items that were successfully downloaded
     pub total_downloaded: Counter,
+    /// The number of out-of-order requests sent by the downloader.
+    /// The consumer of the download stream is able to re-request data (headers or bodies) in case
+    /// it encountered a recoverable error (e.g. during insertion).
+    /// Out-of-order request happen when:
+    ///     - the headers downloader `SyncTarget::Tip` hash is different from the previous sync
+    ///       target hash.
+    ///     - the new download range start for bodies donwloader is less than the last block number
+    ///       returned from the stream.
+    pub out_of_order_requests: Counter,
     /// Number of timeout errors while requesting items
     pub timeout_errors: Counter,
     /// Number of validation errors while requesting items