perf: misc changes (#5701)

crates/primitives/src/trie/hash_builder/mod.rs

@@ -203,20 +203,24 @@ impl HashBuilder {
 
         trace!(target: "trie::hash_builder", ?current, ?succeeding, "updating merkle tree");
 
-        let mut i = 0;
+        let mut i = 0usize;
+        let span = tracing::trace_span!(
+            target: "trie::hash_builder",
+            "loop",
+            i = tracing::field::Empty,
+            current = tracing::field::Empty,
+            build_extensions = tracing::field::Empty,
+        )
+        .entered();
         loop {
-            let span = tracing::span!(
+            if !span.is_disabled() {
-                target: "trie::hash_builder",
+                span.record("i", i);
-                tracing::Level::TRACE,
+                span.record("current", &format!("{current:?}"));
-                "loop",
+                span.record("build_extensions", build_extensions);
-                i,
+            }
-                ?current,
-                ?build_extensions
-            );
-            let _enter = span.enter();
 
             let preceding_exists = !self.groups.is_empty();
-            let preceding_len: usize = self.groups.len().saturating_sub(1);
+            let preceding_len = self.groups.len().saturating_sub(1);
 
             let common_prefix_len = succeeding.common_prefix_length(current.as_slice());
             let len = std::cmp::max(preceding_len, common_prefix_len);
@@ -254,7 +258,7 @@ impl HashBuilder {
             if !succeeding.is_empty() || preceding_exists {
                 len_from += 1;
             }
-            trace!(target: "trie::hash_builder", "skipping {} nibbles", len_from);
+            trace!(target: "trie::hash_builder", "skipping {len_from} nibbles");
 
             // The key without the common prefix
             let short_node_key = current.slice(len_from..);

crates/primitives/src/trie/nibbles.rs

@@ -67,7 +67,6 @@ impl Compact for StoredNibblesSubKey {
 /// The internal representation is a [`SmallVec`] that stores one nibble per byte. This means that
 /// each byte has its upper 4 bits set to zero and the lower 4 bits representing the nibble value.
 #[derive(
-    Clone,
     Default,
     PartialEq,
     Eq,
@@ -83,6 +82,19 @@ impl Compact for StoredNibblesSubKey {
 #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
 pub struct Nibbles(SmallVec<[u8; 64]>);
 
+// Override `SmallVec::from` since it's not specialized for `Copy` types.
+impl Clone for Nibbles {
+    #[inline]
+    fn clone(&self) -> Self {
+        Self(SmallVec::from_slice(&self.0))
+    }
+
+    #[inline]
+    fn clone_from(&mut self, source: &Self) {
+        self.0.clone_from(&source.0);
+    }
+}
+
 impl alloy_rlp::Encodable for Nibbles {
     #[inline]
     fn length(&self) -> usize {

crates/primitives/src/trie/nodes/mod.rs

@@ -28,5 +28,9 @@ fn rlp_node(rlp: &[u8]) -> Vec<u8> {
 // TODO: this could return [u8; 33] but Vec is needed everywhere this function is used
 #[inline]
 pub fn word_rlp(word: &B256) -> Vec<u8> {
-    [&[EMPTY_STRING_CODE + B256::len_bytes() as u8][..], &word[..]].concat()
+    // Gets optimized to alloc + write directly into it: https://godbolt.org/z/rfWGG6ebq
+    let mut arr = [0; 33];
+    arr[0] = EMPTY_STRING_CODE + 32;
+    arr[1..].copy_from_slice(word.as_slice());
+    arr.to_vec()
 }

crates/storage/codecs/src/lib.rs

@@ -53,6 +53,7 @@ pub trait Compact: Sized {
     fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]);
 
     /// "Optional": If there's no good reason to use it, don't.
+    #[inline]
     fn specialized_to_compact<B>(self, buf: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
@@ -61,6 +62,7 @@ pub trait Compact: Sized {
     }
 
     /// "Optional": If there's no good reason to use it, don't.
+    #[inline]
     fn specialized_from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
         Self::from_compact(buf, len)
     }
@@ -70,6 +72,7 @@ macro_rules! impl_uint_compact {
     ($($name:tt),+) => {
         $(
             impl Compact for $name {
+                #[inline]
                 fn to_compact<B>(self, buf: &mut B) -> usize
                     where B: bytes::BufMut + AsMut<[u8]>
                 {
@@ -78,16 +81,16 @@ macro_rules! impl_uint_compact {
                     std::mem::size_of::<$name>() - leading
                 }
 
+                #[inline]
                 fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) {
-                    if len > 0 {
+                    if len == 0 {
-                        let mut arr = [0; std::mem::size_of::<$name>()];
+                        return (0, buf);
-                        arr[std::mem::size_of::<$name>() - len..].copy_from_slice(&buf[..len]);
-
-                        buf.advance(len);
-
-                        return ($name::from_be_bytes(arr), buf)
                     }
-                    (0, buf)
+
+                    let mut arr = [0; std::mem::size_of::<$name>()];
+                    arr[std::mem::size_of::<$name>() - len..].copy_from_slice(&buf[..len]);
+                    buf.advance(len);
+                    ($name::from_be_bytes(arr), buf)
                 }
             }
         )+
@@ -101,6 +104,7 @@ where
     T: Compact,
 {
     /// Returns 0 since we won't include it in the `StructFlags`.
+    #[inline]
     fn to_compact<B>(self, buf: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
@@ -122,6 +126,7 @@ where
         0
     }
 
+    #[inline]
     fn from_compact(buf: &[u8], _: usize) -> (Self, &[u8]) {
         let (length, mut buf) = decode_varuint(buf);
         let mut list = Vec::with_capacity(length);
@@ -139,6 +144,7 @@ where
     }
 
     /// To be used by fixed sized types like `Vec<B256>`.
+    #[inline]
     fn specialized_to_compact<B>(self, buf: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
@@ -151,6 +157,7 @@ where
     }
 
     /// To be used by fixed sized types like `Vec<B256>`.
+    #[inline]
     fn specialized_from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
         let (length, mut buf) = decode_varuint(buf);
         let mut list = Vec::with_capacity(length);
@@ -170,25 +177,27 @@ where
     T: Compact,
 {
     /// Returns 0 for `None` and 1 for `Some(_)`.
+    #[inline]
     fn to_compact<B>(self, buf: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
     {
+        let Some(element) = self else {
+            return 0;
+        };
+
+        // We don't know the length of the element until we compact it.
         let mut tmp = Vec::with_capacity(64);
+        let length = element.to_compact(&mut tmp);
 
-        if let Some(element) = self {
+        encode_varuint(length, buf);
-            // We don't know the length until we compact it
-            let length = element.to_compact(&mut tmp);
 
-            encode_varuint(length, buf);
+        buf.put_slice(&tmp);
 
-            buf.put_slice(&tmp);
+        1
-
-            return 1
-        }
-        0
     }
 
+    #[inline]
     fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
         if len == 0 {
             return (None, buf)
@@ -203,53 +212,58 @@ where
     }
 
     /// To be used by fixed sized types like `Option<B256>`.
+    #[inline]
     fn specialized_to_compact<B>(self, buf: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
     {
         if let Some(element) = self {
             element.to_compact(buf);
-            return 1
+            1
+        } else {
+            0
         }
-        0
     }
 
     /// To be used by fixed sized types like `Option<B256>`.
+    #[inline]
     fn specialized_from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
         if len == 0 {
             return (None, buf)
         }
 
         let (element, buf) = T::from_compact(buf, len);
-
         (Some(element), buf)
     }
 }
 
 impl Compact for U256 {
+    #[inline]
     fn to_compact<B>(self, buf: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
     {
-        let inner: [u8; 32] = self.to_be_bytes();
+        let inner = self.to_be_bytes::<32>();
         let size = 32 - (self.leading_zeros() / 8);
         buf.put_slice(&inner[32 - size..]);
         size
     }
 
+    #[inline]
     fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) {
-        if len > 0 {
+        if len == 0 {
-            let mut arr = [0; 32];
+            return (U256::ZERO, buf);
-            arr[(32 - len)..].copy_from_slice(&buf[..len]);
-            buf.advance(len);
-            return (U256::from_be_bytes(arr), buf)
         }
 
-        (U256::ZERO, buf)
+        let mut arr = [0; 32];
+        arr[(32 - len)..].copy_from_slice(&buf[..len]);
+        buf.advance(len);
+        (U256::from_be_bytes(arr), buf)
     }
 }
 
 impl Compact for Bytes {
+    #[inline]
     fn to_compact<B>(self, buf: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
@@ -259,12 +273,14 @@ impl Compact for Bytes {
         len
     }
 
+    #[inline]
     fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) {
         (buf.copy_to_bytes(len).into(), buf)
     }
 }
 
 impl<const N: usize> Compact for [u8; N] {
+    #[inline]
     fn to_compact<B>(self, buf: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
@@ -273,6 +289,7 @@ impl<const N: usize> Compact for [u8; N] {
         N
     }
 
+    #[inline]
     fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) {
         if len == 0 {
             return ([0; N], buf)
@@ -290,6 +307,7 @@ macro_rules! impl_compact_for_bytes {
     ($($name:tt),+) => {
         $(
             impl Compact for $name {
+                #[inline]
                 fn to_compact<B>(self, buf: &mut B) -> usize
                 where
                     B: bytes::BufMut + AsMut<[u8]>
@@ -297,6 +315,7 @@ macro_rules! impl_compact_for_bytes {
                     self.0.to_compact(buf)
                 }
 
+                #[inline]
                 fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
                     let (v, buf) = <[u8; std::mem::size_of::<$name>()]>::from_compact(buf, len);
                     (Self::from(v), buf)
@@ -310,6 +329,7 @@ impl_compact_for_bytes!(Address, B256, B512, Bloom);
 
 impl Compact for bool {
     /// `bool` vars go directly to the `StructFlags` and are not written to the buffer.
+    #[inline]
     fn to_compact<B>(self, _: &mut B) -> usize
     where
         B: bytes::BufMut + AsMut<[u8]>,
@@ -318,6 +338,7 @@ impl Compact for bool {
     }
 
     /// `bool` expects the real value to come in `len`, and does not advance the cursor.
+    #[inline]
     fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
         (len != 0, buf)
     }
@@ -334,19 +355,24 @@ where
     buf.put_u8(n as u8);
 }
 
-fn decode_varuint(mut buf: &[u8]) -> (usize, &[u8]) {
+fn decode_varuint(buf: &[u8]) -> (usize, &[u8]) {
-    let mut value: usize = 0;
+    let mut value = 0;
 
-    for i in 0usize..33 {
+    for i in 0..33 {
-        let byte = buf.get_u8();
+        let byte = buf[i];
-        if byte < 128 {
+        value |= usize::from(byte & 0x7F) << (i * 7);
-            value |= usize::from(byte) << (i * 7);
+        if byte < 0x80 {
-            return (value, buf)
+            return (value, &buf[i + 1..]);
-        } else {
-            value |= usize::from(byte & 0x7F) << (i * 7);
         }
     }
-    panic!("Could not correctly decode value.");
+
+    decode_varuint_panic();
+}
+
+#[inline(never)]
+#[cold]
+const fn decode_varuint_panic() -> ! {
+    panic!("could not decode varuint");
 }
 
 #[cfg(test)]