feature: Binary transcoding via trait blanket impl (#165)

* feature: Binary transcoding via trait blanket impl

Ensures independence from a certain library (e.g. bincode).

* chore: Incorporate changes in zkvm elf

crates/common/src/hashchain.rs

@@ -1,5 +1,6 @@
 use anyhow::{anyhow, bail, ensure, Result};
 use prism_keys::{Signature, SigningKey, VerifyingKey};
+use prism_serde::binary::BinaryTranscodable;
 use serde::{Deserialize, Serialize};
 use std::ops::{Deref, DerefMut};
 
@@ -253,7 +254,7 @@ impl HashchainEntry {
         key_idx: usize,
     ) -> Self {
         let serialized_operation =
-            bincode::serialize(&operation).expect("Serializing operation should work");
+            operation.encode_to_bytes().expect("Serializing operation should work");
         let hash =
             Digest::hash_items(&[serialized_operation.as_slice(), &previous_hash.to_bytes()]);
 
@@ -341,7 +342,7 @@ impl HashchainEntry {
     pub fn validate_hash(&self) -> Result<()> {
         let pristine_entry = self.without_signature();
 
-        let serialized_operation = bincode::serialize(&pristine_entry.operation)?;
+        let serialized_operation = pristine_entry.operation.encode_to_bytes()?;
         let pristine_entry_hash = Digest::hash_items(&[
             serialized_operation.as_slice(),
             &pristine_entry.previous_hash.to_bytes(),

crates/common/src/transaction.rs

@@ -1,5 +1,5 @@
-use anyhow::anyhow;
 use celestia_types::Blob;
+use prism_serde::binary::BinaryTranscodable;
 use serde::{Deserialize, Serialize};
 
 use crate::hashchain::HashchainEntry;
@@ -14,7 +14,6 @@ impl TryFrom<&Blob> for Transaction {
     type Error = anyhow::Error;
 
     fn try_from(value: &Blob) -> Result<Self, Self::Error> {
-        bincode::deserialize(&value.data)
-            .map_err(|e| anyhow!("Failed to decode blob into Transaction: error: {}", e))
+        Transaction::decode_from_bytes(&value.data)
     }
 }

crates/common/src/tree/key_directory_tree.rs

@@ -1,13 +1,12 @@
 use anyhow::{anyhow, Result};
-use bincode;
 use jmt::{
     self,
     storage::{NodeBatch, TreeReader, TreeUpdateBatch, TreeWriter},
     JellyfishMerkleTree, KeyHash, RootHash,
 };
 use std::sync::Arc;
 
-use crate::{digest::Digest, hashchain::Hashchain, hasher::Hasher};
+use crate::{digest::Digest, hasher::Hasher};
 
 pub const SPARSE_MERKLE_PLACEHOLDER_HASH: Digest =
     Digest::new(*b"SPARSE_MERKLE_PLACEHOLDER_HASH__");
@@ -78,13 +77,4 @@ where
     pub fn get_current_root(&self) -> Result<RootHash> {
         self.jmt.get_root_hash(self.epoch).map_err(|e| anyhow!("Failed to get root hash: {}", e))
     }
-
-    pub(crate) fn serialize_value(value: &Hashchain) -> Result<Vec<u8>> {
-        bincode::serialize(value).map_err(|e| anyhow!("Failed to serialize value: {}", e))
-    }
-
-    pub(crate) fn deserialize_value(bytes: &[u8]) -> Result<Hashchain> {
-        bincode::deserialize::<Hashchain>(bytes)
-            .map_err(|e| anyhow!("Failed to deserialize value: {}", e))
-    }
 }

crates/common/src/tree/proofs.rs

@@ -1,9 +1,9 @@
 use anyhow::{Context, Result};
-use bincode;
 use jmt::{
     proof::{SparseMerkleProof, UpdateMerkleProof},
     KeyHash, RootHash,
 };
+use prism_serde::binary::BinaryTranscodable;
 use serde::{Deserialize, Serialize};
 use std::convert::Into;
 
@@ -55,7 +55,7 @@ impl InsertProof {
         self.non_membership_proof.verify().context("Invalid NonMembershipProof")?;
 
         let hashchain = Hashchain::from_entry(self.new_entry.clone())?;
-        let serialized_hashchain = bincode::serialize(&hashchain)?;
+        let serialized_hashchain = hashchain.encode_to_bytes()?;
 
         self.membership_proof.clone().verify_existence(
             self.new_root.into(),
@@ -88,15 +88,15 @@ impl UpdateProof {
     pub fn verify(&self) -> Result<()> {
         // Verify existence of old value.
         // Otherwise, any arbitrary hashchain could be set as old_hashchain.
-        let old_serialized_hashchain = bincode::serialize(&self.old_hashchain)?;
+        let old_serialized_hashchain = self.old_hashchain.encode_to_bytes()?;
         self.inclusion_proof.verify_existence(self.old_root, self.key, old_serialized_hashchain)?;
 
         let mut hashchain_after_update = self.old_hashchain.clone();
         // Append the new entry and verify it's validity
         hashchain_after_update.add_entry(self.new_entry.clone())?;
 
         // Ensure the update proof corresponds to the new hashchain value
-        let new_serialized_hashchain = bincode::serialize(&hashchain_after_update)?;
+        let new_serialized_hashchain = hashchain_after_update.encode_to_bytes()?;
         self.update_proof.clone().verify_update(
             self.old_root,
             self.new_root,
@@ -117,7 +117,7 @@ pub struct MembershipProof {
 
 impl MembershipProof {
     pub fn verify(&self) -> Result<()> {
-        let value = bincode::serialize(&self.value)?;
+        let value = self.value.encode_to_bytes()?;
         self.proof.verify_existence(self.root.into(), self.key, value)
     }
 }

crates/common/src/tree/snarkable_tree.rs

@@ -1,10 +1,10 @@
 use anyhow::{bail, ensure, Result};
-use bincode;
 use jmt::{
     storage::{TreeReader, TreeWriter},
     KeyHash,
 };
 use prism_errors::DatabaseError;
+use prism_serde::binary::BinaryTranscodable;
 use std::convert::Into;
 
 use crate::{
@@ -124,7 +124,7 @@ where
         };
 
         let hashchain = Hashchain::from_entry(entry.clone())?;
-        let serialized_hashchain = Self::serialize_value(&hashchain)?;
+        let serialized_hashchain = hashchain.encode_to_bytes()?;
 
         // the update proof just contains another nm proof
         let (new_root, _, tree_update_batch) = self
@@ -151,12 +151,12 @@ where
             bail!("Key does not exist");
         };
 
-        let old_hashchain: Hashchain = bincode::deserialize(old_serialized_hashchain.as_slice())?;
+        let old_hashchain = Hashchain::decode_from_bytes(&old_serialized_hashchain)?;
 
         let mut new_hashchain = old_hashchain.clone();
         new_hashchain.add_entry(entry.clone())?;
 
-        let serialized_value = Self::serialize_value(&new_hashchain)?;
+        let serialized_value = new_hashchain.encode_to_bytes()?;
 
         let (new_root, update_proof, tree_update_batch) = self.jmt.put_value_set_with_proof(
             vec![(key, Some(serialized_value.clone()))],
@@ -182,7 +182,7 @@ where
 
         match value {
             Some(serialized_value) => {
-                let deserialized_value = Self::deserialize_value(&serialized_value)?;
+                let deserialized_value = Hashchain::decode_from_bytes(&serialized_value)?;
                 let membership_proof = MembershipProof {
                     root,
                     proof,

crates/da/Cargo.toml

@@ -23,4 +23,5 @@ celestia-types = { workspace = true }
 anyhow = { workspace = true }
 prism-common = { workspace = true }
 prism-errors = { workspace = true }
+prism-serde = { workspace = true }
 sp1-sdk = { workspace = true }

crates/da/src/celestia.rs

@@ -6,6 +6,7 @@ use celestia_types::{nmt::Namespace, Blob, TxConfig};
 use log::{debug, error, trace, warn};
 use prism_common::transaction::Transaction;
 use prism_errors::{DataAvailabilityError, GeneralError};
+use prism_serde::binary::BinaryTranscodable;
 use serde::{Deserialize, Serialize};
 use std::{
     self,
@@ -16,15 +17,15 @@ use std::{
 };
 use tokio::{sync::broadcast, task::spawn};
 
-use bincode;
-
 impl TryFrom<&Blob> for FinalizedEpoch {
     type Error = anyhow::Error;
 
     fn try_from(value: &Blob) -> Result<Self, Self::Error> {
-        bincode::deserialize(&value.data).context(format!(
-            "Failed to decode blob into FinalizedEpoch: {value:?}"
-        ))
+        FinalizedEpoch::decode_from_bytes(&value.data).map_err(|_| {
+            anyhow!(format!(
+                "Failed to decode blob into FinalizedEpoch: {value:?}"
+            ))
+        })
     }
 }
 
@@ -149,7 +150,7 @@ impl DataAvailabilityLayer for CelestiaConnection {
     async fn submit_finalized_epoch(&self, epoch: FinalizedEpoch) -> Result<u64> {
         debug!("posting {}th epoch to da layer", epoch.height);
 
-        let data = bincode::serialize(&epoch).map_err(|e| {
+        let data = epoch.encode_to_bytes().map_err(|e| {
             DataAvailabilityError::GeneralError(GeneralError::ParsingError(format!(
                 "serializing epoch {}: {}",
                 epoch.height, e
@@ -208,7 +209,8 @@ impl DataAvailabilityLayer for CelestiaConnection {
         let blobs: Result<Vec<Blob>, _> = transactions
             .iter()
             .map(|transaction| {
-                let data = bincode::serialize(transaction)
+                let data = transaction
+                    .encode_to_bytes()
                     .context(format!("Failed to serialize transaction {:?}", transaction))
                     .map_err(|e| {
                         DataAvailabilityError::GeneralError(GeneralError::ParsingError(

crates/da/src/lib.rs

@@ -2,6 +2,7 @@ use anyhow::Result;
 use async_trait::async_trait;
 use ed25519_consensus::{Signature, SigningKey, VerificationKey as VerifyingKey};
 use prism_common::{digest::Digest, transaction::Transaction};
+use prism_serde::binary::BinaryTranscodable;
 use serde::{Deserialize, Serialize};
 use sp1_sdk::SP1ProofWithPublicValues;
 use tokio::sync::broadcast;
@@ -22,7 +23,7 @@ pub struct FinalizedEpoch {
 
 impl FinalizedEpoch {
     pub fn insert_signature(&mut self, key: &SigningKey) {
-        let plaintext = bincode::serialize(&self).unwrap();
+        let plaintext = self.encode_to_bytes().unwrap();
         let signature = key.sign(&plaintext);
         self.signature = Some(hex::encode(signature.to_bytes()));
     }
@@ -36,7 +37,8 @@ impl FinalizedEpoch {
             signature: None,
         };
 
-        let message = bincode::serialize(&epoch_without_signature)
+        let message = epoch_without_signature
+            .encode_to_bytes()
             .map_err(|e| anyhow::anyhow!("Failed to serialize epoch: {}", e))?;
 
         let signature =

crates/serde/Cargo.toml

@@ -7,7 +7,12 @@ homepage.workspace = true
 repository.workspace = true
 
 [dependencies]
+anyhow.workspace = true
+
 # serde
 base64.workspace = true
 serde.workspace = true
 hex.workspace = true
+
+# binary
+bincode.workspace = true

crates/serde/src/binary.rs

@@ -0,0 +1,20 @@
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+
+pub trait BinaryTranscodable<'de>: Sized {
+    fn encode_to_bytes(&self) -> Result<Vec<u8>>;
+    fn decode_from_bytes<B: AsRef<[u8]>>(bytes: &'de B) -> Result<Self>;
+}
+
+impl<'de, T> BinaryTranscodable<'de> for T
+where
+    T: Serialize + Deserialize<'de>,
+{
+    fn encode_to_bytes(&self) -> Result<Vec<u8>> {
+        bincode::serialize(self).map_err(Into::<anyhow::Error>::into)
+    }
+
+    fn decode_from_bytes<B: AsRef<[u8]>>(bytes: &'de B) -> Result<Self> {
+        bincode::deserialize(bytes.as_ref()).map_err(Into::<anyhow::Error>::into)
+    }
+}

crates/serde/src/lib.rs

@@ -1,4 +1,7 @@
 #![allow(dead_code)]
+
+pub mod binary;
+
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize)]

crates/storage/Cargo.toml

@@ -20,6 +20,7 @@ hex = { workspace = true }
 jmt = { workspace = true }
 prism-errors = { workspace = true }
 prism-common = { workspace = true }
+prism-serde = { workspace = true }
 auto_impl = { workspace = true }
 rocksdb = { workspace = true }
 

crates/storage/src/redis.rs

@@ -4,6 +4,7 @@ use jmt::{
     KeyHash, OwnedValue, Version,
 };
 use prism_common::digest::Digest;
+use prism_serde::binary::BinaryTranscodable;
 use redis::{Client, Commands, Connection};
 use serde::{Deserialize, Serialize};
 use std::{
@@ -77,9 +78,9 @@ impl RedisConnection {
 impl TreeReader for RedisConnection {
     fn get_node_option(&self, node_key: &NodeKey) -> Result<Option<Node>> {
         let mut con = self.lock_connection()?;
-        let serialized_key = hex::encode(bincode::serialize(node_key)?);
+        let serialized_key = hex::encode(node_key.encode_to_bytes()?);
         let node_data: Option<Vec<u8>> = con.get(format!("node:{}", serialized_key))?;
-        Ok(node_data.map(|data| bincode::deserialize(&data).unwrap()))
+        Ok(node_data.map(|data| Node::decode_from_bytes(&data).unwrap()))
     }
 
     fn get_rightmost_leaf(&self) -> Result<Option<(NodeKey, LeafNode)>> {
@@ -89,10 +90,10 @@ impl TreeReader for RedisConnection {
 
         for key in keys {
             let node_data: Vec<u8> = con.get(&key)?;
-            let node: Node = bincode::deserialize(&node_data)?;
+            let node = Node::decode_from_bytes(&node_data)?;
             if let Node::Leaf(leaf_node) = node {
                 let node_key_bytes = hex::decode(key.strip_prefix("node:").unwrap())?;
-                let node_key: NodeKey = bincode::deserialize(&node_key_bytes)?;
+                let node_key = NodeKey::decode_from_bytes(&node_key_bytes)?;
                 if rightmost.is_none()
                     || leaf_node.key_hash() > rightmost.as_ref().unwrap().1.key_hash()
                 {
@@ -132,8 +133,8 @@ impl TreeWriter for RedisConnection {
         let mut pipe = redis::pipe();
 
         for (node_key, node) in node_batch.nodes() {
-            let serialized_key = hex::encode(bincode::serialize(node_key)?);
-            let node_data = bincode::serialize(node)?;
+            let serialized_key = hex::encode(node_key.encode_to_bytes()?);
+            let node_data = node.encode_to_bytes()?;
             pipe.set(format!("node:{}", serialized_key), node_data);
         }