diff --git a/Cargo.lock b/Cargo.lock index 576e8eff..77ac33ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4833,6 +4833,7 @@ dependencies = [ "log", "prism-common", "prism-errors", + "prism-serde", "serde", "sp1-sdk", "tokio", @@ -4903,7 +4904,9 @@ dependencies = [ name = "prism-serde" version = "0.1.0" dependencies = [ + "anyhow", "base64 0.22.1", + "bincode", "hex", "serde", ] @@ -4928,6 +4931,7 @@ dependencies = [ "log", "prism-common", "prism-errors", + "prism-serde", "redis", "rocksdb", "serde", diff --git a/crates/common/src/hashchain.rs b/crates/common/src/hashchain.rs index 8693fcd3..7423c2b6 100644 --- a/crates/common/src/hashchain.rs +++ b/crates/common/src/hashchain.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, bail, ensure, Result}; use prism_keys::{Signature, SigningKey, VerifyingKey}; +use prism_serde::binary::BinaryTranscodable; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -253,7 +254,7 @@ impl HashchainEntry { key_idx: usize, ) -> Self { let serialized_operation = - bincode::serialize(&operation).expect("Serializing operation should work"); + operation.encode_to_bytes().expect("Serializing operation should work"); let hash = Digest::hash_items(&[serialized_operation.as_slice(), &previous_hash.to_bytes()]); @@ -341,7 +342,7 @@ impl HashchainEntry { pub fn validate_hash(&self) -> Result<()> { let pristine_entry = self.without_signature(); - let serialized_operation = bincode::serialize(&pristine_entry.operation)?; + let serialized_operation = pristine_entry.operation.encode_to_bytes()?; let pristine_entry_hash = Digest::hash_items(&[ serialized_operation.as_slice(), &pristine_entry.previous_hash.to_bytes(), diff --git a/crates/common/src/transaction.rs b/crates/common/src/transaction.rs index 1990d4e6..eac85db4 100644 --- a/crates/common/src/transaction.rs +++ b/crates/common/src/transaction.rs @@ -1,5 +1,5 @@ -use anyhow::anyhow; use celestia_types::Blob; +use prism_serde::binary::BinaryTranscodable; use serde::{Deserialize, Serialize}; use crate::hashchain::HashchainEntry; @@ -14,7 +14,6 @@ impl TryFrom<&Blob> for Transaction { type Error = anyhow::Error; fn try_from(value: &Blob) -> Result { - bincode::deserialize(&value.data) - .map_err(|e| anyhow!("Failed to decode blob into Transaction: error: {}", e)) + Transaction::decode_from_bytes(&value.data) } } diff --git a/crates/common/src/tree/key_directory_tree.rs b/crates/common/src/tree/key_directory_tree.rs index 2045a241..9ca02cce 100644 --- a/crates/common/src/tree/key_directory_tree.rs +++ b/crates/common/src/tree/key_directory_tree.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -use bincode; use jmt::{ self, storage::{NodeBatch, TreeReader, TreeUpdateBatch, TreeWriter}, @@ -7,7 +6,7 @@ use jmt::{ }; use std::sync::Arc; -use crate::{digest::Digest, hashchain::Hashchain, hasher::Hasher}; +use crate::{digest::Digest, hasher::Hasher}; pub const SPARSE_MERKLE_PLACEHOLDER_HASH: Digest = Digest::new(*b"SPARSE_MERKLE_PLACEHOLDER_HASH__"); @@ -78,13 +77,4 @@ where pub fn get_current_root(&self) -> Result { self.jmt.get_root_hash(self.epoch).map_err(|e| anyhow!("Failed to get root hash: {}", e)) } - - pub(crate) fn serialize_value(value: &Hashchain) -> Result> { - bincode::serialize(value).map_err(|e| anyhow!("Failed to serialize value: {}", e)) - } - - pub(crate) fn deserialize_value(bytes: &[u8]) -> Result { - bincode::deserialize::(bytes) - .map_err(|e| anyhow!("Failed to deserialize value: {}", e)) - } } diff --git a/crates/common/src/tree/proofs.rs b/crates/common/src/tree/proofs.rs index 6f9088da..26b35bd6 100644 --- a/crates/common/src/tree/proofs.rs +++ b/crates/common/src/tree/proofs.rs @@ -1,9 +1,9 @@ use anyhow::{Context, Result}; -use bincode; use jmt::{ proof::{SparseMerkleProof, UpdateMerkleProof}, KeyHash, RootHash, }; +use prism_serde::binary::BinaryTranscodable; use serde::{Deserialize, Serialize}; use std::convert::Into; @@ -55,7 +55,7 @@ impl InsertProof { self.non_membership_proof.verify().context("Invalid NonMembershipProof")?; let hashchain = Hashchain::from_entry(self.new_entry.clone())?; - let serialized_hashchain = bincode::serialize(&hashchain)?; + let serialized_hashchain = hashchain.encode_to_bytes()?; self.membership_proof.clone().verify_existence( self.new_root.into(), @@ -88,7 +88,7 @@ impl UpdateProof { pub fn verify(&self) -> Result<()> { // Verify existence of old value. // Otherwise, any arbitrary hashchain could be set as old_hashchain. - let old_serialized_hashchain = bincode::serialize(&self.old_hashchain)?; + let old_serialized_hashchain = self.old_hashchain.encode_to_bytes()?; self.inclusion_proof.verify_existence(self.old_root, self.key, old_serialized_hashchain)?; let mut hashchain_after_update = self.old_hashchain.clone(); @@ -96,7 +96,7 @@ impl UpdateProof { hashchain_after_update.add_entry(self.new_entry.clone())?; // Ensure the update proof corresponds to the new hashchain value - let new_serialized_hashchain = bincode::serialize(&hashchain_after_update)?; + let new_serialized_hashchain = hashchain_after_update.encode_to_bytes()?; self.update_proof.clone().verify_update( self.old_root, self.new_root, @@ -117,7 +117,7 @@ pub struct MembershipProof { impl MembershipProof { pub fn verify(&self) -> Result<()> { - let value = bincode::serialize(&self.value)?; + let value = self.value.encode_to_bytes()?; self.proof.verify_existence(self.root.into(), self.key, value) } } diff --git a/crates/common/src/tree/snarkable_tree.rs b/crates/common/src/tree/snarkable_tree.rs index e51f29b1..37e5e1c5 100644 --- a/crates/common/src/tree/snarkable_tree.rs +++ b/crates/common/src/tree/snarkable_tree.rs @@ -1,10 +1,10 @@ use anyhow::{bail, ensure, Result}; -use bincode; use jmt::{ storage::{TreeReader, TreeWriter}, KeyHash, }; use prism_errors::DatabaseError; +use prism_serde::binary::BinaryTranscodable; use std::convert::Into; use crate::{ @@ -124,7 +124,7 @@ where }; let hashchain = Hashchain::from_entry(entry.clone())?; - let serialized_hashchain = Self::serialize_value(&hashchain)?; + let serialized_hashchain = hashchain.encode_to_bytes()?; // the update proof just contains another nm proof let (new_root, _, tree_update_batch) = self @@ -151,12 +151,12 @@ where bail!("Key does not exist"); }; - let old_hashchain: Hashchain = bincode::deserialize(old_serialized_hashchain.as_slice())?; + let old_hashchain = Hashchain::decode_from_bytes(&old_serialized_hashchain)?; let mut new_hashchain = old_hashchain.clone(); new_hashchain.add_entry(entry.clone())?; - let serialized_value = Self::serialize_value(&new_hashchain)?; + let serialized_value = new_hashchain.encode_to_bytes()?; let (new_root, update_proof, tree_update_batch) = self.jmt.put_value_set_with_proof( vec![(key, Some(serialized_value.clone()))], @@ -182,7 +182,7 @@ where match value { Some(serialized_value) => { - let deserialized_value = Self::deserialize_value(&serialized_value)?; + let deserialized_value = Hashchain::decode_from_bytes(&serialized_value)?; let membership_proof = MembershipProof { root, proof, diff --git a/crates/da/Cargo.toml b/crates/da/Cargo.toml index 9413b640..fa7a72c7 100644 --- a/crates/da/Cargo.toml +++ b/crates/da/Cargo.toml @@ -23,4 +23,5 @@ celestia-types = { workspace = true } anyhow = { workspace = true } prism-common = { workspace = true } prism-errors = { workspace = true } +prism-serde = { workspace = true } sp1-sdk = { workspace = true } diff --git a/crates/da/src/celestia.rs b/crates/da/src/celestia.rs index e26ef540..ab567c57 100644 --- a/crates/da/src/celestia.rs +++ b/crates/da/src/celestia.rs @@ -6,6 +6,7 @@ use celestia_types::{nmt::Namespace, Blob, TxConfig}; use log::{debug, error, trace, warn}; use prism_common::transaction::Transaction; use prism_errors::{DataAvailabilityError, GeneralError}; +use prism_serde::binary::BinaryTranscodable; use serde::{Deserialize, Serialize}; use std::{ self, @@ -16,15 +17,15 @@ use std::{ }; use tokio::{sync::broadcast, task::spawn}; -use bincode; - impl TryFrom<&Blob> for FinalizedEpoch { type Error = anyhow::Error; fn try_from(value: &Blob) -> Result { - bincode::deserialize(&value.data).context(format!( - "Failed to decode blob into FinalizedEpoch: {value:?}" - )) + FinalizedEpoch::decode_from_bytes(&value.data).map_err(|_| { + anyhow!(format!( + "Failed to decode blob into FinalizedEpoch: {value:?}" + )) + }) } } @@ -149,7 +150,7 @@ impl DataAvailabilityLayer for CelestiaConnection { async fn submit_finalized_epoch(&self, epoch: FinalizedEpoch) -> Result { debug!("posting {}th epoch to da layer", epoch.height); - let data = bincode::serialize(&epoch).map_err(|e| { + let data = epoch.encode_to_bytes().map_err(|e| { DataAvailabilityError::GeneralError(GeneralError::ParsingError(format!( "serializing epoch {}: {}", epoch.height, e @@ -208,7 +209,8 @@ impl DataAvailabilityLayer for CelestiaConnection { let blobs: Result, _> = transactions .iter() .map(|transaction| { - let data = bincode::serialize(transaction) + let data = transaction + .encode_to_bytes() .context(format!("Failed to serialize transaction {:?}", transaction)) .map_err(|e| { DataAvailabilityError::GeneralError(GeneralError::ParsingError( diff --git a/crates/da/src/lib.rs b/crates/da/src/lib.rs index 8b4ea30a..54b03946 100644 --- a/crates/da/src/lib.rs +++ b/crates/da/src/lib.rs @@ -2,6 +2,7 @@ use anyhow::Result; use async_trait::async_trait; use ed25519_consensus::{Signature, SigningKey, VerificationKey as VerifyingKey}; use prism_common::{digest::Digest, transaction::Transaction}; +use prism_serde::binary::BinaryTranscodable; use serde::{Deserialize, Serialize}; use sp1_sdk::SP1ProofWithPublicValues; use tokio::sync::broadcast; @@ -22,7 +23,7 @@ pub struct FinalizedEpoch { impl FinalizedEpoch { pub fn insert_signature(&mut self, key: &SigningKey) { - let plaintext = bincode::serialize(&self).unwrap(); + let plaintext = self.encode_to_bytes().unwrap(); let signature = key.sign(&plaintext); self.signature = Some(hex::encode(signature.to_bytes())); } @@ -36,7 +37,8 @@ impl FinalizedEpoch { signature: None, }; - let message = bincode::serialize(&epoch_without_signature) + let message = epoch_without_signature + .encode_to_bytes() .map_err(|e| anyhow::anyhow!("Failed to serialize epoch: {}", e))?; let signature = diff --git a/crates/serde/Cargo.toml b/crates/serde/Cargo.toml index 3a1ae061..58285ec1 100644 --- a/crates/serde/Cargo.toml +++ b/crates/serde/Cargo.toml @@ -7,7 +7,12 @@ homepage.workspace = true repository.workspace = true [dependencies] +anyhow.workspace = true + # serde base64.workspace = true serde.workspace = true hex.workspace = true + +# binary +bincode.workspace = true diff --git a/crates/serde/src/binary.rs b/crates/serde/src/binary.rs new file mode 100644 index 00000000..cbf91b1f --- /dev/null +++ b/crates/serde/src/binary.rs @@ -0,0 +1,20 @@ +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +pub trait BinaryTranscodable<'de>: Sized { + fn encode_to_bytes(&self) -> Result>; + fn decode_from_bytes>(bytes: &'de B) -> Result; +} + +impl<'de, T> BinaryTranscodable<'de> for T +where + T: Serialize + Deserialize<'de>, +{ + fn encode_to_bytes(&self) -> Result> { + bincode::serialize(self).map_err(Into::::into) + } + + fn decode_from_bytes>(bytes: &'de B) -> Result { + bincode::deserialize(bytes.as_ref()).map_err(Into::::into) + } +} diff --git a/crates/serde/src/lib.rs b/crates/serde/src/lib.rs index 43113b92..bfea20df 100644 --- a/crates/serde/src/lib.rs +++ b/crates/serde/src/lib.rs @@ -1,4 +1,7 @@ #![allow(dead_code)] + +pub mod binary; + use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index 1f445205..1741f4c0 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -20,6 +20,7 @@ hex = { workspace = true } jmt = { workspace = true } prism-errors = { workspace = true } prism-common = { workspace = true } +prism-serde = { workspace = true } auto_impl = { workspace = true } rocksdb = { workspace = true } diff --git a/crates/storage/src/redis.rs b/crates/storage/src/redis.rs index 6cf32a55..51aa7f42 100644 --- a/crates/storage/src/redis.rs +++ b/crates/storage/src/redis.rs @@ -4,6 +4,7 @@ use jmt::{ KeyHash, OwnedValue, Version, }; use prism_common::digest::Digest; +use prism_serde::binary::BinaryTranscodable; use redis::{Client, Commands, Connection}; use serde::{Deserialize, Serialize}; use std::{ @@ -77,9 +78,9 @@ impl RedisConnection { impl TreeReader for RedisConnection { fn get_node_option(&self, node_key: &NodeKey) -> Result> { let mut con = self.lock_connection()?; - let serialized_key = hex::encode(bincode::serialize(node_key)?); + let serialized_key = hex::encode(node_key.encode_to_bytes()?); let node_data: Option> = con.get(format!("node:{}", serialized_key))?; - Ok(node_data.map(|data| bincode::deserialize(&data).unwrap())) + Ok(node_data.map(|data| Node::decode_from_bytes(&data).unwrap())) } fn get_rightmost_leaf(&self) -> Result> { @@ -89,10 +90,10 @@ impl TreeReader for RedisConnection { for key in keys { let node_data: Vec = con.get(&key)?; - let node: Node = bincode::deserialize(&node_data)?; + let node = Node::decode_from_bytes(&node_data)?; if let Node::Leaf(leaf_node) = node { let node_key_bytes = hex::decode(key.strip_prefix("node:").unwrap())?; - let node_key: NodeKey = bincode::deserialize(&node_key_bytes)?; + let node_key = NodeKey::decode_from_bytes(&node_key_bytes)?; if rightmost.is_none() || leaf_node.key_hash() > rightmost.as_ref().unwrap().1.key_hash() { @@ -132,8 +133,8 @@ impl TreeWriter for RedisConnection { let mut pipe = redis::pipe(); for (node_key, node) in node_batch.nodes() { - let serialized_key = hex::encode(bincode::serialize(node_key)?); - let node_data = bincode::serialize(node)?; + let serialized_key = hex::encode(node_key.encode_to_bytes()?); + let node_data = node.encode_to_bytes()?; pipe.set(format!("node:{}", serialized_key), node_data); } diff --git a/elf/riscv32im-succinct-zkvm-elf b/elf/riscv32im-succinct-zkvm-elf index d440de0f..66fd256b 100755 Binary files a/elf/riscv32im-succinct-zkvm-elf and b/elf/riscv32im-succinct-zkvm-elf differ