Merge pull request #431 from filecoin-project/proof-serde

Proof serde

.gitignore

@@ -5,3 +5,4 @@ Cargo.lock
 **/*.h
 heaptrack*
 .bencher
+logging-toolkit

storage-proofs/Cargo.toml

@@ -33,6 +33,9 @@ tempfile = "3"
 fs2 = "0.4"
 rayon = "1.0.0"
 slog = { version = "2.4.1", features = ["max_level_trace", "release_max_level_trace"] }
+serde = "1.0"
+serde_derive = "1.0"
+base64 = "0.10.0"
 
 [dependencies.pairing]
 version = "0.14.2"
@@ -48,6 +51,7 @@ asm = ["sha2/sha2-asm", "blake2/simd_asm"]
 proptest = "0.7"
 criterion = "0.2"
 sector-base = { path = "../sector-base" }
+serde_json = "1.0"
 
 [[bench]]
 name = "pedersen"

storage-proofs/src/batchpost.rs

@@ -3,6 +3,8 @@ use std::marker::PhantomData;
 use byteorder::{LittleEndian, WriteBytesExt};
 use num_bigint::BigUint;
 use num_traits::cast::ToPrimitive;
+use serde::de::Deserialize;
+use serde::ser::Serialize;
 
 use crate::crypto::blake2s::blake2s;
 use crate::error::Result;
@@ -23,8 +25,12 @@ pub struct PublicParams {
 #[derive(Debug)]
 pub struct SetupParams {}
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Proof<H: Hasher> {
+    #[serde(bound(
+        serialize = "merklepor::Proof<H>: Serialize",
+        deserialize = "merklepor::Proof<H>: Deserialize<'de>"
+    ))]
     pub proofs: Vec<merklepor::Proof<H>>,
     pub challenges: Vec<usize>,
 }

storage-proofs/src/beacon_post.rs

@@ -2,6 +2,8 @@ use std::marker::PhantomData;
 use std::{thread, time};
 
 use byteorder::{ByteOrder, LittleEndian};
+use serde::de::Deserialize;
+use serde::ser::Serialize;
 
 use crate::error::{Error, Result};
 use crate::hasher::{Domain, HashFunction, Hasher};
@@ -62,8 +64,14 @@ impl<'a, H: 'a + Hasher> PrivateInputs<'a, H> {
 /// Beacon-PoSt
 /// This is one construction of a Proof-of-Spacetime.
 /// It currently only supports proving over a single sector.
-#[derive(Clone, Debug)]
-pub struct Proof<'a, H: Hasher + 'a, V: Vdf<H::Domain>>(Vec<hvh_post::Proof<'a, H, V>>);
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Proof<'a, H: Hasher + 'a, V: Vdf<H::Domain>>(
+    #[serde(bound(
+        serialize = "hvh_post::Proof<'a, H, V>: Serialize",
+        deserialize = "hvh_post::Proof<'a, H, V>: Deserialize<'de>"
+    ))]
+    Vec<hvh_post::Proof<'a, H, V>>,
+);
 
 impl<'a, H: Hasher + 'a, V: Vdf<H::Domain>> Proof<'a, H, V> {
     pub fn proofs(&self) -> &[hvh_post::Proof<'a, H, V>] {

storage-proofs/src/circuit/hvh_post.rs

@@ -4,7 +4,7 @@ use sapling_crypto::circuit::num;
 use sapling_crypto::jubjub::JubjubEngine;
 
 use crate::circuit::constraint;
-use crate::circuit::porc::{self, PoRCCompound};
+use crate::circuit::porc;
 use crate::circuit::sloth;
 use crate::compound_proof::{CircuitComponent, CompoundProof};
 use crate::hasher::Hasher;

storage-proofs/src/drgporep.rs

@@ -1,6 +1,8 @@
 use std::marker::PhantomData;
 
 use byteorder::{LittleEndian, WriteBytesExt};
+use serde::de::Deserialize;
+use serde::ser::Serialize;
 
 use crate::drgraph::Graph;
 use crate::error::Result;
@@ -83,8 +85,12 @@ where
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DataProof<H: Hasher> {
+    #[serde(bound(
+        serialize = "MerkleProof<H>: Serialize",
+        deserialize = "MerkleProof<H>: Deserialize<'de>"
+    ))]
     pub proof: MerkleProof<H>,
     pub data: H::Domain,
 }
@@ -122,12 +128,32 @@ impl<H: Hasher> DataProof<H> {
 
 pub type ReplicaParents<H> = Vec<(usize, DataProof<H>)>;
 
-#[derive(Default, Debug, Clone)]
+#[derive(Default, Debug, Clone, Serialize, Deserialize)]
 pub struct Proof<H: Hasher> {
+    #[serde(bound(
+        serialize = "H::Domain: Serialize",
+        deserialize = "H::Domain: Deserialize<'de>"
+    ))]
     pub data_root: H::Domain,
+    #[serde(bound(
+        serialize = "H::Domain: Serialize",
+        deserialize = "H::Domain: Deserialize<'de>"
+    ))]
     pub replica_root: H::Domain,
+    #[serde(bound(
+        serialize = "DataProof<H>: Serialize",
+        deserialize = "DataProof<H>: Deserialize<'de>"
+    ))]
     pub replica_nodes: Vec<DataProof<H>>,
+    #[serde(bound(
+        serialize = "H::Domain: Serialize",
+        deserialize = "H::Domain: Deserialize<'de>"
+    ))]
     pub replica_parents: Vec<ReplicaParents<H>>,
+    #[serde(bound(
+        serialize = "H::Domain: Serialize",
+        deserialize = "H::Domain: Deserialize<'de>"
+    ))]
     pub nodes: Vec<DataProof<H>>,
 }
 

storage-proofs/src/hasher/digest.rs

@@ -86,7 +86,7 @@ impl<D: Digester> StdHasher for DigestFunction<D> {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, Debug, PartialOrd, Ord, Default)]
+#[derive(Copy, Clone, PartialEq, Eq, Debug, PartialOrd, Ord, Default, Serialize, Deserialize)]
 pub struct DigestDomain(pub [u8; 32]);
 
 impl DigestDomain {

storage-proofs/src/hasher/pedersen.rs

@@ -1,11 +1,14 @@
 use std::hash::Hasher as StdHasher;
 
 use bitvec::{self, BitVec};
+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
 use merkle_light::hash::{Algorithm as LightAlgorithm, Hashable};
 use pairing::bls12_381::{Bls12, Fr, FrRepr};
 use pairing::{PrimeField, PrimeFieldRepr};
 use rand::{Rand, Rng};
 use sapling_crypto::pedersen_hash::{pedersen_hash, Personalization};
+use serde::de::{Deserialize, Deserializer};
+use serde::ser::Serializer;
 
 use super::{Domain, HashFunction, Hasher};
 use crate::crypto::{kdf, pedersen, sloth};
@@ -61,8 +64,52 @@ impl Hashable<PedersenFunction> for PedersenDomain {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, Debug)]
-pub struct PedersenDomain(pub FrRepr);
+#[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+pub struct PedersenDomain(#[serde(with = "FrReprDef")] pub FrRepr);
+
+pub struct FrReprDef(pub [u64; 4]);
+
+impl FrReprDef {
+    fn serialize<S>(__self: &FrRepr, serializer: S) -> ::std::result::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut writer = Vec::with_capacity(32);
+
+        for digit in __self.0.as_ref().iter() {
+            writer.write_u64::<LittleEndian>(*digit).unwrap();
+        }
+
+        if serializer.is_human_readable() {
+            serializer.collect_str(&base64::display::Base64Display::with_config(
+                &writer,
+                base64::STANDARD,
+            ))
+        } else {
+            serializer.serialize_bytes(&writer)
+        }
+    }
+
+    fn deserialize<'de, D>(deserializer: D) -> ::std::result::Result<FrRepr, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let arr: Vec<u8> = if deserializer.is_human_readable() {
+            let raw = String::deserialize(deserializer)?;
+            base64::decode(&raw).unwrap()
+        } else {
+            Vec::deserialize(deserializer)?
+        };
+
+        let mut digits = [0u64; 4];
+        let mut source = ::std::io::Cursor::new(arr);
+        for digit in digits.iter_mut() {
+            *digit = source.read_u64::<LittleEndian>().unwrap();
+        }
+
+        Ok(FrRepr(digits))
+    }
+}
 
 impl Default for PedersenDomain {
     fn default() -> PedersenDomain {
@@ -324,4 +371,15 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_serialize() {
+        let repr = FrRepr([1, 2, 3, 4]);
+        let val = PedersenDomain(repr);
+
+        let ser = serde_json::to_string(&val).unwrap();
+        let val_back = serde_json::from_str(&ser).unwrap();
+
+        assert_eq!(val, val_back);
+    }
 }

storage-proofs/src/hasher/types.rs

@@ -2,6 +2,8 @@ use crate::error::Result;
 use merkle_light::hash::{Algorithm as LightAlgorithm, Hashable as LightHashable};
 use pairing::bls12_381::Fr;
 use rand::Rand;
+use serde::de::DeserializeOwned;
+use serde::ser::Serialize;
 
 pub trait Domain:
     Ord
@@ -16,6 +18,8 @@ pub trait Domain:
     + From<Fr>
     + Into<Fr>
     + Rand
+    + Serialize
+    + DeserializeOwned
 {
     fn serialize(&self) -> Vec<u8>;
     fn into_bytes(&self) -> Vec<u8>;

storage-proofs/src/hvh_post.rs

@@ -1,6 +1,8 @@
 use std::marker::PhantomData;
 
 use byteorder::{ByteOrder, LittleEndian};
+use serde::de::Deserialize;
+use serde::ser::Serialize;
 
 use crate::error::{Error, Result};
 use crate::hasher::{Domain, HashFunction, Hasher};
@@ -80,12 +82,24 @@ impl<'a, H: 'a + Hasher> PrivateInputs<'a, H> {
 /// HVH-PoSt
 /// This is one construction of a Proof-of-Spacetime.
 /// It currently only supports proving over a single sector.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Proof<'a, H: Hasher + 'a, V: Vdf<H::Domain>> {
     /// `post_iteration` online Proof-of-Replication proofs.
+    #[serde(bound(
+        serialize = "V::Proof: Serialize",
+        deserialize = "V::Proof: Deserialize<'de>"
+    ))]
     pub porep_proofs: Vec<<PoRC<'a, H> as ProofScheme<'a>>::Proof>,
     /// `post_epochs - 1` VDF proofs
+    #[serde(bound(
+        serialize = "V::Proof: Serialize",
+        deserialize = "V::Proof: Deserialize<'de>"
+    ))]
     pub vdf_proofs: Vec<V::Proof>,
+    #[serde(bound(
+        serialize = "H::Domain: Serialize",
+        deserialize = "H::Domain: Deserialize<'de>"
+    ))]
     pub ys: Vec<H::Domain>,
 }
 

storage-proofs/src/layered_drgporep.rs

@@ -2,6 +2,8 @@ use std::sync::mpsc::channel;
 
 use crate::merkle::MerkleTree;
 use crossbeam_utils::thread;
+use serde::de::Deserialize;
+use serde::ser::Serialize;
 use slog::*;
 
 use crate::challenge_derivation::derive_challenges;
@@ -108,8 +110,12 @@ pub struct PrivateInputs<'a, H: Hasher> {
     pub tau: Vec<porep::Tau<H::Domain>>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Proof<H: Hasher> {
+    #[serde(bound(
+        serialize = "EncodingProof<H>: Serialize",
+        deserialize = "EncodingProof<H>: Deserialize<'de>"
+    ))]
     pub encoding_proofs: Vec<EncodingProof<H>>,
     pub tau: Vec<porep::Tau<H::Domain>>,
 }

storage-proofs/src/lib.rs

@@ -41,6 +41,9 @@ extern crate pbr;
 extern crate rayon;
 #[macro_use]
 extern crate slog;
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
 
 #[macro_use]
 pub mod test_helper;

storage-proofs/src/merkle.rs

@@ -13,12 +13,13 @@ use crate::hasher::{Domain, Hasher};
 /// Representation of a merkle proof.
 /// Each element in the `path` vector consists of a tuple `(hash, is_right)`, with `hash` being the the hash of the node at the current level and `is_right` a boolean indicating if the path is taking the right path.
 /// The first element is the hash of leaf itself, and the last is the root hash.
-#[derive(Default, Debug, Clone)]
+#[derive(Default, Debug, Clone, Serialize, Deserialize)]
 pub struct MerkleProof<H: Hasher> {
     pub root: H::Domain,
     path: Vec<(H::Domain, bool)>,
     leaf: H::Domain,
 
+    #[serde(skip)]
     _h: PhantomData<H>,
 }
 

storage-proofs/src/porc.rs

@@ -2,6 +2,8 @@ use std::marker::PhantomData;
 
 use num_bigint::BigUint;
 use num_traits::ToPrimitive;
+use serde::de::Deserialize;
+use serde::ser::Serialize;
 
 use crate::drgraph::graph_height;
 use crate::error::{Error, Result};
@@ -50,8 +52,14 @@ pub struct PrivateInputs<'a, H: 'a + Hasher> {
     pub trees: &'a [&'a MerkleTree<H::Domain, H::Function>],
 }
 
-#[derive(Debug, Clone)]
-pub struct Proof<H: Hasher>(Vec<MerkleProof<H>>);
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Proof<H: Hasher>(
+    #[serde(bound(
+        serialize = "MerkleProof<H>: Serialize",
+        deserialize = "MerkleProof<H>: Deserialize<'de>"
+    ))]
+    Vec<MerkleProof<H>>,
+);
 
 impl<H: Hasher> Proof<H> {
     pub fn leafs(&self) -> Vec<&H::Domain> {

storage-proofs/src/porep.rs

@@ -8,8 +8,8 @@ pub struct PublicParams {
     pub time: usize,
 }
 
-#[derive(Debug, Clone, Copy)]
-pub struct Tau<T: Domain> {
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct Tau<T> {
     pub comm_r: T,
     pub comm_d: T,
 }

storage-proofs/src/proof.rs

@@ -1,12 +1,14 @@
 use crate::error::Result;
+use serde::de::DeserializeOwned;
+use serde::ser::Serialize;
 
 /// The ProofScheme trait provides the methods that any proof scheme needs to implement.
 pub trait ProofScheme<'a> {
     type PublicParams: Clone;
     type SetupParams;
     type PublicInputs: Clone;
     type PrivateInputs;
-    type Proof: Clone;
+    type Proof: Clone + Serialize + DeserializeOwned;
 
     /// setup is used to generate public parameters from setup parameters in order to specialize
     /// a ProofScheme to the specific parameters required by a consumer.