verify_post should be passed cfg + add FFI tests for PoSt generation/…

…verification (#508)

* plumb ConfiguredStore instead of sector bytes

* FFI test suite to include verify_post and generate_post calls

* modify paramcache to optionally generate PoSt parameters

* generate PoSt parameters before FFI tests run

* paramcache to be configured with arg-flags

* ensure paramcache runs before FFI tests (using test config)

* rustfmt

* run cache populator on same channel as FFI test

.circleci/config.yml

@@ -93,12 +93,17 @@ jobs:
           keys:
             - cargo-v7-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
             - parameter-cache-{{ .Revision }}
-      - restore_cache:
-          keys:
-            - parameter-cache-{{ .Revision }}
+      - run:
+          name: Ensure cache is hydrated with PoRep and PoSt Groth parameters (for test)
+          command: |
+            cargo run --release --color=always --package filecoin-proofs --bin paramcache -- --include-post --test-only
       - run:
           name: run regression tests (examples) against FFI-exposed filecoin-proofs API
           command: RUSTFLAGS="-Z sanitizer=leak" cargo run --release --package filecoin-proofs --example ffi --target x86_64-unknown-linux-gnu
+      - save_cache:
+          key: parameter-cache-{{ .Revision }}
+          paths:
+          - /root/.filecoin-parameter-cache
 
   test_ignored_release:
     docker:

filecoin-proofs/examples/ffi/main.rs

@@ -19,6 +19,7 @@ use rand::{thread_rng, Rng};
 use std::env;
 use std::error::Error;
 use std::ptr;
+use std::slice::from_raw_parts;
 use std::sync::atomic::AtomicPtr;
 use std::sync::mpsc;
 use std::thread;
@@ -312,6 +313,50 @@ unsafe fn sector_builder_lifecycle(use_live_store: bool) -> Result<(), Box<Error
         assert_eq!(1, (*resp).sectors_len);
     }
 
+    // generate and then verify a proof-of-spacetime for the sealed sectors
+    {
+        let resp = get_sealed_sectors(sector_builder_b);
+        defer!(destroy_get_sealed_sectors_response(resp));
+
+        if (*resp).status_code != 0 {
+            panic!("{}", c_str_to_rust_str((*resp).error_msg))
+        }
+
+        let sealed_sector_metadata: FFISealedSectorMetadata =
+            from_raw_parts((*resp).sectors_ptr, (*resp).sectors_len)[0];
+        let sealed_sector_replica_commitment: [u8; 32] = sealed_sector_metadata.comm_r;
+        let challenge_seed: [u8; 32] = [0; 32];
+
+        let resp = generate_post(
+            sector_builder_b,
+            &sealed_sector_replica_commitment[0],
+            32,
+            &challenge_seed,
+        );
+        defer!(destroy_generate_post_response(resp));
+
+        if (*resp).status_code != 0 {
+            panic!("{}", c_str_to_rust_str((*resp).error_msg))
+        }
+
+        let resp = verify_post(
+            &sizes.store,
+            &sealed_sector_replica_commitment[0],
+            32,
+            &challenge_seed,
+            &((*resp).proof),
+            (*resp).faults_ptr,
+            (*resp).faults_len,
+        );
+        defer!(destroy_verify_post_response(resp));
+
+        if (*resp).status_code != 0 {
+            panic!("{}", c_str_to_rust_str((*resp).error_msg))
+        }
+
+        assert!((*resp).is_valid)
+    }
+
     // after sealing, read the bytes (causes unseal) and compare with what we
     // added to the sector
     {

filecoin-proofs/src/api/mod.rs

@@ -136,13 +136,13 @@ pub unsafe extern "C" fn generate_post(
 ///
 #[no_mangle]
 pub unsafe extern "C" fn verify_post(
+    _cfg_ptr: *const ConfiguredStore,
     _flattened_comm_rs_ptr: *const u8,
     _flattened_comm_rs_len: libc::size_t,
     _challenge_seed: &[u8; 32],
     proof: &[u8; API_POST_PROOF_BYTES],
     _faults_ptr: *const u64,
     _faults_len: libc::size_t,
-    _sector_bytes: u64,
 ) -> *mut responses::VerifyPoSTResponse {
     let mut response: responses::VerifyPoSTResponse = Default::default();
 

filecoin-proofs/src/bin/paramcache.rs

@@ -4,8 +4,8 @@ extern crate sector_base;
 extern crate storage_proofs;
 
 use filecoin_proofs::api::internal;
+use filecoin_proofs::FCP_LOG;
 use pairing::bls12_381::Bls12;
-
 use sector_base::api::bytes_amount::PaddedBytesAmount;
 use sector_base::api::disk_backed_storage::{LIVE_SECTOR_SIZE, TEST_SECTOR_SIZE};
 use storage_proofs::circuit::vdf_post::{VDFPoStCircuit, VDFPostCompound};
@@ -16,9 +16,12 @@ use storage_proofs::parameter_cache::CacheableParameters;
 use storage_proofs::vdf_post::VDFPoSt;
 use storage_proofs::vdf_sloth::Sloth;
 
-const GENERATE_POST_PARAMS: bool = false;
+use clap::{App, Arg};
+use slog::*;
+
+fn cache_porep_params(sector_size: u64) {
+    info!(FCP_LOG, "begin PoRep parameter-cache check/populate routine for {}-byte sectors", sector_size; "target" => "paramcache");
 
-fn cache_params(sector_size: u64) {
     let bytes_amount = PaddedBytesAmount(sector_size);
 
     let public_params = internal::public_params(bytes_amount);
@@ -31,33 +34,68 @@ fn cache_params(sector_size: u64) {
         let circuit = ZigZagCompound::blank_circuit(&public_params, &internal::ENGINE_PARAMS);
         let _ = ZigZagCompound::get_verifying_key(circuit, &public_params);
     }
+}
 
-    if GENERATE_POST_PARAMS {
-        let post_public_params = internal::post_public_params(bytes_amount);
-        {
-            let post_circuit: VDFPoStCircuit<Bls12> =
-                <VDFPostCompound as CompoundProof<
-                    Bls12,
-                    VDFPoSt<PedersenHasher, Sloth>,
-                    VDFPoStCircuit<Bls12>,
-                >>::blank_circuit(&post_public_params, &internal::ENGINE_PARAMS);
-            let _ = VDFPostCompound::get_groth_params(post_circuit, &post_public_params);
-        }
-        {
-            let post_circuit: VDFPoStCircuit<Bls12> =
-                <VDFPostCompound as CompoundProof<
-                    Bls12,
-                    VDFPoSt<PedersenHasher, Sloth>,
-                    VDFPoStCircuit<Bls12>,
-                >>::blank_circuit(&post_public_params, &internal::ENGINE_PARAMS);
-
-            let _ = VDFPostCompound::get_verifying_key(post_circuit, &post_public_params);
-        }
+fn cache_post_params(sector_size: u64) {
+    info!(FCP_LOG, "begin PoSt parameter-cache check/populate routine for {}-byte sectors", sector_size; "target" => "paramcache");
+
+    let bytes_amount = PaddedBytesAmount(sector_size);
+
+    let post_public_params = internal::post_public_params(bytes_amount);
+    {
+        let post_circuit: VDFPoStCircuit<Bls12> =
+            <VDFPostCompound as CompoundProof<
+                Bls12,
+                VDFPoSt<PedersenHasher, Sloth>,
+                VDFPoStCircuit<Bls12>,
+            >>::blank_circuit(&post_public_params, &internal::ENGINE_PARAMS);
+        let _ = VDFPostCompound::get_groth_params(post_circuit, &post_public_params);
+    }
+    {
+        let post_circuit: VDFPoStCircuit<Bls12> =
+            <VDFPostCompound as CompoundProof<
+                Bls12,
+                VDFPoSt<PedersenHasher, Sloth>,
+                VDFPoStCircuit<Bls12>,
+            >>::blank_circuit(&post_public_params, &internal::ENGINE_PARAMS);
+
+        let _ = VDFPostCompound::get_verifying_key(post_circuit, &post_public_params);
     }
 }
 
 // Run this from the command-line to pre-generate the groth parameters used by the API.
 pub fn main() {
-    cache_params(TEST_SECTOR_SIZE);
-    cache_params(LIVE_SECTOR_SIZE);
+    let matches = App::new("paramcache")
+        .version("0.1")
+        .about("Generate and persist Groth parameters")
+        .arg(
+            Arg::with_name("test-only")
+                .long("test-only")
+                .help("generate only parameters useful for testing")
+                .takes_value(false),
+        )
+        .arg(
+            Arg::with_name("include-post")
+                .long("include-post")
+                .help("generate PoSt parameters in addition to PoRep")
+                .takes_value(false),
+        )
+        .get_matches();
+
+    let include_post: bool = matches.is_present("include-post");
+    let test_only: bool = matches.is_present("test-only");
+
+    cache_porep_params(TEST_SECTOR_SIZE);
+
+    if !test_only {
+        cache_porep_params(LIVE_SECTOR_SIZE);
+    }
+
+    if include_post {
+        cache_post_params(TEST_SECTOR_SIZE);
+
+        if !test_only {
+            cache_post_params(LIVE_SECTOR_SIZE);
+        }
+    }
 }