diff --git a/Cargo.lock b/Cargo.lock index 7de213f6..a562f30e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -952,6 +953,7 @@ dependencies = [ name = "floresta-wire" version = "0.3.0" dependencies = [ + "ahash", "bitcoin 0.32.4", "dns-lookup 1.0.8", "floresta-chain", @@ -2007,9 +2009,9 @@ checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustreexo" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b4e6061c2c36ec0f30c8c2e737efc0b69ffc22b57a79a28aefc849c35a1523a" +checksum = "64bce4e1e36c12aaee94f9604187b1dbb2b826bbcb6807f757f41b1cc9946bc7" dependencies = [ "bitcoin_hashes 0.14.0", ] diff --git a/crates/floresta-chain/Cargo.toml b/crates/floresta-chain/Cargo.toml index 0bc62e0a..4e2621bc 100644 --- a/crates/floresta-chain/Cargo.toml +++ b/crates/floresta-chain/Cargo.toml @@ -18,7 +18,7 @@ categories = ["cryptography::cryptocurrencies", "database"] crate-type = ["cdylib", "rlib"] [dependencies] -rustreexo = "0.3.0" +rustreexo = "0.4" sha2 = "^0.10.6" log = "0.4" kv = "0.24.0" diff --git a/crates/floresta-chain/src/pruned_utreexo/chain_state.rs b/crates/floresta-chain/src/pruned_utreexo/chain_state.rs index c4637b9a..3e8073cd 100644 --- a/crates/floresta-chain/src/pruned_utreexo/chain_state.rs +++ b/crates/floresta-chain/src/pruned_utreexo/chain_state.rs @@ -30,7 +30,7 @@ use log::trace; use log::warn; #[cfg(feature = "metrics")] use metrics; -use rustreexo::accumulator::node_hash::NodeHash; +use rustreexo::accumulator::node_hash::BitcoinNodeHash; use rustreexo::accumulator::proof::Proof; use rustreexo::accumulator::stump::Stump; use spin::RwLock; @@ -634,7 +634,7 @@ impl ChainState { assert_eq!(acc.len() % 32, 0); while acc.len() >= 32 { let root = acc.drain(0..32).collect::>(); - let root = NodeHash::from(&*root); + let root = BitcoinNodeHash::from(&*root); roots.push(root); } Stump { leaves, roots } @@ -831,7 +831,7 @@ impl BlockchainInterface for ChainState>(); if !acc.verify(&proof, &del_hashes)? { @@ -1170,7 +1170,7 @@ impl UpdatableChainstate for ChainState Vec { + fn get_root_hashes(&self) -> Vec { let inner = read_lock!(self); inner.acc.roots.clone() } diff --git a/crates/floresta-chain/src/pruned_utreexo/chainparams.rs b/crates/floresta-chain/src/pruned_utreexo/chainparams.rs index ae70ed49..5e455ee4 100644 --- a/crates/floresta-chain/src/pruned_utreexo/chainparams.rs +++ b/crates/floresta-chain/src/pruned_utreexo/chainparams.rs @@ -7,7 +7,7 @@ use bitcoin::p2p::ServiceFlags; use bitcoin::params::Params; use bitcoin::Block; use bitcoin::BlockHash; -use rustreexo::accumulator::node_hash::NodeHash; +use rustreexo::accumulator::node_hash::BitcoinNodeHash; use crate::prelude::*; use crate::AssumeValidArg; @@ -74,7 +74,7 @@ pub struct AssumeUtreexoValue { /// Same as block_hash, but in height pub height: u32, /// The roots of the Utreexo accumulator at this block - pub roots: Vec, + pub roots: Vec, /// The number of leaves in the Utreexo accumulator at this block pub leaves: u64, } @@ -109,7 +109,7 @@ impl ChainParams { "972ea2c7472c22e4eab49e9c2db5757a048b271b6251883ce89ccfeaa38b47ab", ] .into_iter() - .map(|x| NodeHash::from_str(x).unwrap()) + .map(|x| BitcoinNodeHash::from_str(x).unwrap()) .collect(), leaves: 2587882501, }, diff --git a/crates/floresta-chain/src/pruned_utreexo/consensus.rs b/crates/floresta-chain/src/pruned_utreexo/consensus.rs index c0d42f3b..5f57724f 100644 --- a/crates/floresta-chain/src/pruned_utreexo/consensus.rs +++ b/crates/floresta-chain/src/pruned_utreexo/consensus.rs @@ -7,11 +7,9 @@ extern crate alloc; use core::ffi::c_uint; use bitcoin::block::Header as BlockHeader; -use bitcoin::consensus::Encodable; use bitcoin::hashes::sha256; use bitcoin::hashes::Hash; use bitcoin::Block; -use bitcoin::BlockHash; use bitcoin::CompactTarget; use bitcoin::OutPoint; use bitcoin::ScriptBuf; @@ -21,15 +19,14 @@ use bitcoin::TxIn; use bitcoin::TxOut; use bitcoin::Txid; use floresta_common::prelude::*; -use rustreexo::accumulator::node_hash::NodeHash; +use rustreexo::accumulator::node_hash::BitcoinNodeHash; use rustreexo::accumulator::proof::Proof; use rustreexo::accumulator::stump::Stump; -use sha2::Digest; -use sha2::Sha512_256; use super::chainparams::ChainParams; use super::error::BlockValidationErrors; use super::error::BlockchainError; +use super::udata; use crate::TransactionError; /// The value of a single coin in satoshis. @@ -69,37 +66,6 @@ impl Consensus { subsidy } - /// Returns the hash of a leaf node in the utreexo accumulator. - #[inline] - fn get_leaf_hashes( - transaction: &Transaction, - vout: u32, - height: u32, - block_hash: BlockHash, - ) -> sha256::Hash { - let header_code = height << 1; - - let mut ser_utxo = Vec::new(); - let utxo = transaction.output.get(vout as usize).unwrap(); - utxo.consensus_encode(&mut ser_utxo).unwrap(); - let header_code = if transaction.is_coinbase() { - header_code | 1 - } else { - header_code - }; - - let leaf_hash = Sha512_256::new() - .chain_update(UTREEXO_TAG_V1) - .chain_update(UTREEXO_TAG_V1) - .chain_update(block_hash) - .chain_update(transaction.compute_txid()) - .chain_update(vout.to_le_bytes()) - .chain_update(header_code.to_le_bytes()) - .chain_update(ser_utxo) - .finalize(); - sha256::Hash::from_slice(leaf_hash.as_slice()) - .expect("parent_hash: Engines shouldn't be Err") - } /// Verify if all transactions in a block are valid. Here we check the following: /// - The block must contain at least one transaction, and this transaction must be coinbase /// - The first transaction in the block must be coinbase @@ -279,6 +245,7 @@ impl Consensus { CompactTarget::from_next_work_required(first_block.bits, actual_timespan as u64, params) .into() } + /// Updates our accumulator with the new block. This is done by calculating the new /// root hash of the accumulator, and then verifying the proof of inclusion of the /// deleted nodes. If the proof is valid, we return the new accumulator. Otherwise, @@ -292,56 +259,17 @@ impl Consensus { del_hashes: Vec, ) -> Result { let block_hash = block.block_hash(); - let mut leaf_hashes = Vec::new(); let del_hashes = del_hashes .iter() - .map(|hash| NodeHash::from(hash.as_byte_array())) + .map(|hash| BitcoinNodeHash::from(hash.as_byte_array())) .collect::>(); - // Get inputs from the block, we'll need this HashSet to check if an output is spent - // in the same block. If it is, we don't need to add it to the accumulator. - let mut block_inputs = HashSet::new(); - for transaction in block.txdata.iter() { - for input in transaction.input.iter() { - block_inputs.insert((input.previous_output.txid, input.previous_output.vout)); - } - } - // Get all leaf hashes that will be added to the accumulator - for transaction in block.txdata.iter() { - for (i, output) in transaction.output.iter().enumerate() { - if !Self::is_unspendable(&output.script_pubkey) - && !block_inputs.contains(&(transaction.compute_txid(), i as u32)) - { - leaf_hashes.push(Self::get_leaf_hashes( - transaction, - i as u32, - height, - block_hash, - )) - } - } - } - // Convert the leaf hashes to NodeHashes used in Rustreexo - let hashes: Vec = leaf_hashes - .iter() - .map(|&hash| NodeHash::from(hash.as_byte_array())) - .collect(); + let adds = udata::proof_util::get_block_adds(block, height, block_hash); + // Update the accumulator - let acc = acc.modify(&hashes, &del_hashes, &proof)?.0; + let acc = acc.modify(&adds, &del_hashes, &proof)?.0; Ok(acc) } - - fn is_unspendable(script: &ScriptBuf) -> bool { - if script.len() > 10_000 { - return true; - } - - if !script.is_empty() && script.as_bytes()[0] == 0x6a { - return true; - } - - false - } } #[cfg(test)] mod tests { diff --git a/crates/floresta-chain/src/pruned_utreexo/mod.rs b/crates/floresta-chain/src/pruned_utreexo/mod.rs index 0822074d..9c53281e 100644 --- a/crates/floresta-chain/src/pruned_utreexo/mod.rs +++ b/crates/floresta-chain/src/pruned_utreexo/mod.rs @@ -18,7 +18,7 @@ use bitcoin::BlockHash; use bitcoin::OutPoint; use bitcoin::Transaction; use bitcoin::TxOut; -use rustreexo::accumulator::node_hash::NodeHash; +use rustreexo::accumulator::node_hash::BitcoinNodeHash; use rustreexo::accumulator::proof::Proof; use rustreexo::accumulator::stump::Stump; @@ -132,7 +132,7 @@ pub trait UpdatableChainstate { /// marked as invalid. fn mark_block_as_valid(&self, block: BlockHash) -> Result<(), BlockchainError>; /// Returns the root hashes of our utreexo forest - fn get_root_hashes(&self) -> Vec; + fn get_root_hashes(&self) -> Vec; /// Returns a partial chainstate from a range of blocks. /// /// [PartialChainState] is a simplified version of `ChainState` that is used during IBD. @@ -223,7 +223,7 @@ impl UpdatableChainstate for Arc { T::accept_header(self, header) } - fn get_root_hashes(&self) -> Vec { + fn get_root_hashes(&self) -> Vec { T::get_root_hashes(self) } diff --git a/crates/floresta-chain/src/pruned_utreexo/partial_chain.rs b/crates/floresta-chain/src/pruned_utreexo/partial_chain.rs index c7950599..9ec8dbc5 100644 --- a/crates/floresta-chain/src/pruned_utreexo/partial_chain.rs +++ b/crates/floresta-chain/src/pruned_utreexo/partial_chain.rs @@ -22,6 +22,7 @@ //! threads, as long as the origin thread gives away the ownership. use bitcoin::BlockHash; use floresta_common::prelude::*; +use rustreexo::accumulator::node_hash::BitcoinNodeHash; extern crate alloc; use core::cell::UnsafeCell; @@ -318,7 +319,7 @@ impl UpdatableChainstate for PartialChainState { .process_block(block, proof, inputs, del_hashes) } - fn get_root_hashes(&self) -> Vec { + fn get_root_hashes(&self) -> Vec { self.inner().current_acc.roots.clone() } @@ -506,7 +507,7 @@ mod tests { use bitcoin::block::Header; use bitcoin::consensus::deserialize; use bitcoin::Block; - use rustreexo::accumulator::node_hash::NodeHash; + use rustreexo::accumulator::node_hash::BitcoinNodeHash; use rustreexo::accumulator::proof::Proof; use rustreexo::accumulator::stump::Stump; @@ -641,7 +642,7 @@ mod tests { "bedb648c9a3c5741660f926c1552d83ebb4cb1842cca6855b6d1089bb4951ce1", ] .iter() - .map(|hash| NodeHash::from_str(hash).unwrap()) + .map(|hash| BitcoinNodeHash::from_str(hash).unwrap()) .collect(); let acc2 = Stump { roots, leaves: 100 }; @@ -682,7 +683,7 @@ mod tests { "1864a4982532447dcb3d9a5d2fea9f8ed4e3b1e759d55b8a427fb599fed0c302", ] .iter() - .map(|x| NodeHash::from(hex::decode(x).unwrap().as_slice())) + .map(|x| BitcoinNodeHash::from(hex::decode(x).unwrap().as_slice())) .collect::>(); let expected_acc: Stump = Stump { leaves: 150, roots }; diff --git a/crates/floresta-chain/src/pruned_utreexo/udata.rs b/crates/floresta-chain/src/pruned_utreexo/udata.rs index ead19e73..c425592f 100644 --- a/crates/floresta-chain/src/pruned_utreexo/udata.rs +++ b/crates/floresta-chain/src/pruned_utreexo/udata.rs @@ -283,9 +283,12 @@ impl From for UtreexoBlock { pub mod proof_util { use bitcoin::blockdata::script::Instruction; + use bitcoin::consensus::Encodable; use bitcoin::hashes::sha256; use bitcoin::hashes::Hash; use bitcoin::Amount; + use bitcoin::Block; + use bitcoin::BlockHash; use bitcoin::OutPoint; use bitcoin::PubkeyHash; use bitcoin::ScriptBuf; @@ -295,11 +298,14 @@ pub mod proof_util { use bitcoin::TxOut; use bitcoin::WPubkeyHash; use bitcoin::WScriptHash; - use rustreexo::accumulator::node_hash::NodeHash; + use rustreexo::accumulator::node_hash::BitcoinNodeHash; use rustreexo::accumulator::proof::Proof; + use sha2::Digest; + use sha2::Sha512_256; use super::LeafData; use crate::prelude::*; + use crate::pruned_utreexo::consensus::UTREEXO_TAG_V1; use crate::pruned_utreexo::BlockchainInterface; use crate::CompactLeafData; use crate::ScriptPubkeyType; @@ -310,6 +316,26 @@ pub mod proof_util { EmptyStack, } + pub fn get_script_type(script: &ScriptBuf) -> ScriptPubkeyType { + if script.is_p2pkh() { + return ScriptPubkeyType::PubKeyHash; + } + + if script.is_p2sh() { + return ScriptPubkeyType::ScriptHash; + } + + if script.is_p2wpkh() { + return ScriptPubkeyType::WitnessV0PubKeyHash; + } + + if script.is_p2wsh() { + return ScriptPubkeyType::WitnessV0ScriptHash; + } + + ScriptPubkeyType::Other(script.to_bytes().into_boxed_slice()) + } + pub fn reconstruct_leaf_data( leaf: &CompactLeafData, input: &TxIn, @@ -328,6 +354,85 @@ pub mod proof_util { }) } + fn is_unspendable(script: &ScriptBuf) -> bool { + if script.len() > 10_000 { + return true; + } + + if !script.is_empty() && script.as_bytes()[0] == 0x6a { + return true; + } + + false + } + + /// Returns the hash of a leaf node in the utreexo accumulator. + #[inline] + fn get_leaf_hashes( + transaction: &Transaction, + vout: u32, + height: u32, + block_hash: BlockHash, + ) -> sha256::Hash { + let header_code = height << 1; + + let mut ser_utxo = Vec::new(); + let utxo = transaction.output.get(vout as usize).unwrap(); + utxo.consensus_encode(&mut ser_utxo).unwrap(); + let header_code = if transaction.is_coinbase() { + header_code | 1 + } else { + header_code + }; + + let leaf_hash = Sha512_256::new() + .chain_update(UTREEXO_TAG_V1) + .chain_update(UTREEXO_TAG_V1) + .chain_update(block_hash) + .chain_update(transaction.compute_txid()) + .chain_update(vout.to_le_bytes()) + .chain_update(header_code.to_le_bytes()) + .chain_update(ser_utxo) + .finalize(); + sha256::Hash::from_slice(leaf_hash.as_slice()) + .expect("parent_hash: Engines shouldn't be Err") + } + + /// From a block, gets the roots that will be included on the acc, certifying + /// that any utxo will not be spend in the same block. + pub fn get_block_adds( + block: &Block, + height: u32, + block_hash: BlockHash, + ) -> Vec { + let mut leaf_hashes = Vec::new(); + // Get inputs from the block, we'll need this HashSet to check if an output is spent + // in the same block. If it is, we don't need to add it to the accumulator. + let mut block_inputs = HashSet::new(); + for transaction in block.txdata.iter() { + for input in transaction.input.iter() { + block_inputs.insert((input.previous_output.txid, input.previous_output.vout)); + } + } + + // Get all leaf hashes that will be added to the accumulator + for transaction in block.txdata.iter() { + for (i, output) in transaction.output.iter().enumerate() { + if !is_unspendable(&output.script_pubkey) + && !block_inputs.contains(&(transaction.compute_txid(), i as u32)) + { + leaf_hashes.push(get_leaf_hashes(transaction, i as u32, height, block_hash)) + } + } + } + + // Convert the leaf hashes to NodeHashes used in Rustreexo + leaf_hashes + .iter() + .map(|&hash| BitcoinNodeHash::from(hash.as_byte_array())) + .collect() + } + #[allow(clippy::type_complexity)] pub fn process_proof( udata: &UData, @@ -339,7 +444,7 @@ pub mod proof_util { .proof .hashes .iter() - .map(|hash| NodeHash::Some(*hash.as_byte_array())) + .map(|hash| BitcoinNodeHash::Some(*hash.as_byte_array())) .collect(); let proof = Proof::new(targets, hashes); let mut hashes = Vec::new(); @@ -378,7 +483,10 @@ pub mod proof_util { Ok((proof, hashes, inputs)) } - fn reconstruct_script_pubkey(leaf: &CompactLeafData, input: &TxIn) -> Result { + pub fn reconstruct_script_pubkey( + leaf: &CompactLeafData, + input: &TxIn, + ) -> Result { match &leaf.spk_ty { ScriptPubkeyType::Other(spk) => Ok(ScriptBuf::from(spk.clone().into_vec())), ScriptPubkeyType::PubKeyHash => { diff --git a/crates/floresta-electrum/Cargo.toml b/crates/floresta-electrum/Cargo.toml index 06d6d32a..9e515fd3 100644 --- a/crates/floresta-electrum/Cargo.toml +++ b/crates/floresta-electrum/Cargo.toml @@ -21,7 +21,7 @@ floresta-watch-only = { path = "../floresta-watch-only" } floresta-compact-filters = { path = "../floresta-compact-filters" } floresta-wire = { path = "../floresta-wire" } -rustreexo = "0.3.0" +rustreexo = "0.4" sha2 = "^0.10.6" tokio = { version = "1.0", features = ["full"] } tokio-rustls = "0.22" diff --git a/crates/floresta-electrum/src/electrum_protocol.rs b/crates/floresta-electrum/src/electrum_protocol.rs index 3fb4c034..c8667527 100644 --- a/crates/floresta-electrum/src/electrum_protocol.rs +++ b/crates/floresta-electrum/src/electrum_protocol.rs @@ -919,6 +919,7 @@ mod test { use futures::executor::block_on; use rcgen::generate_simple_self_signed; use rcgen::CertifiedKey; + use rustreexo::accumulator::pollard::Pollard; use serde_json::json; use serde_json::Number; use serde_json::Value; @@ -926,6 +927,7 @@ mod test { use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; use tokio::net::TcpStream; + use tokio::sync::Mutex; use tokio::task; use tokio::time::timeout; use tokio_rustls::rustls::Certificate; @@ -1061,7 +1063,7 @@ mod test { UtreexoNode::new( u_config, chain.clone(), - Arc::new(tokio::sync::RwLock::new(Mempool::new())), + Arc::new(Mutex::new(Mempool::new(Pollard::default(), 0))), None, ) .unwrap(); diff --git a/crates/floresta-wire/Cargo.toml b/crates/floresta-wire/Cargo.toml index 68678fac..40e5920c 100644 --- a/crates/floresta-wire/Cargo.toml +++ b/crates/floresta-wire/Cargo.toml @@ -16,7 +16,7 @@ categories = ["cryptography::cryptocurrencies", "network-programming"] [dependencies] -rustreexo = "0.3.0" +rustreexo = "0.4" sha2 = "^0.10.6" tokio = { version = "1", features = ["full"] } log = "0.4" @@ -33,6 +33,7 @@ floresta-compact-filters = { path = "../floresta-compact-filters" } thiserror = "1.0" floresta-common = { path = "../floresta-common" } oneshot = "0.1.5" +ahash = "0.8.11" [dev-dependencies] zstd = "0.13.2" diff --git a/crates/floresta-wire/src/p2p_wire/chain_selector.rs b/crates/floresta-wire/src/p2p_wire/chain_selector.rs index 9daa63cd..07ac3ef0 100644 --- a/crates/floresta-wire/src/p2p_wire/chain_selector.rs +++ b/crates/floresta-wire/src/p2p_wire/chain_selector.rs @@ -59,7 +59,7 @@ use floresta_chain::UtreexoBlock; use floresta_common::service_flags; use log::info; use log::warn; -use rustreexo::accumulator::node_hash::NodeHash; +use rustreexo::accumulator::node_hash::BitcoinNodeHash; use rustreexo::accumulator::stump::Stump; use tokio::sync::RwLock; use tokio::time::timeout; @@ -185,7 +185,7 @@ where let slice = acc.drain(0..32); let mut root = [0u8; 32]; root.copy_from_slice(&slice.collect::>()); - roots.push(NodeHash::from(root)); + roots.push(BitcoinNodeHash::from(root)); } Ok(Stump { leaves, roots }) } diff --git a/crates/floresta-wire/src/p2p_wire/mempool.rs b/crates/floresta-wire/src/p2p_wire/mempool.rs index 3b8730a0..8a2fdfea 100644 --- a/crates/floresta-wire/src/p2p_wire/mempool.rs +++ b/crates/floresta-wire/src/p2p_wire/mempool.rs @@ -2,56 +2,1045 @@ //! A simple mempool that keeps our transactions in memory. It try to rebroadcast //! our transactions every 1 hour. //! Once our transaction is included in a block, we remove it from the mempool. +use std::collections::BTreeSet; use std::collections::HashMap; use std::time::Duration; use std::time::Instant; +use bitcoin::block::Header; +use bitcoin::block::Version; +use bitcoin::hashes::Hash; +use bitcoin::Amount; use bitcoin::Block; +use bitcoin::BlockHash; +use bitcoin::CompactTarget; +use bitcoin::OutPoint; use bitcoin::Transaction; +use bitcoin::TxMerkleNode; +use bitcoin::TxOut; use bitcoin::Txid; +use floresta_chain::proof_util; +use floresta_chain::pruned_utreexo::BlockchainInterface; +use floresta_chain::CompactLeafData; +use floresta_chain::LeafData; +use rustreexo::accumulator::node_hash::BitcoinNodeHash; +use rustreexo::accumulator::pollard::Pollard; +use rustreexo::accumulator::pollard::PollardAddition; +use rustreexo::accumulator::proof::Proof; + +/// A short transaction id that we use to identify transactions in the mempool. +/// +/// We use this to keep track of dependencies between transactions, since keeping the full txid +/// would be too expensive. This value is computed using a keyed hash function, with a local key +/// that only we know. This way, peers can't cause collisions and make our mempool slow. +type ShortTxid = u64; + +#[derive(Debug)] +/// A transaction in the mempool. +/// +/// This struct holds the transaction itself, the time when we added it to the mempool, the +/// transactions that depend on it, and the transactions that it depends on. We need those extra +/// informations to make decisions when to include or not a transaction in mempool or in a block. +struct MempoolTransaction { + transaction: Transaction, + time: Instant, + depends: Vec, + children: Vec, +} + +pub trait BlockHashOracle { + fn get_block_hash(&self, height: u32) -> Option; +} + +impl BlockHashOracle for T { + fn get_block_hash(&self, height: u32) -> Option { + self.get_block_hash(height).ok() + } +} + /// Holds the transactions that we broadcasted and are still in the mempool. -#[derive(Debug, Default)] -pub struct Mempool(HashMap); +#[derive(Debug)] +pub struct Mempool { + /// A list of all transactions we currently have in the mempool. + /// + /// Transactions are kept as a map of their transaction id to the transaction itself, we + /// also keep track of when we added the transaction to the mempool to be able to remove + /// stale transactions. + transactions: HashMap, + /// How much memory (in bytes) does the mempool currently use. + mempool_size: usize, + /// The maximum size of the mempool in bytes. + max_mempool_size: usize, + /// The accumulator that we use to verify proofs. + /// + /// This is a Pollard, a light version of a Utreexo accumulator that allows you to hold some + /// leaves, but not all of them. We use this to keep track of mempool proofs so we don't need + /// to re-download them. + acc: Pollard, + /// A map of all the prevouts that are being spent by transactions in the mempool. + /// + /// Since we don't have a full UTXO set, we need to keep track of the outputs that are being + /// spent in order to perform validation and fee calculation. + prevouts: HashMap, + /// A queue of transaction we know about, but don't have a proof for + queue: Vec, + /// A hasher that we use to compute the short transaction ids. + hasher: ahash::RandomState, +} + +unsafe impl Send for Mempool {} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// An error returned when we try to add a transaction to the mempool. +pub enum AcceptToMempoolError { + /// The proof provided is invalid. + InvalidProof, + /// The transaction is trying to spend an output that we don't have. + InvalidPrevout, + /// Memory usage is too high. + MemoryUsageTooHigh, + /// We couldn't find a prevout in the mempool. + /// + /// This error only happens when we try to add a transaction without a proof, and we don't have + /// the prevouts in the mempool. + PrevoutNotFound, + /// The transaction is conflicting with another transaction in the mempool. + ConflictingTransaction, + /// An error happened while trying to get a proof from the accumulator. + Rustreexo(String), + /// The transaction has duplicate inputs. + DuplicateInput, + BlockNotFound, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// A proof for a transaction in the mempool. +pub struct MempoolProof { + /// The actual utreexo proof + pub proof: Proof, + /// The target hashes that we are trying to prove. + pub target_hashes: Vec, + /// The leaf data for the targets we are proving + pub leaves: Vec, +} impl Mempool { - pub fn new() -> Mempool { - Mempool(HashMap::new()) + /// Creates a new mempool with a given maximum size and accumulator. + /// + /// The acculator should have the same roots as the one inside our chainstate, or we won't be + /// able to validate proofs. + pub fn new(acc: Pollard, max_mempool_size: usize) -> Mempool { + let a = rand::random(); + let b = rand::random(); + let c = rand::random(); + let d = rand::random(); + + let hasher = ahash::RandomState::with_seeds(a, b, c, d); + + Mempool { + transactions: HashMap::new(), + prevouts: HashMap::new(), + queue: Vec::new(), + mempool_size: 0, + max_mempool_size, + acc, + hasher, + } + } + + /// List transactions we are pending to process. + /// + /// Usually, we don't have a proof for these transactions, so we can't add them to the mempool, + /// a wire implementation should call this method and try to get a proof for these + /// transactions. + pub fn list_unprocessed(&self) -> Vec { + self.queue.clone() + } + + /// List all transactions we've accepted to the mempool. + /// + /// This won't count transactions that are still in the queue. + pub fn list_mempool(&self) -> Vec { + self.transactions + .keys() + .map(|id| self.transactions[id].transaction.compute_txid()) + .collect() + } + + /// Returns the data of the prevouts that are being spent by a transaction. + /// + /// This data isn't part of the actual transaction, usually we would fetch it from the UTXO + /// set, but we don't have one. Instead, we keep track of the prevouts that are being spent by + /// transactions in the mempool and use this method to get the data. + pub fn get_prevouts(&self, tx: &Transaction) -> Vec { + tx.input + .iter() + .filter_map(|input| { + let leaf_data = self.prevouts.get(&input.previous_output)?; + let script_pubkey = proof_util::reconstruct_script_pubkey(leaf_data, input).ok()?; + Some(TxOut { + value: Amount::from_sat(leaf_data.amount), + script_pubkey, + }) + }) + .collect() + } + + /// Proves that a mempool transaction is valid for the latest accumulator state. + /// + /// This should return a proof that the transaction is valid, and the data for the prevouts + /// that are being spent by the transaction. + pub fn try_prove( + &self, + tx: &Transaction, + block_hash: &impl BlockHashOracle, + ) -> Result { + let mut target_hashes = Vec::new(); + let mut leaves = Vec::new(); + for input in tx.input.iter() { + let prevout = self + .prevouts + .get(&input.previous_output) + .ok_or(AcceptToMempoolError::PrevoutNotFound)?; + + // The block hash of the block that commited the prevout. + let block_hash = block_hash.get_block_hash(prevout.header_code >> 1).unwrap(); + let leaf_data: LeafData = proof_util::reconstruct_leaf_data(prevout, input, block_hash) + .map_err(|_| AcceptToMempoolError::InvalidPrevout)?; + + let hash = leaf_data._get_leaf_hashes(); + let hash = BitcoinNodeHash::Some(hash.to_byte_array()); + + target_hashes.push(hash); + leaves.push(prevout.clone()); + } + + let proof = self + .acc + .batch_proof(&target_hashes) + .map_err(AcceptToMempoolError::Rustreexo)?; + + Ok(MempoolProof { + proof, + target_hashes, + leaves, + }) } - /// Find all transactions that are in the mempool and were included in the given block. - /// Remove them from the mempool and return them. - pub fn consume_block(&mut self, block: &Block) -> Vec { - if self.0.is_empty() { - return Vec::new(); + + /// Returns a list of transactions that are in the mempool up to the block weight limit. + /// + /// Returns a candidate block to be mined. + pub fn get_block_template( + &self, + version: Version, + prev_blockhash: BlockHash, + time: u32, + bits: CompactTarget, + ) -> Block { + // add transactions until we reach the block limit + let mut size = 0; + + let mut txs = Vec::new(); + for (_, tx) in self.transactions.iter() { + let tx_size = tx.transaction.weight().to_wu(); + if size + tx_size > 4_000_000 { + break; + } + + if txs.contains(&tx.transaction) { + continue; + } + + size += tx_size; + let short_txid = self.hasher.hash_one(tx.transaction.compute_txid()); + self.add_transaction_to_block(&mut txs, short_txid); } - let mut delta = Vec::new(); - for tx in block.txdata.iter() { - if self.0.contains_key(&tx.compute_txid()) { - delta.push(self.0.remove(&tx.compute_txid())); + + let mut block = Block { + header: Header { + version, + prev_blockhash, + merkle_root: TxMerkleNode::all_zeros(), + time, + bits, + nonce: 0, + }, + txdata: txs, + }; + + block.header.merkle_root = block.compute_merkle_root().unwrap(); + block + } + + /// Utility method that grabs one transaction and all its dependencies, then adds them to a tx + /// list. + fn add_transaction_to_block( + &self, + block_transactions: &mut Vec, + short_txid: ShortTxid, + ) { + let transaction = self.transactions.get(&short_txid).unwrap(); + if block_transactions.contains(&transaction.transaction) { + return; + } + + let depends_on = transaction.depends.clone(); + + for depend in depends_on { + self.add_transaction_to_block(block_transactions, depend); + } + + block_transactions.push(transaction.transaction.clone()); + } + + /// Consume a block and remove all transactions that were included in it. + /// + /// This method will remove all transactions that is in the block from the mempool, + /// update the local accumulator and return the txids that were included in the block. + pub fn consume_block( + &mut self, + block: &Block, + proof: Proof, + adds: &[PollardAddition], + del_hashes: &[BitcoinNodeHash], + block_height: u32, + remember_all: bool, + ) -> Result, AcceptToMempoolError> { + self.acc + .modify(adds, del_hashes, proof) + .map_err(AcceptToMempoolError::Rustreexo)?; + + if remember_all { + // add the newly created UTXOs to the prevouts + for tx in block.txdata.iter() { + let is_coinbase = tx.is_coinbase(); + for (vout, output) in tx.output.iter().enumerate() { + let leaf_data = CompactLeafData { + amount: output.value.to_sat(), + spk_ty: proof_util::get_script_type(&output.script_pubkey), + header_code: (block_height << 1) | is_coinbase as u32, + }; + let prevout = OutPoint { + txid: tx.compute_txid(), + vout: vout as u32, + }; + self.prevouts.insert(prevout, leaf_data); + } + + for input in tx.input.iter() { + if self.prevouts.remove(&input.previous_output).is_none() { + return Err(AcceptToMempoolError::PrevoutNotFound); + } + } + } + } + + Ok(block + .txdata + .iter() + .map(|tx| { + let short_txid = self.hasher.hash_one(tx.compute_txid()); + self.transactions + .remove(&short_txid) + .map(|tx| tx.transaction); + + tx.compute_txid() + }) + .collect()) + } + /// Proves all transactions included in a block. + pub fn get_block_proof( + &self, + block: &Block, + get_block_hash: impl BlockHashOracle, + ) -> Result { + let (del_hashes, leaves): (Vec<_>, Vec<_>) = block + .txdata + .iter() + .flat_map(|tx| { + tx.input + .iter() + .flat_map(|input| { + let prevout = self + .prevouts + .get(&input.previous_output) + .ok_or(AcceptToMempoolError::PrevoutNotFound)?; + + let block_height = prevout.header_code >> 1; + let block_hash = get_block_hash + .get_block_hash(block_height) + .ok_or(AcceptToMempoolError::BlockNotFound)?; + let node_hash = BitcoinNodeHash::Some( + proof_util::reconstruct_leaf_data(prevout, input, block_hash) + .unwrap() + ._get_leaf_hashes() + .to_byte_array(), + ); + + Ok::<_, AcceptToMempoolError>((node_hash, prevout.clone())) + }) + .collect::>() + }) + .unzip(); + + let proof = self + .acc + .batch_proof(&del_hashes) + .map_err(AcceptToMempoolError::Rustreexo)?; + + Ok(MempoolProof { + proof, + target_hashes: del_hashes, + leaves, + }) + } + + /// Checks if a outpoint is already spent in the mempool. + /// + /// This can be used to find conficts before adding a transaction to the mempool. + fn is_already_spent(&self, outpoint: &OutPoint) -> bool { + let short_txid = self.hasher.hash_one(outpoint.txid); + let Some(tx) = self.transactions.get(&short_txid) else { + return false; + }; + + tx.children.iter().any(|child| { + let Some(child_tx) = self.transactions.get(child) else { + return false; + }; + + child_tx.transaction.input.iter().any(|input| { + input.previous_output.txid == outpoint.txid + && input.previous_output.vout == outpoint.vout + }) + }) + } + + /// Performs some very basic sanity checks on a transaction before adding it to the mempool. + /// + /// This method checks if the transaction doesn't have conflicting inputs, if it doesn't spend + /// the same output twice, and if it doesn't exceed the memory usage limit. + /// + /// TODO: Move this to floresta-wire + fn sanity_check_transaction( + &self, + transaction: &Transaction, + ) -> Result<(), AcceptToMempoolError> { + let tx_size = transaction.total_size(); + if self.mempool_size + tx_size > self.max_mempool_size { + return Err(AcceptToMempoolError::MemoryUsageTooHigh); + } + + // check for duplicate inputs + let inputs = transaction + .input + .iter() + .map(|input| input.previous_output) + .collect::>(); + + if inputs.len() != transaction.input.len() { + return Err(AcceptToMempoolError::DuplicateInput); + } + + for input in transaction.input.iter() { + if self.is_already_spent(&input.previous_output) { + return Err(AcceptToMempoolError::ConflictingTransaction); + } + } + + Ok(()) + } + + /// Internal utility to add a transaction to the mempool. + /// + /// This method should never be called for transactions coming from the wire, since it doesn't + /// check if the transaction is valid other than basic constraint checks. This method is used + /// by the mempool itself to add transactions that are already known to be valid, such as + /// wallet transactions. For transactions coming from the wire, use `accept_to_mempool`. + pub fn accept_to_mempool_no_acc( + &mut self, + transaction: Transaction, + ) -> Result<(), AcceptToMempoolError> { + let tx_size = transaction.total_size(); + let short_txid = self.hasher.hash_one(transaction.compute_txid()); + let depends = self.find_mempool_depends(&transaction); + + if self.transactions.contains_key(&short_txid) { + return Ok(()); + } + + if self.mempool_size + tx_size > self.max_mempool_size { + return Err(AcceptToMempoolError::MemoryUsageTooHigh); + } + + self.mempool_size += tx_size; + + // this function should only be called if it spends unconfirmed outputs + // Check if the inputs are actually in the mempool + for input in transaction.input.iter() { + if self.prevouts.contains_key(&input.previous_output) { + continue; + } + + let short_txid = self.hasher.hash_one(input.previous_output.txid); + if self.transactions.contains_key(&short_txid) { + continue; } + + return Err(AcceptToMempoolError::PrevoutNotFound); } - delta.into_iter().flat_map(|tx| Some(tx?.0)).collect() + + self.sanity_check_transaction(&transaction)?; + + for depend in depends.iter() { + let tx = self.transactions.get_mut(depend).unwrap(); + tx.children.push(short_txid); + } + + self.transactions.insert( + short_txid, + MempoolTransaction { + time: Instant::now(), + depends, + transaction, + children: Vec::new(), + }, + ); + + Ok(()) } + + /// From a transaction that is already in the mempool, computes which transaction it depends. + fn find_mempool_depends(&self, tx: &Transaction) -> Vec { + tx.input + .iter() + .filter_map(|input| { + let short_txid = self.hasher.hash_one(input.previous_output.txid); + self.transactions.get(&short_txid).map(|_| short_txid) + }) + .collect() + } + /// Add a transaction to the mempool. - pub fn accept_to_mempool(&mut self, transaction: Transaction) { - self.0 - .insert(transaction.compute_txid(), (transaction, Instant::now())); + pub fn accept_to_mempool( + &mut self, + transaction: Transaction, + proof: Proof, + prevouts: &[(OutPoint, CompactLeafData)], + del_hashes: &[BitcoinNodeHash], + remembers: &[u64], + ) -> Result<(), AcceptToMempoolError> { + let tx_size = transaction.total_size(); + let short_txid = self.hasher.hash_one(transaction.compute_txid()); + + if self.transactions.contains_key(&short_txid) { + return Ok(()); + } + + if self.mempool_size + tx_size > self.max_mempool_size { + return Err(AcceptToMempoolError::MemoryUsageTooHigh); + } + + self.acc + .verify_and_ingest(proof, del_hashes, remembers) + .map_err(|_| AcceptToMempoolError::InvalidProof)?; + + self.prevouts.extend(prevouts.iter().cloned()); + + let depends = self.find_mempool_depends(&transaction); + + for depend in depends.iter() { + // check if the input is already spent + for input in transaction.input.iter() { + if self.is_already_spent(&input.previous_output) { + return Err(AcceptToMempoolError::ConflictingTransaction); + } + } + + self.transactions.entry(*depend).and_modify(|tx| { + tx.children.push(short_txid); + }); + } + + self.transactions.insert( + short_txid, + MempoolTransaction { + time: Instant::now(), + depends, + transaction, + children: Vec::new(), + }, + ); + + Ok(()) } + /// Get a transaction from the mempool. - pub fn get_from_mempool(&self, id: &Txid) -> Option<&Transaction> { - if let Some(tx) = self.0.get(id) { - return Some(&tx.0); - } - None + pub fn get_from_mempool<'a>(&'a self, id: &Txid) -> Option<&'a Transaction> { + let id = self.hasher.hash_one(id); + self.transactions.get(&id).map(|tx| &tx.transaction) } - /// Get all transactions that were in the mempool for more than 1 hour. + + /// Get all transactions that were in the mempool for more than 1 hour, if any pub fn get_stale(&mut self) -> Vec { - let mut stale = Vec::new(); - for (txid, transaction) in self.0.iter_mut() { - if transaction.1.elapsed() > Duration::from_secs(60 * 60) { - transaction.1 = Instant::now(); - stale.push(*txid); + self.transactions + .iter() + .filter_map(|(_, tx)| { + let txid = tx.transaction.compute_txid(); + match tx.time.elapsed() > Duration::from_secs(3600) { + true => Some(txid), + false => None, + } + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::collections::HashSet; + use std::str::FromStr; + + use bitcoin::absolute; + use bitcoin::block; + use bitcoin::consensus::encode::deserialize_hex; + use bitcoin::hashes::Hash; + use bitcoin::transaction::Version; + use bitcoin::Block; + use bitcoin::BlockHash; + use bitcoin::OutPoint; + use bitcoin::Script; + use bitcoin::ScriptBuf; + use bitcoin::Sequence; + use bitcoin::Target; + use bitcoin::Transaction; + use bitcoin::Witness; + use floresta_chain::proof_util; + use floresta_chain::CompactLeafData; + use floresta_chain::LeafData; + use rand::Rng; + use rand::SeedableRng; + use rustreexo::accumulator::node_hash::BitcoinNodeHash; + use rustreexo::accumulator::pollard::Pollard; + use rustreexo::accumulator::pollard::PollardAddition; + use rustreexo::accumulator::proof::Proof; + + use super::BlockHashOracle; + use super::Mempool; + use crate::mempool::MempoolProof; + + struct BlockHashProvider { + block_hash: HashMap, + } + + impl BlockHashOracle for BlockHashProvider { + fn get_block_hash(&self, height: u32) -> Option { + self.block_hash.get(&height).cloned() + } + } + + /// builds a list of transactions in a pseudo-random way + /// + /// We use those transactions in mempool tests + fn build_transactions(seed: u64, conflict: bool) -> Vec { + let mut rng = rand::rngs::StdRng::seed_from_u64(seed); + let mut transactions = Vec::new(); + + let n = rng.gen_range(1..1_000); + let mut outputs = Vec::new(); + + for _ in 0..n { + let mut tx = bitcoin::Transaction { + version: Version::ONE, + lock_time: absolute::LockTime::from_consensus(0), + input: Vec::new(), + output: Vec::new(), + }; + + let inputs = rng.gen_range(1..10); + for _ in 0..inputs { + if outputs.is_empty() { + break; + } + + let index = rng.gen_range(0..outputs.len()); + let previous_output: OutPoint = match conflict { + false => outputs.remove(index), + true => *outputs.get(index).unwrap(), + }; + + let input = bitcoin::TxIn { + previous_output, + script_sig: bitcoin::Script::new().into(), + sequence: Sequence::MAX, + witness: Witness::new(), + }; + + tx.input.push(input); + } + + let n = rng.gen_range(1..10); + + for _ in 0..n { + let script = rng.gen::<[u8; 32]>(); + let output = bitcoin::TxOut { + value: bitcoin::Amount::from_sat(rng.gen_range(0..100_000_000)), + script_pubkey: bitcoin::Script::from_bytes(&script).into(), + }; + + tx.output.push(output); + } + + outputs.extend(tx.output.iter().enumerate().map(|(vout, _)| OutPoint { + txid: tx.compute_txid(), + vout: vout as u32, + })); + + transactions.push(tx); + } + + transactions + } + + #[test] + fn test_block_proof() { + let mut mempool = super::Mempool::new( + rustreexo::accumulator::pollard::Pollard::default(), + 10_000_000, + ); + + let coinbase_spk: ScriptBuf = Script::from_bytes(&[0x6a]).into(); + + let coinbase = bitcoin::Transaction { + version: Version::ONE, + lock_time: absolute::LockTime::from_consensus(0), + input: Vec::new(), + output: vec![bitcoin::TxOut { + value: bitcoin::Amount::from_sat(50_000_000), + script_pubkey: coinbase_spk.clone(), + }], + }; + + let coinbase_id = coinbase.compute_txid(); + + let block = Block { + header: bitcoin::block::Header { + version: bitcoin::block::Version::ONE, + prev_blockhash: bitcoin::BlockHash::all_zeros(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: 0, + bits: bitcoin::CompactTarget::from_consensus(0x1d00ffff), + nonce: 0, + }, + txdata: vec![coinbase], + }; + + let coinbase_out_leaf = LeafData { + prevout: OutPoint { + txid: coinbase_id, + vout: 0, + }, + utxo: bitcoin::TxOut { + value: bitcoin::Amount::from_sat(50_000_000), + script_pubkey: coinbase_spk.clone(), + }, + block_hash: block.block_hash(), + header_code: 0, + }; + + let coinbase_out = PollardAddition:: { + hash: coinbase_out_leaf._get_leaf_hashes().into(), + remember: true, + }; + + mempool + .consume_block(&block, Proof::default(), &[coinbase_out], &[], 0, true) + .expect("failed to consume block"); + + let spending_tx = bitcoin::Transaction { + version: Version::ONE, + lock_time: absolute::LockTime::from_consensus(0), + input: vec![bitcoin::TxIn { + previous_output: OutPoint { + txid: coinbase_id, + vout: 0, + }, + script_sig: ScriptBuf::default(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![bitcoin::TxOut { + value: bitcoin::Amount::from_sat(50_000_000), + script_pubkey: coinbase_spk.clone(), + }], + }; + + let hashes = BlockHashProvider { + block_hash: [(0, block.block_hash())].iter().cloned().collect(), + }; + + mempool + .accept_to_mempool_no_acc(spending_tx) + .expect("failed to accept to mempool"); + + let block = mempool.get_block_template( + block::Version::ONE, + block.block_hash(), + 0, + Target::MAX_ATTAINABLE_REGTEST.to_compact_lossy(), + ); + + let MempoolProof { + proof, + target_hashes, + .. + } = mempool + .get_block_proof(&block, hashes) + .expect("failed to get block proof"); + + assert!(mempool.acc.verify(&proof, &target_hashes).is_ok()); + } + + #[test] + fn test_random() { + // just sanity check for build_transactions + let transactions = build_transactions(42, true); + assert!(!transactions.is_empty()); + + let transactions2 = build_transactions(42, true); + assert!(!transactions2.is_empty()); + assert_eq!(transactions, transactions2); + + let transactions3 = build_transactions(43, true); + assert!(!transactions3.is_empty()); + assert_ne!(transactions, transactions3); + } + + #[test] + fn test_mepool_accept_no_acc() { + let mut mempool = super::Mempool::new( + rustreexo::accumulator::pollard::Pollard::default(), + 10_000_000, + ); + + let transactions = build_transactions(42, false); + let len = transactions.len(); + + for tx in transactions { + mempool + .accept_to_mempool_no_acc(tx) + .expect("failed to accept to mempool"); + } + + assert_eq!(mempool.transactions.len(), len); + } + + #[test] + fn test_gbt_with_conflict() { + let mut mempool = super::Mempool::new( + rustreexo::accumulator::pollard::Pollard::default(), + 10_000_000, + ); + + let transactions = build_transactions(21, true); + + let mut did_confict = false; + for tx in transactions { + if mempool.accept_to_mempool_no_acc(tx).is_ok() { + did_confict = true; + } + } + + // we expect at least one conflict + assert!(did_confict); + + let target = Target::MAX_ATTAINABLE_REGTEST; + let block = mempool.get_block_template( + block::Version::ONE, + bitcoin::BlockHash::all_zeros(), + 0, + target.to_compact_lossy(), + ); + + assert!(block.check_merkle_root()); + + check_block_transactions(block); + } + + fn check_block_transactions(block: Block) { + // make sure that all outputs are spent after being created, and only once + let mut outputs = HashSet::new(); + for tx in block.txdata.iter() { + for input in tx.input.iter() { + if input.previous_output.txid == bitcoin::Txid::all_zeros() { + continue; + } + + assert!( + outputs.remove(&input.previous_output), + "double spend {input:?}" + ); + } + + for (vout, _) in tx.output.iter().enumerate() { + let output = OutPoint { + txid: tx.compute_txid(), + vout: vout as u32, + }; + outputs.insert(output); } } - stale + } + + #[test] + fn test_gbt_first_transaction() { + // this test will recreate the network state on block 269, and then submit the famous + // first non-coinbase transaction to the mempool, and then create a block template + // builds a proof for it, and then consumes the block. After that, we'll have a network at + // block 270, with the transaction confirmed. + + let roots = [ + "69482b799cf46ed514b01ce0573730a89c537018636b8c52a8864d5968b917f3", + "53c92fa0792c9af1c19793b1149e7fe209c69b320ea054338f53f8fd8535f2e8", + "6096c8421c1f86a9caa26e972dccdb964e280164fb060a576d51f5844e259569", + "fd46029ebb0c19e2d468a9b24d20519c64ccc342e6a32b95c86a57489b6d2504", + ] + .into_iter() + .map(|x| BitcoinNodeHash::from_str(x).unwrap()) + .collect::>(); + + let acc = Pollard::from_roots(roots, 169); + let proof_hashes = [ + "8be90393e71aa65710270b51857b538458dabd7769d801d6bbcbabe32c317251", + "5ae3964e9cc3c9e188de778c5b5fb19eaa60bce98facf1e9e68b3c1257d08c00", + "2c8dbc0642bd41cd8625344f99ef6513e5e68c03e184fcd401bddce6eba97674", + "1a55f3d560fa01fb6114842e7b4d7a0b8461f399e646f1762e6edf4be055b4dd", + "d1d2e49bce194f31dc9f3ec1cb8df3b95e097633ef42fd3723e629c9bed85ae5", + "15aba691713052033954935777d8089f4ca6b0573c7ad89fe1d0d85bbbe21846", + "8f22055465f568fd2bf9d19b285fcf2539ffea59a3cb096a3a0645366adea1b0", + ] + .into_iter() + .map(|x| BitcoinNodeHash::from_str(x).unwrap()) + .collect::>(); + + let proof = Proof::new(vec![8], proof_hashes); + let del_hashes = ["427aceafd82c11cb53a2b78f408ece6fcacf2a5b9feb5fc45cdcf36627d68d76"] + .into_iter() + .map(|x| BitcoinNodeHash::from_str(x).unwrap()) + .collect::>(); + + let prevout: LeafData = deserialize_hex("0508085c47cc849eb80ea905cc7800a3be674ffc57263cf210c59d8d00000000c997a5e56e104102fa209c6a852dd90660a20b2d9c352423edce25857fcd3704000000001300000000f2052a0100000043410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac").unwrap(); + + let mut mempool = Mempool::new(acc, 10_000_000); + + let tx_hex = "0100000001c997a5e56e104102fa209c6a852dd90660a20b2d9c352423edce25857fcd3704000000004847304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901ffffffff0200ca9a3b00000000434104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84cac00286bee0000000043410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac00000000"; + let tx: Transaction = deserialize_hex(tx_hex).unwrap(); + + let compact_leaf = CompactLeafData { + amount: prevout.utxo.value.to_sat(), + spk_ty: proof_util::get_script_type(&prevout.utxo.script_pubkey), + header_code: (9 << 1) | 1_u32, + }; + + let targets = proof.targets.clone(); + mempool + .accept_to_mempool( + tx.clone(), + proof, + &[(prevout.prevout, compact_leaf.clone())], + &del_hashes, + &targets, + ) + .expect("failed to accept to mempool"); + + let block = mempool.get_block_template( + block::Version::ONE, + bitcoin::BlockHash::from_str( + "000000002a22cfee1f2c846adbd12b3e183d4f97683f85dad08a79780a84bd55", + ) + .unwrap(), + 1231731025, + Target::MAX_ATTAINABLE_MAINNET.to_compact_lossy(), + ); + + let MempoolProof { + proof, + target_hashes, + .. + } = mempool + .get_block_proof( + &block, + BlockHashProvider { + block_hash: [( + 9, + bitcoin::BlockHash::from_str( + "000000008d9dc510f23c2657fc4f67bea30078cc05a90eb89e84cc475c080805", + ) + .unwrap(), + )] + .iter() + .cloned() + .collect(), + }, + ) + .expect("failed to get block proof"); + + let adds = tx + .output + .iter() + .enumerate() + .map(|(vout, output)| { + let leaf_data = LeafData { + prevout: OutPoint { + txid: tx.compute_txid(), + vout: vout as u32, + }, + utxo: output.clone(), + block_hash: block.block_hash(), + header_code: 170 << 1, + }; + + PollardAddition { + hash: leaf_data._get_leaf_hashes().into(), + remember: true, + } + }) + .collect::>(); + + assert!(mempool.acc.verify(&proof, &del_hashes).is_ok()); + mempool + .consume_block(&block, proof, &adds, &target_hashes, 170, true) + .expect("failed to consume block"); + } + + #[test] + fn test_gbt() { + let mut mempool = super::Mempool::new( + rustreexo::accumulator::pollard::Pollard::default(), + 10_000_000, + ); + + let transactions = build_transactions(42, false); + let len = transactions.len(); + + for tx in transactions { + mempool + .accept_to_mempool_no_acc(tx) + .expect("failed to accept to mempool"); + } + + let target = Target::MAX_ATTAINABLE_REGTEST; + let block = mempool.get_block_template( + block::Version::ONE, + bitcoin::BlockHash::all_zeros(), + 0, + target.to_compact_lossy(), + ); + + assert_eq!(block.txdata.len(), len); + assert!(block.check_merkle_root()); + + check_block_transactions(block); } } diff --git a/crates/floresta-wire/src/p2p_wire/node.rs b/crates/floresta-wire/src/p2p_wire/node.rs index 1f161485..4dad1733 100644 --- a/crates/floresta-wire/src/p2p_wire/node.rs +++ b/crates/floresta-wire/src/p2p_wire/node.rs @@ -9,7 +9,6 @@ use std::net::SocketAddr; use std::ops::Deref; use std::ops::DerefMut; use std::sync::Arc; -use std::sync::Mutex; use std::time::Duration; use std::time::Instant; use std::time::SystemTime; @@ -39,7 +38,7 @@ use tokio::spawn; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::mpsc::UnboundedSender; -use tokio::sync::RwLock; +use tokio::sync::Mutex; use tokio::time::timeout; use super::address_man::AddressMan; @@ -48,6 +47,7 @@ use super::address_man::LocalAddress; use super::error::AddrParseError; use super::error::WireError; use super::mempool::Mempool; +use super::mempool::MempoolProof; use super::node_context::NodeContext; use super::node_interface::NodeInterface; use super::node_interface::PeerInfo; @@ -134,7 +134,7 @@ impl Default for RunningNode { last_feeler: Instant::now(), last_address_rearrange: Instant::now(), user_requests: Arc::new(NodeInterface { - requests: Mutex::new(Vec::new()), + requests: std::sync::Mutex::new(Vec::new()), }), last_invs: HashMap::default(), inflight_filters: BTreeMap::new(), @@ -146,7 +146,7 @@ pub struct NodeCommon { // 1. Core Blockchain and Transient Data pub(crate) chain: Chain, pub(crate) blocks: HashMap, - pub(crate) mempool: Arc>, + pub(crate) mempool: Arc>, pub(crate) block_filters: Option>>, pub(crate) last_filter: BlockHash, @@ -217,7 +217,7 @@ where pub fn new( config: UtreexoNodeConfig, chain: Chain, - mempool: Arc>, + mempool: Arc>, block_filters: Option>>, ) -> Result { let (node_tx, node_rx) = unbounded_channel(); @@ -735,16 +735,54 @@ where for transaction in transactions { let txid = transaction.compute_txid(); - self.mempool.write().await.accept_to_mempool(transaction); - peer.channel - .send(NodeRequest::BroadcastTransaction(txid)) - .map_err(WireError::ChannelSend)?; - } - let stale = self.mempool.write().await.get_stale(); - for tx in stale { - peer.channel - .send(NodeRequest::BroadcastTransaction(tx)) - .map_err(WireError::ChannelSend)?; + let mut mempool = self.mempool.lock().await; + + if self.network == Network::Regtest { + match mempool.try_prove(&transaction, &self.chain) { + Ok(proof) => { + let MempoolProof { + proof, + target_hashes, + leaves, + } = proof; + + let leaves = transaction + .input + .iter() + .cloned() + .map(|input| input.previous_output) + .zip(leaves.into_iter()) + .collect::>(); + + let targets = proof.targets.clone(); + try_and_log!(mempool.accept_to_mempool( + transaction, + proof, + &leaves, + &target_hashes, + &targets, + )); + } + Err(e) => { + log::error!( + "Could not prove tx {} because: {:?}", + transaction.compute_txid(), + e + ); + } + } + + peer.channel + .send(NodeRequest::BroadcastTransaction(txid)) + .map_err(WireError::ChannelSend)?; + } + + let stale = self.mempool.lock().await.get_stale(); + for tx in stale { + peer.channel + .send(NodeRequest::BroadcastTransaction(tx)) + .map_err(WireError::ChannelSend)?; + } } } Ok(()) @@ -906,7 +944,7 @@ where address: LocalAddress, requests_rx: UnboundedReceiver, peer_id_count: u32, - mempool: Arc>, + mempool: Arc>, network: bitcoin::Network, node_tx: UnboundedSender, user_agent: String, @@ -949,7 +987,7 @@ where pub(crate) async fn open_proxy_connection( proxy: SocketAddr, kind: ConnectionKind, - mempool: Arc>, + mempool: Arc>, network: bitcoin::Network, node_tx: UnboundedSender, peer_id: usize, diff --git a/crates/floresta-wire/src/p2p_wire/peer.rs b/crates/floresta-wire/src/p2p_wire/peer.rs index e1b776cf..a9f3958d 100644 --- a/crates/floresta-wire/src/p2p_wire/peer.rs +++ b/crates/floresta-wire/src/p2p_wire/peer.rs @@ -36,7 +36,7 @@ use tokio::spawn; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::mpsc::UnboundedSender; -use tokio::sync::RwLock; +use tokio::sync::Mutex; use self::peer_utils::make_pong; use super::mempool::Mempool; @@ -148,7 +148,7 @@ pub fn create_tcp_stream_actor( } pub struct Peer { - mempool: Arc>, + mempool: Arc>, network: Network, blocks_only: bool, services: ServiceFlags, @@ -582,13 +582,13 @@ impl Peer { pub async fn handle_get_data(&mut self, inv: Inventory) -> Result<()> { match inv { Inventory::WitnessTransaction(txid) => { - let tx = self.mempool.read().await.get_from_mempool(&txid).cloned(); + let tx = self.mempool.lock().await.get_from_mempool(&txid).cloned(); if let Some(tx) = tx { self.write(NetworkMessage::Tx(tx)).await?; } } Inventory::Transaction(txid) => { - let tx = self.mempool.read().await.get_from_mempool(&txid).cloned(); + let tx = self.mempool.lock().await.get_from_mempool(&txid).cloned(); if let Some(tx) = tx { self.write(NetworkMessage::Tx(tx)).await?; } @@ -601,7 +601,7 @@ impl Peer { #[allow(clippy::too_many_arguments)] pub async fn create_peer( id: u32, - mempool: Arc>, + mempool: Arc>, network: Network, node_tx: UnboundedSender, node_requests: UnboundedReceiver, diff --git a/crates/floresta-wire/src/p2p_wire/running_node.rs b/crates/floresta-wire/src/p2p_wire/running_node.rs index dc7e0b88..d0c536b7 100644 --- a/crates/floresta-wire/src/p2p_wire/running_node.rs +++ b/crates/floresta-wire/src/p2p_wire/running_node.rs @@ -9,6 +9,7 @@ use std::time::Duration; use std::time::Instant; use bitcoin::bip158::BlockFilter; +use bitcoin::hashes::Hash; use bitcoin::p2p::address::AddrV2; use bitcoin::p2p::address::AddrV2Message; use bitcoin::p2p::message_blockdata::Inventory; @@ -25,6 +26,8 @@ use log::debug; use log::error; use log::info; use log::warn; +use rustreexo::accumulator::node_hash::BitcoinNodeHash; +use rustreexo::accumulator::pollard::PollardAddition; use rustreexo::accumulator::stump::Stump; use tokio::sync::RwLock; use tokio::time::timeout; @@ -666,9 +669,10 @@ where let (proof, del_hashes, inputs) = floresta_chain::proof_util::process_proof(udata, &block.block.txdata, &self.chain)?; - if let Err(e) = self - .chain - .connect_block(&block.block, proof, inputs, del_hashes) + + if let Err(e) = + self.chain + .connect_block(&block.block, proof.clone(), inputs, del_hashes.clone()) { error!("Invalid block received by peer {} reason: {:?}", peer, e); if let BlockchainError::BlockValidation(e) = e { @@ -712,7 +716,39 @@ where } if !self.chain.is_in_idb() { - let mempool_delta = self.mempool.write().await.consume_block(&block.block); + let del_hashes: Vec = del_hashes + .iter() + .map(|hash| BitcoinNodeHash::from(hash.as_byte_array())) + .collect(); + + let block_height = self + .chain + .get_block_height(&block.block.block_hash())? + .unwrap(); + + let block_hash = block.block.block_hash(); + + let adds = floresta_chain::proof_util::get_block_adds( + &block.block, + block_height, + block_hash, + ); + + let adds: Vec> = adds + .into_iter() + .map(|add| PollardAddition { + remember: true, + hash: add, + }) + .collect(); + + let mempool_delta = self + .mempool + .lock() + .await + .consume_block(&block.block, proof, &adds, &del_hashes, block_height, false) + .unwrap_or(Vec::new()); + debug!( "Block {} accepted, confirmed transactions: {:?}", block.block.block_hash(), @@ -726,6 +762,7 @@ where Ok(_next_block) => next_block = _next_block, Err(_) => break, } + debug!("accepted block {}", block.block.block_hash()); } diff --git a/crates/floresta-wire/src/p2p_wire/tests/sync_node.rs b/crates/floresta-wire/src/p2p_wire/tests/sync_node.rs index 9b706709..550c4357 100644 --- a/crates/floresta-wire/src/p2p_wire/tests/sync_node.rs +++ b/crates/floresta-wire/src/p2p_wire/tests/sync_node.rs @@ -12,6 +12,8 @@ mod tests_utils { use floresta_chain::ChainState; use floresta_chain::KvChainStore; use floresta_chain::UtreexoBlock; + use rustreexo::accumulator::pollard::Pollard; + use tokio::sync::Mutex; use tokio::sync::RwLock; use tokio::time::timeout; @@ -33,7 +35,7 @@ mod tests_utils { ) -> Arc>> { let datadir = format!("./tmp-db/{}.sync_node", rand::random::()); let chainstore = KvChainStore::new(datadir.clone()).unwrap(); - let mempool = Arc::new(RwLock::new(Mempool::new())); + let mempool = Arc::new(Mutex::new(Mempool::new(Pollard::default(), 1000))); let chain = ChainState::new(chainstore, network, AssumeValidArg::Disabled); let chain = Arc::new(chain); diff --git a/crates/floresta/Cargo.toml b/crates/floresta/Cargo.toml index 747b4a07..b315b133 100644 --- a/crates/floresta/Cargo.toml +++ b/crates/floresta/Cargo.toml @@ -30,7 +30,7 @@ floresta-watch-only = { path = "../floresta-watch-only", optional = true, versio floresta-electrum = { path = "../floresta-electrum", optional = true, version = "0.3.0" } [dev-dependencies] -rustreexo = "0.3.0" +rustreexo = "0.4" miniscript = "11" tokio = { version = "1", features = ["full"] } bitcoin = { version = "0.32", features = [ diff --git a/crates/floresta/examples/node.rs b/crates/floresta/examples/node.rs index 8ae53426..00fb86db 100644 --- a/crates/floresta/examples/node.rs +++ b/crates/floresta/examples/node.rs @@ -19,6 +19,8 @@ use floresta_chain::AssumeValidArg; use floresta_wire::node_interface::NodeMethods; use floresta_wire::running_node::RunningNode; use floresta_wire::UtreexoNodeConfig; +use rustreexo::accumulator::pollard::Pollard; +use tokio::sync::Mutex; use tokio::sync::RwLock; const DATA_DIR: &str = "./tmp-db"; @@ -60,7 +62,7 @@ async fn main() { let p2p: UtreexoNode>, RunningNode> = UtreexoNode::new( config, chain.clone(), - Arc::new(RwLock::new(Mempool::new())), + Arc::new(Mutex::new(Mempool::new(Pollard::default(), 1000))), None, ) .unwrap(); diff --git a/florestad/Cargo.toml b/florestad/Cargo.toml index 8db78b7a..7ca8482c 100644 --- a/florestad/Cargo.toml +++ b/florestad/Cargo.toml @@ -4,7 +4,7 @@ version = "0.7.0" edition = "2021" [dependencies] -rustreexo = "0.3.0" +rustreexo = "0.4" clap = { version = "4.0.29", features = ["derive"] } sha2 = "^0.10.6" tokio = { version = "1", features = ["full"] } diff --git a/florestad/src/florestad.rs b/florestad/src/florestad.rs index 0f73c581..7e43499c 100644 --- a/florestad/src/florestad.rs +++ b/florestad/src/florestad.rs @@ -45,6 +45,7 @@ use log::warn; use log::Record; #[cfg(feature = "metrics")] use metrics; +use rustreexo::accumulator::pollard::Pollard; use tokio::net::TcpListener; use tokio::sync::RwLock; use tokio::task; @@ -402,11 +403,12 @@ impl Florestad { user_agent: self.config.user_agent.clone(), }; + let acc = Pollard::new(); // Chain Provider (p2p) let chain_provider = UtreexoNode::new( config, blockchain_state.clone(), - Arc::new(tokio::sync::RwLock::new(Mempool::new())), + Arc::new(tokio::sync::Mutex::new(Mempool::new(acc, 300_000_000))), cfilters.clone(), ) .expect("Could not create a chain provider");