diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aefcb0d9a9..781b9e8f19 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -213,7 +213,7 @@ jobs: strategy: fail-fast: false matrix: - e2e-type: [cosmwasm, non-cosmwasm] + e2e-type: [cosmwasm, evm, sealevel] steps: - uses: actions/setup-node@v4 with: @@ -240,7 +240,7 @@ jobs: save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} workspaces: | ./rust/main - ${{ matrix.e2e-type == 'non-cosmwasm' && './rust/sealevel' || '' }} + ${{ matrix.e2e-type == 'sealevel' && './rust/sealevel' || '' }} - name: Free disk space run: | @@ -269,13 +269,6 @@ jobs: - name: Checkout registry uses: ./.github/actions/checkout-registry - - name: agent tests (CosmWasm) - run: cargo test --release --package run-locally --bin run-locally --features cosmos test-utils -- cosmos::test --nocapture - if: matrix.e2e-type == 'cosmwasm' - working-directory: ./rust/main - env: - RUST_BACKTRACE: 'full' - - name: Check for Rust file changes id: check-rust-changes run: | @@ -286,16 +279,31 @@ jobs: echo "rust_changes=false" >> $GITHUB_OUTPUT fi - - name: agent tests (EVM and Sealevel) + - name: agent tests (Sealevel) + run: cargo test --release --package run-locally --bin run-locally --features sealevel -- sealevel::test --nocapture + if: ${{ matrix.e2e-type == 'sealevel' && steps.check-rust-changes.outputs.rust_changes == 'true' }} + working-directory: ./rust/main + env: + E2E_CI_MODE: 'true' + E2E_CI_TIMEOUT_SEC: '600' + RUST_BACKTRACE: 'full' + + - name: agent tests (EVM) run: cargo run --release --bin run-locally --features test-utils - if: matrix.e2e-type == 'non-cosmwasm' + if: ${{ matrix.e2e-type == 'evm' }} working-directory: ./rust/main env: E2E_CI_MODE: 'true' E2E_CI_TIMEOUT_SEC: '600' E2E_KATHY_MESSAGES: '20' RUST_BACKTRACE: 'full' - SEALEVEL_ENABLED: ${{ steps.check-rust-changes.outputs.rust_changes }} + + - name: agent tests (CosmWasm) + run: cargo test --release --package run-locally --bin run-locally --features cosmos -- cosmos::test --nocapture + if: ${{ matrix.e2e-type == 'cosmwasm' }} + working-directory: ./rust/main + env: + RUST_BACKTRACE: 'full' env-test: runs-on: ubuntu-latest diff --git a/rust/README.md b/rust/README.md index 2e4577deac..658bec7ae8 100644 --- a/rust/README.md +++ b/rust/README.md @@ -112,10 +112,18 @@ validator. By default, this test will run indefinitely, but can be stopped with To run the tests for a specific VM, use the `--features` flag. +##### Cosmos E2E Test + ```bash cargo test --release --package run-locally --bin run-locally --features cosmos -- cosmos::test --nocapture ``` +##### Sealevel E2E Test + +```bash +cargo test --release --package run-locally --bin run-locally --features sealevel -- sealevel::test --nocapture +``` + ### Building Agent Docker Images There exists a docker build for the agent binaries. These docker images are used for deploying the agents in a diff --git a/rust/main/utils/run-locally/Cargo.toml b/rust/main/utils/run-locally/Cargo.toml index a994324687..5b2e1199cd 100644 --- a/rust/main/utils/run-locally/Cargo.toml +++ b/rust/main/utils/run-locally/Cargo.toml @@ -44,3 +44,4 @@ vergen = { version = "8.3.2", features = ["build", "git", "gitcl"] } [features] cosmos = [] +sealevel = [] diff --git a/rust/main/utils/run-locally/src/config.rs b/rust/main/utils/run-locally/src/config.rs index c03f859ba8..31eaae1613 100644 --- a/rust/main/utils/run-locally/src/config.rs +++ b/rust/main/utils/run-locally/src/config.rs @@ -7,7 +7,6 @@ pub struct Config { pub ci_mode: bool, pub ci_mode_timeout: u64, pub kathy_messages: u64, - pub sealevel_enabled: bool, // TODO: Include count of sealevel messages in a field separate from `kathy_messages`? } @@ -28,9 +27,6 @@ impl Config { .map(|r| r.parse::().unwrap()); r.unwrap_or(16) }, - sealevel_enabled: env::var("SEALEVEL_ENABLED") - .map(|k| k.parse::().unwrap()) - .unwrap_or(true), }) } } diff --git a/rust/main/utils/run-locally/src/cosmos/mod.rs b/rust/main/utils/run-locally/src/cosmos/mod.rs index d786853338..71e6c63bd0 100644 --- a/rust/main/utils/run-locally/src/cosmos/mod.rs +++ b/rust/main/utils/run-locally/src/cosmos/mod.rs @@ -30,7 +30,9 @@ use crate::cosmos::link::link_networks; use crate::logging::log; use crate::metrics::agent_balance_sum; use crate::program::Program; -use crate::utils::{as_task, concat_path, stop_child, AgentHandles, TaskHandle}; +use crate::utils::{ + as_task, concat_path, get_workspace_path, stop_child, AgentHandles, TaskHandle, +}; use crate::{fetch_metric, AGENT_BIN_PATH}; use cli::{OsmosisCLI, OsmosisEndpoint}; @@ -345,10 +347,12 @@ fn run_locally() { const TIMEOUT_SECS: u64 = 60 * 10; let debug = false; + let workspace_path = get_workspace_path(); + log!("Building rust..."); Program::new("cargo") .cmd("build") - .working_dir("../../") + .working_dir(&workspace_path) .arg("features", "test-utils") .arg("bin", "relayer") .arg("bin", "validator") @@ -529,7 +533,8 @@ fn run_locally() { // give things a chance to fully start. sleep(Duration::from_secs(10)); - let starting_relayer_balance: f64 = agent_balance_sum(hpl_rly_metrics_port).unwrap(); + let starting_relayer_balance: f64 = + agent_balance_sum(hpl_rly_metrics_port).expect("Failed to get relayer agent balance"); // dispatch the second batch of messages (after agents start) dispatched_messages += dispatch(&osmosisd, linker, &nodes); @@ -664,7 +669,8 @@ fn termination_invariants_met( return Ok(false); } - let ending_relayer_balance: f64 = agent_balance_sum(relayer_metrics_port).unwrap(); + let ending_relayer_balance: f64 = + agent_balance_sum(relayer_metrics_port).expect("Failed to get relayer agent balance"); // Make sure the balance was correctly updated in the metrics. // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here: diff --git a/rust/main/utils/run-locally/src/ethereum/mod.rs b/rust/main/utils/run-locally/src/ethereum/mod.rs index 8abed1f85c..ec82fb9207 100644 --- a/rust/main/utils/run-locally/src/ethereum/mod.rs +++ b/rust/main/utils/run-locally/src/ethereum/mod.rs @@ -10,15 +10,18 @@ use crate::config::Config; use crate::ethereum::multicall::{DEPLOYER_ADDRESS, SIGNED_DEPLOY_MULTICALL_TX}; use crate::logging::log; use crate::program::Program; -use crate::utils::{as_task, AgentHandles, TaskHandle}; -use crate::{INFRA_PATH, MONOREPO_ROOT_PATH}; +use crate::utils::{as_task, get_ts_infra_path, get_workspace_path, AgentHandles, TaskHandle}; mod multicall; #[apply(as_task)] pub fn start_anvil(config: Arc) -> AgentHandles { log!("Installing typescript dependencies..."); - let yarn_monorepo = Program::new("yarn").working_dir(MONOREPO_ROOT_PATH); + + let workspace_path = get_workspace_path(); + let ts_infra_path = get_ts_infra_path(&workspace_path); + + let yarn_monorepo = Program::new("yarn").working_dir(workspace_path); if !config.is_ci_env { // test.yaml workflow installs dependencies yarn_monorepo.clone().cmd("install").run().join(); @@ -42,7 +45,7 @@ pub fn start_anvil(config: Arc) -> AgentHandles { sleep(Duration::from_secs(10)); - let yarn_infra = Program::new("yarn").working_dir(INFRA_PATH); + let yarn_infra = Program::new("yarn").working_dir(&ts_infra_path); log!("Deploying hyperlane ism contracts..."); yarn_infra.clone().cmd("deploy-ism").run().join(); diff --git a/rust/main/utils/run-locally/src/invariants/common.rs b/rust/main/utils/run-locally/src/invariants/common.rs deleted file mode 100644 index 1f603db53b..0000000000 --- a/rust/main/utils/run-locally/src/invariants/common.rs +++ /dev/null @@ -1,4 +0,0 @@ -// This number should be even, so the messages can be split into two equal halves -// sent before and after the relayer spins up, to avoid rounding errors. -pub const SOL_MESSAGES_EXPECTED: u32 = 20; -pub const SOL_MESSAGES_WITH_NON_MATCHING_IGP: u32 = 1; diff --git a/rust/main/utils/run-locally/src/invariants.rs b/rust/main/utils/run-locally/src/invariants/mod.rs similarity index 51% rename from rust/main/utils/run-locally/src/invariants.rs rename to rust/main/utils/run-locally/src/invariants/mod.rs index 13fb465b5f..69b63442e3 100644 --- a/rust/main/utils/run-locally/src/invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/mod.rs @@ -1,7 +1,5 @@ -pub use common::SOL_MESSAGES_EXPECTED; pub use post_startup_invariants::post_startup_invariants; -pub use termination_invariants::termination_invariants_met; +pub use termination_invariants::*; -mod common; mod post_startup_invariants; mod termination_invariants; diff --git a/rust/main/utils/run-locally/src/invariants/post_startup_invariants.rs b/rust/main/utils/run-locally/src/invariants/post_startup_invariants.rs index 2d7053f301..809a8f28cb 100644 --- a/rust/main/utils/run-locally/src/invariants/post_startup_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/post_startup_invariants.rs @@ -3,7 +3,7 @@ use std::io::BufReader; use hyperlane_base::AgentMetadata; -use crate::DynPath; +use crate::{log, DynPath}; pub fn post_startup_invariants(checkpoints_dirs: &[DynPath]) -> bool { post_startup_validator_metadata_written(checkpoints_dirs) @@ -17,6 +17,10 @@ fn post_startup_validator_metadata_written(checkpoints_dirs: &[DynPath]) -> bool .map(|path| metadata_file_check(expected_git_sha, path)) .any(|b| !b); + if failed_metadata { + log!("Error: Metadata git hash mismatch, maybe try `cargo clean` and try again"); + } + !failed_metadata } diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index 65546f6762..9ad92531e5 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -1,5 +1,4 @@ use std::fs::File; -use std::path::Path; use crate::config::Config; use crate::metrics::agent_balance_sum; @@ -7,9 +6,7 @@ use crate::utils::get_matching_lines; use maplit::hashmap; use relayer::GAS_EXPENDITURE_LOG_MESSAGE; -use crate::invariants::common::{SOL_MESSAGES_EXPECTED, SOL_MESSAGES_WITH_NON_MATCHING_IGP}; use crate::logging::log; -use crate::solana::solana_termination_invariants_met; use crate::{ fetch_metric, AGENT_LOGGING_DIR, RELAYER_METRICS_PORT, SCRAPER_METRICS_PORT, ZERO_MERKLE_INSERTION_KATHY_MESSAGES, @@ -21,24 +18,70 @@ use crate::{ pub fn termination_invariants_met( config: &Config, starting_relayer_balance: f64, - solana_cli_tools_path: Option<&Path>, - solana_config_path: Option<&Path>, ) -> eyre::Result { let eth_messages_expected = (config.kathy_messages / 2) as u32 * 2; - let sol_messages_expected = if config.sealevel_enabled { - SOL_MESSAGES_EXPECTED - } else { - 0 - }; - let sol_messages_with_non_matching_igp = if config.sealevel_enabled { - SOL_MESSAGES_WITH_NON_MATCHING_IGP - } else { - 0 - }; // this is total messages expected to be delivered - let total_messages_expected = eth_messages_expected + sol_messages_expected; - let total_messages_dispatched = total_messages_expected + sol_messages_with_non_matching_igp; + let total_messages_expected = eth_messages_expected; + + // Also ensure the counter is as expected (total number of messages), summed + // across all mailboxes. + let msg_processed_count = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_messages_processed_count", + &hashmap! {}, + )? + .iter() + .sum::(); + + let gas_payment_events_count = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "gas_payments"}, + )? + .iter() + .sum::(); + + if !relayer_termination_invariants_met( + config, + msg_processed_count, + gas_payment_events_count, + total_messages_expected, + eth_messages_expected, + ZERO_MERKLE_INSERTION_KATHY_MESSAGES, + total_messages_expected + (config.kathy_messages as u32 / 4) * 2, + )? { + return Ok(false); + } + + if !scraper_termination_invariants_met( + gas_payment_events_count, + total_messages_expected + ZERO_MERKLE_INSERTION_KATHY_MESSAGES, + total_messages_expected, + )? { + return Ok(false); + } + + if !relayer_balance_check(starting_relayer_balance)? { + return Ok(false); + } + + log!("Termination invariants have been meet"); + Ok(true) +} + +/// returns false if invariants are not met +/// returns true if invariants are met +pub fn relayer_termination_invariants_met( + config: &Config, + msg_processed_count: u32, + gas_payment_events_count: u32, + total_messages_expected: u32, + total_messages_dispatched: u32, + submitter_queue_length_expected: u32, + non_zero_sequence_count_expected: u32, +) -> eyre::Result { + log!("Checking relayer termination invariants"); let lengths = fetch_metric( RELAYER_METRICS_PORT, @@ -46,9 +89,7 @@ pub fn termination_invariants_met( &hashmap! {}, )?; assert!(!lengths.is_empty(), "Could not find queue length metric"); - if lengths.iter().sum::() - != ZERO_MERKLE_INSERTION_KATHY_MESSAGES + sol_messages_with_non_matching_igp - { + if lengths.iter().sum::() != submitter_queue_length_expected { log!( "Relayer queues contain more messages than the zero-merkle-insertion ones. Lengths: {:?}", lengths @@ -56,15 +97,6 @@ pub fn termination_invariants_met( return Ok(false); }; - // Also ensure the counter is as expected (total number of messages), summed - // across all mailboxes. - let msg_processed_count = fetch_metric( - RELAYER_METRICS_PORT, - "hyperlane_messages_processed_count", - &hashmap! {}, - )? - .iter() - .sum::(); if msg_processed_count != total_messages_expected { log!( "Relayer has {} processed messages, expected {}", @@ -74,14 +106,6 @@ pub fn termination_invariants_met( return Ok(false); } - let gas_payment_events_count = fetch_metric( - RELAYER_METRICS_PORT, - "hyperlane_contract_sync_stored_events", - &hashmap! {"data_type" => "gas_payments"}, - )? - .iter() - .sum::(); - let log_file_path = AGENT_LOGGING_DIR.join("RLY-output.log"); const STORING_NEW_MESSAGE_LOG_MESSAGE: &str = "Storing new message in db"; const LOOKING_FOR_EVENTS_LOG_MESSAGE: &str = "Looking for events in index range"; @@ -153,7 +177,7 @@ pub fn termination_invariants_met( config.kathy_messages ); assert!( - log_counts.get(&hyper_incoming_body_line_filter).is_none(), + !log_counts.contains_key(&hyper_incoming_body_line_filter), "Verbose logs not expected at the log level set in e2e" ); @@ -180,19 +204,19 @@ pub fn termination_invariants_met( merkle_tree_max_sequence.iter().filter(|&x| *x > 0).count() as u32; assert_eq!( merkle_tree_max_sequence.iter().sum::() + non_zero_sequence_count, - total_messages_expected - + sol_messages_with_non_matching_igp - + (config.kathy_messages as u32 / 4) * 2 + non_zero_sequence_count_expected, ); + Ok(true) +} - if let Some((solana_cli_tools_path, solana_config_path)) = - solana_cli_tools_path.zip(solana_config_path) - { - if !solana_termination_invariants_met(solana_cli_tools_path, solana_config_path) { - log!("Solana termination invariants not met"); - return Ok(false); - } - } +/// returns false if invariants are not met +/// returns true if invariants are met +pub fn scraper_termination_invariants_met( + gas_payment_events_count: u32, + total_messages_dispatched: u32, + delivered_messages_scraped_expected: u32, +) -> eyre::Result { + log!("Checking scraper termination invariants"); let dispatched_messages_scraped = fetch_metric( SCRAPER_METRICS_PORT, @@ -201,13 +225,11 @@ pub fn termination_invariants_met( )? .iter() .sum::(); - if dispatched_messages_scraped - != total_messages_dispatched + ZERO_MERKLE_INSERTION_KATHY_MESSAGES - { + if dispatched_messages_scraped != total_messages_dispatched { log!( "Scraper has scraped {} dispatched messages, expected {}", dispatched_messages_scraped, - total_messages_dispatched + ZERO_MERKLE_INSERTION_KATHY_MESSAGES, + total_messages_dispatched, ); return Ok(false); } @@ -235,16 +257,21 @@ pub fn termination_invariants_met( )? .iter() .sum::(); - if delivered_messages_scraped != total_messages_expected { + if delivered_messages_scraped != delivered_messages_scraped_expected { log!( "Scraper has scraped {} delivered messages, expected {}", delivered_messages_scraped, - total_messages_expected + sol_messages_with_non_matching_igp + delivered_messages_scraped_expected, ); return Ok(false); } - let ending_relayer_balance: f64 = agent_balance_sum(9092).unwrap(); + Ok(true) +} + +pub fn relayer_balance_check(starting_relayer_balance: f64) -> eyre::Result { + let ending_relayer_balance: f64 = + agent_balance_sum(9092).expect("Failed to get relayer agent balance"); // Make sure the balance was correctly updated in the metrics. if starting_relayer_balance <= ending_relayer_balance { log!( @@ -254,7 +281,5 @@ pub fn termination_invariants_met( ); return Ok(false); } - - log!("Termination invariants have been meet"); Ok(true) } diff --git a/rust/main/utils/run-locally/src/main.rs b/rust/main/utils/run-locally/src/main.rs index e19fa1564d..31085a6509 100644 --- a/rust/main/utils/run-locally/src/main.rs +++ b/rust/main/utils/run-locally/src/main.rs @@ -13,7 +13,6 @@ //! the end conditions are met, the test is a failure. Defaults to 10 min. //! - `E2E_KATHY_MESSAGES`: Number of kathy messages to dispatch. Defaults to 16 if CI mode is enabled. //! else false. -//! - `SEALEVEL_ENABLED`: true/false, enables sealevel testing. Defaults to true. use std::{ collections::HashMap, @@ -36,27 +35,31 @@ use program::Program; use relayer::msg::pending_message::RETRIEVED_MESSAGE_LOG; use tempfile::{tempdir, TempDir}; use utils::get_matching_lines; +use utils::{get_ts_infra_path, get_workspace_path}; use crate::{ config::Config, ethereum::start_anvil, - invariants::{post_startup_invariants, termination_invariants_met, SOL_MESSAGES_EXPECTED}, + invariants::{post_startup_invariants, termination_invariants_met}, metrics::agent_balance_sum, - solana::*, utils::{concat_path, make_static, stop_child, AgentHandles, ArbitraryData, TaskHandle}, }; mod config; -mod cosmos; mod ethereum; mod invariants; mod logging; mod metrics; mod program; mod server; -mod solana; mod utils; +#[cfg(feature = "cosmos")] +mod cosmos; + +#[cfg(feature = "sealevel")] +mod sealevel; + pub static AGENT_LOGGING_DIR: Lazy<&Path> = Lazy::new(|| { let dir = Path::new("/tmp/test_logs"); fs::create_dir_all(dir).unwrap(); @@ -71,10 +74,6 @@ const RELAYER_KEYS: &[&str] = &[ "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97", // test3 "0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356", - // sealeveltest1 - "0x892bf6949af4233e62f854cb3618bc1a3ee3341dc71ada08c4d5deca239acf4f", - // sealeveltest2 - "0x892bf6949af4233e62f854cb3618bc1a3ee3341dc71ada08c4d5deca239acf4f", ]; /// These private keys are from hardhat/anvil's testing accounts. /// These must be consistent with the ISM config for the test. @@ -85,15 +84,7 @@ const ETH_VALIDATOR_KEYS: &[&str] = &[ "0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e", ]; -const SEALEVEL_VALIDATOR_KEYS: &[&str] = &[ - // sealevel - "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", -]; - const AGENT_BIN_PATH: &str = "target/debug"; -const SOLANA_AGNET_BIN_PATH: &str = "../sealevel/target/debug/"; -const INFRA_PATH: &str = "../../typescript/infra"; -const MONOREPO_ROOT_PATH: &str = "../../"; const ZERO_MERKLE_INSERTION_KATHY_MESSAGES: u32 = 10; @@ -117,6 +108,7 @@ struct State { impl State { fn push_agent(&mut self, handles: AgentHandles) { + log!("Pushing {} agent handles", handles.0); self.agents.insert(handles.0, (handles.1, handles.5)); self.watchers.push(handles.2); self.watchers.push(handles.3); @@ -132,18 +124,29 @@ impl Drop for State { log!("Stopping child {}", name); stop_child(&mut agent); } - log!("Joining watchers..."); RUN_LOG_WATCHERS.store(false, Ordering::Relaxed); - for w in self.watchers.drain(..) { + + log!("Joining watchers..."); + let watchers_count = self.watchers.len(); + for (i, w) in self.watchers.drain(..).enumerate() { + log!("Joining {}/{}", i + 1, watchers_count); w.join_box(); } + + log!("Dropping data..."); // drop any held data self.data.reverse(); for data in self.data.drain(..) { drop(data) } - fs::remove_dir_all(SOLANA_CHECKPOINT_LOCATION).unwrap_or_default(); + #[cfg(feature = "sealevel")] + { + // use sealevel::solana::SOLANA_CHECKPOINT_LOCATION; + // fs::remove_dir_all(SOLANA_CHECKPOINT_LOCATION).unwrap_or_default(); + } fs::remove_dir_all::<&Path>(AGENT_LOGGING_DIR.as_ref()).unwrap_or_default(); + + log!("Done..."); } } @@ -158,21 +161,15 @@ fn main() -> ExitCode { let config = Config::load(); log!("Running with config: {:?}", config); - let mut validator_origin_chains = ["test1", "test2", "test3"].to_vec(); - let mut validator_keys = ETH_VALIDATOR_KEYS.to_vec(); - let mut validator_count: usize = validator_keys.len(); - let mut checkpoints_dirs: Vec = (0..validator_count) + let workspace_path = get_workspace_path(); + let ts_infra_path = get_ts_infra_path(&workspace_path); + + let validator_origin_chains = ["test1", "test2", "test3"].to_vec(); + let validator_keys = ETH_VALIDATOR_KEYS.to_vec(); + let validator_count: usize = validator_keys.len(); + let checkpoints_dirs: Vec = (0..validator_count) .map(|_| Box::new(tempdir().unwrap()) as DynPath) .collect(); - if config.sealevel_enabled { - validator_origin_chains.push("sealeveltest1"); - let mut sealevel_keys = SEALEVEL_VALIDATOR_KEYS.to_vec(); - validator_keys.append(&mut sealevel_keys); - let solana_checkpoint_path = Path::new(SOLANA_CHECKPOINT_LOCATION); - fs::remove_dir_all(solana_checkpoint_path).unwrap_or_default(); - checkpoints_dirs.push(Box::new(solana_checkpoint_path) as DynPath); - validator_count += 1; - } assert_eq!(validator_origin_chains.len(), validator_keys.len()); let rocks_db_dir = tempdir().unwrap(); @@ -182,7 +179,7 @@ fn main() -> ExitCode { .collect::>(); let common_agent_env = create_common_agent(); - let relayer_env = create_relayer(&config, &rocks_db_dir); + let relayer_env = create_relayer(&rocks_db_dir); let base_validator_env = common_agent_env .clone() @@ -231,15 +228,8 @@ fn main() -> ExitCode { .hyp_env( "DB", "postgresql://postgres:47221c18c610@localhost:5432/postgres", - ); - let scraper_env = if config.sealevel_enabled { - scraper_env.hyp_env( - "CHAINSTOSCRAPE", - "test1,test2,test3,sealeveltest1,sealeveltest2", ) - } else { - scraper_env.hyp_env("CHAINSTOSCRAPE", "test1,test2,test3") - }; + .hyp_env("CHAINSTOSCRAPE", "test1,test2,test3"); let mut state = State::default(); @@ -260,19 +250,6 @@ fn main() -> ExitCode { // Ready to run... // - let solana_paths = if config.sealevel_enabled { - let (solana_path, solana_path_tempdir) = install_solana_cli_tools( - SOLANA_CONTRACTS_CLI_RELEASE_URL.to_owned(), - SOLANA_CONTRACTS_CLI_VERSION.to_owned(), - ) - .join(); - state.data.push(Box::new(solana_path_tempdir)); - let solana_program_builder = build_solana_programs(solana_path.clone()); - Some((solana_program_builder.join(), solana_path)) - } else { - None - }; - // this task takes a long time in the CI so run it in parallel log!("Building rust..."); let build_main = Program::new("cargo") @@ -299,37 +276,6 @@ fn main() -> ExitCode { state.push_agent(postgres); build_main.join(); - if config.sealevel_enabled { - Program::new("cargo") - .working_dir("../sealevel") - .cmd("build") - .arg("bin", "hyperlane-sealevel-client") - .filter_logs(|l| !l.contains("workspace-inheritance")) - .run() - .join(); - } - - let solana_ledger_dir = tempdir().unwrap(); - let solana_config_path = if let Some((solana_program_path, _)) = solana_paths.clone() { - // use the agave 2.x validator version to ensure mainnet compatibility - let (solana_path, solana_path_tempdir) = install_solana_cli_tools( - SOLANA_NETWORK_CLI_RELEASE_URL.to_owned(), - SOLANA_NETWORK_CLI_VERSION.to_owned(), - ) - .join(); - state.data.push(Box::new(solana_path_tempdir)); - let start_solana_validator = start_solana_test_validator( - solana_path.clone(), - solana_program_path, - solana_ledger_dir.as_ref().to_path_buf(), - ); - - let (solana_config_path, solana_validator) = start_solana_validator.join(); - state.push_agent(solana_validator); - Some(solana_config_path) - } else { - None - }; state.push_agent(start_anvil.join()); @@ -346,14 +292,14 @@ fn main() -> ExitCode { // Send half the kathy messages before starting the rest of the agents let kathy_env_single_insertion = Program::new("yarn") - .working_dir(INFRA_PATH) + .working_dir(&ts_infra_path) .cmd("kathy") .arg("messages", (config.kathy_messages / 4).to_string()) .arg("timeout", "1000"); kathy_env_single_insertion.clone().run().join(); let kathy_env_zero_insertion = Program::new("yarn") - .working_dir(INFRA_PATH) + .working_dir(&ts_infra_path) .cmd("kathy") .arg( "messages", @@ -366,7 +312,7 @@ fn main() -> ExitCode { kathy_env_zero_insertion.clone().run().join(); let kathy_env_double_insertion = Program::new("yarn") - .working_dir(INFRA_PATH) + .working_dir(&ts_infra_path) .cmd("kathy") .arg("messages", (config.kathy_messages / 4).to_string()) .arg("timeout", "1000") @@ -375,16 +321,6 @@ fn main() -> ExitCode { .arg("required-hook", "merkleTreeHook"); kathy_env_double_insertion.clone().run().join(); - if let Some((solana_config_path, (_, solana_path))) = - solana_config_path.clone().zip(solana_paths.clone()) - { - // Send some sealevel messages before spinning up the agents, to test the backward indexing cursor - for _i in 0..(SOL_MESSAGES_EXPECTED / 2) { - initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone()) - .join(); - } - } - // spawn the rest of the validators for (i, validator_env) in validator_envs.into_iter().enumerate().skip(1) { let validator = validator_env.spawn( @@ -396,21 +332,6 @@ fn main() -> ExitCode { state.push_agent(relayer_env.spawn("RLY", Some(&AGENT_LOGGING_DIR))); - if let Some((solana_config_path, (_, solana_path))) = - solana_config_path.clone().zip(solana_paths.clone()) - { - // Send some sealevel messages before spinning up the agents, to test the backward indexing cursor - for _i in 0..(SOL_MESSAGES_EXPECTED / 2) { - initiate_solana_hyperlane_transfer(solana_path.clone(), solana_config_path.clone()) - .join(); - } - initiate_solana_non_matching_igp_paying_transfer( - solana_path.clone(), - solana_config_path.clone(), - ) - .join(); - } - log!("Setup complete! Agents running in background..."); log!("Ctrl+C to end execution..."); @@ -440,17 +361,7 @@ fn main() -> ExitCode { let mut test_passed = wait_for_condition( &config, loop_start, - || { - termination_invariants_met( - &config, - starting_relayer_balance, - solana_paths - .clone() - .map(|(_, solana_path)| solana_path) - .as_deref(), - solana_config_path.as_deref(), - ) - }, + || termination_invariants_met(&config, starting_relayer_balance), || !SHUTDOWN.load(Ordering::Relaxed), || long_running_processes_exited_check(&mut state), ); @@ -462,7 +373,7 @@ fn main() -> ExitCode { // Here we want to restart the relayer and validate // its restart behaviour. - restart_relayer(&config, &mut state, &rocks_db_dir); + restart_relayer(&mut state, &rocks_db_dir); // give relayer a chance to fully restart. sleep(Duration::from_secs(20)); @@ -494,14 +405,14 @@ fn create_common_agent() -> Program { .hyp_env("CHAINS_TEST3_INDEX_CHUNK", "1") } -fn create_relayer(config: &Config, rocks_db_dir: &TempDir) -> Program { +fn create_relayer(rocks_db_dir: &TempDir) -> Program { let relayer_db = concat_path(rocks_db_dir, "relayer"); let common_agent_env = create_common_agent(); let multicall_address_string: String = format!("0x{}", hex::encode(MULTICALL_ADDRESS)); - let relayer_env = common_agent_env + common_agent_env .clone() .bin(concat_path(AGENT_BIN_PATH, "relayer")) .hyp_env("CHAINS_TEST1_RPCCONSENSUSTYPE", "fallback") @@ -532,8 +443,6 @@ fn create_relayer(config: &Config, rocks_db_dir: &TempDir) -> Program { .hyp_env("DB", relayer_db.to_str().unwrap()) .hyp_env("CHAINS_TEST1_SIGNER_KEY", RELAYER_KEYS[0]) .hyp_env("CHAINS_TEST2_SIGNER_KEY", RELAYER_KEYS[1]) - .hyp_env("CHAINS_SEALEVELTEST1_SIGNER_KEY", RELAYER_KEYS[3]) - .hyp_env("CHAINS_SEALEVELTEST2_SIGNER_KEY", RELAYER_KEYS[4]) .hyp_env("RELAYCHAINS", "invalidchain,otherinvalid") .hyp_env("ALLOWLOCALCHECKPOINTSYNCERS", "true") .hyp_env( @@ -548,25 +457,18 @@ fn create_relayer(config: &Config, rocks_db_dir: &TempDir) -> Program { "http://127.0.0.1:8545,http://127.0.0.1:8545,http://127.0.0.1:8545", ) // default is used for TEST3 - .arg("defaultSigner.key", RELAYER_KEYS[2]); - if config.sealevel_enabled { - relayer_env.arg( - "relayChains", - "test1,test2,test3,sealeveltest1,sealeveltest2", - ) - } else { - relayer_env.arg("relayChains", "test1,test2,test3") - } + .arg("defaultSigner.key", RELAYER_KEYS[2]) + .arg("relayChains", "test1,test2,test3") } /// Kills relayer in State and respawns the relayer again -fn restart_relayer(config: &Config, state: &mut State, rocks_db_dir: &TempDir) { +fn restart_relayer(state: &mut State, rocks_db_dir: &TempDir) { log!("Stopping relayer..."); let (child, _) = state.agents.get_mut("RLY").expect("No relayer agent found"); child.kill().expect("Failed to stop relayer"); log!("Restarting relayer..."); - let relayer_env = create_relayer(config, rocks_db_dir); + let relayer_env = create_relayer(rocks_db_dir); state.push_agent(relayer_env.spawn("RLY", Some(&AGENT_LOGGING_DIR))); log!("Restarted relayer..."); } @@ -606,7 +508,7 @@ fn relayer_restart_invariants_met() -> eyre::Result { Ok(true) } -fn wait_for_condition( +pub fn wait_for_condition( config: &Config, start_time: Instant, condition_fn: F1, @@ -630,7 +532,7 @@ where } if check_ci_timed_out(config.ci_mode_timeout, start_time) { // we ran out of time - log!("CI timeout reached before invariants were met"); + log!("Error: CI timeout reached before invariants were met"); return false; } if shutdown_criteria_fn() { @@ -663,7 +565,7 @@ fn long_running_processes_exited_check(state: &mut State) -> bool { false } -fn report_test_result(passed: bool) -> ExitCode { +pub fn report_test_result(passed: bool) -> ExitCode { if passed { log!("E2E tests passed"); ExitCode::SUCCESS diff --git a/rust/main/utils/run-locally/src/program.rs b/rust/main/utils/run-locally/src/program.rs index 3775ef8e99..c59a4c813f 100644 --- a/rust/main/utils/run-locally/src/program.rs +++ b/rust/main/utils/run-locally/src/program.rs @@ -130,6 +130,7 @@ impl Program { } /// Assumes an arg in the format of `--$ARG1 $ARG2 $ARG3`, args should exclude quoting, equal sign, and the leading hyphens. + #[allow(dead_code)] pub fn arg3( self, arg1: impl AsRef, @@ -171,6 +172,7 @@ impl Program { /// Remember some arbitrary data until either this program args goes out of scope or until the /// agent/child process exits. This is useful for preventing something from dropping. + #[allow(dead_code)] pub fn remember(mut self, data: impl ArbitraryData) -> Self { self.arbitrary_data.push(Arc::new(data)); self @@ -183,6 +185,9 @@ impl Program { .unwrap(), ); if let Some(wd) = &self.working_dir { + if !wd.exists() { + panic!("Working directory does not exist: {:?}", wd.as_path()); + } cmd.current_dir(wd.as_path()); } for (k, v) in self.env.iter() { diff --git a/rust/main/utils/run-locally/src/sealevel/mod.rs b/rust/main/utils/run-locally/src/sealevel/mod.rs new file mode 100644 index 0000000000..6f0553f438 --- /dev/null +++ b/rust/main/utils/run-locally/src/sealevel/mod.rs @@ -0,0 +1,328 @@ +pub mod solana; +pub mod termination_invariant; + +use std::{ + fs, + path::Path, + sync::atomic::Ordering, + thread::sleep, + time::{Duration, Instant}, +}; + +use tempfile::tempdir; + +use crate::SHUTDOWN; +use crate::{ + config::Config, + invariants::post_startup_invariants, + logging::log, + long_running_processes_exited_check, + metrics::agent_balance_sum, + program::Program, + sealevel::{solana::*, termination_invariant::*}, + utils::{ + concat_path, get_sealevel_path, get_ts_infra_path, get_workspace_path, make_static, + TaskHandle, + }, + wait_for_condition, State, AGENT_LOGGING_DIR, RELAYER_METRICS_PORT, SCRAPER_METRICS_PORT, +}; + +// This number should be even, so the messages can be split into two equal halves +// sent before and after the relayer spins up, to avoid rounding errors. +pub const SOL_MESSAGES_EXPECTED: u32 = 20; +pub const SOL_MESSAGES_WITH_NON_MATCHING_IGP: u32 = 1; + +/// These private keys are from hardhat/anvil's testing accounts. +const RELAYER_KEYS: &[&str] = &[ + // sealeveltest1 + "0x892bf6949af4233e62f854cb3618bc1a3ee3341dc71ada08c4d5deca239acf4f", + // sealeveltest2 + "0x892bf6949af4233e62f854cb3618bc1a3ee3341dc71ada08c4d5deca239acf4f", +]; + +const SEALEVEL_VALIDATOR_KEYS: &[&str] = &[ + // sealevel + "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", +]; + +type DynPath = Box>; + +#[allow(dead_code)] +fn run_locally() { + // on sigint we want to trigger things to stop running + ctrlc::set_handler(|| { + log!("Terminating..."); + SHUTDOWN.store(true, Ordering::Relaxed); + }) + .unwrap(); + + let config = Config::load(); + log!("Running with config: {:?}", config); + + let workspace_path = get_workspace_path(); + let sealevel_path = get_sealevel_path(&workspace_path); + let ts_infra_path = get_ts_infra_path(&workspace_path); + log!( + "Paths:\n{:?}\n{:?}\n{:?}", + workspace_path, + sealevel_path, + ts_infra_path + ); + + let validator_origin_chains = ["sealeveltest1"].to_vec(); + let validator_keys = SEALEVEL_VALIDATOR_KEYS.to_vec(); + let validator_count: usize = validator_keys.len(); + + let solana_checkpoint_path = Path::new(SOLANA_CHECKPOINT_LOCATION); + fs::remove_dir_all(solana_checkpoint_path).unwrap_or_default(); + let checkpoints_dirs: Vec = vec![Box::new(solana_checkpoint_path) as DynPath]; + + assert_eq!(validator_origin_chains.len(), validator_keys.len()); + + let rocks_db_dir = tempdir().expect("Failed to create tempdir for rocksdb"); + let relayer_db = concat_path(&rocks_db_dir, "relayer"); + let validator_dbs = (0..validator_count) + .map(|i| concat_path(&rocks_db_dir, format!("validator{i}"))) + .collect::>(); + + let common_agent_env = Program::default() + .env("RUST_BACKTRACE", "full") + .hyp_env("LOG_FORMAT", "compact") + .hyp_env("LOG_LEVEL", "debug"); + + let relayer_env = common_agent_env + .clone() + .bin(concat_path(&workspace_path, "target/debug/relayer")) + .working_dir(&workspace_path) + .hyp_env("METRICSPORT", RELAYER_METRICS_PORT) + .hyp_env("DB", relayer_db.to_str().unwrap()) + .hyp_env("CHAINS_SEALEVELTEST1_SIGNER_KEY", RELAYER_KEYS[0]) + .hyp_env("CHAINS_SEALEVELTEST2_SIGNER_KEY", RELAYER_KEYS[1]) + .hyp_env("RELAYCHAINS", "invalidchain,otherinvalid") + .hyp_env("ALLOWLOCALCHECKPOINTSYNCERS", "true") + .hyp_env( + "GASPAYMENTENFORCEMENT", + r#"[{ + "type": "minimum", + "payment": "1" + }]"#, + ) + .arg("defaultSigner.key", RELAYER_KEYS[0]) + .arg("relayChains", "sealeveltest1,sealeveltest2"); + + let base_validator_env = common_agent_env + .clone() + .bin(concat_path(&workspace_path, "target/debug/validator")) + .working_dir(&workspace_path) + .hyp_env("INTERVAL", "5") + .hyp_env("CHECKPOINTSYNCER_TYPE", "localStorage"); + + let validator_envs = (0..validator_count) + .map(|i| { + base_validator_env + .clone() + .hyp_env("METRICSPORT", (9094 + i).to_string()) + .hyp_env("DB", validator_dbs[i].to_str().unwrap()) + .hyp_env("ORIGINCHAINNAME", validator_origin_chains[i]) + .hyp_env("VALIDATOR_KEY", validator_keys[i]) + .hyp_env( + "CHECKPOINTSYNCER_PATH", + (*checkpoints_dirs[i]).as_ref().to_str().unwrap(), + ) + }) + .collect::>(); + + log!("Relayer DB in {}", relayer_db.display()); + (0..validator_count).for_each(|i| { + log!("Validator {} DB in {}", i + 1, validator_dbs[i].display()); + }); + + let scraper_env = common_agent_env + .bin(concat_path(&workspace_path, "target/debug/scraper")) + .working_dir(&workspace_path) + .hyp_env("METRICSPORT", SCRAPER_METRICS_PORT) + .hyp_env( + "DB", + "postgresql://postgres:47221c18c610@localhost:5432/postgres", + ) + .hyp_env("CHAINSTOSCRAPE", "sealeveltest1,sealeveltest2"); + + let mut state = State::default(); + + log!( + "Signed checkpoints in {}", + checkpoints_dirs + .iter() + .map(|d| (**d).as_ref().display().to_string()) + .collect::>() + .join(", ") + ); + + // + // Ready to run... + // + + let (solana_programs_path, hyperlane_solana_programs_path) = { + let solana_path_tempdir = tempdir().expect("Failed to create solana temp dir"); + let solana_bin_path = install_solana_cli_tools( + SOLANA_CONTRACTS_CLI_RELEASE_URL.to_owned(), + SOLANA_CONTRACTS_CLI_VERSION.to_owned(), + solana_path_tempdir.path().to_path_buf(), + ) + .join(); + state.data.push(Box::new(solana_path_tempdir)); + + let solana_program_builder = build_solana_programs(solana_bin_path.clone()); + (solana_bin_path, solana_program_builder.join()) + }; + + // this task takes a long time in the CI so run it in parallel + log!("Building rust..."); + let build_main = Program::new("cargo") + .cmd("build") + .working_dir(&workspace_path) + .arg("features", "test-utils") + .arg("bin", "relayer") + .arg("bin", "validator") + .arg("bin", "scraper") + .arg("bin", "init-db") + .filter_logs(|l| !l.contains("workspace-inheritance")) + .run(); + + log!("Running postgres db..."); + let postgres = Program::new("docker") + .cmd("run") + .flag("rm") + .arg("name", "scraper-testnet-postgres") + .arg("env", "POSTGRES_PASSWORD=47221c18c610") + .arg("publish", "5432:5432") + .cmd("postgres:14") + .spawn("SQL", None); + state.push_agent(postgres); + + build_main.join(); + + log!("Building hyperlane-sealevel-client..."); + Program::new("cargo") + .working_dir(&sealevel_path) + .cmd("build") + .arg("bin", "hyperlane-sealevel-client") + .filter_logs(|l| !l.contains("workspace-inheritance")) + .run() + .join(); + + let solana_ledger_dir = tempdir().expect("Failed to create solana ledger dir"); + let (solana_cli_tools_path, solana_config_path) = { + // use the agave 2.x validator version to ensure mainnet compatibility + let solana_tools_dir = tempdir().expect("Failed to create solana tools dir"); + let solana_bin_path = install_solana_cli_tools( + SOLANA_NETWORK_CLI_RELEASE_URL.to_owned(), + SOLANA_NETWORK_CLI_VERSION.to_owned(), + solana_tools_dir.path().to_path_buf(), + ) + .join(); + state.data.push(Box::new(solana_tools_dir)); + + let start_solana_validator = start_solana_test_validator( + solana_bin_path.clone(), + hyperlane_solana_programs_path.clone(), + solana_ledger_dir.as_ref().to_path_buf(), + ); + + let (solana_config_path, solana_validator) = start_solana_validator.join(); + state.push_agent(solana_validator); + (solana_bin_path, solana_config_path) + }; + + sleep(Duration::from_secs(5)); + + log!("Init postgres db..."); + Program::new(concat_path(&workspace_path, "target/debug/init-db")) + .working_dir(&workspace_path) + .run() + .join(); + state.push_agent(scraper_env.spawn("SCR", None)); + + // spawn the rest of the validators + for (i, validator_env) in validator_envs.into_iter().enumerate() { + let validator = validator_env.spawn( + make_static(format!("VL{}", 1 + i)), + Some(AGENT_LOGGING_DIR.as_ref()), + ); + state.push_agent(validator); + } + + // Send some sealevel messages before spinning up the agents, to test the backward indexing cursor + for _i in 0..(SOL_MESSAGES_EXPECTED / 2) { + initiate_solana_hyperlane_transfer( + solana_cli_tools_path.clone(), + solana_config_path.clone(), + ) + .join(); + } + + state.push_agent(relayer_env.spawn("RLY", Some(&AGENT_LOGGING_DIR))); + + // Send some sealevel messages before spinning up the agents, to test the backward indexing cursor + for _i in 0..(SOL_MESSAGES_EXPECTED / 2) { + initiate_solana_hyperlane_transfer( + solana_cli_tools_path.clone(), + solana_config_path.clone(), + ) + .join(); + } + + initiate_solana_non_matching_igp_paying_transfer( + solana_cli_tools_path.clone(), + solana_config_path.clone(), + ) + .join(); + + log!("Setup complete! Agents running in background..."); + log!("Ctrl+C to end execution..."); + + let loop_start = Instant::now(); + // give things a chance to fully start. + sleep(Duration::from_secs(10)); + + if !post_startup_invariants(&checkpoints_dirs) { + panic!("Failure: Post startup invariants are not met"); + } else { + log!("Success: Post startup invariants are met"); + } + + let starting_relayer_balance: f64 = agent_balance_sum(9092).unwrap(); + + // wait for CI invariants to pass + let test_passed = wait_for_condition( + &config, + loop_start, + || { + termination_invariants_met( + &config, + starting_relayer_balance, + &solana_programs_path, + &solana_config_path, + ) + }, + || !SHUTDOWN.load(Ordering::Relaxed), + || long_running_processes_exited_check(&mut state), + ); + + if !test_passed { + panic!("Failure occurred during E2E"); + } + log!("E2E tests passed"); +} + +#[cfg(test)] +#[cfg(feature = "sealevel")] +mod test { + + #[test] + fn test_run() { + use crate::sealevel::run_locally; + + run_locally() + } +} diff --git a/rust/main/utils/run-locally/src/solana.rs b/rust/main/utils/run-locally/src/sealevel/solana.rs similarity index 77% rename from rust/main/utils/run-locally/src/solana.rs rename to rust/main/utils/run-locally/src/sealevel/solana.rs index fdb315f0b8..c247314992 100644 --- a/rust/main/utils/run-locally/src/solana.rs +++ b/rust/main/utils/run-locally/src/sealevel/solana.rs @@ -10,8 +10,11 @@ use tempfile::{tempdir, NamedTempFile}; use crate::logging::log; use crate::program::Program; -use crate::utils::{as_task, concat_path, AgentHandles, ArbitraryData, TaskHandle}; -use crate::SOLANA_AGNET_BIN_PATH; +use crate::utils::{ + as_task, concat_path, get_sealevel_path, get_workspace_path, AgentHandles, TaskHandle, +}; + +pub const SOLANA_AGENT_BIN_PATH: &str = "target/debug"; /// Solana CLI version for compiling programs pub const SOLANA_CONTRACTS_CLI_VERSION: &str = "1.14.20"; @@ -54,13 +57,12 @@ const SOLANA_HYPERLANE_PROGRAMS: &[&str] = &[ "hyperlane-sealevel-igp", ]; -const SOLANA_KEYPAIR: &str = "../main/config/test-sealevel-keys/test_deployer-keypair.json"; -const SOLANA_DEPLOYER_ACCOUNT: &str = - "../main/config/test-sealevel-keys/test_deployer-account.json"; +const SOLANA_KEYPAIR: &str = "config/test-sealevel-keys/test_deployer-keypair.json"; +const SOLANA_DEPLOYER_ACCOUNT: &str = "config/test-sealevel-keys/test_deployer-account.json"; const SOLANA_WARPROUTE_TOKEN_CONFIG_FILE: &str = - "../sealevel/environments/local-e2e/warp-routes/testwarproute/token-config.json"; -const SOLANA_CHAIN_CONFIG_FILE: &str = "../sealevel/environments/local-e2e/chain-config.json"; -const SOLANA_ENVS_DIR: &str = "../sealevel/environments"; + "environments/local-e2e/warp-routes/testwarproute/token-config.json"; +const SOLANA_CHAIN_CONFIG_FILE: &str = "environments/local-e2e/chain-config.json"; +const SOLANA_ENVS_DIR: &str = "environments"; const SOLANA_ENV_NAME: &str = "local-e2e"; @@ -76,17 +78,16 @@ const SEALEVELTEST2_IGP_PROGRAM_ID: &str = "FArd4tEikwz2fk3MB7S9kC82NGhkgT6f9aXi pub const SOLANA_CHECKPOINT_LOCATION: &str = "/tmp/test_sealevel_checkpoints_0x70997970c51812dc3a010c7d01b50e0d17dc79c8"; -const SOLANA_GAS_ORACLE_CONFIG_FILE: &str = - "../sealevel/environments/local-e2e/gas-oracle-configs.json"; +const SOLANA_GAS_ORACLE_CONFIG_FILE: &str = "environments/local-e2e/gas-oracle-configs.json"; // Install the CLI tools and return the path to the bin dir. #[apply(as_task)] pub fn install_solana_cli_tools( release_url: String, release_version: String, -) -> (PathBuf, impl ArbitraryData) { + tools_dir: PathBuf, +) -> PathBuf { let solana_download_dir = tempdir().unwrap(); - let solana_tools_dir = tempdir().unwrap(); log!( "Downloading solana cli release v{} from {}", release_version, @@ -132,19 +133,20 @@ pub fn install_solana_cli_tools( fs::rename( concat_path(&solana_download_dir, "solana-release"), - &solana_tools_dir, + &tools_dir, ) .expect("Failed to move solana-release dir"); - (concat_path(&solana_tools_dir, "bin"), solana_tools_dir) + concat_path(&tools_dir, "bin") } #[apply(as_task)] pub fn build_solana_programs(solana_cli_tools_path: PathBuf) -> PathBuf { - let out_path = Path::new(SBF_OUT_PATH); + let workspace_path = get_workspace_path(); + let out_path = concat_path(&workspace_path, SBF_OUT_PATH); if out_path.exists() { - fs::remove_dir_all(out_path).expect("Failed to remove solana program deploy dir"); + fs::remove_dir_all(&out_path).expect("Failed to remove solana program deploy dir"); } - fs::create_dir_all(out_path).expect("Failed to create solana program deploy dir"); + fs::create_dir_all(&out_path).expect("Failed to create solana program deploy dir"); let out_path = out_path.canonicalize().unwrap(); Program::new("curl") @@ -167,19 +169,18 @@ pub fn build_solana_programs(solana_cli_tools_path: PathBuf) -> PathBuf { fs::remove_file(concat_path(&out_path, "spl.tar.gz")) .expect("Failed to remove solana program archive"); - let build_sbf = Program::new( - concat_path(&solana_cli_tools_path, "cargo-build-sbf") - .to_str() - .unwrap(), - ) - .env("PATH", updated_path(&solana_cli_tools_path)) - .env("SBF_OUT_PATH", out_path.to_str().unwrap()); + let bin_path = concat_path(&solana_cli_tools_path, "cargo-build-sbf"); + let build_sbf = Program::new(bin_path).env("SBF_OUT_PATH", out_path.to_str().unwrap()); + + let workspace_path = get_workspace_path(); + let sealevel_path = get_sealevel_path(&workspace_path); + let sealevel_programs = concat_path(sealevel_path, "programs"); // build our programs for &path in SOLANA_HYPERLANE_PROGRAMS { build_sbf .clone() - .working_dir(concat_path("../sealevel/programs", path)) + .working_dir(concat_path(&sealevel_programs, path)) .run() .join(); } @@ -193,11 +194,34 @@ pub fn start_solana_test_validator( solana_programs_path: PathBuf, ledger_dir: PathBuf, ) -> (PathBuf, AgentHandles) { + let workspace_path = get_workspace_path(); + let sealevel_path = get_sealevel_path(&workspace_path); + + let solana_deployer_account = concat_path(&workspace_path, SOLANA_DEPLOYER_ACCOUNT); + let solana_deployer_account_str = solana_deployer_account.to_string_lossy(); + + let solana_env_dir = concat_path(&sealevel_path, SOLANA_ENVS_DIR); + let solana_env_dir_str = solana_env_dir.to_string_lossy(); + + let solana_chain_config_file = concat_path(&sealevel_path, SOLANA_CHAIN_CONFIG_FILE); + let solana_chain_config_file_str = solana_chain_config_file.to_string_lossy(); + + let solana_warproute_token_config_file = + concat_path(&sealevel_path, SOLANA_WARPROUTE_TOKEN_CONFIG_FILE); + let solana_warproute_token_config_file_str = + solana_warproute_token_config_file.to_string_lossy(); + + let solana_gas_oracle_config_file = concat_path(&sealevel_path, SOLANA_GAS_ORACLE_CONFIG_FILE); + let solana_gas_oracle_config_file_str = solana_gas_oracle_config_file.to_string_lossy(); + + let build_so_dir = concat_path(&workspace_path, SBF_OUT_PATH); + let build_so_dir_str = build_so_dir.to_string_lossy(); // init solana config let solana_config = NamedTempFile::new().unwrap().into_temp_path(); let solana_config_path = solana_config.to_path_buf(); + Program::new(concat_path(&solana_cli_tools_path, "solana")) - .arg("config", solana_config.to_str().unwrap()) + .arg("config", solana_config_path.to_string_lossy()) .cmd("config") .cmd("set") .arg("url", "localhost") @@ -212,7 +236,7 @@ pub fn start_solana_test_validator( .arg3( "account", "E9VrvAdGRvCguN2XgXsgu9PNmMM3vZsU8LSUrM68j8ty", - SOLANA_DEPLOYER_ACCOUNT, + solana_deployer_account_str.clone(), ) .remember(solana_config); for &(address, lib) in SOLANA_PROGRAMS { @@ -226,16 +250,16 @@ pub fn start_solana_test_validator( sleep(Duration::from_secs(5)); log!("Deploying the hyperlane programs to solana"); - let sealevel_client = sealevel_client(&solana_cli_tools_path, &solana_config_path); + let sealevel_client = sealevel_client(&solana_cli_tools_path, &solana_config_path); let sealevel_client_deploy_core = sealevel_client .clone() .arg("compute-budget", "200000") .cmd("core") .cmd("deploy") .arg("environment", SOLANA_ENV_NAME) - .arg("environments-dir", SOLANA_ENVS_DIR) - .arg("built-so-dir", SBF_OUT_PATH); + .arg("environments-dir", solana_env_dir_str.clone()) + .arg("built-so-dir", build_so_dir_str.clone()); // Deploy sealeveltest1 core sealevel_client_deploy_core @@ -256,8 +280,11 @@ pub fn start_solana_test_validator( .clone() .cmd("igp") .cmd("configure") - .arg("gas-oracle-config-file", SOLANA_GAS_ORACLE_CONFIG_FILE) - .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE); + .arg( + "gas-oracle-config-file", + solana_gas_oracle_config_file_str.clone(), + ) + .arg("chain-config-file", solana_chain_config_file_str.clone()); // Configure sealeveltest1 IGP igp_configure_command @@ -280,10 +307,13 @@ pub fn start_solana_test_validator( .cmd("warp-route") .cmd("deploy") .arg("environment", SOLANA_ENV_NAME) - .arg("environments-dir", SOLANA_ENVS_DIR) - .arg("built-so-dir", SBF_OUT_PATH) + .arg("environments-dir", solana_env_dir_str.clone()) + .arg("built-so-dir", build_so_dir_str.clone()) .arg("warp-route-name", "testwarproute") - .arg("token-config-file", SOLANA_WARPROUTE_TOKEN_CONFIG_FILE) + .arg( + "token-config-file", + solana_warproute_token_config_file_str.clone(), + ) .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE) .arg("ata-payer-funding-amount", "1000000000") .run() @@ -325,7 +355,7 @@ pub fn start_solana_test_validator( .cmd("init-igp-account") .arg("program-id", SEALEVELTEST1_IGP_PROGRAM_ID) .arg("environment", SOLANA_ENV_NAME) - .arg("environments-dir", SOLANA_ENVS_DIR) + .arg("environments-dir", solana_env_dir_str) .arg("chain", "sealeveltest1") .arg("account-salt", ALTERNATIVE_SALT) .run() @@ -348,7 +378,7 @@ pub fn start_solana_test_validator( .cmd("igp") .cmd("configure") .arg("program-id", SEALEVELTEST1_IGP_PROGRAM_ID) - .arg("gas-oracle-config-file", SOLANA_GAS_ORACLE_CONFIG_FILE) + .arg("gas-oracle-config-file", solana_gas_oracle_config_file_str) .arg("chain-config-file", SOLANA_CHAIN_CONFIG_FILE) .arg("chain", "sealeveltest1") .arg("account-salt", ALTERNATIVE_SALT) @@ -366,9 +396,13 @@ pub fn initiate_solana_hyperlane_transfer( solana_cli_tools_path: PathBuf, solana_config_path: PathBuf, ) -> String { + let workspace_path = get_workspace_path(); + let solana_keypair = concat_path(workspace_path, SOLANA_KEYPAIR); + let solana_keypair_str = solana_keypair.to_string_lossy(); + let sender = Program::new(concat_path(&solana_cli_tools_path, "solana")) .arg("config", solana_config_path.to_str().unwrap()) - .arg("keypair", SOLANA_KEYPAIR) + .arg("keypair", solana_keypair_str.clone()) .cmd("address") .run_with_output() .join() @@ -380,7 +414,7 @@ pub fn initiate_solana_hyperlane_transfer( let output = sealevel_client(&solana_cli_tools_path, &solana_config_path) .cmd("token") .cmd("transfer-remote") - .cmd(SOLANA_KEYPAIR) + .cmd(solana_keypair_str.clone()) .cmd("10000000000") .cmd(SOLANA_REMOTE_CHAIN_ID) .cmd(sender) // send to self @@ -411,9 +445,13 @@ pub fn initiate_solana_non_matching_igp_paying_transfer( solana_cli_tools_path: PathBuf, solana_config_path: PathBuf, ) -> String { + let workspace_path = get_workspace_path(); + let solana_keypair = concat_path(workspace_path, SOLANA_KEYPAIR); + let solana_keypair_str = solana_keypair.to_string_lossy(); + let sender = Program::new(concat_path(&solana_cli_tools_path, "solana")) .arg("config", solana_config_path.to_str().unwrap()) - .arg("keypair", SOLANA_KEYPAIR) + .arg("keypair", solana_keypair_str.clone()) .cmd("address") .run_with_output() .join() @@ -425,7 +463,7 @@ pub fn initiate_solana_non_matching_igp_paying_transfer( let output = sealevel_client(&solana_cli_tools_path, &solana_config_path) .cmd("token") .cmd("transfer-remote") - .cmd(SOLANA_KEYPAIR) + .cmd(solana_keypair_str) .cmd("10000000000") .cmd(SOLANA_REMOTE_CHAIN_ID) .cmd(sender) // send to self @@ -491,17 +529,21 @@ pub fn solana_termination_invariants_met( .contains("Message delivered") } fn sealevel_client(solana_cli_tools_path: &Path, solana_config_path: &Path) -> Program { + let workspace_path = get_workspace_path(); + let sealevel_path = get_sealevel_path(&workspace_path); + + let solana_keypair = concat_path(workspace_path, SOLANA_KEYPAIR); + let solana_keypair_str = solana_keypair.to_string_lossy(); + Program::new(concat_path( - SOLANA_AGNET_BIN_PATH, - "hyperlane-sealevel-client", + &sealevel_path, + format!("{}/hyperlane-sealevel-client", SOLANA_AGENT_BIN_PATH), )) + .working_dir(sealevel_path.clone()) .env("PATH", updated_path(solana_cli_tools_path)) .env("RUST_BACKTRACE", "1") .arg("config", solana_config_path.to_str().unwrap()) - .arg( - "keypair", - "config/test-sealevel-keys/test_deployer-keypair.json", - ) + .arg("keypair", solana_keypair_str) } fn updated_path(solana_cli_tools_path: &Path) -> String { diff --git a/rust/main/utils/run-locally/src/sealevel/termination_invariant.rs b/rust/main/utils/run-locally/src/sealevel/termination_invariant.rs new file mode 100644 index 0000000000..d4a3df489a --- /dev/null +++ b/rust/main/utils/run-locally/src/sealevel/termination_invariant.rs @@ -0,0 +1,82 @@ +use std::path::Path; + +use maplit::hashmap; + +use crate::{ + config::Config, + fetch_metric, + invariants::{ + relayer_balance_check, relayer_termination_invariants_met, + scraper_termination_invariants_met, + }, + logging::log, + sealevel::{solana::*, SOL_MESSAGES_EXPECTED, SOL_MESSAGES_WITH_NON_MATCHING_IGP}, + RELAYER_METRICS_PORT, +}; + +/// Use the metrics to check if the relayer queues are empty and the expected +/// number of messages have been sent. +#[allow(clippy::unnecessary_get_then_check)] // TODO: `rustc` 1.80.1 clippy issue +pub fn termination_invariants_met( + config: &Config, + starting_relayer_balance: f64, + solana_cli_tools_path: &Path, + solana_config_path: &Path, +) -> eyre::Result { + let sol_messages_expected = SOL_MESSAGES_EXPECTED; + let sol_messages_with_non_matching_igp = SOL_MESSAGES_WITH_NON_MATCHING_IGP; + + // this is total messages expected to be delivered + let total_messages_expected = sol_messages_expected; + let total_messages_dispatched = total_messages_expected + sol_messages_with_non_matching_igp; + + // Also ensure the counter is as expected (total number of messages), summed + // across all mailboxes. + let msg_processed_count = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_messages_processed_count", + &hashmap! {}, + )? + .iter() + .sum::(); + + let gas_payment_events_count = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_contract_sync_stored_events", + &hashmap! {"data_type" => "gas_payments"}, + )? + .iter() + .sum::(); + + if !relayer_termination_invariants_met( + config, + msg_processed_count, + gas_payment_events_count, + total_messages_expected, + total_messages_dispatched, + sol_messages_with_non_matching_igp, + total_messages_expected + sol_messages_with_non_matching_igp, + )? { + return Ok(false); + } + + if !solana_termination_invariants_met(solana_cli_tools_path, solana_config_path) { + log!("Solana termination invariants not met"); + return Ok(false); + } + + if !scraper_termination_invariants_met( + gas_payment_events_count, + total_messages_dispatched, + total_messages_expected, + )? { + return Ok(false); + } + + if !relayer_balance_check(starting_relayer_balance)? { + return Ok(false); + } + + log!("Termination invariants have been meet"); + Ok(true) +} diff --git a/rust/main/utils/run-locally/src/utils.rs b/rust/main/utils/run-locally/src/utils.rs index ebb5a70245..55988ddf27 100644 --- a/rust/main/utils/run-locally/src/utils.rs +++ b/rust/main/utils/run-locally/src/utils.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::fs::File; use std::io::{self, BufRead}; use std::path::{Path, PathBuf}; -use std::process::Child; +use std::process::{Child, Command}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; @@ -147,3 +147,46 @@ pub fn get_matching_lines<'a>( } matches } + +/// Returns absolute path to rust workspace +/// `/<...>/hyperlane-monorepo/rust/main`. +/// This allows us to have a more reliable way of generating +/// relative paths such path to sealevel directory +pub fn get_workspace_path() -> PathBuf { + let output = Command::new(env!("CARGO")) + .arg("locate-project") + .arg("--workspace") + .arg("--message-format=plain") + .output() + .expect("Failed to get workspace path") + .stdout; + let path_str = String::from_utf8(output).expect("Failed to parse workspace path"); + let mut workspace_path = PathBuf::from(path_str); + // pop Cargo.toml from path + workspace_path.pop(); + workspace_path +} + +/// Returns absolute path to sealevel directory +/// `/<...>/hyperlane-monorepo/rust/sealevel` +#[cfg(feature = "sealevel")] +pub fn get_sealevel_path(workspace_path: &Path) -> PathBuf { + concat_path( + workspace_path + .parent() + .expect("workspace path has no parent"), + "sealevel", + ) +} + +/// Returns absolute path to typescript infra directory +/// `/<...>/hyperlane-monorepo/typescript/infra` +pub fn get_ts_infra_path(workspace_path: &Path) -> PathBuf { + concat_path( + workspace_path + .parent() + .and_then(|p| p.parent()) + .expect("workspace path has no parent x2"), + "typescript/infra", + ) +}