feat: run sealevel E2E in parallel with EVM

.github/workflows/test.yml

@@ -213,7 +213,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        e2e-type: [cosmwasm, non-cosmwasm]
+        e2e-type: [cosmwasm, evm, sealevel]
     steps:
       - uses: actions/setup-node@v4
         with:
@@ -240,7 +240,7 @@ jobs:
           save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }}
           workspaces: |
             ./rust/main
-            ${{ matrix.e2e-type == 'non-cosmwasm' && './rust/sealevel' || '' }}
+            ${{ matrix.e2e-type == 'sealevel' && './rust/sealevel' || '' }}
 
       - name: Free disk space
         run: |
@@ -269,13 +269,6 @@ jobs:
       - name: Checkout registry
         uses: ./.github/actions/checkout-registry
 
-      - name: agent tests (CosmWasm)
-        run: cargo test --release --package run-locally --bin run-locally --features cosmos test-utils -- cosmos::test --nocapture
-        if: matrix.e2e-type == 'cosmwasm'
-        working-directory: ./rust/main
-        env:
-          RUST_BACKTRACE: 'full'
-
       - name: Check for Rust file changes
         id: check-rust-changes
         run: |
@@ -286,16 +279,31 @@ jobs:
             echo "rust_changes=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: agent tests (EVM and Sealevel)
+      - name: agent tests (Sealevel)
+        run: cargo test --release --package run-locally --bin run-locally --features sealevel -- sealevel::test --nocapture
+        if: ${{ matrix.e2e-type == 'sealevel' && steps.check-rust-changes.outputs.rust_changes == 'true' }}
+        working-directory: ./rust/main
+        env:
+          E2E_CI_MODE: 'true'
+          E2E_CI_TIMEOUT_SEC: '600'
+          RUST_BACKTRACE: 'full'
+
+      - name: agent tests (EVM)
         run: cargo run --release --bin run-locally --features test-utils
-        if: matrix.e2e-type == 'non-cosmwasm'
+        if: ${{ matrix.e2e-type == 'evm' }}
         working-directory: ./rust/main
         env:
           E2E_CI_MODE: 'true'
           E2E_CI_TIMEOUT_SEC: '600'
           E2E_KATHY_MESSAGES: '20'
           RUST_BACKTRACE: 'full'
-          SEALEVEL_ENABLED: ${{ steps.check-rust-changes.outputs.rust_changes }}
+
+      - name: agent tests (CosmWasm)
+        run: cargo test --release --package run-locally --bin run-locally --features cosmos -- cosmos::test --nocapture
+        if: ${{ matrix.e2e-type == 'cosmwasm' }}
+        working-directory: ./rust/main
+        env:
+          RUST_BACKTRACE: 'full'
 
   env-test:
     runs-on: ubuntu-latest

rust/README.md

@@ -112,10 +112,18 @@ validator. By default, this test will run indefinitely, but can be stopped with
 
 To run the tests for a specific VM, use the `--features` flag.
 
+##### Cosmos E2E Test
+
 ```bash
 cargo test --release --package run-locally --bin run-locally --features cosmos -- cosmos::test --nocapture
 ```
 
+##### Sealevel E2E Test
+
+```bash
+cargo test --release --package run-locally --bin run-locally --features sealevel -- sealevel::test --nocapture
+```
+
 ### Building Agent Docker Images
 
 There exists a docker build for the agent binaries. These docker images are used for deploying the agents in a

rust/main/utils/run-locally/Cargo.toml

@@ -44,3 +44,4 @@ vergen = { version = "8.3.2", features = ["build", "git", "gitcl"] }
 
 [features]
 cosmos = []
+sealevel = []

rust/main/utils/run-locally/src/config.rs

@@ -7,7 +7,6 @@ pub struct Config {
     pub ci_mode: bool,
     pub ci_mode_timeout: u64,
     pub kathy_messages: u64,
-    pub sealevel_enabled: bool,
     // TODO: Include count of sealevel messages in a field separate from `kathy_messages`?
 }
 
@@ -28,9 +27,6 @@ impl Config {
                     .map(|r| r.parse::<u64>().unwrap());
                 r.unwrap_or(16)
             },
-            sealevel_enabled: env::var("SEALEVEL_ENABLED")
-                .map(|k| k.parse::<bool>().unwrap())
-                .unwrap_or(true),
         })
     }
 }

rust/main/utils/run-locally/src/cosmos/mod.rs

@@ -30,7 +30,9 @@ use crate::cosmos::link::link_networks;
 use crate::logging::log;
 use crate::metrics::agent_balance_sum;
 use crate::program::Program;
-use crate::utils::{as_task, concat_path, stop_child, AgentHandles, TaskHandle};
+use crate::utils::{
+    as_task, concat_path, get_workspace_path, stop_child, AgentHandles, TaskHandle,
+};
 use crate::{fetch_metric, AGENT_BIN_PATH};
 use cli::{OsmosisCLI, OsmosisEndpoint};
 
@@ -345,10 +347,12 @@ fn run_locally() {
     const TIMEOUT_SECS: u64 = 60 * 10;
     let debug = false;
 
+    let workspace_path = get_workspace_path();
+
     log!("Building rust...");
     Program::new("cargo")
         .cmd("build")
-        .working_dir("../../")
+        .working_dir(&workspace_path)
         .arg("features", "test-utils")
         .arg("bin", "relayer")
         .arg("bin", "validator")
@@ -529,7 +533,8 @@ fn run_locally() {
     // give things a chance to fully start.
     sleep(Duration::from_secs(10));
 
-    let starting_relayer_balance: f64 = agent_balance_sum(hpl_rly_metrics_port).unwrap();
+    let starting_relayer_balance: f64 =
+        agent_balance_sum(hpl_rly_metrics_port).expect("Failed to get relayer agent balance");
 
     // dispatch the second batch of messages (after agents start)
     dispatched_messages += dispatch(&osmosisd, linker, &nodes);
@@ -664,7 +669,8 @@ fn termination_invariants_met(
         return Ok(false);
     }
 
-    let ending_relayer_balance: f64 = agent_balance_sum(relayer_metrics_port).unwrap();
+    let ending_relayer_balance: f64 =
+        agent_balance_sum(relayer_metrics_port).expect("Failed to get relayer agent balance");
 
     // Make sure the balance was correctly updated in the metrics.
     // Ideally, make sure that the difference is >= gas_per_tx * gas_cost, set here:

rust/main/utils/run-locally/src/ethereum/mod.rs

@@ -10,15 +10,18 @@ use crate::config::Config;
 use crate::ethereum::multicall::{DEPLOYER_ADDRESS, SIGNED_DEPLOY_MULTICALL_TX};
 use crate::logging::log;
 use crate::program::Program;
-use crate::utils::{as_task, AgentHandles, TaskHandle};
-use crate::{INFRA_PATH, MONOREPO_ROOT_PATH};
+use crate::utils::{as_task, get_ts_infra_path, get_workspace_path, AgentHandles, TaskHandle};
 
 mod multicall;
 
 #[apply(as_task)]
 pub fn start_anvil(config: Arc<Config>) -> AgentHandles {
     log!("Installing typescript dependencies...");
-    let yarn_monorepo = Program::new("yarn").working_dir(MONOREPO_ROOT_PATH);
+
+    let workspace_path = get_workspace_path();
+    let ts_infra_path = get_ts_infra_path(&workspace_path);
+
+    let yarn_monorepo = Program::new("yarn").working_dir(workspace_path);
     if !config.is_ci_env {
         // test.yaml workflow installs dependencies
         yarn_monorepo.clone().cmd("install").run().join();
@@ -42,7 +45,7 @@ pub fn start_anvil(config: Arc<Config>) -> AgentHandles {
 
     sleep(Duration::from_secs(10));
 
-    let yarn_infra = Program::new("yarn").working_dir(INFRA_PATH);
+    let yarn_infra = Program::new("yarn").working_dir(&ts_infra_path);
 
     log!("Deploying hyperlane ism contracts...");
     yarn_infra.clone().cmd("deploy-ism").run().join();

rust/main/utils/run-locally/src/invariants.rs → rust/main/utils/run-locally/src/invariants/mod.rs

@@ -1,7 +1,5 @@
-pub use common::SOL_MESSAGES_EXPECTED;
 pub use post_startup_invariants::post_startup_invariants;
-pub use termination_invariants::termination_invariants_met;
+pub use termination_invariants::*;
 
-mod common;
 mod post_startup_invariants;
 mod termination_invariants;

rust/main/utils/run-locally/src/invariants/post_startup_invariants.rs

@@ -3,7 +3,7 @@ use std::io::BufReader;
 
 use hyperlane_base::AgentMetadata;
 
-use crate::DynPath;
+use crate::{log, DynPath};
 
 pub fn post_startup_invariants(checkpoints_dirs: &[DynPath]) -> bool {
     post_startup_validator_metadata_written(checkpoints_dirs)
@@ -17,6 +17,10 @@ fn post_startup_validator_metadata_written(checkpoints_dirs: &[DynPath]) -> bool
         .map(|path| metadata_file_check(expected_git_sha, path))
         .any(|b| !b);
 
+    if failed_metadata {
+        log!("Error: Metadata git hash mismatch, maybe try `cargo clean` and try again");
+    }
+
     !failed_metadata
 }