-
Notifications
You must be signed in to change notification settings - Fork 441
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: run sealevel E2E in parallel with EVM
- separate evm e2e tests with sealevel e2e tests - refactor relative paths code to use more absolute paths - update working directory code to be more reliable - update github CI to run these 2 tests in parallel
Showing
14 changed files
with
844 additions
and
239 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -43,3 +43,4 @@ vergen = { version = "8.3.2", features = ["build", "git", "gitcl"] } | |
|
||
[features] | ||
cosmos = [] | ||
sealevel = [] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
.../main/utils/run-locally/src/invariants.rs → ...n/utils/run-locally/src/invariants/mod.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
223 changes: 223 additions & 0 deletions
223
rust/main/utils/run-locally/src/sealevel/termination_invariant.rs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,223 @@ | ||
use std::{fs::File, path::Path}; | ||
|
||
use maplit::hashmap; | ||
use relayer::GAS_EXPENDITURE_LOG_MESSAGE; | ||
|
||
use crate::{ | ||
config::Config, | ||
fetch_metric, | ||
invariants::{SOL_MESSAGES_EXPECTED, SOL_MESSAGES_WITH_NON_MATCHING_IGP}, | ||
logging::log, | ||
metrics::agent_balance_sum, | ||
sealevel::solana::*, | ||
utils::get_matching_lines, | ||
AGENT_LOGGING_DIR, RELAYER_METRICS_PORT, SCRAPER_METRICS_PORT, | ||
}; | ||
|
||
/// Use the metrics to check if the relayer queues are empty and the expected | ||
/// number of messages have been sent. | ||
#[allow(clippy::unnecessary_get_then_check)] // TODO: `rustc` 1.80.1 clippy issue | ||
pub fn termination_invariants_met( | ||
config: &Config, | ||
starting_relayer_balance: f64, | ||
solana_cli_tools_path: &Path, | ||
solana_config_path: &Path, | ||
) -> eyre::Result<bool> { | ||
let sol_messages_expected = SOL_MESSAGES_EXPECTED; | ||
let sol_messages_with_non_matching_igp = SOL_MESSAGES_WITH_NON_MATCHING_IGP; | ||
|
||
// this is total messages expected to be delivered | ||
let total_messages_expected = sol_messages_expected; | ||
let total_messages_dispatched = total_messages_expected + sol_messages_with_non_matching_igp; | ||
|
||
let lengths = fetch_metric( | ||
RELAYER_METRICS_PORT, | ||
"hyperlane_submitter_queue_length", | ||
&hashmap! {}, | ||
)?; | ||
assert!(!lengths.is_empty(), "Could not find queue length metric"); | ||
if lengths.iter().sum::<u32>() != sol_messages_with_non_matching_igp { | ||
log!( | ||
"Relayer queues contain more messages than the zero-merkle-insertion ones. Lengths: {:?}", | ||
lengths | ||
); | ||
return Ok(false); | ||
}; | ||
|
||
// Also ensure the counter is as expected (total number of messages), summed | ||
// across all mailboxes. | ||
let msg_processed_count = fetch_metric( | ||
RELAYER_METRICS_PORT, | ||
"hyperlane_messages_processed_count", | ||
&hashmap! {}, | ||
)? | ||
.iter() | ||
.sum::<u32>(); | ||
if msg_processed_count != total_messages_expected { | ||
log!( | ||
"Relayer has {} processed messages, expected {}", | ||
msg_processed_count, | ||
total_messages_expected | ||
); | ||
return Ok(false); | ||
} | ||
|
||
let gas_payment_events_count = fetch_metric( | ||
RELAYER_METRICS_PORT, | ||
"hyperlane_contract_sync_stored_events", | ||
&hashmap! {"data_type" => "gas_payments"}, | ||
)? | ||
.iter() | ||
.sum::<u32>(); | ||
|
||
let log_file_path = AGENT_LOGGING_DIR.join("RLY-output.log"); | ||
const STORING_NEW_MESSAGE_LOG_MESSAGE: &str = "Storing new message in db"; | ||
const LOOKING_FOR_EVENTS_LOG_MESSAGE: &str = "Looking for events in index range"; | ||
const HYPER_INCOMING_BODY_LOG_MESSAGE: &str = "incoming body completed"; | ||
|
||
const TX_ID_INDEXING_LOG_MESSAGE: &str = "Found log(s) for tx id"; | ||
|
||
let relayer_logfile = File::open(log_file_path)?; | ||
let invariant_logs = &[ | ||
STORING_NEW_MESSAGE_LOG_MESSAGE, | ||
LOOKING_FOR_EVENTS_LOG_MESSAGE, | ||
GAS_EXPENDITURE_LOG_MESSAGE, | ||
HYPER_INCOMING_BODY_LOG_MESSAGE, | ||
TX_ID_INDEXING_LOG_MESSAGE, | ||
]; | ||
let log_counts = get_matching_lines(&relayer_logfile, invariant_logs); | ||
// Zero insertion messages don't reach `submit` stage where gas is spent, so we only expect these logs for the other messages. | ||
// TODO: Sometimes we find more logs than expected. This may either mean that gas is deducted twice for the same message due to a bug, | ||
// or that submitting the message transaction fails for some messages. Figure out which is the case and convert this check to | ||
// strict equality. | ||
// EDIT: Having had a quick look, it seems like there are some legitimate reverts happening in the confirm step | ||
// (`Transaction attempting to process message either reverted or was reorged`) | ||
// in which case more gas expenditure logs than messages are expected. | ||
let gas_expenditure_log_count = log_counts.get(GAS_EXPENDITURE_LOG_MESSAGE).unwrap(); | ||
assert!( | ||
gas_expenditure_log_count >= &total_messages_expected, | ||
"Didn't record gas payment for all delivered messages. Got {} gas payment logs, expected at least {}", | ||
gas_expenditure_log_count, | ||
total_messages_expected | ||
); | ||
// These tests check that we fixed https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/3915, where some logs would not show up | ||
assert!( | ||
log_counts.get(STORING_NEW_MESSAGE_LOG_MESSAGE).unwrap() > &0, | ||
"Didn't find any logs about storing messages in db" | ||
); | ||
assert!( | ||
log_counts.get(LOOKING_FOR_EVENTS_LOG_MESSAGE).unwrap() > &0, | ||
"Didn't find any logs about looking for events in index range" | ||
); | ||
let total_tx_id_log_count = log_counts.get(TX_ID_INDEXING_LOG_MESSAGE).unwrap(); | ||
assert!( | ||
// there are 3 txid-indexed events: | ||
// - relayer: merkle insertion and gas payment | ||
// - scraper: gas payment | ||
// some logs are emitted for multiple events, so requiring there to be at least | ||
// `config.kathy_messages` logs is a reasonable approximation, since all three of these events | ||
// are expected to be logged for each message. | ||
*total_tx_id_log_count as u64 >= config.kathy_messages, | ||
"Didn't find as many tx id logs as expected. Found {} and expected {}", | ||
total_tx_id_log_count, | ||
config.kathy_messages | ||
); | ||
assert!( | ||
log_counts.get(HYPER_INCOMING_BODY_LOG_MESSAGE).is_none(), | ||
"Verbose logs not expected at the log level set in e2e" | ||
); | ||
|
||
// TestSendReceiver randomly breaks gas payments up into | ||
// two. So we expect at least as many gas payments as messages. | ||
if gas_payment_events_count < total_messages_dispatched { | ||
log!( | ||
"Relayer has {} gas payment events, expected at least {}", | ||
gas_payment_events_count, | ||
total_messages_dispatched | ||
); | ||
return Ok(false); | ||
} | ||
|
||
let merkle_tree_max_sequence = fetch_metric( | ||
RELAYER_METRICS_PORT, | ||
"hyperlane_cursor_max_sequence", | ||
&hashmap! {"event_type" => "merkle_tree_insertion"}, | ||
)?; | ||
// check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions | ||
// LHS: sum(merkle_tree_max_sequence) + len(merkle_tree_max_sequence) (each is index so we add 1 to each) | ||
// RHS: total_messages_expected + non_matching_igp_messages + (config.kathy_messages as u32 / 4) * 2 (double insertions) | ||
let non_zero_sequence_count = | ||
merkle_tree_max_sequence.iter().filter(|&x| *x > 0).count() as u32; | ||
assert_eq!( | ||
merkle_tree_max_sequence.iter().sum::<u32>() + non_zero_sequence_count, | ||
total_messages_expected + sol_messages_with_non_matching_igp | ||
); | ||
|
||
if !solana_termination_invariants_met(solana_cli_tools_path, solana_config_path) { | ||
log!("Solana termination invariants not met"); | ||
return Ok(false); | ||
} | ||
|
||
let dispatched_messages_scraped = fetch_metric( | ||
SCRAPER_METRICS_PORT, | ||
"hyperlane_contract_sync_stored_events", | ||
&hashmap! {"data_type" => "message_dispatch"}, | ||
)? | ||
.iter() | ||
.sum::<u32>(); | ||
if dispatched_messages_scraped != total_messages_dispatched { | ||
log!( | ||
"Scraper has scraped {} dispatched messages, expected {}", | ||
dispatched_messages_scraped, | ||
total_messages_dispatched, | ||
); | ||
return Ok(false); | ||
} | ||
|
||
let gas_payments_scraped = fetch_metric( | ||
SCRAPER_METRICS_PORT, | ||
"hyperlane_contract_sync_stored_events", | ||
&hashmap! {"data_type" => "gas_payment"}, | ||
)? | ||
.iter() | ||
.sum::<u32>(); | ||
if gas_payments_scraped != gas_payment_events_count { | ||
log!( | ||
"Scraper has scraped {} gas payments, expected {}", | ||
gas_payments_scraped, | ||
gas_payment_events_count | ||
); | ||
return Ok(false); | ||
} | ||
|
||
let delivered_messages_scraped = fetch_metric( | ||
SCRAPER_METRICS_PORT, | ||
"hyperlane_contract_sync_stored_events", | ||
&hashmap! {"data_type" => "message_delivery"}, | ||
)? | ||
.iter() | ||
.sum::<u32>(); | ||
if delivered_messages_scraped != total_messages_expected { | ||
log!( | ||
"Scraper has scraped {} delivered messages, expected {}", | ||
delivered_messages_scraped, | ||
total_messages_expected + sol_messages_with_non_matching_igp | ||
); | ||
return Ok(false); | ||
} | ||
|
||
let ending_relayer_balance: f64 = | ||
agent_balance_sum(9092).expect("Failed to get relayer agent balance"); | ||
// Make sure the balance was correctly updated in the metrics. | ||
if starting_relayer_balance <= ending_relayer_balance { | ||
log!( | ||
"Expected starting relayer balance to be greater than ending relayer balance, but got {} <= {}", | ||
starting_relayer_balance, | ||
ending_relayer_balance | ||
); | ||
return Ok(false); | ||
} | ||
|
||
log!("Termination invariants have been meet"); | ||
Ok(true) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters