From d7813104d15e2653163919aac348042a090396d9 Mon Sep 17 00:00:00 2001 From: Danil Nemirovsky Date: Wed, 15 Jan 2025 11:19:50 +0000 Subject: [PATCH 1/7] feat: Add logging to agents (#5167) ### Description Add logging to agents * When an agent start * When cursor indexing task is exiting * When cursor sync method is exiting ### Related issues - Contributes into https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/5098 ### Backward compatibility Yes ### Testing E2E tests --------- Co-authored-by: Danil Nemirovsky <4614623+ameten@users.noreply.github.com> --- rust/main/Cargo.lock | 12 ++++---- rust/main/agents/relayer/src/main.rs | 3 ++ rust/main/agents/relayer/src/relayer.rs | 28 +++++++++---------- rust/main/agents/scraper/src/main.rs | 3 ++ rust/main/agents/validator/src/main.rs | 3 ++ rust/main/agents/validator/src/validator.rs | 8 +++--- rust/main/hyperlane-base/src/agent.rs | 3 ++ .../hyperlane-base/src/contract_sync/mod.rs | 10 +++++++ 8 files changed, 46 insertions(+), 24 deletions(-) diff --git a/rust/main/Cargo.lock b/rust/main/Cargo.lock index aea92da917..6a6f44e6c9 100644 --- a/rust/main/Cargo.lock +++ b/rust/main/Cargo.lock @@ -5264,9 +5264,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libloading" @@ -9838,9 +9838,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -9867,9 +9867,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2 1.0.86", "quote 1.0.37", diff --git a/rust/main/agents/relayer/src/main.rs b/rust/main/agents/relayer/src/main.rs index f9ac628aca..b00ca8dade 100644 --- a/rust/main/agents/relayer/src/main.rs +++ b/rust/main/agents/relayer/src/main.rs @@ -18,6 +18,9 @@ mod memory_profiler; #[tokio::main(flavor = "multi_thread", worker_threads = 20)] async fn main() -> Result<()> { + // Logging is not initialised at this point, so, using `println!` + println!("Relayer starting up..."); + let agent_main_fut = agent_main::(); #[cfg(feature = "memory-profiling")] diff --git a/rust/main/agents/relayer/src/relayer.rs b/rust/main/agents/relayer/src/relayer.rs index b1f013b6ae..d91117c1d7 100644 --- a/rust/main/agents/relayer/src/relayer.rs +++ b/rust/main/agents/relayer/src/relayer.rs @@ -465,11 +465,11 @@ impl Relayer { return tokio::spawn(async {}).instrument(info_span!("MessageSync")); } }; + let origin_name = origin.name().to_string(); tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { - contract_sync - .clone() - .sync("dispatched_messages", cursor.into()) - .await + let label = "dispatched_messages"; + contract_sync.clone().sync(label, cursor.into()).await; + info!(chain = origin_name, label, "contract sync task exit"); })) .instrument(info_span!("MessageSync")) } @@ -496,14 +496,14 @@ impl Relayer { return tokio::spawn(async {}).instrument(info_span!("IgpSync")); } }; + let origin_name = origin.name().to_string(); tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { + let label = "gas_payments"; contract_sync .clone() - .sync( - "gas_payments", - SyncOptions::new(Some(cursor), tx_id_receiver), - ) - .await + .sync(label, SyncOptions::new(Some(cursor), tx_id_receiver)) + .await; + info!(chain = origin_name, label, "contract sync task exit"); })) .instrument(info_span!("IgpSync")) } @@ -526,14 +526,14 @@ impl Relayer { return tokio::spawn(async {}).instrument(info_span!("MerkleTreeHookSync")); } }; + let origin_name = origin.name().to_string(); tokio::spawn(TaskMonitor::instrument(&task_monitor, async move { + let label = "merkle_tree_hook"; contract_sync .clone() - .sync( - "merkle_tree_hook", - SyncOptions::new(Some(cursor), tx_id_receiver), - ) - .await + .sync(label, SyncOptions::new(Some(cursor), tx_id_receiver)) + .await; + info!(chain = origin_name, label, "contract sync task exit"); })) .instrument(info_span!("MerkleTreeHookSync")) } diff --git a/rust/main/agents/scraper/src/main.rs b/rust/main/agents/scraper/src/main.rs index f9b0d5971f..f09297cde6 100644 --- a/rust/main/agents/scraper/src/main.rs +++ b/rust/main/agents/scraper/src/main.rs @@ -26,5 +26,8 @@ mod store; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<()> { + // Logging is not initialised at this point, so, using `println!` + println!("Scraper agent starting up..."); + agent_main::().await } diff --git a/rust/main/agents/validator/src/main.rs b/rust/main/agents/validator/src/main.rs index ebc24974c2..14056046e0 100644 --- a/rust/main/agents/validator/src/main.rs +++ b/rust/main/agents/validator/src/main.rs @@ -16,5 +16,8 @@ mod validator; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<()> { + // Logging is not initialised at this point, so, using `println!` + println!("Validator starting up..."); + agent_main::().await } diff --git a/rust/main/agents/validator/src/validator.rs b/rust/main/agents/validator/src/validator.rs index f7a8b43f59..f599fa8e39 100644 --- a/rust/main/agents/validator/src/validator.rs +++ b/rust/main/agents/validator/src/validator.rs @@ -228,11 +228,11 @@ impl Validator { self.origin_chain ) }); + let origin = self.origin_chain.name().to_string(); tokio::spawn(async move { - contract_sync - .clone() - .sync("merkle_tree_hook", cursor.into()) - .await; + let label = "merkle_tree_hook"; + contract_sync.clone().sync(label, cursor.into()).await; + info!(chain = origin, label, "contract sync task exit"); }) .instrument(info_span!("MerkleTreeHookSyncer")) } diff --git a/rust/main/hyperlane-base/src/agent.rs b/rust/main/hyperlane-base/src/agent.rs index 5ee00c1eb9..87ad60f46c 100644 --- a/rust/main/hyperlane-base/src/agent.rs +++ b/rust/main/hyperlane-base/src/agent.rs @@ -80,6 +80,9 @@ pub async fn agent_main() -> Result<()> { // the variable defaults to "VERGEN_IDEMPOTENT_OUTPUT". let git_sha = env!("VERGEN_GIT_SHA").to_owned(); + // Logging is not initialised at this point, so, using `println!` + println!("Agent {} starting up with version {git_sha}", A::AGENT_NAME); + let agent_metadata = AgentMetadata::new(git_sha); let settings = A::Settings::load()?; diff --git a/rust/main/hyperlane-base/src/contract_sync/mod.rs b/rust/main/hyperlane-base/src/contract_sync/mod.rs index c9048b4808..26f6c9d673 100644 --- a/rust/main/hyperlane-base/src/contract_sync/mod.rs +++ b/rust/main/hyperlane-base/src/contract_sync/mod.rs @@ -107,7 +107,17 @@ where self.fetch_logs_with_cursor(cursor, &stored_logs_metric, &indexed_height_metric) .await; } + + // Added so that we confuse compiler that it is an infinite loop + if false { + break; + } } + + // Although the above loop should never end (unless by panicking), + // we put log here to make sure that we see when this method returns normally. + // Hopefully, compiler will not optimise this code out. + info!(chain = chain_name, label, "contract sync loop exit"); } #[instrument(fields(domain=self.domain().name()), skip(self, recv, stored_logs_metric))] From af0dd49a677c86e043e6d720431be07a9936932d Mon Sep 17 00:00:00 2001 From: Trevor Porter Date: Wed, 15 Jan 2025 11:56:33 +0000 Subject: [PATCH 2/7] chore: re-enable Neutron scraping (#5179) ### Description - The chain is back, scraping it again now after its removal in #5165 ### Drive-by changes ### Related issues ### Backward compatibility ### Testing --- typescript/infra/config/environments/mainnet3/agent.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/typescript/infra/config/environments/mainnet3/agent.ts b/typescript/infra/config/environments/mainnet3/agent.ts index d143d5849c..d2bf764c7c 100644 --- a/typescript/infra/config/environments/mainnet3/agent.ts +++ b/typescript/infra/config/environments/mainnet3/agent.ts @@ -374,8 +374,7 @@ export const hyperlaneContextAgentChainConfig: AgentChainConfig< moonbeam: true, morph: true, nero: true, - // Jan 14th - temporarily disabled while Neutron chain is down and RPCs give issues, causing scraper startup problems - neutron: false, + neutron: true, oortmainnet: true, optimism: true, orderly: true, From c47511b0d86a9d06e90aaa0c8432e802bf7f223f Mon Sep 17 00:00:00 2001 From: Jeff Date: Wed, 15 Jan 2025 22:18:00 +0800 Subject: [PATCH 3/7] feat: don't crash relayer due to flaky RPCs (#5115) ### Description - instead of crashing relayer if any RPC calls fail during `build_mailboxes` and `build_validator_announces`, we just log the error and metric - relayer will continue running without those chains ### Drive-by changes ### Related issues Fixes https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/4887 ### Backward compatibility Are these changes backward compatible? Are there any infrastructure implications, e.g. changes that would prohibit deploying older commits using this infra tooling? Yes ### Testing Added unittests to test ensure functions run correctly and log errors in metrics --------- Co-authored-by: Danil Nemirovsky Co-authored-by: Trevor Porter --- rust/main/Cargo.lock | 2 + rust/main/agents/relayer/Cargo.toml | 2 + rust/main/agents/relayer/src/relayer.rs | 297 +++++++++++++++++- rust/main/agents/relayer/src/settings/mod.rs | 2 +- .../src/metrics/agent_metrics.rs | 2 +- rust/main/hyperlane-base/src/settings/base.rs | 12 +- 6 files changed, 295 insertions(+), 22 deletions(-) diff --git a/rust/main/Cargo.lock b/rust/main/Cargo.lock index 6a6f44e6c9..da153bf564 100644 --- a/rust/main/Cargo.lock +++ b/rust/main/Cargo.lock @@ -7101,6 +7101,7 @@ dependencies = [ "dhat", "ethers", "ethers-contract", + "ethers-prometheus", "eyre", "futures", "futures-util", @@ -7126,6 +7127,7 @@ dependencies = [ "tokio-test", "tracing", "tracing-futures", + "tracing-test", "typetag", ] diff --git a/rust/main/agents/relayer/Cargo.toml b/rust/main/agents/relayer/Cargo.toml index 5a891d912c..ac0e198945 100644 --- a/rust/main/agents/relayer/Cargo.toml +++ b/rust/main/agents/relayer/Cargo.toml @@ -56,9 +56,11 @@ hyperlane-ethereum = { path = "../../chains/hyperlane-ethereum" } once_cell.workspace = true mockall.workspace = true tokio-test.workspace = true +tracing-test.workspace = true hyperlane-test = { path = "../../hyperlane-test" } hyperlane-base = { path = "../../hyperlane-base", features = ["test-utils"] } hyperlane-core = { path = "../../hyperlane-core", features = ["agent", "async", "test-utils"] } +ethers-prometheus = { path = "../../ethers-prometheus", features = ["serde"] } [features] default = ["color-eyre", "oneline-errors"] diff --git a/rust/main/agents/relayer/src/relayer.rs b/rust/main/agents/relayer/src/relayer.rs index d91117c1d7..e05549fb61 100644 --- a/rust/main/agents/relayer/src/relayer.rs +++ b/rust/main/agents/relayer/src/relayer.rs @@ -18,8 +18,8 @@ use hyperlane_base::{ }; use hyperlane_core::{ rpc_clients::call_and_retry_n_times, ChainCommunicationError, ContractSyncCursor, - HyperlaneDomain, HyperlaneMessage, InterchainGasPayment, MerkleTreeInsertion, QueueOperation, - H512, U256, + HyperlaneDomain, HyperlaneMessage, InterchainGasPayment, Mailbox, MerkleTreeInsertion, + QueueOperation, ValidatorAnnounce, H512, U256, }; use tokio::{ sync::{ @@ -135,12 +135,10 @@ impl BaseAgent for Relayer { .map(|origin| (origin.clone(), HyperlaneRocksDB::new(origin, db.clone()))) .collect::>(); - let mailboxes = settings - .build_mailboxes(settings.destination_chains.iter(), &core_metrics) - .await?; - let validator_announces = settings - .build_validator_announces(settings.origin_chains.iter(), &core_metrics) - .await?; + let mailboxes = Self::build_mailboxes(&settings, &core_metrics, &chain_metrics).await; + + let validator_announces = + Self::build_validator_announces(&settings, &core_metrics, &chain_metrics).await; let contract_sync_metrics = Arc::new(ContractSyncMetrics::new(&core_metrics)); @@ -236,7 +234,9 @@ impl BaseAgent for Relayer { let mut msg_ctxs = HashMap::new(); let mut destination_chains = HashMap::new(); - for destination in &settings.destination_chains { + + // only iterate through destination chains that were successfully instantiated + for (destination, dest_mailbox) in mailboxes.iter() { let destination_chain_setup = core.settings.chain_setup(destination).unwrap().clone(); destination_chains.insert(destination.clone(), destination_chain_setup.clone()); let transaction_gas_limit: Option = @@ -246,18 +246,19 @@ impl BaseAgent for Relayer { transaction_gas_limit }; - for origin in &settings.origin_chains { + // only iterate through origin chains that were successfully instantiated + for (origin, validator_announce) in validator_announces.iter() { let db = dbs.get(origin).unwrap().clone(); let metadata_builder = BaseMetadataBuilder::new( origin.clone(), destination_chain_setup.clone(), prover_syncs[origin].clone(), - validator_announces[origin].clone(), + validator_announce.clone(), settings.allow_local_checkpoint_syncers, core.metrics.clone(), db, IsmAwareAppContextClassifier::new( - mailboxes[destination].clone(), + dest_mailbox.clone(), settings.metric_app_contexts.clone(), ), ); @@ -268,7 +269,7 @@ impl BaseAgent for Relayer { destination: destination.id(), }, Arc::new(MessageContext { - destination_mailbox: mailboxes[destination].clone(), + destination_mailbox: dest_mailbox.clone(), origin_db: dbs.get(origin).unwrap().clone(), metadata_builder: Arc::new(metadata_builder), origin_gas_payment_enforcer: gas_payment_enforcers[origin].clone(), @@ -620,7 +621,275 @@ impl Relayer { })) .instrument(span) } + + /// Helper function to build and return a hashmap of mailboxes. + /// Any chains that fail to build mailbox will not be included + /// in the hashmap. Errors will be logged and chain metrics + /// will be updated for chains that fail to build mailbox. + pub async fn build_mailboxes( + settings: &RelayerSettings, + core_metrics: &CoreMetrics, + chain_metrics: &ChainMetrics, + ) -> HashMap> { + settings + .build_mailboxes(settings.destination_chains.iter(), core_metrics) + .await + .into_iter() + .filter_map(|(origin, mailbox_res)| match mailbox_res { + Ok(mailbox) => Some((origin, mailbox)), + Err(err) => { + error!(?err, origin=?origin, "Critical error when building mailbox"); + chain_metrics.set_critical_error(origin.name(), true); + None + } + }) + .collect() + } + + /// Helper function to build and return a hashmap of validator announces. + /// Any chains that fail to build validator announce will not be included + /// in the hashmap. Errors will be logged and chain metrics + /// will be updated for chains that fail to build validator announce. + pub async fn build_validator_announces( + settings: &RelayerSettings, + core_metrics: &CoreMetrics, + chain_metrics: &ChainMetrics, + ) -> HashMap> { + settings + .build_validator_announces(settings.origin_chains.iter(), core_metrics) + .await + .into_iter() + .filter_map(|(origin, mailbox_res)| match mailbox_res { + Ok(mailbox) => Some((origin, mailbox)), + Err(err) => { + error!(?err, origin=?origin, "Critical error when building validator announce"); + chain_metrics.set_critical_error(origin.name(), true); + None + } + }) + .collect() + } } #[cfg(test)] -mod test {} +mod test { + use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + }; + + use crate::settings::{matching_list::MatchingList, RelayerSettings}; + use ethers::utils::hex; + use ethers_prometheus::middleware::PrometheusMiddlewareConf; + use hyperlane_base::{ + settings::{ + ChainConf, ChainConnectionConf, CoreContractAddresses, IndexSettings, Settings, + TracingConfig, + }, + ChainMetrics, CoreMetrics, BLOCK_HEIGHT_HELP, BLOCK_HEIGHT_LABELS, CRITICAL_ERROR_HELP, + CRITICAL_ERROR_LABELS, + }; + use hyperlane_core::{ + config::OperationBatchConfig, HyperlaneDomain, IndexMode, KnownHyperlaneDomain, + ReorgPeriod, H256, + }; + use hyperlane_ethereum as h_eth; + use prometheus::{opts, IntGaugeVec, Registry}; + use reqwest::Url; + + use super::Relayer; + + /// Builds a test RelayerSetting + fn generate_test_relayer_settings() -> RelayerSettings { + let chains = [( + "arbitrum".to_string(), + ChainConf { + domain: HyperlaneDomain::Known(KnownHyperlaneDomain::Arbitrum), + signer: None, + reorg_period: ReorgPeriod::None, + addresses: CoreContractAddresses { + mailbox: H256::from_slice( + hex::decode( + "000000000000000000000000598facE78a4302f11E3de0bee1894Da0b2Cb71F8", + ) + .unwrap() + .as_slice(), + ), + interchain_gas_paymaster: H256::from_slice( + hex::decode( + "000000000000000000000000c756cFc1b7d0d4646589EDf10eD54b201237F5e8", + ) + .unwrap() + .as_slice(), + ), + validator_announce: H256::from_slice( + hex::decode( + "0000000000000000000000001b33611fCc073aB0737011d5512EF673Bff74962", + ) + .unwrap() + .as_slice(), + ), + merkle_tree_hook: H256::from_slice( + hex::decode( + "000000000000000000000000AD34A66Bf6dB18E858F6B686557075568c6E031C", + ) + .unwrap() + .as_slice(), + ), + }, + connection: ChainConnectionConf::Ethereum(h_eth::ConnectionConf { + rpc_connection: h_eth::RpcConnectionConf::Http { + url: Url::parse("https://sepolia-rollup.arbitrum.io/rpc").unwrap(), + }, + transaction_overrides: h_eth::TransactionOverrides { + gas_price: None, + gas_limit: None, + max_fee_per_gas: None, + max_priority_fee_per_gas: None, + }, + operation_batch: OperationBatchConfig { + batch_contract_address: None, + max_batch_size: 1, + }, + }), + metrics_conf: PrometheusMiddlewareConf { + contracts: HashMap::new(), + chain: None, + }, + index: IndexSettings { + from: 0, + chunk_size: 1, + mode: IndexMode::Block, + }, + }, + )]; + + RelayerSettings { + base: Settings { + chains: chains.into_iter().collect(), + metrics_port: 5000, + tracing: TracingConfig::default(), + }, + db: PathBuf::new(), + origin_chains: [ + HyperlaneDomain::Known(KnownHyperlaneDomain::Arbitrum), + HyperlaneDomain::Known(KnownHyperlaneDomain::Ethereum), + HyperlaneDomain::Known(KnownHyperlaneDomain::Optimism), + ] + .into_iter() + .collect(), + destination_chains: [ + HyperlaneDomain::Known(KnownHyperlaneDomain::Arbitrum), + HyperlaneDomain::Known(KnownHyperlaneDomain::Ethereum), + HyperlaneDomain::Known(KnownHyperlaneDomain::Optimism), + ] + .into_iter() + .collect(), + gas_payment_enforcement: Vec::new(), + whitelist: MatchingList::default(), + blacklist: MatchingList::default(), + address_blacklist: Vec::new(), + transaction_gas_limit: None, + skip_transaction_gas_limit_for: HashSet::new(), + allow_local_checkpoint_syncers: true, + metric_app_contexts: Vec::new(), + } + } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_failed_build_mailboxes() { + let settings = generate_test_relayer_settings(); + + let registry = Registry::new(); + let core_metrics = CoreMetrics::new("relayer", 4000, registry).unwrap(); + let chain_metrics = ChainMetrics { + block_height: IntGaugeVec::new( + opts!("block_height", BLOCK_HEIGHT_HELP), + BLOCK_HEIGHT_LABELS, + ) + .unwrap(), + gas_price: None, + critical_error: IntGaugeVec::new( + opts!("critical_error", CRITICAL_ERROR_HELP), + CRITICAL_ERROR_LABELS, + ) + .unwrap(), + }; + + let mailboxes = Relayer::build_mailboxes(&settings, &core_metrics, &chain_metrics).await; + + assert_eq!(mailboxes.len(), 1); + assert!(mailboxes.contains_key(&HyperlaneDomain::Known(KnownHyperlaneDomain::Arbitrum))); + + // Arbitrum chain should not have any errors because it's ChainConf exists + let metric = chain_metrics + .critical_error + .get_metric_with_label_values(&["arbitrum"]) + .unwrap(); + assert_eq!(metric.get(), 0); + + // Ethereum chain should error because it is missing ChainConf + let metric = chain_metrics + .critical_error + .get_metric_with_label_values(&["ethereum"]) + .unwrap(); + assert_eq!(metric.get(), 1); + + // Optimism chain should error because it is missing ChainConf + let metric = chain_metrics + .critical_error + .get_metric_with_label_values(&["optimism"]) + .unwrap(); + assert_eq!(metric.get(), 1); + } + + #[tokio::test] + #[tracing_test::traced_test] + async fn test_failed_build_validator_announces() { + let settings = generate_test_relayer_settings(); + + let registry = Registry::new(); + let core_metrics = CoreMetrics::new("relayer", 4000, registry).unwrap(); + let chain_metrics = ChainMetrics { + block_height: IntGaugeVec::new( + opts!("block_height", BLOCK_HEIGHT_HELP), + BLOCK_HEIGHT_LABELS, + ) + .unwrap(), + gas_price: None, + critical_error: IntGaugeVec::new( + opts!("critical_error", CRITICAL_ERROR_HELP), + CRITICAL_ERROR_LABELS, + ) + .unwrap(), + }; + + let mailboxes = + Relayer::build_validator_announces(&settings, &core_metrics, &chain_metrics).await; + + assert_eq!(mailboxes.len(), 1); + assert!(mailboxes.contains_key(&HyperlaneDomain::Known(KnownHyperlaneDomain::Arbitrum))); + + // Arbitrum chain should not have any errors because it's ChainConf exists + let metric = chain_metrics + .critical_error + .get_metric_with_label_values(&["arbitrum"]) + .unwrap(); + assert_eq!(metric.get(), 0); + + // Ethereum chain should error because it is missing ChainConf + let metric = chain_metrics + .critical_error + .get_metric_with_label_values(&["ethereum"]) + .unwrap(); + assert_eq!(metric.get(), 1); + + // Optimism chain should error because it is missing ChainConf + let metric = chain_metrics + .critical_error + .get_metric_with_label_values(&["optimism"]) + .unwrap(); + assert_eq!(metric.get(), 1); + } +} diff --git a/rust/main/agents/relayer/src/settings/mod.rs b/rust/main/agents/relayer/src/settings/mod.rs index 2d3d9375de..a8a5c1ca4c 100644 --- a/rust/main/agents/relayer/src/settings/mod.rs +++ b/rust/main/agents/relayer/src/settings/mod.rs @@ -33,7 +33,7 @@ pub struct RelayerSettings { #[as_mut] #[deref] #[deref_mut] - base: Settings, + pub base: Settings, /// Database path pub db: PathBuf, diff --git a/rust/main/hyperlane-base/src/metrics/agent_metrics.rs b/rust/main/hyperlane-base/src/metrics/agent_metrics.rs index ad1320488f..2f4862d4aa 100644 --- a/rust/main/hyperlane-base/src/metrics/agent_metrics.rs +++ b/rust/main/hyperlane-base/src/metrics/agent_metrics.rs @@ -94,7 +94,7 @@ pub struct ChainMetrics { pub gas_price: Option, /// Boolean marker for critical errors on a chain, signalling loss of liveness. - critical_error: IntGaugeVec, + pub critical_error: IntGaugeVec, } impl ChainMetrics { diff --git a/rust/main/hyperlane-base/src/settings/base.rs b/rust/main/hyperlane-base/src/settings/base.rs index 7024d67cc6..d298a3c94f 100644 --- a/rust/main/hyperlane-base/src/settings/base.rs +++ b/rust/main/hyperlane-base/src/settings/base.rs @@ -1,9 +1,9 @@ use std::{collections::HashMap, fmt::Debug, hash::Hash, sync::Arc}; use eyre::{eyre, Context, Result}; -use futures_util::future::try_join_all; +use futures_util::future::join_all; use hyperlane_core::{ - HyperlaneChain, HyperlaneDomain, HyperlaneLogStore, HyperlaneProvider, + HyperlaneDomain, HyperlaneLogStore, HyperlaneProvider, HyperlaneSequenceAwareIndexerStoreReader, HyperlaneWatermarkedLogStore, InterchainGasPaymaster, Mailbox, MerkleTreeHook, MultisigIsm, SequenceAwareIndexer, ValidatorAnnounce, H256, }; @@ -132,11 +132,11 @@ macro_rules! build_contract_fns { &self, domains: impl Iterator, metrics: &CoreMetrics, - ) -> Result>> { - try_join_all(domains.map(|d| self.$singular(d, metrics))) - .await? + ) -> HashMap>> { + join_all(domains.map(|d| async { (d.clone(), self.$singular(d, metrics).await) })) + .await .into_iter() - .map(|i| Ok((i.domain().clone(), Arc::from(i)))) + .map(|(d, future)| (d, future.map(|f| Arc::from(f)))) .collect() } }; From 9eb19cac7a382a5bc1c9b88ebd52f1ec90330839 Mon Sep 17 00:00:00 2001 From: Jason Guo <33064781+Xaroz@users.noreply.github.com> Date: Wed, 15 Jan 2025 09:13:26 -0600 Subject: [PATCH 4/7] feat: explorer link in cli logs (#5170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description Add link to explorer message for hyperlane CLI `warp send` and `send message` commands ### Drive-by changes No ### Related issues ### Backward compatibility Yes ### Testing CLI send message command: ![Screenshot 2025-01-14 at 3 30 51 PM](https://github.com/user-attachments/assets/dac9a9ae-0b19-43a2-8d12-6c15551a1b50) warp send command: ![Screenshot 2025-01-14 at 3 32 32 PM](https://github.com/user-attachments/assets/afb114fd-9397-40bd-a29a-3d6e07f5938d) --- .changeset/three-walls-count.md | 5 +++++ typescript/cli/src/consts.ts | 1 + typescript/cli/src/send/message.ts | 3 ++- typescript/cli/src/send/transfer.ts | 3 ++- 4 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 .changeset/three-walls-count.md diff --git a/.changeset/three-walls-count.md b/.changeset/three-walls-count.md new file mode 100644 index 0000000000..37145b9838 --- /dev/null +++ b/.changeset/three-walls-count.md @@ -0,0 +1,5 @@ +--- +'@hyperlane-xyz/cli': minor +--- + +Add explorer link to warp send and send message commands diff --git a/typescript/cli/src/consts.ts b/typescript/cli/src/consts.ts index 9fb27df9f2..9c931cab26 100644 --- a/typescript/cli/src/consts.ts +++ b/typescript/cli/src/consts.ts @@ -3,3 +3,4 @@ export const MINIMUM_WARP_DEPLOY_GAS = (3e7).toString(); export const MINIMUM_TEST_SEND_GAS = (3e5).toString(); export const MINIMUM_AVS_GAS = (3e6).toString(); export const PROXY_DEPLOYED_URL = 'https://proxy.hyperlane.xyz'; +export const EXPLORER_URL = 'https://explorer.hyperlane.xyz'; diff --git a/typescript/cli/src/send/message.ts b/typescript/cli/src/send/message.ts index e43d08fe2c..b6d75b4e51 100644 --- a/typescript/cli/src/send/message.ts +++ b/typescript/cli/src/send/message.ts @@ -3,7 +3,7 @@ import { stringify as yamlStringify } from 'yaml'; import { ChainName, HyperlaneCore, HyperlaneRelayer } from '@hyperlane-xyz/sdk'; import { addressToBytes32, timeout } from '@hyperlane-xyz/utils'; -import { MINIMUM_TEST_SEND_GAS } from '../consts.js'; +import { EXPLORER_URL, MINIMUM_TEST_SEND_GAS } from '../consts.js'; import { CommandContext, WriteCommandContext } from '../context/types.js'; import { runPreflightChecksForChains } from '../deploy/utils.js'; import { errorRed, log, logBlue, logGreen } from '../logger.js'; @@ -102,6 +102,7 @@ async function executeDelivery({ ); logBlue(`Sent message from ${origin} to ${recipient} on ${destination}.`); logBlue(`Message ID: ${message.id}`); + logBlue(`Explorer Link: ${EXPLORER_URL}/message/${message.id}`); log(`Message:\n${indentYamlOrJson(yamlStringify(message, null, 2), 4)}`); if (selfRelay) { diff --git a/typescript/cli/src/send/transfer.ts b/typescript/cli/src/send/transfer.ts index 8aa4cca59b..1fabc3c853 100644 --- a/typescript/cli/src/send/transfer.ts +++ b/typescript/cli/src/send/transfer.ts @@ -14,7 +14,7 @@ import { } from '@hyperlane-xyz/sdk'; import { parseWarpRouteMessage, timeout } from '@hyperlane-xyz/utils'; -import { MINIMUM_TEST_SEND_GAS } from '../consts.js'; +import { EXPLORER_URL, MINIMUM_TEST_SEND_GAS } from '../consts.js'; import { WriteCommandContext } from '../context/types.js'; import { runPreflightChecksForChains } from '../deploy/utils.js'; import { log, logBlue, logGreen, logRed } from '../logger.js'; @@ -167,6 +167,7 @@ async function executeDelivery({ `Sent transfer from sender (${signerAddress}) on ${origin} to recipient (${recipient}) on ${destination}.`, ); logBlue(`Message ID: ${message.id}`); + logBlue(`Explorer Link: ${EXPLORER_URL}/message/${message.id}`); log(`Message:\n${indentYamlOrJson(yamlStringify(message, null, 2), 4)}`); log(`Body:\n${indentYamlOrJson(yamlStringify(parsed, null, 2), 4)}`); From b9bc57fbaaa06a1fbf9d5193be3b90d49cb8c9c2 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:34:15 +0000 Subject: [PATCH 5/7] chore: retryable errors should still be logged as warnings (#5184) ### Description Retryable errors often indicate that something abnormal is happening (such as we falling back to other providers, which may be causing issues). It's much easier to notice this if the logs are emitted are `warn` level. Non-retryable errors should be logged at `error` level to highlight the relative severity to retryable ones --- rust/main/chains/hyperlane-ethereum/src/rpc_clients/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/main/chains/hyperlane-ethereum/src/rpc_clients/mod.rs b/rust/main/chains/hyperlane-ethereum/src/rpc_clients/mod.rs index d7cdb80cd4..375f5cedda 100644 --- a/rust/main/chains/hyperlane-ethereum/src/rpc_clients/mod.rs +++ b/rust/main/chains/hyperlane-ethereum/src/rpc_clients/mod.rs @@ -1,5 +1,5 @@ use ethers::providers::HttpClientError; -use tracing::{info, trace, warn}; +use tracing::{error, info, trace, warn}; pub use self::{fallback::*, provider::*, retrying::*, trait_builder::*}; @@ -86,12 +86,12 @@ fn categorize_client_response( // We don't want to retry errors that are probably not going to work if we keep // retrying them or that indicate an error in higher-order logic and not // transient provider (connection or other) errors. - warn!(error=%e, "Non-retryable JsonRpcError in http provider"); + error!(error=%e, "Non-retryable JsonRpcError in http provider"); NonRetryableErr(JsonRpcError(e)) } else { // the assumption is this is not a "provider error" but rather an invalid // request, e.g. nonce too low, not enough gas, ... - info!(error=%e, "Retryable JsonRpcError in http provider"); + warn!(error=%e, "Retryable JsonRpcError in http provider"); RetryableErr(JsonRpcError(e)) } } From 1536ea57022b3dcee1ae56dc485311c030ec1766 Mon Sep 17 00:00:00 2001 From: Paul Balaji <10051819+paulbalaji@users.noreply.github.com> Date: Wed, 15 Jan 2025 19:36:01 +0000 Subject: [PATCH 6/7] fix: print displayName instead of chain name in cli log (#5185) ### Description fix: print displayName instead of chain name in cli log ### Drive-by changes ### Related issues ### Backward compatibility ### Testing --- .changeset/green-schools-attack.md | 5 +++++ typescript/cli/src/deploy/utils.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/green-schools-attack.md diff --git a/.changeset/green-schools-attack.md b/.changeset/green-schools-attack.md new file mode 100644 index 0000000000..e795b807f0 --- /dev/null +++ b/.changeset/green-schools-attack.md @@ -0,0 +1,5 @@ +--- +'@hyperlane-xyz/cli': patch +--- + +Print displayName instead of chain name in signer validity logs. diff --git a/typescript/cli/src/deploy/utils.ts b/typescript/cli/src/deploy/utils.ts index 0378ab7243..1f11f52596 100644 --- a/typescript/cli/src/deploy/utils.ts +++ b/typescript/cli/src/deploy/utils.ts @@ -51,7 +51,7 @@ export async function runPreflightChecksForChains({ throw new Error('Only Ethereum chains are supported for now'); const signer = multiProvider.getSigner(chain); assertSigner(signer); - logGreen(`✅ ${chain} signer is valid`); + logGreen(`✅ ${metadata.displayName ?? chain} signer is valid`); } logGreen('✅ Chains are valid'); From 769cec1f93227816b765a3d13d7cc608cc87523f Mon Sep 17 00:00:00 2001 From: J M Rossy Date: Wed, 15 Jan 2025 17:50:22 -0300 Subject: [PATCH 7/7] chore: Add/clarify log in ISM and Hook deployer (#5188) ### Description Since there's currently no other good way to track deployment progress, the Deploy App watches SDK log messages for its progress indicator. This adds a few logs to help it. ### Backward compatibility Yes --- typescript/sdk/src/hook/EvmHookModule.ts | 2 ++ typescript/sdk/src/ism/HyperlaneIsmFactory.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/typescript/sdk/src/hook/EvmHookModule.ts b/typescript/sdk/src/hook/EvmHookModule.ts index eddbb1c788..45235304a6 100644 --- a/typescript/sdk/src/hook/EvmHookModule.ts +++ b/typescript/sdk/src/hook/EvmHookModule.ts @@ -631,6 +631,8 @@ export class EvmHookModule extends HyperlaneModule< ); } + this.logger.debug(`Deploying hook of type ${config.type}`); + switch (config.type) { case HookType.MERKLE_TREE: return this.deployer.deployContract(this.chain, HookType.MERKLE_TREE, [ diff --git a/typescript/sdk/src/ism/HyperlaneIsmFactory.ts b/typescript/sdk/src/ism/HyperlaneIsmFactory.ts index c8c987a5d9..515a2da0a9 100644 --- a/typescript/sdk/src/ism/HyperlaneIsmFactory.ts +++ b/typescript/sdk/src/ism/HyperlaneIsmFactory.ts @@ -131,7 +131,7 @@ export class HyperlaneIsmFactory extends HyperlaneApp { const logger = this.logger.child({ destination, ismType }); logger.debug( - `Deploying ${ismType} to ${destination} ${ + `Deploying ISM of type ${ismType} to ${destination} ${ origin ? `(for verifying ${origin})` : '' }`, );