From abb5a8bc1850c2fea4f5c00ddc3c7d972d5e1244 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:10:24 +0000 Subject: [PATCH 1/2] fix: use middleware submission tweaks (#5100) ### Description Uses the short-term fixes described in this [design doc](https://www.notion.so/hyperlanexyz/State-of-EVM-provider-middleware-16c6d35200d680d8a6b5f0d32cd8c66b?pvs=4) and implemented in https://github.com/hyperlane-xyz/ethers-rs/pull/23 These fixes are expected to make the relayer self-recover from nonce gaps. An expected drawback is that tx hashes are dropped more often due to the nonce manager resyncing to nonces that end up being too low by 1, which can slow down submissions. After 12h of testing, any submit queue spikes were only 1 min long and the biggest one had 7 messages - so short enough not to trigger the "rising submit queue" alert which stays pending for 6min: https://abacusworks.grafana.net/alerting/de6hwplr7eakgc ### Drive-by changes - Removes verbosity from a bunch of logs (we had `HyperlaneMessage` in a single log occur 6-7 times, but wasn't noticeable in `span`s because it was originating from args named `message` which have a clash with inner workings of the `tracing` lib) - rewords some logs because they were misleading ### Related issues Fixes: https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/5107 ### Backward compatibility Yes ### Testing E2E with a gas escalator that randomly dropped every other tx - the nonce manager was able to recover but was slower to submit. Then evaluated on RC for 12h as described above. --- rust/main/Cargo.lock | 20 +++++++++---------- rust/main/Cargo.toml | 10 +++++----- .../relayer/src/msg/metadata/aggregation.rs | 2 +- .../agents/relayer/src/msg/metadata/base.rs | 4 ++-- .../relayer/src/msg/metadata/ccip_read.rs | 2 +- .../relayer/src/msg/metadata/routing.rs | 2 +- .../agents/relayer/src/msg/op_submitter.rs | 2 +- .../agents/relayer/src/msg/pending_message.rs | 2 +- .../src/ism/aggregation_ism.rs | 2 +- .../src/ism/multisig_ism.rs | 2 +- .../hyperlane-ethereum/src/ism/routing_ism.rs | 2 +- .../src/invariants/termination_invariants.rs | 5 ++++- 12 files changed, 29 insertions(+), 26 deletions(-) diff --git a/rust/main/Cargo.lock b/rust/main/Cargo.lock index 078b19fac9..aea92da917 100644 --- a/rust/main/Cargo.lock +++ b/rust/main/Cargo.lock @@ -2955,7 +2955,7 @@ dependencies = [ [[package]] name = "ethers" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "ethers-addressbook", "ethers-contract", @@ -2969,7 +2969,7 @@ dependencies = [ [[package]] name = "ethers-addressbook" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "ethers-core", "once_cell", @@ -2980,7 +2980,7 @@ dependencies = [ [[package]] name = "ethers-contract" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "ethers-contract-abigen", "ethers-contract-derive", @@ -2998,7 +2998,7 @@ dependencies = [ [[package]] name = "ethers-contract-abigen" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "Inflector", "cfg-if", @@ -3022,7 +3022,7 @@ dependencies = [ [[package]] name = "ethers-contract-derive" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "ethers-contract-abigen", "ethers-core", @@ -3036,7 +3036,7 @@ dependencies = [ [[package]] name = "ethers-core" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "arrayvec", "bytes", @@ -3066,7 +3066,7 @@ dependencies = [ [[package]] name = "ethers-etherscan" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "ethers-core", "getrandom 0.2.15", @@ -3082,7 +3082,7 @@ dependencies = [ [[package]] name = "ethers-middleware" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "async-trait", "auto_impl 0.5.0", @@ -3131,7 +3131,7 @@ dependencies = [ [[package]] name = "ethers-providers" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "async-trait", "auto_impl 1.2.0", @@ -3167,7 +3167,7 @@ dependencies = [ [[package]] name = "ethers-signers" version = "1.0.2" -source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-09#0afa80630b90fbc82ef42a0b3f26dd72e68285e4" +source = "git+https://github.com/hyperlane-xyz/ethers-rs?tag=2025-01-10#3ce1ce74f2d014a2f9df213084a700106e567740" dependencies = [ "async-trait", "coins-bip32 0.7.0", diff --git a/rust/main/Cargo.toml b/rust/main/Cargo.toml index db0ddd70ec..832bee0c42 100644 --- a/rust/main/Cargo.toml +++ b/rust/main/Cargo.toml @@ -198,27 +198,27 @@ overflow-checks = true [workspace.dependencies.ethers] features = [] git = "https://github.com/hyperlane-xyz/ethers-rs" -tag = "2025-01-09" +tag = "2025-01-10" [workspace.dependencies.ethers-contract] features = ["legacy"] git = "https://github.com/hyperlane-xyz/ethers-rs" -tag = "2025-01-09" +tag = "2025-01-10" [workspace.dependencies.ethers-core] features = [] git = "https://github.com/hyperlane-xyz/ethers-rs" -tag = "2025-01-09" +tag = "2025-01-10" [workspace.dependencies.ethers-providers] features = [] git = "https://github.com/hyperlane-xyz/ethers-rs" -tag = "2025-01-09" +tag = "2025-01-10" [workspace.dependencies.ethers-signers] features = ["aws"] git = "https://github.com/hyperlane-xyz/ethers-rs" -tag = "2025-01-09" +tag = "2025-01-10" [patch.crates-io.curve25519-dalek] branch = "v3.2.2-relax-zeroize" diff --git a/rust/main/agents/relayer/src/msg/metadata/aggregation.rs b/rust/main/agents/relayer/src/msg/metadata/aggregation.rs index 072ce62b65..1d4ecbc732 100644 --- a/rust/main/agents/relayer/src/msg/metadata/aggregation.rs +++ b/rust/main/agents/relayer/src/msg/metadata/aggregation.rs @@ -117,7 +117,7 @@ impl AggregationIsmMetadataBuilder { #[async_trait] impl MetadataBuilder for AggregationIsmMetadataBuilder { - #[instrument(err, skip(self), ret)] + #[instrument(err, skip(self, message), ret)] #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue async fn build( &self, diff --git a/rust/main/agents/relayer/src/msg/metadata/base.rs b/rust/main/agents/relayer/src/msg/metadata/base.rs index be10af6110..b2e4e5d185 100644 --- a/rust/main/agents/relayer/src/msg/metadata/base.rs +++ b/rust/main/agents/relayer/src/msg/metadata/base.rs @@ -189,7 +189,7 @@ impl Deref for MessageMetadataBuilder { #[async_trait] impl MetadataBuilder for MessageMetadataBuilder { - #[instrument(err, skip(self), fields(destination_domain=self.destination_domain().name()))] + #[instrument(err, skip(self, message), fields(destination_domain=self.destination_domain().name()))] async fn build( &self, ism_address: H256, @@ -228,7 +228,7 @@ impl MessageMetadataBuilder { } } - #[instrument(err, skip(self), fields(destination_domain=self.destination_domain().name()), ret)] + #[instrument(err, skip(self, message), fields(destination_domain=self.destination_domain().name()), ret)] pub async fn build_ism_and_metadata( &self, ism_address: H256, diff --git a/rust/main/agents/relayer/src/msg/metadata/ccip_read.rs b/rust/main/agents/relayer/src/msg/metadata/ccip_read.rs index 507271c123..49b15837df 100644 --- a/rust/main/agents/relayer/src/msg/metadata/ccip_read.rs +++ b/rust/main/agents/relayer/src/msg/metadata/ccip_read.rs @@ -27,7 +27,7 @@ pub struct CcipReadIsmMetadataBuilder { #[async_trait] impl MetadataBuilder for CcipReadIsmMetadataBuilder { - #[instrument(err, skip(self))] + #[instrument(err, skip(self, message))] async fn build( &self, ism_address: H256, diff --git a/rust/main/agents/relayer/src/msg/metadata/routing.rs b/rust/main/agents/relayer/src/msg/metadata/routing.rs index 0f319e8251..f8d5a145bc 100644 --- a/rust/main/agents/relayer/src/msg/metadata/routing.rs +++ b/rust/main/agents/relayer/src/msg/metadata/routing.rs @@ -14,7 +14,7 @@ pub struct RoutingIsmMetadataBuilder { #[async_trait] impl MetadataBuilder for RoutingIsmMetadataBuilder { - #[instrument(err, skip(self), ret)] + #[instrument(err, skip(self, message), ret)] #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue async fn build( &self, diff --git a/rust/main/agents/relayer/src/msg/op_submitter.rs b/rust/main/agents/relayer/src/msg/op_submitter.rs index c1e295a24a..1f0a54234c 100644 --- a/rust/main/agents/relayer/src/msg/op_submitter.rs +++ b/rust/main/agents/relayer/src/msg/op_submitter.rs @@ -543,7 +543,7 @@ impl OperationBatch { }; if !excluded_ops.is_empty() { - warn!(excluded_ops=?excluded_ops, "Either the batch tx would revert, or the operations would revert in the batch. Falling back to serial submission."); + warn!(excluded_ops=?excluded_ops, "Either operations reverted in the batch or the txid wasn't included. Falling back to serial submission."); OperationBatch::new(excluded_ops, self.domain) .submit_serially(prepare_queue, confirm_queue, metrics) .await; diff --git a/rust/main/agents/relayer/src/msg/pending_message.rs b/rust/main/agents/relayer/src/msg/pending_message.rs index 3c03094fc5..92d279856d 100644 --- a/rust/main/agents/relayer/src/msg/pending_message.rs +++ b/rust/main/agents/relayer/src/msg/pending_message.rs @@ -32,7 +32,7 @@ pub const CONFIRM_DELAY: Duration = if cfg!(any(test, feature = "test-utils")) { // Wait 5 seconds after submitting the message before confirming in test mode Duration::from_secs(5) } else { - // Wait 1 min after submitting the message before confirming in normal/production mode + // Wait 10 min after submitting the message before confirming in normal/production mode Duration::from_secs(60 * 10) }; diff --git a/rust/main/chains/hyperlane-ethereum/src/ism/aggregation_ism.rs b/rust/main/chains/hyperlane-ethereum/src/ism/aggregation_ism.rs index 61b371f3b7..0d7bae757f 100644 --- a/rust/main/chains/hyperlane-ethereum/src/ism/aggregation_ism.rs +++ b/rust/main/chains/hyperlane-ethereum/src/ism/aggregation_ism.rs @@ -92,7 +92,7 @@ impl AggregationIsm for EthereumAggregationIsm where M: Middleware + 'static, { - #[instrument(err)] + #[instrument(err, skip(self, message))] #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue async fn modules_and_threshold( &self, diff --git a/rust/main/chains/hyperlane-ethereum/src/ism/multisig_ism.rs b/rust/main/chains/hyperlane-ethereum/src/ism/multisig_ism.rs index 2c71c900c5..b9acb2075d 100644 --- a/rust/main/chains/hyperlane-ethereum/src/ism/multisig_ism.rs +++ b/rust/main/chains/hyperlane-ethereum/src/ism/multisig_ism.rs @@ -98,7 +98,7 @@ impl MultisigIsm for EthereumMultisigIsm where M: Middleware + 'static, { - #[instrument(err)] + #[instrument(err, skip(self, message))] #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue async fn validators_and_threshold( &self, diff --git a/rust/main/chains/hyperlane-ethereum/src/ism/routing_ism.rs b/rust/main/chains/hyperlane-ethereum/src/ism/routing_ism.rs index 39aaf7ecdb..4a9e8204d2 100644 --- a/rust/main/chains/hyperlane-ethereum/src/ism/routing_ism.rs +++ b/rust/main/chains/hyperlane-ethereum/src/ism/routing_ism.rs @@ -89,7 +89,7 @@ impl RoutingIsm for EthereumRoutingIsm where M: Middleware + 'static, { - #[instrument(err)] + #[instrument(err, skip(self, message))] #[allow(clippy::blocks_in_conditions)] // TODO: `rustc` 1.80.1 clippy issue async fn route(&self, message: &HyperlaneMessage) -> ChainResult { let ism = self diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index cb851af8d2..b6e299c1bc 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -39,7 +39,10 @@ pub fn termination_invariants_met( )?; assert!(!lengths.is_empty(), "Could not find queue length metric"); if lengths.iter().sum::() != ZERO_MERKLE_INSERTION_KATHY_MESSAGES { - log!("Relayer queues not empty. Lengths: {:?}", lengths); + log!( + "Relayer queues contain more messages than the zero-merkle-insertion ones. Lengths: {:?}", + lengths + ); return Ok(false); }; From 26ec628085c6e83ce944c789105e5096954479c2 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:29:37 +0000 Subject: [PATCH 2/2] chore: new relayer images (#5157) ### Description To include the changes from https://github.com/hyperlane-xyz/hyperlane-monorepo/pull/5100 --- typescript/infra/config/environments/mainnet3/agent.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/typescript/infra/config/environments/mainnet3/agent.ts b/typescript/infra/config/environments/mainnet3/agent.ts index 32898263bc..19b331cd90 100644 --- a/typescript/infra/config/environments/mainnet3/agent.ts +++ b/typescript/infra/config/environments/mainnet3/agent.ts @@ -652,7 +652,7 @@ const hyperlane: RootAgentConfig = { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: '53fafa6-20250110-125541', + tag: 'abb5a8b-20250113-122226', }, blacklist, gasPaymentEnforcement: gasPaymentEnforcement, @@ -687,7 +687,7 @@ const releaseCandidate: RootAgentConfig = { rpcConsensusType: RpcConsensusType.Fallback, docker: { repo, - tag: 'df9c1ed-20250109-151923', + tag: 'abb5a8b-20250113-122226', }, blacklist, // We're temporarily (ab)using the RC relayer as a way to increase