diff --git a/scripts/fork_fuzz_bots.py b/scripts/fork_fuzz_bots.py index d6df3492bb..7f1c8850ec 100644 --- a/scripts/fork_fuzz_bots.py +++ b/scripts/fork_fuzz_bots.py @@ -228,6 +228,8 @@ def main(argv: Sequence[str] | None = None) -> None: crash_log_level=logging.ERROR, crash_report_additional_info={"rng_seed": rng_seed}, gas_limit=int(3e6), # Plenty of gas limit for transactions + # There's an issue with oracles getting out of date, so we don't advance time when fuzzing + block_timestamp_interval=0, ) while True: diff --git a/src/agent0/core/hyperdrive/interactive/local_hyperdrive.py b/src/agent0/core/hyperdrive/interactive/local_hyperdrive.py index 6752a87346..259da046e6 100644 --- a/src/agent0/core/hyperdrive/interactive/local_hyperdrive.py +++ b/src/agent0/core/hyperdrive/interactive/local_hyperdrive.py @@ -549,6 +549,7 @@ def _create_checkpoint( self.chain.get_deployer_account(), checkpoint_time=checkpoint_time, gas_limit=gas_limit, + preview=self.chain.config.preview_before_trade, ) except AssertionError as exc: # Adding additional context to the "Transaction receipt has no logs" error diff --git a/src/agent0/hyperfuzz/fork_fuzz/accrue_interest_ezeth.py b/src/agent0/hyperfuzz/fork_fuzz/accrue_interest_ezeth.py index 8986b5783a..0614599195 100644 --- a/src/agent0/hyperfuzz/fork_fuzz/accrue_interest_ezeth.py +++ b/src/agent0/hyperfuzz/fork_fuzz/accrue_interest_ezeth.py @@ -18,7 +18,9 @@ DEPOSIT_QUEUE_ADDR = "0xf2F305D14DCD8aaef887E0428B3c9534795D0d60" -def accrue_interest_ezeth(interface: HyperdriveReadWriteInterface, variable_rate: FixedPoint) -> None: +def accrue_interest_ezeth( + interface: HyperdriveReadWriteInterface, variable_rate: FixedPoint, block_number_before_advance: int +) -> None: """Function to accrue interest in the ezeth pool when fork fuzzing. Arguments @@ -27,58 +29,50 @@ def accrue_interest_ezeth(interface: HyperdriveReadWriteInterface, variable_rate The interface to the Hyperdrive pool. variable_rate: FixedPoint The variable rate of the pool. + block_number_before_advance: int + The block number before time was advanced. """ + # pylint: disable=too-many-locals assert variable_rate > FixedPoint(0) # TODO we may want to build these objects once and store them restake_manager = IRestakeManagerContract.factory(w3=interface.web3)(Web3.to_checksum_address(RESTAKE_MANAGER_ADDR)) deposit_queue = IDepositQueueContract.factory(w3=interface.web3)(Web3.to_checksum_address(DEPOSIT_QUEUE_ADDR)) + # TODO we probably just want to do this once + response = interface.web3.provider.make_request( + method=RPCEndpoint("anvil_impersonateAccount"), params=[RESTAKE_MANAGER_ADDR] + ) + # ensure response is valid + if "result" not in response: + raise KeyError("Response did not have a result.") # There's a current issue with pypechain where it breaks if the called function returns a double nested list, # e.g., in calculateTVLs. We fall back to using pure web3 for this. # https://github.com/delvtech/pypechain/issues/147 - total_tvl = restake_manager.get_function_by_name("calculateTVLs")().call()[2] - - # Build accrue_interest_data - accrue_interest_data = { - "block_timestamp": interface.get_block_timestamp(interface.get_current_block()), - # 3rd arg is total tvl - "total_tvl": FixedPoint(scaled_value=total_tvl), - } - - # TODO we hack in a stateful variable into the interface here to check - # how much time has advanced between subsequent calls. - # Initial call, we look to see if the attribute exists - previous_accrue_interest_data: dict | None = getattr(interface, "_accrue_interest_data", None) - # Always set the new state here - setattr(interface, "_previous_interest_accrual_time", accrue_interest_data) - - if previous_accrue_interest_data is None: - # Skip this check on initial call, not a failure - # On initial call, we impersonate the restake manager - response = interface.web3.provider.make_request( - method=RPCEndpoint("anvil_impersonateAccount"), params=[RESTAKE_MANAGER_ADDR] - ) - # ensure response is valid - if "result" not in response: - raise KeyError("Response did not have a result.") - - return + previous_total_tvl = FixedPoint( + scaled_value=restake_manager.get_function_by_name("calculateTVLs")().call( + block_identifier=block_number_before_advance + )[2] + ) + previous_timestamp = interface.get_block_timestamp(interface.get_block(block_number_before_advance)) + current_timestamp = interface.get_block_timestamp(interface.get_block("latest")) + time_delta = current_timestamp - previous_timestamp adjusted_variable_rate = FixedPoint( - scaled_value=FixedPointIntegerMath.mul_div_down( - variable_rate.scaled_value, interface.pool_config.position_duration, SECONDS_IN_YEAR - ) + scaled_value=FixedPointIntegerMath.mul_div_down(variable_rate.scaled_value, time_delta, SECONDS_IN_YEAR) ) - previous_total_tvl: FixedPoint = previous_accrue_interest_data["total_tvl"] - assert isinstance(previous_total_tvl, FixedPoint) eth_to_add = previous_total_tvl * adjusted_variable_rate # Give eth to restake manager curr_balance = FixedPoint(scaled_value=get_account_balance(interface.web3, RESTAKE_MANAGER_ADDR)) - _ = set_account_balance(interface.web3, RESTAKE_MANAGER_ADDR, (curr_balance + eth_to_add).scaled_value) + _ = set_account_balance( + interface.web3, + RESTAKE_MANAGER_ADDR, + # Give a little bit extra for gas + (curr_balance + eth_to_add + FixedPoint(0.01)).scaled_value, + ) # TODO we need a "transact and wait" function in pypechain, as we don't need to sign due to impersonation tx_func = deposit_queue.functions.depositETHFromProtocol() diff --git a/src/agent0/hyperfuzz/fork_fuzz/accrue_interest_fork.py b/src/agent0/hyperfuzz/fork_fuzz/accrue_interest_fork.py index 72c357c3b2..aa884a4390 100644 --- a/src/agent0/hyperfuzz/fork_fuzz/accrue_interest_fork.py +++ b/src/agent0/hyperfuzz/fork_fuzz/accrue_interest_fork.py @@ -7,7 +7,9 @@ from .accrue_interest_ezeth import accrue_interest_ezeth -def accrue_interest_fork(interface: HyperdriveReadWriteInterface, variable_rate: FixedPoint) -> None: +def accrue_interest_fork( + interface: HyperdriveReadWriteInterface, variable_rate: FixedPoint, block_number_before_advance: int +) -> None: """Function to accrue interest in underlying yields when fork fuzzing. This function looks at the kind of pool defined in the interface, and matches the correct accrual function to call. @@ -18,6 +20,8 @@ def accrue_interest_fork(interface: HyperdriveReadWriteInterface, variable_rate: The interface to the Hyperdrive pool. variable_rate: FixedPoint The variable rate of the pool. + block_number_before_advance: int + The block number before time was advanced. """ # Switch case for pool types for interest accrual @@ -25,4 +29,4 @@ def accrue_interest_fork(interface: HyperdriveReadWriteInterface, variable_rate: hyperdrive_kind = interface.hyperdrive_kind match hyperdrive_kind: case interface.HyperdriveKind.EZETH: - accrue_interest_ezeth(interface, variable_rate) + accrue_interest_ezeth(interface, variable_rate, block_number_before_advance) diff --git a/src/agent0/hyperfuzz/system_fuzz/invariant_checks.py b/src/agent0/hyperfuzz/system_fuzz/invariant_checks.py index 22469f5d5c..504faa70c0 100644 --- a/src/agent0/hyperfuzz/system_fuzz/invariant_checks.py +++ b/src/agent0/hyperfuzz/system_fuzz/invariant_checks.py @@ -22,6 +22,7 @@ LP_SHARE_PRICE_EPSILON = 1e-4 TOTAL_SHARES_EPSILON = 1e-9 NEGATIVE_INTEREST_EPSILON = FixedPoint(scaled_value=10) # 10 wei +PRESENT_VALUE_EPSILON = FixedPoint(scaled_value=1) # 1 wei def run_invariant_checks( @@ -114,7 +115,7 @@ def run_invariant_checks( # Info _check_base_balances(pool_state, interface.base_is_yield), # Critical (after diving down into steth failure) - _check_total_shares(pool_state), + _check_total_shares(interface, pool_state), # Critical _check_minimum_share_reserves(pool_state), # Critical @@ -140,7 +141,7 @@ def run_invariant_checks( results = [ _check_eth_balances(pool_state), _check_base_balances(pool_state, interface.base_is_yield), - _check_total_shares(pool_state), + _check_total_shares(interface, pool_state), _check_minimum_share_reserves(pool_state), _check_solvency(pool_state), _check_present_value_greater_than_idle_shares(interface, pool_state), @@ -403,7 +404,7 @@ def _check_minimum_share_reserves(pool_state: PoolState) -> InvariantCheckResult return InvariantCheckResults(failed, exception_message, exception_data, log_level=log_level) -def _check_total_shares(pool_state: PoolState) -> InvariantCheckResults: +def _check_total_shares(interface: HyperdriveReadInterface, pool_state: PoolState) -> InvariantCheckResults: # Total shares is correctly calculated failed = False exception_message = "" @@ -423,9 +424,15 @@ def _check_total_shares(pool_state: PoolState) -> InvariantCheckResults: ) actual_vault_shares = pool_state.vault_shares + # We use a slightly bigger tolerance for the wsteth-usda pool + if interface.hyperdrive_name == "ElementDAO 182 Day Morpho Blue wstETH/USDA Hyperdrive": + shares_epsilon = 1e-5 + else: + shares_epsilon = TOTAL_SHARES_EPSILON + # While the expected vault shares is a bit inaccurate, we're testing # solvency here, hence, we ensure that the actual vault shares >= expected vault shares - if actual_vault_shares < (expected_vault_shares - FixedPoint(str(TOTAL_SHARES_EPSILON))): + if actual_vault_shares < (expected_vault_shares - FixedPoint(str(shares_epsilon))): difference_in_wei = abs(expected_vault_shares.scaled_value - actual_vault_shares.scaled_value) exception_message = ( f"{actual_vault_shares=} is expected to be greater than {expected_vault_shares=}. {difference_in_wei=}. " @@ -471,7 +478,7 @@ def _check_present_value_greater_than_idle_shares( failed=True, exception_message=repr(e), exception_data=exception_data, log_level=logging.CRITICAL ) - if not present_value >= idle_shares: + if not present_value >= (idle_shares - PRESENT_VALUE_EPSILON): difference_in_wei = abs(present_value.scaled_value - idle_shares.scaled_value) exception_message = f"{present_value=} < {idle_shares=}, {difference_in_wei=}" exception_data["invariance_check:idle_shares"] = idle_shares diff --git a/src/agent0/hyperfuzz/system_fuzz/run_fuzz_bots.py b/src/agent0/hyperfuzz/system_fuzz/run_fuzz_bots.py index c2f100c4c2..535d1cd8c3 100644 --- a/src/agent0/hyperfuzz/system_fuzz/run_fuzz_bots.py +++ b/src/agent0/hyperfuzz/system_fuzz/run_fuzz_bots.py @@ -9,6 +9,7 @@ from fixedpointmath import FixedPoint from numpy.random import Generator from pypechain.core import PypechainCallException +from web3.types import RPCEndpoint from agent0 import Chain, Hyperdrive, LocalChain, LocalHyperdrive, PolicyZoo from agent0.chainsync.db.hyperdrive import get_trade_events @@ -18,7 +19,7 @@ from agent0.ethpy.hyperdrive import HyperdriveReadWriteInterface from agent0.hyperfuzz import FuzzAssertionException from agent0.hyperfuzz.system_fuzz.invariant_checks import run_invariant_checks -from agent0.hyperlogs.rollbar_utilities import log_rollbar_exception, log_rollbar_message +from agent0.hyperlogs.rollbar_utilities import log_rollbar_exception ONE_HOUR_IN_SECONDS = 60 * 60 ONE_DAY_IN_SECONDS = ONE_HOUR_IN_SECONDS * 24 @@ -197,9 +198,9 @@ def _check_trades_made_on_pool( ) if has_err: - error_message = "FuzzBots: " + error_message logging.error(error_message) - log_rollbar_message(error_message, logging.ERROR) + # We log message to get rollbar to group these messages together + log_rollbar_exception(ValueError(error_message), logging.ERROR, rollbar_log_prefix="FuzzBots:") def run_fuzz_bots( @@ -223,7 +224,7 @@ def run_fuzz_bots( num_iterations: int | None = None, lp_share_price_test: bool = False, whale_accounts: dict[ChecksumAddress, ChecksumAddress] | None = None, - accrue_interest_func: Callable[[HyperdriveReadWriteInterface, FixedPoint], None] | None = None, + accrue_interest_func: Callable[[HyperdriveReadWriteInterface, FixedPoint, int], None] | None = None, accrue_interest_rate: FixedPoint | None = None, ) -> None: """Runs fuzz bots on a hyperdrive pool. @@ -276,10 +277,10 @@ def run_fuzz_bots( A mapping between token -> whale addresses to use to fund the fuzz agent. If the token is not in the mapping, fuzzing will attempt to call `mint` on the token contract. Defaults to an empty mapping. - accrue_interest_func: Callable[[HyperdriveReadWriteInterface, FixedPoint], None] | None, optional + accrue_interest_func: Callable[[HyperdriveReadWriteInterface, FixedPoint, int], None] | None, optional A function that will accrue interest on the hyperdrive pool. This function will get called - before and after each set of trades, with the pool's hyperdrive interface and the variable rate - as an argument. It's up to the function itself to maintain the last time interest was accrued. + after advancing time, with the following signature: + `accrue_interest_func(hyperdrive_interface, variable_rate, block_number_before_advance)`. accrue_interest_rate: FixedPoint | None, optional The variable rate to be passed into the accrue_interest_func. Note this value is only used when forking, as variable interest is handled by a mock yield source when simulating. @@ -402,10 +403,6 @@ def run_fuzz_bots( if check_invariance and lp_share_price_test: pending_pool_state = pool.interface.get_hyperdrive_state("pending") - # Accrue interest before and after making the trades - if accrue_interest_func is not None and accrue_interest_rate is not None: - accrue_interest_func(pool.interface, accrue_interest_rate) - # Execute trades agent_trade = [] try: @@ -428,10 +425,6 @@ def run_fuzz_bots( # Otherwise, we ignore crashes, we want the bot to keep trading # These errors will get logged regardless - # Accrue interest before and after making the trades - if accrue_interest_func is not None and accrue_interest_rate is not None: - accrue_interest_func(pool.interface, accrue_interest_rate) - # Check invariance on every iteration if we're not doing lp_share_price_test. # Only check invariance if a trade was executed for lp_share_price_test. # This is because the lp_share_price_test requires a trade to be executed @@ -505,8 +498,31 @@ def run_fuzz_bots( if random_advance_time: # We only allow random advance time if the chain connected to the pool is a # LocalChain object - if isinstance(chain, LocalChain): - # The deployer pays gas for advancing time + if not isinstance(chain, LocalChain): + raise ValueError("Random advance time only allowed for pools deployed on LocalChain") + + # RNG should always exist, config's post_init should always + # initialize an rng object + assert chain.config.rng is not None + random_time = int(chain.config.rng.integers(*ADVANCE_TIME_SECONDS_RANGE)) + + if accrue_interest_func is not None and accrue_interest_rate is not None: + # Get block number before we advance time + block_number_before_advance = chain.block_number() + # There's issues around generating intermittent checkpoints with + # an out of date oracle, and we can't do any other contract calls + # until we accrue interest. Hence, we break up the call + # to accrue interest, then update the db. + # chain._advance_chain_time(random_time) # pylint: disable=protected-access + chain._web3.provider.make_request(method=RPCEndpoint("evm_increaseTime"), params=[random_time]) + + for pool in hyperdrive_pools: + accrue_interest_func(pool.interface, accrue_interest_rate, block_number_before_advance) + assert isinstance(pool, LocalHyperdrive) + pool._maybe_run_blocking_data_pipeline() # pylint: disable=protected-access + + else: + # The deployer pays gas for creating checkpoints when advancing time # We check the eth balance and refund if it runs low deployer_account = chain.get_deployer_account() deployer_agent_eth = hyperdrive_pools[0].interface.get_eth_base_balances(deployer_account)[0] @@ -514,11 +530,4 @@ def run_fuzz_bots( _ = set_account_balance( hyperdrive_pools[0].interface.web3, deployer_account.address, eth_budget_per_bot.scaled_value ) - # RNG should always exist, config's post_init should always - # initialize an rng object - assert chain.config.rng is not None - # TODO should there be an upper bound for advancing time? - random_time = int(chain.config.rng.integers(*ADVANCE_TIME_SECONDS_RANGE)) chain.advance_time(random_time, create_checkpoints=True) - else: - raise ValueError("Random advance time only allowed for pools deployed on LocalChain")