Minor fixes to fork fuzz (#1721)

- Set block timestamp interval to be 0 (which increments block time by 1
second per transaction) to avoid advancing time when fork fuzzing. This
is to bypass the `oraclePriceExpired` issue on ezETH.
- Use chain config preview before trade call when creating checkpoints.
- Using call on block for accruing interest for ezeth function instead
of hacking in stateful variable into interface (not currently being
used).
- Only accrue interest when advancing time (not currently being used).
- Increasing total shares epsilon for wstETH/USDA pool in invariance
checks.
- Adding an epsilon of 1 wei for present value invariance check.
- Logging "no trades on fork fuzz pool" as an exception to have rollbar
combine items.

scripts/fork_fuzz_bots.py

@@ -228,6 +228,8 @@ def main(argv: Sequence[str] | None = None) -> None:
         crash_log_level=logging.ERROR,
         crash_report_additional_info={"rng_seed": rng_seed},
         gas_limit=int(3e6),  # Plenty of gas limit for transactions
+        # There's an issue with oracles getting out of date, so we don't advance time when fuzzing
+        block_timestamp_interval=0,
     )
 
     while True:

src/agent0/core/hyperdrive/interactive/local_hyperdrive.py

@@ -549,6 +549,7 @@ def _create_checkpoint(
                 self.chain.get_deployer_account(),
                 checkpoint_time=checkpoint_time,
                 gas_limit=gas_limit,
+                preview=self.chain.config.preview_before_trade,
             )
         except AssertionError as exc:
             # Adding additional context to the "Transaction receipt has no logs" error

src/agent0/hyperfuzz/fork_fuzz/accrue_interest_ezeth.py

@@ -18,7 +18,9 @@
 DEPOSIT_QUEUE_ADDR = "0xf2F305D14DCD8aaef887E0428B3c9534795D0d60"
 
 
-def accrue_interest_ezeth(interface: HyperdriveReadWriteInterface, variable_rate: FixedPoint) -> None:
+def accrue_interest_ezeth(
+    interface: HyperdriveReadWriteInterface, variable_rate: FixedPoint, block_number_before_advance: int
+) -> None:
     """Function to accrue interest in the ezeth pool when fork fuzzing.
 
     Arguments
@@ -27,58 +29,50 @@ def accrue_interest_ezeth(interface: HyperdriveReadWriteInterface, variable_rate
         The interface to the Hyperdrive pool.
     variable_rate: FixedPoint
         The variable rate of the pool.
+    block_number_before_advance: int
+        The block number before time was advanced.
     """
+    # pylint: disable=too-many-locals
 
     assert variable_rate > FixedPoint(0)
 
     # TODO we may want to build these objects once and store them
     restake_manager = IRestakeManagerContract.factory(w3=interface.web3)(Web3.to_checksum_address(RESTAKE_MANAGER_ADDR))
     deposit_queue = IDepositQueueContract.factory(w3=interface.web3)(Web3.to_checksum_address(DEPOSIT_QUEUE_ADDR))
+    # TODO we probably just want to do this once
+    response = interface.web3.provider.make_request(
+        method=RPCEndpoint("anvil_impersonateAccount"), params=[RESTAKE_MANAGER_ADDR]
+    )
+    # ensure response is valid
+    if "result" not in response:
+        raise KeyError("Response did not have a result.")
 
     # There's a current issue with pypechain where it breaks if the called function returns a double nested list,
     # e.g., in calculateTVLs. We fall back to using pure web3 for this.
     # https://github.com/delvtech/pypechain/issues/147
-    total_tvl = restake_manager.get_function_by_name("calculateTVLs")().call()[2]
-
-    # Build accrue_interest_data
-    accrue_interest_data = {
-        "block_timestamp": interface.get_block_timestamp(interface.get_current_block()),
-        # 3rd arg is total tvl
-        "total_tvl": FixedPoint(scaled_value=total_tvl),
-    }
-
-    # TODO we hack in a stateful variable into the interface here to check
-    # how much time has advanced between subsequent calls.
-    # Initial call, we look to see if the attribute exists
-    previous_accrue_interest_data: dict | None = getattr(interface, "_accrue_interest_data", None)
-    # Always set the new state here
-    setattr(interface, "_previous_interest_accrual_time", accrue_interest_data)
-
-    if previous_accrue_interest_data is None:
-        # Skip this check on initial call, not a failure
-        # On initial call, we impersonate the restake manager
-        response = interface.web3.provider.make_request(
-            method=RPCEndpoint("anvil_impersonateAccount"), params=[RESTAKE_MANAGER_ADDR]
-        )
-        # ensure response is valid
-        if "result" not in response:
-            raise KeyError("Response did not have a result.")
-
-        return
+    previous_total_tvl = FixedPoint(
+        scaled_value=restake_manager.get_function_by_name("calculateTVLs")().call(
+            block_identifier=block_number_before_advance
+        )[2]
+    )
+    previous_timestamp = interface.get_block_timestamp(interface.get_block(block_number_before_advance))
+    current_timestamp = interface.get_block_timestamp(interface.get_block("latest"))
+    time_delta = current_timestamp - previous_timestamp
 
     adjusted_variable_rate = FixedPoint(
-        scaled_value=FixedPointIntegerMath.mul_div_down(
-            variable_rate.scaled_value, interface.pool_config.position_duration, SECONDS_IN_YEAR
-        )
+        scaled_value=FixedPointIntegerMath.mul_div_down(variable_rate.scaled_value, time_delta, SECONDS_IN_YEAR)
     )
 
-    previous_total_tvl: FixedPoint = previous_accrue_interest_data["total_tvl"]
-    assert isinstance(previous_total_tvl, FixedPoint)
     eth_to_add = previous_total_tvl * adjusted_variable_rate
 
     # Give eth to restake manager
     curr_balance = FixedPoint(scaled_value=get_account_balance(interface.web3, RESTAKE_MANAGER_ADDR))
-    _ = set_account_balance(interface.web3, RESTAKE_MANAGER_ADDR, (curr_balance + eth_to_add).scaled_value)
+    _ = set_account_balance(
+        interface.web3,
+        RESTAKE_MANAGER_ADDR,
+        # Give a little bit extra for gas
+        (curr_balance + eth_to_add + FixedPoint(0.01)).scaled_value,
+    )
 
     # TODO we need a "transact and wait" function in pypechain, as we don't need to sign due to impersonation
     tx_func = deposit_queue.functions.depositETHFromProtocol()

src/agent0/hyperfuzz/fork_fuzz/accrue_interest_fork.py

@@ -7,7 +7,9 @@
 from .accrue_interest_ezeth import accrue_interest_ezeth
 
 
-def accrue_interest_fork(interface: HyperdriveReadWriteInterface, variable_rate: FixedPoint) -> None:
+def accrue_interest_fork(
+    interface: HyperdriveReadWriteInterface, variable_rate: FixedPoint, block_number_before_advance: int
+) -> None:
     """Function to accrue interest in underlying yields when fork fuzzing.
     This function looks at the kind of pool defined in the interface, and
     matches the correct accrual function to call.
@@ -18,11 +20,13 @@ def accrue_interest_fork(interface: HyperdriveReadWriteInterface, variable_rate:
         The interface to the Hyperdrive pool.
     variable_rate: FixedPoint
         The variable rate of the pool.
+    block_number_before_advance: int
+        The block number before time was advanced.
     """
 
     # Switch case for pool types for interest accrual
     # TODO do we need to switch chain?
     hyperdrive_kind = interface.hyperdrive_kind
     match hyperdrive_kind:
         case interface.HyperdriveKind.EZETH:
-            accrue_interest_ezeth(interface, variable_rate)
+            accrue_interest_ezeth(interface, variable_rate, block_number_before_advance)

src/agent0/hyperfuzz/system_fuzz/invariant_checks.py

@@ -22,6 +22,7 @@
 LP_SHARE_PRICE_EPSILON = 1e-4
 TOTAL_SHARES_EPSILON = 1e-9
 NEGATIVE_INTEREST_EPSILON = FixedPoint(scaled_value=10)  # 10 wei
+PRESENT_VALUE_EPSILON = FixedPoint(scaled_value=1)  # 1 wei
 
 
 def run_invariant_checks(
@@ -114,7 +115,7 @@ def run_invariant_checks(
             # Info
             _check_base_balances(pool_state, interface.base_is_yield),
             # Critical (after diving down into steth failure)
-            _check_total_shares(pool_state),
+            _check_total_shares(interface, pool_state),
             # Critical
             _check_minimum_share_reserves(pool_state),
             # Critical
@@ -140,7 +141,7 @@ def run_invariant_checks(
             results = [
                 _check_eth_balances(pool_state),
                 _check_base_balances(pool_state, interface.base_is_yield),
-                _check_total_shares(pool_state),
+                _check_total_shares(interface, pool_state),
                 _check_minimum_share_reserves(pool_state),
                 _check_solvency(pool_state),
                 _check_present_value_greater_than_idle_shares(interface, pool_state),
@@ -403,7 +404,7 @@ def _check_minimum_share_reserves(pool_state: PoolState) -> InvariantCheckResult
     return InvariantCheckResults(failed, exception_message, exception_data, log_level=log_level)
 
 
-def _check_total_shares(pool_state: PoolState) -> InvariantCheckResults:
+def _check_total_shares(interface: HyperdriveReadInterface, pool_state: PoolState) -> InvariantCheckResults:
     # Total shares is correctly calculated
     failed = False
     exception_message = ""
@@ -423,9 +424,15 @@ def _check_total_shares(pool_state: PoolState) -> InvariantCheckResults:
     )
     actual_vault_shares = pool_state.vault_shares
 
+    # We use a slightly bigger tolerance for the wsteth-usda pool
+    if interface.hyperdrive_name == "ElementDAO 182 Day Morpho Blue wstETH/USDA Hyperdrive":
+        shares_epsilon = 1e-5
+    else:
+        shares_epsilon = TOTAL_SHARES_EPSILON
+
     # While the expected vault shares is a bit inaccurate, we're testing
     # solvency here, hence, we ensure that the actual vault shares >= expected vault shares
-    if actual_vault_shares < (expected_vault_shares - FixedPoint(str(TOTAL_SHARES_EPSILON))):
+    if actual_vault_shares < (expected_vault_shares - FixedPoint(str(shares_epsilon))):
         difference_in_wei = abs(expected_vault_shares.scaled_value - actual_vault_shares.scaled_value)
         exception_message = (
             f"{actual_vault_shares=} is expected to be greater than {expected_vault_shares=}. {difference_in_wei=}. "
@@ -471,7 +478,7 @@ def _check_present_value_greater_than_idle_shares(
             failed=True, exception_message=repr(e), exception_data=exception_data, log_level=logging.CRITICAL
         )
 
-    if not present_value >= idle_shares:
+    if not present_value >= (idle_shares - PRESENT_VALUE_EPSILON):
         difference_in_wei = abs(present_value.scaled_value - idle_shares.scaled_value)
         exception_message = f"{present_value=} < {idle_shares=}, {difference_in_wei=}"
         exception_data["invariance_check:idle_shares"] = idle_shares

src/agent0/hyperfuzz/system_fuzz/run_fuzz_bots.py

@@ -9,6 +9,7 @@
 from fixedpointmath import FixedPoint
 from numpy.random import Generator
 from pypechain.core import PypechainCallException
+from web3.types import RPCEndpoint
 
 from agent0 import Chain, Hyperdrive, LocalChain, LocalHyperdrive, PolicyZoo
 from agent0.chainsync.db.hyperdrive import get_trade_events
@@ -18,7 +19,7 @@
 from agent0.ethpy.hyperdrive import HyperdriveReadWriteInterface
 from agent0.hyperfuzz import FuzzAssertionException
 from agent0.hyperfuzz.system_fuzz.invariant_checks import run_invariant_checks
-from agent0.hyperlogs.rollbar_utilities import log_rollbar_exception, log_rollbar_message
+from agent0.hyperlogs.rollbar_utilities import log_rollbar_exception
 
 ONE_HOUR_IN_SECONDS = 60 * 60
 ONE_DAY_IN_SECONDS = ONE_HOUR_IN_SECONDS * 24
@@ -197,9 +198,9 @@ def _check_trades_made_on_pool(
                     )
 
             if has_err:
-                error_message = "FuzzBots: " + error_message
                 logging.error(error_message)
-                log_rollbar_message(error_message, logging.ERROR)
+                # We log message to get rollbar to group these messages together
+                log_rollbar_exception(ValueError(error_message), logging.ERROR, rollbar_log_prefix="FuzzBots:")
 
 
 def run_fuzz_bots(
@@ -223,7 +224,7 @@ def run_fuzz_bots(
     num_iterations: int | None = None,
     lp_share_price_test: bool = False,
     whale_accounts: dict[ChecksumAddress, ChecksumAddress] | None = None,
-    accrue_interest_func: Callable[[HyperdriveReadWriteInterface, FixedPoint], None] | None = None,
+    accrue_interest_func: Callable[[HyperdriveReadWriteInterface, FixedPoint, int], None] | None = None,
     accrue_interest_rate: FixedPoint | None = None,
 ) -> None:
     """Runs fuzz bots on a hyperdrive pool.
@@ -276,10 +277,10 @@ def run_fuzz_bots(
         A mapping between token -> whale addresses to use to fund the fuzz agent.
         If the token is not in the mapping, fuzzing will attempt to call `mint` on
         the token contract. Defaults to an empty mapping.
-    accrue_interest_func: Callable[[HyperdriveReadWriteInterface, FixedPoint], None] | None, optional
+    accrue_interest_func: Callable[[HyperdriveReadWriteInterface, FixedPoint, int], None] | None, optional
         A function that will accrue interest on the hyperdrive pool. This function will get called
-        before and after each set of trades, with the pool's hyperdrive interface and the variable rate
-        as an argument. It's up to the function itself to maintain the last time interest was accrued.
+        after advancing time, with the following signature:
+        `accrue_interest_func(hyperdrive_interface, variable_rate, block_number_before_advance)`.
     accrue_interest_rate: FixedPoint | None, optional
         The variable rate to be passed into the accrue_interest_func. Note this value is only used when
         forking, as variable interest is handled by a mock yield source when simulating.
@@ -402,10 +403,6 @@ def run_fuzz_bots(
                 if check_invariance and lp_share_price_test:
                     pending_pool_state = pool.interface.get_hyperdrive_state("pending")
 
-                # Accrue interest before and after making the trades
-                if accrue_interest_func is not None and accrue_interest_rate is not None:
-                    accrue_interest_func(pool.interface, accrue_interest_rate)
-
                 # Execute trades
                 agent_trade = []
                 try:
@@ -428,10 +425,6 @@ def run_fuzz_bots(
                     # Otherwise, we ignore crashes, we want the bot to keep trading
                     # These errors will get logged regardless
 
-                # Accrue interest before and after making the trades
-                if accrue_interest_func is not None and accrue_interest_rate is not None:
-                    accrue_interest_func(pool.interface, accrue_interest_rate)
-
                 # Check invariance on every iteration if we're not doing lp_share_price_test.
                 # Only check invariance if a trade was executed for lp_share_price_test.
                 # This is because the lp_share_price_test requires a trade to be executed
@@ -505,20 +498,36 @@ def run_fuzz_bots(
         if random_advance_time:
             # We only allow random advance time if the chain connected to the pool is a
             # LocalChain object
-            if isinstance(chain, LocalChain):
-                # The deployer pays gas for advancing time
+            if not isinstance(chain, LocalChain):
+                raise ValueError("Random advance time only allowed for pools deployed on LocalChain")
+
+            # RNG should always exist, config's post_init should always
+            # initialize an rng object
+            assert chain.config.rng is not None
+            random_time = int(chain.config.rng.integers(*ADVANCE_TIME_SECONDS_RANGE))
+
+            if accrue_interest_func is not None and accrue_interest_rate is not None:
+                # Get block number before we advance time
+                block_number_before_advance = chain.block_number()
+                # There's issues around generating intermittent checkpoints with
+                # an out of date oracle, and we can't do any other contract calls
+                # until we accrue interest. Hence, we break up the call
+                # to accrue interest, then update the db.
+                # chain._advance_chain_time(random_time)  # pylint: disable=protected-access
+                chain._web3.provider.make_request(method=RPCEndpoint("evm_increaseTime"), params=[random_time])
+
+                for pool in hyperdrive_pools:
+                    accrue_interest_func(pool.interface, accrue_interest_rate, block_number_before_advance)
+                    assert isinstance(pool, LocalHyperdrive)
+                    pool._maybe_run_blocking_data_pipeline()  # pylint: disable=protected-access
+
+            else:
+                # The deployer pays gas for creating checkpoints when advancing time
                 # We check the eth balance and refund if it runs low
                 deployer_account = chain.get_deployer_account()
                 deployer_agent_eth = hyperdrive_pools[0].interface.get_eth_base_balances(deployer_account)[0]
                 if deployer_agent_eth < minimum_avg_agent_eth:
                     _ = set_account_balance(
                         hyperdrive_pools[0].interface.web3, deployer_account.address, eth_budget_per_bot.scaled_value
                     )
-                # RNG should always exist, config's post_init should always
-                # initialize an rng object
-                assert chain.config.rng is not None
-                # TODO should there be an upper bound for advancing time?
-                random_time = int(chain.config.rng.integers(*ADVANCE_TIME_SECONDS_RANGE))
                 chain.advance_time(random_time, create_checkpoints=True)
-            else:
-                raise ValueError("Random advance time only allowed for pools deployed on LocalChain")