Merge pull request #3521 from vitaliy-io/task/improve-evm-tracer-perf…

…ormance

Task/improve evm tracer performance

packages/evm/evmutil/tx.go

@@ -0,0 +1,19 @@
+package evmutil
+
+import (
+	"slices"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+func IsFakeTransaction(tx *types.Transaction) bool {
+	sender, err := GetSender(tx)
+
+	// the error will fire when the transaction is invalid. This is most of the time a fake evm tx we use for internal calls, therefore it's fine to assume both.
+	if slices.Equal(sender.Bytes(), common.Address{}.Bytes()) || err != nil {
+		return true
+	}
+
+	return false
+}

packages/evm/jsonrpc/evmchain.go

@@ -10,7 +10,6 @@ import (
 	"math"
 	"math/big"
 	"path"
-	"slices"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
@@ -666,17 +665,6 @@ func (e *EVMChain) iscRequestsInBlock(evmBlockNumber uint64) (*blocklog.BlockInf
 	return blocklog.GetRequestsInBlock(blocklogStatePartition, iscBlockIndex)
 }
 
-func (e *EVMChain) isFakeTransaction(tx *types.Transaction) bool {
-	sender, err := evmutil.GetSender(tx)
-
-	// the error will fire when the transaction is invalid. This is most of the time a fake evm tx we use for internal calls, therefore it's fine to assume both.
-	if slices.Equal(sender.Bytes(), common.Address{}.Bytes()) || err != nil {
-		return true
-	}
-
-	return false
-}
-
 // traceTransaction allows the tracing of a single EVM transaction.
 // "Fake" transactions that are emitted e.g. for L1 deposits return some mocked trace.
 func (e *EVMChain) traceTransaction(
@@ -699,12 +687,12 @@ func (e *EVMChain) traceTransaction(
 		BlockNumber: new(big.Int).SetUint64(blockNumber),
 		TxIndex:     int(txIndex),
 		TxHash:      tx.Hash(),
-	}, config.TracerConfig)
+	}, config.TracerConfig, false, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	if e.isFakeTransaction(tx) {
+	if evmutil.IsFakeTransaction(tx) {
 		return tracer.TraceFakeTx(tx)
 	}
 
@@ -729,32 +717,36 @@ func (e *EVMChain) debugTraceBlock(config *tracers.TraceConfig, block *types.Blo
 		return nil, err
 	}
 
-	blockTxs, err := e.txsByBlockNumber(new(big.Int).SetUint64(block.NumberU64()))
+	tracerType := "callTracer"
+	if config.Tracer != nil {
+		tracerType = *config.Tracer
+	}
+
+	blockNumber := uint64(iscBlock.BlockIndex())
+
+	blockTxs := block.Transactions()
+
+	tracer, err := newTracer(tracerType, &tracers.Context{
+		BlockHash:   block.Hash(),
+		BlockNumber: new(big.Int).SetUint64(blockNumber),
+	}, config.TracerConfig, true, blockTxs)
 	if err != nil {
 		return nil, err
 	}
 
-	results := make([]TxTraceResult, 0)
-	for i, tx := range blockTxs {
-		result, err := e.traceTransaction(
-			config,
-			iscBlock,
-			iscRequestsInBlock,
-			tx,
-			uint64(i),
-			block.Hash(),
-		)
-
-		// Transactions which failed tracing will be omitted, so the rest of the block can be returned
-		if err == nil {
-			results = append(results, TxTraceResult{
-				TxHash: tx.Hash(),
-				Result: result,
-			})
-		}
+	err = e.backend.EVMTrace(
+		iscBlock.PreviousAliasOutput,
+		iscBlock.Timestamp,
+		iscRequestsInBlock,
+		nil,
+		&blockNumber,
+		tracer.Tracer,
+	)
+	if err != nil {
+		return nil, err
 	}
 
-	return results, nil
+	return tracer.GetResult()
 }
 
 func (e *EVMChain) TraceTransaction(txHash common.Hash, config *tracers.TraceConfig) (any, error) {

packages/evm/jsonrpc/jsonrpctest/jsonrpc_test.go

@@ -4,9 +4,12 @@
 package jsonrpctest
 
 import (
+	"bytes"
 	"context"
+	"crypto/ecdsa"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"math/big"
 	"slices"
 	"strings"
@@ -624,20 +627,136 @@ func TestRPCTraceEVMDeposit(t *testing.T) {
 	require.NoError(t, err)
 	require.EqualValues(t, types.ReceiptStatusSuccessful, rc.Status)
 
-	trace, err := env.traceTransactionWithCallTracer(tx.Hash())
+	t.Run("callTracer_tx", func(t *testing.T) {
+		var trace jsonrpc.CallFrame
+		trace, err = env.traceTransactionWithCallTracer(tx.Hash())
+		require.NoError(t, err)
+		require.Equal(t, evmAddr.String(), trace.To.String())
+		require.Equal(t, hexutil.EncodeUint64(isc.NewAssetsBaseTokens(1000).BaseTokens*1e12), trace.Value.String())
+	})
+
+	t.Run("prestateTracer_tx", func(t *testing.T) {
+		var prestate jsonrpc.PrestateAccountMap
+		prestate, err = env.traceTransactionWithPrestate(tx.Hash())
+		require.NoError(t, err)
+		require.Empty(t, prestate)
+	})
+
+	t.Run("prestateTracerDiff_tx", func(t *testing.T) {
+		var prestateDiff jsonrpc.PrestateDiffResult
+		prestateDiff, err = env.traceTransactionWithPrestateDiff(tx.Hash())
+		require.NoError(t, err)
+		require.Empty(t, prestateDiff.Pre)
+		require.Empty(t, prestateDiff.Post)
+	})
+
+	t.Run("callTracer_block", func(t *testing.T) {
+		callTracer := "callTracer"
+		var res1 json.RawMessage
+		// we have to use the raw client, because the normal client does not support debug methods
+		err = env.RawClient.CallContext(
+			context.Background(),
+			&res1,
+			"debug_traceBlockByNumber",
+			hexutil.Uint64(env.BlockNumber()).String(),
+			tracers.TraceConfig{Tracer: &callTracer},
+		)
+		require.NoError(t, err)
+
+		traces := make([]jsonrpc.TxTraceResult, 0)
+		err = json.Unmarshal(res1, &traces)
+		require.NoError(t, err)
+		require.Len(t, traces, 1)
+		require.Equal(t, tx.Hash(), traces[0].TxHash)
+
+		cs := jsonrpc.CallFrame{}
+		err = json.Unmarshal(traces[0].Result, &cs)
+		require.NoError(t, err)
+		require.Equal(t, evmAddr.String(), cs.To.String())
+		require.Equal(t, hexutil.EncodeUint64(isc.NewAssetsBaseTokens(1000).BaseTokens*1e12), cs.Value.String())
+	})
+
+	t.Run("prestateTracer_block", func(t *testing.T) {
+		tracer := "prestateTracer"
+		var res1 json.RawMessage
+		// we have to use the raw client, because the normal client does not support debug methods
+		err = env.RawClient.CallContext(
+			context.Background(),
+			&res1,
+			"debug_traceBlockByNumber",
+			hexutil.Uint64(env.BlockNumber()).String(),
+			tracers.TraceConfig{Tracer: &tracer},
+		)
+		require.NoError(t, err)
+
+		traces := make([]jsonrpc.TxTraceResult, 0)
+		err = json.Unmarshal(res1, &traces)
+		require.NoError(t, err)
+		require.Len(t, traces, 1)
+		require.Equal(t, tx.Hash(), traces[0].TxHash)
+
+		prestate := jsonrpc.PrestateAccountMap{}
+		err = json.Unmarshal(traces[0].Result, &prestate)
+		require.NoError(t, err)
+		require.Empty(t, prestate)
+	})
+}
+
+func addNRequests(n int, env *soloTestEnv, creator *ecdsa.PrivateKey, creatorAddress common.Address, contractABI abi.ABI, contractAddress common.Address) {
+	rqs := make([]isc.Request, 0, n)
+	for i := 0; i < n; i++ {
+		tx1 := types.MustSignNewTx(creator, types.NewEIP155Signer(big.NewInt(int64(env.ChainID))),
+			&types.LegacyTx{
+				Nonce:    env.NonceAt(creatorAddress) + uint64(i),
+				To:       &contractAddress,
+				Value:    big.NewInt(123),
+				Gas:      100000,
+				GasPrice: big.NewInt(10000000000),
+				Data:     lo.Must(contractABI.Pack("sendTo", common.Address{0x1}, big.NewInt(2))),
+			})
+
+		req1 := lo.Must(isc.NewEVMOffLedgerTxRequest(env.soloChain.ChainID, tx1))
+		rqs = append(rqs, req1)
+	}
+
+	env.soloChain.WaitForRequestsMark()
+	env.soloChain.Env.AddRequestsToMempool(env.soloChain, rqs)
+}
+
+// TestRPCTraceBlockForLargeN requires a large number of requests to be added to the mempool, for that set solo.MaxRequestsInBlock to a large value (>500)
+func TestRPCTraceBlockForLargeN(t *testing.T) {
+	t.Skip("skipping because it requires solo parameters to be set")
+
+	n := 400
+	env := newSoloTestEnv(t)
+	creator, creatorAddress := env.soloChain.NewEthereumAccountWithL2Funds()
+	contractABI, err := abi.JSON(strings.NewReader(evmtest.ISCTestContractABI))
 	require.NoError(t, err)
+	_, _, contractAddress := env.DeployEVMContract(creator, contractABI, evmtest.ISCTestContractBytecode)
+
+	addNRequests(n, env, creator, creatorAddress, contractABI, contractAddress)
 
-	require.Equal(t, evmAddr.String(), trace.To.String())
-	require.Equal(t, hexutil.EncodeUint64(isc.NewAssetsBaseTokens(1000).BaseTokens*1e12), trace.Value.String())
+	require.True(t, env.soloChain.WaitForRequestsThrough(n, 5*time.Minute))
 
-	prestate, err := env.traceTransactionWithPrestate(tx.Hash())
+	bi := env.soloChain.GetLatestBlockInfo()
+	require.EqualValues(t, n, bi.NumSuccessfulRequests)
+
+	callTracer := "callTracer"
+	var res1 json.RawMessage
+	// we have to use the raw client, because the normal client does not support debug methods
+	err = env.RawClient.CallContext(
+		context.Background(),
+		&res1,
+		"debug_traceBlockByNumber",
+		hexutil.Uint64(env.BlockNumber()).String(),
+		tracers.TraceConfig{Tracer: &callTracer},
+	)
 	require.NoError(t, err)
-	require.Empty(t, prestate)
 
-	prestateDiff, err := env.traceTransactionWithPrestateDiff(tx.Hash())
+	var prettyJSON bytes.Buffer
+	err = json.Indent(&prettyJSON, res1, "", "    ")
 	require.NoError(t, err)
-	require.Empty(t, prestateDiff.Pre)
-	require.Empty(t, prestateDiff.Post)
+	fmt.Println(prettyJSON.String())
 }
 
 func TestRPCTraceBlock(t *testing.T) {

packages/evm/jsonrpc/tracer.go

@@ -6,25 +6,70 @@ import (
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/tracers"
+
+	"github.com/iotaledger/wasp/packages/evm/evmutil"
 )
 
 type Tracer struct {
 	*tracers.Tracer
 	TraceFakeTx func(tx *types.Transaction) (json.RawMessage, error)
 }
 
-type tracerFactory func(*tracers.Context, json.RawMessage) (*Tracer, error)
+type tracerFactory func(traceCtx *tracers.Context, cfg json.RawMessage, traceBlock bool, initValue any) (*Tracer, error)
 
 var allTracers = map[string]tracerFactory{}
 
 func registerTracer(tracerType string, fn tracerFactory) {
 	allTracers[tracerType] = fn
 }
 
-func newTracer(tracerType string, ctx *tracers.Context, cfg json.RawMessage) (*Tracer, error) {
+func newTracer(tracerType string, ctx *tracers.Context, cfg json.RawMessage, traceBlock bool, initValue any) (*Tracer, error) {
 	fn := allTracers[tracerType]
 	if fn == nil {
 		return nil, fmt.Errorf("unsupported tracer type: %s", tracerType)
 	}
-	return fn(ctx, cfg)
+	return fn(ctx, cfg, traceBlock, initValue)
+}
+
+func GetTraceResults(
+	blockTxs []*types.Transaction,
+	traceBlock bool,
+	getFakeTxTrace func(tx *types.Transaction) (json.RawMessage, error),
+	getTxTrace func(tx *types.Transaction) (json.RawMessage, error),
+	getSingleTxTrace func() (json.RawMessage, error),
+	reason error,
+) (json.RawMessage, error) {
+	var traceResult []byte
+	var err error
+	if traceBlock {
+		results := make([]TxTraceResult, 0, len(blockTxs))
+		var jsResult json.RawMessage
+		for _, tx := range blockTxs {
+			if evmutil.IsFakeTransaction(tx) {
+				jsResult, err = getFakeTxTrace(tx)
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				jsResult, err = getTxTrace(tx)
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			results = append(results, TxTraceResult{TxHash: tx.Hash(), Result: jsResult})
+		}
+
+		traceResult, err = json.Marshal(results)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		traceResult, err = getSingleTxTrace()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return traceResult, reason
 }