From 8cefbefb538ec01e3501815f108881eed10322e0 Mon Sep 17 00:00:00 2001 From: envestcc Date: Thu, 9 Jan 2025 22:48:56 +0800 Subject: [PATCH] write erigon within statedb --- .../protocol/execution/evm/erigonadapter.go | 53 ++- action/protocol/execution/evm/evm.go | 28 +- .../execution/evm/evmstatedbadapter.go | 1 + action/protocol/managers.go | 5 + api/coreservice.go | 4 +- api/web3server.go | 2 +- chainservice/builder.go | 15 +- gasstation/gasstattion.go | 9 +- state/factory/factory.go | 4 +- state/factory/history.go | 15 +- state/factory/statedb.go | 118 ++++++- state/factory/workingset.go | 33 +- state/factory/workingsetstore.go | 9 +- state/factory/workingsetstore_factory.go | 4 +- state/factory/workingsetstore_history.go | 313 ++++++++++++++++++ state/factory/workingsetstore_statedb.go | 4 +- 16 files changed, 564 insertions(+), 53 deletions(-) create mode 100644 state/factory/workingsetstore_history.go diff --git a/action/protocol/execution/evm/erigonadapter.go b/action/protocol/execution/evm/erigonadapter.go index 69e234edea..7032e3104e 100644 --- a/action/protocol/execution/evm/erigonadapter.go +++ b/action/protocol/execution/evm/erigonadapter.go @@ -37,14 +37,15 @@ type ErigonStateDBAdapter struct { snDiff int } +type ErigonStateDBAdapterDryrun struct { + *ErigonStateDBAdapter +} + func NewErigonStateDBAdapter(adapter *StateDBAdapter, rw erigonstate.StateWriter, intra *erigonstate.IntraBlockState, chainRules *erigonchain.Rules, ) *ErigonStateDBAdapter { - adapter.newContract = func(addr hash.Hash160, account *state.Account) (Contract, error) { - return newContractV2(addr, account, adapter.sm, intra) - } return &ErigonStateDBAdapter{ StateDBAdapter: adapter, rw: rw, @@ -53,6 +54,19 @@ func NewErigonStateDBAdapter(adapter *StateDBAdapter, } } +func NewErigonStateDBAdapterDryrun(adapter *StateDBAdapter, + rw erigonstate.StateWriter, + intra *erigonstate.IntraBlockState, + chainRules *erigonchain.Rules, +) *ErigonStateDBAdapterDryrun { + adapter.newContract = func(addr hash.Hash160, account *state.Account) (Contract, error) { + return newContractV2(addr, account, adapter.sm, intra) + } + return &ErigonStateDBAdapterDryrun{ + NewErigonStateDBAdapter(adapter, rw, intra, chainRules), + } +} + func (s *ErigonStateDBAdapter) CreateAccount(evmAddr common.Address) { s.StateDBAdapter.CreateAccount(evmAddr) s.intra.CreateAccount(libcommon.Address(evmAddr), true) @@ -104,7 +118,11 @@ func (s *ErigonStateDBAdapter) Selfdestruct6780(evmAddr common.Address) { func (s *ErigonStateDBAdapter) CommitContracts() error { log.L().Debug("intraBlockState Committing contracts", zap.Uint64("height", s.StateDBAdapter.blockHeight)) - err := s.intra.FinalizeTx(s.chainRules, s.rw) + err := s.StateDBAdapter.CommitContracts() + if err != nil { + return err + } + err = s.intra.FinalizeTx(s.chainRules, s.rw) if err != nil { return errors.Wrap(err, "failed to finalize tx") } @@ -112,17 +130,30 @@ func (s *ErigonStateDBAdapter) CommitContracts() error { } func (s *ErigonStateDBAdapter) RevertToSnapshot(sn int) { + log.L().Debug("erigon adapter revert to snapshot", zap.Int("sn", sn), zap.Int("isn", sn+s.snDiff)) s.StateDBAdapter.RevertToSnapshot(sn) - s.intra.RevertToSnapshot(sn + s.snDiff) + // s.intra.RevertToSnapshot(sn + s.snDiff) } func (s *ErigonStateDBAdapter) Snapshot() int { sn := s.StateDBAdapter.Snapshot() - isn := s.intra.Snapshot() - diff := isn - sn - if s.snDiff != 0 && diff != s.snDiff { - log.L().Panic("snapshot diff changed", zap.Int("old", s.snDiff), zap.Int("new", diff)) - } - s.snDiff = diff + // isn := s.intra.Snapshot() + // log.L().Debug("erigon adapter snapshot", zap.Int("sn", sn), zap.Int("isn", isn)) + // diff := isn - sn + // if s.snDiff != 0 && diff != s.snDiff { + // log.L().Panic("snapshot diff changed", zap.Int("old", s.snDiff), zap.Int("new", diff)) + // } + // s.snDiff = diff return sn } + +func (stateDB *ErigonStateDBAdapterDryrun) GetCode(evmAddr common.Address) []byte { + return stateDB.intra.GetCode(libcommon.Address(evmAddr)) +} + +// GetCodeSize gets the code size saved in hash +func (stateDB *ErigonStateDBAdapterDryrun) GetCodeSize(evmAddr common.Address) int { + code := stateDB.intra.GetCodeSize(libcommon.Address(evmAddr)) + log.T(stateDB.ctx).Debug("Called GetCodeSize.", log.Hex("addrHash", evmAddr[:])) + return code +} diff --git a/action/protocol/execution/evm/evm.go b/action/protocol/execution/evm/evm.go index 6de2316a4e..fe846be53a 100644 --- a/action/protocol/execution/evm/evm.go +++ b/action/protocol/execution/evm/evm.go @@ -267,16 +267,26 @@ func ExecuteContract( return nil, nil, err } if erigonsm, ok := sm.(interface { - StateWriter() erigonstate.StateWriter - Intra() *erigonstate.IntraBlockState + Erigon() (erigonstate.StateWriter, *erigonstate.IntraBlockState, bool) }); ok { - rules := ps.chainConfig.Rules(ps.context.BlockNumber, ps.genesis.IsSumatra(uint64(ps.context.BlockNumber.Int64())), ps.context.Time) - stateDB = NewErigonStateDBAdapter( - stateDB.(*StateDBAdapter), - erigonsm.StateWriter(), - erigonsm.Intra(), - NewErigonRules(&rules), - ) + if sw, in, dryrun := erigonsm.Erigon(); sw != nil && in != nil { + rules := ps.chainConfig.Rules(ps.context.BlockNumber, ps.genesis.IsSumatra(uint64(ps.context.BlockNumber.Int64())), ps.context.Time) + if dryrun { + stateDB = NewErigonStateDBAdapterDryrun( + stateDB.(*StateDBAdapter), + sw, + in, + NewErigonRules(&rules), + ) + } else { + stateDB = NewErigonStateDBAdapter( + stateDB.(*StateDBAdapter), + sw, + in, + NewErigonRules(&rules), + ) + } + } } retval, depositGas, remainingGas, contractAddress, statusCode, err := executeInEVM(ctx, ps, stateDB) diff --git a/action/protocol/execution/evm/evmstatedbadapter.go b/action/protocol/execution/evm/evmstatedbadapter.go index ea5951d17c..127e7acee9 100644 --- a/action/protocol/execution/evm/evmstatedbadapter.go +++ b/action/protocol/execution/evm/evmstatedbadapter.go @@ -1013,6 +1013,7 @@ func (stateDB *StateDBAdapter) SetCode(evmAddr common.Address, code []byte) { stateDB.logError(err) return } + log.T(stateDB.ctx).Debug("Called SetCode", log.Hex("addrHash", evmAddr[:])) contract.SetCode(hash.Hash256b(code), code) } diff --git a/action/protocol/managers.go b/action/protocol/managers.go index 60defe5ef3..f71d10bf02 100644 --- a/action/protocol/managers.go +++ b/action/protocol/managers.go @@ -86,6 +86,11 @@ type ( Dock } + StateManagerWithCloser interface { + StateManager + Close() + } + // Dock defines an interface for protocol to read/write their private data in StateReader/Manager // data are stored as interface{}, user needs to type-assert on their own upon Unload() Dock interface { diff --git a/api/coreservice.go b/api/coreservice.go index 9f6b65d3af..3eb04b148f 100644 --- a/api/coreservice.go +++ b/api/coreservice.go @@ -381,7 +381,7 @@ func (core *coreService) AccountAt(addr address.Address, height uint64) (*iotext } span.AddEvent("accountutil.AccountStateWithHeight") ctx = genesis.WithGenesisContext(ctx, core.bc.Genesis()) - ws, err := core.history.StateManagerAt(ctx, height) + ws, err := core.sf.WorkingSetAtHeight(ctx, height) if err != nil { return nil, nil, status.Error(codes.NotFound, err.Error()) } @@ -2089,7 +2089,7 @@ func (core *coreService) simulateExecutionAt(ctx context.Context, addr address.A GetBlockTime: core.getBlockTime, DepositGasFunc: rewarding.DepositGas, }) - ws, err := core.history.StateManagerAt(ctx, height) + ws, err := core.sf.WorkingSetAtHeight(ctx, height) if err != nil { return nil, nil, status.Error(codes.Internal, err.Error()) } diff --git a/api/web3server.go b/api/web3server.go index 14e35e09b9..2df539cac7 100644 --- a/api/web3server.go +++ b/api/web3server.go @@ -159,7 +159,7 @@ func (svr *web3Handler) handleWeb3Req(ctx context.Context, web3Req *gjson.Result ) defer func(start time.Time) { svr.coreService.Track(ctx, start, method.(string), int64(size), err == nil) }(time.Now()) - // log.T(ctx).Debug("handleWeb3Req", zap.String("method", method.(string)), zap.String("requestParams", fmt.Sprintf("%+v", web3Req))) + log.T(ctx).Debug("handleWeb3Req", zap.String("method", method.(string)), zap.String("requestParams", fmt.Sprintf("%+v", web3Req))) _web3ServerMtc.WithLabelValues(method.(string)).Inc() _web3ServerMtc.WithLabelValues("requests_total").Inc() switch method { diff --git a/chainservice/builder.go b/chainservice/builder.go index 02ded51aab..cad2881ae4 100644 --- a/chainservice/builder.go +++ b/chainservice/builder.go @@ -153,11 +153,11 @@ func (builder *Builder) buildFactory(forTest bool) error { return errors.Wrapf(err, "failed to create state factory") } builder.cs.factory = factory - history, err := builder.createHistoryIndex() - if err != nil { - return errors.Wrapf(err, "failed to create history index") - } - builder.cs.historyIndex = history + // history, err := builder.createHistoryIndex() + // if err != nil { + // return errors.Wrapf(err, "failed to create history index") + // } + // builder.cs.historyIndex = history return nil } @@ -820,6 +820,11 @@ func (builder *Builder) buildBlockTimeCalculator() (err error) { if builder.cs.historyIndex != nil { builder.cs.historyIndex.SetGetBlockTime(builder.cs.blockTimeCalculator.CalculateBlockTime) } + if f, ok := builder.cs.factory.(interface { + SetGetBlockTime(func(uint64) (time.Time, error)) + }); ok { + f.SetGetBlockTime(builder.cs.blockTimeCalculator.CalculateBlockTime) + } return err } diff --git a/gasstation/gasstattion.go b/gasstation/gasstattion.go index b21ffdbdf8..d7f0dfa67d 100644 --- a/gasstation/gasstattion.go +++ b/gasstation/gasstattion.go @@ -137,6 +137,9 @@ func (gs *GasStation) FeeHistory(ctx context.Context, blocks, lastBlock uint64, log.T(ctx).Warn("Sanitizing fee history length", zap.Uint64("requested", blocks), zap.Uint64("truncated", maxFeeHistory)) blocks = maxFeeHistory } + if blocks > lastBlock { + blocks = lastBlock + } for i, p := range rewardPercentiles { if p < 0 || p > 100 { return 0, nil, nil, nil, nil, nil, status.Error(codes.InvalidArgument, "percentile must be in [0, 100]") @@ -199,7 +202,11 @@ func (gs *GasStation) FeeHistory(ctx context.Context, blocks, lastBlock uint64, } fees := make([]*big.Int, 0, len(receipts)) for _, r := range receipts { - fees = append(fees, r.PriorityFee()) + if pf := r.PriorityFee(); pf != nil { + fees = append(fees, pf) + } else { + fees = append(fees, big.NewInt(0)) + } } sort.Slice(fees, func(i, j int) bool { return fees[i].Cmp(fees[j]) < 0 diff --git a/state/factory/factory.go b/state/factory/factory.go index 37cd5a7b62..a885c8d819 100644 --- a/state/factory/factory.go +++ b/state/factory/factory.go @@ -87,7 +87,7 @@ type ( NewBlockBuilder(context.Context, actpool.ActPool, func(action.Envelope) (*action.SealedEnvelope, error)) (*block.Builder, error) PutBlock(context.Context, *block.Block) error WorkingSet(context.Context) (protocol.StateManager, error) - WorkingSetAtHeight(context.Context, uint64) (protocol.StateManager, error) + WorkingSetAtHeight(context.Context, uint64) (protocol.StateManagerWithCloser, error) } // factory implements StateFactory interface, tracks changes to account/contract and batch-commits to DB @@ -408,7 +408,7 @@ func (sf *factory) WorkingSet(ctx context.Context) (protocol.StateManager, error return sf.newWorkingSet(ctx, sf.currentChainHeight+1) } -func (sf *factory) WorkingSetAtHeight(ctx context.Context, height uint64) (protocol.StateManager, error) { +func (sf *factory) WorkingSetAtHeight(ctx context.Context, height uint64) (protocol.StateManagerWithCloser, error) { if !sf.saveHistory { return nil, ErrNoArchiveData } diff --git a/state/factory/history.go b/state/factory/history.go index ee38b6cad1..29ffb515bd 100644 --- a/state/factory/history.go +++ b/state/factory/history.go @@ -42,7 +42,10 @@ type HistoryState interface { // 1. query history account, impl state reader // 2. query history storage, impl state manager // 3. simulate in history, impl state manager - StateManagerAt(height uint64) protocol.StateManager + StateManagerAt(height uint64) interface { + protocol.StateManager + Close() + } } type HistoryStateIndex struct { @@ -107,7 +110,7 @@ func (h *HistoryStateIndex) Start(ctx context.Context) error { defer tx.Rollback() r, tsw := erigonstate.NewPlainStateReader(tx), erigonstate.NewPlainStateWriter(tx, tx, 0) intraBlockState := erigonstate.New(r) - intraBlockState.SetTrace(true) + intraBlockState.SetTrace(false) hws := &historyWorkingSetRo{ historyWorkingSet: &historyWorkingSet{ workingSet: ws, @@ -160,7 +163,7 @@ func (h *HistoryStateIndex) PutBlock(ctx context.Context, blk *block.Block) erro defer tx.Rollback() r, tsw := erigonstate.NewPlainStateReader(tx), erigonstate.NewPlainStateWriter(tx, tx, blk.Height()) intraBlockState := erigonstate.New(r) - intraBlockState.SetTrace(true) + intraBlockState.SetTrace(false) hws := &historyWorkingSet{ workingSet: ws, @@ -190,7 +193,7 @@ func (h *HistoryStateIndex) commit(ctx context.Context, tx kv.RwTx, tsw *erigons if err != nil { return err } - intraBlockState.Print(*rules) + // intraBlockState.Print(*rules) err = tsw.WriteChangeSets() if err != nil { @@ -428,3 +431,7 @@ func (ws *historyWorkingSetRo) PutState(s interface{}, opts ...protocol.StateOpt } return h, nil } + +func (ws *historyWorkingSetRo) Close() { + ws.cleanup() +} diff --git a/state/factory/statedb.go b/state/factory/statedb.go index 638debf7ab..750826f875 100644 --- a/state/factory/statedb.go +++ b/state/factory/statedb.go @@ -12,6 +12,10 @@ import ( "sync" "time" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/mdbx" + erigonstate "github.com/ledgerwatch/erigon/core/state" + erigonlog "github.com/ledgerwatch/log/v3" "github.com/pkg/errors" "go.uber.org/zap" @@ -53,6 +57,10 @@ type ( protocolView protocol.View skipBlockValidationOnPut bool ps *patchStore + + // erigon + rw kv.RwDB + getBlockTime func(uint64) (time.Time, error) } ) @@ -120,11 +128,29 @@ func NewStateDB(cfg Config, dao db.KVStore, opts ...StateDBOption) (Factory, err return &sdb, nil } +func (sdb *stateDB) SetGetBlockTime(getBlockTime func(uint64) (time.Time, error)) { + sdb.getBlockTime = getBlockTime +} + func (sdb *stateDB) Start(ctx context.Context) error { ctx = protocol.WithRegistry(ctx, sdb.registry) if err := sdb.dao.Start(ctx); err != nil { return err } + // start erigon + if len(sdb.cfg.Chain.HistoryIndexPath) > 0 { + log.L().Info("starting history state index") + lg := erigonlog.New() + lg.SetHandler(erigonlog.StdoutHandler) + rw, err := mdbx.NewMDBX(lg).Path(sdb.cfg.Chain.HistoryIndexPath).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { + defaultBuckets[systemNS] = kv.TableCfgItem{} + return defaultBuckets + }).Open(ctx) + if err != nil { + return errors.Wrap(err, "failed to open history state index") + } + sdb.rw = rw + } // check factory height h, err := sdb.dao.getHeight() switch errors.Cause(err) { @@ -197,10 +223,63 @@ func (sdb *stateDB) newWorkingSet(ctx context.Context, height uint64) (*workingS if err := store.Start(ctx); err != nil { return nil, err } - return newWorkingSet(height, store), nil } +func (sdb *stateDB) newErigonStore(ctx context.Context, height uint64) (*erigonStore, error) { + tx, err := sdb.rw.BeginRw(ctx) + if err != nil { + return nil, err + } + r, tsw := erigonstate.NewPlainStateReader(tx), erigonstate.NewPlainStateWriter(tx, tx, height) + intraBlockState := erigonstate.New(r) + // debug: enable trace + intraBlockState.SetTrace(false) + return &erigonStore{ + tsw: tsw, + tx: tx, + intraBlockState: intraBlockState, + getBlockTime: sdb.getBlockTime, + }, nil +} + +func (sdb *stateDB) newWorkingSetWithErigonOutput(ctx context.Context, height uint64) (*workingSet, error) { + ws, err := sdb.newWorkingSet(ctx, height) + if err != nil { + return nil, err + } + e, err := sdb.newErigonStore(ctx, height) + if err != nil { + return nil, err + } + ws.store = newStateDBWorkingSetStoreWithErigonOutput( + ws.store.(*stateDBWorkingSetStore), + e, + ) + return ws, nil +} + +func (sdb *stateDB) newWorkingSetWithErigonDryrun(ctx context.Context, height uint64) (*workingSet, error) { + ws, err := sdb.newWorkingSet(ctx, height) + if err != nil { + return nil, err + } + tx, err := sdb.rw.BeginRo(ctx) + if err != nil { + return nil, err + } + tsw := erigonstate.NewPlainState(tx, height, nil) + intraBlockState := erigonstate.New(tsw) + e := &erigonStore{ + tsw: tsw, + tx: tx, + intraBlockState: intraBlockState, + getBlockTime: sdb.getBlockTime, + } + ws.store = newStateDBWorkingSetStoreWithErigonDryrun(ws.store.(*stateDBWorkingSetStore), e) + return ws, nil +} + func (sdb *stateDB) Register(p protocol.Protocol) error { return p.Register(sdb.registry) } @@ -236,7 +315,15 @@ func (sdb *stateDB) NewBlockBuilder( sdb.mutex.RLock() currHeight := sdb.currentChainHeight sdb.mutex.RUnlock() - ws, err := sdb.newWorkingSet(ctx, currHeight+1) + var ( + ws *workingSet + err error + ) + if len(sdb.cfg.Chain.HistoryIndexPath) > 0 { + ws, err = sdb.newWorkingSetWithErigonOutput(ctx, currHeight+1) + } else { + ws, err = sdb.newWorkingSet(ctx, currHeight+1) + } if err != nil { return nil, err } @@ -270,9 +357,12 @@ func (sdb *stateDB) WorkingSet(ctx context.Context) (protocol.StateManager, erro return sdb.newWorkingSet(ctx, height+1) } -func (sdb *stateDB) WorkingSetAtHeight(ctx context.Context, height uint64) (protocol.StateManager, error) { +func (sdb *stateDB) WorkingSetAtHeight(ctx context.Context, height uint64) (protocol.StateManagerWithCloser, error) { // TODO: implement archive mode - return sdb.newWorkingSet(ctx, height) + if sdb.rw == nil { + return nil, errors.New("archive mode is not enabled") + } + return sdb.newWorkingSetWithErigonDryrun(ctx, height) } // PutBlock persists all changes in RunActions() into the DB @@ -406,7 +496,15 @@ func (sdb *stateDB) state(h uint64, ns string, addr []byte, s interface{}) error } func (sdb *stateDB) createGenesisStates(ctx context.Context) error { - ws, err := sdb.newWorkingSet(ctx, 0) + var ( + ws *workingSet + err error + ) + if len(sdb.cfg.Chain.HistoryIndexPath) > 0 { + ws, err = sdb.newWorkingSetWithErigonOutput(ctx, 0) + } else { + ws, err = sdb.newWorkingSet(ctx, 0) + } if err != nil { return err } @@ -429,6 +527,14 @@ func (sdb *stateDB) getFromWorkingSets(ctx context.Context, key hash.Hash256) (* sdb.mutex.RLock() currHeight := sdb.currentChainHeight sdb.mutex.RUnlock() - tx, err := sdb.newWorkingSet(ctx, currHeight+1) + var ( + tx *workingSet + err error + ) + if len(sdb.cfg.Chain.HistoryIndexPath) > 0 { + tx, err = sdb.newWorkingSetWithErigonOutput(ctx, currHeight+1) + } else { + tx, err = sdb.newWorkingSet(ctx, currHeight+1) + } return tx, false, err } diff --git a/state/factory/workingset.go b/state/factory/workingset.go index 890ae6d0f4..cd16611d82 100644 --- a/state/factory/workingset.go +++ b/state/factory/workingset.go @@ -7,6 +7,7 @@ package factory import ( "context" + "fmt" "math/big" "sort" "time" @@ -15,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/iotexproject/go-pkgs/hash" "github.com/iotexproject/iotex-proto/golang/iotextypes" + erigonstate "github.com/ledgerwatch/erigon/core/state" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -277,11 +279,11 @@ func (ws *workingSet) checkContract(ctx context.Context, to *common.Address) (bo return sender.IsContract(), false, false, nil } -func (ws *workingSet) finalize() error { +func (ws *workingSet) finalize(ctx context.Context) error { if ws.finalized { return errors.New("Cannot finalize a working set twice") } - if err := ws.store.Finalize(ws.height); err != nil { + if err := ws.store.Finalize(ctx, ws.height); err != nil { return err } ws.finalized = true @@ -325,7 +327,7 @@ func (ws *workingSet) Commit(ctx context.Context) error { if err := protocolPreCommit(ctx, ws); err != nil { return err } - if err := ws.store.Commit(); err != nil { + if err := ws.store.Commit(ctx); err != nil { return err } if err := protocolCommit(ctx, ws); err != nil { @@ -383,6 +385,7 @@ func (ws *workingSet) PutState(s interface{}, opts ...protocol.StateOption) (uin if err != nil { return ws.height, errors.Wrapf(err, "failed to convert account %v to bytes", s) } + log.L().Debug("workingSet.PutState", zap.String("namespace", cfg.Namespace), log.Hex("key", cfg.Key), zap.Any("state", s), zap.String("store", fmt.Sprintf("%+T", ws.store))) return ws.height, ws.store.Put(cfg.Namespace, cfg.Key, ss) } @@ -393,6 +396,7 @@ func (ws *workingSet) DelState(opts ...protocol.StateOption) (uint64, error) { if err != nil { return ws.height, err } + log.L().Debug("workingSet.DelState", zap.String("namespace", cfg.Namespace), log.Hex("key", cfg.Key), zap.String("store", fmt.Sprintf("%+T", ws.store))) return ws.height, ws.store.Delete(cfg.Namespace, cfg.Key) } @@ -422,6 +426,21 @@ func (ws *workingSet) Reset() { ws.dock.Reset() } +func (ws *workingSet) Close() { + ws.store.Close() +} + +func (ws *workingSet) Erigon() (erigonstate.StateWriter, *erigonstate.IntraBlockState, bool) { + switch st := ws.store.(type) { + case *stateDBWorkingSetStoreWithErigonOutput: + return st.erigonStore.tsw, st.erigonStore.intraBlockState, false + case *stateDBWorkingSetStoreWithErigonDryrun: + return st.erigonStore.tsw, st.erigonStore.intraBlockState, true + default: + return nil, nil, false + } +} + // CreateGenesisStates initialize the genesis states func (ws *workingSet) CreateGenesisStates(ctx context.Context) error { if reg, ok := protocol.GetRegistry(ctx); ok { @@ -434,7 +453,7 @@ func (ws *workingSet) CreateGenesisStates(ctx context.Context) error { } } - return ws.finalize() + return ws.finalize(ctx) } func (ws *workingSet) validateNonce(ctx context.Context, blk *block.Block) error { @@ -562,7 +581,7 @@ func (ws *workingSet) processWithCorrectOrder(ctx context.Context, actions []*ac updateReceiptIndex(receipts) } ws.receipts = receipts - return ws.finalize() + return ws.finalize(ctx) } func (ws *workingSet) process(ctx context.Context, actions []*action.SealedEnvelope) error { @@ -597,7 +616,7 @@ func (ws *workingSet) process(ctx context.Context, actions []*action.SealedEnvel return err } ws.receipts = receipts - return ws.finalize() + return ws.finalize(ctx) } func (ws *workingSet) generateSystemActions(ctx context.Context) ([]action.Envelope, error) { @@ -792,7 +811,7 @@ func (ws *workingSet) pickAndRunActions( } ws.receipts = receipts - return executedActions, ws.finalize() + return executedActions, ws.finalize(ctx) } func updateReceiptIndex(receipts []*action.Receipt) { diff --git a/state/factory/workingsetstore.go b/state/factory/workingsetstore.go index a6445df079..42feb85777 100644 --- a/state/factory/workingsetstore.go +++ b/state/factory/workingsetstore.go @@ -6,6 +6,8 @@ package factory import ( + "context" + "github.com/iotexproject/go-pkgs/hash" "github.com/iotexproject/iotex-core/v2/action/protocol" @@ -15,15 +17,16 @@ import ( type ( workingSetStore interface { db.KVStoreBasic - Commit() error + Commit(context.Context) error States(string, [][]byte) ([][]byte, [][]byte, error) Digest() hash.Hash256 - Finalize(uint64) error + Finalize(context.Context, uint64) error Snapshot() int RevertSnapshot(int) error ResetSnapshots() ReadView(string) (interface{}, error) WriteView(string, interface{}) error + Close() } workingSetStoreCommon struct { view protocol.View @@ -53,7 +56,7 @@ func (store *workingSetStoreCommon) Digest() hash.Hash256 { return hash.Hash256b(store.flusher.SerializeQueue()) } -func (store *workingSetStoreCommon) Commit() error { +func (store *workingSetStoreCommon) Commit(context.Context) error { _dbBatchSizelMtc.WithLabelValues().Set(float64(store.flusher.KVStoreWithBuffer().Size())) return store.flusher.Flush() } diff --git a/state/factory/workingsetstore_factory.go b/state/factory/workingsetstore_factory.go index bd259f5861..729b5e1b89 100644 --- a/state/factory/workingsetstore_factory.go +++ b/state/factory/workingsetstore_factory.go @@ -94,7 +94,7 @@ func (store *factoryWorkingSetStore) States(ns string, keys [][]byte) ([][]byte, return readStatesFromTLT(store.tlt, ns, keys) } -func (store *factoryWorkingSetStore) Finalize(h uint64) error { +func (store *factoryWorkingSetStore) Finalize(_ context.Context, h uint64) error { rootHash, err := store.tlt.RootHash() if err != nil { return err @@ -136,3 +136,5 @@ func (store *factoryWorkingSetStore) ResetSnapshots() { store.workingSetStoreCommon.ResetSnapshots() store.trieRoots = make(map[int][]byte) } + +func (store *factoryWorkingSetStore) Close() {} diff --git a/state/factory/workingsetstore_history.go b/state/factory/workingsetstore_history.go new file mode 100644 index 0000000000..2adbd7d413 --- /dev/null +++ b/state/factory/workingsetstore_history.go @@ -0,0 +1,313 @@ +package factory + +import ( + "context" + "fmt" + "math/big" + "time" + + "github.com/holiman/uint256" + "github.com/iotexproject/go-pkgs/hash" + libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + erigonstate "github.com/ledgerwatch/erigon/core/state" + "github.com/pkg/errors" + "go.uber.org/zap" + + "github.com/iotexproject/iotex-core/v2/action/protocol" + "github.com/iotexproject/iotex-core/v2/action/protocol/execution/evm" + "github.com/iotexproject/iotex-core/v2/blockchain/genesis" + "github.com/iotexproject/iotex-core/v2/pkg/log" + "github.com/iotexproject/iotex-core/v2/state" +) + +type reader interface { + Get(string, []byte) ([]byte, error) + States(string, [][]byte) ([][]byte, [][]byte, error) + Digest() hash.Hash256 + ReadView(string) (interface{}, error) +} + +type writer interface { + WriteView(name string, value interface{}) error + Put(ns string, key []byte, value []byte) error + Delete(ns string, key []byte) error + Snapshot() int + RevertSnapshot(snapshot int) error + ResetSnapshots() +} + +// treat erigon as 3rd output, still read from statedb +// it's used for PutBlock, generating historical states in erigon +type stateDBWorkingSetStoreWithErigonOutput struct { + reader + store *stateDBWorkingSetStore + erigonStore *erigonStore + snMap map[int]int +} + +type erigonStore struct { + tsw erigonstate.StateWriter + intraBlockState *erigonstate.IntraBlockState + tx kv.Tx + getBlockTime func(uint64) (time.Time, error) +} + +func newStateDBWorkingSetStoreWithErigonOutput(store *stateDBWorkingSetStore, erigonStore *erigonStore) *stateDBWorkingSetStoreWithErigonOutput { + return &stateDBWorkingSetStoreWithErigonOutput{ + reader: store, + store: store, + erigonStore: erigonStore, + snMap: make(map[int]int), + } +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) Start(context.Context) error { + return nil +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) Stop(context.Context) error { + return nil +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) Finalize(ctx context.Context, height uint64) error { + if err := store.store.Finalize(ctx, height); err != nil { + return err + } + return store.erigonStore.finalize(ctx, height, uint64(protocol.MustGetBlockCtx(ctx).BlockTimeStamp.Unix())) +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) WriteView(name string, value interface{}) error { + return store.store.WriteView(name, value) +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) Put(ns string, key []byte, value []byte) error { + if err := store.store.Put(ns, key, value); err != nil { + return err + } + return store.erigonStore.put(ns, key, value) +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) Delete(ns string, key []byte) error { + // delete won't happen in account and contract + return store.store.Delete(ns, key) +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) Commit(ctx context.Context) error { + if err := store.store.Commit(ctx); err != nil { + return err + } + return store.erigonStore.commit(ctx) +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) Snapshot() int { + sn := store.store.Snapshot() + isn := store.erigonStore.intraBlockState.Snapshot() + store.snMap[sn] = isn + return sn +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) RevertSnapshot(sn int) error { + store.store.RevertSnapshot(sn) + if isn, ok := store.snMap[sn]; ok { + store.erigonStore.intraBlockState.RevertToSnapshot(isn) + delete(store.snMap, sn) + } else { + panic(fmt.Sprintf("no isn for sn %d", sn)) + } + return nil +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) ResetSnapshots() { + store.store.ResetSnapshots() + store.snMap = make(map[int]int) +} + +func (store *stateDBWorkingSetStoreWithErigonOutput) Close() { + store.erigonStore.tx.Rollback() +} + +func (store *erigonStore) finalize(ctx context.Context, height uint64, ts uint64) error { + g := genesis.MustExtractGenesisContext(ctx) + chainCfg, err := evm.NewChainConfig(g.Blockchain, height, protocol.MustGetBlockchainCtx(ctx).EvmNetworkID, store.getBlockTime) + if err != nil { + return err + } + + chainRules := chainCfg.Rules(big.NewInt(int64(height)), g.IsSumatra(height), uint64(ts)) + rules := evm.NewErigonRules(&chainRules) + log.L().Debug("intraBlockState Commit block", zap.Uint64("height", height)) + err = store.intraBlockState.CommitBlock(rules, store.tsw) + if err != nil { + return err + } + log.L().Debug("erigon store finalize", zap.Uint64("height", height), zap.String("tsw", fmt.Sprintf("%+T", store.tsw))) + // store.intraBlockState.Print(*rules) + + if c, ok := store.tsw.(erigonstate.WriterWithChangeSets); ok { + log.L().Debug("erigon store write changesets", zap.Uint64("height", height)) + err = c.WriteChangeSets() + if err != nil { + return err + } + err = c.WriteHistory() + if err != nil { + return err + } + } + if tx, ok := store.tx.(kv.RwTx); ok { + log.L().Debug("erigon store commit tx", zap.Uint64("height", height)) + err = tx.Put(systemNS, heightKey, uint256.NewInt(height).Bytes()) + if err != nil { + return err + } + } + return nil +} + +func (store *erigonStore) commit(ctx context.Context) error { + defer store.tx.Rollback() + err := store.tx.Commit() + if err != nil { + return err + } + return nil +} + +func (store *erigonStore) put(ns string, key []byte, value []byte) (err error) { + // only handling account, contract storage handled by evm adapter + // others are ignored + if ns != AccountKVNamespace { + return nil + } + defer func() { + if r := recover(); r != nil { + log.L().Warn("store no account in account namespace", zap.Any("recover", r), log.Hex("key", key), zap.String("ns", ns), zap.ByteString("value", value)) + err = nil + } + }() + acc := &state.Account{} + if err := acc.Deserialize(value); err != nil { + // should be legacy rewarding funds + log.L().Warn("store no account in account namespace", log.Hex("key", key), zap.String("ns", ns), zap.ByteString("value", value)) + return nil + } + addr := libcommon.Address(key) + if !store.intraBlockState.Exist(addr) { + store.intraBlockState.CreateAccount(addr, false) + } + store.intraBlockState.SetBalance(addr, uint256.MustFromBig(acc.Balance)) + store.intraBlockState.SetNonce(addr, acc.PendingNonce()) // TODO(erigon): not sure if this is correct + return nil +} + +func (store *erigonStore) get(ns string, key []byte) ([]byte, error) { + switch ns { + case AccountKVNamespace: + acc := &state.Account{} + addr := libcommon.Address(key) + if !store.intraBlockState.Exist(addr) { + return nil, state.ErrStateNotExist + } + balance := store.intraBlockState.GetBalance(addr) + acc.Balance = balance.ToBig() + acc.SetPendingNonce(store.intraBlockState.GetNonce(addr)) + if ch := store.intraBlockState.GetCodeHash(addr); len(ch) > 0 { + acc.CodeHash = store.intraBlockState.GetCodeHash(addr).Bytes() + } + return acc.Serialize() + case evm.CodeKVNameSpace: + addr := libcommon.Address(key) + if !store.intraBlockState.Exist(addr) { + return nil, state.ErrStateNotExist + } + return store.intraBlockState.GetCode(addr), nil + default: + return nil, errors.Errorf("unexpected erigon get namespace %s, key %x", ns, key) + } +} + +// used in historical states query +// account & contract read & write on erigon +type stateDBWorkingSetStoreWithErigonDryrun struct { + writer + store *stateDBWorkingSetStore // fallback to statedb for staking, rewarding and poll + erigonStore *erigonStore +} + +func newStateDBWorkingSetStoreWithErigonDryrun(store *stateDBWorkingSetStore, erigonStore *erigonStore) *stateDBWorkingSetStoreWithErigonDryrun { + return &stateDBWorkingSetStoreWithErigonDryrun{ + store: store, + erigonStore: erigonStore, + writer: newStateDBWorkingSetStoreWithErigonOutput(store, erigonStore), + } +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) Start(context.Context) error { + return nil +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) Stop(context.Context) error { + return nil +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) Get(ns string, key []byte) ([]byte, error) { + switch ns { + case AccountKVNamespace, evm.CodeKVNameSpace: + return store.erigonStore.get(ns, key) + default: + return store.store.Get(ns, key) + } +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) States(ns string, keys [][]byte) ([][]byte, [][]byte, error) { + // currently only used for staking & poll, no need to read from erigon + return store.store.States(ns, keys) +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) Finalize(_ context.Context, height uint64) error { + // do nothing for dryrun + return nil +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) ReadView(name string) (interface{}, error) { + // only used for staking + return store.store.ReadView(name) +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) Digest() hash.Hash256 { + return store.store.Digest() +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) Commit(context.Context) error { + // do nothing for dryrun + return nil +} + +func (store *stateDBWorkingSetStoreWithErigonDryrun) Close() { + store.erigonStore.tx.Rollback() +} + +// func (store *stateDBWorkingSetStoreWithErigonDryrun) WriteView(name string, value interface{}) error { +// return store.outer.WriteView(name, value) +// } + +// func (store *stateDBWorkingSetStoreWithErigonDryrun) Put(ns string, key []byte, value []byte) error { +// return store.outer.Put(ns, key, value) +// } + +// func (store *stateDBWorkingSetStoreWithErigonDryrun) Delete(ns string, key []byte) error { +// return store.outer.Delete(ns, key) +// } + +// func (store *stateDBWorkingSetStoreWithErigonDryrun) Snapshot() int { +// return store.outer.Snapshot() +// } + +// func (store *stateDBWorkingSetStoreWithErigonDryrun) RevertSnapshot(sn int) error { +// return store.outer.RevertSnapshot(sn) +// } + +// func (store *stateDBWorkingSetStoreWithErigonDryrun) ResetSnapshots() { +// store.outer.ResetSnapshots() +// } diff --git a/state/factory/workingsetstore_statedb.go b/state/factory/workingsetstore_statedb.go index 4f96237204..876efd0eb4 100644 --- a/state/factory/workingsetstore_statedb.go +++ b/state/factory/workingsetstore_statedb.go @@ -58,7 +58,7 @@ func (store *stateDBWorkingSetStore) States(ns string, keys [][]byte) ([][]byte, return readStates(store.flusher.BaseKVStore(), ns, keys) } -func (store *stateDBWorkingSetStore) Finalize(height uint64) error { +func (store *stateDBWorkingSetStore) Finalize(_ context.Context, height uint64) error { // Persist current chain Height store.flusher.KVStoreWithBuffer().MustPut( AccountKVNamespace, @@ -67,3 +67,5 @@ func (store *stateDBWorkingSetStore) Finalize(height uint64) error { ) return nil } + +func (store *stateDBWorkingSetStore) Close() {}