eth/traces,core: support tracing transactions while the chain is synchronizing and store results in db.

cmd/geth/main.go

@@ -84,6 +84,8 @@ var (
 		utils.GCModeFlag,
 		utils.SnapshotFlag,
 		utils.TxLookupLimitFlag,
+		utils.TxTraceEnabledFlag,
+		utils.TxTraceConfigPathFlag,
 		utils.LightServeFlag,
 		utils.LightIngressFlag,
 		utils.LightEgressFlag,

cmd/utils/flags.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"crypto/ecdsa"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
@@ -243,6 +244,17 @@ var (
 		Value:    ethconfig.Defaults.TxLookupLimit,
 		Category: flags.EthCategory,
 	}
+	TxTraceEnabledFlag = &cli.BoolFlag{
+		Name:     "txtrace",
+		Usage:    "Enable record transaction trace while evm processing",
+		Category: flags.EthCategory,
+	}
+	TxTraceConfigPathFlag = &cli.PathFlag{
+		Name:      "txtrace.cfg",
+		Usage:     "Path to txtrace config file, if not set, use openEthereum's style as default config",
+		TakesFile: true,
+		Category:  flags.EthCategory,
+	}
 	LightKDFFlag = &cli.BoolFlag{
 		Name:     "lightkdf",
 		Usage:    "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@@ -1849,6 +1861,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if err := kzg4844.UseCKZG(ctx.String(CryptoKZGFlag.Name) == "ckzg"); err != nil {
 		Fatalf("Failed to set KZG library implementation to %s: %v", ctx.String(CryptoKZGFlag.Name), err)
 	}
+	if ctx.IsSet(TxTraceEnabledFlag.Name) {
+		cfg.EnableTxTraceRecording = true
+		cfg.TxTraceConfig = MakeTxTraceConfig(ctx)
+	}
 }
 
 // SetDNSDiscoveryDefaults configures DNS discovery with the given URL if
@@ -2166,3 +2182,28 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
 	}
 	return preloads
 }
+
+// MakeTxtraceConfig reads the txtrace config file (the same traceconfig format as the debug_traceXXX APIs)
+// and returns the TraceConfig specified by the global --txtrace.cfg flag.
+func MakeTxTraceConfig(ctx *cli.Context) tracers.TraceConfig {
+	path := ctx.Path(TxTraceConfigPathFlag.Name)
+	var traceConfig tracers.TraceConfig
+	if path == "" {
+		tracer := "flatCallTracer"
+		tracerConfigdata, _ := json.Marshal(map[string]interface{}{
+			"convertParityErrors": true,
+			"includePrecompiles":  true,
+		})
+		traceConfig.Tracer = &tracer
+		traceConfig.TracerConfig = tracerConfigdata
+		return traceConfig
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		Fatalf("Failed to read txtrace file: %v", err)
+	}
+	if err := json.Unmarshal(data, &traceConfig); err != nil {
+		Fatalf("Failed to parse txtrace config file: %v", err)
+	}
+	return traceConfig
+}

common/json_compare.go

@@ -0,0 +1,40 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package common
+
+import (
+	"encoding/json"
+	"reflect"
+)
+
+// CmpJson compares the JSON in two byte slices.
+func CmpJson(a, b []byte) bool {
+	if a == nil {
+		return b == nil
+	}
+	if b == nil {
+		return false
+	}
+	var j1, j2 interface{}
+	if err := json.Unmarshal(a, &j1); err != nil {
+		return false
+	}
+	if err := json.Unmarshal(b, &j2); err != nil {
+		return false
+	}
+	return reflect.DeepEqual(j1, j2)
+}

core/rawdb/accessors_trace.go

@@ -0,0 +1,42 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// ReadTxTrace retrieves the transaction trace for the given hash from the database.
+func ReadTxTrace(db ethdb.KeyValueReader, hash common.Hash) ([]byte, error) {
+	return db.Get(txTraceKey(hash))
+}
+
+// WriteTxTrace stores the transaction trace for the given hash to the database.
+func WriteTxTrace(db ethdb.KeyValueWriter, hash common.Hash, trace []byte) {
+	if err := db.Put(txTraceKey(hash), trace); err != nil {
+		log.Crit("Failed to store transaction trace", "err", err)
+	}
+}
+
+// DeleteTxTrace deletes the transaction trace for the given hash from the database.
+func DeleteTxTrace(db ethdb.KeyValueWriter, hash common.Hash) {
+	if err := db.Delete(txTraceKey(hash)); err != nil {
+		log.Crit("Failed to delete transaction trace", "err", err)
+	}
+}

core/rawdb/database.go

@@ -472,6 +472,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		bloomBits       stat
 		beaconHeaders   stat
 		cliqueSnaps     stat
+		txtraces        stat
 
 		// Les statistic
 		chtTrieNodes   stat
@@ -536,6 +537,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			bytes.HasPrefix(key, BloomTrieIndexPrefix) ||
 			bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub
 			bloomTrieNodes.Add(size)
+		case bytes.HasPrefix(key, TracePrefix) && len(key) == (len(TracePrefix)+common.HashLength):
+			txtraces.Add(size)
 		default:
 			var accounted bool
 			for _, meta := range [][]byte{
@@ -569,6 +572,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()},
 		{"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()},
 		{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
+		{"Key-Value store", "Transaction traces", txtraces.Size(), txtraces.Count()},
 		{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
 		{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
 		{"Key-Value store", "Trie nodes", tries.Size(), tries.Count()},

core/rawdb/schema.go

@@ -122,6 +122,8 @@ var (
 
 	CliqueSnapshotPrefix = []byte("clique-")
 
+	TracePrefix = []byte("tr-")
+
 	preimageCounter    = metrics.NewRegisteredCounter("db/preimage/total", nil)
 	preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
 )
@@ -181,6 +183,11 @@ func txLookupKey(hash common.Hash) []byte {
 	return append(txLookupPrefix, hash.Bytes()...)
 }
 
+// txTraceKey = TracePrefix + hash
+func txTraceKey(hash common.Hash) []byte {
+	return append(TracePrefix, hash.Bytes()...)
+}
+
 // accountSnapshotKey = SnapshotAccountPrefix + hash
 func accountSnapshotKey(hash common.Hash) []byte {
 	return append(SnapshotAccountPrefix, hash.Bytes()...)

core/state_processor.go

@@ -24,10 +24,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -78,6 +80,20 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 	)
 	// Iterate over and process the individual transactions
 	for i, tx := range block.Transactions() {
+		if cfg.EnableTxTraceRecording {
+			txctx := &vm.TraceContext{
+				BlockHash:   blockHash,
+				BlockNumber: blockNumber,
+				TxIndex:     i,
+				TxHash:      tx.Hash(),
+			}
+			tracer, err := cfg.TxTracerCreateFn(cfg.TxTracerName, txctx, cfg.TxTracerConfig)
+			if err != nil {
+				return nil, nil, 0, fmt.Errorf("could not create txtracer: %w", err)
+			}
+			cfg.Tracer = tracer
+			vmenv = vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg)
+		}
 		msg, err := TransactionToMessage(tx, signer, header.BaseFee)
 		if err != nil {
 			return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err)
@@ -89,6 +105,19 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 		}
 		receipts = append(receipts, receipt)
 		allLogs = append(allLogs, receipt.Logs...)
+		if cfg.EnableTxTraceRecording {
+			tracer := (cfg.Tracer).(vm.EVMLoggerWithResult)
+			if tracer != nil {
+				res, err := tracer.GetResult()
+				if err != nil {
+					log.Error("could not get result from tracer", "err", err)
+				} else {
+					if res != nil {
+						rawdb.WriteTxTrace(p.bc.db, tx.Hash(), res)
+					}
+				}
+			}
+		}
 	}
 	// Fail if Shanghai not enabled and len(withdrawals) is non-zero.
 	withdrawals := block.Withdrawals()

core/vm/interpreter.go

@@ -17,6 +17,8 @@
 package vm
 
 import (
+	"encoding/json"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -29,6 +31,11 @@ type Config struct {
 	NoBaseFee               bool      // Forces the EIP-1559 baseFee to 0 (needed for 0 price calls)
 	EnablePreimageRecording bool      // Enables recording of SHA3/keccak preimages
 	ExtraEips               []int     // Additional EIPS that are to be enabled
+
+	EnableTxTraceRecording bool                                                                       // Enables recording trace of tranascations while evm processing
+	TxTracerName           *string                                                                    // Name of the tracer
+	TxTracerConfig         json.RawMessage                                                            // Json config of the tracer
+	TxTracerCreateFn       func(*string, *TraceContext, json.RawMessage) (EVMLoggerWithResult, error) // fn for create tracer
 }
 
 // ScopeContext contains the things that are per-call, such as stack and memory,

core/vm/logger.go

@@ -17,6 +17,7 @@
 package vm
 
 import (
+	"encoding/json"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -41,3 +42,19 @@ type EVMLogger interface {
 	CaptureState(pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error)
 	CaptureFault(pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, depth int, err error)
 }
+
+// TraceContext contains some contextual infos for a transaction execution that is not
+// available from within the EVM object.
+type TraceContext struct {
+	BlockHash   common.Hash // Hash of the block the tx is contained within (zero if dangling tx or call)
+	BlockNumber *big.Int    // Number of the block the tx is contained within (zero if dangling tx or call)
+	TxIndex     int         // Index of the transaction within a block (zero if dangling tx or call)
+	TxHash      common.Hash // Hash of the transaction being traced (zero if dangling call)
+}
+
+// EVMLoggerWithResult interface extends vm.EVMLogger and additionally
+// allows collecting the tracing result.
+type EVMLoggerWithResult interface {
+	EVMLogger
+	GetResult() (json.RawMessage, error)
+}

eth/api_backend.go

@@ -18,7 +18,9 @@ package eth
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"math/big"
 	"time"
 
@@ -318,6 +320,24 @@ func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash)
 	return tx, blockHash, blockNumber, index, nil
 }
 
+func (b *EthAPIBackend) GetTransactionTrace(ctx context.Context, traceCfg *tracers.TraceConfig, txHash common.Hash) (json.RawMessage, error) {
+	vmCfg := b.eth.BlockChain().GetVMConfig()
+	if vmCfg.EnableTxTraceRecording &&
+		*vmCfg.TxTracerName == *traceCfg.Tracer &&
+		common.CmpJson(vmCfg.TxTracerConfig, traceCfg.TracerConfig) {
+		data, err := rawdb.ReadTxTrace(b.eth.ChainDb(), txHash)
+		if err != nil {
+			return nil, err
+		}
+		if len(data) == 0 {
+			return nil, fmt.Errorf("transaction trace %s not found", txHash.Hex())
+		}
+		return data, nil
+	}
+
+	return nil, fmt.Errorf("transaction trace config not support")
+}
+
 func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) {
 	return b.eth.txPool.Nonce(addr), nil
 }

eth/backend.go

@@ -18,6 +18,7 @@
 package eth
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
@@ -43,6 +44,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/gasprice"
 	"github.com/ethereum/go-ethereum/eth/protocols/eth"
 	"github.com/ethereum/go-ethereum/eth/protocols/snap"
+	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
@@ -181,6 +183,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	var (
 		vmConfig = vm.Config{
 			EnablePreimageRecording: config.EnablePreimageRecording,
+			EnableTxTraceRecording:  config.EnableTxTraceRecording,
 		}
 		cacheConfig = &core.CacheConfig{
 			TrieCleanLimit:      config.TrieCleanCache,
@@ -192,6 +195,20 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 			Preimages:           config.Preimages,
 		}
 	)
+	if vmConfig.EnableTxTraceRecording {
+		vmConfig.TxTracerName = config.TxTraceConfig.Tracer
+		vmConfig.TxTracerConfig = config.TxTraceConfig.TracerConfig
+		// If TxTraceRecording is enabled, we need to create a tracer for each transaction.
+		vmConfig.TxTracerCreateFn = func(name *string, tc *vm.TraceContext, config json.RawMessage) (vm.EVMLoggerWithResult, error) {
+			return tracers.DefaultDirectory.New(*name, &tracers.Context{
+				BlockHash:   tc.BlockHash,
+				BlockNumber: tc.BlockNumber,
+				TxIndex:     tc.TxIndex,
+				TxHash:      tc.TxHash,
+			}, config)
+		}
+	}
+
 	// Override the chain config with provided settings.
 	var overrides core.ChainOverrides
 	if config.OverrideCancun != nil {