diff --git a/beacon/light/request/server.go b/beacon/light/request/server.go index a06dec99ae..d39570b8e5 100644 --- a/beacon/light/request/server.go +++ b/beacon/light/request/server.go @@ -438,14 +438,11 @@ func (s *serverWithLimits) fail(desc string) { // failLocked calculates the dynamic failure delay and applies it. func (s *serverWithLimits) failLocked(desc string) { log.Debug("Server error", "description", desc) - s.failureDelay *= 2 now := s.clock.Now() if now > s.failureDelayEnd { s.failureDelay *= math.Pow(2, -float64(now-s.failureDelayEnd)/float64(maxFailureDelay)) } - if s.failureDelay < float64(minFailureDelay) { - s.failureDelay = float64(minFailureDelay) - } + s.failureDelay = max(min(s.failureDelay*2, float64(maxFailureDelay)), float64(minFailureDelay)) s.failureDelayEnd = now + mclock.AbsTime(s.failureDelay) s.delay(time.Duration(s.failureDelay)) } diff --git a/beacon/light/sync/update_sync.go b/beacon/light/sync/update_sync.go index 9549ee5992..d84a3d64da 100644 --- a/beacon/light/sync/update_sync.go +++ b/beacon/light/sync/update_sync.go @@ -62,7 +62,6 @@ const ( ssNeedParent // cp header slot %32 != 0, need parent to check epoch boundary ssParentRequested // cp parent header requested ssPrintStatus // has all necessary info, print log message if init still not successful - ssDone // log message printed, no more action required ) type serverState struct { @@ -180,7 +179,8 @@ func (s *CheckpointInit) Process(requester request.Requester, events []request.E default: log.Error("blsync: checkpoint not available, but reported as finalized; specified checkpoint hash might be too old", "server", server.Name()) } - s.serverState[server] = serverState{state: ssDone} + s.serverState[server] = serverState{state: ssDefault} + requester.Fail(server, "checkpoint init failed") } } diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 95e9ab1ed9..f1a3279de5 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -292,15 +292,25 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa } } - // EIP-7843 SLOTNUM: post-Amsterdam headers must carry a slotNumber, pre-Amsterdam - // must not. Amsterdam is dormant on Bor (AmsterdamBlock nil), so IsAmsterdam is - // false and headers must not carry a slotNumber. + // Verify the existence / non-existence of Amsterdam-specific header fields. + // EIP-7843 SLOTNUM and the EIP-7928 block access list hash both key off + // Amsterdam, which is dormant on Bor (AmsterdamBlock nil), so IsAmsterdam is + // false and headers must carry neither field. amsterdam := chain.Config().IsAmsterdam(header.Number) - if amsterdam && header.SlotNumber == nil { - return errors.New("header is missing slotNumber") - } - if !amsterdam && header.SlotNumber != nil { - return fmt.Errorf("invalid slotNumber: have %d, expected nil", *header.SlotNumber) + if amsterdam { + if header.BlockAccessListHash == nil { + return errors.New("header is missing block access list hash") + } + if header.SlotNumber == nil { + return errors.New("header is missing slotNumber") + } + } else { + if header.BlockAccessListHash != nil { + return fmt.Errorf("invalid block access list hash: have %x, expected nil", *header.BlockAccessListHash) + } + if header.SlotNumber != nil { + return fmt.Errorf("invalid slotNumber: have %d, expected nil", *header.SlotNumber) + } } return nil } diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index bcb9476219..f25d8b7595 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -626,7 +626,7 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { } // StateIndexProgress returns the historical state indexing progress. -func (bc *BlockChain) StateIndexProgress() (uint64, error) { +func (bc *BlockChain) StateIndexProgress() (uint64, uint64, error) { return bc.triedb.IndexProgress() } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index d8de5a4f26..e4373c0fb9 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/types/bal" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -828,6 +829,55 @@ func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { } } +// HasAccessList verifies the existence of a block access list for a block. +func HasAccessList(db ethdb.Reader, hash common.Hash, number uint64) bool { + has, _ := db.Has(accessListKey(number, hash)) + return has +} + +// ReadAccessListRLP retrieves the RLP-encoded block access list for a block from KV. +func ReadAccessListRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + data, _ := db.Get(accessListKey(number, hash)) + return data +} + +// ReadAccessList retrieves and decodes the block access list for a block. +func ReadAccessList(db ethdb.Reader, hash common.Hash, number uint64) *bal.BlockAccessList { + data := ReadAccessListRLP(db, hash, number) + if len(data) == 0 { + return nil + } + b := new(bal.BlockAccessList) + if err := rlp.DecodeBytes(data, b); err != nil { + log.Error("Invalid BAL RLP", "hash", hash, "err", err) + return nil + } + return b +} + +// WriteAccessList RLP-encodes and stores a block access list in the active KV store. +func WriteAccessList(db ethdb.KeyValueWriter, hash common.Hash, number uint64, b *bal.BlockAccessList) { + bytes, err := rlp.EncodeToBytes(b) + if err != nil { + log.Crit("Failed to encode BAL", "err", err) + } + WriteAccessListRLP(db, hash, number, bytes) +} + +// WriteAccessListRLP stores a pre-encoded block access list in the active KV store. +func WriteAccessListRLP(db ethdb.KeyValueWriter, hash common.Hash, number uint64, encoded rlp.RawValue) { + if err := db.Put(accessListKey(number, hash), encoded); err != nil { + log.Crit("Failed to store BAL", "err", err) + } +} + +// DeleteAccessList removes a block access list from the active KV store. +func DeleteAccessList(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { + if err := db.Delete(accessListKey(number, hash)); err != nil { + log.Crit("Failed to delete BAL", "err", err) + } +} + // ReceiptLogs is a barebone version of ReceiptForStorage which only keeps // the list of logs. When decoding a stored receipt into this object we // avoid creating the bloom filter. @@ -883,13 +933,25 @@ func ReadBlock(db ethdb.Reader, hash common.Hash, number uint64) *types.Block { if body == nil { return nil } - return types.NewBlockWithHeader(header).WithBody(*body) + block := types.NewBlockWithHeader(header).WithBody(*body) + + // Best-effort assembly of the block access list from the database. + if header.BlockAccessListHash != nil { + al := ReadAccessList(db, hash, number) + block = block.WithAccessListUnsafe(al) + } + return block } // WriteBlock serializes a block into the database, header and body separately. func WriteBlock(db ethdb.KeyValueWriter, block *types.Block) { - WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) + hash, number := block.Hash(), block.NumberU64() + WriteBody(db, hash, number, block.Body()) WriteHeader(db, block.Header()) + + if accessList := block.AccessList(); accessList != nil { + WriteAccessList(db, hash, number, accessList) + } } // WriteAncientBlocks writes entire block data into ancient store and returns the total written size. diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index a9d737926f..2c67527678 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -29,11 +29,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/types/bal" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" ) // Tests block header storage and retrieval operations. @@ -1224,3 +1226,78 @@ func TestWriteBlockPruneHead(t *testing.T) { } } } + +func makeTestBAL(t *testing.T) (rlp.RawValue, *bal.BlockAccessList) { + t.Helper() + + cb := bal.NewConstructionBlockAccessList() + addr := common.HexToAddress("0x1111111111111111111111111111111111111111") + cb.AccountRead(addr) + cb.StorageRead(addr, common.BytesToHash([]byte{0x01})) + cb.StorageWrite(0, addr, common.BytesToHash([]byte{0x02}), common.BytesToHash([]byte{0xaa})) + cb.BalanceChange(0, addr, uint256.NewInt(100)) + cb.NonceChange(addr, 0, 1) + + var buf bytes.Buffer + if err := cb.EncodeRLP(&buf); err != nil { + t.Fatalf("failed to encode BAL: %v", err) + } + encoded := buf.Bytes() + + var decoded bal.BlockAccessList + if err := rlp.DecodeBytes(encoded, &decoded); err != nil { + t.Fatalf("failed to decode BAL: %v", err) + } + return encoded, &decoded +} + +// TestBALStorage tests write/read/delete of BALs in the KV store. +func TestBALStorage(t *testing.T) { + db := NewMemoryDatabase() + + hash := common.BytesToHash([]byte{0x03, 0x14}) + number := uint64(42) + + // Check that no BAL exists in a new database. + if HasAccessList(db, hash, number) { + t.Fatal("BAL found in new database") + } + if b := ReadAccessList(db, hash, number); b != nil { + t.Fatalf("non existent BAL returned: %v", b) + } + + // Write a BAL and verify it can be read back. + encoded, testBAL := makeTestBAL(t) + WriteAccessList(db, hash, number, testBAL) + + if !HasAccessList(db, hash, number) { + t.Fatal("HasAccessList returned false after write") + } + if blob := ReadAccessListRLP(db, hash, number); len(blob) == 0 { + t.Fatal("ReadAccessListRLP returned empty after write") + } + if b := ReadAccessList(db, hash, number); b == nil { + t.Fatal("ReadAccessList returned nil after write") + } else if b.Hash() != testBAL.Hash() { + t.Fatalf("BAL hash mismatch: got %x, want %x", b.Hash(), testBAL.Hash()) + } + + // Also test WriteAccessListRLP with pre-encoded data. + hash2 := common.BytesToHash([]byte{0x03, 0x15}) + WriteAccessListRLP(db, hash2, number, encoded) + if b := ReadAccessList(db, hash2, number); b == nil { + t.Fatal("ReadAccessList returned nil after WriteAccessListRLP") + } else if b.Hash() != testBAL.Hash() { + t.Fatalf("BAL hash mismatch after WriteAccessListRLP: got %x, want %x", b.Hash(), testBAL.Hash()) + } + + // Delete the BAL and verify it's gone. + DeleteAccessList(db, hash, number) + + if HasAccessList(db, hash, number) { + t.Fatal("HasAccessList returned true after delete") + } + if b := ReadAccessList(db, hash, number); b != nil { + t.Fatalf("deleted BAL returned: %v", b) + } +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 8751487d83..0a90b3e14a 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -650,6 +650,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { tds stat numHashPairings stat hashNumPairings stat + blockAccessList stat legacyTries stat stateLookups stat accountTries stat @@ -721,6 +722,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { numHashPairings.add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairings.add(size) + case bytes.HasPrefix(key, accessListPrefix) && len(key) == len(accessListPrefix)+8+common.HashLength: + blockAccessList.add(size) + case IsLegacyTrieNode(key, it.Value()): legacyTries.add(size) case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: @@ -862,6 +866,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Difficulties", tds.sizeString(), tds.countString()}, {"Key-Value store", "Block number->hash", numHashPairings.sizeString(), numHashPairings.countString()}, {"Key-Value store", "Block hash->number", hashNumPairings.sizeString(), hashNumPairings.countString()}, + {"Key-Value store", "Block accessList", blockAccessList.sizeString(), blockAccessList.countString()}, {"Key-Value store", "Transaction index", txLookups.sizeString(), txLookups.countString()}, {"Key-Value store", "Log index filter-map rows", filterMapRows.sizeString(), filterMapRows.countString()}, {"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.sizeString(), filterMapLastBlock.countString()}, diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 4f5cb05bf0..4dba648930 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -127,6 +127,8 @@ var ( BlockPruneCursorKey = []byte("blockPruneCursorKey") BlockPruneHeadKey = []byte("blockPruneHeadKey") + accessListPrefix = []byte("j") // accessListPrefix + num (uint64 big endian) + hash -> block access list + txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value @@ -251,6 +253,11 @@ func blockReceiptsKey(number uint64, hash common.Hash) []byte { return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) } +// accessListKey = accessListPrefix + num (uint64 big endian) + hash +func accessListKey(number uint64, hash common.Hash) []byte { + return append(append(accessListPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + // txLookupKey = txLookupPrefix + hash func txLookupKey(hash common.Hash) []byte { return append(txLookupPrefix, hash.Bytes()...) diff --git a/core/state/database.go b/core/state/database.go index 291a586dbd..652279e055 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -48,6 +48,10 @@ type Database interface { // Reader returns a state reader associated with the specified state root. Reader(root common.Hash) (Reader, error) + // Iteratee returns a state iteratee associated with the specified state root, + // through which the account iterator and storage iterator can be created. + Iteratee(root common.Hash) (Iteratee, error) + // OpenTrie opens the main account trie. OpenTrie(root common.Hash) (Trie, error) @@ -337,6 +341,11 @@ func (db *CachingDB) Snapshot() *snapshot.Tree { return db.snap } +// Iteratee returns a state iteratee associated with the specified state root. +func (db *CachingDB) Iteratee(root common.Hash) (Iteratee, error) { + return newStateIteratee(!db.triedb.IsVerkle(), root, db.triedb, db.snap) +} + // mustCopyTrie returns a deep-copied trie. func mustCopyTrie(t Trie) Trie { switch t := t.(type) { diff --git a/core/state/database_history.go b/core/state/database_history.go index 7a2be8fe4f..fed8b6530d 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -17,6 +17,7 @@ package state import ( + "errors" "fmt" "sync" @@ -298,3 +299,8 @@ func (db *HistoricDB) TrieDB() *triedb.Database { func (db *HistoricDB) Snapshot() *snapshot.Tree { return nil } + +// Iteratee returns a state iteratee associated with the specified state root. +func (db *HistoricDB) Iteratee(root common.Hash) (Iteratee, error) { + return nil, errors.New("not implemented") +} diff --git a/core/state/database_iterator.go b/core/state/database_iterator.go new file mode 100644 index 0000000000..8fad66a1e8 --- /dev/null +++ b/core/state/database_iterator.go @@ -0,0 +1,435 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" +) + +// Iterator is an iterator to step over all the accounts or the specific +// storage in the specific state. +type Iterator interface { + // Next steps the iterator forward one element. It returns false if the iterator + // is exhausted or if an error occurs. Any error encountered is retained and + // can be retrieved via Error(). + Next() bool + + // Error returns any failure that occurred during iteration, which might have + // caused a premature iteration exit. + Error() error + + // Hash returns the hash of the account or storage slot the iterator is + // currently at. + Hash() common.Hash + + // Release releases associated resources. Release should always succeed and + // can be called multiple times without causing error. + Release() +} + +// AccountIterator is an iterator to step over all the accounts in the +// specific state. +type AccountIterator interface { + Iterator + + // Address returns the raw account address the iterator is currently at. + // An error will be returned if the preimage is not available. + Address() (common.Address, error) + + // Account returns the RLP encoded account the iterator is currently at. + // An error will be retained if the iterator becomes invalid. + Account() []byte +} + +// StorageIterator is an iterator to step over the specific storage in the +// specific state. +type StorageIterator interface { + Iterator + + // Key returns the raw storage slot key the iterator is currently at. + // An error will be returned if the preimage is not available. + Key() (common.Hash, error) + + // Slot returns the storage slot the iterator is currently at. An error will + // be retained if the iterator becomes invalid. + Slot() []byte +} + +// Iteratee wraps the NewIterator methods for traversing the accounts and +// storages of the specific state. +type Iteratee interface { + // NewAccountIterator creates an account iterator for the state specified by + // the given root. It begins at a specified starting position, corresponding + // to a particular initial key (or the next key if the specified one does + // not exist). + // + // The starting position here refers to the hash of the account address. + NewAccountIterator(start common.Hash) (AccountIterator, error) + + // NewStorageIterator creates a storage iterator for the state specified by + // the address hash. It begins at a specified starting position, corresponding + // to a particular initial key (or the next key if the specified one does + // not exist). + // + // The starting position here refers to the hash of the slot key. + NewStorageIterator(addressHash common.Hash, start common.Hash) (StorageIterator, error) +} + +// PreimageReader wraps the function Preimage for accessing the preimage of +// a given hash. +type PreimageReader interface { + // Preimage returns the preimage of associated hash. + Preimage(hash common.Hash) []byte +} + +// flatAccountIterator is a wrapper around the underlying flat state iterator. +// Before returning data from the iterator, it performs an additional conversion +// to bridge the slim encoding with the full encoding format. +type flatAccountIterator struct { + err error + it snapshot.AccountIterator + preimage PreimageReader +} + +// newFlatAccountIterator constructs the account iterator with the provided +// flat state iterator. +func newFlatAccountIterator(it snapshot.AccountIterator, preimage PreimageReader) *flatAccountIterator { + return &flatAccountIterator{it: it, preimage: preimage} +} + +// Next steps the iterator forward one element. It returns false if the iterator +// is exhausted or if an error occurs. Any error encountered is retained and +// can be retrieved via Error(). +func (ai *flatAccountIterator) Next() bool { + if ai.err != nil { + return false + } + return ai.it.Next() +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit. +func (ai *flatAccountIterator) Error() error { + if ai.err != nil { + return ai.err + } + return ai.it.Error() +} + +// Hash returns the hash of the account or storage slot the iterator is +// currently at. +func (ai *flatAccountIterator) Hash() common.Hash { + return ai.it.Hash() +} + +// Release releases associated resources. Release should always succeed and +// can be called multiple times without causing error. +func (ai *flatAccountIterator) Release() { + ai.it.Release() +} + +// Address returns the raw account address the iterator is currently at. +// An error will be returned if the preimage is not available. +func (ai *flatAccountIterator) Address() (common.Address, error) { + if ai.preimage == nil { + return common.Address{}, errors.New("account address is not available") + } + preimage := ai.preimage.Preimage(ai.Hash()) + if preimage == nil { + return common.Address{}, errors.New("account address is not available") + } + return common.BytesToAddress(preimage), nil +} + +// Account returns the account data the iterator is currently at. The account +// data is encoded as slim format from the underlying iterator, the conversion +// is required. +func (ai *flatAccountIterator) Account() []byte { + data, err := types.FullAccountRLP(ai.it.Account()) + if err != nil { + ai.err = err + return nil + } + return data +} + +// flatStorageIterator is a wrapper around the underlying flat state iterator. +type flatStorageIterator struct { + it snapshot.StorageIterator + preimage PreimageReader +} + +// newFlatStorageIterator constructs the storage iterator with the provided +// flat state iterator. +func newFlatStorageIterator(it snapshot.StorageIterator, preimage PreimageReader) *flatStorageIterator { + return &flatStorageIterator{it: it, preimage: preimage} +} + +// Next steps the iterator forward one element. It returns false if the iterator +// is exhausted or if an error occurs. Any error encountered is retained and +// can be retrieved via Error(). +func (si *flatStorageIterator) Next() bool { + return si.it.Next() +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit. +func (si *flatStorageIterator) Error() error { + return si.it.Error() +} + +// Hash returns the hash of the account or storage slot the iterator is +// currently at. +func (si *flatStorageIterator) Hash() common.Hash { + return si.it.Hash() +} + +// Release releases associated resources. Release should always succeed and +// can be called multiple times without causing error. +func (si *flatStorageIterator) Release() { + si.it.Release() +} + +// Key returns the raw storage slot key the iterator is currently at. +// An error will be returned if the preimage is not available. +func (si *flatStorageIterator) Key() (common.Hash, error) { + if si.preimage == nil { + return common.Hash{}, errors.New("slot key is not available") + } + preimage := si.preimage.Preimage(si.Hash()) + if preimage == nil { + return common.Hash{}, errors.New("slot key is not available") + } + return common.BytesToHash(preimage), nil +} + +// Slot returns the storage slot data the iterator is currently at. +func (si *flatStorageIterator) Slot() []byte { + return si.it.Slot() +} + +// merkleIterator implements the Iterator interface, providing functions to traverse +// the accounts or storages with the manner of Merkle-Patricia-Trie. +type merkleIterator struct { + tr Trie + it *trie.Iterator + account bool +} + +// newMerkleTrieIterator constructs the iterator with the given trie and starting position. +func newMerkleTrieIterator(tr Trie, start common.Hash, account bool) (*merkleIterator, error) { + it, err := tr.NodeIterator(start.Bytes()) + if err != nil { + return nil, err + } + return &merkleIterator{ + tr: tr, + it: trie.NewIterator(it), + account: account, + }, nil +} + +// Next steps the iterator forward one element. It returns false if the iterator +// is exhausted or if an error occurs. Any error encountered is retained and +// can be retrieved via Error(). +func (ti *merkleIterator) Next() bool { + return ti.it.Next() +} + +// Error returns any failure that occurred during iteration, which might have +// caused a premature iteration exit. +func (ti *merkleIterator) Error() error { + return ti.it.Err +} + +// Hash returns the hash of the account or storage slot the iterator is +// currently at. +func (ti *merkleIterator) Hash() common.Hash { + return common.BytesToHash(ti.it.Key) +} + +// Release releases associated resources. Release should always succeed and +// can be called multiple times without causing error. +func (ti *merkleIterator) Release() {} + +// Address returns the raw account address the iterator is currently at. +// An error will be returned if the preimage is not available. +func (ti *merkleIterator) Address() (common.Address, error) { + if !ti.account { + return common.Address{}, errors.New("account address is not available") + } + preimage := ti.tr.GetKey(ti.it.Key) + if preimage == nil { + return common.Address{}, errors.New("account address is not available") + } + return common.BytesToAddress(preimage), nil +} + +// Account returns the account data the iterator is currently at. +func (ti *merkleIterator) Account() []byte { + if !ti.account { + return nil + } + return ti.it.Value +} + +// Key returns the raw storage slot key the iterator is currently at. +// An error will be returned if the preimage is not available. +func (ti *merkleIterator) Key() (common.Hash, error) { + if ti.account { + return common.Hash{}, errors.New("slot key is not available") + } + preimage := ti.tr.GetKey(ti.it.Key) + if preimage == nil { + return common.Hash{}, errors.New("slot key is not available") + } + return common.BytesToHash(preimage), nil +} + +// Slot returns the storage slot the iterator is currently at. +func (ti *merkleIterator) Slot() []byte { + if ti.account { + return nil + } + return ti.it.Value +} + +// stateIteratee implements Iteratee interface, providing the state traversal +// functionalities of a specific state. +type stateIteratee struct { + merkle bool + root common.Hash + triedb *triedb.Database + snap *snapshot.Tree +} + +func newStateIteratee(merkle bool, root common.Hash, triedb *triedb.Database, snap *snapshot.Tree) (*stateIteratee, error) { + return &stateIteratee{ + merkle: merkle, + root: root, + triedb: triedb, + snap: snap, + }, nil +} + +// NewAccountIterator creates an account iterator for the state specified by +// the given root. It begins at a specified starting position, corresponding +// to a particular initial key (or the next key if the specified one does +// not exist). +// +// The starting position here refers to the hash of the account address. +func (si *stateIteratee) NewAccountIterator(start common.Hash) (AccountIterator, error) { + // If the external snapshot is available (hash scheme), try to initialize + // the account iterator from there first. + if si.snap != nil { + it, err := si.snap.AccountIterator(si.root, start) + if err == nil { + return newFlatAccountIterator(it, si.triedb), nil + } + } + // If the external snapshot is not available, try to initialize the + // account iterator from the trie database (path scheme) + it, err := si.triedb.AccountIterator(si.root, start) + if err == nil { + return newFlatAccountIterator(it, si.triedb), nil + } + if !si.merkle { + return nil, fmt.Errorf("state %x is not available for account traversal", si.root) + } + // The snapshot is not usable so far, construct the account iterator from + // the trie as the fallback. It's not as efficient as the flat state iterator. + tr, err := trie.NewStateTrie(trie.StateTrieID(si.root), si.triedb) + if err != nil { + return nil, err + } + return newMerkleTrieIterator(tr, start, true) +} + +// NewStorageIterator creates a storage iterator for the state specified by +// the address hash. It begins at a specified starting position, corresponding +// to a particular initial key (or the next key if the specified one does not exist). +// +// The starting position here refers to the hash of the slot key. +func (si *stateIteratee) NewStorageIterator(addressHash common.Hash, start common.Hash) (StorageIterator, error) { + // If the external snapshot is available (hash scheme), try to initialize + // the storage iterator from there first. + if si.snap != nil { + it, err := si.snap.StorageIterator(si.root, addressHash, start) + if err == nil { + return newFlatStorageIterator(it, si.triedb), nil + } + } + // If the external snapshot is not available, try to initialize the + // storage iterator from the trie database (path scheme) + it, err := si.triedb.StorageIterator(si.root, addressHash, start) + if err == nil { + return newFlatStorageIterator(it, si.triedb), nil + } + if !si.merkle { + return nil, fmt.Errorf("state %x is not available for storage traversal", si.root) + } + // The snapshot is not usable so far, construct the storage iterator from + // the trie as the fallback. It's not as efficient as the flat state iterator. + tr, err := trie.NewStateTrie(trie.StateTrieID(si.root), si.triedb) + if err != nil { + return nil, err + } + acct, err := tr.GetAccountByHash(addressHash) + if err != nil { + return nil, err + } + if acct == nil || acct.Root == types.EmptyRootHash { + return &exhaustedIterator{}, nil + } + storageTr, err := trie.NewStateTrie(trie.StorageTrieID(si.root, addressHash, acct.Root), si.triedb) + if err != nil { + return nil, err + } + return newMerkleTrieIterator(storageTr, start, false) +} + +type exhaustedIterator struct{} + +func (e exhaustedIterator) Next() bool { + return false +} + +func (e exhaustedIterator) Error() error { + return nil +} + +func (e exhaustedIterator) Hash() common.Hash { + return common.Hash{} +} + +func (e exhaustedIterator) Release() { +} + +func (e exhaustedIterator) Key() (common.Hash, error) { + return common.Hash{}, nil +} + +func (e exhaustedIterator) Slot() []byte { + return nil +} diff --git a/core/state/database_iterator_test.go b/core/state/database_iterator_test.go new file mode 100644 index 0000000000..87819e5526 --- /dev/null +++ b/core/state/database_iterator_test.go @@ -0,0 +1,262 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// TestExhaustedIterator verifies the exhaustedIterator sentinel: Next is false, +// Error is nil, Hash/Key are zero, Slot is nil, and double Release is safe. +func TestExhaustedIterator(t *testing.T) { + var it exhaustedIterator + + if it.Next() { + t.Fatal("Next() returned true") + } + if err := it.Error(); err != nil { + t.Fatalf("Error() = %v, want nil", err) + } + if hash := it.Hash(); hash != (common.Hash{}) { + t.Fatalf("Hash() = %x, want zero", hash) + } + if key, err := it.Key(); key != (common.Hash{}) || err != nil { + t.Fatalf("Key() = %x, %v; want zero, nil", key, err) + } + if slot := it.Slot(); slot != nil { + t.Fatalf("Slot() = %x, want nil", slot) + } + it.Release() + it.Release() +} + +// TestAccountIterator tests the account iterator: correct count, ascending +// hash order, valid full-format RLP, data integrity, address preimage +// resolution, and seek behavior. +func TestAccountIterator(t *testing.T) { + testAccountIterator(t, rawdb.HashScheme) + testAccountIterator(t, rawdb.PathScheme) +} + +func testAccountIterator(t *testing.T, scheme string) { + _, sdb, ndb, root, accounts := makeTestState(scheme) + ndb.Commit(root, false) + + iteratee, err := sdb.Iteratee(root) + if err != nil { + t.Fatalf("(%s) failed to create iteratee: %v", scheme, err) + } + // Build lookups from address hash. + addrByHash := make(map[common.Hash]*testAccount) + for _, acc := range accounts { + addrByHash[crypto.Keccak256Hash(acc.address.Bytes())] = acc + } + + // --- Full iteration: count, ordering, RLP validity, data integrity, address resolution --- + acctIt, err := iteratee.NewAccountIterator(common.Hash{}) + if err != nil { + t.Fatalf("(%s) failed to create account iterator: %v", scheme, err) + } + var ( + hashes []common.Hash + prevHash common.Hash + ) + for acctIt.Next() { + hash := acctIt.Hash() + if hash == (common.Hash{}) { + t.Fatalf("(%s) zero hash at position %d", scheme, len(hashes)) + } + if len(hashes) > 0 && bytes.Compare(prevHash.Bytes(), hash.Bytes()) >= 0 { + t.Fatalf("(%s) hashes not ascending: %x >= %x", scheme, prevHash, hash) + } + prevHash = hash + hashes = append(hashes, hash) + + // Decode and verify account data. + blob := acctIt.Account() + if blob == nil { + t.Fatalf("(%s) nil account at %x", scheme, hash) + } + var decoded types.StateAccount + if err := rlp.DecodeBytes(blob, &decoded); err != nil { + t.Fatalf("(%s) bad RLP at %x: %v", scheme, hash, err) + } + acc := addrByHash[hash] + if decoded.Nonce != acc.nonce { + t.Fatalf("(%s) nonce %x: got %d, want %d", scheme, hash, decoded.Nonce, acc.nonce) + } + if decoded.Balance.Cmp(acc.balance) != 0 { + t.Fatalf("(%s) balance %x: got %v, want %v", scheme, hash, decoded.Balance, acc.balance) + } + // Verify address preimage resolution. + addr, err := acctIt.Address() + if err != nil { + t.Fatalf("(%s) failed to address: %v", scheme, err) + } + if addr != acc.address { + t.Fatalf("(%s) Address() = %x, want %x", scheme, addr, acc.address) + } + } + acctIt.Release() + + if err := acctIt.Error(); err != nil { + t.Fatalf("(%s) iteration error: %v", scheme, err) + } + if len(hashes) != len(accounts) { + t.Fatalf("(%s) iterated %d accounts, want %d", scheme, len(hashes), len(accounts)) + } + + // --- Seek: starting from midpoint should skip earlier entries --- + mid := hashes[len(hashes)/2] + seekIt, err := iteratee.NewAccountIterator(mid) + if err != nil { + t.Fatalf("(%s) failed to create seeked iterator: %v", scheme, err) + } + seekCount := 0 + for seekIt.Next() { + if bytes.Compare(seekIt.Hash().Bytes(), mid.Bytes()) < 0 { + t.Fatalf("(%s) seeked iterator returned hash before start", scheme) + } + seekCount++ + } + seekIt.Release() + + if seekCount != len(hashes)/2 { + t.Fatalf("(%s) unexpected seeked count, %d != %d", scheme, seekCount, len(hashes)/2) + } +} + +// TestStorageIterator tests the storage iterator: correct slot counts against +// the trie, ascending hash order, non-nil slot data, key preimage resolution, +// seek behavior, and empty-storage accounts. +func TestStorageIterator(t *testing.T) { + testStorageIterator(t, rawdb.HashScheme) + testStorageIterator(t, rawdb.PathScheme) +} + +func testStorageIterator(t *testing.T, scheme string) { + _, sdb, ndb, root, accounts := makeTestState(scheme) + ndb.Commit(root, false) + + iteratee, err := sdb.Iteratee(root) + if err != nil { + t.Fatalf("(%s) failed to create iteratee: %v", scheme, err) + } + + // --- Slot count and ordering for every account --- + var withStorage common.Hash // remember an account that has storage for seek test + for _, acc := range accounts { + addrHash := crypto.Keccak256Hash(acc.address.Bytes()) + expected := countStorageSlots(t, scheme, sdb, root, addrHash) + + storageIt, err := iteratee.NewStorageIterator(addrHash, common.Hash{}) + if err != nil { + t.Fatalf("(%s) failed to create storage iterator for %x: %v", scheme, acc.address, err) + } + count := 0 + var prevHash common.Hash + for storageIt.Next() { + hash := storageIt.Hash() + if count > 0 && bytes.Compare(prevHash.Bytes(), hash.Bytes()) >= 0 { + t.Fatalf("(%s) storage hashes not ascending for %x", scheme, acc.address) + } + prevHash = hash + if storageIt.Slot() == nil { + t.Fatalf("(%s) nil slot at %x", scheme, hash) + } + // Check key preimage resolution on first slot. + if _, err := storageIt.Key(); err != nil { + t.Fatalf("(%s) Key() failed to resolve", scheme) + } + count++ + } + if err := storageIt.Error(); err != nil { + t.Fatalf("(%s) storage iteration error for %x: %v", scheme, acc.address, err) + } + storageIt.Release() + + if count != expected { + t.Fatalf("(%s) account %x: %d slots, want %d", scheme, acc.address, count, expected) + } + if count > 0 { + withStorage = addrHash + } + } + + // --- Seek: starting from second slot should skip the first --- + if withStorage == (common.Hash{}) { + t.Fatalf("(%s) no account with storage found", scheme) + } + fullIt, err := iteratee.NewStorageIterator(withStorage, common.Hash{}) + if err != nil { + t.Fatalf("(%s) failed to create full storage iterator: %v", scheme, err) + } + var slotHashes []common.Hash + for fullIt.Next() { + slotHashes = append(slotHashes, fullIt.Hash()) + } + fullIt.Release() + + seekIt, err := iteratee.NewStorageIterator(withStorage, slotHashes[1]) + if err != nil { + t.Fatalf("(%s) failed to create seeked storage iterator: %v", scheme, err) + } + seekCount := 0 + for seekIt.Next() { + if bytes.Compare(seekIt.Hash().Bytes(), slotHashes[1].Bytes()) < 0 { + t.Fatalf("(%s) seeked storage iterator returned hash before start", scheme) + } + seekCount++ + } + seekIt.Release() + + if seekCount != len(slotHashes)-1 { + t.Fatalf("(%s) unexpected seeked storage count %d != %d", scheme, seekCount, len(slotHashes)-1) + } +} + +// countStorageSlots counts storage slots for an account by opening the +// storage trie directly. +func countStorageSlots(t *testing.T, scheme string, sdb Database, root common.Hash, addrHash common.Hash) int { + t.Helper() + accTrie, err := trie.NewStateTrie(trie.StateTrieID(root), sdb.TrieDB()) + if err != nil { + t.Fatalf("(%s) failed to open account trie: %v", scheme, err) + } + acct, err := accTrie.GetAccountByHash(addrHash) + if err != nil || acct == nil || acct.Root == types.EmptyRootHash { + return 0 + } + storageTrie, err := trie.NewStateTrie(trie.StorageTrieID(root, addrHash, acct.Root), sdb.TrieDB()) + if err != nil { + t.Fatalf("(%s) failed to open storage trie for %x: %v", scheme, addrHash, err) + } + it := trie.NewIterator(storageTrie.MustNodeIterator(nil)) + count := 0 + for it.Next() { + count++ + } + return count +} diff --git a/core/state/dump.go b/core/state/dump.go index 130e4bfa9f..8fcbdb1fea 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/bintrie" ) @@ -45,6 +44,7 @@ type DumpConfig struct { type DumpCollector interface { // OnRoot is called with the state root OnRoot(common.Hash) + // OnAccount is called once for each account in the trie OnAccount(*common.Address, DumpAccount) } @@ -65,9 +65,10 @@ type DumpAccount struct { type Dump struct { Root string `json:"root"` Accounts map[string]DumpAccount `json:"accounts"` + // Next can be set to represent that this dump is only partial, and Next // is where an iterator should be positioned in order to continue the dump. - Next []byte `json:"next,omitempty"` // nil if no more accounts + Next hexutil.Bytes `json:"next,omitempty"` // nil if no more accounts } // OnRoot implements DumpCollector interface @@ -115,9 +116,6 @@ func (d iterativeDump) OnRoot(root common.Hash) { // DumpToCollector iterates the state according to the given options and inserts // the items into a collector for aggregation or serialization. -// -// The state iterator is still trie-based and can be converted to snapshot-based -// once the state snapshot is fully integrated into database. TODO(rjl493456442). func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) { // Sanitize the input to allow nil configs if conf == nil { @@ -133,20 +131,23 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] log.Info("Trie dumping started", "root", s.originalRoot) c.OnRoot(s.originalRoot) - tr, err := s.db.OpenTrie(s.originalRoot) + iteratee, err := s.db.Iteratee(s.originalRoot) if err != nil { return nil } - trieIt, err := tr.NodeIterator(conf.Start) + var startHash common.Hash + if conf.Start != nil { + startHash = common.BytesToHash(conf.Start) + } + acctIt, err := iteratee.NewAccountIterator(startHash) if err != nil { - log.Error("Trie dumping error", "err", err) return nil } - it := trie.NewIterator(trieIt) + defer acctIt.Release() - for it.Next() { + for acctIt.Next() { var data types.StateAccount - if err := rlp.DecodeBytes(it.Value, &data); err != nil { + if err := rlp.DecodeBytes(acctIt.Account(), &data); err != nil { panic(err) } var ( @@ -155,24 +156,22 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] Nonce: data.Nonce, Root: data.Root[:], CodeHash: data.CodeHash, - AddressHash: it.Key, + AddressHash: acctIt.Hash().Bytes(), } - address *common.Address - addr common.Address - addrBytes = tr.GetKey(it.Key) + address *common.Address ) - if addrBytes == nil { + addrBytes, err := acctIt.Address() + if err != nil { missingPreimages++ if conf.OnlyWithAddresses { continue } } else { - addr = common.BytesToAddress(addrBytes) - address = &addr + address = &addrBytes account.Address = address } - obj := newObject(s, addr, &data) + obj := newObject(s, addrBytes, &data) if !conf.SkipCode { account.Code = obj.Code() } @@ -180,42 +179,35 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) - storageTr, err := s.db.OpenStorageTrie(s.originalRoot, addr, obj.Root(), tr) + storageIt, err := iteratee.NewStorageIterator(acctIt.Hash(), common.Hash{}) if err != nil { log.Error("Failed to load storage trie", "err", err) continue } - trieIt, err := storageTr.NodeIterator(nil) - if err != nil { - log.Error("Failed to create trie iterator", "err", err) - continue - } - storageIt := trie.NewIterator(trieIt) for storageIt.Next() { - _, content, _, err := rlp.Split(storageIt.Value) + _, content, _, err := rlp.Split(storageIt.Slot()) if err != nil { log.Error("Failed to decode the value returned by iterator", "error", err) continue } - key := storageTr.GetKey(storageIt.Key) - if key == nil { + key, err := storageIt.Key() + if err != nil { continue } - account.Storage[common.BytesToHash(key)] = common.Bytes2Hex(content) + account.Storage[key] = common.Bytes2Hex(content) } + storageIt.Release() } c.OnAccount(address, account) accounts++ if time.Since(logged) > 8*time.Second { - log.Info("Trie dumping in progress", "at", common.Bytes2Hex(it.Key), "accounts", accounts, - "elapsed", common.PrettyDuration(time.Since(start))) - + log.Info("Trie dumping in progress", "at", acctIt.Hash().Hex(), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } if conf.Max > 0 && accounts >= conf.Max { - if it.Next() { - nextKey = it.Key + if acctIt.Next() { + nextKey = acctIt.Hash().Bytes() } break @@ -225,10 +217,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] if missingPreimages > 0 { log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } - - log.Info("Trie dumping complete", "accounts", accounts, - "elapsed", common.PrettyDuration(time.Since(start))) - + log.Info("Trie dumping complete", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) return nextKey } diff --git a/core/state/statedb.go b/core/state/statedb.go index d1ea3d877f..ca7a4497b8 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -33,6 +33,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/blockstm" "github.com/ethereum/go-ethereum/core/rawdb" + + // Upstream dropped this import in #33102; Bor still needs it for the + // BlockSTM-only NewWithMVHashmap constructor. "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/tracing" @@ -1615,10 +1618,12 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if err := s.trie.UpdateStorage(addr, key[:], common.TrimLeftZeroes(value[:])); err != nil { s.setError(err) } + s.StorageUpdated.Add(1) } else { if err := s.trie.DeleteStorage(addr, key[:]); err != nil { s.setError(err) } + s.StorageDeleted.Add(1) } } } @@ -1778,31 +1783,32 @@ func (s *StateDB) clearJournalAndRefund() { s.refund = 0 } -// fastDeleteStorage is the function that efficiently deletes the storage trie -// of a specific account. It leverages the associated state snapshot for fast -// storage iteration and constructs trie node deletion markers by creating -// stack trie with iterated slots. -func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { - iter, err := snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) - if err != nil { - return nil, nil, nil, err - } - defer iter.Release() - +// deleteStorage is designed to delete the storage trie of a designated account. +func (s *StateDB) deleteStorage(addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { var ( nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil) storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil) storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot ) + iteratee, err := s.db.Iteratee(s.originalRoot) + if err != nil { + return nil, nil, nil, err + } + it, err := iteratee.NewStorageIterator(addrHash, common.Hash{}) + if err != nil { + return nil, nil, nil, err + } + defer it.Release() + stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { nodes.AddNode(path, trienode.NewDeletedWithPrev(blob)) }) - for iter.Next() { - slot := common.CopyBytes(iter.Slot()) - if err := iter.Error(); err != nil { // error might occur after Slot function + for it.Next() { + slot := common.CopyBytes(it.Slot()) + if err := it.Error(); err != nil { // error might occur after Slot function return nil, nil, nil, err } - key := iter.Hash() + key := it.Hash() storages[key] = nil storageOrigins[key] = slot @@ -1810,7 +1816,7 @@ func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, return nil, nil, nil, err } } - if err := iter.Error(); err != nil { // error might occur during iteration + if err := it.Error(); err != nil { // error might occur during iteration return nil, nil, nil, err } if stack.Hash() != root { @@ -1819,68 +1825,6 @@ func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, return storages, storageOrigins, nodes, nil } -// slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage," -// employed when the associated state snapshot is not available. It iterates the -// storage slots along with all internal trie nodes via trie directly. -func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { - tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) - } - it, err := tr.NodeIterator(nil) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) - } - var ( - nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil) - storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil) - storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot - ) - for it.Next(true) { - if it.Leaf() { - key := common.BytesToHash(it.LeafKey()) - storages[key] = nil - storageOrigins[key] = common.CopyBytes(it.LeafBlob()) - continue - } - if it.Hash() == (common.Hash{}) { - continue - } - nodes.AddNode(it.Path(), trienode.NewDeletedWithPrev(it.NodeBlob())) - } - if err := it.Error(); err != nil { - return nil, nil, nil, err - } - return storages, storageOrigins, nodes, nil -} - -// deleteStorage is designed to delete the storage trie of a designated account. -// The function will make an attempt to utilize an efficient strategy if the -// associated state snapshot is reachable; otherwise, it will resort to a less -// efficient approach. -func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { - var ( - err error - nodes *trienode.NodeSet // the set for trie node mutations (value is nil) - storages map[common.Hash][]byte // the set for storage mutations (value is nil) - storageOrigins map[common.Hash][]byte // the set for tracking the original value of slot - ) - // The fast approach can be failed if the snapshot is not fully - // generated, or it's internally corrupted. Fallback to the slow - // one just in case. - snaps := s.db.Snapshot() - if snaps != nil { - storages, storageOrigins, nodes, err = s.fastDeleteStorage(snaps, addrHash, root) - } - if snaps == nil || err != nil { - storages, storageOrigins, nodes, err = s.slowDeleteStorage(addr, addrHash, root) - } - if err != nil { - return nil, nil, nil, err - } - return storages, storageOrigins, nodes, nil -} - // handleDestruction processes all destruction markers and deletes the account // and associated storage slots if necessary. There are four potential scenarios // as following: @@ -1931,7 +1875,7 @@ func (s *StateDB) handleDestruction(noStorageWiping bool) (map[common.Hash]*acco return nil, nil, fmt.Errorf("unexpected storage wiping, %x", addr) } // Remove storage slots belonging to the account. - storages, storagesOrigin, set, err := s.deleteStorage(addr, addrHash, prev.Root) + storages, storagesOrigin, set, err := s.deleteStorage(addrHash, prev.Root) if err != nil { return nil, nil, fmt.Errorf("failed to delete storage, err: %w", err) } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 4c1ece5e52..f5a32bc1d1 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1962,12 +1962,12 @@ func TestDeleteStorage(t *testing.T) { obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root - _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, _, fastNodes, err := fastState.deleteStorage(crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } - _, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, _, slowNodes, err := slowState.deleteStorage(crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } diff --git a/core/state_processor.go b/core/state_processor.go index 1d49d8d153..2b9699fb78 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -365,6 +365,9 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, evm *vm.EVM) { evm.SetTxContext(NewEVMTxContext(msg)) evm.StateDB.AddAddressToAccessList(params.BeaconRootsAddress) _, _, _ = evm.Call(msg.From, *msg.To, msg.Data, 30_000_000, common.U2560) + if evm.StateDB.AccessEvents() != nil { + evm.StateDB.AccessEvents().Merge(evm.AccessEvents) + } evm.StateDB.Finalise(true) } @@ -428,6 +431,9 @@ func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte evm.SetTxContext(NewEVMTxContext(msg)) evm.StateDB.AddAddressToAccessList(addr) ret, _, err := evm.Call(msg.From, *msg.To, msg.Data, 30_000_000, common.U2560) + if evm.StateDB.AccessEvents() != nil { + evm.StateDB.AccessEvents().Merge(evm.AccessEvents) + } evm.StateDB.Finalise(true) if err != nil { return fmt.Errorf("system call failed to execute: %v", err) diff --git a/core/types/bal/bal_encoding.go b/core/types/bal/bal_encoding.go index f4b5d9faa9..1e9155a038 100644 --- a/core/types/bal/bal_encoding.go +++ b/core/types/bal/bal_encoding.go @@ -351,9 +351,12 @@ func (e *BlockAccessList) PrettyPrint() string { } // Copy returns a deep copy of the access list -func (e *BlockAccessList) Copy() (res BlockAccessList) { +func (e *BlockAccessList) Copy() *BlockAccessList { + cpy := &BlockAccessList{ + Accesses: make([]AccountAccess, 0, len(e.Accesses)), + } for _, accountAccess := range e.Accesses { - res.Accesses = append(res.Accesses, accountAccess.Copy()) + cpy.Accesses = append(cpy.Accesses, accountAccess.Copy()) } - return + return cpy } diff --git a/core/types/bal/bal_test.go b/core/types/bal/bal_test.go index a20390dcc6..9e86623285 100644 --- a/core/types/bal/bal_test.go +++ b/core/types/bal/bal_test.go @@ -191,8 +191,8 @@ func makeTestAccountAccess(sort bool) AccountAccess { } } -func makeTestBAL(sort bool) BlockAccessList { - list := BlockAccessList{} +func makeTestBAL(sort bool) *BlockAccessList { + list := &BlockAccessList{} for i := 0; i < 5; i++ { list.Accesses = append(list.Accesses, makeTestAccountAccess(sort)) } diff --git a/core/types/block.go b/core/types/block.go index 90a0606aa0..e38f51d56e 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types/bal" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -111,6 +112,9 @@ type Header struct { // RequestsHash was added by EIP-7685 and is ignored in legacy headers. RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + // BlockAccessListHash was added by EIP-7928 and is ignored in legacy headers. + BlockAccessListHash *common.Hash `json:"balHash" rlp:"optional"` + // SlotNumber was added by EIP-7843 and is ignored in legacy headers. SlotNumber *uint64 `json:"slotNumber" rlp:"optional"` } @@ -293,6 +297,7 @@ type Block struct { uncles []*Header transactions Transactions withdrawals Withdrawals + accessList *bal.BlockAccessList // caches hash atomic.Pointer[common.Hash] @@ -411,6 +416,10 @@ func CopyHeader(h *Header) *Header { cpy.RequestsHash = new(common.Hash) *cpy.RequestsHash = *h.RequestsHash } + if h.BlockAccessListHash != nil { + cpy.BlockAccessListHash = new(common.Hash) + *cpy.BlockAccessListHash = *h.BlockAccessListHash + } if h.SlotNumber != nil { cpy.SlotNumber = new(uint64) *cpy.SlotNumber = *h.SlotNumber @@ -452,9 +461,10 @@ func (b *Block) Body() *Body { // Accessors for body data. These do not return a copy because the content // of the body slices does not affect the cached hash/size in block. -func (b *Block) Uncles() []*Header { return b.uncles } -func (b *Block) Transactions() Transactions { return b.transactions } -func (b *Block) Withdrawals() Withdrawals { return b.withdrawals } +func (b *Block) Uncles() []*Header { return b.uncles } +func (b *Block) Transactions() Transactions { return b.transactions } +func (b *Block) Withdrawals() Withdrawals { return b.withdrawals } +func (b *Block) AccessList() *bal.BlockAccessList { return b.accessList } func (b *Block) Transaction(hash common.Hash) *Transaction { for _, transaction := range b.transactions { @@ -714,6 +724,7 @@ func (b *Block) WithSeal(header *Header) *Block { transactions: b.transactions, uncles: b.uncles, withdrawals: b.withdrawals, + accessList: b.accessList, } } @@ -725,6 +736,7 @@ func (b *Block) WithBody(body Body) *Block { transactions: slices.Clone(body.Transactions), uncles: make([]*Header, len(body.Uncles)), withdrawals: slices.Clone(body.Withdrawals), + accessList: b.accessList, } for i := range body.Uncles { block.uncles[i] = CopyHeader(body.Uncles[i]) @@ -732,6 +744,24 @@ func (b *Block) WithBody(body Body) *Block { return block } +// WithAccessList returns a copy of the block with the given access list embedded. +func (b *Block) WithAccessList(accessList *bal.BlockAccessList) *Block { + return b.WithAccessListUnsafe(accessList.Copy()) +} + +// WithAccessListUnsafe returns a copy of the block with the given access list +// embedded. Note that the access list is not deep-copied; use WithAccessList +// if the provided list may be modified by other actors. +func (b *Block) WithAccessListUnsafe(accessList *bal.BlockAccessList) *Block { + return &Block{ + header: b.header, + transactions: b.transactions, + uncles: b.uncles, + withdrawals: b.withdrawals, + accessList: accessList, + } +} + // Hash returns the keccak256 hash of b's header. // The hash is computed on the first call and cached thereafter. func (b *Block) Hash() common.Hash { diff --git a/core/types/gen_header_json.go b/core/types/gen_header_json.go index 16fb03f612..2e2f1cdca5 100644 --- a/core/types/gen_header_json.go +++ b/core/types/gen_header_json.go @@ -16,29 +16,30 @@ var _ = (*headerMarshaling)(nil) // MarshalJSON marshals as JSON. func (h Header) MarshalJSON() ([]byte, error) { type Header struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - UncleHash common.Hash `json:"sha3Uncles" gencodec:"required"` - Coinbase common.Address `json:"miner"` - Root common.Hash `json:"stateRoot" gencodec:"required"` - TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` - ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` - Bloom Bloom `json:"logsBloom" gencodec:"required"` - Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` - Number *hexutil.Big `json:"number" gencodec:"required"` - GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Time hexutil.Uint64 `json:"timestamp" gencodec:"required"` - Extra hexutil.Bytes `json:"extraData" gencodec:"required"` - MixDigest common.Hash `json:"mixHash"` - Nonce BlockNonce `json:"nonce"` - BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` - WithdrawalsHash *common.Hash `json:"withdrawalsRoot" rlp:"optional"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` - ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` - RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` - SlotNumber *hexutil.Uint64 `json:"slotNumber" rlp:"optional"` - Hash common.Hash `json:"hash"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + UncleHash common.Hash `json:"sha3Uncles" gencodec:"required"` + Coinbase common.Address `json:"miner"` + Root common.Hash `json:"stateRoot" gencodec:"required"` + TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` + ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` + Bloom Bloom `json:"logsBloom" gencodec:"required"` + Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` + Number *hexutil.Big `json:"number" gencodec:"required"` + GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Time hexutil.Uint64 `json:"timestamp" gencodec:"required"` + Extra hexutil.Bytes `json:"extraData" gencodec:"required"` + MixDigest common.Hash `json:"mixHash"` + Nonce BlockNonce `json:"nonce"` + BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` + WithdrawalsHash *common.Hash `json:"withdrawalsRoot" rlp:"optional"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` + ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + BlockAccessListHash *common.Hash `json:"balHash" rlp:"optional"` + SlotNumber *hexutil.Uint64 `json:"slotNumber" rlp:"optional"` + Hash common.Hash `json:"hash"` } var enc Header enc.ParentHash = h.ParentHash @@ -62,6 +63,7 @@ func (h Header) MarshalJSON() ([]byte, error) { enc.ExcessBlobGas = (*hexutil.Uint64)(h.ExcessBlobGas) enc.ParentBeaconRoot = h.ParentBeaconRoot enc.RequestsHash = h.RequestsHash + enc.BlockAccessListHash = h.BlockAccessListHash enc.SlotNumber = (*hexutil.Uint64)(h.SlotNumber) enc.Hash = h.Hash() return json.Marshal(&enc) @@ -70,28 +72,29 @@ func (h Header) MarshalJSON() ([]byte, error) { // UnmarshalJSON unmarshals from JSON. func (h *Header) UnmarshalJSON(input []byte) error { type Header struct { - ParentHash *common.Hash `json:"parentHash" gencodec:"required"` - UncleHash *common.Hash `json:"sha3Uncles" gencodec:"required"` - Coinbase *common.Address `json:"miner"` - Root *common.Hash `json:"stateRoot" gencodec:"required"` - TxHash *common.Hash `json:"transactionsRoot" gencodec:"required"` - ReceiptHash *common.Hash `json:"receiptsRoot" gencodec:"required"` - Bloom *Bloom `json:"logsBloom" gencodec:"required"` - Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` - Number *hexutil.Big `json:"number" gencodec:"required"` - GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Time *hexutil.Uint64 `json:"timestamp" gencodec:"required"` - Extra *hexutil.Bytes `json:"extraData" gencodec:"required"` - MixDigest *common.Hash `json:"mixHash"` - Nonce *BlockNonce `json:"nonce"` - BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` - WithdrawalsHash *common.Hash `json:"withdrawalsRoot" rlp:"optional"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` - ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` - RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` - SlotNumber *hexutil.Uint64 `json:"slotNumber" rlp:"optional"` + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + UncleHash *common.Hash `json:"sha3Uncles" gencodec:"required"` + Coinbase *common.Address `json:"miner"` + Root *common.Hash `json:"stateRoot" gencodec:"required"` + TxHash *common.Hash `json:"transactionsRoot" gencodec:"required"` + ReceiptHash *common.Hash `json:"receiptsRoot" gencodec:"required"` + Bloom *Bloom `json:"logsBloom" gencodec:"required"` + Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` + Number *hexutil.Big `json:"number" gencodec:"required"` + GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Time *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + Extra *hexutil.Bytes `json:"extraData" gencodec:"required"` + MixDigest *common.Hash `json:"mixHash"` + Nonce *BlockNonce `json:"nonce"` + BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` + WithdrawalsHash *common.Hash `json:"withdrawalsRoot" rlp:"optional"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` + ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + BlockAccessListHash *common.Hash `json:"balHash" rlp:"optional"` + SlotNumber *hexutil.Uint64 `json:"slotNumber" rlp:"optional"` } var dec Header if err := json.Unmarshal(input, &dec); err != nil { @@ -172,6 +175,9 @@ func (h *Header) UnmarshalJSON(input []byte) error { if dec.RequestsHash != nil { h.RequestsHash = dec.RequestsHash } + if dec.BlockAccessListHash != nil { + h.BlockAccessListHash = dec.BlockAccessListHash + } if dec.SlotNumber != nil { h.SlotNumber = (*uint64)(dec.SlotNumber) } diff --git a/core/types/gen_header_rlp.go b/core/types/gen_header_rlp.go index 943815546f..d7b1d73c99 100644 --- a/core/types/gen_header_rlp.go +++ b/core/types/gen_header_rlp.go @@ -46,8 +46,9 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { _tmp4 := obj.ExcessBlobGas != nil _tmp5 := obj.ParentBeaconRoot != nil _tmp6 := obj.RequestsHash != nil - _tmp7 := obj.SlotNumber != nil - if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + _tmp7 := obj.BlockAccessListHash != nil + _tmp8 := obj.SlotNumber != nil + if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 || _tmp8 { if obj.BaseFee == nil { w.Write(rlp.EmptyString) } else { @@ -57,42 +58,49 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { w.WriteBigInt(obj.BaseFee) } } - if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 || _tmp8 { if obj.WithdrawalsHash == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.WithdrawalsHash[:]) } } - if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { + if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 || _tmp8 { if obj.BlobGasUsed == nil { w.Write([]byte{0x80}) } else { w.WriteUint64((*obj.BlobGasUsed)) } } - if _tmp4 || _tmp5 || _tmp6 || _tmp7 { + if _tmp4 || _tmp5 || _tmp6 || _tmp7 || _tmp8 { if obj.ExcessBlobGas == nil { w.Write([]byte{0x80}) } else { w.WriteUint64((*obj.ExcessBlobGas)) } } - if _tmp5 || _tmp6 || _tmp7 { + if _tmp5 || _tmp6 || _tmp7 || _tmp8 { if obj.ParentBeaconRoot == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.ParentBeaconRoot[:]) } } - if _tmp6 || _tmp7 { + if _tmp6 || _tmp7 || _tmp8 { if obj.RequestsHash == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.RequestsHash[:]) } } - if _tmp7 { + if _tmp7 || _tmp8 { + if obj.BlockAccessListHash == nil { + w.Write([]byte{0x80}) + } else { + w.WriteBytes(obj.BlockAccessListHash[:]) + } + } + if _tmp8 { if obj.SlotNumber == nil { w.Write([]byte{0x80}) } else { diff --git a/eth/api_backend.go b/eth/api_backend.go index 17871fea42..f92e766079 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -513,9 +513,10 @@ func (b *EthAPIBackend) SyncProgress(ctx context.Context) ethereum.SyncProgress prog.TxIndexFinishedBlocks = txProg.Indexed prog.TxIndexRemainingBlocks = txProg.Remaining } - remain, err := b.eth.blockchain.StateIndexProgress() + stateRemain, trienodeRemain, err := b.eth.blockchain.StateIndexProgress() if err == nil { - prog.StateIndexRemaining = remain + prog.StateIndexRemaining = stateRemain + prog.TrienodeIndexRemaining = trienodeRemain } return prog } @@ -604,12 +605,12 @@ func (b *EthAPIBackend) StartMining() error { return b.eth.StartMining() } -func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { - return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) +func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { + return b.eth.stateAtBlock(ctx, block, base, readOnly, preferDisk) } -func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*types.Transaction, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { - return b.eth.stateAtTransaction(ctx, block, txIndex, reexec) +func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int) (*types.Transaction, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { + return b.eth.stateAtTransaction(ctx, block, txIndex) } func (b *EthAPIBackend) GetWhitelistedCheckpoint() (bool, uint64, common.Hash) { diff --git a/eth/api_debug.go b/eth/api_debug.go index 24c96690a0..2ea483da3b 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -221,7 +221,7 @@ func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockNrOrHash rpc.Block if block == nil { return StorageRangeResult{}, fmt.Errorf("block %v not found", blockNrOrHash) } - _, _, statedb, release, err := api.eth.stateAtTransaction(ctx, block, txIndex, 0) + _, _, statedb, release, err := api.eth.stateAtTransaction(ctx, block, txIndex) if err != nil { return StorageRangeResult{}, err } @@ -235,6 +235,8 @@ func storageRangeAt(statedb *state.StateDB, root common.Hash, address common.Add if storageRoot == types.EmptyRootHash || storageRoot == (common.Hash{}) { return StorageRangeResult{}, nil // empty storage } + // TODO(rjl493456442) it's problematic for traversing the state with in-memory + // state mutations, specifically txIndex != 0. id := trie.StorageTrieID(root, crypto.Keccak256Hash(address.Bytes()), storageRoot) tr, err := trie.NewStateTrie(id, statedb.Database().TrieDB()) if err != nil { diff --git a/eth/downloader/api.go b/eth/downloader/api.go index f97371de5f..1fea35775e 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -81,9 +81,10 @@ func (api *DownloaderAPI) eventLoop() { prog.TxIndexFinishedBlocks = txProg.Indexed prog.TxIndexRemainingBlocks = txProg.Remaining } - remain, err := api.chain.StateIndexProgress() + stateRemain, trienodeRemain, err := api.chain.StateIndexProgress() if err == nil { - prog.StateIndexRemaining = remain + prog.StateIndexRemaining = stateRemain + prog.TrienodeIndexRemaining = trienodeRemain } return prog } diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index c3fd479aa6..f0777f4196 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/internal/ethapi/override" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) @@ -38,11 +37,12 @@ import ( // these together, it would be excessively hard to test. Splitting the parts out // allows testing without needing a proper live chain. type Options struct { - Config *params.ChainConfig // Chain configuration for hard fork selection - Chain core.ChainContext // Chain context to access past block hashes - Header *types.Header // Header defining the block context to execute in - State *state.StateDB // Pre-state on top of which to estimate the gas - BlockOverrides *override.BlockOverrides // Block overrides to apply during the estimation + Config *params.ChainConfig // Chain configuration for hard fork selection + Chain core.ChainContext // Chain context to access past block hashes + Header *types.Header // Header defining the block context to execute in + State *state.StateDB // Pre-state on top of which to estimate the gas + + BlobBaseFee *big.Int // BlobBaseFee optionally overrides the blob base fee in the execution context. ErrorRatio float64 // Allowed overestimation ratio for faster estimation termination } @@ -64,14 +64,10 @@ func Estimate(ctx context.Context, call *core.Message, opts *Options, gasCap uin // Cap the maximum gas allowance according to EIP-7825 if the estimation targets Osaka if hi > params.MaxTxGas { - blockNumber := opts.Header.Number - if opts.BlockOverrides != nil { - if opts.BlockOverrides.Number != nil { - blockNumber = opts.BlockOverrides.Number.ToInt() - } - } - isOsaka := opts.Config.IsOsaka(blockNumber) - isMadhugiri := opts.Config.Bor != nil && opts.Config.Bor.IsMadhugiri(blockNumber) + // Block overrides are applied to the header by the caller (#34081), so the + // header is already the effective block context here. + isOsaka := opts.Config.IsOsaka(opts.Header.Number) + isMadhugiri := opts.Config.Bor != nil && opts.Config.Bor.IsMadhugiri(opts.Header.Number) if isOsaka || isMadhugiri { hi = params.MaxTxGas } @@ -240,10 +236,8 @@ func run(ctx context.Context, call *core.Message, opts *Options) (*core.Executio evmContext = core.NewEVMBlockContext(opts.Header, opts.Chain, nil) dirtyState = opts.State.Copy() ) - if opts.BlockOverrides != nil { - if err := opts.BlockOverrides.Apply(&evmContext); err != nil { - return nil, err - } + if opts.BlobBaseFee != nil { + evmContext.BlobBaseFee = new(big.Int).Set(opts.BlobBaseFee) } // Lower the basefee to 0 to avoid breaking EVM // invariants (basefee < feecap). diff --git a/eth/state_accessor.go b/eth/state_accessor.go index a685d8e33d..d6becd70ed 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -38,7 +38,11 @@ import ( // for releasing state. var noopReleaser = tracers.StateReleaseFunc(func() {}) -func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { +// reexecLimit is the maximum number of ancestor blocks to walk back when +// attempting to reconstruct missing historical state for hash-scheme nodes. +const reexecLimit = uint64(128) + +func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { var ( current *types.Block database state.Database @@ -103,7 +107,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u } } // Database does not have the state for the given block, try to regenerate - for i := uint64(0); i < reexec; i++ { + for i := uint64(0); i < reexecLimit; i++ { if err := ctx.Err(); err != nil { return nil, nil, err } @@ -128,7 +132,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u if err != nil { switch err.(type) { case *trie.MissingNodeError: - return nil, nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) + return nil, nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexecLimit) default: return nil, nil, err } @@ -201,10 +205,9 @@ func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), erro } // stateAtBlock retrieves the state database associated with a certain block. -// If no state is locally available for the given block, a number of blocks -// are attempted to be reexecuted to generate the desired state. The optional -// base layer statedb can be provided which is regarded as the statedb of the -// parent block. +// If no state is locally available for the given block, up to reexecLimit ancestor +// blocks are reexecuted to generate the desired state. The optional base layer +// statedb can be provided which is regarded as the statedb of the parent block. // // An additional release function will be returned if the requested state is // available. Release is expected to be invoked when the returned state is no @@ -213,7 +216,6 @@ func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), erro // // Parameters: // - block: The block for which we want the state(state = block.Root) -// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state // - base: If the caller is tracing multiple blocks, the caller can provide the parent // state continuously from the callsite. // - readOnly: If true, then the live 'blockchain' state database is used. No mutation should @@ -222,9 +224,9 @@ func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), erro // - preferDisk: This arg can be used by the caller to signal that even though the 'base' is // provided, it would be preferable to start from a fresh state, if we have it // on disk. -func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { +func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { if eth.blockchain.TrieDB().Scheme() == rawdb.HashScheme { - return eth.hashState(ctx, block, reexec, base, readOnly, preferDisk) + return eth.hashState(ctx, block, base, readOnly, preferDisk) } return eth.pathState(block) } @@ -236,7 +238,7 @@ func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexe // function will return the state of block after the pre-block operations have // been completed (e.g. updating system contracts), but before post-block // operations are completed (e.g. processing withdrawals). -func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*types.Transaction, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { +func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, txIndex int) (*types.Transaction, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { // Short circuit if it's genesis block. if block.NumberU64() == 0 { return nil, vm.BlockContext{}, nil, nil, errors.New("no transaction in genesis") @@ -252,7 +254,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, } // Lookup the statedb of parent block from the live database, // otherwise regenerate it on the flight. - statedb, release, err := eth.stateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := eth.stateAtBlock(ctx, parent, nil, true, false) if err != nil { return nil, vm.BlockContext{}, nil, nil, err } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 825e057834..5b6095534c 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -53,11 +53,6 @@ const ( // by default before being forcefully aborted. defaultTraceTimeout = 5 * time.Second - // defaultTraceReexec is the number of blocks the tracer is willing to go back - // and reexecute to produce missing historical state necessary to run a specific - // trace. - defaultTraceReexec = uint64(128) - // defaultTracechainMemLimit is the size of the triedb, at which traceChain // switches over and tries to use a disk-backed database instead of building // on top of memory. @@ -98,8 +93,8 @@ type Backend interface { ChainConfig() *params.ChainConfig Engine() consensus.Engine ChainDb() ethdb.Database - StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, StateReleaseFunc, error) - StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*types.Transaction, vm.BlockContext, *state.StateDB, StateReleaseFunc, error) + StateAtBlock(ctx context.Context, block *types.Block, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, StateReleaseFunc, error) + StateAtTransaction(ctx context.Context, block *types.Block, txIndex int) (*types.Transaction, vm.BlockContext, *state.StateDB, StateReleaseFunc, error) } // API is the collection of tracing APIs exposed over the private debugging endpoint. @@ -171,7 +166,6 @@ type TraceConfig struct { *logger.Config Tracer *string Timeout *string - Reexec *uint64 // gasBailout suppresses the synthetic upfront gas debit/refund used by // trace_call while retaining the normal balance validation. gasBailout bool @@ -192,7 +186,6 @@ type TraceCallConfig struct { // StdTraceConfig holds extra parameters to standard-json trace functions. type StdTraceConfig struct { logger.Config - Reexec *uint64 TxHash common.Hash } @@ -270,11 +263,6 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf // The tracing procedure should be aborted in case the closed signal is received. // nolint:gocognit func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed <-chan error) chan *blockTraceResult { - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - blocks := int(end.NumberU64() - start.NumberU64()) threads := runtime.NumCPU() if threads > blocks { @@ -409,8 +397,7 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed s1, s2, s3 := statedb.Database().TrieDB().Size() preferDisk = s1+s2+s3 > defaultTracechainMemLimit } - - statedb, release, err = api.backend.StateAtBlock(ctx, block, reexec, statedb, false, preferDisk) + statedb, release, err = api.backend.StateAtBlock(ctx, block, statedb, false, preferDisk) if err != nil { failed = err break @@ -574,13 +561,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config if err != nil { return nil, err } - - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - - statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, nil, true, false) if err != nil { return nil, err } @@ -657,11 +638,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac if err != nil { return nil, err } - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, nil, true, false) if err != nil { return nil, err } @@ -854,13 +831,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block if err != nil { return nil, err } - - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - - statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, nil, true, false) if err != nil { return nil, err } @@ -1062,15 +1033,10 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc return nil, err } // try to recompute the state - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - if config != nil && config.TxIndex != nil { - _, _, statedb, release, err = api.backend.StateAtTransaction(ctx, block, int(*config.TxIndex), reexec) + _, _, statedb, release, err = api.backend.StateAtTransaction(ctx, block, int(*config.TxIndex)) } else { - statedb, release, err = api.backend.StateAtBlock(ctx, block, reexec, nil, true, false) + statedb, release, err = api.backend.StateAtBlock(ctx, block, nil, true, false) } if err != nil { return nil, err @@ -1196,12 +1162,6 @@ func (api *API) TraceCallMany(ctx context.Context, bundles []Bundle, simulateCon if err != nil { return nil, err } - // try to recompute the state - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - // Default tx index is "-1" which means full block var txIndex = -1 if simulateContext.TransactionIndex != nil { @@ -1218,9 +1178,9 @@ func (api *API) TraceCallMany(ctx context.Context, bundles []Bundle, simulateCon } if txIndex == -1 { - statedb, release, err = api.backend.StateAtBlock(ctx, block, reexec, nil, true, false) + statedb, release, err = api.backend.StateAtBlock(ctx, block, nil, true, false) } else { - _, _, statedb, release, err = api.backend.StateAtTransaction(ctx, block, txIndex, reexec) + _, _, statedb, release, err = api.backend.StateAtTransaction(ctx, block, txIndex) } if err != nil { return nil, err diff --git a/eth/tracers/api_statesync_test.go b/eth/tracers/api_statesync_test.go index 9b204a95e4..b86bb0d736 100644 --- a/eth/tracers/api_statesync_test.go +++ b/eth/tracers/api_statesync_test.go @@ -469,10 +469,10 @@ func TestIntermediateRoots_WithStateSyncTx(t *testing.T) { } } -// TestIntermediateRoots_WithReexecOverride exercises the `config.Reexec` override branch -// Passing a non-nil config with Reexec set must not change the result vs the default +// TestIntermediateRoots_WithNonNilConfig exercises the non-nil-config branch +// Passing a non-nil config must not change the result vs the default // (no config). Covers the trivial-but-untouched config-handling path. -func TestIntermediateRoots_WithReexecOverride(t *testing.T) { +func TestIntermediateRoots_WithNonNilConfig(t *testing.T) { t.Parallel() backend, api, stateSyncBlock := newStateSyncTestSetup(t, 3, 2) @@ -481,8 +481,7 @@ func TestIntermediateRoots_WithReexecOverride(t *testing.T) { block, _ := backend.BlockByNumber(context.Background(), rpc.BlockNumber(stateSyncBlock)) require.NotNil(t, block) - reexec := uint64(8) - withConfig, err := api.IntermediateRoots(context.Background(), block.Hash(), &TraceConfig{Reexec: &reexec}) + withConfig, err := api.IntermediateRoots(context.Background(), block.Hash(), &TraceConfig{}) require.NoError(t, err) withoutConfig, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 12c9b9ed47..8f1c76afbc 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -162,7 +162,7 @@ func (b *testBackend) teardown() { b.chain.Stop() } -func (b *testBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, StateReleaseFunc, error) { +func (b *testBackend) StateAtBlock(ctx context.Context, block *types.Block, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, StateReleaseFunc, error) { statedb, err := b.chain.StateAt(block.Root()) if err != nil { return nil, nil, errStateNotFound @@ -181,13 +181,12 @@ func (b *testBackend) StateAtBlock(ctx context.Context, block *types.Block, reex return statedb, release, nil } -func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*types.Transaction, vm.BlockContext, *state.StateDB, StateReleaseFunc, error) { +func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int) (*types.Transaction, vm.BlockContext, *state.StateDB, StateReleaseFunc, error) { parent := b.chain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { return nil, vm.BlockContext{}, nil, nil, errBlockNotFound } - - statedb, release, err := b.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := b.StateAtBlock(ctx, parent, nil, true, false) if err != nil { return nil, vm.BlockContext{}, nil, nil, errStateNotFound } @@ -230,11 +229,11 @@ type prunedTestBackend struct { *testBackend } -func (b *prunedTestBackend) StateAtBlock(_ context.Context, _ *types.Block, _ uint64, _ *state.StateDB, _ bool, _ bool) (*state.StateDB, StateReleaseFunc, error) { +func (b *prunedTestBackend) StateAtBlock(_ context.Context, _ *types.Block, _ *state.StateDB, _ bool, _ bool) (*state.StateDB, StateReleaseFunc, error) { return nil, nil, errStateNotFound } -func (b *prunedTestBackend) StateAtTransaction(_ context.Context, _ *types.Block, _ int, _ uint64) (*types.Transaction, vm.BlockContext, *state.StateDB, StateReleaseFunc, error) { +func (b *prunedTestBackend) StateAtTransaction(_ context.Context, _ *types.Block, _ int) (*types.Transaction, vm.BlockContext, *state.StateDB, StateReleaseFunc, error) { return nil, vm.BlockContext{}, nil, nil, errStateNotFound } @@ -244,6 +243,18 @@ type stateTracer struct { Storage map[common.Address]map[common.Hash]common.Hash } +type tracedOpcodeLog struct { + Op string `json:"op"` + Refund *uint64 `json:"refund,omitempty"` + Storage map[string]string `json:"storage,omitempty"` +} + +type tracedOpcodeResult struct { + Failed bool `json:"failed"` + ReturnValue string `json:"returnValue"` + StructLogs []tracedOpcodeLog `json:"structLogs"` +} + func newStateTracer(ctx *Context, cfg json.RawMessage, chainCfg *params.ChainConfig) (*Tracer, error) { t := &stateTracer{ Balance: make(map[common.Address]*hexutil.Big), @@ -1601,6 +1612,176 @@ func TestTracingWithOverrides(t *testing.T) { } } +func TestTraceTransactionRefundAndStorageSnapshots(t *testing.T) { + t.Parallel() + + accounts := newAccounts(1) + contract := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + slot0 := common.BigToHash(big.NewInt(0)) + txSigner := types.HomesteadSigner{} + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + contract: { + Nonce: 1, + Code: []byte{ + byte(vm.PUSH1), 0x00, + byte(vm.SLOAD), + byte(vm.POP), + byte(vm.PUSH1), 0x00, + byte(vm.PUSH1), 0x00, + byte(vm.SSTORE), + byte(vm.STOP), + }, + Storage: map[common.Hash]common.Hash{ + slot0: common.BigToHash(big.NewInt(1)), + }, + }, + }, + } + var target common.Hash + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &contract, + Value: big.NewInt(0), + Gas: 100000, + GasPrice: b.BaseFee(), + }), txSigner, accounts[0].key) + b.AddTx(tx) + target = tx.Hash() + }) + defer backend.teardown() + + api := NewAPI(backend) + result, err := api.TraceTransaction(context.Background(), target, nil) + if err != nil { + t.Fatalf("failed to trace refunding transaction: %v", err) + } + var traced tracedOpcodeResult + if err := json.Unmarshal(result.(json.RawMessage), &traced); err != nil { + t.Fatalf("failed to unmarshal trace result: %v", err) + } + if traced.Failed { + t.Fatal("expected refunding transaction to succeed") + } + if traced.ReturnValue != "0x" { + t.Fatalf("unexpected return value: have %s want 0x", traced.ReturnValue) + } + slotHex := slot0.Hex() + oneHex := common.BigToHash(big.NewInt(1)).Hex() + zeroHex := common.Hash{}.Hex() + var ( + foundSloadSnapshot bool + foundSstoreSnapshot bool + foundRefund bool + ) + for _, log := range traced.StructLogs { + switch log.Op { + case "SLOAD": + if got := log.Storage[slotHex]; got == oneHex { + foundSloadSnapshot = true + } + case "SSTORE": + if got := log.Storage[slotHex]; got == zeroHex { + foundSstoreSnapshot = true + } + } + if log.Refund != nil && *log.Refund > 0 { + foundRefund = true + } + } + if !foundSloadSnapshot { + t.Fatal("expected SLOAD snapshot to include the pre-existing non-zero storage value") + } + if !foundSstoreSnapshot { + t.Fatal("expected SSTORE snapshot to include the post-write zeroed storage value") + } + if !foundRefund { + t.Fatal("expected at least one structLog entry with a non-zero refund field") + } +} + +func TestTraceTransactionFailureReturnValues(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + code []byte + wantReturnValue string + }{ + { + name: "revert preserves return data", + code: []byte{ + byte(vm.PUSH1), 0x2a, + byte(vm.PUSH1), 0x00, + byte(vm.MSTORE), + byte(vm.PUSH1), 0x20, + byte(vm.PUSH1), 0x00, + byte(vm.REVERT), + }, + wantReturnValue: "0x000000000000000000000000000000000000000000000000000000000000002a", + }, + { + name: "hard failure clears return data", + code: []byte{ + byte(vm.INVALID), + }, + wantReturnValue: "0x", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + accounts := newAccounts(1) + contract := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + txSigner := types.HomesteadSigner{} + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + contract: { + Nonce: 1, + Code: tc.code, + }, + }, + } + var target common.Hash + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &contract, + Value: big.NewInt(0), + Gas: 100000, + GasPrice: b.BaseFee(), + }), txSigner, accounts[0].key) + b.AddTx(tx) + target = tx.Hash() + }) + defer backend.teardown() + + api := NewAPI(backend) + result, err := api.TraceTransaction(context.Background(), target, nil) + if err != nil { + t.Fatalf("failed to trace transaction: %v", err) + } + var traced tracedOpcodeResult + if err := json.Unmarshal(result.(json.RawMessage), &traced); err != nil { + t.Fatalf("failed to unmarshal trace result: %v", err) + } + if !traced.Failed { + t.Fatal("expected traced transaction to fail") + } + if traced.ReturnValue != tc.wantReturnValue { + t.Fatalf("unexpected returnValue: have %s want %s", traced.ReturnValue, tc.wantReturnValue) + } + if len(traced.StructLogs) == 0 { + t.Fatal("expected failing trace to still include structLogs") + } + }) + } +} + type Account struct { key *ecdsa.PrivateKey addr common.Address diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index 02a15dca03..b56a5706a4 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -150,7 +150,7 @@ type structLogLegacy struct { Gas uint64 `json:"gas"` GasCost uint64 `json:"gasCost"` Depth int `json:"depth"` - Error string `json:"error,omitempty"` + Error string `json:"error,omitempty,omitzero"` Stack *[]string `json:"stack,omitempty"` ReturnData string `json:"returnData,omitempty"` Memory *[]string `json:"memory,omitempty"` @@ -158,6 +158,15 @@ type structLogLegacy struct { RefundCounter uint64 `json:"refund,omitempty"` } +func formatMemoryWord(chunk []byte) string { + if len(chunk) == 32 { + return hexutil.Encode(chunk) + } + var word [32]byte + copy(word[:], chunk) + return hexutil.Encode(word[:]) +} + // toLegacyJSON converts the structLog to legacy json-encoded legacy form. func (s *StructLog) toLegacyJSON() json.RawMessage { msg := structLogLegacy{ @@ -177,7 +186,7 @@ func (s *StructLog) toLegacyJSON() json.RawMessage { msg.Stack = &stack } if len(s.ReturnData) > 0 { - msg.ReturnData = hexutil.Bytes(s.ReturnData).String() + msg.ReturnData = hexutil.Encode(s.ReturnData) } if len(s.Memory) > 0 { memory := make([]string, 0, (len(s.Memory)+31)/32) @@ -186,14 +195,14 @@ func (s *StructLog) toLegacyJSON() json.RawMessage { if end > len(s.Memory) { end = len(s.Memory) } - memory = append(memory, fmt.Sprintf("%x", s.Memory[i:end])) + memory = append(memory, formatMemoryWord(s.Memory[i:end])) } msg.Memory = &memory } if len(s.Storage) > 0 { storage := make(map[string]string) for i, storageValue := range s.Storage { - storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue) + storage[i.Hex()] = storageValue.Hex() } msg.Storage = &storage } diff --git a/eth/tracers/logger/logger_test.go b/eth/tracers/logger/logger_test.go index a1a7323440..796ec24e01 100644 --- a/eth/tracers/logger/logger_test.go +++ b/eth/tracers/logger/logger_test.go @@ -106,3 +106,46 @@ func TestStructLogMarshalingOmitEmpty(t *testing.T) { }) } } + +func TestStructLogLegacyJSONSpecFormatting(t *testing.T) { + tests := []struct { + name string + log *StructLog + want string + }{ + { + name: "omits empty error and pads memory/storage", + log: &StructLog{ + Pc: 7, + Op: vm.SSTORE, + Gas: 100, + GasCost: 20, + Memory: []byte{0xaa, 0xbb}, + Storage: map[common.Hash]common.Hash{common.BigToHash(big.NewInt(1)): common.BigToHash(big.NewInt(2))}, + Depth: 1, + ReturnData: []byte{0x12, 0x34}, + }, + want: `{"pc":7,"op":"SSTORE","gas":100,"gasCost":20,"depth":1,"returnData":"0x1234","memory":["0xaabb000000000000000000000000000000000000000000000000000000000000"],"storage":{"0x0000000000000000000000000000000000000000000000000000000000000001":"0x0000000000000000000000000000000000000000000000000000000000000002"}}`, + }, + { + name: "includes error only when present", + log: &StructLog{ + Pc: 1, + Op: vm.STOP, + Gas: 2, + GasCost: 3, + Depth: 1, + Err: errors.New("boom"), + }, + want: `{"pc":1,"op":"STOP","gas":2,"gasCost":3,"depth":1,"error":"boom"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + have := string(tt.log.toLegacyJSON()) + if have != tt.want { + t.Fatalf("mismatched results\n\thave: %v\n\twant: %v", have, tt.want) + } + }) + } +} diff --git a/eth/tracers/parity.go b/eth/tracers/parity.go index 941211783f..dcb118be23 100644 --- a/eth/tracers/parity.go +++ b/eth/tracers/parity.go @@ -174,7 +174,7 @@ type parityBlockExec struct { // system calls (beacon root, parent block hash) required before replaying the // block's transactions. The caller MUST invoke the returned release function; it // is non-nil only on success (error returns leave state unallocated). -func (api *API) setupParityBlockExec(ctx context.Context, block *types.Block, reexec uint64) (*parityBlockExec, StateReleaseFunc, error) { +func (api *API) setupParityBlockExec(ctx context.Context, block *types.Block) (*parityBlockExec, StateReleaseFunc, error) { if block.NumberU64() == 0 { return nil, nil, errors.New("genesis block is not traceable") } @@ -184,7 +184,7 @@ func (api *API) setupParityBlockExec(ctx context.Context, block *types.Block, re return nil, nil, fmt.Errorf("failed to get parent block: %w", err) } - statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, nil, true, false) if err != nil { return nil, nil, fmt.Errorf("failed to get state at block %d: %w (archive node required for historical blocks)", parent.NumberU64(), err) } @@ -228,7 +228,7 @@ func (e *parityBlockExec) txInput(txIndex int, tx *types.Transaction, cumulative } // parityPhaseConfig builds the TraceConfig for one Parity output phase (trace, -// stateDiff or vmTrace) forcing the given tracer. Only Reexec/Timeout are +// stateDiff or vmTrace) forcing the given tracer. Only Timeout is // honoured from the caller's config, and an unset timeout falls back to the // Parity-specific (longer) default — every phase re-executes the transaction, // so each must get the full budget rather than defaultTraceTimeout (5s). This @@ -239,7 +239,6 @@ func parityPhaseConfig(tracerName string, tracerCfg json.RawMessage, base *Trace TracerConfig: tracerCfg, } if base != nil { - cfg.Reexec = base.Reexec cfg.Timeout = base.Timeout cfg.gasBailout = base.gasBailout } @@ -351,7 +350,7 @@ func (api *API) parityTxFrameCtx(in parityExecInput, wrapped parityCallResult, c // from txctx; when includeTxMeta is false they are cleared (trace_call / // trace_replay* semantics). The callTracer is always forced regardless of any // tracer set on the input config, since the Parity conversion requires a -// structured call frame; only Reexec/Timeout are honoured from it. +// structured call frame; only Timeout is honoured from it. func (api *API) parityTraceTx(ctx context.Context, in parityExecInput, includeTxMeta bool) ([]*ParityTrace, *hexutil.Bytes, uint64, error) { res, gasUsed, err := api.traceTx(ctx, in.tx, in.msg, in.txctx, in.vmctx, in.statedb, parityTraceConfig(in.config), nil) if err != nil { @@ -411,17 +410,12 @@ func (api *API) canonicalTxTraceEnv(ctx context.Context, hash common.Hash, confi return parityExecInput{}, nil, errors.New("genesis is not traceable") } - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - block, err := api.blockByNumberAndHash(ctx, rpc.BlockNumber(blockNumber), blockHash) if err != nil { return parityExecInput{}, nil, err } - tx, vmctx, statedb, release, err := api.backend.StateAtTransaction(ctx, block, int(index), reexec) + tx, vmctx, statedb, release, err := api.backend.StateAtTransaction(ctx, block, int(index)) if err != nil { return parityExecInput{}, nil, err } diff --git a/eth/tracers/parity_block.go b/eth/tracers/parity_block.go index afe71cbe20..581cac86ef 100644 --- a/eth/tracers/parity_block.go +++ b/eth/tracers/parity_block.go @@ -107,12 +107,7 @@ func (api *API) traceBlockParityByHash(ctx context.Context, hash common.Hash, co return nil, fmt.Errorf("failed to get block by hash: %w", err) } - reexec := defaultTraceReexec - if config != nil && config.Reexec != nil { - reexec = *config.Reexec - } - - exec, release, err := api.setupParityBlockExec(ctx, block, reexec) + exec, release, err := api.setupParityBlockExec(ctx, block) if err != nil { return nil, err } diff --git a/eth/tracers/parity_call.go b/eth/tracers/parity_call.go index 74b47615df..b4bf8409d0 100644 --- a/eth/tracers/parity_call.go +++ b/eth/tracers/parity_call.go @@ -146,7 +146,7 @@ func (api *API) traceCallState(ctx context.Context, blockNrOrHash rpc.BlockNumbe return nil, nil, nil, err } - statedb, release, err := api.backend.StateAtBlock(ctx, block, defaultTraceReexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, block, nil, true, false) if err != nil { return nil, nil, nil, err } diff --git a/eth/tracers/parity_replay_block.go b/eth/tracers/parity_replay_block.go index dbfa2def67..f84c2b76cb 100644 --- a/eth/tracers/parity_replay_block.go +++ b/eth/tracers/parity_replay_block.go @@ -53,7 +53,7 @@ func (api *TraceAPI) ReplayBlockTransactions(ctx context.Context, blockNrOrHash // traceBlockParityByHash, but emits one ReplayResult per transaction (with // trace metadata stripped) instead of a flat list of block traces. func (api *API) replayBlockTransactions(ctx context.Context, block *types.Block, set traceTypeSet) ([]*ReplayResult, error) { - exec, release, err := api.setupParityBlockExec(ctx, block, defaultTraceReexec) + exec, release, err := api.setupParityBlockExec(ctx, block) if err != nil { return nil, err } diff --git a/eth/tracers/parity_replay_tx.go b/eth/tracers/parity_replay_tx.go index cfa09aab4b..27c979825c 100644 --- a/eth/tracers/parity_replay_tx.go +++ b/eth/tracers/parity_replay_tx.go @@ -36,7 +36,7 @@ func (api *TraceAPI) ReplayTransaction(ctx context.Context, txHash common.Hash, return nil, err } - tx, vmctx, statedb, release, err := api.backend.StateAtTransaction(ctx, block, int(index), defaultTraceReexec) + tx, vmctx, statedb, release, err := api.backend.StateAtTransaction(ctx, block, int(index)) if err != nil { return nil, err } diff --git a/eth/tracers/parity_test.go b/eth/tracers/parity_test.go index 1a361f0aa3..f6f1256386 100644 --- a/eth/tracers/parity_test.go +++ b/eth/tracers/parity_test.go @@ -195,15 +195,11 @@ func TestParityTraceConfig(t *testing.T) { } }) - t.Run("caller reexec and timeout honoured, tracer forced", func(t *testing.T) { + t.Run("caller timeout honoured, tracer forced", func(t *testing.T) { t.Parallel() - reexec := uint64(42) timeout := "9s" userTracer := "callTracer" - cfg := parityTraceConfig(&TraceConfig{Reexec: &reexec, Timeout: &timeout, Tracer: &userTracer}) - if cfg.Reexec == nil || *cfg.Reexec != 42 { - t.Errorf("reexec = %v, want 42", cfg.Reexec) - } + cfg := parityTraceConfig(&TraceConfig{Timeout: &timeout, Tracer: &userTracer}) if cfg.Timeout == nil || *cfg.Timeout != "9s" { t.Errorf("timeout = %v, want 9s", cfg.Timeout) } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 70260559d6..3f05521c08 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -535,7 +535,11 @@ func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuer func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { arg := map[string]interface{}{} - if q.Addresses != nil { + // Only include "address" when there are actual address filters. + // An empty slice is treated the same as nil (no filter), and omitting + // the field avoids sending "address":[] to nodes that reject empty arrays + // (e.g. Hedera, some non-Geth implementations). + if len(q.Addresses) > 0 { arg["address"] = q.Addresses } if q.Topics != nil { @@ -900,6 +904,7 @@ type rpcProgress struct { TxIndexFinishedBlocks hexutil.Uint64 TxIndexRemainingBlocks hexutil.Uint64 StateIndexRemaining hexutil.Uint64 + TrienodeIndexRemaining hexutil.Uint64 } func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress { @@ -928,6 +933,7 @@ func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress { TxIndexFinishedBlocks: uint64(p.TxIndexFinishedBlocks), TxIndexRemainingBlocks: uint64(p.TxIndexRemainingBlocks), StateIndexRemaining: uint64(p.StateIndexRemaining), + TrienodeIndexRemaining: uint64(p.TrienodeIndexRemaining), } } diff --git a/ethclient/types_test.go b/ethclient/types_test.go index dcb9a579b7..8820b11162 100644 --- a/ethclient/types_test.go +++ b/ethclient/types_test.go @@ -53,6 +53,22 @@ func TestToFilterArg(t *testing.T) { }, nil, }, + { + // empty Addresses slice must be treated same as nil: + // the "address" field must be omitted so that non-Geth nodes + // (e.g. Hedera) do not reject the request with an error. + "with empty addresses slice", + ethereum.FilterQuery{ + Addresses: []common.Address{}, + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + }, + map[string]interface{}{ + "fromBlock": "0x1", + "toBlock": "0x2", + }, + nil, + }, { "without BlockHash", ethereum.FilterQuery{ diff --git a/graphql/graphql.go b/graphql/graphql.go index 4e5c8c67ca..68a359cea7 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -1531,6 +1531,9 @@ func (s *SyncState) TxIndexRemainingBlocks() hexutil.Uint64 { func (s *SyncState) StateIndexRemaining() hexutil.Uint64 { return hexutil.Uint64(s.progress.StateIndexRemaining) } +func (s *SyncState) TrienodeIndexRemaining() hexutil.Uint64 { + return hexutil.Uint64(s.progress.TrienodeIndexRemaining) +} // Syncing returns false in case the node is currently not syncing with the network. It can be up-to-date or has not // yet received the latest block headers from its peers. In case it is synchronizing: diff --git a/interfaces.go b/interfaces.go index 9312789917..c0a1d71869 100644 --- a/interfaces.go +++ b/interfaces.go @@ -139,8 +139,9 @@ type SyncProgress struct { TxIndexFinishedBlocks uint64 // Number of blocks whose transactions are already indexed TxIndexRemainingBlocks uint64 // Number of blocks whose transactions are not indexed yet - // "historical state indexing" fields - StateIndexRemaining uint64 // Number of states remain unindexed + // "historical data indexing" fields + StateIndexRemaining uint64 // Number of states remain unindexed + TrienodeIndexRemaining uint64 // Number of trienodes remain unindexed } // Done returns the indicator if the initial sync is finished or not. @@ -148,7 +149,7 @@ func (prog SyncProgress) Done() bool { if prog.CurrentBlock < prog.HighestBlock { return false } - return prog.TxIndexRemainingBlocks == 0 && prog.StateIndexRemaining == 0 + return prog.TxIndexRemainingBlocks == 0 && prog.StateIndexRemaining == 0 && prog.TrienodeIndexRemaining == 0 } // ChainSyncReader wraps access to the node's current sync status. If there's no diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 332aa67d40..6098783178 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -259,6 +259,7 @@ func (api *EthereumAPI) Syncing(ctx context.Context) (interface{}, error) { "txIndexFinishedBlocks": hexutil.Uint64(progress.TxIndexFinishedBlocks), "txIndexRemainingBlocks": hexutil.Uint64(progress.TxIndexRemainingBlocks), "stateIndexRemaining": hexutil.Uint64(progress.StateIndexRemaining), + "trienodeIndexRemaining": hexutil.Uint64(progress.TrienodeIndexRemaining), }, nil } @@ -1162,6 +1163,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr if err := blockOverrides.Apply(&blockCtx); err != nil { return 0, err } + header = blockOverrides.MakeHeader(header) } rules := b.ChainConfig().Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time) precompiles := vm.ActivePrecompiledContracts(rules) @@ -1169,13 +1171,17 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr return 0, err } // Construct the gas estimator option from the user input + var blobBaseFee *big.Int + if blockOverrides != nil && blockOverrides.BlobBaseFee != nil { + blobBaseFee = blockOverrides.BlobBaseFee.ToInt() + } opts := &gasestimator.Options{ - Config: b.ChainConfig(), - Chain: NewChainContext(ctx, b), - Header: header, - BlockOverrides: blockOverrides, - State: state, - ErrorRatio: estimateGasErrorRatio, + Config: b.ChainConfig(), + Chain: NewChainContext(ctx, b), + Header: header, + State: state, + BlobBaseFee: blobBaseFee, + ErrorRatio: estimateGasErrorRatio, } // Set any required transaction default, but make sure the gas cap itself is not messed with // if it was not specified in the original argument list. diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 90c5211ad3..cc1a5dcaea 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -1005,6 +1005,17 @@ func TestEstimateGas(t *testing.T) { expectErr: core.ErrInsufficientFunds, want: 21000, }, + // block override gas limit should bound estimation search space. + { + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{ + From: &accounts[0].addr, + Input: hex2Bytes("6080604052348015600f57600080fd5b50483a1015601c57600080fd5b60003a111560315760004811603057600080fd5b5b603f80603e6000396000f3fe6080604052600080fdfea264697066735822122060729c2cee02b10748fae5200f1c9da4661963354973d9154c13a8e9ce9dee1564736f6c63430008130033"), + Gas: func() *hexutil.Uint64 { v := hexutil.Uint64(0); return &v }(), + }, + blockOverrides: override.BlockOverrides{GasLimit: func() *hexutil.Uint64 { v := hexutil.Uint64(50000); return &v }()}, + expectErr: errors.New("gas required exceeds allowance (50000)"), + }, // empty create { blockNumber: rpc.LatestBlockNumber, @@ -1086,6 +1097,19 @@ func TestEstimateGas(t *testing.T) { }, want: 21000, }, + // blob base fee block override should be applied during estimation. + { + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{ + From: &accounts[0].addr, + To: &accounts[1].addr, + Value: (*hexutil.Big)(big.NewInt(1)), + BlobHashes: []common.Hash{{0x01, 0x22}}, + BlobFeeCap: (*hexutil.Big)(big.NewInt(1)), + }, + blockOverrides: override.BlockOverrides{BlobBaseFee: (*hexutil.Big)(big.NewInt(2))}, + expectErr: core.ErrBlobFeeCapTooLow, + }, // // SPDX-License-Identifier: GPL-3.0 //pragma solidity >=0.8.2 <0.9.0; // diff --git a/triedb/database.go b/triedb/database.go index 07fe798bef..6531a6808f 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -376,10 +376,10 @@ func (db *Database) StorageIterator(root common.Hash, account common.Hash, seek // IndexProgress returns the indexing progress made so far. It provides the // number of states that remain unindexed. -func (db *Database) IndexProgress() (uint64, error) { +func (db *Database) IndexProgress() (uint64, uint64, error) { pdb, ok := db.backend.(*pathdb.Database) if !ok { - return 0, errors.New("not supported") + return 0, 0, errors.New("not supported") } return pdb.IndexProgress() } diff --git a/triedb/generate.go b/triedb/generate.go new file mode 100644 index 0000000000..259e139848 --- /dev/null +++ b/triedb/generate.go @@ -0,0 +1,108 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package triedb + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/triedb/internal" +) + +// kvAccountIterator wraps an ethdb.Iterator to iterate over account snapshot +// entries in the database, implementing internal.AccountIterator. +type kvAccountIterator struct { + it ethdb.Iterator + hash common.Hash +} + +func newKVAccountIterator(db ethdb.Iteratee) *kvAccountIterator { + it := rawdb.NewKeyLengthIterator( + db.NewIterator(rawdb.SnapshotAccountPrefix, nil), + len(rawdb.SnapshotAccountPrefix)+common.HashLength, + ) + return &kvAccountIterator{it: it} +} + +func (it *kvAccountIterator) Next() bool { + if !it.it.Next() { + return false + } + key := it.it.Key() + copy(it.hash[:], key[len(rawdb.SnapshotAccountPrefix):]) + return true +} + +func (it *kvAccountIterator) Hash() common.Hash { return it.hash } +func (it *kvAccountIterator) Account() []byte { return it.it.Value() } +func (it *kvAccountIterator) Error() error { return it.it.Error() } +func (it *kvAccountIterator) Release() { it.it.Release() } + +// kvStorageIterator wraps an ethdb.Iterator to iterate over storage snapshot +// entries for a specific account, implementing internal.StorageIterator. +type kvStorageIterator struct { + it ethdb.Iterator + hash common.Hash +} + +func newKVStorageIterator(db ethdb.Iteratee, accountHash common.Hash) *kvStorageIterator { + it := rawdb.IterateStorageSnapshots(db, accountHash) + return &kvStorageIterator{it: it} +} + +func (it *kvStorageIterator) Next() bool { + if !it.it.Next() { + return false + } + key := it.it.Key() + copy(it.hash[:], key[len(rawdb.SnapshotStoragePrefix)+common.HashLength:]) + return true +} + +func (it *kvStorageIterator) Hash() common.Hash { return it.hash } +func (it *kvStorageIterator) Slot() []byte { return it.it.Value() } +func (it *kvStorageIterator) Error() error { return it.it.Error() } +func (it *kvStorageIterator) Release() { it.it.Release() } + +// GenerateTrie rebuilds all tries (storage + account) from flat snapshot data +// in the database. It reads account and storage snapshots from the KV store, +// builds tries using StackTrie with streaming node writes, and verifies the +// computed state root matches the expected root. +func GenerateTrie(db ethdb.Database, scheme string, root common.Hash) error { + acctIt := newKVAccountIterator(db) + defer acctIt.Release() + + got, err := internal.GenerateTrieRoot(db, scheme, acctIt, common.Hash{}, internal.StackTrieGenerate, func(dst ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *internal.GenerateStats) (common.Hash, error) { + storageIt := newKVStorageIterator(db, accountHash) + defer storageIt.Release() + + hash, err := internal.GenerateTrieRoot(dst, scheme, storageIt, accountHash, internal.StackTrieGenerate, nil, stat, false) + if err != nil { + return common.Hash{}, err + } + return hash, nil + }, internal.NewGenerateStats(), true) + if err != nil { + return err + } + if got != root { + return fmt.Errorf("state root mismatch: got %x, want %x", got, root) + } + return nil +} diff --git a/triedb/generate_test.go b/triedb/generate_test.go new file mode 100644 index 0000000000..42bccd9aa3 --- /dev/null +++ b/triedb/generate_test.go @@ -0,0 +1,178 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package triedb + +import ( + "bytes" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" +) + +// testAccount is a helper for building test state with deterministic ordering. +type testAccount struct { + hash common.Hash + account types.StateAccount + storage []testSlot // must be sorted by hash +} + +type testSlot struct { + hash common.Hash + value []byte +} + +// buildExpectedRoot computes the state root from sorted test accounts using +// StackTrie (which requires sorted key insertion). +func buildExpectedRoot(t *testing.T, accounts []testAccount) common.Hash { + t.Helper() + // Sort accounts by hash + sort.Slice(accounts, func(i, j int) bool { + return bytes.Compare(accounts[i].hash[:], accounts[j].hash[:]) < 0 + }) + acctTrie := trie.NewStackTrie(nil) + for i := range accounts { + data, err := rlp.EncodeToBytes(&accounts[i].account) + if err != nil { + t.Fatal(err) + } + acctTrie.Update(accounts[i].hash[:], data) + } + return acctTrie.Hash() +} + +// computeStorageRoot computes the storage trie root from sorted slots. +func computeStorageRoot(slots []testSlot) common.Hash { + sort.Slice(slots, func(i, j int) bool { + return bytes.Compare(slots[i].hash[:], slots[j].hash[:]) < 0 + }) + st := trie.NewStackTrie(nil) + for _, s := range slots { + st.Update(s.hash[:], s.value) + } + return st.Hash() +} + +func TestGenerateTrieEmpty(t *testing.T) { + db := rawdb.NewMemoryDatabase() + if err := GenerateTrie(db, rawdb.HashScheme, types.EmptyRootHash); err != nil { + t.Fatalf("GenerateTrie on empty state failed: %v", err) + } +} + +func TestGenerateTrieAccountsOnly(t *testing.T) { + db := rawdb.NewMemoryDatabase() + + accounts := []testAccount{ + { + hash: common.HexToHash("0x01"), + account: types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(100), + Root: types.EmptyRootHash, + CodeHash: types.EmptyCodeHash.Bytes(), + }, + }, + { + hash: common.HexToHash("0x02"), + account: types.StateAccount{ + Nonce: 2, + Balance: uint256.NewInt(200), + Root: types.EmptyRootHash, + CodeHash: types.EmptyCodeHash.Bytes(), + }, + }, + } + for _, a := range accounts { + rawdb.WriteAccountSnapshot(db, a.hash, types.SlimAccountRLP(a.account)) + } + root := buildExpectedRoot(t, accounts) + + if err := GenerateTrie(db, rawdb.HashScheme, root); err != nil { + t.Fatalf("GenerateTrie failed: %v", err) + } +} + +func TestGenerateTrieWithStorage(t *testing.T) { + db := rawdb.NewMemoryDatabase() + + slots := []testSlot{ + {hash: common.HexToHash("0xaa"), value: []byte{0x01, 0x02, 0x03}}, + {hash: common.HexToHash("0xbb"), value: []byte{0x04, 0x05, 0x06}}, + } + storageRoot := computeStorageRoot(slots) + + accounts := []testAccount{ + { + hash: common.HexToHash("0x01"), + account: types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(100), + Root: storageRoot, + CodeHash: types.EmptyCodeHash.Bytes(), + }, + storage: slots, + }, + { + hash: common.HexToHash("0x02"), + account: types.StateAccount{ + Nonce: 0, + Balance: uint256.NewInt(50), + Root: types.EmptyRootHash, + CodeHash: types.EmptyCodeHash.Bytes(), + }, + }, + } + // Write account snapshots + for _, a := range accounts { + rawdb.WriteAccountSnapshot(db, a.hash, types.SlimAccountRLP(a.account)) + } + // Write storage snapshots + for _, a := range accounts { + for _, s := range a.storage { + rawdb.WriteStorageSnapshot(db, a.hash, s.hash, s.value) + } + } + root := buildExpectedRoot(t, accounts) + + if err := GenerateTrie(db, rawdb.HashScheme, root); err != nil { + t.Fatalf("GenerateTrie failed: %v", err) + } +} + +func TestGenerateTrieRootMismatch(t *testing.T) { + db := rawdb.NewMemoryDatabase() + + acct := types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(100), + Root: types.EmptyRootHash, + CodeHash: types.EmptyCodeHash.Bytes(), + } + rawdb.WriteAccountSnapshot(db, common.HexToHash("0x01"), types.SlimAccountRLP(acct)) + + wrongRoot := common.HexToHash("0xdeadbeef") + err := GenerateTrie(db, rawdb.HashScheme, wrongRoot) + if err == nil { + t.Fatal("expected error for root mismatch, got nil") + } +} diff --git a/triedb/internal/conversion.go b/triedb/internal/conversion.go new file mode 100644 index 0000000000..b331b63e21 --- /dev/null +++ b/triedb/internal/conversion.go @@ -0,0 +1,363 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package internal contains shared trie generation utilities used by both +// triedb and triedb/pathdb. All code is ported from +// core/state/snapshot/conversion.go (with exported names) unless noted. +package internal + +import ( + "encoding/binary" + "fmt" + "math" + "runtime" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" +) + +// Iterator is an iterator to step over all the accounts or the specific +// storage in a snapshot which may or may not be composed of multiple layers. +type Iterator interface { + // Next steps the iterator forward one element, returning false if exhausted, + // or an error if iteration failed for some reason (e.g. root being iterated + // becomes stale and garbage collected). + Next() bool + + // Error returns any failure that occurred during iteration, which might have + // caused a premature iteration exit (e.g. snapshot stack becoming stale). + Error() error + + // Hash returns the hash of the account or storage slot the iterator is + // currently at. + Hash() common.Hash + + // Release releases associated resources. Release should always succeed and + // can be called multiple times without causing error. + Release() +} + +// AccountIterator is an iterator to step over all the accounts in a snapshot, +// which may or may not be composed of multiple layers. +type AccountIterator interface { + Iterator + + // Account returns the RLP encoded slim account the iterator is currently at. + // An error will be returned if the iterator becomes invalid + Account() []byte +} + +// StorageIterator is an iterator to step over the specific storage in a snapshot, +// which may or may not be composed of multiple layers. +type StorageIterator interface { + Iterator + + // Slot returns the storage slot the iterator is currently at. An error will + // be returned if the iterator becomes invalid + Slot() []byte +} + +// TrieKV represents a trie key-value pair. +type TrieKV struct { + Key common.Hash + Value []byte +} + +type ( + // TrieGeneratorFn is the interface of trie generation which can + // be implemented by different trie algorithm. + TrieGeneratorFn func(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan (TrieKV), out chan (common.Hash)) + + // LeafCallbackFn is the callback invoked at the leaves of the trie, + // returns the subtrie root with the specified subtrie identifier. + LeafCallbackFn func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *GenerateStats) (common.Hash, error) +) + +// GenerateStats is a collection of statistics gathered by the trie generator +// for logging purposes. +type GenerateStats struct { + head common.Hash + start time.Time + + accounts uint64 // Number of accounts done (including those being crawled) + slots uint64 // Number of storage slots done (including those being crawled) + + slotsStart map[common.Hash]time.Time // Start time for account slot crawling + slotsHead map[common.Hash]common.Hash // Slot head for accounts being crawled + + lock sync.RWMutex +} + +// NewGenerateStats creates a new generator stats. +func NewGenerateStats() *GenerateStats { + return &GenerateStats{ + slotsStart: make(map[common.Hash]time.Time), + slotsHead: make(map[common.Hash]common.Hash), + start: time.Now(), + } +} + +// ProgressAccounts updates the generator stats for the account range. +func (stat *GenerateStats) ProgressAccounts(account common.Hash, done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.accounts += done + stat.head = account +} + +// FinishAccounts updates the generator stats for the finished account range. +func (stat *GenerateStats) FinishAccounts(done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.accounts += done +} + +// ProgressContract updates the generator stats for a specific in-progress contract. +func (stat *GenerateStats) ProgressContract(account common.Hash, slot common.Hash, done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.slots += done + stat.slotsHead[account] = slot + if _, ok := stat.slotsStart[account]; !ok { + stat.slotsStart[account] = time.Now() + } +} + +// FinishContract updates the generator stats for a specific just-finished contract. +func (stat *GenerateStats) FinishContract(account common.Hash, done uint64) { + stat.lock.Lock() + defer stat.lock.Unlock() + + stat.slots += done + delete(stat.slotsHead, account) + delete(stat.slotsStart, account) +} + +// Report prints the cumulative progress statistic smartly. +func (stat *GenerateStats) Report() { + stat.lock.RLock() + defer stat.lock.RUnlock() + + ctx := []interface{}{ + "accounts", stat.accounts, + "slots", stat.slots, + "elapsed", common.PrettyDuration(time.Since(stat.start)), + } + if stat.accounts > 0 { + if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 { + var ( + left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts + eta = common.CalculateETA(done, left, time.Since(stat.start)) + ) + // If there are large contract crawls in progress, estimate their finish time + for acc, head := range stat.slotsHead { + start := stat.slotsStart[acc] + if done := binary.BigEndian.Uint64(head[:8]); done > 0 { + left := math.MaxUint64 - binary.BigEndian.Uint64(head[:8]) + + // Override the ETA if larger than the largest until now + if slotETA := common.CalculateETA(done, left, time.Since(start)); eta < slotETA { + eta = slotETA + } + } + } + ctx = append(ctx, []interface{}{ + "eta", common.PrettyDuration(eta), + }...) + } + } + log.Info("Iterating state snapshot", ctx...) +} + +// ReportDone prints the last log when the whole generation is finished. +func (stat *GenerateStats) ReportDone() { + stat.lock.RLock() + defer stat.lock.RUnlock() + + var ctx []interface{} + ctx = append(ctx, []interface{}{"accounts", stat.accounts}...) + if stat.slots != 0 { + ctx = append(ctx, []interface{}{"slots", stat.slots}...) + } + ctx = append(ctx, []interface{}{"elapsed", common.PrettyDuration(time.Since(stat.start))}...) + log.Info("Iterated snapshot", ctx...) +} + +// RunReport periodically prints the progress information. +func RunReport(stats *GenerateStats, stop chan bool) { + timer := time.NewTimer(0) + defer timer.Stop() + + for { + select { + case <-timer.C: + stats.Report() + timer.Reset(time.Second * 8) + case success := <-stop: + if success { + stats.ReportDone() + } + return + } + } +} + +// GenerateTrieRoot generates the trie hash based on the snapshot iterator. +// It can be used for generating account trie, storage trie or even the +// whole state which connects the accounts and the corresponding storages. +func GenerateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, account common.Hash, generatorFn TrieGeneratorFn, leafCallback LeafCallbackFn, stats *GenerateStats, report bool) (common.Hash, error) { + var ( + in = make(chan TrieKV) // chan to pass leaves + out = make(chan common.Hash, 1) // chan to collect result + stoplog = make(chan bool, 1) // 1-size buffer, works when logging is not enabled + wg sync.WaitGroup + ) + // Spin up a go-routine for trie hash re-generation + wg.Add(1) + go func() { + defer wg.Done() + generatorFn(db, scheme, account, in, out) + }() + // Spin up a go-routine for progress logging + if report && stats != nil { + wg.Add(1) + go func() { + defer wg.Done() + RunReport(stats, stoplog) + }() + } + // Create a semaphore to assign tasks and collect results through. We'll pre- + // fill it with nils, thus using the same channel for both limiting concurrent + // processing and gathering results. + threads := runtime.NumCPU() + results := make(chan error, threads) + for i := 0; i < threads; i++ { + results <- nil // fill the semaphore + } + // stop is a helper function to shutdown the background threads + // and return the re-generated trie hash. + stop := func(fail error) (common.Hash, error) { + close(in) + result := <-out + for i := 0; i < threads; i++ { + if err := <-results; err != nil && fail == nil { + fail = err + } + } + stoplog <- fail == nil + + wg.Wait() + return result, fail + } + var ( + logged = time.Now() + processed = uint64(0) + leaf TrieKV + ) + // Start to feed leaves + for it.Next() { + if account == (common.Hash{}) { + var ( + err error + fullData []byte + ) + if leafCallback == nil { + fullData, err = types.FullAccountRLP(it.(AccountIterator).Account()) + if err != nil { + return stop(err) + } + } else { + // Wait until the semaphore allows us to continue, aborting if + // a sub-task failed + if err := <-results; err != nil { + results <- nil // stop will drain the results, add a noop back for this error we just consumed + return stop(err) + } + // Fetch the next account and process it concurrently + account, err := types.FullAccount(it.(AccountIterator).Account()) + if err != nil { + return stop(err) + } + go func(hash common.Hash) { + subroot, err := leafCallback(db, hash, common.BytesToHash(account.CodeHash), stats) + if err != nil { + results <- err + return + } + if account.Root != subroot { + results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot) + return + } + results <- nil + }(it.Hash()) + fullData, err = rlp.EncodeToBytes(account) + if err != nil { + return stop(err) + } + } + leaf = TrieKV{it.Hash(), fullData} + } else { + leaf = TrieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} + } + in <- leaf + + // Accumulate the generation statistic if it's required. + processed++ + if time.Since(logged) > 3*time.Second && stats != nil { + if account == (common.Hash{}) { + stats.ProgressAccounts(it.Hash(), processed) + } else { + stats.ProgressContract(account, it.Hash(), processed) + } + logged, processed = time.Now(), 0 + } + } + // Commit the last part statistic. + if processed > 0 && stats != nil { + if account == (common.Hash{}) { + stats.FinishAccounts(processed) + } else { + stats.FinishContract(account, processed) + } + } + return stop(nil) +} + +// StackTrieGenerate is the trie generation function that creates a StackTrie +// and persists nodes via rawdb.WriteTrieNode. +func StackTrieGenerate(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan TrieKV, out chan common.Hash) { + var onTrieNode trie.OnTrieNode + if db != nil { + onTrieNode = func(path []byte, hash common.Hash, blob []byte) { + rawdb.WriteTrieNode(db, owner, path, hash, blob, scheme) + } + } + t := trie.NewStackTrie(onTrieNode) + for leaf := range in { + t.Update(leaf.Key[:], leaf.Value) + } + out <- t.Hash() +} diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 410a1b698d..e9c763309d 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -630,11 +630,26 @@ func (db *Database) HistoryRange() (uint64, uint64, error) { // IndexProgress returns the indexing progress made so far. It provides the // number of states that remain unindexed. -func (db *Database) IndexProgress() (uint64, error) { - if db.stateIndexer == nil { - return 0, nil +func (db *Database) IndexProgress() (uint64, uint64, error) { + var ( + stateProgress uint64 + trieProgress uint64 + ) + if db.stateIndexer != nil { + prog, err := db.stateIndexer.progress() + if err != nil { + return 0, 0, err + } + stateProgress = prog + } + if db.trienodeIndexer != nil { + prog, err := db.trienodeIndexer.progress() + if err != nil { + return 0, 0, err + } + trieProgress = prog } - return db.stateIndexer.progress() + return stateProgress, trieProgress, nil } // AccountIterator creates a new account iterator for the specified root hash and diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 307088ffce..2de5847124 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -987,7 +987,7 @@ func TestDatabaseIndexRecovery(t *testing.T) { t.Fatalf("Unexpected state history found, %d", i) } } - remain, err := env.db.IndexProgress() + remain, _, err := env.db.IndexProgress() if err != nil { t.Fatalf("Failed to obtain the progress, %v", err) } @@ -1001,7 +1001,7 @@ func TestDatabaseIndexRecovery(t *testing.T) { panic(fmt.Errorf("failed to update state changes, err: %w", err)) } } - remain, err = env.db.IndexProgress() + remain, _, err = env.db.IndexProgress() if err != nil { t.Fatalf("Failed to obtain the progress, %v", err) } diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 4272309108..9ed89788b2 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -413,6 +413,11 @@ func (dl *diskLayer) writeHistory(typ historyType, diff *diffLayer) (bool, error if err != nil { return false, err } + // Notify the index pruner about the new tail so that stale index + // blocks referencing the pruned histories can be cleaned up. + if indexer != nil && pruned > 0 { + indexer.prune(newFirst) + } log.Debug("Pruned history", "type", typ, "items", pruned, "tailid", newFirst) return false, nil } diff --git a/triedb/pathdb/history_index_pruner.go b/triedb/pathdb/history_index_pruner.go new file mode 100644 index 0000000000..c9be3618e8 --- /dev/null +++ b/triedb/pathdb/history_index_pruner.go @@ -0,0 +1,385 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "encoding/binary" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +const ( + // indexPruningThreshold defines the number of pruned histories that must + // accumulate before triggering index pruning. This helps avoid scheduling + // index pruning too frequently. + indexPruningThreshold = 90000 + + // iteratorReopenInterval is how long the iterator is kept open before + // being released and re-opened. Long-lived iterators hold a read snapshot + // that blocks LSM compaction; periodically re-opening avoids stalling the + // compactor during a large scan. + iteratorReopenInterval = 30 * time.Second +) + +// indexPruner is responsible for pruning stale index data from the tail side +// when old history objects are removed. It runs as a background goroutine and +// processes pruning signals whenever the history tail advances. +// +// The pruning operates at the block level: for each state element's index +// metadata, leading index blocks whose maximum history ID falls below the +// new tail are removed entirely. This avoids the need to decode individual +// block contents and is efficient because index blocks store monotonically +// increasing history IDs. +type indexPruner struct { + disk ethdb.KeyValueStore + typ historyType + tail atomic.Uint64 // Tail below which index entries can be pruned + lastRun uint64 // The tail in the last pruning run + trigger chan struct{} // Non-blocking signal that tail has advanced + closed chan struct{} + wg sync.WaitGroup + log log.Logger + + pauseReq chan chan struct{} // Pause request; caller sends ack channel, pruner closes it when paused + resumeCh chan struct{} // Resume signal sent by caller after indexSingle/unindexSingle completes +} + +// newIndexPruner creates and starts a new index pruner for the given history type. +func newIndexPruner(disk ethdb.KeyValueStore, typ historyType) *indexPruner { + p := &indexPruner{ + disk: disk, + typ: typ, + trigger: make(chan struct{}, 1), + closed: make(chan struct{}), + log: log.New("type", typ.String()), + pauseReq: make(chan chan struct{}), + resumeCh: make(chan struct{}), + } + p.wg.Add(1) + go p.run() + return p +} + +// prune signals the pruner that the history tail has advanced to the given ID. +// All index entries referencing history IDs below newTail can be removed. +func (p *indexPruner) prune(newTail uint64) { + // Only update if the tail is actually advancing + for { + old := p.tail.Load() + if newTail <= old { + return + } + if p.tail.CompareAndSwap(old, newTail) { + break + } + } + // Non-blocking signal + select { + case p.trigger <- struct{}{}: + default: + } +} + +// pause requests the pruner to flush all pending writes and pause. It blocks +// until the pruner has acknowledged the pause. This must be paired with a +// subsequent call to resume. +func (p *indexPruner) pause() { + ack := make(chan struct{}) + select { + case p.pauseReq <- ack: + <-ack // wait for the pruner to flush and acknowledge + case <-p.closed: + } +} + +// resume unblocks a previously paused pruner, allowing it to continue +// processing. +func (p *indexPruner) resume() { + select { + case p.resumeCh <- struct{}{}: + case <-p.closed: + } +} + +// close shuts down the pruner and waits for it to finish. +func (p *indexPruner) close() { + select { + case <-p.closed: + return + default: + close(p.closed) + p.wg.Wait() + } +} + +// run is the main loop of the pruner. It waits for trigger signals and +// processes a small batch of entries on each trigger, advancing the cursor. +func (p *indexPruner) run() { + defer p.wg.Done() + + for { + select { + case <-p.trigger: + tail := p.tail.Load() + if tail < p.lastRun || tail-p.lastRun < indexPruningThreshold { + continue + } + if err := p.process(tail); err != nil { + p.log.Error("Failed to prune index", "tail", tail, "err", err) + } else { + p.lastRun = tail + } + + case ack := <-p.pauseReq: + // Pruner is idle, acknowledge immediately and wait for resume. + close(ack) + select { + case <-p.resumeCh: + case <-p.closed: + return + } + + case <-p.closed: + return + } + } +} + +// process iterates all index metadata entries for the history type and prunes +// leading blocks whose max history ID is below the given tail. +func (p *indexPruner) process(tail uint64) error { + var ( + err error + pruned int + start = time.Now() + ) + switch p.typ { + case typeStateHistory: + n, err := p.prunePrefix(rawdb.StateHistoryAccountMetadataPrefix, typeAccount, tail) + if err != nil { + return err + } + pruned += n + + n, err = p.prunePrefix(rawdb.StateHistoryStorageMetadataPrefix, typeStorage, tail) + if err != nil { + return err + } + pruned += n + statePruneHistoryIndexTimer.UpdateSince(start) + + case typeTrienodeHistory: + pruned, err = p.prunePrefix(rawdb.TrienodeHistoryMetadataPrefix, typeTrienode, tail) + if err != nil { + return err + } + trienodePruneHistoryIndexTimer.UpdateSince(start) + + default: + panic("unknown history type") + } + if pruned > 0 { + p.log.Info("Pruned stale index blocks", "pruned", pruned, "tail", tail, "elapsed", common.PrettyDuration(time.Since(start))) + } + return nil +} + +// prunePrefix scans all metadata entries under the given prefix and prunes +// leading index blocks below the tail. The iterator is periodically released +// and re-opened to avoid holding a read snapshot that blocks LSM compaction. +func (p *indexPruner) prunePrefix(prefix []byte, elemType elementType, tail uint64) (int, error) { + var ( + pruned int + opened = time.Now() + it = p.disk.NewIterator(prefix, nil) + batch = p.disk.NewBatchWithSize(ethdb.IdealBatchSize) + ) + for { + // Terminate if iterator is exhausted + if !it.Next() { + it.Release() + break + } + // Check termination or pause request + select { + case <-p.closed: + // Terminate the process if indexer is closed + it.Release() + if batch.ValueSize() > 0 { + return pruned, batch.Write() + } + return pruned, nil + + case ack := <-p.pauseReq: + // Save the current position so that after resume the + // iterator can be re-opened from where it left off. + start := common.CopyBytes(it.Key()[len(prefix):]) + it.Release() + + // Flush all pending writes before acknowledging the pause. + var flushErr error + if batch.ValueSize() > 0 { + if err := batch.Write(); err != nil { + flushErr = err + } + batch.Reset() + } + close(ack) + + // Block until resumed or closed. Always wait here even if + // the flush failed — returning early would cause resume() + // to deadlock since nobody would receive on resumeCh. + select { + case <-p.resumeCh: + if flushErr != nil { + return 0, flushErr + } + // Re-open the iterator from the saved position so the + // pruner sees the current database state (including any + // writes made by indexer during the pause). + it = p.disk.NewIterator(prefix, start) + opened = time.Now() + continue + case <-p.closed: + return pruned, flushErr + } + + default: + // Keep processing + } + + // Prune the index data block + key, value := it.Key(), it.Value() + ident, bsize := p.identFromKey(key, prefix, elemType) + n, err := p.pruneEntry(batch, ident, value, bsize, tail) + if err != nil { + p.log.Warn("Failed to prune index entry", "ident", ident, "err", err) + continue + } + pruned += n + + // Flush the batch if there are too many accumulated + if batch.ValueSize() >= ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + it.Release() + return 0, err + } + batch.Reset() + } + + // Periodically release the iterator so the LSM compactor + // is not blocked by the read snapshot we hold. + if time.Since(opened) >= iteratorReopenInterval { + opened = time.Now() + + start := common.CopyBytes(it.Key()[len(prefix):]) + it.Release() + it = p.disk.NewIterator(prefix, start) + } + } + if batch.ValueSize() > 0 { + if err := batch.Write(); err != nil { + return 0, err + } + } + return pruned, nil +} + +// identFromKey reconstructs the stateIdent and bitmapSize from a metadata key. +func (p *indexPruner) identFromKey(key []byte, prefix []byte, elemType elementType) (stateIdent, int) { + rest := key[len(prefix):] + + switch elemType { + case typeAccount: + // key = prefix + addressHash(32) + var addrHash common.Hash + copy(addrHash[:], rest[:32]) + return newAccountIdent(addrHash), 0 + + case typeStorage: + // key = prefix + addressHash(32) + storageHash(32) + var addrHash, storHash common.Hash + copy(addrHash[:], rest[:32]) + copy(storHash[:], rest[32:64]) + return newStorageIdent(addrHash, storHash), 0 + + case typeTrienode: + // key = prefix + addressHash(32) + path(variable) + var addrHash common.Hash + copy(addrHash[:], rest[:32]) + path := string(rest[32:]) + ident := newTrienodeIdent(addrHash, path) + return ident, ident.bloomSize() + + default: + panic("unknown element type") + } +} + +// pruneEntry checks a single metadata entry and removes leading index blocks +// whose max < tail. Returns the number of blocks pruned. +func (p *indexPruner) pruneEntry(batch ethdb.Batch, ident stateIdent, blob []byte, bsize int, tail uint64) (int, error) { + // Fast path: the first 8 bytes of the metadata encode the max history ID + // of the first index block (big-endian uint64). If it is >= tail, no + // blocks can be pruned and we skip the full parse entirely. + if len(blob) >= 8 && binary.BigEndian.Uint64(blob[:8]) >= tail { + return 0, nil + } + descList, err := parseIndex(blob, bsize) + if err != nil { + return 0, err + } + // Find the number of leading blocks that can be entirely pruned. + // A block can be pruned if its max history ID is strictly below + // the tail. + var count int + for _, desc := range descList { + if desc.max < tail { + count++ + } else { + break // blocks are ordered, no more to prune + } + } + if count == 0 { + return 0, nil + } + // Delete the pruned index blocks + for i := 0; i < count; i++ { + deleteStateIndexBlock(ident, batch, descList[i].id) + } + // Update or delete the metadata + remaining := descList[count:] + if len(remaining) == 0 { + // All blocks pruned, remove the metadata entry entirely + deleteStateIndex(ident, batch) + } else { + // Rewrite the metadata with the remaining blocks + size := indexBlockDescSize + bsize + buf := make([]byte, 0, size*len(remaining)) + for _, desc := range remaining { + buf = append(buf, desc.encode()...) + } + writeStateIndex(ident, batch, buf) + } + return count, nil +} diff --git a/triedb/pathdb/history_index_pruner_test.go b/triedb/pathdb/history_index_pruner_test.go new file mode 100644 index 0000000000..b3094de3e6 --- /dev/null +++ b/triedb/pathdb/history_index_pruner_test.go @@ -0,0 +1,355 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "math" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +func writeMultiBlockIndex(t *testing.T, db ethdb.Database, ident stateIdent, bitmapSize int, startID uint64) []*indexBlockDesc { + t.Helper() + + if startID == 0 { + startID = 1 + } + iw, _ := newIndexWriter(db, ident, 0, bitmapSize) + + for i := 0; i < 10000; i++ { + if err := iw.append(startID+uint64(i), randomExt(bitmapSize, 5)); err != nil { + t.Fatalf("Failed to append element %d: %v", i, err) + } + } + batch := db.NewBatch() + iw.finish(batch) + if err := batch.Write(); err != nil { + t.Fatalf("Failed to write batch: %v", err) + } + + blob := readStateIndex(ident, db) + descList, err := parseIndex(blob, bitmapSize) + if err != nil { + t.Fatalf("Failed to parse index: %v", err) + } + return descList +} + +// TestPruneEntryBasic verifies that pruneEntry correctly removes leading index +// blocks whose max is below the given tail. +func TestPruneEntryBasic(t *testing.T) { + db := rawdb.NewMemoryDatabase() + ident := newAccountIdent(common.Hash{0xa}) + descList := writeMultiBlockIndex(t, db, ident, 0, 1) + + // Prune with a tail that is above the first block's max but below the second + firstBlockMax := descList[0].max + + pruner := newIndexPruner(db, typeStateHistory) + defer pruner.close() + + if err := pruner.process(firstBlockMax + 1); err != nil { + t.Fatalf("Failed to process pruning: %v", err) + } + + // Verify the first block was removed + blob := readStateIndex(ident, db) + if len(blob) == 0 { + t.Fatal("Index metadata should not be empty after partial prune") + } + remaining, err := parseIndex(blob, 0) + if err != nil { + t.Fatalf("Failed to parse index after prune: %v", err) + } + if len(remaining) != len(descList)-1 { + t.Fatalf("Expected %d blocks remaining, got %d", len(descList)-1, len(remaining)) + } + // The first remaining block should be what was previously the second block + if remaining[0].id != descList[1].id { + t.Fatalf("Expected first remaining block id %d, got %d", descList[1].id, remaining[0].id) + } + + // Verify the pruned block data is actually deleted + blockData := readStateIndexBlock(ident, db, descList[0].id) + if len(blockData) != 0 { + t.Fatal("Pruned block data should have been deleted") + } + + // Remaining blocks should still have their data + for _, desc := range remaining { + blockData = readStateIndexBlock(ident, db, desc.id) + if len(blockData) == 0 { + t.Fatalf("Block %d data should still exist", desc.id) + } + } +} + +// TestPruneEntryBasicTrienode is the same as TestPruneEntryBasic but for +// trienode index entries with a non-zero bitmapSize. +func TestPruneEntryBasicTrienode(t *testing.T) { + db := rawdb.NewMemoryDatabase() + addrHash := common.Hash{0xa} + path := string([]byte{0x0, 0x0, 0x0}) + ident := newTrienodeIdent(addrHash, path) + + descList := writeMultiBlockIndex(t, db, ident, ident.bloomSize(), 1) + firstBlockMax := descList[0].max + + pruner := newIndexPruner(db, typeTrienodeHistory) + defer pruner.close() + + if err := pruner.process(firstBlockMax + 1); err != nil { + t.Fatalf("Failed to process pruning: %v", err) + } + + blob := readStateIndex(ident, db) + remaining, err := parseIndex(blob, ident.bloomSize()) + if err != nil { + t.Fatalf("Failed to parse index after prune: %v", err) + } + if len(remaining) != len(descList)-1 { + t.Fatalf("Expected %d blocks remaining, got %d", len(descList)-1, len(remaining)) + } + if remaining[0].id != descList[1].id { + t.Fatalf("Expected first remaining block id %d, got %d", descList[1].id, remaining[0].id) + } + blockData := readStateIndexBlock(ident, db, descList[0].id) + if len(blockData) != 0 { + t.Fatal("Pruned block data should have been deleted") + } +} + +// TestPruneEntryComplete verifies that when all blocks are pruned, the metadata +// entry is also deleted. +func TestPruneEntryComplete(t *testing.T) { + db := rawdb.NewMemoryDatabase() + ident := newAccountIdent(common.Hash{0xb}) + iw, _ := newIndexWriter(db, ident, 0, 0) + + for i := 1; i <= 10; i++ { + if err := iw.append(uint64(i), nil); err != nil { + t.Fatalf("Failed to append: %v", err) + } + } + batch := db.NewBatch() + iw.finish(batch) + if err := batch.Write(); err != nil { + t.Fatalf("Failed to write: %v", err) + } + + pruner := newIndexPruner(db, typeStateHistory) + defer pruner.close() + + // Prune with tail above all elements + if err := pruner.process(11); err != nil { + t.Fatalf("Failed to process: %v", err) + } + + // Metadata entry should be deleted + blob := readStateIndex(ident, db) + if len(blob) != 0 { + t.Fatal("Index metadata should be empty after full prune") + } +} + +// TestPruneNoop verifies that pruning does nothing when the tail is below all +// block maximums. +func TestPruneNoop(t *testing.T) { + db := rawdb.NewMemoryDatabase() + ident := newAccountIdent(common.Hash{0xc}) + iw, _ := newIndexWriter(db, ident, 0, 0) + + for i := 100; i <= 200; i++ { + if err := iw.append(uint64(i), nil); err != nil { + t.Fatalf("Failed to append: %v", err) + } + } + batch := db.NewBatch() + iw.finish(batch) + if err := batch.Write(); err != nil { + t.Fatalf("Failed to write: %v", err) + } + + blob := readStateIndex(ident, db) + origLen := len(blob) + + pruner := newIndexPruner(db, typeStateHistory) + defer pruner.close() + + if err := pruner.process(50); err != nil { + t.Fatalf("Failed to process: %v", err) + } + + // Nothing should have changed + blob = readStateIndex(ident, db) + if len(blob) != origLen { + t.Fatalf("Expected no change, original len %d, got %d", origLen, len(blob)) + } +} + +// TestPrunePreservesReadability verifies that after pruning, the remaining +// index data is still readable and returns correct results. +func TestPrunePreservesReadability(t *testing.T) { + db := rawdb.NewMemoryDatabase() + ident := newAccountIdent(common.Hash{0xe}) + descList := writeMultiBlockIndex(t, db, ident, 0, 1) + firstBlockMax := descList[0].max + + pruner := newIndexPruner(db, typeStateHistory) + defer pruner.close() + + if err := pruner.process(firstBlockMax + 1); err != nil { + t.Fatalf("Failed to process: %v", err) + } + + // Read the remaining index and verify lookups still work + ir, err := newIndexReader(db, ident, 0) + if err != nil { + t.Fatalf("Failed to create reader: %v", err) + } + + // Looking for something greater than firstBlockMax should still work + result, err := ir.readGreaterThan(firstBlockMax) + if err != nil { + t.Fatalf("Failed to read: %v", err) + } + if result != firstBlockMax+1 { + t.Fatalf("Expected %d, got %d", firstBlockMax+1, result) + } + + // Looking for the last element should return MaxUint64 + result, err = ir.readGreaterThan(20000) + if err != nil { + t.Fatalf("Failed to read: %v", err) + } + if result != math.MaxUint64 { + t.Fatalf("Expected MaxUint64, got %d", result) + } +} + +// TestPrunePauseResume verifies the pause/resume mechanism: +// - The pruner pauses mid-iteration and flushes its batch +// - Data written while the pruner is paused (simulating indexSingle) is +// visible after resume via a fresh iterator +// - Pruning still completes correctly after resume +func TestPrunePauseResume(t *testing.T) { + db := rawdb.NewMemoryDatabase() + + // Create many accounts with multi-block indexes so the pruner is still + // iterating when the pause request arrives. + var firstBlockMax uint64 + for i := 0; i < 200; i++ { + hash := common.Hash{byte(i)} + ident := newAccountIdent(hash) + descList := writeMultiBlockIndex(t, db, ident, 0, 1) + if i == 0 { + firstBlockMax = descList[0].max + } + } + // Target account at the end of the key space — the pruner should not + // have visited it yet when the pause is acknowledged. + targetIdent := newAccountIdent(common.Hash{0xff}) + targetDescList := writeMultiBlockIndex(t, db, targetIdent, 0, 1) + + tail := firstBlockMax + 1 + + // Construct the pruner without starting run(). We call process() + // directly to exercise the mid-iteration pause path deterministically. + pruner := &indexPruner{ + disk: db, + typ: typeStateHistory, + log: log.New("type", "account"), + closed: make(chan struct{}), + pauseReq: make(chan chan struct{}, 1), // buffered so we can pre-deposit + resumeCh: make(chan struct{}), + } + + // Pre-deposit a pause request before process() starts. Because + // pauseReq is buffered, this succeeds immediately. When prunePrefix's + // select checks the channel on an early iteration, it will find the + // pending request and pause — no scheduling race is possible. + ack := make(chan struct{}) + pruner.pauseReq <- ack + + // Run process() in the background. + errCh := make(chan error, 1) + go func() { + errCh <- pruner.process(tail) + }() + + // Block until the pruner has flushed pending writes and acknowledged. + <-ack + + // While paused, append a new element to the target account's index, + // simulating what indexSingle would do during the pause window. + lastMax := targetDescList[len(targetDescList)-1].max + newID := lastMax + 10000 + iw, err := newIndexWriter(db, targetIdent, lastMax, 0) + if err != nil { + t.Fatalf("Failed to create index writer: %v", err) + } + if err := iw.append(newID, nil); err != nil { + t.Fatalf("Failed to append: %v", err) + } + batch := db.NewBatch() + iw.finish(batch) + if err := batch.Write(); err != nil { + t.Fatalf("Failed to write batch: %v", err) + } + + // Resume the pruner. + pruner.resume() + + // Wait for process() to complete. + if err := <-errCh; err != nil { + t.Fatalf("process() failed: %v", err) + } + + // Verify: the entry written during the pause must still be accessible. + // If the pruner used a stale iterator snapshot, it would overwrite the + // target's metadata and lose the new entry. + ir, err := newIndexReader(db, targetIdent, 0) + if err != nil { + t.Fatalf("Failed to create index reader: %v", err) + } + result, err := ir.readGreaterThan(newID - 1) + if err != nil { + t.Fatalf("Failed to read: %v", err) + } + if result != newID { + t.Fatalf("Entry written during pause was lost: want %d, got %d", newID, result) + } + + // Verify: pruning actually occurred on an early account. + earlyIdent := newAccountIdent(common.Hash{0x00}) + earlyBlob := readStateIndex(earlyIdent, db) + if len(earlyBlob) == 0 { + t.Fatal("Early account index should not be completely empty") + } + earlyRemaining, err := parseIndex(earlyBlob, 0) + if err != nil { + t.Fatalf("Failed to parse early account index: %v", err) + } + // The first block (id=0) should have been pruned. + if earlyRemaining[0].id == 0 { + t.Fatal("First block of early account should have been pruned") + } +} diff --git a/triedb/pathdb/history_indexer.go b/triedb/pathdb/history_indexer.go index 4f8b2205b2..d2f6764c52 100644 --- a/triedb/pathdb/history_indexer.go +++ b/triedb/pathdb/history_indexer.go @@ -720,6 +720,7 @@ func (i *indexIniter) recover() bool { // state history. type historyIndexer struct { initer *indexIniter + pruner *indexPruner typ historyType disk ethdb.KeyValueStore freezer ethdb.AncientStore @@ -775,6 +776,7 @@ func newHistoryIndexer(disk ethdb.Database, freezer ethdb.AncientStore, lastHist checkVersion(disk, typ) return &historyIndexer{ initer: newIndexIniter(disk, freezer, typ, lastHistoryID, noWait), + pruner: newIndexPruner(disk, typ), typ: typ, disk: disk, freezer: freezer, @@ -783,6 +785,7 @@ func newHistoryIndexer(disk ethdb.Database, freezer ethdb.AncientStore, lastHist func (i *historyIndexer) close() { i.initer.close() + i.pruner.close() } // inited returns a flag indicating whether the existing state histories @@ -803,6 +806,8 @@ func (i *historyIndexer) extend(historyID uint64) error { case <-i.initer.closed: return errors.New("indexer is closed") case <-i.initer.done: + i.pruner.pause() + defer i.pruner.resume() return indexSingle(historyID, i.disk, i.freezer, i.typ) case i.initer.interrupt <- signal: return <-signal.result @@ -820,12 +825,27 @@ func (i *historyIndexer) shorten(historyID uint64) error { case <-i.initer.closed: return errors.New("indexer is closed") case <-i.initer.done: + i.pruner.pause() + defer i.pruner.resume() return unindexSingle(historyID, i.disk, i.freezer, i.typ) case i.initer.interrupt <- signal: return <-signal.result } } +// prune signals the pruner that the history tail has advanced to the given ID, +// so that stale index blocks referencing pruned histories can be removed. +func (i *historyIndexer) prune(newTail uint64) { + select { + case <-i.initer.closed: + log.Debug("Ignored the pruning signal", "reason", "closed") + case <-i.initer.done: + i.pruner.prune(newTail) + default: + log.Debug("Ignored the pruning signal", "reason", "busy") + } +} + // progress returns the indexing progress made so far. It provides the number // of states that remain unindexed. func (i *historyIndexer) progress() (uint64, error) { diff --git a/triedb/pathdb/iterator.go b/triedb/pathdb/iterator.go index 8ca8247206..2d333dfa1b 100644 --- a/triedb/pathdb/iterator.go +++ b/triedb/pathdb/iterator.go @@ -24,48 +24,15 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/triedb/internal" ) -// Iterator is an iterator to step over all the accounts or the specific -// storage in a snapshot which may or may not be composed of multiple layers. -type Iterator interface { - // Next steps the iterator forward one element, returning false if exhausted, - // or an error if iteration failed for some reason (e.g. root being iterated - // becomes stale and garbage collected). - Next() bool - - // Error returns any failure that occurred during iteration, which might have - // caused a premature iteration exit (e.g. layer stack becoming stale). - Error() error - - // Hash returns the hash of the account or storage slot the iterator is - // currently at. - Hash() common.Hash - - // Release releases associated resources. Release should always succeed and - // can be called multiple times without causing error. - Release() -} - -// AccountIterator is an iterator to step over all the accounts in a snapshot, -// which may or may not be composed of multiple layers. -type AccountIterator interface { - Iterator - - // Account returns the RLP encoded slim account the iterator is currently at. - // An error will be returned if the iterator becomes invalid - Account() []byte -} - -// StorageIterator is an iterator to step over the specific storage in a snapshot, -// which may or may not be composed of multiple layers. -type StorageIterator interface { - Iterator - - // Slot returns the storage slot the iterator is currently at. An error will - // be returned if the iterator becomes invalid - Slot() []byte -} +// Type aliases for the iterator interfaces defined in triedb/internal. +type ( + Iterator = internal.Iterator + AccountIterator = internal.AccountIterator + StorageIterator = internal.StorageIterator +) type ( // loadAccount is the function to retrieve the account from the associated diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go index ec45257db5..0d7ca5a5b4 100644 --- a/triedb/pathdb/layertree.go +++ b/triedb/pathdb/layertree.go @@ -151,6 +151,15 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6 if root == parentRoot { return errors.New("layer cycle") } + // If a layer with this root already exists, skip the insertion. Fork blocks + // can produce the same state root as the canonical block (same parent, same + // coinbase, zero txs); overwriting tree.layers[root] would corrupt the parent + // chain for any child layers already built on top of the existing one, and + // appending a duplicate root to the lookup indices causes accountTip/storageTip + // to resolve the wrong layer. + if tree.get(root) != nil { + return nil + } parent := tree.get(parentRoot) if parent == nil { return fmt.Errorf("triedb parent [%#x] layer missing", parentRoot) diff --git a/triedb/pathdb/layertree_test.go b/triedb/pathdb/layertree_test.go index a0ef141850..5004ae60c9 100644 --- a/triedb/pathdb/layertree_test.go +++ b/triedb/pathdb/layertree_test.go @@ -575,6 +575,40 @@ func TestDescendant(t *testing.T) { } } +func TestDuplicateRootLookup(t *testing.T) { + // Chain: + // C1->C2->C3 (HEAD) + tr := newTestLayerTree() // base = 0x1 + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, NewNodeSetWithOrigin(nil, nil), + NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x1"}}, nil), nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, NewNodeSetWithOrigin(nil, nil), + NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x1"}}, nil), nil, nil, false)) + + // A fork block with the same state root as C2; inserting it must not + // pollute the lookup history for the canonical descendant C3. + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, NewNodeSetWithOrigin(nil, nil), + NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x1"}}, nil), nil, nil, false)) + if n := tr.len(); n != 3 { + t.Fatalf("duplicate root insert changed layer count, got %d, want 3", n) + } + + l, err := tr.lookupAccount(common.HexToHash("0xa"), common.Hash{0x3}) + if err != nil { + t.Fatalf("account lookup failed: %v", err) + } + if l.rootHash() != (common.Hash{0x3}) { + t.Errorf("unexpected account tip, want %x, got %x", common.Hash{0x3}, l.rootHash()) + } + + l, err = tr.lookupStorage(common.HexToHash("0xa"), common.HexToHash("0x1"), common.Hash{0x3}) + if err != nil { + t.Fatalf("storage lookup failed: %v", err) + } + if l.rootHash() != (common.Hash{0x3}) { + t.Errorf("unexpected storage tip, want %x, got %x", common.Hash{0x3}, l.rootHash()) + } +} + func TestAccountLookup(t *testing.T) { // Chain: // C1->C2->C3->C4 (HEAD) diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go index a0a626f9b5..e01dfdfb86 100644 --- a/triedb/pathdb/metrics.go +++ b/triedb/pathdb/metrics.go @@ -77,10 +77,12 @@ var ( trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil) trienodeHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/index", nil) - stateIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/index/time", nil) - stateUnindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/unindex/time", nil) - trienodeIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/index/time", nil) - trienodeUnindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/unindex/time", nil) + stateIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/index/time", nil) + stateUnindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/unindex/time", nil) + statePruneHistoryIndexTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/prune/time", nil) + trienodeIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/index/time", nil) + trienodeUnindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/unindex/time", nil) + trienodePruneHistoryIndexTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/prune/time", nil) lookupAddLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil) lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil) diff --git a/triedb/pathdb/verifier.go b/triedb/pathdb/verifier.go index a69b10f4f3..c53590f2fd 100644 --- a/triedb/pathdb/verifier.go +++ b/triedb/pathdb/verifier.go @@ -17,36 +17,15 @@ package pathdb import ( - "encoding/binary" "errors" "fmt" - "math" - "runtime" - "sync" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" -) - -// trieKV represents a trie key-value pair -type trieKV struct { - key common.Hash - value []byte -} - -type ( - // trieHasherFn is the interface of trie hasher which can be implemented - // by different trie algorithm. - trieHasherFn func(in chan trieKV, out chan common.Hash) - - // leafCallbackFn is the callback invoked at the leaves of the trie, - // returns the subtrie root with the specified subtrie identifier. - leafCallbackFn func(accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) + "github.com/ethereum/go-ethereum/triedb/internal" ) // VerifyState traverses the flat states specified by the given state root and @@ -58,7 +37,7 @@ func (db *Database) VerifyState(root common.Hash) error { } defer acctIt.Release() - got, err := generateTrieRoot(acctIt, common.Hash{}, stackTrieHasher, func(accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + got, err := internal.GenerateTrieRoot(nil, "", acctIt, common.Hash{}, stackTrieHasher, func(_ ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *internal.GenerateStats) (common.Hash, error) { // Migrate the code first, commit the contract code into the tmp db. if codeHash != types.EmptyCodeHash { code := rawdb.ReadCode(db.diskdb, codeHash) @@ -73,12 +52,12 @@ func (db *Database) VerifyState(root common.Hash) error { } defer storageIt.Release() - hash, err := generateTrieRoot(storageIt, accountHash, stackTrieHasher, nil, stat, false) + hash, err := internal.GenerateTrieRoot(nil, "", storageIt, accountHash, stackTrieHasher, nil, stat, false) if err != nil { return common.Hash{}, err } return hash, nil - }, newGenerateStats(), true) + }, internal.NewGenerateStats(), true) if err != nil { return err @@ -89,264 +68,10 @@ func (db *Database) VerifyState(root common.Hash) error { return nil } -// generateStats is a collection of statistics gathered by the trie generator -// for logging purposes. -type generateStats struct { - head common.Hash - start time.Time - - accounts uint64 // Number of accounts done (including those being crawled) - slots uint64 // Number of storage slots done (including those being crawled) - - slotsStart map[common.Hash]time.Time // Start time for account slot crawling - slotsHead map[common.Hash]common.Hash // Slot head for accounts being crawled - - lock sync.RWMutex -} - -// newGenerateStats creates a new generator stats. -func newGenerateStats() *generateStats { - return &generateStats{ - slotsStart: make(map[common.Hash]time.Time), - slotsHead: make(map[common.Hash]common.Hash), - start: time.Now(), - } -} - -// progressAccounts updates the generator stats for the account range. -func (stat *generateStats) progressAccounts(account common.Hash, done uint64) { - stat.lock.Lock() - defer stat.lock.Unlock() - - stat.accounts += done - stat.head = account -} - -// finishAccounts updates the generator stats for the finished account range. -func (stat *generateStats) finishAccounts(done uint64) { - stat.lock.Lock() - defer stat.lock.Unlock() - - stat.accounts += done -} - -// progressContract updates the generator stats for a specific in-progress contract. -func (stat *generateStats) progressContract(account common.Hash, slot common.Hash, done uint64) { - stat.lock.Lock() - defer stat.lock.Unlock() - - stat.slots += done - stat.slotsHead[account] = slot - if _, ok := stat.slotsStart[account]; !ok { - stat.slotsStart[account] = time.Now() - } -} - -// finishContract updates the generator stats for a specific just-finished contract. -func (stat *generateStats) finishContract(account common.Hash, done uint64) { - stat.lock.Lock() - defer stat.lock.Unlock() - - stat.slots += done - delete(stat.slotsHead, account) - delete(stat.slotsStart, account) -} - -// report prints the cumulative progress statistic smartly. -func (stat *generateStats) report() { - stat.lock.RLock() - defer stat.lock.RUnlock() - - ctx := []interface{}{ - "accounts", stat.accounts, - "slots", stat.slots, - "elapsed", common.PrettyDuration(time.Since(stat.start)), - } - if stat.accounts > 0 { - // If there's progress on the account trie, estimate the time to finish crawling it - if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 { - var ( - left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts - eta = common.CalculateETA(done, left, time.Since(stat.start)) - ) - // If there are large contract crawls in progress, estimate their finish time - for acc, head := range stat.slotsHead { - start := stat.slotsStart[acc] - if done := binary.BigEndian.Uint64(head[:8]); done > 0 { - left := math.MaxUint64 - binary.BigEndian.Uint64(head[:8]) - - // Override the ETA if larger than the largest until now - if slotETA := common.CalculateETA(done, left, time.Since(start)); eta < slotETA { - eta = slotETA - } - } - } - ctx = append(ctx, []interface{}{ - "eta", common.PrettyDuration(eta), - }...) - } - } - log.Info("Iterating state snapshot", ctx...) -} - -// reportDone prints the last log when the whole generation is finished. -func (stat *generateStats) reportDone() { - stat.lock.RLock() - defer stat.lock.RUnlock() - - var ctx []interface{} - ctx = append(ctx, []interface{}{"accounts", stat.accounts}...) - if stat.slots != 0 { - ctx = append(ctx, []interface{}{"slots", stat.slots}...) - } - ctx = append(ctx, []interface{}{"elapsed", common.PrettyDuration(time.Since(stat.start))}...) - log.Info("Iterated snapshot", ctx...) -} - -// runReport periodically prints the progress information. -func runReport(stats *generateStats, stop chan bool) { - timer := time.NewTimer(0) - defer timer.Stop() - - for { - select { - case <-timer.C: - stats.report() - timer.Reset(time.Second * 8) - case success := <-stop: - if success { - stats.reportDone() - } - return - } - } -} - -// generateTrieRoot generates the trie hash based on the snapshot iterator. -// It can be used for generating account trie, storage trie or even the -// whole state which connects the accounts and the corresponding storages. -func generateTrieRoot(it Iterator, account common.Hash, generatorFn trieHasherFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { - var ( - in = make(chan trieKV) // chan to pass leaves - out = make(chan common.Hash, 1) // chan to collect result - stoplog = make(chan bool, 1) // 1-size buffer, works when logging is not enabled - wg sync.WaitGroup - ) - // Spin up a go-routine for trie hash re-generation - wg.Add(1) - go func() { - defer wg.Done() - generatorFn(in, out) - }() - // Spin up a go-routine for progress logging - if report && stats != nil { - wg.Add(1) - go func() { - defer wg.Done() - runReport(stats, stoplog) - }() - } - // Create a semaphore to assign tasks and collect results through. We'll pre- - // fill it with nils, thus using the same channel for both limiting concurrent - // processing and gathering results. - threads := runtime.NumCPU() - results := make(chan error, threads) - for i := 0; i < threads; i++ { - results <- nil // fill the semaphore - } - // stop is a helper function to shutdown the background threads - // and return the re-generated trie hash. - stop := func(fail error) (common.Hash, error) { - close(in) - result := <-out - for i := 0; i < threads; i++ { - if err := <-results; err != nil && fail == nil { - fail = err - } - } - stoplog <- fail == nil - - wg.Wait() - return result, fail - } - var ( - logged = time.Now() - processed = uint64(0) - leaf trieKV - ) - // Start to feed leaves - for it.Next() { - if account == (common.Hash{}) { - var ( - err error - fullData []byte - ) - if leafCallback == nil { - fullData, err = types.FullAccountRLP(it.(AccountIterator).Account()) - if err != nil { - return stop(err) - } - } else { - // Wait until the semaphore allows us to continue, aborting if - // a sub-task failed - if err := <-results; err != nil { - results <- nil // stop will drain the results, add a noop back for this error we just consumed - return stop(err) - } - // Fetch the next account and process it concurrently - account, err := types.FullAccount(it.(AccountIterator).Account()) - if err != nil { - return stop(err) - } - go func(hash common.Hash) { - subroot, err := leafCallback(hash, common.BytesToHash(account.CodeHash), stats) - if err != nil { - results <- err - return - } - if account.Root != subroot { - results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot) - return - } - results <- nil - }(it.Hash()) - fullData, err = rlp.EncodeToBytes(account) - if err != nil { - return stop(err) - } - } - leaf = trieKV{it.Hash(), fullData} - } else { - leaf = trieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} - } - in <- leaf - - // Accumulate the generation statistic if it's required. - processed++ - if time.Since(logged) > 3*time.Second && stats != nil { - if account == (common.Hash{}) { - stats.progressAccounts(it.Hash(), processed) - } else { - stats.progressContract(account, it.Hash(), processed) - } - logged, processed = time.Now(), 0 - } - } - // Commit the last part statistic. - if processed > 0 && stats != nil { - if account == (common.Hash{}) { - stats.finishAccounts(processed) - } else { - stats.finishContract(account, processed) - } - } - return stop(nil) -} - -func stackTrieHasher(in chan trieKV, out chan common.Hash) { +func stackTrieHasher(_ ethdb.KeyValueWriter, _ string, _ common.Hash, in chan internal.TrieKV, out chan common.Hash) { t := trie.NewStackTrie(nil) for leaf := range in { - t.Update(leaf.key[:], leaf.value) + t.Update(leaf.Key[:], leaf.Value) } out <- t.Hash() }