diff --git a/accounts/abi/bind/v2/dep_tree_test.go b/accounts/abi/bind/v2/dep_tree_test.go index c4c0b09237..26df3da888 100644 --- a/accounts/abi/bind/v2/dep_tree_test.go +++ b/accounts/abi/bind/v2/dep_tree_test.go @@ -159,10 +159,10 @@ func testLinkCase(tcInput linkTestCaseInput) error { overrideAddrs = make(map[rune]common.Address) ) // generate deterministic addresses for the override set. - rand.Seed(42) + rng := rand.New(rand.NewSource(42)) for contract := range tcInput.overrides { var addr common.Address - rand.Read(addr[:]) + rng.Read(addr[:]) overrideAddrs[contract] = addr overridesAddrs[addr] = struct{}{} } diff --git a/accounts/accounts.go b/accounts/accounts.go index 6d46a0f76c..b41f1595fa 100644 --- a/accounts/accounts.go +++ b/accounts/accounts.go @@ -21,11 +21,10 @@ import ( "fmt" "math/big" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/event" ) @@ -198,7 +197,7 @@ func TextHash(data []byte) []byte { // This gives context to the signed message and prevents signing of transactions. func TextAndHash(data []byte) ([]byte, string) { msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) - hasher := sha3.NewLegacyKeccak256() + hasher := keccak.NewLegacyKeccak256() hasher.Write([]byte(msg)) return hasher.Sum(nil), msg diff --git a/accounts/keystore/keystore.go b/accounts/keystore/keystore.go index 444bb8c7dc..79e1e0e9d1 100644 --- a/accounts/keystore/keystore.go +++ b/accounts/keystore/keystore.go @@ -441,7 +441,7 @@ func (ks *KeyStore) Export(a accounts.Account, passphrase, newPassphrase string) if err != nil { return nil, err } - + defer zeroKey(key.PrivateKey) var N, P int if store, ok := ks.storage.(*keyStorePassphrase); ok { N, P = store.scryptN, store.scryptP @@ -508,7 +508,7 @@ func (ks *KeyStore) Update(a accounts.Account, passphrase, newPassphrase string) if err != nil { return err } - + defer zeroKey(key.PrivateKey) return ks.storage.StoreKey(a.URL.Path, key, newPassphrase) } diff --git a/accounts/keystore/presale.go b/accounts/keystore/presale.go index c2fa8063ca..7d439a2ddd 100644 --- a/accounts/keystore/presale.go +++ b/accounts/keystore/presale.go @@ -89,7 +89,9 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error */ passBytes := []byte(password) derivedKey := pbkdf2.Key(passBytes, passBytes, 2000, 16, sha256.New) - + if len(cipherText)%aes.BlockSize != 0 { + return nil, errors.New("ciphertext must be a multiple of block size") + } plainText, err := aesCBCDecrypt(derivedKey, cipherText, iv) if err != nil { return nil, err diff --git a/accounts/scwallet/securechannel.go b/accounts/scwallet/securechannel.go index 552d39cc1c..d82b3d9ecc 100644 --- a/accounts/scwallet/securechannel.go +++ b/accounts/scwallet/securechannel.go @@ -313,6 +313,10 @@ func (s *SecureChannelSession) decryptAPDU(data []byte) ([]byte, error) { return nil, err } + if len(data) == 0 || len(data)%aes.BlockSize != 0 { + return nil, fmt.Errorf("invalid ciphertext length: %d", len(data)) + } + ret := make([]byte, len(data)) crypter := cipher.NewCBCDecrypter(a, s.iv) diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index 585ae0a1c5..caddbee983 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -44,6 +44,14 @@ const refreshCycle = time.Second // trashing. const refreshThrottling = 500 * time.Millisecond +const ( + // deviceUsagePage identifies Ledger devices by HID usage page (0xffa0) on Windows and macOS. + // See: https://github.com/LedgerHQ/ledger-live/blob/05a2980e838955a11a1418da638ef8ac3df4fb74/libs/ledgerjs/packages/hw-transport-node-hid-noevents/src/TransportNodeHid.ts + deviceUsagePage = 0xffa0 + // deviceInterface identifies Ledger devices by USB interface number (0) on Linux. + deviceInterface = 0 +) + // Hub is a accounts.Backend that can find and handle generic USB hardware wallets. type Hub struct { scheme string // Protocol scheme prefixing account and wallet URLs. @@ -83,6 +91,7 @@ func NewLedgerHub() (*Hub, error) { 0x0005, /* Ledger Nano S Plus */ 0x0006, /* Ledger Nano FTS */ 0x0007, /* Ledger Flex */ + 0x0008, /* Ledger Nano Gen5 */ 0x0000, /* WebUSB Ledger Blue */ 0x1000, /* WebUSB Ledger Nano S */ @@ -90,7 +99,8 @@ func NewLedgerHub() (*Hub, error) { 0x5000, /* WebUSB Ledger Nano S Plus */ 0x6000, /* WebUSB Ledger Nano FTS */ 0x7000, /* WebUSB Ledger Flex */ - }, 0xffa0, 0, newLedgerDriver) + 0x8000, /* WebUSB Ledger Nano Gen5 */ + }, deviceUsagePage, deviceInterface, newLedgerDriver) } // NewTrezorHubWithHID creates a new hardware wallet manager for Trezor devices. diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 09a308c94c..be41596293 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -190,7 +190,7 @@ func (w *ledgerDriver) SignTypedMessage(path accounts.DerivationPath, domainHash return nil, accounts.ErrWalletClosed } // Ensure the wallet is capable of signing the given transaction - if w.version[0] < 1 && w.version[1] < 5 { + if w.version[0] < 1 || (w.version[0] == 1 && w.version[1] < 5) { //lint:ignore ST1005 brand name displayed on the console return nil, fmt.Errorf("Ledger version >= 1.5.0 required for EIP-712 signing (found version v%d.%d.%d)", w.version[0], w.version[1], w.version[2]) } diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 04b17e5d71..59865f048b 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -662,7 +662,7 @@ func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID // data is not supported for Ledger wallets, so this method will always return // an error. func (w *wallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) { - return w.SignText(account, accounts.TextHash(text)) + return w.SignText(account, text) } // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given diff --git a/beacon/blsync/engineclient.go b/beacon/blsync/engineclient.go index f9821fc6f3..9fc6a18a57 100644 --- a/beacon/blsync/engineclient.go +++ b/beacon/blsync/engineclient.go @@ -101,7 +101,16 @@ func (ec *engineClient) callNewPayload(fork string, event types.ChainHeadEvent) params = []any{execData} ) switch fork { - case "electra": + case "altair", "bellatrix": + method = "engine_newPayloadV1" + case "capella": + method = "engine_newPayloadV2" + case "deneb": + method = "engine_newPayloadV3" + parentBeaconRoot := event.BeaconHead.ParentRoot + blobHashes := collectBlobHashes(event.Block) + params = append(params, blobHashes, parentBeaconRoot) + default: // electra, fulu and above method = "engine_newPayloadV4" parentBeaconRoot := event.BeaconHead.ParentRoot blobHashes := collectBlobHashes(event.Block) @@ -110,15 +119,6 @@ func (ec *engineClient) callNewPayload(fork string, event types.ChainHeadEvent) hexRequests[i] = hexutil.Bytes(event.ExecRequests[i]) } params = append(params, blobHashes, parentBeaconRoot, hexRequests) - case "deneb": - method = "engine_newPayloadV3" - parentBeaconRoot := event.BeaconHead.ParentRoot - blobHashes := collectBlobHashes(event.Block) - params = append(params, blobHashes, parentBeaconRoot) - case "capella": - method = "engine_newPayloadV2" - default: - method = "engine_newPayloadV1" } ctx, cancel := context.WithTimeout(ec.rootCtx, time.Second*5) @@ -145,12 +145,12 @@ func (ec *engineClient) callForkchoiceUpdated(fork string, event types.ChainHead var method string switch fork { - case "deneb", "electra": - method = "engine_forkchoiceUpdatedV3" + case "altair", "bellatrix": + method = "engine_forkchoiceUpdatedV1" case "capella": method = "engine_forkchoiceUpdatedV2" - default: - method = "engine_forkchoiceUpdatedV1" + default: // deneb, electra, fulu and above + method = "engine_forkchoiceUpdatedV3" } ctx, cancel := context.WithTimeout(ec.rootCtx, time.Second*5) diff --git a/beacon/engine/gen_ed.go b/beacon/engine/gen_ed.go index 50cb0ad37c..6893d64a16 100644 --- a/beacon/engine/gen_ed.go +++ b/beacon/engine/gen_ed.go @@ -17,26 +17,24 @@ var _ = (*executableDataMarshaling)(nil) // MarshalJSON marshals as JSON. func (e ExecutableData) MarshalJSON() ([]byte, error) { type ExecutableData struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Random common.Hash `json:"prevRandao" gencodec:"required"` - Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` - GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` - ExtraData hexutil.Bytes `json:"extraData" gencodec:"required"` - BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` + Number hexutil.Uint64 `json:"blockNumber" gencodec:"required"` + GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ExtraData hexutil.Bytes `json:"extraData" gencodec:"required"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` } - var enc ExecutableData enc.ParentHash = e.ParentHash enc.FeeRecipient = e.FeeRecipient @@ -51,136 +49,102 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { enc.ExtraData = e.ExtraData enc.BaseFeePerGas = (*hexutil.Big)(e.BaseFeePerGas) enc.BlockHash = e.BlockHash - if e.Transactions != nil { enc.Transactions = make([]hexutil.Bytes, len(e.Transactions)) for k, v := range e.Transactions { enc.Transactions[k] = v } } - enc.Withdrawals = e.Withdrawals enc.BlobGasUsed = (*hexutil.Uint64)(e.BlobGasUsed) enc.ExcessBlobGas = (*hexutil.Uint64)(e.ExcessBlobGas) - enc.ExecutionWitness = e.ExecutionWitness return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (e *ExecutableData) UnmarshalJSON(input []byte) error { type ExecutableData struct { - ParentHash *common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient *common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` - Random *common.Hash `json:"prevRandao" gencodec:"required"` - Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` - GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` - ExtraData *hexutil.Bytes `json:"extraData" gencodec:"required"` - BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` - BlockHash *common.Hash `json:"blockHash" gencodec:"required"` - Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient *common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot *common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot *common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom *hexutil.Bytes `json:"logsBloom" gencodec:"required"` + Random *common.Hash `json:"prevRandao" gencodec:"required"` + Number *hexutil.Uint64 `json:"blockNumber" gencodec:"required"` + GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Timestamp *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + ExtraData *hexutil.Bytes `json:"extraData" gencodec:"required"` + BaseFeePerGas *hexutil.Big `json:"baseFeePerGas" gencodec:"required"` + BlockHash *common.Hash `json:"blockHash" gencodec:"required"` + Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` } - var dec ExecutableData if err := json.Unmarshal(input, &dec); err != nil { return err } - if dec.ParentHash == nil { return errors.New("missing required field 'parentHash' for ExecutableData") } - e.ParentHash = *dec.ParentHash - if dec.FeeRecipient == nil { return errors.New("missing required field 'feeRecipient' for ExecutableData") } - e.FeeRecipient = *dec.FeeRecipient - if dec.StateRoot == nil { return errors.New("missing required field 'stateRoot' for ExecutableData") } - e.StateRoot = *dec.StateRoot - if dec.ReceiptsRoot == nil { return errors.New("missing required field 'receiptsRoot' for ExecutableData") } - e.ReceiptsRoot = *dec.ReceiptsRoot - if dec.LogsBloom == nil { return errors.New("missing required field 'logsBloom' for ExecutableData") } - e.LogsBloom = *dec.LogsBloom - if dec.Random == nil { return errors.New("missing required field 'prevRandao' for ExecutableData") } - e.Random = *dec.Random - if dec.Number == nil { return errors.New("missing required field 'blockNumber' for ExecutableData") } - e.Number = uint64(*dec.Number) - if dec.GasLimit == nil { return errors.New("missing required field 'gasLimit' for ExecutableData") } - e.GasLimit = uint64(*dec.GasLimit) - if dec.GasUsed == nil { return errors.New("missing required field 'gasUsed' for ExecutableData") } - e.GasUsed = uint64(*dec.GasUsed) - if dec.Timestamp == nil { return errors.New("missing required field 'timestamp' for ExecutableData") } - e.Timestamp = uint64(*dec.Timestamp) - if dec.ExtraData == nil { return errors.New("missing required field 'extraData' for ExecutableData") } - e.ExtraData = *dec.ExtraData - if dec.BaseFeePerGas == nil { return errors.New("missing required field 'baseFeePerGas' for ExecutableData") } - e.BaseFeePerGas = (*big.Int)(dec.BaseFeePerGas) - if dec.BlockHash == nil { return errors.New("missing required field 'blockHash' for ExecutableData") } - e.BlockHash = *dec.BlockHash - if dec.Transactions == nil { return errors.New("missing required field 'transactions' for ExecutableData") } - e.Transactions = make([][]byte, len(dec.Transactions)) for k, v := range dec.Transactions { e.Transactions[k] = v } - if dec.Withdrawals != nil { e.Withdrawals = dec.Withdrawals } @@ -190,8 +154,5 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { if dec.ExcessBlobGas != nil { e.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas) } - if dec.ExecutionWitness != nil { - e.ExecutionWitness = dec.ExecutionWitness - } return nil } diff --git a/beacon/engine/types.go b/beacon/engine/types.go index 6e1a7bbb6d..da9b6568f2 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -73,24 +73,23 @@ type payloadAttributesMarshaling struct { // ExecutableData is the data necessary to execute an EL payload. type ExecutableData struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` - StateRoot common.Hash `json:"stateRoot" gencodec:"required"` - ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` - LogsBloom []byte `json:"logsBloom" gencodec:"required"` - Random common.Hash `json:"prevRandao" gencodec:"required"` - Number uint64 `json:"blockNumber" gencodec:"required"` - GasLimit uint64 `json:"gasLimit" gencodec:"required"` - GasUsed uint64 `json:"gasUsed" gencodec:"required"` - Timestamp uint64 `json:"timestamp" gencodec:"required"` - ExtraData []byte `json:"extraData" gencodec:"required"` - BaseFeePerGas *big.Int `json:"baseFeePerGas" gencodec:"required"` - BlockHash common.Hash `json:"blockHash" gencodec:"required"` - Transactions [][]byte `json:"transactions" gencodec:"required"` - Withdrawals []*types.Withdrawal `json:"withdrawals"` - BlobGasUsed *uint64 `json:"blobGasUsed"` - ExcessBlobGas *uint64 `json:"excessBlobGas"` - ExecutionWitness *types.ExecutionWitness `json:"executionWitness,omitempty"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` + StateRoot common.Hash `json:"stateRoot" gencodec:"required"` + ReceiptsRoot common.Hash `json:"receiptsRoot" gencodec:"required"` + LogsBloom []byte `json:"logsBloom" gencodec:"required"` + Random common.Hash `json:"prevRandao" gencodec:"required"` + Number uint64 `json:"blockNumber" gencodec:"required"` + GasLimit uint64 `json:"gasLimit" gencodec:"required"` + GasUsed uint64 `json:"gasUsed" gencodec:"required"` + Timestamp uint64 `json:"timestamp" gencodec:"required"` + ExtraData []byte `json:"extraData" gencodec:"required"` + BaseFeePerGas *big.Int `json:"baseFeePerGas" gencodec:"required"` + BlockHash common.Hash `json:"blockHash" gencodec:"required"` + Transactions [][]byte `json:"transactions" gencodec:"required"` + Withdrawals []*types.Withdrawal `json:"withdrawals"` + BlobGasUsed *uint64 `json:"blobGasUsed"` + ExcessBlobGas *uint64 `json:"excessBlobGas"` } // JSON type overrides for executableData. @@ -192,7 +191,6 @@ func (b *PayloadID) UnmarshalText(input []byte) error { if err != nil { return fmt.Errorf("invalid payload id %q: %w", input, err) } - return nil } @@ -212,22 +210,18 @@ func encodeTransactions(txs []*types.Transaction) [][]byte { for i, tx := range txs { enc[i], _ = tx.MarshalBinary() } - return enc } func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { var txs = make([]*types.Transaction, len(enc)) - for i, encTx := range enc { var tx types.Transaction if err := tx.UnmarshalBinary(encTx); err != nil { return nil, fmt.Errorf("invalid transaction %d: %v", i, err) } - txs[i] = &tx } - return txs, nil } @@ -321,8 +315,7 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H RequestsHash: requestsHash, } return types.NewBlockWithHeader(header). - WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}). - WithWitness(data.ExecutionWitness), + WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}), nil } @@ -330,24 +323,23 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H // fields from the given block. It assumes the given block is post-merge block. func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.BlobTxSidecar, requests [][]byte) *ExecutionPayloadEnvelope { data := &ExecutableData{ - BlockHash: block.Hash(), - ParentHash: block.ParentHash(), - FeeRecipient: block.Coinbase(), - StateRoot: block.Root(), - Number: block.NumberU64(), - GasLimit: block.GasLimit(), - GasUsed: block.GasUsed(), - BaseFeePerGas: block.BaseFee(), - Timestamp: block.Time(), - ReceiptsRoot: block.ReceiptHash(), - LogsBloom: block.Bloom().Bytes(), - Transactions: encodeTransactions(block.Transactions()), - Random: block.MixDigest(), - ExtraData: block.Extra(), - Withdrawals: block.Withdrawals(), - BlobGasUsed: block.BlobGasUsed(), - ExcessBlobGas: block.ExcessBlobGas(), - ExecutionWitness: block.ExecutionWitness(), + BlockHash: block.Hash(), + ParentHash: block.ParentHash(), + FeeRecipient: block.Coinbase(), + StateRoot: block.Root(), + Number: block.NumberU64(), + GasLimit: block.GasLimit(), + GasUsed: block.GasUsed(), + BaseFeePerGas: block.BaseFee(), + Timestamp: block.Time(), + ReceiptsRoot: block.ReceiptHash(), + LogsBloom: block.Bloom().Bytes(), + Transactions: encodeTransactions(block.Transactions()), + Random: block.MixDigest(), + ExtraData: block.Extra(), + Withdrawals: block.Withdrawals(), + BlobGasUsed: block.BlobGasUsed(), + ExcessBlobGas: block.ExcessBlobGas(), } // Add blobs. diff --git a/beacon/light/canonical.go b/beacon/light/canonical.go index b5371493b4..56622425b2 100644 --- a/beacon/light/canonical.go +++ b/beacon/light/canonical.go @@ -69,7 +69,10 @@ func newCanonicalStore[T any](db ethdb.Iteratee, keyPrefix []byte) (*canonicalSt // databaseKey returns the database key belonging to the given period. func (cs *canonicalStore[T]) databaseKey(period uint64) []byte { - return binary.BigEndian.AppendUint64(append([]byte{}, cs.keyPrefix...), period) + key := make([]byte, len(cs.keyPrefix)+8) + copy(key, cs.keyPrefix) + binary.BigEndian.PutUint64(key[len(cs.keyPrefix):], period) + return key } // add adds the given item to the database. It also ensures that the range remains diff --git a/beacon/light/sync/head_sync.go b/beacon/light/sync/head_sync.go index 5e41258053..7189767d9c 100644 --- a/beacon/light/sync/head_sync.go +++ b/beacon/light/sync/head_sync.go @@ -105,6 +105,7 @@ func (s *HeadSync) Process(requester request.Requester, events []request.Event) delete(s.serverHeads, event.Server) delete(s.unvalidatedOptimistic, event.Server) delete(s.unvalidatedFinality, event.Server) + delete(s.reqFinalityEpoch, event.Server) } } } diff --git a/beacon/params/checkpoint_hoodi.hex b/beacon/params/checkpoint_hoodi.hex index 2885d7c996..7bac591f96 100644 --- a/beacon/params/checkpoint_hoodi.hex +++ b/beacon/params/checkpoint_hoodi.hex @@ -1 +1 @@ -0x1bbf958008172591b6cbdb3d8d52e26998258e83d4bdb9eec10969d84519a6bd \ No newline at end of file +0xbb7a7f3c40d8ea0b450f91587db65d0f1c079669277e01a0426c8911702a863a \ No newline at end of file diff --git a/beacon/params/checkpoint_mainnet.hex b/beacon/params/checkpoint_mainnet.hex index 417e69a24b..c2886cc564 100644 --- a/beacon/params/checkpoint_mainnet.hex +++ b/beacon/params/checkpoint_mainnet.hex @@ -1 +1 @@ -0x2fe39a39b6f7cbd549e0f74d259de6db486005a65bd3bd92840dd6ce21d6f4c8 \ No newline at end of file +0x2af778d703186526a1b6304b423f338f11556206f618643c3f7fa0d7b1ef5c9b \ No newline at end of file diff --git a/beacon/params/checkpoint_sepolia.hex b/beacon/params/checkpoint_sepolia.hex index 02faf72187..55842f8ac0 100644 --- a/beacon/params/checkpoint_sepolia.hex +++ b/beacon/params/checkpoint_sepolia.hex @@ -1 +1 @@ -0x86686b2b366e24134e0e3969a9c5f3759f92e5d2b04785b42e22cc7d468c2107 \ No newline at end of file +0x48a89c9ea7ba19de2931797974cf8722344ab231c0edada278b108ef74125478 \ No newline at end of file diff --git a/beacon/params/config.go b/beacon/params/config.go index 235decf1ea..134f3a3fad 100644 --- a/beacon/params/config.go +++ b/beacon/params/config.go @@ -39,7 +39,7 @@ import ( // across signing different data structures. const syncCommitteeDomain = 7 -var knownForks = []string{"GENESIS", "ALTAIR", "BELLATRIX", "CAPELLA", "DENEB"} +var knownForks = []string{"GENESIS", "ALTAIR", "BELLATRIX", "CAPELLA", "DENEB", "ELECTRA", "FULU"} // ClientConfig contains beacon light client configuration. type ClientConfig struct { @@ -104,6 +104,9 @@ func (c *ChainConfig) LoadForks(file []byte) error { epochs["GENESIS"] = 0 for key, value := range config { + if value == nil { + continue + } if strings.HasSuffix(key, "_FORK_VERSION") { name := key[:len(key)-len("_FORK_VERSION")] switch version := value.(type) { diff --git a/beacon/params/config_test.go b/beacon/params/config_test.go index 41e120469b..0b569b604c 100644 --- a/beacon/params/config_test.go +++ b/beacon/params/config_test.go @@ -15,6 +15,9 @@ ALTAIR_FORK_EPOCH: 1 EIP7928_FORK_VERSION: 0xb0000038 EIP7928_FORK_EPOCH: 18446744073709551615 +EIP7XXX_FORK_VERSION: +EIP7XXX_FORK_EPOCH: + BLOB_SCHEDULE: [] ` c := &ChainConfig{} diff --git a/beacon/params/networks.go b/beacon/params/networks.go index b35db34fd6..5dcf08cc5d 100644 --- a/beacon/params/networks.go +++ b/beacon/params/networks.go @@ -40,36 +40,39 @@ var ( GenesisTime: 1606824023, Checkpoint: common.HexToHash(checkpointMainnet), }). - AddFork("GENESIS", 0, []byte{0, 0, 0, 0}). - AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}). - AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}). - AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}). - AddFork("DENEB", 269568, []byte{4, 0, 0, 0}). - AddFork("ELECTRA", 364032, []byte{5, 0, 0, 0}) + AddFork("GENESIS", 0, common.FromHex("0x00000000")). + AddFork("ALTAIR", 74240, common.FromHex("0x01000000")). + AddFork("BELLATRIX", 144896, common.FromHex("0x02000000")). + AddFork("CAPELLA", 194048, common.FromHex("0x03000000")). + AddFork("DENEB", 269568, common.FromHex("0x04000000")). + AddFork("ELECTRA", 364032, common.FromHex("0x05000000")). + AddFork("FULU", 411392, common.FromHex("0x06000000")) SepoliaLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"), GenesisTime: 1655733600, Checkpoint: common.HexToHash(checkpointSepolia), }). - AddFork("GENESIS", 0, []byte{144, 0, 0, 105}). - AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}). - AddFork("BELLATRIX", 100, []byte{144, 0, 0, 113}). - AddFork("CAPELLA", 56832, []byte{144, 0, 0, 114}). - AddFork("DENEB", 132608, []byte{144, 0, 0, 115}). - AddFork("ELECTRA", 222464, []byte{144, 0, 0, 116}) + AddFork("GENESIS", 0, common.FromHex("0x90000069")). + AddFork("ALTAIR", 50, common.FromHex("0x90000070")). + AddFork("BELLATRIX", 100, common.FromHex("0x90000071")). + AddFork("CAPELLA", 56832, common.FromHex("0x90000072")). + AddFork("DENEB", 132608, common.FromHex("0x90000073")). + AddFork("ELECTRA", 222464, common.FromHex("0x90000074")). + AddFork("FULU", 272640, common.FromHex("0x90000075")) HoleskyLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1"), GenesisTime: 1695902400, Checkpoint: common.HexToHash(checkpointHolesky), }). - AddFork("GENESIS", 0, []byte{1, 1, 112, 0}). - AddFork("ALTAIR", 0, []byte{2, 1, 112, 0}). - AddFork("BELLATRIX", 0, []byte{3, 1, 112, 0}). - AddFork("CAPELLA", 256, []byte{4, 1, 112, 0}). - AddFork("DENEB", 29696, []byte{5, 1, 112, 0}). - AddFork("ELECTRA", 115968, []byte{6, 1, 112, 0}) + AddFork("GENESIS", 0, common.FromHex("0x01017000")). + AddFork("ALTAIR", 0, common.FromHex("0x02017000")). + AddFork("BELLATRIX", 0, common.FromHex("0x03017000")). + AddFork("CAPELLA", 256, common.FromHex("0x04017000")). + AddFork("DENEB", 29696, common.FromHex("0x05017000")). + AddFork("ELECTRA", 115968, common.FromHex("0x06017000")). + AddFork("FULU", 165120, common.FromHex("0x07017000")) HoodiLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0x212f13fc4df078b6cb7db228f1c8307566dcecf900867401a92023d7ba99cb5f"), @@ -82,5 +85,5 @@ var ( AddFork("CAPELLA", 0, common.FromHex("0x40000910")). AddFork("DENEB", 0, common.FromHex("0x50000910")). AddFork("ELECTRA", 2048, common.FromHex("0x60000910")). - AddFork("FULU", 18446744073709551615, common.FromHex("0x70000910")) + AddFork("FULU", 50688, common.FromHex("0x70000910")) ) diff --git a/beacon/types/beacon_block.go b/beacon/types/beacon_block.go index 1191740962..59e8fbd19d 100644 --- a/beacon/types/beacon_block.go +++ b/beacon/types/beacon_block.go @@ -53,7 +53,7 @@ func BlockFromJSON(forkName string, data []byte) (*BeaconBlock, error) { obj = new(capella.BeaconBlock) case "deneb": obj = new(deneb.BeaconBlock) - case "electra": + case "electra", "fulu": obj = new(electra.BeaconBlock) default: return nil, fmt.Errorf("unsupported fork: %s", forkName) diff --git a/beacon/types/exec_header.go b/beacon/types/exec_header.go index 37c54da03f..0a48c7e79f 100644 --- a/beacon/types/exec_header.go +++ b/beacon/types/exec_header.go @@ -46,7 +46,7 @@ func ExecutionHeaderFromJSON(forkName string, data []byte) (*ExecutionHeader, er switch forkName { case "capella": obj = new(capella.ExecutionPayloadHeader) - case "deneb", "electra": // note: the payload type was not changed in electra + case "deneb", "electra", "fulu": // note: the payload type was not changed in electra/fulu obj = new(deneb.ExecutionPayloadHeader) default: return nil, fmt.Errorf("unsupported fork: %s", forkName) diff --git a/build/ci.go b/build/ci.go index a112033543..ddeb547177 100644 --- a/build/ci.go +++ b/build/ci.go @@ -107,6 +107,18 @@ var ( Tags: "ziren", Env: map[string]string{"GOMIPS": "softfloat", "CGO_ENABLED": "0"}, }, + { + Name: "wasm-js", + GOOS: "js", + GOARCH: "wasm", + Tags: "example", + }, + { + Name: "wasm-wasi", + GOOS: "wasip1", + GOARCH: "wasm", + Tags: "example", + }, { Name: "example", Tags: "example", @@ -331,6 +343,10 @@ func buildFlags(env build.Environment, staticLinking bool, buildTags []string) ( } ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") } + // TODO(gballet): revisit after the input api has been defined + if runtime.GOARCH == "wasm" { + ld = append(ld, "-gcflags=all=-d=softfloat") + } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } @@ -446,9 +462,14 @@ func doCheckGenerate() { ) pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")} + excludes := []string{"tests/testdata", "build/cache", ".git"} + for i := range excludes { + excludes[i] = filepath.FromSlash(excludes[i]) + } + for _, mod := range goModules { // Compute the origin hashes of all the files - hashes, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"}) + hashes, err := build.HashFolder(mod, excludes) if err != nil { log.Fatal("Error computing hashes", "err", err) } @@ -458,7 +479,7 @@ func doCheckGenerate() { c.Dir = mod build.MustRun(c) // Check if generate file hashes have changed - generated, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"}) + generated, err := build.HashFolder(mod, excludes) if err != nil { log.Fatalf("Error re-computing hashes: %v", err) } diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 39ff5d83c6..0000000000 --- a/circle.yml +++ /dev/null @@ -1,32 +0,0 @@ -machine: - services: - - docker - -dependencies: - cache_directories: - - "~/.ethash" # Cache the ethash DAG generated by hive for consecutive builds - - "~/.docker" # Cache all docker images manually to avoid lengthy rebuilds - override: - # Restore all previously cached docker images - - mkdir -p ~/.docker - - for img in `ls ~/.docker`; do docker load -i ~/.docker/$img; done - - # Pull in and hive, restore cached ethash DAGs and do a dry run - - go get -u github.com/karalabe/hive - - (cd ~/.go_workspace/src/github.com/karalabe/hive && mkdir -p workspace/ethash/ ~/.ethash) - - (cd ~/.go_workspace/src/github.com/karalabe/hive && cp -r ~/.ethash/. workspace/ethash/) - - (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=NONE --test=. --sim=. --loglevel=6) - - # Cache all the docker images and the ethash DAGs - - for img in `docker images | grep -v "^" | tail -n +2 | awk '{print $1}'`; do docker save $img > ~/.docker/`echo $img | tr '/' ':'`.tar; done - - cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/ethash/. ~/.ethash - -test: - override: - # Build Geth and move into a known folder - - make geth - - cp ./build/bin/geth $HOME/geth - - # Run hive and move all generated logs into the public artifacts folder - - (cd ~/.go_workspace/src/github.com/karalabe/hive && hive --docker-noshell --client=go-ethereum:local --override=$HOME/geth --test=. --sim=.) - - cp -r ~/.go_workspace/src/github.com/karalabe/hive/workspace/logs/* $CIRCLE_ARTIFACTS diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index 8114df2d35..408974ff9d 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -17,16 +17,18 @@ package main import ( + "bufio" "encoding/json" - "errors" "fmt" "maps" "os" "regexp" "slices" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" @@ -35,33 +37,52 @@ import ( var blockTestCommand = &cli.Command{ Action: blockTestCmd, Name: "blocktest", - Usage: "Executes the given blockchain tests", + Usage: "Executes the given blockchain tests. Filenames can be fed via standard input (batch mode) or as an argument (one-off execution).", ArgsUsage: "", Flags: slices.Concat([]cli.Flag{ DumpFlag, HumanReadableFlag, RunFlag, WitnessCrossCheckFlag, + FuzzFlag, }, traceFlags), } func blockTestCmd(ctx *cli.Context) error { path := ctx.Args().First() - if len(path) == 0 { - return errors.New("path argument required") + + // If path is provided, run the tests at that path. + if len(path) != 0 { + var ( + collected = collectFiles(path) + results []testResult + ) + for _, fname := range collected { + r, err := runBlockTest(ctx, fname) + if err != nil { + return err + } + results = append(results, r...) + } + report(ctx, results) + return nil } - var ( - collected = collectFiles(path) - results []testResult - ) - for _, fname := range collected { - r, err := runBlockTest(ctx, fname) + // Otherwise, read filenames from stdin and execute back-to-back. + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + fname := scanner.Text() + if len(fname) == 0 { + return nil + } + results, err := runBlockTest(ctx, fname) if err != nil { return err } - results = append(results, r...) + // During fuzzing, we report the result after every block + if !ctx.IsSet(FuzzFlag.Name) { + report(ctx, results) + } } - report(ctx, results) return nil } @@ -82,6 +103,11 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { } tracer := tracerFromFlags(ctx) + // Suppress INFO logs during fuzzing + if ctx.IsSet(FuzzFlag.Name) { + log.SetDefault(log.NewLogger(log.DiscardHandler())) + } + // Pull out keys to sort and ensure tests are run in order. keys := slices.Sorted(maps.Keys(tests)) @@ -91,16 +117,35 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { if !re.MatchString(name) { continue } + test := tests[name] result := &testResult{Name: name, Pass: true} - if err := tests[name].Run(false, rawdb.PathScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) { + var finalRoot *common.Hash + if err := test.Run(false, rawdb.PathScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) { if ctx.Bool(DumpFlag.Name) { if s, _ := chain.State(); s != nil { result.State = dump(s) } } + // Capture final state root for end marker + if chain != nil { + root := chain.CurrentBlock().Root + finalRoot = &root + } }); err != nil { result.Pass, result.Error = false, err.Error() } + + // Always assign fork (regardless of pass/fail or tracer) + result.Fork = test.Network() + // Assign root if test succeeded + if result.Pass && finalRoot != nil { + result.Root = finalRoot + } + + // When fuzzing, write results after every block + if ctx.IsSet(FuzzFlag.Name) { + report(ctx, []testResult{*result}) + } results = append(results, *result) } return results, nil diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index ce49224f41..3c6790225f 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -18,10 +18,10 @@ package t8ntool import ( "fmt" + stdmath "math" "math/big" "github.com/holiman/uint256" - "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -44,8 +45,9 @@ import ( ) type Prestate struct { - Env stEnv `json:"env"` - Pre types.GenesisAlloc `json:"pre"` + Env stEnv `json:"env"` + Pre types.GenesisAlloc `json:"pre"` + TreeLeaves map[common.Hash]hexutil.Bytes `json:"vkt,omitempty"` } //go:generate go run github.com/fjl/gencodec -type ExecutionResult -field-override executionResultMarshaling -out gen_execresult.go @@ -147,7 +149,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } var ( - statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre) + isEIP4762 = chainConfig.IsVerkle(big.NewInt(int64(pre.Env.Number))) + statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762) signer = types.MakeSigner(chainConfig, new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) gaspool = new(core.GasPool) blockHash = common.Hash{0x13, 0x37} @@ -311,6 +314,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, // Amount is in gwei, turn into wei amount := new(big.Int).Mul(new(big.Int).SetUint64(w.Amount), big.NewInt(params.GWei)) statedb.AddBalance(w.Address, uint256.MustFromBig(amount), tracing.BalanceIncreaseWithdrawal) + + if isEIP4762 { + statedb.AccessEvents().AddAccount(w.Address, true, stdmath.MaxUint64) + } } // Polygon/bor: EIP-6110, EIP-7002, and EIP-7251 are not supported @@ -374,8 +381,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, execRs.Requests = requests } - // Re-create statedb instance with new root upon the updated database - // for accessing latest states. + // Re-create statedb instance with new root for MPT mode statedb, err = state.New(root, statedb.Database()) if err != nil { return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err)) @@ -384,12 +390,17 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return statedb, execRs, body, nil } -func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB { - tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true}) +func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, isBintrie bool) *state.StateDB { + tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true, IsVerkle: isBintrie}) sdb := state.NewDatabase(tdb, nil) - statedb, err := state.New(types.EmptyRootHash, sdb) + + root := types.EmptyRootHash + if isBintrie { + root = types.EmptyBinaryHash + } + statedb, err := state.New(root, sdb) if err != nil { - panic(fmt.Errorf("failed to create initial state: %v", err)) + panic(fmt.Errorf("failed to create initial statedb: %v", err)) } for addr, a := range accounts { statedb.SetCode(addr, a.Code, tracing.CodeChangeUnspecified) @@ -400,10 +411,15 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB } } // Commit and re-open to start with a clean state. - root, err := statedb.Commit(0, false, false) + root, err = statedb.Commit(0, false, false) if err != nil { panic(fmt.Errorf("failed to commit initial state: %v", err)) } + // If bintrie mode started, check if conversion happened + if isBintrie { + return statedb + } + // For MPT mode, reopen the state with the committed root statedb, err = state.New(root, sdb) if err != nil { panic(fmt.Errorf("failed to reopen state after commit: %v", err)) @@ -411,8 +427,8 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB return statedb } -func rlpHash(x interface{}) (h common.Hash) { - hw := sha3.NewLegacyKeccak256() +func rlpHash(x any) (h common.Hash) { + hw := keccak.NewLegacyKeccak256() rlp.Encode(hw, x) hw.Sum(h[:0]) diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index cd136815f1..90dcc2bf3f 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -89,6 +89,14 @@ var ( "\t - into the file ", Value: "block.json", } + OutputBTFlag = &cli.StringFlag{ + Name: "output.vkt", + Usage: "Determines where to put the `BT` of the post-state.\n" + + "\t`stdout` - into the stdout output\n" + + "\t`stderr` - into the stderr output\n" + + "\t - into the file ", + Value: "vkt.json", + } InputAllocFlag = &cli.StringFlag{ Name: "input.alloc", Usage: "`stdin` or file name of where to find the prestate alloc to use.", @@ -124,6 +132,11 @@ var ( Usage: "`stdin` or file name of where to find the transactions list in RLP form.", Value: "txs.rlp", } + // TODO(@CPerezz): rename `Name` of the file in a follow-up PR (relays on EEST -> https://github.com/ethereum/execution-spec-tests/tree/verkle/main) + InputBTFlag = &cli.StringFlag{ + Name: "input.vkt", + Usage: "`stdin` or file name of where to find the prestate BT.", + } SealCliqueFlag = &cli.StringFlag{ Name: "seal.clique", Usage: "Seal block with Clique. `stdin` or file name of where to find the Clique sealing data.", diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index c71828257f..b94f41d6a2 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -126,10 +126,6 @@ func Transaction(ctx *cli.Context) error { var results []result for it.Next() { - if err := it.Err(); err != nil { - return NewError(ErrorIO, err) - } - var tx types.Transaction err := rlp.DecodeBytes(it.Value(), &tx) @@ -213,6 +209,9 @@ func Transaction(ctx *cli.Context) error { results = append(results, r) } + if err := it.Err(); err != nil { + return NewError(ErrorIO, err) + } out, err := json.MarshalIndent(results, "", " ") fmt.Println(string(out)) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index fc52752802..fbd78977cf 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -30,15 +30,22 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/tests" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/database" + "github.com/holiman/uint256" ) const ( @@ -76,10 +83,11 @@ var ( ) type input struct { - Alloc types.GenesisAlloc `json:"alloc,omitempty"` - Env *stEnv `json:"env,omitempty"` - Txs []*txWithKey `json:"txs,omitempty"` - TxRlp string `json:"txsRlp,omitempty"` + Alloc types.GenesisAlloc `json:"alloc,omitempty"` + Env *stEnv `json:"env,omitempty"` + BT map[common.Hash]hexutil.Bytes `json:"vkt,omitempty"` + Txs []*txWithKey `json:"txs,omitempty"` + TxRlp string `json:"txsRlp,omitempty"` } func Transition(ctx *cli.Context) error { @@ -91,16 +99,16 @@ func Transition(ctx *cli.Context) error { // stdin input or in files. // Check if anything needs to be read from stdin var ( - prestate Prestate - txIt txIterator // txs to apply - allocStr = ctx.String(InputAllocFlag.Name) - + prestate Prestate + txIt txIterator // txs to apply + allocStr = ctx.String(InputAllocFlag.Name) + btStr = ctx.String(InputBTFlag.Name) envStr = ctx.String(InputEnvFlag.Name) txStr = ctx.String(InputTxsFlag.Name) inputData = &input{} ) // Figure out the prestate alloc - if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { + if allocStr == stdinSelector || btStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector { decoder := json.NewDecoder(os.Stdin) if err := decoder.Decode(inputData); err != nil { return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err)) @@ -115,6 +123,13 @@ func Transition(ctx *cli.Context) error { prestate.Pre = inputData.Alloc + if btStr != stdinSelector && btStr != "" { + if err := readFile(btStr, "BT", &inputData.BT); err != nil { + return err + } + } + prestate.TreeLeaves = inputData.BT + // Set the block environment if envStr != stdinSelector { var env stEnv @@ -188,9 +203,21 @@ func Transition(ctx *cli.Context) error { return err } // Dump the execution result - collector := make(Alloc) - s.DumpToCollector(collector, nil) - return dispatchOutput(ctx, baseDir, result, collector, body) + var ( + collector = make(Alloc) + btleaves map[common.Hash]hexutil.Bytes + ) + isBinary := chainConfig.IsVerkle(big.NewInt(int64(prestate.Env.Number))) + if !isBinary { + s.DumpToCollector(collector, nil) + } else { + btleaves = make(map[common.Hash]hexutil.Bytes) + if err := s.DumpBinTrieLeaves(btleaves); err != nil { + return err + } + } + + return dispatchOutput(ctx, baseDir, result, collector, body, btleaves) } func applyLondonChecks(env *stEnv, chainConfig *params.ChainConfig) error { @@ -319,7 +346,7 @@ func saveFile(baseDir, filename string, data interface{}) error { // dispatchOutput writes the output data to either stderr or stdout, or to the specified // files -func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes) error { +func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes, bt map[common.Hash]hexutil.Bytes) error { stdOutObject := make(map[string]interface{}) stdErrObject := make(map[string]interface{}) @@ -350,6 +377,12 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a if err := dispatch(baseDir, ctx.String(OutputBodyFlag.Name), "body", body); err != nil { return err } + // Only write bt output if we actually have binary trie leaves + if bt != nil { + if err := dispatch(baseDir, ctx.String(OutputBTFlag.Name), "vkt", bt); err != nil { + return err + } + } if len(stdOutObject) > 0 { b, err := json.MarshalIndent(stdOutObject, "", " ") @@ -373,3 +406,168 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a return nil } + +// BinKey computes the tree key given an address and an optional slot number. +func BinKey(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 2 { + return errors.New("invalid number of arguments: expecting an address and an optional slot number") + } + + addr, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + + if ctx.Args().Len() == 2 { + slot, err := hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + return fmt.Errorf("error decoding slot: %w", err) + } + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyStorageSlot(common.BytesToAddress(addr), slot)) + } else { + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyBasicData(common.BytesToAddress(addr))) + } + return nil +} + +// BinKeys computes a set of tree keys given a genesis alloc. +func BinKeys(ctx *cli.Context) error { + var allocStr = ctx.String(InputAllocFlag.Name) + var alloc core.GenesisAlloc + // Figure out the prestate alloc + if allocStr == stdinSelector { + decoder := json.NewDecoder(os.Stdin) + if err := decoder.Decode(&alloc); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + } + } + if allocStr != stdinSelector { + if err := readFile(allocStr, "alloc", &alloc); err != nil { + return err + } + } + db := triedb.NewDatabase(rawdb.NewMemoryDatabase(), triedb.VerkleDefaults) + defer db.Close() + + bt, err := genBinTrieFromAlloc(alloc, db) + if err != nil { + return fmt.Errorf("error generating bt: %w", err) + } + + collector := make(map[common.Hash]hexutil.Bytes) + it, err := bt.NodeIterator(nil) + if err != nil { + panic(err) + } + for it.Next(true) { + if it.Leaf() { + collector[common.BytesToHash(it.LeafKey())] = it.LeafBlob() + } + } + + output, err := json.MarshalIndent(collector, "", "") + if err != nil { + return fmt.Errorf("error outputting tree: %w", err) + } + + fmt.Println(string(output)) + + return nil +} + +// BinTrieRoot computes the root of a Binary Trie from a genesis alloc. +func BinTrieRoot(ctx *cli.Context) error { + var allocStr = ctx.String(InputAllocFlag.Name) + var alloc core.GenesisAlloc + if allocStr == stdinSelector { + decoder := json.NewDecoder(os.Stdin) + if err := decoder.Decode(&alloc); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + } + } + if allocStr != stdinSelector { + if err := readFile(allocStr, "alloc", &alloc); err != nil { + return err + } + } + db := triedb.NewDatabase(rawdb.NewMemoryDatabase(), triedb.VerkleDefaults) + defer db.Close() + + bt, err := genBinTrieFromAlloc(alloc, db) + if err != nil { + return fmt.Errorf("error generating bt: %w", err) + } + fmt.Println(bt.Hash().Hex()) + + return nil +} + +// TODO(@CPerezz): Should this go to `bintrie` module? +func genBinTrieFromAlloc(alloc core.GenesisAlloc, db database.NodeDatabase) (*bintrie.BinaryTrie, error) { + bt, err := bintrie.NewBinaryTrie(types.EmptyBinaryHash, db) + if err != nil { + return nil, err + } + for addr, acc := range alloc { + for slot, value := range acc.Storage { + err := bt.UpdateStorage(addr, slot.Bytes(), value.Big().Bytes()) + if err != nil { + return nil, fmt.Errorf("error inserting storage: %w", err) + } + } + account := &types.StateAccount{ + Balance: uint256.MustFromBig(acc.Balance), + Nonce: acc.Nonce, + CodeHash: crypto.Keccak256Hash(acc.Code).Bytes(), + Root: common.Hash{}, + } + err := bt.UpdateAccount(addr, account, len(acc.Code)) + if err != nil { + return nil, fmt.Errorf("error inserting account: %w", err) + } + err = bt.UpdateContractCode(addr, common.BytesToHash(account.CodeHash), acc.Code) + if err != nil { + return nil, fmt.Errorf("error inserting code: %w", err) + } + } + return bt, nil +} + +// BinaryCodeChunkKey computes the tree key of a code-chunk for a given address. +func BinaryCodeChunkKey(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 2 { + return errors.New("invalid number of arguments: expecting an address and an code-chunk number") + } + + addr, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + chunkNumberBytes, err := hexutil.Decode(ctx.Args().Get(1)) + if err != nil { + return fmt.Errorf("error decoding chunk number: %w", err) + } + var chunkNumber uint256.Int + chunkNumber.SetBytes(chunkNumberBytes) + + fmt.Printf("%#x\n", bintrie.GetBinaryTreeKeyCodeChunk(common.BytesToAddress(addr), &chunkNumber)) + + return nil +} + +// BinaryCodeChunkCode returns the code chunkification for a given code. +func BinaryCodeChunkCode(ctx *cli.Context) error { + if ctx.Args().Len() == 0 || ctx.Args().Len() > 1 { + return errors.New("invalid number of arguments: expecting a bytecode") + } + + bytecode, err := hexutil.Decode(ctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("error decoding address: %w", err) + } + + chunkedCode := bintrie.ChunkifyCode(bytecode) + fmt.Printf("%#x\n", chunkedCode) + + return nil +} diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 3155221c47..3a5576437d 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -56,6 +56,11 @@ var ( Usage: "benchmark the execution", Category: flags.VMCategory, } + FuzzFlag = &cli.BoolFlag{ + Name: "fuzz", + Usage: "adapts output format for fuzzing", + Category: flags.VMCategory, + } WitnessCrossCheckFlag = &cli.BoolFlag{ Name: "cross-check", Aliases: []string{"xc"}, @@ -147,16 +152,63 @@ var ( t8ntool.TraceEnableCallFramesFlag, t8ntool.OutputBasedir, t8ntool.OutputAllocFlag, + t8ntool.OutputBTFlag, t8ntool.OutputResultFlag, t8ntool.OutputBodyFlag, t8ntool.InputAllocFlag, t8ntool.InputEnvFlag, + t8ntool.InputBTFlag, t8ntool.InputTxsFlag, t8ntool.ForknameFlag, t8ntool.ChainIDFlag, t8ntool.RewardFlag, }, } + + verkleCommand = &cli.Command{ + Name: "verkle", + Aliases: []string{"vkt"}, + Usage: "Binary Trie helpers", + Subcommands: []*cli.Command{ + { + Name: "tree-keys", + Aliases: []string{"v"}, + Usage: "compute a set of binary trie keys, given their source addresses and optional slot numbers", + Action: t8ntool.BinKeys, + Flags: []cli.Flag{ + t8ntool.InputAllocFlag, + }, + }, + { + Name: "single-key", + Aliases: []string{"vk"}, + Usage: "compute the binary trie key given an address and optional slot number", + Action: t8ntool.BinKey, + }, + { + Name: "code-chunk-key", + Aliases: []string{"vck"}, + Usage: "compute the binary trie key given an address and chunk number", + Action: t8ntool.BinaryCodeChunkKey, + }, + { + Name: "chunkify-code", + Aliases: []string{"vcc"}, + Usage: "chunkify a given bytecode for a binary trie", + Action: t8ntool.BinaryCodeChunkCode, + }, + { + Name: "state-root", + Aliases: []string{"vsr"}, + Usage: "compute the state-root of a binary trie for the given alloc", + Action: t8ntool.BinTrieRoot, + Flags: []cli.Flag{ + t8ntool.InputAllocFlag, + }, + }, + }, + } + transactionCommand = &cli.Command{ Name: "transaction", Aliases: []string{"t9n"}, @@ -211,6 +263,7 @@ func init() { stateTransitionCommand, transactionCommand, blockBuilderCommand, + verkleCommand, } app.Before = func(ctx *cli.Context) error { flags.MigrateGlobalFlags(ctx) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index fb22b58e3d..cd9c1546f2 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -121,6 +121,8 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.LogNoHistoryFlag, utils.LogExportCheckpointsFlag, utils.StateHistoryFlag, + utils.TrienodeHistoryFlag, + utils.TrienodeHistoryFullValueCheckpointFlag, }, utils.DatabaseFlags, debug.Flags), Before: func(ctx *cli.Context) error { flags.MigrateGlobalFlags(ctx) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index f6aab1a3a4..7fa01a737f 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -40,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/beacon/blsync" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" @@ -268,11 +267,11 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { // Configure synchronization override service var synctarget common.Hash if ctx.IsSet(utils.SyncTargetFlag.Name) { - hex := hexutil.MustDecode(ctx.String(utils.SyncTargetFlag.Name)) - if len(hex) != common.HashLength { - utils.Fatalf("invalid sync target length: have %d, want %d", len(hex), common.HashLength) + target := ctx.String(utils.SyncTargetFlag.Name) + if !common.IsHexHash(target) { + utils.Fatalf("sync target hash is not a valid hex hash: %s", target) } - synctarget = common.BytesToHash(hex) + synctarget = common.HexToHash(target) } utils.RegisterSyncOverrideService(stack, eth, synctarget, ctx.Bool(utils.ExitWhenSyncedFlag.Name)) @@ -393,9 +392,9 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { ctx.IsSet(utils.MetricsInfluxDBBucketFlag.Name) if enableExport && v2FlagIsSet { - utils.Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2") + utils.Fatalf("Flags --%s, --%s, --%s are only available for influxdb-v2", utils.MetricsInfluxDBOrganizationFlag.Name, utils.MetricsInfluxDBTokenFlag.Name, utils.MetricsInfluxDBBucketFlag.Name) } else if enableExportV2 && v1FlagIsSet { - utils.Fatalf("Flags --influxdb.metrics.username, --influxdb.metrics.password are only available for influxdb-v1") + utils.Fatalf("Flags --%s, --%s are only available for influxdb-v1", utils.MetricsInfluxDBUsernameFlag.Name, utils.MetricsInfluxDBPasswordFlag.Name) } } } diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index d6be6e0ca3..3555b42d45 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -39,8 +39,9 @@ const ( // child g gets a temporary data directory. func runMinimalGeth(t *testing.T, args ...string) *testgeth { // --holesky to make the 'writing genesis to disk' faster (no accounts) + // --networkid=1337 to avoid cache bump // --syncmode=full to avoid allocating fast sync bloom - allArgs := []string{"--holesky", "--authrpc.port", "0", "--syncmode=full", "--port", "0", + allArgs := []string{"--holesky", "--networkid", "1337", "--authrpc.port", "0", "--syncmode=full", "--port", "0", "--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64", "--datadir.minfreedisk", "0"} diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 0e29e7e311..45c7eb101a 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -28,7 +28,6 @@ import ( "syscall" "time" - "github.com/olekukonko/tablewriter" "github.com/urfave/cli/v2" "github.com/ethereum/go-ethereum/cmd/utils" @@ -830,8 +829,7 @@ func showMetaData(ctx *cli.Context) error { data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (%#x)", h.Number, h.Number)}) } - - table := tablewriter.NewWriter(os.Stdout) + table := rawdb.NewTableWriter(os.Stdout) table.SetHeader([]string{"Field", "Value"}) table.AppendBulk(data) table.Render() diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 8e417a7de8..3640da493f 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -103,6 +103,8 @@ var ( utils.LogNoHistoryFlag, utils.LogExportCheckpointsFlag, utils.StateHistoryFlag, + utils.TrienodeHistoryFlag, + utils.TrienodeHistoryFullValueCheckpointFlag, utils.LightKDFFlag, utils.EthRequiredBlocksFlag, utils.LegacyWhitelistFlag, // deprecated @@ -251,7 +253,6 @@ func init() { javascriptCommand, // See misccmd.go: versionCommand, - versionCheckCommand, licenseCommand, // See config.go dumpConfigCommand, @@ -261,8 +262,6 @@ func init() { utils.ShowDeprecated, // See snapshot.go snapshotCommand, - // See verkle.go - verkleCommand, } if logTestCommand != nil { app.Commands = append(app.Commands, logTestCommand) diff --git a/cmd/geth/misccmd.go b/cmd/geth/misccmd.go index 49a8989829..3cc773dc32 100644 --- a/cmd/geth/misccmd.go +++ b/cmd/geth/misccmd.go @@ -28,16 +28,6 @@ import ( ) var ( - VersionCheckUrlFlag = &cli.StringFlag{ - Name: "check.url", - Usage: "URL to use when checking vulnerabilities", - Value: "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json", - } - VersionCheckVersionFlag = &cli.StringFlag{ - Name: "check.version", - Usage: "Version to check", - Value: version.ClientName(clientIdentifier), - } versionCommand = &cli.Command{ Action: printVersion, Name: "version", @@ -45,20 +35,6 @@ var ( ArgsUsage: " ", Description: ` The output of this command is supposed to be machine-readable. -`, - } - versionCheckCommand = &cli.Command{ - Action: versionCheck, - Flags: []cli.Flag{ - VersionCheckUrlFlag, - VersionCheckVersionFlag, - }, - Name: "version-check", - Usage: "Checks (online) for known Geth security vulnerabilities", - ArgsUsage: "", - Description: ` -The version-check command fetches vulnerability-information from https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json, -and displays information about any security vulnerabilities that affect the currently executing version. `, } licenseCommand = &cli.Command{ diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 1b02628e14..faf2b578da 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -693,11 +693,11 @@ func snapshotExportPreimages(ctx *cli.Context) error { var root common.Hash if ctx.NArg() > 1 { - rootBytes := common.FromHex(ctx.Args().Get(1)) - if len(rootBytes) != common.HashLength { + hash := ctx.Args().Get(1) + if !common.IsHexHash(hash) { return fmt.Errorf("invalid hash: %s", ctx.Args().Get(1)) } - root = common.BytesToHash(rootBytes) + root = common.HexToHash(hash) } else { headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { diff --git a/cmd/geth/testdata/vcheck/data.json b/cmd/geth/testdata/vcheck/data.json deleted file mode 100644 index e52fd84e67..0000000000 --- a/cmd/geth/testdata/vcheck/data.json +++ /dev/null @@ -1,202 +0,0 @@ -[ - { - "name": "CorruptedDAG", - "uid": "GETH-2020-01", - "summary": "Mining nodes will generate erroneous PoW on epochs > `385`.", - "description": "A mining flaw could cause miners to erroneously calculate PoW, due to an index overflow, if DAG size is exceeding the maximum 32 bit unsigned value.\n\nThis occurred on the ETC chain on 2020-11-06. This is likely to trigger for ETH mainnet around block `11550000`/epoch `385`, slated to occur early January 2021.\n\nThis issue is relevant only for miners, non-mining nodes are unaffected, since non-mining nodes use a smaller verification cache instead of a full DAG.", - "links": [ - "https://github.com/ethereum/go-ethereum/pull/21793", - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/commit/567d41d9363706b4b13ce0903804e8acf214af49", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-v592-xf75-856p" - ], - "introduced": "v1.6.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Medium", - "CVE": "CVE-2020-26240", - "check": "Geth\\/v1\\.(6|7|8)\\..*|Geth\\/v1\\.9\\.\\d-.*|Geth\\/v1\\.9\\.1.*|Geth\\/v1\\.9\\.2(0|1|2|3)-.*" - }, - { - "name": "Denial of service due to Go CVE-2020-28362", - "uid": "GETH-2020-02", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing, due to an underlying bug in Go (CVE-2020-28362) versions < `1.15.5`, or `<1.14.12`", - "description": "The DoS issue can be used to crash all Geth nodes during block processing, the effects of which would be that a major part of the Ethereum network went offline.\n\nOutside of Go-Ethereum, the issue is most likely relevant for all forks of Geth (such as TurboGeth or ETC’s core-geth) which is built with versions of Go which contains the vulnerability.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://groups.google.com/g/golang-announce/c/NpBGTTmKzpM", - "https://github.com/golang/go/issues/42552", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-m6gx-rhvj-fh52" - ], - "introduced": "v0.0.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-28362", - "check": "Geth.*\\/go1\\.(11(.*)|12(.*)|13(.*)|14|14\\.(\\d|10|11|)|15|15\\.[0-4])$" - }, - { - "name": "ShallowCopy", - "uid": "GETH-2020-03", - "summary": "A consensus flaw in Geth, related to `datacopy` precompile", - "description": "Geth erroneously performed a 'shallow' copy when the precompiled `datacopy` (at `0x00...04`) was invoked. An attacker could deploy a contract that uses the shallow copy to corrupt the contents of the `RETURNDATA`, thus causing a consensus failure.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-69v6-xc2j-r2jf" - ], - "introduced": "v1.9.7", - "fixed": "v1.9.17", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26241", - "check": "Geth\\/v1\\.9\\.(7|8|9|10|11|12|13|14|15|16).*$" - }, - { - "name": "Geth DoS via MULMOD", - "uid": "GETH-2020-04", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing", - "description": "Affected versions suffer from a vulnerability which can be exploited through the `MULMOD` operation, by specifying a modulo of `0`: `mulmod(a,b,0)`, causing a `panic` in the underlying library. \nThe crash was in the `uint256` library, where a buffer [underflowed](https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L442).\n\n\tif `d == 0`, `dLen` remains `0`\n\nand https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L451 will try to access index `[-1]`.\n\nThe `uint256` library was first merged in this [commit](https://github.com/ethereum/go-ethereum/commit/cf6674539c589f80031f3371a71c6a80addbe454), on 2020-06-08. \nExploiting this vulnerabilty would cause all vulnerable nodes to drop off the network. \n\nThe issue was brought to our attention through a [bug report](https://github.com/ethereum/go-ethereum/issues/21367), showing a `panic` occurring on sync from genesis on the Ropsten network.\n \nIt was estimated that the least obvious way to fix this would be to merge the fix into `uint256`, make a new release of that library and then update the geth-dependency.\n", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-jm5c-rv3w-w83m", - "https://github.com/holiman/uint256/releases/tag/v1.1.1", - "https://github.com/holiman/uint256/pull/80", - "https://github.com/ethereum/go-ethereum/pull/21368" - ], - "introduced": "v1.9.16", - "fixed": "v1.9.18", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26242", - "check": "Geth\\/v1\\.9.(16|17).*$" - }, - { - "name": "LES Server DoS via GetProofsV2", - "uid": "GETH-2020-05", - "summary": "A DoS vulnerability can make a LES server crash.", - "description": "A DoS vulnerability can make a LES server crash via malicious GetProofsV2 request from a connected LES client.\n\nThe vulnerability was patched in #21896.\n\nThis vulnerability only concern users explicitly running geth as a light server", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-r33q-22hv-j29q", - "https://github.com/ethereum/go-ethereum/pull/21896" - ], - "introduced": "v1.8.0", - "fixed": "v1.9.25", - "published": "2020-12-10", - "severity": "Medium", - "CVE": "CVE-2020-26264", - "check": "(Geth\\/v1\\.8\\.*)|(Geth\\/v1\\.9\\.\\d-.*)|(Geth\\/v1\\.9\\.1\\d-.*)|(Geth\\/v1\\.9\\.(20|21|22|23|24)-.*)$" - }, - { - "name": "SELFDESTRUCT-recreate consensus flaw", - "uid": "GETH-2020-06", - "introduced": "v1.9.4", - "fixed": "v1.9.20", - "summary": "A consensus-vulnerability in Geth could cause a chain split, where vulnerable versions refuse to accept the canonical chain.", - "description": "A flaw was repoted at 2020-08-11 by John Youngseok Yang (Software Platform Lab), where a particular sequence of transactions could cause a consensus failure.\n\n- Tx 1:\n - `sender` invokes `caller`.\n - `caller` invokes `0xaa`. `0xaa` has 3 wei, does a self-destruct-to-self\n - `caller` does a `1 wei` -call to `0xaa`, who thereby has 1 wei (the code in `0xaa` still executed, since the tx is still ongoing, but doesn't redo the selfdestruct, it takes a different path if callvalue is non-zero)\n\n-Tx 2:\n - `sender` does a 5-wei call to 0xaa. No exec (since no code). \n\nIn geth, the result would be that `0xaa` had `6 wei`, whereas OE reported (correctly) `5` wei. Furthermore, in geth, if the second tx was not executed, the `0xaa` would be destructed, resulting in `0 wei`. Thus obviously wrong. \n\nIt was determined that the root cause was this [commit](https://github.com/ethereum/go-ethereum/commit/223b950944f494a5b4e0957fd9f92c48b09037ad) from [this PR](https://github.com/ethereum/go-ethereum/pull/19953). The semantics of `createObject` was subtly changd, into returning a non-nil object (with `deleted=true`) where it previously did not if the account had been destructed. This return value caused the new object to inherit the old `balance`.\n", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-xw37-57qp-9mm4" - ], - "published": "2020-12-10", - "severity": "High", - "CVE": "CVE-2020-26265", - "check": "(Geth\\/v1\\.9\\.(4|5|6|7|8|9)-.*)|(Geth\\/v1\\.9\\.1\\d-.*)$" - }, - { - "name": "Not ready for London upgrade", - "uid": "GETH-2021-01", - "summary": "The client is not ready for the 'London' technical upgrade, and will deviate from the canonical chain when the London upgrade occurs (at block '12965000' around August 4, 2021.", - "description": "At (or around) August 4, Ethereum will undergo a technical upgrade called 'London'. Clients not upgraded will fail to progress on the canonical chain.", - "links": [ - "https://github.com/ethereum/eth1.0-specs/blob/master/network-upgrades/mainnet-upgrades/london.md", - "https://notes.ethereum.org/@timbeiko/ropsten-postmortem" - ], - "introduced": "v1.10.1", - "fixed": "v1.10.6", - "published": "2021-07-22", - "severity": "High", - "check": "(Geth\\/v1\\.10\\.(1|2|3|4|5)-.*)$" - }, - { - "name": "RETURNDATA corruption via datacopy", - "uid": "GETH-2021-02", - "summary": "A consensus-flaw in the Geth EVM could cause a node to deviate from the canonical chain.", - "description": "A memory-corruption bug within the EVM can cause a consensus error, where vulnerable nodes obtain a different `stateRoot` when processing a maliciously crafted transaction. This, in turn, would lead to the chain being split: mainnet splitting in two forks.\n\nAll Geth versions supporting the London hard fork are vulnerable (the bug is older than London), so all users should update.\n\nThis bug was exploited on Mainnet at block 13107518.\n\nCredits for the discovery go to @guidovranken (working for Sentnl during an audit of the Telos EVM) and reported via bounty@ethereum.org.", - "links": [ - "https://github.com/ethereum/go-ethereum/blob/master/docs/postmortems/2021-08-22-split-postmortem.md", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-9856-9gg9-qcmq", - "https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.8", - "published": "2021-08-24", - "severity": "High", - "CVE": "CVE-2021-39137", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7)-.*)$" - }, - { - "name": "DoS via malicious `snap/1` request", - "uid": "GETH-2021-03", - "summary": "A vulnerable node is susceptible to crash when processing a maliciously crafted message from a peer, via the snap/1 protocol. The crash can be triggered by sending a malicious snap/1 GetTrieNodes package.", - "description": "The `snap/1` protocol handler contains two vulnerabilities related to the `GetTrieNodes` packet, which can be exploited to crash the node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/23657" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.9", - "published": "2021-10-24", - "severity": "Medium", - "CVE": "CVE-2021-41173", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2022-01", - "summary": "A vulnerable node can crash via p2p messages sent from an attacker node, if running with non-default log options.", - "description": "A vulnerable node, if configured to use high verbosity logging, can be made to crash when handling specially crafted p2p messages sent from an attacker node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/24507" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.17", - "published": "2022-05-11", - "severity": "Low", - "CVE": "CVE-2022-29177", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2023-01", - "summary": "A vulnerable node can be made to consume unbounded amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "The p2p handler spawned a new goroutine to respond to ping requests. By flooding a node with ping requests, an unbounded number of goroutines can be created, leading to resource exhaustion and potentially crash due to OOM.", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-ppjg-v974-84cm", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.12.1", - "published": "2023-09-06", - "severity": "High", - "CVE": "CVE-2023-40591", - "check": "(Geth\\/v1\\.(10|11)\\..*)|(Geth\\/v1\\.12\\.0-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2024-01", - "summary": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node. Full details will be available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.13.15", - "published": "2024-05-06", - "severity": "High", - "CVE": "CVE-2024-32972", - "check": "(Geth\\/v1\\.(10|11|12)\\..*)|(Geth\\/v1\\.13\\.\\d-.*)|(Geth\\/v1\\.13\\.1(0|1|2|3|4)-.*)$" - } -] diff --git a/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig b/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig deleted file mode 100644 index 987ffe92bb..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: signature from minisign secret key -RUQkliYstQBOKHklFEYCUjepz81dyUuDmIAxjAvXa+icjGuKcjtVfV06G7qfOMSpplS5EcntU12n+AnGNyuOM8zIctaIWcfG2w0= -trusted comment: timestamp:1752094689 file:data.json hashed -u2e4wo4HBTU6viQTSY/NVBHoWoPFJnnTvLZS0FYl3JdvSOYi6+qpbEsDhAIFqq/n8VmlS/fPqqf7vKCNiAgjAA== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 deleted file mode 100644 index 6b6aa900e3..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.1 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: signature from minisign secret key -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: timestamp:1752094703 file:data.json -cNyq3ZGlqo785HtWODb9ejWqF0HhSeXuLGXzC7z1IhnDrBObWBJngYd3qBG1dQcYlHQ+bgB/On5mSyMFn4UoCQ== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 deleted file mode 100644 index 704437de39..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.2 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: Here's a comment -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: Here's a trusted comment -dL7lO8sqFFCOXJO/u8SgoDk2nlXGWPRDbOTJkChMbmtUp9PB7sG831basXkZ/0CQ/l/vG7AbPyMNEVZyJn5NCg== diff --git a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 b/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 deleted file mode 100644 index 806cd07316..0000000000 --- a/cmd/geth/testdata/vcheck/minisig-sigs/vulnerabilities.json.minisig.3 +++ /dev/null @@ -1,4 +0,0 @@ -untrusted comment: One more (untrusted™) comment -RWQkliYstQBOKNoyq2O98hPmeVJQ6ShQLM58+4n0gkY0y0trFMDAsHuN/l4IyHfh8dDQ1ry0+IuZVrf/i8M/P3YFzFfAymDYCQ0= -trusted comment: Here's a trusted comment -dL7lO8sqFFCOXJO/u8SgoDk2nlXGWPRDbOTJkChMbmtUp9PB7sG831basXkZ/0CQ/l/vG7AbPyMNEVZyJn5NCg== diff --git a/cmd/geth/testdata/vcheck/minisign.pub b/cmd/geth/testdata/vcheck/minisign.pub deleted file mode 100644 index 183dce5f6b..0000000000 --- a/cmd/geth/testdata/vcheck/minisign.pub +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: minisign public key 284E00B52C269624 -RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp diff --git a/cmd/geth/testdata/vcheck/minisign.sec b/cmd/geth/testdata/vcheck/minisign.sec deleted file mode 100644 index 5c50715b20..0000000000 --- a/cmd/geth/testdata/vcheck/minisign.sec +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: minisign encrypted secret key -RWRTY0Iyz8kmPMKrqk6DCtlO9a33akKiaOQG1aLolqDxs52qvPoAAAACAAAAAAAAAEAAAAAArEiggdvyn6+WzTprirLtgiYQoU+ihz/HyGgjhuF+Pz2ddMduyCO+xjCHeq+vgVVW039fbsI8hW6LRGJZLBKV5/jdxCXAVVQE7qTQ6xpEdO0z8Z731/pV1hlspQXG2PNd16NMtwd9dWw= diff --git a/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig b/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig deleted file mode 100644 index d704af7709..0000000000 --- a/cmd/geth/testdata/vcheck/signify-sigs/data.json.sig +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: verify with signifykey.pub -RWSKLNhZb0KdARbMcGN40hbHzKQYZDgDOFhEUT1YpzMnqre/mbKJ8td/HVlG03Am1YCszATiI0DbnljjTy4iNHYwqBfzrFUqUg0= diff --git a/cmd/geth/testdata/vcheck/signifykey.pub b/cmd/geth/testdata/vcheck/signifykey.pub deleted file mode 100644 index 328f973ab4..0000000000 --- a/cmd/geth/testdata/vcheck/signifykey.pub +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: signify public key -RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/ diff --git a/cmd/geth/testdata/vcheck/signifykey.sec b/cmd/geth/testdata/vcheck/signifykey.sec deleted file mode 100644 index 3279a2e58b..0000000000 --- a/cmd/geth/testdata/vcheck/signifykey.sec +++ /dev/null @@ -1,2 +0,0 @@ -untrusted comment: signify secret key -RWRCSwAAACpLQDLawSQCtI7eAVIvaiHzjTsTyJsfV5aKLNhZb0KdAWeICXJGa93/bHAcsY6jUh9I8RdEcDWEoGxmaXZC+IdVBPxDpkix9fBRGEUdKWHi3dOfqME0YRzErWI5AVg3cRw= diff --git a/cmd/geth/testdata/vcheck/vulnerabilities.json b/cmd/geth/testdata/vcheck/vulnerabilities.json deleted file mode 100644 index 28514eadad..0000000000 --- a/cmd/geth/testdata/vcheck/vulnerabilities.json +++ /dev/null @@ -1,202 +0,0 @@ -[ - { - "name": "CorruptedDAG", - "uid": "GETH-2020-01", - "summary": "Mining nodes will generate erroneous PoW on epochs > `385`.", - "description": "A mining flaw could cause miners to erroneously calculate PoW, due to an index overflow, if DAG size is exceeding the maximum 32 bit unsigned value.\n\nThis occurred on the ETC chain on 2020-11-06. This is likely to trigger for ETH mainnet around block `11550000`/epoch `385`, slated to occur early January 2021.\n\nThis issue is relevant only for miners, non-mining nodes are unaffected, since non-mining nodes use a smaller verification cache instead of a full DAG.", - "links": [ - "https://github.com/ethereum/go-ethereum/pull/21793", - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/commit/567d41d9363706b4b13ce0903804e8acf214af49", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-v592-xf75-856p" - ], - "introduced": "v1.6.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Medium", - "CVE": "CVE-2020-26240", - "check": "Geth\\/v1\\.(6|7|8)\\..*|Geth\\/v1\\.9\\.\\d-.*|Geth\\/v1\\.9\\.1.*|Geth\\/v1\\.9\\.2(0|1|2|3)-.*" - }, - { - "name": "Denial of service due to Go CVE-2020-28362", - "uid": "GETH-2020-02", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing, due to an underlying bug in Go (CVE-2020-28362) versions < `1.15.5`, or `<1.14.12`", - "description": "The DoS issue can be used to crash all Geth nodes during block processing, the effects of which would be that a major part of the Ethereum network went offline.\n\nOutside of Go-Ethereum, the issue is most likely relevant for all forks of Geth (such as TurboGeth or ETC’s core-geth) which is built with versions of Go which contains the vulnerability.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://groups.google.com/g/golang-announce/c/NpBGTTmKzpM", - "https://github.com/golang/go/issues/42552", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-m6gx-rhvj-fh52" - ], - "introduced": "v0.0.0", - "fixed": "v1.9.24", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-28362", - "check": "Geth.*\\/go1\\.(11(.*)|12(.*)|13(.*)|14|14\\.(\\d|10|11|)|15|15\\.[0-4])$" - }, - { - "name": "ShallowCopy", - "uid": "GETH-2020-03", - "summary": "A consensus flaw in Geth, related to `datacopy` precompile", - "description": "Geth erroneously performed a 'shallow' copy when the precompiled `datacopy` (at `0x00...04`) was invoked. An attacker could deploy a contract that uses the shallow copy to corrupt the contents of the `RETURNDATA`, thus causing a consensus failure.", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-69v6-xc2j-r2jf" - ], - "introduced": "v1.9.7", - "fixed": "v1.9.17", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26241", - "check": "Geth\\/v1\\.9\\.(7|8|9|10|11|12|13|14|15|16).*$" - }, - { - "name": "Geth DoS via MULMOD", - "uid": "GETH-2020-04", - "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing", - "description": "Affected versions suffer from a vulnerability which can be exploited through the `MULMOD` operation, by specifying a modulo of `0`: `mulmod(a,b,0)`, causing a `panic` in the underlying library. \nThe crash was in the `uint256` library, where a buffer [underflowed](https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L442).\n\n\tif `d == 0`, `dLen` remains `0`\n\nand https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L451 will try to access index `[-1]`.\n\nThe `uint256` library was first merged in this [commit](https://github.com/ethereum/go-ethereum/commit/cf6674539c589f80031f3371a71c6a80addbe454), on 2020-06-08. \nExploiting this vulnerability would cause all vulnerable nodes to drop off the network. \n\nThe issue was brought to our attention through a [bug report](https://github.com/ethereum/go-ethereum/issues/21367), showing a `panic` occurring on sync from genesis on the Ropsten network.\n \nIt was estimated that the least obvious way to fix this would be to merge the fix into `uint256`, make a new release of that library and then update the geth-dependency.\n", - "links": [ - "https://blog.ethereum.org/2020/11/12/geth-security-release", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-jm5c-rv3w-w83m", - "https://github.com/holiman/uint256/releases/tag/v1.1.1", - "https://github.com/holiman/uint256/pull/80", - "https://github.com/ethereum/go-ethereum/pull/21368" - ], - "introduced": "v1.9.16", - "fixed": "v1.9.18", - "published": "2020-11-12", - "severity": "Critical", - "CVE": "CVE-2020-26242", - "check": "Geth\\/v1\\.9.(16|17).*$" - }, - { - "name": "LES Server DoS via GetProofsV2", - "uid": "GETH-2020-05", - "summary": "A DoS vulnerability can make a LES server crash.", - "description": "A DoS vulnerability can make a LES server crash via malicious GetProofsV2 request from a connected LES client.\n\nThe vulnerability was patched in #21896.\n\nThis vulnerability only concern users explicitly running geth as a light server", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-r33q-22hv-j29q", - "https://github.com/ethereum/go-ethereum/pull/21896" - ], - "introduced": "v1.8.0", - "fixed": "v1.9.25", - "published": "2020-12-10", - "severity": "Medium", - "CVE": "CVE-2020-26264", - "check": "(Geth\\/v1\\.8\\.*)|(Geth\\/v1\\.9\\.\\d-.*)|(Geth\\/v1\\.9\\.1\\d-.*)|(Geth\\/v1\\.9\\.(20|21|22|23|24)-.*)$" - }, - { - "name": "SELFDESTRUCT-recreate consensus flaw", - "uid": "GETH-2020-06", - "introduced": "v1.9.4", - "fixed": "v1.9.20", - "summary": "A consensus-vulnerability in Geth could cause a chain split, where vulnerable versions refuse to accept the canonical chain.", - "description": "A flaw was repoted at 2020-08-11 by John Youngseok Yang (Software Platform Lab), where a particular sequence of transactions could cause a consensus failure.\n\n- Tx 1:\n - `sender` invokes `caller`.\n - `caller` invokes `0xaa`. `0xaa` has 3 wei, does a self-destruct-to-self\n - `caller` does a `1 wei` -call to `0xaa`, who thereby has 1 wei (the code in `0xaa` still executed, since the tx is still ongoing, but doesn't redo the selfdestruct, it takes a different path if callvalue is non-zero)\n\n-Tx 2:\n - `sender` does a 5-wei call to 0xaa. No exec (since no code). \n\nIn geth, the result would be that `0xaa` had `6 wei`, whereas OE reported (correctly) `5` wei. Furthermore, in geth, if the second tx was not executed, the `0xaa` would be destructed, resulting in `0 wei`. Thus obviously wrong. \n\nIt was determined that the root cause was this [commit](https://github.com/ethereum/go-ethereum/commit/223b950944f494a5b4e0957fd9f92c48b09037ad) from [this PR](https://github.com/ethereum/go-ethereum/pull/19953). The semantics of `createObject` was subtly changd, into returning a non-nil object (with `deleted=true`) where it previously did not if the account had been destructed. This return value caused the new object to inherit the old `balance`.\n", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-xw37-57qp-9mm4" - ], - "published": "2020-12-10", - "severity": "High", - "CVE": "CVE-2020-26265", - "check": "(Geth\\/v1\\.9\\.(4|5|6|7|8|9)-.*)|(Geth\\/v1\\.9\\.1\\d-.*)$" - }, - { - "name": "Not ready for London upgrade", - "uid": "GETH-2021-01", - "summary": "The client is not ready for the 'London' technical upgrade, and will deviate from the canonical chain when the London upgrade occurs (at block '12965000' around August 4, 2021.", - "description": "At (or around) August 4, Ethereum will undergo a technical upgrade called 'London'. Clients not upgraded will fail to progress on the canonical chain.", - "links": [ - "https://github.com/ethereum/eth1.0-specs/blob/master/network-upgrades/mainnet-upgrades/london.md", - "https://notes.ethereum.org/@timbeiko/ropsten-postmortem" - ], - "introduced": "v1.10.1", - "fixed": "v1.10.6", - "published": "2021-07-22", - "severity": "High", - "check": "(Geth\\/v1\\.10\\.(1|2|3|4|5)-.*)$" - }, - { - "name": "RETURNDATA corruption via datacopy", - "uid": "GETH-2021-02", - "summary": "A consensus-flaw in the Geth EVM could cause a node to deviate from the canonical chain.", - "description": "A memory-corruption bug within the EVM can cause a consensus error, where vulnerable nodes obtain a different `stateRoot` when processing a maliciously crafted transaction. This, in turn, would lead to the chain being split: mainnet splitting in two forks.\n\nAll Geth versions supporting the London hard fork are vulnerable (the bug is older than London), so all users should update.\n\nThis bug was exploited on Mainnet at block 13107518.\n\nCredits for the discovery go to @guidovranken (working for Sentnl during an audit of the Telos EVM) and reported via bounty@ethereum.org.", - "links": [ - "https://github.com/ethereum/go-ethereum/blob/master/docs/postmortems/2021-08-22-split-postmortem.md", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-9856-9gg9-qcmq", - "https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.8", - "published": "2021-08-24", - "severity": "High", - "CVE": "CVE-2021-39137", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7)-.*)$" - }, - { - "name": "DoS via malicious `snap/1` request", - "uid": "GETH-2021-03", - "summary": "A vulnerable node is susceptible to crash when processing a maliciously crafted message from a peer, via the snap/1 protocol. The crash can be triggered by sending a malicious snap/1 GetTrieNodes package.", - "description": "The `snap/1` protocol handler contains two vulnerabilities related to the `GetTrieNodes` packet, which can be exploited to crash the node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/23657" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.9", - "published": "2021-10-24", - "severity": "Medium", - "CVE": "CVE-2021-41173", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2022-01", - "summary": "A vulnerable node can crash via p2p messages sent from an attacker node, if running with non-default log options.", - "description": "A vulnerable node, if configured to use high verbosity logging, can be made to crash when handling specially crafted p2p messages sent from an attacker node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities", - "https://github.com/ethereum/go-ethereum/pull/24507" - ], - "introduced": "v1.10.0", - "fixed": "v1.10.17", - "published": "2022-05-11", - "severity": "Low", - "CVE": "CVE-2022-29177", - "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16)-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2023-01", - "summary": "A vulnerable node can be made to consume unbounded amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "The p2p handler spawned a new goroutine to respond to ping requests. By flooding a node with ping requests, an unbounded number of goroutines can be created, leading to resource exhaustion and potentially crash due to OOM.", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-ppjg-v974-84cm", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.12.1", - "published": "2023-09-06", - "severity": "High", - "CVE": "CVE-2023-40591", - "check": "(Geth\\/v1\\.(10|11)\\..*)|(Geth\\/v1\\.12\\.0-.*)$" - }, - { - "name": "DoS via malicious p2p message", - "uid": "GETH-2024-01", - "summary": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node.", - "description": "A vulnerable node can be made to consume very large amounts of memory when handling specially crafted p2p messages sent from an attacker node. Full details will be available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652)", - "links": [ - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-4xc9-8hmq-j652", - "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities" - ], - "introduced": "v1.10.0", - "fixed": "v1.13.15", - "published": "2024-05-06", - "severity": "High", - "CVE": "CVE-2024-32972", - "check": "(Geth\\/v1\\.(10|11|12)\\..*)|(Geth\\/v1\\.13\\.\\d-.*)|(Geth\\/v1\\.13\\.1(0|1|2|3|4)-.*)$" - } -] diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go deleted file mode 100644 index f8fb487196..0000000000 --- a/cmd/geth/verkle.go +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "bytes" - "encoding/hex" - "errors" - "fmt" - "os" - "slices" - - "github.com/ethereum/go-verkle" - cli "github.com/urfave/cli/v2" - - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/log" -) - -var ( - zero [32]byte - - verkleCommand = &cli.Command{ - Name: "verkle", - Usage: "A set of experimental verkle tree management commands", - Description: "", - Subcommands: []*cli.Command{ - { - Name: "verify", - Usage: "verify the conversion of a MPT into a verkle tree", - ArgsUsage: "", - Action: verifyVerkle, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), - Description: ` -geth verkle verify -This command takes a root commitment and attempts to rebuild the tree. - `, - }, - { - Name: "dump", - Usage: "Dump a verkle tree to a DOT file", - ArgsUsage: " [ ...]", - Action: expandVerkle, - Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), - Description: ` -geth verkle dump [ ...] -This command will produce a dot file representing the tree, rooted at . -in which key1, key2, ... are expanded. - `, - }, - }, - } -) - -// recurse into each child to ensure they can be loaded from the db. The tree isn't rebuilt -// (only its nodes are loaded) so there is no need to flush them, the garbage collector should -// take care of that for us. -func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error { - switch node := root.(type) { - case *verkle.InternalNode: - for i, child := range node.Children() { - childC := child.Commit().Bytes() - - childS, err := resolver(childC[:]) - - if bytes.Equal(childC[:], zero[:]) { - continue - } - - if err != nil { - return fmt.Errorf("could not find child %x in db: %w", childC, err) - } - // depth is set to 0, the tree isn't rebuilt so it's not a problem - childN, err := verkle.ParseNode(childS, 0) - if err != nil { - return fmt.Errorf("decode error child %x in db: %w", child.Commitment().Bytes(), err) - } - - if err := checkChildren(childN, resolver); err != nil { - return fmt.Errorf("%x%w", i, err) // write the path to the erroring node - } - } - case *verkle.LeafNode: - // sanity check: ensure at least one value is non-zero - for i := 0; i < verkle.NodeWidth; i++ { - if len(node.Value(i)) != 0 { - return nil - } - } - return errors.New("both balance and nonce are 0") - case verkle.Empty: - // nothing to do - default: - return fmt.Errorf("unsupported type encountered %v", root) - } - - return nil -} - -func verifyVerkle(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chaindb := utils.MakeChainDatabase(ctx, stack, true, false) - defer chaindb.Close() - headBlock := rawdb.ReadHeadBlock(chaindb) - - if headBlock == nil { - log.Error("Failed to load head block") - return errors.New("no head block") - } - - if ctx.NArg() > 1 { - log.Error("Too many arguments given") - return errors.New("too many arguments") - } - - var ( - rootC common.Hash - err error - ) - - if ctx.NArg() == 1 { - rootC, err = parseRoot(ctx.Args().First()) - if err != nil { - log.Error("Failed to resolve state root", "error", err) - return err - } - - log.Info("Rebuilding the tree", "root", rootC) - } else { - rootC = headBlock.Root() - log.Info("Rebuilding the tree", "root", rootC, "number", headBlock.NumberU64()) - } - - serializedRoot, err := chaindb.Get(rootC[:]) - if err != nil { - return err - } - root, err := verkle.ParseNode(serializedRoot, 0) - if err != nil { - return err - } - - if err := checkChildren(root, chaindb.Get); err != nil { - log.Error("Could not rebuild the tree from the database", "err", err) - return err - } - - log.Info("Tree was rebuilt from the database") - - return nil -} - -func expandVerkle(ctx *cli.Context) error { - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chaindb := utils.MakeChainDatabase(ctx, stack, true, false) - defer chaindb.Close() - var ( - rootC common.Hash - keylist [][]byte - err error - ) - - if ctx.NArg() >= 2 { - rootC, err = parseRoot(ctx.Args().First()) - if err != nil { - log.Error("Failed to resolve state root", "error", err) - return err - } - - keylist = make([][]byte, 0, ctx.Args().Len()-1) - - args := ctx.Args().Slice() - for i := range args[1:] { - key, err := hex.DecodeString(args[i+1]) - log.Info("decoded key", "arg", args[i+1], "key", key) - - if err != nil { - return fmt.Errorf("error decoding key #%d: %w", i+1, err) - } - - keylist = append(keylist, key) - } - - log.Info("Rebuilding the tree", "root", rootC) - } else { - return fmt.Errorf("usage: %s root key1 [key 2...]", ctx.App.Name) - } - - serializedRoot, err := chaindb.Get(rootC[:]) - if err != nil { - return err - } - root, err := verkle.ParseNode(serializedRoot, 0) - if err != nil { - return err - } - - for i, key := range keylist { - log.Info("Reading key", "index", i, "key", key) - root.Get(key, chaindb.Get) - } - - if err := os.WriteFile("dump.dot", []byte(verkle.ToDot(root)), 0600); err != nil { - log.Error("Failed to dump file", "err", err) - } else { - log.Info("Tree was dumped to file", "file", "dump.dot") - } - - return nil -} diff --git a/cmd/geth/version_check.go b/cmd/geth/version_check.go deleted file mode 100644 index 48746ee547..0000000000 --- a/cmd/geth/version_check.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "regexp" - "strings" - - "github.com/jedisct1/go-minisign" - "github.com/urfave/cli/v2" - - "github.com/ethereum/go-ethereum/log" -) - -var gethPubKeys []string = []string{ - //@holiman, minisign public key FB1D084D39BAEC24 - "RWQk7Lo5TQgd+wxBNZM+Zoy+7UhhMHaWKzqoes9tvSbFLJYZhNTbrIjx", - //minisign public key 138B1CA303E51687 - "RWSHFuUDoxyLEzjszuWZI1xStS66QTyXFFZG18uDfO26CuCsbckX1e9J", - //minisign public key FD9813B2D2098484 - "RWSEhAnSshOY/b+GmaiDkObbCWefsAoavjoLcPjBo1xn71yuOH5I+Lts", -} - -type vulnJson struct { - Name string - Uid string - Summary string - Description string - Links []string - Introduced string - Fixed string - Published string - Severity string - Check string - CVE string -} - -func versionCheck(ctx *cli.Context) error { - url := ctx.String(VersionCheckUrlFlag.Name) - version := ctx.String(VersionCheckVersionFlag.Name) - log.Info("Checking vulnerabilities", "version", version, "url", url) - - return checkCurrent(url, version) -} - -func checkCurrent(url, current string) error { - var ( - data []byte - sig []byte - err error - ) - - if data, err = fetch(url); err != nil { - return fmt.Errorf("could not retrieve data: %w", err) - } - - if sig, err = fetch(fmt.Sprintf("%v.minisig", url)); err != nil { - return fmt.Errorf("could not retrieve signature: %w", err) - } - - if err = verifySignature(gethPubKeys, data, sig); err != nil { - return err - } - - var vulns []vulnJson - if err = json.Unmarshal(data, &vulns); err != nil { - return err - } - - allOk := true - - for _, vuln := range vulns { - r, err := regexp.Compile(vuln.Check) - if err != nil { - return err - } - - if r.MatchString(current) { - allOk = false - - fmt.Printf("## Vulnerable to %v (%v)\n\n", vuln.Uid, vuln.Name) - fmt.Printf("Severity: %v\n", vuln.Severity) - fmt.Printf("Summary : %v\n", vuln.Summary) - fmt.Printf("Fixed in: %v\n", vuln.Fixed) - - if len(vuln.CVE) > 0 { - fmt.Printf("CVE: %v\n", vuln.CVE) - } - - if len(vuln.Links) > 0 { - fmt.Printf("References:\n") - - for _, ref := range vuln.Links { - fmt.Printf("\t- %v\n", ref) - } - } - - fmt.Println() - } - } - - if allOk { - fmt.Println("No vulnerabilities found") - } - - return nil -} - -// fetch makes an HTTP request to the given url and returns the response body -func fetch(url string) ([]byte, error) { - if filep := strings.TrimPrefix(url, "file://"); filep != url { - return os.ReadFile(filep) - } - - res, err := http.Get(url) - if err != nil { - return nil, err - } - - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, err - } - - return body, nil -} - -// verifySignature checks that the sigData is a valid signature of the given -// data, for pubkey GethPubkey -func verifySignature(pubkeys []string, data, sigdata []byte) error { - sig, err := minisign.DecodeSignature(string(sigdata)) - if err != nil { - return err - } - // find the used key - var key *minisign.PublicKey - - for _, pubkey := range pubkeys { - pub, err := minisign.NewPublicKey(pubkey) - if err != nil { - // our pubkeys should be parseable - return err - } - - if pub.KeyId != sig.KeyId { - continue - } - - key = &pub - - break - } - - if key == nil { - log.Info("Signing key not trusted", "keyid", keyID(sig.KeyId), "error", err) - return errors.New("signature could not be verified") - } - - if ok, err := key.Verify(data, sig); !ok || err != nil { - log.Info("Verification failed error", "keyid", keyID(key.KeyId), "error", err) - return errors.New("signature could not be verified") - } - - return nil -} - -// keyID turns a binary minisign key ID into a hex string. -// Note: key IDs are printed in reverse byte order. -func keyID(id [8]byte) string { - var rev [8]byte - for i := range id { - rev[len(rev)-1-i] = id[i] - } - - return fmt.Sprintf("%X", rev) -} diff --git a/cmd/geth/version_check_test.go b/cmd/geth/version_check_test.go deleted file mode 100644 index e7db593379..0000000000 --- a/cmd/geth/version_check_test.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "testing" - - "github.com/jedisct1/go-minisign" -) - -func TestVerification(t *testing.T) { - t.Parallel() - // Signatures generated with `minisign`. Legacy format, not pre-hashed file. - t.Run("minisig-legacy", func(t *testing.T) { - t.Parallel() - // For this test, the pubkey is in testdata/vcheck/minisign.pub - // (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' ) - // 1. `minisign -S -l -s ./minisign.sec -m data.json -x ./minisig-sigs/vulnerabilities.json.minisig.1 -c "signature from minisign secret key"` - // 2. `minisign -S -l -s ./minisign.sec -m vulnerabilities.json -x ./minisig-sigs/vulnerabilities.json.minisig.2 -c "Here's a comment" -t "Here's a trusted comment"` - // 3. minisign -S -l -s ./minisign.sec -m vulnerabilities.json -x ./minisig-sigs/vulnerabilities.json.minisig.3 -c "One more (untrusted™) comment" -t "Here's a trusted comment" - pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp" - testVerification(t, pub, "./testdata/vcheck/minisig-sigs/") - }) - t.Run("minisig-new", func(t *testing.T) { - t.Parallel() - // For this test, the pubkey is in testdata/vcheck/minisign.pub - // (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' ) - // `minisign -S -s ./minisign.sec -m data.json -x ./minisig-sigs-new/data.json.minisig` - pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp" - testVerification(t, pub, "./testdata/vcheck/minisig-sigs-new/") - }) - // Signatures generated with `signify-openbsd` - t.Run("signify-openbsd", func(t *testing.T) { - t.Parallel() - t.Skip("This currently fails, minisign expects 4 lines of data, signify provides only 2") - // For this test, the pubkey is in testdata/vcheck/signifykey.pub - // (the privkey is `signifykey.sec`, if we want to expand this test. Password 'test' ) - // `signify -S -s signifykey.sec -m data.json -x ./signify-sigs/data.json.sig` - pub := "RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/" - testVerification(t, pub, "./testdata/vcheck/signify-sigs/") - }) -} - -func testVerification(t *testing.T, pubkey, sigdir string) { - // Data to verify - data, err := os.ReadFile("./testdata/vcheck/data.json") - if err != nil { - t.Fatal(err) - } - // Signatures, with and without comments, both trusted and untrusted - files, err := os.ReadDir(sigdir) - if err != nil { - t.Fatal(err) - } - if len(files) == 0 { - t.Fatal("Missing tests") - } - for _, f := range files { - sig, err := os.ReadFile(filepath.Join(sigdir, f.Name())) - if err != nil { - t.Fatal(err) - } - - err = verifySignature([]string{pubkey}, data, sig) - if err != nil { - t.Fatal(err) - } - } -} - -func versionUint(v string) int { - mustInt := func(s string) int { - a, err := strconv.Atoi(s) - if err != nil { - panic(v) - } - - return a - } - components := strings.Split(strings.TrimPrefix(v, "v"), ".") - a := mustInt(components[0]) - b := mustInt(components[1]) - c := mustInt(components[2]) - - return a*100*100 + b*100 + c -} - -// TestMatching can be used to check that the regexps are correct -func TestMatching(t *testing.T) { - t.Parallel() - data, _ := os.ReadFile("./testdata/vcheck/vulnerabilities.json") - - var vulns []vulnJson - - if err := json.Unmarshal(data, &vulns); err != nil { - t.Fatal(err) - } - - check := func(version string) { - vFull := fmt.Sprintf("Geth/%v-unstable-15339cf1-20201204/linux-amd64/go1.15.4", version) - - for _, vuln := range vulns { - r, err := regexp.Compile(vuln.Check) - vulnIntro := versionUint(vuln.Introduced) - vulnFixed := versionUint(vuln.Fixed) - current := versionUint(version) - - if err != nil { - t.Fatal(err) - } - - if vuln.Name == "Denial of service due to Go CVE-2020-28362" { - // this one is not tied to geth-versions - continue - } - - if vulnIntro <= current && vulnFixed > current { - // Should be vulnerable - if !r.MatchString(vFull) { - t.Errorf("Should be vulnerable, version %v, intro: %v, fixed: %v %v %v", - version, vuln.Introduced, vuln.Fixed, vuln.Name, vuln.Check) - } - } else { - if r.MatchString(vFull) { - t.Errorf("Should not be flagged vulnerable, version %v, intro: %v, fixed: %v %v %d %d %d", - version, vuln.Introduced, vuln.Fixed, vuln.Name, vulnIntro, current, vulnFixed) - } - } - } - } - - for major := 1; major < 2; major++ { - for minor := 0; minor < 30; minor++ { - for patch := 0; patch < 30; patch++ { - vShort := fmt.Sprintf("v%d.%d.%d", major, minor, patch) - check(vShort) - } - } - } -} - -func TestGethPubKeysParseable(t *testing.T) { - t.Parallel() - for _, pubkey := range gethPubKeys { - _, err := minisign.NewPublicKey(pubkey) - if err != nil { - t.Errorf("Should be parseable") - } - } -} - -func TestKeyID(t *testing.T) { - t.Parallel() - type args struct { - id [8]byte - } - - tests := []struct { - name string - args args - want string - }{ - {"@holiman key", args{id: extractKeyId(gethPubKeys[0])}, "FB1D084D39BAEC24"}, - {"second key", args{id: extractKeyId(gethPubKeys[1])}, "138B1CA303E51687"}, - {"third key", args{id: extractKeyId(gethPubKeys[2])}, "FD9813B2D2098484"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - if got := keyID(tt.args.id); got != tt.want { - t.Errorf("keyID() = %v, want %v", got, tt.want) - } - }) - } -} - -func extractKeyId(pubkey string) [8]byte { - p, _ := minisign.NewPublicKey(pubkey) - return p.KeyId -} diff --git a/core/rawdb/database_tablewriter_unix.go b/cmd/keeper/getpayload_wasm.go similarity index 70% rename from core/rawdb/database_tablewriter_unix.go rename to cmd/keeper/getpayload_wasm.go index 8bec5396e8..b912678825 100644 --- a/core/rawdb/database_tablewriter_unix.go +++ b/cmd/keeper/getpayload_wasm.go @@ -1,4 +1,4 @@ -// Copyright 2025 The go-ethereum Authors +// Copyright 2026 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,20 +14,23 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !tinygo -// +build !tinygo +//go:build wasm +// +build wasm -package rawdb +package main import ( - "io" - - "github.com/olekukonko/tablewriter" + "unsafe" ) -// Re-export the real tablewriter types and functions -type Table = tablewriter.Table +//go:wasmimport geth_io len +func hintLen() uint32 + +//go:wasmimport geth_io read +func hintRead(data unsafe.Pointer) -func newTableWriter(w io.Writer) *Table { - return tablewriter.NewWriter(w) +func getInput() []byte { + data := make([]byte, hintLen()) + hintRead(unsafe.Pointer(&data[0])) + return data } diff --git a/cmd/keeper/go.mod b/cmd/keeper/go.mod index ee64e07c73..83527a7b60 100644 --- a/cmd/keeper/go.mod +++ b/cmd/keeper/go.mod @@ -15,15 +15,15 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/gnark-crypto v0.19.2 // indirect github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect - github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/emicklei/dot v1.9.2 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/ethereum/c-kzg-4844/v2 v2.1.5 // indirect github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab // indirect - github.com/ethereum/go-verkle v0.2.2 // indirect github.com/ferranbt/fastssz v0.1.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/gofrs/flock v0.13.0 // indirect github.com/golang/snappy v1.0.0 // indirect @@ -46,6 +46,10 @@ require ( github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect diff --git a/cmd/keeper/go.sum b/cmd/keeper/go.sum index 10ab1ee020..7170ecf042 100644 --- a/cmd/keeper/go.sum +++ b/cmd/keeper/go.sum @@ -32,8 +32,6 @@ github.com/consensys/gnark-crypto v0.19.2 h1:qrEAIXq3T4egxqiliFFoNrepkIWVEeIYwt3 github.com/consensys/gnark-crypto v0.19.2/go.mod h1:rT23F0XSZqE0mUA0+pRtnL56IbPxs6gp4CeRsBk4XS0= github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -52,14 +50,17 @@ github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3 github.com/ethereum/c-kzg-4844/v2 v2.1.5/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= -github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= -github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/getsentry/sentry-go v0.39.0 h1:uhnexj8PNCyCve37GSqxXOeXHh4cJNLNNB4w70Jtgo0= github.com/getsentry/sentry-go v0.39.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -87,8 +88,8 @@ github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= -github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -150,6 +151,14 @@ github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9R github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= diff --git a/cmd/keeper/stubs.go b/cmd/keeper/stubs.go index 04a3bc735b..407a21a145 100644 --- a/cmd/keeper/stubs.go +++ b/cmd/keeper/stubs.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !example && !ziren +//go:build !example && !ziren && !wasm +// +build !example,!ziren,!wasm package main diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e582de2031..2051c4b2c1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -116,7 +116,7 @@ var ( Usage: "Root directory for era1 history (default = inside ancient/chain)", Category: flags.EthCategory, } - MinFreeDiskSpaceFlag = &flags.DirectoryFlag{ + MinFreeDiskSpaceFlag = &cli.IntFlag{ Name: "datadir.minfreedisk", Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", Category: flags.EthCategory, @@ -139,7 +139,7 @@ var ( } NetworkIdFlag = &cli.Uint64Flag{ Name: "networkid", - Usage: "Explicitly set network id (integer)(For testnets: use --sepolia, --holesky, --hoodi instead)", + Usage: "Explicitly set network ID (integer)(For testnets: use --sepolia, --holesky, --hoodi instead)", Value: ethconfig.Defaults.NetworkId, Category: flags.EthCategory, } @@ -310,6 +310,18 @@ var ( Value: ethconfig.Defaults.StateHistory, Category: flags.StateCategory, } + TrienodeHistoryFlag = &cli.Int64Flag{ + Name: "history.trienode", + Usage: "Number of recent blocks to retain trienode history for, only relevant in state.scheme=path (default/negative = disabled, 0 = entire chain)", + Value: ethconfig.Defaults.TrienodeHistory, + Category: flags.StateCategory, + } + TrienodeHistoryFullValueCheckpointFlag = &cli.UintFlag{ + Name: "history.trienode.full-value-checkpoint", + Usage: "The frequency of full-value encoding. Every n-th node is stored in full-value format; all other nodes are stored as diffs relative to their predecessor", + Value: uint(ethconfig.Defaults.NodeFullValueCheckpoint), + Category: flags.StateCategory, + } TransactionHistoryFlag = &cli.Uint64Flag{ Name: "history.transactions", Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", @@ -717,7 +729,6 @@ var ( Usage: "Disables db compaction after import", Category: flags.LoggingCategory, } - // MISC settings SyncTargetFlag = &cli.StringFlag{ Name: "synctarget", @@ -909,14 +920,14 @@ var ( Aliases: []string{"discv4"}, Usage: "Enables the V4 discovery mechanism", Category: flags.NetworkingCategory, - Value: true, + Value: node.DefaultConfig.P2P.DiscoveryV4, } DiscoveryV5Flag = &cli.BoolFlag{ Name: "discovery.v5", Aliases: []string{"discv5"}, Usage: "Enables the V5 discovery mechanism", Category: flags.NetworkingCategory, - Value: true, + Value: node.DefaultConfig.P2P.DiscoveryV5, } NetrestrictFlag = &cli.StringFlag{ Name: "netrestrict", @@ -1455,13 +1466,17 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { cfg.MaxPendingPeers = ctx.Int(MaxPendingPeersFlag.Name) } if ctx.IsSet(NoDiscoverFlag.Name) { - cfg.NoDiscovery = true + cfg.NoDiscovery = ctx.Bool(NoDiscoverFlag.Name) } flags.CheckExclusive(ctx, DiscoveryV4Flag, NoDiscoverFlag) flags.CheckExclusive(ctx, DiscoveryV5Flag, NoDiscoverFlag) - cfg.DiscoveryV4 = ctx.Bool(DiscoveryV4Flag.Name) - cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) + if ctx.IsSet(DiscoveryV4Flag.Name) { + cfg.DiscoveryV4 = ctx.Bool(DiscoveryV4Flag.Name) + } + if ctx.IsSet(DiscoveryV5Flag.Name) { + cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) + } if netrestrict := ctx.String(NetrestrictFlag.Name); netrestrict != "" { list, err := netutil.ParseNetlist(netrestrict) @@ -1530,7 +1545,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { } if ctx.IsSet(DeveloperFlag.Name) { - cfg.UseLightweightKDF = true + cfg.UseLightweightKDF = ctx.Bool(DeveloperFlag.Name) } if ctx.IsSet(LightKDFFlag.Name) { @@ -1748,8 +1763,8 @@ func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { - // Avoid conflicting network flags, don't allow network id override on preset networks - flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag, NetworkIdFlag, OverrideGenesisFlag) + // Avoid conflicting network flags + flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag, OverrideGenesisFlag) flags.CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer // Set configurations from CLI flags @@ -1798,10 +1813,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } } - if ctx.IsSet(NetworkIdFlag.Name) { - cfg.NetworkId = ctx.Uint64(NetworkIdFlag.Name) - } - if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheDatabaseFlag.Name) { cfg.DatabaseCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 } @@ -1827,9 +1838,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(CacheNoPrefetchFlag.Name) { cfg.NoPrefetch = ctx.Bool(CacheNoPrefetchFlag.Name) } - // Read the value from the flag no matter if it's set or not. - cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) - + if ctx.IsSet(CachePreimagesFlag.Name) { + cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) + } if cfg.NoPruning && !cfg.Preimages { cfg.Preimages = true @@ -1838,6 +1849,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateHistoryFlag.Name) { cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) } + if ctx.IsSet(TrienodeHistoryFlag.Name) { + cfg.TrienodeHistory = ctx.Int64(TrienodeHistoryFlag.Name) + } + if ctx.IsSet(TrienodeHistoryFullValueCheckpointFlag.Name) { + cfg.NodeFullValueCheckpoint = uint32(ctx.Uint(TrienodeHistoryFullValueCheckpointFlag.Name)) + } if ctx.IsSet(StateSchemeFlag.Name) { cfg.StateScheme = ctx.String(StateSchemeFlag.Name) } @@ -1863,7 +1880,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.LogHistory = ctx.Uint64(LogHistoryFlag.Name) } if ctx.IsSet(LogNoHistoryFlag.Name) { - cfg.LogNoHistory = true + cfg.LogNoHistory = ctx.Bool(LogNoHistoryFlag.Name) } if ctx.IsSet(LogExportCheckpointsFlag.Name) { cfg.LogExportCheckpoints = ctx.String(LogExportCheckpointsFlag.Name) @@ -2071,11 +2088,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.Genesis = genesis default: - if cfg.NetworkId == 1 { + if ctx.Uint64(NetworkIdFlag.Name) == 1 { SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash) } } - + if ctx.IsSet(NetworkIdFlag.Name) { + // Typically it's best to automatically set the network ID to the chainID, + // by not passing the --networkid flag at all. Emit a warning when set + // explicitly in case overriding the network ID is not the user's intention. + id := ctx.Uint64(NetworkIdFlag.Name) + log.Warn("Setting network ID with command-line flag", "id", id) + cfg.NetworkId = id + } // Set any dangling config values if ctx.String(CryptoKZGFlag.Name) != "gokzg" && ctx.String(CryptoKZGFlag.Name) != "ckzg" { Fatalf("--%s flag must be 'gokzg' or 'ckzg'", CryptoKZGFlag.Name) @@ -2475,15 +2499,18 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh Fatalf("%v", err) } options := &core.BlockChainConfig{ - TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, - NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), - TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, - ArchiveMode: ctx.String(GCModeFlag.Name) == "archive", - TrieTimeLimit: ethconfig.Defaults.TrieTimeout, - SnapshotLimit: ethconfig.Defaults.SnapshotCache, - Preimages: ctx.Bool(CachePreimagesFlag.Name), - StateScheme: scheme, - StateHistory: ctx.Uint64(StateHistoryFlag.Name), + TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, + NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), + TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, + ArchiveMode: ctx.String(GCModeFlag.Name) == "archive", + TrieTimeLimit: ethconfig.Defaults.TrieTimeout, + SnapshotLimit: ethconfig.Defaults.SnapshotCache, + Preimages: ctx.Bool(CachePreimagesFlag.Name), + StateScheme: scheme, + StateHistory: ctx.Uint64(StateHistoryFlag.Name), + TrienodeHistory: ctx.Int64(TrienodeHistoryFlag.Name), + NodeFullValueCheckpoint: uint32(ctx.Uint(TrienodeHistoryFullValueCheckpointFlag.Name)), + // Disable transaction indexing/unindexing. TxLookupLimit: -1, diff --git a/cmd/workload/README.md b/cmd/workload/README.md index 1b84dd05db..ee1d6acbc9 100644 --- a/cmd/workload/README.md +++ b/cmd/workload/README.md @@ -34,4 +34,5 @@ the following commands (in this directory) against a synced mainnet node: > go run . filtergen --queries queries/filter_queries_mainnet.json http://host:8545 > go run . historygen --history-tests queries/history_mainnet.json http://host:8545 > go run . tracegen --trace-tests queries/trace_mainnet.json --trace-start 4000000 --trace-end 4000100 http://host:8545 +> go run . proofgen --proof-tests queries/proof_mainnet.json --proof-states 3000 http://host:8545 ``` diff --git a/cmd/workload/main.go b/cmd/workload/main.go index 6d2bab9378..a525ba0c5a 100644 --- a/cmd/workload/main.go +++ b/cmd/workload/main.go @@ -49,6 +49,7 @@ func init() { historyGenerateCommand, filterGenerateCommand, traceGenerateCommand, + proofGenerateCommand, filterPerfCommand, filterFuzzCommand, } diff --git a/cmd/workload/prooftest.go b/cmd/workload/prooftest.go new file mode 100644 index 0000000000..dcc063d30e --- /dev/null +++ b/cmd/workload/prooftest.go @@ -0,0 +1,105 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "os" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/utesting" + "github.com/urfave/cli/v2" +) + +// proofTest is the content of a state-proof test. +type proofTest struct { + BlockNumbers []uint64 `json:"blockNumbers"` + Addresses [][]common.Address `json:"addresses"` + StorageKeys [][][]string `json:"storageKeys"` + Results [][]common.Hash `json:"results"` +} + +type proofTestSuite struct { + cfg testConfig + tests proofTest + invalidDir string +} + +func newProofTestSuite(cfg testConfig, ctx *cli.Context) *proofTestSuite { + s := &proofTestSuite{ + cfg: cfg, + invalidDir: ctx.String(proofTestInvalidOutputFlag.Name), + } + if err := s.loadTests(); err != nil { + exit(err) + } + return s +} + +func (s *proofTestSuite) loadTests() error { + file, err := s.cfg.fsys.Open(s.cfg.proofTestFile) + if err != nil { + // If not found in embedded FS, try to load it from disk + if !os.IsNotExist(err) { + return err + } + file, err = os.OpenFile(s.cfg.proofTestFile, os.O_RDONLY, 0666) + if err != nil { + return fmt.Errorf("can't open proofTestFile: %v", err) + } + } + defer file.Close() + if err := json.NewDecoder(file).Decode(&s.tests); err != nil { + return fmt.Errorf("invalid JSON in %s: %v", s.cfg.proofTestFile, err) + } + if len(s.tests.BlockNumbers) == 0 { + return fmt.Errorf("proofTestFile %s has no test data", s.cfg.proofTestFile) + } + return nil +} + +func (s *proofTestSuite) allTests() []workloadTest { + return []workloadTest{ + newArchiveWorkloadTest("Proof/GetProof", s.getProof), + } +} + +func (s *proofTestSuite) getProof(t *utesting.T) { + ctx := context.Background() + for i, blockNumber := range s.tests.BlockNumbers { + for j := 0; j < len(s.tests.Addresses[i]); j++ { + res, err := s.cfg.client.Geth.GetProof(ctx, s.tests.Addresses[i][j], s.tests.StorageKeys[i][j], big.NewInt(int64(blockNumber))) + if err != nil { + t.Errorf("State proving fails, blockNumber: %d, address: %x, keys: %v, err: %v\n", blockNumber, s.tests.Addresses[i][j], strings.Join(s.tests.StorageKeys[i][j], " "), err) + continue + } + blob, err := json.Marshal(res) + if err != nil { + t.Fatalf("State proving fails: error %v", err) + continue + } + if crypto.Keccak256Hash(blob) != s.tests.Results[i][j] { + t.Errorf("State proof mismatch, %d, number: %d, address: %x, keys: %v: invalid result", i, blockNumber, s.tests.Addresses[i][j], strings.Join(s.tests.StorageKeys[i][j], " ")) + } + } + } +} diff --git a/cmd/workload/prooftestgen.go b/cmd/workload/prooftestgen.go new file mode 100644 index 0000000000..5d92eea114 --- /dev/null +++ b/cmd/workload/prooftestgen.go @@ -0,0 +1,355 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see + +package main + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "math/rand" + "os" + "path/filepath" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/eth/tracers/native" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/testrand" + "github.com/ethereum/go-ethereum/log" + "github.com/urfave/cli/v2" +) + +var ( + proofGenerateCommand = &cli.Command{ + Name: "proofgen", + Usage: "Generates tests for state proof verification", + ArgsUsage: "", + Action: generateProofTests, + Flags: []cli.Flag{ + proofTestFileFlag, + proofTestResultOutputFlag, + proofTestStatesFlag, + proofTestStartBlockFlag, + proofTestEndBlockFlag, + }, + } + + proofTestFileFlag = &cli.StringFlag{ + Name: "proof-tests", + Usage: "JSON file containing proof test queries", + Value: "proof_tests.json", + Category: flags.TestingCategory, + } + proofTestResultOutputFlag = &cli.StringFlag{ + Name: "proof-output", + Usage: "Folder containing detailed trace output files", + Value: "", + Category: flags.TestingCategory, + } + proofTestStatesFlag = &cli.Int64Flag{ + Name: "proof-states", + Usage: "Number of states to generate proof against", + Value: 10000, + Category: flags.TestingCategory, + } + proofTestInvalidOutputFlag = &cli.StringFlag{ + Name: "proof-invalid", + Usage: "Folder containing the mismatched state proof output files", + Value: "", + Category: flags.TestingCategory, + } + proofTestStartBlockFlag = &cli.Uint64Flag{ + Name: "proof-start", + Usage: "The number of starting block for proof verification (included)", + Category: flags.TestingCategory, + } + proofTestEndBlockFlag = &cli.Uint64Flag{ + Name: "proof-end", + Usage: "The number of ending block for proof verification (excluded)", + Category: flags.TestingCategory, + } +) + +type proofGenerator func(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) + +func genAccountProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + nAccounts int + ctx = context.Background() + start = time.Now() + ) + chainID, err := cli.Eth.ChainID(ctx) + if err != nil { + return nil, nil, nil, err + } + signer := types.LatestSignerForChainID(chainID) + + for { + if nAccounts >= number { + break + } + blockNumber := uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + + block, err := cli.Eth.BlockByNumber(context.Background(), big.NewInt(int64(blockNumber))) + if err != nil { + continue + } + var ( + addresses []common.Address + keys [][]string + gather = func(address common.Address) { + addresses = append(addresses, address) + keys = append(keys, nil) + nAccounts++ + } + ) + for _, tx := range block.Transactions() { + if nAccounts >= number { + break + } + sender, err := signer.Sender(tx) + if err != nil { + log.Error("Failed to resolve the sender address", "hash", tx.Hash(), "err", err) + continue + } + gather(sender) + + if tx.To() != nil { + gather(*tx.To()) + } + } + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, keys) + } + log.Info("Generated tests for account proof", "blocks", len(blockNumbers), "accounts", nAccounts, "elapsed", common.PrettyDuration(time.Since(start))) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genNonExistentAccountProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + total int + ) + for i := 0; i < number/5; i++ { + var ( + addresses []common.Address + keys [][]string + blockNumber = uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + ) + for j := 0; j < 5; j++ { + addresses = append(addresses, testrand.Address()) + keys = append(keys, nil) + } + total += len(addresses) + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, keys) + } + log.Info("Generated tests for non-existing account proof", "blocks", len(blockNumbers), "accounts", total) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genStorageProof(cli *client, startBlock uint64, endBlock uint64, number int) ([]uint64, [][]common.Address, [][][]string, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + + nAccounts int + nStorages int + start = time.Now() + ) + for { + if nAccounts+nStorages >= number { + break + } + blockNumber := uint64(rand.Intn(int(endBlock-startBlock))) + startBlock + + block, err := cli.Eth.BlockByNumber(context.Background(), big.NewInt(int64(blockNumber))) + if err != nil { + continue + } + var ( + addresses []common.Address + slots [][]string + tracer = "prestateTracer" + configBlob, _ = json.Marshal(native.PrestateTracerConfig{ + DiffMode: false, + DisableCode: true, + DisableStorage: false, + }) + ) + for _, tx := range block.Transactions() { + if nAccounts+nStorages >= number { + break + } + if tx.To() == nil { + continue + } + ret, err := cli.Geth.TraceTransaction(context.Background(), tx.Hash(), &tracers.TraceConfig{ + Tracer: &tracer, + TracerConfig: configBlob, + }) + if err != nil { + log.Error("Failed to trace the transaction", "blockNumber", blockNumber, "hash", tx.Hash(), "err", err) + continue + } + blob, err := json.Marshal(ret) + if err != nil { + log.Error("Failed to marshal data", "err", err) + continue + } + var accounts map[common.Address]*types.Account + if err := json.Unmarshal(blob, &accounts); err != nil { + log.Error("Failed to decode trace result", "blockNumber", blockNumber, "hash", tx.Hash(), "err", err) + continue + } + for addr, account := range accounts { + if len(account.Storage) == 0 { + continue + } + addresses = append(addresses, addr) + nAccounts += 1 + + var keys []string + for k := range account.Storage { + keys = append(keys, k.Hex()) + } + nStorages += len(keys) + + var emptyKeys []string + for i := 0; i < 3; i++ { + emptyKeys = append(emptyKeys, testrand.Hash().Hex()) + } + nStorages += len(emptyKeys) + + slots = append(slots, append(keys, emptyKeys...)) + } + } + blockNumbers = append(blockNumbers, blockNumber) + accountAddresses = append(accountAddresses, addresses) + storageKeys = append(storageKeys, slots) + } + log.Info("Generated tests for storage proof", "blocks", len(blockNumbers), "accounts", nAccounts, "storages", nStorages, "elapsed", common.PrettyDuration(time.Since(start))) + return blockNumbers, accountAddresses, storageKeys, nil +} + +func genProofRequests(cli *client, startBlock, endBlock uint64, states int) (*proofTest, error) { + var ( + blockNumbers []uint64 + accountAddresses [][]common.Address + storageKeys [][][]string + ) + ratio := []float64{0.2, 0.1, 0.7} + for i, fn := range []proofGenerator{genAccountProof, genNonExistentAccountProof, genStorageProof} { + numbers, addresses, keys, err := fn(cli, startBlock, endBlock, int(float64(states)*ratio[i])) + if err != nil { + return nil, err + } + blockNumbers = append(blockNumbers, numbers...) + accountAddresses = append(accountAddresses, addresses...) + storageKeys = append(storageKeys, keys...) + } + return &proofTest{ + BlockNumbers: blockNumbers, + Addresses: accountAddresses, + StorageKeys: storageKeys, + }, nil +} + +func generateProofTests(clictx *cli.Context) error { + var ( + client = makeClient(clictx) + ctx = context.Background() + states = clictx.Int(proofTestStatesFlag.Name) + outputFile = clictx.String(proofTestFileFlag.Name) + outputDir = clictx.String(proofTestResultOutputFlag.Name) + startBlock = clictx.Uint64(proofTestStartBlockFlag.Name) + endBlock = clictx.Uint64(proofTestEndBlockFlag.Name) + ) + head, err := client.Eth.BlockNumber(ctx) + if err != nil { + exit(err) + } + if startBlock > head || endBlock > head { + return fmt.Errorf("chain is out of proof range, head %d, start: %d, limit: %d", head, startBlock, endBlock) + } + if endBlock == 0 { + endBlock = head + } + log.Info("Generating proof states", "startBlock", startBlock, "endBlock", endBlock, "states", states) + + test, err := genProofRequests(client, startBlock, endBlock, states) + if err != nil { + exit(err) + } + for i, blockNumber := range test.BlockNumbers { + var hashes []common.Hash + for j := 0; j < len(test.Addresses[i]); j++ { + res, err := client.Geth.GetProof(ctx, test.Addresses[i][j], test.StorageKeys[i][j], big.NewInt(int64(blockNumber))) + if err != nil { + log.Error("Failed to prove the state", "number", blockNumber, "address", test.Addresses[i][j], "slots", len(test.StorageKeys[i][j]), "err", err) + continue + } + blob, err := json.Marshal(res) + if err != nil { + return err + } + hashes = append(hashes, crypto.Keccak256Hash(blob)) + + writeStateProof(outputDir, blockNumber, test.Addresses[i][j], res) + } + test.Results = append(test.Results, hashes) + } + writeJSON(outputFile, test) + return nil +} + +func writeStateProof(dir string, blockNumber uint64, address common.Address, result any) { + if dir == "" { + return + } + // Ensure the directory exists + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + exit(fmt.Errorf("failed to create directories: %w", err)) + } + fname := fmt.Sprintf("%d-%x", blockNumber, address) + name := filepath.Join(dir, fname) + file, err := os.Create(name) + if err != nil { + exit(fmt.Errorf("error creating %s: %v", name, err)) + return + } + defer file.Close() + + data, _ := json.MarshalIndent(result, "", " ") + _, err = file.Write(data) + if err != nil { + exit(fmt.Errorf("error writing %s: %v", name, err)) + return + } +} diff --git a/cmd/workload/testsuite.go b/cmd/workload/testsuite.go index 185f086b28..56ed1203f0 100644 --- a/cmd/workload/testsuite.go +++ b/cmd/workload/testsuite.go @@ -51,7 +51,9 @@ var ( filterQueryFileFlag, historyTestFileFlag, traceTestFileFlag, + proofTestFileFlag, traceTestInvalidOutputFlag, + proofTestInvalidOutputFlag, }, } testPatternFlag = &cli.StringFlag{ @@ -96,6 +98,7 @@ type testConfig struct { historyTestFile string historyPruneBlock *uint64 traceTestFile string + proofTestFile string } var errPrunedHistory = errors.New("attempt to access pruned history") @@ -146,6 +149,12 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { } else { cfg.traceTestFile = "queries/trace_mainnet.json" } + if ctx.IsSet(proofTestFileFlag.Name) { + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) + } else { + cfg.proofTestFile = "queries/proof_mainnet.json" + } + cfg.historyPruneBlock = new(uint64) *cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber case ctx.Bool(testSepoliaFlag.Name): @@ -165,6 +174,12 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { } else { cfg.traceTestFile = "queries/trace_sepolia.json" } + if ctx.IsSet(proofTestFileFlag.Name) { + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) + } else { + cfg.proofTestFile = "queries/proof_sepolia.json" + } + cfg.historyPruneBlock = new(uint64) *cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber default: @@ -172,6 +187,7 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) cfg.historyTestFile = ctx.String(historyTestFileFlag.Name) cfg.traceTestFile = ctx.String(traceTestFileFlag.Name) + cfg.proofTestFile = ctx.String(proofTestFileFlag.Name) } return cfg } @@ -223,11 +239,13 @@ func runTestCmd(ctx *cli.Context) error { filterSuite := newFilterTestSuite(cfg) historySuite := newHistoryTestSuite(cfg) traceSuite := newTraceTestSuite(cfg, ctx) + proofSuite := newProofTestSuite(cfg, ctx) // Filter test cases. tests := filterSuite.allTests() tests = append(tests, historySuite.allTests()...) tests = append(tests, traceSuite.allTests()...) + tests = append(tests, proofSuite.allTests()...) utests := filterTests(tests, ctx.String(testPatternFlag.Name), func(t workloadTest) bool { if t.Slow && !ctx.Bool(testSlowFlag.Name) { diff --git a/common/bitutil/bitutil.go b/common/bitutil/bitutil.go index 1a8b15f00d..578da1cf49 100644 --- a/common/bitutil/bitutil.go +++ b/common/bitutil/bitutil.go @@ -8,6 +8,7 @@ package bitutil import ( + "crypto/subtle" "runtime" "unsafe" ) @@ -17,53 +18,16 @@ const supportsUnaligned = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" | // XORBytes xors the bytes in a and b. The destination is assumed to have enough // space. Returns the number of bytes xor'd. +// +// If dst does not have length at least n, +// XORBytes panics without writing anything to dst. +// +// dst and x or y may overlap exactly or not at all, +// otherwise XORBytes may panic. +// +// Deprecated: use crypto/subtle.XORBytes func XORBytes(dst, a, b []byte) int { - if supportsUnaligned { - return fastXORBytes(dst, a, b) - } - - return safeXORBytes(dst, a, b) -} - -// fastXORBytes xors in bulk. It only works on architectures that support -// unaligned read/writes. -func fastXORBytes(dst, a, b []byte) int { - n := len(a) - if len(b) < n { - n = len(b) - } - - w := n / wordSize - if w > 0 { - dw := *(*[]uintptr)(unsafe.Pointer(&dst)) - aw := *(*[]uintptr)(unsafe.Pointer(&a)) - bw := *(*[]uintptr)(unsafe.Pointer(&b)) - - for i := 0; i < w; i++ { - dw[i] = aw[i] ^ bw[i] - } - } - - for i := n - n%wordSize; i < n; i++ { - dst[i] = a[i] ^ b[i] - } - - return n -} - -// safeXORBytes xors one by one. It works on all architectures, independent if -// it supports unaligned read/writes or not. -func safeXORBytes(dst, a, b []byte) int { - n := len(a) - if len(b) < n { - n = len(b) - } - - for i := 0; i < n; i++ { - dst[i] = a[i] ^ b[i] - } - - return n + return subtle.XORBytes(dst, a, b) } // ANDBytes ands the bytes in a and b. The destination is assumed to have enough @@ -72,7 +36,6 @@ func ANDBytes(dst, a, b []byte) int { if supportsUnaligned { return fastANDBytes(dst, a, b) } - return safeANDBytes(dst, a, b) } @@ -83,22 +46,18 @@ func fastANDBytes(dst, a, b []byte) int { if len(b) < n { n = len(b) } - w := n / wordSize if w > 0 { dw := *(*[]uintptr)(unsafe.Pointer(&dst)) aw := *(*[]uintptr)(unsafe.Pointer(&a)) bw := *(*[]uintptr)(unsafe.Pointer(&b)) - for i := 0; i < w; i++ { dw[i] = aw[i] & bw[i] } } - for i := n - n%wordSize; i < n; i++ { dst[i] = a[i] & b[i] } - return n } @@ -109,11 +68,9 @@ func safeANDBytes(dst, a, b []byte) int { if len(b) < n { n = len(b) } - for i := 0; i < n; i++ { dst[i] = a[i] & b[i] } - return n } @@ -123,7 +80,6 @@ func ORBytes(dst, a, b []byte) int { if supportsUnaligned { return fastORBytes(dst, a, b) } - return safeORBytes(dst, a, b) } @@ -134,22 +90,18 @@ func fastORBytes(dst, a, b []byte) int { if len(b) < n { n = len(b) } - w := n / wordSize if w > 0 { dw := *(*[]uintptr)(unsafe.Pointer(&dst)) aw := *(*[]uintptr)(unsafe.Pointer(&a)) bw := *(*[]uintptr)(unsafe.Pointer(&b)) - for i := 0; i < w; i++ { dw[i] = aw[i] | bw[i] } } - for i := n - n%wordSize; i < n; i++ { dst[i] = a[i] | b[i] } - return n } @@ -160,11 +112,9 @@ func safeORBytes(dst, a, b []byte) int { if len(b) < n { n = len(b) } - for i := 0; i < n; i++ { dst[i] = a[i] | b[i] } - return n } @@ -173,7 +123,6 @@ func TestBytes(p []byte) bool { if supportsUnaligned { return fastTestBytes(p) } - return safeTestBytes(p) } @@ -181,7 +130,6 @@ func TestBytes(p []byte) bool { // support unaligned read/writes. func fastTestBytes(p []byte) bool { n := len(p) - w := n / wordSize if w > 0 { pw := *(*[]uintptr)(unsafe.Pointer(&p)) @@ -191,13 +139,11 @@ func fastTestBytes(p []byte) bool { } } } - for i := n - n%wordSize; i < n; i++ { if p[i] != 0 { return true } } - return false } @@ -209,6 +155,5 @@ func safeTestBytes(p []byte) bool { return true } } - return false } diff --git a/common/bitutil/bitutil_test.go b/common/bitutil/bitutil_test.go index 6908fab218..1748029794 100644 --- a/common/bitutil/bitutil_test.go +++ b/common/bitutil/bitutil_test.go @@ -22,17 +22,14 @@ func TestXOR(t *testing.T) { for i := 0; i < len(p); i++ { p[i] = byte(i) } - for i := 0; i < len(q); i++ { q[i] = byte(len(q) - i) } - d1 := make([]byte, 1023+alignD)[alignD:] d2 := make([]byte, 1023+alignD)[alignD:] XORBytes(d1, p, q) - safeXORBytes(d2, p, q) - + naiveXOR(d2, p, q) if !bytes.Equal(d1, d2) { t.Error("not equal", d1, d2) } @@ -41,6 +38,18 @@ func TestXOR(t *testing.T) { } } +// naiveXOR xors bytes one by one. +func naiveXOR(dst, a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + dst[i] = a[i] ^ b[i] + } + return n +} + // Tests that bitwise AND works for various alignments. func TestAND(t *testing.T) { for alignP := 0; alignP < 2; alignP++ { @@ -52,17 +61,14 @@ func TestAND(t *testing.T) { for i := 0; i < len(p); i++ { p[i] = byte(i) } - for i := 0; i < len(q); i++ { q[i] = byte(len(q) - i) } - d1 := make([]byte, 1023+alignD)[alignD:] d2 := make([]byte, 1023+alignD)[alignD:] ANDBytes(d1, p, q) safeANDBytes(d2, p, q) - if !bytes.Equal(d1, d2) { t.Error("not equal") } @@ -82,17 +88,14 @@ func TestOR(t *testing.T) { for i := 0; i < len(p); i++ { p[i] = byte(i) } - for i := 0; i < len(q); i++ { q[i] = byte(len(q) - i) } - d1 := make([]byte, 1023+alignD)[alignD:] d2 := make([]byte, 1023+alignD)[alignD:] ORBytes(d1, p, q) safeORBytes(d2, p, q) - if !bytes.Equal(d1, d2) { t.Error("not equal") } @@ -143,7 +146,7 @@ func benchmarkBaseXOR(b *testing.B, size int) { p, q := make([]byte, size), make([]byte, size) for i := 0; i < b.N; i++ { - safeXORBytes(p, p, q) + naiveXOR(p, p, q) } } @@ -209,11 +212,9 @@ func BenchmarkFastTest4KB(b *testing.B) { benchmarkFastTest(b, 4096) } func benchmarkFastTest(b *testing.B, size int) { p := make([]byte, size) a := false - for i := 0; i < b.N; i++ { a = a != TestBytes(p) } - GloBool = a // Use of benchmark "result" to prevent total dead code elimination. } @@ -225,10 +226,8 @@ func BenchmarkBaseTest4KB(b *testing.B) { benchmarkBaseTest(b, 4096) } func benchmarkBaseTest(b *testing.B, size int) { p := make([]byte, size) a := false - for i := 0; i < b.N; i++ { a = a != safeTestBytes(p) } - GloBool = a // Use of benchmark "result" to prevent total dead code elimination. } diff --git a/common/size.go b/common/size.go index 097b6304a8..e7f504a082 100644 --- a/common/size.go +++ b/common/size.go @@ -26,13 +26,13 @@ type StorageSize float64 // String implements the stringer interface. func (s StorageSize) String() string { - if s > 1099511627776 { + if s >= 1099511627776 { return fmt.Sprintf("%.2f TiB", s/1099511627776) - } else if s > 1073741824 { + } else if s >= 1073741824 { return fmt.Sprintf("%.2f GiB", s/1073741824) - } else if s > 1048576 { + } else if s >= 1048576 { return fmt.Sprintf("%.2f MiB", s/1048576) - } else if s > 1024 { + } else if s >= 1024 { return fmt.Sprintf("%.2f KiB", s/1024) } else { return fmt.Sprintf("%.2f B", s) @@ -42,13 +42,13 @@ func (s StorageSize) String() string { // TerminalString implements log.TerminalStringer, formatting a string for console // output during logging. func (s StorageSize) TerminalString() string { - if s > 1099511627776 { + if s >= 1099511627776 { return fmt.Sprintf("%.2fTiB", s/1099511627776) - } else if s > 1073741824 { + } else if s >= 1073741824 { return fmt.Sprintf("%.2fGiB", s/1073741824) - } else if s > 1048576 { + } else if s >= 1048576 { return fmt.Sprintf("%.2fMiB", s/1048576) - } else if s > 1024 { + } else if s >= 1024 { return fmt.Sprintf("%.2fKiB", s/1024) } else { return fmt.Sprintf("%.2fB", s) diff --git a/common/types.go b/common/types.go index f9112324c2..bcc822750d 100644 --- a/common/types.go +++ b/common/types.go @@ -29,9 +29,8 @@ import ( "strconv" "strings" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto/keccak" ) // Lengths of hashes and addresses in bytes. @@ -80,6 +79,15 @@ func HexToRefHash(s string) *Hash { return &v } +// IsHexHash verifies whether a string can represent a valid hex-encoded +// Ethereum hash or not. +func IsHexHash(s string) bool { + if has0xPrefix(s) { + s = s[2:] + } + return len(s) == 2*HashLength && isHex(s) +} + // Cmp compares two hashes. func (h Hash) Cmp(other Hash) int { return bytes.Compare(h[:], other[:]) @@ -284,7 +292,7 @@ func (a *Address) checksumHex() []byte { buf := a.hex() // compute checksum - sha := sha3.NewLegacyKeccak256() + sha := keccak.NewLegacyKeccak256() sha.Write(buf[2:]) hash := sha.Sum(nil) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 444c327134..ac2a4c67a3 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -277,11 +277,11 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if !cancun { switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) } } else { if header.ParentBeaconRoot == nil { diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index e03214b42a..80c159bab0 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -27,8 +27,6 @@ import ( "sync" "time" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -41,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -325,11 +324,11 @@ func (c *Clique) verifyHeader(chain consensus.ChainHeaderReader, header *types.H // Verify the non-existence of cancun-specific header fields switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) } // All basic checks passed, verify cascading fields return c.verifyCascadingFields(chain, header, parents) @@ -779,7 +778,7 @@ func (c *Clique) APIs(chain consensus.ChainHeaderReader) []rpc.API { // SealHash returns the hash of a block prior to it being sealed. func SealHash(header *types.Header) (hash common.Hash) { - hasher := sha3.NewLegacyKeccak256() + hasher := keccak.NewLegacyKeccak256() encodeSigHeader(hasher, header) hasher.(crypto.KeccakState).Read(hash[:]) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 58a5509e31..3a64865f2d 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -24,7 +24,6 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/holiman/uint256" - "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -34,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -280,11 +280,11 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, pa // Verify the non-existence of cancun-specific header fields switch { case header.ExcessBlobGas != nil: - return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", *header.ExcessBlobGas) case header.BlobGasUsed != nil: - return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: - return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) } // Add some fake checks for tests if ethash.fakeDelay != nil { @@ -534,7 +534,7 @@ func (ethash *Ethash) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea // SealHash returns the hash of a block prior to it being sealed. func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { - hasher := sha3.NewLegacyKeccak256() + hasher := keccak.NewLegacyKeccak256() enc := []interface{}{ header.ParentHash, diff --git a/consensus/misc/eip4844/eip4844.go b/consensus/misc/eip4844/eip4844.go index edb83501da..217122de41 100644 --- a/consensus/misc/eip4844/eip4844.go +++ b/consensus/misc/eip4844/eip4844.go @@ -55,9 +55,9 @@ func (bc *BlobConfig) blobPrice(excessBlobGas uint64) *big.Int { return new(big.Int).Mul(f, big.NewInt(params.BlobTxBlobGasPerBlob)) } -func latestBlobConfig(cfg *params.ChainConfig, _ uint64) *BlobConfig { +func latestBlobConfig(cfg *params.ChainConfig, _ uint64) (BlobConfig, error) { if cfg.BlobScheduleConfig == nil { - return nil + return BlobConfig{}, errors.New("no blob config") } var ( london = cfg.LondonBlock @@ -72,14 +72,14 @@ func latestBlobConfig(cfg *params.ChainConfig, _ uint64) *BlobConfig { case cfg.IsCancun(london) && s.Cancun != nil: bc = s.Cancun default: - return nil + return BlobConfig{}, errors.New("no blob config") } - return &BlobConfig{ + return BlobConfig{ Target: bc.Target, Max: bc.Max, UpdateFraction: bc.UpdateFraction, - } + }, nil } // VerifyEIP4844Header verifies the presence of the excessBlobGas field and that @@ -90,8 +90,8 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade panic("bad header pair") } - bcfg := latestBlobConfig(config, header.Time) - if bcfg == nil { + bcfg, err := latestBlobConfig(config, header.Time) + if err != nil { panic("called before EIP-4844 is active") } @@ -107,7 +107,7 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, bcfg.maxBlobGas()) } if *header.BlobGasUsed%params.BlobTxBlobGasPerBlob != 0 { - return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob) + return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", *header.BlobGasUsed, params.BlobTxBlobGasPerBlob) } // Verify the excessBlobGas is correct based on the parent header @@ -122,14 +122,14 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade // blobs on top of the excess blob gas. func CalcExcessBlobGas(config *params.ChainConfig, parent *types.Header, headTimestamp uint64) uint64 { isOsaka := config.IsOsaka(config.LondonBlock) - bcfg := latestBlobConfig(config, headTimestamp) - if bcfg == nil { + bcfg, err := latestBlobConfig(config, headTimestamp) + if err != nil { return 0 } return calcExcessBlobGas(isOsaka, bcfg, parent) } -func calcExcessBlobGas(isOsaka bool, bcfg *BlobConfig, parent *types.Header) uint64 { +func calcExcessBlobGas(isOsaka bool, bcfg BlobConfig, parent *types.Header) uint64 { var parentExcessBlobGas, parentBlobGasUsed uint64 if parent.ExcessBlobGas != nil { parentExcessBlobGas = *parent.ExcessBlobGas @@ -150,8 +150,8 @@ func calcExcessBlobGas(isOsaka bool, bcfg *BlobConfig, parent *types.Header) uin // CalcBlobFee calculates the blobfee from the header's excess blob gas field. func CalcBlobFee(config *params.ChainConfig, header *types.Header) *big.Int { - blobConfig := latestBlobConfig(config, header.Time) - if blobConfig == nil { + blobConfig, err := latestBlobConfig(config, header.Time) + if err != nil { panic("calculating blob fee on unsupported fork") } return blobConfig.blobBaseFee(*header.ExcessBlobGas) @@ -159,8 +159,8 @@ func CalcBlobFee(config *params.ChainConfig, header *types.Header) *big.Int { // MaxBlobsPerBlock returns the max blobs per block for a block at the given timestamp. func MaxBlobsPerBlock(cfg *params.ChainConfig, time uint64) int { - blobConfig := latestBlobConfig(cfg, time) - if blobConfig == nil { + blobConfig, err := latestBlobConfig(cfg, time) + if err != nil { return 0 } return blobConfig.Max @@ -174,13 +174,22 @@ func MaxBlobGasPerBlock(cfg *params.ChainConfig, time uint64) uint64 { // LatestMaxBlobsPerBlock returns the latest max blobs per block defined by the // configuration, regardless of the currently active fork. func LatestMaxBlobsPerBlock(cfg *params.ChainConfig) int { - bcfg := latestBlobConfig(cfg, math.MaxUint64) - if bcfg == nil { + bcfg, err := latestBlobConfig(cfg, math.MaxUint64) + if err != nil { return 0 } return bcfg.Max } +// TargetBlobsPerBlock returns the target blobs per block for a block at the given timestamp. +func TargetBlobsPerBlock(cfg *params.ChainConfig, time uint64) int { + blobConfig, err := latestBlobConfig(cfg, time) + if err != nil { + return 0 + } + return blobConfig.Target +} + // fakeExponential approximates factor * e ** (numerator / denominator) using // Taylor expansion. func fakeExponential(factor, numerator, denominator *big.Int) *big.Int { diff --git a/console/prompt/prompter.go b/console/prompt/prompter.go index 4ec465e80b..be3a40a4f1 100644 --- a/console/prompt/prompter.go +++ b/console/prompt/prompter.go @@ -152,7 +152,7 @@ func (p *terminalPrompter) PromptPassword(prompt string) (passwd string, err err // PromptConfirm displays the given prompt to the user and requests a boolean // choice to be made, returning that choice. func (p *terminalPrompter) PromptConfirm(prompt string) (bool, error) { - input, err := p.Prompt(prompt + " [y/n] ") + input, err := p.PromptInput(prompt + " [y/n] ") if len(input) > 0 && strings.EqualFold(input[:1], "y") { return true, nil } diff --git a/core/blockchain.go b/core/blockchain.go index f6873cdb1a..412f10b1eb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -223,6 +223,17 @@ type BlockChainConfig struct { // 0 = unlimited (legacy behavior), default = 1MB/s PreloadRateLimit int64 + // Number of blocks from the chain head for which trienode histories are retained. + // If set to 0, all trienode histories across the entire chain will be retained; + // If set to -1, no trienode history will be retained; + TrienodeHistory int64 + + // The frequency of full-value encoding. For example, a value of 16 means + // that, on average, for a given trie node across its 16 consecutive historical + // versions, only one version is stored in full format, while the others + // are stored in diff mode for storage compression. + NodeFullValueCheckpoint uint32 + // State snapshot related options SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory SnapshotNoBuild bool // Whether the background generation is allowed @@ -320,7 +331,9 @@ func (cfg *BlockChainConfig) triedbConfig(isVerkle bool) *triedb.Config { if cfg.StateScheme == rawdb.PathScheme { config.PathDB = &pathdb.Config{ StateHistory: cfg.StateHistory, + TrienodeHistory: cfg.TrienodeHistory, EnableStateIndexing: cfg.ArchiveMode, + FullValueCheckpoint: cfg.NodeFullValueCheckpoint, TrieCleanSize: cfg.TrieCleanLimit * 1024 * 1024, StateCleanSize: cfg.SnapshotLimit * 1024 * 1024, MaxDiffLayers: cfg.GetTriesInMemory(), diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index 3c097b5909..2d05d34500 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -142,29 +142,6 @@ func (it *insertIterator) next() (*types.Block, error) { return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index]) } -// peek returns the next block in the iterator, along with any potential validation -// error for that block, but does **not** advance the iterator. -// -// Both header and body validation errors (nil too) is cached into the iterator -// to avoid duplicating work on the following next() call. -// nolint:unused -func (it *insertIterator) peek() (*types.Block, error) { - // If we reached the end of the chain, abort - if it.index+1 >= len(it.chain) { - return nil, nil - } - // Wait for verification result if not yet done - if len(it.errors) <= it.index+1 { - it.errors = append(it.errors, <-it.results) - } - - if it.errors[it.index+1] != nil { - return it.chain[it.index+1], it.errors[it.index+1] - } - // Block header valid, ignore body validation since we don't have a parent anyway - return it.chain[it.index+1], nil -} - // previous returns the previous header that was being processed, or nil. func (it *insertIterator) previous() *types.Header { if it.index < 1 { diff --git a/core/chain_makers.go b/core/chain_makers.go index b504b598ed..370feb1e03 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -20,7 +20,6 @@ import ( "fmt" "math/big" - "github.com/ethereum/go-verkle" "github.com/holiman/uint256" "github.com/ethereum/go-ethereum/common" @@ -442,7 +441,11 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } // Forcibly use hash-based state scheme for retaining all nodes in disk. - triedb := triedb.NewDatabase(db, triedb.HashDefaults) + var triedbConfig *triedb.Config = triedb.HashDefaults + if config.IsVerkle(config.ChainID) { + triedbConfig = triedb.VerkleDefaults + } + triedb := triedb.NewDatabase(db, triedbConfig) defer triedb.Close() for i := 0; i < n; i++ { @@ -487,7 +490,11 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // then generate chain on top. func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { db := rawdb.NewMemoryDatabase() - triedb := triedb.NewDatabase(db, triedb.HashDefaults) + var triedbConfig *triedb.Config = triedb.HashDefaults + if genesis.Config != nil && genesis.Config.IsVerkle(genesis.Config.ChainID) { + triedbConfig = triedb.VerkleDefaults + } + triedb := triedb.NewDatabase(db, triedbConfig) defer triedb.Close() _, err := genesis.Commit(db, triedb) if err != nil { @@ -499,118 +506,6 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, return db, blocks, receipts } -func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, trdb *triedb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { - if config == nil { - config = params.TestChainConfig - } - proofs := make([]*verkle.VerkleProof, 0, n) - keyvals := make([]verkle.StateDiff, 0, n) - cm := newChainMaker(parent, config, engine) - - genblock := func(i int, parent *types.Block, triedb *triedb.Database, statedb *state.StateDB) (*types.Block, types.Receipts) { - b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine} - b.header = cm.makeHeader(parent, statedb, b.engine) - - // TODO uncomment when proof generation is merged - // Save pre state for proof generation - // preState := statedb.Copy() - - blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) - blockContext.Random = &common.Hash{} // enable post-merge instruction set - evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) - ProcessParentBlockHash(b.header.ParentHash, evm) - - // Execute any user modifications to the block. - if gen != nil { - gen(i, b) - } - - requests := b.collectRequests(false) - if requests != nil { - reqHash := types.CalcRequestsHash(requests) - b.header.RequestsHash = &reqHash - } - - body := &types.Body{ - Transactions: b.txs, - Uncles: b.uncles, - Withdrawals: b.withdrawals, - } - var block *types.Block - var err error - block, b.receipts, _, err = b.engine.FinalizeAndAssemble(cm, b.header, statedb, body, b.receipts) - if err != nil { - panic(err) - } - - // Write state changes to DB. - root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number), config.IsCancun(b.header.Number)) - if err != nil { - panic(fmt.Sprintf("state write error: %v", err)) - } - if err = triedb.Commit(root, false); err != nil { - panic(fmt.Sprintf("trie write error: %v", err)) - } - - proofs = append(proofs, block.ExecutionWitness().VerkleProof) - keyvals = append(keyvals, block.ExecutionWitness().StateDiff) - - return block, b.receipts - } - - sdb := state.NewDatabase(trdb, nil) - - for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), sdb) - if err != nil { - panic(err) - } - block, receipts := genblock(i, parent, trdb, statedb) - - // Post-process the receipts. - // Here we assign the final block hash and other info into the receipt. - // In order for DeriveFields to work, the transaction and receipt lists need to be - // of equal length. If AddUncheckedTx or AddUncheckedReceipt are used, there will be - // extra ones, so we just trim the lists here. - receiptsCount := len(receipts) - txs := block.Transactions() - if len(receipts) > len(txs) { - receipts = receipts[:len(txs)] - } else if len(receipts) < len(txs) { - txs = txs[:len(receipts)] - } - var blobGasPrice *big.Int - if block.ExcessBlobGas() != nil && cm.config.BlobScheduleConfig != nil { - blobGasPrice = eip4844.CalcBlobFee(cm.config, block.Header()) - } - if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs); err != nil { - panic(err) - } - - // Re-expand to ensure all receipts are returned. - receipts = receipts[:receiptsCount] - - // Advance the chain. - cm.add(block, receipts) - parent = block - } - return cm.chain, cm.receipts, proofs, keyvals -} - -func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (common.Hash, ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { - db := rawdb.NewMemoryDatabase() - cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) - cacheConfig.SnapshotLimit = 0 - triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) - defer triedb.Close() - genesisBlock, err := genesis.Commit(db, triedb) - if err != nil { - panic(err) - } - blocks, receipts, proofs, keyvals := GenerateVerkleChain(genesis.Config, genesisBlock, engine, db, triedb, n, gen) - return genesisBlock.Hash(), db, blocks, receipts, proofs, keyvals -} - func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { time := parent.Time() + 10 // block time is fixed at 10 seconds parentHeader := parent.Header() diff --git a/core/evm.go b/core/evm.go index a74c098e00..0c16464d6d 100644 --- a/core/evm.go +++ b/core/evm.go @@ -118,19 +118,16 @@ func EthereumTransfer(db vm.StateDB, sender, recipient common.Address, amount *u func NewEVMTxContext(msg *Message) vm.TxContext { ctx := vm.TxContext{ Origin: msg.From, - GasPrice: new(big.Int).Set(msg.GasPrice), + GasPrice: uint256.MustFromBig(msg.GasPrice), BlobHashes: msg.BlobHashes, } - if msg.BlobGasFeeCap != nil { - ctx.BlobFeeCap = new(big.Int).Set(msg.BlobGasFeeCap) - } return ctx } // NewEVMTxContextForStateSync returns a minimal TxContext for executing // state-sync transactions. func NewEVMTxContextForStateSync() vm.TxContext { - return vm.TxContext{GasPrice: big.NewInt(0)} + return vm.TxContext{GasPrice: uint256.NewInt(0)} } // GetHashFn returns a GetHashFunc which retrieves header hashes by number diff --git a/core/filtermaps/checkpoints_mainnet.json b/core/filtermaps/checkpoints_mainnet.json index 2ea065ddb7..795967405d 100644 --- a/core/filtermaps/checkpoints_mainnet.json +++ b/core/filtermaps/checkpoints_mainnet.json @@ -288,5 +288,32 @@ {"blockNumber": 22958100, "blockId": "0xe38e0ff7b0c4065ca42ea577bc32f2566ca46f2ddeedcc4bc1f8fb00e7f26329", "firstIndex": 19260242424}, {"blockNumber": 22988600, "blockId": "0x04ca74758b22e0ea54b8c992022ff21c16a2af9c45144c3b0f80de921a7eee82", "firstIndex": 19327351273}, {"blockNumber": 23018392, "blockId": "0x61cc979b00bc97b48356f986a5b9ec997d674bc904c2a2e4b0f17de08e50b3bb", "firstIndex": 19394459627}, -{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073} +{"blockNumber": 23048524, "blockId": "0x489de15d95739ede4ab15e8b5151d80d4dc85ae10e7be800b1a4723094a678df", "firstIndex": 19461570073}, +{"blockNumber": 23078983, "blockId": "0xd64da4fe45a0a349101b8ce1a6336fb099d8b00cc274d0eb59356e134190b8f2", "firstIndex": 19528679292}, +{"blockNumber": 23109224, "blockId": "0xbb6c29f91820fcf6caef881bdfe61eb690f9796f8f139c56eaf27aa601fe5ed2", "firstIndex": 19595787850}, +{"blockNumber": 23136690, "blockId": "0xc1915739edff731d469ec1500ad05102c0d68cc1ef062411d2e9741b8ebdc571", "firstIndex": 19662896969}, +{"blockNumber": 23164324, "blockId": "0x97d0078f2a22a8fbde4660d5f11846d00a669c606e42291df55e863190914a9f", "firstIndex": 19730004997}, +{"blockNumber": 23192975, "blockId": "0x8463749ec09f55fccdd962498f93d6468d744faedc35da8097642cd4609e08f2", "firstIndex": 19797111604}, +{"blockNumber": 23219805, "blockId": "0x76ca51d01b5724f1b0f6c3acb50dfcdb00e1648a475dd7599174b301c25e5517", "firstIndex": 19864222312}, +{"blockNumber": 23236331, "blockId": "0x388f9e36b3ec2120d2b4adfdcdb0c0b8de139eef107e327670dd77fc581c2c5f", "firstIndex": 19931331831}, +{"blockNumber": 23260849, "blockId": "0xe4a467164dbc8f9beebf0c8478a9e7ee16dfce65e2da423b1048601831222ba7", "firstIndex": 19998441286}, +{"blockNumber": 23282795, "blockId": "0x74ad210aa1bfdd4bcf61298ebff622da758c36c38d62d001f2440d09e73ef6c7", "firstIndex": 20065548083}, +{"blockNumber": 23300759, "blockId": "0xa405f5ea21a5207d3cde718a1e3fb7f0ce3dd87ac6040a0db52da0e9488e63f6", "firstIndex": 20132623334}, +{"blockNumber": 23319772, "blockId": "0xafae645dd057af450eddf69c7604bde0136524abc5b7d6697426427ef2d30724", "firstIndex": 20199767615}, +{"blockNumber": 23342113, "blockId": "0x8482c4be13294cfd862d500051c8b4efb95b50f4a79717da6aeabb2a6ff3e199", "firstIndex": 20266874986}, +{"blockNumber": 23366974, "blockId": "0xf6047cafea5da7aaf20691df700d759fe84f5aa2176d9dd42c6ae138899a29ea", "firstIndex": 20333983351}, +{"blockNumber": 23397579, "blockId": "0xe94815fe0278659a26e15b023c0c3877abf6f1265710c5dfddf161ee8af01b40", "firstIndex": 20401094335}, +{"blockNumber": 23425940, "blockId": "0xa98d6d48b93d9ef848c340b47cf9d191f36e76375dd747d8adcb39d72251822d", "firstIndex": 20468200506}, +{"blockNumber": 23452402, "blockId": "0x5f26ff308c0883a04f4a92411bcd05e50538d95a96532452f43b71fde4af908d", "firstIndex": 20535311178}, +{"blockNumber": 23478884, "blockId": "0x23a81186c21218f7034ee81dcd72db71edcbc0e97da26703b8026b6d4a81f693", "firstIndex": 20602384106}, +{"blockNumber": 23507136, "blockId": "0xb1d4a2bd836c5d78b678f0bfffd651bdfeae8b7965639b640f8745193b142323", "firstIndex": 20669529935}, +{"blockNumber": 23535463, "blockId": "0x1886a719a6376352753242ec5093c434938b9a90356facdbdaafd2670da97d82", "firstIndex": 20736637374}, +{"blockNumber": 23564278, "blockId": "0xcdc5a1349f45430771fa2080555f20b80b2d84f575728dbb8bd3b633fbb0a00b", "firstIndex": 20803747399}, +{"blockNumber": 23594711, "blockId": "0xbbf41e407367feeb865b305e38aee8bef84ba62b66efbd6fcd3b16a5d50cc055", "firstIndex": 20870854523}, +{"blockNumber": 23627417, "blockId": "0x1b3c6bd39c5aa7c73101e7e946964b3d7afecf3f253efda8de5610dd99cbee82", "firstIndex": 20937963176}, +{"blockNumber": 23660347, "blockId": "0xadc8ac88c281f50c3aaf1d08149100e8f91a16bbe09b28ac86665bcea681d41b", "firstIndex": 21005072865}, +{"blockNumber": 23692228, "blockId": "0xeb4040468161d9d5b6863eb62e4f21f72a1944f910ecfa95ee6a9dbc39e92ef0", "firstIndex": 21072181753}, +{"blockNumber": 23722331, "blockId": "0xdfc9be1b43488148868da0c8ac56e99e0ffde18bcc418755c4765e745fe74024", "firstIndex": 21139291342}, +{"blockNumber": 23752866, "blockId": "0xb9cf0dfee429a1450f5fb7a237729cf7d0c49e7e35c00dc3709f197c091f8b39", "firstIndex": 21206399901}, +{"blockNumber": 23784485, "blockId": "0x92f5f119078b4ef7ad99273e2ae6f874cfb663e80d741a821e0bb7c25c0369c7", "firstIndex": 21273505141} ] diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index fede54df57..f6b1ef26d0 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -434,7 +434,7 @@ func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, ha lastLogPrinted = start ) switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool { - if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 { + if deleted && (!logPrinted || time.Since(lastLogPrinted) > time.Second*10) { log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start))) logPrinted, lastLogPrinted = true, time.Now() } diff --git a/core/genesis.go b/core/genesis.go index 20e3d8e8a2..f4b3e8008d 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -207,7 +207,7 @@ func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database) (common.Hash, e return common.Hash{}, err } // Commit newly generated states into disk if it's not empty. - if root != types.EmptyRootHash { + if root != emptyRoot { if err := triedb.Commit(root, true); err != nil { return common.Hash{}, err } diff --git a/core/genesis_test.go b/core/genesis_test.go index 0502e252c7..25064331b3 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -331,7 +331,7 @@ func TestVerkleGenesisCommit(t *testing.T) { }, } - expected := common.FromHex("018d20eebb130b5e2b796465fe36aafab650650729a92435aec071bf2386f080") + expected := common.FromHex("b94812c1674dcf4f2bc98f4503d15f4cc674265135bcf3be6e4417b60881042a") got := genesis.ToBlock().Root().Bytes() if !bytes.Equal(got, expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) diff --git a/core/overlay/state_transition.go b/core/overlay/state_transition.go index 67ca0f9671..a52d9139c9 100644 --- a/core/overlay/state_transition.go +++ b/core/overlay/state_transition.go @@ -97,7 +97,7 @@ func LoadTransitionState(db ethdb.KeyValueReader, root common.Hash, isVerkle boo // Initialize the first transition state, with the "ended" // field set to true if the database was created // as a verkle database. - log.Debug("no transition state found, starting fresh", "is verkle", db) + log.Debug("no transition state found, starting fresh", "verkle", isVerkle) // Start with a fresh state ts = &TransitionState{Ended: isVerkle} diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index d8de5a4f26..1a2c78dc31 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -580,7 +580,13 @@ func WriteBodyRLP(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rlp // HasBody verifies the existence of a block body corresponding to the hash. func HasBody(db ethdb.Reader, hash common.Hash, number uint64) bool { if isCanon(db, number, hash) { - return true + // Block is in ancient store, but bodies can be pruned. + // Check if the block number is above the pruning tail. + tail, _ := db.Tail() + if number >= tail { + return true + } + return false } if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { @@ -678,7 +684,13 @@ func DeleteTd(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { // to a block. func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { if isCanon(db, number, hash) { - return true + // Block is in ancient store, but receipts can be pruned. + // Check if the block number is above the pruning tail. + tail, _ := db.Tail() + if number >= tail { + return true + } + return false } if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 28e5b92f1c..a9d737926f 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -27,11 +27,10 @@ import ( "testing" "time" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -58,10 +57,7 @@ func TestHeaderStorage(t *testing.T) { if entry := ReadHeaderRLP(db, header.Hash(), header.Number.Uint64()); entry == nil { t.Fatalf("Stored header RLP not found") } else { - hasher := sha3.NewLegacyKeccak256() - hasher.Write(entry) - - if hash := common.BytesToHash(hasher.Sum(nil)); hash != header.Hash() { + if hash := crypto.Keccak256Hash(entry); hash != header.Hash() { t.Fatalf("Retrieved RLP header mismatch: have %v, want %v", entry, header) } } @@ -79,8 +75,7 @@ func TestBodyStorage(t *testing.T) { // Create a test body to move around the database and make sure it's really new body := &types.Body{Uncles: []*types.Header{{Extra: []byte("test header")}}} - - hasher := sha3.NewLegacyKeccak256() + hasher := keccak.NewLegacyKeccak256() rlp.Encode(hasher, body) hash := common.BytesToHash(hasher.Sum(nil)) @@ -99,10 +94,7 @@ func TestBodyStorage(t *testing.T) { if entry := ReadBodyRLP(db, hash, 0); entry == nil { t.Fatalf("Stored body RLP not found") } else { - hasher := sha3.NewLegacyKeccak256() - hasher.Write(entry) - - if calc := common.BytesToHash(hasher.Sum(nil)); calc != hash { + if calc := crypto.Keccak256Hash(entry); calc != hash { t.Fatalf("Retrieved RLP body mismatch: have %v, want %v", entry, body) } } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 7996b78f26..2b1d0b99b9 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -208,9 +208,6 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans } txIndex := uint64(0) for iter.Next() { - if iter.Err() != nil { - return nil, 0, err - } // The preimage for the hash calculation of legacy transactions // is just their RLP encoding. For typed (EIP-2718) transactions, // which are encoded as byte arrays, the preimage is the content of @@ -232,6 +229,9 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans } txIndex++ } + if iter.Err() != nil { + return nil, 0, iter.Err() + } return nil, 0, errors.New("transaction not found") } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 9b4d300667..e2abe86cce 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -34,14 +34,10 @@ type freezerInfo struct { name string // The identifier of freezer head uint64 // The number of last stored item in the freezer tail uint64 // The number of first stored item in the freezer + count uint64 // The number of stored items in the freezer sizes []tableSize // The storage size per table } -// count returns the number of stored items in the freezer. -func (info *freezerInfo) count() uint64 { - return info.head - info.tail + 1 -} - // size returns the storage size of the entire freezer. func (info *freezerInfo) size() common.StorageSize { var total common.StorageSize @@ -66,7 +62,11 @@ func inspect(name string, order map[string]freezerTableConfig, reader ethdb.Anci if err != nil { return freezerInfo{}, err } - info.head = ancients - 1 + if ancients > 0 { + info.head = ancients - 1 + } else { + info.head = 0 + } // Retrieve the number of first stored item tail, err := reader.Tail() @@ -74,6 +74,12 @@ func inspect(name string, order map[string]freezerTableConfig, reader ethdb.Anci return freezerInfo{}, err } info.tail = tail + + if ancients == 0 { + info.count = 0 + } else { + info.count = info.head - info.tail + 1 + } return info, nil } @@ -147,6 +153,8 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s path, tables = resolveChainFreezerDir(ancient), chainFreezerTableConfigs case MerkleStateFreezerName, VerkleStateFreezerName: path, tables = filepath.Join(ancient, freezerName), stateFreezerTableConfigs + case MerkleTrienodeFreezerName, VerkleTrienodeFreezerName: + path, tables = filepath.Join(ancient, freezerName), trienodeFreezerTableConfigs default: return fmt.Errorf("unknown freezer, supported ones: %v", freezers) } @@ -165,7 +173,7 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s if err != nil { return err } - + defer table.Close() table.dumpIndexStdout(start, end) return nil diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 4c0f5b105a..6276e6a94c 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -106,6 +106,7 @@ func adjustRangeForBor(db ethdb.Database, from uint64) uint64 { type blockTxHashes struct { number uint64 hashes []common.Hash + err error } // iterateTransactions iterates over all transactions in the (canon) block @@ -172,19 +173,22 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool for data := range rlpCh { var body types.Body + var result *blockTxHashes if err := rlp.DecodeBytes(data.rlp, &body); err != nil { log.Warn("Failed to decode block body", "block", data.number, "error", err) - return - } - - var hashes []common.Hash - for _, tx := range body.Transactions { - hashes = append(hashes, tx.Hash()) - } - - result := &blockTxHashes{ - hashes: hashes, - number: data.number, + result = &blockTxHashes{ + number: data.number, + err: err, + } + } else { + hashes := make([]common.Hash, len(body.Transactions)) + for i, tx := range body.Transactions { + hashes[i] = tx.Hash() + } + result = &blockTxHashes{ + hashes: hashes, + number: data.number, + } } // Feed the block to the aggregator, or abort on interrupt select { @@ -253,6 +257,10 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan // Next block available, pop it off and index it delivery := queue.PopItem() lastNum = delivery.number + if delivery.err != nil { + log.Warn("Skipping tx indexing for block with missing/corrupt body", "block", delivery.number, "error", delivery.err) + continue + } WriteTxLookupEntries(batch, delivery.number, delivery.hashes) blocks++ @@ -356,6 +364,10 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch delivery := queue.PopItem() nextNum = delivery.number + 1 + if delivery.err != nil { + log.Warn("Skipping tx unindexing for block with missing/corrupt body", "block", delivery.number, "error", delivery.err) + continue + } DeleteTxLookupEntries(batch, delivery.hashes) txs += len(delivery.hashes) blocks++ @@ -395,9 +407,9 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch } select { case <-interrupt: - logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", nextNum, "elapsed", common.PrettyDuration(time.Since(start))) default: - logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", nextNum, "elapsed", common.PrettyDuration(time.Since(start))) } } diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index dab2c54dd9..2cd427f556 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -236,6 +236,36 @@ func TestIndexTransactions(t *testing.T) { verify(0, 8, false, 8) } +func TestUnindexTransactionsMissingBody(t *testing.T) { + // Construct test chain db + chainDB := NewMemoryDatabase() + blocks, _ := initDatabaseWithTransactions(chainDB) + + // Index the entire chain. + lastBlock := blocks[len(blocks)-1].NumberU64() + IndexTransactions(chainDB, 0, lastBlock+1, nil, false) + + // Prove that block 2 body exists in the database. + if raw := ReadCanonicalBodyRLP(chainDB, 2, nil); len(raw) == 0 { + t.Fatalf("Block 2 body does not exist in the database.") + } + + // Delete body for block 2. This simulates a corrupted database. + key := blockBodyKey(2, blocks[2].Hash()) + if err := chainDB.Delete(key); err != nil { + t.Fatalf("Failed to delete block body %v", err) + } + + // Unindex blocks [0, 3) + UnindexTransactions(chainDB, 0, 3, nil, false) + + // Verify that tx index tail is updated to 3. + tail := ReadTxIndexTail(chainDB) + if tail == nil || *tail != 3 { + t.Fatalf("The tx index tail is wrong: got %v want %d", *tail, 3) + } +} + func TestPruneTransactionIndex(t *testing.T) { chainDB := NewMemoryDatabase() blocks, _ := initDatabaseWithTransactions(chainDB) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 234a373537..2b88809d5a 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -501,6 +501,7 @@ func Open(db ethdb.KeyValueStore, opts OpenOptions) (ethdb.Database, error) { } ethDb := &freezerdb{ + readOnly: opts.ReadOnly, ancientRoot: opts.Ancient, KeyValueStore: db, chainFreezer: frdb, @@ -666,7 +667,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { filterMapBlockLV stat // Path-mode archive data - stateIndex stat + stateIndex stat + trienodeIndex stat // Verkle statistics verkleTries stat @@ -761,8 +763,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { bloomBits.add(size) // Path-based historic state indexes - case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: + case bytes.HasPrefix(key, StateHistoryAccountMetadataPrefix) && len(key) == len(StateHistoryAccountMetadataPrefix)+common.HashLength: stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryStorageMetadataPrefix) && len(key) == len(StateHistoryStorageMetadataPrefix)+2*common.HashLength: + stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryAccountBlockPrefix) && len(key) == len(StateHistoryAccountBlockPrefix)+common.HashLength+4: + stateIndex.add(size) + case bytes.HasPrefix(key, StateHistoryStorageBlockPrefix) && len(key) == len(StateHistoryStorageBlockPrefix)+2*common.HashLength+4: + stateIndex.add(size) + + case bytes.HasPrefix(key, TrienodeHistoryMetadataPrefix) && len(key) >= len(TrienodeHistoryMetadataPrefix)+common.HashLength: + trienodeIndex.add(size) + case bytes.HasPrefix(key, TrienodeHistoryBlockPrefix) && len(key) >= len(TrienodeHistoryBlockPrefix)+common.HashLength+4: + trienodeIndex.add(size) // Verkle trie data is detected, determine the sub-category case bytes.HasPrefix(key, VerklePrefix): @@ -859,12 +872,13 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Path trie state lookups", stateLookups.sizeString(), stateLookups.countString()}, {"Key-Value store", "Path trie account nodes", accountTries.sizeString(), accountTries.countString()}, {"Key-Value store", "Path trie storage nodes", storageTries.sizeString(), storageTries.countString()}, - {"Key-Value store", "Path state history indexes", stateIndex.sizeString(), stateIndex.countString()}, {"Key-Value store", "Verkle trie nodes", verkleTries.sizeString(), verkleTries.countString()}, {"Key-Value store", "Verkle trie state lookups", verkleStateLookups.sizeString(), verkleStateLookups.countString()}, {"Key-Value store", "Trie preimages", preimages.sizeString(), preimages.countString()}, {"Key-Value store", "Account snapshot", accountSnaps.sizeString(), accountSnaps.countString()}, {"Key-Value store", "Storage snapshot", storageSnaps.sizeString(), storageSnaps.countString()}, + {"Key-Value store", "Historical state index", stateIndex.sizeString(), stateIndex.countString()}, + {"Key-Value store", "Historical trie index", trienodeIndex.sizeString(), trienodeIndex.countString()}, {"Key-Value store", "Beacon sync headers", beaconHeaders.sizeString(), beaconHeaders.countString()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.sizeString(), cliqueSnaps.countString()}, {"Key-Value store", "Singleton metadata", metadata.sizeString(), metadata.countString()}, @@ -885,14 +899,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { //nolint:staticcheck strings.Title(table.name), table.size.String(), - fmt.Sprintf("%d", ancient.count()), + fmt.Sprintf("%d", ancient.count), }) } total.Add(uint64(ancient.size())) } - table := newTableWriter(os.Stdout) + table := NewTableWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) table.SetFooter([]string{"", "Total", common.StorageSize(total.Load()).String(), fmt.Sprintf("%d", count.Load())}) table.AppendBulk(stats) @@ -915,7 +929,7 @@ var knownMetadataKeys = [][]byte{ snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey, - filterMapsRangeKey, headStateHistoryIndexKey, bytecodeSyncLastBlockKey, VerkleTransitionStatePrefix, + filterMapsRangeKey, headStateHistoryIndexKey, bytecodeSyncLastBlockKey, headTrienodeHistoryIndexKey, VerkleTransitionStatePrefix, } // printChainMetadata prints out chain metadata to stderr. diff --git a/core/rawdb/database_tablewriter_tinygo.go b/core/rawdb/database_tablewriter.go similarity index 90% rename from core/rawdb/database_tablewriter_tinygo.go rename to core/rawdb/database_tablewriter.go index 2f8e456fd5..5f4054ae47 100644 --- a/core/rawdb/database_tablewriter_tinygo.go +++ b/core/rawdb/database_tablewriter.go @@ -14,10 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// TODO: naive stub implementation for tablewriter - -//go:build tinygo -// +build tinygo +// Naive stub implementation for tablewriter package rawdb @@ -40,7 +37,7 @@ type Table struct { rows [][]string } -func newTableWriter(w io.Writer) *Table { +func NewTableWriter(w io.Writer) *Table { return &Table{out: w} } @@ -89,6 +86,7 @@ func (t *Table) render() error { rowSeparator := t.buildRowSeparator(widths) if len(t.headers) > 0 { + fmt.Fprintln(t.out, rowSeparator) t.printRow(t.headers, widths) fmt.Fprintln(t.out, rowSeparator) } @@ -100,6 +98,7 @@ func (t *Table) render() error { if len(t.footer) > 0 { fmt.Fprintln(t.out, rowSeparator) t.printRow(t.footer, widths) + fmt.Fprintln(t.out, rowSeparator) } return nil @@ -150,15 +149,20 @@ func (t *Table) calculateColumnWidths() []int { } // Find max width needed for data cells in each column + // (explicit comparison: the rawdb package defines a uint64 max that shadows the builtin). for _, row := range t.rows { for i, cell := range row { - widths[i] = max(widths[i], len(cell)) + if l := len(cell); l > widths[i] { + widths[i] = l + } } } // Find max width needed for footer in each column for i, f := range t.footer { - widths[i] = max(widths[i], len(f)) + if l := len(f); l > widths[i] { + widths[i] = l + } } for i := range widths { @@ -172,21 +176,22 @@ func (t *Table) calculateColumnWidths() []int { // // It generates a string with dashes (-) for each column width, joined by plus signs (+). // -// Example output: "----------+--------+-----------" +// Example output: "+----------+--------+-----------+" func (t *Table) buildRowSeparator(widths []int) string { parts := make([]string, len(widths)) for i, w := range widths { parts[i] = strings.Repeat("-", w) } - return strings.Join(parts, "+") + return "+" + strings.Join(parts, "+") + "+" } // printRow outputs a single row to the table writer. // // Each cell is padded with spaces and separated by pipe characters (|). // -// Example output: " Database | Size | Items " +// Example output: "| Database | Size | Items |" func (t *Table) printRow(row []string, widths []int) { + fmt.Fprintf(t.out, "|") for i, cell := range row { if i > 0 { fmt.Fprint(t.out, "|") @@ -204,5 +209,6 @@ func (t *Table) printRow(row []string, widths []int) { fmt.Fprintf(t.out, "%s%s%s", leftPadding, cell, rightPadding) } + fmt.Fprintf(t.out, "|") fmt.Fprintln(t.out) } diff --git a/core/rawdb/database_tablewriter_tinygo_test.go b/core/rawdb/database_tablewriter_test.go similarity index 94% rename from core/rawdb/database_tablewriter_tinygo_test.go rename to core/rawdb/database_tablewriter_test.go index 3bcf93832b..e9de5d8ce8 100644 --- a/core/rawdb/database_tablewriter_tinygo_test.go +++ b/core/rawdb/database_tablewriter_test.go @@ -14,9 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build tinygo -// +build tinygo - package rawdb import ( @@ -27,7 +24,7 @@ import ( func TestTableWriterTinyGo(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) headers := []string{"Database", "Size", "Items", "Status"} rows := [][]string{ @@ -51,7 +48,7 @@ func TestTableWriterValidationErrors(t *testing.T) { // Test missing headers t.Run("MissingHeaders", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) rows := [][]string{{"x", "y", "z"}} @@ -66,7 +63,7 @@ func TestTableWriterValidationErrors(t *testing.T) { t.Run("NotEnoughRowColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) headers := []string{"A", "B", "C"} badRows := [][]string{ @@ -85,7 +82,7 @@ func TestTableWriterValidationErrors(t *testing.T) { t.Run("TooManyRowColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) headers := []string{"A", "B", "C"} badRows := [][]string{ @@ -105,7 +102,7 @@ func TestTableWriterValidationErrors(t *testing.T) { // Test mismatched footer columns t.Run("MismatchedFooterColumns", func(t *testing.T) { var buf bytes.Buffer - table := newTableWriter(&buf) + table := NewTableWriter(&buf) headers := []string{"A", "B", "C"} rows := [][]string{{"x", "y", "z"}} diff --git a/core/rawdb/eradb/eradb.go b/core/rawdb/eradb/eradb.go index 29e658798e..a552b94da9 100644 --- a/core/rawdb/eradb/eradb.go +++ b/core/rawdb/eradb/eradb.go @@ -303,6 +303,7 @@ func (db *Store) openEraFile(epoch uint64) (*era.Era, error) { } // Sanity-check start block. if e.Start()%uint64(era.MaxEra1Size) != 0 { + e.Close() return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxEra1Size) } log.Debug("Opened era1 file", "epoch", epoch) diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go index 3cc5ca29af..17e3a2d9ae 100644 --- a/core/rawdb/freezer_batch.go +++ b/core/rawdb/freezer_batch.go @@ -54,12 +54,18 @@ func newFreezerBatch(f *Freezer) *freezerBatch { // Append adds an RLP-encoded item of the given kind. func (batch *freezerBatch) Append(kind string, num uint64, item interface{}) error { - return batch.tables[kind].Append(num, item) + if table := batch.tables[kind]; table != nil { + return table.Append(num, item) + } + return errUnknownTable } // AppendRaw adds an item of the given kind. func (batch *freezerBatch) AppendRaw(kind string, num uint64, item []byte) error { - return batch.tables[kind].AppendRaw(num, item) + if table := batch.tables[kind]; table != nil { + return table.AppendRaw(num, item) + } + return errUnknownTable } // reset initializes the batch. diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index 96740d0c6e..962340cbaf 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -92,6 +92,13 @@ func (t *memoryTable) truncateHead(items uint64) error { if items < t.offset { return errors.New("truncation below tail") } + for i := int(items - t.offset); i < len(t.data); i++ { + if t.size > uint64(len(t.data[i])) { + t.size -= uint64(len(t.data[i])) + } else { + t.size = 0 + } + } t.data = t.data[:items-t.offset] t.items = items return nil @@ -109,6 +116,13 @@ func (t *memoryTable) truncateTail(items uint64) error { if t.items < items { return errors.New("truncation above head") } + for i := uint64(0); i < items-t.offset; i++ { + if t.size > uint64(len(t.data[i])) { + t.size -= uint64(len(t.data[i])) + } else { + t.size = 0 + } + } t.data = t.data[items-t.offset:] t.offset = items return nil diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index fb9b51f24a..38bc3cc390 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -239,16 +239,12 @@ func cleanup(path string) error { if err != nil { return err } + defer dir.Close() names, err := dir.Readdirnames(0) if err != nil { return err } - - if cerr := dir.Close(); cerr != nil { - return cerr - } - for _, name := range names { if name == filepath.Base(path)+tmpSuffix { log.Info("Removed leftover freezer directory", "name", name) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index f5fdaa81c4..b53d8daa17 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -1267,8 +1267,7 @@ func (t *freezerTable) sizeNolock() (uint64, error) { } // advanceHead should be called when the current head file would outgrow the file limits, -// and a new file must be opened. The caller of this method must hold the write-lock -// before calling this method. +// and a new file must be opened. This method acquires the write-lock internally. func (t *freezerTable) advanceHead() error { t.lock.Lock() defer t.lock.Unlock() @@ -1291,7 +1290,9 @@ func (t *freezerTable) advanceHead() error { } t.releaseFile(t.headId) - t.openFile(t.headId, openFreezerFileForReadOnly) + if _, err := t.openFile(t.headId, openFreezerFileForReadOnly); err != nil { + return err + } // Swap out the current head. t.head = newHead diff --git a/core/rlp_test.go b/core/rlp_test.go index 79246c207e..090ed95990 100644 --- a/core/rlp_test.go +++ b/core/rlp_test.go @@ -21,12 +21,11 @@ import ( "math/big" "testing" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -162,7 +161,7 @@ func BenchmarkHashing(b *testing.B) { var got common.Hash - var hasher = sha3.NewLegacyKeccak256() + var hasher = keccak.NewLegacyKeccak256() b.Run("iteratorhashing", func(b *testing.B) { for b.Loop() { diff --git a/core/state/access_events.go b/core/state/access_events.go index 109767bfa2..86f44bd623 100644 --- a/core/state/access_events.go +++ b/core/state/access_events.go @@ -20,12 +20,11 @@ import ( "maps" gomath "math" - "github.com/holiman/uint256" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/holiman/uint256" ) // mode specifies how a tree location has been accessed @@ -46,15 +45,12 @@ var zeroTreeIndex uint256.Int type AccessEvents struct { branches map[branchAccessKey]mode chunks map[chunkAccessKey]mode - - pointCache *utils.PointCache } -func NewAccessEvents(pointCache *utils.PointCache) *AccessEvents { +func NewAccessEvents() *AccessEvents { return &AccessEvents{ - branches: make(map[branchAccessKey]mode), - chunks: make(map[chunkAccessKey]mode), - pointCache: pointCache, + branches: make(map[branchAccessKey]mode), + chunks: make(map[chunkAccessKey]mode), } } @@ -76,8 +72,11 @@ func (ae *AccessEvents) Keys() [][]byte { // TODO: consider if parallelizing this is worth it, probably depending on len(ae.chunks). keys := make([][]byte, 0, len(ae.chunks)) for chunk := range ae.chunks { - basePoint := ae.pointCache.Get(chunk.addr[:]) - key := utils.GetTreeKeyWithEvaluatedAddress(basePoint, &chunk.treeIndex, chunk.leafKey) + var offset [32]byte + treeIndexBytes := chunk.treeIndex.Bytes32() + copy(offset[:31], treeIndexBytes[1:]) + offset[31] = chunk.leafKey + key := bintrie.GetBinaryTreeKey(chunk.addr, offset[:]) keys = append(keys, key) } return keys @@ -85,9 +84,8 @@ func (ae *AccessEvents) Keys() [][]byte { func (ae *AccessEvents) Copy() *AccessEvents { cpy := &AccessEvents{ - branches: maps.Clone(ae.branches), - chunks: maps.Clone(ae.chunks), - pointCache: ae.pointCache, + branches: maps.Clone(ae.branches), + chunks: maps.Clone(ae.chunks), } return cpy } @@ -96,12 +94,12 @@ func (ae *AccessEvents) Copy() *AccessEvents { // member fields of an account. func (ae *AccessEvents) AddAccount(addr common.Address, isWrite bool, availableGas uint64) uint64 { var gas uint64 // accumulate the consumed gas - consumed, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, isWrite, availableGas) + consumed, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, isWrite, availableGas) if consumed < expected { return expected } gas += consumed - consumed, expected = ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, isWrite, availableGas-consumed) + consumed, expected = ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, isWrite, availableGas-consumed) if consumed < expected { return expected + gas } @@ -113,7 +111,7 @@ func (ae *AccessEvents) AddAccount(addr common.Address, isWrite bool, availableG // cold member fields of an account, that need to be touched when making a message // call to that account. func (ae *AccessEvents) MessageCallGas(destination common.Address, availableGas uint64) uint64 { - _, expected := ae.touchAddressAndChargeGas(destination, zeroTreeIndex, utils.BasicDataLeafKey, false, availableGas) + _, expected := ae.touchAddressAndChargeGas(destination, zeroTreeIndex, bintrie.BasicDataLeafKey, false, availableGas) if expected == 0 { expected = params.WarmStorageReadCostEIP2929 } @@ -123,11 +121,11 @@ func (ae *AccessEvents) MessageCallGas(destination common.Address, availableGas // ValueTransferGas returns the gas to be charged for each of the currently // cold balance member fields of the caller and the callee accounts. func (ae *AccessEvents) ValueTransferGas(callerAddr, targetAddr common.Address, availableGas uint64) uint64 { - _, expected1 := ae.touchAddressAndChargeGas(callerAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas) + _, expected1 := ae.touchAddressAndChargeGas(callerAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas) if expected1 > availableGas { return expected1 } - _, expected2 := ae.touchAddressAndChargeGas(targetAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas-expected1) + _, expected2 := ae.touchAddressAndChargeGas(targetAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas-expected1) if expected1+expected2 == 0 { return params.WarmStorageReadCostEIP2929 } @@ -139,8 +137,8 @@ func (ae *AccessEvents) ValueTransferGas(callerAddr, targetAddr common.Address, // address collision is done before the transfer, and so no write // are guaranteed to happen at this point. func (ae *AccessEvents) ContractCreatePreCheckGas(addr common.Address, availableGas uint64) uint64 { - consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, false, availableGas) - _, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, false, availableGas-consumed) + consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, false, availableGas) + _, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, false, availableGas-consumed) return expected1 + expected2 } @@ -148,9 +146,9 @@ func (ae *AccessEvents) ContractCreatePreCheckGas(addr common.Address, available // a contract creation. func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, availableGas uint64) (uint64, uint64) { var gas uint64 - consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, true, availableGas) + consumed, expected1 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, availableGas) gas += consumed - consumed, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, true, availableGas-consumed) + consumed, expected2 := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, true, availableGas-consumed) gas += consumed return gas, expected1 + expected2 } @@ -158,20 +156,20 @@ func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, availableGas // AddTxOrigin adds the member fields of the sender account to the access event list, // so that cold accesses are not charged, since they are covered by the 21000 gas. func (ae *AccessEvents) AddTxOrigin(originAddr common.Address) { - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.BasicDataLeafKey, true, gomath.MaxUint64) - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.CodeHashLeafKey, false, gomath.MaxUint64) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, bintrie.BasicDataLeafKey, true, gomath.MaxUint64) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, bintrie.CodeHashLeafKey, false, gomath.MaxUint64) } // AddTxDestination adds the member fields of the sender account to the access event list, // so that cold accesses are not charged, since they are covered by the 21000 gas. func (ae *AccessEvents) AddTxDestination(addr common.Address, sendsValue, doesntExist bool) { - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, sendsValue, gomath.MaxUint64) - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, doesntExist, gomath.MaxUint64) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, sendsValue, gomath.MaxUint64) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, doesntExist, gomath.MaxUint64) } // SlotGas returns the amount of gas to be charged for a cold storage access. func (ae *AccessEvents) SlotGas(addr common.Address, slot common.Hash, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - treeIndex, subIndex := utils.StorageIndex(slot.Bytes()) + treeIndex, subIndex := bintrie.StorageIndex(slot.Bytes()) _, expected := ae.touchAddressAndChargeGas(addr, *treeIndex, subIndex, isWrite, availableGas) if expected == 0 && chargeWarmCosts { expected = params.WarmStorageReadCostEIP2929 @@ -314,7 +312,7 @@ func (ae *AccessEvents) CodeChunksRangeGas(contractAddr common.Address, startPC, // Note that an access in write mode implies an access in read mode, whereas an // access in read mode does not imply an access in write mode. func (ae *AccessEvents) BasicDataGas(addr common.Address, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BasicDataLeafKey, isWrite, availableGas) + _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.BasicDataLeafKey, isWrite, availableGas) if expected == 0 && chargeWarmCosts { if availableGas < params.WarmStorageReadCostEIP2929 { return availableGas @@ -330,7 +328,7 @@ func (ae *AccessEvents) BasicDataGas(addr common.Address, isWrite bool, availabl // Note that an access in write mode implies an access in read mode, whereas an access in // read mode does not imply an access in write mode. func (ae *AccessEvents) CodeHashGas(addr common.Address, isWrite bool, availableGas uint64, chargeWarmCosts bool) uint64 { - _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeHashLeafKey, isWrite, availableGas) + _, expected := ae.touchAddressAndChargeGas(addr, zeroTreeIndex, bintrie.CodeHashLeafKey, isWrite, availableGas) if expected == 0 && chargeWarmCosts { if availableGas < params.WarmStorageReadCostEIP2929 { return availableGas diff --git a/core/state/access_events_test.go b/core/state/access_events_test.go index e80859a0b4..0b39130e8d 100644 --- a/core/state/access_events_test.go +++ b/core/state/access_events_test.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" ) var ( @@ -38,7 +37,7 @@ func init() { } func TestAccountHeaderGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() // Check cold read cost gas := ae.BasicDataGas(testAddr, false, math.MaxUint64, false) @@ -93,7 +92,7 @@ func TestAccountHeaderGas(t *testing.T) { // TestContractCreateInitGas checks that the gas cost of contract creation is correctly // calculated. func TestContractCreateInitGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() var testAddr [20]byte for i := byte(0); i < 20; i++ { @@ -116,7 +115,7 @@ func TestContractCreateInitGas(t *testing.T) { // TestMessageCallGas checks that the gas cost of message calls is correctly // calculated. func TestMessageCallGas(t *testing.T) { - ae := NewAccessEvents(utils.NewPointCache(1024)) + ae := NewAccessEvents() // Check cold read cost, without a value gas := ae.MessageCallGas(testAddr, math.MaxUint64) diff --git a/core/state/database.go b/core/state/database.go index 580f3cddbf..291a586dbd 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -29,8 +29,9 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" "github.com/ethereum/go-ethereum/triedb" ) @@ -40,9 +41,6 @@ const ( // Cache size granted for caching clean code. codeCacheSize = 256 * 1024 * 1024 - - // Number of address->curve point associations to keep. - pointCacheSize = 4096 ) // Database wraps access to tries and contract code. @@ -56,9 +54,6 @@ type Database interface { // OpenStorageTrie opens the storage trie of an account. OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error) - // PointCache returns the cache holding points used in verkle tree key computation - PointCache() *utils.PointCache - // TrieDB returns the underlying trie database for managing trie nodes. TrieDB() *triedb.Database @@ -160,7 +155,6 @@ type CachingDB struct { snap *snapshot.Tree codeCache *lru.SizeConstrainedCache[common.Hash, []byte] codeSizeCache *lru.Cache[common.Hash, int] - pointCache *utils.PointCache snapMu sync.RWMutex // Protects useSnapInReader useSnapInReader bool @@ -176,7 +170,6 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { snap: snap, codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - pointCache: utils.NewPointCache(pointCacheSize), TransitionStatePerRoot: lru.NewCache[common.Hash, *overlay.TransitionState](1000), useSnapInReader: true, } @@ -228,7 +221,7 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { } // Configure the trie reader, which is expected to be available as the // gatekeeper unless the state is corrupted. - tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache) + tr, err := newTrieReader(stateRoot, db.triedb) if err != nil { return nil, err } @@ -245,7 +238,7 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { // snapshot layers. Useful for V2 parallel execution where the snapshot reader // may have thread-safety issues under concurrent access from multiple workers. func (db *CachingDB) ReaderTrieOnly(stateRoot common.Hash) (Reader, error) { - tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache) + tr, err := newTrieReader(stateRoot, db.triedb) if err != nil { return nil, err } @@ -287,10 +280,12 @@ func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { if db.triedb.IsVerkle() { ts := overlay.LoadTransitionState(db.TrieDB().Disk(), root, db.triedb.IsVerkle()) if ts.InTransition() { - panic("transition isn't supported yet") + panic("state tree transition isn't supported yet") } if ts.Transitioned() { - return trie.NewVerkleTrie(root, db.triedb, db.pointCache) + // Use BinaryTrie instead of VerkleTrie when IsVerkle is set + // (IsVerkle actually means Binary Trie mode in this codebase) + return bintrie.NewBinaryTrie(root, db.triedb) } } tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) @@ -337,11 +332,6 @@ func (db *CachingDB) TrieDB() *triedb.Database { return db.triedb } -// PointCache returns the cache of evaluated curve points. -func (db *CachingDB) PointCache() *utils.PointCache { - return db.pointCache -} - // Snapshot returns the underlying state snapshot. func (db *CachingDB) Snapshot() *snapshot.Tree { return db.snap @@ -352,9 +342,7 @@ func mustCopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.StateTrie: return t.Copy() - case *trie.VerkleTrie: - return t.Copy() - case *trie.TransitionTrie: + case *transitiontrie.TransitionTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) diff --git a/core/state/database_history.go b/core/state/database_history.go index 314c56c470..7a2be8fe4f 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -17,41 +17,39 @@ package state import ( - "errors" + "fmt" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" + "github.com/ethereum/go-ethereum/triedb/database" "github.com/ethereum/go-ethereum/triedb/pathdb" ) -// historicReader wraps a historical state reader defined in path database, -// providing historic state serving over the path scheme. -// -// TODO(rjl493456442): historicReader is not thread-safe and does not fully -// comply with the StateReader interface requirements, needs to be fixed. -// Currently, it is only used in a non-concurrent context, so it is safe for now. -type historicReader struct { +// historicStateReader implements StateReader, wrapping a historical state reader +// defined in path database and provide historic state serving over the path scheme. +type historicStateReader struct { reader *pathdb.HistoricalStateReader + lock sync.Mutex // Lock for protecting concurrent read } -// newHistoricReader constructs a reader for historic state serving. -func newHistoricReader(r *pathdb.HistoricalStateReader) *historicReader { - return &historicReader{reader: r} +// newHistoricStateReader constructs a reader for historical state serving. +func newHistoricStateReader(r *pathdb.HistoricalStateReader) *historicStateReader { + return &historicStateReader{reader: r} } // Account implements StateReader, retrieving the account specified by the address. -// -// An error will be returned if the associated snapshot is already stale or -// the requested account is not yet covered by the snapshot. -// -// The returned account might be nil if it's not existent. -func (r *historicReader) Account(addr common.Address) (*types.StateAccount, error) { +func (r *historicStateReader) Account(addr common.Address) (*types.StateAccount, error) { + r.lock.Lock() + defer r.lock.Unlock() + account, err := r.reader.Account(addr) if err != nil { return nil, err @@ -81,7 +79,10 @@ func (r *historicReader) Account(addr common.Address) (*types.StateAccount, erro // the requested storage slot is not yet covered by the snapshot. // // The returned storage slot might be empty if it's not existent. -func (r *historicReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { +func (r *historicStateReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { + r.lock.Lock() + defer r.lock.Unlock() + blob, err := r.reader.Storage(addr, key) if err != nil { return common.Hash{}, err @@ -98,6 +99,125 @@ func (r *historicReader) Storage(addr common.Address, key common.Hash) (common.H return slot, nil } +// historicTrieOpener is a wrapper of pathdb.HistoricalNodeReader, implementing +// the database.NodeDatabase by adding NodeReader function. +type historicTrieOpener struct { + root common.Hash + reader *pathdb.HistoricalNodeReader +} + +// newHistoricTrieOpener constructs the historical trie opener. +func newHistoricTrieOpener(root common.Hash, reader *pathdb.HistoricalNodeReader) *historicTrieOpener { + return &historicTrieOpener{ + root: root, + reader: reader, + } +} + +// NodeReader implements database.NodeDatabase, returning a node reader of a +// specified state. +func (o *historicTrieOpener) NodeReader(root common.Hash) (database.NodeReader, error) { + if root != o.root { + return nil, fmt.Errorf("state %x is not available", root) + } + return o.reader, nil +} + +// historicalTrieReader wraps a historical node reader defined in path database, +// providing historical node serving over the path scheme. +type historicalTrieReader struct { + root common.Hash + opener *historicTrieOpener + tr Trie + + subRoots map[common.Address]common.Hash // Set of storage roots, cached when the account is resolved + subTries map[common.Address]Trie // Group of storage tries, cached when it's resolved + lock sync.Mutex // Lock for protecting concurrent read +} + +// newHistoricalTrieReader constructs a reader for historical trie node serving. +func newHistoricalTrieReader(root common.Hash, r *pathdb.HistoricalNodeReader) (*historicalTrieReader, error) { + opener := newHistoricTrieOpener(root, r) + tr, err := trie.NewStateTrie(trie.StateTrieID(root), opener) + if err != nil { + return nil, err + } + return &historicalTrieReader{ + root: root, + opener: opener, + tr: tr, + subRoots: make(map[common.Address]common.Hash), + subTries: make(map[common.Address]Trie), + }, nil +} + +// account is the inner version of Account and assumes the r.lock is already held. +func (r *historicalTrieReader) account(addr common.Address) (*types.StateAccount, error) { + account, err := r.tr.GetAccount(addr) + if err != nil { + return nil, err + } + if account == nil { + r.subRoots[addr] = types.EmptyRootHash + } else { + r.subRoots[addr] = account.Root + } + return account, nil +} + +// Account implements StateReader, retrieving the account specified by the address. +// +// An error will be returned if the associated snapshot is already stale or +// the requested account is not yet covered by the snapshot. +// +// The returned account might be nil if it's not existent. +func (r *historicalTrieReader) Account(addr common.Address) (*types.StateAccount, error) { + r.lock.Lock() + defer r.lock.Unlock() + + return r.account(addr) +} + +// Storage implements StateReader, retrieving the storage slot specified by the +// address and slot key. +// +// An error will be returned if the associated snapshot is already stale or +// the requested storage slot is not yet covered by the snapshot. +// +// The returned storage slot might be empty if it's not existent. +func (r *historicalTrieReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { + r.lock.Lock() + defer r.lock.Unlock() + + tr, found := r.subTries[addr] + if !found { + root, ok := r.subRoots[addr] + + // The storage slot is accessed without account caching. It's unexpected + // behavior but try to resolve the account first anyway. + if !ok { + _, err := r.account(addr) + if err != nil { + return common.Hash{}, err + } + root = r.subRoots[addr] + } + var err error + tr, err = trie.NewStateTrie(trie.StorageTrieID(r.root, crypto.Keccak256Hash(addr.Bytes()), root), r.opener) + if err != nil { + return common.Hash{}, err + } + r.subTries[addr] = tr + } + ret, err := tr.GetStorage(addr, key.Bytes()) + if err != nil { + return common.Hash{}, err + } + var value common.Hash + value.SetBytes(ret) + return value, nil +} + // HistoricDB is the implementation of Database interface, with the ability to // access historical state. type HistoricDB struct { @@ -105,7 +225,6 @@ type HistoricDB struct { triedb *triedb.Database codeCache *lru.SizeConstrainedCache[common.Hash, []byte] codeSizeCache *lru.Cache[common.Hash, int] - pointCache *utils.PointCache } // NewHistoricDatabase creates a historic state database. @@ -115,33 +234,59 @@ func NewHistoricDatabase(disk ethdb.KeyValueStore, triedb *triedb.Database) *His triedb: triedb, codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - pointCache: utils.NewPointCache(pointCacheSize), } } // Reader implements Database interface, returning a reader of the specific state. func (db *HistoricDB) Reader(stateRoot common.Hash) (Reader, error) { - hr, err := db.triedb.HistoricReader(stateRoot) + var readers []StateReader + sr, err := db.triedb.HistoricStateReader(stateRoot) + if err == nil { + readers = append(readers, newHistoricStateReader(sr)) + } + nr, err := db.triedb.HistoricNodeReader(stateRoot) + if err == nil { + tr, err := newHistoricalTrieReader(stateRoot, nr) + if err == nil { + readers = append(readers, tr) + } + } + if len(readers) == 0 { + return nil, fmt.Errorf("historical state %x is not available", stateRoot) + } + combined, err := newMultiStateReader(readers...) if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), newHistoricReader(hr)), nil + return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil } // OpenTrie opens the main account trie. It's not supported by historic database. func (db *HistoricDB) OpenTrie(root common.Hash) (Trie, error) { - return nil, errors.New("not implemented") + nr, err := db.triedb.HistoricNodeReader(root) + if err != nil { + return nil, err + } + tr, err := trie.NewStateTrie(trie.StateTrieID(root), newHistoricTrieOpener(root, nr)) + if err != nil { + return nil, err + } + return tr, nil } // OpenStorageTrie opens the storage trie of an account. It's not supported by // historic database. -func (db *HistoricDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error) { - return nil, errors.New("not implemented") -} - -// PointCache returns the cache holding points used in verkle tree key computation -func (db *HistoricDB) PointCache() *utils.PointCache { - return db.pointCache +func (db *HistoricDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, _ Trie) (Trie, error) { + nr, err := db.triedb.HistoricNodeReader(stateRoot) + if err != nil { + return nil, err + } + id := trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root) + tr, err := trie.NewStateTrie(id, newHistoricTrieOpener(stateRoot, nr)) + if err != nil { + return nil, err + } + return tr, nil } // TrieDB returns the underlying trie database for managing trie nodes. diff --git a/core/state/dump.go b/core/state/dump.go index 6009be3360..130e4bfa9f 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -18,6 +18,7 @@ package state import ( "encoding/json" + "errors" "fmt" "time" @@ -27,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" ) // DumpConfig is a set of options to control what portions of the state will be @@ -230,6 +232,28 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] return nextKey } +// DumpBinTrieLeaves collects all binary trie leaf nodes into the provided map. +func (s *StateDB) DumpBinTrieLeaves(collector map[common.Hash]hexutil.Bytes) error { + tr, err := s.db.OpenTrie(s.originalRoot) + if err != nil { + return err + } + btr, ok := tr.(*bintrie.BinaryTrie) + if !ok { + return errors.New("trie is not a binary trie") + } + it, err := btr.NodeIterator(nil) + if err != nil { + return err + } + for it.Next(true) { + if it.Leaf() { + collector[common.BytesToHash(it.LeafKey())] = it.LeafBlob() + } + } + return nil +} + // RawDump returns the state. If the processing is aborted e.g. due to options // reaching Max, the `Next` key is set on the returned Dump. func (s *StateDB) RawDump(opts *DumpConfig) Dump { diff --git a/core/state/journal.go b/core/state/journal.go index 5cd47e9f2f..9d56b66f84 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -41,7 +41,8 @@ type journalEntry interface { revert(*StateDB) // dirtied returns the Ethereum address modified by this journal entry. - dirtied() *common.Address + // indicates false if no address was changed. + dirtied() (common.Address, bool) // copy returns a deep-copied journal entry. copy() journalEntry @@ -102,8 +103,8 @@ func (j *journal) revertToSnapshot(revid int, s *StateDB) { // append inserts a new modification entry to the end of the change journal. func (j *journal) append(entry journalEntry) { j.entries = append(j.entries, entry) - if addr := entry.dirtied(); addr != nil { - j.dirties[*addr]++ + if addr, dirty := entry.dirtied(); dirty { + j.dirties[addr]++ } } @@ -115,9 +116,9 @@ func (j *journal) revert(statedb *StateDB, snapshot int) { j.entries[i].revert(statedb) // Drop any dirty tracking induced by the change - if addr := j.entries[i].dirtied(); addr != nil { - if j.dirties[*addr]--; j.dirties[*addr] == 0 { - delete(j.dirties, *addr) + if addr, dirty := j.entries[i].dirtied(); dirty { + if j.dirties[addr]--; j.dirties[addr] == 0 { + delete(j.dirties, addr) } } } @@ -297,8 +298,8 @@ func (ch createObjectChange) revert(s *StateDB) { delete(s.stateObjects, ch.account) } -func (ch createObjectChange) dirtied() *common.Address { - return &ch.account +func (ch createObjectChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch createObjectChange) copy() journalEntry { @@ -311,8 +312,8 @@ func (ch createContractChange) revert(s *StateDB) { s.getStateObject(ch.account).newContract = false } -func (ch createContractChange) dirtied() *common.Address { - return nil +func (ch createContractChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch createContractChange) copy() journalEntry { @@ -329,8 +330,8 @@ func (ch selfDestructChange) revert(s *StateDB) { } } -func (ch selfDestructChange) dirtied() *common.Address { - return &ch.account +func (ch selfDestructChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch selfDestructChange) copy() journalEntry { @@ -344,8 +345,8 @@ var ripemd = common.HexToAddress("0000000000000000000000000000000000000003") func (ch touchChange) revert(s *StateDB) { } -func (ch touchChange) dirtied() *common.Address { - return &ch.account +func (ch touchChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch touchChange) copy() journalEntry { @@ -358,8 +359,8 @@ func (ch balanceChange) revert(s *StateDB) { s.getStateObject(ch.account).setBalance(ch.prev) } -func (ch balanceChange) dirtied() *common.Address { - return &ch.account +func (ch balanceChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch balanceChange) copy() journalEntry { @@ -373,8 +374,8 @@ func (ch nonceChange) revert(s *StateDB) { s.getStateObject(ch.account).setNonce(ch.prev) } -func (ch nonceChange) dirtied() *common.Address { - return &ch.account +func (ch nonceChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch nonceChange) copy() journalEntry { @@ -388,8 +389,8 @@ func (ch codeChange) revert(s *StateDB) { s.getStateObject(ch.account).setCode(crypto.Keccak256Hash(ch.prevCode), ch.prevCode) } -func (ch codeChange) dirtied() *common.Address { - return &ch.account +func (ch codeChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch codeChange) copy() journalEntry { @@ -403,8 +404,8 @@ func (ch storageChange) revert(s *StateDB) { s.getStateObject(ch.account).setState(ch.key, ch.prevvalue, ch.origvalue) } -func (ch storageChange) dirtied() *common.Address { - return &ch.account +func (ch storageChange) dirtied() (common.Address, bool) { + return ch.account, true } func (ch storageChange) copy() journalEntry { @@ -420,8 +421,8 @@ func (ch transientStorageChange) revert(s *StateDB) { s.setTransientState(ch.account, ch.key, ch.prevalue) } -func (ch transientStorageChange) dirtied() *common.Address { - return nil +func (ch transientStorageChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch transientStorageChange) copy() journalEntry { @@ -436,8 +437,8 @@ func (ch refundChange) revert(s *StateDB) { s.refund = ch.prev } -func (ch refundChange) dirtied() *common.Address { - return nil +func (ch refundChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch refundChange) copy() journalEntry { @@ -457,8 +458,8 @@ func (ch addLogChange) revert(s *StateDB) { s.logSize-- } -func (ch addLogChange) dirtied() *common.Address { - return nil +func (ch addLogChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch addLogChange) copy() journalEntry { @@ -480,8 +481,8 @@ func (ch accessListAddAccountChange) revert(s *StateDB) { s.accessList.DeleteAddress(ch.address) } -func (ch accessListAddAccountChange) dirtied() *common.Address { - return nil +func (ch accessListAddAccountChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch accessListAddAccountChange) copy() journalEntry { @@ -494,8 +495,8 @@ func (ch accessListAddSlotChange) revert(s *StateDB) { s.accessList.DeleteSlot(ch.address, ch.slot) } -func (ch accessListAddSlotChange) dirtied() *common.Address { - return nil +func (ch accessListAddSlotChange) dirtied() (common.Address, bool) { + return common.Address{}, false } func (ch accessListAddSlotChange) copy() journalEntry { diff --git a/core/state/parallel_statedb.go b/core/state/parallel_statedb.go index 15dd33f303..5d7579f862 100644 --- a/core/state/parallel_statedb.go +++ b/core/state/parallel_statedb.go @@ -14,7 +14,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" ) // --------------------------------------------------------------------------- @@ -1233,8 +1232,6 @@ func (s *ParallelStateDB) Finalise(deleteEmptyObjects bool) {} // Inner returns the underlying StateDB. Required by Bor consensus. func (s *ParallelStateDB) Inner() *StateDB { return s.rawBase } -func (s *ParallelStateDB) PointCache() *utils.PointCache { return s.rawBase.PointCache() } - // Witness returns the shared *Witness so BLOCKHASH writes from V2 workers // land where finalDB sees them. The share is established by the caller // after statedb.Copy() (which deep-copies). Witness locks its own mutations. diff --git a/core/state/parallel_statedb_coverage_test.go b/core/state/parallel_statedb_coverage_test.go index 491f93e9c5..fce3898b23 100644 --- a/core/state/parallel_statedb_coverage_test.go +++ b/core/state/parallel_statedb_coverage_test.go @@ -615,7 +615,7 @@ func TestPDB_TransientStorage(t *testing.T) { } // --------------------------------------------------------------------------- -// Inner / PointCache / Witness / AccessEvents trivial accessors +// Inner / Witness / AccessEvents trivial accessors // --------------------------------------------------------------------------- // TestPDB_InnerAccessors exercises wrappers used by Bor consensus hooks. @@ -624,7 +624,6 @@ func TestPDB_InnerAccessors(t *testing.T) { if pdb.Inner() == nil { t.Fatal("Inner returned nil") } - _ = pdb.PointCache() // may be nil; just must not panic if pdb.Witness() != nil { t.Fatal("Witness: V2 always returns nil") } diff --git a/core/state/reader.go b/core/state/reader.go index 4831a0a059..59c1724e28 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -30,7 +30,8 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/triedb" "github.com/ethereum/go-ethereum/triedb/database" ) @@ -264,7 +265,7 @@ type trieReader struct { // newTrieReader constructs a trie reader of the specific state. An error will be // returned if the associated trie specified by root is not existent. -func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache) (*trieReader, error) { +func newTrieReader(root common.Hash, db *triedb.Database) (*trieReader, error) { var ( tr Trie err error @@ -272,7 +273,11 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach if !db.IsVerkle() { tr, err = trie.NewStateTrie(trie.StateTrieID(root), db) } else { - tr, err = trie.NewVerkleTrie(root, db, cache) + // When IsVerkle() is true, create a BinaryTrie wrapped in TransitionTrie + binTrie, binErr := bintrie.NewBinaryTrie(root, db) + if binErr != nil { + return nil, binErr + } // Based on the transition status, determine if the overlay // tree needs to be created, or if a single, target tree is @@ -283,7 +288,22 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach if err != nil { return nil, err } - tr = trie.NewTransitionTrie(mpt, tr.(*trie.VerkleTrie), false) + tr = transitiontrie.NewTransitionTrie(mpt, binTrie, false) + } else { + // HACK: Use TransitionTrie with nil base as a wrapper to make BinaryTrie + // satisfy the Trie interface. This works around the import cycle between + // trie and trie/bintrie packages. + // + // TODO: In future PRs, refactor the package structure to avoid this hack: + // - Option 1: Move common interfaces (Trie, NodeIterator) to a separate + // package that both trie and trie/bintrie can import + // - Option 2: Create a factory function in the trie package that returns + // BinaryTrie as a Trie interface without direct import + // - Option 3: Move BinaryTrie to the main trie package + // + // The current approach works but adds unnecessary overhead and complexity + // by using TransitionTrie when there's no actual transition happening. + tr = transitiontrie.NewTransitionTrie(nil, binTrie, false) } } if err != nil { diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index b8aa5237c7..e7ddf9e1d9 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -477,6 +477,6 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash { storageList := slices.SortedFunc(maps.Keys(dl.storageData[accountHash]), common.Hash.Cmp) dl.storageList[accountHash] = storageList - dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) + dl.memory += uint64(len(storageList)*common.HashLength + common.HashLength) return storageList } diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index aa93df0426..032efee057 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -210,6 +210,39 @@ func TestInsertAndMerge(t *testing.T) { } } +// TestStorageListMemoryAccounting ensures that StorageList increases dl.memory +// proportionally to the number of storage slots in the requested account and +// does not change memory usage on repeated calls for the same account. +func TestStorageListMemoryAccounting(t *testing.T) { + parent := newDiffLayer(emptyLayer(), common.Hash{}, nil, nil) + account := common.HexToHash("0x01") + + slots := make(map[common.Hash][]byte) + for i := 0; i < 3; i++ { + slots[randomHash()] = []byte{0x01} + } + storage := map[common.Hash]map[common.Hash][]byte{ + account: slots, + } + dl := newDiffLayer(parent, common.Hash{}, nil, storage) + + before := dl.memory + list := dl.StorageList(account) + if have, want := len(list), len(slots); have != want { + t.Fatalf("StorageList length mismatch: have %d, want %d", have, want) + } + expectedDelta := uint64(len(list)*common.HashLength + common.HashLength) + if have, want := dl.memory-before, expectedDelta; have != want { + t.Fatalf("StorageList memory delta mismatch: have %d, want %d", have, want) + } + + before = dl.memory + _ = dl.StorageList(account) + if dl.memory != before { + t.Fatalf("StorageList changed memory on cached call: have %d, want %d", dl.memory, before) + } +} + func emptyLayer() *diskLayer { return &diskLayer{ diskdb: memorydb.New(), diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 1785a53a39..1d0223af19 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -23,11 +23,11 @@ import ( "time" "github.com/holiman/uint256" - "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -39,7 +39,7 @@ import ( ) func hashData(input []byte) common.Hash { - var hasher = sha3.NewLegacyKeccak256() + var hasher = keccak.NewLegacyKeccak256() var hash common.Hash diff --git a/core/state/state_object.go b/core/state/state_object.go index 4c48084531..05010f2af9 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -32,6 +32,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" + "github.com/ethereum/go-ethereum/trie/transitiontrie" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -48,11 +50,11 @@ func (s Storage) Copy() Storage { // - Account values as well as storages can be accessed and modified through the object. // - Finally, call commit to return the changes of storage trie and update account data. type stateObject struct { - db *StateDB - address common.Address // address of ethereum account - addrHash common.Hash // hash of ethereum address of the account - origin *types.StateAccount // Account original data without any change applied, nil means it was not existent - data types.StateAccount // Account data with all mutations applied in the scope of block + db *StateDB + address common.Address // address of ethereum account + addressHash *common.Hash // hash of ethereum address of the account + origin *types.StateAccount // Account original data without any change applied, nil means it was not existent + data types.StateAccount // Account data with all mutations applied in the scope of block // Write caches. trie Trie // storage trie, which becomes non-nil on first access @@ -104,7 +106,6 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s return &stateObject{ db: db, address: address, - addrHash: crypto.Keccak256Hash(address[:]), origin: origin, data: *acct, originStorage: make(Storage), @@ -114,6 +115,14 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s } } +func (s *stateObject) addrHash() common.Hash { + if s.addressHash == nil { + h := crypto.Keccak256Hash(s.address[:]) + s.addressHash = &h + } + return *s.addressHash +} + func (s *stateObject) markSelfdestructed() { s.selfDestructed = true } @@ -154,7 +163,7 @@ func (s *stateObject) getPrefetchedTrie() Trie { return nil } // Attempt to retrieve the trie from the prefetcher - return s.db.prefetcher.trie(s.addrHash, s.data.Root) + return s.db.prefetcher.trie(s.addrHash(), s.data.Root) } // GetState retrieves a value associated with the given storage key. @@ -210,7 +219,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { // Schedule the resolved storage slots for prefetching if it's enabled. if s.db.prefetcher != nil && s.data.Root != types.EmptyRootHash { - if err = s.db.prefetcher.prefetch(s.addrHash, s.origin.Root, s.address, nil, []common.Hash{key}, true); err != nil { + if err = s.db.prefetcher.prefetch(s.addrHash(), s.origin.Root, s.address, nil, []common.Hash{key}, true); err != nil { log.Error("Failed to prefetch storage slot", "addr", s.address, "key", key, "err", err) } } @@ -272,7 +281,7 @@ func (s *stateObject) finalise() { s.pendingStorage[key] = value } if s.db.prefetcher != nil && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash { - if err := s.db.prefetcher.prefetch(s.addrHash, s.data.Root, s.address, nil, slotsToPrefetch, false); err != nil { + if err := s.db.prefetcher.prefetch(s.addrHash(), s.data.Root, s.address, nil, slotsToPrefetch, false); err != nil { log.Error("Failed to prefetch slots", "addr", s.address, "slots", len(slotsToPrefetch), "err", err) } } @@ -368,7 +377,7 @@ func (s *stateObject) updateTrie() (Trie, error) { } if s.db.prefetcher != nil { - s.db.prefetcher.used(s.addrHash, s.data.Root, nil, used) + s.db.prefetcher.used(s.addrHash(), s.data.Root, nil, used) } s.uncommittedStorage = make(Storage) // empties the commit markers return tr, nil @@ -494,7 +503,7 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { obj := &stateObject{ db: db, address: s.address, - addrHash: s.addrHash, + addressHash: nil, origin: s.origin, data: s.data, code: s.code, @@ -508,11 +517,11 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { } switch s.trie.(type) { - case *trie.VerkleTrie: - // Verkle uses only one tree, and the copy has already been + case *bintrie.BinaryTrie: + // UBT uses only one tree, and the copy has already been // made in mustCopyTrie. obj.trie = db.trie - case *trie.TransitionTrie: + case *transitiontrie.TransitionTrie: // Same thing for the transition tree, since the MPT is // read-only. obj.trie = db.trie diff --git a/core/state/state_sizer.go b/core/state/state_sizer.go index 3e63b572aa..79bcbed02a 100644 --- a/core/state/state_sizer.go +++ b/core/state/state_sizer.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/triedb" ) @@ -49,6 +50,21 @@ var ( codeKeySize = int64(len(rawdb.CodePrefix) + common.HashLength) ) +// State size metrics +var ( + stateSizeChainHeightGauge = metrics.NewRegisteredGauge("state/height", nil) + stateSizeAccountsCountGauge = metrics.NewRegisteredGauge("state/accounts/count", nil) + stateSizeAccountsBytesGauge = metrics.NewRegisteredGauge("state/accounts/bytes", nil) + stateSizeStoragesCountGauge = metrics.NewRegisteredGauge("state/storages/count", nil) + stateSizeStoragesBytesGauge = metrics.NewRegisteredGauge("state/storages/bytes", nil) + stateSizeAccountTrieNodesCountGauge = metrics.NewRegisteredGauge("state/trienodes/account/count", nil) + stateSizeAccountTrieNodesBytesGauge = metrics.NewRegisteredGauge("state/trienodes/account/bytes", nil) + stateSizeStorageTrieNodesCountGauge = metrics.NewRegisteredGauge("state/trienodes/storage/count", nil) + stateSizeStorageTrieNodesBytesGauge = metrics.NewRegisteredGauge("state/trienodes/storage/bytes", nil) + stateSizeContractsCountGauge = metrics.NewRegisteredGauge("state/contracts/count", nil) + stateSizeContractsBytesGauge = metrics.NewRegisteredGauge("state/contracts/bytes", nil) +) + // SizeStats represents either the current state size statistics or the size // differences resulting from a state transition. type SizeStats struct { @@ -77,6 +93,20 @@ func (s SizeStats) String() string { ) } +func (s SizeStats) publish() { + stateSizeChainHeightGauge.Update(int64(s.BlockNumber)) + stateSizeAccountsCountGauge.Update(s.Accounts) + stateSizeAccountsBytesGauge.Update(s.AccountBytes) + stateSizeStoragesCountGauge.Update(s.Storages) + stateSizeStoragesBytesGauge.Update(s.StorageBytes) + stateSizeAccountTrieNodesCountGauge.Update(s.AccountTrienodes) + stateSizeAccountTrieNodesBytesGauge.Update(s.AccountTrienodeBytes) + stateSizeStorageTrieNodesCountGauge.Update(s.StorageTrienodes) + stateSizeStorageTrieNodesBytesGauge.Update(s.StorageTrienodeBytes) + stateSizeContractsCountGauge.Update(s.ContractCodes) + stateSizeContractsBytesGauge.Update(s.ContractCodeBytes) +} + // add applies the given state diffs and produces a new version of the statistics. func (s SizeStats) add(diff SizeStats) SizeStats { s.StateRoot = diff.StateRoot @@ -310,8 +340,12 @@ func (t *SizeTracker) run() { stats[u.root] = stat last = u.root + // Publish statistics to metric system + stat.publish() + + // Evict the stale statistics heap.Push(&h, stats[u.root]) - for u.blockNumber-h[0].BlockNumber > statEvictThreshold { + for len(h) > 0 && u.blockNumber-h[0].BlockNumber > statEvictThreshold { delete(stats, h[0].StateRoot) heap.Pop(&h) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 5e349d7b0c..274d972149 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -42,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" ) // TriesInMemory represents the number of layers that are kept in RAM. @@ -209,7 +208,7 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro transientStorage: newTransientStorage(), } if db.TrieDB().IsVerkle() { - sdb.accessEvents = NewAccessEvents(db.PointCache()) + sdb.accessEvents = NewAccessEvents() } return sdb, nil @@ -814,7 +813,6 @@ func (s *StateDB) Logs() []*types.Log { for _, lgs := range s.logs { logs = append(logs, lgs...) } - return logs } @@ -1603,13 +1601,13 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { witness := trie.Witness() s.witness.AddState(witness) if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) + s.witnessStats.Add(witness, obj.addrHash()) } } else if obj.trie != nil { witness := obj.trie.Witness() s.witness.AddState(witness) if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) + s.witnessStats.Add(witness, obj.addrHash()) } } } @@ -1627,13 +1625,13 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { witness := trie.Witness() s.witness.AddState(witness) if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) + s.witnessStats.Add(witness, obj.addrHash()) } } else if obj.trie != nil { witness := obj.trie.Witness() s.witness.AddState(witness) if s.witnessStats != nil { - s.witnessStats.Add(witness, obj.addrHash) + s.witnessStats.Add(witness, obj.addrHash()) } } } @@ -2024,7 +2022,7 @@ func (s *StateDB) commit(deleteEmptyObjects bool, noStorageWiping bool, blockNum return err } lock.Lock() - updates[obj.addrHash] = update + updates[obj.addrHash()] = update s.StorageCommits = time.Since(start) // overwrite with the longest storage commit runtime lock.Unlock() return nil @@ -2362,7 +2360,7 @@ func (s *StateDB) FinaliseFastWithPrefetch(deleteEmptyObjects bool) { if obj == nil { continue } - _ = s.prefetcher.prefetch(obj.addrHash, obj.data.Root, as.addr, nil, as.slots, false) + _ = s.prefetcher.prefetch(obj.addrHash(), obj.data.Root, as.addr, nil, as.slots, false) } } s.FinaliseFast(deleteEmptyObjects) @@ -2454,11 +2452,6 @@ func (s *StateDB) SkipTimers() { s.skipTimers = true } -// PointCache returns the point cache used by verkle tree. -func (s *StateDB) PointCache() *utils.PointCache { - return s.db.PointCache() -} - // Witness retrieves the current state witness being collected. func (s *StateDB) Witness() *stateless.Witness { return s.witness diff --git a/core/state/statedb_hooked.go b/core/state/statedb_hooked.go index 9042660801..5e3f301e0e 100644 --- a/core/state/statedb_hooked.go +++ b/core/state/statedb_hooked.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" ) // hookedStateDB represents a statedb which emits calls to tracing-hooks @@ -134,10 +133,6 @@ func (s *hookedStateDB) AddSlotToAccessList(addr common.Address, slot common.Has s.inner.AddSlotToAccessList(addr, slot) } -func (s *hookedStateDB) PointCache() *utils.PointCache { - return s.inner.PointCache() -} - func (s *hookedStateDB) Prepare(rules params.Rules, sender, coinbase common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) { s.inner.Prepare(rules, sender, coinbase, dest, precompiles, txAccesses) } @@ -233,7 +228,7 @@ func (s *hookedStateDB) SelfDestruct(address common.Address) uint256.Int { var prevCode []byte var prevCodeHash common.Hash - if s.hooks.OnCodeChange != nil { + if s.hooks.OnCodeChange != nil || s.hooks.OnCodeChangeV2 != nil { prevCode = s.inner.GetCode(address) prevCodeHash = s.inner.GetCodeHash(address) } @@ -259,7 +254,7 @@ func (s *hookedStateDB) SelfDestruct6780(address common.Address) (uint256.Int, b var prevCode []byte var prevCodeHash common.Hash - if s.hooks.OnCodeChange != nil { + if s.hooks.OnCodeChange != nil || s.hooks.OnCodeChangeV2 != nil { prevCodeHash = s.inner.GetCodeHash(address) prevCode = s.inner.GetCode(address) } diff --git a/core/state/statedb_hooked_test.go b/core/state/statedb_hooked_test.go index e23db9b3c0..8e36704524 100644 --- a/core/state/statedb_hooked_test.go +++ b/core/state/statedb_hooked_test.go @@ -123,6 +123,47 @@ func TestHooks(t *testing.T) { sdb.AddLog(&types.Log{ Address: common.Address{0xbb}, }) + + if len(result) != len(wants) { + t.Fatalf("number of tracing events wrong, have %d want %d", len(result), len(wants)) + } + + for i, want := range wants { + if have := result[i]; have != want { + t.Fatalf("error event %d, have\n%v\nwant%v\n", i, have, want) + } + } +} + +func TestHooks_OnCodeChangeV2(t *testing.T) { + inner, _ := New(types.EmptyRootHash, NewDatabaseForTesting()) + + var result []string + var wants = []string{ + "0xaa00000000000000000000000000000000000000.code: (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) ->0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) ContractCreation", + "0xaa00000000000000000000000000000000000000.code: 0x1325 (0xa12ae05590de0c93a00bc7ac773c2fdb621e44f814985e72194f921c0050f728) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct", + "0xbb00000000000000000000000000000000000000.code: (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) ->0x1326 (0x3c54516221d604e623f358bc95996ca3242aaa109bddabcebda13db9b3f90dcb) ContractCreation", + "0xbb00000000000000000000000000000000000000.code: 0x1326 (0x3c54516221d604e623f358bc95996ca3242aaa109bddabcebda13db9b3f90dcb) -> (0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470) SelfDestruct", + } + emitF := func(format string, a ...any) { + result = append(result, fmt.Sprintf(format, a...)) + } + sdb := NewHookedState(inner, &tracing.Hooks{ + OnCodeChangeV2: func(addr common.Address, prevCodeHash common.Hash, prevCode []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) { + emitF("%v.code: %#x (%v) ->%#x (%v) %s", addr, prevCode, prevCodeHash, code, codeHash, reason) + }, + }) + sdb.SetCode(common.Address{0xaa}, []byte{0x13, 37}, tracing.CodeChangeContractCreation) + sdb.SelfDestruct(common.Address{0xaa}) + + sdb.SetCode(common.Address{0xbb}, []byte{0x13, 38}, tracing.CodeChangeContractCreation) + sdb.CreateContract(common.Address{0xbb}) + sdb.SelfDestruct6780(common.Address{0xbb}) + + if len(result) != len(wants) { + t.Fatalf("number of tracing events wrong, have %d want %d", len(result), len(wants)) + } + for i, want := range wants { if have := result[i]; have != want { t.Fatalf("error event %d, have\n%v\nwant%v\n", i, have, want) diff --git a/core/state/v2_method_parity_test.go b/core/state/v2_method_parity_test.go index e258a851e4..d5510935fc 100644 --- a/core/state/v2_method_parity_test.go +++ b/core/state/v2_method_parity_test.go @@ -116,10 +116,11 @@ var pdbExemptMethods = map[string]pdbExemptCategory{ "ValidateKnownAccounts": catLowLevel, // Debug / introspection — purely for tooling. - "Dump": catDebug, - "DumpToCollector": catDebug, - "RawDump": catDebug, - "IterativeDump": catDebug, + "Dump": catDebug, + "DumpToCollector": catDebug, + "RawDump": catDebug, + "IterativeDump": catDebug, + "DumpBinTrieLeaves": catDebug, } // TestPDBMethodParity fails when *StateDB grows an exported method that diff --git a/core/state/witness_prewalk_test.go b/core/state/witness_prewalk_test.go index 1fe5283e77..3bc125dfd3 100644 --- a/core/state/witness_prewalk_test.go +++ b/core/state/witness_prewalk_test.go @@ -43,7 +43,7 @@ func prewalkFixture(t *testing.T) (*StateDB, *readerWithCache, common.Address, * if err != nil { t.Fatalf("flat inner reader: %v", err) } - tr, err := newTrieReader(root, tdb, nil) + tr, err := newTrieReader(root, tdb) if err != nil { t.Fatalf("trie reader: %v", err) } diff --git a/core/state/witness_readset_test.go b/core/state/witness_readset_test.go index 3d23a01032..aaf01b2556 100644 --- a/core/state/witness_readset_test.go +++ b/core/state/witness_readset_test.go @@ -91,7 +91,7 @@ func TestCollectStateWitnessIncludesFlatServedReads(t *testing.T) { if err != nil { t.Fatalf("flat inner reader: %v", err) } - tr, err := newTrieReader(root, tdb, nil) + tr, err := newTrieReader(root, tdb) if err != nil { t.Fatalf("trie reader: %v", err) } diff --git a/core/state_processor.go b/core/state_processor.go index 0ac0af66e6..184a546950 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -80,9 +80,14 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg interruptCtx = context.Background() } + var tracingStateDB = vm.StateDB(statedb) + if hooks := cfg.Tracer; hooks != nil { + tracingStateDB = state.NewHookedState(statedb, hooks) + } + // Mutate the block and state according to any hard-fork specs if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(block.Number()) == 0 { - misc.ApplyDAOHardFork(statedb) + misc.ApplyDAOHardFork(tracingStateDB) } var ( context vm.BlockContext @@ -90,12 +95,8 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg ) // Apply pre-execution system calls. - var tracingStateDB = vm.StateDB(statedb) - if hooks := cfg.Tracer; hooks != nil { - tracingStateDB = state.NewHookedState(statedb, hooks) - } context = NewEVMBlockContext(header, p.chain, author) - evm := vm.NewEVM(context, tracingStateDB, p.chainConfig(), cfg) + evm := vm.NewEVM(context, tracingStateDB, config, cfg) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { ProcessBeaconBlockRoot(*beaconRoot, evm) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index a4824757d4..cddd1e60c6 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -22,8 +22,6 @@ import ( "math/big" "testing" - "golang.org/x/crypto/sha3" - "github.com/holiman/uint256" "github.com/ethereum/go-ethereum/common" @@ -35,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) @@ -400,7 +399,7 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr var receipts []*types.Receipt // The post-state result doesn't need to be correct (this is a bad block), but we do need something there // Preferably something unique. So let's use a combo of blocknum + txhash - hasher := sha3.NewLegacyKeccak256() + hasher := keccak.NewLegacyKeccak256() hasher.Write(header.Number.Bytes()) var cumulativeGas uint64 diff --git a/core/stateless/stats.go b/core/stateless/stats.go index 94f5587f99..73ce031bff 100644 --- a/core/stateless/stats.go +++ b/core/stateless/stats.go @@ -62,10 +62,17 @@ func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) { // If current path is a prefix of the next path, it's not a leaf. // The last path is always a leaf. if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) { + depth := len(path) if owner == (common.Hash{}) { - s.accountTrieLeaves[len(path)] += 1 + if depth >= len(s.accountTrieLeaves) { + depth = len(s.accountTrieLeaves) - 1 + } + s.accountTrieLeaves[depth] += 1 } else { - s.storageTrieLeaves[len(path)] += 1 + if depth >= len(s.storageTrieLeaves) { + depth = len(s.storageTrieLeaves) - 1 + } + s.storageTrieLeaves[depth] += 1 } } } diff --git a/core/tracing/gen_balance_change_reason_stringer.go b/core/tracing/gen_balance_change_reason_stringer.go index 250b521193..f0edfad872 100644 --- a/core/tracing/gen_balance_change_reason_stringer.go +++ b/core/tracing/gen_balance_change_reason_stringer.go @@ -31,8 +31,9 @@ const _BalanceChangeReason_name = "UnspecifiedBalanceIncreaseRewardMineUncleBala var _BalanceChangeReason_index = [...]uint16{0, 11, 41, 71, 96, 125, 160, 181, 205, 231, 256, 264, 276, 303, 330, 361, 367} func (i BalanceChangeReason) String() string { - if i >= BalanceChangeReason(len(_BalanceChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_BalanceChangeReason_index)-1 { return "BalanceChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _BalanceChangeReason_name[_BalanceChangeReason_index[i]:_BalanceChangeReason_index[i+1]] + return _BalanceChangeReason_name[_BalanceChangeReason_index[idx]:_BalanceChangeReason_index[idx+1]] } diff --git a/core/tracing/gen_code_change_reason_stringer.go b/core/tracing/gen_code_change_reason_stringer.go index 9372954063..2531b10471 100644 --- a/core/tracing/gen_code_change_reason_stringer.go +++ b/core/tracing/gen_code_change_reason_stringer.go @@ -22,8 +22,9 @@ const _CodeChangeReason_name = "UnspecifiedContractCreationGenesisAuthorizationA var _CodeChangeReason_index = [...]uint8{0, 11, 27, 34, 47, 65, 77, 83} func (i CodeChangeReason) String() string { - if i >= CodeChangeReason(len(_CodeChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_CodeChangeReason_index)-1 { return "CodeChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _CodeChangeReason_name[_CodeChangeReason_index[i]:_CodeChangeReason_index[i+1]] + return _CodeChangeReason_name[_CodeChangeReason_index[idx]:_CodeChangeReason_index[idx+1]] } diff --git a/core/tracing/gen_nonce_change_reason_stringer.go b/core/tracing/gen_nonce_change_reason_stringer.go index f775c1f3a6..f617d75280 100644 --- a/core/tracing/gen_nonce_change_reason_stringer.go +++ b/core/tracing/gen_nonce_change_reason_stringer.go @@ -22,8 +22,9 @@ const _NonceChangeReason_name = "UnspecifiedGenesisEoACallContractCreatorNewCont var _NonceChangeReason_index = [...]uint8{0, 11, 18, 25, 40, 51, 64, 70} func (i NonceChangeReason) String() string { - if i >= NonceChangeReason(len(_NonceChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_NonceChangeReason_index)-1 { return "NonceChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _NonceChangeReason_name[_NonceChangeReason_index[i]:_NonceChangeReason_index[i+1]] + return _NonceChangeReason_name[_NonceChangeReason_index[idx]:_NonceChangeReason_index[idx+1]] } diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index 7e25d0a7c9..417a3fa598 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -377,6 +377,9 @@ const ( // NonceChangeRevert is emitted when the nonce is reverted back to a previous value due to call failure. // It is only emitted when the tracer has opted in to use the journaling wrapper (WrapWithJournal). NonceChangeRevert NonceChangeReason = 6 + + // NonceChangeSelfdestruct is emitted when the nonce is reset to zero due to a self-destruct + NonceChangeSelfdestruct NonceChangeReason = 7 ) // CodeChangeReason is used to indicate the reason for a code change. diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 4211e9169e..158b592169 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -21,12 +21,10 @@ import ( "container/heap" "errors" "fmt" - "maps" "math" "math/big" "os" "path/filepath" - "slices" "sort" "sync" "sync/atomic" @@ -98,11 +96,15 @@ const ( // store. storeVersion = 1 - // conversionTimeWindow defines the period after the Osaka fork during which - // the pool will still accept and convert legacy blob transactions. After this - // window, all legacy blob transactions will be rejected. - //nolint:unused - conversionTimeWindow = time.Hour * 2 + // gappedLifetime is the approximate duration for which nonce-gapped transactions + // are kept before being dropped. Since gapped is only a reorder buffer and it + // is expected that the original transactions were inserted in the mempool in + // nonce order, the duration is kept short to avoid DoS vectors. + gappedLifetime = 1 * time.Minute + + // maxGappedTxs is the maximum number of gapped transactions kept overall. + // This is a safety limit to avoid DoS vectors. + maxGapped = 128 ) // blobTxMeta is the minimal subset of types.BlobTx necessary to validate and @@ -339,9 +341,11 @@ type BlobPool struct { stored uint64 // Useful data size of all transactions on disk limbo *limbo // Persistent data store for the non-finalized blobs - signer types.Signer // Transaction signer to use for sender recovery - chain BlockChain // Chain object to access the state through - cQueue *conversionQueue // The queue for performing legacy sidecar conversion (TODO: remove after Osaka) + gapped map[common.Address][]*types.Transaction // Transactions that are currently gapped (nonce too high) + gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion + + signer types.Signer // Transaction signer to use for sender recovery + chain BlockChain // Chain object to access the state through head atomic.Pointer[types.Header] // Current head of the chain state *state.StateDB // Current state at the head of the chain @@ -370,16 +374,22 @@ func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bo hasPendingAuth: hasPendingAuth, signer: types.LatestSigner(chain.Config()), chain: chain, - cQueue: newConversionQueue(), // Deprecate it after the osaka fork lookup: newLookup(), index: make(map[common.Address][]*blobTxMeta), spent: make(map[common.Address]*uint256.Int), + gapped: make(map[common.Address][]*types.Transaction), + gappedSource: make(map[common.Hash]common.Address), } } // Filter returns whether the given transaction can be consumed by the blob pool. func (p *BlobPool) Filter(tx *types.Transaction) bool { - return tx.Type() == types.BlobTxType + return p.FilterType(tx.Type()) +} + +// FilterType returns whether the blob pool supports the given transaction type. +func (p *BlobPool) FilterType(kind byte) bool { + return kind == types.BlobTxType } // Init sets the gas price needed to keep a transaction in the pool and the chain @@ -416,7 +426,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser p.state = state // Create new slotter for pre-Osaka blob configuration. - slotter := newSlotter(eip4844.LatestMaxBlobsPerBlock(p.chain.Config())) + slotter := newSlotter(params.BlobTxMaxBlobs) // See if we need to migrate the queue blob store after fusaka slotter, err = tryMigrate(p.chain.Config(), slotter, queuedir) @@ -487,9 +497,6 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser // Close closes down the underlying persistent store. func (p *BlobPool) Close() error { - // Terminate the conversion queue - p.cQueue.close() - var errs []error if p.limbo != nil { // Close might be invoked due to error in constructor, before p,limbo is set if err := p.limbo.Close(); err != nil { @@ -843,6 +850,9 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { resettimeHist.Update(time.Since(start).Nanoseconds()) }(time.Now()) + // Handle reorg buffer timeouts evicting old gapped transactions + p.evictGapped() + statedb, err := p.chain.StateAt(newHead.Root) if err != nil { log.Error("Failed to reset blobpool state", "err", err) @@ -887,172 +897,6 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { basefeeGauge.Update(int64(basefee.Uint64())) blobfeeGauge.Update(int64(blobfee.Uint64())) p.updateStorageMetrics() - - // Perform the conversion logic at the fork boundary - if !p.chain.Config().IsOsaka(oldHead.Number) && p.chain.Config().IsOsaka(newHead.Number) { - // Deep copy all indexed transaction metadata. - var ( - ids = make(map[common.Address]map[uint64]uint64) - txs = make(map[common.Address]map[uint64]common.Hash) - ) - for sender, list := range p.index { - ids[sender] = make(map[uint64]uint64) - txs[sender] = make(map[uint64]common.Hash) - for _, m := range list { - ids[sender][m.nonce] = m.id - txs[sender][m.nonce] = m.hash - } - } - // Initiate the background conversion thread. - p.cQueue.launchBillyConversion(func() { - p.convertLegacySidecars(ids, txs) - }) - } -} - -// compareAndSwap checks if the specified transaction is still tracked in the pool -// and replace the metadata accordingly. It should only be used in the fork boundary -// bulk conversion. If it fails for some reason, the subsequent txs won't be dropped -// for simplicity which we assume it's very likely to happen. -// -// The returned flag indicates whether the replacement succeeded. -func (p *BlobPool) compareAndSwap(address common.Address, hash common.Hash, blob []byte, oldID uint64, oldStorageSize uint32) bool { - p.lock.Lock() - defer p.lock.Unlock() - - newId, err := p.store.Put(blob) - if err != nil { - log.Error("Failed to store transaction", "hash", hash, "err", err) - return false - } - newSize := uint64(len(blob)) - newStorageSize := p.store.Size(newId) - - // Terminate the procedure if the transaction was already evicted. The - // newly added blob should be removed before return. - if !p.lookup.update(hash, newId, newSize) { - if derr := p.store.Delete(newId); derr != nil { - log.Error("Failed to delete the dangling blob tx", "err", derr) - } else { - log.Warn("Deleted the dangling blob tx", "id", newId) - } - return false - } - // Update the metadata of blob transaction - for _, meta := range p.index[address] { - if meta.hash == hash { - meta.id = newId - meta.version = types.BlobSidecarVersion1 - meta.storageSize = newStorageSize - meta.size = newSize - - p.stored += uint64(newStorageSize) - p.stored -= uint64(oldStorageSize) - break - } - } - if err := p.store.Delete(oldID); err != nil { - log.Error("Failed to delete the legacy transaction", "hash", hash, "id", oldID, "err", err) - } - return true -} - -// convertLegacySidecar fetches transaction data from the store, performs an -// on-the-fly conversion. This function is intended for use only during the -// Osaka fork transition period. -// -// The returned flag indicates whether the replacement succeeds or not. -func (p *BlobPool) convertLegacySidecar(sender common.Address, hash common.Hash, id uint64) bool { - start := time.Now() - - // Retrieves the legacy blob transaction from the underlying store with - // read lock held, preventing any potential data race around the slot - // specified by the id. - p.lock.RLock() - data, err := p.store.Get(id) - if err != nil { - p.lock.RUnlock() - // The transaction may have been evicted simultaneously, safe to skip conversion. - log.Debug("Blob transaction is missing", "hash", hash, "id", id, "err", err) - return false - } - oldStorageSize := p.store.Size(id) - p.lock.RUnlock() - - // Decode the transaction, the failure is not expected and report the error - // loudly if possible. If the blob transaction in this slot is corrupted, - // leave it in the store, it will be dropped during the next pool - // initialization. - var tx types.Transaction - if err = rlp.DecodeBytes(data, &tx); err != nil { - log.Error("Blob transaction is corrupted", "hash", hash, "id", id, "err", err) - return false - } - - // Skip conversion if the transaction does not match the expected hash, or if it was - // already converted. This can occur if the original transaction was evicted from the - // pool and the slot was reused by a new one. - if tx.Hash() != hash { - log.Warn("Blob transaction was replaced", "hash", hash, "id", id, "stored", tx.Hash()) - return false - } - sc := tx.BlobTxSidecar() - if sc.Version >= types.BlobSidecarVersion1 { - log.Debug("Skipping conversion of blob tx", "hash", hash, "id", id) - return false - } - - // Perform the sidecar conversion, the failure is not expected and report the error - // loudly if possible. - if err := tx.BlobTxSidecar().ToV1(); err != nil { - log.Error("Failed to convert blob transaction", "hash", hash, "err", err) - return false - } - - // Encode the converted transaction, the failure is not expected and report - // the error loudly if possible. - blob, err := rlp.EncodeToBytes(&tx) - if err != nil { - log.Error("Failed to encode blob transaction", "hash", tx.Hash(), "err", err) - return false - } - - // Replace the legacy blob transaction with the converted format. - if !p.compareAndSwap(sender, hash, blob, id, oldStorageSize) { - log.Error("Failed to replace the legacy transaction", "hash", hash) - return false - } - log.Debug("Converted legacy blob transaction", "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) - return true -} - -// convertLegacySidecars converts all given transactions to sidecar version 1. -// -// If any of them fails to be converted, the subsequent transactions will still -// be processed, as we assume the failure is very unlikely to happen. If happens, -// these transactions will be stuck in the pool until eviction. -func (p *BlobPool) convertLegacySidecars(ids map[common.Address]map[uint64]uint64, txs map[common.Address]map[uint64]common.Hash) { - var ( - start = time.Now() - success int - failure int - ) - for addr, list := range txs { - // Transactions evicted from the pool must be contiguous, if in any case, - // the transactions are gapped with each other, they will be discarded. - nonces := slices.Collect(maps.Keys(list)) - slices.Sort(nonces) - - // Convert the txs with nonce order - for _, nonce := range nonces { - if p.convertLegacySidecar(addr, list[nonce], ids[addr][nonce]) { - success++ - } else { - failure++ - } - } - } - log.Info("Completed blob transaction conversion", "discarded", failure, "injected", success, "elapsed", common.PrettyDuration(time.Since(start))) } // reorg assembles all the transactors and missing transactions between an old @@ -1371,7 +1215,9 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { State: p.state, FirstNonceGap: func(addr common.Address) uint64 { - // Nonce gaps are not permitted in the blob pool, the first gap will + // Nonce gaps are permitted in the blob pool, but only as part of the + // in-memory 'gapped' buffer. We expose the gap here to validateTx, + // then handle the error by adding to the buffer. The first gap will // be the next nonce shifted by however many transactions we already // have pooled. return p.state.GetNonce(addr) + uint64(len(p.index[addr])) @@ -1450,7 +1296,9 @@ func (p *BlobPool) Has(hash common.Hash) bool { p.lock.RLock() defer p.lock.RUnlock() - return p.lookup.exists(hash) + poolHas := p.lookup.exists(hash) + _, gapped := p.gappedSource[hash] + return poolHas || gapped } func (p *BlobPool) getRLP(hash common.Hash) []byte { @@ -1532,8 +1380,8 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { // // The version argument specifies the type of proofs to return, either the // blob proofs (version 0) or the cell proofs (version 1). Proofs conversion is -// CPU intensive, so only done if explicitly requested with the convert flag. -func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ([]*kzg4844.Blob, []kzg4844.Commitment, [][]kzg4844.Proof, error) { +// CPU intensive and prohibited in the blobpool explicitly. +func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte) ([]*kzg4844.Blob, []kzg4844.Commitment, [][]kzg4844.Proof, error) { var ( blobs = make([]*kzg4844.Blob, len(vhashes)) commitments = make([]kzg4844.Commitment, len(vhashes)) @@ -1584,7 +1432,7 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ( } // Mark hash as seen. filled[hash] = struct{}{} - if sidecar.Version != version && !convert { + if sidecar.Version != version { // Skip blobs with incompatible version. Note we still track the blob hash // in `filled` here, ensuring that we do not resolve this tx another time. continue @@ -1593,29 +1441,13 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte, convert bool) ( var pf []kzg4844.Proof switch version { case types.BlobSidecarVersion0: - if sidecar.Version == types.BlobSidecarVersion0 { - pf = []kzg4844.Proof{sidecar.Proofs[i]} - } else { - proof, err := kzg4844.ComputeBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i]) - if err != nil { - return nil, nil, nil, err - } - pf = []kzg4844.Proof{proof} - } + pf = []kzg4844.Proof{sidecar.Proofs[i]} case types.BlobSidecarVersion1: - if sidecar.Version == types.BlobSidecarVersion0 { - cellProofs, err := kzg4844.ComputeCellProofs(&sidecar.Blobs[i]) - if err != nil { - return nil, nil, nil, err - } - pf = cellProofs - } else { - cellProofs, err := sidecar.CellProofsAt(i) - if err != nil { - return nil, nil, nil, err - } - pf = cellProofs + cellProofs, err := sidecar.CellProofsAt(i) + if err != nil { + return nil, nil, nil, err } + pf = cellProofs } for _, index := range list { blobs[index] = &sidecar.Blobs[i] @@ -1642,44 +1474,6 @@ func (p *BlobPool) AvailableBlobs(vhashes []common.Hash) int { return available } -// preCheck performs the static validation upon the provided tx and converts -// the legacy sidecars if Osaka fork has been activated with a short time window. -// -// This function is pure static and lock free. -func (p *BlobPool) preCheck(tx *types.Transaction) error { - var ( - head = p.head.Load() - isOsaka = p.chain.Config().IsOsaka(head.Number) - deadline time.Time - ) - // Validate the transaction statically at first to avoid unnecessary - // conversion. This step doesn't require lock protection. - if err := p.ValidateTxBasics(tx); err != nil { - return err - } - // Before the Osaka fork, reject the blob txs with cell proofs - if !isOsaka { - if tx.BlobTxSidecar().Version == types.BlobSidecarVersion0 { - return nil - } else { - return errors.New("cell proof is not supported yet") - } - } - // After the Osaka fork, reject the legacy blob txs if the conversion - // time window is passed. - if tx.BlobTxSidecar().Version == types.BlobSidecarVersion1 { - return nil - } - if head.Time > uint64(deadline.Unix()) { - return errors.New("legacy blob tx is not supported") - } - // Convert the legacy sidecar after Osaka fork. This could be a long - // procedure which takes a few seconds, even minutes if there is a long - // queue. Fortunately it will only block the routine of the source peer - // announcing the tx, without affecting other parts. - return p.cQueue.convert(tx) -} - // Add inserts a set of blob transactions into the pool if they pass validation (both // consensus validity and pool restrictions). func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error { @@ -1688,17 +1482,13 @@ func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error { adds = make([]*types.Transaction, 0, len(txs)) ) for i, tx := range txs { - if errs[i] = p.preCheck(tx); errs[i] != nil { + if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil { continue } if errs[i] = p.add(tx); errs[i] == nil { adds = append(adds, tx.WithoutBlobTxSidecar()) } } - if len(adds) > 0 { - p.discoverFeed.Send(core.NewTxsEvent{Txs: adds}) - p.insertFeed.Send(core.NewTxsEvent{Txs: adds}) - } return errs } @@ -1717,6 +1507,13 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { addtimeHist.Update(time.Since(start).Nanoseconds()) }(time.Now()) + return p.addLocked(tx, true) +} + +// addLocked inserts a new blob transaction into the pool if it passes validation (both +// consensus validity and pool restrictions). It must be called with the pool lock held. +// Only for internal use. +func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error) { // Ensure the transaction is valid from all perspectives if err := p.validateTx(tx); err != nil { log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err) @@ -1729,6 +1526,21 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { addStaleMeter.Mark(1) case errors.Is(err, core.ErrNonceTooHigh): addGappedMeter.Mark(1) + // Store the tx in memory, and revalidate later + from, _ := types.Sender(p.signer, tx) + allowance := p.gappedAllowance(from) + if allowance >= 1 && len(p.gapped) < maxGapped { + p.gapped[from] = append(p.gapped[from], tx) + p.gappedSource[tx.Hash()] = from + log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + return nil + } else { + // if maxGapped is reached, it is better to give time to gapped + // transactions by keeping the old and dropping this one. + // Thus replacing a gapped transaction with another gapped transaction + // is discouraged. + log.Trace("no gapped blob queue allowance", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + } case errors.Is(err, core.ErrInsufficientFunds): addOverdraftedMeter.Mark(1) case errors.Is(err, txpool.ErrAccountLimitExceeded): @@ -1866,6 +1678,58 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { p.updateStorageMetrics() addValidMeter.Mark(1) + + // Notify all listeners of the new arrival + p.discoverFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + p.insertFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + + //check the gapped queue for this account and try to promote + if gtxs, ok := p.gapped[from]; checkGapped && ok && len(gtxs) > 0 { + // We have to add in nonce order, but we want to stable sort to cater for situations + // where transactions are replaced, keeping the original receive order for same nonce + sort.SliceStable(gtxs, func(i, j int) bool { + return gtxs[i].Nonce() < gtxs[j].Nonce() + }) + for len(gtxs) > 0 { + stateNonce := p.state.GetNonce(from) + firstgap := stateNonce + uint64(len(p.index[from])) + + if gtxs[0].Nonce() > firstgap { + // Anything beyond the first gap is not addable yet + break + } + + // Drop any buffered transactions that became stale in the meantime (included in chain or replaced) + // If we arrive to the transaction in the pending range (between the state Nonce and first gap, we + // try to add them now while removing from here. + tx := gtxs[0] + gtxs[0] = nil + gtxs = gtxs[1:] + delete(p.gappedSource, tx.Hash()) + + if tx.Nonce() < stateNonce { + // Stale, drop it. Eventually we could add to limbo here if hash matches. + log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from])) + continue + } + + if tx.Nonce() <= firstgap { + // If we hit the pending range, including the first gap, add it and continue to try to add more. + // We do not recurse here, but continue to loop instead. + // We are under lock, so we can add the transaction directly. + if err := p.addLocked(tx, false); err == nil { + log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) + } else { + log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err) + } + } + } + if len(gtxs) == 0 { + delete(p.gapped, from) + } else { + p.gapped[from] = gtxs + } + } return nil } @@ -2106,6 +1970,50 @@ func (p *BlobPool) Nonce(addr common.Address) uint64 { return p.state.GetNonce(addr) } +// gappedAllowance returns the number of gapped transactions still +// allowed for the given account. Allowance is based on a slow-start +// logic, allowing more gaps (resource usage) to accounts with a +// higher nonce. Can also return negative values. +func (p *BlobPool) gappedAllowance(addr common.Address) int { + // Gaps happen, but we don't want to allow too many. + // Use log10(nonce+1) as the allowance, with a minimum of 0. + nonce := p.state.GetNonce(addr) + allowance := int(math.Log10(float64(nonce + 1))) + // Cap the allowance to the remaining pool space + return min(allowance, maxTxsPerAccount-len(p.index[addr])) - len(p.gapped[addr]) +} + +// evictGapped removes the old transactions from the gapped reorder buffer. +// Concurrency: The caller must hold the pool lock before calling this function. +func (p *BlobPool) evictGapped() { + cutoff := time.Now().Add(-gappedLifetime) + for from, txs := range p.gapped { + nonce := p.state.GetNonce(from) + // Reuse the original slice to avoid extra allocations. + // This is safe because we only keep references to the original gappedTx objects, + // and we overwrite the slice for this account after filtering. + keep := txs[:0] + for i, gtx := range txs { + if gtx.Time().Before(cutoff) || gtx.Nonce() < nonce { + // Evict old or stale transactions + // Should we add stale to limbo here if it would belong? + delete(p.gappedSource, gtx.Hash()) + txs[i] = nil // Explicitly nil out evicted element + } else { + keep = append(keep, gtx) + } + } + if len(keep) < len(txs) { + log.Trace("Evicting old gapped blob transactions", "count", len(txs)-len(keep), "from", from) + } + if len(keep) == 0 { + delete(p.gapped, from) + } else { + p.gapped[from] = keep + } + } +} + // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. func (p *BlobPool) Stats() (int, int) { @@ -2140,9 +2048,15 @@ func (p *BlobPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*ty // Status returns the known status (unknown/pending/queued) of a transaction // identified by their hashes. func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus { - if p.Has(hash) { + p.lock.RLock() + defer p.lock.RUnlock() + + if p.lookup.exists(hash) { return txpool.TxStatusPending } + if _, gapped := p.gappedSource[hash]; gapped { + return txpool.TxStatusQueued + } return txpool.TxStatusUnknown } @@ -2192,6 +2106,11 @@ func (p *BlobPool) Clear() { p.index = make(map[common.Address][]*blobTxMeta) p.spent = make(map[common.Address]*uint256.Int) + // Reset counters and the gapped buffer + p.stored = 0 + p.gapped = make(map[common.Address][]*types.Transaction) + p.gappedSource = make(map[common.Hash]common.Address) + var ( basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head.Load())) blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index ae0639d9da..f02d1fd193 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -104,11 +104,6 @@ type testBlockChain struct { blockTime *uint64 } -//nolint:unused -func (bc *testBlockChain) setHeadTime(time uint64) { - bc.blockTime = &time -} - func (bc *testBlockChain) Config() *params.ChainConfig { return bc.config } @@ -447,11 +442,11 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) { hashes = append(hashes, tx.vhashes...) } } - blobs1, _, proofs1, err := pool.GetBlobs(hashes, types.BlobSidecarVersion0, false) + blobs1, _, proofs1, err := pool.GetBlobs(hashes, types.BlobSidecarVersion0) if err != nil { t.Fatal(err) } - blobs2, _, proofs2, err := pool.GetBlobs(hashes, types.BlobSidecarVersion1, false) + blobs2, _, proofs2, err := pool.GetBlobs(hashes, types.BlobSidecarVersion1) if err != nil { t.Fatal(err) } @@ -1360,7 +1355,7 @@ func TestBlobCountLimit(t *testing.T) { // Check that first succeeds second fails. if errs[0] != nil { - t.Fatalf("expected tx with 7 blobs to succeed") + t.Fatalf("expected tx with 7 blobs to succeed, got %v", errs[0]) } if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) { t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1]) @@ -1387,9 +1382,10 @@ func TestAdd(t *testing.T) { } // addtx is a helper sender/tx tuple to represent a new tx addition type addtx struct { - from string - tx *types.BlobTx - err error + from string + tx *types.BlobTx + err error + check func(*BlobPool, *types.Transaction) bool } tests := []struct { @@ -1406,6 +1402,7 @@ func TestAdd(t *testing.T) { "bob": {balance: 21100 + blobSize, nonce: 1}, "claire": {balance: 21100 + blobSize}, "dave": {balance: 21100 + blobSize, nonce: 1}, + "eve": {balance: 21100 + blobSize, nonce: 10}, // High nonce to test gapped acceptance }, adds: []addtx{ { // New account, no previous txs: accept nonce 0 @@ -1433,6 +1430,14 @@ func TestAdd(t *testing.T) { tx: makeUnsignedTx(2, 1, 1, 1), err: core.ErrNonceTooHigh, }, + { // Old account, 10 txs in chain: 0 pending: accept nonce 11 as gapped + from: "eve", + tx: makeUnsignedTx(11, 1, 1, 1), + err: nil, + check: func(pool *BlobPool, tx *types.Transaction) bool { + return pool.Status(tx.Hash()) == txpool.TxStatusQueued + }, + }, }, }, // Transactions from already pooled accounts should only be accepted if @@ -1793,15 +1798,28 @@ func TestAdd(t *testing.T) { t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, errs[0], add.err) } if add.err == nil { - size, exist := pool.lookup.sizeOfTx(signed.Hash()) - if !exist { - t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j) + // first check if tx is in the pool (reorder queue or pending) + if !pool.Has(signed.Hash()) { + t.Errorf("test %d, tx %d: added transaction not found in pool", i, j) + } + // if it is pending, check if size matches + if pool.Status(signed.Hash()) == txpool.TxStatusPending { + size, exist := pool.lookup.sizeOfTx(signed.Hash()) + if !exist { + t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j) + } + if size != signed.Size() { + t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v", + i, j, size, signed.Size()) + } } - if size != signed.Size() { - t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v", - i, j, size, signed.Size()) + } + if add.check != nil { + if !add.check(pool, signed) { + t.Errorf("test %d, tx %d: custom check failed", i, j) } } + // Verify the pool internals after each addition verifyPoolInternals(t, pool) } verifyPoolInternals(t, pool) @@ -1923,7 +1941,6 @@ func TestGetBlobs(t *testing.T) { limit int fillRandom bool // Whether to randomly fill some of the requested blobs with unknowns version byte // Blob sidecar version to request - convert bool // Whether to convert version on retrieval }{ { start: 0, limit: 6, @@ -1989,11 +2006,6 @@ func TestGetBlobs(t *testing.T) { start: 0, limit: 18, fillRandom: true, version: types.BlobSidecarVersion1, }, - { - start: 0, limit: 18, fillRandom: true, - version: types.BlobSidecarVersion1, - convert: true, // Convert some version 0 blobs to version 1 while retrieving - }, } for i, c := range cases { var ( @@ -2015,7 +2027,7 @@ func TestGetBlobs(t *testing.T) { filled[len(vhashes)] = struct{}{} vhashes = append(vhashes, testrand.Hash()) } - blobs, _, proofs, err := pool.GetBlobs(vhashes, c.version, c.convert) + blobs, _, proofs, err := pool.GetBlobs(vhashes, c.version) if err != nil { t.Errorf("Unexpected error for case %d, %v", i, err) } @@ -2041,8 +2053,7 @@ func TestGetBlobs(t *testing.T) { // If an item is missing, but shouldn't, error if blobs[j] == nil || proofs[j] == nil { // This is only an error if there was no version mismatch - if c.convert || - (c.version == types.BlobSidecarVersion1 && 6 <= testBlobIndex && testBlobIndex < 12) || + if (c.version == types.BlobSidecarVersion1 && 6 <= testBlobIndex && testBlobIndex < 12) || (c.version == types.BlobSidecarVersion0 && (testBlobIndex < 6 || 12 <= testBlobIndex)) { t.Errorf("tracked blob retrieval failed: item %d, hash %x", j, vhashes[j]) } @@ -2069,188 +2080,6 @@ func TestGetBlobs(t *testing.T) { _ = pool.Close() } -// TestSidecarConversion will verify that after the Osaka fork, all legacy -// sidecars in the pool are successfully convert to v1 sidecars. -func TestSidecarConversion(t *testing.T) { - t.Skip("bor: skipping as it's not relevant to bor") - // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) - - // Create a temporary folder for the persistent backend - storage := t.TempDir() - _ = os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) - - var ( - preOsakaTxs = make(types.Transactions, 10) - postOsakaTxs = make(types.Transactions, 3) - keys = make([]*ecdsa.PrivateKey, len(preOsakaTxs)+len(postOsakaTxs)) - addrs = make([]common.Address, len(preOsakaTxs)+len(postOsakaTxs)) - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - ) - for i := range keys { - keys[i], _ = crypto.GenerateKey() - addrs[i] = crypto.PubkeyToAddress(keys[i].PublicKey) - statedb.AddBalance(addrs[i], uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) - } - for i := range preOsakaTxs { - preOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 2, 0, keys[i], types.BlobSidecarVersion0) - } - for i := range postOsakaTxs { - if i == 0 { - // First has a v0 sidecar. - postOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 1, 0, keys[len(preOsakaTxs)+i], types.BlobSidecarVersion0) - } - postOsakaTxs[i] = makeMultiBlobTx(0, 1, 1000, 100, 1, 0, keys[len(preOsakaTxs)+i], types.BlobSidecarVersion1) - } - _, _ = statedb.Commit(0, true, false) - - // Test plan: - // 1) Create a bunch v0 sidecar txs and add to pool before Osaka. - // 2) Pass in new Osaka header to activate the conversion thread. - // 3) Continue adding both v0 and v1 transactions to the pool. - // 4) Verify that as additional blocks come in, transactions involved in the - // migration are correctly discarded. - - config := ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - LondonBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - CancunBlock: big.NewInt(0), - PragueBlock: big.NewInt(0), - OsakaBlock: big.NewInt(1), - BlobScheduleConfig: params.DefaultBlobSchedule, - } - chain := &testBlockChain{ - config: config, - basefee: uint256.NewInt(1050), - blobfee: uint256.NewInt(105), - statedb: statedb, - blocks: make(map[uint64]*types.Block), - } - - // Create 3 blocks: - // - the current block, before Osaka - // - the first block after Osaka - // - another post-Osaka block with several transactions in it - header0 := chain.CurrentBlock() - header0.Time = 0 - chain.blocks[0] = types.NewBlockWithHeader(header0) - - header1 := chain.CurrentBlock() - header1.Number = big.NewInt(1) - header1.Time = 1 - chain.blocks[1] = types.NewBlockWithHeader(header1) - - header2 := chain.CurrentBlock() - header2.Time = 2 - header2.Number = big.NewInt(2) - - // Make a copy of one of the pre-Osaka transactions and convert it to v1 here - // so that we can add it to the pool later and ensure a duplicate is not added - // by the conversion queue. - tx := preOsakaTxs[len(preOsakaTxs)-1] - sc := *tx.BlobTxSidecar() // copy sidecar - _ = sc.ToV1() - tx.WithBlobTxSidecar(&sc) - - block2 := types.NewBlockWithHeader(header2).WithBody(types.Body{Transactions: append(postOsakaTxs, tx)}) - chain.blocks[2] = block2 - - pool := New(Config{Datadir: storage}, chain, nil) - if err := pool.Init(1, header0, newReserver()); err != nil { - t.Fatalf("failed to create blob pool: %v", err) - } - - errs := pool.Add(preOsakaTxs, true) - for i, err := range errs { - if err != nil { - t.Errorf("failed to insert blob tx from %s: %s", addrs[i], errs[i]) - } - } - - // Kick off migration. - pool.Reset(header0, header1) - - // Add the v0 sidecar tx, but don't block so we can keep doing other stuff - // while it converts the sidecar. - addDone := make(chan struct{}) - go func() { - pool.Add(types.Transactions{postOsakaTxs[0]}, false) - close(addDone) - }() - - // Add the post-Osaka v1 sidecar txs. - errs = pool.Add(postOsakaTxs[1:], false) - for _, err := range errs { - if err != nil { - t.Fatalf("expected tx add to succeed: %v", err) - } - } - - // Wait for the first tx's conversion to complete, then check that all - // transactions added after Osaka can be accounted for in the pool. - <-addDone - pending := pool.Pending(txpool.PendingFilter{BlobTxs: true, BlobVersion: types.BlobSidecarVersion1}, nil) - for _, tx := range postOsakaTxs { - from, _ := pool.signer.Sender(tx) - if len(pending[from]) != 1 || pending[from][0].Hash != tx.Hash() { - t.Fatalf("expected post-Osaka txs to be pending") - } - } - - // Now update the pool with the next block. This should cause the pool to - // clear out the post-Osaka txs since they were included in block 2. Since the - // test blockchain doesn't manage nonces, we'll just do that manually before - // the reset is called. Don't forget about the pre-Osaka transaction we also - // added to block 2! - for i := range postOsakaTxs { - statedb.SetNonce(addrs[len(preOsakaTxs)+i], 1, tracing.NonceChangeEoACall) - } - statedb.SetNonce(addrs[len(preOsakaTxs)-1], 1, tracing.NonceChangeEoACall) - pool.Reset(header1, block2.Header()) - - // Now verify no post-Osaka transactions are tracked by the pool. - for i, tx := range postOsakaTxs { - if pool.Get(tx.Hash()) != nil { - t.Fatalf("expected txs added post-osaka to have been placed in limbo due to inclusion in a block: index %d, hash %s", i, tx.Hash()) - } - } - - // Wait for the pool migration to complete. - <-pool.cQueue.anyBillyConversionDone - - // Verify all transactions in the pool were converted and verify the - // subsequent cell proofs. - count, _ := pool.Stats() - if count != len(preOsakaTxs)-1 { - t.Errorf("expected pending count to match initial tx count: pending=%d, expected=%d", count, len(preOsakaTxs)-1) - } - for addr, acc := range pool.index { - for _, m := range acc { - if m.version != types.BlobSidecarVersion1 { - t.Errorf("expected sidecar to have been converted: from %s, hash %s", addr, m.hash) - } - tx := pool.Get(m.hash) - if tx == nil { - t.Errorf("failed to get tx by hash: %s", m.hash) - } - sc := tx.BlobTxSidecar() - if sc != nil { - if err := kzg4844.VerifyCellProofs(sc.Blobs, sc.Commitments, sc.Proofs); err != nil { - t.Errorf("failed to verify cell proofs for tx %s after conversion: %s", m.hash, err) - } - } - } - } - - verifyPoolInternals(t, pool) - - // Launch conversion a second time. - // This is just a sanity check to ensure we can handle it. - pool.Reset(header0, header1) - - _ = pool.Close() -} - // fakeBilly is a billy.Database implementation which just drops data on the floor. type fakeBilly struct { billy.Database @@ -2332,6 +2161,7 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) { MinTip: uint256.NewInt(1), BaseFee: chain.basefee, BlobFee: chain.blobfee, + BlobTxs: true, }, nil) if len(p) != int(capacity) { b.Fatalf("have %d want %d", len(p), capacity) diff --git a/core/txpool/blobpool/conversion.go b/core/txpool/blobpool/conversion.go deleted file mode 100644 index 95828d83b2..0000000000 --- a/core/txpool/blobpool/conversion.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2025 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package blobpool - -import ( - "errors" - "slices" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" -) - -// maxPendingConversionTasks caps the number of pending conversion tasks. This -// prevents excessive memory usage; the worst-case scenario (2k transactions -// with 6 blobs each) would consume approximately 1.5GB of memory. -const maxPendingConversionTasks = 2048 - -// txConvert represents a conversion task with an attached legacy blob transaction. -type txConvert struct { - tx *types.Transaction // Legacy blob transaction - done chan error // Channel for signaling back if the conversion succeeds -} - -// conversionQueue is a dedicated queue for converting legacy blob transactions -// received from the network after the Osaka fork. Since conversion is expensive, -// it is performed in the background by a single thread, ensuring the main Geth -// process is not overloaded. -type conversionQueue struct { - tasks chan *txConvert - startBilly chan func() - quit chan struct{} - closed chan struct{} - - billyQueue []func() - billyTaskDone chan struct{} - - // This channel will be closed when the first billy conversion finishes. - // It's added for unit tests to synchronize with the conversion progress. - anyBillyConversionDone chan struct{} -} - -// newConversionQueue constructs the conversion queue. -func newConversionQueue() *conversionQueue { - q := &conversionQueue{ - tasks: make(chan *txConvert), - startBilly: make(chan func()), - quit: make(chan struct{}), - closed: make(chan struct{}), - anyBillyConversionDone: make(chan struct{}), - } - go q.loop() - return q -} - -// convert accepts a legacy blob transaction with version-0 blobs and queues it -// for conversion. -// -// This function may block for a long time until the transaction is processed. -func (q *conversionQueue) convert(tx *types.Transaction) error { - done := make(chan error, 1) - select { - case q.tasks <- &txConvert{tx: tx, done: done}: - return <-done - case <-q.closed: - return errors.New("conversion queue closed") - } -} - -// launchBillyConversion starts a conversion task in the background. -func (q *conversionQueue) launchBillyConversion(fn func()) error { - select { - case q.startBilly <- fn: - return nil - case <-q.closed: - return errors.New("conversion queue closed") - } -} - -// close terminates the conversion queue. -func (q *conversionQueue) close() { - select { - case <-q.closed: - return - default: - close(q.quit) - <-q.closed - } -} - -// run converts a batch of legacy blob txs to the new cell proof format. -func (q *conversionQueue) run(tasks []*txConvert, done chan struct{}, interrupt *atomic.Int32) { - defer close(done) - - for _, t := range tasks { - if interrupt != nil && interrupt.Load() != 0 { - t.done <- errors.New("conversion is interrupted") - continue - } - sidecar := t.tx.BlobTxSidecar() - if sidecar == nil { - t.done <- errors.New("tx without sidecar") - continue - } - // Run the conversion, the original sidecar will be mutated in place - start := time.Now() - err := sidecar.ToV1() - t.done <- err - log.Trace("Converted legacy blob tx", "hash", t.tx.Hash(), "err", err, "elapsed", common.PrettyDuration(time.Since(start))) - } -} - -func (q *conversionQueue) loop() { - defer close(q.closed) - - var ( - done chan struct{} // Non-nil if background routine is active - interrupt *atomic.Int32 // Flag to signal conversion interruption - - // The pending tasks for sidecar conversion. We assume the number of legacy - // blob transactions requiring conversion will not be excessive. However, - // a hard cap is applied as a protective measure. - txTasks []*txConvert - - firstBilly = true - ) - - for { - select { - case t := <-q.tasks: - if len(txTasks) >= maxPendingConversionTasks { - t.done <- errors.New("conversion queue is overloaded") - continue - } - txTasks = append(txTasks, t) - - // Launch the background conversion thread if it's idle - if done == nil { - done, interrupt = make(chan struct{}), new(atomic.Int32) - - tasks := slices.Clone(txTasks) - txTasks = txTasks[:0] - go q.run(tasks, done, interrupt) - } - - case <-done: - done, interrupt = nil, nil - - case fn := <-q.startBilly: - q.billyQueue = append(q.billyQueue, fn) - q.runNextBillyTask() - - case <-q.billyTaskDone: - if firstBilly { - close(q.anyBillyConversionDone) - firstBilly = false - } - q.runNextBillyTask() - - case <-q.quit: - if done != nil { - log.Debug("Waiting for blob proof conversion to exit") - interrupt.Store(1) - <-done - } - if q.billyTaskDone != nil { - log.Debug("Waiting for blobpool billy conversion to exit") - <-q.billyTaskDone - } - return - } - } -} - -func (q *conversionQueue) runNextBillyTask() { - if len(q.billyQueue) == 0 { - q.billyTaskDone = nil - return - } - - fn := q.billyQueue[0] - q.billyQueue = append(q.billyQueue[:0], q.billyQueue[1:]...) - - done := make(chan struct{}) - go func() { defer close(done); fn() }() - q.billyTaskDone = done -} diff --git a/core/txpool/blobpool/conversion_test.go b/core/txpool/blobpool/conversion_test.go deleted file mode 100644 index dd18cea900..0000000000 --- a/core/txpool/blobpool/conversion_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2025 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package blobpool - -import ( - "crypto/ecdsa" - "crypto/sha256" - "testing" - - "github.com/holiman/uint256" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/kzg4844" - "github.com/ethereum/go-ethereum/params" -) - -// createV1BlobTx creates a blob transaction with version 1 sidecar for testing. -func createV1BlobTx(nonce uint64, key *ecdsa.PrivateKey) *types.Transaction { - blob := &kzg4844.Blob{byte(nonce)} - commitment, _ := kzg4844.BlobToCommitment(blob) - cellProofs, _ := kzg4844.ComputeCellProofs(blob) - - blobtx := &types.BlobTx{ - ChainID: uint256.MustFromBig(params.MainnetChainConfig.ChainID), - Nonce: nonce, - GasTipCap: uint256.NewInt(1), - GasFeeCap: uint256.NewInt(1000), - Gas: 21000, - BlobFeeCap: uint256.NewInt(100), - BlobHashes: []common.Hash{kzg4844.CalcBlobHashV1(sha256.New(), &commitment)}, - Value: uint256.NewInt(100), - Sidecar: types.NewBlobTxSidecar(types.BlobSidecarVersion1, []kzg4844.Blob{*blob}, []kzg4844.Commitment{commitment}, cellProofs), - } - return types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx) -} - -func TestConversionQueueBasic(t *testing.T) { - queue := newConversionQueue() - defer queue.close() - - key, _ := crypto.GenerateKey() - tx := makeTx(0, 1, 1, 1, key) - if err := queue.convert(tx); err != nil { - t.Fatalf("Expected successful conversion, got error: %v", err) - } - if tx.BlobTxSidecar().Version != types.BlobSidecarVersion1 { - t.Errorf("Expected sidecar version to be %d, got %d", types.BlobSidecarVersion1, tx.BlobTxSidecar().Version) - } -} - -func TestConversionQueueV1BlobTx(t *testing.T) { - t.Skip("bor: skipping as it's not relevant to bor") - queue := newConversionQueue() - defer queue.close() - - key, _ := crypto.GenerateKey() - tx := createV1BlobTx(0, key) - version := tx.BlobTxSidecar().Version - - err := queue.convert(tx) - if err != nil { - t.Fatalf("Expected successful conversion, got error: %v", err) - } - if tx.BlobTxSidecar().Version != version { - t.Errorf("Expected sidecar version to remain %d, got %d", version, tx.BlobTxSidecar().Version) - } -} - -func TestConversionQueueClosed(t *testing.T) { - queue := newConversionQueue() - - // Close the queue first - queue.close() - key, _ := crypto.GenerateKey() - tx := makeTx(0, 1, 1, 1, key) - - err := queue.convert(tx) - if err == nil { - t.Fatal("Expected error when converting on closed queue, got nil") - } -} - -func TestConversionQueueDoubleClose(t *testing.T) { - queue := newConversionQueue() - queue.close() - queue.close() // Should not panic -} diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go index d0d12bca61..4772e2f70c 100644 --- a/core/txpool/blobpool/limbo.go +++ b/core/txpool/blobpool/limbo.go @@ -22,7 +22,6 @@ import ( "github.com/holiman/billy" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -58,7 +57,7 @@ func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) { } // Create new slotter for pre-Osaka blob configuration. - slotter := newSlotter(eip4844.LatestMaxBlobsPerBlock(config)) + slotter := newSlotter(params.BlobTxMaxBlobs) // See if we need to migrate the limbo after fusaka. slotter, err := tryMigrate(config, slotter, datadir) diff --git a/core/txpool/blobpool/lookup.go b/core/txpool/blobpool/lookup.go index 874ca85b8c..7607cd487a 100644 --- a/core/txpool/blobpool/lookup.go +++ b/core/txpool/blobpool/lookup.go @@ -110,13 +110,3 @@ func (l *lookup) untrack(tx *blobTxMeta) { } } } - -// update updates the transaction index. It should only be used in the conversion. -func (l *lookup) update(hash common.Hash, id uint64, size uint64) bool { - meta, exists := l.txIndex[hash] - if !exists { - return false - } - meta.id, meta.size = id, size - return true -} diff --git a/core/txpool/blobpool/slotter.go b/core/txpool/blobpool/slotter.go index 0b1d964078..a462e234e2 100644 --- a/core/txpool/blobpool/slotter.go +++ b/core/txpool/blobpool/slotter.go @@ -19,7 +19,6 @@ package blobpool import ( "github.com/holiman/billy" - "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/params" ) @@ -56,7 +55,7 @@ func tryMigrate(config *params.ChainConfig, slotter billy.SlotSizeFn, datadir st store.Close() } // Set the slotter to the format now that the Osaka is active. - slotter = newSlotterEIP7594(eip4844.LatestMaxBlobsPerBlock(config)) + slotter = newSlotterEIP7594(params.BlobTxMaxBlobs) } return slotter, nil } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index ea90e641a0..8ba714ca2a 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -123,6 +123,9 @@ var ( queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) slotsGauge = metrics.NewRegisteredGauge("txpool/slots", nil) + pendingAddrsGauge = metrics.NewRegisteredGauge("txpool/pending/accounts", nil) + queuedAddrsGauge = metrics.NewRegisteredGauge("txpool/queued/accounts", nil) + resetCacheGauge = metrics.NewRegisteredGauge("txpool/resetcache", nil) reheapTimer = metrics.NewRegisteredTimer("txpool/reheap", nil) urgentHeapInitTimer = metrics.NewRegisteredTimer("txpool/heapinit/urgent", nil) @@ -188,7 +191,7 @@ type Config struct { AccountQueue uint64 // Maximum number of non-executable transaction slots permitted per account GlobalQueue uint64 // Maximum number of non-executable transaction slots for all accounts - Lifetime time.Duration // Maximum amount of time non-executable transaction are queued + Lifetime time.Duration // Maximum amount of time an account can remain stale in the non-executable pool AllowUnprotectedTxs bool // Allow non-EIP-155 transactions // Transaction filtering configuration @@ -381,7 +384,12 @@ func New(config Config, chain BlockChain, options ...func(pool *LegacyPool)) *Le // Filter returns whether the given transaction can be consumed by the legacy // pool, specifically, whether it is a Legacy, AccessList or Dynamic transaction. func (pool *LegacyPool) Filter(tx *types.Transaction) bool { - switch tx.Type() { + return pool.FilterType(tx.Type()) +} + +// FilterType returns whether the legacy pool supports the given transaction type. +func (pool *LegacyPool) FilterType(kind byte) bool { + switch kind { case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType, types.SetCodeTxType: return true default: @@ -1154,7 +1162,7 @@ func (pool *LegacyPool) add(tx *types.Transaction, async bool) (replaced bool, e pool.queueTxEvent(tx) log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) - // Successful promotion, bump the heartbeat + // Successful replacement. If needed, bump the heartbeat giving more time to queued txs. pool.queue.bump(from) stage2Duration = time.Since(stage2Time) return old != nil, nil @@ -1226,6 +1234,7 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ // Try to insert the transaction into the pending queue if pool.pending[addr] == nil { pool.pending[addr] = newList(true) + pendingAddrsGauge.Inc(1) } list := pool.pending[addr] @@ -1251,7 +1260,7 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ // Set the potentially new pending nonce and notify any subsystems of the new tx pool.pendingNonces.set(addr, tx.Nonce()+1) - // Successful promotion, bump the heartbeat + // Successful promotion, bump the heartbeat, giving more time to queued txs. pool.queue.bump(addr) return true } @@ -1472,6 +1481,7 @@ func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool, unreserve bo // If no more pending transactions are left, remove the list if pending.Empty() { delete(pool.pending, addr) + pendingAddrsGauge.Dec(1) } // Postpone any invalidated transactions for _, tx := range invalids { @@ -1975,6 +1985,8 @@ func (pool *LegacyPool) demoteUnexecutables() { // Internal shuffle shouldn't touch the lookup set. pool.enqueueTx(hash, tx, false) } + pool.priced.Removed(len(olds) + len(drops)) + // bor: Drop all transactions that no longer have valid TxOptions txConditionalsRemoved := list.FilterTxConditional(pool.currentState, currentHeader) @@ -2001,6 +2013,7 @@ func (pool *LegacyPool) demoteUnexecutables() { if list.Empty() { // Delete the entire pending entry if it became empty. delete(pool.pending, addr) + pendingAddrsGauge.Dec(1) if _, ok := pool.queue.get(addr); !ok { pool.reserver.Release(addr) } @@ -2265,6 +2278,13 @@ func (pool *LegacyPool) Clear() { pool.pending = make(map[common.Address]*list) pool.queue = newQueue(pool.config, pool.signer) pool.pendingNonces = newNoncer(pool.currentState) + + // Reset gauges + pendingGauge.Update(0) + queuedGauge.Update(0) + slotsGauge.Update(0) + pendingAddrsGauge.Update(0) + queuedAddrsGauge.Update(0) } // HasPendingAuth returns a flag indicating whether there are pending diff --git a/core/txpool/legacypool/queue.go b/core/txpool/legacypool/queue.go index a889debe37..35beaded12 100644 --- a/core/txpool/legacypool/queue.go +++ b/core/txpool/legacypool/queue.go @@ -88,8 +88,12 @@ func (q *queue) get(addr common.Address) (*list, bool) { return l, ok } +// bump updates the heartbeat for the given account address. +// If the address is unknown, the call is a no-op. func (q *queue) bump(addr common.Address) { - q.beats[addr] = time.Now() + if _, ok := q.beats[addr]; ok { + q.beats[addr] = time.Now() + } } func (q *queue) addresses() []common.Address { @@ -114,6 +118,7 @@ func (q *queue) remove(addr common.Address, tx *types.Transaction) { if future.Empty() { delete(q.queued, addr) delete(q.beats, addr) + queuedAddrsGauge.Dec(1) } } } @@ -123,6 +128,7 @@ func (q *queue) add(tx *types.Transaction) (*common.Hash, error) { from, _ := types.Sender(q.signer, tx) // already validated if q.queued[from] == nil { q.queued[from] = newList(false) + queuedAddrsGauge.Inc(1) } inserted, old := q.queued[from].Add(tx, q.config.PriceBump) if !inserted { @@ -200,6 +206,7 @@ func (q *queue) promoteExecutables(accounts []common.Address, gasLimit uint64, c if list.Empty() { delete(q.queued, addr) delete(q.beats, addr) + queuedAddrsGauge.Dec(1) removedAddresses = append(removedAddresses, addr) } } diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 4b2e258930..f876217e89 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -102,6 +102,9 @@ type SubPool interface { // to this particular subpool. Filter(tx *types.Transaction) bool + // FilterType returns whether the subpool supports the given transaction type. + FilterType(kind byte) bool + // Init sets the base parameters of the subpool, allowing it to load any saved // transactions from disk and also permitting internal maintenance routines to // start up. diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 33ffb7e550..0d595b2f34 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -553,3 +553,14 @@ func (p *TxPool) Clear() { subpool.Clear() } } + +// FilterType returns whether a transaction with the given type is supported +// (can be added) by the pool. +func (p *TxPool) FilterType(kind byte) bool { + for _, subpool := range p.subpools { + if subpool.FilterType(kind) { + return true + } + } + return false +} diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 73ab55bd97..e93acff6c6 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -164,6 +164,15 @@ func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationO if sidecar == nil { return errors.New("missing sidecar in blob transaction") } + // Ensure the sidecar is constructed with the correct version, consistent + // with the current fork. + version := types.BlobSidecarVersion0 + if opts.Config.IsOsaka(head.Number) { + version = types.BlobSidecarVersion1 + } + if sidecar.Version != version { + return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", version, sidecar.Version) + } // Ensure the blob fee cap satisfies the minimum blob gas price if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) diff --git a/core/types/block.go b/core/types/block.go index 3226ce8851..277903c9d2 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -28,8 +28,6 @@ import ( "sync/atomic" "time" - "github.com/ethereum/go-verkle" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/log" @@ -71,13 +69,6 @@ func (n *BlockNonce) UnmarshalText(input []byte) error { return hexutil.UnmarshalFixedText("BlockNonce", input, n[:]) } -// ExecutionWitness represents the witness + proof used in a verkle context, -// to provide the ability to execute a block statelessly. -type ExecutionWitness struct { - StateDiff verkle.StateDiff `json:"stateDiff"` - VerkleProof *verkle.VerkleProof `json:"verkleProof"` -} - //go:generate go run github.com/fjl/gencodec -type Header -field-override headerMarshaling -out gen_header_json.go //go:generate go run ../../rlp/rlpgen -type Header -out gen_header_rlp.go @@ -299,11 +290,6 @@ type Block struct { transactions Transactions withdrawals Withdrawals - // witness is not an encoded part of the block body. - // It is held in Block in order for easy relaying to the places - // that process it. - witness *ExecutionWitness - // caches hash atomic.Pointer[common.Hash] size atomic.Uint64 @@ -646,9 +632,6 @@ func (b *Block) BlobGasUsed() *uint64 { return nil } -// ExecutionWitness returns the verkle execution witneess + proof for a block -func (b *Block) ExecutionWitness() *ExecutionWitness { return b.witness } - // Size returns the true RLP encoded storage size of the block, either by encoding // and returning it, or returning a previously cached value. func (b *Block) Size() uint64 { @@ -714,7 +697,6 @@ func (b *Block) WithSeal(header *Header) *Block { transactions: b.transactions, uncles: b.uncles, withdrawals: b.withdrawals, - witness: b.witness, } } @@ -726,7 +708,6 @@ func (b *Block) WithBody(body Body) *Block { transactions: slices.Clone(body.Transactions), uncles: make([]*Header, len(body.Uncles)), withdrawals: slices.Clone(body.Withdrawals), - witness: b.witness, } for i := range body.Uncles { block.uncles[i] = CopyHeader(body.Uncles[i]) @@ -734,16 +715,6 @@ func (b *Block) WithBody(body Body) *Block { return block } -func (b *Block) WithWitness(witness *ExecutionWitness) *Block { - return &Block{ - header: b.header, - transactions: b.transactions, - uncles: b.uncles, - withdrawals: b.withdrawals, - witness: witness, - } -} - // Hash returns the keccak256 hash of b's header. // The hash is computed on the first call and cached thereafter. func (b *Block) Hash() common.Hash { diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index f412b337a4..9e7b486e5d 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -425,7 +425,7 @@ func (s eip2930Signer) Hash(tx *Transaction) common.Hash { // EIP155Signer implements Signer using the EIP-155 rules. This accepts transactions which // are replay-protected as well as unprotected homestead transactions. type EIP155Signer struct { - chainId, chainIdMul *big.Int + chainId *big.Int } func NewEIP155Signer(chainId *big.Int) EIP155Signer { @@ -433,8 +433,7 @@ func NewEIP155Signer(chainId *big.Int) EIP155Signer { chainId = new(big.Int) } return EIP155Signer{ - chainId: chainId, - chainIdMul: new(big.Int).Mul(chainId, big.NewInt(2)), + chainId: chainId, } } @@ -460,7 +459,8 @@ func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) { return common.Address{}, fmt.Errorf("%w: have %d want %d", ErrInvalidChainId, tx.ChainId(), s.chainId) } V, R, S := tx.RawSignatureValues() - V = new(big.Int).Sub(V, s.chainIdMul) + V = new(big.Int).Sub(V, s.chainId) + V = new(big.Int).Sub(V, s.chainId) V.Sub(V, big8) return recoverPlain(s.Hash(tx), R, S, V, true) } @@ -474,7 +474,8 @@ func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big R, S, V = decodeSignature(sig) if s.chainId.Sign() != 0 { V = big.NewInt(int64(sig[64] + 35)) - V.Add(V, s.chainIdMul) + V.Add(V, s.chainId) + V.Add(V, s.chainId) } return R, S, V, nil } diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go index 328e05b494..bab7802418 100644 --- a/core/types/tx_blob.go +++ b/core/types/tx_blob.go @@ -119,8 +119,8 @@ func (sc *BlobTxSidecar) ToV1() error { } if sc.Version == BlobSidecarVersion0 { proofs := make([]kzg4844.Proof, 0, len(sc.Blobs)*kzg4844.CellProofsPerBlob) - for _, blob := range sc.Blobs { - cellProofs, err := kzg4844.ComputeCellProofs(&blob) + for i := range sc.Blobs { + cellProofs, err := kzg4844.ComputeCellProofs(&sc.Blobs[i]) if err != nil { return err } diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go deleted file mode 100644 index 0e38e5994f..0000000000 --- a/core/verkle_witness_test.go +++ /dev/null @@ -1,1119 +0,0 @@ -// Copyright 2024 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "bytes" - "encoding/binary" - "encoding/hex" - "fmt" - "math/big" - "slices" - "testing" - - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/beacon" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/tracing" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/ethereum/go-ethereum/triedb" -) - -var ( - testVerkleChainConfig = ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), - ShanghaiBlock: big.NewInt(0), - VerkleBlock: big.NewInt(0), - TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, - BlobScheduleConfig: ¶ms.BlobScheduleConfig{ - Verkle: params.DefaultPragueBlobConfig, - }, - // TODO uncomment when proof generation is merged - // ProofInBlocks: true, - } - testKaustinenLikeChainConfig = ¶ms.ChainConfig{ - ChainID: big.NewInt(69420), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), - ShanghaiBlock: big.NewInt(0), - VerkleBlock: big.NewInt(0), - TerminalTotalDifficulty: common.Big0, - EnableVerkleAtGenesis: true, - BlobScheduleConfig: ¶ms.BlobScheduleConfig{ - Verkle: params.DefaultPragueBlobConfig, - }, - } -) - -func TestProcessVerkle(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - - var ( - code = common.FromHex(`6060604052600a8060106000396000f360606040526008565b00`) - intrinsicContractCreationGas, _ = IntrinsicGas(code, nil, nil, true, true, true, true) - // A contract creation that calls EXTCODECOPY in the constructor. Used to ensure that the witness - // will not contain that copied data. - // Source: https://gist.github.com/gballet/a23db1e1cb4ed105616b5920feb75985 - codeWithExtCodeCopy = common.FromHex(`0x60806040526040516100109061017b565b604051809103906000f08015801561002c573d6000803e3d6000fd5b506000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555034801561007857600080fd5b5060008067ffffffffffffffff8111156100955761009461024a565b5b6040519080825280601f01601f1916602001820160405280156100c75781602001600182028036833780820191505090505b50905060008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690506020600083833c81610101906101e3565b60405161010d90610187565b61011791906101a3565b604051809103906000f080158015610133573d6000803e3d6000fd5b50600160006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550505061029b565b60d58061046783390190565b6102068061053c83390190565b61019d816101d9565b82525050565b60006020820190506101b86000830184610194565b92915050565b6000819050602082019050919050565b600081519050919050565b6000819050919050565b60006101ee826101ce565b826101f8846101be565b905061020381610279565b925060208210156102435761023e7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8360200360080261028e565b831692505b5050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600061028582516101d9565b80915050919050565b600082821b905092915050565b6101bd806102aa6000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f566852414610030575b600080fd5b61003861004e565b6040516100459190610146565b60405180910390f35b6000600160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166381ca91d36040518163ffffffff1660e01b815260040160206040518083038186803b1580156100b857600080fd5b505afa1580156100cc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906100f0919061010a565b905090565b60008151905061010481610170565b92915050565b6000602082840312156101205761011f61016b565b5b600061012e848285016100f5565b91505092915050565b61014081610161565b82525050565b600060208201905061015b6000830184610137565b92915050565b6000819050919050565b600080fd5b61017981610161565b811461018457600080fd5b5056fea2646970667358221220a6a0e11af79f176f9c421b7b12f441356b25f6489b83d38cc828a701720b41f164736f6c63430008070033608060405234801561001057600080fd5b5060b68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063ab5ed15014602d575b600080fd5b60336047565b604051603e9190605d565b60405180910390f35b60006001905090565b6057816076565b82525050565b6000602082019050607060008301846050565b92915050565b600081905091905056fea26469706673582212203a14eb0d5cd07c277d3e24912f110ddda3e553245a99afc4eeefb2fbae5327aa64736f6c63430008070033608060405234801561001057600080fd5b5060405161020638038061020683398181016040528101906100329190610063565b60018160001c6100429190610090565b60008190555050610145565b60008151905061005d8161012e565b92915050565b60006020828403121561007957610078610129565b5b60006100878482850161004e565b91505092915050565b600061009b826100f0565b91506100a6836100f0565b9250827fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff038211156100db576100da6100fa565b5b828201905092915050565b6000819050919050565b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600080fd5b610137816100e6565b811461014257600080fd5b50565b60b3806101536000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806381ca91d314602d575b600080fd5b60336047565b604051603e9190605a565b60405180910390f35b60005481565b6054816073565b82525050565b6000602082019050606d6000830184604d565b92915050565b600081905091905056fea26469706673582212209bff7098a2f526de1ad499866f27d6d0d6f17b74a413036d6063ca6a0998ca4264736f6c63430008070033`) - intrinsicCodeWithExtCodeCopyGas, _ = IntrinsicGas(codeWithExtCodeCopy, nil, nil, true, true, true, true) - signer = types.LatestSigner(testVerkleChainConfig) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - bcdb = rawdb.NewMemoryDatabase() // Database for the blockchain - coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") - gspec = &Genesis{ - Config: testVerkleChainConfig, - Alloc: GenesisAlloc{ - coinbase: { - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - params.BeaconRootsAddress: {Nonce: 1, Code: params.BeaconRootsCode, Balance: common.Big0}, - params.HistoryStorageAddress: {Nonce: 1, Code: params.HistoryStorageCode, Balance: common.Big0}, - params.WithdrawalQueueAddress: {Nonce: 1, Code: params.WithdrawalQueueCode, Balance: common.Big0}, - params.ConsolidationQueueAddress: {Nonce: 1, Code: params.ConsolidationQueueCode, Balance: common.Big0}, - }, - } - ) - // Verkle trees use the snapshot, which must be enabled before the - // data is saved into the tree+database. - // genesis := gspec.MustCommit(bcdb, triedb) - options := DefaultConfig().WithStateScheme(rawdb.PathScheme) - options.SnapshotLimit = 0 - blockchain, _ := NewBlockChain(bcdb, gspec, beacon.New(ethash.NewFaker()), options) - defer blockchain.Stop() - - txCost1 := params.TxGas - txCost2 := params.TxGas - contractCreationCost := intrinsicContractCreationGas + - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* creation with value */ - 739 /* execution costs */ - codeWithExtCodeCopyGas := intrinsicCodeWithExtCodeCopyGas + - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (tx) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at pc=0x20) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ - params.WitnessChunkReadCost + /* SLOAD in constructor */ - params.WitnessChunkWriteCost + /* SSTORE in constructor */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + params.WitnessBranchReadCost + params.WitnessBranchWriteCost + /* creation (CREATE at PC=0x121) */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #0 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #1 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #2 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #3 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #4 */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* code chunk #5 */ - params.WitnessChunkReadCost + /* SLOAD in constructor */ - params.WitnessChunkWriteCost + /* SSTORE in constructor */ - params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash for tx creation */ - 15*(params.WitnessChunkReadCost+params.WitnessChunkWriteCost) + /* code chunks #0..#14 */ - uint64(4844) /* execution costs */ - blockGasUsagesExpected := []uint64{ - txCost1*2 + txCost2, - txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas, - } - _, _, chain, _, proofs, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - // TODO need to check that the tx cost provided is the exact amount used (no remaining left-over) - tx, _ := types.SignTx(types.NewTransaction(uint64(i)*3, common.Address{byte(i), 2, 3}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+1, common.Address{}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+2, common.Address{}, big.NewInt(0), txCost2, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - - // Add two contract creations in block #2 - if i == 1 { - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 6, - Value: big.NewInt(16), - Gas: 3000000, - GasPrice: big.NewInt(875000000), - Data: code, - }) - gen.AddTx(tx) - - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 7, - Value: big.NewInt(0), - Gas: 3000000, - GasPrice: big.NewInt(875000000), - Data: codeWithExtCodeCopy, - }) - gen.AddTx(tx) - } - }) - - // Check proof for both blocks - err := verkle.Verify(proofs[0], gspec.ToBlock().Root().Bytes(), chain[0].Root().Bytes(), statediffs[0]) - if err != nil { - t.Fatal(err) - } - err = verkle.Verify(proofs[1], chain[0].Root().Bytes(), chain[1].Root().Bytes(), statediffs[1]) - if err != nil { - t.Fatal(err) - } - - t.Log("verified verkle proof, inserting blocks into the chain") - - for i, b := range chain { - fmt.Printf("%d %x\n", i, b.Root()) - } - endnum, err := blockchain.InsertChain(chain, false) - if err != nil { - t.Fatalf("block %d imported with error: %v", endnum, err) - } - - for i := range 2 { - b := blockchain.GetBlockByNumber(uint64(i) + 1) - if b == nil { - t.Fatalf("expected block %d to be present in chain", i+1) - } - if b.Hash() != chain[i].Hash() { - t.Fatalf("block #%d not found at expected height", b.NumberU64()) - } - if b.GasUsed() != blockGasUsagesExpected[i] { - t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), blockGasUsagesExpected[i], b.GasUsed()) - } - } -} - -func TestProcessParentBlockHash(t *testing.T) { - // This test uses blocks where, - // block 1 parent hash is 0x0100.... - // block 2 parent hash is 0x0200.... - // etc - checkBlockHashes := func(statedb *state.StateDB, isVerkle bool) { - statedb.SetNonce(params.HistoryStorageAddress, 1, tracing.NonceChangeUnspecified) - statedb.SetCode(params.HistoryStorageAddress, params.HistoryStorageCode, tracing.CodeChangeUnspecified) - // Process n blocks, from 1 .. num - var num = 2 - for i := 1; i <= num; i++ { - header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} - chainConfig := params.MergedTestChainConfig - if isVerkle { - chainConfig = testVerkleChainConfig - } - vmContext := NewEVMBlockContext(header, nil, new(common.Address)) - evm := vm.NewEVM(vmContext, statedb, chainConfig, vm.Config{}) - ProcessParentBlockHash(header.ParentHash, evm) - } - // Read block hashes for block 0 .. num-1 - for i := 0; i < num; i++ { - have, want := getContractStoredBlockHash(statedb, uint64(i), isVerkle), common.Hash{byte(i + 1)} - if have != want { - t.Errorf("block %d, verkle=%v, have parent hash %v, want %v", i, isVerkle, have, want) - } - } - } - t.Run("MPT", func(t *testing.T) { - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - checkBlockHashes(statedb, false) - }) - t.Run("Verkle", func(t *testing.T) { - db := rawdb.NewMemoryDatabase() - cacheConfig := DefaultConfig().WithStateScheme(rawdb.PathScheme) - cacheConfig.SnapshotLimit = 0 - triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) - statedb, _ := state.New(types.EmptyVerkleHash, state.NewDatabase(triedb, nil)) - checkBlockHashes(statedb, true) - }) -} - -// getContractStoredBlockHash is a utility method which reads the stored parent blockhash for block 'number' -func getContractStoredBlockHash(statedb *state.StateDB, number uint64, isVerkle bool) common.Hash { - ringIndex := number % params.HistoryServeWindow - var key common.Hash - binary.BigEndian.PutUint64(key[24:], ringIndex) - if isVerkle { - return statedb.GetState(params.HistoryStorageAddress, key) - } - return statedb.GetState(params.HistoryStorageAddress, key) -} - -// TestProcessVerkleInvalidContractCreation checks for several modes of contract creation failures -func TestProcessVerkleInvalidContractCreation(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - var ( - account1 = common.HexToAddress("0x687704DB07e902e9A8B3754031D168D46E3D586e") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(testKaustinenLikeChainConfig) - ) - // slightly modify it to suit the live txs from the testnet - gspec.Alloc[account2] = types.Account{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 1, - } - - // Create two blocks that reproduce what is happening on kaustinen. - // - The first block contains two failing contract creation transactions, that - // write to storage before they revert. - // - // - The second block contains a single failing contract creation transaction, - // that fails right off the bat. - genesisH, _, chain, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - for _, rlpData := range []string{ - // SSTORE at slot 41 and reverts - "f8d48084479c2c18830186a08080b8806000602955bda3f9600060ca55600060695523b360006039551983576000601255b0620c2fde2c592ac2600060bc55e0ac6000606455a63e22600060e655eb607e605c5360a2605d5360c7605e53601d605f5360eb606053606b606153608e60625360816063536079606453601e60655360fc60665360b7606753608b60685383021e7ca0cc20c65a97d2e526b8ec0f4266e8b01bdcde43b9aeb59d8bfb44e8eb8119c109a07a8e751813ae1b2ce734960dbc39a4f954917d7822a2c5d1dca18b06c584131f", - // SSTORE at slot 133 and reverts - "02f8db83010f2c01843b9aca0084479c2c18830186a08080b88060006085553fad6000600a55600060565555600060b55506600060cf557f1b8b38183e7bd1bdfaa7123c5a4976e54cce0e42049d841411978fd3595e25c66019527f0538943712953cf08900aae40222a40b2d5a4ac8075ad8cf0870e2be307edbb96039527f9f3174ff85024747041ae7a611acffb987c513c088d90ab288aec080a0cd6ac65ce2cb0a912371f6b5a551ba8caffc22ec55ad4d3cb53de41d05eb77b6a02e0dfe8513dfa6ec7bfd7eda6f5c0dac21b39b982436045e128cec46cfd3f960", - // this one is a simple transfer that succeeds, necessary to get the correct nonce in the other block. - "f8e80184479c2c18830186a094bbbbde4ca27f83fc18aa108170547ff57675936a80b8807ff71f7c15faadb969a76a5f54a81a0117e1e743cb7f24e378eda28442ea4c6eb6604a527fb5409e5718d44e23bfffac926e5ea726067f772772e7e19446acba0c853f62f5606a526020608a536088608b536039608c536004608d5360af608e537f7f7675d9f210e0a61564e6d11e7cd75f5bc9009ac9f6b94a0fc63035441a83021e7ba04a4a172d81ebb02847829b76a387ac09749c8b65668083699abe20c887fb9efca07c5b1a990702ec7b31a5e8e3935cd9a77649f8c25a84131229e24ab61aec6093", - } { - var tx = new(types.Transaction) - if err := tx.UnmarshalBinary(common.Hex2Bytes(rlpData)); err != nil { - t.Fatal(err) - } - gen.AddTx(tx) - } - } else { - var tx = new(types.Transaction) - // immediately reverts - if err := tx.UnmarshalBinary(common.Hex2Bytes("01f8d683010f2c028443ad7d0e830186a08080b880b00e7fa3c849dce891cce5fae8a4c46cbb313d6aec0c0ffe7863e05fb7b22d4807674c6055527ffbfcb0938f3e18f7937aa8fa95d880afebd5c4cec0d85186095832d03c85cf8a60755260ab60955360cf6096536066609753606e60985360fa609953609e609a53608e609b536024609c5360f6609d536072609e5360a4609fc080a08fc6f7101f292ff1fb0de8ac69c2d320fbb23bfe61cf327173786ea5daee6e37a044c42d91838ef06646294bf4f9835588aee66243b16a66a2da37641fae4c045f")); err != nil { - t.Fatal(err) - } - gen.AddTx(tx) - } - }) - - tx1ContractAddress := crypto.CreateAddress(account1, 0) - tx1ContractStem := utils.GetTreeKey(tx1ContractAddress[:], uint256.NewInt(0), 105) - tx1ContractStem = tx1ContractStem[:31] - - tx2ContractAddress := crypto.CreateAddress(account2, 1) - tx2SlotKey := [32]byte{} - tx2SlotKey[31] = 133 - tx2ContractStem := utils.StorageSlotKey(tx2ContractAddress[:], tx2SlotKey[:]) - tx2ContractStem = tx2ContractStem[:31] - - eip2935Stem := utils.GetTreeKey(params.HistoryStorageAddress[:], uint256.NewInt(0), 0) - eip2935Stem = eip2935Stem[:31] - - // Check that the witness contains what we expect: a storage entry for each of the two contract - // creations that failed: one at 133 for the 2nd tx, and one at 105 for the first tx. - for _, stemStateDiff := range statediffs[0] { - // Check that the slot number 133, which is overflowing the account header, - // is present. Note that the offset of the 2nd group (first group after the - // header) is skipping the first 64 values, hence we still have an offset - // of 133, and not 133 - 64. - if bytes.Equal(stemStateDiff.Stem[:], tx2ContractStem[:]) { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix != 133 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - if suffixDiff.CurrentValue != nil { - t.Fatalf("invalid prestate value found for %x in block #1: %v != nil\n", stemStateDiff.Stem, suffixDiff.CurrentValue) - } - if suffixDiff.NewValue != nil { - t.Fatalf("invalid poststate value found for %x in block #1: %v != nil\n", stemStateDiff.Stem, suffixDiff.NewValue) - } - } - } else if bytes.Equal(stemStateDiff.Stem[:], tx1ContractStem) { - // For this contract creation, check that only the account header and storage slot 41 - // are found in the witness. - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix != 105 && suffixDiff.Suffix != 0 && suffixDiff.Suffix != 1 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } else if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // Check the eip 2935 group of leaves. - // Check that only one leaf was accessed, and is present in the witness. - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - // Check that this leaf is the first storage slot - if stemStateDiff.SuffixDiffs[0].Suffix != 64 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract: %d != 64", stemStateDiff.SuffixDiffs[0].Suffix) - } - // check that the prestate value is nil and that the poststate value isn't. - if stemStateDiff.SuffixDiffs[0].CurrentValue != nil { - t.Fatalf("non-nil current value in BLOCKHASH contract insert: %x", stemStateDiff.SuffixDiffs[0].CurrentValue) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("nil new value in BLOCKHASH contract insert") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != genesisH { - t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, genesisH) - } - } else { - // For all other entries present in the witness, check that nothing beyond - // the account header was accessed. - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix > 2 { - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } - } - - // Check that no account has a value above 4 in the 2nd block as no storage nor - // code should make it to the witness. - for _, stemStateDiff := range statediffs[1] { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // BLOCKHASH contract stem - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract at block #2: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - if stemStateDiff.SuffixDiffs[0].Suffix != 65 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract at block #2: %d != 65", stemStateDiff.SuffixDiffs[0].Suffix) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("missing post state value for BLOCKHASH contract at block #2") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != chain[0].Hash() { - t.Fatalf("invalid post state value for BLOCKHASH contract at block #2: %x != %x", chain[0].Hash(), (*stemStateDiff.SuffixDiffs[0].NewValue)[:]) - } - } else if suffixDiff.Suffix > 4 { - t.Fatalf("invalid suffix diff found for %x in block #2: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } -} - -func verkleTestGenesis(config *params.ChainConfig) *Genesis { - var ( - coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") - account1 = common.HexToAddress("0x687704DB07e902e9A8B3754031D168D46E3D586e") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - ) - return &Genesis{ - Config: config, - Alloc: GenesisAlloc{ - coinbase: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - account1: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - account2: GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 3, - }, - params.BeaconRootsAddress: {Nonce: 1, Code: params.BeaconRootsCode, Balance: common.Big0}, - params.HistoryStorageAddress: {Nonce: 1, Code: params.HistoryStorageCode, Balance: common.Big0}, - params.WithdrawalQueueAddress: {Nonce: 1, Code: params.WithdrawalQueueCode, Balance: common.Big0}, - params.ConsolidationQueueAddress: {Nonce: 1, Code: params.ConsolidationQueueCode, Balance: common.Big0}, - }, - } -} - -// TestProcessVerkleContractWithEmptyCode checks that the witness contains all valid -// entries, if the initcode returns an empty code. -func TestProcessVerkleContractWithEmptyCode(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - gspec := verkleTestGenesis(&config) - - genesisH, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - var tx types.Transaction - // a transaction that does some PUSH1n but returns a 0-sized contract - txpayload := common.Hex2Bytes("02f8db83010f2d03843b9aca008444cf6a05830186a08080b8807fdfbbb59f2371a76485ce557fd0de00c298d3ede52a3eab56d35af674eb49ec5860335260826053536001605453604c60555360f3605653606060575360446058536096605953600c605a5360df605b5360f3605c5360fb605d53600c605e53609a605f53607f60605360fe606153603d60625360f4606353604b60645360cac001a0486b6dc55b8a311568b7239a2cae1d77e7446dba71df61eaafd53f73820a138fa010bd48a45e56133ac4c5645142c2ea48950d40eb35050e9510b6bad9e15c5865") - if err := tx.UnmarshalBinary(txpayload); err != nil { - t.Fatal(err) - } - gen.AddTx(&tx) - }) - - eip2935Stem := utils.GetTreeKey(params.HistoryStorageAddress[:], uint256.NewInt(0), 0) - eip2935Stem = eip2935Stem[:31] - - for _, stemStateDiff := range statediffs[0] { - // Handle the case of the history contract: make sure only the correct - // slots are added to the witness. - if bytes.Equal(stemStateDiff.Stem[:], eip2935Stem) { - // BLOCKHASH contract stem - if len(stemStateDiff.SuffixDiffs) > 1 { - t.Fatalf("invalid suffix diff count found for BLOCKHASH contract: %d != 1", len(stemStateDiff.SuffixDiffs)) - } - if stemStateDiff.SuffixDiffs[0].Suffix != 64 { - t.Fatalf("invalid suffix diff value found for BLOCKHASH contract: %d != 64", stemStateDiff.SuffixDiffs[0].Suffix) - } - // check that the "current value" is nil and that the new value isn't. - if stemStateDiff.SuffixDiffs[0].CurrentValue != nil { - t.Fatalf("non-nil current value in BLOCKHASH contract insert: %x", stemStateDiff.SuffixDiffs[0].CurrentValue) - } - if stemStateDiff.SuffixDiffs[0].NewValue == nil { - t.Fatalf("nil new value in BLOCKHASH contract insert") - } - if *stemStateDiff.SuffixDiffs[0].NewValue != genesisH { - t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, genesisH) - } - } else { - for _, suffixDiff := range stemStateDiff.SuffixDiffs { - if suffixDiff.Suffix > 2 { - // if d8898012c484fb48610ecb7963886339207dab004bce968b007b616ffa18e0 shows up, it means that the PUSHn - // in the transaction above added entries into the witness, when they should not have since they are - // part of a contract deployment. - t.Fatalf("invalid suffix diff found for %x in block #1: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) - } - } - } - } -} - -// TestProcessVerkleExtCodeHashOpcode verifies that calling EXTCODEHASH on another -// deployed contract, creates all the right entries in the witness. -func TestProcessVerkleExtCodeHashOpcode(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - ) - dummyContract := []byte{ - byte(vm.PUSH1), 2, - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), - - byte(vm.PUSH1), 2, - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), - - byte(vm.PUSH1), 42, - } - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - dummyContractAddr := crypto.CreateAddress(deployer, 0) - - // contract that calls EXTCODEHASH on the dummy contract - extCodeHashContract := []byte{ - byte(vm.PUSH1), 22, - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), - - byte(vm.PUSH1), 22, - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), - - byte(vm.PUSH20), - 0x3a, 0x22, 0x0f, 0x35, 0x12, 0x52, 0x08, 0x9d, 0x38, 0x5b, 0x29, 0xbe, 0xca, 0x14, 0xe2, 0x7f, 0x20, 0x4c, 0x29, 0x6a, - byte(vm.EXTCODEHASH), - } - extCodeHashContractAddr := crypto.CreateAddress(deployer, 1) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - // Create dummy contract. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: dummyContract, - }) - gen.AddTx(tx) - - // Create contract with EXTCODEHASH opcode. - tx, _ = types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 1, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: extCodeHashContract}) - gen.AddTx(tx) - } else { - tx, _ := types.SignTx(types.NewTransaction(2, extCodeHashContractAddr, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - contractKeccakTreeKey := utils.CodeHashKey(dummyContractAddr[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], contractKeccakTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - codeHashStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - // Check location of code hash was accessed - if codeHashStateDiff.Suffix != utils.CodeHashLeafKey { - t.Fatalf("code hash invalid suffix") - } - // check the code hash wasn't present in the prestate, as - // the contract was deployed in this block. - if codeHashStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - // check the poststate value corresponds to the code hash - // of the deployed contract. - expCodeHash := crypto.Keccak256Hash(dummyContract[12:]) - if *codeHashStateDiff.CurrentValue != expCodeHash { - t.Fatalf("codeHash.CurrentValue unexpected code hash") - } - if codeHashStateDiff.NewValue != nil { - t.Fatalf("codeHash.NewValue must be nil") - } -} - -// TestProcessVerkleBalanceOpcode checks that calling balance -// on another contract will add the correct entries to the witness. -func TestProcessVerkleBalanceOpcode(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - txData := slices.Concat( - []byte{byte(vm.PUSH20)}, - common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d").Bytes(), - []byte{byte(vm.BALANCE)}) - - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(0), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: txData}) - gen.AddTx(tx) - }) - - account2BalanceTreeKey := utils.BasicDataKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], account2BalanceTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - var zero [32]byte - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("invalid suffix diff") - } - // check the prestate balance wasn't 0 or missing - if balanceStateDiff.CurrentValue == nil || *balanceStateDiff.CurrentValue == zero { - t.Fatalf("invalid current value %v", *balanceStateDiff.CurrentValue) - } - // check that the poststate witness value for the balance is nil, - // meaning that it didn't get updated. - if balanceStateDiff.NewValue != nil { - t.Fatalf("invalid new value") - } -} - -// TestProcessVerkleSelfDestructInSeparateTx controls the contents of the witness after -// a non-eip6780-compliant selfdestruct occurs. -func TestProcessVerkleSelfDestructInSeparateTx(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - - // runtime code: selfdestruct ( 0x6177843db3138ae69679A54b95cf345ED759450d ) - runtimeCode := slices.Concat( - []byte{byte(vm.PUSH20)}, - account2.Bytes(), - []byte{byte(vm.SELFDESTRUCT)}) - - //The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in a previous transaction. - selfDestructContract := slices.Concat([]byte{ - byte(vm.PUSH1), byte(len(runtimeCode)), - byte(vm.PUSH1), 12, - byte(vm.PUSH1), 0x00, - byte(vm.CODECOPY), // Codecopy( to-offset: 0, code offset: 12, length: 22 ) - - byte(vm.PUSH1), byte(len(runtimeCode)), - byte(vm.PUSH1), 0x00, - byte(vm.RETURN), // Return ( 0 : len(runtimecode) - }, - runtimeCode) - - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - - if i == 0 { - // Create selfdestruct contract, sending 42 wei. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - } else { - // Call it. - tx, _ := types.SignTx(types.NewTransaction(1, contract, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - var zero [32]byte - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - - // The original balance was 42. - var oldBalance [16]byte - oldBalance[15] = 42 - if !bytes.Equal((*balanceStateDiff.CurrentValue)[utils.BasicDataBalanceOffset:], oldBalance[:]) { - t.Fatalf("the pre-state balance before self-destruct must be %x, got %x", oldBalance, *balanceStateDiff.CurrentValue) - } - - // The new balance must be 0. - if !bytes.Equal((*balanceStateDiff.NewValue)[utils.BasicDataBalanceOffset:], zero[utils.BasicDataBalanceOffset:]) { - t.Fatalf("the post-state balance after self-destruct must be 0") - } - } - { // Check self-destructed target in the witness. - selfDestructTargetTreeKey := utils.CodeHashKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructTargetTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - if balanceStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - if balanceStateDiff.NewValue == nil { - t.Fatalf("codeHash.NewValue must not be empty") - } - preStateBalance := binary.BigEndian.Uint64(balanceStateDiff.CurrentValue[utils.BasicDataBalanceOffset+8:]) - postStateBalance := binary.BigEndian.Uint64(balanceStateDiff.NewValue[utils.BasicDataBalanceOffset+8:]) - if postStateBalance-preStateBalance != 42 { - t.Fatalf("the post-state balance after self-destruct must be 42, got %d-%d=%d", postStateBalance, preStateBalance, postStateBalance-preStateBalance) - } - } -} - -// TestProcessVerkleSelfDestructInSameTx controls the contents of the witness after -// a eip6780-compliant selfdestruct occurs. -func TestProcessVerkleSelfDestructInSameTx(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") - gspec = verkleTestGenesis(&config) - ) - - // The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in **the same** transaction sending the remaining - // balance to an external (i.e: not itself) account. - - selfDestructContract := slices.Concat( - []byte{byte(vm.PUSH20)}, - account2.Bytes(), - []byte{byte(vm.SELFDESTRUCT)}) - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - - if balanceStateDiff.CurrentValue != nil { - t.Fatalf("the pre-state balance before must be nil, since the contract didn't exist") - } - - if balanceStateDiff.NewValue != nil { - t.Fatalf("the post-state balance after self-destruct must be nil since the contract shouldn't be created at all") - } - } - { // Check self-destructed target in the witness. - selfDestructTargetTreeKey := utils.CodeHashKey(account2[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[0] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructTargetTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatalf("no state diff found for stem") - } - - balanceStateDiff := statediffs[0][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatalf("balance invalid suffix") - } - if balanceStateDiff.CurrentValue == nil { - t.Fatalf("codeHash.CurrentValue must not be empty") - } - if balanceStateDiff.NewValue == nil { - t.Fatalf("codeHash.NewValue must not be empty") - } - preStateBalance := binary.BigEndian.Uint64(balanceStateDiff.CurrentValue[utils.BasicDataBalanceOffset+8:]) - postStateBalance := binary.BigEndian.Uint64(balanceStateDiff.NewValue[utils.BasicDataBalanceOffset+8:]) - if postStateBalance-preStateBalance != 42 { - t.Fatalf("the post-state balance after self-destruct must be 42. got %d", postStateBalance) - } - } -} - -// TestProcessVerkleSelfDestructInSeparateTxWithSelfBeneficiary checks the content of the witness -// if a selfdestruct occurs in a different tx than the one that created it, but the beneficiary -// is the selfdestructed account. -func TestProcessVerkleSelfDestructInSeparateTxWithSelfBeneficiary(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - ) - // The goal of this test is to test SELFDESTRUCT that happens in a contract - // execution which is created in a *previous* transaction sending the remaining - // balance to itself. - selfDestructContract := []byte{ - byte(vm.PUSH1), 2, // PUSH1 2 - byte(vm.PUSH1), 10, // PUSH1 12 - byte(vm.PUSH0), // PUSH0 - byte(vm.CODECOPY), // Codecopy ( to offset 0, code@offset: 10, length: 2) - - byte(vm.PUSH1), 22, - byte(vm.PUSH0), - byte(vm.RETURN), // RETURN( memory[0:2] ) - - // Deployed code - byte(vm.ADDRESS), - byte(vm.SELFDESTRUCT), - } - deployer := crypto.PubkeyToAddress(testKey.PublicKey) - contract := crypto.CreateAddress(deployer, 0) - - _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { - gen.SetPoS() - if i == 0 { - // Create self-destruct contract, sending 42 wei. - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - } else { - // Call it. - tx, _ := types.SignTx(types.NewTransaction(1, contract, big.NewInt(0), 100_000, big.NewInt(875000000), nil), signer, testKey) - gen.AddTx(tx) - } - }) - - { - // Check self-destructed contract in the witness. - // The way 6780 is implemented today, it always SubBalance from the self-destructed contract, and AddBalance - // to the beneficiary. In this case both addresses are the same, thus this might be optimizable from a gas - // perspective. But until that happens, we need to honor this "balance reading" adding it to the witness. - - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range statediffs[1] { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - - balanceStateDiff := statediffs[1][stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - - // The original balance was 42. - var oldBalance [16]byte - oldBalance[15] = 42 - if !bytes.Equal((*balanceStateDiff.CurrentValue)[utils.BasicDataBalanceOffset:], oldBalance[:]) { - t.Fatal("the pre-state balance before self-destruct must be 42") - } - - // Note that the SubBalance+AddBalance net effect is a 0 change, so NewValue - // must be nil. - if balanceStateDiff.NewValue != nil { - t.Fatal("the post-state balance after self-destruct must be empty") - } - } -} - -// TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiary checks the content of the witness -// if a selfdestruct occurs in the same tx as the one that created it, but the beneficiary -// is the selfdestructed account. -func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiary(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - deployer = crypto.PubkeyToAddress(testKey.PublicKey) - contract = crypto.CreateAddress(deployer, 0) - ) - - // The goal of this test is to test SELFDESTRUCT that happens while executing - // the init code of a contract creation, that occurs in **the same** transaction. - // The balance is sent to itself. - t.Logf("Contract: %v", contract.String()) - - selfDestructContract := []byte{byte(vm.ADDRESS), byte(vm.SELFDESTRUCT)} - - _, _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - stateDiff := stateDiffs[0] // state difference of block 1 - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range stateDiff { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - balanceStateDiff := stateDiff[stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - if balanceStateDiff.CurrentValue != nil { - t.Fatal("the pre-state balance before must be nil, since the contract didn't exist") - } - // Ensure that the value is burnt, and therefore that the balance of the self-destructed - // contract isn't modified (it should remain missing from the state) - if balanceStateDiff.NewValue != nil { - t.Fatal("the post-state balance after self-destruct must be nil since the contract shouldn't be created at all") - } - } -} - -// TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount checks the -// content of the witness if a selfdestruct occurs in the same tx as the one that created it, -// it, but the beneficiary is the selfdestructed account. The difference with the test above, -// is that the created account is prefunded and so the final value should be 0. -func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount(t *testing.T) { - t.Skip("verkle trie is not yet supported in bor") - // The test txs were taken from a secondary testnet with chain id 69421 - config := *testKaustinenLikeChainConfig - config.ChainID = new(big.Int).SetUint64(69421) - - var ( - signer = types.LatestSigner(&config) - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - gspec = verkleTestGenesis(&config) - deployer = crypto.PubkeyToAddress(testKey.PublicKey) - contract = crypto.CreateAddress(deployer, 0) - ) - // Prefund the account, at an address that the contract will be deployed at, - // before it selfdestrucs. We can therefore check that the account itseld is - // NOT destroyed, which is what the current version of the spec requires. - // TODO(gballet) revisit after the spec has been modified. - gspec.Alloc[contract] = types.Account{ - Balance: big.NewInt(100), - } - - selfDestructContract := []byte{byte(vm.ADDRESS), byte(vm.SELFDESTRUCT)} - - _, _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { - gen.SetPoS() - tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, - Value: big.NewInt(42), - Gas: 100_000, - GasPrice: big.NewInt(875000000), - Data: selfDestructContract, - }) - gen.AddTx(tx) - }) - stateDiff := stateDiffs[0] // state difference of block 1 - - { // Check self-destructed contract in the witness - selfDestructContractTreeKey := utils.CodeHashKey(contract[:]) - - var stateDiffIdx = -1 - for i, stemStateDiff := range stateDiff { - if bytes.Equal(stemStateDiff.Stem[:], selfDestructContractTreeKey[:31]) { - stateDiffIdx = i - break - } - } - if stateDiffIdx == -1 { - t.Fatal("no state diff found for stem") - } - balanceStateDiff := stateDiff[stateDiffIdx].SuffixDiffs[0] - if balanceStateDiff.Suffix != utils.BasicDataLeafKey { - t.Fatal("balance invalid suffix") - } - expected, _ := hex.DecodeString("0000000000000000000000000000000000000000000000000000000000000064") - if balanceStateDiff.CurrentValue == nil || !bytes.Equal(balanceStateDiff.CurrentValue[:], expected) { - t.Fatalf("incorrect prestate balance: %x != %x", *balanceStateDiff.CurrentValue, expected) - } - // Ensure that the value is burnt, and therefore that the balance of the self-destructed - // contract isn't modified (it should remain missing from the state) - expected = make([]byte, 32) - if balanceStateDiff.NewValue == nil { - t.Fatal("incorrect nil poststate balance") - } - if !bytes.Equal(balanceStateDiff.NewValue[:], expected[:]) { - t.Fatalf("incorrect poststate balance: %x != %x", *balanceStateDiff.NewValue, expected[:]) - } - } -} diff --git a/core/vm/contracts.go b/core/vm/contracts.go index abe998c1f9..f75c73adaf 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -457,11 +457,11 @@ func (c *ecrecover) Run(input []byte) ([]byte, error) { } // We must make sure not to modify the 'input', so placing the 'v' along with // the signature needs to be done on a new allocation - sig := make([]byte, 65) - copy(sig, input[64:128]) + var sig [65]byte + copy(sig[:], input[64:128]) sig[64] = v // v needs to be at the end for libsecp256k1 - pubKey, err := crypto.Ecrecover(input[:32], sig) + pubKey, err := crypto.Ecrecover(input[:32], sig[:]) // make sure the public key is a valid one if err != nil { return nil, nil @@ -751,7 +751,6 @@ func (c *bigModExp) RequiredGas(input []byte) uint64 { if expLen > 32 { expHead.SetBytes(getData(input, baseLen, 32)) } else { - // TODO: Check that if expLen < baseLen, then getData will return an empty slice expHead.SetBytes(getData(input, baseLen, expLen)) } } diff --git a/core/vm/dispatch_bench_test.go b/core/vm/dispatch_bench_test.go index f50b556109..616ccac6ef 100644 --- a/core/vm/dispatch_bench_test.go +++ b/core/vm/dispatch_bench_test.go @@ -76,7 +76,7 @@ func benchSnailtracer(b *testing.B, switchDispatch bool) { evm := NewEVM(bctx, db, benchChainConfig, Config{EnableEVMSwitchDispatch: switchDispatch}) evm.SetTxContext(TxContext{ Origin: caller, - GasPrice: big.NewInt(1), + GasPrice: uint256.NewInt(1), }) evm.Call(caller, addr, calldata, 1_000_000_000, new(uint256.Int)) } diff --git a/core/vm/dispatch_test.go b/core/vm/dispatch_test.go index f359a13e52..d23d90badf 100644 --- a/core/vm/dispatch_test.go +++ b/core/vm/dispatch_test.go @@ -201,7 +201,7 @@ func execPathResultWithConfig( evm := NewEVM(bctx, db, chainCfg, cfg) evm.SetTxContext(TxContext{ Origin: origin, - GasPrice: big.NewInt(66), + GasPrice: uint256.NewInt(66), BlobHashes: blobHashes, }) ret, gasLeft, err := evm.Call(caller, addr, input, gas, uint256.NewInt(77)) @@ -1553,7 +1553,7 @@ func makeEVM(code []byte, gas uint64, switchDispatch bool) (*EVM, common.Address db.Prepare(rules, caller, common.Address{}, &addr, ActivePrecompiles(rules), nil) evm := NewEVM(bctx, db, diffChainConfig, Config{EnableEVMSwitchDispatch: switchDispatch}) - evm.SetTxContext(TxContext{Origin: origin, GasPrice: big.NewInt(66)}) + evm.SetTxContext(TxContext{Origin: origin, GasPrice: uint256.NewInt(66)}) return evm, addr } diff --git a/core/vm/eips.go b/core/vm/eips.go index 64e01584c8..e68f775f56 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -43,6 +43,7 @@ var activators = map[int]func(*JumpTable){ 4762: enable4762, 7702: enable7702, 7939: enable7939, + 8024: enable8024, } // EnableEIP enables the given EIP on the config. @@ -366,6 +367,28 @@ func enable6780(jt *JumpTable) { } } +// enable8024 applies EIP-8024 (DUPN, SWAPN, EXCHANGE) +func enable8024(jt *JumpTable) { + jt[DUPN] = &operation{ + execute: opDupN, + constantGas: GasFastestStep, + minStack: minStack(1, 0), + maxStack: maxStack(0, 1), + } + jt[SWAPN] = &operation{ + execute: opSwapN, + constantGas: GasFastestStep, + minStack: minStack(2, 0), + maxStack: maxStack(0, 0), + } + jt[EXCHANGE] = &operation{ + execute: opExchange, + constantGas: GasFastestStep, + minStack: minStack(2, 0), + maxStack: maxStack(0, 0), + } +} + func opExtCodeCopyEIP4762(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { var ( stack = scope.Stack diff --git a/core/vm/evm.go b/core/vm/evm.go index 5f19d0e5b5..f6ce516f1c 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -127,9 +127,8 @@ type BlockContext struct { type TxContext struct { // Message information Origin common.Address // Provides information for ORIGIN - GasPrice *big.Int // Provides information for GASPRICE (and is used to zero the basefee if NoBaseFee is set) + GasPrice *uint256.Int // Provides information for GASPRICE (and is used to zero the basefee if NoBaseFee is set) BlobHashes []common.Hash // Provides information for BLOBHASH - BlobFeeCap *big.Int // Is used to zero the blobbasefee if NoBaseFee is set AccessEvents *state.AccessEvents // Capture all state accesses for this tx } @@ -179,9 +178,6 @@ type EVM struct { // jumpDests stores results of JUMPDEST analysis. jumpDests JumpDestCache - hasher crypto.KeccakState // Keccak256 hasher instance shared across opcodes - hasherBuf common.Hash // Keccak256 hasher result array shared across opcodes - readOnly bool // Whether to throw on stateful modifications returnData []byte // Last CALL's return data for subsequent reuse @@ -214,7 +210,6 @@ func NewEVM(blockCtx BlockContext, statedb StateDB, chainConfig *params.ChainCon chainConfig: chainConfig, chainRules: chainConfig.Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time), jumpDests: jd, - hasher: crypto.NewKeccakState(), } evm.precompiles = activePrecompiledContracts(evm.chainRules) @@ -289,7 +284,7 @@ func (evm *EVM) SetJumpDestCache(jumpDests JumpDestCache) { // This is not threadsafe and should only be done very cautiously. func (evm *EVM) SetTxContext(txCtx TxContext) { if evm.chainRules.IsEIP4762 { - txCtx.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) + txCtx.AccessEvents = state.NewAccessEvents() } evm.TxContext = txCtx } @@ -715,7 +710,7 @@ func (evm *EVM) Create(caller common.Address, code []byte, gas uint64, value *ui // The different between Create2 with Create is Create2 uses keccak256(0xff ++ msg.sender ++ salt ++ keccak256(init_code))[12:] // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at. func (evm *EVM) Create2(caller common.Address, code []byte, gas uint64, endowment *uint256.Int, salt *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { - inithash := crypto.HashData(evm.hasher, code) + inithash := crypto.Keccak256Hash(code) contractAddr = crypto.CreateAddress2(caller, salt.Bytes32(), inithash[:]) return evm.create(caller, code, gas, endowment, contractAddr, CREATE2) } diff --git a/core/vm/instructions.go b/core/vm/instructions.go index db0d396dd8..12fd8e8cdd 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" ) @@ -260,33 +261,26 @@ func opKeccak256(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { offset, size := scope.Stack.pop(), scope.Stack.peek() data := scope.Memory.GetPtr(offset.Uint64(), size.Uint64()) - // Fast path: cache 64-byte keccak256 (Solidity mapping slot lookups). - // These are the most common SHA3 calls — keccak256(key ++ slot_number). + // Fast path: cache 64-byte keccak256 results (Solidity mapping slot lookups), + // the most common SHA3 calls — keccak256(key ++ slot_number). + var hash common.Hash if len(data) == 64 && evm.Config.Keccak256Cache != nil { var key [64]byte copy(key[:], data) if cached, ok := evm.Config.Keccak256Cache.Load(key); ok { - h := cached.(common.Hash) - if evm.Config.EnablePreimageRecording { - evm.StateDB.AddPreimage(h, data) - } - size.SetBytes32(h[:]) - return nil, nil + hash = cached.(common.Hash) + } else { + hash = crypto.Keccak256Hash(data) + evm.Config.Keccak256Cache.Store(key, hash) } - evm.hasher.Reset() - evm.hasher.Write(data) - evm.hasher.Read(evm.hasherBuf[:]) - evm.Config.Keccak256Cache.Store(key, evm.hasherBuf) } else { - evm.hasher.Reset() - evm.hasher.Write(data) - evm.hasher.Read(evm.hasherBuf[:]) + hash = crypto.Keccak256Hash(data) } if evm.Config.EnablePreimageRecording { - evm.StateDB.AddPreimage(evm.hasherBuf, data) + evm.StateDB.AddPreimage(hash, data) } - size.SetBytes(evm.hasherBuf[:]) + size.SetBytes(hash[:]) return nil, nil } @@ -473,9 +467,7 @@ func opExtCodeHash(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opGasprice(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - v, _ := uint256.FromBig(evm.GasPrice) - scope.Stack.push(v) - + scope.Stack.push(evm.GasPrice.Clone()) return nil, nil } @@ -1009,6 +1001,115 @@ func opSelfdestruct6780(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, erro return nil, errStopToken } +func decodeSingle(x byte) int { + if x <= 90 { + return int(x) + 17 + } + return int(x) - 20 +} + +func decodePair(x byte) (int, int) { + var k int + if x <= 79 { + k = int(x) + } else { + k = int(x) - 48 + } + q, r := k/16, k%16 + if q < r { + return q + 1, r + 1 + } + return r + 1, 29 - q +} + +func opDupN(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded to preserve compatibility with existing opcodes. + if x > 90 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n := decodeSingle(x) + + // DUPN duplicates the n'th stack item, so the stack must contain at least n elements. + if scope.Stack.len() < n { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: n} + } + + //The n‘th stack item is duplicated at the top of the stack. + scope.Stack.push(scope.Stack.Back(n - 1)) + *pc += 1 + return nil, nil +} + +func opSwapN(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded to preserve compatibility with existing opcodes. + if x > 90 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n := decodeSingle(x) + + // SWAPN operates on the top and n+1 stack items, so the stack must contain at least n+1 elements. + if scope.Stack.len() < n+1 { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: n + 1} + } + + // The (n+1)‘th stack item is swapped with the top of the stack. + indexTop := scope.Stack.len() - 1 + indexN := scope.Stack.len() - 1 - n + scope.Stack.data[indexTop], scope.Stack.data[indexN] = scope.Stack.data[indexN], scope.Stack.data[indexTop] + *pc += 1 + return nil, nil +} + +func opExchange(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + code := scope.Contract.Code + i := *pc + 1 + + // If the immediate byte is missing, treat as 0x00 (same convention as PUSHn). + var x byte + if i < uint64(len(code)) { + x = code[i] + } + + // This range is excluded both to preserve compatibility with existing opcodes + // and to keep decode_pair’s 16-aligned arithmetic mapping valid (0–79, 128–255). + if x > 79 && x < 128 { + return nil, &ErrInvalidOpCode{opcode: OpCode(x)} + } + n, m := decodePair(x) + need := max(n, m) + 1 + + // EXCHANGE operates on the (n+1)'th and (m+1)'th stack items, + // so the stack must contain at least max(n, m)+1 elements. + if scope.Stack.len() < need { + return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: need} + } + + // The (n+1)‘th stack item is swapped with the (m+1)‘th stack item. + indexN := scope.Stack.len() - 1 - n + indexM := scope.Stack.len() - 1 - m + scope.Stack.data[indexN], scope.Stack.data[indexM] = scope.Stack.data[indexM], scope.Stack.data[indexN] + *pc += 1 + return nil, nil +} + // following functions are used by the instruction jump table // make log instruction function diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 055f7e03a6..ebdedf987c 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -19,6 +19,7 @@ package vm import ( "bytes" "encoding/json" + "errors" "fmt" "math/big" "os" @@ -1079,3 +1080,222 @@ func TestOpCLZ(t *testing.T) { } } } + +func TestEIP8024_Execution(t *testing.T) { + evm := NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + + tests := []struct { + name string + codeHex string + wantErr error + wantOpcode OpCode + wantVals []uint64 + }{ + { + name: "DUPN", + codeHex: "60016000808080808080808080808080808080e600", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, + }, + }, + { + name: "DUPN_MISSING_IMMEDIATE", + codeHex: "60016000808080808080808080808080808080e6", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, + }, + }, + { + name: "SWAPN", + codeHex: "600160008080808080808080808080808080806002e700", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, + }, + }, + { + name: "SWAPN_MISSING_IMMEDIATE", + codeHex: "600160008080808080808080808080808080806002e7", + wantVals: []uint64{ + 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, + }, + }, + { + name: "EXCHANGE", + codeHex: "600060016002e801", + wantVals: []uint64{2, 0, 1}, + }, + { + name: "EXCHANGE_MISSING_IMMEDIATE", + codeHex: "600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060016002e8", + wantVals: []uint64{ + 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, + }, + }, + { + name: "INVALID_SWAPN_LOW", + codeHex: "e75b", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: SWAPN, + }, + { + name: "JUMP over INVALID_DUPN", + codeHex: "600456e65b", + wantErr: nil, + }, + { + name: "UNDERFLOW_DUPN_1", + codeHex: "6000808080808080808080808080808080e600", + wantErr: &ErrStackUnderflow{}, + wantOpcode: DUPN, + }, + // Additional test cases + { + name: "INVALID_DUPN_LOW", + codeHex: "e65b", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: DUPN, + }, + { + name: "INVALID_EXCHANGE_LOW", + codeHex: "e850", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, + }, + { + name: "INVALID_DUPN_HIGH", + codeHex: "e67f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: DUPN, + }, + { + name: "INVALID_SWAPN_HIGH", + codeHex: "e77f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: SWAPN, + }, + { + name: "INVALID_EXCHANGE_HIGH", + codeHex: "e87f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, + }, + { + name: "UNDERFLOW_DUPN_2", + codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe600", // (n=17, need 17 items, have 16) + wantErr: &ErrStackUnderflow{}, + wantOpcode: DUPN, + }, + { + name: "UNDERFLOW_SWAPN", + codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe700", // (n=17, need 18 items, have 17) + wantErr: &ErrStackUnderflow{}, + wantOpcode: SWAPN, + }, + { + name: "UNDERFLOW_EXCHANGE", + codeHex: "60016002e801", // (n,m)=(1,2), need 3 items, have 2 + wantErr: &ErrStackUnderflow{}, + wantOpcode: EXCHANGE, + }, + { + name: "PC_INCREMENT", + codeHex: "600060006000e80115", + wantVals: []uint64{1, 0, 0}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code := common.FromHex(tc.codeHex) + stack := newstack() + pc := uint64(0) + scope := &ScopeContext{Stack: stack, Contract: &Contract{Code: code}} + var err error + var errOp OpCode + for pc < uint64(len(code)) && err == nil { + op := code[pc] + switch OpCode(op) { + case STOP: + return + case PUSH1: + _, err = opPush1(&pc, evm, scope) + case DUP1: + dup1 := makeDup(1) + _, err = dup1(&pc, evm, scope) + case JUMP: + _, err = opJump(&pc, evm, scope) + case JUMPDEST: + _, err = opJumpdest(&pc, evm, scope) + case ISZERO: + _, err = opIszero(&pc, evm, scope) + case PUSH0: + _, err = opPush0(&pc, evm, scope) + case DUPN: + _, err = opDupN(&pc, evm, scope) + case SWAPN: + _, err = opSwapN(&pc, evm, scope) + case EXCHANGE: + _, err = opExchange(&pc, evm, scope) + default: + t.Fatalf("unexpected opcode %s at pc=%d", OpCode(op), pc) + } + if err != nil { + errOp = OpCode(op) + } + pc++ + } + if tc.wantErr != nil { + // Fail because we wanted an error, but didn't get one. + if err == nil { + t.Fatalf("expected error, got nil") + } + // Fail if the wrong opcode threw an error. + if errOp != tc.wantOpcode { + t.Fatalf("expected error from opcode %s, got %s", tc.wantOpcode, errOp) + } + // Fail if we don't get the error we expect. + switch tc.wantErr.(type) { + case *ErrInvalidOpCode: + var want *ErrInvalidOpCode + if !errors.As(err, &want) { + t.Fatalf("expected ErrInvalidOpCode, got %v", err) + } + case *ErrStackUnderflow: + var want *ErrStackUnderflow + if !errors.As(err, &want) { + t.Fatalf("expected ErrStackUnderflow, got %v", err) + } + default: + t.Fatalf("unsupported wantErr type %T", tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := make([]uint64, 0, stack.len()) + for i := stack.len() - 1; i >= 0; i-- { + got = append(got, stack.data[i].Uint64()) + } + if len(got) != len(tc.wantVals) { + t.Fatalf("stack len=%d; want %d", len(got), len(tc.wantVals)) + } + for i := range got { + if got[i] != tc.wantVals[i] { + t.Fatalf("[%s] stack[%d]=%d; want %d\nstack=%v", + tc.name, i, got[i], tc.wantVals[i], got) + } + } + }) + } +} diff --git a/core/vm/interface.go b/core/vm/interface.go index 815844a984..a996de3e61 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie/utils" ) // StateDB is an EVM database for full state querying. @@ -86,9 +85,6 @@ type StateDB interface { // even if the feature/fork is not active yet AddSlotToAccessList(addr common.Address, slot common.Hash) - // PointCache returns the point cache used in computations - PointCache() *utils.PointCache - Prepare(rules params.Rules, sender, coinbase common.Address, dest *common.Address, precompiles []common.Address, txAccesses types.AccessList) RevertToSnapshot(int) diff --git a/core/vm/memory.go b/core/vm/memory.go index 76d480adef..678fcef19b 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -44,6 +44,7 @@ func (m *Memory) Free() { // To reduce peak allocation, return only smaller memory instances to the pool. const maxBufferSize = 16 << 10 if cap(m.store) <= maxBufferSize { + clear(m.store) m.store = m.store[:0] m.lastGasCost = 0 memoryPool.Put(m) @@ -77,10 +78,14 @@ func (m *Memory) Set32(offset uint64, val *uint256.Int) { val.PutUint256(m.store[offset:]) } -// Resize resizes the memory to size +// Resize grows the memory to the requested size. func (m *Memory) Resize(size uint64) { - if uint64(m.Len()) < size { - m.store = append(m.store, make([]byte, size-uint64(m.Len()))...) + if uint64(len(m.store)) < size { + if uint64(cap(m.store)) >= size { + m.store = m.store[:size] + } else { + m.store = append(m.store, make([]byte, size-uint64(len(m.store)))...) + } } } diff --git a/core/vm/memory_test.go b/core/vm/memory_test.go index 41389b729a..3890d18cb5 100644 --- a/core/vm/memory_test.go +++ b/core/vm/memory_test.go @@ -83,3 +83,10 @@ func TestMemoryCopy(t *testing.T) { } } } + +func BenchmarkResize(b *testing.B) { + memory := NewMemory() + for i := range b.N { + memory.Resize(uint64(i)) + } +} diff --git a/core/vm/runtime/env.go b/core/vm/runtime/env.go index e54041f7e2..4181e08ee8 100644 --- a/core/vm/runtime/env.go +++ b/core/vm/runtime/env.go @@ -19,14 +19,14 @@ package runtime import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" ) func NewEnv(cfg *Config) *vm.EVM { txContext := vm.TxContext{ Origin: cfg.Origin, - GasPrice: cfg.GasPrice, + GasPrice: uint256.MustFromBig(cfg.GasPrice), BlobHashes: cfg.BlobHashes, - BlobFeeCap: cfg.BlobFeeCap, } blockContext := vm.BlockContext{ CanTransfer: core.CanTransfer, diff --git a/crypto/ecies/ecies.go b/crypto/ecies/ecies.go index 6876ad7814..db8a9de013 100644 --- a/crypto/ecies/ecies.go +++ b/crypto/ecies/ecies.go @@ -131,7 +131,6 @@ func (prv *PrivateKey) GenerateShared(pub *PublicKey, skLen, macLen int) (sk []b if pub.X == nil || pub.Y == nil || !pub.Curve.IsOnCurve(pub.X, pub.Y) { return nil, ErrInvalidPublicKey } - if skLen+macLen > MaxSharedKeyLength(pub) { return nil, ErrSharedKeyTooBig } diff --git a/crypto/keccak.go b/crypto/keccak.go index 0ad79a63c1..3fafddc92e 100644 --- a/crypto/keccak.go +++ b/crypto/keccak.go @@ -22,17 +22,17 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" - "golang.org/x/crypto/sha3" + "github.com/ethereum/go-ethereum/crypto/keccak" ) // NewKeccakState creates a new KeccakState func NewKeccakState() KeccakState { - return sha3.NewLegacyKeccak256().(KeccakState) + return keccak.NewLegacyKeccak256().(KeccakState) } var hasherPool = sync.Pool{ New: func() any { - return sha3.NewLegacyKeccak256().(KeccakState) + return keccak.NewLegacyKeccak256().(KeccakState) }, } diff --git a/crypto/keccak/LICENSE b/crypto/keccak/LICENSE new file mode 100644 index 0000000000..2a7cf70da6 --- /dev/null +++ b/crypto/keccak/LICENSE @@ -0,0 +1,27 @@ +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crypto/keccak/README.md b/crypto/keccak/README.md new file mode 100644 index 0000000000..295a5b958c --- /dev/null +++ b/crypto/keccak/README.md @@ -0,0 +1,6 @@ +This is a vendored and modified copy of golang.org/x/crypto/sha3, with an assembly +implementation of keccak256. We wish to retain the assembly implementation, +which was removed in v0.44.0. + +Ethereum uses a 'legacy' variant of Keccak, which was defined before it became SHA3. As +such, we cannot use the standard library crypto/sha3 package. diff --git a/crypto/keccak/hashes.go b/crypto/keccak/hashes.go new file mode 100644 index 0000000000..c78c5fe992 --- /dev/null +++ b/crypto/keccak/hashes.go @@ -0,0 +1,44 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +// This file provides functions for creating instances of the SHA-3 +// and SHAKE hash functions, as well as utility functions for hashing +// bytes. + +import ( + "hash" +) + +const ( + dsbyteSHA3 = 0b00000110 + dsbyteKeccak = 0b00000001 + dsbyteShake = 0b00011111 + dsbyteCShake = 0b00000100 + + // rateK[c] is the rate in bytes for Keccak[c] where c is the capacity in + // bits. Given the sponge size is 1600 bits, the rate is 1600 - c bits. + rateK256 = (1600 - 256) / 8 + rateK448 = (1600 - 448) / 8 + rateK512 = (1600 - 512) / 8 + rateK768 = (1600 - 768) / 8 + rateK1024 = (1600 - 1024) / 8 +) + +// NewLegacyKeccak256 creates a new Keccak-256 hash. +// +// Only use this function if you require compatibility with an existing cryptosystem +// that uses non-standard padding. All other users should use New256 instead. +func NewLegacyKeccak256() hash.Hash { + return &state{rate: rateK512, outputLen: 32, dsbyte: dsbyteKeccak} +} + +// NewLegacyKeccak512 creates a new Keccak-512 hash. +// +// Only use this function if you require compatibility with an existing cryptosystem +// that uses non-standard padding. All other users should use New512 instead. +func NewLegacyKeccak512() hash.Hash { + return &state{rate: rateK1024, outputLen: 64, dsbyte: dsbyteKeccak} +} diff --git a/crypto/keccak/keccakf.go b/crypto/keccak/keccakf.go new file mode 100644 index 0000000000..82694fa4a3 --- /dev/null +++ b/crypto/keccak/keccakf.go @@ -0,0 +1,414 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !amd64 || purego || !gc + +package keccak + +import "math/bits" + +// rc stores the round constants for use in the ι step. +var rc = [24]uint64{ + 0x0000000000000001, + 0x0000000000008082, + 0x800000000000808A, + 0x8000000080008000, + 0x000000000000808B, + 0x0000000080000001, + 0x8000000080008081, + 0x8000000000008009, + 0x000000000000008A, + 0x0000000000000088, + 0x0000000080008009, + 0x000000008000000A, + 0x000000008000808B, + 0x800000000000008B, + 0x8000000000008089, + 0x8000000000008003, + 0x8000000000008002, + 0x8000000000000080, + 0x000000000000800A, + 0x800000008000000A, + 0x8000000080008081, + 0x8000000000008080, + 0x0000000080000001, + 0x8000000080008008, +} + +// keccakF1600 applies the Keccak permutation to a 1600b-wide +// state represented as a slice of 25 uint64s. +func keccakF1600(a *[25]uint64) { + // Implementation translated from Keccak-inplace.c + // in the keccak reference code. + var t, bc0, bc1, bc2, bc3, bc4, d0, d1, d2, d3, d4 uint64 + + for i := 0; i < 24; i += 4 { + // Combines the 5 steps in each round into 2 steps. + // Unrolls 4 rounds per loop and spreads some steps across rounds. + + // Round 1 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[6] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[12] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[18] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[24] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i] + a[6] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[16] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[22] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[3] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[10] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[1] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[7] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[19] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[20] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[11] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[23] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[4] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[5] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[2] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[8] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[14] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[15] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + // Round 2 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[16] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[7] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[23] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[14] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+1] + a[16] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[11] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[2] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[18] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[20] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[6] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[22] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[4] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[15] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[1] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[8] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[24] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[10] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[12] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[3] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[19] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[5] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + // Round 3 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[11] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[22] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[8] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[19] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+2] + a[11] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[1] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[12] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[23] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[15] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[16] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[2] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[24] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[5] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[6] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[3] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[14] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[20] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[7] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[18] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[4] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[10] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + // Round 4 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[1] ^ d1 + bc1 = bits.RotateLeft64(t, 44) + t = a[2] ^ d2 + bc2 = bits.RotateLeft64(t, 43) + t = a[3] ^ d3 + bc3 = bits.RotateLeft64(t, 21) + t = a[4] ^ d4 + bc4 = bits.RotateLeft64(t, 14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+3] + a[1] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc2 = bits.RotateLeft64(t, 3) + t = a[6] ^ d1 + bc3 = bits.RotateLeft64(t, 45) + t = a[7] ^ d2 + bc4 = bits.RotateLeft64(t, 61) + t = a[8] ^ d3 + bc0 = bits.RotateLeft64(t, 28) + t = a[9] ^ d4 + bc1 = bits.RotateLeft64(t, 20) + a[5] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc4 = bits.RotateLeft64(t, 18) + t = a[11] ^ d1 + bc0 = bits.RotateLeft64(t, 1) + t = a[12] ^ d2 + bc1 = bits.RotateLeft64(t, 6) + t = a[13] ^ d3 + bc2 = bits.RotateLeft64(t, 25) + t = a[14] ^ d4 + bc3 = bits.RotateLeft64(t, 8) + a[10] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc1 = bits.RotateLeft64(t, 36) + t = a[16] ^ d1 + bc2 = bits.RotateLeft64(t, 10) + t = a[17] ^ d2 + bc3 = bits.RotateLeft64(t, 15) + t = a[18] ^ d3 + bc4 = bits.RotateLeft64(t, 56) + t = a[19] ^ d4 + bc0 = bits.RotateLeft64(t, 27) + a[15] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc3 = bits.RotateLeft64(t, 41) + t = a[21] ^ d1 + bc4 = bits.RotateLeft64(t, 2) + t = a[22] ^ d2 + bc0 = bits.RotateLeft64(t, 62) + t = a[23] ^ d3 + bc1 = bits.RotateLeft64(t, 55) + t = a[24] ^ d4 + bc2 = bits.RotateLeft64(t, 39) + a[20] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + } +} diff --git a/crypto/keccak/keccakf_amd64.go b/crypto/keccak/keccakf_amd64.go new file mode 100644 index 0000000000..cb6eca44c3 --- /dev/null +++ b/crypto/keccak/keccakf_amd64.go @@ -0,0 +1,13 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build amd64 && !purego && gc + +package keccak + +// This function is implemented in keccakf_amd64.s. + +//go:noescape + +func keccakF1600(a *[25]uint64) diff --git a/crypto/keccak/keccakf_amd64.s b/crypto/keccak/keccakf_amd64.s new file mode 100644 index 0000000000..99e2f16e97 --- /dev/null +++ b/crypto/keccak/keccakf_amd64.s @@ -0,0 +1,5419 @@ +// Code generated by command: go run keccakf_amd64_asm.go -out ../keccakf_amd64.s -pkg sha3. DO NOT EDIT. + +//go:build amd64 && !purego && gc + +// func keccakF1600(a *[25]uint64) +TEXT ·keccakF1600(SB), $200-8 + MOVQ a+0(FP), DI + + // Convert the user state into an internal state + NOTQ 8(DI) + NOTQ 16(DI) + NOTQ 64(DI) + NOTQ 96(DI) + NOTQ 136(DI) + NOTQ 160(DI) + + // Execute the KeccakF permutation + MOVQ (DI), SI + MOVQ 8(DI), BP + MOVQ 32(DI), R15 + XORQ 40(DI), SI + XORQ 48(DI), BP + XORQ 72(DI), R15 + XORQ 80(DI), SI + XORQ 88(DI), BP + XORQ 112(DI), R15 + XORQ 120(DI), SI + XORQ 128(DI), BP + XORQ 152(DI), R15 + XORQ 160(DI), SI + XORQ 168(DI), BP + MOVQ 176(DI), DX + MOVQ 184(DI), R8 + XORQ 192(DI), R15 + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000008082, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000000000808a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008000, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000808b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008081, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008009, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000008a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000000000088, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080008009, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000008000000a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000008000808b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000000000008b, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008089, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008003, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008002, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000000080, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x000000000000800a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x800000008000000a, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008081, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000000008080, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + MOVQ R12, BP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + XORQ R10, R15 + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + XORQ R11, R15 + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(DI), R12 + XORQ 56(DI), DX + XORQ R15, BX + XORQ 96(DI), R12 + XORQ 136(DI), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(DI), R13 + XORQ 64(DI), R8 + XORQ SI, CX + XORQ 104(DI), R13 + XORQ 144(DI), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (DI), R10 + MOVQ 48(DI), R11 + XORQ R13, R9 + MOVQ 96(DI), R12 + MOVQ 144(DI), R13 + MOVQ 192(DI), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x0000000080000001, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (SP) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(SP) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(SP) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(SP) + MOVQ R12, 8(SP) + MOVQ R12, BP + + // Result g + MOVQ 72(DI), R11 + XORQ R9, R11 + MOVQ 80(DI), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(DI), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(DI), R13 + MOVQ 176(DI), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(SP) + XORQ AX, SI + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(SP) + XORQ AX, BP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(SP) + NOTQ R14 + XORQ R10, R15 + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(SP) + + // Result k + MOVQ 8(DI), R10 + MOVQ 56(DI), R11 + MOVQ 104(DI), R12 + MOVQ 152(DI), R13 + MOVQ 160(DI), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(SP) + XORQ AX, SI + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(SP) + XORQ AX, BP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(SP) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(SP) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(SP) + XORQ R10, R15 + + // Result m + MOVQ 40(DI), R11 + XORQ BX, R11 + MOVQ 88(DI), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(DI), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(DI), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(DI), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(SP) + XORQ AX, SI + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(SP) + XORQ AX, BP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(SP) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(SP) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(SP) + XORQ R11, R15 + + // Result s + MOVQ 16(DI), R10 + MOVQ 64(DI), R11 + MOVQ 112(DI), R12 + XORQ DX, R10 + MOVQ 120(DI), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(DI), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(SP) + ROLQ $0x27, R12 + XORQ R9, R15 + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(SP) + XORQ BX, SI + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(SP) + XORQ CX, BP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(SP) + MOVQ R8, 184(SP) + + // Prepare round + MOVQ BP, BX + ROLQ $0x01, BX + MOVQ 16(SP), R12 + XORQ 56(SP), DX + XORQ R15, BX + XORQ 96(SP), R12 + XORQ 136(SP), DX + XORQ DX, R12 + MOVQ R12, CX + ROLQ $0x01, CX + MOVQ 24(SP), R13 + XORQ 64(SP), R8 + XORQ SI, CX + XORQ 104(SP), R13 + XORQ 144(SP), R8 + XORQ R8, R13 + MOVQ R13, DX + ROLQ $0x01, DX + MOVQ R15, R8 + XORQ BP, DX + ROLQ $0x01, R8 + MOVQ SI, R9 + XORQ R12, R8 + ROLQ $0x01, R9 + + // Result b + MOVQ (SP), R10 + MOVQ 48(SP), R11 + XORQ R13, R9 + MOVQ 96(SP), R12 + MOVQ 144(SP), R13 + MOVQ 192(SP), R14 + XORQ CX, R11 + ROLQ $0x2c, R11 + XORQ DX, R12 + XORQ BX, R10 + ROLQ $0x2b, R12 + MOVQ R11, SI + MOVQ $0x8000000080008008, AX + ORQ R12, SI + XORQ R10, AX + XORQ AX, SI + MOVQ SI, (DI) + XORQ R9, R14 + ROLQ $0x0e, R14 + MOVQ R10, R15 + ANDQ R11, R15 + XORQ R14, R15 + MOVQ R15, 32(DI) + XORQ R8, R13 + ROLQ $0x15, R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 16(DI) + NOTQ R12 + ORQ R10, R14 + ORQ R13, R12 + XORQ R13, R14 + XORQ R11, R12 + MOVQ R14, 24(DI) + MOVQ R12, 8(DI) + NOP + + // Result g + MOVQ 72(SP), R11 + XORQ R9, R11 + MOVQ 80(SP), R12 + ROLQ $0x14, R11 + XORQ BX, R12 + ROLQ $0x03, R12 + MOVQ 24(SP), R10 + MOVQ R11, AX + ORQ R12, AX + XORQ R8, R10 + MOVQ 128(SP), R13 + MOVQ 176(SP), R14 + ROLQ $0x1c, R10 + XORQ R10, AX + MOVQ AX, 40(DI) + NOP + XORQ CX, R13 + ROLQ $0x2d, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 48(DI) + NOP + XORQ DX, R14 + ROLQ $0x3d, R14 + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 64(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 72(DI) + NOTQ R14 + NOP + ORQ R14, R13 + XORQ R12, R13 + MOVQ R13, 56(DI) + + // Result k + MOVQ 8(SP), R10 + MOVQ 56(SP), R11 + MOVQ 104(SP), R12 + MOVQ 152(SP), R13 + MOVQ 160(SP), R14 + XORQ DX, R11 + ROLQ $0x06, R11 + XORQ R8, R12 + ROLQ $0x19, R12 + MOVQ R11, AX + ORQ R12, AX + XORQ CX, R10 + ROLQ $0x01, R10 + XORQ R10, AX + MOVQ AX, 80(DI) + NOP + XORQ R9, R13 + ROLQ $0x08, R13 + MOVQ R12, AX + ANDQ R13, AX + XORQ R11, AX + MOVQ AX, 88(DI) + NOP + XORQ BX, R14 + ROLQ $0x12, R14 + NOTQ R13 + MOVQ R13, AX + ANDQ R14, AX + XORQ R12, AX + MOVQ AX, 96(DI) + MOVQ R14, AX + ORQ R10, AX + XORQ R13, AX + MOVQ AX, 104(DI) + ANDQ R11, R10 + XORQ R14, R10 + MOVQ R10, 112(DI) + NOP + + // Result m + MOVQ 40(SP), R11 + XORQ BX, R11 + MOVQ 88(SP), R12 + ROLQ $0x24, R11 + XORQ CX, R12 + MOVQ 32(SP), R10 + ROLQ $0x0a, R12 + MOVQ R11, AX + MOVQ 136(SP), R13 + ANDQ R12, AX + XORQ R9, R10 + MOVQ 184(SP), R14 + ROLQ $0x1b, R10 + XORQ R10, AX + MOVQ AX, 120(DI) + NOP + XORQ DX, R13 + ROLQ $0x0f, R13 + MOVQ R12, AX + ORQ R13, AX + XORQ R11, AX + MOVQ AX, 128(DI) + NOP + XORQ R8, R14 + ROLQ $0x38, R14 + NOTQ R13 + MOVQ R13, AX + ORQ R14, AX + XORQ R12, AX + MOVQ AX, 136(DI) + ORQ R10, R11 + XORQ R14, R11 + MOVQ R11, 152(DI) + ANDQ R10, R14 + XORQ R13, R14 + MOVQ R14, 144(DI) + NOP + + // Result s + MOVQ 16(SP), R10 + MOVQ 64(SP), R11 + MOVQ 112(SP), R12 + XORQ DX, R10 + MOVQ 120(SP), R13 + ROLQ $0x3e, R10 + XORQ R8, R11 + MOVQ 168(SP), R14 + ROLQ $0x37, R11 + XORQ R9, R12 + MOVQ R10, R9 + XORQ CX, R14 + ROLQ $0x02, R14 + ANDQ R11, R9 + XORQ R14, R9 + MOVQ R9, 192(DI) + ROLQ $0x27, R12 + NOP + NOTQ R11 + XORQ BX, R13 + MOVQ R11, BX + ANDQ R12, BX + XORQ R10, BX + MOVQ BX, 160(DI) + NOP + ROLQ $0x29, R13 + MOVQ R12, CX + ORQ R13, CX + XORQ R11, CX + MOVQ CX, 168(DI) + NOP + MOVQ R13, DX + MOVQ R14, R8 + ANDQ R14, DX + ORQ R10, R8 + XORQ R12, DX + XORQ R13, R8 + MOVQ DX, 176(DI) + MOVQ R8, 184(DI) + + // Revert the internal state to the user state + NOTQ 8(DI) + NOTQ 16(DI) + NOTQ 64(DI) + NOTQ 96(DI) + NOTQ 136(DI) + NOTQ 160(DI) + RET diff --git a/crypto/keccak/sha3.go b/crypto/keccak/sha3.go new file mode 100644 index 0000000000..a554323244 --- /dev/null +++ b/crypto/keccak/sha3.go @@ -0,0 +1,244 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +import ( + "crypto/subtle" + "encoding/binary" + "errors" + "unsafe" + + "golang.org/x/sys/cpu" +) + +// spongeDirection indicates the direction bytes are flowing through the sponge. +type spongeDirection int + +const ( + // spongeAbsorbing indicates that the sponge is absorbing input. + spongeAbsorbing spongeDirection = iota + // spongeSqueezing indicates that the sponge is being squeezed. + spongeSqueezing +) + +type state struct { + a [1600 / 8]byte // main state of the hash + + // a[n:rate] is the buffer. If absorbing, it's the remaining space to XOR + // into before running the permutation. If squeezing, it's the remaining + // output to produce before running the permutation. + n, rate int + + // dsbyte contains the "domain separation" bits and the first bit of + // the padding. Sections 6.1 and 6.2 of [1] separate the outputs of the + // SHA-3 and SHAKE functions by appending bitstrings to the message. + // Using a little-endian bit-ordering convention, these are "01" for SHA-3 + // and "1111" for SHAKE, or 00000010b and 00001111b, respectively. Then the + // padding rule from section 5.1 is applied to pad the message to a multiple + // of the rate, which involves adding a "1" bit, zero or more "0" bits, and + // a final "1" bit. We merge the first "1" bit from the padding into dsbyte, + // giving 00000110b (0x06) and 00011111b (0x1f). + // [1] http://csrc.nist.gov/publications/drafts/fips-202/fips_202_draft.pdf + // "Draft FIPS 202: SHA-3 Standard: Permutation-Based Hash and + // Extendable-Output Functions (May 2014)" + dsbyte byte + + outputLen int // the default output size in bytes + state spongeDirection // whether the sponge is absorbing or squeezing +} + +// BlockSize returns the rate of sponge underlying this hash function. +func (d *state) BlockSize() int { return d.rate } + +// Size returns the output size of the hash function in bytes. +func (d *state) Size() int { return d.outputLen } + +// Reset clears the internal state by zeroing the sponge state and +// the buffer indexes, and setting Sponge.state to absorbing. +func (d *state) Reset() { + // Zero the permutation's state. + for i := range d.a { + d.a[i] = 0 + } + d.state = spongeAbsorbing + d.n = 0 +} + +func (d *state) clone() *state { + ret := *d + return &ret +} + +// permute applies the KeccakF-1600 permutation. +func (d *state) permute() { + var a *[25]uint64 + if cpu.IsBigEndian { + a = new([25]uint64) + for i := range a { + a[i] = binary.LittleEndian.Uint64(d.a[i*8:]) + } + } else { + a = (*[25]uint64)(unsafe.Pointer(&d.a)) + } + + keccakF1600(a) + d.n = 0 + + if cpu.IsBigEndian { + for i := range a { + binary.LittleEndian.PutUint64(d.a[i*8:], a[i]) + } + } +} + +// pads appends the domain separation bits in dsbyte, applies +// the multi-bitrate 10..1 padding rule, and permutes the state. +func (d *state) padAndPermute() { + // Pad with this instance's domain-separator bits. We know that there's + // at least one byte of space in the sponge because, if it were full, + // permute would have been called to empty it. dsbyte also contains the + // first one bit for the padding. See the comment in the state struct. + d.a[d.n] ^= d.dsbyte + // This adds the final one bit for the padding. Because of the way that + // bits are numbered from the LSB upwards, the final bit is the MSB of + // the last byte. + d.a[d.rate-1] ^= 0x80 + // Apply the permutation + d.permute() + d.state = spongeSqueezing +} + +// Write absorbs more data into the hash's state. It panics if any +// output has already been read. +func (d *state) Write(p []byte) (n int, err error) { + if d.state != spongeAbsorbing { + panic("sha3: Write after Read") + } + + n = len(p) + + for len(p) > 0 { + x := subtle.XORBytes(d.a[d.n:d.rate], d.a[d.n:d.rate], p) + d.n += x + p = p[x:] + + // If the sponge is full, apply the permutation. + if d.n == d.rate { + d.permute() + } + } + + return +} + +// Read squeezes an arbitrary number of bytes from the sponge. +func (d *state) Read(out []byte) (n int, err error) { + // If we're still absorbing, pad and apply the permutation. + if d.state == spongeAbsorbing { + d.padAndPermute() + } + + n = len(out) + + // Now, do the squeezing. + for len(out) > 0 { + // Apply the permutation if we've squeezed the sponge dry. + if d.n == d.rate { + d.permute() + } + + x := copy(out, d.a[d.n:d.rate]) + d.n += x + out = out[x:] + } + + return +} + +// Sum applies padding to the hash state and then squeezes out the desired +// number of output bytes. It panics if any output has already been read. +func (d *state) Sum(in []byte) []byte { + if d.state != spongeAbsorbing { + panic("sha3: Sum after Read") + } + + // Make a copy of the original hash so that caller can keep writing + // and summing. + dup := d.clone() + hash := make([]byte, dup.outputLen, 64) // explicit cap to allow stack allocation + dup.Read(hash) + return append(in, hash...) +} + +const ( + magicSHA3 = "sha\x08" + magicShake = "sha\x09" + magicCShake = "sha\x0a" + magicKeccak = "sha\x0b" + // magic || rate || main state || n || sponge direction + marshaledSize = len(magicSHA3) + 1 + 200 + 1 + 1 +) + +func (d *state) MarshalBinary() ([]byte, error) { + return d.AppendBinary(make([]byte, 0, marshaledSize)) +} + +func (d *state) AppendBinary(b []byte) ([]byte, error) { + switch d.dsbyte { + case dsbyteSHA3: + b = append(b, magicSHA3...) + case dsbyteShake: + b = append(b, magicShake...) + case dsbyteCShake: + b = append(b, magicCShake...) + case dsbyteKeccak: + b = append(b, magicKeccak...) + default: + panic("unknown dsbyte") + } + // rate is at most 168, and n is at most rate. + b = append(b, byte(d.rate)) + b = append(b, d.a[:]...) + b = append(b, byte(d.n), byte(d.state)) + return b, nil +} + +func (d *state) UnmarshalBinary(b []byte) error { + if len(b) != marshaledSize { + return errors.New("sha3: invalid hash state") + } + + magic := string(b[:len(magicSHA3)]) + b = b[len(magicSHA3):] + switch { + case magic == magicSHA3 && d.dsbyte == dsbyteSHA3: + case magic == magicShake && d.dsbyte == dsbyteShake: + case magic == magicCShake && d.dsbyte == dsbyteCShake: + case magic == magicKeccak && d.dsbyte == dsbyteKeccak: + default: + return errors.New("sha3: invalid hash state identifier") + } + + rate := int(b[0]) + b = b[1:] + if rate != d.rate { + return errors.New("sha3: invalid hash state function") + } + + copy(d.a[:], b) + b = b[len(d.a):] + + n, state := int(b[0]), spongeDirection(b[1]) + if n > d.rate { + return errors.New("sha3: invalid hash state") + } + d.n = n + if state != spongeAbsorbing && state != spongeSqueezing { + return errors.New("sha3: invalid hash state") + } + d.state = state + + return nil +} diff --git a/crypto/keccak/sha3_test.go b/crypto/keccak/sha3_test.go new file mode 100644 index 0000000000..28a20ec72d --- /dev/null +++ b/crypto/keccak/sha3_test.go @@ -0,0 +1,210 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keccak + +// Tests include all the ShortMsgKATs provided by the Keccak team at +// https://github.com/gvanas/KeccakCodePackage +// +// They only include the zero-bit case of the bitwise testvectors +// published by NIST in the draft of FIPS-202. + +import ( + "bytes" + "compress/flate" + "encoding" + "encoding/hex" + "encoding/json" + "hash" + "math/rand" + "os" + "strings" + "testing" +) + +const ( + testString = "brekeccakkeccak koax koax" + katFilename = "testdata/keccakKats.json.deflate" +) + +// testDigests contains functions returning hash.Hash instances +// with output-length equal to the KAT length for SHA-3, Keccak +// and SHAKE instances. +var testDigests = map[string]func() hash.Hash{ + "Keccak-256": NewLegacyKeccak256, + "Keccak-512": NewLegacyKeccak512, +} + +// decodeHex converts a hex-encoded string into a raw byte string. +func decodeHex(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +// structs used to marshal JSON test-cases. +type KeccakKats struct { + Kats map[string][]struct { + Digest string `json:"digest"` + Length int64 `json:"length"` + Message string `json:"message"` + + // Defined only for cSHAKE + N string `json:"N"` + S string `json:"S"` + } +} + +// TestKeccakKats tests the SHA-3 and Shake implementations against all the +// ShortMsgKATs from https://github.com/gvanas/KeccakCodePackage +// (The testvectors are stored in keccakKats.json.deflate due to their length.) +func TestKeccakKats(t *testing.T) { + // Read the KATs. + deflated, err := os.Open(katFilename) + if err != nil { + t.Errorf("error opening %s: %s", katFilename, err) + } + file := flate.NewReader(deflated) + dec := json.NewDecoder(file) + var katSet KeccakKats + err = dec.Decode(&katSet) + if err != nil { + t.Errorf("error decoding KATs: %s", err) + } + + for algo, function := range testDigests { + d := function() + for _, kat := range katSet.Kats[algo] { + d.Reset() + in, err := hex.DecodeString(kat.Message) + if err != nil { + t.Errorf("error decoding KAT: %s", err) + } + d.Write(in[:kat.Length/8]) + got := strings.ToUpper(hex.EncodeToString(d.Sum(nil))) + if got != kat.Digest { + t.Errorf("function=%s, length=%d\nmessage:\n %s\ngot:\n %s\nwanted:\n %s", + algo, kat.Length, kat.Message, got, kat.Digest) + t.Logf("wanted %+v", kat) + t.FailNow() + } + continue + } + } +} + +// TestKeccak does a basic test of the non-standardized Keccak hash functions. +func TestKeccak(t *testing.T) { + tests := []struct { + fn func() hash.Hash + data []byte + want string + }{ + { + NewLegacyKeccak256, + []byte("abc"), + "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45", + }, + { + NewLegacyKeccak512, + []byte("abc"), + "18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", + }, + } + + for _, u := range tests { + h := u.fn() + h.Write(u.data) + got := h.Sum(nil) + want := decodeHex(u.want) + if !bytes.Equal(got, want) { + t.Errorf("unexpected hash for size %d: got '%x' want '%s'", h.Size()*8, got, u.want) + } + } +} + +// TestUnalignedWrite tests that writing data in an arbitrary pattern with +// small input buffers. +func TestUnalignedWrite(t *testing.T) { + buf := sequentialBytes(0x10000) + for alg, df := range testDigests { + d := df() + d.Reset() + d.Write(buf) + want := d.Sum(nil) + d.Reset() + for i := 0; i < len(buf); { + // Cycle through offsets which make a 137 byte sequence. + // Because 137 is prime this sequence should exercise all corner cases. + offsets := [17]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1} + for _, j := range offsets { + if v := len(buf) - i; v < j { + j = v + } + d.Write(buf[i : i+j]) + i += j + } + } + got := d.Sum(nil) + if !bytes.Equal(got, want) { + t.Errorf("Unaligned writes, alg=%s\ngot %q, want %q", alg, got, want) + } + } +} + +// sequentialBytes produces a buffer of size consecutive bytes 0x00, 0x01, ..., used for testing. +// +// The alignment of each slice is intentionally randomized to detect alignment +// issues in the implementation. See https://golang.org/issue/37644. +// Ideally, the compiler should fuzz the alignment itself. +// (See https://golang.org/issue/35128.) +func sequentialBytes(size int) []byte { + alignmentOffset := rand.Intn(8) + result := make([]byte, size+alignmentOffset)[alignmentOffset:] + for i := range result { + result[i] = byte(i) + } + return result +} + +func TestMarshalUnmarshal(t *testing.T) { + t.Run("Keccak-256", func(t *testing.T) { testMarshalUnmarshal(t, NewLegacyKeccak256()) }) + t.Run("Keccak-512", func(t *testing.T) { testMarshalUnmarshal(t, NewLegacyKeccak512()) }) +} + +// TODO(filippo): move this to crypto/internal/cryptotest. +func testMarshalUnmarshal(t *testing.T, h hash.Hash) { + buf := make([]byte, 200) + rand.Read(buf) + n := rand.Intn(200) + h.Write(buf) + want := h.Sum(nil) + h.Reset() + h.Write(buf[:n]) + b, err := h.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + t.Errorf("MarshalBinary: %v", err) + } + h.Write(bytes.Repeat([]byte{0}, 200)) + if err := h.(encoding.BinaryUnmarshaler).UnmarshalBinary(b); err != nil { + t.Errorf("UnmarshalBinary: %v", err) + } + h.Write(buf[n:]) + got := h.Sum(nil) + if !bytes.Equal(got, want) { + t.Errorf("got %x, want %x", got, want) + } +} + +// BenchmarkPermutationFunction measures the speed of the permutation function +// with no input data. +func BenchmarkPermutationFunction(b *testing.B) { + b.SetBytes(int64(200)) + var lanes [25]uint64 + for i := 0; i < b.N; i++ { + keccakF1600(&lanes) + } +} diff --git a/crypto/keccak/testdata/keccakKats.json.deflate b/crypto/keccak/testdata/keccakKats.json.deflate new file mode 100644 index 0000000000..7a94c2f8bc Binary files /dev/null and b/crypto/keccak/testdata/keccakKats.json.deflate differ diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index 2cb5af84ac..f9cefeca1d 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -144,9 +144,9 @@ func ckzgComputeCellProofs(blob *Blob) ([]Proof, error) { if err != nil { return []Proof{}, err } - var p []Proof - for _, proof := range proofs { - p = append(p, (Proof)(proof)) + p := make([]Proof, len(proofs)) + for i, proof := range proofs { + p[i] = (Proof)(proof) } return p, nil } diff --git a/crypto/kzg4844/kzg4844_gokzg.go b/crypto/kzg4844/kzg4844_gokzg.go index e9676ff1b8..03627ebafb 100644 --- a/crypto/kzg4844/kzg4844_gokzg.go +++ b/crypto/kzg4844/kzg4844_gokzg.go @@ -108,9 +108,9 @@ func gokzgComputeCellProofs(blob *Blob) ([]Proof, error) { if err != nil { return []Proof{}, err } - var p []Proof - for _, proof := range proofs { - p = append(p, (Proof)(proof)) + p := make([]Proof, len(proofs)) + for i, proof := range proofs { + p[i] = (Proof)(proof) } return p, nil } diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index ae6352e356..9affa82c8a 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -43,6 +43,9 @@ func sigToPub(hash, sig []byte) (*secp256k1.PublicKey, error) { if len(sig) != SignatureLength { return nil, errors.New("invalid signature") } + if len(hash) != DigestLength { + return nil, fmt.Errorf("hash is required to be exactly %d bytes (%d)", DigestLength, len(hash)) + } // Convert to secp256k1 input format with 'recovery id' v at the beginning. btcsig := make([]byte, SignatureLength) btcsig[0] = sig[RecoveryIDOffset] + 27 @@ -76,8 +79,8 @@ func SigToPub(hash, sig []byte) (*ecdsa.PublicKey, error) { // // The produced signature is in the [R || S || V] format where V is 0 or 1. func Sign(hash []byte, prv *ecdsa.PrivateKey) ([]byte, error) { - if len(hash) != 32 { - return nil, fmt.Errorf("hash is required to be exactly 32 bytes (%d)", len(hash)) + if len(hash) != DigestLength { + return nil, fmt.Errorf("hash is required to be exactly %d bytes (%d)", DigestLength, len(hash)) } if prv.Curve != S256() { return nil, errors.New("private key curve is not secp256k1") diff --git a/crypto/signify/signify_fuzz.go b/crypto/signify/signify_fuzz.go index 239a2134df..d11125c697 100644 --- a/crypto/signify/signify_fuzz.go +++ b/crypto/signify/signify_fuzz.go @@ -56,7 +56,7 @@ func Fuzz(data []byte) int { fmt.Printf("untrusted: %v\n", untrustedComment) fmt.Printf("trusted: %v\n", trustedComment) - err = SignifySignFile(tmpFile.Name(), tmpFile.Name()+".sig", testSecKey, untrustedComment, trustedComment) + err = SignFile(tmpFile.Name(), tmpFile.Name()+".sig", testSecKey, untrustedComment, trustedComment) if err != nil { panic(err) } @@ -68,7 +68,7 @@ func Fuzz(data []byte) int { signify = path } - _, err := exec.LookPath(signify) + _, err = exec.LookPath(signify) if err != nil { panic(err) } diff --git a/docs/upstream-merges/v1.17.4/fork-register.md b/docs/upstream-merges/v1.17.4/fork-register.md new file mode 100644 index 0000000000..63fe7267fa --- /dev/null +++ b/docs/upstream-merges/v1.17.4/fork-register.md @@ -0,0 +1,118 @@ +# Fork / EIP register: go-ethereum v1.17.4 sync + +Per-batch record of every upstream hardfork or EIP encountered during the sync +(invariant 9). **Default policy: any hardfork/EIP introduced upstream is merged +disabled by default in Bor** — its activation gate stays nil/unset so the code +is dormant on every Bor network. Enabling is always a separate, explicit +PoS-team decision (its own change + review + N-1/N/N+1 activation tests). + +`decision` ∈ {`never-enable`, `defer`, `adopt-later`, `already-ours`}. +`verified-dormant` = confirmed the Bor gate is nil/unset **and** the behavior is +gate-guarded (not ungated / unconditional). + +| EIP / fork | upstream PR | batch | touches | Bor gate (value) | decision | verified-dormant | +| ---------- | ----------- | ----- | ------- | ---------------- | -------- | ---------------- | +| Verkle / binary-trie transition | verkle/UBT groundwork (`cf93077fa`); go-verkle removal `#33461` (`710008450`) | 3 (v1.17.0 2/12) intro → removed batch 6 (`b66d3f157`) | `core/state`, `core/vm`, `core/types/block.go`, `trie/*`, `cmd/*` | `VerkleBlock = nil` (all Bor presets); `IsVerkle(num)` block-based — **fork gate KEPT dormant** | **go-verkle impl REMOVED** (2026-07-23, `b66d3f157`, mirroring #33461); binary tree kept; fork gate kept dormant — see Notes | yes — go-verkle/go-ipa deleted; `VerkleTrie`/`PointCache`/verkle `ExecutionWitness` gone; `AccessEvents` key derivation now `bintrie`. `VerkleBlock: nil` on every preset so `IsVerkle()` never activates; upstream's follow-up verkle stub/rename PRs to be mirrored in later batches. | +| EIP-8024 (DUPN, SWAPN, EXCHANGE opcodes) | boundary `a122dbe45` | 4 (v1.17.0 3/12) | `core/vm/eips.go` (activator), `core/vm/jump_table.go` | registered in the `activators` map only; **not wired into any fork instruction set**; reachable solely via `ChainConfig.ExtraEips`, which is unset on every Bor preset | **defer** — dormant, not adopted | yes — `enable8024` is never invoked by any `new*InstructionSet()` (0 refs in `jump_table.go`); no Bor preset sets `ExtraEips`, so the DUPN/SWAPN/EXCHANGE opcodes are absent from every Bor network's jump table. Not ungated. | +| EIP-8024 PC-increment fix (#33361) | boundary `228933a66` | 5 (v1.17.0 4/12) | `core/vm/instructions.go`, `core/vm/instructions_test.go` | same as above — no gate change; corrects the opcode implementation only | **defer** (unchanged) | yes — re-confirmed `enable8024` still unreferenced by any instruction set and no preset sets `ExtraEips`. The fix corrects the (still-dormant) DUPN/SWAPN/EXCHANGE PC handling; it does not enable them. Auto-merged clean. | +| EIP-8024 missing-immediate-byte update (#33614) | boundary `845009f68` | 10 (v1.17.0 9/12) | `core/vm/instructions.go`, `core/vm/instructions_test.go` | same as above — no gate change; spec-clarification only (missing immediate byte after DUPN/SWAPN/EXCHANGE now treated as `0x00`, matching PUSHn, instead of `ErrInvalidOpCode`) | **defer** (unchanged) | yes — re-confirmed `enable8024` registered only in the `activators` map (`eips.go:46`), never invoked by any `new*InstructionSet()`; no Bor preset sets `ExtraEips`. The opcodes stay absent from every Bor network's jump table; the behavior change is unreachable. Auto-merged clean; `TestEIP8024_Execution` (incl. new `DUPN_MISSING_IMMEDIATE`) passes. | +| Fulu (Ethereum CL fork) beacon types (#33349) | boundary `228933a66` | 5 (v1.17.0 4/12) | `beacon/types/beacon_block.go`, `beacon/types/exec_header.go` | n/a — CL block/header type definitions; not a Bor execution-layer fork gate | **already-ours / inert** | yes — Fulu is an Ethereum consensus-layer fork; `beacon/types` carries CL block/header shapes used by the beacon light client, not Bor's PoA consensus path. No `params/config.go` gate, no execution behavior. Auto-merged clean, inert for Bor networks. | +| Catalyst getPayload fork-timestamp checks (#32754) | boundary `228933a66` | 5 (v1.17.0 4/12) | `eth/catalyst/api.go` | n/a — Engine API `engine_getPayloadVX` version guard against configured fork timestamps | **already-ours / inert** | yes — the Engine API (`eth/catalyst`) is inherited-but-unused in Bor's PoA consensus (per repo `CLAUDE.md`). Adds a version-vs-fork-timestamp guard on getPayload; introduces no new fork and flips no gate. Auto-merged clean; the only catalyst conflicts were sync-mode (`ConfigSyncMode`→`api.eth.SyncMode()`), unrelated to this guard. | + +## Notes + +- **go-verkle scaffolding REMOVED (adopted #33461)** — **decided 2026-07-23 (user-approved).** + go-ethereum abandoned the go-verkle/go-ipa verkle implementation (#33461), and Bor's + verkle was never enabled (`VerkleBlock=nil`). Rather than carry orphaned scaffolding + and re-revert #33461 every batch, Bor **mirrors upstream's removal patch-by-patch**: + a dedicated commit on top of batch 6 (`85a1750c5`) mirrors #33461 — deletes + `trie/verkle*.go`, `trie/utils/verkle.go` (incl. `PointCache`), `cmd/geth/verkle.go`; + removes the `ExecutionWitness` verkle fields from `core/types/block.go`, `PointCache` + from the `state.Database`/`vm.StateDB` interfaces + all `core/state` implementers + (incl. Bor's `ParallelStateDB`), `NewAccessEvents()` drops its `PointCache` arg, + `AccessEvents` key derivation switches to `bintrie` (binary tree kept); drops + go-verkle/go-ipa from `go.{mod,sum}` + `cmd/keeper`; removes `verkleCommand`. + **Kept:** the Verkle **fork gate** (`VerkleBlock`/`IsVerkle`/`OverrideVerkle`, + dormant) — upstream also keeps `isVerkle` vars per #33461's author note; and Bor's + stateless client (`stateless.Witness`, distinct from the verkle `ExecutionWitness`). + **Going forward:** upstream's follow-up verkle stub PRs are mirrored as they arrive + in later batches (Bor tracks upstream's removal trajectory, staying in lockstep). +- **v1.17.0 batch 4/12 (`5dfcffcf3`, batch 5)** — no hardfork/EIP introduced. + #33352 removed the blobpool's Osaka legacy-blob-sidecar conversion machinery; + Osaka remains **dormant** in Bor (`IsOsaka(num)` block-based, unset on every + preset), so this is dead-code cleanup, not a gate change. The blob-sidecar + version check in `core/txpool/validation.go` was kept on Bor's block-based + `IsOsaka(head.Number)`. Fulu (`beacon/types`, #33349) is an Ethereum CL fork, + inert for Bor's PoA path. +- **v1.17.0 batch 9/12 (`845009f68`, batch 10)** — no hardfork/EIP introduced. + One fork surface touched, dormant: EIP-8024 missing-immediate-byte update + (#33614, `251b86310`) refines the DUPN/SWAPN/EXCHANGE opcode behavior but + changes no gate — `enable8024` remains `ExtraEips`-only (unset on every Bor + preset), so the opcodes stay unreachable (see table row). The batch's two real + features (eth_getProofs-for-history #32727, callTracer log index #33629) are + RPC/tracing, not forks; the adopted trienode `FullValueCheckpoint` / + `NodeFullValueCheckpoint` knob is a storage-compression tuning parameter + (dormant while `TrienodeHistory = -1`), not a fork. The deferred code-stats + (#33659) and slow-block (#33655) lines touch no gate. Nothing enters the + register. +- **v1.17.0 batch 10/12 (`c12959dc8`, batch 11)** — no hardfork/EIP introduced; + no fork gating, precompile set, or gas schedule changed. EIP-8024's only touch + this batch is test-only (#33785 `bc0db302e` adds a PUSH0 handler to the EIP-8024 + *test* mini-interpreter in `core/vm/instructions_test.go`); `enable8024` stays + `ExtraEips`-only and dormant. The `params.Rules` change was an upstream refactor + (ChainID field removal + timestamp param name), reconciled preserving Bor's + block-based `Is(num)` gating and Bor fork bools — no activation moved, and + Bor's `ChainID` field was retained (see ledger + `TestReinforceMultiClientPreCompilesTest`). + The `crypto/keccak` vendoring, the `vm.TxContext.GasPrice`/`stateObject.addrHash` + type migrations, and the metrics/rlp/eip4844-API reworks are non-fork. Nothing + enters the register. +- **v1.17.0 batch 12/12 (`0cf3d3ba4`, batch 13, v1.17.0 release)** — no hardfork/EIP + introduced. `#33860` (consensus/misc header hardening — defensive parent-baseFee + nil-check in `VerifyEIP1559Header`) is **already present in Bor HEAD** (`eip1559.go:57`), + merged as a no-op (verified present, not dropped — it's consensus-security). `9b78f45e3` + (secp256k1 coordinate check) is the **v1.16.9 duplicate** the plan flagged — already + applied in the v1.16.9 milestone (`a06dbd7d2`), comment-only conflict, took ours. The + batch's two large features (OTel newPayload #33521, delayed p2p decoding #33835) are + declined/deferred and non-fork; `#33842` (download progress bar) is non-fork. Nothing + enters the register. **All v1.17.0 EIP surfaces remain dormant** (EIP-8024 `ExtraEips`-only, + Verkle gate nil). +- **v1.17.0 batch 11/12 (`c50e5edfa`, batch 12)** — no hardfork/EIP introduced; + no fork gating, precompile set, or gas schedule changed. EIP-8024's only touch + this batch is test-only (#33787 — the DUPN/SWAPN/EXCHANGE test table in + `core/vm/instructions_test.go` now asserts explicit error *types* + expected + opcode instead of `wantErr bool`); `enable8024` stays `ExtraEips`-only and + dormant, no production `core/vm`/`params` code touched. The batch's two large + features (OpenTelemetry CLI wiring #33484/#33772, EraE format #32157) are + declined/deferred and non-fork; the adopted fixes (trie/rlp/rawdb/pathdb/ + gethclient/node) are non-fork. Nothing enters the register. +- **v1.17.0 batch 8/12 (`8fad02ac6`, batch 9)** — no hardfork/EIP introduced. + This batch touched `core/vm` gas/opcode surfaces via four upstream PRs, all + **deferred** to Bor HEAD (see `needs-wiring.md`), so Bor's EVM behavior is + unchanged and no gate is affected: `#33281`/`#33637` relocate EIP-214 + write-protection into the gas handlers (unconditional, no fork gate); + `#33450` is an EIP-2929 cold-access gas early-return (existing fork, no new + gate); `#32919` reworks selfdestruct balance handling + tracer hooks + (net-state-neutral, clashes with Bor's BlockSTM `SelfDestruct`). None + introduces or enables a fork; nothing enters the register. Trienode-history + (`#32621`/`#33551`/`#33584`) was adopted but is a storage feature, not a + fork — defaulted to `-1` (disabled). +- **v1.17.0 batch 7/12 (`94710f79a`, batch 8)** — no hardfork/EIP introduced. + Three fork-adjacent commits, none of which touch a fork gate: `5b99d2bba` + (drop peers on invalid KZG proofs — txpool peer-drop robustness, no gate), + `b993cb6f3` #32717 (allow gaps in blobpool — queue behavior, no gate), and + `a32851fac` #33542 (graphql GasPrice for blob/setcode txs — RPC formatting). + Osaka/blob forks remain dormant (`IsOsaka(num)` block-based, unset on every + Bor preset). The batch's large state-tracking PRs (#33490 state update hook, + #33532 code-reader cache stats) introduce no fork gating and were deferred + wholesale to Bor HEAD (see `needs-wiring.md`), so nothing enters the register. +- **v1.16.9 (batch 1)** — crypto security backports (ECIES invalid-curve, + secp256k1 coordinate check). No hardfork/EIP activation; nothing to register. +- **v1.17.0 batch 1/12 (`f23d506b7`, batch 2)** — bugfix sweep (rawdb / state / + pathdb / RPC / blobpool / filters). No hardfork/EIP introduced; nothing to + register. +- **Upcoming (from the plan's risk flags), all to be entered dormant/disabled + unless the PoS team decides otherwise:** EIP-8024 (batches 4, 10, 15), + EIP-7843 SLOTNUM (15), EIP-7778 (16), EIP-7954 (17), EIP-7708 (19), EIP-7976 + (23), EIP-7981 (25), EIP-8037 (32), Amsterdam jump-table / precompile wiring + (14–16), Osaka/BPO blob-schedule forks, gas-vector / gas-budget reworks + (21, 23), jumpdest cache (29). diff --git a/docs/upstream-merges/v1.17.4/ledger.md b/docs/upstream-merges/v1.17.4/ledger.md index c860c693e5..7cb03c81a6 100644 --- a/docs/upstream-merges/v1.17.4/ledger.md +++ b/docs/upstream-merges/v1.17.4/ledger.md @@ -7,3 +7,417 @@ Append-only. One line per non-trivial resolution: file — class — upstream PR - `crypto/ecies/ecies.go` — class 3 — #33669 — took upstream's `IsOnCurve` validation in `GenerateShared` — genuine invalid-curve guard absent from Bor; expect the duplicate commit (`c974722dc`, v1.17.0 batch 10) to now merge clean. - `crypto/secp256k1/curve.go` — class 3 — upstream `895a8597c` — kept Bor's text — Bor pre-applied the identical coordinate check via `d9fac7a4c`; only delta was Bor's comment. - `crypto/secp256k1/ext.h` — class 3 — upstream `895a8597c` — took upstream's combined `if (!a || !b)` form over Bor's equivalent two-if variant — zero behavior delta; converges text with upstream to de-conflict future syncs. + +## v1.17.0 + +### batch 1/12 (boundary `f23d506b7`, 20 commits) + +- `core/state/statedb.go` — class 3 ⚠ — #33082 — kept Bor's `Logs()` unsorted (dropped upstream's `sort.Slice` + now-unused `sort` import) — Bor sorts HF-gated at `blockchain.go:2380` (pre-Madhugiri only) and relies on `Logs()` = receipt logs ++ state-sync logs tail order (`blockLogs[len(logs):]`); upstream's unconditional in-`Logs()` sort would reorder post-Madhugiri where Bor deliberately does not. Consensus-relevant; `core/state` tests green. +- `eth/filters/filter_system.go` — class 3 — #33108 — took upstream's `txHashes map[common.Hash]struct{}` + kept Bor's `stateSyncData` field — Bor's own `hashSet` already builds `struct{}`, the sole read (`filter.go:601`) is comma-ok/type-agnostic; `eth/filters` tests green. +- `core/txpool/blobpool/blobpool_test.go` — class 3 (test) — #33161 — union: added upstream's `BlobTxs: true` field + kept Bor's `, nil` 2nd arg to `Pending()` — orthogonal; blobpool tests green. +- `core/blockchain_insert.go` — class 3 — #33155 — took upstream's deletion of unused `insertIterator.peek()` — provenance: Bor side was a blank-line churn over base (`nolint:unused`, no callers); converges and de-conflicts. +- `core/rawdb/database.go` — class 1 — #33025 — kept Bor's `ethDb := &freezerdb{…}` form + added upstream's `readOnly: opts.ReadOnly` field — Bor needs the `ethDb` ref for the witness-store setup below; `readOnly` struct field auto-merged; build + rawdb tests green. +- `common/types.go` — class 1 — #32998 — kept both `HexToRefHash` (Bor) and `IsHexHash` (upstream) — orthogonal additions at the same location. +- `cmd/geth/config.go` — class 1 — #32998 — removed both unused imports (`crypto`, `hexutil`) — upstream reworked config.go to use `common.IsHexHash`; the merged Bor body references neither; build-confirmed. +- `rpc/http.go` — class 2 — #33122 — took upstream's `cleanlyCloseBody(respBody)` — provenance: Bor side was lint/merge churn (`f3ffacf2d wsl lint`), no deliberate divergence; converges; rpc tests green. +- `rpc/http_test.go` — class 2 — #33122 — took upstream's `cleanlyCloseBody(resp.Body)` — provenance: Bor side was upstream code carried via a prior merge (`f3c19b1a1`, #30384); converges. +- `version/version.go` — class 1 — v1.16.8 dev cycle — kept HEAD (`1.16.9`/`stable`) — avoids a mid-stack version regression; the v1.17.0 release batch (12/12) sets `1.17.0`. + +### batch 2/12 (boundary `cf93077fa`, 20 commits) — verkle/UBT groundwork, rawdb+rlp hardening + +Consensus / concurrency (full write-up): + +- `core/state_processor.go` — class 3 ⚠ — #33124 (tracer relocation) — **merged**: kept Bor's `interruptCtx` nil-init and `NewEVMBlockContext(header, p.chain, author)` (Bor passes the author), adopted upstream's relocation of `tracingStateDB` to before the DAO-fork block (required — `ApplyDAOHardFork` now takes it) and dropped the second, now-duplicate definition. `config` == `p.chainConfig()`. Bounded `core` Process tests green. +- `triedb/pathdb/history_index.go` — class 3 ⚠ concurrency — upstream refactored `readGreaterThan` to `return it.ID(), nil` (iterator-based) — **took upstream's refactor and PORTED Bor's `#1875` RWMutex** into `newIterator`'s closure (the new home of the `r.readers` access), keeping `refresh()`'s existing lock; imports keep `sync`, drop now-unused `sort`. Validated with `go test -race ./triedb/pathdb` (green, no data race). + +Bor-owned surfaces kept (class 1): + +- `core/rawdb/accessors_chain.go` — class 1 — upstream removed 4 accessors — **kept Bor's HEAD**: `ReadAllHashesInRange` is used in prod (`blockchain.go`, witness/block pruners); `FindCommonAncestor` is used by Bor's `eth/bor_checkpoint_verifier_test.go`; `ReadAllCanonicalHashes`/`DeleteBadBlocks` used in rawdb tests. Bor depends on all four. +- `core/rawdb/accessors_chain_test.go` — class 1 — kept HEAD (tests for the retained accessors); re-added dropped `reflect` import. +- `internal/ethapi/simulate.go` — class 3 — kept Bor's `remaining`-based gas-limit check returning `(bool, error)` (deliberate signature/logic divergence); upstream's change was only a `*gasUsed` fmt fix to code Bor replaced. +- `tests/init.go` — class 3 — kept ours (Bor's `params.ChainConfig` lacks the BPO time-based fields upstream's new test fork configs reference). + +Take-theirs / converge (class 2): + +- `rlp/iterator.go` — class 2 — took upstream's `it.err = nil` (clear transient error after a successful `next`); Bor matched base modulo a blank line. +- `accounts/keystore/keystore.go` — class 2 — took upstream's `defer zeroKey(key.PrivateKey)` (×2, key hygiene); Bor matched base. +- `cmd/geth/dbcmd.go` — class 2 — took upstream's `rawdb.NewTableWriter` (replaces `olekukonko/tablewriter`); dropped the now-unused `tablewriter` import + a duplicate `cli`. +- `cmd/geth/verkle.go` — class 2 — took upstream's `resolver(childC[:])` reordering. +- `cmd/evm/internal/t8ntool/transition.go` — class 2/3 — took upstream's binary-trie (`bt`) output support + imports; dropped a duplicate `cli`; adapted `IsVerkle` call to Bor's 1-arg signature. + +Accept upstream deletion / removal: + +- `core/chain_makers.go` — class 3 — accepted upstream's deletion of `GenerateVerkleChain`/`GenerateVerkleChainWithGenesis` (zero usage outside the also-deleted verkle test); dropped `go-verkle`, de-duplicated `uint256`, adapted two `IsVerkle` calls to Bor's 1-arg signature. +- `core/verkle_witness_test.go` — class 3 — accepted upstream's deletion (used the removed `GenerateVerkleChain`). +- `trie/bintrie/iterator_test.go` — class 3 — accepted upstream's deletion (Bor's only change was `167e5aee4` reformat churn). +- `core/bintrie_witness_test.go` — class 3 (**coverage gap flagged**) — **removed** upstream's newly-added binary-trie witness test: it can't compile against Bor (uses time-based `ShanghaiTime`/`VerkleTime` config fields + `u64` helper Bor lacks, and Bor's different `InsertChain` signature). Re-add and adapt if Bor ever adopts the binary-trie transition. + +Merge-integration fixes (silently-merged upstream code vs Bor APIs; caught by build/test tier): + +- `core/rawdb/database_tablewriter.go` (upstream-added) — replaced builtin `max(int,int)` usage with explicit comparisons (Bor's rawdb defines a `max(uint64,uint64)` in `pruner.go` that shadows the builtin). `database_tablewriter_unix.go` deleted by upstream (auto-merged). +- `trie/transitiontrie/transition.go`, `triedb/pathdb/database.go` — dropped now-unused `go-verkle` imports (verkle-removal artifacts). +- `cmd/evm/internal/t8ntool/execution.go`, `tests/block_test_util.go` — adapted upstream code to Bor's block-based verkle API (`IsVerkle` 1-arg; `IsVerkleGenesis()` instead of `VerkleTime`). +- `go.mod`/`go.sum`, `cmd/keeper/go.{mod,sum}` — kept ours (Bor's dep graph is ahead: `liner` v1.2.2, `stun/v3`, `go-toml`, bumped `cpuid`/`mapstructure`), then `go mod tidy` on both modules to reconcile. + +### batch 3/12 (boundary `a122dbe45`, 20 commits) — EIP-8024, miner `--miner.maxblobs`, gnark-crypto bump + +Fork/EIP (see `fork-register.md`): + +- **EIP-8024** (DUPN/SWAPN/EXCHANGE) — **dormant / defer**. Registered in `core/vm/eips.go` `activators` map but not wired into any fork instruction set and no Bor preset sets `ExtraEips`; opcodes absent from every Bor jump table. Disabled by default (invariant 9). + +Deferred feature (needs-wiring, tracked for milestone triage): + +- `--miner.maxblobs` — **deferred, not adopted this batch.** Upstream added a configurable per-block blob cap (`MinerMaxBlobsFlag`, `miner.Config.MaxBlobsPerBlock`, `(*Miner).maxBlobsPerBlock`, replacing `eip4844.MaxBlobsPerBlock(...)` call sites). Bor's miner is heavily diverged (`worker` struct with `w.chainConfig`, restructured `commitTransaction`, and it deleted the isCancun blob-space-left block), so wiring the knob requires adapting it to Bor's worker structure + flag→config plumbing — out of scope for a merge batch and not consensus-relevant. Kept Bor's behavior (blob cap = `eip4844.MaxBlobsPerBlock` protocol default; backward-compatible). Resolutions: `miner/miner.go`, `miner/worker.go` (4 hunks), `cmd/utils/flags.go` (flag def) → took ours; removed the auto-merged plumbing that referenced the dropped flag/field — `cmd/utils/flags.go` `setMiner` block and `cmd/geth/main.go` flag registration. + +Other resolutions: + +- `eth/downloader/queue.go` — class 2 — took upstream's removal of the `proc` throttle counter (3 hunks); Bor matched base modulo whitespace (no deliberate divergence). +- `node/defaults.go` — class 1 — merged: kept Bor's `TxArrivalWait: 500ms` + adopted upstream's `DiscoveryV4/V5: true` defaults. +- `.github/workflows/validate_pr.yml` — non-code — kept Bor's deletion (modify/delete; Bor removed this upstream CI workflow, has its own). +- `go.mod`/`go.sum`, `cmd/keeper/go.{mod,sum}` — kept ours + `go mod tidy` (both modules). + +Merge-integration fix: + +- `tests/block_test_util.go` — restored Bor's single `run(..., useV2 bool)` method (Forks lookup inside). The auto-merge had spliced upstream's `run(config, ...)` refactor in, producing a **duplicate `run` method** and orphaning `useV2` in the implementation (a compile error). Kept Bor's V2-BlockSTM (`Run`/`RunV2`) structure. + +### batch 4/12 (boundary `228933a66`, 20 commits) — catalyst getPayload fork checks, pebble config, slow-block stats, downloader syncmode + +Conflicts: 20 files (18 content, 2 modify/delete). Central theme = upstream's #33157 "keep current syncmode in downloader only" refactor colliding with Bor's deep downloader fork; plus two independent observability refactors (#32812 slow-block, #33254 reader stats) that Bor diverged from. All three refactors **deferred** (tracked in `needs-wiring.md`); the batch's genuinely-independent bugfixes (#33331, #33330, #32999, #33353, #33361, #33280, #33320, #33336, #33329, #33058, #33338, #33355) were adopted. + +Fork/EIP (see `fork-register.md`): EIP-8024 PC-increment fix (#33361, still dormant), Fulu beacon types (#33349, inert — CL fork), catalyst getPayload fork-timestamp guard (#32754, inert — Engine API unused in Bor). No gate flipped; invariant 9 holds. + +Big decision — **defer #33157 (syncModer refactor) entirely**: + +- Bor keeps `SyncMode` in `eth/downloader/modes.go` (with Bor-only `StatelessSync`/`BytecodeOnlySnapSync`); upstream moved the type to `eth/ethconfig` and pushed mode selection into a new `syncModer`. Porting would relocate the type (breaking Bor's extra modes' package home) and rework Bor's forked `bor_downloader.go` — out of scope, non-consensus. Kept Bor's mode-taking `synchronise`/`BeaconSync`/`BeaconExtend`/`BeaconDevSync` signatures, `h.snapSync.Load()` checks, and `Ethereum.SyncMode()`. +- Removed the added `eth/downloader/syncmode.go`. +- Reverted to Bor HEAD (their entire delta this batch was #33157): `eth/handler.go`, `eth/handler_test.go`, `eth/handler_eth_test.go`, `eth/sync_test.go`, `eth/downloader/beaconsync.go`, `eth/downloader/beacondevsync.go`. Reverting `handler.go` also preserved Bor's auto-disable-snap logic in `enableSyncedFeatures` (upstream removed it, relying on the deferred `syncModer.disableSnap`). +- `eth/backend.go` — class 1 — took ours (kept Bor's `SyncMode()` method that upstream removed); the auto-merged `SlowBlockThreshold` line reverted with the #32812 deferral. +- `eth/syncer/syncer.go` — class 1 — took ours (`BeaconDevSync(downloader.FullSync, target)`, dropped upstream's `ConfigSyncMode` guard). +- `eth/catalyst/api.go` — class 1 — took ours on the 2 sync-mode conflicts (`api.eth.SyncMode()` vs upstream `Downloader().ConfigSyncMode()`); restored Bor's `mode` arg on the auto-merged `BeaconSync`/`BeaconExtend` call sites (build-caught). `#32754` getPayload guard auto-merged, kept (inert). + +**Defer #32812 (slow-block stats) entirely** — logging entangled with upstream's `blockProcessingResult.stats`/`logSlow`, which Bor's tuple-returning `ProcessBlock` doesn't share. Removed `core/blockchain_stats.go` (new upstream file) and the `--debug.logslowblock` flag (`cmd/utils/flags.go` + `cmd/geth/{main,chaincmd}.go` registrations — build-caught dangling refs); reverted `core/blockchain.go`, `eth/backend.go`, `eth/ethconfig/{config,gen_config}.go` to Bor HEAD (their only delta this batch was #32812). + +**Defer #33254 (reader cache-stats rename)** — Bor's `core/state/reader.go` carries richer prefetch-attribution instrumentation (the pipelined-SRC work) using the old field names, consumed by `core/blockchain.go` via `GetStats()`. Half-merged rename would break the build; reverted `core/state/reader.go` + `core/state/statedb.go` to Bor HEAD. + +Independent conflicts resolved (Bor had no meaningful divergence → converged to upstream): + +- `common/bitutil/bitutil.go` + `_test.go` — class 2 — took theirs: `XORBytes` now delegates to `crypto/subtle.XORBytes` (#33331); Bor's only divergence was blank-line reformat. +- `ethdb/pebble/pebble.go` — class 2 (1 hunk) — took theirs: `MemTableStopWritesThreshold: memTableNumber * 2` (#33353). Bor had independently set `* 2` (`memTableLimit`); upstream renamed the var to `memTableNumber` (auto-merged) and adopted the same `* 2`, so Bor's intent converged. Bor's custom pebble amplification/IO/WAL metrics (auto-merged block) preserved. +- `p2p/rlpx/rlpx.go` — class 2 (1 hunk, + build fix) — body auto-merged to `subtle.XORBytes`; the import conflict's "theirs" side was empty (upstream dropped `bitutil`), which also dropped Bor's `snappy`/`sha3`. Corrected: kept `snappy` + `sha3` (Bor's body uses both), dropped `bitutil`. +- `cmd/utils/flags.go` — class 2 (3 hunks) — took theirs: `--networkid` override relocation + explicit-set warning (#32999), and the preimages config-file bugfix (`if ctx.IsSet(CachePreimagesFlag.Name)`, #33330). Bor matched base on all three. +- `eth/api_debug.go` — class 1 (4 hunks) — took ours: Bor's `ProcessBlock(parentBlock, header, nil, nil)` (BlockSTM-V2/stateless) signature; base == theirs (upstream didn't touch these lines). + +Verification (per-batch tier): `go build ./...` pass; `go vet` clean on all touched + `eth/…` packages (test files compile); `go test` pass on `common/bitutil`, `ethdb/pebble`, `p2p/rlpx`, `core/filtermaps`, `eth/filters`, `p2p/nat`, `core/vm` (+`program`,`runtime`), `triedb/pathdb`. gofmt clean. `misc/` not staged. + +### batch 5/12 (boundary `5dfcffcf3`, 20 commits) — state/code-read metrics, tx fetcher validation, blobpool legacy sidecar removal + +Conflicts: 15 files (13 content, 2 modify/delete). No hardfork/EIP introduced (see `fork-register.md`). Three refactors deferred (Bor-diverged, tracked in `needs-wiring.md`); several independent fixes adopted; one latent HEAD test break fixed. + +Adopted (Bor had no meaningful divergence, or clean bugfix): + +- **#33352 blobpool legacy sidecar removal** — took theirs on `core/txpool/blobpool/blobpool.go` (3 hunks: removed `conversionTimeWindow`, `convertLegacySidecar(s)`, `compareAndSwap`, `preCheck`), adopted the auto-merged deletion of `conversion.go`, and removed `conversion_test.go` (modify/delete: upstream deleted it; Bor's only change was an import reorder + one `t.Skip`). Bor's copy was dormant dead code (`//nolint:unused`; Osaka block-gated and inactive), merely carried from upstream — not a deliberate divergence, so converged. `blobpool_test.go` resolved per-hunk (theirs, theirs, **ours**): removed `setHeadTime` + `TestSidecarConversion` (test the removed machinery), **kept Bor's `TestSubscribeRebroadcastTransactions`**; upstream's now-unused `newUint64` dropped with it. +- **#33370** `p2p/tracker/tracker.go` — took theirs (the `wasHead` fix; the `wasHead :=` precompute auto-merged, and Bor's `req.expire.Prev()==nil` re-check after `Remove` was the pre-fix bug). +- **#33447** `tests/fuzzers/rangeproof/rangeproof-fuzzer.go` — took theirs (kv struct lost its unused 3rd field; the struct def auto-merged to 2 fields). +- `ethclient/gethclient/gethclient.go` — took theirs (`return &result, nil`). +- Auto-merged clean and kept: #33280, #33320, #33336, #33329, #33044, #33344, #33381, #33363, #33058, #33338, #32247/#32637 (state proof / overlay), #33440 (catalyst timestamp log), fulu beacon types #33349. + +Preserved Bor divergence: + +- `core/txpool/validation.go` — took **ours** on the EIP-7623 floor-data-gas gate: `opts.Config.IsPrague(head.Number) && opts.Config.Bor != nil` (block-based + Bor-chain guard) over upstream's `rules.IsPrague`. Fixed the auto-merged blob-sidecar-version check `opts.Config.IsOsaka(head.Number, head.Time)` → block-based `IsOsaka(head.Number)` (Bor's 1-arg signature). +- `tests/block_test_util.go` — took **ours** (Bor's `Run`/`RunV2`/`run(...,useV2 bool)` V2-BlockSTM differential harness) over #33383's inlining of `run` into `Run`; removed the duplicate `Network()` method the merge left (upstream added one; Bor already had it). + +Deferred (Bor-diverged refactors → `needs-wiring.md`): + +- **#33378 tx-announcement metadata validation** — upstream replaced the fetcher's `hasTx func(common.Hash) bool` callback with `validateMeta func(common.Hash, byte) error`, converting all call sites. Bor's fetcher is diverged (`txMetadata` type, 34 old-signature test call sites). Reverted `eth/fetcher/tx_fetcher.go`, `tx_fetcher_test.go`, `eth/handler.go`, `eth/handler_test.go`, `tests/fuzzers/txfetcher/txfetcher_fuzzer.go` to Bor HEAD (their entire batch delta was #33378 + the already-deferred #33157). Security-positive; good adopt candidate later. +- **State code-metrics changeset (#33442 code-read stats, #33376 code-metric fix, #33415 code-existence fix, #33400, #33254)** — entangled with Bor's diverged state package (`CommitWithUpdate`/`stateUpdate`, prefetch-attribution instrumentation, BlockSTM-V2, direct-timer metrics) and its `Reader` interface / `commitAndFlush` signature. First attempted to adopt just the `CodeReads`/`CodeLoaded` fields, but the auto-merge had silently replaced Bor's consensus-critical `CommitWithUpdate` with upstream's `CommitAndTrack` and cascaded into test-helper breaks (`Reader.Has`, `commitAndFlush` arity). Deferred the whole changeset: reverted `core/state/{statedb,state_object,reader,state_sizer,stateupdate}.go`, `core/blockchain.go`, and their tests to Bor HEAD. +- **#33157 syncModer** (from batch 4) recurred via `eth/handler.go`/`handler_test.go`; kept deferred (reverted to HEAD). + +Latent HEAD fix: + +- `core/blockchain_test.go` — Bor HEAD called `blockchain.reportBadBlock(...)` but Bor's `blockchain.go` defines `reportBlock` (upstream's rename was declined in batch 4; the test wasn't updated and batch 4's verification didn't vet `core/` test files). Fixed the 2 call sites to `reportBlock`. This makes the `core` test package compile again. + +Verification (per-batch tier): `go build ./...` pass; `go vet` clean except a pre-existing `//nolint:govet`-suppressed copylocks in `core/parallel_state_processor.go` (Bor BlockSTM-V1 rerun path; golangci-lint honors the nolint, raw `go vet` doesn't); `go test` pass on `core/txpool/blobpool`, `core/txpool`, `core/txpool/legacypool`, `ethclient/gethclient`, `triedb/pathdb`, `eth/tracers/native`; `core` test binary compiles. gofmt clean; `misc/` not staged. + +### batch 6/12 (boundary `710008450`, 20 commits) — snap-sync locking, getBlobsV3, pathdb history indexing, go-verkle removal + +Large batch — ~40 conflicts (content + many modify/deletes). Dominated by three cross-cutting upstream removals colliding with Bor divergences: go-verkle/go-ipa removal (#33461), deprecated vuln-check command removal (#33498), and the recurring syncmode/downloader refactors. No hardfork/EIP introduced (see `fork-register.md`). + +Adopted (Bor no divergence / clean): + +- **#33498 remove deprecated `version-check` command** — Bor's copy was inherited (version_check.go diff vs base was cosmetic; upstream also deleted the whole `vcheck` testdata dir, so keeping Bor's test would break). Removed `cmd/geth/version_check.go`, `version_check_test.go`, `testdata/vcheck/vulnerabilities.json` (+ auto-deleted testdata), took theirs on `misccmd.go` (drop `versionCheckCommand`), and dropped the `versionCheckCommand` registration in `main.go` (kept `verkleCommand`). +- **#33474 blobpool slotter** — Bor deliberately disabled the EIP-7594 slotter migration (`// TODO enable and fix?`, commented out; Osaka dormant); took ours and removed the now-unused `eip4844` import. +- `SECURITY.md` — took ours (Bor's Polygon security-contact doc). +- Auto-merged clean & kept: #33404 getBlobsV3 (+ new `eth/catalyst/metrics.go`), #33196, #33322, #33440, #33303 (pathdb history indexing), #33044, #33505, #33225, #33444, #33541 (in reverted files), #33483, #33395's non-Bor parts. + +Preserved Bor divergence — **kept Bor's Verkle (declined #33461)**: + +- Upstream #33461 removes all go-verkle/go-ipa references (go-ethereum abandoned the go-verkle-based verkle implementation). Bor **deferred** Verkle (user-directed; `VerkleBlock=nil`, dormant) and its `core/types/block.go` `ExecutionWitness` (`verkle.StateDiff`/`VerkleProof`), `core/state` trie readers (`newTrieReader` `PointCache`), and trie scaffolding still use go-verkle. Removing Bor's dormant verkle is a deliberate deferred-feature decision, not a merge call — so reverted all 15 #33461-touched Bor files to HEAD (`core/types/block.go`, `core/state/{statedb,statedb_hooked,database,access_events,reader,state_object,database_history,access_events_test}.go`, `core/vm/{evm,interface}.go`, `consensus/beacon/consensus.go`, `beacon/engine/{types,gen_ed}.go`, `trie/bintrie/{trie,key_encoding}.go`), kept the verkle files (`trie/verkle*.go`, `trie/utils/verkle*.go`, `cmd/geth/verkle.go`), kept go-verkle/go-ipa in `go.mod` (took ours + `go mod tidy`), and kept `verkleCommand` in `main.go`. **Flagged for the team** (`fork-register.md` + `needs-wiring.md`): keeping Bor's dormant go-verkle now costs a full #33461-file revert every batch and the impl is upstream-orphaned; the team should decide whether to drop Bor's verkle. + +Deferred (Bor-diverged / entangled → `needs-wiring.md`): + +- **#33157 syncModer** (recurring) — reverted `eth/handler.go` to HEAD. +- **State code-metrics** (recurring #33442/#33376/#32812) — auto-merged as ours (Bor's no-metrics version). +- **#33486 finalized-block cleanup on rewind** — initially took theirs, but that **collaterally dropped Bor's own `SnapSyncCommitHead`** (adjacent Bor method required by `bor_downloader.go`'s interface) — reverted `core/blockchain.go` to HEAD. Correctness fix; adopt later into Bor's forked setHeadBeyondRoot. +- **#33428 snap-sync lock protection** — touches Bor's forked `blockchain.go`/`skeleton.go`/`beaconsync.go`; reverted those to HEAD. Concurrency-safety fix; adopt later. +- **#33481 stale beacon header deletion** — Bor's forked `skeleton.go`/`beaconsync.go`; reverted to HEAD. Correctness fix; adopt later. +- **#33395 ethstats newPayload processing time** — needs a `newPayloadFeed` in Bor's forked `blockchain.go`; reverted `ethstats.go`, `core/events.go`, `blockchain_reader.go`, `api_backend.go` to HEAD and surgically removed the auto-merged `SendNewPayloadEvent` emit + `start`/`processingTime` from `eth/catalyst/api.go`. + +Deferred-recurrence modify/deletes (kept Bor's deletion): `.github/workflows/validate_pr.yml`, `core/blockchain_stats.go` (#32812), `eth/downloader/downloader.go` (Bor's bor_downloader rename), `eth/downloader/syncmode.go` (#33157). + +Tests: took ours — `core/blockchain_test.go` (Bor's `TestStatelessModeRewind`), `eth/catalyst/api_test.go` (Bor's skipped `TestGetBlobsV2`). + +Verification (per-batch tier): `go build ./...` pass; `go vet ./core/... ./eth/... ./cmd/geth/... ./trie/... ./consensus/... ./ethstats/...` clean except two pre-existing `//nolint:govet` copylocks (`core/parallel_state_processor.go`, `trie/secure_trie.go` — Bor concurrency divergences, golangci-lint honors the nolint); `go test` pass on `core/txpool/blobpool`, `trie`, `trie/utils`; `core`/`eth/catalyst`/`cmd/geth`/`eth/downloader` test binaries compile. `go mod tidy` clean (go-verkle retained). gofmt clean; `misc/` not staged. + +### follow-up commit (on top of batch 6/12 `85a1750c5`) — remove dormant go-verkle scaffolding (mirror upstream #33461) + +User-approved 2026-07-23. go-ethereum abandoned the go-verkle/go-ipa verkle implementation (#33461); Bor never enabled Verkle (`VerkleBlock=nil`). Instead of re-reverting #33461 every batch (orphaned, unmaintained), Bor **mirrors upstream's removal patch-by-patch** — this commit mirrors #33461; future upstream verkle stub PRs get mirrored as they arrive in later batches. + +Mirrored #33461 onto Bor's diverged files (preserving BlockSTM / `CommitWithUpdate` / prefetch / stateless): + +- **Deleted:** `trie/verkle.go`, `trie/verkle_test.go`, `trie/utils/verkle.go` (incl. `PointCache`), `cmd/geth/verkle.go`. +- **`core/types/block.go`** — removed the verkle `ExecutionWitness` type (`StateDiff`/`VerkleProof`), the `witness` field, `WithWitness`, `ExecutionWitness()`, and the go-verkle import. (Bor's *stateless* client uses `stateless.Witness`, untouched.) +- **`core/vm/interface.go`** — removed `PointCache()` from the `StateDB` interface + `trie/utils` import. **`core/vm/evm.go`** — `NewAccessEvents(evm.StateDB.PointCache())` → `NewAccessEvents()`. +- **`core/state`** — removed `PointCache` field/method/interface-member and `trie/utils` imports from `database.go`, `database_history.go`, `reader.go` (`newTrieReader` drops the `cache` param), `statedb.go`, `statedb_hooked.go`, and Bor's `parallel_statedb.go`; removed the `trie.VerkleTrie` `mustCopyTrie`/`deepCopy` cases (→ `bintrie.BinaryTrie`). Took theirs for `access_events.go`/`access_events_test.go` (key derivation switches from verkle `PointCache` to `bintrie.GetBinaryTreeKey`; no Bor divergence) and `trie/bintrie/{key_encoding,trie}.go` (binary tree kept + extended — provides the key funcs `AccessEvents` now uses). Fixed Bor's `parallel_statedb_coverage_test.go` (dropped the `PointCache()` accessor assertion). +- **`beacon/engine/{types,gen_ed}.go`** — took theirs (removed the verkle `ExecutionWitness` payload field; low Bor divergence). `consensus/beacon/consensus.go` — **no change needed** (Bor's version already has no verkle witness-attachment; its `consensus.Engine` diverges heavily, so left at HEAD). +- **`go.mod`/`go.sum` + `cmd/keeper`** — removed `go-verkle`/`go-ipa`; `go mod tidy`. **`cmd/geth/main.go`** — removed `verkleCommand` registration. +- **Kept:** the dormant Verkle fork gate (`VerkleBlock`/`IsVerkle`/`OverrideVerkle`) — upstream keeps `isVerkle` too per #33461's author note. + +Verification: `go build ./...` pass; `go vet ./core/... ./eth/... ./trie/... ./consensus/... ./beacon/... ./cmd/geth/...` clean except the two pre-existing `//nolint:govet` copylocks. `go test` pass: `core/state` (excl. pre-existing `TestPDBMethodParity` drift — see below), `trie`, `trie/bintrie`, `trie/trienode`, `core/state/snapshot`, `core/stateless`. gofmt clean; `go mod tidy` clean. + +**Pre-existing latent failures found (NOT caused by this change; confirmed via a worktree at `85a1750c5`):** +- `core/state` `TestPDBMethodParity` — `*StateDB.DumpBinTrieLeaves` has no `*ParallelStateDB` equivalent/exemption (drift introduced by an earlier merge #32445; fails at the batch-6 tip too). Fix: add `DumpBinTrieLeaves` to `pdbExemptMethods`. Tracked for milestone verification. +- `core/vm` `TestInterruptDuringExecution`/`TestAbortDuringJump` — **flaky** (timing/interrupt-based; fail ~1/3 runs, pass at batch-6 tip and on retry). +- Root cause both slipped through: per-batch tier compiled but did not *run* the full `core/state`/`core/vm` suites. Milestone tier must run them; consider adding `core/state` to the per-batch run. + +## v1.17.0 batch 7/12 (`94710f79a`, batch 8) + +Merge `94710f79a` into `ppatil-upstream-v1.17.0` (on top of the go-verkle-removal commit `b66d3f157`). 20 first-parent commits: keystore panic fix, ecies aes blocksize, KZG-proof peer-drop, v1.17.0 release-cycle version bump, rpc buffer limit, RPC tx formatter refactor, rawdb tx-unindex robustness, chainHeadFeed dedup, ethclient BlockReceipts, blobpool gaps, trienode-history compression, pathdb history-index extension, state update hook (#33490), code-reader cache stats (#33532). 15 conflicted files. + +No hardfork/EIP introduced (see `fork-register.md`). + +Converged with upstream (class 2 — Bor had no meaningful divergence / only reformat): + +- **`version/version.go`** — took theirs (`1.17.0 unstable`). Bor's `1.16.9 stable` came solely from merging upstream's v1.16.9 release commit; this file tracks the upstream base, so it advances onto the v1.17.0 line (matches batch-1 precedent). +- **`accounts/keystore/presale.go`** — took theirs (the `len(cipherText)%aes.BlockSize` panic guard, #33602). Bor's only divergence was a stray blank line (reformat). Security-positive. +- **`core/rawdb/chain_iterator.go`** — took theirs on the decode-error hunk (#33573: build a `blockTxHashes{err}` and skip missing bodies instead of `return`). Bor's `var result` is upstream's (auto-merged before the conflict), so the `result :=` short-decl side couldn't compile. Preserved Bor's ancient-pruning divergence (`adjustRangeForBor`, `AncientOffSet`, `ItemAmountInAncient`, `to <= from`) — Bor-only code the merge base never had, so it survived auto-merge untouched (verified in the working tree). +- **`internal/ethapi/api.go`** (x4 hunks) — took theirs (#33582's `flattenTxs(...)` helper). Helper auto-merged in and calls the same `NewRPCPendingTransaction`, so Bor's path is preserved transitively. Bor's only divergence was a blank line. + +Preserved Bor divergence (class 1 — kept ours): + +- **`eth/fetcher/tx_fetcher.go`** — kept Bor's richer protocol-violation log (`"direct"`, `"hashes"` fields from Bor's peer-jailing #2283) over upstream's slimmer line. Logging-only superset, no consensus impact; all referenced `txDelivery` fields exist. + +Combined both intents (class 3): + +- **`triedb/pathdb/history_index.go` + `history_index_iterator.go`** — upstream #33399 added an `indexReader.bitmapSize` field and (via #32981) relocated `newIterator` into `history_index_iterator.go` with a new `newIterator(filter)` signature, deleting the old no-arg method. Bor #1875 added `mu sync.RWMutex` to `indexReader` for thread-safe `readers`-map access. Resolution: (1) struct — kept `mu` and added `bitmapSize`; (2) `refresh()` — took upstream's unconditional `delete(r.readers, last.id)` under Bor's `r.mu.Lock/Unlock`; (3) took upstream's deletion of the old `newIterator` (relocated); (4) ported Bor's #1875 mutex onto the relocated cache access in `history_index_iterator.go`'s `newBlockIter` (`r.mu.RLock` around the read, `r.mu.Lock` around the write) — without this, deferring conflict 3 alone would have moved `indexReader.readers` access to an unprotected site and reintroduced the concurrent-map race #1875 fixed (state-security). `history_reader.go`'s #1875 mutex survived auto-merge intact (no action). `newIndexIterator` fully removed upstream (no orphan). Verified: `go test -race` on the pathdb history-index/iterator tests passes. + +Deferred (Bor-diverged / entangled -> `needs-wiring.md`): + +- **#33490 new state update hook** (`01b39c96b`) — entangled state-tracking continuation of the batch-6 code-metrics line, colliding with Bor's `CommitWithUpdate`/`stateUpdate`/BlockSTM divergence. Every one of its 14 touched files had #33490 as its sole upstream PR this batch (verified), so reverted all 14 to Bor HEAD: `core/blockchain.go`, `core/genesis.go`, `core/genesis_test.go`, `core/state/{statedb,state_sizer,stateupdate,state_object}.go`, `core/tracing/hooks.go`, `cmd/geth/chaincmd.go`, `core/chain_makers.go`, `core/headerchain_test.go`, `eth/filters/{filter_system_test,filter_test}.go`, `tests/block_test_util.go`. The `SetupGenesisBlockWithOverride` `tracer`-arg signature change had auto-merged into genesis.go/blockchain.go/chaincmd.go call sites — reverting all together keeps the signature consistent (no half-applied arg). +- **#33532 code-reader cache stats** (`9623dcbca`) — same reader-instrumentation surface Bor diverges on (batch-5 #33254). Reverted `core/state/database.go` + `core/state/reader.go` to HEAD (each #33532-only this batch). +- **#33563 duplicate `chainHeadFeed.Send` dedup** (`127d1f42b`) — trivial `sendChainHeadEvent()` extraction that auto-merged cleanly, but shares `core/blockchain.go` with #33490. blockchain.go reverted wholesale to Bor HEAD for safety (most consensus-critical file), deferring this clean cleanup with it. No behavior change; easy re-apply. + +Merge artifact fixed: + +- **`eth/fetcher/tx_fetcher_test.go`** — upstream `5b99d2bba` (KZG peer-drop) re-added `makeInvalidBlobTx` + a running `TestTransactionProtocolViolation` using #33378's `func(common.Hash, byte) error` `NewTxFetcher` signature; Bor already carries an adapted copy (same helper + a `t.Skip("bor: not relevant, ...")` test against Bor's retained `func(common.Hash) bool` signature). Auto-merge left duplicate `makeInvalidBlobTx`/`TestTransactionProtocolViolation` declarations (vet: "redeclared in this block"). Deleted upstream's duplicate copy (which wouldn't compile against the merged `NewTxFetcher`); kept Bor's skipped copy (matches the merged signature). Recorded in `needs-wiring.md` (wontfix, depends on the #33378 fetcher-signature adoption). + +Verification (per-batch tier): `go build ./...` pass (rc=0); `go vet ./core/... ./eth/... ./triedb/... ./internal/ethapi/... ./accounts/keystore/... ./version/...` clean except the two pre-existing `//nolint:govet` copylocks (`core/parallel_state_processor.go`, `trie/secure_trie.go`); unit tests pass on `accounts/keystore`, `core/rawdb`, `internal/ethapi`, `version`, `eth/fetcher` (137s), `core/state/snapshot`, and `triedb/pathdb` (fresh `-count=1 -race`). `core/state` fails only on the pre-existing `TestPDBMethodParity` (`DumpBinTrieLeaves` drift from #32445 — present at HEAD/batch-6 tip, not a batch-7 regression; milestone fix = add to `pdbExemptMethods`). No leftover conflict markers; `git diff --cached --check` clean; `misc/` not staged. + +## v1.17.0 batch 8/12 (`8fad02ac6`, batch 9) + +Merge `8fad02ac6` into `ppatil-upstream-v1.17.0`. 20 first-parent commits, 25 conflicted files across ~7 feature clusters. No hardfork/EIP introduced (see `fork-register.md`). Large, entanglement-dense batch — resolved by adopting clean standalone features and deferring the ones tangled with Bor's consensus-critical divergences. + +Adopted: + +- **Trienode-history (#32621 enable, #33551 indexing, #33584 big-endian bitmap)** — files already existed in Bor HEAD (prior-batch groundwork), mostly auto-merged; 2 conflicts combined: `triedb/pathdb/buffer.go` `flush` signature (upstream's `freezers []ethdb.AncientWriter` + Bor's `nodesCache *AddressBiasedCache` — the auto-merged body already uses `syncHistory(freezers...)` and Bor's AddressBiasedCache), and `history_indexer.go` imports (added `golang.org/x/exp/maps`; deduped `errgroup` already at line 27). `BlockChainConfig` (`core/blockchain.go`) + backend init (`eth/backend.go`) combined Bor's `AddressCacheSizes`/`PreloadRateLimit` with upstream's `TrienodeHistory`. Default `TrienodeHistory: -1` (disabled/dormant). `gen_config.go` TOML marshaling for the field deferred to milestone regen (`needs-wiring.md`). `-race` pathdb tests pass. +- **Grafana Pyroscope profiling (#33623)** — `internal/debug/flags.go` + new `internal/debug/pyroscope.go` (auto-merged); `go mod tidy` added `github.com/grafana/pyroscope-go v1.4.1`. +- **legacypool per-account metric (#33646)** — combined: kept Bor's extensive txpool metrics block + added `pendingAddrsGauge`/`queuedAddrsGauge` (used by the auto-merged body). + +Declined — already ours / equivalent (`needs-wiring.md` wontfix): + +- **rpc.rangelimit (#33163)** — Bor already has `RPCBlockRangeLimit` → `filters.Config.RangeLimit` (covers eth_getLogs *and* bor_getLogs) via Bor's own `RPCGlobalRangeLimitFlag` on the same `rpc.rangelimit` CLI name. Auto-merge produced duplicate flag defs (`flags.go` 672 & 690), duplicate setters (setting the declined `cfg.RangeLimit`), and duplicate main.go registration — two flags with the same name would panic at startup. Removed upstream's duplicate flag/setter/registration + `ethconfig.RangeLimit` field + `NewRangeFilter` `rangeLimit` param; reverted `eth/filters/{api,filter,filter_system,filter_test}.go` + `graphql/graphql.go` to HEAD; kept Bor's config/flag/gen_config. + +Deferred — entangled with Bor consensus divergence (`needs-wiring.md`): + +- **core/vm write-protection + selfdestruct cluster (#33281 gas-handler write-protection, #33637 per-opcode read-only, #33450 selfdestruct cold-access gas, #32919 selfdestruct tracer hooks)** — intertwined across `gas_table.go`/`operations_acl.go`/`instructions.go`/`interface.go`. #32919 makes `StateDB.SelfDestruct` void, removes `SelfDestruct6780` from the `vm.StateDB` interface, adds `IsNewContract` — clashes with Bor's BlockSTM `*StateDB.SelfDestruct` (returns prevBalance + `MVWrite`; `SelfDestruct6780` consumes the return). All four intertwined → reverted `core/vm/{gas_table,operations_acl,instructions,interface}.go`, `core/state/{statedb,statedb_hooked,statedb_hooked_test}.go`, `core/tracing/{hooks,gen_nonce_change_reason_stringer}.go` to HEAD; removed the new `eth/tracers/internal/tracetest/selfdestruct_state_test.go` + `selfdestruct_test_contracts/*.yul`. Bor's existing write-protection + gas accounting (EIP-214 unconditional; EIP-2929 cold-access) are correct and tested (`core/vm` tests pass). +- **#33644 deterministic hook emission order** — `statedb_hooked.go`; builds on the batch-8-deferred #33490 hook infra. Reverted to HEAD. +- **OpenTelemetry JSON-RPC tracing (#33452)** — new `internal/telemetry` pkg + `tracerProvider` threaded through `rpc/{client,handler,server,service}.go` (`newHandler` collides with Bor's `pool *SafePool`; `serviceRegistry.callback` → 3 returns). Reverted the 4 rpc files to HEAD; removed `internal/telemetry/telemetry.go` + `rpc/tracing_test.go`; otel deps dropped by tidy. +- **#33647 signature-length panic fix** — Bor keeps its per-fork signer chain (`pragueSigner`/`cancunSigner`) + 3-return `decodeSignature`; the fix is embedded in the `modernSigner` refactor + `tx_setcode.go`. Reverted `core/types/{transaction_signing,transaction_signing_test,tx_setcode}.go` to HEAD. Bor's `decodeSignature` still panics on bad length, but only on the signing path (crypto-generated 65-byte sigs), not the peer-facing recover path — low external exploitability. +- **#33610 fetcher test refactor** — depends on the batch-6-declined #33378 `NewTxFetcher(func(common.Hash, byte) error, …)` signature; reverted `eth/fetcher/tx_fetcher_test.go` to HEAD. + +Deps: took Bor's `go.mod`/`go.sum` + `cmd/keeper/go.{mod,sum}`; `go mod tidy` reconciled (added pyroscope, no otel). + +Verification (per-batch tier): `go build ./...` pass (rc=0); `gofmt` clean on edited files; `go vet ./core/... ./eth/... ./triedb/... ./cmd/... ./rpc/... ./internal/... ./graphql/... ./accounts/...` clean except the two pre-existing `//nolint:govet` copylocks. `go test` pass: `triedb/pathdb` (29s, incl. `-race` earlier), `core/txpool/legacypool` (18s), `eth/ethconfig`, `core/vm` (5s), `core/types`, `core/state/snapshot`, `eth/filters`. `core/state` fails only on the pre-existing `TestPDBMethodParity` (`DumpBinTrieLeaves`/#32445; present at HEAD; milestone fix). No leftover conflict markers; `git diff --cached --check` clean; `misc/` not staged. + +## v1.17.0 batch 9/12 (`845009f68`, batch 10) + +Merge `845009f68` into `ppatil-upstream-v1.17.0`. 20 first-parent commits, 17 conflicted paths (16 `UU` + 1 `DU`). Fork surface: EIP-8024 immediate-byte update (#33614) — dormant, no gate touched (see `fork-register.md`). Two real features (eth_getProofs-for-history, callTracer log index) mostly auto-merged; the recurring code-stats / slow-block deferred lines resurfaced as base-vs-HEAD conflicts. + +Adopted: + +- **eth_getProofs for history (#32727)** — mostly auto-merged (`core/state/database_history.go` +199; `triedb/pathdb/{history_reader.go +201, reader.go +87, history_index_iterator.go +45, history_trienode.go, nodes.go, disklayer.go, metrics.go}`). Conflicts: `triedb/pathdb/history_indexer.go` — removed the now-**unused** `golang.org/x/exp/maps` import (#33681 dropped its usage; HEAD had it unused, exactly why upstream removed it), kept `errgroup`; `triedb/pathdb/history_reader.go` — added `sync/atomic` (auto-merged code uses it); `core/rawdb/database.go` — combined known-metadata keys (Bor's `bytecodeSyncLastBlockKey` + upstream's `headTrienodeHistoryIndexKey`). +- **NodeFullValueCheckpoint / trienode `FullValueCheckpoint` compression knob (#32727)** — consistent with batch-8's adopted trienode-history. `triedb/pathdb/config.go`: combined Bor's `defaultStateReservation`/`defaultPreloadRateLimit` consts + `StateReservation` field with upstream's `maxFullValueCheckpoint`/`defaultFullValueCheckpoint` consts + `FullValueCheckpoint` field + sanitization block (deduped against the auto-merged struct-top fields that upstream relocated). `FullValueCheckpoint` is consumed by auto-merged `disklayer.go`/`reader.go`. `NodeFullValueCheckpoint` field + CLI flag (`--history.trienode.full-value-checkpoint`) auto-merged into `ethconfig.Config`/`cmd/utils/flags.go`; added `Defaults.NodeFullValueCheckpoint = pathdb.Defaults.FullValueCheckpoint` + the `core.BlockChainConfig` literal wiring (`eth/backend.go`, `core/blockchain.go`, historical-state group). Dormant while `TrienodeHistory = -1`. TOML marshaling → milestone `gen_config` regen (`needs-wiring.md`). +- **callTracer log index (#33629)** — added `Index` to the tracer `callLog` (`eth/tracers/native/call.go`, auto-merged) **and** the test-local `callLog` type (`calltrace_test.go`, was missing it → `want` dropped index on round-trip). Regenerated all 6 `call_tracer_withLog` golden fixtures from actual Bor tracer output, **preserving Bor's synthetic `0x1010` fee-transfer logs** (the `_withLog` harness injects `BorConfig`). Fixed the inline `TestInternals` "Mem expansion in LOG0" want (LOG0 = index `0x0`, Bor fee log = `0x1`). Regen used a temporary env-gated hook (`REGEN_WITHLOG`), since removed; the hook only rewrote each fixture's `result`, preserving `genesis`/`context`/`input`/`tracerConfig` verbatim. +- **legacypool non-executable heartbeat fix (#33704)** — added the missing `pool.priced.Removed(len(olds) + len(drops))` to `demoteUnexecutables` and adopted the clarified `Lifetime` comment; kept Bor's `txConditionalsRemoved` handling. +- **crypto/ecies (#33669)** — v1.16.9 duplicate; the `IsOnCurve` invalid-curve check is already present at HEAD. Converged the stray blank-line conflict to upstream form. +- Auto-merged: alloc reductions (#33698/#33690/#33689/#33703), #33697 pebble seek-compaction, #33711/#33694 trie fixes, #33712 heap-eviction bounds check, #33686 keeper wasm (`cmd/keeper/getpayload_wasm.go`, new), #33693 ethclient timeout, #33653/#33654 legacypool gauge/counter, #33681 trienode reader. + +Deferred — entangled with Bor divergence (`needs-wiring.md`): + +- **#33659 extend the code reader statistics** — continuation of the batch-5 (#33254) / batch-6 (#33442 line) code-read-metrics deferrals. Clashes with Bor's `PrefetchStats` / pipelined-SRC reader instrumentation + direct-`*Timer.Update()` stats. Reverted `core/state/{reader,state_object,statedb}.go` to HEAD; kept `core/blockchain_stats.go` deleted (Bor removed it earlier — `DU` modify/delete, kept deleted); did not add `codeReadTimer`/`codeReadBytesTimer`. `core/blockchain.go` conflicts (metrics timers, `BlockChainConfig` field, `processBlock` signature, per-block stats block) all resolved to Bor HEAD, preserving the adopted `FullValueCheckpoint` wiring. +- **#33655 standardize slow-block JSON output** — the slow-block feature was deferred wholesale in batch 5 (#32812; Bor's tuple-returning `ProcessBlock` lacks upstream's `stats` struct). This batch's format refinement kept deferred: `cmd/utils/flags.go`, `core/blockchain.go`, `eth/ethconfig/{config,gen_config}.go` resolved to HEAD (no `SlowBlockThreshold`/`LogSlowBlockFlag`). + +Verification (per-batch tier): `go build ./...` pass (rc=0); `gofmt` clean on edited files; `go vet ./core/... ./eth/... ./triedb/... ./cmd/... ./internal/... ./crypto/...` clean except the pre-existing `//nolint:govet` copylocks (`core/parallel_state_processor.go:341`). `go test` pass: `triedb/pathdb` (54s), `core/txpool/legacypool` (20s), `eth/ethconfig`, `eth/tracers/...` (all incl. `tracetest` `_withLog`), `crypto/ecies`, `core/rawdb` (57s), `core/state/snapshot`, `core/vm` (EIP-8024 + continuity + jump-table). `core/state` fails only on the pre-existing `TestPDBMethodParity` (`DumpBinTrieLeaves`/#32445; milestone fix). `core/vm` `TestInterruptDuringExecution`/`TestAbortDuringJump` flaky (2/3 pass) — `dispatch_test.go` unchanged by this batch, `time.Sleep(5ms)`-based interrupt race, pre-existing. No leftover conflict markers; `git diff --cached --check` clean; `misc/` not staged. + +## v1.17.0 batch 10/12 (`c12959dc8`, batch 11) + +Merge `c12959dc8` into `ppatil-upstream-v1.17.0`. 20 first-parent commits, 22 conflicted paths (all `UU`). **No new fork/EIP surface**; no fork gating / precompile / gas-schedule change (see `fork-register.md`). The dominant theme is the `crypto/keccak` vendoring (#33323) rippling across every Bor-touched hasher file, plus two batch-wide upstream type migrations (`vm.TxContext.GasPrice` and `stateObject.addrHash`) that surfaced in Bor-only code the merge couldn't auto-fix. + +Adopted (converged to upstream, Bor divergences preserved): + +- **keccak vendoring (#33323)** — mechanical `golang.org/x/crypto/sha3` → `github.com/ethereum/go-ethereum/crypto/keccak` rename. 12 conflicted files resolved to Bor's side + `sha3.`→`keccak.` + `goimports` (preserves Bor's `-local` grouping): `common/types.go`, `internal/blocktest/test_hash.go`, `consensus/ethash/consensus.go`, `cmd/evm/internal/t8ntool/execution.go` (`rlpHash`), `core/rawdb/accessors_chain_test.go`, `core/rlp_test.go`, `core/state_processor_test.go`, `core/state/snapshot/generate_test.go`, `eth/protocols/snap/sync_test.go`, `p2p/rlpx/rlpx.go`, `tests/state_test_util.go`, `trie/trie_test.go`. Ripple (auto-merged call sites left dangling `sha3` imports) fixed via `goimports`: `accounts/accounts.go`, `p2p/enode/idscheme.go`, `p2p/dnsdisc/tree.go`, `consensus/clique/clique.go`. Bor's own `consensus/bor/*` + `tests/bor/*` keep `x/crypto/sha3` (not a conflict; Bor-authored, dep retained). +- **metrics registry API (#33699/#33748/#33749)** — upstream moved `getOrRegister(name, ctor, r)` → `r.GetOrRegister(name, func() any {…}).(T)`. `metrics/histogram.go` (2 hunks) + `metrics/runtimehistogram.go` resolved to upstream: upstream's `if r == nil { r = DefaultRegistry }` subsumes Bor's Yoda `nil == r` nil-guard (semantically identical); Bor's divergence was cosmetic (blank lines / Yoda style). +- **rlp `RawList` / iterator offset (#33755)** — `rlp/iterator.go` (`listIterator`→`Iterator` + `offset` field) and `rlp/iterator_test.go` (additive `Offset()` assertion) resolved to upstream; Bor divergence was blank-line-only. +- **`vm.TxContext.GasPrice` `*big.Int`→`*uint256.Int` migration** — struct field + main `NewEVMTxContext` bridge (`uint256.MustFromBig`) auto-merged. Adapted Bor-only sites the merge couldn't reach: `core/evm.go` `NewEVMTxContextForStateSync` (`uint256.NewInt(0)`); `core/vm/dispatch_test.go` (×2) + `core/vm/dispatch_bench_test.go` (Bor EVM-switch-dispatch tests). `opGasprice` (`core/vm/instructions.go`) → upstream `scope.Stack.push(evm.GasPrice.Clone())` (Bor divergence was a stray blank line). Non-`TxContext` `GasPrice` fields (Miner/Message/CallMsg/args) correctly left `*big.Int`. +- **`stateObject.addrHash` field→method (`addrHash() common.Hash`, lazy-hash alloc reduction)** — upstream call sites auto-merged; adapted Bor's pipelined-SRC prefetch loop in `core/state/statedb.go` (`obj.addrHash` → `obj.addrHash()`). +- **eip4844 blob config API (`latestBlobConfig` `*BlobConfig`→`(BlobConfig, error)`)** — the function body + 5 of 7 callers auto-merged to upstream's error form, so keeping Bor's pointer signature would have broken them. Resolved the 2 conflict hunks to upstream's return type while preserving Bor's divergences: signature keeps Bor's `_ uint64` (block-based, no timestamp); body keeps Bor's single-arg `IsOsaka/IsPrague/IsCancun(london)` gating + no BPO forks (auto-merged correctly); `CalcExcessBlobGas` keeps Bor's **`return 0` on no-config** (consensus-safe: Bor can call this before blob activation on its block schedule) instead of upstream's `panic`, adapted to `if err != nil`. +- **freezer table close (#33776)** — `core/rawdb/ancient_utils.go` took upstream's `defer table.Close()`. +- Auto-merged (no conflict): metrics alloc/GetAll cases across `counter*/gauge*/meter/timer/registry`, `rlp/raw.go` `RawList`, `signer/core/apitypes` cell proofs (#32910) + 128KB-blob copy avoidance (#33717), Ledger Gen5 (#33297), eth_getTransactionByHash timestamp (#33709), eth_simulateV1 revert code (#33007), bintrie witness fix (#33739), miner block-building alloc (#33375), pathdb encode/decode preallocation (#33736/#33715). + +Kept Bor divergence (declined upstream change): + +- **`params.Rules.ChainID` retained** — upstream removed the `ChainID` field from `Rules` (struct-def deletion auto-merged; nothing in Bor reads `rules.ChainID`, build-confirmed). Restored it because Bor's `TestReinforceMultiClientPreCompilesTest` guard (`core/vm/contracts_test.go`) reflects over `Rules` fields and lists `ChainID` as expected — the guard exists specifically to force review of Bor↔Erigon precompile/Rules parity, so silently editing its expected list during a merge is exactly what it forbids. Restored the field + `chainID` local + `ChainID: new(big.Int).Set(chainID)` initializer; kept Bor's block-based `Rules()` (`_ uint64` param, single-arg `Is(num)`, Bor fork bools, no `IsAmsterdam`). +- **`opKeccak256` `Keccak256Cache` (`core/vm/instructions.go`)** — upstream retired the reused-`evm.hasher` pattern for `hash := crypto.Keccak256Hash(data)` and deleted the `hasher`/`hasherBuf` EVM struct fields (deletion auto-merged). Kept Bor's actual optimization (the 64-byte Solidity-slot result cache) and adapted it to upstream's shape: cache stores/loads `common.Hash`, miss path uses `crypto.Keccak256Hash`, preimage recording preserved via the shared tail. `evm.go` kept Bor's `jumpDests: jd` and dropped the now-removed `hasher:` init. + +Deferred/declined — entangled with Bor divergence (`needs-wiring.md`): + +- **legacypool alloc reduction (#33701)** — upstream reworked `addTxsLocked(txs, errs []error)` (caller-provided errs slice, in-place error set, skip pre-errored). Entangled with Bor's async txpool divergence (`addTxs(txs, async bool)` + `pool.add(tx, async)` + nilSlot merge in `Add`); the refactor's non-conflicting parts auto-merged and clashed with Bor's async signature. #33701 was the sole batch-10 commit touching `legacypool.go`, so restored the file wholesale to HEAD (declined). Micro-opt (one slice alloc/batch); revisit harmonizing into Bor's async path. + +Verification (per-batch tier): `go build ./...` pass (rc=0); `gofmt` clean on all edited files. `go vet` on the touched set clean except the pre-existing `//nolint:govet` copylocks (`core/parallel_state_processor.go:341`; `atomic.Int64` fields predate this batch; raw `go vet` doesn't honor the nolint). `go test` pass: `metrics`, `rlp`, `common`, `params`, `consensus/misc/eip4844`, `consensus/ethash`, `p2p/rlpx`, `p2p/dnsdisc`, `p2p/enode`, `consensus/clique`, `core/rawdb`, `trie`, `core/state/snapshot`, `eth/protocols/snap`, `eth/tracers/...`, `accounts`, `core/vm` (incl. `Keccak256Cache` preimage test + precompile guard, minus the flaky). Pre-existing failures (all reproduced at pre-merge HEAD `0f76e0c30`, none batch-introduced): `core/state TestPDBMethodParity` (`DumpBinTrieLeaves`/#32445; milestone fix); `core/vm TestAbortDuringJump`/`TestInterruptDuringExecution` (5ms-sleep interrupt race, 4/5 pass); `cmd/evm TestT8n`/`TestEvmRun`/`TestEVMTracing`/`TestEvmRunRegEx` (t8n eip-validation/golden drift — `invalid eip number 1346` — confirmed identical failure at HEAD via throwaway worktree). No leftover conflict markers; `misc/` not staged. + +## v1.17.0 batch 11/12 (`c50e5edfa`, plan row 12) — merged `2c0608fc6` + +20 first-parent commits, 17 conflicts (15 `UU`, 2 `DU`). No new fork/EIP surface +(EIP-8024 touch test-only, #33787). Theme: EraE format + rlp iterators + HTTP/2 +JSON-RPC + OpenTelemetry CLI wiring — the two large features both declined/deferred. + +Adopted (correctness fixes): + +- **trie/node.go (#33803)** — `decodeNodeUnsafe` now surfaces the `rlp.CountValues` + error (`invalid node list`) instead of discarding it. Bor divergence was a + cosmetic blank line; took upstream (lower `case c == 17:`/`default:` already + merged to upstream's form). RLP robustness on trie decode. +- **core/rawdb/freezer_resettable.go (#33798)** — `cleanup()` moves the dir close + to `defer dir.Close()` after `os.Open`, closing the fd on the `Readdirnames` + error path. Bor divergence cosmetic; took upstream. +- **cmd/evm/internal/t8ntool/transaction.go + core/rawdb/accessors_indexes.go + (#33820)** — relocated the RLP-iterator `Err()` check from inside the loop to + after it (only meaningful once iteration terminates). Uses Bor's unchanged + iterator API; `accessors_indexes.go` auto-merged the same relocation. +- **rlp/iterator_test.go (#33841)** — added the `txit.Count()==2` assertion for + the re-added `Iterator.Count`; Bor divergence was a stray blank line. +- **eth/backend.go (#33810)** — nil-guard on `chainView` in `updateFilterMapsHeads` + (filtermaps/log-index, non-consensus). + +Adopted (auto-merged, no Bor divergence): ethclient/gethclient callTracer methods +(#31510, added the missing `context` import the auto-merge left out); node HTTP/2 +JSON-RPC (#33812); triedb/pathdb nodeLoc-by-value (#33819, didn't touch Bor's +`AddressBiasedCache`/history-mutex); trie/bintrie storage-slot key mapping (#33807, +dormant); rlp `RawList` `AppendRaw`/validate+cache (#33834/#33840/#33818); influxdb +metrics error-message fix (#33804, kept — telemetry co-located in the same file +stripped); eth/tracers bad-block tracing tests (#33821); core/tracing gen stringers; +ethdb/pebble comment (#33805). + +Declined — continuation of the batch-9 OTel deferral (`needs-wiring.md`): + +- **OpenTelemetry JSON-RPC CLI wiring (#33484) + spanEnd-by-pointer fix (#33772)** + — re-adds the `internal/telemetry` package (absent at HEAD since #33452 was + deferred in batch 9), the `RPCTelemetry*` flags, `node.Config.OpenTelemetry`, + `tracesetup.SetupTelemetry`, and threads the tracer through `rpc/handler.go`. + Declined: reverted `rpc/handler.go`, `node/config.go`, `cmd/geth/main.go`, + `cmd/utils/flags.go` to HEAD; removed `internal/telemetry/telemetry.go`, + `internal/telemetry/tracesetup/setup.go`, `rpc/tracing_test.go`; stripped the + telemetry block from `cmd/geth/config.go`. Bor's own pre-existing gRPC-server + OTel (`internal/cli/server/server.go`) is a separate feature, untouched. + +Deferred — class 3, entangled with Bor divergence (`needs-wiring.md`): + +- **EraE archive-format rewrite (#32157/#33827)** — 15-file `internal/era` + restructure (onedb/execdb split, `proof.go`, `SlimReceipt`, `--era.format` flag, + `era.Era`/`Builder`/`ReadAtSeekCloser` interface). Blocked by two Bor divergences: + upstream's `ExportHistory` assumes TD is no longer DB-stored (Bor still reads + `bc.GetTd`), and Bor's `ImportHistory(chain, db, dir, network)` adds a `db` param + + explicit `InsertHeaderChain` upstream lacks. Reverted the whole footprint to + HEAD; `git rm` of new `onedb`/`execdb`/`proof.go`; `core/types/receipt.go`+test + reverted (the orphaned `SlimReceipt`). + +Verification (per-batch tier): `go build ./...` pass (rc=0); `gofmt` clean; +`go mod tidy` no changes (go.mod/go.sum revert to HEAD; declined features add no +deps). `go vet` clean except the pre-existing `//nolint:govet` copylocks +(`trie/secure_trie.go:88`, `core/parallel_state_processor.go:341`; both at HEAD). +`go test` pass: `trie`, `trie/bintrie`, `core/rawdb`, `rlp`, `triedb/pathdb`, +`ethclient/gethclient`, `node`, `core/types`, `core/tracing`, `ethdb/pebble`, +`core/` (183s), `tests/bor` (51s). Pre-existing failures (unchanged since HEAD +`f4f2c07fd`, none batch-introduced): `core/state TestPDBMethodParity` +(`DumpBinTrieLeaves`/#32445; milestone fix); `core/vm TestAbortDuringJump`/ +`TestInterruptDuringExecution` (5ms interrupt race; `dispatch_test.go` untouched; +both pass on isolated re-run). No leftover conflict markers; docs + `misc/` not staged. + +## v1.17.0 batch 12/12 (`0cf3d3ba4`, plan row 13) — merged `70994671a` — MILESTONE-CLOSING + +7 first-parent commits, 28 conflicts (25 `UU`, 3 `DU`). The v1.17.0 release batch. +No new fork/EIP. Two large features declined/deferred; the only net code change is a +one-line CLI fix. The merge still advances the merge base past all 7 commits. + +Adopted: + +- **internal/download/download.go (#33842)** — `DownloadFile` wraps the writer in the + progress-bar `downloadWriter` only when `resp.ContentLength > 0`. Clean auto-merge; + isolated CLI tooling. The only net code change in this batch. +- **secp256k1/curve.go (`9b78f45e3`, v1.16.9 dup)** — Bor already carries the + coordinate-check fix (applied in the v1.16.9 milestone `a06dbd7d2`); conflict was a + comment-only divergence. Took ours. +- **consensus/misc/eip1559.go (#33860)** — the defensive parent-baseFee nil-check in + `VerifyEIP1559Header` is already present in Bor HEAD (`eip1559.go:57`); merged as a + no-op (converged). Consensus-security hardening, so verified present, not dropped. + +Declined — OpenTelemetry newPayload tracing (#33521) + telemetry spans (#33780), +continuation of the batch-9 (#33452) / batch-11 (#33484) OTel line: + +- #33521 threads `ctx context.Context` through the whole block-processing chain + (`Processor.Process`, `BlockChain.{insertChain,ProcessBlock,insertSideChain}`, + `ExecuteStateless`, all call sites) solely to carry `telemetry.StartSpan` server + spans for `engine_newPayload`. Entangled with Bor's diverged signatures: Bor's + `Process(block, statedb, cfg, author, interruptCtx)` (BlockSTM interrupt ctx + author), + 6-return `ProcessBlock(block, header, …)`, 5-return `ExecuteStateless(…, author, + consensus, diskdb)`. Reverted to HEAD: `core/types.go`, `core/state_processor.go` + (kept Bor's Prague `Bor == nil` gating vs upstream's `postExecution` wrap), + `core/blockchain.go` (kept Bor's `reportBlock` naming), `core/stateless.go`, + `eth/state_accessor.go`, `eth/api_debug.go`, `eth/catalyst/{api,simulated_beacon, + api_test}.go`, `cmd/keeper/main.go`, `core/{blockchain,block_validator}_test.go`; + removed `internal/telemetry/{telemetry,telemetry_test}.go`. `version/version.go` + reverted to HEAD (Bor keeps its own version, not upstream's `Meta = "stable"`). + +Deferred (class 3) — delayed p2p message decoding (#33835): + +- 26-file eth/snap protocol refactor: `BlockBodiesResponse`/`GetTrieNodesPacket` + fields → `rlp.RawList[...]` (decode-on-demand DoS hardening), `p2p/tracker` API → + typed `Track(req Request) error`/`Fulfil(resp Response) error`, all response + handlers reworked. Entangled with Bor's `ExcludeStateSyncReceipt()` receipt-interface + divergence (state-sync consensus), Bor's `newBlockPushIntervalTimer`/ + `lastNewBlockPushUnix` NewBlock push metrics, the snap `requestTracker` singleton, + Bor's forked downloader + wit protocol. Security-positive; good future-adopt. Reverted + the full footprint to HEAD (`eth/protocols/eth/*`, `eth/protocols/snap/*` incl. + restoring `snap/tracker.go` upstream deleted, `eth/downloader/{queue,queue_test, + bor_fetchers_concurrent_bodies}.go`, `eth/handler_eth{,_test}.go`, `p2p/tracker/tracker.go`, + `cmd/devp2p/internal/ethtest/*`); `git rm` of `eth/downloader/downloader_test.go` (DU), + `eth/catalyst/witness.go` (DU), new `p2p/tracker/tracker_test.go`. + +Verification (per-batch tier): `go build ./...` pass (rc=0); `gofmt` clean; +`go mod tidy` no changes (root go.mod/go.sum unchanged vs HEAD; declined features add +no deps). `go vet` clean except pre-existing `//nolint:govet` copylocks +(`trie/secure_trie.go:88`, `core/parallel_state_processor.go:341`). `go test` pass: +`eth/protocols/eth`, `eth/protocols/snap`, `eth/downloader` (+whitelist), `eth/catalyst`, +`internal/download`, `core/` (195s), `tests/bor`. Pre-existing failures unchanged since +HEAD `2c0608fc6` (only net code change is `internal/download`): `core/state +TestPDBMethodParity`, `core/vm TestAbortDuringJump`/`TestInterruptDuringExecution` +(5ms flake). No leftover conflict markers; docs + `misc/` not staged. diff --git a/docs/upstream-merges/v1.17.4/needs-wiring.md b/docs/upstream-merges/v1.17.4/needs-wiring.md new file mode 100644 index 0000000000..0c2349d066 --- /dev/null +++ b/docs/upstream-merges/v1.17.4/needs-wiring.md @@ -0,0 +1,56 @@ +# Needs-wiring / deferred backlog: go-ethereum v1.17.4 sync + +Durable backlog of upstream changes intentionally **not adopted** during the +sync — features/tests that merged clean (or were deferred at a conflict) and +kept out of Bor to preserve behavior, but which the PoS team could pick up +later. Populated per batch (invariant: nothing dropped silently). Fork/EIP +enable-decisions live in `fork-register.md`; this file is for the *wiring work* +a future adoption would require. + +`status` ∈ {`deferred`, `coverage-gap`, `adopted`, `wontfix`}. + +| Item | source (batch / boundary) | status | why deferred | to adopt (wiring required) | pointers | +| ---- | ------------------------- | ------ | ------------ | -------------------------- | -------- | +| `--miner.maxblobs` (configurable per-block blob cap) | batch 3 (v1.17.0 3/12, `a122dbe45`) | deferred | Bor's miner is heavily diverged (`worker` struct with `w.chainConfig`, restructured `commitTransaction`, removed the isCancun blob-space-left block); it's an operator knob, not consensus. Wiring it needs adaptation to Bor's worker structure + flag→config plumbing — out of scope for a merge batch. Bor keeps the protocol-default cap (`eip4844.MaxBlobsPerBlock`), backward-compatible. | 1. add `MaxBlobsPerBlock int` to `miner.Config`; 2. add a `maxBlobsPerBlock(time)` helper reachable from Bor's `worker` (`w`), falling back to `eip4844.MaxBlobsPerBlock` when 0; 3. replace the `eip4844.MaxBlobsPerBlock(w.chainConfig, time)` call sites in `miner/worker.go`; 4. re-add `MinerMaxBlobsFlag` in `cmd/utils/flags.go`, its `setMiner` read, and the `cmd/geth/main.go` registration. | `miner/worker.go`, `miner/miner.go`, `cmd/utils/flags.go` (`setMiner`), `cmd/geth/main.go`; upstream boundary `a122dbe45` | +| `core/bintrie_witness_test.go` (binary-trie witness test) | batch 2 (v1.17.0 2/12, `cf93077fa`) | coverage-gap | Upstream's newly-added test can't compile against Bor: uses time-based `ShanghaiTime`/`VerkleTime` config fields + a `u64` helper Bor lacks, and Bor's different `InsertChain` signature. Removed to keep the tree building; Bor doesn't run the verkle/binary-trie transition. | Only if Bor adopts the binary-trie transition (see Verkle in `fork-register.md`): re-add the file and adapt it to Bor's block-based config (`VerkleBlock`/`IsVerkleGenesis()`), define/borrow the `u64` helper, and fix the `InsertChain` call to Bor's signature. | removed file (recoverable from upstream `cf93077fa`); relates to Verkle row in `fork-register.md` | +| syncModer refactor — "keep current syncmode in downloader only" (#33157) | batch 5 (v1.17.0 4/12, `228933a66`) | deferred | Upstream moved sync-mode selection into the downloader via a new `syncModer` (`eth/downloader/syncmode.go`), dropped the `mode` param from `synchronise`/`BeaconSync`/`BeaconExtend`/`BeaconDevSync`, added `(*Downloader).ConfigSyncMode()`, auto-disables snap after the first cycle, and moved the `SyncMode` **type** fully into `eth/ethconfig`. Bor's downloader is a deep fork (`bor_downloader.go`, `eth/downloader/modes.go` with Bor-only `StatelessSync`/`BytecodeOnlySnapSync`, witness sync, whitelist, fast-forward) and keeps `SyncMode` in the `downloader` package. Porting would require relocating the type (breaking Bor's extra modes' package home) and reworking Bor's mode handling — out of scope and high-risk for a merge batch; sync-mode is operational, not consensus. Kept Bor's mode-taking signatures + `h.snapSync.Load()` checks + `Ethereum.SyncMode()`; reverted `handler.go`/`syncer.go`/`beaconsync.go`/`beacondevsync.go`/`sync_test.go`/`handler_test.go`/`handler_eth_test.go` to Bor HEAD; removed the added `eth/downloader/syncmode.go`. | Decide whether Bor adopts syncModer: (1) move `SyncMode` type to `ethconfig` while preserving Bor's `StatelessSync`/`BytecodeOnlySnapSync`; (2) introduce `syncModer` adapted to `bor_downloader.go`'s `synchronise(id,hash,td,ttd,mode,beaconMode,ping)`; (3) add `ConfigSyncMode()` and re-slim `BeaconSync`/`BeaconExtend`/`BeaconDevSync`; (4) restore the auto-disable-snap-after-first-cycle behavior (currently in Bor's `enableSyncedFeatures`). | upstream boundary `228933a66`; `eth/downloader/{bor_downloader.go,modes.go,beaconsync.go,beacondevsync.go}`, `eth/{handler.go,backend.go,syncer/syncer.go,catalyst/api.go}` | +| Slow-block detailed statistics logging (#32812) | batch 5 (v1.17.0 4/12, `228933a66`) | deferred | Upstream added `--debug.logslowblock` + `ethconfig.SlowBlockThreshold` + `core.BlockChainConfig.SlowBlockThreshold` + a `core/blockchain_stats.go` (`blockProcessingStats.logSlow`/`reportMetrics`) and calls `res.stats.logSlow(block, bc.slowBlockThreshold)` in `insertChain`. The logging is entangled with upstream's `blockProcessingResult.stats` structure, which Bor's heavily-diverged `ProcessBlock` (tuple-returning, BlockSTM-V2, state-sync feed, milestone validation) does not share. Config plumbing is trivial but the actual logging can't be grafted mechanically. Deferred the whole feature: removed `core/blockchain_stats.go`, the `--debug.logslowblock` flag (`cmd/utils/flags.go` + `cmd/geth/{main,chaincmd}.go` registrations), and reverted `ethconfig/{config,gen_config}.go`, `eth/backend.go`, `core/blockchain.go` to Bor HEAD. | Adapt slow-block logging to Bor's `insertChain`: (1) re-add `ethconfig.SlowBlockThreshold` (+ default `2s`) and the `--debug.logslowblock` flag; (2) thread it into `core.BlockChainConfig` + `BlockChain.slowBlockThreshold` + backend wiring; (3) port `logSlow`/`reportMetrics` onto Bor's per-block stats (Bor uses direct `*Timer.Update(...)` calls, not a `stats` struct) or add a minimal Bor-native slow-block log gated on the threshold. Bor operators want this (see slow-block incidents), so a good adopt candidate. | upstream boundary `228933a66`; commit `042c47ce1`; `core/blockchain.go` (`insertChain`, `ProcessBlock`), `core/blockchain_stats.go` (upstream), `eth/backend.go`, `eth/ethconfig/*`, `cmd/utils/flags.go`, `cmd/geth/{main,chaincmd}.go` | +| Reader cache-stats rename / export (#33254) | batch 5 (v1.17.0 4/12, `228933a66`) | deferred | Upstream renamed `ReaderStats` fields `AccountHit`→`AccountCacheHit` (etc.), renamed the `readerWithCacheStats` internals, and added a `ReaderStats.String()` + metric export. Bor's `core/state/reader.go` carries a richer, deliberately-diverged prefetch-attribution instrumentation (role-based hit/miss, prefetch-origin attribution, unique-insert counters — the pipelined-SRC work) that uses the old names and its own fields, and `core/blockchain.go` reads them via `GetStats()` with the old names. Adopting the rename would churn Bor's instrumentation for no functional gain. Reverted `core/state/reader.go` + `core/state/statedb.go` to Bor HEAD. | If converging Bor's reader instrumentation to upstream: rename Bor's fields to the `*CacheHit`/`*CacheMiss` scheme, reconcile with Bor's prefetch-attribution counters, add the `String()`/export, and update `GetStats()` call sites in `core/blockchain.go`. Low priority — Bor's version is a superset. | upstream boundary `228933a66`; commit(s) in `core/state/reader.go`, `core/state/statedb.go`; consumers in `core/blockchain.go` | +| tx-announcement metadata validation (#33378) | batch 6 (v1.17.0 5/12, `5dfcffcf3`) | deferred | Upstream replaced the fetcher's `hasTx func(common.Hash) bool` callback with `validateMeta func(common.Hash, byte) error` (validates the announced tx type/metadata against the local txpool and drops invalid announcements), converting the `NewTxFetcher`/`NewTxFetcherForTests` signature and every call site. Bor's fetcher is diverged (`txMetadata` type vs upstream `txMetadataWithSeq`; `hasTx`-based; 34 old-signature call sites in `tx_fetcher_test.go`). Adopting means rewriting the production caller (`eth/handler.go`) + 34 test sites + the fuzzer, with subtle per-test semantic mapping (hasTx→validateMeta). Reverted `eth/fetcher/tx_fetcher.go`, `tx_fetcher_test.go`, `eth/handler.go`, `eth/handler_test.go`, `tests/fuzzers/txfetcher/txfetcher_fuzzer.go` to Bor HEAD. Security-positive (rejects malformed announcements) — worth adopting. | 1. change the callback to `validateMeta func(common.Hash, byte) error`; 2. update `NewTxFetcher`/`NewTxFetcherForTests` + the `TxFetcher` struct (`hasTx`→`validateMeta`); 3. wire the production caller in `eth/handler.go` (build the validateMeta closure over the txpool); 4. migrate all 34 test call sites + the fuzzer to the new signature; 5. adopt the announcement-loop refactor (validate → `continue` on invalid). | upstream boundary `5dfcffcf3`; commit `56d201b0f`; `eth/fetcher/tx_fetcher.go`, `eth/handler.go`, `tests/fuzzers/txfetcher/txfetcher_fuzzer.go` | +| State code-read metrics changeset (#33442, #33376, #33415, #33400) | batch 6 (v1.17.0 5/12, `5dfcffcf3`) | deferred | Upstream added `CodeReads`/`CodeLoaded` StateDB metrics (#33442), fixed contract-code metric counting (#33376) and code-existence marking (#33415, which also added `Has` to the `state.Reader` interface), plus test-readability (#33400). Entangled with Bor's diverged state package: the auto-merge silently swapped Bor's consensus-critical `CommitWithUpdate`/`stateUpdate` for upstream's `Commit`/`CommitAndTrack`, and cascaded into test-helper breaks (`recordingReader` missing `Has`, `commitAndFlush` arity). Bor also uses direct `*Timer.Update()` metrics, not upstream's `blockProcessingResult.stats` struct. Reverted `core/state/{statedb,state_object,reader,state_sizer,stateupdate}.go`, `core/blockchain.go`, and their tests to Bor HEAD. | Adopt carefully, preserving Bor's `CommitWithUpdate` and prefetch instrumentation: add `CodeReads`/`CodeLoaded` fields + their population in `state_object.go`; add `Has` to Bor's `Reader` implementers + test helpers; wire `codeReadTimer.Update(statedb.CodeReads)` into Bor's direct-timer blocks (`insertChain`/`processBlock`) and fold `CodeReads` into `blockExecutionTimer`. Relates to the #33254 row (same instrumentation surface). | upstream boundary `5dfcffcf3`; commits `ffe9dc97e`/`9a346873b`/`15f52a293`/`3a5560fa9`; `core/state/*.go`, `core/blockchain.go` | +| `BlockTest.run` inlined into `BlockTest.Run` (#33383) | batch 6 (v1.17.0 5/12, `5dfcffcf3`) | wontfix | Upstream inlined the `run` helper into `Run` and removed the separate method. Bor deliberately keeps `Run`/`RunV2`/`run(..., useV2 bool)` for V2-BlockSTM differential testing (`TestExecutionSpecBlocktestsV2`). Kept Bor's harness (took ours). | n/a — Bor's split is intentional; do not inline. | `tests/block_test_util.go`; upstream commit `c21fe5475` | +| Drop Bor's go-verkle scaffolding (adopt #33461) | batch 7 (v1.17.0 6/12, `710008450`) | **adopted** (2026-07-23, user-approved) | go-ethereum abandoned the go-verkle/go-ipa impl (#33461); Bor never enabled Verkle. Mirrored #33461 in a dedicated commit on top of batch 6 (deleted `trie/verkle*.go`, `trie/utils/verkle.go`, `cmd/geth/verkle.go`; removed `ExecutionWitness` verkle fields, `PointCache` from `state.Database`/`vm.StateDB` interfaces + implementers incl. `ParallelStateDB`; `AccessEvents` key derivation → `bintrie`; dropped go-verkle/go-ipa deps; removed `verkleCommand`). Kept the dormant Verkle fork gate + Bor's `stateless.Witness`. | **Future work: mirror upstream's follow-up verkle stub PRs as they arrive in later batches** — #33461 is the first of several; Bor now tracks upstream's removal trajectory patch-by-patch. Watch for the promised "stubs to replace code when the keeper build is detected." | `fork-register.md` Verkle notes; upstream commit `3f641dba8` | +| Snap-sync lock protection (#33428) | batch 7 (v1.17.0 6/12, `710008450`) | deferred | Adds lock protection in snap sync across `core/blockchain.go`, `eth/downloader/skeleton.go`, `beaconsync.go` — all deeply Bor-forked. Concurrency-safety fix (p2p-security relevant). Reverted those files to HEAD. | Port the lock additions into Bor's forked snap-sync paths (`skeleton.go`, `bor_downloader.go`) and `blockchain.go`; verify against Bor's BlockSTM concurrency model. | upstream boundary `710008450`; commit `bf141fbfb` | +| Stale beacon header deletion (#33481) | batch 7 (v1.17.0 6/12, `710008450`) | deferred | Fixes stale beacon-header deletion in the skeleton syncer; Bor's `skeleton.go`/`beaconsync.go` are forked. Correctness fix. Reverted to HEAD. | Port the fix into Bor's forked skeleton syncer; add the debug logging in `findBeaconAncestor`. | upstream boundary `710008450`; commit `4531bfebe` | +| Finalized-block cleanup on rewind in setHeadBeyondRoot (#33486) | batch 7 (v1.17.0 6/12, `710008450`) | deferred | Uses the resolved head block number (not the requested `head`) for the finalized-invalidation check, and reorders the head-block set after snapshot regen. Bor's `setHeadBeyondRoot` diverges (chainmu.TryLock wrapper, adjacent `SnapSyncCommitHead`); taking theirs collaterally dropped `SnapSyncCommitHead`. Reverted `core/blockchain.go` to HEAD. Correctness fix. | Port the `headBlock.Number` finalized check + head-set reorder into Bor's `setHeadBeyondRoot`, preserving `SnapSyncCommitHead` and the chainmu handling. | upstream boundary `710008450`; commit `32fea008d`; `core/blockchain.go` | +| ethstats newPayload processing-time reporting (#33395) | batch 7 (v1.17.0 6/12, `710008450`) | deferred | Adds a `NewPayloadEvent` + `newPayloadFeed` to `core.BlockChain` (Bor-forked), `Subscribe/SendNewPayloadEvent` methods, and ethstats consumption. Reverted `ethstats.go`/`core/events.go`/`blockchain_reader.go`/`api_backend.go` to HEAD and removed the auto-merged emit in `eth/catalyst/api.go`. Observability. | Add `newPayloadFeed` + `NewPayloadEvent` to Bor's `blockchain.go`, the pub/sub methods, emit in the newPayload/insert path, and wire ethstats. | upstream boundary `710008450`; commit `a8a480489`; `core/blockchain.go`, `ethstats/ethstats.go`, `core/events.go` | +| New `OnStateUpdate` state-commit hook (#33490) | batch 8 (v1.17.0 7/12, `94710f79a`) | deferred | Adds an `OnStateUpdate` tracing hook invoked after state commit, threading `update.ToTracingUpdate()` through `writeBlockWithState` and a `tracer` arg through `SetupGenesisBlockWithOverride`. Entangled with Bor's consensus-critical state divergence: it restructures `writeBlockWithState` around upstream's `CommitWithUpdate`/`CommitAndTrack` (Bor has its own `CommitWithUpdate` returning `*stateUpdate` + timing metrics + `WriteBytecodeSyncLastBlock` + `[]*types.Log` return), reworks `contractCode`/`stateUpdate` structs (`originHash`/`duplicate`/`originBlob`), and is the continuation of the state code-metrics line already deferred in batch 6 (#33442/#33376/#33415/#33400). Reverted all 14 touched files to Bor HEAD: `core/blockchain.go`, `core/genesis.go`, `core/genesis_test.go`, `core/state/{statedb,state_sizer,stateupdate,state_object}.go`, `core/tracing/hooks.go`, `cmd/geth/chaincmd.go`, `core/chain_makers.go`, `core/headerchain_test.go`, `eth/filters/{filter_system_test,filter_test}.go`, `tests/block_test_util.go`. | Adopt alongside the batch-6 state code-metrics row (same `stateUpdate`/`CommitWithUpdate` surface): thread `tracer` into Bor's `SetupGenesisBlockWithOverride`; add the `OnStateUpdate` hook + `ToTracingUpdate()` onto Bor's `stateUpdate`; graft the hook call into Bor's `writeBlockWithState` next to its existing `stateSizer.Notify`. Do the two together (they share `stateupdate.go`). | upstream boundary `94710f79a`; commit `01b39c96b`; relates to the batch-6 state code-metrics row | +| Contract-code reader cache statistics (#33532) | batch 8 (v1.17.0 7/12, `94710f79a`) | deferred | Adds hit/miss cache statistics to the contract-code reader (`core/state/{database,reader}.go`). Same reader-instrumentation surface as the batch-5 #33254 row, which Bor deliberately diverges on (richer prefetch-attribution instrumentation from the pipelined-SRC work). Reverted `core/state/database.go` + `core/state/reader.go` to Bor HEAD. | Fold upstream's code-reader cache stats into Bor's existing reader instrumentation (reconcile with the prefetch-attribution counters), same as the #33254 adoption. Low priority — Bor's instrumentation is a superset. | upstream boundary `94710f79a`; commit `9623dcbca`; relates to the batch-5 #33254 row | +| Duplicate `chainHeadFeed.Send` dedup (#33563) | batch 8 (v1.17.0 7/12, `94710f79a`) | deferred | Trivial cleanup extracting `sendChainHeadEvent()` from `SetHead`/`SetHeadWithTimestamp`. Auto-merged cleanly, but shares `core/blockchain.go` with the entangled #33490 change; blockchain.go was reverted wholesale to Bor HEAD for safety (single most consensus-critical file), so this clean cleanup was deferred with it. No behavior change either way. | Re-apply the `sendChainHeadEvent()` extraction in `SetHead`/`SetHeadWithTimestamp` (the auto-merge already produced a correct form). Trivial; adopt whenever blockchain.go is next touched. | upstream boundary `94710f79a`; commit `127d1f42b`; `core/blockchain.go` | +| KZG-proof peer-drop test (`makeInvalidBlobTx`/`TestTransactionProtocolViolation`, from #33602's KZG series) | batch 8 (v1.17.0 7/12, `94710f79a`) | wontfix | Upstream's `5b99d2bba` (drop peers on invalid KZG proofs) re-added `makeInvalidBlobTx` + a *running* `TestTransactionProtocolViolation` using the new `NewTxFetcher(func(common.Hash, byte) error, …)` signature (#33378). Bor already carries an adapted copy: the same helper + a `t.Skip("bor: not relevant, transaction type not supported")` test against Bor's retained `func(common.Hash) bool` signature. Auto-merge produced duplicate `makeInvalidBlobTx`/`TestTransactionProtocolViolation` decls; kept Bor's skipped copy, deleted upstream's duplicate. | n/a — the running test depends on adopting #33378's fetcher-signature change (see the batch-6 #33378 row). If #33378 is adopted, un-skip Bor's copy and drop the `func(common.Hash) bool` shim. | upstream boundary `94710f79a`; commit `5b99d2bba`; `eth/fetcher/tx_fetcher_test.go`; depends on the batch-6 #33378 row | + +| core/vm write-protection relocation (#33281 + #33637) | batch 9 (v1.17.0 8/12, `8fad02ac6`) | deferred | Upstream moved EIP-214 write-protection enforcement out of the interpreter/opcodes into the gas handlers (#33281: `gas_table.go` gasCall, `operations_acl.go` selfdestruct gas; instructions.go −12) and added per-opcode read-only checks (#33637). Intertwined in `instructions.go`/`operations_acl.go` with #32919 (selfdestruct rework, deferred) and #33450. Bor's existing write-protection works and is tested; reverted `core/vm/{gas_table,operations_acl,instructions,interface}.go` to HEAD. Behavior-neutral (EIP-214 is unconditional). | Re-apply the gas-handler write-protection relocation onto Bor's `core/vm` once #32919 is untangled (they share instructions.go/operations_acl.go). Low priority — no behavior change. | upstream boundary `8fad02ac6`; commits `23c349883`/`500931bc8` | +| Selfdestruct cold-access gas early-return (#33450) | batch 9 (v1.17.0 8/12, `8fad02ac6`) | deferred | EIP-2929 correctness: return early in the selfdestruct gas fn if gas can't cover cold account access. `operations_acl.go` (+4), intertwined with #33281/#33637 there; reverted with the write-protection cluster. Gas-accounting fix worth adopting. | Port the early-return into Bor's `makeSelfdestructGasFn` in `operations_acl.go` alongside the write-protection relocation. Verify against Bor's gas tests. | upstream boundary `8fad02ac6`; commit `b6fb79cdf` | +| Selfdestruct tracer hooks during finalisation (#32919) | batch 9 (v1.17.0 8/12, `8fad02ac6`) | deferred | Reworks `opSelfdestruct` to explicit `SubBalance`, makes `StateDB.SelfDestruct` **void**, **removes** `SelfDestruct6780` from the `vm.StateDB` interface, adds `IsNewContract`, and fires selfdestruct tracer hooks in `Finalise`. Clashes with Bor's BlockSTM `*StateDB.SelfDestruct` (returns `uint256.Int` prevBalance + `MVWrite` instrumentation; `SelfDestruct6780` consumes the return). Reverted `core/state/{statedb,statedb_hooked,statedb_hooked_test}.go`, `core/vm/{instructions,interface}.go`, `core/tracing/{hooks,gen_nonce_change_reason_stringer}.go` to HEAD; removed the new `eth/tracers/internal/tracetest/selfdestruct_state_test.go` + `selfdestruct_test_contracts/*.yul`. | Adopt with Bor's BlockSTM in mind: keep MVWrite instrumentation, decide whether to keep Bor's returning `SelfDestruct`/`SelfDestruct6780` or converge to upstream's void + `IsNewContract`; graft the finalisation tracer-hook emission. Do alongside the #33490 hook infra (batch-8 row). | upstream boundary `8fad02ac6`; commit `715bf8e81` | +| Deterministic hook emission order in Finalise (#33644) | batch 9 (v1.17.0 8/12, `8fad02ac6`) | deferred | Sorts hook emission deterministically in `Finalise` (`statedb_hooked.go` +14). Builds on the #33490 `OnStateUpdate`/hook infra deferred in batch 8. Reverted `statedb_hooked.go` to HEAD (also carried #32919). | Adopt together with #33490 + #32919 (shared `statedb_hooked.go`); ensure the ordering is deterministic under Bor's BlockSTM. | upstream boundary `8fad02ac6`; commit `2eb1ccc6c` | +| OpenTelemetry JSON-RPC tracing (#33452) | batch 9 (v1.17.0 8/12, `8fad02ac6`) | deferred | Adds OTel tracing to the JSON-RPC server: new `internal/telemetry` package, `tracerProvider` threaded through `rpc/{client,handler,server,service}.go` (`newHandler`, `serviceRegistry.callback` → 3 returns), + otel deps. `newHandler` collides with Bor's `pool *SafePool` execution-pool divergence. Observability, not consensus. Reverted the 4 rpc files to HEAD; removed `internal/telemetry/telemetry.go` + `rpc/tracing_test.go`; otel deps dropped by `go mod tidy`. | Thread `tracerProvider` through Bor's `newHandler` (alongside `pool`), re-add `internal/telemetry`, restore `serviceRegistry.callback`'s span handling, re-add otel deps. Observability nice-to-have. | upstream boundary `8fad02ac6`; commit `a9acb3ff9` | +| Signature-length panic fix (#33647) | batch 9 (v1.17.0 8/12, `8fad02ac6`) | deferred | Changes `decodeSignature` to return an `error` (instead of `panic`) on wrong signature length, threading the error through `modernSigner.SignatureValues` + `tx_setcode.go`. Bor keeps its per-fork signer chain (`pragueSigner`/`cancunSigner`) alongside `modernSigner` and a 3-return `decodeSignature`; the fix is embedded in the modernSigner refactor. Reverted `core/types/{transaction_signing,transaction_signing_test,tx_setcode}.go` to HEAD. Bor's `decodeSignature` still panics on bad length — but only on the **signing** path (`SignatureValues`, fed crypto-generated 65-byte sigs), not the peer-facing `Sender`/recover path, so low external exploitability. | Port the `decodeSignature` error-return + `SignatureValues`/`tx_setcode.go` error handling onto Bor's signer (defensive hardening). | upstream boundary `8fad02ac6`; commit `8fad02ac6` | +| Duplicate `rpc.rangelimit` flag (#33163) | batch 9 (v1.17.0 8/12, `8fad02ac6`) | wontfix | Adds an `--rpc.rangelimit` flag + `ethconfig.RangeLimit` + a `rangeLimit` param on `NewRangeFilter` to cap `eth_getLogs` block ranges. **Bor already has this feature** (`RPCBlockRangeLimit` → `filters.Config.RangeLimit`, covering eth_getLogs *and* bor_getLogs, via Bor's own `RPCGlobalRangeLimitFlag` on the same `rpc.rangelimit` CLI name). Two flags with the same name would panic at startup. Declined upstream's duplicate: removed its flag def/setter/registration/`NewRangeFilter` param + `ethconfig.RangeLimit` field; reverted `eth/filters/{api,filter,filter_system,filter_test}.go` + `graphql/graphql.go` to HEAD; kept Bor's config/flag/gen_config. | n/a — Bor's `RPCBlockRangeLimit` is the equivalent (and broader — covers bor_getLogs). If ever converging to upstream's naming, rename Bor's field/flag and drop the duplicate. | upstream boundary `8fad02ac6`; commit `3d78da917` | +| Fetcher test-code refactor (#33610) | batch 9 (v1.17.0 8/12, `8fad02ac6`) | wontfix | Refactors `tx_fetcher_test.go` to a `newTestTxFetcher` helper built on #33378's `NewTxFetcher(func(common.Hash, byte) error, …)` signature. Bor declined #33378 (batch-6 row), keeping `func(common.Hash) bool`, so the refactor can't compile against Bor's fetcher. Reverted `eth/fetcher/tx_fetcher_test.go` to HEAD. | Adopt only if #33378 is adopted (batch-6 row); then migrate to `newTestTxFetcher`. | upstream boundary `8fad02ac6`; commit `9ba13b609`; depends on the batch-6 #33378 row | +| TrienodeHistory + NodeFullValueCheckpoint TOML marshaling (gen_config) | batch 9 (v1.17.0 8/12, `8fad02ac6`); extended batch 10 (`845009f68`) | coverage-gap | Trienode-history (#32621/#33551/#33584, batch 8) and `NodeFullValueCheckpoint` (#32727, batch 10) were **adopted** (see ledger). `eth/ethconfig/gen_config.go` was resolved to Bor's HEAD both times (declining #33163's `RangeLimit` marshaling), so neither `TrienodeHistory` (default `-1`) nor `NodeFullValueCheckpoint` (default `pathdb.Defaults.FullValueCheckpoint`, also settable via `--history.trienode.full-value-checkpoint`) is yet in the generated TOML (un)marshaling. Both function at their defaults / via CLI; only TOML round-trip is missing. | Regenerate `gen_config.go` (`go generate ./eth/ethconfig/`) at the v1.17.0 milestone chores to add `TrienodeHistory` + `NodeFullValueCheckpoint` (and confirm no `RangeLimit`). | upstream boundary `8fad02ac6`/`845009f68`; `eth/ethconfig/gen_config.go` | +| Extend the code reader statistics (#33659) | batch 10 (v1.17.0 9/12, `845009f68`) | deferred | Continuation of the batch-5 (#33254) / batch-6 (#33442 line) code-read-metrics deferrals. Upstream adds `ContractCodeReaderStats` (hit/miss + `CacheHitBytes`/`CacheMissBytes`) + `HitRate()`, reworks `ReaderStats`, changes `cachingCodeReader.GetStats()` to return the struct, adds `hitBytes`/`missBytes` atomics, and `CodeLoadBytes`/`CodeUpdated`/`CodeUpdateBytes` on `StateDB`/`blockProcessingStats`. Clashes with Bor's deliberately-diverged `PrefetchStats` / pipelined-SRC reader instrumentation (`readerWithCacheStats`, `AccountHitFromPrefetch`, unique-insert counters) and Bor's direct-`*Timer.Update()` block stats (not upstream's `stats` struct). Reverted `core/state/{reader,state_object,statedb}.go` to HEAD; kept `core/blockchain_stats.go` deleted; `core/blockchain.go` code-read timers (`codeReadTimer`/`codeReadBytesTimer`) not added. | Fold upstream's code-reader byte accounting into Bor's reader instrumentation (reconcile with the prefetch-attribution counters), add `CodeLoadBytes`/`CodeUpdated`/`CodeUpdateBytes` to Bor's stats and wire `codeReadTimer`/`codeReadBytesTimer` into Bor's direct-timer blocks. Do together with the batch-5 #33254 and batch-6 #33442 rows (same surface). Low priority — Bor's instrumentation is a superset. | upstream boundary `845009f68`; commit `c2595381b`; `core/state/{reader,state_object,statedb}.go`, `core/blockchain.go`; relates to the batch-5 #33254 + batch-6 #33442 rows | +| Standardize slow-block JSON output (#33655) | batch 10 (v1.17.0 9/12, `845009f68`) | deferred | Refines the slow-block statistic logging output format for cross-client metrics (negative `SlowBlockThreshold` = disabled default, zero = log all, positive = filter). The whole slow-block feature was deferred in batch 5 (#32812) — Bor's diverged tuple-returning `ProcessBlock` (BlockSTM-V2, state-sync feed, milestone validation) doesn't carry upstream's `blockProcessingResult.stats` struct that the logging hangs off. This format refinement kept deferred with it: reverted `cmd/utils/flags.go` (`LogSlowBlockFlag`), `core/blockchain.go` (`SlowBlockThreshold` field), `eth/ethconfig/{config,gen_config}.go` (`SlowBlockThreshold` default/field) to HEAD. | Adopt together with the batch-5 #32812 row: when slow-block logging is grafted onto Bor's `insertChain`, use the #33655 output convention (negative disabled / zero all / positive filter). | upstream boundary `845009f68`; commit `1e9dfd5bb`; relates to the batch-5 #32812 row | +| legacypool add-path alloc reduction (#33701) | batch 10 (v1.17.0 10/12, `c12959dc8`) | deferred | Upstream reworked `addTxsLocked` to take a caller-provided `errs []error` (write errors in place, `Requires len(txs)==len(errs)`, skip pre-errored txs) and return only `*accountSet`, saving one `[]error` alloc per batch. Entangled with Bor's async txpool divergence: Bor renamed `addTxsLocked`→`addTxs(txs, async bool)` (returns `([]error, *accountSet)`, threads `async` into `pool.add(tx, async)` to avoid locking, merges errors in `Add` via a `nilSlot` loop). The refactor's non-conflicting parts auto-merged and clashed with Bor's async signature (`errs` undefined, wrong return arity). #33701 was the sole batch-10 commit touching `legacypool.go`, so restored the file wholesale to Bor HEAD. | Harmonize upstream's caller-provided-errs pattern into Bor's async `addTxs`: pass a pre-sized `errs` slice through `addTxs`/`pool.add`, drop the `Add` `nilSlot` merge, and skip pre-errored slots — preserving Bor's `async` flag and lock-avoidance. Micro-opt (one slice alloc/batch); low priority. | upstream boundary `c12959dc8`; commit `bba41f807`; `core/txpool/legacypool/legacypool.go` (`Add`, `addTxs`) | +| OpenTelemetry JSON-RPC CLI wiring (#33484) + spanEnd-by-pointer fix (#33772) | batch 11 (v1.17.0 11/12, `c50e5edfa`) | deferred | Continuation of the batch-9 OTel deferral (#33452). #33484 re-adds the `internal/telemetry` package (absent at HEAD since #33452 was deferred), `internal/telemetry/tracesetup.SetupTelemetry`, the `RPCTelemetry*` CLI flags + `setOpenTelemetry`, a `node.Config.OpenTelemetry` field + `OpenTelemetryConfig`, and threads the tracer through `rpc/handler.go`'s span code; #33772 fixes that span code to pass errors by pointer. `newHandler` collides with Bor's `SafePool` execution-pool divergence, and Bor already runs its own gRPC-server OTel (`internal/cli/server/server.go`) — a separate feature. Observability, not consensus. Declined: reverted `rpc/handler.go`, `node/config.go`, `cmd/geth/main.go`, `cmd/utils/flags.go` to HEAD; removed `internal/telemetry/telemetry.go`, `internal/telemetry/tracesetup/setup.go`, `rpc/tracing_test.go`; stripped the `tracesetup.SetupTelemetry` block + import from `cmd/geth/config.go` (kept its independent #33804 influxdb error-message fix). | Adopt together with the batch-9 #33452 row: thread `tracerProvider` through Bor's `newHandler` (alongside `SafePool`), re-add `internal/telemetry` + `tracesetup`, restore the span handling in `rpc/{handler,server,service}.go`, re-add the `RPCTelemetry*` flags + `node.Config.OpenTelemetry`, and reconcile with Bor's existing server-side OTel so the two don't double-configure. Observability nice-to-have. | upstream boundary `c50e5edfa`; commits `#33484`/`#33772`; relates to the batch-9 #33452 row; `rpc/handler.go`, `node/config.go`, `cmd/geth/{main,config}.go`, `cmd/utils/flags.go`, `internal/telemetry/*` | +| OpenTelemetry newPayload tracing + ctx-threading (#33521, #33780) | batch 12 (v1.17.0 12/12, `0cf3d3ba4`) | deferred | Continuation of the OTel line (batch-9 #33452 / batch-11 #33484). #33521 threads `ctx context.Context` through the entire block-processing chain (`Processor.Process`, `BlockChain.{insertChain,ProcessBlock,insertSideChain}`, `ExecuteStateless` + all call sites incl. `eth/state_accessor.go`, `eth/api_debug.go`, `eth/catalyst/{api,simulated_beacon}.go`, `cmd/keeper/main.go`) purely to carry `telemetry.StartSpan` server spans for `engine_newPayload`; #33780 tweaks span-parent handling. Entangled with Bor's diverged signatures — Bor's `Process(block, statedb, cfg, author *common.Address, interruptCtx context.Context)` (BlockSTM interrupt ctx + author), 6-return `ProcessBlock(block, header, …)`, 5-return `ExecuteStateless(config, vmconfig, block, witness, author, consensus, diskdb)` — and the payoff is telemetry spans being declined. Reverted the whole chain to HEAD (kept Bor's Prague `Bor == nil` gating + `reportBlock` naming); removed `internal/telemetry/{telemetry,telemetry_test}.go`. | Adopt together with the batch-9 #33452 + batch-11 #33484 OTel rows: re-add `internal/telemetry`, decide a ctx-threading strategy compatible with Bor's `interruptCtx`/`author` params (either add a leading `ctx` alongside them or reuse `interruptCtx`), thread it through Bor's diverged `Process`/`ProcessBlock`/`ExecuteStateless` + call sites, and add the newPayload server span in `eth/catalyst`. Observability nice-to-have. | upstream boundary `0cf3d3ba4`; commits `#33521`/`#33780`; relates to batch-9 #33452 + batch-11 #33484; `core/{types,state_processor,blockchain,stateless}.go`, `eth/{state_accessor,api_debug,catalyst/*}.go` | +| Delayed p2p message decoding (#33835) | batch 12 (v1.17.0 12/12, `0cf3d3ba4`) | deferred | 26-file eth/snap protocol refactor: `BlockBodiesResponse`/`GetTrieNodesPacket` response fields become `rlp.RawList[...]` (decode-on-demand — limits decode work on unmatched/oversized peer messages, a DoS-hardening measure), the `p2p/tracker` API changes from `Track(peer, version, reqCode, resCode, id)` to typed `Track(req Request) error` / `Fulfil(resp Response) error` (+ `ErrNoMatchingRequest`/`ErrCodeMismatch`), the receipt interface gains `Derivable() types.DerivableList` + `EncodeForStorage() (rlp.RawValue, error)`, and every eth/snap response handler is reworked around lazy decoding. Entangled with several Bor divergences: the receipt interface's Bor-only `ExcludeStateSyncReceipt()` (state-sync consensus), Bor's `newBlockPushIntervalTimer`/`lastNewBlockPushUnix` NewBlock push metrics in `handlers.go`, the snap `requestTracker` singleton (`snap/tracker.go`), and Bor's forked downloader (`bor_downloader.go`, `bor_fetchers_concurrent_bodies.go`) + wit protocol. Security-positive but a large cross-file rework on the consensus-feeding p2p path — out of scope for a merge batch. Reverted the full footprint to HEAD (`eth/protocols/eth/*`, `eth/protocols/snap/*` incl. restoring `snap/tracker.go`, `eth/downloader/{queue,queue_test,bor_fetchers_concurrent_bodies}.go`, `eth/handler_eth{,_test}.go`, `p2p/tracker/tracker.go`, `cmd/devp2p/internal/ethtest/*`); `git rm` of `eth/downloader/downloader_test.go` (DU) + new `p2p/tracker/tracker_test.go`. | Adopt the delayed-decoding rework onto Bor's protocol layer: (1) migrate `p2p/tracker` to the typed `Track(Request)`/`Fulfil(Response)` API + update all Bor callers; (2) convert `BlockBodiesResponse`/`GetTrieNodesPacket` to `rlp.RawList` while preserving Bor's `Unpack()` semantics; (3) reconcile the receipt interface with Bor's `ExcludeStateSyncReceipt()`; (4) re-add Bor's NewBlock push metrics + snap `requestTracker` behavior on top; (5) port into Bor's forked downloader. Security-positive (DoS hardening) — good adopt candidate when the protocol layer is next revisited. | upstream boundary `0cf3d3ba4`; commit `0cba803fb` (#33835); `eth/protocols/eth/*`, `eth/protocols/snap/*`, `eth/downloader/*`, `p2p/tracker/*` | +| EraE archive-format rewrite (#32157, #33827) | batch 11 (v1.17.0 11/12, `c50e5edfa`) | deferred | 15-file restructure of `internal/era`: splits the era1 monolith into `internal/era/onedb` + a new `internal/era/execdb`, adds `internal/era/proof.go`, a `types.SlimReceipt` RLP wrapper "used for era files", a `--era.format` (`era1`/`erae`) flag, and an `era.Era`/`era.Builder`/`ReadAtSeekCloser` interface layer. Entangled with two Bor divergences: (1) upstream's `ExportHistory` is built on "TD is no longer stored in the database" and recomputes initial TD by accumulating difficulty, but Bor HEAD still reads TD via `bc.GetTd`; (2) Bor's `ImportHistory(chain, db, dir, network)` adds a `db ethdb.Database` param + an explicit `InsertHeaderChain` step upstream's signature lacks. Adopting requires reworking Bor's TD storage + history-import path — out of scope for a merge batch, and TD-adjacent (consensus-sensitive). Reverted the full footprint to HEAD (`internal/era/{era,accumulator,builder,era_test,iterator}.go`, `internal/era/eradl/eradl.go`, `cmd/era/main.go`, `core/rawdb/eradb/eradb.go`, `cmd/utils/{cmd,history_test}.go`, `cmd/geth/chaincmd.go`, `core/types/receipt.go`+test); `git rm` of new `internal/era/{onedb/reader.go,onedb/builder.go,onedb/builder_test.go,onedb/iterator.go,execdb/*,proof.go}`. | Decide whether Bor adopts the EraE format: (1) reconcile `ExportHistory` with Bor's TD-in-DB (keep `bc.GetTd`, or adopt upstream's difficulty-accumulation only if Bor drops TD storage); (2) preserve Bor's `db` param + `InsertHeaderChain` in `ImportHistory` while adding the `from func(era.ReadAtSeekCloser) (era.Era, error)` format selector; (3) bring in `onedb`/`execdb` + `proof.go` + `SlimReceipt` + `--era.format`; (4) migrate `internal/era/eradl`, `cmd/era`, `core/rawdb/eradb` to the interface layer. Operational (history archive), not consensus. | upstream boundary `c50e5edfa`; commits `#32157`/`#33827`; `internal/era/*`, `cmd/utils/cmd.go`, `cmd/geth/chaincmd.go`, `core/types/receipt.go` | + +## Notes + +- Fork-schedule adoptions (EIP-8024, Verkle, and the upcoming Osaka/BPO/ + Amsterdam EIPs listed in `fork-register.md`) are **not** in this file's scope + as line items — enabling any of them is a deliberate fork-schedule change + (its own PR + N-1/N/N+1 tests), tracked in `fork-register.md`. This backlog + is for the surrounding *code wiring* (flags, config plumbing, tests, adapters) + a future adoption would need. diff --git a/docs/upstream-merges/v1.17.4/plan.md b/docs/upstream-merges/v1.17.4/plan.md index e7bf4b51aa..62dfa53274 100644 --- a/docs/upstream-merges/v1.17.4/plan.md +++ b/docs/upstream-merges/v1.17.4/plan.md @@ -23,18 +23,18 @@ Status legend: `pending` / `in-progress` / `merged` / `skipped`. | # | Milestone | Batch | Boundary | Commits | Theme | Status | | -- | --------- | ----- | ----------- | ------- | ---------------------------------------------------------------------------------------------- | ------- | | 1 | v1.16.9 | 1/1 | `95665d570` | 3 | crypto security backports (ECIES invalid-curve, secp256k1 coordinate check) + release | merged (`a06dbd7d2`) | -| 2 | v1.17.0 | 1/12 | `f23d506b7` | 20 | bugfix sweep: rawdb/state/pathdb fixes, RPC framing, blobpool, filters | pending | -| 3 | v1.17.0 | 2/12 | `cf93077fa` | 20 | verkle/UBT groundwork, rawdb+rlp hardening, fusaka beacon update | pending | -| 4 | v1.17.0 | 3/12 | `a122dbe45` | 20 | **EIP-8024 introduced** (`core/vm`), miner `--miner.maxblobs`, gnark-crypto bump | pending | -| 5 | v1.17.0 | 4/12 | `228933a66` | 20 | catalyst getPayload fork checks, pebble config, slow-block stats, downloader syncmode | pending | -| 6 | v1.17.0 | 5/12 | `5dfcffcf3` | 20 | state/code-read metrics, tx fetcher validation, blobpool legacy sidecar removal | pending | -| 7 | v1.17.0 | 6/12 | `710008450` | 20 | snap-sync locking, getBlobsV3, pathdb history indexing, go-verkle removal | pending | -| 8 | v1.17.0 | 7/12 | `94710f79a` | 20 | state update hook, trienode history compression, blobpool gaps; KZG peer-drop fix | pending | -| 9 | v1.17.0 | 8/12 | `8fad02ac6` | 20 | OpenTelemetry RPC tracing, **read-only gas handlers (`core/vm`)**, trienode history enable | pending | -| 10 | v1.17.0 | 9/12 | `845009f68` | 20 | eth_getProofs for history, **EIP-8024 immediate-byte update**, alloc reductions | pending | -| 11 | v1.17.0 | 10/12 | `c12959dc8` | 20 | perf/alloc sweep, keccak vendoring, Ledger Gen5, rlp RawList | pending | -| 12 | v1.17.0 | 11/12 | `c50e5edfa` | 20 | EraE format, rlp iterators, HTTP/2 JSON-RPC, telemetry CLI wiring | pending | -| 13 | v1.17.0 | 12/12 | `0cf3d3ba4` | 7 | delayed p2p decoding, header-verification hardening, **v1.17.0 release** | pending | +| 2 | v1.17.0 | 1/12 | `f23d506b7` | 20 | bugfix sweep: rawdb/state/pathdb fixes, RPC framing, blobpool, filters | merged (`2a85bf611`) | +| 3 | v1.17.0 | 2/12 | `cf93077fa` | 20 | verkle/UBT groundwork, rawdb+rlp hardening, fusaka beacon update | merged (`ad0f54e9c`) | +| 4 | v1.17.0 | 3/12 | `a122dbe45` | 20 | **EIP-8024 introduced** (`core/vm`), miner `--miner.maxblobs`, gnark-crypto bump | merged (`7278fa559`) | +| 5 | v1.17.0 | 4/12 | `228933a66` | 20 | catalyst getPayload fork checks, pebble config, slow-block stats, downloader syncmode | merged (`8b4a6095d`) | +| 6 | v1.17.0 | 5/12 | `5dfcffcf3` | 20 | state/code-read metrics, tx fetcher validation, blobpool legacy sidecar removal | merged (`4e17d3dc0`) | +| 7 | v1.17.0 | 6/12 | `710008450` | 20 | snap-sync locking, getBlobsV3, pathdb history indexing, go-verkle removal | merged (`85a1750c5`) | +| 8 | v1.17.0 | 7/12 | `94710f79a` | 20 | state update hook, trienode history compression, blobpool gaps; KZG peer-drop fix | merged (`09cddf9b7`) | +| 9 | v1.17.0 | 8/12 | `8fad02ac6` | 20 | OpenTelemetry RPC tracing, **read-only gas handlers (`core/vm`)**, trienode history enable | merged (`c1537496c`) | +| 10 | v1.17.0 | 9/12 | `845009f68` | 20 | eth_getProofs for history, **EIP-8024 immediate-byte update**, alloc reductions | merged (`0f76e0c30`) | +| 11 | v1.17.0 | 10/12 | `c12959dc8` | 20 | perf/alloc sweep, keccak vendoring, Ledger Gen5, rlp RawList | merged (`f4f2c07fd`) | +| 12 | v1.17.0 | 11/12 | `c50e5edfa` | 20 | EraE format, rlp iterators, HTTP/2 JSON-RPC, telemetry CLI wiring | merged (`2c0608fc6`) | +| 13 | v1.17.0 | 12/12 | `0cf3d3ba4` | 7 | delayed p2p decoding, header-verification hardening, **v1.17.0 release** | merged (`70994671a`) | | 14 | v1.17.1 | 1/2 | `9ecb6c4ae` | 20 | **syscall value-transfer disable (`core/vm`)**, BAL type changes, **Amsterdam precompile touch** | pending | | 15 | v1.17.1 | 2/2 | `16783c167` | 20 | **eth/68 protocol drop**, **EIP-7843 SLOTNUM**, **8024 enabled in Amsterdam**, Go 1.26; release | pending | | 16 | v1.17.2 | 1/4 | `00540f946` | 20 | **EIP-7778 block gas accounting**, miner prefetcher, **amsterdam jump table**, default cache 4096 | pending | @@ -82,7 +82,7 @@ Status legend: `pending` / `in-progress` / `merged` / `skipped`. ## Resume anchors -- `PROGRESS_TIP`: none yet (no batch merged). +- `PROGRESS_TIP`: `70994671a` (v1.17.0 batch 12/12 `0cf3d3ba4` merged). **All 12 v1.17.0 batches merged.** Next = v1.17.0 **milestone chores** (durable-docs commit + `gen_config.go` regen + `pdbExemptMethods` fix + full integration/kurtosis/diffguard/govulncheck + PR triage), then the v1.17.0 stacked draft PR, then milestone v1.17.1 (batches 14–15, boundary `9ecb6c4ae`). - Working branch for milestone 1 (`v1.16.9`): `ppatil-upstream-v1.16.9` — cut from `upstream-merge-v1.17.4` on the milestone's first batch. - Full first-parent logs per milestone captured in the planning run directory: diff --git a/eth/api_debug.go b/eth/api_debug.go index 7bcf149e07..24c96690a0 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -499,7 +499,6 @@ func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumber) (*stateless.ExtWitness if err != nil { return &stateless.ExtWitness{}, fmt.Errorf("block number %v not found", bn) } - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) if parent == nil { return &stateless.ExtWitness{}, fmt.Errorf("block number %v found, but parent missing", bn) @@ -520,7 +519,6 @@ func (api *DebugAPI) ExecutionWitnessByHash(hash common.Hash) (*stateless.ExtWit if block == nil { return &stateless.ExtWitness{}, fmt.Errorf("block hash %x not found", hash) } - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) if parent == nil { return &stateless.ExtWitness{}, fmt.Errorf("block number %x found, but parent missing", hash) diff --git a/eth/backend.go b/eth/backend.go index d6414dd0b1..13d0be69dc 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -307,22 +307,24 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { var ( options = &core.BlockChainConfig{ - TrieCleanLimit: config.TrieCleanCache, - NoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - ArchiveMode: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - StateScheme: scheme, - TriesInMemory: config.TriesInMemory, - ChainHistoryMode: config.HistoryMode, - TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), - AddressCacheSizes: config.AddressCacheSizes, - PreloadRateLimit: config.PreloadRateLimit, - VmConfig: vmCfg, - Stateless: config.SyncMode == downloader.StatelessSync, + TrieCleanLimit: config.TrieCleanCache, + NoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + ArchiveMode: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + TrienodeHistory: config.TrienodeHistory, + NodeFullValueCheckpoint: config.NodeFullValueCheckpoint, + StateScheme: scheme, + TriesInMemory: config.TriesInMemory, + ChainHistoryMode: config.HistoryMode, + TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), + AddressCacheSizes: config.AddressCacheSizes, + PreloadRateLimit: config.PreloadRateLimit, + VmConfig: vmCfg, + Stateless: config.SyncMode == downloader.StatelessSync, // Enables file journaling for the trie database. The journal files will be stored // within the data directory. The corresponding paths will be either: // - DATADIR/triedb/merkle.journal @@ -963,6 +965,9 @@ func (s *Ethereum) updateFilterMapsHeads() { if head == nil || newHead.Hash() != head.Hash() { head = newHead chainView := s.newChainView(head) + if chainView == nil { + return + } historyCutoff, _ := s.blockchain.HistoryPruningCutoff() var finalBlock uint64 if fb := s.blockchain.CurrentFinalBlock(); fb != nil { diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 22bdce8f9f..290e9ba3f6 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -38,7 +38,6 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -83,20 +82,6 @@ const ( beaconUpdateWarnFrequency = 5 * time.Minute ) -var ( - // Number of blobs requested via getBlobsV2 - getBlobsRequestedCounter = metrics.NewRegisteredCounter("engine/getblobs/requested", nil) - - // Number of blobs requested via getBlobsV2 that are present in the blobpool - getBlobsAvailableCounter = metrics.NewRegisteredCounter("engine/getblobs/available", nil) - - // Number of times getBlobsV2 responded with “hit” - getBlobsV2RequestHit = metrics.NewRegisteredCounter("engine/getblobs/hit", nil) - - // Number of times getBlobsV2 responded with “miss” - getBlobsV2RequestMiss = metrics.NewRegisteredCounter("engine/getblobs/miss", nil) -) - type ConsensusAPI struct { eth *eth.Ethereum @@ -139,6 +124,9 @@ type ConsensusAPI struct { // NewConsensusAPI creates a new consensus api for the given backend. // The underlying blockchain needs to have a valid terminal total difficulty set. +// +// This function creates a long-lived object with an attached background thread. +// For testing or other short-term use cases, please use newConsensusAPIWithoutHeartbeat. func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { api := newConsensusAPIWithoutHeartbeat(eth) go api.heartbeat() @@ -165,9 +153,6 @@ func newConsensusAPIWithoutHeartbeat(eth *eth.Ethereum) *ConsensusAPI { // // We try to set our blockchain to the headBlock. // -// If the method is called with an empty head block: we return success, which can be used -// to check if the engine API is enabled. -// // If the total difficulty was not reached: we return INVALID. // // If the finalizedBlockHash is set: we check if we have the finalizedBlockHash in our db, @@ -476,10 +461,12 @@ func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config engine.Transit // GetPayloadV1 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.ExecutableData, error) { - if !payloadID.Is(engine.PayloadV1) { - return nil, engine.UnsupportedFork - } - data, err := api.getPayload(payloadID, false) + data, err := api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV1}, + nil, + ) if err != nil { return nil, err } @@ -488,35 +475,34 @@ func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.Execu // GetPayloadV2 returns a cached payload by id. func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - // executionPayload: ExecutionPayloadV1 | ExecutionPayloadV2 where: - // - // - ExecutionPayloadV1 MUST be returned if the payload timestamp is lower - // than the Shanghai timestamp - // - // - ExecutionPayloadV2 MUST be returned if the payload timestamp is greater - // or equal to the Shanghai timestamp - if !payloadID.Is(engine.PayloadV1, engine.PayloadV2) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV1, engine.PayloadV2}, + []forks.Fork{forks.Shanghai}, + ) } // GetPayloadV3 returns a cached payload by id. This endpoint should only // be used for the Cancun fork. func (api *ConsensusAPI) GetPayloadV3(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{forks.Cancun}, + ) } // GetPayloadV4 returns a cached payload by id. This endpoint should only // be used for the Prague fork. func (api *ConsensusAPI) GetPayloadV4(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{forks.Prague}, + ) } // GetPayloadV5 returns a cached payload by id. This endpoint should only @@ -525,18 +511,37 @@ func (api *ConsensusAPI) GetPayloadV4(payloadID engine.PayloadID) (*engine.Execu // This method follows the same specification as engine_getPayloadV4 with // changes of returning BlobsBundleV2 with BlobSidecar version 1. func (api *ConsensusAPI) GetPayloadV5(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - if !payloadID.Is(engine.PayloadV3) { - return nil, engine.UnsupportedFork - } - return api.getPayload(payloadID, false) + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV3}, + []forks.Fork{ + forks.Osaka, + forks.BPO1, + forks.BPO2, + forks.BPO3, + forks.BPO4, + forks.BPO5, + }) } -func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID, full bool) (*engine.ExecutionPayloadEnvelope, error) { +// getPayload will retrieve the specified payload and verify it conforms to the +// endpoint's allowed payload versions and forks. +// +// Note passing nil `forks`, `versions` disables the respective check. +func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID, full bool, versions []engine.PayloadVersion, forks []forks.Fork) (*engine.ExecutionPayloadEnvelope, error) { log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) + if versions != nil && !payloadID.Is(versions...) { + return nil, engine.UnsupportedFork + } data := api.localBlocks.get(payloadID, full) if data == nil { return nil, engine.UnknownPayload } + if forks != nil && !api.checkFork(data.ExecutionPayload.Timestamp, forks...) { + return nil, engine.UnsupportedFork + } + return data, nil } @@ -571,7 +576,7 @@ func (api *ConsensusAPI) GetBlobsV1(hashes []common.Hash) ([]*engine.BlobAndProo if len(hashes) > 128 { return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes))) } - blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion0, false) + blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion0) if err != nil { return nil, engine.InvalidParams.With(err) } @@ -617,8 +622,25 @@ func (api *ConsensusAPI) GetBlobsV1(hashes []common.Hash) ([]*engine.BlobAndProo func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) { head := api.eth.BlockChain().CurrentHeader() if api.config().LatestFork(head.Time) < forks.Osaka { - return nil, unsupportedForkErr("engine_getBlobsV2 is not available before Osaka fork") + return nil, nil } + return api.getBlobs(hashes, true) +} + +// GetBlobsV3 returns a set of blobs from the transaction pool. Same as +// GetBlobsV2, except will return partial responses in case there is a missing +// blob. +func (api *ConsensusAPI) GetBlobsV3(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) { + head := api.eth.BlockChain().CurrentHeader() + if api.config().LatestFork(head.Time) < forks.Osaka { + return nil, nil + } + return api.getBlobs(hashes, false) +} + +// getBlobs returns all available blobs. In v2, partial responses are not allowed, +// while v3 supports partial responses. +func (api *ConsensusAPI) getBlobs(hashes []common.Hash, v2 bool) ([]*engine.BlobAndProofV2, error) { if len(hashes) > 128 { return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes))) } @@ -626,28 +648,30 @@ func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProo getBlobsRequestedCounter.Inc(int64(len(hashes))) getBlobsAvailableCounter.Inc(int64(available)) - // Optimization: check first if all blobs are available, if not, return empty response - if available != len(hashes) { - getBlobsV2RequestMiss.Inc(1) + // Short circuit if partial response is not allowed + if v2 && available != len(hashes) { + getBlobsRequestMiss.Inc(1) return nil, nil } - - blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion1, false) + // Retrieve blobs from the pool. This operation is expensive and may involve + // heavy disk I/O. + blobs, _, proofs, err := api.eth.BlobTxPool().GetBlobs(hashes, types.BlobSidecarVersion1) if err != nil { return nil, engine.InvalidParams.With(err) } - - // To comply with API spec, check again that we really got all data needed - for _, blob := range blobs { - if blob == nil { - getBlobsV2RequestMiss.Inc(1) - return nil, nil - } - } - getBlobsV2RequestHit.Inc(1) - + // Validate the blobs from the pool and assemble the response res := make([]*engine.BlobAndProofV2, len(hashes)) - for i := 0; i < len(blobs); i++ { + for i := range blobs { + // The blob has been evicted since the last AvailableBlobs call. + // Return null if partial response is not allowed. + if blobs[i] == nil { + if !v2 { + continue + } else { + getBlobsRequestMiss.Inc(1) + return nil, nil + } + } var cellProofs []hexutil.Bytes for _, proof := range proofs[i] { cellProofs = append(cellProofs, proof[:]) @@ -657,6 +681,13 @@ func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProo CellProofs: cellProofs, } } + if len(res) == len(hashes) { + getBlobsRequestCompleteHit.Inc(1) + } else if len(res) > 0 { + getBlobsRequestPartialHit.Inc(1) + } else { + getBlobsRequestMiss.Inc(1) + } return res, nil } @@ -826,7 +857,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe return engine.INVALID_TERMINAL_BLOCK, nil } if block.Time() <= parent.Time() { - log.Warn("Invalid timestamp", "parent", block.Time(), "block", block.Time()) + log.Warn("Invalid timestamp", "parent", parent.Time(), "block", block.Time()) return api.invalid(errors.New("invalid timestamp"), parent.Header()), nil } // Another corner case: if the node is in snap sync mode, but the CL client @@ -1122,7 +1153,7 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) engine.PayloadSt return engine.PayloadStatusV1{Status: engine.SYNCING} } // Either no beacon sync was started yet, or it rejected the delivered - // payload as non-integratable on top of the existing sync. We'll just + // payload as non-integrate on top of the existing sync. We'll just // have to rely on the beacon client to forcefully update the head with // a forkchoice update request. if api.eth.SyncMode() == downloader.FullSync { @@ -1220,8 +1251,6 @@ func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) engine.Pa // heartbeat loops indefinitely, and checks if there have been beacon client updates // received in the last while. If not - or if they but strange ones - it warns the // user that something might be off with their consensus node. -// -// TODO(karalabe): Spin this goroutine down somehow func (api *ConsensusAPI) heartbeat() { // Sleep a bit on startup since there's obviously no beacon client yet // attached, so no need to print scary warnings to the user. diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 15b2f0455f..7c50289eae 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -225,7 +225,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { BeaconRoot: blockParams.BeaconRoot, Version: engine.PayloadV1, }).Id() - execData, err := api.getPayload(payloadID, true) + execData, err := api.getPayload(payloadID, true, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } @@ -450,7 +450,7 @@ func TestEth2DeepReorg(t *testing.T) { } // startEthService creates a full node instance for testing. -func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) (*node.Node, *eth.Ethereum) { +func startEthService(t testing.TB, genesis *core.Genesis, blocks []*types.Block) (*node.Node, *eth.Ethereum) { t.Helper() n, err := node.New(&node.Config{ @@ -664,7 +664,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { t.Fatalf("error preparing payload, invalid status: %v", resp.PayloadStatus.Status) } // give the payload some time to be built - if payload, err = api.getPayload(*resp.PayloadID, true); err != nil { + if payload, err = api.getPayload(*resp.PayloadID, true, nil, nil); err != nil { t.Fatalf("can't get payload: %v", err) } if len(payload.ExecutionPayload.Transactions) > 0 { @@ -1319,7 +1319,7 @@ func TestNilWithdrawals(t *testing.T) { Random: test.blockParams.Random, Version: payloadVersion, }).Id() - execData, err := api.GetPayloadV2(payloadID) + execData, err := api.getPayload(payloadID, false, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } @@ -1780,7 +1780,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { BeaconRoot: blockParams.BeaconRoot, Version: engine.PayloadV3, }).Id() - envelope, err := api.getPayload(payloadID, true) + envelope, err := api.getPayload(payloadID, true, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } @@ -1980,7 +1980,7 @@ func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount in return types.MustSignNewTx(key, types.LatestSigner(chainConfig), blobtx) } -func newGetBlobEnv(t *testing.T, version byte) (*node.Node, *ConsensusAPI) { +func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) { var ( // Create a database pre-initialize with a genesis block config = *params.MergedTestChainConfig @@ -2155,36 +2155,64 @@ func TestGetBlobsV2(t *testing.T) { }, } for i, suite := range suites { - // Fill the request for retrieving blobs - var ( - vhashes []common.Hash - expect []*engine.BlobAndProofV2 - ) - // fill missing blob - if suite.fillRandom { - vhashes = append(vhashes, testrand.Hash()) - } - for j := suite.start; j < suite.limit; j++ { - vhashes = append(vhashes, testBlobVHashes[j]) - var cellProofs []hexutil.Bytes - for _, proof := range testBlobCellProofs[j] { - cellProofs = append(cellProofs, proof[:]) + runGetBlobs(t, api.GetBlobsV2, suite.start, suite.limit, suite.fillRandom, false, fmt.Sprintf("GetBlobsV2 suite=%d", i)) + runGetBlobs(t, api.GetBlobsV3, suite.start, suite.limit, suite.fillRandom, true, fmt.Sprintf("GetBlobsV3 suite=%d %v", i, suite)) + } +} + +// Benchmark GetBlobsV2 internals +// Note that this is not an RPC-level benchmark, so JSON-RPC overhead is not included. +func BenchmarkGetBlobsV2(b *testing.B) { + n, api := newGetBlobEnv(b, 1) + defer n.Close() + + // for blobs in [1, 2, 4, 6], print string and run benchmark + for _, blobs := range []int{1, 2, 4, 6} { + name := fmt.Sprintf("blobs=%d", blobs) + b.Run(name, func(b *testing.B) { + for b.Loop() { + runGetBlobs(b, api.GetBlobsV2, 0, blobs, false, false, name) } - expect = append(expect, &engine.BlobAndProofV2{ - Blob: testBlobs[j][:], - CellProofs: cellProofs, - }) - } - result, err := api.GetBlobsV2(vhashes) - if err != nil { - t.Errorf("Unexpected error for case %d, %v", i, err) - } - // null is responded if any blob is missing - if suite.fillRandom { + }) + } +} + +type getBlobsFn func(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) + +func runGetBlobs(t testing.TB, getBlobs getBlobsFn, start, limit int, fillRandom bool, expectPartialResponse bool, name string) { + // Fill the request for retrieving blobs + var ( + vhashes []common.Hash + expect []*engine.BlobAndProofV2 + ) + for j := start; j < limit; j++ { + vhashes = append(vhashes, testBlobVHashes[j]) + var cellProofs []hexutil.Bytes + for _, proof := range testBlobCellProofs[j] { + cellProofs = append(cellProofs, proof[:]) + } + expect = append(expect, &engine.BlobAndProofV2{ + Blob: testBlobs[j][:], + CellProofs: cellProofs, + }) + } + // fill missing blob + if fillRandom { + vhashes = append(vhashes, testrand.Hash()) + } + result, err := getBlobs(vhashes) + if err != nil { + t.Errorf("Unexpected error for case %s, %v", name, err) + } + if fillRandom { + if expectPartialResponse { + expect = append(expect, nil) + } else { + // Nil is expected if getBlobs can not return a partial response expect = nil } - if !reflect.DeepEqual(result, expect) { - t.Fatalf("Unexpected result for case %d", i) - } + } + if !reflect.DeepEqual(result, expect) { + t.Fatalf("Unexpected result for case %s", name) } } diff --git a/eth/catalyst/metrics.go b/eth/catalyst/metrics.go new file mode 100644 index 0000000000..01a24191b0 --- /dev/null +++ b/eth/catalyst/metrics.go @@ -0,0 +1,37 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import "github.com/ethereum/go-ethereum/metrics" + +var ( + // Number of blobs requested via getBlobsV2 + getBlobsRequestedCounter = metrics.NewRegisteredCounter("engine/getblobs/requested", nil) + + // Number of blobs requested via getBlobsV2 that are present in the blobpool + getBlobsAvailableCounter = metrics.NewRegisteredCounter("engine/getblobs/available", nil) + + // Number of times getBlobsV2/V3 responded with all blobs + getBlobsRequestCompleteHit = metrics.NewRegisteredCounter("engine/getblobs/hit", nil) + + // Number of times getBlobsV2/V3 responded with no blobs. V2 will return no + // blobs if it doesn't have all the blobs (all or nothing). + getBlobsRequestMiss = metrics.NewRegisteredCounter("engine/getblobs/miss", nil) + + // Number of times getBlobsV3 responded with some, but not all, blobs + getBlobsRequestPartialHit = metrics.NewRegisteredCounter("engine/getblobs/partial", nil) +) diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 157b5e1ba3..25a136f049 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -214,7 +214,7 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u return nil } - envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true) + envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true, nil, nil) if err != nil { return err } @@ -280,9 +280,8 @@ func (c *SimulatedBeacon) loop() { case <-timer.C: if err := c.sealBlock(c.withdrawals.pop(10), uint64(time.Now().Unix())); err != nil { log.Warn("Error performing sealing work", "err", err) - } else { - timer.Reset(time.Second * time.Duration(c.period)) } + timer.Reset(time.Second * time.Duration(c.period)) } } } diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 75a02a53b0..f7184c0c40 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -613,8 +613,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common skip := make([]*types.Header, 0) progress := false throttled := false - - for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ { + for len(send) < count && !taskQueue.Empty() { // the task queue will pop items in order, so the highest prio block // is also the lowest block number. header, _ := taskQueue.Peek() @@ -631,9 +630,6 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common progress = true delete(taskPool, header.Hash()) - - proc = proc - 1 - log.Error("Fetch reservation already delivered", "number", header.Number.Uint64()) continue @@ -659,8 +655,6 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common // If it's a noop, we can skip this task delete(taskPool, header.Hash()) taskQueue.PopItem() - - proc = proc - 1 progress = true continue diff --git a/eth/dropper.go b/eth/dropper.go index d69a0b27f9..4cdc7de3fe 100644 --- a/eth/dropper.go +++ b/eth/dropper.go @@ -144,6 +144,9 @@ func randomDuration(min, max time.Duration) time.Duration { if min > max { panic("min duration must be less than or equal to max duration") } + if min == max { + return min + } return time.Duration(mrand.Int63n(int64(max-min)) + int64(min)) } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index c7643976f3..a139b1ebe2 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -47,6 +47,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/triedb/pathdb" ) // parseURLs splits a comma-separated URL string into a trimmed, non-empty slice. @@ -80,33 +81,35 @@ var FullNodeGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - SyncMode: downloader.SnapSync, - HistoryMode: history.KeepAll, - NetworkId: 0, // enable auto configuration of networkID == chainID - TxLookupLimit: 2350000, - TransactionHistory: 2350000, // Note: used in bor cli - LogHistory: 2350000, // Note: used in bor cli - StateHistory: params.FullImmutabilityThreshold, - DatabaseCache: 512, - TrieCleanCache: 154, - TrieDirtyCache: 256, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 102, - FilterLogCacheSize: 32, - LogQueryLimit: 1000, - Miner: miner.DefaultConfig, - TxPool: legacypool.DefaultConfig, - BlobPool: blobpool.DefaultConfig, - RPCGasCap: 50000000, - RPCEVMTimeout: 5 * time.Second, - GPO: FullNodeGPO, - RPCTxFeeCap: 1, // 1 ether - FastForwardThreshold: 6400, - WitnessPruneThreshold: 64000, - WitnessPruneInterval: 120 * time.Second, - WitnessAPIEnabled: false, - TxSyncDefaultTimeout: 20 * time.Second, - TxSyncMaxTimeout: 1 * time.Minute, + SyncMode: downloader.SnapSync, + HistoryMode: history.KeepAll, + NetworkId: 0, // enable auto configuration of networkID == chainID + TxLookupLimit: 2350000, + TransactionHistory: 2350000, // Note: used in bor cli + LogHistory: 2350000, // Note: used in bor cli + StateHistory: params.FullImmutabilityThreshold, + TrienodeHistory: -1, + NodeFullValueCheckpoint: pathdb.Defaults.FullValueCheckpoint, + DatabaseCache: 512, + TrieCleanCache: 154, + TrieDirtyCache: 256, + TrieTimeout: 60 * time.Minute, + SnapshotCache: 102, + FilterLogCacheSize: 32, + LogQueryLimit: 1000, + Miner: miner.DefaultConfig, + TxPool: legacypool.DefaultConfig, + BlobPool: blobpool.DefaultConfig, + RPCGasCap: 50000000, + RPCEVMTimeout: 5 * time.Second, + GPO: FullNodeGPO, + RPCTxFeeCap: 1, // 1 ether + FastForwardThreshold: 6400, + WitnessPruneThreshold: 64000, + WitnessPruneInterval: 120 * time.Second, + WitnessAPIEnabled: false, + TxSyncDefaultTimeout: 20 * time.Second, + TxSyncMaxTimeout: 1 * time.Minute, } //go:generate go run github.com/fjl/gencodec -type Config -formats toml -out gen_config.go @@ -142,6 +145,13 @@ type Config struct { LogNoHistory bool `toml:",omitempty"` // No log search index is maintained. LogExportCheckpoints string // export log index checkpoints to file StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. + TrienodeHistory int64 `toml:",omitempty"` // Number of blocks from the chain head for which trienode histories are retained + + // The frequency of full-value encoding. For example, a value of 16 means + // that, on average, for a given trie node across its 16 consecutive historical + // versions, only one version is stored in full format, while the others + // are stored in diff mode for storage compression. + NodeFullValueCheckpoint uint32 `toml:",omitempty"` // State scheme represents the scheme used to store ethereum states and trie // nodes on top. It can be 'hash', 'path', or none which means use the scheme diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index aa318fb2f6..219f23fb31 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -34,6 +34,8 @@ func (c Config) MarshalTOML() (interface{}, error) { LogNoHistory bool `toml:",omitempty"` LogExportCheckpoints string StateHistory uint64 `toml:",omitempty"` + TrienodeHistory int64 `toml:",omitempty"` + NodeFullValueCheckpoint uint32 `toml:",omitempty"` StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` SkipBcVersionCheck bool `toml:"-"` @@ -84,6 +86,7 @@ func (c Config) MarshalTOML() (interface{}, error) { BorLogs bool ParallelEVM core.ParallelEVMConfig `toml:",omitempty"` WitnessProtocol bool + NoSnapServing bool SyncWithWitnesses bool SyncAndProduceWitnesses bool DevFakeAuthor bool `hcl:"devfakeauthor,optional" toml:"devfakeauthor,optional"` @@ -122,6 +125,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.LogNoHistory = c.LogNoHistory enc.LogExportCheckpoints = c.LogExportCheckpoints enc.StateHistory = c.StateHistory + enc.TrienodeHistory = c.TrienodeHistory + enc.NodeFullValueCheckpoint = c.NodeFullValueCheckpoint enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks enc.SkipBcVersionCheck = c.SkipBcVersionCheck @@ -172,6 +177,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.BorLogs = c.BorLogs enc.ParallelEVM = c.ParallelEVM enc.WitnessProtocol = c.WitnessProtocol + enc.NoSnapServing = c.NoSnapServing enc.SyncWithWitnesses = c.SyncWithWitnesses enc.SyncAndProduceWitnesses = c.SyncAndProduceWitnesses enc.DevFakeAuthor = c.DevFakeAuthor @@ -214,6 +220,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { LogNoHistory *bool `toml:",omitempty"` LogExportCheckpoints *string StateHistory *uint64 `toml:",omitempty"` + TrienodeHistory *int64 `toml:",omitempty"` + NodeFullValueCheckpoint *uint32 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` SkipBcVersionCheck *bool `toml:"-"` @@ -264,6 +272,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { BorLogs *bool ParallelEVM *core.ParallelEVMConfig `toml:",omitempty"` WitnessProtocol *bool + NoSnapServing *bool SyncWithWitnesses *bool SyncAndProduceWitnesses *bool DevFakeAuthor *bool `hcl:"devfakeauthor,optional" toml:"devfakeauthor,optional"` @@ -333,6 +342,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.StateHistory != nil { c.StateHistory = *dec.StateHistory } + if dec.TrienodeHistory != nil { + c.TrienodeHistory = *dec.TrienodeHistory + } + if dec.NodeFullValueCheckpoint != nil { + c.NodeFullValueCheckpoint = *dec.NodeFullValueCheckpoint + } if dec.StateScheme != nil { c.StateScheme = *dec.StateScheme } @@ -483,6 +498,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.WitnessProtocol != nil { c.WitnessProtocol = *dec.WitnessProtocol } + if dec.NoSnapServing != nil { + c.NoSnapServing = *dec.NoSnapServing + } if dec.SyncWithWitnesses != nil { c.SyncWithWitnesses = *dec.SyncWithWitnesses } diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 666d919549..e4f94b20d7 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -357,9 +357,9 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) otherRejectMeter.Mark(otherreject) // If 'other reject' is >25% of the deliveries in any batch, sleep a bit. - if otherreject > addTxsBatchSize/4 { + if otherreject > int64((len(batch)+3)/4) { + log.Debug("Peer delivering stale or invalid transactions", "peer", peer, "rejected", otherreject) time.Sleep(200 * time.Millisecond) - log.Debug("Peer delivering stale transactions", "peer", peer, "rejected", otherreject) } // If we encountered a protocol violation, disconnect this peer. if violation != nil { diff --git a/eth/filters/api.go b/eth/filters/api.go index 3ca35276fa..5c58d5d561 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -36,18 +36,30 @@ import ( ) var ( - errInvalidTopic = errors.New("invalid topic(s)") - errFilterNotFound = errors.New("filter not found") - errInvalidBlockRange = errors.New("invalid block range params") + errInvalidTopic = invalidParamsErr("invalid topic(s)") + errInvalidBlockRange = invalidParamsErr("invalid block range params") + errBlockRangeIntoFuture = invalidParamsErr("block range extends beyond current head block") + errBlockHashWithRange = invalidParamsErr("can't specify fromBlock/toBlock with blockHash") + errPendingLogsUnsupported = invalidParamsErr("pending logs are not supported") errUnknownBlock = errors.New("unknown block") - errBlockHashWithRange = errors.New("can't specify fromBlock/toBlock with blockHash") - errPendingLogsUnsupported = errors.New("pending logs are not supported") + errFilterNotFound = errors.New("filter not found") errExceedMaxTopics = errors.New("exceed max topics") errExceedLogQueryLimit = errors.New("exceed max addresses or topics per search position") errExceedMaxTxHashes = errors.New("exceed max number of transaction hashes allowed per transactionReceipts subscription") errExceedBlockRangeLimit = errors.New("block range exceeds configured limit") ) +type invalidParamsError struct { + err error +} + +func (e invalidParamsError) Error() string { return e.err.Error() } +func (e invalidParamsError) ErrorCode() int { return -32602 } + +func invalidParamsErr(format string, args ...any) error { + return invalidParamsError{fmt.Errorf(format, args...)} +} + const ( // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0 maxTopics = 4 diff --git a/eth/filters/filter.go b/eth/filters/filter.go index b449813af8..a3936993bc 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -223,9 +223,12 @@ func (s *searchSession) updateChainView() error { if lastBlock == math.MaxUint64 { lastBlock = head } - if firstBlock > lastBlock || lastBlock > head { + if firstBlock > lastBlock { return errInvalidBlockRange } + if lastBlock > head { + return errBlockRangeIntoFuture + } s.searchRange = common.NewRange(firstBlock, lastBlock+1-firstBlock) // Trim existing match set in case a reorg may have invalidated some results @@ -576,7 +579,7 @@ type ReceiptWithTx struct { // In addition to returning receipts, it also returns the corresponding transactions. // This is because receipts only contain low-level data, while user-facing data // may require additional information from the Transaction. -func filterReceipts(txHashes map[common.Hash]bool, ev core.ChainEvent) []*ReceiptWithTx { +func filterReceipts(txHashes map[common.Hash]struct{}, ev core.ChainEvent) []*ReceiptWithTx { var ret []*ReceiptWithTx receipts := ev.Receipts @@ -598,7 +601,7 @@ func filterReceipts(txHashes map[common.Hash]bool, ev core.ChainEvent) []*Receip } } else { for i, receipt := range receipts { - if txHashes[receipt.TxHash] { + if _, ok := txHashes[receipt.TxHash]; ok { ret = append(ret, &ReceiptWithTx{ Receipt: receipt, Transaction: txs[i], diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 85360ec55f..58b3633a9b 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -210,7 +210,7 @@ type subscription struct { txs chan []*types.Transaction headers chan *types.Header receipts chan []*ReceiptWithTx - txHashes map[common.Hash]bool // contains transaction hashes for transactionReceipts subscription filtering + txHashes map[common.Hash]struct{} // contains transaction hashes for transactionReceipts subscription filtering installed chan struct{} // closed when the filter is installed err chan error // closed when the filter is uninstalled stateSyncData chan *types.StateSyncData // channel for state sync data @@ -444,9 +444,9 @@ func (es *EventSystem) SubscribePendingTxs(txs chan []*types.Transaction) *Subsc // transactions when they are included in blocks. If txHashes is provided, only receipts // for those specific transaction hashes will be delivered. func (es *EventSystem) SubscribeTransactionReceipts(txHashes []common.Hash, receipts chan []*ReceiptWithTx) *Subscription { - hashSet := make(map[common.Hash]bool) + hashSet := make(map[common.Hash]struct{}, len(txHashes)) for _, h := range txHashes { - hashSet[h] = true + hashSet[h] = struct{}{} } sub := &subscription{ id: rpc.NewID(), diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 4b7ef60d9b..d5c01ec7f0 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "github.com/holiman/uint256" ) const sampleNumber = 3 // Number of transactions sampled in a block @@ -292,12 +293,12 @@ func (oracle *Oracle) getBlockValues(ctx context.Context, blockNum uint64, limit sortedTxs := make([]*types.Transaction, len(txs)) copy(sortedTxs, txs) baseFee := block.BaseFee() + baseFee256 := new(uint256.Int) + if baseFee != nil { + baseFee256.SetFromBig(baseFee) + } slices.SortFunc(sortedTxs, func(a, b *types.Transaction) int { - // It's okay to discard the error because a tx would never be - // accepted into a block with an invalid effective tip. - tip1, _ := a.EffectiveGasTip(baseFee) - tip2, _ := b.EffectiveGasTip(baseFee) - return tip1.Cmp(tip2) + return a.EffectiveGasTipCmp(b, baseFee256) }) var prices []*big.Int diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index cb522863ef..98af9edd32 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -461,7 +461,7 @@ func (p *Peer) RequestReceipts(hashes []common.Hash, sink chan *Response) (*Requ // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { - p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) + p.Log().Trace("Fetching batch of transactions", "count", len(hashes)) id := rand.Uint64() requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 779b7ded0b..62474dbefc 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -29,12 +29,12 @@ import ( "time" "github.com/holiman/uint256" - "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/log" @@ -58,7 +58,7 @@ func TestHashing(t *testing.T) { var want, got string var old = func() { - hasher := sha3.NewLegacyKeccak256() + hasher := keccak.NewLegacyKeccak256() for i := 0; i < len(bytecodes); i++ { hasher.Reset() hasher.Write(bytecodes[i]) @@ -95,7 +95,7 @@ func BenchmarkHashing(b *testing.B) { } var old = func() { - hasher := sha3.NewLegacyKeccak256() + hasher := keccak.NewLegacyKeccak256() for i := 0; i < len(bytecodes); i++ { hasher.Reset() hasher.Write(bytecodes[i]) diff --git a/eth/syncer/syncer.go b/eth/syncer/syncer.go index 9411133e00..2c6dbeba88 100644 --- a/eth/syncer/syncer.go +++ b/eth/syncer/syncer.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" @@ -133,13 +134,35 @@ func (s *Syncer) run() { } case <-ticker.C: - if target == nil || !s.exitWhenSynced { + if target == nil { continue } - if block := s.backend.BlockChain().GetBlockByHash(target.Hash()); block != nil { - log.Info("Sync target reached", "number", block.NumberU64(), "hash", block.Hash()) - go s.stack.Close() // async since we need to close ourselves - return + + // Terminate the node if the target has been reached + if s.exitWhenSynced { + if block := s.backend.BlockChain().GetBlockByHash(target.Hash()); block != nil { + log.Info("Sync target reached", "number", block.NumberU64(), "hash", block.Hash()) + go s.stack.Close() // async since we need to close ourselves + return + } + } + + // Set the finalized and safe markers relative to the current head. + // The finalized marker is set two epochs behind the target, + // and the safe marker is set one epoch behind the target. + head := s.backend.BlockChain().CurrentHeader() + if head == nil { + continue + } + if header := s.backend.BlockChain().GetHeaderByNumber(head.Number.Uint64() - params.EpochLength*2); header != nil { + if final := s.backend.BlockChain().CurrentFinalBlock(); final == nil || final.Number.Cmp(header.Number) < 0 { + s.backend.BlockChain().SetFinalized(header) + } + } + if header := s.backend.BlockChain().GetHeaderByNumber(head.Number.Uint64() - params.EpochLength); header != nil { + if safe := s.backend.BlockChain().CurrentSafeBlock(); safe == nil || safe.Number.Cmp(header.Number) < 0 { + s.backend.BlockChain().SetSafe(header) + } } case <-s.closed: diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 8dac55cfa4..2c182e78f4 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -1110,7 +1110,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc } var ( msg = args.ToMessage(blockContext.BaseFee, true) - tx = args.ToTransaction(types.LegacyTxType) + tx = args.ToTransaction(types.DynamicFeeTxType) traceConfig *TraceConfig ) // Lower the basefee to 0 to avoid breaking EVM diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 4b1658b730..fdabe2925c 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -1927,6 +1927,353 @@ func TestStandardTraceBlockToFile(t *testing.T) { } } +func TestTraceBadBlock(t *testing.T) { + t.Parallel() + + var ( + accounts = newAccounts(2) + storageContract = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer = types.HomesteadSigner{} + txHashs = make([]common.Hash, 0, 2) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + storageContract: { + Nonce: 1, + Balance: big.NewInt(0), + Code: []byte{ + byte(vm.PUSH1), 0x2a, // push 42 + byte(vm.PUSH1), 0x00, // push slot 0 + byte(vm.SSTORE), // sstore(0, 42) + byte(vm.STOP), + }, + }, + }, + } + ) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // tx 0: plain transfer + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + + // tx 1: call storage contract (executes PUSH1, PUSH1, SSTORE, STOP) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &storageContract, + Value: big.NewInt(0), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + }) + defer backend.teardown() + + // Write the block as a bad block so parent state is available + block := backend.chain.GetBlockByNumber(1) + rawdb.WriteBadBlock(backend.chaindb, block) + + api := NewAPI(backend) + result, err := api.TraceBadBlock(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error, have %v", err) + } + if len(result) != 2 { + t.Fatalf("expected 2 tx traces, got %d", len(result)) + } + + // First tx: plain transfer + have, _ := json.Marshal(result) + var traces []struct { + TxHash common.Hash `json:"txHash"` + Result struct { + Gas uint64 `json:"gas"` + Failed bool `json:"failed"` + StructLogs []json.RawMessage `json:"structLogs"` + } `json:"result"` + } + if err := json.Unmarshal(have, &traces); err != nil { + t.Fatalf("failed to unmarshal traces: %v", err) + } + if traces[0].TxHash != txHashs[0] { + t.Errorf("tx 0: hash mismatch, have %v, want %v", traces[0].TxHash, txHashs[0]) + } + if traces[0].Result.Gas != params.TxGas { + t.Errorf("tx 0: gas mismatch, have %d, want %d", traces[0].Result.Gas, params.TxGas) + } + if len(traces[0].Result.StructLogs) != 0 { + t.Errorf("tx 0: expected empty structLogs for plain transfer, got %d entries", len(traces[0].Result.StructLogs)) + } + + // Second tx: contract call + if traces[1].TxHash != txHashs[1] { + t.Errorf("tx 1: hash mismatch, have %v, want %v", traces[1].TxHash, txHashs[1]) + } + if traces[1].Result.Failed { + t.Error("tx 1: expected success, got failed") + } + // Contract has 4 opcodes: PUSH1, PUSH1, SSTORE, STOP + if len(traces[1].Result.StructLogs) != 4 { + t.Errorf("tx 1: expected 4 structLog entries for contract call, got %d", len(traces[1].Result.StructLogs)) + } + + // Non-existent bad block + _, err = api.TraceBadBlock(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent bad block, have none") + } + wantErr := fmt.Sprintf("bad block %#x not found", common.Hash{42}) + if err.Error() != wantErr { + t.Errorf("error mismatch, want '%s', have '%v'", wantErr, err) + } +} + +func TestIntermediateRoots(t *testing.T) { + t.Parallel() + + // Initialize test accounts and a contract that writes to storage. + var ( + accounts = newAccounts(2) + storageContract = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer = types.HomesteadSigner{} + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + // Contract: SSTORE(CALLVALUE, CALLVALUE) + storageContract: { + Nonce: 1, + Balance: big.NewInt(0), + Code: []byte{ + byte(vm.CALLVALUE), + byte(vm.CALLVALUE), + byte(vm.SSTORE), + byte(vm.STOP), + }, + }, + }, + } + ) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // tx 0: plain transfer + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + + // tx 1: sstore(1, 1) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &storageContract, + Value: big.NewInt(1), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + + // tx 2: sstore(2, 2) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 2, + To: &storageContract, + Value: big.NewInt(2), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + }) + defer backend.teardown() + + api := NewAPI(backend) + block := backend.chain.GetBlockByNumber(1) + + // Should return one root per tx + roots, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error, have %v", err) + } + if len(roots) != 3 { + t.Fatalf("root count mismatch, have %d, want 3", len(roots)) + } + for i, root := range roots { + if root == (common.Hash{}) { + t.Errorf("root[%d] should not be zero", i) + } + } + if roots[0] == roots[1] { + t.Error("root[0] and root[1] should differ (transfer vs sstore)") + } + if roots[1] == roots[2] { + t.Error("root[1] and root[2] should differ (sstore to different slots)") + } + + // Intermediate roots of a bad block + rawdb.WriteBadBlock(backend.chaindb, block) + badRoots, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error for bad block fallback, have %v", err) + } + if !reflect.DeepEqual(roots, badRoots) { + t.Errorf("bad block roots mismatch, have %v, want %v", badRoots, roots) + } + + // Genesis block: should return error + genesisBlock := backend.chain.GetBlockByNumber(0) + _, err = api.IntermediateRoots(context.Background(), genesisBlock.Hash(), nil) + if err == nil || err.Error() != "genesis is not traceable" { + t.Fatalf("want 'genesis is not traceable' error, have %v", err) + } + + // Non-existent block: should return error + _, err = api.IntermediateRoots(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent block, have none") + } + wantErr := fmt.Sprintf("block %#x not found", common.Hash{42}) + if err.Error() != wantErr { + t.Errorf("error mismatch, want '%s', have '%v'", wantErr, err) + } +} + +func TestStandardTraceBadBlockToFile(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + + aa = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f43") + aaCode = []byte{byte(vm.PUSH1), 0x00, byte(vm.POP)} + + bb = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f44") + bbCode = []byte{byte(vm.PUSH2), 0x00, 0x01, byte(vm.POP)} + ) + + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + address: {Balance: funds}, + aa: { + Code: aaCode, + Nonce: 1, + Balance: big.NewInt(0), + }, + bb: { + Code: bbCode, + Nonce: 1, + Balance: big.NewInt(0), + }, + }, + } + txHashs := make([]common.Hash, 0, 2) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + b.SetCoinbase(common.Address{1}) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &aa, + Value: big.NewInt(0), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil, + }), types.HomesteadSigner{}, key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &bb, + Value: big.NewInt(1), + Gas: 100000, + GasPrice: b.BaseFee(), + Data: nil, + }), types.HomesteadSigner{}, key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + }) + defer backend.teardown() + + // Write the block as a bad block + block := backend.chain.GetBlockByNumber(1) + rawdb.WriteBadBlock(backend.chaindb, block) + + var testSuite = []struct { + config *StdTraceConfig + want []string + }{ + { + // All txs traced + config: nil, + want: []string{ + `{"pc":0,"op":96,"gas":"0x7148","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH1"} +{"pc":2,"op":80,"gas":"0x7145","gasCost":"0x2","memSize":0,"stack":["0x0"],"depth":1,"refund":0,"opName":"POP"} +{"pc":3,"op":0,"gas":"0x7143","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + `{"pc":0,"op":97,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH2"} +{"pc":3,"op":80,"gas":"0x13495","gasCost":"0x2","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"POP"} +{"pc":4,"op":0,"gas":"0x13493","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + }, + }, + { + // Specific tx traced + config: &StdTraceConfig{TxHash: txHashs[1]}, + want: []string{ + `{"pc":0,"op":97,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH2"} +{"pc":3,"op":80,"gas":"0x13495","gasCost":"0x2","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"POP"} +{"pc":4,"op":0,"gas":"0x13493","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + }, + }, + } + + api := NewAPI(backend) + for i, tc := range testSuite { + txTraces, err := api.StandardTraceBadBlockToFile(context.Background(), block.Hash(), tc.config) + if err != nil { + t.Fatalf("test %d: unexpected error %v", i, err) + } + if len(txTraces) != len(tc.want) { + t.Fatalf("test %d: file count mismatch, have %d, want %d", i, len(txTraces), len(tc.want)) + } + for j, traceFileName := range txTraces { + defer os.Remove(traceFileName) + traceReceived, err := os.ReadFile(traceFileName) + if err != nil { + t.Fatalf("test %d: could not read trace file: %v", i, err) + } + if tc.want[j] != string(traceReceived) { + t.Fatalf("test %d, trace %d: result mismatch\nhave:\n%s\nwant:\n%s", i, j, string(traceReceived), tc.want[j]) + } + } + } + + // Non-existent bad block + _, err := api.StandardTraceBadBlockToFile(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent bad block, have none") + } +} + func TestTraceBlockParity(t *testing.T) { t.Parallel() diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index e8a29e15bf..212319680c 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -43,6 +43,7 @@ type callLog struct { Address common.Address `json:"address"` Topics []common.Hash `json:"topics"` Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` Position hexutil.Uint `json:"position"` } @@ -331,7 +332,7 @@ func TestInternals(t *testing.T) { byte(vm.LOG0), }, tracer: mkTracer("callTracer", json.RawMessage(`{ "withLog": true }`)), - want: fmt.Sprintf(`{"from":"%s","gas":"0x13880","gasUsed":"0x5b9e","to":"0x00000000000000000000000000000000deadbeef","input":"0x","logs":[{"address":"0x00000000000000000000000000000000deadbeef","topics":[],"data":"0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","position":"0x0"},{"address":"0x0000000000000000000000000000000000001010","topics":["0x4dfe1bbbcf077ddc3e01291eea2d5c70c2b422b415d95645b9adcfd678cb1d63","0x0000000000000000000000000000000000000000000000000000000000001010","0x00000000000000000000000071562b71999873db5b286df957af199ec94617f7","0x0000000000000000000000000000000000000000000000000000000000000000"],"data":"0x0000000000000000000000000000000000000000000000000000000000005b9e0000000000000000000000000000000000000000000000000001c6bf5263400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001c6bf5262e4620000000000000000000000000000000000000000000000000000000000005b9e","position":"0x0"}],"value":"0x0","type":"CALL"}`, originHex), + want: fmt.Sprintf(`{"from":"%s","gas":"0x13880","gasUsed":"0x5b9e","to":"0x00000000000000000000000000000000deadbeef","input":"0x","logs":[{"address":"0x00000000000000000000000000000000deadbeef","topics":[],"data":"0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","index":"0x0","position":"0x0"},{"address":"0x0000000000000000000000000000000000001010","topics":["0x4dfe1bbbcf077ddc3e01291eea2d5c70c2b422b415d95645b9adcfd678cb1d63","0x0000000000000000000000000000000000000000000000000000000000001010","0x00000000000000000000000071562b71999873db5b286df957af199ec94617f7","0x0000000000000000000000000000000000000000000000000000000000000000"],"data":"0x0000000000000000000000000000000000000000000000000000000000005b9e0000000000000000000000000000000000000000000000000001c6bf5263400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001c6bf5262e4620000000000000000000000000000000000000000000000000000000000005b9e","index":"0x1","position":"0x0"}],"value":"0x0","type":"CALL"}`, originHex), }, { // Leads to OOM on the prestate tracer diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json index 1a678b7c80..279cc26c97 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/calldata.json @@ -94,6 +94,7 @@ "0xe1c52dc63b719ade82e8bea94cc41a0d5d28e4aaf536adb5e9cccc9ff8c1aeda" ], "data": "0x0000000000000000000000004f5777744b500616697cb655dcb02ee6cd51deb5be96016bb57376da7a6d296e0a405ee1501778227dfa604df0a81cb1ae018598", + "index": "0x1", "position": "0x0" }, { @@ -102,6 +103,7 @@ "0xacbdb084c721332ac59f9b8e392196c9eb0e4932862da8eb9beaf0dad4f550da" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000000", + "index": "0x2", "position": "0x0" } ], @@ -119,6 +121,7 @@ "0x000000000000000000000000200edd17f30485a8735878661960cd7a9a95733f" ], "data": "0x0000000000000000000000000000000000000000000000008ac7230489e80000000000000000000000000000000000000000000000000000b0767db57cd070aa000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000025af5ab0f2e870aa0000000000000000000000000000000000000000000000008ac7230489e80000", + "index": "0x0", "position": "0x0" }, { @@ -130,6 +133,7 @@ "0x000000000000000000000000f8b483dba2c3b7176a3da549ad41a48bb3121069" ], "data": "0x00000000000000000000000000000000000000000000000000119b54eb98e126000000000000000000000000000000000000000000000000b0983f1b83eec29000000000000000000000000000000000000000000000016969a0ba2c2d384d07000000000000000000000000000000000000000000000000b086a3c69855e16a00000000000000000000000000000000000000000000016969b2558118d12e2d", + "index": "0x3", "position": "0x1" } ], diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json index 3e4c548840..bdd596997d 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/delegatecall.json @@ -163,8 +163,8 @@ "to": "0xef3487d24a0702703e04a26cef479e313c8fc7ae", "input": "0x24d4e90a0000000000000000000000000000000000000000000000020000000000000000", "output": "0x000000000000000000000000000000000000000000000000b17217f7d1cf79ab", - "type": "DELEGATECALL", - "value": "0x0" + "value": "0x0", + "type": "DELEGATECALL" }, { "from": "0x8695e5e79dab06fbbb05f445316fa4edb0da30f0", @@ -173,8 +173,8 @@ "to": "0xef3487d24a0702703e04a26cef479e313c8fc7ae", "input": "0x872fb2b5000000000000000000000000000000000000000000000000c330b3f7006420b8", "output": "0x00000000000000000000000000000000000000000000000224bf7df2c80f0878", - "type": "DELEGATECALL", - "value": "0x0" + "value": "0x0", + "type": "DELEGATECALL" }, { "from": "0x8695e5e79dab06fbbb05f445316fa4edb0da30f0", @@ -183,8 +183,8 @@ "to": "0xef3487d24a0702703e04a26cef479e313c8fc7ae", "input": "0x872fb2b50000000000000000000000000000000000000000000000000000000000000000", "output": "0x00000000000000000000000000000000000000000000000100000016aee6e8ef", - "type": "DELEGATECALL", - "value": "0x0" + "value": "0x0", + "type": "DELEGATECALL" }, { "from": "0x8695e5e79dab06fbbb05f445316fa4edb0da30f0", @@ -193,8 +193,8 @@ "to": "0xef3487d24a0702703e04a26cef479e313c8fc7ae", "input": "0x24d4e90a00000000000000000000000000000000000000000000000324bf7e0976f5f167", "output": "0x0000000000000000000000000000000000000000000000012535c5e5f87ee0d2", - "type": "DELEGATECALL", - "value": "0x0" + "value": "0x0", + "type": "DELEGATECALL" }, { "from": "0x8695e5e79dab06fbbb05f445316fa4edb0da30f0", @@ -203,8 +203,8 @@ "to": "0xef3487d24a0702703e04a26cef479e313c8fc7ae", "input": "0x872fb2b5000000000000000000000000000000000000000000000000c330b3f7006420b8", "output": "0x00000000000000000000000000000000000000000000000224bf7df2c80f0878", - "type": "DELEGATECALL", - "value": "0x0" + "value": "0x0", + "type": "DELEGATECALL" }, { "from": "0x8695e5e79dab06fbbb05f445316fa4edb0da30f0", @@ -213,8 +213,8 @@ "to": "0xef3487d24a0702703e04a26cef479e313c8fc7ae", "input": "0x872fb2b500000000000000000000000000000000000000000000000237d37fe5d297a500", "output": "0x0000000000000000000000000000000000000000000000093088c407fcbbce38", - "type": "DELEGATECALL", - "value": "0x0" + "value": "0x0", + "type": "DELEGATECALL" }, { "from": "0x8695e5e79dab06fbbb05f445316fa4edb0da30f0", @@ -223,8 +223,8 @@ "to": "0xef3487d24a0702703e04a26cef479e313c8fc7ae", "input": "0x24d4e90a00000000000000000000000000000000000000000000000b554841fac4cad6b0", "output": "0x0000000000000000000000000000000000000000000000026d7fc130d6a74cbe", - "type": "DELEGATECALL", - "value": "0x0" + "value": "0x0", + "type": "DELEGATECALL" } ], "value": "0x0", @@ -256,6 +256,7 @@ "0x0000000000000000000000006ca7f214ab2ddbb9a8e1a1e2c8550e3164e9dba5" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x0", "position": "0x0" } ], @@ -278,6 +279,7 @@ "0x0000000000000000000000005aae5c59d642e5fd45b427df6ed478b49d55fefd" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x1", "position": "0x0" } ], @@ -308,6 +310,7 @@ "0x0000000000000000000000005aae5c59d642e5fd45b427df6ed478b49d55fefd" ], "data": "0x00000000000000000000000000000000000000000000000080d29fa5cccfadac", + "index": "0x2", "position": "0x0" } ], @@ -330,6 +333,7 @@ "0x000000000000000000000000950ca4a06c78934a148b7a3ff3ea8fc366f77a06" ], "data": "0x0000000000000000000000000000000000000000000000000041f50e27d56848", + "index": "0x3", "position": "0x0" } ], @@ -394,11 +398,12 @@ "0x0000000000000000000000003de712784baf97260455ae25fb74f574ec9c1add" ], "data": "0x000000000000000000000000000000000000000000000000de0b6b3a76400000", + "index": "0x4", "position": "0x0" } ], - "type": "DELEGATECALL", - "value": "0x0" + "value": "0x0", + "type": "DELEGATECALL" } ], "value": "0x0", @@ -415,6 +420,7 @@ "0x00000000000000000000000061c808d82a3ac53231750dadc13c777b59310bd9" ], "data": "0x000000000000000000000000000000000000000000000000000b9e774e8d580000000000000000000000000000000000000000000000000023c8352f33854625000000000000000000000000000000000000000000000090a7af5d475598456100000000000000000000000000000000000000000000000023bc96b7e4f7ee25000000000000000000000000000000000000000000000090a7bafbbea4259d61", + "index": "0x5", "position": "0x8" } ], diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/frontier_create_outofstorage.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/frontier_create_outofstorage.json index f384e2c2d3..0d225d1a18 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/frontier_create_outofstorage.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/frontier_create_outofstorage.json @@ -177,6 +177,7 @@ "0x1f28d876aff267c3302a63cd25ebcca53e6f60691049df42275b6d06ab455c67" ], "data": "0x55bc8431ce52389ac668a9b14a0943290cb7263732251186e960bc8b249b5f32000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000030d41000000000000000000000000000000000000000000000000000000000000000355524c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004c6a736f6e2868747470733a2f2f6170692e6b72616b656e2e636f6d2f302f7075626c69632f5469636b65723f706169723d455448584254292e726573756c742e58455448585842542e632e300000000000000000000000000000000000000000", + "index": "0x0", "position": "0x3" } ], @@ -194,6 +195,7 @@ "0x000000000000000000000000e48430c4e88a929bba0ee3dce284866a9937b609" ], "data": "0x000000000000000000000000000000000000000000000000006a8fb3d104cc0000000000000000000000000000000000000000000000000044f5ced08fe37cf70000000000000000000000000000000000000000000000026758774d51d8677a000000000000000000000000000000000000000000000000448b3f1cbedeb0f700000000000000000000000000000000000000000000000267c3070122dd337a", + "index": "0x1", "position": "0x6" } ], diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json index e821b0fead..ba9fa0fe9a 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multi_contracts.json @@ -356,6 +356,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x00000000000000000000000000000000000000000001819451f999d617dafa93", + "index": "0x0", "position": "0x0" } ], @@ -370,6 +371,7 @@ "0x69ca02dd4edd7bf0a4abb9ed3b7af3f14778db5d61921c7dc7cd545266326de2" ], "data": "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd00000000000000000000000000000000000000000001819451f999d617dafa93", + "index": "0x1", "position": "0x1" } ], @@ -492,6 +494,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x00000000000000000000000000000000000000000001819451f999d617dafa76", + "index": "0x2", "position": "0x0" } ], @@ -506,6 +509,7 @@ "0x69ca02dd4edd7bf0a4abb9ed3b7af3f14778db5d61921c7dc7cd545266326de2" ], "data": "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd00000000000000000000000000000000000000000001819451f999d617dafa76", + "index": "0x3", "position": "0x1" } ], @@ -695,6 +699,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000181a7ae53ea2f0bef8ccd", + "index": "0x5", "position": "0x0" } ], @@ -721,6 +726,7 @@ "0x000000000000000000000000ad3ecf23c0c8983b07163708be6d763b5f056193" ], "data": "0x000000000000000000000000000000000000000000000004563918244f400000000000000000000000000000000000000000000000000004563918244f40000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004563918244f400000", + "index": "0x7", "position": "0x0" } ], @@ -891,6 +897,7 @@ "0x0000000000000000000000004fd27b205895e698fa350f7ea57cec8a21927fcd" ], "data": "0x0000000000000000000000000000000000000000000181a7ae53ea2f0bef8ccc", + "index": "0xa", "position": "0x0" } ], @@ -908,6 +915,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000000022b1c8c12279fffff0000000000000000000000000000000000000000000000022b1c8c1227a000010000000000000000000000000000000000000000000000022b1c8c12279fffff0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000004563918244f3ffffe", + "index": "0x9", "position": "0x0" } ], @@ -923,6 +931,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000000022b1c8c12279fffff", + "index": "0xb", "position": "0x1" } ], @@ -944,6 +953,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000000022b1c8c12279fffff000000000000000000000000000000000000000000000004563918244f40000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022b1c8c1227a000010000000000000000000000000000000000000000000000022b1c8c12279fffff", + "index": "0x8", "position": "0x0" } ], @@ -959,6 +969,7 @@ "0x0000000000000000000000006e715ab4f598eacf0016b9b35ef33e4141844ccc" ], "data": "0x0000000000000000000000000000000000000000000000022b1c8c12279fffff", + "index": "0xc", "position": "0x1" } ], @@ -985,6 +996,7 @@ "0x0000000000000000000000006dbfc63479ffc031f23e94dc91befa38bec2c25f" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000001", + "index": "0xd", "position": "0x0" } ], @@ -999,6 +1011,7 @@ "0x07cf7e805770612a8b2ee8e0bcbba8aa908df5f85fbc4f9e2ef384cf75315038" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000000", + "index": "0x4", "position": "0x6" }, { @@ -1007,6 +1020,7 @@ "0x7027eecbd2a688fc1fa281702b311ed7168571514adfd17014a55d828cb43382" ], "data": "0x000000000000000000000000000000000000000000000004563918244f400000", + "index": "0x6", "position": "0x8" } ], @@ -1084,6 +1098,7 @@ "0x0000000000000000000000006dbfc63479ffc031f23e94dc91befa38bec2c25f" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000063", + "index": "0xe", "position": "0x0" } ], @@ -1212,6 +1227,7 @@ "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e96526" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000064", + "index": "0x11", "position": "0x0" } ], @@ -1226,6 +1242,7 @@ "0x4b0bc4f25f8d0b92d2e12b686ba96cd75e4e69325e6cf7b1f3119d14eaf2cbdf" ], "data": "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e96526", + "index": "0xf", "position": "0x6" }, { @@ -1234,6 +1251,7 @@ "0xf340c079d598119636d42046c6a2d2faf7a68c04aecee516f0e0b8a9e79b8666" ], "data": "0x000000000000000000000000da4a4626d3e16e094de3225a751aab7128e9652600000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000000", + "index": "0x10", "position": "0x9" } ], @@ -1284,6 +1302,7 @@ "0x0000000000000000000000007498bb5749c9801f1f7e490baf5f966dbfe4e97b" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000001", + "index": "0x12", "position": "0x0" } ], @@ -1378,6 +1397,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000001" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x13", "position": "0x2" } ], @@ -1472,6 +1492,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000002" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x14", "position": "0x2" } ], @@ -1566,6 +1587,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000003" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x15", "position": "0x2" } ], @@ -1660,6 +1682,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000004" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x16", "position": "0x2" } ], @@ -1754,6 +1777,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000005" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x17", "position": "0x2" } ], @@ -1848,6 +1872,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000006" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x18", "position": "0x2" } ], @@ -1942,6 +1967,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000007" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x19", "position": "0x2" } ], @@ -2036,6 +2062,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000008" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x1a", "position": "0x2" } ], @@ -2130,6 +2157,7 @@ "0x0000000000000000000000000000000000000000000000000000000000000009" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x1b", "position": "0x2" } ], @@ -2224,6 +2252,7 @@ "0x000000000000000000000000000000000000000000000000000000000000000a" ], "data": "0x000000000000000000000000be3ae5cb97c253dda67181c6e34e43f5c275e08b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000", + "index": "0x1c", "position": "0x2" } ], @@ -2277,6 +2306,7 @@ "0x0000000000000000000000007ccbc69292c7a6d7b538c91f3b283de97906cf30" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1d", "position": "0x0" } ], @@ -2299,6 +2329,7 @@ "0x0000000000000000000000001b9ec8ba24630b75a7a958153ffff56dd6d4b6a2" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1f", "position": "0x0" } ], @@ -2321,6 +2352,7 @@ "0x000000000000000000000000c3a2c744ad1f5253c736875b93bacce5b01b060b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x21", "position": "0x0" } ], @@ -2335,6 +2367,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x1e", "position": "0x2" }, { @@ -2343,6 +2376,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x20", "position": "0x3" }, { @@ -2351,6 +2385,7 @@ "0xc6d8c0af6d21f291e7c359603aa97e0ed500f04db6e983b9fce75a91c6b8da6b" ], "data": "0x00000000000000000000000000000000000000000001010d8bfbbbe40fe7518c", + "index": "0x22", "position": "0x4" } ], @@ -2368,6 +2403,7 @@ "0x000000000000000000000000ea674fdde714fd979de3edf0f56aa9716b898ec8" ], "data": "0x00000000000000000000000000000000000000000000000000e25384e63aa60000000000000000000000000000000000000000000000000167d285b38143c04f00000000000000000000000000000000000000000000004ab3566739e7b2437100000000000000000000000000000000000000000000000166f0322e9b091a4f00000000000000000000000000000000000000000000004ab438babecdece971", + "index": "0x23", "position": "0x23" } ], diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json index 63081f7161..e450023ee5 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/multilogs.json @@ -180,6 +180,7 @@ "0x000000000000000000000000350e0ffc780a6a75b44cc52e1ff9092870668945" ], "data": "0x00000000000000000000000000000000000000000000000003782dace9d9000000000000000000000000000000000000000000000000000067f963ad0ca28400000000000000000000000000000000000000000000000000e37111b7c79406c00000000000000000000000000000000000000000000000006481360022c98400000000000000000000000000000000000000000000000000e6e93f64b16d06c0", + "index": "0x0", "position": "0x0" }, { @@ -188,6 +189,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1", "position": "0x0" }, { @@ -196,6 +198,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2", "position": "0x0" }, { @@ -204,6 +207,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffebebeb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x3", "position": "0x0" }, { @@ -212,6 +216,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8888880000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x4", "position": "0x0" }, { @@ -220,6 +225,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb3b3b30000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x5", "position": "0x0" }, { @@ -228,6 +234,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffcfcfc0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x6", "position": "0x0" }, { @@ -236,6 +243,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x7", "position": "0x0" }, { @@ -244,6 +252,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe3e3e30000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x8", "position": "0x0" }, { @@ -252,6 +261,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff3e3e3e0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x9", "position": "0x0" }, { @@ -260,6 +270,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xa", "position": "0x0" }, { @@ -268,6 +279,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xb", "position": "0x0" }, { @@ -276,6 +288,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xc", "position": "0x0" }, { @@ -284,6 +297,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdbdbdb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xd", "position": "0x0" }, { @@ -292,6 +306,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xe", "position": "0x0" }, { @@ -300,6 +315,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4f4f40000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0xf", "position": "0x0" }, { @@ -308,6 +324,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x10", "position": "0x0" }, { @@ -316,6 +333,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x11", "position": "0x0" }, { @@ -324,6 +342,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff00000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x12", "position": "0x0" }, { @@ -332,6 +351,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034100000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbfbfb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x13", "position": "0x0" }, { @@ -340,6 +360,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002ff000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x14", "position": "0x0" }, { @@ -348,6 +369,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000341000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x15", "position": "0x0" }, { @@ -356,6 +378,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb0b0b00000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x16", "position": "0x0" }, { @@ -364,6 +387,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x17", "position": "0x0" }, { @@ -372,6 +396,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa0a0a00000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x18", "position": "0x0" }, { @@ -380,6 +405,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5b5b5b0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x19", "position": "0x0" }, { @@ -388,6 +414,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbababa0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1a", "position": "0x0" }, { @@ -396,6 +423,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000390000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1b", "position": "0x0" }, { @@ -404,6 +432,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffeaeaea0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1c", "position": "0x0" }, { @@ -412,6 +441,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000360000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa9a9a90000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1d", "position": "0x0" }, { @@ -420,6 +450,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb9b9b90000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1e", "position": "0x0" }, { @@ -428,6 +459,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbfbfb0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x1f", "position": "0x0" }, { @@ -436,6 +468,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x20", "position": "0x0" }, { @@ -444,6 +477,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000350000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffefefe0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x21", "position": "0x0" }, { @@ -452,6 +486,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffbababa0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x22", "position": "0x0" }, { @@ -460,6 +495,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff6363630000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x23", "position": "0x0" }, { @@ -468,6 +504,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x24", "position": "0x0" }, { @@ -476,6 +513,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000340000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9f9f90000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x25", "position": "0x0" }, { @@ -484,6 +522,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffeaeaea0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x26", "position": "0x0" }, { @@ -492,6 +531,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003c0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff9c9c9c0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x27", "position": "0x0" }, { @@ -500,6 +540,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8f8f80000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x28", "position": "0x0" }, { @@ -508,6 +549,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000330000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x29", "position": "0x0" }, { @@ -516,6 +558,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2a", "position": "0x0" }, { @@ -524,6 +567,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2b", "position": "0x0" }, { @@ -532,6 +576,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe00000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2c", "position": "0x0" }, { @@ -540,6 +585,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034200000000000000000000000000000000000000000000000000000000000000320000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffcfcfc0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2d", "position": "0x0" }, { @@ -548,6 +594,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fe000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffdfdfd0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2e", "position": "0x0" }, { @@ -556,6 +603,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x0000000000000000000000000000000000000000000000000000000000000342000000000000000000000000000000000000000000000000000000000000003e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x2f", "position": "0x0" }, { @@ -564,6 +612,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fd00000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4d4e530000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x30", "position": "0x0" }, { @@ -572,6 +621,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x000000000000000000000000000000000000000000000000000000000000034300000000000000000000000000000000000000000000000000000000000000380000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x31", "position": "0x0" }, { @@ -580,6 +630,7 @@ "0xcacb62d8acea4678658eb5dc4aaa889b34d893b967c96a5f8c066e6549fa3f42" ], "data": "0x00000000000000000000000000000000000000000000000000000000000002fd00000000000000000000000000000000000000000000000000000000000000370000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003fcb0342353c541e210013aaddc2e740b9a33d08ffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4f494b0000000000000000000000000000000000000000000000000011c37937e08000", + "index": "0x32", "position": "0x0" }, { @@ -591,6 +642,7 @@ "0x0000000000000000000000002a65aca4d5fc5b5c859090a6c34d164135398226" ], "data": "0x00000000000000000000000000000000000000000000000001b3dd214f1b8c000000000000000000000000000000000000000000000000006a0e4be198f18400000000000000000000000000000000000000000000000044dc051cccdfd2e132000000000000000000000000000000000000000000000000685a6ec049d5f800000000000000000000000000000000000000000000000044ddb8f9ee2eee6d32", + "index": "0x33", "position": "0x0" } ], diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json index a121621923..73bbd6f2e1 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/notopic.json @@ -268,6 +268,7 @@ "0x00000000000000000000000088e1315687aec48a72786c6b3b3f075208b62713" ], "data": "0x00000000000000000000000000000000000000000000000000179d63013c5654000000000000000000000000000000000000000000000006ad16c019b81a414100000000000000000000000000000000000000000000000024b9f2c5dc266dc6000000000000000000000000000000000000000000000006acff22b6b6ddeaed00000000000000000000000000000000000000000000000024d19028dd62c41a", + "index": "0x2", "position": "0x0" }, { @@ -276,6 +277,7 @@ "0xaf30e4d66b2f1f23e63ef4591058a897f67e6867233e33ca3508b982dcc4129b" ], "data": "0x00000000000000000000000050739060a2c32dc076e507ae1a893aab28ecfe68d1b13c1538a940417bf0e73b2498634436753c854c7fb971224d971bd2ae3e8800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000120000000000000000000000000000000000000000000000000000000000000016000000000000000000000000000000000000000000000000000000000000001e000000000000000000000000000000000000000000000000000000000000249f011000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000355524c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000436a736f6e2868747470733a2f2f6170692e72616e646f6d2e6f72672f6a736f6e2d7270632f312f696e766f6b65292e726573756c742e72616e646f6d2e646174612e300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000012c4244584a68725670424a35336f3243786c4a526c51745a4a4b5a714c5974354951652b37335944533448744e6a5335486f64624942337476666f773755717579416b303835566b4c6e4c3945704b67777157517a375a4c64477673516c526432734b78496f6c4e673944626e6650737047714c684c62625953566e4e38437776736a7041586353536f33632b34634e774339307946346f4e69626b764433797461706f5a37676f5453796f5559546677536a6e773374692b484a5648374e332b633069774f43715a6a4464734751556358336d33532f494857624f4f5151356f734f344c626a33476730783155644e7466557a5943465937396e7a596757495145464375524249306e364e42764251573732372b4f73445259304a2f392f676a74387563696248576963303d0000000000000000000000000000000000000000", + "index": "0x3", "position": "0x4" } ], @@ -293,12 +295,14 @@ "0x00000000000000000000000050739060a2c32dc076e507ae1a893aab28ecfe68" ], "data": "0x0000000000000000000000000000000000000000000000000429d069189e00000000000000000000000000000000000000000000000000000fa6f01b59360e70000000000000000000000000000000000000000000000006a8ecefb09f7c41410000000000000000000000000000000000000000000000000b7d1fb240980e70000000000000000000000000000000000000000000000006ad16c019b81a4141", + "index": "0x0", "position": "0x0" }, { "address": "0x50739060a2c32dc076e507ae1a893aab28ecfe68", "topics": [], "data": "0x62616e6b726f6c6c5f6d69736d61746368", + "index": "0x1", "position": "0x2" }, { @@ -310,6 +314,7 @@ "0x00000000000000000000000061c808d82a3ac53231750dadc13c777b59310bd9" ], "data": "0x000000000000000000000000000000000000000000000000000b30d0148e06000000000000000000000000000000000000000000000000000fb5dbfc0d448e7000000000000000000000000000000000000000000000012f621ea72fef44f8480000000000000000000000000000000000000000000000000faaab2bf8b6887000000000000000000000000000000000000000000000012f6229d80003d2fe48", + "index": "0x4", "position": "0x9" } ], diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json index b938fb381e..604245c1ab 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/simple.json @@ -74,6 +74,7 @@ "0x000000000000000000000000dbf03b407c01e7cd3cbea99509d93f8dddc8c6fb" ], "data": "0x0000000000000000000000000000000000000000000000000000000000989680", + "index": "0x0", "position": "0x0" }, { @@ -85,6 +86,7 @@ "0x000000000000000000000000e2fe6b13287f28e193333fdfe7fedf2f6df6124a" ], "data": "0x00000000000000000000000000000000000000000000000000090886d609c4000000000000000000000000000000000000000000000000014203bee2ea6fbe8c0000000000000000000000000000000000000000000002717a9c870a286f435000000000000000000000000000000000000000000000000141fab65c1465fa8c0000000000000000000000000000000000000000000002717aa58f90fe790750", + "index": "0x1", "position": "0x0" } ], diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/tx_partial_failed.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/tx_partial_failed.json index e7c0e6782c..587b1cd4b1 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/tx_partial_failed.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/tx_partial_failed.json @@ -97,6 +97,7 @@ "0x92ca3a80853e6663fa31fa10b99225f18d4902939b4c53a9caae9043f6efd004" ], "data": "0x00000000000000000000000001115b41bd2731353dd3e6abf44818fdc035aaf10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000bb9bc244d798123fde783fcc1c72d3bb8c1894130000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000008861393035396362623030303030303030303030303030303030303030303030303930643363313831326465323636396266383037626437373538636562316533343937616337653430303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303031633662663532363334303030", + "index": "0x0", "position": "0x0" }, { @@ -108,6 +109,7 @@ "0x00000000000000000000000061c808d82a3ac53231750dadc13c777b59310bd9" ], "data": "0x000000000000000000000000000000000000000000000000000bcd242cec0800000000000000000000000000000000000000000000000000016d99e16e80900000000000000000000000000000000000000000000000006a636960e34bd696f40000000000000000000000000000000000000000000000000161ccbd4194880000000000000000000000000000000000000000000000006a63752e0778c29ef4", + "index": "0x1", "position": "0x1" } ], diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/with_onlyTopCall.json b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/with_onlyTopCall.json index 33e8406a59..628442a3a4 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/with_onlyTopCall.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer_withLog/with_onlyTopCall.json @@ -91,6 +91,7 @@ "0x000000000000000000000000200edd17f30485a8735878661960cd7a9a95733f" ], "data": "0x0000000000000000000000000000000000000000000000008ac7230489e80000000000000000000000000000000000000000000000000000b0767db57cd070aa000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000025af5ab0f2e870aa0000000000000000000000000000000000000000000000008ac7230489e80000", + "index": "0x0", "position": "0x0" }, { @@ -102,6 +103,7 @@ "0x000000000000000000000000f8b483dba2c3b7176a3da549ad41a48bb3121069" ], "data": "0x00000000000000000000000000000000000000000000000000119b54eb98e126000000000000000000000000000000000000000000000000b0983f1b83eec29000000000000000000000000000000000000000000000016969a0ba2c2d384d07000000000000000000000000000000000000000000000000b086a3c69855e16a00000000000000000000000000000000000000000000016969b2558118d12e2d", + "index": "0x3", "position": "0x0" } ], diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 167dea18fb..901b04611e 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -47,7 +47,7 @@ type vmContext struct { } func testCtx() *vmContext { - return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1), BaseFee: big.NewInt(0)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} + return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1), BaseFee: big.NewInt(0)}, txCtx: vm.TxContext{GasPrice: uint256.NewInt(100000)}} } func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { @@ -65,7 +65,7 @@ func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCo contract.Code = contractCode } - tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit, GasPrice: vmctx.txCtx.GasPrice}), contract.Caller()) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit, GasPrice: vmctx.txCtx.GasPrice.ToBig()}), contract.Caller()) tracer.OnEnter(0, byte(vm.CALL), contract.Caller(), contract.Address(), []byte{}, startGas, value.ToBig()) ret, err := evm.Run(contract, []byte{}, false) tracer.OnExit(0, ret, startGas-contract.Gas, err, true) @@ -191,7 +191,7 @@ func TestHaltBetweenSteps(t *testing.T) { Contract: vm.NewContract(common.Address{}, common.Address{}, uint256.NewInt(0), 0, nil), } evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(1)}) + evm.SetTxContext(vm.TxContext{GasPrice: uint256.NewInt(1)}) tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 0, big.NewInt(0)) tracer.OnOpcode(0, 0, 0, 0, scope, nil, 0, nil) @@ -215,7 +215,7 @@ func TestNoStepExec(t *testing.T) { t.Fatal(err) } evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(100)}) + evm.SetTxContext(vm.TxContext{GasPrice: uint256.NewInt(100)}) tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 1000, big.NewInt(0)) tracer.OnExit(0, nil, 0, nil, false) @@ -246,7 +246,7 @@ func TestIsPrecompile(t *testing.T) { chaincfg.ByzantiumBlock = big.NewInt(100) chaincfg.IstanbulBlock = big.NewInt(200) chaincfg.BerlinBlock = big.NewInt(300) - txCtx := vm.TxContext{GasPrice: big.NewInt(100000)} + txCtx := vm.TxContext{GasPrice: uint256.NewInt(100000)} tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil, chaincfg) if err != nil { t.Fatal(err) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index 598e501c24..02a15dca03 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -96,8 +96,8 @@ func (s *StructLog) ErrorString() string { return "" } -// WriteTo writes the human-readable log data into the supplied writer. -func (s *StructLog) WriteTo(writer io.Writer) { +// Write writes the human-readable log data into the supplied writer. +func (s *StructLog) Write(writer io.Writer) { fmt.Fprintf(writer, "%-16spc=%08d gas=%v cost=%v", s.Op, s.Pc, s.Gas, s.GasCost) if s.Err != nil { fmt.Fprintf(writer, " ERROR: %v", s.Err) @@ -327,7 +327,7 @@ func (l *StructLogger) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope l.logs = append(l.logs, entry) return } - log.WriteTo(l.writer) + log.Write(l.writer) } // OnExit is called a call frame finishes processing. @@ -413,7 +413,7 @@ func (l *StructLogger) Output() []byte { return l.output } // @deprecated func WriteTrace(writer io.Writer, logs []StructLog) { for _, log := range logs { - log.WriteTo(writer) + log.Write(writer) } } diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index 54ca7a3864..8dbe341902 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -42,6 +42,7 @@ type callLog struct { Address common.Address `json:"address"` Topics []common.Hash `json:"topics"` Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` // Position of the log relative to subcalls within the same trace // See https://github.com/ethereum/go-ethereum/pull/28389 for details Position hexutil.Uint `json:"position"` @@ -255,6 +256,7 @@ func (t *callTracer) OnLog(log *types.Log) { Address: log.Address, Topics: log.Topics, Data: log.Data, + Index: hexutil.Uint(log.Index), Position: hexutil.Uint(len(t.callstack[len(t.callstack)-1].Calls)), } t.callstack[len(t.callstack)-1].Logs = append(t.callstack[len(t.callstack)-1].Logs, l) diff --git a/eth/tracers/native/erc7562.go b/eth/tracers/native/erc7562.go index 05ebe2350b..463495a420 100644 --- a/eth/tracers/native/erc7562.go +++ b/eth/tracers/native/erc7562.go @@ -514,7 +514,7 @@ func defaultIgnoredOpcodes() []hexutil.Uint64 { ignored := make([]hexutil.Uint64, 0, 64) // Allow all PUSHx, DUPx and SWAPx opcodes as they have sequential codes - for op := vm.PUSH0; op < vm.SWAP16; op++ { + for op := vm.PUSH0; op <= vm.SWAP16; op++ { ignored = append(ignored, hexutil.Uint64(op)) } diff --git a/eth/tracers/native/mux.go b/eth/tracers/native/mux.go index 3dcdf6c5dc..420773881f 100644 --- a/eth/tracers/native/mux.go +++ b/eth/tracers/native/mux.go @@ -159,6 +159,14 @@ func (t *muxTracer) OnCodeChange(a common.Address, prevCodeHash common.Hash, pre } } +func (t *muxTracer) OnCodeChangeV2(a common.Address, prevCodeHash common.Hash, prev []byte, codeHash common.Hash, code []byte, reason tracing.CodeChangeReason) { + for _, t := range t.tracers { + if t.OnCodeChangeV2 != nil { + t.OnCodeChangeV2(a, prevCodeHash, prev, codeHash, code, reason) + } + } +} + func (t *muxTracer) OnStorageChange(a common.Address, k, prev, new common.Hash) { for _, t := range t.tracers { if t.OnStorageChange != nil { diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 4dff547bd9..eb2ed1d8a2 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -66,7 +66,7 @@ type prestateTracer struct { pre stateMap post stateMap to common.Address - config prestateTracerConfig + config PrestateTracerConfig chainConfig *params.ChainConfig interrupt atomic.Bool // Atomic flag to signal execution interruption reason error // Textual reason for the interruption @@ -74,7 +74,7 @@ type prestateTracer struct { deleted map[common.Address]bool } -type prestateTracerConfig struct { +type PrestateTracerConfig struct { DiffMode bool `json:"diffMode"` // If true, this tracer will return state modifications DisableCode bool `json:"disableCode"` // If true, this tracer will not return the contract code DisableStorage bool `json:"disableStorage"` // If true, this tracer will not return the contract storage @@ -82,7 +82,7 @@ type prestateTracerConfig struct { } func newPrestateTracer(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params.ChainConfig) (*tracers.Tracer, error) { - var config prestateTracerConfig + var config PrestateTracerConfig if err := json.Unmarshal(cfg, &config); err != nil { return nil, err } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 7465984ce0..70260559d6 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -132,7 +132,7 @@ func (ec *Client) BlockNumber(ctx context.Context) (uint64, error) { // BlockReceipts returns the receipts of a given block number or hash. func (ec *Client) BlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]*types.Receipt, error) { var r []*types.Receipt - err := ec.c.CallContext(ctx, &r, "eth_getBlockReceipts", blockNrOrHash.String()) + err := ec.c.CallContext(ctx, &r, "eth_getBlockReceipts", blockNrOrHash) if err == nil && r == nil { return nil, ethereum.NotFound } @@ -534,9 +534,12 @@ func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuer } func toFilterArg(q ethereum.FilterQuery) (interface{}, error) { - arg := map[string]interface{}{ - "address": q.Addresses, - "topics": q.Topics, + arg := map[string]interface{}{} + if q.Addresses != nil { + arg["address"] = q.Addresses + } + if q.Topics != nil { + arg["topics"] = q.Topics } if q.BlockHash != nil { arg["blockHash"] = *q.BlockHash @@ -785,9 +788,11 @@ func (ec *Client) SendRawTransactionSync( rawTx []byte, timeout *time.Duration, ) (*types.Receipt, error) { - var ms *hexutil.Uint64 + var ms *uint64 if timeout != nil { - if d := hexutil.Uint64(timeout.Milliseconds()); d > 0 { + msInt := timeout.Milliseconds() + if msInt > 0 { + d := uint64(msInt) ms = &d } } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index f8c760646a..c96c78e648 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -705,6 +705,49 @@ func testTransactionSender(t *testing.T, client *rpc.Client) { } } +func TestBlockReceiptsPreservesCanonicalFlag(t *testing.T) { + srv := rpc.NewServer("", 0, 0) + service := &blockReceiptsTestService{calls: make(chan rpc.BlockNumberOrHash, 1)} + if err := srv.RegisterName("eth", service); err != nil { + t.Fatalf("failed to register service: %v", err) + } + defer srv.Stop() + + client := rpc.DialInProc(srv) + defer client.Close() + + ec := ethclient.NewClient(client) + defer ec.Close() + + hash := common.HexToHash("0x01") + ref := rpc.BlockNumberOrHashWithHash(hash, true) + + if _, err := ec.BlockReceipts(context.Background(), ref); err != nil { + t.Fatalf("BlockReceipts returned error: %v", err) + } + + select { + case call := <-service.calls: + if call.BlockHash == nil || *call.BlockHash != hash { + t.Fatalf("unexpected block hash: got %v, want %v", call.BlockHash, hash) + } + if !call.RequireCanonical { + t.Fatalf("requireCanonical flag was lost: %+v", call) + } + default: + t.Fatal("service was not called") + } +} + +type blockReceiptsTestService struct { + calls chan rpc.BlockNumberOrHash +} + +func (s *blockReceiptsTestService) GetBlockReceipts(ctx context.Context, block rpc.BlockNumberOrHash) ([]*types.Receipt, error) { + s.calls <- block + return []*types.Receipt{}, nil +} + func newCanceledContext() context.Context { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/ethclient/gethclient/gen_callframe_json.go b/ethclient/gethclient/gen_callframe_json.go new file mode 100644 index 0000000000..48df2790cc --- /dev/null +++ b/ethclient/gethclient/gen_callframe_json.go @@ -0,0 +1,104 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package gethclient + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*callFrameMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c CallFrame) MarshalJSON() ([]byte, error) { + type CallFrame0 struct { + Type string `json:"type"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input hexutil.Bytes `json:"input"` + Output hexutil.Bytes `json:"output,omitempty"` + Error string `json:"error,omitempty"` + RevertReason string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + } + var enc CallFrame0 + enc.Type = c.Type + enc.From = c.From + enc.Gas = hexutil.Uint64(c.Gas) + enc.GasUsed = hexutil.Uint64(c.GasUsed) + enc.To = c.To + enc.Input = c.Input + enc.Output = c.Output + enc.Error = c.Error + enc.RevertReason = c.RevertReason + enc.Calls = c.Calls + enc.Logs = c.Logs + enc.Value = (*hexutil.Big)(c.Value) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *CallFrame) UnmarshalJSON(input []byte) error { + type CallFrame0 struct { + Type *string `json:"type"` + From *common.Address `json:"from"` + Gas *hexutil.Uint64 `json:"gas"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input *hexutil.Bytes `json:"input"` + Output *hexutil.Bytes `json:"output,omitempty"` + Error *string `json:"error,omitempty"` + RevertReason *string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + } + var dec CallFrame0 + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Type != nil { + c.Type = *dec.Type + } + if dec.From != nil { + c.From = *dec.From + } + if dec.Gas != nil { + c.Gas = uint64(*dec.Gas) + } + if dec.GasUsed != nil { + c.GasUsed = uint64(*dec.GasUsed) + } + if dec.To != nil { + c.To = dec.To + } + if dec.Input != nil { + c.Input = *dec.Input + } + if dec.Output != nil { + c.Output = *dec.Output + } + if dec.Error != nil { + c.Error = *dec.Error + } + if dec.RevertReason != nil { + c.RevertReason = *dec.RevertReason + } + if dec.Calls != nil { + c.Calls = dec.Calls + } + if dec.Logs != nil { + c.Logs = dec.Logs + } + if dec.Value != nil { + c.Value = (*big.Int)(dec.Value) + } + return nil +} diff --git a/ethclient/gethclient/gen_calllog_json.go b/ethclient/gethclient/gen_calllog_json.go new file mode 100644 index 0000000000..50e25d4bb3 --- /dev/null +++ b/ethclient/gethclient/gen_calllog_json.go @@ -0,0 +1,61 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package gethclient + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*callLogMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c CallLog) MarshalJSON() ([]byte, error) { + type CallLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` + Position hexutil.Uint `json:"position"` + } + var enc CallLog + enc.Address = c.Address + enc.Topics = c.Topics + enc.Data = c.Data + enc.Index = hexutil.Uint(c.Index) + enc.Position = hexutil.Uint(c.Position) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *CallLog) UnmarshalJSON(input []byte) error { + type CallLog struct { + Address *common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data *hexutil.Bytes `json:"data"` + Index *hexutil.Uint `json:"index"` + Position *hexutil.Uint `json:"position"` + } + var dec CallLog + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Address != nil { + c.Address = *dec.Address + } + if dec.Topics != nil { + c.Topics = dec.Topics + } + if dec.Data != nil { + c.Data = *dec.Data + } + if dec.Index != nil { + c.Index = uint(*dec.Index) + } + if dec.Position != nil { + c.Position = uint(*dec.Position) + } + return nil +} diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 74131d84a8..2717e0095c 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -19,10 +19,12 @@ package gethclient import ( "context" + "encoding/json" "fmt" "math/big" "runtime" "runtime/debug" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -106,7 +108,10 @@ func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []s var res accountResult err := ec.c.CallContext(ctx, &res, "eth_getProof", account, keys, toBlockNumArg(blockNumber)) - // Turn hexutils back to normal datatypes + if err != nil { + return nil, err + } + // Turn hexutils back to normal data types storageResults := make([]StorageResult, 0, len(res.StorageProof)) for _, st := range res.StorageProof { storageResults = append(storageResults, StorageResult{ @@ -125,8 +130,7 @@ func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []s StorageHash: res.StorageHash, StorageProof: storageResults, } - - return &result, err + return &result, nil } // CallContract executes a message call transaction, which is directly executed in the VM @@ -234,6 +238,124 @@ func (ec *Client) TraceBlock(ctx context.Context, hash common.Hash, config *trac return result, nil } +// CallTracerConfig configures the call tracer for +// TraceTransactionWithCallTracer and TraceCallWithCallTracer. +type CallTracerConfig struct { + // OnlyTopCall, when true, limits tracing to the main (top-level) call only. + OnlyTopCall bool + // WithLog, when true, includes log emissions in the trace output. + WithLog bool + // Timeout is the maximum duration the tracer may run. + // Zero means the server default (5s). + Timeout time.Duration +} + +//go:generate go run github.com/fjl/gencodec -type CallLog -field-override callLogMarshaling -out gen_calllog_json.go + +// CallLog represents a log emitted during a traced call. +type CallLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data []byte `json:"data"` + Index uint `json:"index"` + Position uint `json:"position"` +} + +type callLogMarshaling struct { + Data hexutil.Bytes + Index hexutil.Uint + Position hexutil.Uint +} + +//go:generate go run github.com/fjl/gencodec -type CallFrame -field-override callFrameMarshaling -out gen_callframe_json.go + +// CallFrame contains the result of a call tracer run. +type CallFrame struct { + Type string `json:"type"` + From common.Address `json:"from"` + Gas uint64 `json:"gas"` + GasUsed uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input []byte `json:"input"` + Output []byte `json:"output,omitempty"` + Error string `json:"error,omitempty"` + RevertReason string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *big.Int `json:"value,omitempty"` +} + +type callFrameMarshaling struct { + Gas hexutil.Uint64 + GasUsed hexutil.Uint64 + Input hexutil.Bytes + Output hexutil.Bytes + Value *hexutil.Big +} + +// TraceTransactionWithCallTracer traces a transaction with the call tracer +// and returns a typed CallFrame. If config is nil, defaults are used. +func (ec *Client) TraceTransactionWithCallTracer(ctx context.Context, txHash common.Hash, config *CallTracerConfig) (*CallFrame, error) { + var result CallFrame + err := ec.c.CallContext(ctx, &result, "debug_traceTransaction", txHash, callTracerConfig(config)) + if err != nil { + return nil, err + } + return &result, nil +} + +// TraceCallWithCallTracer executes a call with the call tracer and returns +// a typed CallFrame. blockNrOrHash selects the block context for the call. +// overrides specifies state overrides (nil for none), blockOverrides specifies +// block header overrides (nil for none), and config configures the tracer +// (nil for defaults). +func (ec *Client) TraceCallWithCallTracer(ctx context.Context, msg ethereum.CallMsg, blockNrOrHash rpc.BlockNumberOrHash, overrides map[common.Address]OverrideAccount, blockOverrides *BlockOverrides, config *CallTracerConfig) (*CallFrame, error) { + var result CallFrame + err := ec.c.CallContext(ctx, &result, "debug_traceCall", toCallArg(msg), blockNrOrHash, callTraceCallConfig(config, overrides, blockOverrides)) + if err != nil { + return nil, err + } + return &result, nil +} + +// callTracerConfig converts a CallTracerConfig to the wire-format TraceConfig. +func callTracerConfig(config *CallTracerConfig) *tracers.TraceConfig { + tracer := "callTracer" + tc := &tracers.TraceConfig{Tracer: &tracer} + if config != nil { + if config.OnlyTopCall || config.WithLog { + cfg, _ := json.Marshal(struct { + OnlyTopCall bool `json:"onlyTopCall"` + WithLog bool `json:"withLog"` + }{config.OnlyTopCall, config.WithLog}) + tc.TracerConfig = cfg + } + if config.Timeout != 0 { + s := config.Timeout.String() + tc.Timeout = &s + } + } + return tc +} + +// callTraceCallConfig builds the wire-format TraceCallConfig for debug_traceCall, +// bundling tracer settings with optional state and block overrides. +func callTraceCallConfig(config *CallTracerConfig, overrides map[common.Address]OverrideAccount, blockOverrides *BlockOverrides) interface{} { + tc := callTracerConfig(config) + // debug_traceCall expects a single config object that includes both + // tracer settings and any state/block overrides. + type traceCallConfig struct { + *tracers.TraceConfig + StateOverrides map[common.Address]OverrideAccount `json:"stateOverrides,omitempty"` + BlockOverrides *BlockOverrides `json:"blockOverrides,omitempty"` + } + return &traceCallConfig{ + TraceConfig: tc, + StateOverrides: overrides, + BlockOverrides: blockOverrides, + } +} + func toBlockNumArg(number *big.Int) string { if number == nil { return "latest" diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 3a1986ce69..0bbe04e7a9 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -18,6 +18,7 @@ package gethclient import ( "bytes" + "context" "encoding/json" "math/big" "strings" @@ -33,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/tracers" + _ "github.com/ethereum/go-ethereum/eth/tracers/native" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -170,6 +172,12 @@ func TestGethClient(t *testing.T) { }, { "TestCallContractWithBlockOverrides", func(t *testing.T) { testCallContractWithBlockOverrides(t, client) }, + }, { + "TestTraceTransactionWithCallTracer", + func(t *testing.T) { testTraceTransactionWithCallTracer(t, client, txHashes) }, + }, { + "TestTraceCallWithCallTracer", + func(t *testing.T) { testTraceCallWithCallTracer(t, client) }, }, // The testaccesslist is a bit time-sensitive: the newTestBackend imports // one block. The `testAccessList` fails if the miner has not yet created a @@ -645,3 +653,60 @@ func testCallContractWithBlockOverrides(t *testing.T, client *rpc.Client) { t.Fatalf("unexpected result: %x", res) } } + +func testTraceTransactionWithCallTracer(t *testing.T, client *rpc.Client, txHashes []common.Hash) { + ec := New(client) + for _, txHash := range txHashes { + // With nil config (defaults). + result, err := ec.TraceTransactionWithCallTracer(context.Background(), txHash, nil) + if err != nil { + t.Fatalf("nil config: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + if result.From == (common.Address{}) { + t.Fatal("from is zero") + } + if result.Gas == 0 { + t.Fatal("gas is zero") + } + + // With explicit config. + result, err = ec.TraceTransactionWithCallTracer(context.Background(), txHash, + &CallTracerConfig{}, + ) + if err != nil { + t.Fatalf("explicit config: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + } +} + +func testTraceCallWithCallTracer(t *testing.T, client *rpc.Client) { + ec := New(client) + msg := ethereum.CallMsg{ + From: testAddr, + To: &common.Address{}, + Gas: 21000, + GasPrice: big.NewInt(1000000000), + Value: big.NewInt(1), + } + result, err := ec.TraceCallWithCallTracer(context.Background(), msg, + rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), nil, nil, nil, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + if result.From == (common.Address{}) { + t.Fatal("from is zero") + } + if result.Gas == 0 { + t.Fatal("gas is zero") + } +} diff --git a/ethclient/types_test.go b/ethclient/types_test.go index 02f9f21758..dcb9a579b7 100644 --- a/ethclient/types_test.go +++ b/ethclient/types_test.go @@ -41,6 +41,18 @@ func TestToFilterArg(t *testing.T) { output interface{} err error }{ + { + "without addresses", + ethereum.FilterQuery{ + FromBlock: big.NewInt(1), + ToBlock: big.NewInt(2), + }, + map[string]interface{}{ + "fromBlock": "0x1", + "toBlock": "0x2", + }, + nil, + }, { "without BlockHash", ethereum.FilterQuery{ diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 7d66d27cd4..7dd2f3e030 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -266,8 +266,8 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( // limit unchanged allows writes to be flushed more smoothly. This helps // avoid compaction spikes and mitigates write stalls caused by heavy // compaction workloads. - memTableLimit := 4 - memTableSize := cache * 1024 * 1024 / 2 / memTableLimit + memTableNumber := 4 + memTableSize := cache * 1024 * 1024 / 2 / memTableNumber // The memory table size is currently capped at maxMemTableSize-1 due to a // known bug in the pebble where maxMemTableSize is not recognized as a @@ -307,13 +307,16 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( // Note, there may have more than two memory tables in the system. MemTableSize: uint64(memTableSize), - // MemTableStopWritesThreshold places a hard limit on the size + // MemTableStopWritesThreshold places a hard limit on the number // of the existent MemTables(including the frozen one). + // // Note, this must be the number of tables not the size of all memtables // according to https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742 // and to https://github.com/cockroachdb/pebble/blob/master/db.go#L1892-L1903. - // Doubling the threshold will reduce write stalls during flush - MemTableStopWritesThreshold: memTableLimit * 2, + // + // MemTableStopWritesThreshold is set to twice the maximum number of + // allowed memtables to accommodate temporary spikes. + MemTableStopWritesThreshold: memTableNumber * 2, // The default compaction concurrency(1 thread), // Here use all available CPUs for faster compaction. @@ -328,7 +331,9 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( {TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, {TargetFileSize: 32 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, {TargetFileSize: 64 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 128 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + + // Pebble doesn't use the Bloom filter at level6 for read efficiency. + {TargetFileSize: 128 * 1024 * 1024}, }, ReadOnly: readonly, EventListener: &pebble.EventListener{ @@ -367,6 +372,16 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( // for more details. opt.Experimental.ReadSamplingMultiplier = -1 + // These two settings define the conditions under which compaction concurrency + // is increased. Specifically, one additional compaction job will be enabled when: + // - there is one more overlapping sub-level0; + // - there is an additional 256 MB of compaction debt; + // + // The maximum concurrency is still capped by MaxConcurrentCompactions, but with + // these settings compactions can scale up more readily. + opt.Experimental.L0CompactionConcurrency = 1 + opt.Experimental.CompactionDebtConcurrency = 1 << 28 // 256MB + // Adaptive compaction scales the workers based on the load instead of always using all CPUs // L0CompactionConcurrency compaction worker per overlapping sublevel opt.Experimental.L0CompactionConcurrency = 1 diff --git a/go.mod b/go.mod index e9c852a6e9..72d02e3b6a 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/cosmos/cosmos-sdk v0.50.15 github.com/cosmos/gogoproto v1.7.2 github.com/crate-crypto/go-eth-kzg v1.4.0 - github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dchest/siphash v1.2.3 github.com/deckarep/golang-set/v2 v2.6.0 @@ -34,7 +33,6 @@ require ( github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 github.com/ethereum/c-kzg-4844/v2 v2.1.5 github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab - github.com/ethereum/go-verkle v0.2.2 github.com/fatih/color v1.17.0 github.com/ferranbt/fastssz v0.1.2 github.com/fsnotify/fsnotify v1.9.0 @@ -46,6 +44,7 @@ require ( github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/grafana/pyroscope-go v1.4.1 github.com/graph-gophers/graphql-go v1.5.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/hashicorp/go-bexpr v0.1.14 @@ -140,7 +139,7 @@ require ( github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.18.1 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -253,6 +252,7 @@ require ( github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.11 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect diff --git a/go.sum b/go.sum index f0230f2cee..f105105ecc 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3 github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= github.com/creachadair/atomicfile v0.4.0 h1:umZ+njabCI7eeMLWXFRuUy83iQSiA8qnUj8MH04pfbM= github.com/creachadair/atomicfile v0.4.0/go.mod h1:OyQCzy3n5KhEXHcHxSzZ7BbRt7QmeVkaiMJmtENydDI= github.com/creachadair/mds v0.25.6 h1:l1MkbdJmhqXTpm92VWYTb0/s0wsiZSKKc/VKU0lf2bA= @@ -430,8 +428,6 @@ github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3 github.com/ethereum/c-kzg-4844/v2 v2.1.5/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= -github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= -github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= @@ -669,6 +665,10 @@ github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/pyroscope-go v1.4.1 h1:SKvuZz1qTFpNXjHY3XGlm92mS9wJeCI/TZR42XcDs9w= +github.com/grafana/pyroscope-go v1.4.1/go.mod h1:WQY21vHNiD2o/icxqVPM0AmofabycbhnPPHoV1/4X6s= +github.com/grafana/pyroscope-go/godeltaprof v0.1.11 h1:el5LYpXissAiCKZ5/6yjlr6mhYVV6Cp5lahTocxraXM= +github.com/grafana/pyroscope-go/godeltaprof v0.1.11/go.mod h1:jl1V8M4cWsXciROCPIDDG7CtjSjT/ECbp6eLVuMxYRI= github.com/graph-gophers/graphql-go v1.5.0 h1:fDqblo50TEpD0LY7RXk/LFVYEVqo3+tXMNMPSVXA1yc= github.com/graph-gophers/graphql-go v1.5.0/go.mod h1:YtmJZDLbF1YYNrlNAuiO5zAStUWc3XZT07iGsVqe1Os= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -816,8 +816,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= -github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -1158,8 +1158,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/graphql/graphql.go b/graphql/graphql.go index 7cf55d1376..3cf69a5720 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -272,7 +272,7 @@ func (t *Transaction) GasPrice(ctx context.Context) hexutil.Big { return hexutil.Big{} } switch tx.Type() { - case types.DynamicFeeTxType: + case types.DynamicFeeTxType, types.BlobTxType, types.SetCodeTxType: if block != nil { if baseFee, _ := block.BaseFeePerGas(ctx); baseFee != nil { // price = min(gasTipCap + baseFee, gasFeeCap) @@ -575,6 +575,9 @@ func (t *Transaction) getLogs(ctx context.Context, hash common.Hash) (*[]*Log, e func (t *Transaction) Type(ctx context.Context) *hexutil.Uint64 { tx, _ := t.resolve(ctx) + if tx == nil { + return nil + } txType := hexutil.Uint64(tx.Type()) return &txType } @@ -704,6 +707,9 @@ func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) { if err != nil { return nil, err } + if b.header == nil { + return nil, nil + } if b.hash == (common.Hash{}) { b.hash = b.header.Hash() } diff --git a/internal/blocktest/test_hash.go b/internal/blocktest/test_hash.go index f79de8b519..5e5b1202de 100644 --- a/internal/blocktest/test_hash.go +++ b/internal/blocktest/test_hash.go @@ -26,9 +26,8 @@ import ( "bytes" "hash" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto/keccak" ) // testHasher is the helper tool for transaction/receipt list hashing. @@ -40,7 +39,7 @@ type testHasher struct { // NewHasher returns a new testHasher instance. func NewHasher() *testHasher { - return &testHasher{hasher: sha3.NewLegacyKeccak256()} + return &testHasher{hasher: keccak.NewLegacyKeccak256()} } // Reset resets the hash state. diff --git a/internal/build/file.go b/internal/build/file.go index 2cd090c42c..b7c00eb842 100644 --- a/internal/build/file.go +++ b/internal/build/file.go @@ -21,20 +21,21 @@ import ( "io" "os" "path/filepath" - "sort" - "strings" + "slices" ) // HashFolder iterates all files under the given directory, computing the hash // of each. -func HashFolder(folder string, exlude []string) (map[string][32]byte, error) { +func HashFolder(folder string, excludes []string) (map[string][32]byte, error) { res := make(map[string][32]byte) err := filepath.WalkDir(folder, func(path string, d os.DirEntry, _ error) error { // Skip anything that's exluded or not a regular file - for _, skip := range exlude { - if strings.HasPrefix(path, filepath.FromSlash(skip)) { + // Skip anything that's excluded or not a regular file + if slices.Contains(excludes, path) { + if d.IsDir() { return filepath.SkipDir } + return nil } if !d.Type().IsRegular() { return nil @@ -71,6 +72,6 @@ func DiffHashes(a map[string][32]byte, b map[string][32]byte) []string { updates = append(updates, file) } } - sort.Strings(updates) + slices.Sort(updates) return updates } diff --git a/internal/debug/api.go b/internal/debug/api.go index ff68d5f109..d3c3b6bebc 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -272,7 +272,7 @@ func (*HandlerT) SetGCPercent(v int) int { // - Geth also allocates memory off-heap, particularly for fastCache and Pebble, // which can be non-trivial (a few gigabytes by default). func (*HandlerT) SetMemoryLimit(limit int64) int64 { - log.Info("Setting memory limit", "size", common.PrettyDuration(limit)) + log.Info("Setting memory limit", "size", common.StorageSize(limit)) return debug.SetMemoryLimit(limit) } diff --git a/internal/debug/flags.go b/internal/debug/flags.go index ccb72cba76..e7429ecc18 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -163,6 +163,11 @@ var Flags = []cli.Flag{ blockprofilerateFlag, cpuprofileFlag, traceFlag, + pyroscopeFlag, + pyroscopeServerFlag, + pyroscopeAuthUsernameFlag, + pyroscopeAuthPasswordFlag, + pyroscopeTagsFlag, } var ( @@ -299,6 +304,14 @@ func Setup(ctx *cli.Context) error { // It cannot be imported because it will cause a cyclical dependency. StartPProf(address, !ctx.IsSet("metrics.addr")) } + + // Pyroscope profiling + if ctx.Bool(pyroscopeFlag.Name) { + if err := startPyroscope(ctx); err != nil { + return err + } + } + if len(logFile) > 0 || rotation { log.Info("Logging configured", context...) } @@ -322,6 +335,7 @@ func StartPProf(address string, withMetrics bool) { // Exit stops all running profiles, flushing their output to the // respective file. func Exit() { + stopPyroscope() Handler.StopCPUProfile() Handler.StopGoTrace() if logOutputFile != nil { diff --git a/internal/debug/pyroscope.go b/internal/debug/pyroscope.go new file mode 100644 index 0000000000..d0804cb891 --- /dev/null +++ b/internal/debug/pyroscope.go @@ -0,0 +1,134 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package debug + +import ( + "fmt" + "strings" + + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/log" + "github.com/grafana/pyroscope-go" + "github.com/urfave/cli/v2" +) + +var ( + pyroscopeFlag = &cli.BoolFlag{ + Name: "pyroscope", + Usage: "Enable Pyroscope profiling", + Value: false, + Category: flags.LoggingCategory, + } + pyroscopeServerFlag = &cli.StringFlag{ + Name: "pyroscope.server", + Usage: "Pyroscope server URL to push profiling data to", + Value: "http://localhost:4040", + Category: flags.LoggingCategory, + } + pyroscopeAuthUsernameFlag = &cli.StringFlag{ + Name: "pyroscope.username", + Usage: "Pyroscope basic authentication username", + Value: "", + Category: flags.LoggingCategory, + } + pyroscopeAuthPasswordFlag = &cli.StringFlag{ + Name: "pyroscope.password", + Usage: "Pyroscope basic authentication password", + Value: "", + Category: flags.LoggingCategory, + } + pyroscopeTagsFlag = &cli.StringFlag{ + Name: "pyroscope.tags", + Usage: "Comma separated list of key=value tags to add to profiling data", + Value: "", + Category: flags.LoggingCategory, + } +) + +// This holds the globally-configured Pyroscope instance. +var pyroscopeProfiler *pyroscope.Profiler + +func startPyroscope(ctx *cli.Context) error { + server := ctx.String(pyroscopeServerFlag.Name) + authUsername := ctx.String(pyroscopeAuthUsernameFlag.Name) + authPassword := ctx.String(pyroscopeAuthPasswordFlag.Name) + + rawTags := ctx.String(pyroscopeTagsFlag.Name) + tags := make(map[string]string) + for tag := range strings.SplitSeq(rawTags, ",") { + tag = strings.TrimSpace(tag) + if tag == "" { + continue + } + k, v, _ := strings.Cut(tag, "=") + tags[k] = v + } + + config := pyroscope.Config{ + ApplicationName: "geth", + ServerAddress: server, + BasicAuthUser: authUsername, + BasicAuthPassword: authPassword, + Logger: &pyroscopeLogger{Logger: log.Root()}, + Tags: tags, + ProfileTypes: []pyroscope.ProfileType{ + // Enabling all profile types + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + } + + profiler, err := pyroscope.Start(config) + if err != nil { + return err + } + pyroscopeProfiler = profiler + log.Info("Enabled Pyroscope") + return nil +} + +func stopPyroscope() { + if pyroscopeProfiler != nil { + pyroscopeProfiler.Stop() + pyroscopeProfiler = nil + } +} + +// Small wrapper for log.Logger to satisfy pyroscope.Logger interface +type pyroscopeLogger struct { + Logger log.Logger +} + +func (l *pyroscopeLogger) Infof(format string, v ...any) { + l.Logger.Info(fmt.Sprintf("Pyroscope: "+format, v...)) +} + +func (l *pyroscopeLogger) Debugf(format string, v ...any) { + l.Logger.Debug(fmt.Sprintf("Pyroscope: "+format, v...)) +} + +func (l *pyroscopeLogger) Errorf(format string, v ...any) { + l.Logger.Error(fmt.Sprintf("Pyroscope: "+format, v...)) +} diff --git a/internal/download/download.go b/internal/download/download.go index 26c7795ce5..c59c8a90c3 100644 --- a/internal/download/download.go +++ b/internal/download/download.go @@ -205,7 +205,10 @@ func (db *ChecksumDB) DownloadFile(url, dstPath string) error { if err != nil { return err } - dst := newDownloadWriter(fd, resp.ContentLength) + var dst io.WriteCloser = fd + if resp.ContentLength > 0 { + dst = newDownloadWriter(fd, resp.ContentLength) + } _, err = io.Copy(dst, resp.Body) dst.Close() if err != nil { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index e816d52e19..575f0b8bb8 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -54,7 +54,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" ) // estimateGasErrorRatio is the amount of overestimation eth_estimateGas is @@ -265,6 +264,15 @@ func NewTxPoolAPI(b Backend) *TxPoolAPI { return &TxPoolAPI{b} } +// flattenTxs builds the RPC transaction map keyed by nonce for a set of pool txs. +func flattenTxs(txs types.Transactions, header *types.Header, cfg *params.ChainConfig) map[string]*RPCTransaction { + dump := make(map[string]*RPCTransaction, len(txs)) + for _, tx := range txs { + dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, header, cfg) + } + return dump +} + // Content returns the transactions contained within the transaction pool. func (api *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction { pending, queue := api.b.TxPoolContent() @@ -275,21 +283,11 @@ func (api *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction curHeader := api.b.CurrentHeader() // Flatten the pending transactions for account, txs := range pending { - dump := make(map[string]*RPCTransaction, len(txs)) - for _, tx := range txs { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - - content["pending"][account.Hex()] = dump + content["pending"][account.Hex()] = flattenTxs(txs, curHeader, api.b.ChainConfig()) } // Flatten the queued transactions for account, txs := range queue { - dump := make(map[string]*RPCTransaction, len(txs)) - for _, tx := range txs { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - - content["queued"][account.Hex()] = dump + content["queued"][account.Hex()] = flattenTxs(txs, curHeader, api.b.ChainConfig()) } return content @@ -302,20 +300,10 @@ func (api *TxPoolAPI) ContentFrom(addr common.Address) map[string]map[string]*RP curHeader := api.b.CurrentHeader() // Build the pending transactions - dump := make(map[string]*RPCTransaction, len(pending)) - for _, tx := range pending { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - - content["pending"] = dump + content["pending"] = flattenTxs(pending, curHeader, api.b.ChainConfig()) // Build the queued transactions - dump = make(map[string]*RPCTransaction, len(queue)) - for _, tx := range queue { - dump[fmt.Sprintf("%d", tx.Nonce())] = NewRPCPendingTransaction(tx, curHeader, api.b.ChainConfig()) - } - - content["queued"] = dump + content["queued"] = flattenTxs(queue, curHeader, api.b.ChainConfig()) return content } @@ -471,9 +459,9 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, // Deserialize all keys. This prevents state access on invalid input. for i, hexKey := range storageKeys { var err error - keys[i], keyLengths[i], err = decodeHash(hexKey) + keys[i], keyLengths[i], err = decodeStorageKey(hexKey) if err != nil { - return nil, err + return nil, &invalidParamsError{fmt.Sprintf("%v: %q", err, hexKey)} } } statedb, header, err := api.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) @@ -486,8 +474,7 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, if len(keys) > 0 { var storageTrie state.Trie if storageRoot != types.EmptyRootHash && storageRoot != (common.Hash{}) { - id := trie.StorageTrieID(header.Root, crypto.Keccak256Hash(address.Bytes()), storageRoot) - st, err := trie.NewStateTrie(id, statedb.Database().TrieDB()) + st, err := statedb.Database().OpenStorageTrie(header.Root, address, storageRoot, nil) if err != nil { return nil, err } @@ -518,7 +505,7 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, } } // Create the accountProof. - tr, err := trie.NewStateTrie(trie.StateTrieID(header.Root), statedb.Database().TrieDB()) + tr, err := statedb.Database().OpenTrie(header.Root) if err != nil { return nil, err } @@ -538,9 +525,10 @@ func (api *BlockChainAPI) GetProof(ctx context.Context, address common.Address, }, statedb.Error() } -// decodeHash parses a hex-encoded 32-byte hash. The input may optionally -// be prefixed by 0x and can have a byte length up to 32. -func decodeHash(s string) (h common.Hash, inputLength int, err error) { +// decodeStorageKey parses a hex-encoded 32-byte hash. +// For legacy compatibility reasons, we parse these keys leniently, +// with the 0x prefix being optional. +func decodeStorageKey(s string) (h common.Hash, inputLength int, err error) { if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { s = s[2:] } @@ -549,11 +537,11 @@ func decodeHash(s string) (h common.Hash, inputLength int, err error) { s = "0" + s } if len(s) > 64 { - return common.Hash{}, len(s) / 2, errors.New("hex string too long, want at most 32 bytes") + return common.Hash{}, len(s) / 2, errors.New("storage key too long (want at most 32 bytes)") } b, err := hex.DecodeString(s) if err != nil { - return common.Hash{}, 0, errors.New("hex string invalid") + return common.Hash{}, 0, errors.New("invalid hex in storage key") } return common.BytesToHash(b), len(b), nil } @@ -731,9 +719,9 @@ func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Addre if state == nil || err != nil { return nil, err } - key, _, err := decodeHash(hexKey) + key, _, err := decodeStorageKey(hexKey) if err != nil { - return nil, fmt.Errorf("unable to decode storage key: %s", err) + return nil, &invalidParamsError{fmt.Sprintf("%v: %q", err, hexKey)} } res := state.GetState(address, key) @@ -1369,6 +1357,7 @@ func RPCMarshalWitness(witness *stateless.Witness) map[string]interface{} { type RPCTransaction struct { BlockHash *common.Hash `json:"blockHash"` BlockNumber *hexutil.Big `json:"blockNumber"` + BlockTimestamp *hexutil.Uint64 `json:"blockTimestamp"` From common.Address `json:"from"` Gas hexutil.Uint64 `json:"gas"` GasPrice *hexutil.Big `json:"gasPrice"` @@ -1416,6 +1405,7 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber if blockHash != (common.Hash{}) { result.BlockHash = &blockHash result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber)) + result.BlockTimestamp = (*hexutil.Uint64)(&blockTime) result.TransactionIndex = (*hexutil.Uint64)(&index) } @@ -2138,7 +2128,7 @@ func (api *TransactionAPI) SendTransaction(ctx context.Context, args Transaction return common.Hash{}, err } // Assemble the transaction and sign with the wallet - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) signed, err := wallet.SignTx(account, tx, api.b.ChainConfig().ChainID) if err != nil { @@ -2160,7 +2150,7 @@ func (api *TransactionAPI) FillTransaction(ctx context.Context, args Transaction return nil, err } // Assemble the transaction and obtain rlp - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) data, err := tx.MarshalBinary() if err != nil { return nil, err @@ -2212,7 +2202,7 @@ func (api *TransactionAPI) SendRawTransaction(ctx context.Context, input hexutil // SendRawTransactionSync will add the signed transaction to the transaction pool // and wait until the transaction has been included in a block and return the receipt, or the timeout. -func (api *TransactionAPI) SendRawTransactionSync(ctx context.Context, input hexutil.Bytes, timeoutMs *hexutil.Uint64) (map[string]interface{}, error) { +func (api *TransactionAPI) SendRawTransactionSync(ctx context.Context, input hexutil.Bytes, timeoutMs *uint64) (map[string]interface{}, error) { tx := new(types.Transaction) if err := tx.UnmarshalBinary(input); err != nil { return nil, err @@ -2454,7 +2444,7 @@ func (api *TransactionAPI) SignTransaction(ctx context.Context, args Transaction return nil, err } // Before actually sign the transaction, ensure the transaction fee is reasonable. - tx := args.ToTransaction(types.LegacyTxType) + tx := args.ToTransaction(types.DynamicFeeTxType) if err := checkTxFee(tx.GasPrice(), tx.Gas(), api.b.RPCTxFeeCap()); err != nil { return nil, err } @@ -2514,7 +2504,7 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, if err := sendArgs.setDefaults(ctx, api.b, sidecarConfig{}); err != nil { return common.Hash{}, err } - matchTx := sendArgs.ToTransaction(types.LegacyTxType) + matchTx := sendArgs.ToTransaction(types.DynamicFeeTxType) // Before replacing the old transaction, ensure the _new_ transaction fee is reasonable. price := matchTx.GasPrice() @@ -2551,7 +2541,7 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, if gasLimit != nil && *gasLimit != 0 { sendArgs.Gas = gasLimit } - signedTx, err := api.sign(sendArgs.from(), sendArgs.ToTransaction(types.LegacyTxType)) + signedTx, err := api.sign(sendArgs.from(), sendArgs.ToTransaction(types.DynamicFeeTxType)) if err != nil { return common.Hash{}, err } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 29357c024c..2004d4584b 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -141,6 +141,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -171,6 +172,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -209,6 +211,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -255,6 +258,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x6", @@ -302,6 +306,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x9", @@ -346,6 +351,7 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x7", "gasPrice": "0x9", @@ -388,6 +394,7 @@ func allBlobTxs(addr common.Address, config *params.ChainConfig) []txData { Want: `{ "blockHash": null, "blockNumber": null, + "blockTimestamp": null, "from": "0x71562b71999873db5b286df957af199ec94617f7", "gas": "0x6", "gasPrice": "0x5", @@ -3571,6 +3578,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3591,6 +3599,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3609,6 +3618,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -3629,6 +3639,7 @@ func TestRPCMarshalBlock(t *testing.T) { { "blockHash": "0x9b73c83b25d0faf7eab854e3684c7e394336d6e135625aafa5c183f27baa8fee", "blockNumber": "0x64", + "blockTimestamp": "0x0", "from": "0x0000000000000000000000000000000000000000", "gas": "0x457", "gasPrice": "0x2b67", @@ -5113,7 +5124,7 @@ func TestSendRawTransactionSync_Timeout(t *testing.T) { raw, _ := makeSelfSignedRaw(t, api, b.acc.Address) - timeout := hexutil.Uint64(200) // 200ms + timeout := uint64(200) // 200ms receipt, err := api.SendRawTransactionSync(context.Background(), raw, &timeout) if receipt != nil { diff --git a/internal/ethapi/errors.go b/internal/ethapi/errors.go index 30711a0167..e406c36d6c 100644 --- a/internal/ethapi/errors.go +++ b/internal/ethapi/errors.go @@ -112,7 +112,6 @@ const ( errCodeClientLimitExceeded = -38026 errCodeInternalError = -32603 errCodeInvalidParams = -32602 - errCodeReverted = -32000 errCodeVMError = -32015 errCodeTxSyncTimeout = 4 ) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 140d759197..11f1e34d34 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -305,7 +305,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if block.BlockOverrides.BlobBaseFee != nil { blockContext.BlobBaseFee = block.BlockOverrides.BlobBaseFee.ToInt() } - precompiles := sim.activePrecompiles(sim.base) + precompiles := sim.activePrecompiles(header) // State overrides are applied prior to execution of a block if err := block.StateOverrides.Apply(sim.state, precompiles); err != nil { return nil, nil, nil, err @@ -391,7 +391,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if errors.Is(result.Err, vm.ErrExecutionReverted) { // If the result contains a revert reason, try to unpack it. revertErr := newRevertError(result.Revert()) - callRes.Error = &callError{Message: revertErr.Error(), Code: errCodeReverted, Data: revertErr.ErrorData().(string)} + callRes.Error = &callError{Message: revertErr.Error(), Code: revertErr.ErrorCode(), Data: revertErr.ErrorData().(string)} } else { msg := result.Err.Error() if gasCapped { diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json index 19024f3e61..228a1b633f 100644 --- a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json @@ -20,6 +20,7 @@ { "blockHash": "0xc6f9009e050c0c4373e1a2865041dd74f32cddf4187318db2f2911edda6250d7", "blockNumber": "0x9", + "blockTimestamp": "0x5a", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0x121a9cca", diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json index 19024f3e61..228a1b633f 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json @@ -20,6 +20,7 @@ { "blockHash": "0xc6f9009e050c0c4373e1a2865041dd74f32cddf4187318db2f2911edda6250d7", "blockNumber": "0x9", + "blockTimestamp": "0x5a", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0x121a9cca", diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json index e4a42de4a0..ee37ebc4b3 100644 --- a/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json +++ b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json @@ -20,6 +20,7 @@ { "blockHash": "0xdbfad986768c46622dbffa32455fee9ae67929dc27da1f97b364516fd004456f", "blockNumber": "0xb", + "blockTimestamp": "0x6e", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0xde56ab3", diff --git a/internal/ethapi/testdata/eth_getTransactionByBlockHashAndIndex-create-contract-with-access-list.json b/internal/ethapi/testdata/eth_getTransactionByBlockHashAndIndex-create-contract-with-access-list.json index 9429f9359d..294b174d3d 100644 --- a/internal/ethapi/testdata/eth_getTransactionByBlockHashAndIndex-create-contract-with-access-list.json +++ b/internal/ethapi/testdata/eth_getTransactionByBlockHashAndIndex-create-contract-with-access-list.json @@ -1,6 +1,7 @@ { "blockHash": "0xd086fa663066cf77a12eb0c180b7e01ecc1c7df833f79743f1a496619244db9e", "blockNumber": "0x5", + "blockTimestamp": "0x32", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0xe2f4", "gasPrice": "0x1ecb3fb4", diff --git a/internal/ethapi/testdata/eth_getTransactionByBlockHashAndIndex-state-sync-tx.json b/internal/ethapi/testdata/eth_getTransactionByBlockHashAndIndex-state-sync-tx.json index bbfd29df3b..bd3ac24266 100644 --- a/internal/ethapi/testdata/eth_getTransactionByBlockHashAndIndex-state-sync-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionByBlockHashAndIndex-state-sync-tx.json @@ -1,6 +1,7 @@ { "blockHash": "0xd086fa663066cf77a12eb0c180b7e01ecc1c7df833f79743f1a496619244db9e", "blockNumber": "0x5", + "blockTimestamp": "0x32", "from": "0x0000000000000000000000000000000000000000", "gas": "0x0", "gasPrice": "0x0", diff --git a/internal/ethapi/testdata/eth_getTransactionByHash-create-contract-tx.json b/internal/ethapi/testdata/eth_getTransactionByHash-create-contract-tx.json index 9e84bcb146..5ecd65927b 100644 --- a/internal/ethapi/testdata/eth_getTransactionByHash-create-contract-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionByHash-create-contract-tx.json @@ -1,6 +1,7 @@ { "blockHash": "0x55e4898b479efff97fc1e578f11a7d01e20891556c2a546ee6bb73040e4fed7a", "blockNumber": "0x2", + "blockTimestamp": "0x14", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0xcf6c", "gasPrice": "0x2db16291", diff --git a/internal/ethapi/testdata/eth_getTransactionByHash-create-contract-with-access-list.json b/internal/ethapi/testdata/eth_getTransactionByHash-create-contract-with-access-list.json index 9429f9359d..294b174d3d 100644 --- a/internal/ethapi/testdata/eth_getTransactionByHash-create-contract-with-access-list.json +++ b/internal/ethapi/testdata/eth_getTransactionByHash-create-contract-with-access-list.json @@ -1,6 +1,7 @@ { "blockHash": "0xd086fa663066cf77a12eb0c180b7e01ecc1c7df833f79743f1a496619244db9e", "blockNumber": "0x5", + "blockTimestamp": "0x32", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0xe2f4", "gasPrice": "0x1ecb3fb4", diff --git a/internal/ethapi/testdata/eth_getTransactionByHash-dynamic-tx-with-logs.json b/internal/ethapi/testdata/eth_getTransactionByHash-dynamic-tx-with-logs.json index 236284d3ee..9e4fb3b03a 100644 --- a/internal/ethapi/testdata/eth_getTransactionByHash-dynamic-tx-with-logs.json +++ b/internal/ethapi/testdata/eth_getTransactionByHash-dynamic-tx-with-logs.json @@ -1,6 +1,7 @@ { "blockHash": "0x2dbc032049f6c345859079f8b217428fa7b1f6492733b20bdc461887ac909276", "blockNumber": "0x4", + "blockTimestamp": "0x28", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0xea60", "gasPrice": "0x2325c42f", diff --git a/internal/ethapi/testdata/eth_getTransactionByHash-normal-transfer-tx.json b/internal/ethapi/testdata/eth_getTransactionByHash-normal-transfer-tx.json index be53ad75e8..3c7ab885e3 100644 --- a/internal/ethapi/testdata/eth_getTransactionByHash-normal-transfer-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionByHash-normal-transfer-tx.json @@ -1,6 +1,7 @@ { "blockHash": "0x8d822dcec517a299730892714efbb08e998cc2dd0549cfd2de8d0a72eb651797", "blockNumber": "0x1", + "blockTimestamp": "0xa", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0x5208", "gasPrice": "0x342770c0", diff --git a/internal/ethapi/testdata/eth_getTransactionByHash-state-sync-tx.json b/internal/ethapi/testdata/eth_getTransactionByHash-state-sync-tx.json index bbfd29df3b..bd3ac24266 100644 --- a/internal/ethapi/testdata/eth_getTransactionByHash-state-sync-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionByHash-state-sync-tx.json @@ -1,6 +1,7 @@ { "blockHash": "0xd086fa663066cf77a12eb0c180b7e01ecc1c7df833f79743f1a496619244db9e", "blockNumber": "0x5", + "blockTimestamp": "0x32", "from": "0x0000000000000000000000000000000000000000", "gas": "0x0", "gasPrice": "0x0", diff --git a/internal/ethapi/testdata/eth_getTransactionByHash-with-logs.json b/internal/ethapi/testdata/eth_getTransactionByHash-with-logs.json index e7dac60faa..48af0bca9a 100644 --- a/internal/ethapi/testdata/eth_getTransactionByHash-with-logs.json +++ b/internal/ethapi/testdata/eth_getTransactionByHash-with-logs.json @@ -1,6 +1,7 @@ { "blockHash": "0xc44c61793228c6bde7ea124b6a386cae812b221016c16fa1a64f3ab6e83b2a9b", "blockNumber": "0x3", + "blockTimestamp": "0x1e", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gas": "0xea60", "gasPrice": "0x281c2585", diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 0193685d09..6a597b7ae1 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -343,8 +343,8 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca commitments = make([]kzg4844.Commitment, n) proofs = make([]kzg4844.Proof, 0, proofLen) ) - for i, b := range args.Blobs { - c, err := kzg4844.BlobToCommitment(&b) + for i := range args.Blobs { + c, err := kzg4844.BlobToCommitment(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) } @@ -352,13 +352,13 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca switch config.blobSidecarVersion { case types.BlobSidecarVersion0: - p, err := kzg4844.ComputeBlobProof(&b, c) + p, err := kzg4844.ComputeBlobProof(&args.Blobs[i], c) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } proofs = append(proofs, p) case types.BlobSidecarVersion1: - ps, err := kzg4844.ComputeCellProofs(&b) + ps, err := kzg4844.ComputeCellProofs(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) } @@ -370,8 +370,8 @@ func (args *TransactionArgs) setBlobTxSidecar(ctx context.Context, config sideca } else { switch config.blobSidecarVersion { case types.BlobSidecarVersion0: - for i, b := range args.Blobs { - if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { + for i := range args.Blobs { + if err := kzg4844.VerifyBlobProof(&args.Blobs[i], args.Commitments[i], args.Proofs[i]); err != nil { return fmt.Errorf("failed to verify blob proof: %v", err) } } diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index c864528eba..25a17c27fa 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -41,7 +41,7 @@ func NewApp(usage string) *cli.App { app.EnableBashCompletion = true app.Version = version.WithCommit(git.Commit, git.Date) app.Usage = usage - app.Copyright = "Copyright 2013-2025 The go-ethereum Authors" + app.Copyright = "Copyright 2013-2026 The go-ethereum Authors" app.Before = func(ctx *cli.Context) error { MigrateGlobalFlags(ctx) return nil diff --git a/metrics/counter.go b/metrics/counter.go index 59d839c8ef..d176eac56d 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -7,7 +7,10 @@ import ( // GetOrRegisterCounter returns an existing Counter or constructs and registers // a new Counter. func GetOrRegisterCounter(name string, r Registry) *Counter { - return getOrRegister(name, NewCounter, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewCounter() }).(*Counter) } // NewCounter constructs a new Counter. diff --git a/metrics/counter_float64.go b/metrics/counter_float64.go index 0749a17ebc..c9a85bdb4e 100644 --- a/metrics/counter_float64.go +++ b/metrics/counter_float64.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterCounterFloat64 returns an existing *CounterFloat64 or constructs and registers // a new CounterFloat64. func GetOrRegisterCounterFloat64(name string, r Registry) *CounterFloat64 { - return getOrRegister(name, NewCounterFloat64, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewCounterFloat64() }).(*CounterFloat64) } // NewCounterFloat64 constructs a new CounterFloat64. diff --git a/metrics/gauge.go b/metrics/gauge.go index 5284f3ce96..36d3d17b78 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -11,7 +11,10 @@ func (g GaugeSnapshot) Value() int64 { return int64(g) } // GetOrRegisterGauge returns an existing Gauge or constructs and registers a // new Gauge. func GetOrRegisterGauge(name string, r Registry) *Gauge { - return getOrRegister(name, NewGauge, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGauge() }).(*Gauge) } // NewGauge constructs a new Gauge. diff --git a/metrics/gauge_float64.go b/metrics/gauge_float64.go index 87247a7fa8..40d1decaeb 100644 --- a/metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a // new GaugeFloat64. func GetOrRegisterGaugeFloat64(name string, r Registry) *GaugeFloat64 { - return getOrRegister(name, NewGaugeFloat64, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGaugeFloat64() }).(*GaugeFloat64) } // GaugeFloat64Snapshot is a read-only copy of a GaugeFloat64. diff --git a/metrics/gauge_info.go b/metrics/gauge_info.go index 34ac917919..30c5114085 100644 --- a/metrics/gauge_info.go +++ b/metrics/gauge_info.go @@ -16,7 +16,10 @@ func (val GaugeInfoValue) String() string { // GetOrRegisterGaugeInfo returns an existing GaugeInfo or constructs and registers a // new GaugeInfo. func GetOrRegisterGaugeInfo(name string, r Registry) *GaugeInfo { - return getOrRegister(name, NewGaugeInfo, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGaugeInfo() }).(*GaugeInfo) } // NewGaugeInfo constructs a new GaugeInfo. diff --git a/metrics/histogram.go b/metrics/histogram.go index 89a60056ce..f61da457ca 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -23,19 +23,19 @@ type Histogram interface { // GetOrRegisterHistogram returns an existing Histogram or constructs and // registers a new StandardHistogram. func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { - if nil == r { + if r == nil { r = DefaultRegistry } - return getOrRegister(name, func() Histogram { return NewHistogram(s) }, r) + return r.GetOrRegister(name, func() any { return NewHistogram(s) }).(Histogram) } // GetOrRegisterHistogramLazy returns an existing Histogram or constructs and // registers a new StandardHistogram. func GetOrRegisterHistogramLazy(name string, r Registry, s func() Sample) Histogram { - if nil == r { + if r == nil { r = DefaultRegistry } - return getOrRegister(name, func() Histogram { return NewHistogram(s()) }, r) + return r.GetOrRegister(name, func() any { return NewHistogram(s()) }).(Histogram) } // NewHistogram constructs a new StandardHistogram from a Sample. diff --git a/metrics/meter.go b/metrics/meter.go index 8b6700b395..1297e66d75 100644 --- a/metrics/meter.go +++ b/metrics/meter.go @@ -12,7 +12,10 @@ import ( // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func GetOrRegisterMeter(name string, r Registry) *Meter { - return getOrRegister(name, NewMeter, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewMeter() }).(*Meter) } // NewMeter constructs a new Meter and launches a goroutine. diff --git a/metrics/registry.go b/metrics/registry.go index d6748a47d7..bcc4e6c31b 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -144,6 +144,8 @@ func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { values["value"] = metric.Snapshot().Value() case *GaugeFloat64: values["value"] = metric.Snapshot().Value() + case *GaugeInfo: + values["value"] = metric.Snapshot().Value() case *Healthcheck: values["error"] = nil @@ -189,6 +191,18 @@ func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { values["5m.rate"] = t.Rate5() values["15m.rate"] = t.Rate15() values["mean.rate"] = t.RateMean() + case *ResettingTimer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = t.Count() + values["min"] = t.Min() + values["max"] = t.Max() + values["mean"] = t.Mean() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] } data[name] = values @@ -337,13 +351,6 @@ func GetOrRegister(name string, i func() interface{}) interface{} { return DefaultRegistry.GetOrRegister(name, i) } -func getOrRegister[T any](name string, ctor func() T, r Registry) T { - if r == nil { - r = DefaultRegistry - } - return r.GetOrRegister(name, func() any { return ctor() }).(T) -} - // Register the given metric under the given name. Returns a ErrDuplicateMetric // if a metric by the given name is already registered. func Register(name string, i interface{}) error { diff --git a/metrics/registry_test.go b/metrics/registry_test.go index 137c5c5246..d09583ed2f 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -15,6 +15,31 @@ func BenchmarkRegistry(b *testing.B) { } } +func BenchmarkRegistryGetOrRegister(b *testing.B) { + sample := func() Sample { return nil } + tests := []struct { + name string + ctor func() any + }{ + {name: "counter", ctor: func() any { return GetOrRegisterCounter("counter", DefaultRegistry) }}, + {name: "gauge", ctor: func() any { return GetOrRegisterGauge("gauge", DefaultRegistry) }}, + {name: "gaugefloat64", ctor: func() any { return GetOrRegisterGaugeFloat64("gaugefloat64", DefaultRegistry) }}, + {name: "histogram", ctor: func() any { return GetOrRegisterHistogram("histogram", DefaultRegistry, sample()) }}, + {name: "meter", ctor: func() any { return GetOrRegisterMeter("meter", DefaultRegistry) }}, + {name: "timer", ctor: func() any { return GetOrRegisterTimer("timer", DefaultRegistry) }}, + {name: "gaugeinfo", ctor: func() any { return GetOrRegisterGaugeInfo("gaugeinfo", DefaultRegistry) }}, + {name: "resettingtimer", ctor: func() any { return GetOrRegisterResettingTimer("resettingtimer", DefaultRegistry) }}, + {name: "runtimehistogramlazy", ctor: func() any { return GetOrRegisterHistogramLazy("runtimehistogramlazy", DefaultRegistry, sample) }}, + } + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + test.ctor() + } + }) + } +} + func BenchmarkRegistryGetOrRegisterParallel_8(b *testing.B) { benchmarkRegistryGetOrRegisterParallel(b, 8) } @@ -306,7 +331,7 @@ func TestPrefixedChildRegistryGet(t *testing.T) { } func TestChildPrefixedRegistryRegister(t *testing.T) { - r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.") + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") err := r.Register("foo", NewCounter()) c := NewCounter() Register("bar", c) diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index e68206e0cd..1f9b895de5 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a // new ResettingTimer. func GetOrRegisterResettingTimer(name string, r Registry) *ResettingTimer { - return getOrRegister(name, NewResettingTimer, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewResettingTimer() }).(*ResettingTimer) } // NewRegisteredResettingTimer constructs and registers a new ResettingTimer. diff --git a/metrics/runtimehistogram.go b/metrics/runtimehistogram.go index 2ed6213bb4..acad10ffd9 100644 --- a/metrics/runtimehistogram.go +++ b/metrics/runtimehistogram.go @@ -11,10 +11,7 @@ func getOrRegisterRuntimeHistogram(name string, scale float64, r Registry) *runt if r == nil { r = DefaultRegistry } - - constructor := func() Histogram { return newRuntimeHistogram(scale) } - - return getOrRegister(name, constructor, r).(*runtimeHistogram) + return r.GetOrRegister(name, func() any { return newRuntimeHistogram(scale) }).(*runtimeHistogram) } // runtimeHistogram wraps a runtime/metrics histogram. diff --git a/metrics/timer.go b/metrics/timer.go index 5b9165be9d..daac41bf85 100644 --- a/metrics/timer.go +++ b/metrics/timer.go @@ -10,7 +10,10 @@ import ( // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func GetOrRegisterTimer(name string, r Registry) *Timer { - return getOrRegister(name, NewTimer, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewTimer() }).(*Timer) } // NewCustomTimer constructs a new Timer from a Histogram and a Meter. diff --git a/node/defaults.go b/node/defaults.go index ce3e636f5a..f4b0a08c3c 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -73,6 +73,8 @@ var DefaultConfig = Config{ ListenAddr: ":30303", MaxPeers: 50, NAT: nat.Any(), + DiscoveryV4: true, + DiscoveryV5: true, TxArrivalWait: 500 * time.Millisecond, }, DBEngine: "", // Use whatever exists, will default to Pebble if non-existent and supported diff --git a/node/rpcstack.go b/node/rpcstack.go index 14426420fa..0b792ef61a 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -149,6 +149,9 @@ func (h *httpServer) start() error { // Initialize the server. h.server = &http.Server{Handler: h} + h.server.Protocols = new(http.Protocols) + h.server.Protocols.SetHTTP1(true) + h.server.Protocols.SetUnencryptedHTTP2(true) if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 2478f8e5d7..17dc169e76 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -646,6 +646,38 @@ func TestHTTPWriteTimeout(t *testing.T) { }) } +func TestHTTP2H2C(t *testing.T) { + srv := createAndStartServer(t, &httpConfig{}, false, &wsConfig{}, nil) + defer srv.stop() + + // Create an HTTP/2 cleartext client. + transport := &http.Transport{} + transport.Protocols = new(http.Protocols) + transport.Protocols.SetUnencryptedHTTP2(true) + client := &http.Client{Transport: transport} + + body := strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"rpc_modules","params":[]}`) + resp, err := client.Post("http://"+srv.listenAddr(), "application/json", body) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + if resp.Proto != "HTTP/2.0" { + t.Fatalf("expected HTTP/2.0, got %s", resp.Proto) + } + result, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(result), "jsonrpc") { + t.Fatalf("unexpected response: %s", result) + } +} + func apis() []rpc.API { return []rpc.API{ { diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 6083ac6f74..73209fd8c8 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -64,7 +64,7 @@ function compile_fuzzer() { go get github.com/holiman/gofuzz-shim/testing if [[ $SANITIZER == *coverage* ]]; then - coverbuild $path $function $fuzzer $coverpkg + coverbuild $path $function $fuzzer else gofuzz-shim --func $function --package $package -f $file -o $fuzzer.a $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 2d5613690a..9ea9595eb6 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -338,7 +338,6 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error // Create header. head := c.makeHeader(toID, flagWhoareyou, 0) - head.AuthData = slices.Clone(c.buf.Bytes()) head.Nonce = packet.Nonce // Encode auth data. diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index 9fa9408b70..6b980c598d 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -27,9 +27,8 @@ import ( "slices" "strings" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" @@ -290,7 +289,7 @@ const ( ) func subdomain(e entry) string { - h := sha3.NewLegacyKeccak256() + h := keccak.NewLegacyKeccak256() io.WriteString(h, e.String()) return b32format.EncodeToString(h.Sum(nil)[:16]) @@ -301,7 +300,7 @@ func (e *rootEntry) String() string { } func (e *rootEntry) sigHash() []byte { - h := sha3.NewLegacyKeccak256() + h := keccak.NewLegacyKeccak256() fmt.Fprintf(h, rootPrefix+" e=%s l=%s seq=%d", e.eroot, e.lroot, e.seq) return h.Sum(nil) diff --git a/p2p/enode/idscheme.go b/p2p/enode/idscheme.go index fbe1815b82..c52280e0e4 100644 --- a/p2p/enode/idscheme.go +++ b/p2p/enode/idscheme.go @@ -21,10 +21,9 @@ import ( "errors" "io" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" ) @@ -50,7 +49,7 @@ func SignV4(r *enr.Record, privkey *ecdsa.PrivateKey) error { cpy.Set(enr.ID("v4")) cpy.Set(Secp256k1(privkey.PublicKey)) - h := sha3.NewLegacyKeccak256() + h := keccak.NewLegacyKeccak256() rlp.Encode(h, cpy.AppendElements(nil)) sig, err := crypto.Sign(h.Sum(nil), privkey) @@ -74,7 +73,7 @@ func (V4ID) Verify(r *enr.Record, sig []byte) error { return errors.New("invalid public key") } - h := sha3.NewLegacyKeccak256() + h := keccak.NewLegacyKeccak256() rlp.Encode(h, r.AppendElements(nil)) if !crypto.VerifySignature(entry, h.Sum(nil), sig) { diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index 5dda7d0efb..674f922757 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -115,30 +115,30 @@ func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.I }) } // For IGDv1 and v1 services we should first try to add with extport. + var lastErr error for i := 0; i < retryCount+1; i++ { - err := n.withRateLimit(func() error { + lastErr = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) - if err == nil { + if lastErr == nil { return uint16(extport), nil } - log.Debug("Failed to add port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", err) + log.Debug("Failed to add port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", lastErr) } // If above fails, we retry with a random port. // We retry several times because of possible port conflicts. - var err error for i := 0; i < randomCount; i++ { extport = n.randomPort() - err := n.withRateLimit(func() error { + lastErr = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) - if err == nil { + if lastErr == nil { return uint16(extport), nil } - log.Debug("Failed to add random port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", err) + log.Debug("Failed to add random port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", lastErr) } - return 0, err + return 0, lastErr } func (n *upnp) randomPort() int { diff --git a/p2p/rlpx/rlpx.go b/p2p/rlpx/rlpx.go index 238bc49ca3..a2e9208ab1 100644 --- a/p2p/rlpx/rlpx.go +++ b/p2p/rlpx/rlpx.go @@ -24,6 +24,7 @@ import ( "crypto/ecdsa" "crypto/hmac" "crypto/rand" + "crypto/subtle" "encoding/binary" "errors" "fmt" @@ -34,11 +35,10 @@ import ( "time" "github.com/golang/snappy" - "golang.org/x/crypto/sha3" - "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/ecies" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/rlp" ) @@ -523,11 +523,11 @@ func (h *handshakeState) secrets(auth, authResp []byte) (Secrets, error) { } // setup sha3 instances for the MACs - mac1 := sha3.NewLegacyKeccak256() + mac1 := keccak.NewLegacyKeccak256() mac1.Write(xor(s.MAC, h.respNonce)) mac1.Write(auth) - mac2 := sha3.NewLegacyKeccak256() + mac2 := keccak.NewLegacyKeccak256() mac2.Write(xor(s.MAC, h.initNonce)) mac2.Write(authResp) @@ -731,6 +731,6 @@ func exportPubkey(pub *ecies.PublicKey) []byte { func xor(one, other []byte) (xor []byte) { xor = make([]byte, len(one)) - bitutil.XORBytes(xor, one, other) + subtle.XORBytes(xor, one, other) return xor } diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index c699b3904f..6c6665d5dd 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -191,10 +191,10 @@ func (t *Tracker) Fulfil(peer string, version uint, code uint64, id uint64) { return } // Everything matches, mark the request serviced and meter it + wasHead := req.expire.Prev() == nil t.expire.Remove(req.expire) delete(t.pending, id) - - if req.expire.Prev() == nil { + if wasHead { if t.wake.Stop() { t.schedule() } diff --git a/params/protocol_params.go b/params/protocol_params.go index f03f7cfcaf..7f8cd458ad 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -33,7 +33,7 @@ const ( MaximumExtraDataSize uint64 = 32 // Maximum size extra data may be after Genesis. ExpByteGas uint64 = 10 // Times ceil(log256(exponent)) for the EXP instruction. - SloadGas uint64 = 50 // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. + SloadGas uint64 = 50 // CallValueTransferGas uint64 = 9000 // Paid for CALL when the value transfer is non-zero. CallNewAccountGas uint64 = 25000 // Paid for CALL when the destination address didn't exist prior. TxGas uint64 = 21000 // Per transaction not creating a contract. NOTE: Not payable on data of calls between transactions. @@ -93,7 +93,7 @@ const ( CallCreateDepth uint64 = 1024 // Maximum depth of call/create stack. ExpGas uint64 = 10 // Once per EXP instruction LogGas uint64 = 375 // Per LOG* operation. - CopyGas uint64 = 3 // + CopyGas uint64 = 3 // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. StackLimit uint64 = 1024 // Maximum size of VM stack allowed. TierStepGas uint64 = 0 // Once per operation, for a selection of them. LogTopicGas uint64 = 375 // Multiplied by the * of the LOG*, per LOG transaction. e.g. LOG0 incurs 0 * c_txLogTopicGas, LOG4 incurs 4 * c_txLogTopicGas. diff --git a/rlp/encode.go b/rlp/encode.go index 1336940132..949ea9782f 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -106,6 +106,29 @@ func EncodeToReader(val interface{}) (size int, r io.Reader, err error) { return buf.size(), &encReader{buf: buf}, nil } +// EncodeToRawList encodes val as an RLP list and returns it as a RawList. +func EncodeToRawList[T any](val []T) (RawList[T], error) { + if len(val) == 0 { + return RawList[T]{}, nil + } + + // Encode the value to an internal buffer. + buf := getEncBuffer() + defer encBufferPool.Put(buf) + if err := buf.encode(val); err != nil { + return RawList[T]{}, err + } + + // Create the RawList. RawList assumes the initial list header is padded + // 9 bytes, so we have to determine the offset where the value should be + // placed. + contentSize := buf.lheads[0].size + bytes := make([]byte, contentSize+9) + offset := 9 - headsize(uint64(contentSize)) + buf.copyTo(bytes[offset:]) + return RawList[T]{enc: bytes, length: len(val)}, nil +} + type listhead struct { offset int // index of this header in string data size int // total size of encoded data (including list headers) diff --git a/rlp/iterator.go b/rlp/iterator.go index 0e1a325236..6a1ea0d7b6 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -16,49 +16,76 @@ package rlp -type listIterator struct { - data []byte - next []byte - err error +// Iterator is an iterator over the elements of an encoded container. +type Iterator struct { + data []byte + next []byte + offset int + err error } -// NewListIterator creates an iterator for the (list) represented by data -func NewListIterator(data RawValue) (*listIterator, error) { +// NewListIterator creates an iterator for the (list) represented by data. +func NewListIterator(data RawValue) (Iterator, error) { k, t, c, err := readKind(data) if err != nil { - return nil, err + return Iterator{}, err } if k != List { - return nil, ErrExpectedList + return Iterator{}, ErrExpectedList } - - it := &listIterator{ - data: data[t : t+c], - } - + it := newIterator(data[t:t+c], int(t)) return it, nil } -// Next forwards the iterator one step, returns true if it was not at end yet -func (it *listIterator) Next() bool { +func newIterator(data []byte, initialOffset int) Iterator { + return Iterator{data: data, offset: initialOffset} +} + +// Next forwards the iterator one step. +// Returns true if there is a next item or an error occurred on this step (check Err()). +// On parse error, the iterator is marked finished and subsequent calls return false. +func (it *Iterator) Next() bool { if len(it.data) == 0 { return false } _, t, c, err := readKind(it.data) - it.next = it.data[:t+c] - it.data = it.data[t+c:] - it.err = err - + if err != nil { + it.next = nil + it.err = err + // Mark iteration as finished to avoid potential infinite loops on subsequent Next calls. + it.data = nil + return true + } + length := t + c + it.next = it.data[:length] + it.data = it.data[length:] + it.offset += int(length) + it.err = nil return true } -// Value returns the current value -func (it *listIterator) Value() []byte { +// Value returns the current value. +func (it *Iterator) Value() []byte { return it.next } -func (it *listIterator) Err() error { +// Count returns the remaining number of items. +// Note this is O(n) and the result may be incorrect if the list data is invalid. +// The returned count is always an upper bound on the remaining items +// that will be visited by the iterator. +func (it *Iterator) Count() int { + count, _ := CountValues(it.data) + return count +} + +// Offset returns the offset of the current value into the list data. +func (it *Iterator) Offset() int { + return it.offset - len(it.next) +} + +// Err returns the error that caused Next to return false, if any. +func (it *Iterator) Err() error { return it.err } diff --git a/rlp/iterator_test.go b/rlp/iterator_test.go index 4dab97e162..b9e08e4e80 100644 --- a/rlp/iterator_test.go +++ b/rlp/iterator_test.go @@ -17,6 +17,7 @@ package rlp import ( + "io" "testing" "github.com/ethereum/go-ethereum/common/hexutil" @@ -39,16 +40,25 @@ func TestIterator(t *testing.T) { } txs := it.Value() + if offset := it.Offset(); offset != 3 { + t.Fatal("wrong offset", offset, "want 3") + } + // Check that uncles exist if !it.Next() { t.Fatal("expected two elems, got one") } + if offset := it.Offset(); offset != 219 { + t.Fatal("wrong offset", offset, "want 219") + } txit, err := NewListIterator(txs) if err != nil { t.Fatal(err) } - + if c := txit.Count(); c != 2 { + t.Fatal("wrong Count:", c) + } var i = 0 for txit.Next() { @@ -63,3 +73,65 @@ func TestIterator(t *testing.T) { t.Errorf("count wrong, expected %d got %d", i, exp) } } + +func TestIteratorErrors(t *testing.T) { + tests := []struct { + input []byte + wantCount int // expected Count before iterating + wantErr error + }{ + // Second item string header claims 3 bytes content, but only 2 remain. + {unhex("C4 01 83AABB"), 2, ErrValueTooLarge}, + // Second item truncated: B9 requires 2 size bytes, none available. + {unhex("C2 01 B9"), 2, io.ErrUnexpectedEOF}, + // 0x05 should be encoded directly, not as 81 05. + {unhex("C3 01 8105"), 2, ErrCanonSize}, + // Long-form string header B8 used for 1-byte content (< 56). + {unhex("C4 01 B801AA"), 2, ErrCanonSize}, + // Long-form list header F8 used for 1-byte content (< 56). + {unhex("C4 01 F80101"), 2, ErrCanonSize}, + } + for _, tt := range tests { + it, err := NewListIterator(tt.input) + if err != nil { + t.Fatal("NewListIterator error:", err) + } + if c := it.Count(); c != tt.wantCount { + t.Fatalf("%x: Count = %d, want %d", tt.input, c, tt.wantCount) + } + n := 0 + for it.Next() { + if it.Err() != nil { + break + } + n++ + } + if wantN := tt.wantCount - 1; n != wantN { + t.Fatalf("%x: got %d valid items, want %d", tt.input, n, wantN) + } + if it.Err() != tt.wantErr { + t.Fatalf("%x: got error %v, want %v", tt.input, it.Err(), tt.wantErr) + } + if it.Next() { + t.Fatalf("%x: Next returned true after error", tt.input) + } + } +} + +func FuzzIteratorCount(f *testing.F) { + examples := [][]byte{unhex("010203"), unhex("018142"), unhex("01830202")} + for _, e := range examples { + f.Add(e) + } + f.Fuzz(func(t *testing.T, in []byte) { + it := newIterator(in, 0) + count := it.Count() + i := 0 + for it.Next() { + i++ + } + if i != count { + t.Fatalf("%x: count %d not equal to %d iterations", in, count, i) + } + }) +} diff --git a/rlp/raw.go b/rlp/raw.go index 03343340e8..782da83d05 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -17,8 +17,10 @@ package rlp import ( + "fmt" "io" "reflect" + "slices" ) // RawValue represents an encoded RLP value and can be used to delay @@ -28,6 +30,144 @@ type RawValue []byte var rawValueType = reflect.TypeFor[RawValue]() +// RawList represents an encoded RLP list. +type RawList[T any] struct { + // The list is stored in encoded form. + // Note this buffer has some special properties: + // + // - if the buffer is nil, it's the zero value, representing + // an empty list. + // - if the buffer is non-nil, it must have a length of at least + // 9 bytes, which is reserved padding for the encoded list header. + // The remaining bytes, enc[9:], store the content bytes of the list. + // + // The implementation code mostly works with the Content method because it + // returns something valid either way. + enc []byte + + // length holds the number of items in the list. + length int +} + +// Content returns the RLP-encoded data of the list. +// This does not include the list-header. +// The return value is a direct reference to the internal buffer, not a copy. +func (r *RawList[T]) Content() []byte { + if r.enc == nil { + return nil + } else { + return r.enc[9:] + } +} + +// EncodeRLP writes the encoded list to the writer. +func (r RawList[T]) EncodeRLP(w io.Writer) error { + _, err := w.Write(r.Bytes()) + return err +} + +// Bytes returns the RLP encoding of the list. +// Note the return value aliases the internal buffer. +func (r *RawList[T]) Bytes() []byte { + if r == nil || r.enc == nil { + return []byte{0xC0} // zero value encodes as empty list + } + n := puthead(r.enc, 0xC0, 0xF7, uint64(len(r.Content()))) + copy(r.enc[9-n:], r.enc[:n]) + return r.enc[9-n:] +} + +// DecodeRLP decodes the list. This does not perform validation of the items! +func (r *RawList[T]) DecodeRLP(s *Stream) error { + k, size, err := s.Kind() + if err != nil { + return err + } + if k != List { + return fmt.Errorf("%w for %T", ErrExpectedList, r) + } + enc := make([]byte, 9+size) + if err := s.readFull(enc[9:]); err != nil { + return err + } + n, err := CountValues(enc[9:]) + if err != nil { + if err == ErrValueTooLarge { + return ErrElemTooLarge + } + return err + } + *r = RawList[T]{enc: enc, length: n} + return nil +} + +// Items decodes and returns all items in the list. +func (r *RawList[T]) Items() ([]T, error) { + items := make([]T, r.Len()) + it := r.ContentIterator() + for i := 0; it.Next(); i++ { + if err := DecodeBytes(it.Value(), &items[i]); err != nil { + return items[:i], err + } + } + return items, nil +} + +// Len returns the number of items in the list. +func (r *RawList[T]) Len() int { + return r.length +} + +// Size returns the encoded size of the list. +func (r *RawList[T]) Size() uint64 { + return ListSize(uint64(len(r.Content()))) +} + +// ContentIterator returns an iterator over the content of the list. +// Note the offsets returned by iterator.Offset are relative to the +// Content bytes of the list. +func (r *RawList[T]) ContentIterator() Iterator { + return newIterator(r.Content(), 0) +} + +// Append adds an item to the end of the list. +func (r *RawList[T]) Append(item T) error { + if r.enc == nil { + r.enc = make([]byte, 9) + } + + eb := getEncBuffer() + defer encBufferPool.Put(eb) + + if err := eb.encode(item); err != nil { + return err + } + prevEnd := len(r.enc) + end := prevEnd + eb.size() + r.enc = slices.Grow(r.enc, eb.size())[:end] + eb.copyTo(r.enc[prevEnd:end]) + r.length++ + return nil +} + +// AppendRaw adds an encoded item to the list. +// The given byte slice must contain exactly one RLP value. +func (r *RawList[T]) AppendRaw(b []byte) error { + _, tagsize, contentsize, err := readKind(b) + if err != nil { + return err + } + if tagsize+contentsize != uint64(len(b)) { + return fmt.Errorf("rlp: input has trailing bytes in AppendRaw") + } + if r.enc == nil { + r.enc = make([]byte, 9) + } + r.enc = append(r.enc, b...) + r.length++ + return nil +} + // StringSize returns the encoded size of a string. func StringSize(s string) uint64 { switch n := len(s); n { @@ -154,7 +294,7 @@ func CountValues(b []byte) (int, error) { for ; len(b) > 0; i++ { _, tagsize, size, err := readKind(b) if err != nil { - return 0, err + return i + 1, err } b = b[tagsize+size:] @@ -163,6 +303,45 @@ func CountValues(b []byte) (int, error) { return i, nil } +// SplitListValues extracts the raw elements from the list RLP-encoding blob. +// +// Note: the returned slice must not be modified, as it shares the same +// backing array as the original slice. It's acceptable to deep-copy the elements +// out if necessary, but let's stick with this approach for less allocation +// overhead. +func SplitListValues(b []byte) ([][]byte, error) { + b, _, err := SplitList(b) + if err != nil { + return nil, err + } + n, err := CountValues(b) + if err != nil { + return nil, err + } + var elements = make([][]byte, 0, n) + + for len(b) > 0 { + _, tagsize, size, err := readKind(b) + if err != nil { + return nil, err + } + elements = append(elements, b[:tagsize+size]) + b = b[tagsize+size:] + } + return elements, nil +} + +// MergeListValues takes a list of raw elements and rlp-encodes them as list. +func MergeListValues(elems [][]byte) ([]byte, error) { + w := NewEncoderBuffer(nil) + offset := w.List() + for _, elem := range elems { + w.Write(elem) + } + w.ListEnd(offset) + return w.ToBytes(), nil +} + func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) { if len(buf) == 0 { return 0, 0, 0, io.ErrUnexpectedEOF diff --git a/rlp/raw_test.go b/rlp/raw_test.go index e8f4ea705b..7d544c3577 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -19,11 +19,259 @@ package rlp import ( "bytes" "errors" + "fmt" "io" + "reflect" "testing" "testing/quick" ) +type rawListTest[T any] struct { + input string + content string + items []T + length int +} + +func (test rawListTest[T]) name() string { + return fmt.Sprintf("%T-%d", *new(T), test.length) +} + +func (test rawListTest[T]) run(t *testing.T) { + // check decoding and properties + input := unhex(test.input) + inputSize := len(input) + var rl RawList[T] + if err := DecodeBytes(input, &rl); err != nil { + t.Fatal("decode failed:", err) + } + if l := rl.Len(); l != test.length { + t.Fatalf("wrong Len %d, want %d", l, test.length) + } + if sz := rl.Size(); sz != uint64(inputSize) { + t.Fatalf("wrong Size %d, want %d", sz, inputSize) + } + items, err := rl.Items() + if err != nil { + t.Fatal("Items failed:", err) + } + if !reflect.DeepEqual(items, test.items) { + t.Fatal("wrong items:", items) + } + if !bytes.Equal(rl.Content(), unhex(test.content)) { + t.Fatalf("wrong Content %x, want %s", rl.Content(), test.content) + } + if !bytes.Equal(rl.Bytes(), unhex(test.input)) { + t.Fatalf("wrong Bytes %x, want %s", rl.Bytes(), test.input) + } + + // check iterator + it := rl.ContentIterator() + i := 0 + for it.Next() { + var item T + if err := DecodeBytes(it.Value(), &item); err != nil { + t.Fatalf("item %d decode error: %v", i, err) + } + if !reflect.DeepEqual(item, items[i]) { + t.Fatalf("iterator has wrong item %v at %d", item, i) + } + i++ + } + if i != test.length { + t.Fatalf("iterator produced %d values, want %d", i, test.length) + } + if it.Err() != nil { + t.Fatalf("iterator error: %v", it.Err()) + } + + // check encoding round trip + output, err := EncodeToBytes(&rl) + if err != nil { + t.Fatal("encode error:", err) + } + if !bytes.Equal(output, unhex(test.input)) { + t.Fatalf("encoding does not round trip: %x", output) + } + + // check EncodeToRawList on items produces same bytes + encRL, err := EncodeToRawList(test.items) + if err != nil { + t.Fatal("EncodeToRawList error:", err) + } + encRLOutput, err := EncodeToBytes(&encRL) + if err != nil { + t.Fatal("EncodeToBytes of encoded list failed:", err) + } + if !bytes.Equal(encRLOutput, output) { + t.Fatalf("wrong encoding of EncodeToRawList result: %x", encRLOutput) + } +} + +func TestRawList(t *testing.T) { + tests := []interface { + name() string + run(t *testing.T) + }{ + rawListTest[uint64]{ + input: "C0", + content: "", + items: []uint64{}, + length: 0, + }, + rawListTest[uint64]{ + input: "C3010203", + content: "010203", + items: []uint64{1, 2, 3}, + length: 3, + }, + rawListTest[simplestruct]{ + input: "C6C20102C20304", + content: "C20102C20304", + items: []simplestruct{{1, "\x02"}, {3, "\x04"}}, + length: 2, + }, + rawListTest[string]{ + input: "F83C836161618362626283636363836464648365656583666666836767678368686883696969836A6A6A836B6B6B836C6C6C836D6D6D836E6E6E836F6F6F", + content: "836161618362626283636363836464648365656583666666836767678368686883696969836A6A6A836B6B6B836C6C6C836D6D6D836E6E6E836F6F6F", + items: []string{"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj", "kkk", "lll", "mmm", "nnn", "ooo"}, + length: 15, + }, + } + + for _, test := range tests { + t.Run(test.name(), test.run) + } +} + +func TestRawListEmpty(t *testing.T) { + // zero value list + var rl RawList[uint64] + b, _ := EncodeToBytes(&rl) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("empty RawList has wrong encoding %x", b) + } + if rl.Len() != 0 { + t.Fatalf("empty list has Len %d", rl.Len()) + } + if rl.Size() != 1 { + t.Fatalf("empty list has Size %d", rl.Size()) + } + if len(rl.Content()) > 0 { + t.Fatalf("empty list has non-empty Content") + } + if !bytes.Equal(rl.Bytes(), []byte{0xC0}) { + t.Fatalf("empty list has wrong encoding") + } + + // nil pointer + var nilptr *RawList[uint64] + b, _ = EncodeToBytes(nilptr) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("nil pointer to RawList has wrong encoding %x", b) + } +} + +// This checks that *RawList works in an 'optional' context. +func TestRawListOptional(t *testing.T) { + type foo struct { + L *RawList[uint64] `rlp:"optional"` + } + // nil pointer encoding + var empty foo + b, _ := EncodeToBytes(empty) + if !bytes.Equal(b, unhex("C0")) { + t.Fatalf("nil pointer to RawList has wrong encoding %x", b) + } + // decoding + var dec foo + if err := DecodeBytes(unhex("C0"), &dec); err != nil { + t.Fatal(err) + } + if dec.L != nil { + t.Fatal("rawlist was decoded as non-nil") + } +} + +func TestRawListAppend(t *testing.T) { + var rl RawList[simplestruct] + + v1 := simplestruct{1, "one"} + v2 := simplestruct{2, "two"} + if err := rl.Append(v1); err != nil { + t.Fatal("append 1 failed:", err) + } + if err := rl.Append(v2); err != nil { + t.Fatal("append 2 failed:", err) + } + + if rl.Len() != 2 { + t.Fatalf("wrong Len %d", rl.Len()) + } + if rl.Size() != 13 { + t.Fatalf("wrong Size %d", rl.Size()) + } + if !bytes.Equal(rl.Content(), unhex("C501836F6E65 C5028374776F")) { + t.Fatalf("wrong Content %x", rl.Content()) + } + encoded, _ := EncodeToBytes(&rl) + if !bytes.Equal(encoded, unhex("CC C501836F6E65 C5028374776F")) { + t.Fatalf("wrong encoding %x", encoded) + } +} + +func TestRawListAppendRaw(t *testing.T) { + var rl RawList[uint64] + + if err := rl.AppendRaw(unhex("01")); err != nil { + t.Fatal("AppendRaw(01) failed:", err) + } + if err := rl.AppendRaw(unhex("820102")); err != nil { + t.Fatal("AppendRaw(820102) failed:", err) + } + if rl.Len() != 2 { + t.Fatalf("wrong Len %d after valid appends", rl.Len()) + } + + if err := rl.AppendRaw(nil); err == nil { + t.Fatal("AppendRaw(nil) should fail") + } + if err := rl.AppendRaw(unhex("0102")); err == nil { + t.Fatal("AppendRaw(0102) should fail due to trailing bytes") + } + if err := rl.AppendRaw(unhex("8201")); err == nil { + t.Fatal("AppendRaw(8201) should fail due to truncated value") + } + if rl.Len() != 2 { + t.Fatalf("wrong Len %d after invalid appends, want 2", rl.Len()) + } +} + +func TestRawListDecodeInvalid(t *testing.T) { + tests := []struct { + input string + err error + }{ + // Single item with non-canonical size (0x81 wrapping byte <= 0x7F). + {input: "C28142", err: ErrCanonSize}, + // Single item claiming more bytes than available in the list. + {input: "C484020202", err: ErrElemTooLarge}, + // Two items, second has non-canonical size. + {input: "C3018142", err: ErrCanonSize}, + // Two items, second claims more bytes than remain in the list. + {input: "C401830202", err: ErrElemTooLarge}, + // Item is a sub-list whose declared size exceeds available bytes. + {input: "C3C40102", err: ErrElemTooLarge}, + } + for _, test := range tests { + var rl RawList[RawValue] + err := DecodeBytes(unhex(test.input), &rl) + if !errors.Is(err, test.err) { + t.Errorf("input %s: error mismatch: got %v, want %v", test.input, err, test.err) + } + } +} + func TestCountValues(t *testing.T) { tests := []struct { input string // note: spaces in input are stripped by unhex @@ -40,9 +288,9 @@ func TestCountValues(t *testing.T) { {"820101 820202 8403030303 04", 4, nil}, // size errors - {"8142", 0, ErrCanonSize}, - {"01 01 8142", 0, ErrCanonSize}, - {"02 84020202", 0, ErrValueTooLarge}, + {"8142", 1, ErrCanonSize}, + {"01 01 8142", 3, ErrCanonSize}, + {"02 84020202", 2, ErrValueTooLarge}, { input: "A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", @@ -353,3 +601,269 @@ func TestBytesSize(t *testing.T) { } } } + +func TestSplitListValues(t *testing.T) { + tests := []struct { + name string + input string // hex-encoded RLP list + want []string // hex-encoded expected elements + wantErr error + }{ + { + name: "empty list", + input: "C0", + want: []string{}, + }, + { + name: "single byte element", + input: "C101", + want: []string{"01"}, + }, + { + name: "single empty string", + input: "C180", + want: []string{"80"}, + }, + { + name: "two byte elements", + input: "C20102", + want: []string{"01", "02"}, + }, + { + name: "three elements", + input: "C3010203", + want: []string{"01", "02", "03"}, + }, + { + name: "mixed size elements", + input: "C80182020283030303", + want: []string{"01", "820202", "83030303"}, + }, + { + name: "string elements", + input: "C88363617483646F67", + want: []string{"83636174", "83646F67"}, // cat,dog + }, + { + name: "nested list element", + input: "C4C3010203", // [[1,2,3]] + want: []string{"C3010203"}, // [1,2,3] + }, + { + name: "multiple nested lists", + input: "C6C20102C20304", // [[1,2],[3,4]] + want: []string{"C20102", "C20304"}, // [1,2], [3,4] + }, + { + name: "large list", + input: "C6010203040506", + want: []string{"01", "02", "03", "04", "05", "06"}, + }, + { + name: "list with empty strings", + input: "C3808080", + want: []string{"80", "80", "80"}, + }, + // Error cases + { + name: "single byte", + input: "01", + wantErr: ErrExpectedList, + }, + { + name: "string", + input: "83636174", + wantErr: ErrExpectedList, + }, + { + name: "empty input", + input: "", + wantErr: io.ErrUnexpectedEOF, + }, + { + name: "invalid list - value too large", + input: "C60102030405", + wantErr: ErrValueTooLarge, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SplitListValues(unhex(tt.input)) + if !errors.Is(err, tt.wantErr) { + t.Errorf("SplitListValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + if len(got) != len(tt.want) { + t.Errorf("SplitListValues() got %d elements, want %d", len(got), len(tt.want)) + return + } + for i, elem := range got { + want := unhex(tt.want[i]) + if !bytes.Equal(elem, want) { + t.Errorf("SplitListValues() element[%d] = %x, want %x", i, elem, want) + } + } + }) + } +} + +func TestMergeListValues(t *testing.T) { + tests := []struct { + name string + elems []string // hex-encoded RLP elements + want string // hex-encoded expected result + wantErr error + }{ + { + name: "empty list", + elems: []string{}, + want: "C0", + }, + { + name: "single byte element", + elems: []string{"01"}, + want: "C101", + }, + { + name: "single empty string", + elems: []string{"80"}, + want: "C180", + }, + { + name: "two byte elements", + elems: []string{"01", "02"}, + want: "C20102", + }, + { + name: "three elements", + elems: []string{"01", "02", "03"}, + want: "C3010203", + }, + { + name: "mixed size elements", + elems: []string{"01", "820202", "83030303"}, + want: "C80182020283030303", + }, + { + name: "string elements", + elems: []string{"83636174", "83646F67"}, // cat, dog + want: "C88363617483646F67", + }, + { + name: "nested list element", + elems: []string{"C20102", "03"}, // [[1, 2], 3] + want: "C4C2010203", + }, + { + name: "multiple nested lists", + elems: []string{"C20102", "C3030405"}, // [[1,2],[3,4,5]], + want: "C7C20102C3030405", + }, + { + name: "large list", + elems: []string{"01", "02", "03", "04", "05", "06"}, + want: "C6010203040506", + }, + { + name: "list with empty strings", + elems: []string{"80", "80", "80"}, + want: "C3808080", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + elems := make([][]byte, len(tt.elems)) + for i, s := range tt.elems { + elems[i] = unhex(s) + } + got, err := MergeListValues(elems) + if !errors.Is(err, tt.wantErr) { + t.Errorf("MergeListValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + want := unhex(tt.want) + if !bytes.Equal(got, want) { + t.Errorf("MergeListValues() = %x, want %x", got, want) + } + }) + } +} + +func TestSplitMergeList(t *testing.T) { + tests := []struct { + name string + input string // hex-encoded RLP list + }{ + { + name: "empty list", + input: "C0", + }, + { + name: "single byte element", + input: "C101", + }, + { + name: "two byte elements", + input: "C20102", + }, + { + name: "three elements", + input: "C3010203", + }, + { + name: "mixed size elements", + input: "C80182020283030303", + }, + { + name: "string elements", + input: "C88363617483646F67", // [cat, dog] + }, + { + name: "nested list element", + input: "C4C2010203", // [[1,2],3] + }, + { + name: "multiple nested lists", + input: "C6C20102C20304", // [[1,2],[3,4]] + }, + { + name: "large list", + input: "C6010203040506", // [1,2,3,4,5,6] + }, + { + name: "list with empty strings", + input: "C3808080", // ["", "", ""] + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + original := unhex(tt.input) + + // Split the list + elements, err := SplitListValues(original) + if err != nil { + t.Fatalf("SplitListValues() error = %v", err) + } + + // Merge back + merged, err := MergeListValues(elements) + if err != nil { + t.Fatalf("MergeListValues() error = %v", err) + } + + // The merged result should match the original + if !bytes.Equal(merged, original) { + t.Errorf("Round trip failed: original = %x, merged = %x", original, merged) + } + }) + } +} diff --git a/rpc/handler.go b/rpc/handler.go index 2543ebfb12..133b8999ea 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -655,8 +655,11 @@ type limitedBuffer struct { } func (buf *limitedBuffer) Write(data []byte) (int, error) { - avail := max(buf.limit, len(buf.output)) - if len(data) < avail { + avail := buf.limit - len(buf.output) + if avail <= 0 { + return 0, errTruncatedOutput + } + if len(data) <= avail { buf.output = append(buf.output, data...) return len(data), nil } diff --git a/rpc/http.go b/rpc/http.go index d17c9836ff..065c5fa887 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -30,6 +30,9 @@ import ( "strconv" "sync" "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) const ( @@ -169,13 +172,21 @@ func newClientTransportHTTP(endpoint string, cfg *clientConfig) reconnectFunc { } } +// cleanlyCloseBody avoids sending unnecessary RST_STREAM and PING frames by +// ensuring the whole body is read before being closed. +// See https://blog.cloudflare.com/go-and-enhance-your-calm/#reading-bodies-in-go-can-be-unintuitive +func cleanlyCloseBody(body io.ReadCloser) error { + io.Copy(io.Discard, body) + return body.Close() +} + func (c *Client) sendHTTP(ctx context.Context, op *requestOp, msg interface{}) error { hc := c.writeConn.(*httpConn) respBody, err := hc.doRequest(ctx, msg) if err != nil { return err } - defer respBody.Close() + defer cleanlyCloseBody(respBody) var resp jsonrpcMessage batch := [1]*jsonrpcMessage{&resp} @@ -194,7 +205,7 @@ func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonr return err } - defer respBody.Close() + defer cleanlyCloseBody(respBody) var respmsgs []*jsonrpcMessage if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { @@ -239,7 +250,7 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos if _, err := buf.ReadFrom(resp.Body); err == nil { body = buf.Bytes() } - resp.Body.Close() + cleanlyCloseBody(resp.Body) return nil, HTTPError{ Status: resp.Status, StatusCode: resp.StatusCode, @@ -332,6 +343,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() ctx = context.WithValue(ctx, peerInfoContextKey{}, connInfo) + // Extract trace context from incoming headers. + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) + // All checks passed, create a codec that reads directly from the request body // until EOF, writes the response to w, and orders the server to process a // single request. diff --git a/rpc/http_test.go b/rpc/http_test.go index 11d992b68a..37fe4ff5ff 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -114,7 +114,7 @@ func confirmHTTPRequestYieldsStatusCode(t *testing.T, method, contentType, body t.Fatalf("request failed: %v", err) } - resp.Body.Close() + cleanlyCloseBody(resp.Body) confirmStatusCode(t, resp.StatusCode, expectedStatusCode) } diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index b361359b75..451740a977 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -111,6 +111,7 @@ type SendTxArgs struct { BlobHashes []common.Hash `json:"blobVersionedHashes,omitempty"` // For BlobTxType transactions with blob sidecar + BlobVersion byte `json:"blobVersion,omitempty"` Blobs []kzg4844.Blob `json:"blobs,omitempty"` Commitments []kzg4844.Commitment `json:"commitments,omitempty"` Proofs []kzg4844.Proof `json:"proofs,omitempty"` @@ -242,37 +243,58 @@ func (args *SendTxArgs) validateTxSidecar() error { if args.Commitments != nil && len(args.Commitments) != n { return fmt.Errorf("number of blobs and commitments mismatch (have=%d, want=%d)", len(args.Commitments), n) } - if args.Proofs != nil && len(args.Proofs) != n { - return fmt.Errorf("number of blobs and proofs mismatch (have=%d, want=%d)", len(args.Proofs), n) - } if args.BlobHashes != nil && len(args.BlobHashes) != n { return fmt.Errorf("number of blobs and hashes mismatch (have=%d, want=%d)", len(args.BlobHashes), n) } - + if args.Proofs != nil { + if len(args.Proofs) == n { + // v1 transaction + for i := range args.Blobs { + if err := kzg4844.VerifyBlobProof(&args.Blobs[i], args.Commitments[i], args.Proofs[i]); err != nil { + return fmt.Errorf("failed to verify blob proof: %v", err) + } + } + } else if len(args.Proofs) == n*kzg4844.CellProofsPerBlob { + // v2 transaction + if err := kzg4844.VerifyCellProofs(args.Blobs, args.Commitments, args.Proofs); err != nil { + return fmt.Errorf("failed to verify blob proof: %v", err) + } + } else { + return fmt.Errorf("number of proofs and blobs mismatch (have=%d, want=%d or %d)", len(args.Proofs), n, n*kzg4844.CellProofsPerBlob) + } + } if args.Commitments == nil { // Generate commitment and proof. commitments := make([]kzg4844.Commitment, n) - proofs := make([]kzg4844.Proof, n) - for i, b := range args.Blobs { - c, err := kzg4844.BlobToCommitment(&b) + for i := range args.Blobs { + c, err := kzg4844.BlobToCommitment(&args.Blobs[i]) if err != nil { return fmt.Errorf("blobs[%d]: error computing commitment: %v", i, err) } commitments[i] = c - p, err := kzg4844.ComputeBlobProof(&b, c) - if err != nil { - return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) + } + var proofs []kzg4844.Proof + if args.BlobVersion == types.BlobSidecarVersion1 { + proofs = make([]kzg4844.Proof, 0, n*kzg4844.CellProofsPerBlob) + for i := range args.Blobs { + p, err := kzg4844.ComputeCellProofs(&args.Blobs[i]) + if err != nil { + return fmt.Errorf("blobs[%d]: error computing cell proof: %v", i, err) + } + proofs = append(proofs, p...) + } + } else { + proofs = make([]kzg4844.Proof, 0, n) + for i := range args.Blobs { + p, err := kzg4844.ComputeBlobProof(&args.Blobs[i], commitments[i]) + if err != nil { + return fmt.Errorf("blobs[%d]: error computing proof: %v", i, err) + } + proofs = append(proofs, p) } - proofs[i] = p } args.Commitments = commitments args.Proofs = proofs - } else { - for i, b := range args.Blobs { - if err := kzg4844.VerifyBlobProof(&b, args.Commitments[i], args.Proofs[i]); err != nil { - return fmt.Errorf("failed to verify blob proof: %v", err) - } - } } hashes := make([]common.Hash, n) diff --git a/signer/core/apitypes/types_test.go b/signer/core/apitypes/types_test.go index 64e61649c2..f6d6156f5c 100644 --- a/signer/core/apitypes/types_test.go +++ b/signer/core/apitypes/types_test.go @@ -17,6 +17,7 @@ package apitypes import ( + "crypto/rand" "crypto/sha256" "encoding/json" "testing" @@ -230,3 +231,183 @@ func TestType_TypeName(t *testing.T) { } } } + +func TestValidateTxSidecar(t *testing.T) { + t.Parallel() + + // Helper function to create a test blob and its commitment/proof + createTestBlob := func() (kzg4844.Blob, kzg4844.Commitment, kzg4844.Proof, common.Hash) { + b := make([]byte, 31) + rand.Read(b) + var blob kzg4844.Blob + for i := range b { + blob[i+1] = b[i] + } + commitment, err := kzg4844.BlobToCommitment(&blob) + if err != nil { + t.Fatal(err) + } + proof, err := kzg4844.ComputeBlobProof(&blob, commitment) + if err != nil { + t.Fatal(err) + } + hash := kzg4844.CalcBlobHashV1(sha256.New(), &commitment) + return blob, commitment, proof, hash + } + + // Helper function to create test cell proofs for v1 transactions + createTestCellProofs := func(blob kzg4844.Blob) []kzg4844.Proof { + cellProofs, err := kzg4844.ComputeCellProofs(&blob) + if err != nil { + t.Fatal(err) + } + return cellProofs + } + + blob1, commitment1, proof1, hash1 := createTestBlob() + blob2, commitment2, proof2, hash2 := createTestBlob() + + tests := []struct { + name string + args SendTxArgs + wantErr bool + }{ + { + name: "no blobs - should pass", + args: SendTxArgs{}, + wantErr: false, + }, + { + name: "valid blobs with commitments and proofs", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1, blob2}, + Commitments: []kzg4844.Commitment{commitment1, commitment2}, + Proofs: []kzg4844.Proof{proof1, proof2}, + BlobHashes: []common.Hash{hash1, hash2}, + }, + wantErr: false, + }, + { + name: "valid blobs without commitments/proofs - should generate them", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1}, + }, + wantErr: false, + }, + { + name: "valid blobs with v1 cell proofs", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1}, + Commitments: []kzg4844.Commitment{commitment1}, + Proofs: createTestCellProofs(blob1), + BlobHashes: []common.Hash{hash1}, + }, + wantErr: false, + }, + { + name: "blobs with v1 version flag - should generate cell proofs", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1}, + BlobVersion: types.BlobSidecarVersion1, + }, + wantErr: false, + }, + { + name: "proofs provided but commitments not", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1}, + Proofs: []kzg4844.Proof{proof1}, + }, + wantErr: true, + }, + { + name: "commitments provided but proofs not", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1}, + Commitments: []kzg4844.Commitment{commitment1}, + }, + wantErr: true, + }, + { + name: "mismatch between blobs and commitments", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1, blob2}, + Commitments: []kzg4844.Commitment{commitment1}, // Only one commitment for two blobs + Proofs: []kzg4844.Proof{proof1}, + }, + wantErr: true, + }, + { + name: "mismatch between blobs and hashes", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1, blob2}, + Commitments: []kzg4844.Commitment{commitment1, commitment2}, + Proofs: []kzg4844.Proof{proof1, proof2}, + BlobHashes: []common.Hash{hash1}, // Only one hash for two blobs + }, + wantErr: true, + }, + { + name: "wrong number of proofs", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1, blob2}, + Commitments: []kzg4844.Commitment{commitment1, commitment2}, + Proofs: []kzg4844.Proof{proof1, proof2, proof1}, // 3 proofs for 2 blobs + }, + wantErr: true, + }, + { + name: "invalid blob hash", + args: SendTxArgs{ + Blobs: []kzg4844.Blob{blob1}, + Commitments: []kzg4844.Commitment{commitment1}, + Proofs: []kzg4844.Proof{proof1}, + BlobHashes: []common.Hash{hash2}, // Wrong hash + }, + wantErr: true, + }, + { + name: "invalid proof", + args: SendTxArgs{ + BlobVersion: types.BlobSidecarVersion1, + Blobs: []kzg4844.Blob{blob1}, + Commitments: []kzg4844.Commitment{commitment1}, + Proofs: []kzg4844.Proof{proof1, proof2}, // wrong proof + BlobHashes: []common.Hash{hash1}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Make a copy to avoid modifying the original test case + args := tt.args + err := args.validateTxSidecar() + + if tt.wantErr { + if err == nil { + t.Errorf("validateTxSidecar() expected error but got none") + return + } + } else { + if err != nil { + t.Errorf("validateTxSidecar() unexpected error = %v", err) + } + + // For successful cases, verify that commitments and proofs were generated if they weren't provided + if len(args.Blobs) > 0 { + if args.Commitments == nil || len(args.Commitments) != len(args.Blobs) { + t.Errorf("validateTxSidecar() should have generated commitments") + } + if args.Proofs == nil || (len(args.Proofs) != len(args.Blobs) && len(args.Proofs) != len(args.Blobs)*kzg4844.CellProofsPerBlob) { + t.Errorf("validateTxSidecar() should have generated proofs") + } + if args.BlobHashes == nil || len(args.BlobHashes) != len(args.Blobs) { + t.Errorf("validateTxSidecar() should have generated blob hashes") + } + } + } + }) + } +} diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 2da8af302e..df4a635698 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -126,25 +126,31 @@ func (t *BlockTest) RunV2(snapshotter bool, scheme string, witness bool, tracer return t.run(snapshotter, scheme, witness, tracer, postCheck, true) } +// Network returns the network/fork name for this test. +func (t *BlockTest) Network() string { + return t.json.Network +} + func (t *BlockTest) run(snapshotter bool, scheme string, witness bool, tracer *tracing.Hooks, postCheck func(error, *core.BlockChain), useV2 bool) (result error) { config, ok := Forks[t.json.Network] if !ok { return UnsupportedForkError{t.json.Network} } // import pre accounts & construct test genesis block & state root + // Commit genesis state var ( + gspec = t.genesis(config) db = rawdb.NewMemoryDatabase() tconf = &triedb.Config{ Preimages: true, + IsVerkle: gspec.Config.IsVerkleGenesis(), } ) - if scheme == rawdb.PathScheme { + if scheme == rawdb.PathScheme || tconf.IsVerkle { tconf.PathDB = pathdb.Defaults } else { tconf.HashDB = hashdb.Defaults } - // Commit genesis state - gspec := t.genesis(config) // if ttd is not specified, set an arbitrary huge value if gspec.Config.TerminalTotalDifficulty == nil { @@ -289,7 +295,7 @@ func (t *BlockTest) insertBlocks(blockchain *core.BlockChain) ([]btBlock, error) if b.BlockHeader == nil { if data, err := json.MarshalIndent(cb.Header(), "", " "); err == nil { - fmt.Fprintf(os.Stderr, "block (index %d) insertion should have failed due to: %v:\n%v\n", + fmt.Fprintf(os.Stdout, "block (index %d) insertion should have failed due to: %v:\n%v\n", bi, b.ExpectException, string(data)) } diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index a0f3a77209..c6c07ed3ab 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -32,7 +32,6 @@ import ( type kv struct { k, v []byte - t bool } type fuzzer struct { @@ -64,9 +63,8 @@ func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { size := f.readInt() // Fill it with some fluff for i := byte(0); i < byte(size); i++ { - value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} - value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}, false} - + value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}} + value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}} trie.MustUpdate(value.k, value.v) trie.MustUpdate(value2.k, value2.v) @@ -81,7 +79,7 @@ func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { for i := 0; i < n; i++ { k := f.randBytes(32) v := f.randBytes(20) - value := &kv{k, v, false} + value := &kv{k, v} trie.MustUpdate(k, v) vals[string(k)] = value diff --git a/tests/state_test_util.go b/tests/state_test_util.go index c3497f07ce..7ca6dc3772 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -26,7 +26,6 @@ import ( "strings" "github.com/holiman/uint256" - "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -41,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -245,6 +245,20 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo if err != nil { // Here, an error exists but it was expected. // We do not check the post state or logs. + // However, if the test defines a post state root, we should check it. + // In case of an error, the state is reverted to the snapshot, so we need to + // recalculate the root. + post := t.json.Post[subtest.Fork][subtest.Index] + if post.Root != (common.UnprefixedHash{}) { + config, _, err := GetChainConfig(subtest.Fork) + if err != nil { + return fmt.Errorf("failed to get chain config: %w", err) + } + root = st.StateDB.IntermediateRoot(config.IsEIP158(new(big.Int).SetUint64(t.json.Env.Number))) + if root != common.Hash(post.Root) { + return fmt.Errorf("post-state root does not match the pre-state root, indicates an error in the test: got %x, want %x", root, post.Root) + } + } return nil } @@ -515,7 +529,7 @@ func (tx *stTransaction) toMessage(ps stPostState, baseFee *big.Int) (*core.Mess } func rlpHash(x interface{}) (h common.Hash) { - hw := sha3.NewLegacyKeccak256() + hw := keccak.NewLegacyKeccak256() rlp.Encode(hw, x) hw.Sum(h[:0]) diff --git a/trie/bintrie/binary_node.go b/trie/bintrie/binary_node.go index 1c003a6c8f..690489b2aa 100644 --- a/trie/bintrie/binary_node.go +++ b/trie/bintrie/binary_node.go @@ -31,8 +31,11 @@ type ( var zero [32]byte const ( - NodeWidth = 256 // Number of child per leaf node - StemSize = 31 // Number of bytes to travel before reaching a group of leaves + StemNodeWidth = 256 // Number of child per leaf node + StemSize = 31 // Number of bytes to travel before reaching a group of leaves + NodeTypeBytes = 1 // Size of node type prefix in serialization + HashSize = 32 // Size of a hash in bytes + BitmapSize = 32 // Size of the bitmap in a stem node ) const ( @@ -58,25 +61,28 @@ type BinaryNode interface { func SerializeNode(node BinaryNode) []byte { switch n := (node).(type) { case *InternalNode: - var serialized [65]byte + // InternalNode: 1 byte type + 32 bytes left hash + 32 bytes right hash + var serialized [NodeTypeBytes + HashSize + HashSize]byte serialized[0] = nodeTypeInternal copy(serialized[1:33], n.left.Hash().Bytes()) copy(serialized[33:65], n.right.Hash().Bytes()) return serialized[:] case *StemNode: - var serialized [32 + 32 + 256*32]byte + // StemNode: 1 byte type + 31 bytes stem + 32 bytes bitmap + 256*32 bytes values + var serialized [NodeTypeBytes + StemSize + BitmapSize + StemNodeWidth*HashSize]byte serialized[0] = nodeTypeStem - copy(serialized[1:32], node.(*StemNode).Stem) - bitmap := serialized[32:64] - offset := 64 - for i, v := range node.(*StemNode).Values { + copy(serialized[NodeTypeBytes:NodeTypeBytes+StemSize], n.Stem) + bitmap := serialized[NodeTypeBytes+StemSize : NodeTypeBytes+StemSize+BitmapSize] + offset := NodeTypeBytes + StemSize + BitmapSize + for i, v := range n.Values { if v != nil { bitmap[i/8] |= 1 << (7 - (i % 8)) - copy(serialized[offset:offset+32], v) - offset += 32 + copy(serialized[offset:offset+HashSize], v) + offset += HashSize } } - return serialized[:] + // Only return the actual data, not the entire array + return serialized[:offset] default: panic("invalid node type") } @@ -104,21 +110,21 @@ func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { if len(serialized) < 64 { return nil, invalidSerializedLength } - var values [256][]byte - bitmap := serialized[32:64] - offset := 64 + var values [StemNodeWidth][]byte + bitmap := serialized[NodeTypeBytes+StemSize : NodeTypeBytes+StemSize+BitmapSize] + offset := NodeTypeBytes + StemSize + BitmapSize - for i := range 256 { + for i := range StemNodeWidth { if bitmap[i/8]>>(7-(i%8))&1 == 1 { - if len(serialized) < offset+32 { + if len(serialized) < offset+HashSize { return nil, invalidSerializedLength } - values[i] = serialized[offset : offset+32] - offset += 32 + values[i] = serialized[offset : offset+HashSize] + offset += HashSize } } return &StemNode{ - Stem: serialized[1:32], + Stem: serialized[NodeTypeBytes : NodeTypeBytes+StemSize], Values: values[:], depth: depth, }, nil diff --git a/trie/bintrie/binary_node_test.go b/trie/bintrie/binary_node_test.go index b21daaab69..242743ba53 100644 --- a/trie/bintrie/binary_node_test.go +++ b/trie/bintrie/binary_node_test.go @@ -77,12 +77,12 @@ func TestSerializeDeserializeInternalNode(t *testing.T) { // TestSerializeDeserializeStemNode tests serialization and deserialization of StemNode func TestSerializeDeserializeStemNode(t *testing.T) { // Create a stem node with some values - stem := make([]byte, 31) + stem := make([]byte, StemSize) for i := range stem { stem[i] = byte(i) } - var values [256][]byte + var values [StemNodeWidth][]byte // Add some values at different indices values[0] = common.HexToHash("0x0101010101010101010101010101010101010101010101010101010101010101").Bytes() values[10] = common.HexToHash("0x0202020202020202020202020202020202020202020202020202020202020202").Bytes() @@ -103,7 +103,7 @@ func TestSerializeDeserializeStemNode(t *testing.T) { } // Check the stem is correctly serialized - if !bytes.Equal(serialized[1:32], stem) { + if !bytes.Equal(serialized[1:1+StemSize], stem) { t.Errorf("Stem mismatch in serialized data") } @@ -136,7 +136,7 @@ func TestSerializeDeserializeStemNode(t *testing.T) { } // Check that other values are nil - for i := range NodeWidth { + for i := range StemNodeWidth { if i == 0 || i == 10 || i == 255 { continue } @@ -218,15 +218,15 @@ func TestKeyToPath(t *testing.T) { }, { name: "max valid depth", - depth: 31 * 8, - key: make([]byte, 32), - expected: make([]byte, 31*8+1), + depth: StemSize * 8, + key: make([]byte, HashSize), + expected: make([]byte, StemSize*8+1), wantErr: false, }, { name: "depth too large", - depth: 31*8 + 1, - key: make([]byte, 32), + depth: StemSize*8 + 1, + key: make([]byte, HashSize), wantErr: true, }, } diff --git a/trie/bintrie/hashed_node.go b/trie/bintrie/hashed_node.go index 8f9fd66a59..e4d8c2e7ac 100644 --- a/trie/bintrie/hashed_node.go +++ b/trie/bintrie/hashed_node.go @@ -46,8 +46,31 @@ func (h HashedNode) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error return nil, errors.New("attempted to get values from an unresolved node") } -func (h HashedNode) InsertValuesAtStem(key []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { - return nil, errors.New("insertValuesAtStem not implemented for hashed node") +func (h HashedNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { + // Step 1: Generate the path for this node's position in the tree + path, err := keyToPath(depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem path generation error: %w", err) + } + + if resolver == nil { + return nil, errors.New("InsertValuesAtStem resolve error: resolver is nil") + } + + // Step 2: Resolve the hashed node to get the actual node data + data, err := resolver(path, common.Hash(h)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + + // Step 3: Deserialize the resolved data into a concrete node + node, err := DeserializeNode(data, depth) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + + // Step 4: Call InsertValuesAtStem on the resolved concrete node + return node.InsertValuesAtStem(stem, values, resolver, depth) } func (h HashedNode) toDot(parent string, path string) string { @@ -58,7 +81,8 @@ func (h HashedNode) toDot(parent string, path string) string { } func (h HashedNode) CollectNodes([]byte, NodeFlushFn) error { - return errors.New("collectNodes not implemented for hashed node") + // HashedNodes are already persisted in the database and don't need to be collected. + return nil } func (h HashedNode) GetHeight() int { diff --git a/trie/bintrie/hashed_node_test.go b/trie/bintrie/hashed_node_test.go index 0c19ae0c57..f9e6984888 100644 --- a/trie/bintrie/hashed_node_test.go +++ b/trie/bintrie/hashed_node_test.go @@ -17,6 +17,7 @@ package bintrie import ( + "bytes" "testing" "github.com/ethereum/go-ethereum/common" @@ -59,8 +60,8 @@ func TestHashedNodeCopy(t *testing.T) { func TestHashedNodeInsert(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - key := make([]byte, 32) - value := make([]byte, 32) + key := make([]byte, HashSize) + value := make([]byte, HashSize) _, err := node.Insert(key, value, nil, 0) if err == nil { @@ -76,7 +77,7 @@ func TestHashedNodeInsert(t *testing.T) { func TestHashedNodeGetValuesAtStem(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - stem := make([]byte, 31) + stem := make([]byte, StemSize) _, err := node.GetValuesAtStem(stem, nil) if err == nil { t.Fatal("Expected error for GetValuesAtStem on HashedNode") @@ -91,17 +92,85 @@ func TestHashedNodeGetValuesAtStem(t *testing.T) { func TestHashedNodeInsertValuesAtStem(t *testing.T) { node := HashedNode(common.HexToHash("0x1234")) - stem := make([]byte, 31) - values := make([][]byte, 256) + stem := make([]byte, StemSize) + values := make([][]byte, StemNodeWidth) + // Test 1: nil resolver should return an error _, err := node.InsertValuesAtStem(stem, values, nil, 0) if err == nil { - t.Fatal("Expected error for InsertValuesAtStem on HashedNode") + t.Fatal("Expected error for InsertValuesAtStem on HashedNode with nil resolver") } - if err.Error() != "insertValuesAtStem not implemented for hashed node" { + if err.Error() != "InsertValuesAtStem resolve error: resolver is nil" { t.Errorf("Unexpected error message: %v", err) } + + // Test 2: mock resolver returning invalid data should return deserialization error + mockResolver := func(path []byte, hash common.Hash) ([]byte, error) { + // Return invalid/nonsense data that cannot be deserialized + return []byte{0xff, 0xff, 0xff}, nil + } + + _, err = node.InsertValuesAtStem(stem, values, mockResolver, 0) + if err == nil { + t.Fatal("Expected error for InsertValuesAtStem on HashedNode with invalid resolver data") + } + + expectedPrefix := "InsertValuesAtStem node deserialization error:" + if len(err.Error()) < len(expectedPrefix) || err.Error()[:len(expectedPrefix)] != expectedPrefix { + t.Errorf("Expected deserialization error, got: %v", err) + } + + // Test 3: mock resolver returning valid serialized node should succeed + stem = make([]byte, StemSize) + stem[0] = 0xaa + var originalValues [StemNodeWidth][]byte + originalValues[0] = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111").Bytes() + originalValues[1] = common.HexToHash("0x2222222222222222222222222222222222222222222222222222222222222222").Bytes() + + originalNode := &StemNode{ + Stem: stem, + Values: originalValues[:], + depth: 0, + } + + // Serialize the node + serialized := SerializeNode(originalNode) + + // Create a mock resolver that returns the serialized node + validResolver := func(path []byte, hash common.Hash) ([]byte, error) { + return serialized, nil + } + + var newValues [StemNodeWidth][]byte + newValues[2] = common.HexToHash("0x3333333333333333333333333333333333333333333333333333333333333333").Bytes() + + resolvedNode, err := node.InsertValuesAtStem(stem, newValues[:], validResolver, 0) + if err != nil { + t.Fatalf("Expected successful resolution and insertion, got error: %v", err) + } + + resultStem, ok := resolvedNode.(*StemNode) + if !ok { + t.Fatalf("Expected resolved node to be *StemNode, got %T", resolvedNode) + } + + if !bytes.Equal(resultStem.Stem, stem) { + t.Errorf("Stem mismatch: expected %x, got %x", stem, resultStem.Stem) + } + + // Verify the original values are preserved + if !bytes.Equal(resultStem.Values[0], originalValues[0]) { + t.Errorf("Original value at index 0 not preserved: expected %x, got %x", originalValues[0], resultStem.Values[0]) + } + if !bytes.Equal(resultStem.Values[1], originalValues[1]) { + t.Errorf("Original value at index 1 not preserved: expected %x, got %x", originalValues[1], resultStem.Values[1]) + } + + // Verify the new value was inserted + if !bytes.Equal(resultStem.Values[2], newValues[2]) { + t.Errorf("New value at index 2 not inserted correctly: expected %x, got %x", newValues[2], resultStem.Values[2]) + } } // TestHashedNodeToDot tests the toDot method for visualization diff --git a/trie/bintrie/internal_node.go b/trie/bintrie/internal_node.go index f3ddd1aab0..0a7bece521 100644 --- a/trie/bintrie/internal_node.go +++ b/trie/bintrie/internal_node.go @@ -49,14 +49,26 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ } bit := stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 - var child *BinaryNode if bit == 0 { - child = &bt.left - } else { - child = &bt.right + if hn, ok := bt.left.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNode(data, bt.depth+1) + if err != nil { + return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) + } + bt.left = node + } + return bt.left.GetValuesAtStem(stem, resolver) } - if hn, ok := (*child).(HashedNode); ok { + if hn, ok := bt.right.(HashedNode); ok { path, err := keyToPath(bt.depth, stem) if err != nil { return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) @@ -69,9 +81,9 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ if err != nil { return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) } - *child = node + bt.right = node } - return (*child).GetValuesAtStem(stem, resolver) + return bt.right.GetValuesAtStem(stem, resolver) } // Get retrieves the value for the given key. @@ -80,6 +92,9 @@ func (bt *InternalNode) Get(key []byte, resolver NodeResolverFn) ([]byte, error) if err != nil { return nil, fmt.Errorf("get error: %w", err) } + if values == nil { + return nil, nil + } return values[key[31]], nil } @@ -118,17 +133,54 @@ func (bt *InternalNode) Hash() common.Hash { // InsertValuesAtStem inserts a full value group at the given stem in the internal node. // Already-existing values will be overwritten. func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver NodeResolverFn, depth int) (BinaryNode, error) { - var ( - child *BinaryNode - err error - ) + var err error bit := stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 if bit == 0 { - child = &bt.left - } else { - child = &bt.right + if bt.left == nil { + bt.left = Empty{} + } + + if hn, ok := bt.left.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNode(data, bt.depth+1) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + bt.left = node + } + + bt.left, err = bt.left.InsertValuesAtStem(stem, values, resolver, depth+1) + return bt, err + } + + if bt.right == nil { + bt.right = Empty{} } - *child, err = (*child).InsertValuesAtStem(stem, values, resolver, depth+1) + + if hn, ok := bt.right.(HashedNode); ok { + path, err := keyToPath(bt.depth, stem) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + data, err := resolver(path, common.Hash(hn)) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) + } + node, err := DeserializeNode(data, bt.depth+1) + if err != nil { + return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) + } + bt.right = node + } + + bt.right, err = bt.right.InsertValuesAtStem(stem, values, resolver, depth+1) return bt, err } diff --git a/trie/bintrie/iterator.go b/trie/bintrie/iterator.go index a6bab2bcfa..9b863ed1e3 100644 --- a/trie/bintrie/iterator.go +++ b/trie/bintrie/iterator.go @@ -108,6 +108,11 @@ func (it *binaryNodeIterator) Next(descend bool) bool { } // go back to parent to get the next leaf + // Check if we're at the root before popping + if len(it.stack) == 1 { + it.lastErr = errIteratorEnd + return false + } it.stack = it.stack[:len(it.stack)-1] it.current = it.stack[len(it.stack)-1].Node it.stack[len(it.stack)-1].Index++ @@ -183,9 +188,31 @@ func (it *binaryNodeIterator) NodeBlob() []byte { } // Leaf returns true iff the current node is a leaf node. +// In a Binary Trie, a StemNode contains up to 256 leaf values. +// The iterator is only considered to be "at a leaf" when it's positioned +// at a specific non-nil value within the StemNode, not just at the StemNode itself. func (it *binaryNodeIterator) Leaf() bool { - _, ok := it.current.(*StemNode) - return ok + sn, ok := it.current.(*StemNode) + if !ok { + return false + } + + // Check if we have a valid stack position + if len(it.stack) == 0 { + return false + } + + // The Index in the stack state points to the NEXT position after the current value. + // So if Index is 0, we haven't started iterating through the values yet. + // If Index is 5, we're currently at value[4] (the 5th value, 0-indexed). + idx := it.stack[len(it.stack)-1].Index + if idx == 0 || idx > 256 { + return false + } + + // Check if there's actually a value at the current position + currentValueIndex := idx - 1 + return sn.Values[currentValueIndex] != nil } // LeafKey returns the key of the leaf. The method panics if the iterator is not @@ -219,7 +246,7 @@ func (it *binaryNodeIterator) LeafProof() [][]byte { panic("LeafProof() called on an binary node iterator not at a leaf location") } - proof := make([][]byte, 0, len(it.stack)+NodeWidth) + proof := make([][]byte, 0, len(it.stack)+StemNodeWidth) // Build proof by walking up the stack and collecting sibling hashes for i := range it.stack[:len(it.stack)-2] { diff --git a/trie/bintrie/iterator_test.go b/trie/bintrie/iterator_test.go deleted file mode 100644 index 2e1c20c1e2..0000000000 --- a/trie/bintrie/iterator_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2025 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package bintrie - -import ( - "testing" - - "github.com/holiman/uint256" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/triedb" - "github.com/ethereum/go-ethereum/triedb/hashdb" - "github.com/ethereum/go-ethereum/triedb/pathdb" -) - -func newTestDatabase(diskdb ethdb.Database, scheme string) *triedb.Database { - config := &triedb.Config{Preimages: true} - if scheme == rawdb.HashScheme { - config.HashDB = &hashdb.Config{CleanCacheSize: 0} - } else { - config.PathDB = &pathdb.Config{TrieCleanSize: 0, StateCleanSize: 0} - } - return triedb.NewDatabase(diskdb, config) -} - -func TestBinaryIterator(t *testing.T) { - trie, err := NewBinaryTrie(types.EmptyVerkleHash, newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme)) - if err != nil { - t.Fatal(err) - } - account0 := &types.StateAccount{ - Nonce: 1, - Balance: uint256.NewInt(2), - Root: types.EmptyRootHash, - CodeHash: nil, - } - // NOTE: the code size isn't written to the trie via TryUpdateAccount - // so it will be missing from the test nodes. - trie.UpdateAccount(common.Address{}, account0, 0) - account1 := &types.StateAccount{ - Nonce: 1337, - Balance: uint256.NewInt(2000), - Root: types.EmptyRootHash, - CodeHash: nil, - } - // This address is meant to hash to a value that has the same first byte as 0xbf - var clash = common.HexToAddress("69fd8034cdb20934dedffa7dccb4fb3b8062a8be") - trie.UpdateAccount(clash, account1, 0) - - // Manually go over every node to check that we get all - // the correct nodes. - it, err := trie.NodeIterator(nil) - if err != nil { - t.Fatal(err) - } - var leafcount int - for it.Next(true) { - t.Logf("Node: %x", it.Path()) - if it.Leaf() { - leafcount++ - t.Logf("\tLeaf: %x", it.LeafKey()) - } - } - if leafcount != 2 { - t.Fatalf("invalid leaf count: %d != 6", leafcount) - } -} diff --git a/trie/bintrie/key_encoding.go b/trie/bintrie/key_encoding.go index 2c7b66cd9f..94a22d52d0 100644 --- a/trie/bintrie/key_encoding.go +++ b/trie/bintrie/key_encoding.go @@ -20,9 +20,8 @@ import ( "bytes" "crypto/sha256" - "github.com/holiman/uint256" - "github.com/ethereum/go-ethereum/common" + "github.com/holiman/uint256" ) const ( @@ -34,8 +33,17 @@ const ( ) var ( - zeroHash = common.Hash{} - codeOffset = uint256.NewInt(128) + zeroInt = uint256.NewInt(0) + zeroHash = common.Hash{} + verkleNodeWidthLog2 = 8 + headerStorageOffset = uint256.NewInt(64) + codeOffset = uint256.NewInt(128) + codeStorageDelta = uint256.NewInt(0).Sub(codeOffset, headerStorageOffset) + mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(1), 248-uint(verkleNodeWidthLog2)) + CodeOffset = uint256.NewInt(128) + VerkleNodeWidth = uint256.NewInt(256) + HeaderStorageOffset = uint256.NewInt(64) + VerkleNodeWidthLog2 = 8 ) func GetBinaryTreeKey(addr common.Address, key []byte) []byte { @@ -43,11 +51,18 @@ func GetBinaryTreeKey(addr common.Address, key []byte) []byte { hasher.Write(zeroHash[:12]) hasher.Write(addr[:]) hasher.Write(key[:31]) + hasher.Write([]byte{0}) k := hasher.Sum(nil) k[31] = key[31] return k } +func GetBinaryTreeKeyBasicData(addr common.Address) []byte { + var k [32]byte + k[31] = BasicDataLeafKey + return GetBinaryTreeKey(addr, k[:]) +} + func GetBinaryTreeKeyCodeHash(addr common.Address) []byte { var k [32]byte k[31] = CodeHashLeafKey @@ -78,3 +93,38 @@ func GetBinaryTreeKeyCodeChunk(address common.Address, chunknr *uint256.Int) []b chunkOffset := new(uint256.Int).Add(codeOffset, chunknr).Bytes() return GetBinaryTreeKey(address, chunkOffset) } + +func StorageIndex(storageKey []byte) (*uint256.Int, byte) { + // If the storage slot is in the header, we need to add the header offset. + var key uint256.Int + key.SetBytes(storageKey) + if key.Cmp(codeStorageDelta) < 0 { + // This addition is always safe; it can't ever overflow since pos> (7 - (bt.depth % 8)) & 1 n := &InternalNode{depth: bt.depth} @@ -65,26 +65,26 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int } *other = Empty{} } else { - var values [256][]byte - values[key[31]] = value + var values [StemNodeWidth][]byte + values[key[StemSize]] = value *other = &StemNode{ - Stem: slices.Clone(key[:31]), + Stem: slices.Clone(key[:StemSize]), Values: values[:], depth: depth + 1, } } return n, nil } - if len(value) != 32 { + if len(value) != HashSize { return bt, errors.New("invalid insertion: value length") } - bt.Values[key[31]] = value + bt.Values[key[StemSize]] = value return bt, nil } // Copy creates a deep copy of the node. func (bt *StemNode) Copy() BinaryNode { - var values [256][]byte + var values [StemNodeWidth][]byte for i, v := range bt.Values { values[i] = slices.Clone(v) } @@ -102,7 +102,7 @@ func (bt *StemNode) GetHeight() int { // Hash returns the hash of the node. func (bt *StemNode) Hash() common.Hash { - var data [NodeWidth]common.Hash + var data [StemNodeWidth]common.Hash for i, v := range bt.Values { if v != nil { h := sha256.Sum256(v) @@ -112,7 +112,7 @@ func (bt *StemNode) Hash() common.Hash { h := sha256.New() for level := 1; level <= 8; level++ { - for i := range NodeWidth / (1 << level) { + for i := range StemNodeWidth / (1 << level) { h.Reset() if data[i*2] == (common.Hash{}) && data[i*2+1] == (common.Hash{}) { @@ -141,14 +141,17 @@ func (bt *StemNode) CollectNodes(path []byte, flush NodeFlushFn) error { } // GetValuesAtStem retrieves the group of values located at the given stem key. -func (bt *StemNode) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error) { +func (bt *StemNode) GetValuesAtStem(stem []byte, _ NodeResolverFn) ([][]byte, error) { + if !bytes.Equal(bt.Stem, stem) { + return nil, nil + } return bt.Values[:], nil } // InsertValuesAtStem inserts a full value group at the given stem in the internal node. // Already-existing values will be overwritten. func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolverFn, depth int) (BinaryNode, error) { - if !bytes.Equal(bt.Stem, key[:31]) { + if !bytes.Equal(bt.Stem, key[:StemSize]) { bitStem := bt.Stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 n := &InternalNode{depth: bt.depth} @@ -174,7 +177,7 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv *other = Empty{} } else { *other = &StemNode{ - Stem: slices.Clone(key[:31]), + Stem: slices.Clone(key[:StemSize]), Values: values, depth: n.depth + 1, } @@ -206,7 +209,7 @@ func (bt *StemNode) toDot(parent, path string) string { // Key returns the full key for the given index. func (bt *StemNode) Key(i int) []byte { - var ret [32]byte + var ret [HashSize]byte copy(ret[:], bt.Stem) ret[StemSize] = byte(i) return ret[:] diff --git a/trie/bintrie/stem_node_test.go b/trie/bintrie/stem_node_test.go index e0ffd5c3c8..d8d6844427 100644 --- a/trie/bintrie/stem_node_test.go +++ b/trie/bintrie/stem_node_test.go @@ -251,27 +251,23 @@ func TestStemNodeGetValuesAtStem(t *testing.T) { } // Check that all values match - for i := 0; i < 256; i++ { + for i := range 256 { if !bytes.Equal(retrievedValues[i], values[i]) { t.Errorf("Value mismatch at index %d", i) } } - // GetValuesAtStem with different stem also returns the same values - // (implementation ignores the stem parameter) + // GetValuesAtStem with different stem should return nil differentStem := make([]byte, 31) differentStem[0] = 0xFF - retrievedValues2, err := node.GetValuesAtStem(differentStem, nil) + shouldBeNil, err := node.GetValuesAtStem(differentStem, nil) if err != nil { t.Fatalf("Failed to get values with different stem: %v", err) } - // Should still return the same values (stem is ignored) - for i := 0; i < 256; i++ { - if !bytes.Equal(retrievedValues2[i], values[i]) { - t.Errorf("Value mismatch at index %d with different stem", i) - } + if shouldBeNil != nil { + t.Error("Expected nil for different stem, got non-nil") } } diff --git a/trie/bintrie/trie.go b/trie/bintrie/trie.go index 221183f590..ecdd002331 100644 --- a/trie/bintrie/trie.go +++ b/trie/bintrie/trie.go @@ -22,18 +22,92 @@ import ( "errors" "fmt" - "github.com/holiman/uint256" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/triedb/database" + "github.com/holiman/uint256" ) var errInvalidRootType = errors.New("invalid root type") +// ChunkedCode represents a sequence of HashSize-byte chunks of code (StemSize bytes of which +// are actual code, and NodeTypeBytes byte is the pushdata offset). +type ChunkedCode []byte + +// Copy the values here so as to avoid an import cycle +const ( + PUSH1 = byte(0x60) + PUSH32 = byte(0x7f) +) + +// ChunkifyCode generates the chunked version of an array representing EVM bytecode +// according to EIP-7864 specification. +// +// The code is divided into HashSize-byte chunks, where each chunk contains: +// - Byte 0: Metadata byte indicating the number of leading bytes that are PUSHDATA (0-StemSize) +// - Bytes 1-StemSize: Actual code bytes +// +// This format enables stateless clients to validate jump destinations within a chunk +// without requiring additional context. When a PUSH instruction's data spans multiple +// chunks, the metadata byte tells us how many bytes at the start of the chunk are +// part of the previous chunk's PUSH instruction data. +// +// For example: +// - If a chunk starts with regular code: metadata byte = 0 +// - If a PUSH32 instruction starts at byte 30 of chunk N: +// - Chunk N: normal, contains PUSH32 opcode + 1 byte of data +// - Chunk N+1: metadata = StemSize (entire chunk is PUSH data) +// - Chunk N+2: metadata = 1 (first byte is PUSH data, then normal code resumes) +// +// This chunking approach ensures that jump destination validity can be determined +// by examining only the chunk containing the potential JUMPDEST, making it ideal +// for stateless execution and verkle/binary tries. +// +// Reference: https://eips.ethereum.org/EIPS/eip-7864 +func ChunkifyCode(code []byte) ChunkedCode { + var ( + chunkOffset = 0 // offset in the chunk + chunkCount = len(code) / StemSize + codeOffset = 0 // offset in the code + ) + if len(code)%StemSize != 0 { + chunkCount++ + } + chunks := make([]byte, chunkCount*HashSize) + for i := 0; i < chunkCount; i++ { + // number of bytes to copy, StemSize unless the end of the code has been reached. + end := min(len(code), StemSize*(i+1)) + copy(chunks[i*HashSize+1:], code[StemSize*i:end]) // copy the code itself + + // chunk offset = taken from the last chunk. + if chunkOffset > StemSize { + // skip offset calculation if push data covers the whole chunk + chunks[i*HashSize] = StemSize + chunkOffset = 1 + continue + } + chunks[HashSize*i] = byte(chunkOffset) + chunkOffset = 0 + + // Check each instruction and update the offset it should be 0 unless + // a PUSH-N overflows. + for ; codeOffset < end; codeOffset++ { + if code[codeOffset] >= PUSH1 && code[codeOffset] <= PUSH32 { + codeOffset += int(code[codeOffset] - PUSH1 + 1) + if codeOffset+1 >= StemSize*(i+1) { + codeOffset++ + chunkOffset = codeOffset - StemSize*(i+1) + break + } + } + } + } + return chunks +} + // NewBinaryNode creates a new empty binary trie func NewBinaryNode() BinaryNode { return Empty{} @@ -115,7 +189,7 @@ func (t *BinaryTrie) GetAccount(addr common.Address) (*types.StateAccount, error ) switch r := t.root.(type) { case *InternalNode: - values, err = r.GetValuesAtStem(key[:31], t.nodeResolver) + values, err = r.GetValuesAtStem(key[:StemSize], t.nodeResolver) case *StemNode: values = r.Values case Empty: @@ -162,15 +236,15 @@ func (t *BinaryTrie) GetAccount(addr common.Address) (*types.StateAccount, error // not be modified by the caller. If a node was not found in the database, a // trie.MissingNodeError is returned. func (t *BinaryTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { - return t.root.Get(GetBinaryTreeKey(addr, key), t.nodeResolver) + return t.root.Get(GetBinaryTreeKeyStorageSlot(addr, key), t.nodeResolver) } // UpdateAccount updates the account information for the given address. func (t *BinaryTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, codeLen int) error { var ( err error - basicData [32]byte - values = make([][]byte, NodeWidth) + basicData [HashSize]byte + values = make([][]byte, StemNodeWidth) stem = GetBinaryTreeKey(addr, zero[:]) ) binary.BigEndian.PutUint32(basicData[BasicDataCodeSizeOffset-1:], uint32(codeLen)) @@ -178,14 +252,14 @@ func (t *BinaryTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, // Because the balance is a max of 16 bytes, truncate // the extra values. This happens in devmode, where - // 0xff**32 is allocated to the developer account. + // 0xff**HashSize is allocated to the developer account. balanceBytes := acc.Balance.Bytes() // TODO: reduce the size of the allocation in devmode, then panic instead // of truncating. if len(balanceBytes) > 16 { balanceBytes = balanceBytes[16:] } - copy(basicData[32-len(balanceBytes):], balanceBytes[:]) + copy(basicData[HashSize-len(balanceBytes):], balanceBytes[:]) values[BasicDataLeafKey] = basicData[:] values[CodeHashLeafKey] = acc.CodeHash[:] @@ -206,11 +280,11 @@ func (t *BinaryTrie) UpdateStem(key []byte, values [][]byte) error { // database, a trie.MissingNodeError is returned. func (t *BinaryTrie) UpdateStorage(address common.Address, key, value []byte) error { k := GetBinaryTreeKeyStorageSlot(address, key) - var v [32]byte - if len(value) >= 32 { - copy(v[:], value[:32]) + var v [HashSize]byte + if len(value) >= HashSize { + copy(v[:], value[:HashSize]) } else { - copy(v[32-len(value):], value[:]) + copy(v[HashSize-len(value):], value[:]) } root, err := t.root.Insert(k, v[:], t.nodeResolver, 0) if err != nil { @@ -228,8 +302,8 @@ func (t *BinaryTrie) DeleteAccount(addr common.Address) error { // DeleteStorage removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. func (t *BinaryTrie) DeleteStorage(addr common.Address, key []byte) error { - k := GetBinaryTreeKey(addr, key) - var zero [32]byte + k := GetBinaryTreeKeyStorageSlot(addr, key) + var zero [HashSize]byte root, err := t.root.Insert(k, zero[:], t.nodeResolver, 0) if err != nil { return fmt.Errorf("DeleteStorage (%x) error: %v", addr, err) @@ -247,12 +321,12 @@ func (t *BinaryTrie) Hash() common.Hash { // Commit writes all nodes to the trie's memory database, tracking the internal // and external (for account tries) references. func (t *BinaryTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet) { - root := t.root.(*InternalNode) nodeset := trienode.NewNodeSet(common.Hash{}) - err := root.CollectNodes(nil, func(path []byte, node BinaryNode) { + // The root can be any type of BinaryNode (InternalNode, StemNode, etc.) + err := t.root.CollectNodes(nil, func(path []byte, node BinaryNode) { serialized := SerializeNode(node) - nodeset.AddNode(path, trienode.NewNodeWithPrev(common.Hash{}, serialized, t.tracer.Get(path))) + nodeset.AddNode(path, trienode.NewNodeWithPrev(node.Hash(), serialized, t.tracer.Get(path))) }) if err != nil { panic(fmt.Errorf("CollectNodes failed: %v", err)) @@ -300,23 +374,23 @@ func (t *BinaryTrie) IsVerkle() bool { // Note: the basic data leaf needs to have been previously created for this to work func (t *BinaryTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { var ( - chunks = trie.ChunkifyCode(code) + chunks = ChunkifyCode(code) values [][]byte key []byte err error ) - for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+32, chunknr+1 { - groupOffset := (chunknr + 128) % 256 + for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+HashSize, chunknr+1 { + groupOffset := (chunknr + 128) % StemNodeWidth if groupOffset == 0 /* start of new group */ || chunknr == 0 /* first chunk in header group */ { - values = make([][]byte, NodeWidth) - var offset [32]byte + values = make([][]byte, StemNodeWidth) + var offset [HashSize]byte binary.LittleEndian.PutUint64(offset[24:], chunknr+128) key = GetBinaryTreeKey(addr, offset[:]) } - values[groupOffset] = chunks[i : i+32] + values[groupOffset] = chunks[i : i+HashSize] - if groupOffset == 255 || len(chunks)-i <= 32 { - err = t.UpdateStem(key[:31], values) + if groupOffset == StemNodeWidth-1 || len(chunks)-i <= HashSize { + err = t.UpdateStem(key[:StemSize], values) if err != nil { return fmt.Errorf("UpdateContractCode (addr=%x) error: %w", addr[:], err) @@ -350,5 +424,5 @@ func (t *BinaryTrie) PrefetchStorage(addr common.Address, keys [][]byte) error { // Witness returns a set containing all trie nodes that have been accessed. func (t *BinaryTrie) Witness() map[string][]byte { - panic("not implemented") + return t.tracer.Values() } diff --git a/trie/bintrie/trie_test.go b/trie/bintrie/trie_test.go index 84f7689549..256fd218e2 100644 --- a/trie/bintrie/trie_test.go +++ b/trie/bintrie/trie_test.go @@ -22,10 +22,13 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) var ( - zeroKey = [32]byte{} + zeroKey = [HashSize]byte{} oneKey = common.HexToHash("0101010101010101010101010101010101010101010101010101010101010101") twoKey = common.HexToHash("0202020202020202020202020202020202020202020202020202020202020202") threeKey = common.HexToHash("0303030303030303030303030303030303030303030303030303030303030303") @@ -158,8 +161,8 @@ func TestInsertDuplicateKey(t *testing.T) { func TestLargeNumberOfEntries(t *testing.T) { var err error tree := NewBinaryNode() - for i := range 256 { - var key [32]byte + for i := range StemNodeWidth { + var key [HashSize]byte key[0] = byte(i) tree, err = tree.Insert(key[:], ffKey[:], nil, 0) if err != nil { @@ -182,7 +185,7 @@ func TestMerkleizeMultipleEntries(t *testing.T) { common.HexToHash("8100000000000000000000000000000000000000000000000000000000000000").Bytes(), } for i, key := range keys { - var v [32]byte + var v [HashSize]byte binary.LittleEndian.PutUint64(v[:8], uint64(i)) tree, err = tree.Insert(key, v[:], nil, 0) if err != nil { @@ -195,3 +198,97 @@ func TestMerkleizeMultipleEntries(t *testing.T) { t.Fatalf("invalid root, expected=%x, got = %x", expected, got) } } + +// TestStorageRoundTrip verifies that GetStorage and DeleteStorage use the same +// key mapping as UpdateStorage (GetBinaryTreeKeyStorageSlot). This is a regression +// test: previously GetStorage and DeleteStorage used GetBinaryTreeKey directly, +// which produced different tree keys and broke the read/delete path. +func TestStorageRoundTrip(t *testing.T) { + tracer := trie.NewPrevalueTracer() + tr := &BinaryTrie{ + root: NewBinaryNode(), + tracer: tracer, + } + addr := common.HexToAddress("0x1234567890abcdef1234567890abcdef12345678") + + // Create an account first so the root becomes an InternalNode, + // which is the realistic state when storage operations happen. + acc := &types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(1000), + CodeHash: common.HexToHash("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + } + if err := tr.UpdateAccount(addr, acc, 0); err != nil { + t.Fatalf("UpdateAccount error: %v", err) + } + + // Test main storage slots (key[31] >= 64 or key[:31] != 0). + // These produce a different stem than the account data, so after + // UpdateAccount + UpdateStorage the root is an InternalNode. + // Note: header slots (key[31] < 64, key[:31] == 0) share the same + // stem as account data and are covered by GetAccount/UpdateAccount path. + slots := []common.Hash{ + common.HexToHash("00000000000000000000000000000000000000000000000000000000000000FF"), // main storage (slot 255) + common.HexToHash("0100000000000000000000000000000000000000000000000000000000000001"), // main storage (non-zero prefix) + } + val := common.TrimLeftZeroes(common.HexToHash("00000000000000000000000000000000000000000000000000000000deadbeef").Bytes()) + + for _, slot := range slots { + // Write + if err := tr.UpdateStorage(addr, slot[:], val); err != nil { + t.Fatalf("UpdateStorage(%x) error: %v", slot, err) + } + // Read back + got, err := tr.GetStorage(addr, slot[:]) + if err != nil { + t.Fatalf("GetStorage(%x) error: %v", slot, err) + } + if len(got) == 0 { + t.Fatalf("GetStorage(%x) returned empty, expected value", slot) + } + // Verify value (right-justified in 32 bytes) + var expected [HashSize]byte + copy(expected[HashSize-len(val):], val) + if !bytes.Equal(got, expected[:]) { + t.Fatalf("GetStorage(%x) = %x, want %x", slot, got, expected) + } + // Delete + if err := tr.DeleteStorage(addr, slot[:]); err != nil { + t.Fatalf("DeleteStorage(%x) error: %v", slot, err) + } + // Verify deleted (should read as zero, not the old value) + got, err = tr.GetStorage(addr, slot[:]) + if err != nil { + t.Fatalf("GetStorage(%x) after delete error: %v", slot, err) + } + if len(got) > 0 && !bytes.Equal(got, zero[:]) { + t.Fatalf("GetStorage(%x) after delete = %x, expected zero", slot, got) + } + } +} + +func TestBinaryTrieWitness(t *testing.T) { + tracer := trie.NewPrevalueTracer() + + tr := &BinaryTrie{ + root: NewBinaryNode(), + tracer: tracer, + } + if w := tr.Witness(); len(w) != 0 { + t.Fatal("expected empty witness for fresh trie") + } + + tracer.Put([]byte("path1"), []byte("blob1")) + tracer.Put([]byte("path2"), []byte("blob2")) + + witness := tr.Witness() + if len(witness) != 2 { + t.Fatalf("expected 2 witness entries, got %d", len(witness)) + } + if !bytes.Equal(witness[string([]byte("path1"))], []byte("blob1")) { + t.Fatal("unexpected witness value for path1") + } + if !bytes.Equal(witness[string([]byte("path2"))], []byte("blob2")) { + t.Fatal("unexpected witness value for path2") + } +} diff --git a/trie/node.go b/trie/node.go index dc1e82c02a..c9023f6089 100644 --- a/trie/node.go +++ b/trie/node.go @@ -17,6 +17,7 @@ package trie import ( + "bytes" "fmt" "io" "strings" @@ -166,12 +167,14 @@ func decodeNodeUnsafe(hash, buf []byte) (node, error) { if err != nil { return nil, fmt.Errorf("decode error: %v", err) } - - switch c, _ := rlp.CountValues(elems); c { - case 2: + c, err := rlp.CountValues(elems) + switch { + case err != nil: + return nil, fmt.Errorf("invalid node list: %v", err) + case c == 2: n, err := decodeShort(hash, elems) return n, wrapError(err, "short") - case 17: + case c == 17: n, err := decodeFull(hash, elems) return n, wrapError(err, "full") default: @@ -242,7 +245,7 @@ func decodeRef(buf []byte) (node, []byte, error) { case kind == rlp.List: // 'embedded' node reference. The encoding must be smaller // than a hash in order to be valid. - if size := len(buf) - len(rest); size > hashLen { + if size := len(buf) - len(rest); size >= hashLen { err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen) return nil, buf, err } @@ -260,6 +263,74 @@ func decodeRef(buf []byte) (node, []byte, error) { } } +// decodeNodeElements parses the RLP encoding of a trie node and returns all the +// elements in raw byte format. +// +// For full node, it returns a slice of 17 elements; +// For short node, it returns a slice of 2 elements; +func decodeNodeElements(buf []byte) ([][]byte, error) { + if len(buf) == 0 { + return nil, io.ErrUnexpectedEOF + } + return rlp.SplitListValues(buf) +} + +// encodeNodeElements encodes the provided node elements into a rlp list. +func encodeNodeElements(elements [][]byte) ([]byte, error) { + if len(elements) != 2 && len(elements) != 17 { + return nil, fmt.Errorf("invalid number of elements: %d", len(elements)) + } + return rlp.MergeListValues(elements) +} + +// NodeDifference accepts two RLP-encoding nodes and figures out the difference +// between them. +// +// An error is returned if any of the provided blob is nil, or the type of nodes +// are different. +func NodeDifference(oldvalue []byte, newvalue []byte) (int, []int, [][]byte, error) { + oldElems, err := decodeNodeElements(oldvalue) + if err != nil { + return 0, nil, nil, err + } + newElems, err := decodeNodeElements(newvalue) + if err != nil { + return 0, nil, nil, err + } + if len(oldElems) != len(newElems) { + return 0, nil, nil, fmt.Errorf("different node type, old elements: %d, new elements: %d", len(oldElems), len(newElems)) + } + var ( + indices = make([]int, 0, len(oldElems)) + diff = make([][]byte, 0, len(oldElems)) + ) + for i := 0; i < len(oldElems); i++ { + if !bytes.Equal(oldElems[i], newElems[i]) { + indices = append(indices, i) + diff = append(diff, oldElems[i]) + } + } + return len(oldElems), indices, diff, nil +} + +// ReassembleNode accepts a RLP-encoding node along with a set of mutations, +// applying the modification diffs according to the indices and re-assemble. +func ReassembleNode(blob []byte, mutations [][][]byte, indices [][]int) ([]byte, error) { + if len(mutations) == 0 && len(indices) == 0 { + return blob, nil + } + elements, err := decodeNodeElements(blob) + if err != nil { + return nil, err + } + for i := 0; i < len(mutations); i++ { + for j, pos := range indices[i] { + elements[pos] = mutations[i][j] + } + } + return encodeNodeElements(elements) +} + // wraps a decoding error with information about the path to the // invalid child node (for debugging encoding issues). type decodeError struct { diff --git a/trie/node_test.go b/trie/node_test.go index 08b7d65a74..1347d2c6d5 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -18,9 +18,13 @@ package trie import ( "bytes" + "math/rand" + "reflect" + "slices" "testing" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" ) @@ -99,6 +103,286 @@ func TestDecodeFullNode(t *testing.T) { } } +func makeTestLeafNode(small bool) []byte { + l := leafNodeEncoder{} + l.Key = hexToCompact(keybytesToHex(testrand.Bytes(10))) + if small { + l.Val = testrand.Bytes(10) + } else { + l.Val = testrand.Bytes(32) + } + buf := rlp.NewEncoderBuffer(nil) + l.encode(buf) + return buf.ToBytes() +} + +func makeTestFullNode(small bool) []byte { + n := fullnodeEncoder{} + for i := 0; i < 16; i++ { + switch rand.Intn(3) { + case 0: + // write nil + case 1: + // write hash + n.Children[i] = testrand.Bytes(32) + case 2: + // write embedded node + n.Children[i] = makeTestLeafNode(small) + } + } + n.Children[16] = testrand.Bytes(32) // value + buf := rlp.NewEncoderBuffer(nil) + n.encode(buf) + return buf.ToBytes() +} + +func TestEncodeDecodeNodeElements(t *testing.T) { + var nodes [][]byte + nodes = append(nodes, makeTestFullNode(true)) + nodes = append(nodes, makeTestFullNode(false)) + nodes = append(nodes, makeTestLeafNode(true)) + nodes = append(nodes, makeTestLeafNode(false)) + + for _, blob := range nodes { + elements, err := decodeNodeElements(blob) + if err != nil { + t.Fatalf("Failed to decode node elements: %v", err) + } + enc, err := encodeNodeElements(elements) + if err != nil { + t.Fatalf("Failed to encode node elements: %v", err) + } + if !bytes.Equal(enc, blob) { + t.Fatalf("Unexpected encoded node element, want: %v, got: %v", blob, enc) + } + } +} + +func makeTestLeafNodePair() ([]byte, []byte, [][]byte, []int) { + var ( + na = leafNodeEncoder{} + nb = leafNodeEncoder{} + ) + key := keybytesToHex(testrand.Bytes(10)) + na.Key = hexToCompact(key) + nb.Key = hexToCompact(key) + + valA := testrand.Bytes(32) + valB := testrand.Bytes(32) + na.Val = valA + nb.Val = valB + + bufa, bufb := rlp.NewEncoderBuffer(nil), rlp.NewEncoderBuffer(nil) + na.encode(bufa) + nb.encode(bufb) + diff, _ := rlp.EncodeToBytes(valA) + return bufa.ToBytes(), bufb.ToBytes(), [][]byte{diff}, []int{1} +} + +func makeTestFullNodePair() ([]byte, []byte, [][]byte, []int) { + var ( + na = fullnodeEncoder{} + nb = fullnodeEncoder{} + indices []int + values [][]byte + ) + for i := 0; i < 16; i++ { + switch rand.Intn(3) { + case 0: + // write nil + case 1: + // write same + var child []byte + if rand.Intn(2) == 0 { + child = testrand.Bytes(32) // hashnode + } else { + child = makeTestLeafNode(true) // embedded node + } + na.Children[i] = child + nb.Children[i] = child + case 2: + // write different + var ( + va []byte + diff []byte + ) + rnd := rand.Intn(3) + if rnd == 0 { + va = testrand.Bytes(32) // hashnode + diff, _ = rlp.EncodeToBytes(va) + } else if rnd == 1 { + va = makeTestLeafNode(true) // embedded node + diff = va + } else { + va = nil + diff = rlp.EmptyString + } + vb := testrand.Bytes(32) // hashnode + na.Children[i] = va + nb.Children[i] = vb + + indices = append(indices, i) + values = append(values, diff) + } + } + na.Children[16] = nil + nb.Children[16] = nil + + bufa, bufb := rlp.NewEncoderBuffer(nil), rlp.NewEncoderBuffer(nil) + na.encode(bufa) + nb.encode(bufb) + return bufa.ToBytes(), bufb.ToBytes(), values, indices +} + +func TestNodeDifference(t *testing.T) { + type testsuite struct { + old []byte + new []byte + expErr bool + expIndices []int + expValues [][]byte + } + var tests = []testsuite{ + // Invalid node data + { + old: nil, new: nil, expErr: true, + }, + { + old: testrand.Bytes(32), new: nil, expErr: true, + }, + { + old: nil, new: testrand.Bytes(32), expErr: true, + }, + { + old: bytes.Repeat([]byte{0x1}, 32), new: bytes.Repeat([]byte{0x2}, 32), expErr: true, + }, + // Different node type + { + old: makeTestLeafNode(true), new: makeTestFullNode(true), expErr: true, + }, + } + for range 10 { + va, vb, elements, indices := makeTestLeafNodePair() + tests = append(tests, testsuite{ + old: va, + new: vb, + expErr: false, + expIndices: indices, + expValues: elements, + }) + } + for range 10 { + va, vb, elements, indices := makeTestFullNodePair() + tests = append(tests, testsuite{ + old: va, + new: vb, + expErr: false, + expIndices: indices, + expValues: elements, + }) + } + + for i, test := range tests { + _, indices, values, err := NodeDifference(test.old, test.new) + if test.expErr && err == nil { + t.Fatalf("Expect error, got nil %d", i) + } + if !test.expErr && err != nil { + t.Fatalf("Unexpect error, %v", err) + } + if err == nil { + if !slices.Equal(indices, test.expIndices) { + t.Fatalf("Unexpected indices, want: %v, got: %v", test.expIndices, indices) + } + if !slices.EqualFunc(values, test.expValues, bytes.Equal) { + t.Fatalf("Unexpected values, want: %v, got: %v", test.expValues, values) + } + } + } +} + +func TestReassembleFullNode(t *testing.T) { + var fn fullnodeEncoder + for i := 0; i < 16; i++ { + if rand.Intn(2) == 0 { + fn.Children[i] = testrand.Bytes(32) + } + } + buf := rlp.NewEncoderBuffer(nil) + fn.encode(buf) + enc := buf.ToBytes() + + // Generate a list of diffs + var ( + values [][][]byte + indices [][]int + ) + for i := 0; i < 10; i++ { + var ( + pos = make(map[int]struct{}) + poslist []int + valuelist [][]byte + ) + for j := 0; j < 3; j++ { + p := rand.Intn(16) + if _, ok := pos[p]; ok { + continue + } + pos[p] = struct{}{} + + nh := testrand.Bytes(32) + diff, _ := rlp.EncodeToBytes(nh) + poslist = append(poslist, p) + valuelist = append(valuelist, diff) + fn.Children[p] = nh + } + values = append(values, valuelist) + indices = append(indices, poslist) + } + reassembled, err := ReassembleNode(enc, values, indices) + if err != nil { + t.Fatalf("Failed to re-assemble full node %v", err) + } + buf2 := rlp.NewEncoderBuffer(nil) + fn.encode(buf2) + enc2 := buf2.ToBytes() + if !reflect.DeepEqual(enc2, reassembled) { + t.Fatalf("Unexpeted reassembled node") + } +} + +func TestReassembleShortNode(t *testing.T) { + var ln leafNodeEncoder + ln.Key = hexToCompact(keybytesToHex(testrand.Bytes(10))) + ln.Val = testrand.Bytes(10) + buf := rlp.NewEncoderBuffer(nil) + ln.encode(buf) + enc := buf.ToBytes() + + // Generate a list of diffs + var ( + values [][][]byte + indices [][]int + ) + for i := 0; i < 10; i++ { + val := testrand.Bytes(10) + ln.Val = val + diff, _ := rlp.EncodeToBytes(val) + values = append(values, [][]byte{diff}) + indices = append(indices, []int{1}) + } + reassembled, err := ReassembleNode(enc, values, indices) + if err != nil { + t.Fatalf("Failed to re-assemble full node %v", err) + } + buf2 := rlp.NewEncoderBuffer(nil) + ln.encode(buf2) + enc2 := buf2.ToBytes() + if !reflect.DeepEqual(enc2, reassembled) { + t.Fatalf("Unexpeted reassembled node") + } +} + // goos: darwin // goarch: arm64 // pkg: github.com/ethereum/go-ethereum/trie diff --git a/trie/secure_trie.go b/trie/secure_trie.go index ad6df7ab3e..ee010dd267 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -140,9 +140,9 @@ func (t *StateTrie) GetAccountByHash(addrHash common.Hash) (*types.StateAccount, // PrefetchAccount attempts to resolve specific accounts from the database // to accelerate subsequent trie operations. func (t *StateTrie) PrefetchAccount(addresses []common.Address) error { - var keys [][]byte - for _, addr := range addresses { - keys = append(keys, crypto.Keccak256(addr.Bytes())) + keys := make([][]byte, len(addresses)) + for i, addr := range addresses { + keys[i] = crypto.Keccak256(addr.Bytes()) } return t.trie.Prefetch(keys) } @@ -163,9 +163,9 @@ func (t *StateTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) { // PrefetchStorage attempts to resolve specific storage slots from the database // to accelerate subsequent trie operations. func (t *StateTrie) PrefetchStorage(_ common.Address, keys [][]byte) error { - var keylist [][]byte - for _, key := range keys { - keylist = append(keylist, crypto.Keccak256(key)) + keylist := make([][]byte, len(keys)) + for i, key := range keys { + keylist[i] = crypto.Keccak256(key) } return t.trie.Prefetch(keylist) } diff --git a/trie/transition.go b/trie/transitiontrie/transition.go similarity index 87% rename from trie/transition.go rename to trie/transitiontrie/transition.go index 71a471e461..4c73022082 100644 --- a/trie/transition.go +++ b/trie/transitiontrie/transition.go @@ -14,16 +14,16 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package trie +package transitiontrie import ( "fmt" - "github.com/ethereum/go-verkle" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -32,13 +32,16 @@ import ( // first from the overlay trie, and falls back to the base trie if the key isn't // found. All writes go to the overlay trie. type TransitionTrie struct { - overlay *VerkleTrie - base *SecureTrie + overlay *bintrie.BinaryTrie + base *trie.SecureTrie storage bool } // NewTransitionTrie creates a new TransitionTrie. -func NewTransitionTrie(base *SecureTrie, overlay *VerkleTrie, st bool) *TransitionTrie { +// Note: base can be nil when using TransitionTrie as a wrapper for BinaryTrie +// to work around import cycles. This is a temporary hack that should be +// refactored in future PRs (see core/state/reader.go for details). +func NewTransitionTrie(base *trie.SecureTrie, overlay *bintrie.BinaryTrie, st bool) *TransitionTrie { return &TransitionTrie{ overlay: overlay, base: base, @@ -47,12 +50,12 @@ func NewTransitionTrie(base *SecureTrie, overlay *VerkleTrie, st bool) *Transiti } // Base returns the base trie. -func (t *TransitionTrie) Base() *SecureTrie { +func (t *TransitionTrie) Base() *trie.SecureTrie { return t.base } // Overlay returns the overlay trie. -func (t *TransitionTrie) Overlay() *VerkleTrie { +func (t *TransitionTrie) Overlay() *bintrie.BinaryTrie { return t.overlay } @@ -62,7 +65,10 @@ func (t *TransitionTrie) GetKey(key []byte) []byte { if key := t.overlay.GetKey(key); key != nil { return key } - return t.base.GetKey(key) + if t.base != nil { + return t.base.GetKey(key) + } + return nil } // GetStorage returns the value for key stored in the trie. The value bytes must @@ -75,8 +81,11 @@ func (t *TransitionTrie) GetStorage(addr common.Address, key []byte) ([]byte, er if len(val) != 0 { return val, nil } - // TODO also insert value into overlay - return t.base.GetStorage(addr, key) + if t.base != nil { + // TODO also insert value into overlay + return t.base.GetStorage(addr, key) + } + return nil, nil } // PrefetchStorage attempts to resolve specific storage slots from the database @@ -103,7 +112,10 @@ func (t *TransitionTrie) GetAccount(address common.Address) (*types.StateAccount if data != nil { return data, nil } - return t.base.GetAccount(address) + if t.base != nil { + return t.base.GetAccount(address) + } + return nil, nil } // PrefetchAccount attempts to resolve specific accounts from the database @@ -175,7 +187,7 @@ func (t *TransitionTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSe // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. -func (t *TransitionTrie) NodeIterator(startKey []byte) (NodeIterator, error) { +func (t *TransitionTrie) NodeIterator(startKey []byte) (trie.NodeIterator, error) { panic("not implemented") // TODO: Implement } @@ -198,14 +210,10 @@ func (t *TransitionTrie) IsVerkle() bool { // UpdateStem updates a group of values, given the stem they are using. If // a value already exists, it is overwritten. +// TODO: This is Verkle-specific and requires access to private fields. +// Not currently used in the codebase. func (t *TransitionTrie) UpdateStem(key []byte, values [][]byte) error { - trie := t.overlay - switch root := trie.root.(type) { - case *verkle.InternalNode: - return root.InsertValuesAtStem(key, values, t.overlay.nodeResolver) - default: - panic("invalid root type") - } + panic("UpdateStem is not implemented for TransitionTrie") } // Copy creates a deep copy of the transition trie. diff --git a/trie/trie_test.go b/trie/trie_test.go index aa17f81302..4761dbf717 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -32,12 +32,12 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/holiman/uint256" - "golang.org/x/crypto/sha3" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/keccak" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" @@ -1032,7 +1032,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { prng := rand.New(rand.NewSource(int64(count))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{ - sponge: sha3.NewLegacyKeccak256(), + sponge: keccak.NewLegacyKeccak256(), id: "a", values: make(map[string]string), } @@ -1041,7 +1041,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{ - sponge: sha3.NewLegacyKeccak256(), + sponge: keccak.NewLegacyKeccak256(), id: "b", values: make(map[string]string), } @@ -1109,7 +1109,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do. func TestCommitSequenceSmallRoot(t *testing.T) { s := &spongeDb{ - sponge: sha3.NewLegacyKeccak256(), + sponge: keccak.NewLegacyKeccak256(), id: "a", values: make(map[string]string), } @@ -1118,7 +1118,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{ - sponge: sha3.NewLegacyKeccak256(), + sponge: keccak.NewLegacyKeccak256(), id: "b", values: make(map[string]string), } diff --git a/trie/utils/verkle.go b/trie/utils/verkle.go deleted file mode 100644 index 044fca6a7a..0000000000 --- a/trie/utils/verkle.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package utils - -import ( - "encoding/binary" - "sync" - - "github.com/crate-crypto/go-ipa/bandersnatch/fr" - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" - - "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/metrics" -) - -const ( - BasicDataLeafKey = 0 - CodeHashLeafKey = 1 - - BasicDataVersionOffset = 0 - BasicDataCodeSizeOffset = 5 - BasicDataNonceOffset = 8 - BasicDataBalanceOffset = 16 -) - -var ( - zero = uint256.NewInt(0) - verkleNodeWidthLog2 = 8 - headerStorageOffset = uint256.NewInt(64) - codeOffset = uint256.NewInt(128) - verkleNodeWidth = uint256.NewInt(256) - codeStorageDelta = uint256.NewInt(0).Sub(codeOffset, headerStorageOffset) - mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(1), 248-uint(verkleNodeWidthLog2)) - - index0Point *verkle.Point // pre-computed commitment of polynomial [2+256*64] - - // cacheHitGauge is the metric to track how many cache hit occurred. - cacheHitGauge = metrics.NewRegisteredGauge("trie/verkle/cache/hit", nil) - - // cacheMissGauge is the metric to track how many cache miss occurred. - cacheMissGauge = metrics.NewRegisteredGauge("trie/verkle/cache/miss", nil) -) - -func init() { - // The byte array is the Marshalled output of the point computed as such: - // - // var ( - // config = verkle.GetConfig() - // fr verkle.Fr - // ) - // verkle.FromLEBytes(&fr, []byte{2, 64}) - // point := config.CommitToPoly([]verkle.Fr{fr}, 1) - index0Point = new(verkle.Point) - err := index0Point.SetBytes([]byte{34, 25, 109, 242, 193, 5, 144, 224, 76, 52, 189, 92, 197, 126, 9, 145, 27, 152, 199, 130, 165, 3, 210, 27, 193, 131, 142, 28, 110, 26, 16, 191}) - if err != nil { - panic(err) - } -} - -// PointCache is the LRU cache for storing evaluated address commitment. -type PointCache struct { - lru lru.BasicLRU[string, *verkle.Point] - lock sync.RWMutex -} - -// NewPointCache returns the cache with specified size. -func NewPointCache(maxItems int) *PointCache { - return &PointCache{ - lru: lru.NewBasicLRU[string, *verkle.Point](maxItems), - } -} - -// Get returns the cached commitment for the specified address, or computing -// it on the flight. -func (c *PointCache) Get(addr []byte) *verkle.Point { - c.lock.Lock() - defer c.lock.Unlock() - - p, ok := c.lru.Get(string(addr)) - if ok { - cacheHitGauge.Inc(1) - return p - } - cacheMissGauge.Inc(1) - p = evaluateAddressPoint(addr) - c.lru.Add(string(addr), p) - return p -} - -// GetStem returns the first 31 bytes of the tree key as the tree stem. It only -// works for the account metadata whose treeIndex is 0. -func (c *PointCache) GetStem(addr []byte) []byte { - p := c.Get(addr) - return pointToHash(p, 0)[:31] -} - -// GetTreeKey performs both the work of the spec's get_tree_key function, and that -// of pedersen_hash: it builds the polynomial in pedersen_hash without having to -// create a mostly zero-filled buffer and "type cast" it to a 128-long 16-byte -// array. Since at most the first 5 coefficients of the polynomial will be non-zero, -// these 5 coefficients are created directly. -func GetTreeKey(address []byte, treeIndex *uint256.Int, subIndex byte) []byte { - if len(address) < 32 { - var aligned [32]byte - address = append(aligned[:32-len(address)], address...) - } - // poly = [2+256*64, address_le_low, address_le_high, tree_index_le_low, tree_index_le_high] - var poly [5]fr.Element - - // 32-byte address, interpreted as two little endian - // 16-byte numbers. - verkle.FromLEBytes(&poly[1], address[:16]) - verkle.FromLEBytes(&poly[2], address[16:]) - - // treeIndex must be interpreted as a 32-byte aligned little-endian integer. - // e.g: if treeIndex is 0xAABBCC, we need the byte representation to be 0xCCBBAA00...00. - // poly[3] = LE({CC,BB,AA,00...0}) (16 bytes), poly[4]=LE({00,00,...}) (16 bytes). - // - // To avoid unnecessary endianness conversions for go-ipa, we do some trick: - // - poly[3]'s byte representation is the same as the *top* 16 bytes (trieIndexBytes[16:]) of - // 32-byte aligned big-endian representation (BE({00,...,AA,BB,CC})). - // - poly[4]'s byte representation is the same as the *low* 16 bytes (trieIndexBytes[:16]) of - // the 32-byte aligned big-endian representation (BE({00,00,...}). - trieIndexBytes := treeIndex.Bytes32() - verkle.FromBytes(&poly[3], trieIndexBytes[16:]) - verkle.FromBytes(&poly[4], trieIndexBytes[:16]) - - cfg := verkle.GetConfig() - ret := cfg.CommitToPoly(poly[:], 0) - - // add a constant point corresponding to poly[0]=[2+256*64]. - ret.Add(ret, index0Point) - - return pointToHash(ret, subIndex) -} - -// GetTreeKeyWithEvaluatedAddress is basically identical to GetTreeKey, the only -// difference is a part of polynomial is already evaluated. -// -// Specifically, poly = [2+256*64, address_le_low, address_le_high] is already -// evaluated. -func GetTreeKeyWithEvaluatedAddress(evaluated *verkle.Point, treeIndex *uint256.Int, subIndex byte) []byte { - var poly [5]fr.Element - - // little-endian, 32-byte aligned treeIndex - var index [32]byte - for i := 0; i < len(treeIndex); i++ { - binary.LittleEndian.PutUint64(index[i*8:(i+1)*8], treeIndex[i]) - } - verkle.FromLEBytes(&poly[3], index[:16]) - verkle.FromLEBytes(&poly[4], index[16:]) - - cfg := verkle.GetConfig() - ret := cfg.CommitToPoly(poly[:], 0) - - // add the pre-evaluated address - ret.Add(ret, evaluated) - - return pointToHash(ret, subIndex) -} - -// BasicDataKey returns the verkle tree key of the basic data field for -// the specified account. -func BasicDataKey(address []byte) []byte { - return GetTreeKey(address, zero, BasicDataLeafKey) -} - -// CodeHashKey returns the verkle tree key of the code hash field for -// the specified account. -func CodeHashKey(address []byte) []byte { - return GetTreeKey(address, zero, CodeHashLeafKey) -} - -func codeChunkIndex(chunk *uint256.Int) (*uint256.Int, byte) { - var ( - chunkOffset = new(uint256.Int).Add(codeOffset, chunk) - treeIndex, subIndexMod = new(uint256.Int).DivMod(chunkOffset, verkleNodeWidth, new(uint256.Int)) - ) - return treeIndex, byte(subIndexMod.Uint64()) -} - -// CodeChunkKey returns the verkle tree key of the code chunk for the -// specified account. -func CodeChunkKey(address []byte, chunk *uint256.Int) []byte { - treeIndex, subIndex := codeChunkIndex(chunk) - return GetTreeKey(address, treeIndex, subIndex) -} - -func StorageIndex(storageKey []byte) (*uint256.Int, byte) { - // If the storage slot is in the header, we need to add the header offset. - var key uint256.Int - key.SetBytes(storageKey) - if key.Cmp(codeStorageDelta) < 0 { - // This addition is always safe; it can't ever overflow since pos. - -package utils - -import ( - "bytes" - "testing" - - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" -) - -func TestTreeKey(t *testing.T) { - var ( - address = []byte{0x01} - addressEval = evaluateAddressPoint(address) - smallIndex = uint256.NewInt(1) - largeIndex = uint256.NewInt(10000) - smallStorage = []byte{0x1} - largeStorage = bytes.Repeat([]byte{0xff}, 16) - ) - if !bytes.Equal(BasicDataKey(address), BasicDataKeyWithEvaluatedAddress(addressEval)) { - t.Fatal("Unmatched basic data key") - } - if !bytes.Equal(CodeHashKey(address), CodeHashKeyWithEvaluatedAddress(addressEval)) { - t.Fatal("Unmatched code hash key") - } - if !bytes.Equal(CodeChunkKey(address, smallIndex), CodeChunkKeyWithEvaluatedAddress(addressEval, smallIndex)) { - t.Fatal("Unmatched code chunk key") - } - if !bytes.Equal(CodeChunkKey(address, largeIndex), CodeChunkKeyWithEvaluatedAddress(addressEval, largeIndex)) { - t.Fatal("Unmatched code chunk key") - } - if !bytes.Equal(StorageSlotKey(address, smallStorage), StorageSlotKeyWithEvaluatedAddress(addressEval, smallStorage)) { - t.Fatal("Unmatched storage slot key") - } - if !bytes.Equal(StorageSlotKey(address, largeStorage), StorageSlotKeyWithEvaluatedAddress(addressEval, largeStorage)) { - t.Fatal("Unmatched storage slot key") - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkTreeKey -// BenchmarkTreeKey-8 398731 2961 ns/op 32 B/op 1 allocs/op -func BenchmarkTreeKey(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - b.ReportAllocs() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - BasicDataKey([]byte{0x01}) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkTreeKeyWithEvaluation -// BenchmarkTreeKeyWithEvaluation-8 513855 2324 ns/op 32 B/op 1 allocs/op -func BenchmarkTreeKeyWithEvaluation(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - addr := []byte{0x01} - eval := evaluateAddressPoint(addr) - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - BasicDataKeyWithEvaluatedAddress(eval) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkStorageKey -// BenchmarkStorageKey-8 230516 4584 ns/op 96 B/op 3 allocs/op -func BenchmarkStorageKey(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - b.ReportAllocs() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - StorageSlotKey([]byte{0x01}, bytes.Repeat([]byte{0xff}, 32)) - } -} - -// goos: darwin -// goarch: amd64 -// pkg: github.com/ethereum/go-ethereum/trie/utils -// cpu: VirtualApple @ 2.50GHz -// BenchmarkStorageKeyWithEvaluation -// BenchmarkStorageKeyWithEvaluation-8 320125 3753 ns/op 96 B/op 3 allocs/op -func BenchmarkStorageKeyWithEvaluation(b *testing.B) { - // Initialize the IPA settings which can be pretty expensive. - verkle.GetConfig() - - addr := []byte{0x01} - eval := evaluateAddressPoint(addr) - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - StorageSlotKeyWithEvaluatedAddress(eval, bytes.Repeat([]byte{0xff}, 32)) - } -} diff --git a/trie/verkle.go b/trie/verkle.go deleted file mode 100644 index bf36aad70b..0000000000 --- a/trie/verkle.go +++ /dev/null @@ -1,458 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - - "github.com/ethereum/go-verkle" - "github.com/holiman/uint256" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/utils" - "github.com/ethereum/go-ethereum/triedb/database" -) - -var ( - errInvalidRootType = errors.New("invalid node type for root") -) - -// VerkleTrie is a wrapper around VerkleNode that implements the trie.Trie -// interface so that Verkle trees can be reused verbatim. -type VerkleTrie struct { - root verkle.VerkleNode - cache *utils.PointCache - reader *Reader - tracer *PrevalueTracer -} - -// NewVerkleTrie constructs a verkle tree based on the specified root hash. -func NewVerkleTrie(root common.Hash, db database.NodeDatabase, cache *utils.PointCache) (*VerkleTrie, error) { - reader, err := NewReader(root, common.Hash{}, db) - if err != nil { - return nil, err - } - t := &VerkleTrie{ - root: verkle.New(), - cache: cache, - reader: reader, - tracer: NewPrevalueTracer(), - } - // Parse the root verkle node if it's not empty. - if root != types.EmptyVerkleHash && root != types.EmptyRootHash { - blob, err := t.nodeResolver(nil) - if err != nil { - return nil, err - } - node, err := verkle.ParseNode(blob, 0) - if err != nil { - return nil, err - } - t.root = node - } - return t, nil -} - -// GetKey returns the sha3 preimage of a hashed key that was previously used -// to store a value. -func (t *VerkleTrie) GetKey(key []byte) []byte { - return key -} - -// GetAccount implements state.Trie, retrieving the account with the specified -// account address. If the specified account is not in the verkle tree, nil will -// be returned. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) GetAccount(addr common.Address) (*types.StateAccount, error) { - var ( - acc = &types.StateAccount{} - values [][]byte - err error - ) - switch n := t.root.(type) { - case *verkle.InternalNode: - values, err = n.GetValuesAtStem(t.cache.GetStem(addr[:]), t.nodeResolver) - if err != nil { - return nil, fmt.Errorf("GetAccount (%x) error: %v", addr, err) - } - default: - return nil, errInvalidRootType - } - if values == nil { - return nil, nil - } - basicData := values[utils.BasicDataLeafKey] - acc.Nonce = binary.BigEndian.Uint64(basicData[utils.BasicDataNonceOffset:]) - acc.Balance = new(uint256.Int).SetBytes(basicData[utils.BasicDataBalanceOffset : utils.BasicDataBalanceOffset+16]) - acc.CodeHash = values[utils.CodeHashLeafKey] - - // TODO account.Root is leave as empty. How should we handle the legacy account? - return acc, nil -} - -// PrefetchAccount attempts to resolve specific accounts from the database -// to accelerate subsequent trie operations. -func (t *VerkleTrie) PrefetchAccount(addresses []common.Address) error { - for _, addr := range addresses { - if _, err := t.GetAccount(addr); err != nil { - return err - } - } - return nil -} - -// GetStorage implements state.Trie, retrieving the storage slot with the specified -// account address and storage key. If the specified slot is not in the verkle tree, -// nil will be returned. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) - val, err := t.root.Get(k, t.nodeResolver) - if err != nil { - return nil, err - } - return common.TrimLeftZeroes(val), nil -} - -// PrefetchStorage attempts to resolve specific storage slots from the database -// to accelerate subsequent trie operations. -func (t *VerkleTrie) PrefetchStorage(addr common.Address, keys [][]byte) error { - for _, key := range keys { - if _, err := t.GetStorage(addr, key); err != nil { - return err - } - } - return nil -} - -// UpdateAccount implements state.Trie, writing the provided account into the tree. -// If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) UpdateAccount(addr common.Address, acc *types.StateAccount, codeLen int) error { - var ( - err error - basicData [32]byte - values = make([][]byte, verkle.NodeWidth) - stem = t.cache.GetStem(addr[:]) - ) - - // Code size is encoded in BasicData as a 3-byte big-endian integer. Spare bytes are present - // before the code size to support bigger integers in the future. PutUint32(...) requires - // 4 bytes, so we need to shift the offset 1 byte to the left. - binary.BigEndian.PutUint32(basicData[utils.BasicDataCodeSizeOffset-1:], uint32(codeLen)) - binary.BigEndian.PutUint64(basicData[utils.BasicDataNonceOffset:], acc.Nonce) - if acc.Balance.ByteLen() > 16 { - panic("balance too large") - } - acc.Balance.WriteToSlice(basicData[utils.BasicDataBalanceOffset : utils.BasicDataBalanceOffset+16]) - values[utils.BasicDataLeafKey] = basicData[:] - values[utils.CodeHashLeafKey] = acc.CodeHash[:] - - switch root := t.root.(type) { - case *verkle.InternalNode: - err = root.InsertValuesAtStem(stem, values, t.nodeResolver) - default: - return errInvalidRootType - } - if err != nil { - return fmt.Errorf("UpdateAccount (%x) error: %v", addr, err) - } - - return nil -} - -// UpdateStorage implements state.Trie, writing the provided storage slot into -// the tree. If the tree is corrupted, an error will be returned. -func (t *VerkleTrie) UpdateStorage(address common.Address, key, value []byte) error { - // Left padding the slot value to 32 bytes. - var v [32]byte - if len(value) >= 32 { - copy(v[:], value[:32]) - } else { - copy(v[32-len(value):], value[:]) - } - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(address.Bytes()), key) - return t.root.Insert(k, v[:], t.nodeResolver) -} - -// DeleteAccount leaves the account untouched, as no account deletion can happen -// in verkle. -// There is a special corner case, in which an account that is prefunded, CREATE2-d -// and then SELFDESTRUCT-d should see its funds drained. EIP161 says that account -// should be removed, but this is verboten by the verkle spec. This contains a -// workaround in which the method checks for this corner case, and if so, overwrites -// the balance with 0. This will be removed once the spec has been clarified. -func (t *VerkleTrie) DeleteAccount(addr common.Address) error { - k := utils.BasicDataKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes())) - values, err := t.root.(*verkle.InternalNode).GetValuesAtStem(k, t.nodeResolver) - if err != nil { - return fmt.Errorf("Error getting data at %x in delete: %w", k, err) - } - var prefunded bool - for i, v := range values { - switch i { - case 0: - prefunded = len(v) == 32 - case 1: - prefunded = len(v) == 32 && bytes.Equal(v, types.EmptyCodeHash[:]) - default: - prefunded = v == nil - } - if !prefunded { - break - } - } - if prefunded { - t.root.Insert(k, common.Hash{}.Bytes(), t.nodeResolver) - } - return nil -} - -// RollBackAccount removes the account info + code from the tree, unlike DeleteAccount -// that will overwrite it with 0s. The first 64 storage slots are also removed. -func (t *VerkleTrie) RollBackAccount(addr common.Address) error { - var ( - evaluatedAddr = t.cache.Get(addr.Bytes()) - basicDataKey = utils.BasicDataKeyWithEvaluatedAddress(evaluatedAddr) - ) - basicDataBytes, err := t.root.Get(basicDataKey, t.nodeResolver) - if err != nil { - return fmt.Errorf("rollback: error finding code size: %w", err) - } - if len(basicDataBytes) == 0 { - return errors.New("rollback: basic data is not existent") - } - // The code size is encoded in BasicData as a 3-byte big-endian integer. Spare bytes are present - // before the code size to support bigger integers in the future. - // LittleEndian.Uint32(...) expects 4-bytes, so we need to shift the offset 1-byte to the left. - codeSize := binary.BigEndian.Uint32(basicDataBytes[utils.BasicDataCodeSizeOffset-1:]) - - // Delete the account header + first 64 slots + first 128 code chunks - _, err = t.root.(*verkle.InternalNode).DeleteAtStem(basicDataKey[:31], t.nodeResolver) - if err != nil { - return fmt.Errorf("error rolling back account header: %w", err) - } - - // Delete all further code - for i, chunknr := uint64(31*128), uint64(128); i < uint64(codeSize); i, chunknr = i+31*256, chunknr+256 { - // evaluate group key at the start of a new group - offset := uint256.NewInt(chunknr) - key := utils.CodeChunkKeyWithEvaluatedAddress(evaluatedAddr, offset) - - if _, err = t.root.(*verkle.InternalNode).DeleteAtStem(key[:], t.nodeResolver); err != nil { - return fmt.Errorf("error deleting code chunk stem (addr=%x, offset=%d) error: %w", addr[:], offset, err) - } - } - return nil -} - -// DeleteStorage implements state.Trie, deleting the specified storage slot from -// the trie. If the storage slot was not existent in the trie, no error will be -// returned. If the trie is corrupted, an error will be returned. -func (t *VerkleTrie) DeleteStorage(addr common.Address, key []byte) error { - var zero [32]byte - k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) - return t.root.Insert(k, zero[:], t.nodeResolver) -} - -// Hash returns the root hash of the tree. It does not write to the database and -// can be used even if the tree doesn't have one. -func (t *VerkleTrie) Hash() common.Hash { - return t.root.Commit().Bytes() -} - -// Commit writes all nodes to the tree's memory database. -func (t *VerkleTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet) { - root := t.root.(*verkle.InternalNode) - nodes, err := root.BatchSerialize() - if err != nil { - // Error return from this function indicates error in the code logic - // of BatchSerialize, and we fail catastrophically if this is the case. - panic(fmt.Errorf("BatchSerialize failed: %v", err)) - } - nodeset := trienode.NewNodeSet(common.Hash{}) - for _, node := range nodes { - // Hash parameter is not used in pathdb - nodeset.AddNode(node.Path, trienode.NewNodeWithPrev(common.Hash{}, node.SerializedBytes, t.tracer.Get(node.Path))) - } - // Serialize root commitment form - return t.Hash(), nodeset -} - -// NodeIterator implements state.Trie, returning an iterator that returns -// nodes of the trie. Iteration starts at the key after the given start key. -// -// TODO(gballet, rjl493456442) implement it. -func (t *VerkleTrie) NodeIterator(startKey []byte) (NodeIterator, error) { - panic("not implemented") -} - -// Prove implements state.Trie, constructing a Merkle proof for key. The result -// contains all encoded nodes on the path to the value at key. The value itself -// is also included in the last node and can be retrieved by verifying the proof. -// -// If the trie does not contain a value for key, the returned proof contains all -// nodes of the longest existing prefix of the key (at least the root), ending -// with the node that proves the absence of the key. -// -// TODO(gballet, rjl493456442) implement it. -func (t *VerkleTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { - panic("not implemented") -} - -// Copy returns a deep-copied verkle tree. -func (t *VerkleTrie) Copy() *VerkleTrie { - return &VerkleTrie{ - root: t.root.Copy(), - cache: t.cache, - reader: t.reader, - tracer: t.tracer.Copy(), - } -} - -// IsVerkle indicates if the trie is a Verkle trie. -func (t *VerkleTrie) IsVerkle() bool { - return true -} - -// Proof builds and returns the verkle multiproof for keys, built against -// the pre tree. The post tree is passed in order to add the post values -// to that proof. -func (t *VerkleTrie) Proof(posttrie *VerkleTrie, keys [][]byte) (*verkle.VerkleProof, verkle.StateDiff, error) { - var postroot verkle.VerkleNode - if posttrie != nil { - postroot = posttrie.root - } - proof, _, _, _, err := verkle.MakeVerkleMultiProof(t.root, postroot, keys, t.nodeResolver) - if err != nil { - return nil, nil, err - } - p, kvps, err := verkle.SerializeProof(proof) - if err != nil { - return nil, nil, err - } - return p, kvps, nil -} - -// ChunkedCode represents a sequence of 32-bytes chunks of code (31 bytes of which -// are actual code, and 1 byte is the pushdata offset). -type ChunkedCode []byte - -// Copy the values here so as to avoid an import cycle -const ( - PUSH1 = byte(0x60) - PUSH32 = byte(0x7f) -) - -// ChunkifyCode generates the chunked version of an array representing EVM bytecode -func ChunkifyCode(code []byte) ChunkedCode { - var ( - chunkOffset = 0 // offset in the chunk - chunkCount = len(code) / 31 - codeOffset = 0 // offset in the code - ) - if len(code)%31 != 0 { - chunkCount++ - } - chunks := make([]byte, chunkCount*32) - for i := 0; i < chunkCount; i++ { - // number of bytes to copy, 31 unless the end of the code has been reached. - end := 31 * (i + 1) - if len(code) < end { - end = len(code) - } - copy(chunks[i*32+1:], code[31*i:end]) // copy the code itself - - // chunk offset = taken from the last chunk. - if chunkOffset > 31 { - // skip offset calculation if push data covers the whole chunk - chunks[i*32] = 31 - chunkOffset = 1 - continue - } - chunks[32*i] = byte(chunkOffset) - chunkOffset = 0 - - // Check each instruction and update the offset it should be 0 unless - // a PUSH-N overflows. - for ; codeOffset < end; codeOffset++ { - if code[codeOffset] >= PUSH1 && code[codeOffset] <= PUSH32 { - codeOffset += int(code[codeOffset] - PUSH1 + 1) - if codeOffset+1 >= 31*(i+1) { - codeOffset++ - chunkOffset = codeOffset - 31*(i+1) - break - } - } - } - } - return chunks -} - -// UpdateContractCode implements state.Trie, writing the provided contract code -// into the trie. -// Note that the code-size *must* be already saved by a previous UpdateAccount call. -func (t *VerkleTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { - var ( - chunks = ChunkifyCode(code) - values [][]byte - key []byte - err error - ) - for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+32, chunknr+1 { - groupOffset := (chunknr + 128) % 256 - if groupOffset == 0 /* start of new group */ || chunknr == 0 /* first chunk in header group */ { - values = make([][]byte, verkle.NodeWidth) - key = utils.CodeChunkKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), uint256.NewInt(chunknr)) - } - values[groupOffset] = chunks[i : i+32] - - if groupOffset == 255 || len(chunks)-i <= 32 { - switch root := t.root.(type) { - case *verkle.InternalNode: - err = root.InsertValuesAtStem(key[:31], values, t.nodeResolver) - if err != nil { - return fmt.Errorf("UpdateContractCode (addr=%x) error: %w", addr[:], err) - } - default: - return errInvalidRootType - } - } - } - return nil -} - -func (t *VerkleTrie) ToDot() string { - return verkle.ToDot(t.root) -} - -func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) { - blob, err := t.reader.Node(path, common.Hash{}) - if err != nil { - return nil, err - } - t.tracer.Put(path, blob) - return blob, nil -} - -// Witness returns a set containing all trie nodes that have been accessed. -func (t *VerkleTrie) Witness() map[string][]byte { - panic("not implemented") -} diff --git a/trie/verkle_test.go b/trie/verkle_test.go deleted file mode 100644 index b7c68a5438..0000000000 --- a/trie/verkle_test.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "bytes" - "reflect" - "testing" - - "github.com/holiman/uint256" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie/utils" -) - -var ( - accounts = map[common.Address]*types.StateAccount{ - {1}: { - Nonce: 100, - Balance: uint256.NewInt(100), - CodeHash: common.Hash{0x1}.Bytes(), - }, - {2}: { - Nonce: 200, - Balance: uint256.NewInt(200), - CodeHash: common.Hash{0x2}.Bytes(), - }, - } - storages = map[common.Address]map[common.Hash][]byte{ - {1}: { - common.Hash{10}: []byte{10}, - common.Hash{11}: []byte{11}, - common.MaxHash: []byte{0xff}, - }, - {2}: { - common.Hash{20}: []byte{20}, - common.Hash{21}: []byte{21}, - common.MaxHash: []byte{0xff}, - }, - } -) - -func TestVerkleTreeReadWrite(t *testing.T) { - db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) - tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) - - for addr, acct := range accounts { - if err := tr.UpdateAccount(addr, acct, 0); err != nil { - t.Fatalf("Failed to update account, %v", err) - } - for key, val := range storages[addr] { - if err := tr.UpdateStorage(addr, key.Bytes(), val); err != nil { - t.Fatalf("Failed to update storage, %v", err) - } - } - } - - for addr, acct := range accounts { - stored, err := tr.GetAccount(addr) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if !reflect.DeepEqual(stored, acct) { - t.Fatal("account is not matched") - } - for key, val := range storages[addr] { - stored, err := tr.GetStorage(addr, key.Bytes()) - if err != nil { - t.Fatalf("Failed to get storage, %v", err) - } - if !bytes.Equal(stored, val) { - t.Fatal("storage is not matched") - } - } - } -} - -func TestVerkleRollBack(t *testing.T) { - db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.PathScheme) - tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) - - for addr, acct := range accounts { - // create more than 128 chunks of code - code := make([]byte, 129*32) - for i := 0; i < len(code); i += 2 { - code[i] = 0x60 - code[i+1] = byte(i % 256) - } - if err := tr.UpdateAccount(addr, acct, len(code)); err != nil { - t.Fatalf("Failed to update account, %v", err) - } - for key, val := range storages[addr] { - if err := tr.UpdateStorage(addr, key.Bytes(), val); err != nil { - t.Fatalf("Failed to update storage, %v", err) - } - } - hash := crypto.Keccak256Hash(code) - if err := tr.UpdateContractCode(addr, hash, code); err != nil { - t.Fatalf("Failed to update contract, %v", err) - } - } - - // Check that things were created - for addr, acct := range accounts { - stored, err := tr.GetAccount(addr) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if !reflect.DeepEqual(stored, acct) { - t.Fatal("account is not matched") - } - for key, val := range storages[addr] { - stored, err := tr.GetStorage(addr, key.Bytes()) - if err != nil { - t.Fatalf("Failed to get storage, %v", err) - } - if !bytes.Equal(stored, val) { - t.Fatal("storage is not matched") - } - } - } - - // ensure there is some code in the 2nd group of the 1st account - keyOf2ndGroup := utils.CodeChunkKeyWithEvaluatedAddress(tr.cache.Get(common.Address{1}.Bytes()), uint256.NewInt(128)) - chunk, err := tr.root.Get(keyOf2ndGroup, nil) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if len(chunk) == 0 { - t.Fatal("account was not created ") - } - - // Rollback first account and check that it is gone - addr1 := common.Address{1} - err = tr.RollBackAccount(addr1) - if err != nil { - t.Fatalf("error rolling back address 1: %v", err) - } - - // ensure the account is gone - stored, err := tr.GetAccount(addr1) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if stored != nil { - t.Fatal("account was not deleted") - } - - // ensure that the last code chunk is also gone from the tree - chunk, err = tr.root.Get(keyOf2ndGroup, nil) - if err != nil { - t.Fatalf("Failed to get account, %v", err) - } - if len(chunk) != 0 { - t.Fatal("account was not deleted") - } -} diff --git a/triedb/database.go b/triedb/database.go index 85242b2dc2..07fe798bef 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -138,8 +138,8 @@ func (db *Database) StateReader(blockRoot common.Hash) (database.StateReader, er return db.backend.StateReader(blockRoot) } -// HistoricReader constructs a reader for accessing the requested historic state. -func (db *Database) HistoricReader(root common.Hash) (*pathdb.HistoricalStateReader, error) { +// HistoricStateReader constructs a reader for accessing the requested historic state. +func (db *Database) HistoricStateReader(root common.Hash) (*pathdb.HistoricalStateReader, error) { pdb, ok := db.backend.(*pathdb.Database) if !ok { return nil, errors.New("not supported") @@ -147,6 +147,15 @@ func (db *Database) HistoricReader(root common.Hash) (*pathdb.HistoricalStateRea return pdb.HistoricReader(root) } +// HistoricNodeReader constructs a reader for accessing the historical trie node. +func (db *Database) HistoricNodeReader(root common.Hash) (*pathdb.HistoricalNodeReader, error) { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return nil, errors.New("not supported") + } + return pdb.HistoricNodeReader(root) +} + // Update performs a state transition by committing dirty nodes contained in the // given set in order to update state from the specified parent to the specified // root. The held pre-images accumulated up to this point will be flushed in case diff --git a/triedb/pathdb/buffer.go b/triedb/pathdb/buffer.go index 74606c367a..2e29abcf27 100644 --- a/triedb/pathdb/buffer.go +++ b/triedb/pathdb/buffer.go @@ -151,7 +151,7 @@ func (b *buffer) size() uint64 { // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. -func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.AncientWriter, progress []byte, nodesCache *AddressBiasedCache, statesCache *fastcache.Cache, id uint64, postFlush func()) { +func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezers []ethdb.AncientWriter, progress []byte, nodesCache *AddressBiasedCache, statesCache *fastcache.Cache, id uint64, postFlush func()) { if b.done != nil { panic("duplicated flush operation") } @@ -191,11 +191,9 @@ func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezer ethdb.A // // This step is crucial to guarantee that the corresponding state history remains // available for state rollback. - if freezer != nil { - if err := freezer.SyncAncient(); err != nil { - b.flushErr = err - return - } + if err := syncHistory(freezers...); err != nil { + b.flushErr = err + return } nodes := flushNodes.write(batch, nodesCache) accounts, slots := flushStates.write(batch, progress, statesCache) diff --git a/triedb/pathdb/config.go b/triedb/pathdb/config.go index 74108bbdb8..e0c23ce9a9 100644 --- a/triedb/pathdb/config.go +++ b/triedb/pathdb/config.go @@ -60,6 +60,26 @@ const ( // defaultPreloadRateLimit is the default rate limit for address cache preloading // in bytes per second. 1 MB/s balances sync impact with preload performance. defaultPreloadRateLimit = 1024 * 1024 + + // maxFullValueCheckpoint defines the maximum allowed encoding frequency (1/16) + // for storing nodes in full format. With this setting, a node may be written + // to the trienode history as a full value at the specified frequency. + // + // Note that the frequency is not strict: the actual decision is probabilistic. + // Only the overall long-term full-value encoding rate is enforced. + // + // Values beyond this limit are considered ineffective, as the trienode history + // is already well compressed. Increasing it further will only degrade read + // performance linearly. + maxFullValueCheckpoint = 16 + + // defaultFullValueCheckpoint defines the default full-value encoding frequency + // (1/8) for storing nodes in full format. With this setting, nodes may be + // written to the trienode history as full values at the specified rate. + // + // This strikes a balance between effective compression of the trienode history + // and acceptable read performance. + defaultFullValueCheckpoint = 8 ) var ( @@ -70,6 +90,8 @@ var ( // Defaults contains default settings for Ethereum mainnet. var Defaults = &Config{ StateHistory: params.FullImmutabilityThreshold, + TrienodeHistory: -1, + FullValueCheckpoint: defaultFullValueCheckpoint, EnableStateIndexing: false, TrieCleanSize: defaultTrieCleanSize, StateCleanSize: defaultStateCleanSize, @@ -80,21 +102,27 @@ var Defaults = &Config{ // ReadOnly is the config in order to open database in read only mode. var ReadOnly = &Config{ - ReadOnly: true, - TrieCleanSize: defaultTrieCleanSize, - StateCleanSize: defaultStateCleanSize, + ReadOnly: true, + TrienodeHistory: -1, + TrieCleanSize: defaultTrieCleanSize, + StateCleanSize: defaultStateCleanSize, + FullValueCheckpoint: defaultFullValueCheckpoint, } // Config contains the settings for database. type Config struct { + TrieCleanSize int // Maximum memory allowance (in bytes) for caching clean trie data + StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data + WriteBufferSize int // Maximum memory allowance (in bytes) for write buffer + ReadOnly bool // Flag whether the database is opened in read only mode + JournalDirectory string // Absolute path of journal directory (null means the journal data is persisted in key-value store) + + // Historical state configurations StateHistory uint64 // Number of recent blocks to maintain state history for, 0: full chain + TrienodeHistory int64 // Number of recent blocks to maintain trienode history for, 0: full chain, negative: disable EnableStateIndexing bool // Whether to enable state history indexing for external state access - TrieCleanSize int // Maximum memory allowance (in bytes) for caching clean trie data - StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data - WriteBufferSize int // Maximum memory allowance (in bytes) for write buffer StateReservation int // Percentage of write buffer reserved for states (1-100, default 80) - ReadOnly bool // Flag whether the database is opened in read only mode - JournalDirectory string // Absolute path of journal directory (null means the journal data is persisted in key-value store) + FullValueCheckpoint uint32 // The rate at which trie nodes are encoded in full-value format // Address-specific cache configuration for biased caching // Maps account address to cache size in bytes @@ -124,6 +152,14 @@ func (c *Config) sanitize() *Config { log.Warn("Sanitizing invalid state reservation", "provided", conf.StateReservation, "updated", defaultStateReservation) conf.StateReservation = defaultStateReservation } + if conf.FullValueCheckpoint > maxFullValueCheckpoint { + log.Warn("Sanitizing trienode history full value checkpoint", "provided", conf.FullValueCheckpoint, "updated", maxFullValueCheckpoint) + conf.FullValueCheckpoint = maxFullValueCheckpoint + } + if conf.FullValueCheckpoint == 0 { + conf.FullValueCheckpoint = 1 + log.Info("Disabling diff mode trie node history encoding") + } return &conf } @@ -143,6 +179,13 @@ func (c *Config) fields() []interface{} { } else { list = append(list, "state-history", fmt.Sprintf("last %d blocks", c.StateHistory)) } + if c.TrienodeHistory >= 0 { + if c.TrienodeHistory == 0 { + list = append(list, "trie-history", "entire chain") + } else { + list = append(list, "trie-history", fmt.Sprintf("last %d blocks", c.TrienodeHistory)) + } + } if c.EnableStateIndexing { list = append(list, "index-history", true) } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 26a174a90e..5074c04572 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -25,14 +25,13 @@ import ( "sync" "time" - "github.com/ethereum/go-verkle" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -46,7 +45,7 @@ type layer interface { // Note: // - the returned node is not a copy, please don't modify it. // - no error will be returned if the requested node is not found in database. - node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) + node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) // account directly retrieves the account RLP associated with a particular // hash in the slim data format. An error will be returned if the read @@ -98,16 +97,16 @@ func merkleNodeHasher(blob []byte) (common.Hash, error) { return crypto.Keccak256Hash(blob), nil } -// verkleNodeHasher computes the hash of the given verkle node. -func verkleNodeHasher(blob []byte) (common.Hash, error) { +// binaryNodeHasher computes the hash of the given verkle node. +func binaryNodeHasher(blob []byte) (common.Hash, error) { if len(blob) == 0 { return types.EmptyVerkleHash, nil } - n, err := verkle.ParseNode(blob, 0) + n, err := bintrie.DeserializeNode(blob, 0) if err != nil { return common.Hash{}, err } - return n.Commit().Bytes(), nil + return n.Hash(), nil } // Database is a multiple-layered structure for maintaining in-memory states @@ -138,6 +137,9 @@ type Database struct { stateFreezer ethdb.ResettableAncientStore // Freezer for storing state histories, nil possible in tests stateIndexer *historyIndexer // History indexer historical state data, nil possible + trienodeFreezer ethdb.ResettableAncientStore // Freezer for storing trienode histories, nil possible in tests + trienodeIndexer *historyIndexer // History indexer for historical trienode data + lock sync.RWMutex // Lock to prevent mutations from happening at the same time } @@ -164,17 +166,20 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { // compress the shared key prefix. if isVerkle { db.diskdb = rawdb.NewTable(diskdb, string(rawdb.VerklePrefix)) - db.hasher = verkleNodeHasher + db.hasher = binaryNodeHasher } // Construct the layer tree by resolving the in-disk singleton state // and in-memory layer journal. db.tree = newLayerTree(db.loadLayers()) - // Repair the state history, which might not be aligned with the state - // in the key-value store due to an unclean shutdown. - if err := db.repairHistory(); err != nil { - log.Crit("Failed to repair state history", "err", err) + // Repair the history, which might not be aligned with the persistent + // state in the key-value store due to an unclean shutdown. + states, trienodes, err := repairHistory(db.diskdb, isVerkle, db.config.ReadOnly, db.tree.bottom().stateID(), db.config.TrienodeHistory >= 0) + if err != nil { + log.Crit("Failed to repair history", "err", err) } + db.stateFreezer, db.trienodeFreezer = states, trienodes + // Disable database in case node is still in the initial state sync stage. if rawdb.ReadSnapSyncStatusFlag(diskdb) == rawdb.StateSyncRunning && !db.readOnly { if err := db.Disable(); err != nil { @@ -188,11 +193,8 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { if err := db.setStateGenerator(); err != nil { log.Crit("Failed to setup the generator", "err", err) } - // TODO (rjl493456442) disable the background indexing in read-only mode - if db.stateFreezer != nil && db.config.EnableStateIndexing { - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) - log.Info("Enabled state history indexing") - } + db.setHistoryIndexer() + fields := config.fields() if db.isVerkle { fields = append(fields, "verkle", true) @@ -201,59 +203,28 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database { return db } -// repairHistory truncates leftover state history objects, which may occur due -// to an unclean shutdown or other unexpected reasons. -func (db *Database) repairHistory() error { - // Open the freezer for state history. This mechanism ensures that - // only one database instance can be opened at a time to prevent - // accidental mutation. - ancient, err := db.diskdb.AncientDatadir() - if err != nil { - // TODO error out if ancient store is disabled. A tons of unit tests - // disable the ancient store thus the error here will immediately fail - // all of them. Fix the tests first. - return nil - } - freezer, err := rawdb.NewStateFreezer(ancient, db.isVerkle, db.readOnly) - if err != nil { - log.Crit("Failed to open state history freezer", "err", err) +// setHistoryIndexer initializes the indexers for both state history and +// trienode history if available. Note that this function may be called while +// existing indexers are still running, so they must be closed beforehand. +func (db *Database) setHistoryIndexer() { + // TODO (rjl493456442) disable the background indexing in read-only mode + if !db.config.EnableStateIndexing { + return } - db.stateFreezer = freezer - - // Reset the entire state histories if the trie database is not initialized - // yet. This action is necessary because these state histories are not - // expected to exist without an initialized trie database. - id := db.tree.bottom().stateID() - if id == 0 { - frozen, err := db.stateFreezer.Ancients() - if err != nil { - log.Crit("Failed to retrieve head of state history", "err", err) - } - if frozen != 0 { - // Purge all state history indexing data first - batch := db.diskdb.NewBatch() - rawdb.DeleteStateHistoryIndexMetadata(batch) - rawdb.DeleteStateHistoryIndexes(batch) - if err := batch.Write(); err != nil { - log.Crit("Failed to purge state history index", "err", err) - } - if err := db.stateFreezer.Reset(); err != nil { - log.Crit("Failed to reset state histories", "err", err) - } - log.Info("Truncated extraneous state history") + if db.stateFreezer != nil { + if db.stateIndexer != nil { + db.stateIndexer.close() } - return nil - } - // Truncate the extra state histories above in freezer in case it's not - // aligned with the disk layer. It might happen after a unclean shutdown. - pruned, err := truncateFromHead(db.stateFreezer, typeStateHistory, id) - if err != nil { - log.Crit("Failed to truncate extra state histories", "err", err) + db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) + log.Info("Enabled state history indexing") } - if pruned != 0 { - log.Warn("Truncated extra state histories", "number", pruned) + if db.trienodeFreezer != nil { + if db.trienodeIndexer != nil { + db.trienodeIndexer.close() + } + db.trienodeIndexer = newHistoryIndexer(db.diskdb, db.trienodeFreezer, db.tree.bottom().stateID(), typeTrienodeHistory) + log.Info("Enabled trienode history indexing") } - return nil } // setStateGenerator loads the state generation progress marker and potentially @@ -334,8 +305,13 @@ func (db *Database) Update(root common.Hash, parentRoot common.Hash, block uint6 if err := db.modifyAllowed(); err != nil { return err } - // TODO(rjl493456442) tracking the origins in the following PRs. - if err := db.tree.add(root, parentRoot, block, NewNodeSetWithOrigin(nodes.Nodes(), nil), states); err != nil { + var nodesWithOrigins *nodeSetWithOrigin + if db.config.TrienodeHistory >= 0 { + nodesWithOrigins = NewNodeSetWithOrigin(nodes.NodeAndOrigins()) + } else { + nodesWithOrigins = NewNodeSetWithOrigin(nodes.Nodes(), nil) + } + if err := db.tree.add(root, parentRoot, block, nodesWithOrigins, states); err != nil { return err } // Keep 128 diff layers in the memory, persistent layer is 129th. @@ -427,18 +403,9 @@ func (db *Database) Enable(root common.Hash) error { // all root->id mappings should be removed as well. Since // mappings can be huge and might take a while to clear // them, just leave them in disk and wait for overwriting. - if db.stateFreezer != nil { - // Purge all state history indexing data first - batch.Reset() - rawdb.DeleteStateHistoryIndexMetadata(batch) - rawdb.DeleteStateHistoryIndexes(batch) - if err := batch.Write(); err != nil { - return err - } - if err := db.stateFreezer.Reset(); err != nil { - return err - } - } + purgeHistory(db.stateFreezer, db.diskdb, typeStateHistory) + purgeHistory(db.trienodeFreezer, db.diskdb, typeTrienodeHistory) + // Re-enable the database as the final step. db.waitSync = false rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncFinished) @@ -451,11 +418,8 @@ func (db *Database) Enable(root common.Hash) error { // To ensure the history indexer always matches the current state, we must: // 1. Close any existing indexer // 2. Re-initialize the indexer so it starts indexing from the new state root. - if db.stateIndexer != nil && db.stateFreezer != nil && db.config.EnableStateIndexing { - db.stateIndexer.close() - db.stateIndexer = newHistoryIndexer(db.diskdb, db.stateFreezer, db.tree.bottom().stateID(), typeStateHistory) - log.Info("Re-enabled state history indexing") - } + db.setHistoryIndexer() + log.Info("Rebuilt trie database", "root", root) return nil } @@ -511,6 +475,12 @@ func (db *Database) Recover(root common.Hash) error { if err != nil { return err } + if db.trienodeFreezer != nil { + _, err = truncateFromHead(db.trienodeFreezer, typeTrienodeHistory, dl.stateID()) + if err != nil { + return err + } + } log.Debug("Recovered state", "root", root, "elapsed", common.PrettyDuration(time.Since(start))) return nil } @@ -571,11 +541,21 @@ func (db *Database) Close() error { if db.stateIndexer != nil { db.stateIndexer.close() } + if db.trienodeIndexer != nil { + db.trienodeIndexer.close() + } // Close the attached state history freezer. - if db.stateFreezer == nil { - return nil + if db.stateFreezer != nil { + if err := db.stateFreezer.Close(); err != nil { + return err + } + } + if db.trienodeFreezer != nil { + if err := db.trienodeFreezer.Close(); err != nil { + return err + } } - return db.stateFreezer.Close() + return nil } // Size returns the current storage size of the memory cache in front of the diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 40d0554532..c85e62d6e1 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -950,7 +950,7 @@ func TestDatabaseIndexRecovery(t *testing.T) { var ( dIndex int roots = env.roots - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) ) for i, root := range roots { if root == dRoot { @@ -1011,7 +1011,7 @@ func TestDatabaseIndexRecovery(t *testing.T) { // Ensure the truncated state histories become accessible bRoot = env.db.tree.bottom().rootHash() - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) for i, root := range roots { if root == bRoot { break diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index ae523c979c..8ca3a39cf3 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -79,7 +79,7 @@ func (dl *diffLayer) parentLayer() layer { // node implements the layer interface, retrieving the trie node blob with the // provided node information. No error will be returned if the node is not found. -func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) { +func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) { // Hold the lock, ensure the parent won't be changed during the // state accessing. dl.lock.RLock() @@ -91,7 +91,7 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dirtyNodeHitMeter.Mark(1) dirtyNodeHitDepthHist.Update(int64(depth)) dirtyNodeReadMeter.Mark(int64(len(n.Blob))) - return n.Blob, n.Hash, &nodeLoc{loc: locDiffLayer, depth: depth}, nil + return n.Blob, n.Hash, nodeLoc{loc: locDiffLayer, depth: depth}, nil } // Trie node unknown to this layer, resolve from parent return dl.parent.node(owner, path, depth+1) diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index dcba567db9..4272309108 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -116,12 +117,12 @@ func (dl *diskLayer) markStale() { // node implements the layer interface, retrieving the trie node with the // provided node info. No error will be returned if the node is not found. -func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) { +func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) { dl.lock.RLock() defer dl.lock.RUnlock() if dl.stale { - return nil, common.Hash{}, nil, errSnapshotStale + return nil, common.Hash{}, nodeLoc{}, errSnapshotStale } // Try to retrieve the trie node from the not-yet-written node buffer first // (both the live one and the frozen one). Note the buffer is lock free since @@ -133,7 +134,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dirtyNodeHitMeter.Mark(1) dirtyNodeReadMeter.Mark(int64(len(n.Blob))) dirtyNodeHitDepthHist.Update(int64(depth)) - return n.Blob, n.Hash, &nodeLoc{loc: locDirtyCache, depth: depth}, nil + return n.Blob, n.Hash, nodeLoc{loc: locDirtyCache, depth: depth}, nil } } } @@ -145,7 +146,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co if blob := dl.nodes.Get(key); len(blob) > 0 { cleanNodeHitMeter.Mark(1) cleanNodeReadMeter.Mark(int64(len(blob))) - return blob, crypto.Keccak256Hash(blob), &nodeLoc{loc: locCleanCache, depth: depth}, nil + return blob, crypto.Keccak256Hash(blob), nodeLoc{loc: locCleanCache, depth: depth}, nil } cleanNodeMissMeter.Mark(1) } @@ -165,7 +166,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dl.nodes.Set(key, blob) cleanNodeWriteMeter.Mark(int64(len(blob))) } - return blob, crypto.Keccak256Hash(blob), &nodeLoc{loc: locDiskLayer, depth: depth}, nil + return blob, crypto.Keccak256Hash(blob), nodeLoc{loc: locDiskLayer, depth: depth}, nil } // account directly retrieves the account RLP associated with a particular @@ -280,7 +281,7 @@ func (dl *diskLayer) storage(accountHash, storageHash common.Hash, depth int) ([ // If the layer is being generated, ensure the requested storage slot // has already been covered by the generator. - key := append(accountHash[:], storageHash[:]...) + key := storageKeySlice(accountHash, storageHash) marker := dl.genMarker() if marker != nil && bytes.Compare(key, marker) > 0 { return nil, errNotCoveredYet @@ -328,36 +329,61 @@ func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes *no return newDiffLayer(dl, root, id, block, nodes, states) } -// writeStateHistory stores the state history and indexes if indexing is +// writeHistory stores the specified history and indexes if indexing is // permitted. // // What's more, this function also returns a flag indicating whether the // buffer flushing is required, ensuring the persistent state ID is always // greater than or equal to the first history ID. -func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) { - // Short circuit if state history is not permitted - if dl.db.stateFreezer == nil { +func (dl *diskLayer) writeHistory(typ historyType, diff *diffLayer) (bool, error) { + var ( + limit uint64 + freezer ethdb.AncientStore + indexer *historyIndexer + writeFunc func(writer ethdb.AncientWriter, dl *diffLayer) error + ) + switch typ { + case typeStateHistory: + freezer = dl.db.stateFreezer + indexer = dl.db.stateIndexer + writeFunc = writeStateHistory + limit = dl.db.config.StateHistory + case typeTrienodeHistory: + freezer = dl.db.trienodeFreezer + indexer = dl.db.trienodeIndexer + writeFunc = func(writer ethdb.AncientWriter, diff *diffLayer) error { + return writeTrienodeHistory(writer, diff, dl.db.config.FullValueCheckpoint) + } + // Skip the history commit if the trienode history is not permitted + if dl.db.config.TrienodeHistory < 0 { + return false, nil + } + limit = uint64(dl.db.config.TrienodeHistory) + default: + panic(fmt.Sprintf("unknown history type: %v", typ)) + } + // Short circuit if the history freezer is nil + if freezer == nil { return false, nil } // Bail out with an error if writing the state history fails. // This can happen, for example, if the device is full. - err := writeStateHistory(dl.db.stateFreezer, diff) + err := writeFunc(freezer, diff) if err != nil { return false, err } - // Notify the state history indexer for newly created history - if dl.db.stateIndexer != nil { - if err := dl.db.stateIndexer.extend(diff.stateID()); err != nil { + // Notify the history indexer for newly created history + if indexer != nil { + if err := indexer.extend(diff.stateID()); err != nil { return false, err } } // Determine if the persisted history object has exceeded the // configured limitation. - limit := dl.db.config.StateHistory if limit == 0 { return false, nil } - tail, err := dl.db.stateFreezer.Tail() + tail, err := freezer.Tail() if err != nil { return false, err } // firstID = tail+1 @@ -380,14 +406,14 @@ func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) { // These measures ensure the persisted state ID always remains greater // than or equal to the first history ID. if persistentID := rawdb.ReadPersistentStateID(dl.db.diskdb); persistentID < newFirst { - log.Debug("Skip tail truncation", "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit) + log.Debug("Skip tail truncation", "type", typ, "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit) return true, nil } - pruned, err := truncateFromTail(dl.db.stateFreezer, typeStateHistory, newFirst-1) + pruned, err := truncateFromTail(freezer, typ, newFirst-1) if err != nil { return false, err } - log.Debug("Pruned state history", "items", pruned, "tailid", newFirst) + log.Debug("Pruned history", "type", typ, "items", pruned, "tailid", newFirst) return false, nil } @@ -401,10 +427,22 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // Construct and store the state history first. If crash happens after storing // the state history but without flushing the corresponding states(journal), // the stored state history will be truncated from head in the next restart. - flush, err := dl.writeStateHistory(bottom) + flushA, err := dl.writeHistory(typeStateHistory, bottom) if err != nil { return nil, err } + // Construct and store the trienode history first. If crash happens after + // storing the trienode history but without flushing the corresponding + // states(journal), the stored trienode history will be truncated from head + // in the next restart. + flushB, err := dl.writeHistory(typeTrienodeHistory, bottom) + if err != nil { + return nil, err + } + // Since the state history and trienode history may be configured with different + // lengths, the buffer will be flushed once either of them meets its threshold. + flush := flushA || flushB + // Mark the diskLayer as stale before applying any mutations on top. dl.stale = true @@ -453,7 +491,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // Freeze the live buffer and schedule background flushing dl.frozen = combined - dl.frozen.flush(bottom.root, dl.db.diskdb, dl.db.stateFreezer, progress, dl.nodes, dl.states, bottom.stateID(), func() { + dl.frozen.flush(bottom.root, dl.db.diskdb, []ethdb.AncientWriter{dl.db.stateFreezer, dl.db.trienodeFreezer}, progress, dl.nodes, dl.states, bottom.stateID(), func() { // Resume the background generation if it's not completed yet. // The generator is assumed to be available if the progress is // not nil. @@ -521,12 +559,17 @@ func (dl *diskLayer) revert(h *stateHistory) (*diskLayer, error) { dl.stale = true - // Unindex the corresponding state history + // Unindex the corresponding history if dl.db.stateIndexer != nil { if err := dl.db.stateIndexer.shorten(dl.id); err != nil { return nil, err } } + if dl.db.trienodeIndexer != nil { + if err := dl.db.trienodeIndexer.shorten(dl.id); err != nil { + return nil, err + } + } // State change may be applied to node buffer, or the persistent // state, depends on if node buffer is empty or not. If the node // buffer is not empty, it means that the state transition that diff --git a/triedb/pathdb/flush.go b/triedb/pathdb/flush.go index f2b4275124..12cfc288d1 100644 --- a/triedb/pathdb/flush.go +++ b/triedb/pathdb/flush.go @@ -117,15 +117,16 @@ func writeStates(batch ethdb.Batch, genMarker []byte, accountData map[common.Has continue } slots += 1 + key := storageKeySlice(addrHash, storageHash) if len(blob) == 0 { rawdb.DeleteStorageSnapshot(batch, addrHash, storageHash) if clean != nil { - clean.Set(append(addrHash[:], storageHash[:]...), nil) + clean.Set(key, nil) } } else { rawdb.WriteStorageSnapshot(batch, addrHash, storageHash, blob) if clean != nil { - clean.Set(append(addrHash[:], storageHash[:]...), blob) + clean.Set(key, blob) } } } diff --git a/triedb/pathdb/generate.go b/triedb/pathdb/generate.go index dd49eb8d37..a154c452f7 100644 --- a/triedb/pathdb/generate.go +++ b/triedb/pathdb/generate.go @@ -148,6 +148,7 @@ func (g *generator) stop() { g.abort <- ch <-ch g.running = false + log.Debug("Snapshot generation has been terminated") } // completed returns the flag indicating if the whole generation is done. diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index d78999f218..820c3c03bf 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -22,6 +22,7 @@ import ( "iter" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -121,6 +122,20 @@ func (ident stateIdent) String() string { return ident.addressHash.Hex() + ident.path } +func (ident stateIdent) bloomSize() int { + if ident.typ == typeAccount { + return 0 + } + if ident.typ == typeStorage { + return 0 + } + scheme := accountIndexScheme + if ident.addressHash != (common.Hash{}) { + scheme = storageIndexScheme + } + return scheme.getBitmapSize(len(ident.path)) +} + // newAccountIdent constructs a state identifier for an account. func newAccountIdent(addressHash common.Hash) stateIdent { return stateIdent{ @@ -180,17 +195,62 @@ func newStorageIdentQuery(address common.Address, addressHash common.Hash, stora } } -// newTrienodeIdentQuery constructs a state identifier for a trie node. -// the addressHash denotes the address hash of the associated account; -// the path denotes the path of the node within the trie; -// -// nolint:unused -func newTrienodeIdentQuery(addrHash common.Hash, path []byte) stateIdentQuery { - return stateIdentQuery{ - stateIdent: newTrienodeIdent(addrHash, string(path)), +// indexElem defines the element for indexing. +type indexElem interface { + key() stateIdent + ext() []uint16 +} + +type accountIndexElem struct { + addressHash common.Hash +} + +func (a accountIndexElem) key() stateIdent { + return stateIdent{ + typ: typeAccount, + addressHash: a.addressHash, } } +func (a accountIndexElem) ext() []uint16 { + return nil +} + +type storageIndexElem struct { + addressHash common.Hash + storageHash common.Hash +} + +func (a storageIndexElem) key() stateIdent { + return stateIdent{ + typ: typeStorage, + addressHash: a.addressHash, + storageHash: a.storageHash, + } +} + +func (a storageIndexElem) ext() []uint16 { + return nil +} + +type trienodeIndexElem struct { + owner common.Hash + path string + data []uint16 +} + +func (a trienodeIndexElem) key() stateIdent { + return stateIdent{ + typ: typeTrienode, + addressHash: a.owner, + path: a.path, + } +} + +func (a trienodeIndexElem) ext() []uint16 { + return a.data +} + // history defines the interface of historical data, shared by stateHistory // and trienodeHistory. type history interface { @@ -198,7 +258,7 @@ type history interface { typ() historyType // forEach returns an iterator to traverse the state entries in the history. - forEach() iter.Seq[stateIdent] + forEach() iter.Seq[indexElem] } var ( @@ -262,3 +322,133 @@ func truncateFromTail(store ethdb.AncientStore, typ historyType, ntail uint64) ( // Associated root->id mappings are left in the database. return int(ntail - otail), nil } + +// purgeHistory resets the history and also purges the associated index data. +func purgeHistory(store ethdb.ResettableAncientStore, disk ethdb.KeyValueStore, typ historyType) { + if store == nil { + return + } + frozen, err := store.Ancients() + if err != nil { + log.Crit("Failed to retrieve head of history", "type", typ, "err", err) + } + if frozen == 0 { + return + } + // Purge all state history indexing data first + batch := disk.NewBatch() + if typ == typeStateHistory { + rawdb.DeleteStateHistoryIndexMetadata(batch) + rawdb.DeleteStateHistoryIndexes(batch) + } else { + rawdb.DeleteTrienodeHistoryIndexMetadata(batch) + rawdb.DeleteTrienodeHistoryIndexes(batch) + } + if err := batch.Write(); err != nil { + log.Crit("Failed to purge history index", "type", typ, "err", err) + } + if err := store.Reset(); err != nil { + log.Crit("Failed to reset history", "type", typ, "err", err) + } + log.Info("Truncated extraneous history", "type", typ) +} + +// syncHistory explicitly sync the provided history stores. +func syncHistory(stores ...ethdb.AncientWriter) error { + for _, store := range stores { + if store == nil { + continue + } + if err := store.SyncAncient(); err != nil { + return err + } + } + return nil +} + +// repairHistory truncates any leftover history objects in either the state +// history or the trienode history, which may occur due to an unclean shutdown +// or other unexpected events. +// +// Additionally, this mechanism ensures that the state history and trienode +// history remain aligned. Since the trienode history is optional and not +// required by regular users, a gap between the trienode history and the +// persistent state may appear if the trienode history was disabled during the +// previous run. This process detects and resolves such gaps, preventing +// unexpected panics. +func repairHistory(db ethdb.Database, isVerkle bool, readOnly bool, stateID uint64, enableTrienode bool) (ethdb.ResettableAncientStore, ethdb.ResettableAncientStore, error) { + ancient, err := db.AncientDatadir() + if err != nil { + // TODO error out if ancient store is disabled. A tons of unit tests + // disable the ancient store thus the error here will immediately fail + // all of them. Fix the tests first. + return nil, nil, nil + } + // State history is mandatory as it is the key component that ensures + // resilience to deep reorgs. + states, err := rawdb.NewStateFreezer(ancient, isVerkle, readOnly) + if err != nil { + log.Crit("Failed to open state history freezer", "err", err) + } + + // Trienode history is optional and only required for building archive + // node with state proofs. + var trienodes ethdb.ResettableAncientStore + if enableTrienode { + trienodes, err = rawdb.NewTrienodeFreezer(ancient, isVerkle, readOnly) + if err != nil { + log.Crit("Failed to open trienode history freezer", "err", err) + } + } + + // Reset the both histories if the trie database is not initialized yet. + // This action is necessary because these histories are not expected + // to exist without an initialized trie database. + if stateID == 0 { + purgeHistory(states, db, typeStateHistory) + purgeHistory(trienodes, db, typeTrienodeHistory) + return states, trienodes, nil + } + // Truncate excessive history entries in either the state history or + // the trienode history, ensuring both histories remain aligned with + // the state. + head, err := states.Ancients() + if err != nil { + return nil, nil, err + } + if stateID > head { + return nil, nil, fmt.Errorf("gap between state [#%d] and state history [#%d]", stateID, head) + } + if trienodes != nil { + th, err := trienodes.Ancients() + if err != nil { + return nil, nil, err + } + if stateID > th { + return nil, nil, fmt.Errorf("gap between state [#%d] and trienode history [#%d]", stateID, th) + } + if th != head { + log.Info("Histories are not aligned with each other", "state", head, "trienode", th) + head = min(head, th) + } + } + head = min(head, stateID) + + // Truncate the extra history elements above in freezer in case it's not + // aligned with the state. It might happen after an unclean shutdown. + truncate := func(store ethdb.AncientStore, typ historyType, nhead uint64) { + if store == nil { + return + } + pruned, err := truncateFromHead(store, typ, nhead) + if err != nil { + log.Crit("Failed to truncate extra histories", "typ", typ, "err", err) + } + if pruned != 0 { + log.Warn("Truncated extra histories", "typ", typ, "number", pruned) + } + } + truncate(states, typeStateHistory, head) + truncate(trienodes, typeTrienodeHistory, head) + return states, trienodes, nil +} diff --git a/triedb/pathdb/history_index.go b/triedb/pathdb/history_index.go index 28eb857f9e..ebdb8376db 100644 --- a/triedb/pathdb/history_index.go +++ b/triedb/pathdb/history_index.go @@ -20,29 +20,34 @@ import ( "errors" "fmt" "math" - "sort" "sync" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" ) -// parseIndex parses the index data with the supplied byte stream. The index data -// is a list of fixed-sized metadata. Empty metadata is regarded as invalid. -func parseIndex(blob []byte) ([]*indexBlockDesc, error) { +// parseIndex parses the index data from the provided byte stream. The index data +// is a sequence of fixed-size metadata entries, and any empty metadata entry is +// considered invalid. +// +// Each metadata entry consists of two components: the indexBlockDesc and an +// optional extension bitmap. The bitmap length may vary across different categories, +// but must remain consistent within the same category. +func parseIndex(blob []byte, bitmapSize int) ([]*indexBlockDesc, error) { if len(blob) == 0 { return nil, errors.New("empty state history index") } - if len(blob)%indexBlockDescSize != 0 { - return nil, fmt.Errorf("corrupted state index, len: %d", len(blob)) + size := indexBlockDescSize + bitmapSize + if len(blob)%size != 0 { + return nil, fmt.Errorf("corrupted state index, len: %d, bitmap size: %d", len(blob), bitmapSize) } var ( lastID uint32 - descList []*indexBlockDesc + descList = make([]*indexBlockDesc, 0, len(blob)/size) ) - for i := 0; i < len(blob)/indexBlockDescSize; i++ { + for i := 0; i < len(blob)/size; i++ { var desc indexBlockDesc - desc.decode(blob[i*indexBlockDescSize : (i+1)*indexBlockDescSize]) + desc.decode(blob[i*size : (i+1)*size]) if desc.empty() { return nil, errors.New("empty state history index block") } @@ -71,34 +76,36 @@ func parseIndex(blob []byte) ([]*indexBlockDesc, error) { // indexReader is the structure to look up the state history index records // associated with the specific state element. type indexReader struct { - db ethdb.KeyValueReader - descList []*indexBlockDesc - mu sync.RWMutex - readers map[uint32]*blockReader - state stateIdent + db ethdb.KeyValueReader + descList []*indexBlockDesc + mu sync.RWMutex + readers map[uint32]*blockReader + state stateIdent + bitmapSize int } // loadIndexData loads the index data associated with the specified state. -func loadIndexData(db ethdb.KeyValueReader, state stateIdent) ([]*indexBlockDesc, error) { +func loadIndexData(db ethdb.KeyValueReader, state stateIdent, bitmapSize int) ([]*indexBlockDesc, error) { blob := readStateIndex(state, db) if len(blob) == 0 { return nil, nil } - return parseIndex(blob) + return parseIndex(blob, bitmapSize) } // newIndexReader constructs a index reader for the specified state. Reader with // empty data is allowed. -func newIndexReader(db ethdb.KeyValueReader, state stateIdent) (*indexReader, error) { - descList, err := loadIndexData(db, state) +func newIndexReader(db ethdb.KeyValueReader, state stateIdent, bitmapSize int) (*indexReader, error) { + descList, err := loadIndexData(db, state, bitmapSize) if err != nil { return nil, err } return &indexReader{ - descList: descList, - readers: make(map[uint32]*blockReader), - db: db, - state: state, + descList: descList, + readers: make(map[uint32]*blockReader), + db: db, + state: state, + bitmapSize: bitmapSize, }, nil } @@ -109,13 +116,11 @@ func (r *indexReader) refresh() error { // may have been modified by additional elements written to the disk. if len(r.descList) != 0 { last := r.descList[len(r.descList)-1] - if !last.full() { - r.mu.Lock() - delete(r.readers, last.id) - r.mu.Unlock() - } + r.mu.Lock() + delete(r.readers, last.id) + r.mu.Unlock() } - descList, err := loadIndexData(r.db, r.state) + descList, err := loadIndexData(r.db, r.state, r.bitmapSize) if err != nil { return err } @@ -126,32 +131,15 @@ func (r *indexReader) refresh() error { // readGreaterThan locates the first element that is greater than the specified // id. If no such element is found, MaxUint64 is returned. func (r *indexReader) readGreaterThan(id uint64) (uint64, error) { - index := sort.Search(len(r.descList), func(i int) bool { - return id < r.descList[i].max - }) - if index == len(r.descList) { - return math.MaxUint64, nil + it := r.newIterator(nil) + found := it.SeekGT(id) + if err := it.Error(); err != nil { + return 0, err } - desc := r.descList[index] - - r.mu.RLock() - br, ok := r.readers[desc.id] - r.mu.RUnlock() - - if !ok { - var err error - blob := readStateIndexBlock(r.state, r.db, desc.id) - br, err = newBlockReader(blob) - if err != nil { - return 0, err - } - r.mu.Lock() - r.readers[desc.id] = br - r.mu.Unlock() + if !found { + return math.MaxUint64, nil } - // The supplied ID is not greater than block.max, ensuring that an element - // satisfying the condition can be found. - return br.readGreaterThan(id) + return it.ID(), nil } // indexWriter is responsible for writing index data for a specific state (either @@ -161,57 +149,75 @@ func (r *indexReader) readGreaterThan(id uint64) (uint64, error) { // history ids) is stored in these second-layer index blocks, which are size // limited. type indexWriter struct { - descList []*indexBlockDesc // The list of index block descriptions - bw *blockWriter // The live index block writer - frozen []*blockWriter // The finalized index block writers, waiting for flush - lastID uint64 // The ID of the latest tracked history - state stateIdent - db ethdb.KeyValueReader + descList []*indexBlockDesc // The list of index block descriptions + bw *blockWriter // The live index block writer + frozen []*blockWriter // The finalized index block writers, waiting for flush + lastID uint64 // The ID of the latest tracked history + state stateIdent // The identifier of the state being indexed + bitmapSize int // The size of optional extension bitmap + db ethdb.KeyValueReader } -// newIndexWriter constructs the index writer for the specified state. -func newIndexWriter(db ethdb.KeyValueReader, state stateIdent) (*indexWriter, error) { +// newIndexWriter constructs the index writer for the specified state. Additionally, +// it takes an integer as the limit and prunes all existing elements above that ID. +// It's essential as the recovery mechanism after unclean shutdown during the history +// indexing. +func newIndexWriter(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexWriter, error) { blob := readStateIndex(state, db) if len(blob) == 0 { - desc := newIndexBlockDesc(0) - bw, _ := newBlockWriter(nil, desc) + desc := newIndexBlockDesc(0, bitmapSize) + bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */, bitmapSize != 0) return &indexWriter{ - descList: []*indexBlockDesc{desc}, - bw: bw, - state: state, - db: db, + descList: []*indexBlockDesc{desc}, + bw: bw, + state: state, + db: db, + bitmapSize: bitmapSize, }, nil } - descList, err := parseIndex(blob) + descList, err := parseIndex(blob, bitmapSize) if err != nil { return nil, err } + // Trim trailing blocks whose elements all exceed the limit. + for i := len(descList) - 1; i > 0 && descList[i].max > limit; i-- { + // The previous block has the elements that exceed the limit, + // therefore the current block can be entirely dropped. + if descList[i-1].max >= limit { + descList = descList[:i] + } + } + // Take the last block for appending new elements lastDesc := descList[len(descList)-1] indexBlock := readStateIndexBlock(state, db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + + // Construct the writer for the last block. All elements in this block + // that exceed the limit will be truncated. + bw, err := newBlockWriter(indexBlock, lastDesc, limit, bitmapSize != 0) if err != nil { return nil, err } return &indexWriter{ - descList: descList, - lastID: lastDesc.max, - bw: bw, - state: state, - db: db, + descList: descList, + lastID: bw.last(), + bw: bw, + state: state, + db: db, + bitmapSize: bitmapSize, }, nil } // append adds the new element into the index writer. -func (w *indexWriter) append(id uint64) error { +func (w *indexWriter) append(id uint64, ext []uint16) error { if id <= w.lastID { return fmt.Errorf("append element out of order, last: %d, this: %d", w.lastID, id) } - if w.bw.full() { + if w.bw.estimateFull(ext) { if err := w.rotate(); err != nil { return err } } - if err := w.bw.append(id); err != nil { + if err := w.bw.append(id, ext); err != nil { return err } w.lastID = id @@ -224,10 +230,10 @@ func (w *indexWriter) append(id uint64) error { func (w *indexWriter) rotate() error { var ( err error - desc = newIndexBlockDesc(w.bw.desc.id + 1) + desc = newIndexBlockDesc(w.bw.desc.id+1, w.bitmapSize) ) w.frozen = append(w.frozen, w.bw) - w.bw, err = newBlockWriter(nil, desc) + w.bw, err = newBlockWriter(nil, desc, 0 /* useless if the block is empty */, w.bitmapSize != 0) if err != nil { return err } @@ -259,7 +265,8 @@ func (w *indexWriter) finish(batch ethdb.Batch) { } w.frozen = nil // release all the frozen writers - buf := make([]byte, 0, indexBlockDescSize*len(descList)) + size := indexBlockDescSize + w.bitmapSize + buf := make([]byte, 0, size*len(descList)) for _, desc := range descList { buf = append(buf, desc.encode()...) } @@ -268,49 +275,64 @@ func (w *indexWriter) finish(batch ethdb.Batch) { // indexDeleter is responsible for deleting index data for a specific state. type indexDeleter struct { - descList []*indexBlockDesc // The list of index block descriptions - bw *blockWriter // The live index block writer - dropped []uint32 // The list of index block id waiting for deleting - lastID uint64 // The ID of the latest tracked history - state stateIdent - db ethdb.KeyValueReader + descList []*indexBlockDesc // The list of index block descriptions + bw *blockWriter // The live index block writer + dropped []uint32 // The list of index block id waiting for deleting + lastID uint64 // The ID of the latest tracked history + state stateIdent // The identifier of the state being indexed + bitmapSize int // The size of optional extension bitmap + db ethdb.KeyValueReader } // newIndexDeleter constructs the index deleter for the specified state. -func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent) (*indexDeleter, error) { +func newIndexDeleter(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexDeleter, error) { blob := readStateIndex(state, db) if len(blob) == 0 { // TODO(rjl493456442) we can probably return an error here, // deleter with no data is meaningless. - desc := newIndexBlockDesc(0) - bw, _ := newBlockWriter(nil, desc) + desc := newIndexBlockDesc(0, bitmapSize) + bw, _ := newBlockWriter(nil, desc, 0 /* useless if the block is empty */, bitmapSize != 0) return &indexDeleter{ - descList: []*indexBlockDesc{desc}, - bw: bw, - state: state, - db: db, + descList: []*indexBlockDesc{desc}, + bw: bw, + state: state, + bitmapSize: bitmapSize, + db: db, }, nil } - descList, err := parseIndex(blob) + descList, err := parseIndex(blob, bitmapSize) if err != nil { return nil, err } + // Trim trailing blocks whose elements all exceed the limit. + for i := len(descList) - 1; i > 0 && descList[i].max > limit; i-- { + // The previous block has the elements that exceed the limit, + // therefore the current block can be entirely dropped. + if descList[i-1].max >= limit { + descList = descList[:i] + } + } + // Take the block for deleting element from lastDesc := descList[len(descList)-1] indexBlock := readStateIndexBlock(state, db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + + // Construct the writer for the last block. All elements in this block + // that exceed the limit will be truncated. + bw, err := newBlockWriter(indexBlock, lastDesc, limit, bitmapSize != 0) if err != nil { return nil, err } return &indexDeleter{ - descList: descList, - lastID: lastDesc.max, - bw: bw, - state: state, - db: db, + descList: descList, + lastID: bw.last(), + bw: bw, + state: state, + bitmapSize: bitmapSize, + db: db, }, nil } -// empty returns an flag indicating whether the state index is empty. +// empty returns whether the state index is empty. func (d *indexDeleter) empty() bool { return d.bw.empty() && len(d.descList) == 1 } @@ -343,7 +365,7 @@ func (d *indexDeleter) pop(id uint64) error { // Open the previous block writer for deleting lastDesc := d.descList[len(d.descList)-1] indexBlock := readStateIndexBlock(d.state, d.db, lastDesc.id) - bw, err := newBlockWriter(indexBlock, lastDesc) + bw, err := newBlockWriter(indexBlock, lastDesc, lastDesc.max, d.bitmapSize != 0) if err != nil { return err } @@ -369,7 +391,8 @@ func (d *indexDeleter) finish(batch ethdb.Batch) { if d.empty() { deleteStateIndex(d.state, batch) } else { - buf := make([]byte, 0, indexBlockDescSize*len(d.descList)) + size := indexBlockDescSize + d.bitmapSize + buf := make([]byte, 0, size*len(d.descList)) for _, desc := range d.descList { buf = append(buf, desc.encode()...) } diff --git a/triedb/pathdb/history_index_block.go b/triedb/pathdb/history_index_block.go index 5abdee682a..bb823bb13f 100644 --- a/triedb/pathdb/history_index_block.go +++ b/triedb/pathdb/history_index_block.go @@ -17,31 +17,37 @@ package pathdb import ( + "bytes" "encoding/binary" "errors" "fmt" "math" - "sort" + + "github.com/ethereum/go-ethereum/log" ) const ( - indexBlockDescSize = 14 // The size of index block descriptor - indexBlockEntriesCap = 4096 // The maximum number of entries can be grouped in a block - indexBlockRestartLen = 256 // The restart interval length of index block - historyIndexBatch = 512 * 1024 // The number of state history indexes for constructing or deleting as batch + indexBlockDescSize = 14 // The size of index block descriptor + indexBlockMaxSize = 4096 // The maximum size of a single index block + indexBlockRestartLen = 256 // The restart interval length of index block ) // indexBlockDesc represents a descriptor for an index block, which contains a // list of state mutation records associated with a specific state (either an // account or a storage slot). type indexBlockDesc struct { - max uint64 // The maximum state ID retained within the block - entries uint16 // The number of state mutation records retained within the block - id uint32 // The id of the index block + max uint64 // The maximum state ID retained within the block + entries uint16 // The number of state mutation records retained within the block + id uint32 // The id of the index block + extBitmap []byte // Optional fixed-size bitmap for the included extension elements } -func newIndexBlockDesc(id uint32) *indexBlockDesc { - return &indexBlockDesc{id: id} +func newIndexBlockDesc(id uint32, bitmapSize int) *indexBlockDesc { + var bitmap []byte + if bitmapSize > 0 { + bitmap = make([]byte, bitmapSize) + } + return &indexBlockDesc{id: id, extBitmap: bitmap} } // empty indicates whether the block is empty with no element retained. @@ -49,26 +55,33 @@ func (d *indexBlockDesc) empty() bool { return d.entries == 0 } -// full indicates whether the number of elements in the block exceeds the -// preconfigured limit. -func (d *indexBlockDesc) full() bool { - return d.entries >= indexBlockEntriesCap -} - // encode packs index block descriptor into byte stream. func (d *indexBlockDesc) encode() []byte { - var buf [indexBlockDescSize]byte + buf := make([]byte, indexBlockDescSize+len(d.extBitmap)) binary.BigEndian.PutUint64(buf[0:8], d.max) binary.BigEndian.PutUint16(buf[8:10], d.entries) binary.BigEndian.PutUint32(buf[10:14], d.id) + copy(buf[indexBlockDescSize:], d.extBitmap) return buf[:] } -// decode unpacks index block descriptor from byte stream. +// decode unpacks index block descriptor from byte stream. It's unsafe to mutate +// the provided byte stream after the function call. func (d *indexBlockDesc) decode(blob []byte) { d.max = binary.BigEndian.Uint64(blob[:8]) d.entries = binary.BigEndian.Uint16(blob[8:10]) d.id = binary.BigEndian.Uint32(blob[10:14]) + d.extBitmap = blob[indexBlockDescSize:] // no-deep copy! +} + +// copy returns a deep-copied object. +func (d *indexBlockDesc) copy() *indexBlockDesc { + return &indexBlockDesc{ + max: d.max, + entries: d.entries, + id: d.id, + extBitmap: bytes.Clone(d.extBitmap), + } } // parseIndexBlock parses the index block with the supplied byte stream. @@ -96,20 +109,38 @@ func (d *indexBlockDesc) decode(blob []byte) { // A uint16 can cover offsets in the range [0, 65536), which is more than enough // to store 4096 integers. // -// Each chunk begins with the full value of the first integer, followed by -// subsequent integers representing the differences between the current value -// and the preceding one. Integers are encoded with variable-size for best -// storage efficiency. Each chunk can be illustrated as below. +// Each chunk begins with a full integer value for the first element, followed +// by subsequent integers encoded as differences (deltas) from their preceding +// values. All integers use variable-length encoding for optimal space efficiency. +// +// In the updated format, each element in the chunk may optionally include an +// "extension" section. If an extension is present, it starts with a var-size +// integer indicating the length of the remaining extension payload, followed by +// that many bytes. If no extension is present, the element format is identical +// to the original version (i.e., only the integer or delta value is encoded). // -// Restart ---> +----------------+ -// | Full integer | -// +----------------+ -// | Diff with prev | -// +----------------+ -// | ... | -// +----------------+ -// | Diff with prev | -// +----------------+ +// In the trienode history index, the extension field contains the list of +// trie node IDs that fall within this range. For the given state transition, +// these IDs represent the specific nodes in this range that were mutated. +// +// Whether an element includes an extension is determined by the block reader +// based on the specification. Conceptually, a chunk is structured as: +// +// Restart ---> +----------------+ +// | Full integer | +// +----------------+ +// | (Extension?) | +// +----------------+ +// | Diff with prev | +// +----------------+ +// | (Extension?) | +// +----------------+ +// | ... | +// +----------------+ +// | Diff with prev | +// +----------------+ +// | (Extension?) | +// +----------------+ // // Empty index block is regarded as invalid. func parseIndexBlock(blob []byte) ([]uint16, []byte, error) { @@ -147,104 +178,98 @@ func parseIndexBlock(blob []byte) ([]uint16, []byte, error) { type blockReader struct { restarts []uint16 data []byte + hasExt bool } // newBlockReader constructs the block reader with the supplied block data. -func newBlockReader(blob []byte) (*blockReader, error) { +func newBlockReader(blob []byte, hasExt bool) (*blockReader, error) { restarts, data, err := parseIndexBlock(blob) if err != nil { return nil, err } return &blockReader{ restarts: restarts, - data: data, // safe to own the slice + data: data, // safe to own the slice + hasExt: hasExt, // flag whether extension should be resolved }, nil } // readGreaterThan locates the first element in the block that is greater than // the specified value. If no such element is found, MaxUint64 is returned. func (br *blockReader) readGreaterThan(id uint64) (uint64, error) { - var err error - index := sort.Search(len(br.restarts), func(i int) bool { - item, n := binary.Uvarint(br.data[br.restarts[i]:]) - if n <= 0 { - err = fmt.Errorf("failed to decode item at restart %d", br.restarts[i]) - } - return item > id - }) - if err != nil { + it := br.newIterator(nil) + found := it.SeekGT(id) + if err := it.Error(); err != nil { return 0, err } - if index == 0 { - item, _ := binary.Uvarint(br.data[br.restarts[0]:]) - return item, nil - } - var ( - start int - limit int - result uint64 - ) - if index == len(br.restarts) { - // The element being searched falls within the last restart section, - // there is no guarantee such element can be found. - start = int(br.restarts[len(br.restarts)-1]) - limit = len(br.data) - } else { - // The element being searched falls within the non-last restart section, - // such element can be found for sure. - start = int(br.restarts[index-1]) - limit = int(br.restarts[index]) - } - pos := start - for pos < limit { - x, n := binary.Uvarint(br.data[pos:]) - if pos == start { - result = x - } else { - result += x - } - if result > id { - return result, nil - } - pos += n - } - // The element which is greater than specified id is not found. - if index == len(br.restarts) { + if !found { return math.MaxUint64, nil } - // The element which is the first one greater than the specified id - // is exactly the one located at the restart point. - item, _ := binary.Uvarint(br.data[br.restarts[index]:]) - return item, nil + return it.ID(), nil } type blockWriter struct { desc *indexBlockDesc // Descriptor of the block restarts []uint16 // Offsets into the data slice, marking the start of each section data []byte // Aggregated encoded data slice + hasExt bool // Flag whether the extension field for each element exists } -func newBlockWriter(blob []byte, desc *indexBlockDesc) (*blockWriter, error) { +// newBlockWriter constructs a block writer. In addition to the existing data +// and block description, it takes an element ID and prunes all existing elements +// above that ID. It's essential as the recovery mechanism after unclean shutdown +// during the history indexing. +func newBlockWriter(blob []byte, desc *indexBlockDesc, limit uint64, hasExt bool) (*blockWriter, error) { if len(blob) == 0 { return &blockWriter{ - desc: desc, - data: make([]byte, 0, 1024), + desc: desc, + data: make([]byte, 0, 1024), + hasExt: hasExt, }, nil } restarts, data, err := parseIndexBlock(blob) if err != nil { return nil, err } - return &blockWriter{ + writer := &blockWriter{ desc: desc, restarts: restarts, data: data, // safe to own the slice - }, nil + hasExt: hasExt, + } + var trimmed int + for !writer.empty() && writer.last() > limit { + if err := writer.pop(writer.last()); err != nil { + return nil, err + } + trimmed += 1 + } + if trimmed > 0 { + log.Debug("Truncated extraneous elements", "count", trimmed, "limit", limit) + } + return writer, nil +} + +// setBitmap applies the given extension elements into the bitmap. +func (b *blockWriter) setBitmap(ext []uint16) { + for _, n := range ext { + // Node ID zero is intentionally filtered out. Any element in this range + // can indicate that the sub-tree's root node was mutated, so storing zero + // is redundant and saves one byte for bitmap. + if n != 0 { + setBit(b.desc.extBitmap, int(n-1)) + } + } } // append adds a new element to the block. The new element must be greater than // the previous one. The provided ID is assumed to always be greater than 0. -func (b *blockWriter) append(id uint64) error { +// +// ext refers to the optional extension field attached to the appended element. +// This extension mechanism is used by trie-node history and represents a list of +// trie node IDs that fall within the range covered by the index element +// (typically corresponding to a sub-trie in trie-node history). +func (b *blockWriter) append(id uint64, ext []uint16) error { if id == 0 { return errors.New("invalid zero id") } @@ -271,13 +296,29 @@ func (b *blockWriter) append(id uint64) error { // element. b.data = binary.AppendUvarint(b.data, id-b.desc.max) } + // Extension validation + if (len(ext) == 0) != !b.hasExt { + if len(ext) == 0 { + return errors.New("missing extension") + } + return errors.New("unexpected extension") + } + // Append the extension if it is not nil. The extension is prefixed with a + // length indicator, and the block reader MUST understand this scheme and + // decode the extension accordingly. + if len(ext) > 0 { + b.setBitmap(ext) + enc := encodeIDs(ext) + b.data = binary.AppendUvarint(b.data, uint64(len(enc))) + b.data = append(b.data, enc...) + } b.desc.entries++ b.desc.max = id return nil } // scanSection traverses the specified section and terminates if fn returns true. -func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) { +func (b *blockWriter) scanSection(section int, fn func(uint64, int, []uint16) bool) error { var ( value uint64 start = int(b.restarts[section]) @@ -296,27 +337,47 @@ func (b *blockWriter) scanSection(section int, fn func(uint64, int) bool) { } else { value += x } - if fn(value, pos) { - return + // Resolve the extension if exists + var ( + err error + ext []uint16 + extLen int + ) + if b.hasExt { + l, ln := binary.Uvarint(b.data[pos+n:]) + extLen = ln + int(l) + ext, err = decodeIDs(b.data[pos+n+ln : pos+n+extLen]) + } + if err != nil { + return err + } + if fn(value, pos, ext) { + return nil } + // Shift to next position pos += n + pos += extLen } + return nil } // sectionLast returns the last element in the specified section. -func (b *blockWriter) sectionLast(section int) uint64 { +func (b *blockWriter) sectionLast(section int) (uint64, error) { var n uint64 - b.scanSection(section, func(v uint64, _ int) bool { + if err := b.scanSection(section, func(v uint64, _ int, _ []uint16) bool { n = v return false - }) - return n + }); err != nil { + return 0, err + } + return n, nil } // sectionSearch looks up the specified value in the given section, // the position and the preceding value will be returned if found. -func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int) { - b.scanSection(section, func(v uint64, p int) bool { +// It assumes that the preceding element exists in the section. +func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uint64, pos int, err error) { + if err := b.scanSection(section, func(v uint64, p int, _ []uint16) bool { if n == v { pos = p found = true @@ -324,8 +385,24 @@ func (b *blockWriter) sectionSearch(section int, n uint64) (found bool, prev uin } prev = v return false // continue iteration - }) - return found, prev, pos + }); err != nil { + return false, 0, 0, err + } + return found, prev, pos, nil +} + +// rebuildBitmap scans the entire block and rebuilds the bitmap. +func (b *blockWriter) rebuildBitmap() error { + clear(b.desc.extBitmap) + for i := 0; i < len(b.restarts); i++ { + if err := b.scanSection(i, func(v uint64, p int, ext []uint16) bool { + b.setBitmap(ext) + return false // continue iteration + }); err != nil { + return err + } + } + return nil } // pop removes the last element from the block. The assumption is held that block @@ -339,9 +416,9 @@ func (b *blockWriter) pop(id uint64) error { } // If there is only one entry left, the entire block should be reset if b.desc.entries == 1 { - //b.desc.min = 0 b.desc.max = 0 b.desc.entries = 0 + clear(b.desc.extBitmap) b.restarts = nil b.data = b.data[:0] return nil @@ -351,28 +428,45 @@ func (b *blockWriter) pop(id uint64) error { if b.desc.entries%indexBlockRestartLen == 1 { b.data = b.data[:b.restarts[len(b.restarts)-1]] b.restarts = b.restarts[:len(b.restarts)-1] - b.desc.max = b.sectionLast(len(b.restarts) - 1) + last, err := b.sectionLast(len(b.restarts) - 1) + if err != nil { + return err + } + b.desc.max = last b.desc.entries -= 1 - return nil + return b.rebuildBitmap() } // Look up the element preceding the one to be popped, in order to update // the maximum element in the block. - found, prev, pos := b.sectionSearch(len(b.restarts)-1, id) + found, prev, pos, err := b.sectionSearch(len(b.restarts)-1, id) + if err != nil { + return err + } if !found { return fmt.Errorf("pop element is not found, last: %d, this: %d", b.desc.max, id) } b.desc.max = prev b.data = b.data[:pos] b.desc.entries -= 1 - return nil + return b.rebuildBitmap() } func (b *blockWriter) empty() bool { return b.desc.empty() } -func (b *blockWriter) full() bool { - return b.desc.full() +func (b *blockWriter) estimateFull(ext []uint16) bool { + size := 8 + 2*len(ext) + return len(b.data)+size > indexBlockMaxSize +} + +// last returns the last element in the block. It should only be called when +// writer is not empty, otherwise the returned data is meaningless. +func (b *blockWriter) last() uint64 { + if b.empty() { + return 0 + } + return b.desc.max } // finish finalizes the index block encoding by appending the encoded restart points diff --git a/triedb/pathdb/history_index_block_test.go b/triedb/pathdb/history_index_block_test.go index c251cea2ec..923ae29348 100644 --- a/triedb/pathdb/history_index_block_test.go +++ b/triedb/pathdb/history_index_block_test.go @@ -17,6 +17,7 @@ package pathdb import ( + "bytes" "math" "math/rand" "slices" @@ -24,16 +25,36 @@ import ( "testing" ) +func randomExt(bitmapSize int, n int) []uint16 { + if bitmapSize == 0 { + return nil + } + var ( + limit = bitmapSize * 8 + extList []uint16 + ) + for i := 0; i < n; i++ { + extList = append(extList, uint16(rand.Intn(limit+1))) + } + return extList +} + func TestBlockReaderBasic(t *testing.T) { + testBlockReaderBasic(t, 0) + testBlockReaderBasic(t, 2) + testBlockReaderBasic(t, 34) +} + +func testBlockReaderBasic(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -60,18 +81,24 @@ func TestBlockReaderBasic(t *testing.T) { } func TestBlockReaderLarge(t *testing.T) { + testBlockReaderLarge(t, 0) + testBlockReaderLarge(t, 2) + testBlockReaderLarge(t, 34) +} + +func testBlockReaderLarge(t *testing.T, bitmapSize int) { var elements []uint64 for i := 0; i < 1000; i++ { elements = append(elements, rand.Uint64()) } slices.Sort(elements) - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -95,34 +122,91 @@ func TestBlockReaderLarge(t *testing.T) { } func TestBlockWriterBasic(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + testBlockWriteBasic(t, 0) + testBlockWriteBasic(t, 2) + testBlockWriteBasic(t, 34) +} + +func testBlockWriteBasic(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) if !bw.empty() { t.Fatal("expected empty block") } - bw.append(2) - if err := bw.append(1); err == nil { + bw.append(2, randomExt(bitmapSize, 5)) + if err := bw.append(1, randomExt(bitmapSize, 5)); err == nil { t.Fatal("out-of-order insertion is not expected") } + var maxElem uint64 for i := 0; i < 10; i++ { - bw.append(uint64(i + 3)) + bw.append(uint64(i+3), randomExt(bitmapSize, 5)) + maxElem = uint64(i + 3) } - bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0)) + bw, err := newBlockWriter(bw.finish(), newIndexBlockDesc(0, bitmapSize), maxElem, bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block writer, %v", err) } for i := 0; i < 10; i++ { - if err := bw.append(uint64(i + 100)); err != nil { + if err := bw.append(uint64(i+100), randomExt(bitmapSize, 5)); err != nil { t.Fatalf("Failed to append value %d: %v", i, err) } } bw.finish() } +func TestBlockWriterWithLimit(t *testing.T) { + testBlockWriterWithLimit(t, 0) + testBlockWriterWithLimit(t, 2) + testBlockWriterWithLimit(t, 34) +} + +func testBlockWriterWithLimit(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) + + var bitmaps [][]byte + for i := 0; i < indexBlockRestartLen+2; i++ { + bw.append(uint64(i+1), randomExt(bitmapSize, 5)) + bitmaps = append(bitmaps, bytes.Clone(bw.desc.extBitmap)) + } + for i := 0; i < indexBlockRestartLen+2; i++ { + limit := uint64(i + 1) + + desc := bw.desc.copy() + block, err := newBlockWriter(bytes.Clone(bw.finish()), desc, limit, bitmapSize != 0) + if err != nil { + t.Fatalf("Failed to construct the block writer, %v", err) + } + if block.desc.max != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, block.desc.max, limit) + } + if !bytes.Equal(desc.extBitmap, bitmaps[i]) { + t.Fatalf("Test %d, unexpected bitmap, got: %v, want: %v", i, block.desc.extBitmap, bitmaps[i]) + } + + // Re-fill the elements + var maxElem uint64 + for elem := limit + 1; elem < indexBlockRestartLen+4; elem++ { + if err := block.append(elem, randomExt(bitmapSize, 5)); err != nil { + t.Fatalf("Failed to append value %d: %v", elem, err) + } + maxElem = elem + } + if block.desc.max != maxElem { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, block.desc.max, maxElem) + } + } +} + func TestBlockWriterDelete(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + testBlockWriterDelete(t, 0) + testBlockWriterDelete(t, 2) + testBlockWriterDelete(t, 34) +} + +func testBlockWriterDelete(t *testing.T, bitmapSize int) { + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < 10; i++ { - bw.append(uint64(i + 1)) + bw.append(uint64(i+1), randomExt(bitmapSize, 5)) } // Pop unknown id, the request should be rejected if err := bw.pop(100); err == nil { @@ -144,12 +228,18 @@ func TestBlockWriterDelete(t *testing.T) { } func TestBlcokWriterDeleteWithData(t *testing.T) { + testBlcokWriterDeleteWithData(t, 0) + testBlcokWriterDeleteWithData(t, 2) + testBlcokWriterDeleteWithData(t, 34) +} + +func testBlcokWriterDeleteWithData(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, bitmapSize), 0, bitmapSize != 0) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } // Re-construct the block writer with data @@ -158,7 +248,10 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { max: 20, entries: 5, } - bw, err := newBlockWriter(bw.finish(), desc) + if bitmapSize > 0 { + desc.extBitmap = make([]byte, bitmapSize) + } + bw, err := newBlockWriter(bw.finish(), desc, elements[len(elements)-1], bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct block writer %v", err) } @@ -169,7 +262,7 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { newTail := elements[i-1] // Ensure the element can still be queried with no issue - br, err := newBlockReader(bw.finish()) + br, err := newBlockReader(bw.finish(), bitmapSize != 0) if err != nil { t.Fatalf("Failed to construct the block reader, %v", err) } @@ -201,26 +294,60 @@ func TestBlcokWriterDeleteWithData(t *testing.T) { } func TestCorruptedIndexBlock(t *testing.T) { - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 0), 0, false) + + var maxElem uint64 for i := 0; i < 10; i++ { - bw.append(uint64(i + 1)) + bw.append(uint64(i+1), nil) + maxElem = uint64(i + 1) } buf := bw.finish() // Mutate the buffer manually buf[len(buf)-1]++ - _, err := newBlockWriter(buf, newIndexBlockDesc(0)) + _, err := newBlockWriter(buf, newIndexBlockDesc(0, 0), maxElem, false) if err == nil { t.Fatal("Corrupted index block data is not detected") } } // BenchmarkParseIndexBlock benchmarks the performance of parseIndexBlock. +// +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkParseIndexBlock +// BenchmarkParseIndexBlock-8 35829495 34.16 ns/op func BenchmarkParseIndexBlock(b *testing.B) { // Generate a realistic index block blob - bw, _ := newBlockWriter(nil, newIndexBlockDesc(0)) + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 0), 0, false) for i := 0; i < 4096; i++ { - bw.append(uint64(i * 2)) + bw.append(uint64(i*2), nil) + } + blob := bw.finish() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, err := parseIndexBlock(blob) + if err != nil { + b.Fatalf("parseIndexBlock failed: %v", err) + } + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkParseIndexBlockWithExt +// BenchmarkParseIndexBlockWithExt-8 35773242 33.72 ns/op +func BenchmarkParseIndexBlockWithExt(b *testing.B) { + // Generate a realistic index block blob + bw, _ := newBlockWriter(nil, newIndexBlockDesc(0, 34), 0, true) + for i := 0; i < 4096; i++ { + id, ext := uint64(i*2), randomExt(34, 3) + bw.append(id, ext) } blob := bw.finish() @@ -234,19 +361,58 @@ func BenchmarkParseIndexBlock(b *testing.B) { } // BenchmarkBlockWriterAppend benchmarks the performance of indexblock.writer +// +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkBlockWriterAppend +// BenchmarkBlockWriterAppend-8 293611083 4.113 ns/op 3 B/op 0 allocs/op func BenchmarkBlockWriterAppend(b *testing.B) { b.ReportAllocs() b.ResetTimer() - desc := newIndexBlockDesc(0) - writer, _ := newBlockWriter(nil, desc) + var blockID uint32 + desc := newIndexBlockDesc(blockID, 0) + writer, _ := newBlockWriter(nil, desc, 0, false) + + for i := 0; i < b.N; i++ { + if writer.estimateFull(nil) { + blockID += 1 + desc = newIndexBlockDesc(blockID, 0) + writer, _ = newBlockWriter(nil, desc, 0, false) + } + if err := writer.append(writer.desc.max+1, nil); err != nil { + b.Error(err) + } + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/triedb/pathdb +// cpu: Apple M1 Pro +// BenchmarkBlockWriterAppendWithExt +// BenchmarkBlockWriterAppendWithExt-8 11123844 103.6 ns/op 42 B/op 2 allocs/op +func BenchmarkBlockWriterAppendWithExt(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + var ( + bitmapSize = 34 + blockID uint32 + ) + desc := newIndexBlockDesc(blockID, bitmapSize) + writer, _ := newBlockWriter(nil, desc, 0, true) for i := 0; i < b.N; i++ { - if writer.full() { - desc = newIndexBlockDesc(0) - writer, _ = newBlockWriter(nil, desc) + ext := randomExt(bitmapSize, 3) + if writer.estimateFull(ext) { + blockID += 1 + desc = newIndexBlockDesc(blockID, bitmapSize) + writer, _ = newBlockWriter(nil, desc, 0, true) } - if err := writer.append(writer.desc.max + 1); err != nil { + if err := writer.append(writer.desc.max+1, ext); err != nil { b.Error(err) } } diff --git a/triedb/pathdb/history_index_iterator.go b/triedb/pathdb/history_index_iterator.go new file mode 100644 index 0000000000..96fcec23c6 --- /dev/null +++ b/triedb/pathdb/history_index_iterator.go @@ -0,0 +1,661 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see = bitmapElementThresholdTwoLevels { + return false, fmt.Errorf("invalid extension filter %d for 2 bytes bitmap", id) + } + return isBitSet(bitmap, n), nil + case bitmapBytesThreeLevels: + // Bitmap for 3-level trie with at most 16+16*16 elements inside + if n >= bitmapElementThresholdThreeLevels { + return false, fmt.Errorf("invalid extension filter %d for 34 bytes bitmap", id) + } else if n >= bitmapElementThresholdTwoLevels { + return isBitSet(bitmap, n), nil + } else { + // Check the element itself first + if isBitSet(bitmap, n) { + return true, nil + } + // Check descendants: the presence of any descendant implicitly + // represents a mutation of its ancestor. + return bitmap[2+2*n] != 0 || bitmap[3+2*n] != 0, nil + } + default: + return false, fmt.Errorf("unsupported bitmap size %d", len(bitmap)) + } +} + +// blockIterator is the iterator to traverse the indices within a single block. +type blockIterator struct { + // immutable fields + data []byte // Reference to the data segment within the block reader + restarts []uint16 // Offsets pointing to the restart sections within the data + hasExt bool // Flag whether the extension is included in the data + + // Optional extension filter + filter *extFilter // Filters index entries based on the extension field. + + // mutable fields + id uint64 // ID of the element at the iterators current position + ext []byte // Extension field of the element at the iterators current position + dataPtr int // Current read position within the data slice + restartPtr int // Index of the restart section where the iterator is currently positioned + exhausted bool // Flag whether the iterator has been exhausted + err error // Accumulated error during the traversal +} + +func (br *blockReader) newIterator(filter *extFilter) *blockIterator { + it := &blockIterator{ + data: br.data, // hold the slice directly with no deep copy + restarts: br.restarts, // hold the slice directly with no deep copy + hasExt: br.hasExt, // flag whether the extension should be resolved + filter: filter, // optional extension filter + } + it.reset() + return it +} + +func (it *blockIterator) set(dataPtr int, restartPtr int, id uint64, ext []byte) { + it.id = id + it.ext = ext + + it.dataPtr = dataPtr + it.restartPtr = restartPtr + it.exhausted = dataPtr == len(it.data) +} + +func (it *blockIterator) setErr(err error) { + if it.err != nil { + return + } + it.err = err +} + +func (it *blockIterator) reset() { + it.id = 0 + it.ext = nil + + it.dataPtr = -1 + it.restartPtr = -1 + it.exhausted = false + it.err = nil + + // Mark the iterator as exhausted if the associated index block is empty + if len(it.data) == 0 || len(it.restarts) == 0 { + it.exhausted = true + } +} + +func (it *blockIterator) resolveExt(pos int) ([]byte, int, error) { + if !it.hasExt { + return nil, 0, nil + } + length, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + return nil, 0, fmt.Errorf("too short for extension, pos: %d, datalen: %d", pos, len(it.data)) + } + if len(it.data[pos+n:]) < int(length) { + return nil, 0, fmt.Errorf("too short for extension, pos: %d, length: %d, datalen: %d", pos, length, len(it.data)) + } + return it.data[pos+n : pos+n+int(length)], n + int(length), nil +} + +// seekGT moves the iterator to the first element whose id is greater than the +// given number. It returns whether such element exists. +// +// Note, this operation will unset the exhausted status and subsequent traversal +// is allowed. +func (it *blockIterator) seekGT(id uint64) bool { + if it.err != nil { + return false + } + var err error + index := sort.Search(len(it.restarts), func(i int) bool { + item, n := binary.Uvarint(it.data[it.restarts[i]:]) + if n <= 0 { + err = fmt.Errorf("failed to decode item at restart %d", it.restarts[i]) + } + return item > id + }) + if err != nil { + it.setErr(err) + return false + } + if index == 0 { + pos := int(it.restarts[0]) + item, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.restarts[0])) + return false + } + pos = pos + n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + it.set(pos+shift, 0, item, ext) + return true + } + var ( + start int + limit int + restartIndex int // The restart section being searched below + ) + if index == len(it.restarts) { + // The element being searched falls within the last restart section, + // there is no guarantee such element can be found. + start = int(it.restarts[len(it.restarts)-1]) + limit = len(it.data) + restartIndex = len(it.restarts) - 1 + } else { + // The element being searched falls within the non-last restart section, + // such element can be found for sure. + start = int(it.restarts[index-1]) + limit = int(it.restarts[index]) + restartIndex = index - 1 + } + var ( + result uint64 + pos = start + ) + for pos < limit { + x, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", pos)) + return false + } + if pos == start { + result = x + } else { + result += x + } + pos += n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + pos += shift + + if result > id { + if pos == limit { + it.set(pos, restartIndex+1, result, ext) + } else { + it.set(pos, restartIndex, result, ext) + } + return true + } + } + // The element which is greater than specified id is not found. + if index == len(it.restarts) { + it.reset() + return false + } + // The element which is the first one greater than the specified id + // is exactly the one located at the restart point. + pos = int(it.restarts[index]) + item, n := binary.Uvarint(it.data[pos:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.restarts[index])) + return false + } + pos = pos + n + + ext, shift, err := it.resolveExt(pos) + if err != nil { + it.setErr(err) + return false + } + it.set(pos+shift, index, item, ext) + return true +} + +// SeekGT implements HistoryIndexIterator, is the wrapper of the seekGT with +// optional extension filter logic applied. +func (it *blockIterator) SeekGT(id uint64) bool { + if !it.seekGT(id) { + return false + } + if it.filter == nil { + return true + } + for { + found, err := it.filter.exists(it.ext) + if err != nil { + it.setErr(err) + return false + } + if found { + break + } + if !it.next() { + return false + } + } + return true +} + +func (it *blockIterator) init() { + if it.dataPtr != -1 { + return + } + it.dataPtr = 0 + it.restartPtr = 0 +} + +// next moves the iterator to the next element. If the iterator has been exhausted, +// and boolean with false should be returned. +func (it *blockIterator) next() bool { + if it.exhausted || it.err != nil { + return false + } + it.init() + + // Decode the next element pointed by the iterator + v, n := binary.Uvarint(it.data[it.dataPtr:]) + if n <= 0 { + it.setErr(fmt.Errorf("failed to decode item at pos %d", it.dataPtr)) + return false + } + var val uint64 + if it.dataPtr == int(it.restarts[it.restartPtr]) { + val = v + } else { + val = it.id + v + } + + // Decode the extension field + ext, shift, err := it.resolveExt(it.dataPtr + n) + if err != nil { + it.setErr(err) + return false + } + + // Move to the next restart section if the data pointer crosses the boundary + nextRestartPtr := it.restartPtr + if it.restartPtr < len(it.restarts)-1 && it.dataPtr+n+shift == int(it.restarts[it.restartPtr+1]) { + nextRestartPtr = it.restartPtr + 1 + } + it.set(it.dataPtr+n+shift, nextRestartPtr, val, ext) + + return true +} + +// Next implements the HistoryIndexIterator, moving the iterator to the next +// element. It's a wrapper of next with optional extension filter logic applied. +func (it *blockIterator) Next() bool { + if !it.next() { + return false + } + if it.filter == nil { + return true + } + for { + found, err := it.filter.exists(it.ext) + if err != nil { + it.setErr(err) + return false + } + if found { + break + } + if !it.next() { + return false + } + } + return true +} + +// ID implements HistoryIndexIterator, returning the id of the element where the +// iterator is positioned at. +func (it *blockIterator) ID() uint64 { + return it.id +} + +// Error implements HistoryIndexIterator, returning any accumulated error. +// Exhausting all the elements is not considered to be an error. +func (it *blockIterator) Error() error { return it.err } + +// indexIterator is an iterator to traverse the history indices belonging to the +// specific state entry. +type indexIterator struct { + // immutable fields + descList []*indexBlockDesc + reader *indexReader + + // Optional extension filter + filter *extFilter + + // mutable fields + blockIt *blockIterator + blockPtr int + exhausted bool + err error +} + +// newBlockIter initializes the block iterator with the specified block ID. +func (r *indexReader) newBlockIter(id uint32, filter *extFilter) (*blockIterator, error) { + r.mu.RLock() + br, ok := r.readers[id] + r.mu.RUnlock() + + if !ok { + var err error + br, err = newBlockReader(readStateIndexBlock(r.state, r.db, id), r.bitmapSize != 0) + if err != nil { + return nil, err + } + r.mu.Lock() + r.readers[id] = br + r.mu.Unlock() + } + return br.newIterator(filter), nil +} + +// newIterator initializes the index iterator with the specified extension filter. +func (r *indexReader) newIterator(filter *extFilter) *indexIterator { + it := &indexIterator{ + descList: r.descList, + reader: r, + filter: filter, + } + it.reset() + return it +} + +func (it *indexIterator) setErr(err error) { + if it.err != nil { + return + } + it.err = err +} + +func (it *indexIterator) reset() { + it.blockIt = nil + it.blockPtr = -1 + it.exhausted = false + it.err = nil + + if len(it.descList) == 0 { + it.exhausted = true + } +} + +func (it *indexIterator) open(blockPtr int) error { + blockIt, err := it.reader.newBlockIter(it.descList[blockPtr].id, it.filter) + if err != nil { + return err + } + it.blockIt = blockIt + it.blockPtr = blockPtr + return nil +} + +func (it *indexIterator) applyFilter(index int) (int, error) { + if it.filter == nil { + return index, nil + } + for index < len(it.descList) { + found, err := it.filter.contains(it.descList[index].extBitmap) + if err != nil { + return 0, err + } + if found { + break + } + index++ + } + return index, nil +} + +// SeekGT moves the iterator to the first element whose id is greater than the +// given number. It returns whether such element exists. +// +// Note, this operation will unset the exhausted status and subsequent traversal +// is allowed. +func (it *indexIterator) SeekGT(id uint64) bool { + if it.err != nil { + return false + } + index := sort.Search(len(it.descList), func(i int) bool { + return id < it.descList[i].max + }) + index, err := it.applyFilter(index) + if err != nil { + it.setErr(err) + return false + } + if index == len(it.descList) { + return false + } + it.exhausted = false + + if it.blockIt == nil || it.blockPtr != index { + if err := it.open(index); err != nil { + it.setErr(err) + return false + } + } + // Terminate if the element which is greater than the id can be found in the + // last block; otherwise move to the next block. It may happen that all the + // target elements in this block are all less than id. + if it.blockIt.SeekGT(id) { + return true + } + return it.Next() +} + +func (it *indexIterator) init() error { + if it.blockIt != nil { + return nil + } + return it.open(0) +} + +// Next implements the HistoryIndexIterator, moving the iterator to the next +// element. If the iterator has been exhausted, and boolean with false should +// be returned. +func (it *indexIterator) Next() bool { + if it.exhausted || it.err != nil { + return false + } + if err := it.init(); err != nil { + it.setErr(err) + return false + } + if it.blockIt.Next() { + return true + } + it.blockPtr++ + + index, err := it.applyFilter(it.blockPtr) + if err != nil { + it.setErr(err) + return false + } + it.blockPtr = index + + if it.blockPtr == len(it.descList) { + it.exhausted = true + return false + } + if err := it.open(it.blockPtr); err != nil { + it.setErr(err) + return false + } + return it.blockIt.Next() +} + +// Error implements HistoryIndexIterator, returning any accumulated error. +// Exhausting all the elements is not considered to be an error. +func (it *indexIterator) Error() error { + if it.err != nil { + return it.err + } + if it.blockIt != nil { + return it.blockIt.Error() + } + return nil +} + +// ID implements HistoryIndexIterator, returning the id of the element where the +// iterator is positioned at. +func (it *indexIterator) ID() uint64 { + return it.blockIt.ID() +} + +// seqIter provides a simple iterator over a contiguous sequence of +// unsigned integers, ending at end (end is included). +type seqIter struct { + cur uint64 // current position + end uint64 // iteration stops at end-1 + done bool // true when iteration is exhausted +} + +func newSeqIter(last uint64) *seqIter { + return &seqIter{end: last + 1} +} + +// SeekGT positions the iterator at the smallest element > id. +// Returns false if no such element exists. +func (it *seqIter) SeekGT(id uint64) bool { + if id+1 >= it.end { + it.done = true + return false + } + it.cur = id + 1 + it.done = false + return true +} + +// Next advances the iterator. Returns false if exhausted. +func (it *seqIter) Next() bool { + if it.done { + return false + } + if it.cur+1 < it.end { + it.cur++ + return true + } + // this was the last element + it.done = true + return false +} + +// ID returns the id of the element where the iterator is positioned at. +func (it *seqIter) ID() uint64 { return it.cur } + +// Error returns any accumulated error. Exhausting all the elements is not +// considered to be an error. +func (it *seqIter) Error() error { return nil } diff --git a/triedb/pathdb/history_index_iterator_test.go b/triedb/pathdb/history_index_iterator_test.go new file mode 100644 index 0000000000..a11fd17666 --- /dev/null +++ b/triedb/pathdb/history_index_iterator_test.go @@ -0,0 +1,361 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see 0 { + filter = &filters[rand.Intn(len(filters))] + } else { + filter = nil + } + + var input uint64 + if rand.Intn(2) == 0 { + input = elements[rand.Intn(len(elements))] + } else { + input = uint64(rand.Uint32()) + } + + index := sort.Search(len(elements), func(i int) bool { + return elements[i] > input + }) + for index < len(elements) { + if checkExt(filter, ext[index]) { + break + } + index++ + } + + var ( + exp bool + expVal uint64 + remains []uint64 + ) + if index == len(elements) { + exp = false + } else { + exp = true + expVal = elements[index] + + index++ + for index < len(elements) { + if checkExt(filter, ext[index]) { + remains = append(remains, elements[index]) + } + index++ + } + } + + it := newIter(filter) + if err := checkSeekGT(it, input, exp, expVal); err != nil { + t.Fatal(err) + } + if exp { + if err := checkNext(it, remains); err != nil { + t.Fatal(err) + } + } + } +} + +func verifyTraversal(t *testing.T, elements []uint64, ext [][]uint16, newIter func(filter *extFilter) HistoryIndexIterator) { + set := make(map[extFilter]bool) + for _, extList := range ext { + for _, f := range extList { + set[extFilter(f)] = true + } + } + filters := slices.Collect(maps.Keys(set)) + + for i := 0; i < 16; i++ { + var filter *extFilter + if len(filters) > 0 { + filter = &filters[rand.Intn(len(filters))] + } else { + filter = nil + } + it := newIter(filter) + + var ( + pos int + exp []uint64 + ) + for pos < len(elements) { + if checkExt(filter, ext[pos]) { + exp = append(exp, elements[pos]) + } + pos++ + } + if err := checkNext(it, exp); err != nil { + t.Fatal(err) + } + } +} + +func TestBlockIteratorSeekGT(t *testing.T) { + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, indexBlockRestartLen, 3 * indexBlockRestartLen} { + data, elements, ext := makeTestIndexBlock(n, size) + + verifySeekGT(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + br, err := newBlockReader(data, size != 0) + if err != nil { + t.Fatalf("Failed to open the block for reading, %v", err) + } + return br.newIterator(filter) + }) + } + } +} + +func TestIndexIteratorSeekGT(t *testing.T) { + ident := newAccountIdent(common.Hash{0x1}) + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, 4096, 3 * 4096} { + db := rawdb.NewMemoryDatabase() + elements, ext := makeTestIndexBlocks(db, ident, n, size) + + verifySeekGT(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + ir, err := newIndexReader(db, ident, size) + if err != nil { + t.Fatalf("Failed to open the index reader, %v", err) + } + return ir.newIterator(filter) + }) + } + } +} + +func TestBlockIteratorTraversal(t *testing.T) { + /* 0-size index block is not allowed + + data, elements := makeTestIndexBlock(0) + testBlockIterator(t, data, elements) + */ + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, indexBlockRestartLen, 3 * indexBlockRestartLen} { + data, elements, ext := makeTestIndexBlock(n, size) + + verifyTraversal(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + br, err := newBlockReader(data, size != 0) + if err != nil { + t.Fatalf("Failed to open the block for reading, %v", err) + } + return br.newIterator(filter) + }) + } + } +} + +func TestIndexIteratorTraversal(t *testing.T) { + ident := newAccountIdent(common.Hash{0x1}) + + for _, size := range []int{0, 2, 34} { + for _, n := range []int{1, 4096, 3 * 4096} { + db := rawdb.NewMemoryDatabase() + elements, ext := makeTestIndexBlocks(db, ident, n, size) + + verifyTraversal(t, elements, ext, func(filter *extFilter) HistoryIndexIterator { + ir, err := newIndexReader(db, ident, size) + if err != nil { + t.Fatalf("Failed to open the index reader, %v", err) + } + return ir.newIterator(filter) + }) + } + } +} + +func TestSeqIterBasicIteration(t *testing.T) { + it := newSeqIter(5) // iterates over [1..5] + it.SeekGT(0) + + var ( + got []uint64 + expected = []uint64{1, 2, 3, 4, 5} + ) + got = append(got, it.ID()) + for it.Next() { + got = append(got, it.ID()) + } + if len(got) != len(expected) { + t.Fatalf("iteration length mismatch: got %v, expected %v", got, expected) + } + for i := range expected { + if got[i] != expected[i] { + t.Fatalf("element mismatch at %d: got %d, expected %d", i, got[i], expected[i]) + } + } +} + +func TestSeqIterSeekGT(t *testing.T) { + it := newSeqIter(5) + + tests := []struct { + input uint64 + ok bool + expected uint64 + }{ + {0, true, 1}, + {1, true, 2}, + {4, true, 5}, + {5, false, 0}, // 6 is out of range + } + for _, tt := range tests { + ok := it.SeekGT(tt.input) + if ok != tt.ok { + t.Fatalf("SeekGT(%d) ok mismatch: got %v, expected %v", tt.input, ok, tt.ok) + } + if ok && it.ID() != tt.expected { + t.Fatalf("SeekGT(%d) positioned at %d, expected %d", tt.input, it.ID(), tt.expected) + } + } +} diff --git a/triedb/pathdb/history_index_test.go b/triedb/pathdb/history_index_test.go index be9b7c4049..2644db46b5 100644 --- a/triedb/pathdb/history_index_test.go +++ b/triedb/pathdb/history_index_test.go @@ -29,19 +29,25 @@ import ( ) func TestIndexReaderBasic(t *testing.T) { + testIndexReaderBasic(t, 0) + testIndexReaderBasic(t, 2) + testIndexReaderBasic(t, 34) +} + +func testIndexReaderBasic(t *testing.T, bitmapSize int) { elements := []uint64{ 1, 5, 10, 11, 20, } db := rawdb.NewMemoryDatabase() - bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } batch := db.NewBatch() bw.finish(batch) batch.Write() - br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}), bitmapSize) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -68,22 +74,28 @@ func TestIndexReaderBasic(t *testing.T) { } func TestIndexReaderLarge(t *testing.T) { + testIndexReaderLarge(t, 0) + testIndexReaderLarge(t, 2) + testIndexReaderLarge(t, 34) +} + +func testIndexReaderLarge(t *testing.T, bitmapSize int) { var elements []uint64 - for i := 0; i < 10*indexBlockEntriesCap; i++ { + for i := 0; i < 10*4096; i++ { elements = append(elements, rand.Uint64()) } slices.Sort(elements) db := rawdb.NewMemoryDatabase() - bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + bw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) for i := 0; i < len(elements); i++ { - bw.append(elements[i]) + bw.append(elements[i], randomExt(bitmapSize, 5)) } batch := db.NewBatch() bw.finish(batch) batch.Write() - br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(db, newAccountIdent(common.Hash{0xa}), bitmapSize) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -107,7 +119,7 @@ func TestIndexReaderLarge(t *testing.T) { } func TestEmptyIndexReader(t *testing.T) { - br, err := newIndexReader(rawdb.NewMemoryDatabase(), newAccountIdent(common.Hash{0xa})) + br, err := newIndexReader(rawdb.NewMemoryDatabase(), newAccountIdent(common.Hash{0xa}), 0) if err != nil { t.Fatalf("Failed to construct the index reader, %v", err) } @@ -121,57 +133,150 @@ func TestEmptyIndexReader(t *testing.T) { } func TestIndexWriterBasic(t *testing.T) { + testIndexWriterBasic(t, 0) + testIndexWriterBasic(t, 2) + testIndexWriterBasic(t, 34) +} + +func testIndexWriterBasic(t *testing.T, bitmapSize int) { db := rawdb.NewMemoryDatabase() - iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) - iw.append(2) - if err := iw.append(1); err == nil { + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + iw.append(2, randomExt(bitmapSize, 5)) + if err := iw.append(1, randomExt(bitmapSize, 5)); err == nil { t.Fatal("out-of-order insertion is not expected") } + var maxElem uint64 for i := 0; i < 10; i++ { - iw.append(uint64(i + 3)) + iw.append(uint64(i+3), randomExt(bitmapSize, 5)) + maxElem = uint64(i + 3) } batch := db.NewBatch() iw.finish(batch) batch.Write() - iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) + iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), maxElem, bitmapSize) if err != nil { t.Fatalf("Failed to construct the block writer, %v", err) } for i := 0; i < 10; i++ { - if err := iw.append(uint64(i + 100)); err != nil { + if err := iw.append(uint64(i+100), randomExt(bitmapSize, 5)); err != nil { t.Fatalf("Failed to append item, %v", err) } } iw.finish(db.NewBatch()) } -func TestIndexWriterDelete(t *testing.T) { +func TestIndexWriterWithLimit(t *testing.T) { + testIndexWriterWithLimit(t, 0) + testIndexWriterWithLimit(t, 2) + testIndexWriterWithLimit(t, 34) +} + +func testIndexWriterWithLimit(t *testing.T, bitmapSize int) { db := rawdb.NewMemoryDatabase() - iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa})) - for i := 0; i < indexBlockEntriesCap*4; i++ { - iw.append(uint64(i + 1)) + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + } + batch := db.NewBatch() + iw.finish(batch) + batch.Write() + + for i := 0; i < 200; i++ { + limit := uint64(i + 1) + iw, err := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), limit, bitmapSize) + if err != nil { + t.Fatalf("Failed to construct the index writer, %v", err) + } + if iw.lastID != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, limit) + } + // Re-fill the elements + var maxElem uint64 + for elem := limit + 1; elem < 500; elem++ { + if err := iw.append(elem, randomExt(bitmapSize, 5)); err != nil { + t.Fatalf("Failed to append value %d: %v", elem, err) + } + maxElem = elem + } + if iw.lastID != maxElem { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, maxElem) + } + } +} + +func TestIndexDeleterBasic(t *testing.T) { + testIndexDeleterBasic(t, 0) + testIndexDeleterBasic(t, 2) + testIndexDeleterBasic(t, 34) +} + +func testIndexDeleterBasic(t *testing.T, bitmapSize int) { + db := rawdb.NewMemoryDatabase() + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + var maxElem uint64 + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + maxElem = uint64(i + 1) } batch := db.NewBatch() iw.finish(batch) batch.Write() // Delete unknown id, the request should be rejected - id, _ := newIndexDeleter(db, newAccountIdent(common.Hash{0xa})) - if err := id.pop(indexBlockEntriesCap * 5); err == nil { + id, _ := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), maxElem, bitmapSize) + if err := id.pop(500); err == nil { t.Fatal("Expect error to occur for unknown id") } - for i := indexBlockEntriesCap * 4; i >= 1; i-- { + for i := 200; i >= 1; i-- { if err := id.pop(uint64(i)); err != nil { t.Fatalf("Unexpected error for element popping, %v", err) } if id.lastID != uint64(i-1) { t.Fatalf("Unexpected lastID, want: %d, got: %d", uint64(i-1), iw.lastID) } - if rand.Intn(10) == 0 { - batch := db.NewBatch() - id.finish(batch) - batch.Write() + } +} + +func TestIndexDeleterWithLimit(t *testing.T) { + testIndexDeleterWithLimit(t, 0) + testIndexDeleterWithLimit(t, 2) + testIndexDeleterWithLimit(t, 34) +} + +func testIndexDeleterWithLimit(t *testing.T, bitmapSize int) { + db := rawdb.NewMemoryDatabase() + iw, _ := newIndexWriter(db, newAccountIdent(common.Hash{0xa}), 0, bitmapSize) + + // 200 iterations (with around 50 bytes extension) is enough to cross + // the block boundary (4096 bytes) + for i := 0; i < 200; i++ { + iw.append(uint64(i+1), randomExt(bitmapSize, 50)) + } + batch := db.NewBatch() + iw.finish(batch) + batch.Write() + + for i := 0; i < 200; i++ { + limit := uint64(i + 1) + id, err := newIndexDeleter(db, newAccountIdent(common.Hash{0xa}), limit, bitmapSize) + if err != nil { + t.Fatalf("Failed to construct the index writer, %v", err) + } + if id.lastID != limit { + t.Fatalf("Test %d, unexpected max value, got %d, want %d", i, iw.lastID, limit) + } + // Keep removing elements + for elem := id.lastID; elem > 0; elem-- { + if err := id.pop(elem); err != nil { + t.Fatalf("Failed to pop value %d: %v", elem, err) + } } } } @@ -212,7 +317,7 @@ func TestBatchIndexerWrite(t *testing.T) { } } for addrHash, indexes := range accounts { - ir, _ := newIndexReader(db, newAccountIdent(addrHash)) + ir, _ := newIndexReader(db, newAccountIdent(addrHash), 0) for i := 0; i < len(indexes)-1; i++ { n, err := ir.readGreaterThan(indexes[i]) if err != nil { @@ -232,7 +337,7 @@ func TestBatchIndexerWrite(t *testing.T) { } for addrHash, slots := range storages { for slotHash, indexes := range slots { - ir, _ := newIndexReader(db, newStorageIdent(addrHash, slotHash)) + ir, _ := newIndexReader(db, newStorageIdent(addrHash, slotHash), 0) for i := 0; i < len(indexes)-1; i++ { n, err := ir.readGreaterThan(indexes[i]) if err != nil { diff --git a/triedb/pathdb/history_indexer.go b/triedb/pathdb/history_indexer.go index 3492c97090..487e510bd2 100644 --- a/triedb/pathdb/history_indexer.go +++ b/triedb/pathdb/history_indexer.go @@ -35,17 +35,13 @@ import ( const ( // The batch size for reading state histories - historyReadBatch = 1000 + historyReadBatch = 1000 + historyIndexBatch = 8 * 1024 * 1024 // The number of state history indexes for constructing or deleting as batch stateHistoryIndexV0 = uint8(0) // initial version of state index structure stateHistoryIndexVersion = stateHistoryIndexV0 // the current state index version trienodeHistoryIndexV0 = uint8(0) // initial version of trienode index structure trienodeHistoryIndexVersion = trienodeHistoryIndexV0 // the current trienode index version - - // estimations for calculating the batch size for atomic database commit - estimatedStateHistoryIndexSize = 3 // The average size of each state history index entry is approximately 2–3 bytes - estimatedTrienodeHistoryIndexSize = 3 // The average size of each trienode history index entry is approximately 2-3 bytes - estimatedIndexBatchSizeFactor = 32 // The factor counts for the write amplification for each entry ) // indexVersion returns the latest index version for the given history type. @@ -126,18 +122,20 @@ func deleteIndexMetadata(db ethdb.KeyValueWriter, typ historyType) { // batchIndexer is responsible for performing batch indexing or unindexing // of historical data (e.g., state or trie node changes) atomically. type batchIndexer struct { - index map[stateIdent][]uint64 // List of history IDs for tracked state entry - pending int // Number of entries processed in the current batch. - delete bool // Operation mode: true for unindex, false for index. - lastID uint64 // ID of the most recently processed history. - typ historyType // Type of history being processed (e.g., state or trienode). - db ethdb.KeyValueStore // Key-value database used to store or delete index data. + index map[stateIdent][]uint64 // List of history IDs for tracked state entry + ext map[stateIdent][][]uint16 // List of extension for each state element + pending int // Number of entries processed in the current batch. + delete bool // Operation mode: true for unindex, false for index. + lastID uint64 // ID of the most recently processed history. + typ historyType // Type of history being processed (e.g., state or trienode). + db ethdb.KeyValueStore // Key-value database used to store or delete index data. } // newBatchIndexer constructs the batch indexer with the supplied mode. func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batchIndexer { return &batchIndexer{ index: make(map[stateIdent][]uint64), + ext: make(map[stateIdent][][]uint16), delete: delete, typ: typ, db: db, @@ -147,8 +145,10 @@ func newBatchIndexer(db ethdb.KeyValueStore, delete bool, typ historyType) *batc // process traverses the state entries within the provided history and tracks the mutation // records for them. func (b *batchIndexer) process(h history, id uint64) error { - for ident := range h.forEach() { - b.index[ident] = append(b.index[ident], id) + for elem := range h.forEach() { + key := elem.key() + b.index[key] = append(b.index[key], id) + b.ext[key] = append(b.ext[key], elem.ext()) b.pending++ } b.lastID = id @@ -156,22 +156,6 @@ func (b *batchIndexer) process(h history, id uint64) error { return b.finish(false) } -// makeBatch constructs a database batch based on the number of pending entries. -// The batch size is roughly estimated to minimize repeated resizing rounds, -// as accurately predicting the exact size is technically challenging. -func (b *batchIndexer) makeBatch() ethdb.Batch { - var size int - switch b.typ { - case typeStateHistory: - size = estimatedStateHistoryIndexSize - case typeTrienodeHistory: - size = estimatedTrienodeHistoryIndexSize - default: - panic(fmt.Sprintf("unknown history type %d", b.typ)) - } - return b.db.NewBatchWithSize(size * estimatedIndexBatchSizeFactor * b.pending) -} - // finish writes the accumulated state indexes into the disk if either the // memory limitation is reached or it's requested forcibly. func (b *batchIndexer) finish(force bool) error { @@ -182,30 +166,52 @@ func (b *batchIndexer) finish(force bool) error { return nil } var ( - batch = b.makeBatch() - batchMu sync.RWMutex - start = time.Now() - eg errgroup.Group + start = time.Now() + eg errgroup.Group + + batch = b.db.NewBatchWithSize(ethdb.IdealBatchSize) + batchSize int + batchMu sync.RWMutex + + writeBatch = func(fn func(batch ethdb.Batch)) error { + batchMu.Lock() + defer batchMu.Unlock() + + fn(batch) + if batch.ValueSize() >= ethdb.IdealBatchSize { + batchSize += batch.ValueSize() + if err := batch.Write(); err != nil { + return err + } + batch.Reset() + } + return nil + } ) eg.SetLimit(runtime.NumCPU()) + var indexed uint64 + if metadata := loadIndexMetadata(b.db, b.typ); metadata != nil { + indexed = metadata.Last + } for ident, list := range b.index { + ext := b.ext[ident] eg.Go(func() error { if !b.delete { - iw, err := newIndexWriter(b.db, ident) + iw, err := newIndexWriter(b.db, ident, indexed, ident.bloomSize()) if err != nil { return err } - for _, n := range list { - if err := iw.append(n); err != nil { + for i, n := range list { + if err := iw.append(n, ext[i]); err != nil { return err } } - batchMu.Lock() - iw.finish(batch) - batchMu.Unlock() + return writeBatch(func(batch ethdb.Batch) { + iw.finish(batch) + }) } else { - id, err := newIndexDeleter(b.db, ident) + id, err := newIndexDeleter(b.db, ident, indexed, ident.bloomSize()) if err != nil { return err } @@ -214,11 +220,10 @@ func (b *batchIndexer) finish(force bool) error { return err } } - batchMu.Lock() - id.finish(batch) - batchMu.Unlock() + return writeBatch(func(batch ethdb.Batch) { + id.finish(batch) + }) } - return nil }) } if err := eg.Wait(); err != nil { @@ -234,12 +239,16 @@ func (b *batchIndexer) finish(force bool) error { storeIndexMetadata(batch, b.typ, b.lastID-1) } } + batchSize += batch.ValueSize() + if err := batch.Write(); err != nil { return err } - log.Debug("Committed batch indexer", "type", b.typ, "entries", len(b.index), "records", b.pending, "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Committed batch indexer", "type", b.typ, "entries", len(b.index), "records", b.pending, "size", common.StorageSize(batchSize), "elapsed", common.PrettyDuration(time.Since(start))) + b.pending = 0 - b.index = make(map[stateIdent][]uint64) + clear(b.index) + clear(b.ext) return nil } diff --git a/triedb/pathdb/history_inspect.go b/triedb/pathdb/history_inspect.go index 9b4eea27b4..74b8bb8df2 100644 --- a/triedb/pathdb/history_inspect.go +++ b/triedb/pathdb/history_inspect.go @@ -50,12 +50,12 @@ func sanitizeRange(start, end uint64, freezer ethdb.AncientReader) (uint64, uint if err != nil { return 0, 0, err } - last := head - 1 + last := head if end != 0 && end < last { last = end } // Make sure the range is valid - if first >= last { + if first > last { return 0, 0, fmt.Errorf("range is invalid, first: %d, last: %d", first, last) } return first, last, nil @@ -143,7 +143,7 @@ func historyRange(freezer ethdb.AncientReader) (uint64, uint64, error) { if err != nil { return 0, 0, err } - last := head - 1 + last := head fh, err := readStateHistory(freezer, first) if err != nil { diff --git a/triedb/pathdb/history_reader.go b/triedb/pathdb/history_reader.go index 02a9148826..abfbd65413 100644 --- a/triedb/pathdb/history_reader.go +++ b/triedb/pathdb/history_reader.go @@ -22,12 +22,17 @@ import ( "errors" "fmt" "math" + "slices" "sort" "sync" + "sync/atomic" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/sync/errgroup" ) // indexReaderWithLimitTag is a wrapper around indexReader that includes an @@ -41,8 +46,8 @@ type indexReaderWithLimitTag struct { } // newIndexReaderWithLimitTag constructs a index reader with indexing position. -func newIndexReaderWithLimitTag(db ethdb.KeyValueReader, state stateIdent, limit uint64) (*indexReaderWithLimitTag, error) { - r, err := newIndexReader(db, state) +func newIndexReaderWithLimitTag(db ethdb.KeyValueReader, state stateIdent, limit uint64, bitmapSize int) (*indexReaderWithLimitTag, error) { + r, err := newIndexReader(db, state, bitmapSize) if err != nil { return nil, err } @@ -100,17 +105,18 @@ func (r *indexReaderWithLimitTag) readGreaterThan(id uint64, lastID uint64) (uin return r.reader.readGreaterThan(id) } -// historyReader is the structure to access historic state data. -type historyReader struct { +// stateHistoryReader is the structure to access historic state data. +type stateHistoryReader struct { disk ethdb.KeyValueReader freezer ethdb.AncientReader mu sync.RWMutex readers map[string]*indexReaderWithLimitTag } -// newHistoryReader constructs the history reader with the supplied db. -func newHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *historyReader { - return &historyReader{ +// newStateHistoryReader constructs the history reader with the supplied db +// for accessing historical states. +func newStateHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *stateHistoryReader { + return &stateHistoryReader{ disk: disk, freezer: freezer, readers: make(map[string]*indexReaderWithLimitTag), @@ -119,7 +125,7 @@ func newHistoryReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader) *h // readAccountMetadata resolves the account metadata within the specified // state history. -func (r *historyReader) readAccountMetadata(address common.Address, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readAccountMetadata(address common.Address, historyID uint64) ([]byte, error) { blob := rawdb.ReadStateAccountIndex(r.freezer, historyID) if len(blob) == 0 { return nil, fmt.Errorf("account index is truncated, historyID: %d", historyID) @@ -145,7 +151,7 @@ func (r *historyReader) readAccountMetadata(address common.Address, historyID ui // readStorageMetadata resolves the storage slot metadata within the specified // state history. -func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int) ([]byte, error) { +func (r *stateHistoryReader) readStorageMetadata(storageKey common.Hash, storageHash common.Hash, historyID uint64, slotOffset, slotNumber int) ([]byte, error) { data, err := rawdb.ReadStateStorageIndex(r.freezer, historyID, slotIndexSize*slotOffset, slotIndexSize*slotNumber) if err != nil { msg := fmt.Sprintf("id: %d, slot-offset: %d, slot-length: %d", historyID, slotOffset, slotNumber) @@ -180,7 +186,7 @@ func (r *historyReader) readStorageMetadata(storageKey common.Hash, storageHash } // readAccount retrieves the account data from the specified state history. -func (r *historyReader) readAccount(address common.Address, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readAccount(address common.Address, historyID uint64) ([]byte, error) { metadata, err := r.readAccountMetadata(address, historyID) if err != nil { return nil, err @@ -196,7 +202,7 @@ func (r *historyReader) readAccount(address common.Address, historyID uint64) ([ } // readStorage retrieves the storage slot data from the specified state history. -func (r *historyReader) readStorage(address common.Address, storageKey common.Hash, storageHash common.Hash, historyID uint64) ([]byte, error) { +func (r *stateHistoryReader) readStorage(address common.Address, storageKey common.Hash, storageHash common.Hash, historyID uint64) ([]byte, error) { metadata, err := r.readAccountMetadata(address, historyID) if err != nil { return nil, err @@ -226,30 +232,11 @@ func (r *historyReader) readStorage(address common.Address, storageKey common.Ha // stateID: represents the ID of the state of the specified version; // lastID: represents the ID of the latest/newest state history; // latestValue: represents the state value at the current disk layer with ID == lastID; -func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { - tail, err := r.freezer.Tail() +func (r *stateHistoryReader) read(state stateIdentQuery, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { + lastIndexed, err := checkStateAvail(state.stateIdent, typeStateHistory, r.freezer, stateID, lastID, r.disk) if err != nil { return nil, err - } // firstID = tail+1 - - // stateID+1 == firstID is allowed, as all the subsequent state histories - // are present with no gap inside. - if stateID < tail { - return nil, fmt.Errorf("historical state has been pruned, first: %d, state: %d", tail+1, stateID) } - - // To serve the request, all state histories from stateID+1 to lastID - // must be indexed. It's not supposed to happen unless system is very - // wrong. - metadata := loadIndexMetadata(r.disk, toHistoryType(state.typ)) - if metadata == nil || metadata.Last < lastID { - indexed := "null" - if metadata != nil { - indexed = fmt.Sprintf("%d", metadata.Last) - } - return nil, fmt.Errorf("state history is not fully indexed, requested: %d, indexed: %s", stateID, indexed) - } - // Construct the index reader to locate the corresponding history for // state retrieval r.mu.RLock() @@ -257,7 +244,7 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6 r.mu.RUnlock() if !ok { - ir, err = newIndexReaderWithLimitTag(r.disk, state.stateIdent, metadata.Last) + ir, err = newIndexReaderWithLimitTag(r.disk, state.stateIdent, lastIndexed, 0) if err != nil { return nil, err } @@ -284,3 +271,230 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6 } return r.readStorage(state.address, state.storageKey, state.storageHash, historyID) } + +// trienodeReader is the structure to access historical trienode data. +type trienodeReader struct { + disk ethdb.KeyValueReader + freezer ethdb.AncientReader + readConcurrency int // The concurrency used to load trie node data from history +} + +// newTrienodeReader constructs the history reader with the supplied db +// for accessing historical trie nodes. +func newTrienodeReader(disk ethdb.KeyValueReader, freezer ethdb.AncientReader, readConcurrency int) *trienodeReader { + return &trienodeReader{ + disk: disk, + freezer: freezer, + readConcurrency: readConcurrency, + } +} + +// readTrienode retrieves the trienode data from the specified trienode history. +func (r *trienodeReader) readTrienode(addrHash common.Hash, path string, historyID uint64) ([]byte, bool, error) { + tr := newTrienodeHistoryReader(historyID, r.freezer) + return tr.read(addrHash, path) +} + +// assembleNode takes a complete node value as the base and applies a list of +// mutation records to assemble the final node value accordingly. +func assembleNode(blob []byte, elements [][][]byte, indices [][]int) ([]byte, error) { + if len(elements) == 0 && len(indices) == 0 { + return blob, nil + } + children, err := rlp.SplitListValues(blob) + if err != nil { + return nil, err + } + for i := 0; i < len(elements); i++ { + for j, pos := range indices[i] { + children[pos] = elements[i][j] + } + } + return rlp.MergeListValues(children) +} + +type resultQueue struct { + data [][]byte + lock sync.Mutex +} + +func newResultQueue(size int) *resultQueue { + return &resultQueue{ + data: make([][]byte, size, size*2), + } +} + +func (q *resultQueue) set(data []byte, pos int) { + q.lock.Lock() + defer q.lock.Unlock() + + if pos >= len(q.data) { + newSize := pos + 1 + if cap(q.data) < newSize { + newData := make([][]byte, newSize, newSize*2) + copy(newData, q.data) + q.data = newData + } + q.data = q.data[:newSize] + } + q.data[pos] = data +} + +func (r *trienodeReader) readOptimized(state stateIdent, it HistoryIndexIterator, latestValue []byte) ([]byte, error) { + var ( + elements [][][]byte + indices [][]int + blob = latestValue + + eg errgroup.Group + seq int + term atomic.Bool + queue = newResultQueue(r.readConcurrency * 2) + ) + eg.SetLimit(r.readConcurrency) + + for { + id, pos := it.ID(), seq + seq += 1 + + eg.Go(func() error { + data, found, err := r.readTrienode(state.addressHash, state.path, id) + if err != nil { + term.Store(true) + return err + } + // In optimistic readahead mode, it is theoretically possible to encounter a + // NotFound error, where the trie node does not actually exist and the iterator + // reports a false-positive mutation record. Terminate the iterator if so, as + // all the necessary data (checkpoints and all diffs) required has already been + // fetching. + if !found { + term.Store(true) + log.Debug("Failed to read the trienode") + return nil + } + full, _, err := decodeNodeFull(data) + if err != nil { + term.Store(true) + return err + } + if full { + term.Store(true) + } + queue.set(data, pos) + return nil + }) + if term.Load() || !it.Next() { + break + } + } + if err := eg.Wait(); err != nil { + return nil, err + } + if err := it.Error(); err != nil { + return nil, err + } + for i := 0; i < seq; i++ { + isComplete, fullBlob, err := decodeNodeFull(queue.data[i]) + if err != nil { + return nil, err + } + // Terminate the loop is the node with full value has been found + if isComplete { + blob = fullBlob + break + } + // Decode the partial encoded node and keep iterating the node history + // until the node with full value being reached. + element, index, err := decodeNodeCompressed(queue.data[i]) + if err != nil { + return nil, err + } + elements, indices = append(elements, element), append(indices, index) + } + slices.Reverse(elements) + slices.Reverse(indices) + return assembleNode(blob, elements, indices) +} + +// read retrieves the trie node data associated with the stateID. +// stateID: represents the ID of the state of the specified version; +// lastID: represents the ID of the latest/newest trie node history; +// latestValue: represents the trie node value at the current disk layer with ID == lastID; +func (r *trienodeReader) read(state stateIdent, stateID uint64, lastID uint64, latestValue []byte) ([]byte, error) { + _, err := checkStateAvail(state, typeTrienodeHistory, r.freezer, stateID, lastID, r.disk) + if err != nil { + return nil, err + } + // Construct the index iterator to traverse the trienode history + var ( + scheme *indexScheme + it HistoryIndexIterator + ) + if state.addressHash == (common.Hash{}) { + scheme = accountIndexScheme + } else { + scheme = storageIndexScheme + } + if state.addressHash == (common.Hash{}) && state.path == "" { + it = newSeqIter(lastID) + } else { + chunkID, nodeID := scheme.splitPathLast(state.path) + + queryIdent := state + queryIdent.path = chunkID + ir, err := newIndexReader(r.disk, queryIdent, scheme.getBitmapSize(len(chunkID))) + if err != nil { + return nil, err + } + filter := extFilter(nodeID) + it = ir.newIterator(&filter) + } + // Move the iterator to the first element whose id is greater than + // the given number. + found := it.SeekGT(stateID) + if err := it.Error(); err != nil { + return nil, err + } + // The state was not found in the trie node histories, as it has not been + // modified since stateID. Use the data from the associated disk layer + // instead (full value node as always) + if !found { + return latestValue, nil + } + return r.readOptimized(state, it, latestValue) +} + +// checkStateAvail determines whether the requested historical state is available +// for accessing. What's more, it also returns the ID of the latest indexed history +// entry for subsequent usage. +// +// TODO(rjl493456442) it's really expensive to perform the check for every state +// retrieval, please rework this later. +func checkStateAvail(state stateIdent, exptyp historyType, freezer ethdb.AncientReader, stateID uint64, lastID uint64, db ethdb.KeyValueReader) (uint64, error) { + if toHistoryType(state.typ) != exptyp { + return 0, fmt.Errorf("unsupported history type: %d, want: %v", toHistoryType(state.typ), exptyp) + } + // firstID = tail+1 + tail, err := freezer.Tail() + if err != nil { + return 0, err + } + // stateID+1 == firstID is allowed, as all the subsequent history entries + // are present with no gap inside. + if stateID < tail { + return 0, fmt.Errorf("historical state has been pruned, first: %d, state: %d", tail+1, stateID) + } + // To serve the request, all history entries from stateID+1 to lastID + // must be indexed. It's not supposed to happen unless system is very + // wrong. + metadata := loadIndexMetadata(db, exptyp) + if metadata == nil || metadata.Last < lastID { + indexed := "null" + if metadata != nil { + indexed = fmt.Sprintf("%d", metadata.Last) + } + return 0, fmt.Errorf("history is not fully indexed, requested: %d, indexed: %s", stateID, indexed) + } + return metadata.Last, nil +} diff --git a/triedb/pathdb/history_reader_test.go b/triedb/pathdb/history_reader_test.go index 3e1a545ff3..b69fba68cb 100644 --- a/triedb/pathdb/history_reader_test.go +++ b/triedb/pathdb/history_reader_test.go @@ -50,7 +50,7 @@ func stateAvail(id uint64, env *tester) bool { return id+1 >= firstID } -func checkHistoricalState(env *tester, root common.Hash, id uint64, hr *historyReader) error { +func checkHistoricalState(env *tester, root common.Hash, id uint64, hr *stateHistoryReader) error { if !stateAvail(id, env) { return nil } @@ -157,7 +157,7 @@ func testHistoryReader(t *testing.T, historyLimit uint64) { var ( roots = env.roots dl = env.db.tree.bottom() - hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) + hr = newStateHistoryReader(env.db.diskdb, env.db.stateFreezer) ) for i, root := range roots { if root == dl.rootHash() { diff --git a/triedb/pathdb/history_state.go b/triedb/pathdb/history_state.go index bc21915dba..23428b1a54 100644 --- a/triedb/pathdb/history_state.go +++ b/triedb/pathdb/history_state.go @@ -283,11 +283,11 @@ func (h *stateHistory) typ() historyType { // forEach implements the history interface, returning an iterator to traverse the // state entries in the history. -func (h *stateHistory) forEach() iter.Seq[stateIdent] { - return func(yield func(stateIdent) bool) { +func (h *stateHistory) forEach() iter.Seq[indexElem] { + return func(yield func(indexElem) bool) { for _, addr := range h.accountList { addrHash := crypto.Keccak256Hash(addr.Bytes()) - if !yield(newAccountIdent(addrHash)) { + if !yield(accountIndexElem{addrHash}) { return } for _, slotKey := range h.storageList[addr] { @@ -298,7 +298,7 @@ func (h *stateHistory) forEach() iter.Seq[stateIdent] { if h.meta.version != stateHistoryV0 { slotHash = crypto.Keccak256Hash(slotKey.Bytes()) } - if !yield(newStorageIdent(addrHash, slotHash)) { + if !yield(storageIndexElem{addrHash, slotHash}) { return } } diff --git a/triedb/pathdb/history_trienode.go b/triedb/pathdb/history_trienode.go index 3f45b41117..11d6112806 100644 --- a/triedb/pathdb/history_trienode.go +++ b/triedb/pathdb/history_trienode.go @@ -19,9 +19,11 @@ package pathdb import ( "bytes" "encoding/binary" + "errors" "fmt" "iter" "maps" + "math" "slices" "sort" "time" @@ -44,7 +46,10 @@ import ( // - block number (8 bytes) // // - a lexicographically sorted list of trie IDs -// - the corresponding offsets into the key and value sections for each trie data chunk +// - the corresponding offsets into the key and value sections for each trie +// data chunk. The offsets refer to the end position of each chunk, with +// the assumption that the key and value sections for the first data chunk +// start at offset 0. // // Although some fields (e.g., parent state root, block number) are duplicated // between the state history and the trienode history, these two histories @@ -53,19 +58,16 @@ import ( // // # Key section // The key section stores trie node keys (paths) in a compressed format. -// It also contains relative offsets into the value section for resolving -// the corresponding trie node data. Note that these offsets are relative -// to the data chunk for the trie; the chunk offset must be added to obtain -// the absolute position. +// It also contains relative offsets into the value section for locating +// the corresponding trie node data. These offsets are relative to the +// beginning of the trie data chunk, the chunk's base offset must be added +// to obtain the absolute position in the value section. // // # Value section // The value section is a concatenated byte stream of all trie node data. // Each trie node can be retrieved using the offset and length specified // by its index entry. // -// The header and key sections are sufficient for locating a trie node, -// while a partial read of the value section is enough to retrieve its data. - // Header section: // // +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------| @@ -87,9 +89,9 @@ import ( // // +---- key len ----+ // / \ -// +-------+---------+-----------+---------+-----------------------+-----------------+ -// | shared (varint) | not shared (varint) | value length (varlen) | key (varlen) | -// +-----------------+---------------------+-----------------------+-----------------+ +// +-------+---------+-----------+---------+-----------------------+-----------------------+ +// | shared (varint) | not shared (varint) | value length (varlen) | unshared key (varlen) | +// +-----------------+---------------------+-----------------------+-----------------------+ // // trailer: // @@ -99,9 +101,9 @@ import ( // | restart_1 key offset | restart_1 value offset | ... | restart number (4-bytes) | // +----------------------+------------------------+-----+--------------------------+ // -// Note: Both the key offset and the value offset are relative to the start of -// the trie data chunk. To obtain the absolute offset, add the offset of the -// trie data chunk itself. +// Note: Both the key offset and the value offset are relative to the beginning +// of the trie data chunk. The chunk's base offset must be added to obtain the +// absolute position in the value section. // // Value section: // @@ -138,9 +140,12 @@ type trienodeHistory struct { // newTrienodeHistory constructs a trienode history with the provided trie nodes. func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, nodes map[common.Hash]map[string][]byte) *trienodeHistory { - nodeList := make(map[common.Hash][]string) + nodeList := make(map[common.Hash][]string, len(nodes)) for owner, subset := range nodes { - keys := sort.StringSlice(slices.Collect(maps.Keys(subset))) + keys := make(sort.StringSlice, 0, len(subset)) + for k := range subset { + keys = append(keys, k) + } keys.Sort() nodeList[owner] = keys } @@ -157,17 +162,6 @@ func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, node } } -// sharedLen returns the length of the common prefix shared by a and b. -func sharedLen(a, b []byte) int { - n := min(len(a), len(b)) - for i := range n { - if a[i] != b[i] { - return i - } - } - return n -} - // typ implements the history interface, returning the historical data type held. func (h *trienodeHistory) typ() historyType { return typeTrienodeHistory @@ -175,11 +169,35 @@ func (h *trienodeHistory) typ() historyType { // forEach implements the history interface, returning an iterator to traverse the // state entries in the history. -func (h *trienodeHistory) forEach() iter.Seq[stateIdent] { - return func(yield func(stateIdent) bool) { +func (h *trienodeHistory) forEach() iter.Seq[indexElem] { + return func(yield func(indexElem) bool) { for _, owner := range h.owners { - for _, path := range h.nodeList[owner] { - if !yield(newTrienodeIdent(owner, path)) { + var ( + scheme *indexScheme + paths = h.nodeList[owner] + indexes = make(map[string]map[uint16]struct{}) + ) + if owner == (common.Hash{}) { + scheme = accountIndexScheme + } else { + scheme = storageIndexScheme + } + for _, leaf := range findLeafPaths(paths) { + chunks, ids := scheme.splitPath(leaf) + for i := 0; i < len(chunks); i++ { + if _, exists := indexes[chunks[i]]; !exists { + indexes[chunks[i]] = make(map[uint16]struct{}) + } + indexes[chunks[i]][ids[i]] = struct{}{} + } + } + for chunk, ids := range indexes { + elem := trienodeIndexElem{ + owner: owner, + path: chunk, + data: slices.Collect(maps.Keys(ids)), + } + if !yield(elem) { return } } @@ -207,17 +225,22 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) { restarts []uint32 prefixLen int - internalKeyOffset uint32 // key offset for the trie internally - internalValOffset uint32 // value offset for the trie internally + internalKeyOffset uint32 // key offset within the trie data internally + internalValOffset uint32 // value offset within the trie data internally ) for i, path := range h.nodeList[owner] { key := []byte(path) + + // Track the internal key and value offsets at the beginning of the + // restart section. The absolute offsets within the key and value + // sections should first include the offset of the trie chunk itself + // stored in the header section. if i%trienodeDataBlockRestartLen == 0 { restarts = append(restarts, internalKeyOffset) restarts = append(restarts, internalValOffset) prefixLen = 0 } else { - prefixLen = sharedLen(prevKey, key) + prefixLen = commonPrefixLen(prevKey, key) } value := h.nodes[owner][path] @@ -256,18 +279,13 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) { } // Fill the header section with the offsets of the key and value sections. - // Note that the key/value offsets are intentionally tracked *after* encoding - // them into their respective sections, ensuring each offset refers to the end - // position. For n trie chunks, n offset pairs are sufficient to uniquely locate - // the corresponding data. - headerSection.Write(owner.Bytes()) // 32 bytes - binary.Write(&headerSection, binary.BigEndian, uint32(keySection.Len())) // 4 bytes - - // The offset to the value section is theoretically unnecessary, since the - // individual value offset is already tracked in the key section. However, - // we still keep it here for two reasons: - // - It's cheap to store (only 4 bytes for each trie). - // - It can be useful for decoding the trie data when key is not required (e.g., in hash mode). + // Note that key/value offsets are intentionally recorded *after* encoding + // into their respective sections, so each offset refers to an end position. + // For n trie chunks, n offset pairs are sufficient to uniquely locate each + // chunk's data. For example, [0, offset_0] defines the range of trie chunk 0, + // while [offset_{n-2}, offset_{n-1}] defines the range of trie chunk n-1. + headerSection.Write(owner.Bytes()) // 32 bytes + binary.Write(&headerSection, binary.BigEndian, uint32(keySection.Len())) // 4 bytes binary.Write(&headerSection, binary.BigEndian, uint32(valueSection.Len())) // 4 bytes } return headerSection.Bytes(), keySection.Bytes(), valueSection.Bytes(), nil @@ -330,32 +348,68 @@ func decodeHeader(data []byte) (*trienodeMetadata, []common.Hash, []uint32, []ui }, owners, keyOffsets, valOffsets, nil } -func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]string, error) { - var ( - prevKey []byte - items int - keyOffsets []uint32 - valOffsets []uint32 +// decodeKeyEntry resolves a single entry from the key section starting from +// the specified offset. +func decodeKeyEntry(keySection []byte, offset int) (uint64, uint64, []byte, int, error) { + var byteRead int - keyOff int // the key offset within the single trie data - valOff int // the value offset within the single trie data + // Resolve the length of shared key + nShared, nn := binary.Uvarint(keySection[offset:]) // key length shared (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nShared at offset %d", offset) + } + byteRead += nn - keys []string - ) + // Resolve the length of unshared key + nUnshared, nn := binary.Uvarint(keySection[offset+byteRead:]) // key length not shared (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nUnshared at offset %d", offset+byteRead) + } + byteRead += nn + + // Resolve the length of value + nValue, nn := binary.Uvarint(keySection[offset+byteRead:]) // value length (varint) + if nn <= 0 { + return 0, 0, nil, 0, fmt.Errorf("corrupted varint encoding for nValue at offset %d", offset+byteRead) + } + byteRead += nn + + // Validate that the values can fit in an int to prevent overflow on 32-bit systems + if nShared > uint64(math.MaxUint32) || nUnshared > uint64(math.MaxUint32) || nValue > uint64(math.MaxUint32) { + return 0, 0, nil, 0, errors.New("key/value size too large") + } + + // Resolve the unshared key + if offset+byteRead+int(nUnshared) > len(keySection) { + return 0, 0, nil, 0, fmt.Errorf("key length too long, unshared key length: %d, off: %d, section size: %d", nUnshared, offset+byteRead, len(keySection)) + } + unsharedKey := keySection[offset+byteRead : offset+byteRead+int(nUnshared)] + byteRead += int(nUnshared) + + return nShared, nValue, unsharedKey, byteRead, nil +} + +// decodeRestartTrailer resolves all the offsets recorded at the trailer. +func decodeRestartTrailer(keySection []byte) ([]uint32, []uint32, int, error) { // Decode the number of restart section if len(keySection) < 4 { - return nil, fmt.Errorf("key section too short, size: %d", len(keySection)) + return nil, nil, 0, fmt.Errorf("key section too short, size: %d", len(keySection)) } nRestarts := binary.BigEndian.Uint32(keySection[len(keySection)-4:]) + // Decode the trailer + var ( + keyOffsets = make([]uint32, 0, int(nRestarts)) + valOffsets = make([]uint32, 0, int(nRestarts)) + ) if len(keySection) < int(8*nRestarts)+4 { - return nil, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection)) + return nil, nil, 0, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection)) } for i := range int(nRestarts) { o := len(keySection) - 4 - (int(nRestarts)-i)*8 keyOffset := binary.BigEndian.Uint32(keySection[o : o+4]) if i != 0 && keyOffset <= keyOffsets[i-1] { - return nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset) + return nil, nil, 0, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset) } keyOffsets = append(keyOffsets, keyOffset) @@ -363,84 +417,118 @@ func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]st // section have zero-size value. valOffset := binary.BigEndian.Uint32(keySection[o+4 : o+8]) if i != 0 && valOffset < valOffsets[i-1] { - return nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset) + return nil, nil, 0, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset) } valOffsets = append(valOffsets, valOffset) } - keyLimit := len(keySection) - 4 - int(nRestarts)*8 + keyLimit := len(keySection) - 4 - int(nRestarts)*8 // End of key data + return keyOffsets, valOffsets, keyLimit, nil +} +// decodeRestartSection resolves all entries in a restart section. The keyData +// contains the encoded keys for the section. +// +// onValue is the callback function being invoked for each resolved entry. The +// start and limit are the offsets within the restart section, the base value +// offset of the restart section itself should be added by the caller itself. +// What's more, this function should return `aborted == true` if the entry +// resolution should be terminated. +func decodeRestartSection(keyData []byte, onValue func(key []byte, start int, limit int) (bool, error)) error { + var ( + prevKey []byte + items int + + keyOff int // the key offset within the single trie data + valOff int // the value offset within the single trie data + ) // Decode data - for keyOff < keyLimit { - // Validate the key and value offsets within the single trie data chunk - if items%trienodeDataBlockRestartLen == 0 { - restartIndex := items / trienodeDataBlockRestartLen - if restartIndex >= len(keyOffsets) { - return nil, fmt.Errorf("restart index out of range: %d, available restarts: %d", restartIndex, len(keyOffsets)) - } - if keyOff != int(keyOffsets[restartIndex]) { - return nil, fmt.Errorf("key offset is not matched, recorded: %d, want: %d", keyOffsets[restartIndex], keyOff) - } - if valOff != int(valOffsets[restartIndex]) { - return nil, fmt.Errorf("value offset is not matched, recorded: %d, want: %d", valOffsets[restartIndex], valOff) - } + for keyOff < len(keyData) { + nShared, nValue, unsharedKey, nn, err := decodeKeyEntry(keyData, keyOff) + if err != nil { + return err } - // Resolve the entry from key section - nShared, nn := binary.Uvarint(keySection[keyOff:]) // key length shared (varint) keyOff += nn - nUnshared, nn := binary.Uvarint(keySection[keyOff:]) // key length not shared (varint) - keyOff += nn - nValue, nn := binary.Uvarint(keySection[keyOff:]) // value length (varint) - keyOff += nn - - // Resolve unshared key - if keyOff+int(nUnshared) > len(keySection) { - return nil, fmt.Errorf("key length too long, unshared key length: %d, off: %d, section size: %d", nUnshared, keyOff, len(keySection)) - } - unsharedKey := keySection[keyOff : keyOff+int(nUnshared)] - keyOff += int(nUnshared) // Assemble the full key var key []byte if items%trienodeDataBlockRestartLen == 0 { if nShared != 0 { - return nil, fmt.Errorf("unexpected non-zero shared key prefix: %d", nShared) + return fmt.Errorf("unexpected non-zero shared key prefix: %d", nShared) } key = unsharedKey } else { if int(nShared) > len(prevKey) { - return nil, fmt.Errorf("unexpected shared key prefix: %d, prefix key length: %d", nShared, len(prevKey)) + return fmt.Errorf("unexpected shared key prefix: %d, prefix key length: %d", nShared, len(prevKey)) } - key = append([]byte{}, prevKey[:nShared]...) - key = append(key, unsharedKey...) + key = make([]byte, int(nShared)+len(unsharedKey)) + copy(key[:nShared], prevKey[:nShared]) + copy(key[nShared:], unsharedKey) } if items != 0 && bytes.Compare(prevKey, key) >= 0 { - return nil, fmt.Errorf("trienode paths are out of order, prev: %v, cur: %v", prevKey, key) + return fmt.Errorf("trienode paths are out of order, prev: %v, cur: %v", prevKey, key) } prevKey = key - // Resolve value - if onValue != nil { - if err := onValue(key, valOff, valOff+int(nValue)); err != nil { - return nil, err - } + valEnd := valOff + int(nValue) + abort, err := onValue(key, valOff, valEnd) + if err != nil { + return err } - valOff += int(nValue) - + if abort { + return nil + } + valOff = valEnd items++ - keys = append(keys, string(key)) } - if keyOff != keyLimit { - return nil, fmt.Errorf("excessive key data after decoding, offset: %d, size: %d", keyOff, keyLimit) + if keyOff != len(keyData) { + return fmt.Errorf("excessive key data after decoding, offset: %d, size: %d", keyOff, len(keyData)) + } + return nil +} + +// onValue is the callback function being invoked for each resolved entry. The +// start and limit are the offsets within this trie chunk, the base value +// offset of the trie chunk itself should be added by the caller itself. +func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) error { + keyOffsets, valOffsets, keyLimit, err := decodeRestartTrailer(keySection) + if err != nil { + return err + } + for i := 0; i < len(keyOffsets); i++ { + var keyData []byte + if i == len(keyOffsets)-1 { + keyData = keySection[keyOffsets[i]:keyLimit] + } else { + keyData = keySection[keyOffsets[i]:keyOffsets[i+1]] + } + err := decodeRestartSection(keyData, func(key []byte, start int, limit int) (bool, error) { + valStart := int(valOffsets[i]) + start + valLimit := int(valOffsets[i]) + limit + + // Possible in tests + if onValue == nil { + return false, nil + } + if err := onValue(key, valStart, valLimit); err != nil { + return false, err + } + return false, nil // abort=false + }) + if err != nil { + return err + } } - return keys, nil + return nil } func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, map[string][]byte, error) { var ( - offset int - nodes = make(map[string][]byte) + offset int + estimated = len(keySection) / 8 + nodes = make(map[string][]byte, estimated) + paths = make([]string, 0, estimated) ) - paths, err := decodeSingle(keySection, func(key []byte, start int, limit int) error { + err := decodeSingle(keySection, func(key []byte, start int, limit int) error { if start != offset { return fmt.Errorf("gapped value section offset: %d, want: %d", start, offset) } @@ -451,7 +539,9 @@ func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, ma if start > len(valueSection) || limit > len(valueSection) { return fmt.Errorf("value section out of range: start: %d, limit: %d, size: %d", start, limit, len(valueSection)) } - nodes[string(key)] = valueSection[start:limit] + strkey := string(key) + paths = append(paths, strkey) + nodes[strkey] = valueSection[start:limit] offset = limit return nil @@ -477,7 +567,8 @@ func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection h.nodes = make(map[common.Hash]map[string][]byte) for i := range len(owners) { - // Resolve the boundary of key section + // Resolve the boundary of the key section, each offset referring + // to the end position of this trie chunk. var keyStart, keyLimit uint32 if i != 0 { keyStart = keyOffsets[i-1] @@ -487,7 +578,8 @@ func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection return fmt.Errorf("invalid key offsets: keyStart: %d, keyLimit: %d, size: %d", keyStart, keyLimit, len(keySection)) } - // Resolve the boundary of value section + // Resolve the boundary of the value section, each offset referring + // to the end position of this trie chunk. var valStart, valLimit uint32 if i != 0 { valStart = valueOffsets[i-1] @@ -517,14 +609,11 @@ func (ir iRange) len() uint32 { return ir.limit - ir.start } -// singleTrienodeHistoryReader provides read access to a single trie within the -// trienode history. It stores an offset to the trie's position in the history, -// along with a set of per-node offsets that can be resolved on demand. type singleTrienodeHistoryReader struct { - id uint64 - reader ethdb.AncientReader - valueRange iRange // value range within the global value section - valueInternalOffsets map[string]iRange // value offset within the single trie data + id uint64 + reader ethdb.AncientReader + keyData []byte + valueRange iRange } func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRange iRange, valueRange iRange) (*singleTrienodeHistoryReader, error) { @@ -532,121 +621,173 @@ func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRa if err != nil { return nil, err } - valueOffsets := make(map[string]iRange) - _, err = decodeSingle(keyData, func(key []byte, start int, limit int) error { - valueOffsets[string(key)] = iRange{ - start: uint32(start), - limit: uint32(limit), - } - return nil + return &singleTrienodeHistoryReader{ + id: id, + reader: reader, + keyData: keyData, + valueRange: valueRange, + }, nil +} + +// searchSingle searches for a specific trie node identified by the provided +// key within a single trie node chunk. +// +// It returns the node value's offset range (start and limit) within the +// trie node data. An error is returned if the node cannot be found. +func (sr *singleTrienodeHistoryReader) searchSingle(key []byte) (int, int, bool, error) { + keyOffsets, valOffsets, keyLimit, err := decodeRestartTrailer(sr.keyData) + if err != nil { + return 0, 0, false, err + } + // Binary search against the boundary keys for each restart section + var ( + boundFind bool + boundValueLen uint64 + ) + pos := sort.Search(len(keyOffsets), func(i int) bool { + _, nValue, dkey, _, derr := decodeKeyEntry(sr.keyData[keyOffsets[i]:], 0) + if derr != nil { + err = derr + return false + } + n := bytes.Compare(key, dkey) + if n == 0 { + boundFind = true + boundValueLen = nValue + } + return n <= 0 }) if err != nil { - return nil, err + return 0, 0, false, err } - return &singleTrienodeHistoryReader{ - id: id, - reader: reader, - valueRange: valueRange, - valueInternalOffsets: valueOffsets, - }, nil + // The node is found as the boundary of restart section + if boundFind { + start := valOffsets[pos] + limit := valOffsets[pos] + uint32(boundValueLen) + return int(start), int(limit), true, nil + } + // The node is not found as all others have larger key than the target + if pos == 0 { + return 0, 0, false, nil + } + // Search the target node within the restart section + var keyData []byte + if pos == len(keyOffsets) { + keyData = sr.keyData[keyOffsets[pos-1]:keyLimit] // last section + } else { + keyData = sr.keyData[keyOffsets[pos-1]:keyOffsets[pos]] // non-last section + } + var ( + nStart int + nLimit int + found bool + ) + err = decodeRestartSection(keyData, func(ikey []byte, start, limit int) (bool, error) { + if bytes.Equal(key, ikey) { + nStart = int(valOffsets[pos-1]) + start + nLimit = int(valOffsets[pos-1]) + limit + found = true + return true, nil // abort = true + } + return false, nil // abort = false + }) + if err != nil { + return 0, 0, false, err + } + if !found { + return 0, 0, false, nil + } + return nStart, nLimit, true, nil } // read retrieves the trie node data with the provided node path. -func (sr *singleTrienodeHistoryReader) read(path string) ([]byte, error) { - offset, exists := sr.valueInternalOffsets[path] - if !exists { - return nil, fmt.Errorf("trienode %v not found", []byte(path)) +func (sr *singleTrienodeHistoryReader) read(key []byte) ([]byte, bool, error) { + start, limit, found, err := sr.searchSingle(key) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + valStart := uint64(start) + uint64(sr.valueRange.start) + valLen := uint64(limit - start) + value, err := rawdb.ReadTrienodeHistoryValueSection(sr.reader, sr.id, valStart, valLen) + if err != nil { + return nil, false, err } - return rawdb.ReadTrienodeHistoryValueSection(sr.reader, sr.id, uint64(sr.valueRange.start+offset.start), uint64(offset.len())) + return value, true, nil } // trienodeHistoryReader provides read access to node data in the trie node history. // It resolves data from the underlying ancient store only when needed, minimizing // I/O overhead. type trienodeHistoryReader struct { - id uint64 // ID of the associated trienode history - reader ethdb.AncientReader // Database reader of ancient store - keyRanges map[common.Hash]iRange // Key ranges identifying trie chunks - valRanges map[common.Hash]iRange // Value ranges identifying trie chunks - iReaders map[common.Hash]*singleTrienodeHistoryReader // readers for each individual trie chunk + id uint64 // ID of the associated trienode history + reader ethdb.AncientReader // Database reader of ancient store } // newTrienodeHistoryReader constructs the reader for specific trienode history. -func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) (*trienodeHistoryReader, error) { - r := &trienodeHistoryReader{ - id: id, - reader: reader, - keyRanges: make(map[common.Hash]iRange), - valRanges: make(map[common.Hash]iRange), - iReaders: make(map[common.Hash]*singleTrienodeHistoryReader), - } - if err := r.decodeHeader(); err != nil { - return nil, err +func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) *trienodeHistoryReader { + return &trienodeHistoryReader{ + id: id, + reader: reader, } - return r, nil } // decodeHeader decodes the header section of trienode history. -func (r *trienodeHistoryReader) decodeHeader() error { +func (r *trienodeHistoryReader) decodeHeader(owner common.Hash) (iRange, iRange, bool, error) { header, err := rawdb.ReadTrienodeHistoryHeader(r.reader, r.id) if err != nil { - return err + return iRange{}, iRange{}, false, err } _, owners, keyOffsets, valOffsets, err := decodeHeader(header) if err != nil { - return err + return iRange{}, iRange{}, false, err } - for i, owner := range owners { - // Decode the key range for this trie chunk - var keyStart uint32 - if i != 0 { - keyStart = keyOffsets[i-1] - } - r.keyRanges[owner] = iRange{ - start: keyStart, - limit: keyOffsets[i], - } + pos := sort.Search(len(owners), func(i int) bool { + return owner.Cmp(owners[i]) <= 0 + }) + if pos == len(owners) || owners[pos] != owner { + return iRange{}, iRange{}, false, nil + } + var keyRange iRange + if pos != 0 { + keyRange.start = keyOffsets[pos-1] + } + keyRange.limit = keyOffsets[pos] - // Decode the value range for this trie chunk - var valStart uint32 - if i != 0 { - valStart = valOffsets[i-1] - } - r.valRanges[owner] = iRange{ - start: valStart, - limit: valOffsets[i], - } + var valRange iRange + if pos != 0 { + valRange.start = valOffsets[pos-1] } - return nil + valRange.limit = valOffsets[pos] + return keyRange, valRange, true, nil } // read retrieves the trie node data with the provided TrieID and node path. -func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, error) { - ir, ok := r.iReaders[owner] - if !ok { - keyRange, exists := r.keyRanges[owner] - if !exists { - return nil, fmt.Errorf("trie %x is unknown", owner) - } - valRange, exists := r.valRanges[owner] - if !exists { - return nil, fmt.Errorf("trie %x is unknown", owner) - } - var err error - ir, err = newSingleTrienodeHistoryReader(r.id, r.reader, keyRange, valRange) - if err != nil { - return nil, err - } - r.iReaders[owner] = ir +func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, bool, error) { + keyRange, valRange, found, err := r.decodeHeader(owner) + if err != nil { + return nil, false, err + } + if !found { + return nil, false, nil + } + ir, err := newSingleTrienodeHistoryReader(r.id, r.reader, keyRange, valRange) + if err != nil { + return nil, false, err } - return ir.read(path) + return ir.read([]byte(path)) } // writeTrienodeHistory persists the trienode history associated with the given diff layer. -// nolint:unused -func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error { +func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer, rate uint32) error { start := time.Now() - h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, dl.nodes.nodeOrigin) + nodes, err := dl.nodes.encodeNodeHistory(dl.root, rate) + if err != nil { + return err + } + h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, nodes) header, keySection, valueSection, err := h.encode() if err != nil { return err @@ -670,7 +811,6 @@ func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error { } // readTrienodeMetadata resolves the metadata of the specified trienode history. -// nolint:unused func readTrienodeMetadata(reader ethdb.AncientReader, id uint64) (*trienodeMetadata, error) { header, err := rawdb.ReadTrienodeHistoryHeader(reader, id) if err != nil { diff --git a/triedb/pathdb/history_trienode_test.go b/triedb/pathdb/history_trienode_test.go index d6b80f61f5..05278c30b1 100644 --- a/triedb/pathdb/history_trienode_test.go +++ b/triedb/pathdb/history_trienode_test.go @@ -19,6 +19,7 @@ package pathdb import ( "bytes" "encoding/binary" + "fmt" "math/rand" "reflect" "testing" @@ -137,14 +138,11 @@ func TestTrienodeHistoryReader(t *testing.T) { } } for i, h := range hs { - tr, err := newTrienodeHistoryReader(uint64(i+1), freezer) - if err != nil { - t.Fatalf("Failed to construct the history reader: %v", err) - } + tr := newTrienodeHistoryReader(uint64(i+1), freezer) for _, owner := range h.owners { nodes := h.nodes[owner] for key, value := range nodes { - blob, err := tr.read(owner, key) + blob, _, err := tr.read(owner, key) if err != nil { t.Fatalf("Failed to read trienode history: %v", err) } @@ -417,23 +415,23 @@ func TestTrienodeHistoryReaderNonExistentPath(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Try to read a non-existent path - _, err = tr.read(testrand.Hash(), "nonexistent") - if err == nil { - t.Fatal("Expected error for non-existent trie owner") + var ( + err error + found bool + ) + _, found, err = tr.read(testrand.Hash(), "nonexistent") + if found || err != nil { + t.Fatal("Expected not found for non-existent trie owner") } // Try to read from existing owner but non-existent path owner := h.owners[0] - _, err = tr.read(owner, "nonexistent-path") - if err == nil { - t.Fatal("Expected error for non-existent path") + _, found, err = tr.read(owner, "nonexistent-path") + if found || err != nil { + t.Fatal("Expected not found for non-existent path") } } @@ -457,23 +455,19 @@ func TestTrienodeHistoryReaderNilValues(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Test reading nil values - data1, err := tr.read(owner, "nil1") - if err != nil { + data1, found, err := tr.read(owner, "nil1") + if err != nil || !found { t.Fatalf("Failed to read nil value: %v", err) } if len(data1) != 0 { t.Fatal("Expected nil data for nil value") } - data2, err := tr.read(owner, "nil2") - if err != nil { + data2, found, err := tr.read(owner, "nil2") + if err != nil || !found { t.Fatalf("Failed to read nil value: %v", err) } if len(data2) != 0 { @@ -481,8 +475,8 @@ func TestTrienodeHistoryReaderNilValues(t *testing.T) { } // Test reading non-nil value - data3, err := tr.read(owner, "data1") - if err != nil { + data3, found, err := tr.read(owner, "data1") + if err != nil || !found { t.Fatalf("Failed to read non-nil value: %v", err) } if !bytes.Equal(data3, []byte("some data")) { @@ -498,7 +492,7 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { // Add some nil values nodes[owner][""] = []byte("some data") - nodes[owner]["data1"] = []byte("some data") + nodes[owner]["data1"] = []byte("some data1") h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) @@ -509,14 +503,10 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { t.Fatalf("Failed to write trienode history: %v", err) } - - tr, err := newTrienodeHistoryReader(1, freezer) - if err != nil { - t.Fatalf("Failed to construct history reader: %v", err) - } + tr := newTrienodeHistoryReader(1, freezer) // Test reading nil values - data1, err := tr.read(owner, "") + data1, _, err := tr.read(owner, "") if err != nil { t.Fatalf("Failed to read nil value: %v", err) } @@ -525,63 +515,17 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { } // Test reading non-nil value - data2, err := tr.read(owner, "data1") + data2, _, err := tr.read(owner, "data1") if err != nil { t.Fatalf("Failed to read non-nil value: %v", err) } - if !bytes.Equal(data2, []byte("some data")) { + if !bytes.Equal(data2, []byte("some data1")) { t.Fatal("Data mismatch for non-nil key") } } -// TestTrienodeHistoryReaderIterator tests the iterator functionality -func TestTrienodeHistoryReaderIterator(t *testing.T) { - h := makeTrienodeHistory() - - // Count expected entries - expectedCount := 0 - expectedNodes := make(map[stateIdent]bool) - for owner, nodeList := range h.nodeList { - expectedCount += len(nodeList) - for _, node := range nodeList { - expectedNodes[stateIdent{ - typ: typeTrienode, - addressHash: owner, - path: node, - }] = true - } - } - - // Test the iterator - actualCount := 0 - for x := range h.forEach() { - _ = x - actualCount++ - } - if actualCount != expectedCount { - t.Fatalf("Iterator count mismatch: expected %d, got %d", expectedCount, actualCount) - } - - // Test that iterator yields expected state identifiers - seen := make(map[stateIdent]bool) - for ident := range h.forEach() { - if ident.typ != typeTrienode { - t.Fatal("Iterator should only yield trienode history identifiers") - } - key := stateIdent{typ: ident.typ, addressHash: ident.addressHash, path: ident.path} - if seen[key] { - t.Fatal("Iterator yielded duplicate identifier") - } - seen[key] = true - - if !expectedNodes[key] { - t.Fatalf("Unexpected yielded identifier %v", key) - } - } -} - -// TestSharedLen tests the sharedLen helper function -func TestSharedLen(t *testing.T) { +// TestCommonPrefixLen tests the commonPrefixLen helper function +func TestCommonPrefixLen(t *testing.T) { tests := []struct { a, b []byte expected int @@ -610,13 +554,13 @@ func TestSharedLen(t *testing.T) { } for i, test := range tests { - result := sharedLen(test.a, test.b) + result := commonPrefixLen(test.a, test.b) if result != test.expected { t.Errorf("Test %d: sharedLen(%q, %q) = %d, expected %d", i, test.a, test.b, result, test.expected) } // Test commutativity - resultReverse := sharedLen(test.b, test.a) + resultReverse := commonPrefixLen(test.b, test.a) if result != resultReverse { t.Errorf("Test %d: sharedLen is not commutative: sharedLen(a,b)=%d, sharedLen(b,a)=%d", i, result, resultReverse) @@ -678,14 +622,14 @@ func TestDecodeSingleCorruptedData(t *testing.T) { _, keySection, _, _ := h.encode() // Test with empty key section - _, err := decodeSingle([]byte{}, nil) + err := decodeSingle([]byte{}, nil) if err == nil { t.Fatal("Expected error for empty key section") } // Test with key section too small for trailer if len(keySection) > 0 { - _, err := decodeSingle(keySection[:3], nil) // Less than 4 bytes for trailer + err := decodeSingle(keySection[:3], nil) // Less than 4 bytes for trailer if err == nil { t.Fatal("Expected error for key section too small for trailer") } @@ -694,8 +638,11 @@ func TestDecodeSingleCorruptedData(t *testing.T) { // Test with corrupted varint in key section corrupted := make([]byte, len(keySection)) copy(corrupted, keySection) - corrupted[5] = 0xFF // Corrupt varint - _, err = decodeSingle(corrupted, nil) + // Fill first 10 bytes with 0xFF to create a varint overflow (>64 bits) + for i := range 10 { + corrupted[i] = 0xFF + } + err = decodeSingle(corrupted, nil) if err == nil { t.Fatal("Expected error for corrupted varint") } @@ -705,7 +652,7 @@ func TestDecodeSingleCorruptedData(t *testing.T) { copy(corrupted, keySection) // Set restart count to something too large binary.BigEndian.PutUint32(corrupted[len(corrupted)-4:], 10000) - _, err = decodeSingle(corrupted, nil) + err = decodeSingle(corrupted, nil) if err == nil { t.Fatal("Expected error for invalid restart count") } @@ -734,3 +681,57 @@ func testEncodeDecode(t *testing.T, h *trienodeHistory) { t.Fatal("Trienode content mismatch") } } + +func TestSearchSingle(t *testing.T) { + nodes := make(map[common.Hash]map[string][]byte) + ownerA, ownerB := testrand.Hash(), testrand.Hash() + nodes[ownerA] = make(map[string][]byte) + nodes[ownerB] = make(map[string][]byte) + + for i := 0; i < trienodeDataBlockRestartLen*2; i++ { + nodes[ownerA][fmt.Sprintf("%d", 2*i+1)] = testrand.Bytes(rand.Intn(5)) + nodes[ownerB][fmt.Sprintf("%d", 2*i+1)] = testrand.Bytes(rand.Intn(5)) + } + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) + + var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false) + defer freezer.Close() + + header, keySection, valueSection, _ := h.encode() + if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil { + t.Fatalf("Failed to write trienode history: %v", err) + } + tr := newTrienodeHistoryReader(1, freezer) + + // Test reading non-existent entry + keys := []string{ + "0", + "2", + "30", + "32", + "64", + "1000", + } + for _, key := range keys { + _, found, err := tr.read(ownerA, key) + if err != nil || found { + t.Fatalf("Expected non-existent entry %v", err) + } + _, found, err = tr.read(ownerB, key) + if err != nil || found { + t.Fatalf("Expected non-existent entry %v", err) + } + } + + for owner, subnodes := range nodes { + for key, value := range subnodes { + got, found, err := tr.read(owner, key) + if err != nil || !found { + t.Fatal("Failed to read trienode") + } + if bytes.Compare(got, value) != 0 { + t.Fatalf("Unexpected value for key %v, got %v, expected %v", []byte(key), got, value) + } + } + } +} diff --git a/triedb/pathdb/history_trienode_utils.go b/triedb/pathdb/history_trienode_utils.go new file mode 100644 index 0000000000..11107494bb --- /dev/null +++ b/triedb/pathdb/history_trienode_utils.go @@ -0,0 +1,344 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "encoding/binary" + "fmt" + "math/bits" + "slices" + "strings" +) + +// commonPrefixLen returns the length of the common prefix shared by a and b. +func commonPrefixLen(a, b []byte) int { + n := min(len(a), len(b)) + for i := range n { + if a[i] != b[i] { + return i + } + } + return n +} + +// findLeafPaths scans a lexicographically sorted list of paths and returns +// the subset of paths that represent leaves. +// +// A path is considered a leaf if: +// - it is the last element in the list, or +// - the next path does not have the current path as its prefix. +// +// In other words, a leaf is a path that has no children extending it. +// +// Example: +// +// Input: ["a", "ab", "abc", "b", "ba"] +// Output: ["abc", "ba"] +// +// The input must be sorted; otherwise the result is undefined. +func findLeafPaths(paths []string) []string { + var leaves []string + for i := 0; i < len(paths); i++ { + if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) { + leaves = append(leaves, paths[i]) + } + } + return leaves +} + +// hexPathNodeID computes a numeric node ID from the given path. The path is +// interpreted as a sequence of base-16 digits, where each byte of the input +// is treated as one hexadecimal digit in a big-endian number. +// +// The resulting node ID is constructed as: +// +// ID = 1 + 16 + 16^2 + ... + 16^(n-1) + value +// +// where n is the number of bytes in the path, and `value` is the base-16 +// interpretation of the byte sequence. +// +// The offset (1 + 16 + 16^2 + ... + 16^(n-1)) ensures that all IDs of shorter +// paths occupy a lower numeric range, preserving lexicographic ordering between +// differently-length paths. +// +// The numeric node ID is represented by the uint16 with the assumption the length +// of path won't be greater than 3. +func hexPathNodeID(path string) uint16 { + var ( + offset = uint16(0) + pow = uint16(1) + value = uint16(0) + bytes = []byte(path) + ) + for i := 0; i < len(bytes); i++ { + offset += pow + pow *= 16 + } + for i := 0; i < len(bytes); i++ { + value = value*16 + uint16(bytes[i]) + } + return offset + value +} + +// bitmapSize computes the number of bytes required for the marker bitmap +// corresponding to the remaining portion of a path after a cut point. +// The marker is a bitmap where each bit represents the presence of a +// possible element in the remaining path segment. +func bitmapSize(levels int) int { + // Compute: total = 1 + 16 + 16^2 + ... + 16^(segLen-1) + var ( + bits = 0 + pow = 1 + ) + for i := 0; i < levels; i++ { + bits += pow + pow *= 16 + } + // A small adjustment is applied to exclude the root element of this path + // segment, since any existing element would already imply the mutation of + // the root element. This trick can save us 1 byte for each bitmap which is + // non-trivial. + bits -= 1 + return bits / 8 +} + +// indexScheme defines how trie nodes are split into chunks and index them +// at chunk level. +// +// skipRoot indicates whether the root node should be excluded from indexing. +// cutPoints specifies the key length of chunks (in nibbles) extracted from +// each path. +type indexScheme struct { + // skipRoot indicates whether the root node should be excluded from indexing. + // In the account trie, the root is mutated on every state transition, so + // indexing it provides no value. + skipRoot bool + + // cutPoints defines the key lengths of chunks at different positions. + // A single trie node path may span multiple chunks vertically. + cutPoints []int + + // bitmaps specifies the required bitmap size for each chunk. The key is the + // chunk key length, and the value is the corresponding bitmap size. + bitmaps map[int]int +} + +var ( + // Account trie is split into chunks like this: + // + // - root node is excluded from indexing + // - nodes at level1 to level2 are grouped as 16 chunks + // - all other nodes are grouped 3 levels per chunk + // + // Level1 [0] ... [f] 16 chunks + // Level3 [000] ... [fff] 4096 chunks + // Level6 [000000] ... [fffffff] 16777216 chunks + // + // For the chunks at level1, there are 17 nodes per chunk. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // + // For the non-level1 chunks, there are 273 nodes per chunk, + // regardless of the chunk's depth in the trie. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // chunk-level 2 [ 17 ] … … [ 272 ] 256 nodes + accountIndexScheme = newIndexScheme(true) + + // Storage trie is split into chunks like this: (3 levels per chunk) + // + // Level0 [ ROOT ] 1 chunk + // Level3 [000] ... [fff] 4096 chunks + // Level6 [000000] ... [fffffff] 16777216 chunks + // + // Within each chunk, there are 273 nodes in total, regardless of + // the chunk's depth in the trie. + // + // chunk-level 0 [ 0 ] 1 node + // chunk-level 1 [ 1 ] … [ 16 ] 16 nodes + // chunk-level 2 [ 17 ] … … [ 272 ] 256 nodes + storageIndexScheme = newIndexScheme(false) +) + +// newIndexScheme initializes the index scheme. +func newIndexScheme(skipRoot bool) *indexScheme { + var ( + cuts []int + bitmaps = make(map[int]int) + ) + for v := 0; v <= 64; v += 3 { + var ( + levels int + length int + ) + if v == 0 && skipRoot { + length = 1 + levels = 2 + } else { + length = v + levels = 3 + } + cuts = append(cuts, length) + bitmaps[length] = bitmapSize(levels) + } + return &indexScheme{ + skipRoot: skipRoot, + cutPoints: cuts, + bitmaps: bitmaps, + } +} + +// getBitmapSize returns the required bytes for bitmap with chunk's position. +func (s *indexScheme) getBitmapSize(pathLen int) int { + return s.bitmaps[pathLen] +} + +// chunkSpan returns how many chunks should be spanned with the given path. +func (s *indexScheme) chunkSpan(length int) int { + var n int + for _, cut := range s.cutPoints { + if length >= cut { + n++ + continue + } + } + return n +} + +// splitPath applies the indexScheme to the given path and returns two lists: +// +// - chunkIDs: the progressive chunk IDs cuts defined by the scheme +// - innerIDs: the computed node ID for the path segment following each cut +// +// The scheme defines a set of cut points that partition the path. For each cut: +// +// - chunkIDs[i] is path[:cutPoints[i]] +// - innerIDs[i] is the node ID of the segment path[cutPoints[i] : nextCut-1] +func (s *indexScheme) splitPath(path string) ([]string, []uint16) { + // Special case: the root node of the account trie is mutated in every + // state transition, so its mutation records can be ignored. + n := len(path) + if n == 0 && s.skipRoot { + return nil, nil + } + var ( + // Determine how many chunks are spanned by the path + chunks = s.chunkSpan(n) + chunkIDs = make([]string, 0, chunks) + nodeIDs = make([]uint16, 0, chunks) + ) + for i := 0; i < chunks; i++ { + position := s.cutPoints[i] + chunkIDs = append(chunkIDs, path[:position]) + + var limit int + if i != chunks-1 { + limit = s.cutPoints[i+1] - 1 + } else { + limit = len(path) + } + nodeIDs = append(nodeIDs, hexPathNodeID(path[position:limit])) + } + return chunkIDs, nodeIDs +} + +// splitPathLast returns the path prefix of the deepest chunk spanned by the +// given path, along with its corresponding internal node ID. If the path +// spans no chunks, it returns an empty prefix and 0. +// +// nolint:unused +func (s *indexScheme) splitPathLast(path string) (string, uint16) { + chunkIDs, nodeIDs := s.splitPath(path) + if len(chunkIDs) == 0 { + return "", 0 + } + n := len(chunkIDs) + return chunkIDs[n-1], nodeIDs[n-1] +} + +// encodeIDs sorts the given list of uint16 IDs and encodes them into a +// compact byte slice using variable-length unsigned integer encoding. +func encodeIDs(ids []uint16) []byte { + slices.Sort(ids) + buf := make([]byte, 0, len(ids)) + for _, id := range ids { + buf = binary.AppendUvarint(buf, uint64(id)) + } + return buf +} + +// decodeIDs decodes a sequence of variable-length encoded uint16 IDs from the +// given byte slice and returns them as a set. +// +// Returns an error if the input buffer does not contain a complete Uvarint value. +func decodeIDs(buf []byte) ([]uint16, error) { + var res []uint16 + for len(buf) > 0 { + id, n := binary.Uvarint(buf) + if n <= 0 { + return nil, fmt.Errorf("too short for decoding node id, %v", buf) + } + buf = buf[n:] + res = append(res, uint16(id)) + } + return res, nil +} + +// isAncestor reports whether node x is the ancestor of node y. +func isAncestor(x, y uint16) bool { + for y > x { + y = (y - 1) / 16 // parentID(y) = (y - 1) / 16 + if y == x { + return true + } + } + return false +} + +// isBitSet reports whether the bit at `index` in the byte slice `b` is set. +func isBitSet(b []byte, index int) bool { + return b[index/8]&(1<<(7-index%8)) != 0 +} + +// setBit sets the bit at `index` in the byte slice `b` to 1. +func setBit(b []byte, index int) { + b[index/8] |= 1 << (7 - index%8) +} + +// bitPosTwoBytes returns the positions of set bits in a 2-byte bitmap. +// +// The bitmap is interpreted as a big-endian uint16. Bit positions are +// numbered from 0 to 15, where position 0 corresponds to the most +// significant bit of b[0], and position 15 corresponds to the least +// significant bit of b[1]. +func bitPosTwoBytes(b []byte) []int { + if len(b) != 2 { + panic("expect 2 bytes") + } + var ( + pos []int + mask = binary.BigEndian.Uint16(b) + ) + for mask != 0 { + p := bits.LeadingZeros16(mask) + pos = append(pos, p) + mask &^= 1 << (15 - p) + } + return pos +} diff --git a/triedb/pathdb/history_trienode_utils_test.go b/triedb/pathdb/history_trienode_utils_test.go new file mode 100644 index 0000000000..32bd91166d --- /dev/null +++ b/triedb/pathdb/history_trienode_utils_test.go @@ -0,0 +1,584 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "reflect" + "testing" +) + +func TestHexPathNodeID(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + exp uint16 + }{ + { + input: "", + exp: 0, + }, + { + input: string([]byte{0x0}), + exp: 1, + }, + { + input: string([]byte{0xf}), + exp: 16, + }, + { + input: string([]byte{0x0, 0x0}), + exp: 17, + }, + { + input: string([]byte{0x0, 0xf}), + exp: 32, + }, + { + input: string([]byte{0x1, 0x0}), + exp: 33, + }, + { + input: string([]byte{0x1, 0xf}), + exp: 48, + }, + { + input: string([]byte{0xf, 0xf}), + exp: 272, + }, + { + input: string([]byte{0xf, 0xf, 0xf}), + exp: 4368, + }, + } + for _, suite := range suites { + got := hexPathNodeID(suite.input) + if got != suite.exp { + t.Fatalf("Unexpected node ID for %v: got %d, want %d", suite.input, got, suite.exp) + } + } +} + +func TestFindLeafPaths(t *testing.T) { + t.Parallel() + + tests := []struct { + input []string + expect []string + }{ + { + input: nil, + expect: nil, + }, + { + input: []string{"a"}, + expect: []string{"a"}, + }, + { + input: []string{"", "0", "00", "01", "1"}, + expect: []string{ + "00", + "01", + "1", + }, + }, + { + input: []string{"10", "100", "11", "2"}, + expect: []string{ + "100", + "11", + "2", + }, + }, + { + input: []string{"10", "100000000", "11", "111111111", "2"}, + expect: []string{ + "100000000", + "111111111", + "2", + }, + }, + } + for _, test := range tests { + res := findLeafPaths(test.input) + if !reflect.DeepEqual(res, test.expect) { + t.Fatalf("Unexpected result: %v, expected %v", res, test.expect) + } + } +} + +func TestSplitAccountPath(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + expPrefix []string + expID []uint16 + }{ + // Length = 0 + { + "", nil, nil, + }, + // Length = 1 + { + string([]byte{0x0}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 0, + }, + }, + { + string([]byte{0x1}), + []string{ + string([]byte{0x1}), + }, + []uint16{ + 0, + }, + }, + { + string([]byte{0xf}), + []string{ + string([]byte{0xf}), + }, + []uint16{ + 0, + }, + }, + // Length = 2 + { + string([]byte{0x0, 0x0}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 1, + }, + }, + { + string([]byte{0x0, 0x1}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 2, + }, + }, + { + string([]byte{0x0, 0xf}), + []string{ + string([]byte{0x0}), + }, + []uint16{ + 16, + }, + }, + { + string([]byte{0xf, 0xf}), + []string{ + string([]byte{0xf}), + }, + []uint16{ + 16, + }, + }, + // Length = 3 + { + string([]byte{0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 0, + }, + }, + // Length = 3 + { + string([]byte{0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 0, + }, + }, + // Length = 4 + { + string([]byte{0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 1, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 16, + }, + }, + // Length = 5 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 17, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 272, + }, + }, + // Length = 6 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{0x0}), + string([]byte{0x0, 0x0, 0x0}), + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + }, + []uint16{ + 1, 17, 0, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{0xf}), + string([]byte{0xf, 0xf, 0xf}), + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + }, + []uint16{ + 16, 272, 0, + }, + }, + } + for _, suite := range suites { + prefix, id := accountIndexScheme.splitPath(suite.input) + if !reflect.DeepEqual(prefix, suite.expPrefix) { + t.Fatalf("Unexpected prefix for %v: got %v, want %v", suite.input, prefix, suite.expPrefix) + } + if !reflect.DeepEqual(id, suite.expID) { + t.Fatalf("Unexpected ID for %v: got %v, want %v", suite.input, id, suite.expID) + } + } +} + +func TestSplitStoragePath(t *testing.T) { + t.Parallel() + + var suites = []struct { + input string + expPrefix []string + expID []uint16 + }{ + // Length = 0 + { + "", + []string{ + string([]byte{}), + }, + []uint16{ + 0, + }, + }, + // Length = 1 + { + string([]byte{0x0}), + []string{ + string([]byte{}), + }, + []uint16{ + 1, + }, + }, + { + string([]byte{0x1}), + []string{ + string([]byte{}), + }, + []uint16{ + 2, + }, + }, + { + string([]byte{0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 16, + }, + }, + // Length = 2 + { + string([]byte{0x0, 0x0}), + []string{ + string([]byte{}), + }, + []uint16{ + 17, + }, + }, + { + string([]byte{0x0, 0x1}), + []string{ + string([]byte{}), + }, + []uint16{ + 18, + }, + }, + { + string([]byte{0x0, 0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 32, + }, + }, + { + string([]byte{0xf, 0xf}), + []string{ + string([]byte{}), + }, + []uint16{ + 272, + }, + }, + // Length = 3 + { + string([]byte{0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 0, + }, + }, + // Length = 3 + { + string([]byte{0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 0, + }, + }, + // Length = 4 + { + string([]byte{0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 1, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 16, + }, + }, + // Length = 5 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 17, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 272, + }, + }, + // Length = 6 + { + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + []string{ + string([]byte{}), + string([]byte{0x0, 0x0, 0x0}), + string([]byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0}), + }, + []uint16{ + 17, 17, 0, + }, + }, + { + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + []string{ + string([]byte{}), + string([]byte{0xf, 0xf, 0xf}), + string([]byte{0xf, 0xf, 0xf, 0xf, 0xf, 0xf}), + }, + []uint16{ + 272, 272, 0, + }, + }, + } + for i, suite := range suites { + prefix, id := storageIndexScheme.splitPath(suite.input) + if !reflect.DeepEqual(prefix, suite.expPrefix) { + t.Fatalf("Test %d, unexpected prefix for %v: got %v, want %v", i, suite.input, prefix, suite.expPrefix) + } + if !reflect.DeepEqual(id, suite.expID) { + t.Fatalf("Test %d, unexpected ID for %v: got %v, want %v", i, suite.input, id, suite.expID) + } + } +} + +func TestIsAncestor(t *testing.T) { + suites := []struct { + x, y uint16 + want bool + }{ + {0, 1, true}, + {0, 16, true}, + {0, 17, true}, + {0, 272, true}, + + {1, 0, false}, + {1, 2, false}, + {1, 17, true}, + {1, 18, true}, + {17, 273, true}, + {1, 1, false}, + } + for _, tc := range suites { + result := isAncestor(tc.x, tc.y) + if result != tc.want { + t.Fatalf("isAncestor(%d, %d) = %v, want %v", tc.x, tc.y, result, tc.want) + } + } +} + +func TestBitmapSet(t *testing.T) { + suites := []struct { + index int + expect []byte + }{ + { + 0, []byte{0b10000000, 0x0}, + }, + { + 1, []byte{0b01000000, 0x0}, + }, + { + 7, []byte{0b00000001, 0x0}, + }, + { + 8, []byte{0b00000000, 0b10000000}, + }, + { + 15, []byte{0b00000000, 0b00000001}, + }, + } + for _, tc := range suites { + var buf [2]byte + setBit(buf[:], tc.index) + + if !bytes.Equal(buf[:], tc.expect) { + t.Fatalf("bitmap = %v, want %v", buf, tc.expect) + } + if !isBitSet(buf[:], tc.index) { + t.Fatal("bit is not set") + } + } +} + +func TestBitPositions(t *testing.T) { + suites := []struct { + input []byte + expect []int + }{ + { + []byte{0b10000000, 0x0}, []int{0}, + }, + { + []byte{0b01000000, 0x0}, []int{1}, + }, + { + []byte{0b00000001, 0x0}, []int{7}, + }, + { + []byte{0b00000000, 0b10000000}, []int{8}, + }, + { + []byte{0b00000000, 0b00000001}, []int{15}, + }, + { + []byte{0b10000000, 0b00000001}, []int{0, 15}, + }, + { + []byte{0b10000001, 0b00000001}, []int{0, 7, 15}, + }, + { + []byte{0b10000001, 0b10000001}, []int{0, 7, 8, 15}, + }, + { + []byte{0b11111111, 0b11111111}, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + }, + { + []byte{0x0, 0x0}, nil, + }, + } + for _, tc := range suites { + got := bitPosTwoBytes(tc.input) + if !reflect.DeepEqual(got, tc.expect) { + t.Fatalf("Unexpected position set, want: %v, got: %v", tc.expect, got) + } + } +} diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index 29a05e963b..fd5149fb2e 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -338,10 +338,8 @@ func (db *Database) Journal(root common.Hash) error { // but the ancient store is not properly closed, resulting in recent writes // being lost. After a restart, the ancient store would then be misaligned // with the disk layer, causing data corruption. - if db.stateFreezer != nil { - if err := db.stateFreezer.SyncAncient(); err != nil { - return err - } + if err := syncHistory(db.stateFreezer, db.trienodeFreezer); err != nil { + return err } // Store the journal into the database and return var ( diff --git a/triedb/pathdb/lookup.go b/triedb/pathdb/lookup.go index 148f3e4e0f..65877ffb79 100644 --- a/triedb/pathdb/lookup.go +++ b/triedb/pathdb/lookup.go @@ -34,6 +34,13 @@ func storageKey(accountHash common.Hash, slotHash common.Hash) [64]byte { return key } +// storageKeySlice returns a key for uniquely identifying the storage slot in +// the slice format. +func storageKeySlice(accountHash common.Hash, slotHash common.Hash) []byte { + key := storageKey(accountHash, slotHash) + return key[:] +} + // lookup is an internal structure used to efficiently determine the layer in // which a state entry resides. type lookup struct { diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go index 31c40053fc..a0a626f9b5 100644 --- a/triedb/pathdb/metrics.go +++ b/triedb/pathdb/metrics.go @@ -73,11 +73,8 @@ var ( stateHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/data", nil) stateHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/index", nil) - //nolint:unused - trienodeHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/time", nil) - //nolint:unused - trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil) - //nolint:unused + trienodeHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/time", nil) + trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil) trienodeHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/index", nil) stateIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/index/time", nil) @@ -88,8 +85,9 @@ var ( lookupAddLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil) lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil) - historicalAccountReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/account/reads", nil) - historicalStorageReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/storage/reads", nil) + historicalAccountReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/account/reads", nil) + historicalStorageReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/storage/reads", nil) + historicalTrienodeReadTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/reads", nil) ) // Metrics in generation diff --git a/triedb/pathdb/nodes.go b/triedb/pathdb/nodes.go index bf1a8a6732..e507f11524 100644 --- a/triedb/pathdb/nodes.go +++ b/triedb/pathdb/nodes.go @@ -14,12 +14,14 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// nolint:unused package pathdb import ( "bytes" "errors" "fmt" + "hash/fnv" "io" "maps" @@ -29,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -423,3 +426,265 @@ func (s *nodeSetWithOrigin) decode(r *rlp.Stream) error { s.computeSize() return nil } + +// encodeNodeCompressed encodes the trie node differences between two consecutive +// versions into byte stream. The format is as below: +// +// - metadata byte layout (1 byte): +// +// ┌──── Bits (from MSB to LSB) ───┐ +// │ 7 │ 6 │ 5 │ 4 │ 3 │ 2 │ 1 │ 0 │ +// └───────────────────────────────┘ +// │ │ │ │ │ │ │ └─ FlagA: set if value is encoded in compressed format +// │ │ │ │ │ │ └───── FlagB: set if no extended bitmap is present after the metadata byte +// │ │ │ │ │ └───────── FlagC: bitmap for node (only used when flagB == 1) +// │ │ │ │ └───────────── FlagD: bitmap for node (only used when flagB == 1) +// │ │ │ └───────────────── FlagE: reserved (marks the presence of the 16th child in a full node) +// │ │ └───────────────────── FlagF: reserved +// │ └───────────────────────── FlagG: reserved +// └───────────────────────────── FlagH: reserved +// +// Note: +// - If flagB is 1, the node refers to a shortNode; +// - flagC indicates whether the key of the shortNode is recorded. +// - flagD indicates whether the value of the shortNode is recorded. +// +// - If flagB is 0, the node refers to a fullNode; +// - each bit in extended bitmap indicates whether the corresponding +// child have been modified. +// +// Example: +// +// 0b_0000_1011 +// +// Bit0=1, Bit1=1 -> node in compressed format, no extended bitmap +// Bit2=0, Bit3=1 -> the key of a short node is not stored; its value is stored. +// +// - 2 bytes extended bitmap (only if the flagB in metadata is 0), each bit +// represents a corresponding child; +// +// - concatenation of original value of modified children along with its size; +func encodeNodeCompressed(addExtension bool, elements [][]byte, indices []int) []byte { + var ( + enc []byte + flag = byte(1) // The compression format indicator + ) + // Pre-allocate the byte slice for the node encoder + size := 1 + if addExtension { + size += 2 + } + for _, element := range elements { + size += len(element) + 1 + } + enc = make([]byte, 0, size) + + if !addExtension { + flag |= 2 // The embedded bitmap indicator + + // Embedded bitmap + for _, pos := range indices { + flag |= 1 << (pos + 2) + } + enc = append(enc, flag) + } else { + // Extended bitmap + bitmap := make([]byte, 2) // bitmaps for at most 16 children + for _, pos := range indices { + // Children[16] is only theoretically possible in the Merkle-Patricia-trie, + // in practice this field is never used in the Ethereum case. If it occurs, + // use the FlagE for marking the presence. + if pos >= 16 { + log.Warn("Unexpected 16th child encountered in a full node") + flag |= 1 << 4 // Use the reserved flagE + continue + } + setBit(bitmap, pos) + } + enc = append(enc, flag) + enc = append(enc, bitmap...) + } + for _, element := range elements { + enc = append(enc, byte(len(element))) // 1 byte is sufficient for element size + enc = append(enc, element...) + } + return enc +} + +// encodeNodeFull encodes the full trie node value into byte stream. The format is +// as below: +// +// - metadata byte layout (1 byte): 0b0 +// - node value +// +// TODO(rjl493456442) it's not allocation efficient, please improve it. +func encodeNodeFull(value []byte) []byte { + enc := make([]byte, len(value)+1) + copy(enc[1:], value) + return enc +} + +// decodeNodeCompressed decodes the byte stream of compressed trie node +// back to the original elements and their indices. +// +// It assumes the byte stream contains a compressed format node. +func decodeNodeCompressed(data []byte) ([][]byte, []int, error) { + if len(data) < 1 { + return nil, nil, errors.New("invalid data: too short") + } + flag := data[0] + if flag&byte(1) == 0 { + return nil, nil, errors.New("invalid data: full node value") + } + noExtend := flag&byte(2) != 0 + + // Reconstruct indices from bitmap + var indices []int + if noExtend { + if flag&byte(4) != 0 { // flagC + indices = append(indices, 0) + } + if flag&byte(8) != 0 { // flagD + indices = append(indices, 1) + } + data = data[1:] + } else { + if len(data) < 3 { + return nil, nil, errors.New("invalid data: too short") + } + bitmap := data[1:3] + indices = bitPosTwoBytes(bitmap) + if flag&byte(16) != 0 { // flagE + indices = append(indices, 16) + log.Info("Unexpected 16th child encountered in a full node") + } + data = data[3:] + } + // Reconstruct elements + elements := make([][]byte, 0, len(indices)) + for i := 0; i < len(indices); i++ { + if len(data) == 0 { + return nil, nil, errors.New("invalid data: missing size byte") + } + // Read element size + size := int(data[0]) + data = data[1:] + + // Check if we have enough data for the element + if len(data) < size { + return nil, nil, fmt.Errorf("invalid data: expected %d bytes, got %d", size, len(data)) + } + // Extract element + if size == 0 { + elements = append(elements, nil) + + // The zero-size element is practically unexpected, for node deletion + // the rlp.EmptyString is still expected. Log loudly for the potential + // programming error. + log.Error("Empty element from compressed node, please open an issue", "raw", data) + } else { + element := make([]byte, size) + copy(element, data[:size]) + data = data[size:] + elements = append(elements, element) + } + } + // Check if all data is consumed + if len(data) != 0 { + return nil, nil, errors.New("invalid data: trailing bytes") + } + return elements, indices, nil +} + +// decodeNodeFull decodes the byte stream of full value trie node. +func decodeNodeFull(data []byte) (bool, []byte, error) { + if len(data) < 1 { + return false, nil, errors.New("invalid data: too short") + } + flag := data[0] + if flag != byte(0) { + return false, nil, nil + } + return true, data[1:], nil +} + +// encodeNodeHistory encodes the history of a node. Typically, the original values +// of dirty nodes serve as the history, but this can lead to significant storage +// overhead. +// +// For full nodes, which often see only a few modified children during state +// transitions, recording the entire child set (up to 16 children at 32 bytes +// each) is inefficient. For short nodes, which often see only the value is +// modified during the state transition, recording the key part is also unnecessary. +// To compress size, we instead record the diff of the node, rather than the +// full value. It's vital to compress the overall trienode history. +// +// However, recovering a node from a series of diffs requires applying multiple +// history records, which is computationally and IO intensive. To mitigate this, we +// periodically record the full value of a node as a checkpoint. The frequency of +// these checkpoints is a tradeoff between the compression rate and read overhead. +func (s *nodeSetWithOrigin) encodeNodeHistory(root common.Hash, rate uint32) (map[common.Hash]map[string][]byte, error) { + var ( + // the set of all encoded node history elements + nodes = make(map[common.Hash]map[string][]byte) + + // encodeFullValue determines whether a node should be encoded + // in full format with a pseudo-random probabilistic algorithm. + encodeFullValue = func(owner common.Hash, path string) bool { + // For trie nodes at the first two levels of the account trie, it is very + // likely that all children are modified within a single state transition. + // In such cases, do not use diff mode. + if owner == (common.Hash{}) && len(path) < 2 { + return true + } + h := fnv.New32a() + h.Write(root.Bytes()) + h.Write(owner.Bytes()) + h.Write([]byte(path)) + return h.Sum32()%rate == 0 + } + ) + for owner, origins := range s.nodeOrigin { + var posts map[string]*trienode.Node + if owner == (common.Hash{}) { + posts = s.nodeSet.accountNodes + } else { + posts = s.nodeSet.storageNodes[owner] + } + nodes[owner] = make(map[string][]byte) + + for path, oldvalue := range origins { + n, exists := posts[path] + if !exists { + // something not expected + return nil, fmt.Errorf("node with origin is not found, %x-%v", owner, []byte(path)) + } + encodeFull := encodeFullValue(owner, path) + if !encodeFull { + // TODO(rjl493456442) the diff-mode reencoding can take non-trivial + // time, like 1-2ms per block, is there any way to mitigate the overhead? + + // Partial encoding is required, try to find the node diffs and + // fallback to the full-value encoding if fails. + // + // The partial encoding will be failed in these certain cases: + // - the node is deleted or was not-existent; + // - the node type has been changed (e.g, from short to full) + nElem, indices, diffs, err := trie.NodeDifference(oldvalue, n.Blob) + if err != nil { + encodeFull = true // fallback to the full node encoding + } else { + // Encode the node difference as the history element + addExt := nElem != 2 // fullNode + blob := encodeNodeCompressed(addExt, diffs, indices) + nodes[owner][path] = blob + } + } + if encodeFull { + // Encode the entire original value as the history element + nodes[owner][path] = encodeNodeFull(oldvalue) + } + } + } + return nodes, nil +} diff --git a/triedb/pathdb/nodes_test.go b/triedb/pathdb/nodes_test.go index 483dc4b1a6..131d0ab012 100644 --- a/triedb/pathdb/nodes_test.go +++ b/triedb/pathdb/nodes_test.go @@ -18,11 +18,13 @@ package pathdb import ( "bytes" + "math/rand" "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" ) @@ -126,3 +128,49 @@ func TestNodeSetWithOriginEncode(t *testing.T) { t.Fatalf("Unexpected data size, got: %d, want: %d", dec2.size, s.size) } } + +func TestEncodeFullNodeCompressed(t *testing.T) { + var ( + elements [][]byte + indices []int + ) + for i := 0; i <= 16; i++ { + if rand.Intn(2) == 0 { + elements = append(elements, testrand.Bytes(20)) + indices = append(indices, i) + } + } + enc := encodeNodeCompressed(true, elements, indices) + decElements, decIndices, err := decodeNodeCompressed(enc) + if err != nil { + t.Fatalf("Failed to decode node compressed, %v", err) + } + if !reflect.DeepEqual(elements, decElements) { + t.Fatalf("Elements are not matched") + } + if !reflect.DeepEqual(indices, decIndices) { + t.Fatalf("Indices are not matched") + } +} + +func TestEncodeShortNodeCompressed(t *testing.T) { + var ( + elements [][]byte + indices []int + ) + for i := 0; i < 2; i++ { + elements = append(elements, testrand.Bytes(20)) + indices = append(indices, i) + } + enc := encodeNodeCompressed(false, elements, indices) + decElements, decIndices, err := decodeNodeCompressed(enc) + if err != nil { + t.Fatalf("Failed to decode node compressed, %v", err) + } + if !reflect.DeepEqual(elements, decElements) { + t.Fatalf("Elements are not matched") + } + if !reflect.DeepEqual(indices, decIndices) { + t.Fatalf("Indices are not matched") + } +} diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index 70ca558721..e0d86bdeaa 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -47,7 +47,7 @@ type nodeLoc struct { } // string returns the string representation of node location. -func (loc *nodeLoc) string() string { +func (loc nodeLoc) string() string { return fmt.Sprintf("loc: %s, depth: %d", loc.loc, loc.depth) } @@ -231,7 +231,7 @@ func (db *Database) StateReader(root common.Hash) (database.StateReader, error) // historical state. type HistoricalStateReader struct { db *Database - reader *historyReader + reader *stateHistoryReader id uint64 } @@ -265,7 +265,7 @@ func (db *Database) HistoricReader(root common.Hash) (*HistoricalStateReader, er return &HistoricalStateReader{ id: *id, db: db, - reader: newHistoryReader(db.diskdb, db.stateFreezer), + reader: newStateHistoryReader(db.diskdb, db.stateFreezer), }, nil } @@ -349,3 +349,90 @@ func (r *HistoricalStateReader) Storage(address common.Address, key common.Hash) } return r.reader.read(newStorageIdentQuery(address, addrHash, key, keyHash), r.id, dl.stateID(), latest) } + +// HistoricalNodeReader is a wrapper over history reader, providing access to +// historical trie node data. +type HistoricalNodeReader struct { + db *Database + reader *trienodeReader + id uint64 +} + +// HistoricNodeReader constructs a reader for accessing the requested historic state. +func (db *Database) HistoricNodeReader(root common.Hash) (*HistoricalNodeReader, error) { + // Bail out if the state history hasn't been fully indexed + if db.trienodeIndexer == nil || db.trienodeFreezer == nil { + return nil, fmt.Errorf("historical state %x is not available", root) + } + if !db.trienodeIndexer.inited() { + return nil, errors.New("trienode histories haven't been fully indexed yet") + } + // - States at the current disk layer or above are directly accessible + // via `db.NodeReader`. + // + // - States older than the current disk layer (including the disk layer + // itself) are available via `db.HistoricalNodeReader`. + id := rawdb.ReadStateID(db.diskdb, root) + if id == nil { + return nil, fmt.Errorf("state %#x is not available", root) + } + // Ensure the requested trienode history is canonical, states on side chain + // are not accessible. + meta, err := readTrienodeMetadata(db.trienodeFreezer, *id+1) + if err != nil { + return nil, err // e.g., the referred trienode history has been pruned + } + if meta.parent != root { + return nil, fmt.Errorf("state %#x is not canonincal", root) + } + return &HistoricalNodeReader{ + id: *id, + db: db, + reader: newTrienodeReader(db.diskdb, db.trienodeFreezer, int(db.config.FullValueCheckpoint)), + }, nil +} + +// Node directly retrieves the trie node data associated with a particular path, +// within a particular account. An error will be returned if the read operation +// exits abnormally. Specifically, if the layer is already stale. +// +// Note: +// - the returned trie node data is not a copy, please don't modify it. +// - an error will be returned if the requested trie node is not found in database. +func (r *HistoricalNodeReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) { + defer func(start time.Time) { + historicalTrienodeReadTimer.UpdateSince(start) + }(time.Now()) + + // TODO(rjl493456442): Theoretically, the obtained disk layer could become stale + // within a very short time window. + // + // While reading the account data while holding `db.tree.lock` can resolve + // this issue, but it will introduce a heavy contention over the lock. + // + // Let's optimistically assume the situation is very unlikely to happen, + // and try to define a low granularity lock if the current approach doesn't + // work later. + dl := r.db.tree.bottom() + latest, h, _, err := dl.node(owner, path, 0) + if err != nil { + return nil, err + } + if h == hash { + return latest, nil + } + blob, err := r.reader.read(newTrienodeIdent(owner, string(path)), r.id, dl.stateID(), latest) + if err != nil { + return nil, err + } + // Error out if the local one is inconsistent with the target. + if crypto.Keccak256Hash(blob) != hash { + blobHex := "nil" + if len(blob) > 0 { + blobHex = hexutil.Encode(blob) + } + log.Error("Unexpected historical trie node", "owner", owner.Hex(), "path", path, "blob", blobHex) + return nil, fmt.Errorf("unexpected historical trie node: (%x %v), blob: %s", owner, path, blobHex) + } + return blob, nil +} diff --git a/triedb/pathdb/states.go b/triedb/pathdb/states.go index 2e85fe8d6d..9b89233e61 100644 --- a/triedb/pathdb/states.go +++ b/triedb/pathdb/states.go @@ -239,12 +239,13 @@ func (s *stateSet) accountList() []common.Hash { if list != nil { return list } - // No old sorted account list exists, generate a new one. It's possible that - // multiple threads waiting for the write lock may regenerate the list - // multiple times, which is acceptable. s.listLock.Lock() defer s.listLock.Unlock() + // Double check after acquiring the write lock + if list = s.accountListSorted; list != nil { + return list + } list = slices.SortedFunc(maps.Keys(s.accountData), common.Hash.Cmp) s.accountListSorted = list return list @@ -269,12 +270,13 @@ func (s *stateSet) storageList(accountHash common.Hash) []common.Hash { } s.listLock.RUnlock() - // No old sorted account list exists, generate a new one. It's possible that - // multiple threads waiting for the write lock may regenerate the list - // multiple times, which is acceptable. s.listLock.Lock() defer s.listLock.Unlock() + // Double check after acquiring the write lock + if list := s.storageListSorted[accountHash]; list != nil { + return list + } list := slices.SortedFunc(maps.Keys(s.storageData[accountHash]), common.Hash.Cmp) s.storageListSorted[accountHash] = list return list @@ -410,7 +412,10 @@ func (s *stateSet) encode(w io.Writer) error { AddrHashes []common.Hash Accounts [][]byte } - var enc accounts + enc := accounts{ + AddrHashes: make([]common.Hash, 0, len(s.accountData)), + Accounts: make([][]byte, 0, len(s.accountData)), + } for addrHash, blob := range s.accountData { enc.AddrHashes = append(enc.AddrHashes, addrHash) enc.Accounts = append(enc.Accounts, blob) @@ -577,7 +582,10 @@ func (s *StateSetWithOrigin) encode(w io.Writer) error { Addresses []common.Address Accounts [][]byte } - var accounts Accounts + accounts := Accounts{ + Addresses: make([]common.Address, 0, len(s.accountOrigin)), + Accounts: make([][]byte, 0, len(s.accountOrigin)), + } for address, blob := range s.accountOrigin { accounts.Addresses = append(accounts.Addresses, address) accounts.Accounts = append(accounts.Accounts, blob) diff --git a/version/version.go b/version/version.go index d52b5571b0..f81dd0920d 100644 --- a/version/version.go +++ b/version/version.go @@ -19,8 +19,8 @@ package version // This is the version of the go-ethereum const ( - Major = 1 // Major version component of the current release - Minor = 16 // Minor version component of the current release - Patch = 9 // Patch version component of the current release - Meta = "stable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 17 // Minor version component of the current release + Patch = 0 // Patch version component of the current release + Meta = "unstable" // Version metadata to append to the version string )