From ed9a4a6f8dc8cebd34b21e3916c42f3d993b9718 Mon Sep 17 00:00:00 2001 From: Jerry Date: Mon, 10 Aug 2026 15:59:35 -0700 Subject: [PATCH] consensus/bor, eth, miner, internal/cli: add sequence store publisher and consumer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Block producers publish each block's lifecycle (open context, per-tx records, sealed header) to the sequence store as it happens; RPC nodes follow the stream, re-execute it deterministically, and hold preconfirmation receipts for the block being built. Design doc: docs/sequencer-bor.md. Producer (eth/sequencer.Publisher, miner hooks): read-before-write follow model — a foreign unsealed window on our tip is followed, not superseded; the only supersede is the seal flush that makes the store match sealed truth. Pre-seal barrier awaits sequencing; the post-seal gate turns store acks into broadcast verdicts (foreign seal refuses, budget expiry broadcasts for liveness after a recheck). Recovery is the reconcile position ladder (anchor, block-anchor probe, floor read) with delta-only re-anchors; producer rotation adopts the dangling window instead of revoking it. Transport hardening: bounded in-flight sends with ack refill, ack-stall watchdog, and a self-heal redial after prolonged channel silence. Consensus-side: a signer outside the active producer set no longer builds at all, so its sequence can never reach the store. Consumer (eth/sequencer.Consumer): follows the gateway stream with warm/cold resume, verifies the commitment chain per entry, re-executes on canonical or parked speculative state (author-nil EVM context, speculative BLOCKHASH, EIP-2935), cross-checks seals (context, gas, receipts root, state root), voids-and-skips on divergence, and fills a capped receipt index evicted on canonical import. The RPC read path that serves these receipts ships separately. Everything is gated behind the [sequencer] config section; the role derives from the sealer flag (mining node publishes, non-mining node consumes). With the section unset there is no behavior change. Validated on kurtosis devnets: a 12-phase chaos campaign (store component restarts, pauses, 200s outages, flapping, partitions, producer and heimdall kills) ended with zero store gaps, zero revoked or reordered preconfirmations, and zero absent heights across 859k entries; preconfirmation receipts measured at p50 ~100ms against ~2.5-2.9s canonical inclusion at 4s blocks, byte-consistent with canonical receipts after import. Co-Authored-By: Claude Fable 5 --- consensus/bor/bor.go | 10 +- docs/cli/default_config.toml | 6 + docs/cli/server.md | 8 + eth/backend.go | 129 ++- eth/ethconfig/config.go | 13 + eth/sequencer/adoption.go | 848 +++++++++++++++++ eth/sequencer/adoption_test.go | 901 +++++++++++++++++++ eth/sequencer/audit_test.go | 419 +++++++++ eth/sequencer/barrier.go | 203 +++++ eth/sequencer/classify.go | 848 +++++++++++++++++ eth/sequencer/classify_test.go | 474 ++++++++++ eth/sequencer/consumer.go | 428 +++++++++ eth/sequencer/consumer_test.go | 114 +++ eth/sequencer/debt_test.go | 206 +++++ eth/sequencer/degraded_test.go | 194 ++++ eth/sequencer/derive_test.go | 126 +++ eth/sequencer/entry.go | 185 ++++ eth/sequencer/entry_test.go | 145 +++ eth/sequencer/equivocation_test.go | 514 +++++++++++ eth/sequencer/exec.go | 170 ++++ eth/sequencer/gate.go | 376 ++++++++ eth/sequencer/gate_test.go | 949 ++++++++++++++++++++ eth/sequencer/harness_test.go | 61 ++ eth/sequencer/journal.go | 413 +++++++++ eth/sequencer/metrics.go | 56 ++ eth/sequencer/mirror_test.go | 615 +++++++++++++ eth/sequencer/outage_test.go | 187 ++++ eth/sequencer/probes_test.go | 339 +++++++ eth/sequencer/publish.go | 314 +++++++ eth/sequencer/publisher.go | 339 +++++++ eth/sequencer/publisher_test.go | 607 +++++++++++++ eth/sequencer/reader.go | 513 +++++++++++ eth/sequencer/receipts.go | 166 ++++ eth/sequencer/receipts_test.go | 102 +++ eth/sequencer/reconcile.go | 144 +++ eth/sequencer/reconcile_test.go | 543 +++++++++++ eth/sequencer/stall_test.go | 164 ++++ eth/sequencer/stream.go | 581 ++++++++++++ eth/sequencer/stream_test.go | 136 +++ eth/sequencer/twin_test.go | 391 ++++++++ go.mod | 1 + go.sum | 2 + internal/cli/dumpconfig.go | 1 + internal/cli/server/config.go | 78 +- internal/cli/server/config_test.go | 69 ++ internal/cli/server/flags.go | 29 + internal/cli/server/sequencer_flags_test.go | 104 +++ miner/fake_miner.go | 12 +- miner/miner.go | 90 ++ miner/worker.go | 446 ++++++++- miner/worker_finality_test.go | 135 +++ miner/worker_sequencer_test.go | 814 +++++++++++++++++ miner/worker_test.go | 62 +- miner/worker_twin_test.go | 113 +++ 54 files changed, 14817 insertions(+), 76 deletions(-) create mode 100644 eth/sequencer/adoption.go create mode 100644 eth/sequencer/adoption_test.go create mode 100644 eth/sequencer/audit_test.go create mode 100644 eth/sequencer/barrier.go create mode 100644 eth/sequencer/classify.go create mode 100644 eth/sequencer/classify_test.go create mode 100644 eth/sequencer/consumer.go create mode 100644 eth/sequencer/consumer_test.go create mode 100644 eth/sequencer/debt_test.go create mode 100644 eth/sequencer/degraded_test.go create mode 100644 eth/sequencer/derive_test.go create mode 100644 eth/sequencer/entry.go create mode 100644 eth/sequencer/entry_test.go create mode 100644 eth/sequencer/equivocation_test.go create mode 100644 eth/sequencer/exec.go create mode 100644 eth/sequencer/gate.go create mode 100644 eth/sequencer/gate_test.go create mode 100644 eth/sequencer/harness_test.go create mode 100644 eth/sequencer/journal.go create mode 100644 eth/sequencer/metrics.go create mode 100644 eth/sequencer/mirror_test.go create mode 100644 eth/sequencer/outage_test.go create mode 100644 eth/sequencer/probes_test.go create mode 100644 eth/sequencer/publish.go create mode 100644 eth/sequencer/publisher.go create mode 100644 eth/sequencer/publisher_test.go create mode 100644 eth/sequencer/reader.go create mode 100644 eth/sequencer/receipts.go create mode 100644 eth/sequencer/receipts_test.go create mode 100644 eth/sequencer/reconcile.go create mode 100644 eth/sequencer/reconcile_test.go create mode 100644 eth/sequencer/stall_test.go create mode 100644 eth/sequencer/stream.go create mode 100644 eth/sequencer/stream_test.go create mode 100644 eth/sequencer/twin_test.go create mode 100644 internal/cli/server/sequencer_flags_test.go create mode 100644 miner/worker_finality_test.go create mode 100644 miner/worker_sequencer_test.go create mode 100644 miner/worker_twin_test.go diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 52d8f53839..553ce3e78d 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -1129,12 +1129,14 @@ func (c *Bor) Prepare(chain consensus.ChainHeaderReader, header *types.Header, w var succession int // if signer is not empty if currentSigner.signer != (common.Address{}) { + // A signer outside the active set (post-Rio: outside the span's + // producer set) does not build at all — its candidate could never + // seal, and its sequence must never reach the store. Nodes without + // a signer (RPC) skip this check and keep their pending snapshot + // fresh. succession, err = snap.GetSignerSuccessionNumber(currentSigner.signer) if err != nil { - // If the signer is not in the active validator set, use succession 0 - // so that the pending block header is still valid for RPC queries. - // Seal() will independently reject the block if unauthorized. - succession = 0 + return err } } diff --git a/docs/cli/default_config.toml b/docs/cli/default_config.toml index 5ec5b8c01e..1f2d00b74b 100644 --- a/docs/cli/default_config.toml +++ b/docs/cli/default_config.toml @@ -267,3 +267,9 @@ devfakeauthor = false enable-preconfs = false enable-private-tx = false bp-rpc-endpoints = [] + +[sequencer] + enabled = false + publisher-endpoint = "" + consumer-endpoint = "" + poll = "200ms" diff --git a/docs/cli/server.md b/docs/cli/server.md index 37139cb150..72e59f1e8e 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -104,6 +104,14 @@ The ```bor server``` command runs the Bor client. - ```rpc.returndatalimit```: Maximum size (in bytes) a result of an rpc request could have (use 0 for no limits) (default: 100000) +- ```sequencer.consumer-endpoint```: Sequence store consumer service gRPC endpoint (tail reads during reconciliation) + +- ```sequencer.enabled```: Enable the sequence store integration (a mining node publishes the block lifecycle) (default: false) + +- ```sequencer.poll```: Producer txpool poll cadence while a block is open (continuous building); 0 keeps the one-shot fill (default: 200ms) + +- ```sequencer.publisher-endpoint```: Sequence store publisher service gRPC endpoint (publish stream) + - ```snapshot```: Enables the snapshot-database mode (default: true) - ```state.scheme```: Scheme to use for storing ethereum state ('hash' or 'path') (default: path) diff --git a/eth/backend.go b/eth/backend.go index d6414dd0b1..2f7da4655f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -55,6 +55,7 @@ import ( "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/eth/protocols/wit" "github.com/ethereum/go-ethereum/eth/relay" + "github.com/ethereum/go-ethereum/eth/sequencer" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -130,6 +131,11 @@ type Ethereum struct { gasPrice *big.Int etherbase common.Address + // Sequence store integration (design docs/sequencer-bor.md): at most + // one is set, by role. + seqPublisher *sequencer.Publisher // mining node: publishes the block lifecycle + seqConsumer *sequencer.Consumer // RPC node: re-executes the stream for preconf receipts + networkID uint64 netRPCService *ethapi.NetAPI @@ -142,6 +148,45 @@ type Ethereum struct { shutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully } +// attachSequencer wires the sequence store integration by role (design +// docs/sequencer-bor.md): a mining node publishes the block lifecycle, a +// non-mining node follows the stream for preconf receipts. +func (s *Ethereum) attachSequencer(config *ethconfig.Config) error { + switch config.SequencerRole { + case "producer": + if s.miner == nil { + return nil + } + + publisher, err := sequencer.NewPublisher(config.SequencerPublisherEndpoint, + config.SequencerConsumerEndpoint, s.blockchain.Config().ChainID.Uint64(), + config.SequencerPoll, s.blockchain) + if err != nil { + return fmt.Errorf("sequencer publisher: %w", err) + } + + s.seqPublisher = publisher + s.miner.SetSequencer(publisher) + case "consumer": + // A consumer that cannot start (not a bor chain) must not block the + // node: preconfs stay off, everything else runs. + consumer, err := sequencer.NewConsumer(config.SequencerConsumerEndpoint, s.blockchain) + if err != nil { + log.Error("Sequencer consumer disabled", "err", err) + + return nil + } + + s.seqConsumer = consumer + consumer.Start() + case "": + default: + return fmt.Errorf("unknown sequencer role %q", config.SequencerRole) + } + + return nil +} + // New creates a new Ethereum object (including the initialisation of the common Ethereum object), // whose lifecycle will be managed by the provided node. func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { @@ -305,32 +350,30 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { trieJournalDirectory = stack.ResolvePath("triedb") } - var ( - options = &core.BlockChainConfig{ - TrieCleanLimit: config.TrieCleanCache, - NoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - ArchiveMode: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - StateScheme: scheme, - TriesInMemory: config.TriesInMemory, - ChainHistoryMode: config.HistoryMode, - TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), - AddressCacheSizes: config.AddressCacheSizes, - PreloadRateLimit: config.PreloadRateLimit, - VmConfig: vmCfg, - Stateless: config.SyncMode == downloader.StatelessSync, - // Enables file journaling for the trie database. The journal files will be stored - // within the data directory. The corresponding paths will be either: - // - DATADIR/triedb/merkle.journal - // - DATADIR/triedb/verkle.journal - TrieJournalDirectory: trieJournalDirectory, - StateSizeTracking: config.EnableStateSizeTracking, - } - ) + options := &core.BlockChainConfig{ + TrieCleanLimit: config.TrieCleanCache, + NoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + ArchiveMode: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + StateScheme: scheme, + TriesInMemory: config.TriesInMemory, + ChainHistoryMode: config.HistoryMode, + TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), + AddressCacheSizes: config.AddressCacheSizes, + PreloadRateLimit: config.PreloadRateLimit, + VmConfig: vmCfg, + Stateless: config.SyncMode == downloader.StatelessSync, + // Enables file journaling for the trie database. The journal files will be stored + // within the data directory. The corresponding paths will be either: + // - DATADIR/triedb/merkle.journal + // - DATADIR/triedb/verkle.journal + TrieJournalDirectory: trieJournalDirectory, + StateSizeTracking: config.EnableStateSizeTracking, + } checker := whitelist.NewService(chainDb, config.DisableBlindForkValidation, config.MaxBlindForkValidationLimit) @@ -474,6 +517,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.miner.SetPrioAddresses(config.TxPool.Locals) } + if err := eth.attachSequencer(config); err != nil { + return nil, err + } + // 1.14.8: NewOracle function definition was changed to accept (startPrice *big.Int) param. eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, config.GPO, config.Miner.GasPrice) eth.APIBackend.gpo.ProcessCache() @@ -696,13 +743,21 @@ func (s *Ethereum) StopMining() { func (s *Ethereum) IsMining() bool { return s.miner.Mining() } func (s *Ethereum) Miner() *miner.Miner { return s.miner } -func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } -func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } -func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool } -func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool } -func (s *Ethereum) Engine() consensus.Engine { return s.engine } -func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } -func (s *Ethereum) IsListening() bool { return true } // Always listening +func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } +func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } +func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool } +func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool } +func (s *Ethereum) Engine() consensus.Engine { return s.engine } +func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } +func (s *Ethereum) IsListening() bool { return true } // Always listening +// WhitelistedMilestone implements miner.Backend: the newest Heimdall +// milestone the downloader has whitelisted, or false when none has arrived. +// The miner's finality gate compares it against the local chain before a +// producer builds. +func (s *Ethereum) WhitelistedMilestone() (bool, uint64, common.Hash) { + return s.Downloader().ChainValidator.GetWhitelistedMilestone() +} + func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downloader } func (s *Ethereum) Synced() bool { return s.handler.synced.Load() } func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() } @@ -1083,6 +1138,14 @@ func (s *Ethereum) Stop() error { <-ch s.filterMaps.Stop() s.txPool.Close() + if s.seqPublisher != nil { + s.seqPublisher.Close() + } + + if s.seqConsumer != nil { + s.seqConsumer.Close() + } + if s.miner != nil { s.miner.Close() } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index c7643976f3..7fd1cfd358 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -329,7 +329,20 @@ type Config struct { TxSyncDefaultTimeout time.Duration `toml:",omitempty"` TxSyncMaxTimeout time.Duration `toml:",omitempty"` + // Sequence store integration (design docs/sequencer-bor.md). Role is + // derived, not configured: "producer" on a mining node (publishes the + // block lifecycle), "consumer" on a non-mining node (re-executes the + // stream for preconf receipts), empty when disabled. The publisher and + // consumer gRPC services have their own endpoints (the publisher reads + // the tail through the consumer service when it reconciles); Poll is + // the txpool poll cadence while a block is open. + SequencerRole string + SequencerPublisherEndpoint string + SequencerConsumerEndpoint string + SequencerPoll time.Duration + // Preconf / Private transaction relay related settings + EnablePreconfs bool EnablePrivateTx bool BlockProducerRpcEndpoints []string diff --git a/eth/sequencer/adoption.go b/eth/sequencer/adoption.go new file mode 100644 index 0000000000..a330dd9b67 --- /dev/null +++ b/eth/sequencer/adoption.go @@ -0,0 +1,848 @@ +package sequencer + +import ( + "context" + "math/big" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/miner" +) + +// checkTailTimeout bounds the build-start tail read: past it +// the block is built from the pool with the publisher buffering silently. +var checkTailTimeout = 250 * time.Millisecond + +// noHold disables the send loop's send ceiling. +const noHold = ^uint64(0) + +// buildMode is the build treatment for the current height, decided once by +// the build-start classification and consumed by OpenBlock/PublishTx (mute) +// and SealBlock (gate arming). It replaces three formerly loose flags whose +// legal combinations were enforced only by discipline. +type buildMode struct { + kind int + height uint64 // the height the treatment was decided for (modeOpen: unused) +} + +const ( + modeOpen = iota // publish normally + modeMuted // build at/behind a sealed height: publish nothing + modeSealedWait // muted over a store-closed height whose block is still owed + modeRecover // rebuilding a sealed generation the chain never received + // modeOverSealed is muted at a height the store sealed but recovery + // declined: the liveness fallback, whose broadcast the store's standing + // seal must not refuse — refusing every build here would halt the chain + // at this block forever. + modeOverSealed +) + +// keepNone is the keepFrom value for a refold that keeps no suffix: every +// unacked entry is abandoned to supersession convergence. +const keepNone = ^uint64(0) + +// hold is the send loop's send ceiling: entries with seq above after stay +// buffered in the journal. A build-start hold gates only the new build's +// entries (after = last existing seq), so a prior seal flush always +// finishes draining; a mid-block STALE hold gates everything unacked +// (after = ackedSeq). kind decides the release: holdBuild lifts the moment +// the flush it ordered behind is home — batching a window until its own +// seal would defeat mid-block streaming — while holdSticky persists until +// a seal or reconcile resolves the lineage. The cleared state pairs +// noHold with holdNone; use clearedHold(), never the zero value. +type hold struct { + after uint64 + kind int +} + +func clearedHold() hold { return hold{after: noHold, kind: holdNone} } + +func (h hold) active() bool { return h.kind != holdNone } + +func (h hold) gates(seq uint64) bool { return seq > h.after } + +// Send-ceiling kinds. Only the self-release behavior is branched on: +// a build-start hold lifts when the flush it ordered behind drains, while a +// sticky hold (foreign tail, adopt, or mid-block STALE) persists until a +// seal or reconciliation resolves the lineage. +const ( + holdNone = iota + holdBuild + holdSticky +) + +// adoption tracks a store window being adopted — a three-state machine: +// p.adopt == nil (none) -> armed (returned by AdoptWindow, awaiting the +// engaging open; awaitOpen is true exactly then) -> engaged (idx tracks +// the next PublishTx match). It ends by full match, divergence rewind, +// unadopt, or a seal flush. +type adoption struct { + number uint64 + timestamp uint64 + parent common.Hash + gasLimit uint64 + baseFee *big.Int + base commitment.Head // store position the window folds on (its open's prefix) + + txs []*types.Transaction // the window's transactions, decoded from store bytes + idx int // next expected PublishTx match + engaged bool +} + +// buildStartState is the store's relationship to the height about to be +// built, resolved from the parent's seal boundary. +type buildStartState int + +const ( + buildAtBoundary buildStartState = iota // info holds this height's live state + buildSealedPast // the store sealed at or past this height + buildBehind // the store lacks the parent: owed a backfill + buildUnknown // the store could not be read in budget + buildRecover // sealed in the store, absent from the chain + buildSealedWait // sealed in the store, its block still owed to us +) + +// recoverState is how a height sealed in the store but missing from the chain +// should be handled. +type recoverState int + +const ( + recoverNone recoverState = iota // not recoverable: treat as an ordinary sealed height + recoverWait // the block may still arrive; do not rebuild yet + recoverReady // the sealer is presumed dead; rebuild its prefix +) + +// buildStartRead resolves what stands at the height about to be built. It +// anchors at the height itself — probing down for the nearest generation — +// rather than at our own anchor: the anchor-based walk only sees entries +// written after our last confirmed write, and goes blind to a live window +// after a mid-window rebase. This read cannot. +// +// The probe finds generations, sealed or live (the store's block index +// serves both), so the walk disambiguates: a live generation is served from +// its open, and a sealed one from just past its seal. +func (p *Publisher) buildStartRead(ctx context.Context, number uint64) (tailInfo, buildStartState) { + h, found, err := p.read.probeDown(ctx, number) + if err != nil { + return tailInfo{}, buildUnknown + } + + if !found { + // No generation at or below this height. An empty store is a clean + // boundary — a fresh chain, or an operator wipe, where the open + // forward-jumps from the seed by design. A non-empty store without + // our parent is behind. + info, out := p.read.floorRead(ctx) + if out != recOK { + return tailInfo{}, buildUnknown + } + + if number <= 1 || (!info.haveSeal && !info.tipOpen && len(info.window) == 0) { + return info, boundaryOrHold(info) + } + + return info, buildBehind + } + + if h > number { + return tailInfo{}, buildSealedPast + } + + if h < number-1 { + return tailInfo{haveSeal: true, lastSealHeight: h}, buildBehind + } + + // h is our height or our parent's. Walk from its boundary: a live + // generation at h is served from its open, so the window is in view. + info, out, done := p.tryWalk(ctx, blockReq(h), false) + if !done || out != recOK { + return tailInfo{}, buildUnknown + } + + switch { + case info.tipOpen && info.tipOpenHeight == number: + // A live window at our height: the adopt case. + return info, buildAtBoundary + + case info.tipOpen && info.tipOpenHeight < number: + // A dangling unsealed window at our parent's height: its producer + // died before the seal flushed. The store is owed the canonical + // parent — the backfill supersedes the dangling window with it. + return info, buildBehind + + case h == number || info.tipOpen || + (info.haveSeal && info.lastSealHeight >= number): + // Our height is sealed, or the store's newest window is past us: + // the height is closed. Unless the chain never received that block — + // then its producer sealed and died before broadcasting, and this + // build recovers the height instead of muting it away forever. + switch rec, st := p.recoverSealed(ctx, number); st { + case recoverReady: + return rec, buildRecover + case recoverWait: + return info, buildSealedWait + } + + return info, buildSealedPast + } + + // h == number-1, sealed, nothing after it: a clean boundary. Carry the + // probe's knowledge — the parent is sealed — for the dead-build discard + // and the sealed-tip bookkeeping. + if !info.haveSeal { + info.haveSeal, info.lastSealHeight = true, number-1 + } + + return info, boundaryOrHold(info) +} + +// boundaryOrHold refuses to mint a generation on a head we cannot derive. +// +// A boundary read says "nothing stands at this height" — but a head taken on +// the store's word says nothing of the sort, because we never saw what +// produced it. Opening there is how a second generation lands on top of +// another producer's live window: the CAS passes, since passing only means +// we echoed the value we were handed. Holding costs this block's records; +// opening blind costs the other producer's. +func boundaryOrHold(info tailInfo) buildStartState { + if !info.explained { + readUnexplained.Inc(1) + log.Warn("Sequencer holding build: store head could not be derived from entries read") + + return buildUnknown + } + + return buildAtBoundary +} + +// recoverGrace is how long a height sealed in the store may stay missing from +// the chain before another build rebuilds it. One block period: long enough +// that an in-flight broadcast wins the race, short enough that a dead sealer +// costs one slot rather than the chain. +const recoverGrace = 4 * time.Second + +// recoverSealed reconstructs a sealed generation the chain never received. +// +// A producer that got its seal acked and then died leaves the height closed +// in the store and empty on the chain: muting there strands it forever, and +// building fresh content there would orphan every record the dead producer +// already had acked — those are preconfirmations, and the store is the proof +// they were issued. So the recovery build must carry exactly that prefix, +// which means reading the generation back out of the store. +// +// Only the content is recovered. The store already holds the complete +// generation, seal included, so the rebuild publishes nothing — the caller +// mutes it. +func (p *Publisher) recoverSealed(ctx context.Context, number uint64) (tailInfo, recoverState) { + if p.chain == nil || p.chain.GetCanonicalHash(number) != (common.Hash{}) { + return tailInfo{}, recoverNone // the chain has the block: an ordinary loss + } + + entries, err := p.read.generation(ctx, number) + if err != nil { + return tailInfo{}, recoverNone + } + + if len(entries) == 0 || entries[0].GetBlockOpen() == nil { + return tailInfo{}, recoverNone + } + + // Give the sealer its block period to broadcast before rebuilding on its + // behalf: the block may simply still be in flight. Measuring from the + // block's own timestamp rather than from now means a build that starts + // late does not add another period of waiting. + if ts := entries[0].GetBlockOpen().GetBlockTimestamp(); ts != 0 && + time.Now().Before(time.Unix(int64(ts), 0).Add(recoverGrace)) { + return tailInfo{}, recoverWait + } + + // Everything up to (not including) the seal is the window; the seal + // itself stays the store's, not ours to reissue. A generation with no + // seal is a live window, not a phantom — the probe resolves generations + // rather than sealed blocks, so this is the disambiguation. + window := make([]*pb.Entry, 0, len(entries)) + sealed := false + + for _, e := range entries { + if e.GetBlockSeal() != nil { + sealed = true + + break + } + + window = append(window, e) + } + + if !sealed { + return tailInfo{}, recoverNone + } + + cur := commitment.Head(window[0].GetBlockOpen().GetPrefixCommitment()) + + for _, e := range window { + next, err := foldEntry(cur, e) + if err != nil { + return tailInfo{}, recoverNone + } + + cur = next + } + + open := window[0].GetBlockOpen() + + publishRecoverCount.Inc(1) + log.Warn("Recovering a height sealed in the store but absent from the chain", + "number", number, "records", len(window)-1) + + return tailInfo{ + s: cur, + tipOpen: true, + tipOpenHeight: number, + tipOpenParent: common.BytesToHash(open.GetParentHash()), + window: window, + }, recoverReady +} + +// AdoptWindow is the worker's build-start check, and the whole of Rule 1: +// an open may be published only at a seal boundary. The store elects the +// owner of every height — writes are a CAS on its head — and this read is +// how a build respects the election before its open is even attempted: +// +// sealed at/past this height -> mute (the height is closed) +// live window on our parent -> adopt (the store has an owner; extend it) +// clean boundary -> open (we are the owner-elect; the CAS +// settles the simultaneous-open race) +// store behind our parent -> hold + prime the backfill (outage) +// store unreadable -> hold (build and buffer; the flush repairs) +func (p *Publisher) AdoptWindow(number uint64, parent common.Hash) *miner.AdoptedWindow { + if p.failed.Load() { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), checkTailTimeout) + defer cancel() + + info, state := p.buildStartRead(ctx, number) + + // An armed offer the previous work cycle never consumed (its build died + // before opening) stays servable — without it this read would mistake + // our own absorbed window for a clean boundary. + if w, handled := p.reoffer(ctx, number, parent, info); handled { + return w + } + + p.mu.Lock() + defer p.mu.Unlock() + + // Every non-reoffer classification starts disarmed and unmuted, and a + // resync signal armed during the previous cycle dies here: this fresh + // read supersedes whatever that signal saw, so it must not abort the + // build it just classified. + p.adopt = nil + p.mode = buildMode{} + p.resync = false + + p.advanceStoreSealedTipLocked(info, "build-start read") + + // Lost head race, or a reorg rebuilding sealed ground — no phantom open + // on our own sealed tip. If it seals after all, the flush rebuild + // publishes the complete window. + if number <= p.sealedTip { + return p.muteLocked(number) + } + + // A draining flush is not a foreign tail — gate only the new build's + // entries so the flush finishes delivering first. + if _, flushing := p.pendingFlushLocked(); flushing { + p.holdNewLocked(holdBuild) + + return nil + } + + switch state { + case buildUnknown: + // Blind at build start: build and buffer; the transport goroutine + // keeps probing and the seal flush repairs. + log.Warn("Sequencer could not read the store at build start", "number", number) + p.holdNewLocked(holdBuild) + + return nil + + case buildSealedPast: + w := p.muteLocked(number) + // Recovery declined this store-sealed height (the chain already has + // the block, or its generation cannot be fetched). A build that + // still seals here is the liveness fallback the gate must let + // through. + p.mode.kind = modeOverSealed + + return w + + case buildSealedWait: + // The store closed this height and its block is still owed to us. + // Refusing the broadcast is safe only while the rebuild is still + // coming: once the grace elapses this height recovers properly, so + // the refusal cannot outlive it and strand the chain here. + w := p.muteLocked(number) + p.mode.kind = modeSealedWait + + return w + + case buildRecover: + // Rebuild the dead producer's block exactly: its transactions were + // acked, so they are promised at this height. The store already + // holds the whole generation, so publish nothing — mute — and hand + // the content to the worker to seal and broadcast. + a, _, ok := parseWindow(info) + if !ok || a.number != number || a.parent != parent { + return p.muteLocked(number) + } + + p.mode = buildMode{kind: modeRecover, height: number} + + return offerLocked(a) + + case buildBehind: + // The store is owed every block from its seal edge to our parent. + // Build and buffer: the backfill drains oldest-first, and the live + // window follows only once the store reaches our boundary. + p.primeBackfillLocked(info, number) + p.holdNewLocked(holdBuild) + + return nil + } + + // buildAtBoundary: info is exactly this height's live state. + p.discardLostLocked(info) + + if len(info.window) == 0 { + // The parent's seal is the store head: a clean boundary, and the + // open publishes. Sync our lineage to the head when nothing of ours + // is pending, so the open folds onto the true tip. + if p.head == p.anchor && info.s != p.anchor { + p.rebaseLocked(info.s) + } + + p.hold = clearedHold() + + return nil + } + + // A live window at this height: the store already has an owner. + a, items, ok := parseWindow(info) + if ok && a.number == number && a.parent == parent { + return p.adoptWindowLocked(info, a, items) + } + + // Unparseable, or on a different parent (reorg territory): build + // locally, publish nothing; whoever seals resolves the height. + p.holdNewLocked(holdSticky) + + return nil +} + +// primeBackfillLocked registers the store's gap [edge+1, parent] as owed +// from the chain database. The collapse machinery already tracks blocks +// sealed while the store was down; this covers a process that restarted +// mid-outage, whose collapsed range died with its journal. +func (p *Publisher) primeBackfillLocked(info tailInfo, number uint64) { + if number < 2 || p.chain == nil { + return + } + + // The range is based on what this read decoded, never the accumulated + // storeSealedTip: the tip has carried inferred values, and even a real + // tip proves nothing about heights below it. An undecoded boundary + // (probe inference) includes the boundary height itself — it may be a + // partial delivery, and a duplicate generation is cheaper than a hole. + var from uint64 + + switch { + case info.sealDecoded: + from = info.lastSealHeight + 1 + case info.haveSeal: + from = info.lastSealHeight + } + + if from <= 1 { + // The store's seal edge is unknown; priming from genesis would + // republish the world. Leave it to the collapse machinery. + return + } + + to := number - 1 + + if p.pendingFrom != 0 { + if p.pendingFrom < from { + from = p.pendingFrom + } + + if p.pendingTo > to { + to = p.pendingTo + } + } + + if from > to { + return + } + + p.pendingFrom, p.pendingTo = from, to +} + +// holdNewLocked gates entries appended from here on while letting +// everything already in the journal (a draining flush included) deliver. +func (p *Publisher) holdNewLocked(kind int) { + p.hold = hold{after: p.journal.nextSeq - 1, kind: kind} +} + +// muteLocked silences the coming build: OpenBlock and PublishTx append +// nothing, so the doomed lineage never reaches the journal or the store. +func (p *Publisher) muteLocked(number uint64) *miner.AdoptedWindow { + p.mode = buildMode{kind: modeMuted, height: number} + + publishMutedCount.Inc(1) + log.Debug("Sequencer muting build at sealed height", "number", number, "sealedTip", p.sealedTip) + + return nil +} + +// adoptWindowLocked absorbs an unsealed tail window as confirmed lineage: its +// entries enter the journal as published-and-acked, the head moves to S, and +// the window is handed to the miner. Continuations then publish normally, +// so their acks (or STALEs) settle who owns the height. +func (p *Publisher) adoptWindowLocked(info tailInfo, a *adoption, items []journalItem) *miner.AdoptedWindow { + if abandoned := p.unackedLocked(); abandoned > 0 { + publishDropMeter.Mark(int64(abandoned)) + } + + fresh := newJournal() + fresh.nextSeq = p.journal.nextSeq + + for _, it := range items { + fresh.append(it.entry, it.pre, it.post, it.kind, it.height, fresh.nextSeq, it.txHashes) + } + + p.journal = fresh + p.ackedSeq = fresh.nextSeq - 1 + p.head = info.s + p.anchor = info.s + p.anchored, p.confirmed = true, true + p.curHeight = a.number + p.awaitOpen = true + p.adopt = a + + // No hold: continuations publish immediately. Extending the adopted + // window is how ownership is discovered — an ack means we own this + // height and may seal, a STALE means another producer is still + // writing it and the pre-seal barrier must stop us. Holding here + // would hide both answers and strand a legitimate takeover. + p.hold = clearedHold() + + publishQueueGauge.Update(int64(p.unackedLocked())) + reconcileAdopt.Inc(1) + log.Info("Sequencer adopting store window", "number", a.number, "txs", len(a.txs)) + + return offerLocked(a) +} + +// reoffer re-serves or refreshes an armed, unengaged offer for the same +// build. handled=false falls through to normal classification, disarmed. +func (p *Publisher) reoffer(ctx context.Context, number uint64, parent common.Hash, info tailInfo) (*miner.AdoptedWindow, bool) { + p.mu.Lock() + + a := p.adopt + if a == nil || a.engaged || a.number != number || a.parent != parent { + p.mu.Unlock() + + return nil, false + } + + if info.s == p.head { + defer p.mu.Unlock() + + // The offer is re-served exactly as a fresh adoption would be: any + // hold armed since (a STALE from the death throes of the previous + // cycle) is residue of a lineage this adoption replaced, and keeping + // it would gate the continuations that discover ownership. + p.hold = clearedHold() + + return offerLocked(a), true // window unchanged since absorption + } + + p.mu.Unlock() + + return p.readoptGrown(ctx, a, number, parent) +} + +// readoptGrown re-reads a window that kept growing after our snapshot from +// its base and re-adopts it in full — sealing the stale snapshot would +// supersede the extra records at the flush. Any failure (read miss, the +// lineage moved while unlocked, the window is no longer ours) falls +// through to normal classification, disarmed. Reading a.base unlocked is +// race-free: it is written once, before p.adopt publishes it. +func (p *Publisher) readoptGrown(ctx context.Context, a *adoption, number uint64, parent common.Hash) (*miner.AdoptedWindow, bool) { + fresh, out, done := p.tryWalk(ctx, headReq(a.base), false) + if !done || out != recOK { + return nil, false + } + + p.mu.Lock() + defer p.mu.Unlock() + + // Bail if the lineage moved while unlocked (a seal flush landing). + if p.adopt != a || a.engaged { + return nil, false + } + + if a2, items, ok := parseWindow(fresh); ok && a2.number == number && a2.parent == parent { + return p.adoptWindowLocked(fresh, a2, items), true + } + + return nil, false +} + +// offerLocked builds the miner's view of an adopted window. +func offerLocked(a *adoption) *miner.AdoptedWindow { + return &miner.AdoptedWindow{ + Number: a.number, + Timestamp: a.timestamp, + ParentHash: a.parent, + GasLimit: a.gasLimit, + BaseFee: new(big.Int).Set(a.baseFee), + Txs: a.txs, + } +} + +// discardLostLocked drops the unacked suffix when a foreign seal has +// covered all of it, re-anchoring on the store head. Entries above the +// sealed tip (an outage buffer awaiting journal replay) are never touched. +func (p *Publisher) discardLostLocked(info tailInfo) { + lost := p.unackedLocked() + if lost == 0 || !info.haveSeal { + return + } + + for _, it := range p.journal.items { + if it.seq > p.ackedSeq && it.height > info.lastSealHeight { + return + } + } + + publishDropMeter.Mark(int64(lost)) + p.rebaseLocked(info.s) + publishQueueGauge.Update(0) + log.Debug("Sequencer discarded superseded buffer", "entries", lost, "sealedTip", info.lastSealHeight) +} + +// unadoptLocked drops an absorbed-but-abandoned adopt back to the +// window's base: the dead window leaves the journal and the next open folds +// onto the base, so a divergent local build supersedes the incumbent +// window through the normal STALE→reconcile path. +func (p *Publisher) unadoptLocked(a *adoption) { + p.rebaseLocked(a.base) + p.adopt = nil + p.awaitOpen = false + p.curHeight = 0 + p.hold = clearedHold() +} + +// rebaseLocked re-anchors an idle lineage (nothing unconfirmed) onto a +// fresh store head, so the next open extends the true tip. It is the +// primitive under three resets of increasing scope — callers own what it +// deliberately does not touch (adopt, hold, curHeight/awaitOpen): +// +// rebaseLocked journal/head/anchor only (clean tail; between-blocks anchor) +// discardLostLocked + drop metric; onto the store head; hold set by the caller after +// unadoptLocked + clears adopt, awaitOpen, curHeight, hold; onto the WINDOW BASE, +// so the divergent open STALEs onto the counted reconcile path +// installSwapLocked (classify.go) full swap with refolded content; re-derives the window +// dropRefusedFlushLocked (publisher.go) the rewind-side sibling: truncates a refused +// flush off the tail instead of re-anchoring under it +func (p *Publisher) rebaseLocked(s commitment.Head) { + fresh := newJournal() + fresh.nextSeq = p.journal.nextSeq + + p.journal = fresh + p.ackedSeq = fresh.nextSeq - 1 + p.head = s + p.anchor = s + p.anchored, p.confirmed = true, true +} + +// parseWindow decodes a collected tail window into an adoption candidate and +// the journal items representing its entries, verifying the fold chain ends +// at the store head. +func parseWindow(info tailInfo) (*adoption, []journalItem, bool) { + if len(info.window) == 0 { + return nil, nil, false + } + + open := info.window[0].GetBlockOpen() + if open == nil { + return nil, nil, false + } + + a := &adoption{ + number: open.GetBlockNumber(), + timestamp: open.GetBlockTimestamp(), + parent: common.BytesToHash(open.GetParentHash()), + gasLimit: open.GetGasLimit(), + baseFee: new(big.Int).SetBytes(open.GetBaseFee()), + base: commitment.Head(open.GetPrefixCommitment()), + } + + items := make([]journalItem, 0, len(info.window)) + cur := commitment.Head(info.window[0].GetBlockOpen().GetPrefixCommitment()) + + for _, entry := range info.window { + next, err := foldEntry(cur, entry) + if err != nil { + return nil, nil, false + } + + kind := entryOpen + + var hashes []common.Hash + + if rec := entry.GetRecord(); rec != nil { + kind = entryRecord + + for _, raw := range rec.GetTransactions() { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + return nil, nil, false + } + + a.txs = append(a.txs, tx) + hashes = append(hashes, tx.Hash()) + } + } + + items = append(items, journalItem{entry: entry, pre: cur, post: next, kind: kind, height: a.number, txHashes: hashes}) + cur = next + } + + if cur != info.s { + return nil, nil, false + } + + return a, items, true +} + +// matchesOpen reports whether the worker's open reproduces the adopted +// window's context exactly — any mismatch means the worker rejected the +// inherited context. +func (a *adoption) matchesOpen(number, timestamp uint64, parent common.Hash, gasLimit uint64, baseFee *big.Int) bool { + return a.number == number && a.timestamp == timestamp && + a.parent == parent && a.gasLimit == gasLimit && + baseFee != nil && a.baseFee.Cmp(baseFee) == 0 +} + +// adoptOpenLocked resolves an adopted window against the worker's +// OpenBlock. swallow=true means the open engaged the window and must not +// be appended; false drops the adopt and the open proceeds (buffered: +// hold stays until the seal flush). +func (p *Publisher) adoptOpenLocked(number uint64, timestamp uint64, parent common.Hash, gasLimit uint64, baseFee *big.Int) (swallow bool) { + a := p.adopt + if a == nil { + return false + } + + if a.engaged || !a.matchesOpen(number, timestamp, parent, gasLimit, baseFee) { + // The build diverged from the adopted window — the worker + // rejected the inherited context (a bogus or version-skewed + // incumbent window; honest producers derive the same context). + // Undo the absorption back to the window's base so the local open + // folds there and STALEs against the store's incumbent window: + // the supersede then lands on the counted reconcile path + // instead of appending a silent second generation on the store + // head that no reconcile ever sees. + p.unadoptLocked(a) + + return false + } + + a.engaged = true + p.awaitOpen = false + p.curHeight = number + + return true +} + +// adoptTxLocked matches one committed transaction against the adopted +// window. swallow=true: the transaction is already in the store. A +// mismatch (the worker dropped or replaced a window transaction) rewinds +// the lineage to the matched prefix — the dead window tail leaves the +// journal, the head returns to the prefix, and subsequent commits fold onto +// it, buffered; the seal flush resolves the divergence. Nothing is +// published here. +func (p *Publisher) adoptTxLocked(hash common.Hash) (swallow bool) { + a := p.adopt + if a == nil || !a.engaged { + return false + } + + // Only reachable for an empty adopted window: a full match clears + // p.adopt eagerly below. + if a.idx >= len(a.txs) { + p.adopt = nil + + return false + } + + if a.txs[a.idx].Hash() == hash { + a.idx++ + if a.idx == len(a.txs) { + p.adopt = nil // fully adopted; continuations buffer normally + } + + return true + } + + log.Warn("Sequencer adopted window diverged, deferring to the seal flush", + "number", a.number, "matched", a.idx, "of", len(a.txs)) + + p.rewindToPrefixLocked(a.idx) + p.adopt = nil + + return false // the divergent tx folds onto the rewound prefix, buffered +} + +// rewindToPrefixLocked drops the adopted window's unmatched tail from the +// journal and returns the fold head to the end of the matched prefix. At +// call time the journal is exactly the absorbed window (matches append +// nothing; a re-open while engaged unadopts first), so the scan finds the +// window's open first. +func (p *Publisher) rewindToPrefixLocked(matched int) { + cut := -1 + seen := 0 + + for i, it := range p.journal.items { + if it.kind == entryOpen && it.height == p.curHeight { + cut = i // at minimum, keep the window's open + } + + if it.kind != entryRecord { + continue + } + + seen += len(it.entry.GetRecord().GetTransactions()) + if seen >= matched && matched > 0 { + cut = i + + break + } + } + + if cut < 0 { + return + } + + p.rewindJournalLocked(cut + 1) + p.ackedSeq = p.journal.items[cut].seq +} diff --git a/eth/sequencer/adoption_test.go b/eth/sequencer/adoption_test.go new file mode 100644 index 0000000000..6e3635d99b --- /dev/null +++ b/eth/sequencer/adoption_test.go @@ -0,0 +1,901 @@ +package sequencer + +import ( + "math/big" + "testing" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// foreignWindow appends a dangling open window (open + txs records) to the +// store, as an incumbent producer's unsealed work. +func foreignWindow(t *testing.T, h *harness, number uint64, parent common.Hash, txs ...*types.Transaction) (uint64, uint64) { + t.Helper() + + ts := 1700000000 + number + gasLimit := uint64(30_000_000) + + entry := &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: number, + BlockTimestamp: ts, + ParentHash: parent.Bytes(), + GasLimit: gasLimit, + BaseFee: big25gwei(), + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(entry); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign open rejected: %v", status) + } + + for _, tx := range txs { + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + rec := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(rec); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign record rejected: %v", status) + } + } + + return ts, gasLimit +} + +func fee25() *big.Int { + return new(big.Int).SetBytes(big25gwei()) +} + +func windowHeader(number uint64, parent common.Hash, ts uint64, gasLimit uint64) *types.Header { + return &types.Header{ + ParentHash: parent, + Number: new(big.Int).SetUint64(number), + GasLimit: gasLimit, + Time: ts, + BaseFee: fee25(), + Difficulty: big.NewInt(1), + } +} + +// The flagship adopt path: the build-start check finds the incumbent's +// window and adopts it — engage swallows the open, matched transactions +// publish nothing, continuations stay buffered — and the seal flush +// completes the window in place. No supersession anywhere. +func TestFollowCompletesWindowAtSeal(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1, tx2 := testTx(t, 0), testTx(t, 1) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1, tx2) + + adopts := reconcileAdopt.Snapshot().Count() + supersedes := reconcileSupersede.Snapshot().Count() + headBefore := h.store.Head() + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 2 || w.Timestamp != ts || w.GasLimit != gasLimit { + t.Fatalf("AdoptWindow = %+v", w) + } + + if got := reconcileAdopt.Snapshot().Count(); got != adopts+1 { + t.Fatalf("adopt counter = %d, want %d", got, adopts+1) + } + + // The worker builds under the adopted context, re-committing the + // window's transactions: those are matched and swallowed, so the + // store does not move for them. + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + p.PublishTx(w.Txs[1]) + + time.Sleep(100 * time.Millisecond) + + if h.store.Head() != headBefore { + t.Fatal("re-committed window transactions must publish nothing") + } + + // A continuation extends the adopted window immediately — its ack is + // what proves this node owns the height and may seal it. + tx3 := testTx(t, 2) + p.PublishTx(tx3) + waitHead(t, h, p, 5*time.Second) + + // The seal completes the window in place. + header := windowHeader(2, parent, ts, gasLimit) + p.SealBlock(blockFor(header, []*types.Transaction{w.Txs[0], w.Txs[1], tx3})) + waitHead(t, h, p, 5*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("adopted block missing: %v", err) + } + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("adopt completion superseded: counter %d -> %d", supersedes, got) + } +} + +// A worker that drops a window transaction (divergence) publishes nothing +// at the moment of divergence; the seal flush re-anchors — the only +// supersession — and the store converges on the sealed content. +func TestFollowDivergenceResolvesAtSeal(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1, tx2 := testTx(t, 0), testTx(t, 1) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1, tx2) + + headBefore := h.store.Head() + + w := p.AdoptWindow(2, parent) + if w == nil { + t.Fatal("no window to adopt") + } + + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + + other := testTx(t, 7) // tx2 failed to apply; the worker committed another + p.PublishTx(other) + + time.Sleep(100 * time.Millisecond) + + if h.store.Head() != headBefore { + t.Fatal("divergence published before sealing") + } + + header := windowHeader(2, parent, ts, gasLimit) + p.SealBlock(blockFor(header, []*types.Transaction{w.Txs[0], other})) + waitHead(t, h, p, 10*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("sealed block missing after divergence flush: %v", err) + } +} + +// A build that raced the window under its own context (the check returned +// the window but the worker opened differently) buffers silently and the +// seal flush re-anchors to the sealed truth. +func TestFollowMismatchedOpenResolvesAtSeal(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if w := p.AdoptWindow(2, parent); w == nil { + t.Fatal("no window offered") + } + + supersedes := reconcileSupersede.Snapshot().Count() + + // Own context: the adopt disengages, un-adopts back to the window's + // base, and the divergent local build supersedes the incumbent window. + ownTs := uint64(1600000099) + p.OpenBlock(2, ownTs, parent, 30_000_000, fee25()) + tx := testTx(t, 3) + p.PublishTx(tx) + + header := windowHeader(2, parent, ownTs, 30_000_000) + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + waitHead(t, h, p, 10*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("sealed block missing after flush: %v", err) + } + + // The supersede of the incumbent's window is counted, not silent + // (finding: divergent takeover on the clean-append path escaped the + // metric). + if got := reconcileSupersede.Snapshot().Count(); got != supersedes+1 { + t.Fatalf("divergent takeover supersede uncounted: %d -> %d", supersedes, got) + } +} + +// A window at a height the build is not at (chain moved) is not adopted: +// the build buffers and the flush repairs. +func TestCheckTailForeignHeightHolds(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if w := p.AdoptWindow(3, common.Hash{0x99}); w != nil { + t.Fatal("mismatched build must not receive the window") + } + + p.mu.Lock() + ceiling := p.hold.after + next := p.journal.nextSeq + p.mu.Unlock() + + if ceiling == noHold || ceiling != next-1 { + t.Fatalf("foreign window must gate new entries (ceiling=%d next=%d)", ceiling, next) + } +} + +// A record-less window (open only) is adopted with an empty seed; the +// first pool transaction buffers as a continuation and the flush completes +// the window. +func TestFollowEmptyWindow(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + ts, gasLimit := foreignWindow(t, h, 2, parent) // zero records + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 0 { + t.Fatalf("AdoptWindow = %+v, want empty window", w) + } + + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + tx := testTx(t, 0) + p.PublishTx(tx) + + header := windowHeader(2, parent, ts, gasLimit) + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + waitHead(t, h, p, 5*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("adopted block missing: %v", err) + } +} + +// A clean tail (no window) publishes normally — the incumbent path. +func TestCheckTailCleanPublishes(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatalf("clean tail returned a window: %+v", w) + } + + publishBlock(t, p, 2, parent, 1) + waitHead(t, h, p, 5*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("incumbent block missing: %v", err) + } +} + +// A seal arriving while an adopted window stands but was never engaged +// (the build sealed under its own steam) is never dropped: the flush +// rebuilds the window from the block and the store converges. +func TestSealNeverDropped(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if w := p.AdoptWindow(2, parent); w == nil { + t.Fatal("no window offered") + } + + // No OpenBlock at all — the straggler seal arrives first. + header := testHeader(2, parent) + tx := testTx(t, 5) + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + waitHead(t, h, p, 10*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("straggler seal never landed: %v", err) + } +} + +// A build-start check must never strangle the previous block's draining +// seal flush: the send ceiling gates only the new build's entries, so a +// flush buffered during an outage still delivers once the store returns — +// even though the next build's check ran (and held) in between. +func TestBuildStartHoldDoesNotBlockFlush(t *testing.T) { + restore := checkTailTimeout + checkTailTimeout = 100 * time.Millisecond + + t.Cleanup(func() { checkTailTimeout = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Store goes down; block 2 is built and sealed entirely offline — the + // flush entries sit in the journal, undeliverable. + h.stop() + + parent := sealHash(t, sealed) + sealed2 := publishBlock(t, p, 2, parent, 2) + + // The next build's check runs while the flush is still pending (store + // unreachable): it must gate only future entries. + if w := p.AdoptWindow(3, sealed2.Hash()); w != nil { + t.Fatalf("unreachable store returned a window: %+v", w) + } + + // Store returns: the pending flush must drain without any further + // publisher calls. + h.resume() + waitHead(t, h, p, 15*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("flush strangled by build-start hold: %v", err) + } +} + +// An offer whose build dies before opening (rotation churn) is re-served +// at the next work cycle: the absorbed window must be resumed, never +// mistaken for a clean tail and replaced by a fresh generation. +func TestUnconsumedOfferReoffered(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx := testTx(t, 0) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx) + + // First work cycle takes the offer and dies before OpenBlock. + if w := p.AdoptWindow(2, parent); w == nil { + t.Fatal("no window offered") + } + + supersedes := reconcileSupersede.Snapshot().Count() + + // The replacement build for the same height must get the offer again. + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 || w.Timestamp != ts { + t.Fatalf("unconsumed offer not re-served: %+v", w) + } + + // This time the build engages and completes the window in place. + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + p.SealBlock(blockFor(windowHeader(2, parent, ts, gasLimit), []*types.Transaction{w.Txs[0]})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("resumed window superseded: %d -> %d", supersedes, got) + } +} + +// An armed offer whose window kept growing after the snapshot (the +// incumbent streamed more records before dying) is re-read from its base +// and re-adopted in full — sealing the stale snapshot would supersede +// the extra records at the flush. +func TestGrownWindowReadoptedNotStale(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1 := testTx(t, 0) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1) + + // Snapshot taken at one record; the build dies before opening. + if w := p.AdoptWindow(2, parent); w == nil || len(w.Txs) != 1 { + t.Fatalf("first offer wrong: %+v", w) + } + + // The incumbent streams one more record before dying. + tx2 := testTx(t, 1) + raw, err := tx2.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + rec := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: h.store.Head().Bytes(), + }}} + if status := h.store.Append(rec); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("grown record rejected: %v", status) + } + + supersedes := reconcileSupersede.Snapshot().Count() + + // The replacement build must receive the FULL grown window. + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 2 { + t.Fatalf("grown window not re-adopted: %+v", w) + } + + // Engage and complete in place: both records already in the store. + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + p.PublishTx(w.Txs[1]) + p.SealBlock(blockFor(windowHeader(2, parent, ts, gasLimit), []*types.Transaction{w.Txs[0], w.Txs[1]})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("grown-window resume superseded: %d -> %d", supersedes, got) + } +} + +// A producer restarted mid-window resumes it (self-adoption): with no +// persisted position, the fresh publisher relocates the store tail from +// the chain's last imported block, the startup reconcile anchors at the +// window's base, the first build-start check collects and adopts it, and +// the seal completes the same generation — its own preconfs never revoked. +func TestRestartResumesOwnWindow(t *testing.T) { + h := startHarness(t) + + first, err := NewPublisher(h.addr, h.addr, testChainID, 0, nil) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + sealed := publishBlock(t, first, 1, common.Hash{0xef}, 1) + waitHead(t, h, first, 5*time.Second) + + // Mid-window death: open 2 and stream one record, then die unsealed. + parent := sealHash(t, sealed) + tx := testTx(t, 0) + first.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + first.PublishTx(tx) + waitHead(t, h, first, 5*time.Second) + first.Close() + + // A fresh publisher with block 1 as its chain head: the restart probe + // locates the store tail from that height — no local state carried over. + chain := &fakeChain{current: &types.Header{Number: big.NewInt(1)}} + + p, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + waitFor(t, 5*time.Second, func() bool { return p.isAnchored() }) + + supersedes := reconcileSupersede.Snapshot().Count() + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 || w.Txs[0].Hash() != tx.Hash() { + t.Fatalf("restart did not resume own window: %+v", w) + } + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(w.Txs[0]) + p.SealBlock(blockFor(windowHeader(2, parent, 1700000002, 30_000_000), []*types.Transaction{w.Txs[0]})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("self-resume superseded own window: %d -> %d", supersedes, got) + } +} + +// A build-start hold lifts the moment the flush it ordered behind is +// drained: the gated window then streams mid-block instead of batching +// until its own seal — under continuous load a seal-persistent hold +// would ratchet (each flush still draining at the next build's check) +// and turn preconf streaming into at-seal batching. +func TestBuildStartHoldReleasesOnDrain(t *testing.T) { + restore := checkTailTimeout + checkTailTimeout = 100 * time.Millisecond + + t.Cleanup(func() { checkTailTimeout = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Block 2 seals while the store is down: its flush sits in the journal. + h.stop() + + parent := sealHash(t, sealed) + sealed2 := publishBlock(t, p, 2, parent, 1) + + // Build 3 starts against the unreachable store: its window is gated + // behind the pending flush. + if w := p.AdoptWindow(3, sealed2.Hash()); w != nil { + t.Fatalf("unreachable store returned a window: %+v", w) + } + + p.OpenBlock(3, sealed2.Time+2, sealed2.Hash(), sealed2.GasLimit, fee25()) + p.PublishTx(testTx(t, 3)) + + // The store returns: the flush drains, the hold lifts, and block 3's + // window streams to the store with no SealBlock in sight. + h.resume() + waitHead(t, h, p, 15*time.Second) + + p.mu.Lock() + tail := p.journal.items[len(p.journal.items)-1] + held := p.hold.after != noHold + p.mu.Unlock() + + if tail.kind != entryRecord || tail.height != 3 || held { + t.Fatalf("mid-block window not streaming: tail kind=%d height=%d held=%v", tail.kind, tail.height, held) + } +} + +// A build racing its own chain-head update (its height already sealed and +// flushed) is muted: no phantom open dangles on the sealed tip, and the +// next legitimate build publishes normally. +func TestStaleBuildPublishesNothing(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + headBefore := h.store.Head() + muted := publishMutedCount.Snapshot().Count() + + // The worker re-prepares height 1 before the chain head catches up. + if w := p.AdoptWindow(1, common.Hash{0xef}); w != nil { + t.Fatalf("stale build received a window: %+v", w) + } + + if got := publishMutedCount.Snapshot().Count(); got != muted+1 { + t.Fatalf("muted counter = %d, want %d", got, muted+1) + } + + p.OpenBlock(1, sealed.Time, common.Hash{0xef}, sealed.GasLimit, fee25()) + p.PublishTx(testTx(t, 7)) + + p.mu.Lock() + pending := p.journal.nextSeq - 1 - p.ackedSeq + p.mu.Unlock() + + if pending != 0 { + t.Fatalf("muted build appended %d entries", pending) + } + + if h.store.Head() != headBefore { + t.Fatal("muted build reached the store") + } + + // The interrupted build gives way to the real next height. + parent := sealHash(t, sealed) + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatalf("clean tail returned a window: %+v", w) + } + + publishBlock(t, p, 2, parent, 1) + waitHead(t, h, p, 5*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("post-mute block missing: %v", err) + } +} + +// A builder whose published window lost to a foreign seal discards the +// dead buffer at the next work cycle instead of re-STALEing it +// forever, and its next open folds onto the foreign store head. +func TestBuildStartDiscardsSupersededBuffer(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Our window for 2 folds locally but the store never sees it (the + // foreign producer owns the height); its content stays unacked. + parent := sealHash(t, sealed) + h.stop() + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + + // The foreign producer's block 2 lands in the store, sealed. + h.resume() + ts, gasLimit := foreignWindow(t, h, 2, parent, testTx(t, 1)) + appendForeignSeal(t, h, windowHeader(2, parent, ts, gasLimit)) + + p.mu.Lock() + buffered := p.journal.nextSeq - 1 - p.ackedSeq + p.mu.Unlock() + + if buffered == 0 { + t.Fatal("test setup: nothing buffered") + } + + // Next work cycle: the read shows the foreign sealed tip covering our + // whole buffer — it is discarded and the lineage re-anchors. + if w := p.AdoptWindow(3, common.Hash{0x33}); w != nil { + t.Fatalf("AdoptWindow = %+v", w) + } + + p.mu.Lock() + left := p.journal.nextSeq - 1 - p.ackedSeq + rebased := p.head == h.store.Head() && p.anchor == p.head + p.mu.Unlock() + + if left != 0 || !rebased { + t.Fatalf("buffer not discarded: unacked=%d rebased=%v", left, rebased) + } + + // The new build publishes cleanly onto the foreign head. + publishBlock(t, p, 3, common.Hash{0x33}, 1) + waitHead(t, h, p, 5*time.Second) +} + +// A muted build that seals after all (a reorg win) loses nothing: the +// flush rebuilds the complete window from the block — the designed +// supersede-by-seal. +func TestStaleBuildSealFlushRebuilds(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + if w := p.AdoptWindow(1, common.Hash{0xef}); w != nil { + t.Fatalf("stale build received a window: %+v", w) + } + + tx := testTx(t, 9) + p.OpenBlock(1, sealed.Time+2, common.Hash{0xef}, sealed.GasLimit, fee25()) + p.PublishTx(tx) + + header := testHeader(1, common.Hash{0xef}) + header.Time = sealed.Time + 2 + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + waitHead(t, h, p, 10*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 1}); err != nil { + t.Fatalf("reorg seal never landed: %v", err) + } +} + +// A adopt snapshot raced by the dying producer's draining stream: the +// store window gains records after our absorption, so the flush STALEs — +// but the store's copy is a strict prefix of the sealed content, and the +// flush completes it in place. Same generation, nothing revoked. +func TestFlushCompletesExtendedWindowInPlace(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1, tx2 := testTx(t, 0), testTx(t, 1) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1) + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 { + t.Fatalf("offer: %+v", w) + } + + // The dying producer's last in-flight record lands after our snapshot. + raw, err := tx2.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + rec := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: h.store.Head().Bytes(), + }}} + if st := h.store.Append(rec); st != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("extension record rejected: %v", st) + } + + supersedes := reconcileSupersede.Snapshot().Count() + + // The build seeds the snapshot, then commits the extension tx (it is + // in our pool too) and a continuation, and seals all three. + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + p.PublishTx(tx2) + + tx3 := testTx(t, 2) + p.PublishTx(tx3) + p.SealBlock(blockFor(windowHeader(2, parent, ts, gasLimit), []*types.Transaction{w.Txs[0], tx2, tx3})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("extended-window completion superseded: %d -> %d", supersedes, got) + } + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("completed block missing: %v", err) + } +} + +// A adopt inside a live send loop session must not re-send the absorbed +// window: those entries are already in the store, and re-sending them +// guarantees a STALE plus a spurious gap-fill on every rotation adopt. +func TestFollowDoesNotResendAbsorbed(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1 := testTx(t, 0) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1) + + stales := publishStaleCount.Snapshot().Count() + gapfills := reconcileGapfill.Snapshot().Count() + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 { + t.Fatalf("offer: %+v", w) + } + + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + + tx2 := testTx(t, 1) + p.PublishTx(tx2) + p.SealBlock(blockFor(windowHeader(2, parent, ts, gasLimit), []*types.Transaction{w.Txs[0], tx2})) + waitHead(t, h, p, 10*time.Second) + + if got := publishStaleCount.Snapshot().Count(); got != stales { + t.Fatalf("adopt re-sent absorbed entries: stale %d -> %d", stales, got) + } + + if got := reconcileGapfill.Snapshot().Count(); got != gapfills { + t.Fatalf("adopt caused spurious gapfill: %d -> %d", gapfills, got) + } +} + +// A parked adopter (its mid-block build died; no seal ever comes) must +// not starve the idle catch-up: the held dead buffer is refolded away and +// the anchor tracks the store tip. +func TestParkedAdopterIdleRecovers(t *testing.T) { + restore := idleReconcileInterval + idleReconcileInterval = 80 * time.Millisecond + + t.Cleanup(func() { idleReconcileInterval = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Foreign content wins the height mid-build: our window STALEs into a + // stale-hold with no seal ever coming (the parked-adopter shape). + parent := sealHash(t, sealed) + ts, gasLimit := foreignWindow(t, h, 2, parent, testTx(t, 0)) + appendForeignSeal(t, h, windowHeader(2, parent, ts, gasLimit)) + + p.OpenBlock(2, ts+9, parent, gasLimit, fee25()) + p.PublishTx(testTx(t, 5)) + + base := h.store.Head() + + waitFor(t, 10*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.anchor == base && p.unackedLocked() == 0 + }) +} + +// A seal result racing the mute-clear between a build's check and its +// open must not publish an open at or behind the sealed tip. +func TestOpenBlockRefusesSealedHeight(t *testing.T) { + p := barePublisher() + + header := testHeader(1, common.Hash{0xef}) + p.OpenBlock(1, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + + before := len(p.journal.items) + p.OpenBlock(1, header.Time+4, header.ParentHash, header.GasLimit, header.BaseFee) + + if len(p.journal.items) != before { + t.Fatal("open published at a sealed height") + } +} + +// The 30 s idle ticker landing on a healthy producer's drained-open window +// (all records acked, seal pending — unacked==0 with a hold of holdNone) +// must NOT zero curHeight: doing so tagged later records height=0 and +// misclassified the live window as between-blocks. Only a held/parked +// window is cleaned. +func TestIdleTickPreservesLiveWindow(t *testing.T) { + restore := idleReconcileInterval + idleReconcileInterval = 40 * time.Millisecond + + t.Cleanup(func() { idleReconcileInterval = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Open the next window and stream a record; wait until it drains + // (open+record acked) — a healthy incumbent between txs. + parent := sealHash(t, sealed) + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + tx := testTx(t, 0) + p.PublishTx(tx) + waitHead(t, h, p, 5*time.Second) + + p.mu.Lock() + drained := p.unackedLocked() == 0 && p.journal.openStart() >= 0 && p.hold.kind == holdNone + p.mu.Unlock() + + if !drained { + t.Fatal("test setup: window not in the drained-open state") + } + + // Let several idle ticks fire across the drained-open window. + time.Sleep(200 * time.Millisecond) + + p.mu.Lock() + curHeight := p.curHeight + var zeroHeightRecords int + for _, it := range p.journal.items { + if it.kind == entryRecord && it.height == 0 { + zeroHeightRecords++ + } + } + p.mu.Unlock() + + if curHeight != 2 { + t.Fatalf("idle tick zeroed a live window: curHeight=%d, want 2", curHeight) + } + + // A continuation after the ticks must still tag the live height. + p.PublishTx(testTx(t, 1)) + + p.mu.Lock() + for _, it := range p.journal.items { + if it.kind == entryRecord && it.height == 0 { + zeroHeightRecords++ + } + } + p.mu.Unlock() + + if zeroHeightRecords != 0 { + t.Fatalf("records tagged height=0 after idle tick: %d", zeroHeightRecords) + } + + // The window still seals in place — no supersession. + supersedes := reconcileSupersede.Snapshot().Count() + p.SealBlock(blockFor(windowHeader(2, parent, 1700000002, 30_000_000), []*types.Transaction{tx, testTx(t, 1)})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("live window superseded after idle tick: %d -> %d", supersedes, got) + } +} diff --git a/eth/sequencer/audit_test.go b/eth/sequencer/audit_test.go new file mode 100644 index 0000000000..3051484d39 --- /dev/null +++ b/eth/sequencer/audit_test.go @@ -0,0 +1,419 @@ +package sequencer + +import ( + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// The preconfirmation failures storeprobe measures on a devnet. Asserting +// them here closes the gap that let a 40x regression pass a green suite: the +// other tests check what the publisher decides, these check what a consumer +// of the store would actually experience. +type storeAudit struct { + // Revoked: a record the store acked — so it was preconfirmed — that + // never landed in any canonical block. The worst outcome; the promise + // was simply broken. + Revoked []common.Hash + + // Displaced: acked at one height, landed at another. The transaction + // executes, but not where it was promised. + Displaced []common.Hash + + // Reordered: landed at the promised height in a different order. + // Position within a block is part of the promise for anything + // order-sensitive. + Reordered int + + // Mismatch: the store's newest generation at a height disagrees with the + // canonical block. A consumer reading the store's latest view of that + // height gets the wrong answer. + Mismatch []uint64 +} + +func (a storeAudit) clean() bool { + return len(a.Revoked) == 0 && len(a.Displaced) == 0 && + a.Reordered == 0 && len(a.Mismatch) == 0 +} + +// generation is one open..seal span at a height, in store order. +type generation struct { + height uint64 + txs []common.Hash + sealed bool +} + +// readAllGenerations replays the whole log and groups it the way the store's +// readers see it: an open starts a generation at its height, records extend +// the current one, and later generations at a height supersede earlier ones. +func readAllGenerations(t *testing.T, h *harness) []generation { + t.Helper() + + seed := commitment.Seed(testChainID) + + var ( + gens []generation + cur *generation + next = &pb.RangeRequest{ + After: &pb.RangeRequest_Head{Head: seed.Bytes()}, + Limit: 512, + } + ) + + for { + resp, err := h.store.Range(context.Background(), next) + if err != nil { + t.Fatalf("range: %v", err) + } + + for _, e := range resp.GetEntries() { + switch { + case e.GetBlockOpen() != nil: + gens = append(gens, generation{height: e.GetBlockOpen().GetBlockNumber()}) + cur = &gens[len(gens)-1] + case e.GetRecord() != nil && cur != nil: + for _, raw := range e.GetRecord().GetTransactions() { + var tx types.Transaction + if err := tx.UnmarshalBinary(raw); err != nil { + t.Fatalf("decode record tx: %v", err) + } + + cur.txs = append(cur.txs, tx.Hash()) + } + case e.GetBlockSeal() != nil && cur != nil: + cur.sealed = true + } + } + + if resp.GetLive() { + return gens + } + + next = &pb.RangeRequest{ + After: &pb.RangeRequest_Head{Head: resp.GetNext()}, + Limit: 512, + } + } +} + +// auditStore compares everything the store acked against the canonical +// chain the test declares (height -> ordered transaction hashes). +func auditStore(t *testing.T, h *harness, canonical map[uint64][]common.Hash) storeAudit { + t.Helper() + + gens := readAllGenerations(t, h) + + landedAt := map[common.Hash]uint64{} + indexIn := map[common.Hash]int{} + + for height, txs := range canonical { + for i, tx := range txs { + landedAt[tx] = height + indexIn[tx] = i + } + } + + var audit storeAudit + + for _, g := range gens { + for _, tx := range g.txs { + at, ok := landedAt[tx] + switch { + case !ok: + audit.Revoked = append(audit.Revoked, tx) + case at != g.height: + audit.Displaced = append(audit.Displaced, tx) + } + } + } + + // The newest generation at a height is what a reader takes as the truth + // for that height, so that is what has to agree with the block. + newest := map[uint64]generation{} + for _, g := range gens { + newest[g.height] = g + } + + for height, g := range newest { + block, ok := canonical[height] + if !ok { + continue // no block at this height: nothing to disagree with + } + + if len(g.txs) != len(block) { + audit.Mismatch = append(audit.Mismatch, height) + + continue + } + + for i := range g.txs { + if g.txs[i] != block[i] { + audit.Mismatch = append(audit.Mismatch, height) + + break + } + } + + // Order within the promised height, counted pairwise as storeprobe + // does: two records promised in one order must not land reversed. + for i := 0; i < len(g.txs); i++ { + for j := i + 1; j < len(g.txs); j++ { + a, b := g.txs[i], g.txs[j] + if landedAt[a] == height && landedAt[b] == height && + indexIn[a] > indexIn[b] { + audit.Reordered++ + } + } + } + } + + return audit +} + +// The auditor has to fail before it is worth trusting: each failure class is +// constructed deliberately and must be reported. An auditor that only ever +// says "clean" is what a green suite looked like while a devnet burned. +func TestAuditorDetectsEachFailureClass(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Three records acked at height 2, in this order. + t0, t1, t2 := testTx(t, 0), testTx(t, 1), testTx(t, 2) + foreignWindow(t, h, 2, parent, t0, t1, t2) + + block1 := []common.Hash{} + for _, tx := range sealed1Txs(t, h) { + block1 = append(block1, tx) + } + + cases := []struct { + name string + canonical map[uint64][]common.Hash + want func(storeAudit) bool + reason string + }{ + { + name: "clean", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash(), t1.Hash(), t2.Hash()}}, + want: func(a storeAudit) bool { return a.clean() }, + reason: "an exact match must audit clean", + }, + { + name: "revoked", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash(), t1.Hash()}}, + want: func(a storeAudit) bool { return len(a.Revoked) == 1 }, + reason: "a preconfirmed record in no block is a revocation", + }, + { + name: "displaced", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash(), t1.Hash()}, 3: {t2.Hash()}}, + want: func(a storeAudit) bool { return len(a.Displaced) == 1 }, + reason: "acked at 2 but landed at 3 is a displacement", + }, + { + name: "reordered", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash(), t2.Hash(), t1.Hash()}}, + want: func(a storeAudit) bool { return a.Reordered > 0 }, + reason: "same height, swapped order, is a reorder", + }, + { + name: "mismatch", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash()}}, + want: func(a storeAudit) bool { return len(a.Mismatch) == 1 }, + reason: "the newest generation must equal the block at that height", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := auditStore(t, h, tc.canonical) + if !tc.want(got) { + t.Fatalf("%s: audit=%+v", tc.reason, got) + } + }) + } +} + +// sealed1Txs returns the transactions the store holds at height 1, so a test +// can declare a canonical chain that agrees with it. +func sealed1Txs(t *testing.T, h *harness) []common.Hash { + t.Helper() + + for _, g := range readAllGenerations(t, h) { + if g.height == 1 { + return g.txs + } + } + + return nil +} + +// canonicalFrom builds the canonical view a test declares by treating the +// blocks a publisher actually sealed as the chain, which is what happens when +// only one producer closes a height. +func canonicalFrom(blocks map[uint64][]*types.Transaction) map[uint64][]common.Hash { + out := map[uint64][]common.Hash{} + + for height, txs := range blocks { + hashes := make([]common.Hash, 0, len(txs)) + for _, tx := range txs { + hashes = append(hashes, tx.Hash()) + } + + out[height] = hashes + } + + return out +} + +// The end-to-end property, stated in the terms a consumer cares about: a +// single producer's published window must audit clean against the block it +// seals. No revocation, no displacement, no reorder, no mismatch. +func TestSoleProducerAuditsClean(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + txs := []*types.Transaction{testTx(t, 0), testTx(t, 1), testTx(t, 2)} + + header := testHeader(1, common.Hash{0xef}) + p.OpenBlock(1, header.Time, common.Hash{0xef}, header.GasLimit, header.BaseFee) + + for _, tx := range txs { + p.PublishTx(tx) + } + + p.SealBlock(blockFor(header, txs)) + waitHead(t, h, p, 5*time.Second) + + audit := auditStore(t, h, canonicalFrom(map[uint64][]*types.Transaction{1: txs})) + if !audit.clean() { + t.Fatalf("a sole producer broke its own preconfirmations: %+v", audit) + } +} + +// Refusal buys agreement: whichever producer closes the height, the store's +// newest generation there must equal the block. That is the mismatch class, +// and it is what a consumer reading the store's latest view depends on. +func TestRefusalPreventsMismatch(t *testing.T) { + h, a, b := twinPublishers(t) + parent := sealedParent(t, h, a, b) + + header := testHeader(2, parent) + + // Distinct nonces throughout: height 1 already holds nonce 0. + aTxs := []*types.Transaction{testTx(t, 1), testTx(t, 2)} + bTxs := []*types.Transaction{testTx(t, 7)} + + a.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + + for _, tx := range aTxs { + a.PublishTx(tx) + } + + waitDrained(t, a, 5*time.Second) + + b.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + + for _, tx := range bTxs { + b.PublishTx(tx) + } + + waitFor(t, 5*time.Second, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + + return b.hold.kind == holdSticky || b.unackedLocked() == 0 + }) + + sealA, sealB := decide(a), decide(b) + if sealA && sealB { + t.Fatal("both sealed: divergent blocks at one height") + } + + if !sealA && !sealB { + t.Fatal("neither sealed: the height never closes") + } + + winner := bTxs + if sealA { + winner = aTxs + } + + canonical := canonicalFrom(map[uint64][]*types.Transaction{2: winner}) + canonical[1] = sealed1Txs(t, h) + + audit := auditStore(t, h, canonical) + + if len(audit.Mismatch) > 0 { + t.Fatalf("the store's newest generation disagrees with the sealed "+ + "block at %v: a consumer reading the store gets the wrong "+ + "answer for that height", audit.Mismatch) + } +} + +// What refusal does NOT buy, stated plainly so nobody expects it to. Once a +// rival's records are acked they are preconfirmed, and they cannot also be in +// a block built from different content. They land later or not at all. +// +// Note the escape the publisher gets for free in the common case: a second +// producer appending to the same generation STALEs and never acks, so it +// never promises anything. This test uses a rival that opens its own +// generation, which the store accepts by design — the one shape where both +// sides really do hold promises. Nothing available to bor prevents it; only +// the store declining a second open at a live height removes the class. +func TestRivalGenerationStrandsItsOwnRecords(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + header := testHeader(2, parent) + ours := []*types.Transaction{testTx(t, 1), testTx(t, 2)} + + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + + for _, tx := range ours { + p.PublishTx(tx) + } + + waitDrained(t, p, 5*time.Second) + + // A rival opens its own generation on the current head, so its record is + // accepted and therefore preconfirmed. + rival := testTx(t, 9) + foreignWindow(t, h, 2, parent, rival) + + // Only one block exists at height 2, and it holds our content. + canonical := canonicalFrom(map[uint64][]*types.Transaction{2: ours}) + canonical[1] = sealed1Txs(t, h) + + audit := auditStore(t, h, canonical) + + stranded := len(audit.Revoked) + len(audit.Displaced) + if stranded != 1 { + t.Fatalf("expected exactly the rival's record stranded, got %d "+ + "(revoked=%d displaced=%d): more than that means contention is "+ + "costing the winner's promises too", + stranded, len(audit.Revoked), len(audit.Displaced)) + } + + // And the mismatch class is the one bor can still control: the store's + // newest generation at this height is the rival's, so a reader gets the + // wrong answer until the winner's flush corrects it. + if len(audit.Mismatch) != 1 || audit.Mismatch[0] != 2 { + t.Fatalf("expected the rival generation to leave height 2 mismatched, got %v", + audit.Mismatch) + } +} diff --git a/eth/sequencer/barrier.go b/eth/sequencer/barrier.go new file mode 100644 index 0000000000..8e8b285b2f --- /dev/null +++ b/eth/sequencer/barrier.go @@ -0,0 +1,203 @@ +package sequencer + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +// coverageSkip records a seal allowed through without a proven-complete +// window. Each reason is a different way this block can be sealed short of +// what the store holds, so they are worth telling apart rather than +// collapsing into one silent "true". +func coverageSkip(height uint64, reason string, ctx ...any) { + log.Info("Coverage check skipped", + append([]any{"number", height, "reason", reason}, ctx...)...) +} + +// AwaitSequenced blocks until the block about to be sealed is provably the +// store's sequence at this height. The rule is a mirror: seal only a block +// whose content is exactly the store's live window. +// +// The store elects the owner of every height: writes are a compare-and-swap +// on its head, so exactly one producer's open lands and everyone else +// STALEs. The owner's window is the store, its mirror check passes, and it +// seals — liveness by construction, no tie-break needed. A producer that +// finds itself behind adopts the store's window and rebuilds; false from +// here means exactly that (the resync signal is armed). +// +// A store that is merely unreachable, slow, or catching up after an outage +// returns true on the deadline: block production never waits on the store. +func (p *Publisher) AwaitSequenced(timeout time.Duration, number uint64, txs []*types.Transaction) bool { + deadline := time.Now().Add(timeout) + + for { + if p.failed.Load() { + return true // publishing is off; it must not gate production + } + + p.mu.Lock() + unacked := p.unackedLocked() + sticky := p.hold.kind == holdSticky + catchingUp := p.pendingFrom != 0 + p.mu.Unlock() + + switch { + case sticky: + // Another producer owns this height (our writes STALEd). Adopt + // their window instead of sealing beside it: the rebuild's + // boundary read collects it. + p.armResync() + + return false + case catchingUp: + // The store is behind us by construction while the backfill + // drains, so there is nothing here worth comparing against and + // nothing worth waiting for. Seal now; the mirror check resumes + // once the drain finishes. + publishCatchupSkip.Inc(1) + + return true + case unacked == 0: + return p.sealMirror(number, txs) + case time.Now().After(deadline): + publishBarrierTimeout.Inc(1) + + // The drain did not finish in budget, but that says nothing + // about whether this block carries what the store promised — + // and surrendering here drops the content check exactly when + // load makes it matter. A window still draining is a prefix of + // our own block, so the comparison holds mid-drain; it was a + // block with none of a 9523-record window that rode this exit + // out to a broadcast. + return p.sealMirror(number, txs) + } + + time.Sleep(2 * time.Millisecond) + } +} + +// armResync requests a rebuild that follows the store's sequence. Idempotent; +// the worker consumes it via ResyncNeeded. +func (p *Publisher) armResync() { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.resync { + p.resync = true + reconcileResync.Inc(1) + } +} + +// sealMirror reports whether the block about to be sealed is the store's +// sequence at this height. +// +// The comparison is against the block's own transactions, not against our +// journal position. A block built before an adoption leaves the journal +// perfectly in sync with the store while the block itself carries different +// content — position agreement says nothing about the transactions we are +// about to broadcast, and those are what the store promised. +func (p *Publisher) sealMirror(height uint64, txs []*types.Transaction) bool { + if p.unreachable.Load() { + coverageSkip(height, "store unreachable") + + return true // production never waits on a store we cannot reach + } + + ctx, cancel := context.WithTimeout(context.Background(), tailReadTimeout) + defer cancel() + + info, out := p.readTail(ctx) + if out != recOK { + // A window we cannot read is not a window we can prove divergent, + // and production never waits on the store. + coverageSkip(height, "tail unreadable", "outcome", int(out)) + + return true + } + + if info.haveSeal && info.lastSealHeight >= height { + coverageSkip(height, "height already sealed in the store") + + return true // the height moved on without us: the next build mutes + } + + // The store's live window at our height is the sequence this block owes + // its consumers. It must appear in the block, in order, from the front: + // anything else means transactions were promised at this height that + // the block does not deliver there. + if info.tipOpen && info.tipOpenHeight == height { + if storeTxs, ok := windowTxHashes(info.window); ok { + return p.mirrorVerdict(height, storeTxs, txs, len(storeTxs)) + } + } + + // The read shows nothing past the entries we published, so the store's + // window at this height is the one in our journal — already acked, so + // comparing the block against it compares it against the store. + p.mu.Lock() + ourTxs, atHead := p.journalWindowLocked(height), info.s == p.anchor + p.mu.Unlock() + + if atHead { + return p.mirrorVerdict(height, ourTxs, txs, len(ourTxs)) + } + + // The store holds content this block does not — a record extension of + // our window, or a generation we have not absorbed. Either way the + // answer is the same: adopt the store's sequence and rebuild on it. + // Never seal beside it, never republish over it. + p.mu.Lock() + p.resync = true + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + reconcileResync.Inc(1) + ours := len(p.journal.suffixFromHeight(height)) + p.mu.Unlock() + + log.Warn("Store holds content this block does not cover, adopting", + "number", height, "ours", ours, "storeWindow", len(info.window)) + + return false +} + +// ResyncNeeded reports whether a competing producer holds the height this +// node is building, meaning this build must stop — the next work cycle's +// build-start read follows the store's sequence. Reading consumes the +// signal; it fires at most once per height. +func (p *Publisher) ResyncNeeded() bool { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.resync { + return false + } + + p.resync = false + + return true +} + +// mirrorVerdict passes a block that delivers the promised sequence and arms +// a rebuild for one that does not. +func (p *Publisher) mirrorVerdict(height uint64, promised []common.Hash, + txs []*types.Transaction, window int, +) bool { + if windowLeadsHashes(promised, txHashes(txs)) { + return true + } + + barrierDivergedCount.Inc(1) + log.Warn("Sequencer block does not carry the sequence promised at this height, rebuilding", + "number", height, "promised", window, "block", len(txs)) + + p.mu.Lock() + p.resync = true + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + reconcileResync.Inc(1) + p.mu.Unlock() + + return false +} diff --git a/eth/sequencer/classify.go b/eth/sequencer/classify.go new file mode 100644 index 0000000000..c30ceb3d1b --- /dev/null +++ b/eth/sequencer/classify.go @@ -0,0 +1,848 @@ +package sequencer + +import ( + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rlp" +) + +// applyTail classifies one tail read and applies the corrective action. +// Writes are reserved for replay of our own lineage, gap +// crossings, and the seal flush: a foreign tail with our window unsealed +// is held (the flush resolves it), never superseded. +func (p *Publisher) applyTail(info tailInfo) reconcileOutcome { + p.mu.Lock() + defer p.mu.Unlock() + + p.advanceStoreSealedTipLocked(info, "reconcile read") + + // Row 1: the store head is on our lineage — retire through it and let + // the send loop replay the rest. A diverged flush branches *below* the + // anchor (the adopt rewound to a prefix): its suffix does not extend + // the store head and must re-anchor onto it instead. + if info.s == p.anchor { + if fh, ok := p.pendingFlushLocked(); ok && !p.suffixExtendsLocked(info.s) { + return p.refoldLocked(info.s, fh, reconcileSupersede) + } + + return p.finishRow1Locked() + } + + if seq, ok := p.journal.findPost(info.s); ok { + p.ackedSeq = seq + p.anchor = info.s + p.confirmed = true + + return p.finishRow1Locked() + } + + return p.classifyForeignLocked(info) +} + +func (p *Publisher) finishRow1Locked() reconcileOutcome { + if items, covered := p.journal.after(p.ackedSeq); covered && p.pendingFrom == 0 { + if len(items) > 0 { + reconcileGapfill.Inc(1) + } + + p.anchored, p.confirmed = true, true + + return recOK + } + + // Eviction opened a gap behind the replayable suffix: jump, keeping + // everything a flush still owes the store (the oldest undelivered + // seal onward), else the current window (always retained). + keep := p.curHeight + if h := p.oldestPendingSealLocked(); h != 0 { + keep = h + } + + if keep == 0 { + keep = keepNone + } + + return p.refoldLocked(p.anchor, keep, reconcileForwardJump) +} + +// classifyForeignLocked resolves a foreign tail against what this publisher +// holds, in rank order: a flush in flight first, then the between-blocks +// re-anchor, else hold until our seal resolves the height. +func (p *Publisher) classifyForeignLocked(info tailInfo) reconcileOutcome { + if out, handled := p.classifyPendingFlushLocked(info); handled { + return out + } + + if p.curHeight == 0 { + return p.classifyBetweenBlocksLocked(info) + } + + // The store's window at our height may be a prefix of what we hold — + // our own earlier generation, or an identical build. Absorb it and + // deliver only the records it lacks. Republishing the whole window + // instead would write a fresh generation of data the store already + // has, which is pure duplication: every reader already has the prefix. + if p.completeExtendedWindowLocked(info, p.curHeight) { + return recOK + } + + // A window standing at our own height on our own parent means another + // producer is building this slot and reached the store first. Adopt it: + // rebuilding onto their sequence is what keeps the two blocks from + // diverging, and adoption extends their window rather than replacing + // it, so nothing already published is revoked. Only this exact shape + // arms the signal: a transient STALE, a foreign window at another + // height, or one on a different parent must never restart a + // legitimate build. + if ourParent, ok := p.ourOpenParentLocked(); ok && + info.tipOpen && info.tipOpenHeight == p.curHeight && + info.tipOpenParent == ourParent { + p.resync = true + + reconcileResync.Inc(1) + log.Warn("Sequencer found a competing producer at our height, requesting rebuild", + "number", p.curHeight, "parent", ourParent) + } + + // Our window is unsealed: nothing we hold outranks the store yet. + // Buffer everything unacked until the seal flush resolves the height; + // a build the chain moved past heals at the next build-start check. + return p.holdStaleLocked() +} + +// classifyPendingFlushLocked handles a flush in flight: a sealed trailing +// window with undelivered entries — the sealed, broadcast block overrides +// whatever the store shows at its height. handled=false means either no +// flush is pending, or a canonical seal past the flush outranks it and +// classification continues as if no flush were pending. +func (p *Publisher) classifyPendingFlushLocked(info tailInfo) (reconcileOutcome, bool) { + flushHeight, ok := p.pendingFlushLocked() + if !ok { + return recOK, false + } + + // The store's standing window may be OUR window extended by records + // that were still in flight when we read it (a dying producer's drain, + // an adoption snapshot racing the stream). When it is a strict prefix + // of what this flush carries, complete it in place — absorbing the + // store's copy and delivering only the remainder — instead of + // superseding content that matches ours. + if p.completeExtendedWindowLocked(info, flushHeight) { + return recOK, true + } + + // A seal standing at our own flush height is another producer claiming + // the same slot (duplicate signing keys), or our own copy already + // delivered. Superseding it is only right when the chain chose OUR + // block: if the store's seal is the canonical one, overwriting it + // would leave the store's newest generation disagreeing with the + // chain. We lost — yield instead. + if info.haveSeal && info.lastSealHeight == flushHeight && + p.chain != nil && p.foreignSealCanonicalLocked(info) { + return p.yieldFlushLocked(info), true + } + + // A foreign seal already stands at our flush height and our own block + // is still unbroadcast: the store says this height is closed, and the + // head we would chain onto encodes that seal — we cannot claim not to + // know. Publishing our window and seal over it would put a second + // sealed generation at a height the store already closed, for a block + // that may never exist on any chain. + // + // Unconditional while the gate is pending: the chain's opinion is not + // required, and waiting for it is what let a refold slip through in + // the ~140ms before the winner's block imported. Once the gate clears + // (broadcast, or the liveness timeout) this height becomes ordinary + // repair, where canonicality decides supersede versus yield. A decoded + // seal also settles the gate itself: this hold is the verdict, not a + // wait for one. + if info.haveSeal && info.lastSealHeight == flushHeight && + p.gatePendingLocked(flushHeight) { + p.resolveGateFromSealLocked(info) + + return p.holdStaleLocked(), true + } + + // Only a sealed height past the flush outranks it. A foreign unsealed + // open above the flush height is a contender that already lost to this + // seal — stranding the flush on it would leave the height sealed on + // chain but forever open in the store. + if !info.haveSeal || info.lastSealHeight <= flushHeight { + if !p.mayDisplaceWindowLocked(info, flushHeight) { + return p.holdStaleLocked(), true + } + + return p.reanchorFlushLocked(info), true + } + + // A foreign seal stands above our pending flush. Dropping our seal for + // it is right only when that lineage is the canonical chain — we were + // genuinely reorged past. If the chain has not accepted it (a reorg + // loser, or a lineage not yet imported), the store tail is not + // authoritative: hold our seal and let our next flush re-anchor onto + // the canonical chain. Never drop a canonical seal for a non-canonical + // store tail. + if !p.foreignSealCanonicalLocked(info) { + return p.holdStaleLocked(), true + } + + return recOK, false // canonical seal past the flush outranks it +} + +// restoreAbandonedDebtLocked folds the sealed heights a refold is about to +// abandon back into the pending range. The backfill advances pendingFrom at +// build time, before delivery is confirmed; when the batch then loses its +// head race and the next classification abandons it, nothing else remembers +// those heights are owed — a devnet pinned an outage height as a permanent +// hole exactly this way. Restoring the debt costs at most a duplicate +// delivery. +func (p *Publisher) restoreAbandonedDebtLocked(suffix []journalItem) { + kept := uint64(0) + if len(suffix) > 0 { + kept = suffix[0].seq + } + + lo, hi := uint64(0), uint64(0) + + for _, it := range p.journal.items { + if it.seq <= p.ackedSeq || it.kind != entrySeal { + continue + } + + if kept != 0 && it.seq >= kept { + break + } + + if lo == 0 || it.height < lo { + lo = it.height + } + + if it.height > hi { + hi = it.height + } + } + + if lo == 0 { + return + } + + if p.pendingFrom == 0 || lo < p.pendingFrom { + p.pendingFrom = lo + } + + if hi > p.pendingTo { + p.pendingTo = hi + } + + log.Warn("Sequencer restoring abandoned sealed heights to the backfill", + "from", lo, "to", hi) +} + +// mayDisplaceWindowLocked decides whether this flush may overwrite a live +// foreign window, having looked at what that window holds. +// +// A re-anchor rebuilds our own entries onto the store head; it never ingests +// what stands behind that head. So displacing a live window silently drops +// every record it holds that our block does not carry — records the store +// already acked, which is to say preconfirmations. That is only defensible +// when our block is the one the chain kept: then the store must end up +// holding our content or it disagrees with the chain. When our block did not +// win, displacing is pure destruction, and holding lets the winner's flush +// resolve the height. +func (p *Publisher) mayDisplaceWindowLocked(info tailInfo, flushHeight uint64) bool { + // A foreign live window at the flush height is content this flush would + // overwrite; one above it is re-anchored past just as surely — a silent + // fold-past exactly there is how two full acked generations ended up + // buried under an empty sealed block. Either shape needs proof, and a + // flush the chain ratified still proceeds: the re-anchor only moves our + // lineage, so a window above stays the newest generation at its own + // height, where the next build's boundary read adopts it. + foreignWindow := info.tipOpen && info.tipOpenHeight >= flushHeight && + p.relateWindowLocked(info, info.tipOpenHeight) == windowForeign + + sealCandidate := info.sealDecoded && info.lastSealHeight == flushHeight + + if !foreignWindow && !sealCandidate { + return true // nothing foreign standing here to displace + } + + // A sealed foreign generation at our height is a stronger claim than a + // live window, and it gets the same treatment: chaining our flush past + // it reads the head's bytes without understanding what they encode — a + // closed height. Only a decoded seal counts, and our own seal already + // standing there is re-delivery, not displacement. + ours, sealedHere := p.sealedHashAtLocked(flushHeight) + foreignSealed := sealCandidate && !(sealedHere && info.lastSealHash == ours) + + if !foreignWindow && !foreignSealed { + return true // the standing seal is our own re-delivery + } + + if p.chain == nil { + return true + } + + // Only affirmative proof that the chain kept our block licenses a + // displacement. "Not decided yet" is not proof — on a devnet the + // winner's block imported 134ms after a displacement made on exactly + // that reasoning, and the records it destroyed were already acked. + if !sealedHere || p.chain.GetCanonicalHash(flushHeight) != ours { + // With the gate still pending this withhold is the verdict: an + // unbroadcast block can never become canonical, so waiting for the + // proof would deadlock against producing it. Refuse the broadcast + // and the rebuild adopts what stands here instead. The liveness + // fallback's tolerance covers exactly the store's seal — a foreign + // window still refuses it. + if p.gatePendingLocked(flushHeight) && + (foreignWindow || !p.gate.tolerateSealed) { + p.gate.verdict = gateLost + } + + log.Warn("Sequencer withholding flush: the chain has not kept this block", + "number", flushHeight, "ours", ours) + + return false + } + + // Only what this block does not carry is actually orphaned, and only a + // window standing at the flush height itself can be counted truthfully: + // a displaced foreign seal's generation is not in this read, and the + // trailing window of a higher height compared against this height's + // block once reported 1,960 phantom orphans for a displacement that + // destroyed nothing. + if foreignWindow && info.tipOpenHeight == flushHeight { + if orphans := p.orphanedByDisplacementLocked(info, flushHeight); orphans > 0 { + windowDisplacedRecords.Inc(int64(orphans)) + log.Warn("Sequencer flush displacing acked records this block does not carry", + "number", flushHeight, "orphaned", orphans) + } + } + + return true +} + +// orphanedByDisplacementLocked counts the displaced window's transactions +// that our block does not deliver at this height. The caller has already +// established that the canonical block at this height is ours, so the chain +// holds the authoritative copy of what we delivered — the journal has moved +// on by seal time and no longer describes the window. +func (p *Publisher) orphanedByDisplacementLocked(info tailInfo, height uint64) int { + theirs, ok := windowTxHashes(info.window) + if !ok || p.chain == nil { + return 0 + } + + block := p.chain.GetBlockByNumber(height) + if block == nil { + return 0 + } + + ours := make(map[common.Hash]struct{}, block.Transactions().Len()) + for _, tx := range block.Transactions() { + ours[tx.Hash()] = struct{}{} + } + + orphans := 0 + + for _, h := range theirs { + if _, carried := ours[h]; !carried { + orphans++ + } + } + + return orphans +} + +// sealedHashAtLocked returns the block hash our undelivered seal carries for +// a height. +func (p *Publisher) sealedHashAtLocked(height uint64) (common.Hash, bool) { + for i := len(p.journal.items) - 1; i >= 0; i-- { + it := p.journal.items[i] + if it.kind != entrySeal || it.height != height { + continue + } + + header, err := decodeSealHeader(it.entry.GetBlockSeal().GetHeader()) + if err != nil { + return common.Hash{}, false + } + + return header.Hash(), true + } + + return common.Hash{}, false +} + +// yieldFlushLocked abandons our sealed window because the chain chose +// another producer's block at that height. Our content is non-canonical; +// republishing it would overwrite the winner's correct content. Counted +// apart from supersede — here we are the loser, not the winner. +func (p *Publisher) yieldFlushLocked(info tailInfo) reconcileOutcome { + log.Warn("Sequencer yielding to canonical seal at our own height", + "height", info.lastSealHeight, "canonical", info.lastSealHash) + reconcileYield.Inc(1) + + return p.refoldLocked(info.s, keepNone, nil) +} + +// reanchorFlushLocked re-anchors every undelivered seal onto the store +// head, oldest window first — flushes stack behind a reconcile backoff, +// and a skipped one would leave its height unsealed in the store forever. +func (p *Publisher) reanchorFlushLocked(info tailInfo) reconcileOutcome { + keepFrom := p.oldestPendingSealLocked() + + var counter *metrics.Counter + + switch { + case (info.tipOpen && info.tipOpenHeight >= keepFrom) || + (info.haveSeal && info.lastSealHeight >= keepFrom): + counter = reconcileSupersede // foreign content on re-anchored ground is overridden + case info.haveSeal && info.lastSealHeight+1 == keepFrom: + counter = nil // contiguous re-delivery: nothing overridden or abandoned + default: + counter = reconcileForwardJump // sealed ground missing below the flush + } + + return p.refoldLocked(info.s, keepFrom, counter) +} + +// classifyBetweenBlocksLocked resolves a foreign tail with no build in +// progress. A tail ending in an adoptable unsealed window is not history +// to rebase past — it is what the next build-start check adopts (a +// restarted producer resumes its own window this way): anchor at the +// window's base, keeping the window ahead of the anchor where the +// build-start read can collect it. Otherwise rebase onto the store head +// and let the next open extend it; abandoning unconfirmed history here is +// the forward-jump case (counted only when entries are actually left +// behind). +func (p *Publisher) classifyBetweenBlocksLocked(info tailInfo) reconcileOutcome { + if len(info.window) > 0 && p.unackedLocked() == 0 { + base := commitment.Head(info.window[0].GetBlockOpen().GetPrefixCommitment()) + if base != p.anchor { + p.rebaseLocked(base) + } + + p.anchored = true + + return recOK + } + + log.Info("Sequencer rebasing onto store head", "head", info.s) + + return p.refoldLocked(info.s, keepNone, reconcileForwardJump) +} + +// holdStaleLocked buffers everything unacked until a seal flush resolves +// the height. +func (p *Publisher) holdStaleLocked() reconcileOutcome { + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + p.anchored = true + + return recOK +} + +// foreignSealCanonicalLocked reports whether the store's trailing seal is +// the block the local chain considers canonical at that height. A nil +// chain (test publishers) preserves the pre-check behavior. +func (p *Publisher) foreignSealCanonicalLocked(info tailInfo) bool { + if !info.haveSeal { + return false + } + + if p.chain == nil { + return true + } + + return p.chain.GetCanonicalHash(info.lastSealHeight) == info.lastSealHash +} + +// pendingFlushLocked reports whether the lineage carries any undelivered +// seal — a flush in flight, possibly stacked behind retry pacing and +// possibly with the next build's window already opened on top — and the +// height of the newest one. Back-to-back builds leave only a sliver of +// time where a seal is the trailing item, so keying on the trailing item +// alone would hide the stack from nearly every reconcile. +func (p *Publisher) pendingFlushLocked() (uint64, bool) { + for i := len(p.journal.items) - 1; i >= 0; i-- { + it := p.journal.items[i] + if it.seq <= p.ackedSeq { + return 0, false + } + + if it.kind == entrySeal { + return it.height, true + } + } + + return 0, false +} + +// oldestPendingSealLocked returns the height of the first undelivered +// seal — the start of a possibly stacked flush suffix. +func (p *Publisher) oldestPendingSealLocked() uint64 { + for _, it := range p.journal.items { + if it.seq > p.ackedSeq && it.kind == entrySeal { + return it.height + } + } + + return 0 +} + +// suffixExtendsLocked reports whether the first undelivered journal item +// folds directly onto s — the send loop can replay it verbatim. +func (p *Publisher) suffixExtendsLocked(s commitment.Head) bool { + items, covered := p.journal.after(p.ackedSeq) + if !covered || len(items) == 0 { + return true + } + + return items[0].pre == s +} + +// completeExtendedWindowLocked completes a window in place when the store's +// standing window is a prefix of what we hold: the store's copy is absorbed +// as acked and only the records it lacks (plus the seal, once sealed) fold +// onto its head. Nothing already published is re-sent and no new generation +// is written — readers already have the prefix. +func (p *Publisher) completeExtendedWindowLocked(info tailInfo, flushHeight uint64) bool { + ours, items, ok := p.matchExtendedWindowLocked(info, flushHeight) + if !ok { + return false + } + + p.absorbExtendedWindowLocked(info, ours, items, flushHeight) + + return true +} + +// matchExtendedWindowLocked reports whether the store's standing window is +// a strict prefix of the flush at flushHeight (same open, records a +// leading byte-subsequence of ours, same order), returning our flush +// suffix and the store's window parsed as journal items. +func (p *Publisher) matchExtendedWindowLocked(info tailInfo, flushHeight uint64) (ours, items []journalItem, ok bool) { + if !info.tipOpen || info.tipOpenHeight != flushHeight || len(info.window) == 0 { + return nil, nil, false + } + + // The window may still be building (no seal yet): a mid-build + // re-anchor completes in place just as a flush does. + ours = p.journal.suffixFromHeight(flushHeight) + if len(ours) == 0 || ours[0].kind != entryOpen { + return nil, nil, false + } + + if len(info.window) > len(ours)-1 { // window can cover at most our records + return nil, nil, false + } + + for i, entry := range info.window { + if !contentEqual(entry, ours[i].entry) { + return nil, nil, false + } + } + + stored, items, ok := parseWindow(info) + if !ok || stored.number != flushHeight { + return nil, nil, false + } + + return ours, items, true +} + +// absorbExtendedWindowLocked absorbs the store's copy as confirmed lineage +// and re-folds the remainder (records past the stored prefix, and the +// seal) onto its head for the send loop to deliver. A refold failure +// latches fail(); nothing else to classify. +func (p *Publisher) absorbExtendedWindowLocked(info tailInfo, ours, items []journalItem, flushHeight uint64) { + // Older stacked flushes below this window are abandoned by the swap: + // their heights are already represented byte-identically in the store + // lineage this window folds on (the open's parent hash pins it), but + // the entries themselves never deliver — count them. + if abandoned := p.unackedLocked() - len(ours); abandoned > 0 { + publishDropMeter.Mark(int64(abandoned)) + } + + fresh := newJournal() + fresh.nextSeq = p.journal.nextSeq + + for _, it := range items { + fresh.append(it.entry, it.pre, it.post, it.kind, it.height, fresh.nextSeq, it.txHashes) + } + + p.ackedSeq = fresh.nextSeq - 1 + cur := info.s + + for _, item := range ours[len(info.window):] { + entry, next, err := refoldEntry(cur, item) + if err != nil { + p.fail("refold flush remainder", "err", err) + + return + } + + fresh.append(entry, cur, next, item.kind, item.height, 0, item.txHashes) + cur = next + } + + log.Info("Sequencer completing extended window in place", "number", flushHeight, + "stored", len(info.window), "delivering", len(ours)-len(info.window)) + p.installSwapLocked(fresh, cur, info.s) +} + +// backfillLocked drains the oldest owed blocks from the chain database onto +// cur, appending to fresh as undelivered entries, and returns the new fold +// head. One call rebuilds at most a journal byte budget's worth; the +// remainder stays pending, and the drain resumes on the next re-anchor — +// oldest first, so the store's sealed tip never advances past a gap it +// would then refuse to fill. Blocks the store already sealed are not owed; +// blocks missing from the database (pruned) are the one gap backfill cannot +// close, skipped as a counted forward jump. +func (p *Publisher) backfillLocked(fresh *journal, cur commitment.Head) commitment.Head { + if p.pendingFrom == 0 { + return cur + } + + // The whole pending range drains, storeSealedTip notwithstanding. The + // tip is the newest seal, not proof of anything below it: live flushes + // seal heights above the gap while it waits, and a store restart can + // shed acked writes, so "the store has these" was twice false on a + // devnet — pending heights skipped on the tip stayed holes forever. + // Re-delivering a height the store does have only adds an identical + // duplicate generation, which costs churn, not correctness. + lo, hi := p.pendingFrom, p.pendingTo + + log.Info("Sequencer backfill starting", "from", lo, "to", hi, + "storeSealedTip", p.storeSealedTip) + + if lo > hi || p.chain == nil { + if p.chain == nil && hi >= lo { + reconcileForwardJump.Inc(1) + } + + // An inverted range cannot arise anymore — the collapse merges + // rather than clobbers — so hitting one means new accounting is + // broken somewhere. Dropping the debt is still the only safe move + // (a wedged drain is worse), but never a silent one again. + if lo > hi { + log.Warn("Sequencer backfill pending range inverted, dropping the debt", + "from", lo, "to", hi) + } + + p.pendingFrom, p.pendingTo, p.pendingEntries = 0, 0, 0 + + return cur + } + + var ( + budget, rebuilt int + jumped bool + n = lo + started = time.Now() + ) + + for ; n <= hi; n++ { + block := p.chain.GetBlockByNumber(n) + if block == nil { + jumped = true // pruned: unfillable + + continue + } + + // Always take at least one block, or a block larger than the + // budget would wedge the drain forever. + if budget += int(block.Size()); budget > backfillBatchBytes && rebuilt > 0 { + break + } + + before := len(fresh.items) + + next, ok := p.appendBlockLocked(fresh, cur, block) + if !ok { + return cur // fail() latched + } + + rebuilt += len(fresh.items) - before + cur = next + } + + if n > hi { + p.pendingFrom, p.pendingTo, p.pendingEntries = 0, 0, 0 + } else { + p.pendingFrom = n + if p.pendingEntries > rebuilt { + p.pendingEntries -= rebuilt + } else { + p.pendingEntries = 0 + } + + log.Info("Sequencer backfill batch, remainder pending", + "rebuilt-through", n-1, "pending", n, "to", hi, + "batch", time.Since(started)) + + backfillBatchTimer.UpdateSince(started) + } + + if jumped { + reconcileForwardJump.Inc(1) + log.Warn("Sequencer backfill skipped pruned blocks", "pending", lo, "through", hi) + } + + return cur +} + +// appendBlockLocked rebuilds one chain block as journal entries folded onto +// cur — the same encodings the live build publishes, so a rebuilt block is +// byte-identical to the original. +func (p *Publisher) appendBlockLocked(fresh *journal, cur commitment.Head, block *types.Block) (commitment.Head, bool) { + header := block.Header() + n := header.Number.Uint64() + + open := openEntry(commitment.OpenContext{ + Number: n, + Timestamp: header.Time, + ParentHash: header.ParentHash, + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, cur) + + next, err := foldEntry(cur, open) + if err != nil { + p.fail("backfill fold open", "number", n, "err", err) + + return cur, false + } + + fresh.append(open, cur, next, entryOpen, n, 0, nil) + cur = next + + for _, tx := range block.Transactions() { + raw, err := tx.MarshalBinary() + if err != nil { + p.fail("backfill encode transaction", "hash", tx.Hash(), "err", err) + + return cur, false + } + + rec := recordEntry(raw, cur) + next = commitment.FoldTxs(cur, [][]byte{raw}) + fresh.append(rec, cur, next, entryRecord, n, 0, []common.Hash{tx.Hash()}) + cur = next + } + + raw, err := rlp.EncodeToBytes(header) + if err != nil { + p.fail("backfill encode header", "number", n, "err", err) + + return cur, false + } + + seal := sealEntry(raw, cur) + next = commitment.FoldSeal(cur, commitment.SealedHash(raw)) + fresh.append(seal, cur, next, entrySeal, n, 0, nil) + + return next, true +} + +// refoldLocked swaps the lineage: journal items from the first window at or +// above keepFrom are re-prefixed onto s (mid-window republish); +// everything earlier is abandoned to supersession convergence. +func (p *Publisher) refoldLocked(s commitment.Head, keepFrom uint64, counter *metrics.Counter) reconcileOutcome { + suffix := p.journal.suffixFromHeight(keepFrom) + + // Entries behind the kept suffix are abandoned to supersession + // convergence; count the unconfirmed ones as dropped. Collapsed + // entries are excluded — the backfill accounts for them (rebuilt or + // skipped) itself. A re-anchor that abandons nothing (a fresh process + // resuming at the store tail) is not a forward jump — only count the + // jump when entries are left behind. + abandoned := p.unackedLocked() - len(suffix) - p.pendingEntries + if abandoned > 0 { + publishDropMeter.Mark(int64(abandoned)) + p.restoreAbandonedDebtLocked(suffix) + } else if counter == reconcileForwardJump { + counter = nil + } + + fresh := newJournal() + fresh.nextSeq = p.journal.nextSeq + p.ackedSeq = fresh.nextSeq - 1 + + cur := p.backfillLocked(fresh, s) + + // Everything appended so far is the backfill batch; the suffix follows. + batchCeiling := fresh.nextSeq - 1 + + for _, item := range suffix { + entry, next, err := refoldEntry(cur, item) + if err != nil { + p.fail("refold entry", "err", err) + + return recTerminal + } + + // Refolded entries are all undelivered: watermark 0 keeps a + // stacked re-anchor from evicting its own seals. + fresh.append(entry, cur, next, item.kind, item.height, 0, item.txHashes) + cur = next + } + + p.installSwapLocked(fresh, cur, s) + + // An unfinished drain gates the suffix behind the batch: a suffix seal + // landing out of order would advance the store's sealed tip past the + // gap, and the floor above would then skip the middle forever. The + // drain cycle re-arms this ceiling each batch and lifts it with the + // last one. + if p.pendingFrom != 0 { + p.hold = hold{after: batchCeiling, kind: holdBuild} + } + + if counter != nil { + counter.Inc(1) + } + + return recOK +} + +// installSwapLocked commits a rebuilt lineage: it points the journal, head and +// anchor at the swap result and resets every field a lineage swap must clear +// together — the confirmed-anchor latches, any adopted window (folded on +// state that no longer exists), and the send hold — then re-derives the open +// window and wakes the transport. Centralizing this keeps the "reset as a +// unit" invariant in one place across the refold and complete-in-place paths. +func (p *Publisher) installSwapLocked(fresh *journal, head, anchor commitment.Head) { + p.journal = fresh + p.head = head + p.anchor = anchor + p.anchored, p.confirmed = true, true + p.adopt = nil + p.hold = clearedHold() + p.syncWindowLocked() + publishQueueGauge.Update(int64(p.unackedLocked())) + p.signalWake() +} + +// syncWindowLocked recomputes the open-window bookkeeping after a lineage +// swap. A worker mid-window whose entries were all dropped must not have its +// remaining records published without their open (awaitOpen). +func (p *Publisher) syncWindowLocked() { + if idx := p.journal.openStart(); idx >= 0 { + p.curHeight = p.journal.items[idx].height + p.awaitOpen = false + + return + } + + if p.curHeight != 0 { + p.awaitOpen = true + } + + p.curHeight = 0 +} diff --git a/eth/sequencer/classify_test.go b/eth/sequencer/classify_test.go new file mode 100644 index 0000000000..42cb7e26c8 --- /dev/null +++ b/eth/sequencer/classify_test.go @@ -0,0 +1,474 @@ +package sequencer + +import ( + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// lineagePublisher builds a bare publisher holding a confirmed one-block +// lineage plus an open window at height 2 with one record. +func lineagePublisher(t *testing.T, chain chainReader) (*Publisher, commitment.Head) { + t.Helper() + + p := barePublisher() + p.chain = chain + + header := testHeader(1, common.Hash{0xef}) + p.OpenBlock(1, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + + p.OpenBlock(2, header.Time+1, header.Hash(), header.GasLimit, header.BaseFee) + p.PublishTx(testTx(t, 0)) + + // Everything through the seal of block 1 is store-confirmed. + items, _ := p.journal.after(0) + sealItem := items[1] + p.ackedSeq = sealItem.seq + p.anchor = sealItem.post + p.confirmed = true + + return p, sealItem.post +} + +func TestApplyTailRow1Replay(t *testing.T) { + p, anchor := lineagePublisher(t, &fakeChain{}) + + if out := p.applyTail(tailInfo{s: anchor}); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if !p.anchored || p.head == anchor { + t.Fatalf("must anchor and keep the unconfirmed suffix (anchored=%v)", p.anchored) + } +} + +func TestApplyTailRow1FindPost(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) + + // The store confirmed one entry further than we knew: the open of 2. + items, _ := p.journal.after(p.ackedSeq) + openItem := items[0] + + if out := p.applyTail(tailInfo{s: openItem.post}); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.ackedSeq != openItem.seq || p.anchor != openItem.post { + t.Fatalf("frontier not advanced: acked=%d", p.ackedSeq) + } +} + +// A foreign tail while our window is unsealed never writes: the publisher +// holds and the seal flush resolves the height. +func TestApplyTailForeignUnsealedHolds(t *testing.T) { + cases := []struct { + name string + info tailInfo + }{ + {name: "open window at pending", info: tailInfo{s: commitment.Head{0x99}, tipOpen: true, tipOpenHeight: 2}}, + {name: "open window above pending", info: tailInfo{s: commitment.Head{0x33}, tipOpen: true, tipOpenHeight: 5}}, + {name: "sealed past pending", info: tailInfo{s: commitment.Head{0x55}, haveSeal: true, lastSealHeight: 2, lastSealHash: common.Hash{0xcc}}}, + {name: "store behind", info: tailInfo{s: commitment.Head{0x77}, haveSeal: true, lastSealHeight: 1, lastSealHash: common.Hash{0xaa}}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + journalBefore := len(p.journal.items) + + if out := p.applyTail(tc.info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.hold.after != p.ackedSeq { + t.Fatalf("unsealed window against a foreign tail must gate unacked entries (ceiling=%d acked=%d)", p.hold.after, p.ackedSeq) + } + + if len(p.journal.items) != journalBefore { + t.Fatal("hold must not touch the lineage") + } + }) + } +} + +// The same foreign tails with the window sealed (flush in flight) re-anchor +// the sealed window onto the store head — supersede for contention shapes, +// forward jump for a store still behind. +func TestApplyTailSealedFlushReanchors(t *testing.T) { + cases := []struct { + name string + info tailInfo + wantNone bool + }{ + {name: "open window at pending", info: tailInfo{s: commitment.Head{0x99}, tipOpen: true, tipOpenHeight: 2}}, + {name: "sealed divergent at pending", info: tailInfo{s: commitment.Head{0x55}, haveSeal: true, lastSealHeight: 2, lastSealHash: common.Hash{0xcc}}}, + {name: "parent sealed, clean tail", info: tailInfo{s: commitment.Head{0x77}, haveSeal: true, lastSealHeight: 1, lastSealHash: common.Hash{0xaa}}, wantNone: true}, + {name: "unsealed contender above flush", info: tailInfo{s: commitment.Head{0x44}, haveSeal: true, lastSealHeight: 1, lastSealHash: common.Hash{0xaa}, tipOpen: true, tipOpenHeight: 3}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + // Seal the pending window locally: the flush owns the height now. + header2 := testHeader(2, common.Hash{0x01}) + sealOnChain(p, fc, header2, []*types.Transaction{testTx(t, 0)}) + + // The block went out on the gate's liveness path (Unknown): + // from here the flush is post-broadcast repair, which is the + // behavior under test. + p.ConfirmSeal(time.Millisecond) + + supersedes := reconcileSupersede.Snapshot().Count() + jumps := reconcileForwardJump.Snapshot().Count() + + if out := p.applyTail(tc.info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + items, covered := p.journal.after(p.ackedSeq) + if !covered || len(items) == 0 { + t.Fatalf("re-anchored suffix missing: covered=%v len=%d", covered, len(items)) + } + + if items[0].kind != entryOpen || items[0].height != 2 { + t.Fatalf("suffix starts with kind %d height %d", items[0].kind, items[0].height) + } + + if got := commitment.Head(entryPrefix(items[0].entry)); got != tc.info.s { + t.Fatalf("re-prefixed onto %x, want %x", got, tc.info.s) + } + + if items[len(items)-1].kind != entrySeal { + t.Fatal("the flush suffix must end with the seal") + } + + if tc.wantNone { + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("re-delivery counted as supersede: %d -> %d", supersedes, got) + } + + if got := reconcileForwardJump.Snapshot().Count(); got != jumps { + t.Fatalf("re-delivery counted as jump: %d -> %d", jumps, got) + } + } else { + if got := reconcileSupersede.Snapshot().Count(); got != supersedes+1 { + t.Fatalf("supersede counter = %d, want %d", got, supersedes+1) + } + } + }) + } +} + +// Flushes stacked behind a reconcile backoff all re-anchor — oldest +// window first — so no sealed-on-chain height is left unsealed in the +// store (the quad-strand shape). +func TestStackedFlushesAllDeliver(t *testing.T) { + fc2 := &fakeChain{} + p, _ := lineagePublisher(t, fc2) + + // Two blocks seal while nothing delivers: the journal carries two + // complete flush windows beyond the acked prefix. + header2 := testHeader(2, common.Hash{0x01}) + sealOnChain(p, fc2, header2, []*types.Transaction{testTx(t, 0)}) + p.OpenBlock(3, header2.Time+1, header2.Hash(), header2.GasLimit, header2.BaseFee) + tx := testTx(t, 1) + p.PublishTx(tx) + sealOnChain(p, fc2, testHeader(3, header2.Hash()), []*types.Transaction{tx}) + + // The next build has already opened on top of the stack — the flush + // must stay visible through the trailing open (back-to-back builds + // leave almost no trailing-seal moments for a reconcile to land in). + header3 := testHeader(3, common.Hash{0x02}) + p.OpenBlock(4, header3.Time+1, header3.Hash(), header3.GasLimit, header3.BaseFee) + + // A foreign contender window stands above both flushes. + foreign := commitment.Head{0x44} + info := tailInfo{s: foreign, haveSeal: true, lastSealHeight: 1, lastSealHash: common.Hash{0xaa}, tipOpen: true, tipOpenHeight: 3} + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + items, covered := p.journal.after(p.ackedSeq) + if !covered || len(items) == 0 { + t.Fatalf("re-anchored suffix missing: covered=%v len=%d", covered, len(items)) + } + + if items[0].kind != entryOpen || items[0].height != 2 { + t.Fatalf("suffix starts with kind %d height %d, want the oldest flush window", items[0].kind, items[0].height) + } + + if got := commitment.Head(entryPrefix(items[0].entry)); got != foreign { + t.Fatalf("re-prefixed onto %x, want %x", got, foreign) + } + + seals := 0 + + for _, it := range items { + if it.kind == entrySeal { + seals++ + } + } + + if seals != 2 { + t.Fatalf("re-anchored suffix carries %d seals, want both", seals) + } +} + +// A seal flush whose window does not mirror the sealed block rebuilds the +// window — without erasing older flushes still undelivered behind it. +func TestSealRebuildPreservesStackedFlushes(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) + + // First flush: seals the standing window for height 2, undelivered. + header2 := testHeader(2, common.Hash{0x01}) + p.SealBlock(blockFor(header2, []*types.Transaction{testTx(t, 0)})) + + // Second build diverges from what seals (its window will not mirror + // the block), forcing the rebuild path. + p.OpenBlock(3, header2.Time+1, header2.Hash(), header2.GasLimit, header2.BaseFee) + p.PublishTx(testTx(t, 1)) + + sealedTx := testTx(t, 2) + p.SealBlock(blockFor(testHeader(3, header2.Hash()), []*types.Transaction{sealedTx})) + + p.mu.Lock() + defer p.mu.Unlock() + + heights := map[uint64]int{} + + for _, it := range p.journal.items { + if it.kind == entrySeal && it.seq > p.ackedSeq { + heights[it.height]++ + } + } + + if heights[2] != 1 || heights[3] != 1 { + t.Fatalf("stacked flushes lost in rebuild: undelivered seals per height = %v", heights) + } +} + +func TestApplyTailBetweenBlocksRebases(t *testing.T) { + p := barePublisher() + + // The store's sealed lineage is far past our stale flush and is the + // canonical chain locally: the chain moved on, so a rebase is right + // (a canonical sealed height past the flush outranks it). + p.chain = &fakeChain{canonical: map[uint64]common.Hash{8: {0xbb}}} + + header := testHeader(1, common.Hash{0xef}) + p.OpenBlock(1, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + p.curHeight = 0 // between blocks + + foreign := commitment.Head{0x22} + info := tailInfo{s: foreign, haveSeal: true, lastSealHeight: 8, lastSealHash: common.Hash{0xbb}, tipOpen: true, tipOpenHeight: 9} + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.head != foreign || p.anchor != foreign { + t.Fatalf("head/anchor not rebased: %x/%x", p.head, p.anchor) + } + + if items, _ := p.journal.after(p.ackedSeq); len(items) != 0 { + t.Fatal("rebase must purge the lineage") + } +} + +func TestProbeFindsStoreEdge(t *testing.T) { + h := startHarness(t) + + seed := newTestPublisher(t, h, nil) + sealed := publishBlock(t, seed, 1, common.Hash{0xef}, 1) + sealed2 := publishBlock(t, seed, 2, sealed.Hash(), 1) + publishBlock(t, seed, 3, sealed2.Hash(), 1) + waitHead(t, h, seed, 5*time.Second) + seed.Close() + + p := newTestPublisher(t, h, nil) + waitFor(t, 5*time.Second, func() bool { return p.isAnchored() }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + edge, found, err := p.read.probeDown(ctx, 10) + if err != nil || !found || edge != 3 { + t.Fatalf("probeDown = %d/%v/%v, want 3", edge, found, err) + } + + up, err := p.read.probeUp(ctx, 1) + if err != nil || up != 3 { + t.Fatalf("probeUp = %d/%v, want 3", up, err) + } +} + +// A lagging ack from a lineage an adoption has since replaced must not +// regress the frontier — the phantom eviction gap it faked used to force +// a spurious forward-jump superseding a window under silent adopt. +func TestLaggingAckAfterLineageSwapIgnored(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) + + inflight, _ := p.journal.after(p.ackedSeq) + openItem := inflight[0] + + var win []*pb.Entry + for _, it := range inflight { + win = append(win, it.entry) + } + + p.mu.Lock() + info := tailInfo{s: p.head, tipOpen: true, tipOpenHeight: 2, window: win} + + a, items, ok := parseWindow(info) + if !ok { + p.mu.Unlock() + t.Fatal("window unparseable") + } + + p.adoptWindowLocked(info, a, items) + acked := p.ackedSeq + p.mu.Unlock() + + jumps := reconcileForwardJump.Snapshot().Count() + + // The transport's lagging ack for the pre-swap open arrives. + p.retire(openItem, time.Now()) + + p.mu.Lock() + after, anchor := p.ackedSeq, p.anchor + p.mu.Unlock() + + if after != acked { + t.Fatalf("stale ack regressed ackedSeq: %d -> %d", acked, after) + } + + if out := p.applyTail(tailInfo{s: anchor}); out != recOK { + t.Fatalf("row-1 outcome: %v", out) + } + + if got := reconcileForwardJump.Snapshot().Count(); got != jumps { + t.Fatalf("spurious forward-jump: %d -> %d", jumps, got) + } +} + +// completeExtendedWindowLocked abandons older stacked flushes — their +// content is pinned byte-identically in the store lineage the window +// folds on — but must count them as drops. +func TestExtendedCompletionCountsAbandonedStack(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) + + header2 := testHeader(2, common.Hash{0x01}) + p.SealBlock(blockFor(header2, []*types.Transaction{testTx(t, 0)})) + + p.OpenBlock(3, header2.Time+1, header2.Hash(), header2.GasLimit, header2.BaseFee) + tx := testTx(t, 1) + p.PublishTx(tx) + p.SealBlock(blockFor(testHeader(3, header2.Hash()), []*types.Transaction{tx})) + + p.mu.Lock() + defer p.mu.Unlock() + + ours := p.journal.suffixFromHeight(3) + win := []*pb.Entry{ours[0].entry, ours[1].entry} + info := tailInfo{s: ours[1].post, tipOpen: true, tipOpenHeight: 3, window: win} + + drops := publishDropMeter.Snapshot().Count() + + if !p.completeExtendedWindowLocked(info, 3) { + t.Fatal("completion did not fire") + } + + if got := publishDropMeter.Snapshot().Count(); got <= drops { + t.Fatalf("abandoned stack uncounted: %d -> %d", drops, got) + } +} + +// A foreign seal above our pending flush that the local chain does NOT +// consider canonical (a reorg loser, or a lineage not yet imported) must +// not drop our canonical seal: hold and let our next flush re-anchor. +func TestForeignSealAboveNonCanonicalHolds(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) // nil canonical map: seal not canonical + + header2 := testHeader(2, common.Hash{0x01}) + p.SealBlock(blockFor(header2, []*types.Transaction{testTx(t, 0)})) + + forwardBefore := reconcileForwardJump.Snapshot().Count() + + // A foreign seal at height 3 stands above our pending flush for 2, but + // GetCanonicalHash(3) is zero (not this hash) → not canonical. + info := tailInfo{s: commitment.Head{0x22}, haveSeal: true, lastSealHeight: 3, lastSealHash: common.Hash{0xbb}} + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + // Our pending seal must survive (held, not dropped) and no jump counted. + items, covered := p.journal.after(p.ackedSeq) + if !covered || len(items) == 0 || items[len(items)-1].kind != entrySeal { + t.Fatalf("pending seal dropped for a non-canonical foreign tail: covered=%v len=%d", covered, len(items)) + } + + if p.hold.kind != holdSticky { + t.Fatalf("expected holdSticky, got holdKind=%d", p.hold.kind) + } + + if got := reconcileForwardJump.Snapshot().Count(); got != forwardBefore { + t.Fatalf("dropped a canonical seal (forward jump %d -> %d)", forwardBefore, got) + } +} + +// A foreign seal above our pending flush that IS canonical locally means +// we were genuinely reorged past: drop the stale seal and rebase. +func TestForeignSealAboveCanonicalRebases(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{canonical: map[uint64]common.Hash{3: {0xbb}}}) + + header2 := testHeader(2, common.Hash{0x01}) + p.SealBlock(blockFor(header2, []*types.Transaction{testTx(t, 0)})) + + info := tailInfo{s: commitment.Head{0x22}, haveSeal: true, lastSealHeight: 3, lastSealHash: common.Hash{0xbb}} + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.anchor != (commitment.Head{0x22}) { + t.Fatalf("canonical reorg-past must rebase: anchor=%x", p.anchor) + } +} diff --git a/eth/sequencer/consumer.go b/eth/sequencer/consumer.go new file mode 100644 index 0000000000..8d0cd539b5 --- /dev/null +++ b/eth/sequencer/consumer.go @@ -0,0 +1,428 @@ +package sequencer + +import ( + "context" + "errors" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/log" +) + +const consumerRetryDelay = 2 * time.Second + +// Consumer follows the sequence store stream on an RPC node, re-executes it +// on top of canonical state, and fills the preconf receipt Index that +// eth_getTransactionReceipt consults for not-yet-imported transactions. +// +// Position and application are handled separately, per the design's +// chain-everything-apply-selectively rule: only commitment gaps and transport +// errors abandon the stream position (warm resume by running head, falling +// back to a cold block anchor, falling back to the earliest retained entry); +// application problems — unknown parents, unavailable state, execution or +// seal divergence — void the speculative work and skip forward until an open +// record re-anchors on a canonical block. +type Consumer struct { + chain *core.BlockChain + endpoint string + index *Index + + cancel context.CancelFunc + done chan struct{} +} + +// NewConsumer returns a stopped consumer. Determinism preconditions (Rio +// active, coinbase map present) are re-checked per session, not here — a +// node still syncing pre-Rio history becomes eligible once it catches up. +func NewConsumer(endpoint string, chain *core.BlockChain) (*Consumer, error) { + if chain.Config().Bor == nil { + return nil, errors.New("sequencer consumer requires a bor chain") + } + + return &Consumer{ + chain: chain, + endpoint: endpoint, + index: NewIndex(), + }, nil +} + +// Index exposes the preconf receipts for the RPC layer. +func (c *Consumer) Index() *Index { + return c.index +} + +// Start launches the stream-follow loop. +func (c *Consumer) Start() { + ctx, cancel := context.WithCancel(context.Background()) + c.cancel = cancel + c.done = make(chan struct{}) + + go c.run(ctx) + go c.evictLoop(ctx) +} + +// Close stops the consumer and waits for the follow loop to exit. +func (c *Consumer) Close() { + if c.cancel != nil { + c.cancel() + <-c.done + } +} + +// deterministic reports whether the producer's execution context is +// reproducible at the current head: pre-Rio the EVM coinbase is the +// producer's own address, unknowable pre-seal; post-Rio it is +// CalculateCoinbase from the chain config — reproducible only when the +// coinbase map is set. +func (c *Consumer) deterministic() error { + config := c.chain.Config().Bor + + head := c.chain.CurrentBlock().Number + if !config.IsRio(head) { + return errors.New("rio fork not active at current head") + } + + if common.HexToAddress(config.CalculateCoinbase(head.Uint64())) == (common.Address{}) { + return errors.New("chain config has no coinbase map") + } + + return nil +} + +func (c *Consumer) run(ctx context.Context) { + defer close(c.done) + + var sess *session + + for { + var err error + if derr := c.deterministic(); derr != nil { + err = fmt.Errorf("preconf re-execution not deterministic yet: %w", derr) + } else { + sess, err = c.follow(ctx, sess) + } + + if ctx.Err() != nil { + return + } + + log.Warn("Sequence stream session ended", "err", err) + + select { + case <-ctx.Done(): + return + case <-time.After(consumerRetryDelay): + } + } +} + +// evictLoop drops preconf receipts for heights the canonical chain has +// imported — the normal receipt path serves them from there on. +func (c *Consumer) evictLoop(ctx context.Context) { + heads := make(chan core.ChainHeadEvent, 16) + sub := c.chain.SubscribeChainHeadEvent(heads) + + defer sub.Unsubscribe() + + for { + select { + case <-ctx.Done(): + return + case head := <-heads: + c.index.EvictThrough(head.Header.Number.Uint64()) + case <-sub.Err(): + return + } + } +} + +// resumeRequest picks the stream position: warm (the session's running head), +// cold (a block anchor at the canonical head), or from the earliest retained +// entry. attempt counts NOT_FOUND fallbacks within one session start. +func (c *Consumer) resumeRequest(sess *session, attempt int) *pb.StreamRequest { + if sess != nil && sess.seeded && attempt == 0 { + return &pb.StreamRequest{After: &pb.StreamRequest_Head{Head: sess.head.Bytes()}} + } + + if attempt <= 1 { + return &pb.StreamRequest{After: &pb.StreamRequest_Block{Block: c.chain.CurrentBlock().Number.Uint64()}} + } + + return &pb.StreamRequest{} +} + +// follow runs one streaming session. It returns the session for a warm +// resume when the stream position is still valid, or nil when position was +// lost (commitment gap, malformed entry) and the next attempt must re-anchor. +func (c *Consumer) follow(ctx context.Context, sess *session) (*session, error) { + conn, err := grpc.NewClient(c.endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return sess, fmt.Errorf("dial sequence store: %w", err) + } + + defer func() { + if cerr := conn.Close(); cerr != nil { + log.Warn("Sequence store connection close", "err", cerr) + } + }() + + client := pb.NewConsumerServiceClient(conn) + + for attempt := 0; ; attempt++ { + stream, serr := client.Stream(ctx, c.resumeRequest(sess, attempt)) + if serr != nil { + return sess, fmt.Errorf("open stream: %w", serr) + } + + if attempt > 0 || sess == nil { + // Any non-warm position invalidates the old fold state. + sess = &session{consumer: c} + c.index.Reset() + } + + sess, err = c.consume(stream, sess) + if status.Code(err) == codes.NotFound && attempt < 2 { + continue + } + + return sess, err + } +} + +func (c *Consumer) consume(stream pb.ConsumerService_StreamClient, sess *session) (*session, error) { + for { + frame, err := stream.Recv() + if err != nil { + return sess, fmt.Errorf("stream recv: %w", err) + } + + entry := frame.GetEntry() + if entry == nil { + log.Info("Sequence stream live", "head", fmt.Sprintf("%x", sess.head[:8])) + + continue + } + + if err := sess.handle(entry); err != nil { + // Position lost: the caller must re-anchor, not resume. + return nil, err + } + } +} + +// session is one consistent stretch of the stream: a running commitment +// head, the speculative execution state, and the speculative seal hashes for +// BLOCKHASH resolution. env == nil between blocks and while skipping. +type session struct { + consumer *Consumer + head commitment.Head + seeded bool + env *blockEnv // block currently being applied + parked *state.StateDB // post-state of the last speculative seal + tip common.Hash // last speculatively sealed hash + sealed map[uint64]common.Hash +} + +// handle verifies one entry against the commitment chain, folds it, and +// applies it best-effort. An error means the stream position itself is +// invalid; application problems skip instead (void speculative work, wait +// for a re-anchoring open). +func (s *session) handle(entry *pb.Entry) error { + prefix, next, err := s.fold(entry) + if err != nil { + return err + } + + if !s.seeded { + // Cold start: adopt the stream's prefix as the seed; integrity from + // here comes from the chain and the self-authenticating seals. + s.head = prefix + s.seeded = true + + var refold error + if _, next, refold = s.fold(entry); refold != nil { + return refold + } + } else if prefix != s.head { + return fmt.Errorf("commitment gap: entry prefix %x != running head %x", prefix[:8], s.head[:8]) + } + + s.apply(entry) + s.head = next + + return nil +} + +// fold checks an entry's wire shape, computes its post-fold head over the +// running head, and returns the entry's claimed prefix alongside it. +func (s *session) fold(entry *pb.Entry) (commitment.Head, commitment.Head, error) { + prefix := entryPrefix(entry) + if len(prefix) != 32 { + return commitment.Head{}, commitment.Head{}, errors.New("malformed entry prefix") + } + + if open := entry.GetBlockOpen(); open != nil && len(open.GetParentHash()) != 32 { + return commitment.Head{}, commitment.Head{}, errors.New("malformed open parent hash") + } + + next, err := foldEntry(s.head, entry) + if err != nil { + return commitment.Head{}, commitment.Head{}, err + } + + return commitment.Head(prefix), next, nil +} + +func (s *session) apply(entry *pb.Entry) { + switch kind := entry.GetKind().(type) { + case *pb.Entry_BlockOpen: + s.applyOpen(kind.BlockOpen) + case *pb.Entry_Record: + s.applyRecord(kind.Record) + case *pb.Entry_BlockSeal: + s.applySeal(kind.BlockSeal) + } +} + +// skip voids the speculative work from a height upward and waits for the +// next re-anchoring open. +func (s *session) skip(from uint64, reason string, args ...any) { + log.Warn("Preconf application skipped: "+reason, args...) + s.consumer.index.ClearFrom(from) + s.env = nil + s.parked = nil +} + +// applyOpen starts a speculative block. A canonical parent is preferred as +// the base even on the happy path — it bounds how long one speculative +// StateDB lives; the parked post-seal state covers parents the chain hasn't +// imported yet. +func (s *session) applyOpen(open *pb.BlockOpen) { + parent := common.BytesToHash(open.GetParentHash()) + number := open.GetBlockNumber() + + if s.env != nil { + // A new open while a block is still open is a producer rebuild of + // the in-progress height; its state is unusable. + s.skip(s.env.header.Number.Uint64(), "producer rebuilt in-progress block", "number", number) + } + + if header := s.consumer.chain.GetHeaderByHash(parent); header != nil && + s.consumer.chain.GetCanonicalHash(header.Number.Uint64()) == parent { + if number != header.Number.Uint64()+1 { + s.skip(number, "open height is not parent height+1", "number", number, "parent", header.Number) + + return + } + + statedb, err := s.consumer.chain.StateAt(header.Root) + if err != nil { + s.skip(number, "parent state unavailable", "parent", parent, "err", err) + + return + } + + s.consumer.index.ClearFrom(number) + s.pruneSealed(number) + + s.tip = parent + s.parked = nil + s.env = newBlockEnv(s.consumer.chain, statedb, open, s.sealed) + + return + } + + if parent == s.tip && s.parked != nil { + s.env = newBlockEnv(s.consumer.chain, s.parked, open, s.sealed) + s.parked = nil + + return + } + + s.skip(number, "open parent neither canonical nor speculative tip", "parent", parent) +} + +func (s *session) applyRecord(record *pb.Record) { + if s.env == nil { + return // skipping until the next re-anchoring open + } + + for _, raw := range record.GetTransactions() { + start := time.Now() + + tx, receipt, err := s.env.applyRaw(raw) + if err != nil { + s.skip(s.env.header.Number.Uint64(), "re-execution diverged", "err", err) + + return + } + + s.consumer.index.Add(tx, receipt) + preconfApplyTimer.UpdateSince(start) + } +} + +func (s *session) applySeal(seal *pb.BlockSeal) { + if s.env == nil { + return // skipping until the next re-anchoring open + } + + sealed, err := decodeSealHeader(seal.GetHeader()) + if err != nil { + s.skip(s.env.header.Number.Uint64(), "undecodable sealed header", "err", err) + + return + } + + if err := s.env.checkSeal(sealed); err != nil { + s.skip(s.env.header.Number.Uint64(), "seal cross-check failed", "err", err) + + return + } + + sealedHash := common.Hash(commitment.SealedHash(seal.GetHeader())) + number := s.env.header.Number.Uint64() + + s.consumer.index.Seal(number, sealedHash) + + if s.sealed == nil { + s.sealed = map[uint64]common.Hash{} + } + + s.sealed[number] = sealedHash + + // BLOCKHASH reaches at most 256 back; older speculative hashes would + // otherwise accumulate for the whole parked stretch. + for height := range s.sealed { + if height+256 < number { + delete(s.sealed, height) + } + } + + // Park the post-block state as the base for the next open. + s.tip = sealedHash + s.parked = s.env.statedb + s.env = nil +} + +// pruneSealed drops speculative seal hashes that a canonical re-anchor +// superseded (at or above the new height) or that BLOCKHASH can no longer +// reach (more than 256 below it). +func (s *session) pruneSealed(number uint64) { + for height := range s.sealed { + if height >= number || height+256 < number { + delete(s.sealed, height) + } + } +} diff --git a/eth/sequencer/consumer_test.go b/eth/sequencer/consumer_test.go new file mode 100644 index 0000000000..bfd182adbe --- /dev/null +++ b/eth/sequencer/consumer_test.go @@ -0,0 +1,114 @@ +package sequencer + +import ( + "math/big" + "strings" + "testing" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" +) + +func testSession() *session { + return &session{consumer: &Consumer{index: NewIndex()}} +} + +// A cold-started session adopts the first entry's prefix as its seed and +// advances past it, so a mid-stream resume verifies from that point on. +func TestSessionColdSeedAdoptsTheFirstPrefix(t *testing.T) { + seed := commitment.Head{0xaa, 0xbb} + raw := []byte{0x01, 0x02} + sess := testSession() + + if err := sess.handle(recordEntry(raw, seed)); err != nil { + t.Fatalf("cold-seed handle: %v", err) + } + + if want := commitment.FoldTxs(seed, [][]byte{raw}); sess.head != want { + t.Fatalf("head %x, want the seed folded past the entry %x", sess.head[:8], want[:8]) + } +} + +// A seeded session accepts only entries whose prefix extends its running +// head; anything else is a gap and invalidates the stream position. +func TestSessionRejectsACommitmentGap(t *testing.T) { + seed := commitment.Seed(testChainID) + sess := testSession() + + if err := sess.handle(recordEntry([]byte{0x01}, seed)); err != nil { + t.Fatalf("seeding handle: %v", err) + } + + next := recordEntry([]byte{0x02}, sess.head) + if err := sess.handle(next); err != nil { + t.Fatalf("chained handle: %v", err) + } + + gapped := recordEntry([]byte{0x03}, commitment.Head{0xde, 0xad}) + if err := sess.handle(gapped); err == nil || !strings.Contains(err.Error(), "commitment gap") { + t.Fatalf("gapped entry must fail the position, got %v", err) + } +} + +// Wire entries with short commitment or hash fields must fail the fold, not +// panic the fixed-width conversions. +func TestSessionFoldRejectsMalformedEntries(t *testing.T) { + sess := testSession() + + short := recordEntry([]byte{0x01}, commitment.Head{}) + short.GetRecord().PrefixCommitment = []byte{0x01} + + if _, _, err := sess.fold(short); err == nil || !strings.Contains(err.Error(), "malformed entry prefix") { + t.Fatalf("short prefix must fail the fold, got %v", err) + } + + open := openEntry(commitment.OpenContext{Number: 7, BaseFee: big.NewInt(1)}, commitment.Head{}) + open.GetBlockOpen().ParentHash = []byte{0x01, 0x02} + + if _, _, err := sess.fold(open); err == nil || !strings.Contains(err.Error(), "malformed open parent hash") { + t.Fatalf("short parent hash must fail the fold, got %v", err) + } +} + +// The consumer's fold must agree with the publisher-side foldEntry on every +// entry kind — both ends of the wire chain the same bytes. +func TestSessionFoldMatchesFoldEntry(t *testing.T) { + head := commitment.Seed(testChainID) + oc := commitment.OpenContext{ + Number: 3, + Timestamp: 100, + ParentHash: [32]byte{0x0f}, + GasLimit: 30_000_000, + BaseFee: big.NewInt(32), + } + + entries := []*pb.Entry{ + openEntry(oc, head), + recordEntry([]byte{0x01, 0x02}, head), + sealEntry([]byte{0x0a, 0x0b}, head), + } + + for _, entry := range entries { + sess := testSession() + sess.head = head + sess.seeded = true + + prefix, next, err := sess.fold(entry) + if err != nil { + t.Fatalf("fold: %v", err) + } + + if prefix != head { + t.Fatalf("claimed prefix %x, entry was built on %x", prefix[:8], head[:8]) + } + + want, err := foldEntry(head, entry) + if err != nil { + t.Fatalf("foldEntry: %v", err) + } + + if next != want { + t.Fatalf("fold %x diverges from foldEntry %x", next[:8], want[:8]) + } + } +} diff --git a/eth/sequencer/debt_test.go b/eth/sequencer/debt_test.go new file mode 100644 index 0000000000..be310d7049 --- /dev/null +++ b/eth/sequencer/debt_test.go @@ -0,0 +1,206 @@ +package sequencer + +import ( + "testing" + + "github.com/0xPolygon/sequence-store-proto/commitment" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// A probe-inferred boundary is not a decoded seal: recording it as the +// sealed tip told every backfill the height was owed nothing, and an +// outage's partial delivery at exactly that height became a permanent hole +// (devnet height 364: partial 340/1399, skipped by every repairer). +func TestInferredBoundaryDoesNotAdvanceTheSealedTip(t *testing.T) { + p := barePublisher() + p.chain = &fakeChain{} + + p.applyTail(tailInfo{haveSeal: true, lastSealHeight: 9}) // no sealDecoded + + p.mu.Lock() + tip := p.storeSealedTip + p.mu.Unlock() + + if tip != 0 { + t.Fatalf("storeSealedTip = %d from an inferred boundary, want 0: "+ + "only a decoded seal proves anything is sealed", tip) + } +} + +// Priming from an undecoded boundary includes the boundary height itself: it +// may be a partial delivery, and a duplicate generation is cheaper than a +// hole. +func TestPrimeIncludesAnUndecodedBoundary(t *testing.T) { + p := barePublisher() + p.chain = &fakeChain{} + + p.mu.Lock() + p.primeBackfillLocked(tailInfo{haveSeal: true, lastSealHeight: 7}, 10) + from, to := p.pendingFrom, p.pendingTo + p.mu.Unlock() + + if from != 7 || to != 9 { + t.Fatalf("pending = [%d,%d], want [7,9]: an inferred boundary at 7 "+ + "may be a partial window and must be re-delivered", from, to) + } +} + +// And a decoded seal at the boundary starts the range above it, as before. +func TestPrimeExcludesADecodedSeal(t *testing.T) { + p := barePublisher() + p.chain = &fakeChain{} + + p.mu.Lock() + p.primeBackfillLocked(tailInfo{haveSeal: true, sealDecoded: true, lastSealHeight: 7}, 10) + from, to := p.pendingFrom, p.pendingTo + p.mu.Unlock() + + if from != 8 || to != 9 { + t.Fatalf("pending = [%d,%d], want [8,9]", from, to) + } +} + +// A refold that abandons unacked sealed heights must put them back in the +// pending range: the backfill advanced past them at build time, and without +// restoration nothing remembers the debt. +func TestAbandonedBackfillHeightsReturnToPending(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + parent := common.Hash{0xef} + for n := uint64(4); n <= 6; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, nil) + parent = header.Hash() + } + + // The journal holds rebuilt-but-unacked seals for 4..5, as after a + // backfill batch whose delivery lost its head race. pendingFrom has + // already advanced past them. + p.mu.Lock() + cur := commitment.Seed(testChainID) + fresh := newJournal() + for n := uint64(4); n <= 5; n++ { + next, ok := p.appendBlockLocked(fresh, cur, chain.blocks[n]) + if !ok { + p.mu.Unlock() + t.Fatal("appendBlock failed") + } + cur = next + } + p.journal = fresh + p.ackedSeq = 0 + p.pendingFrom, p.pendingTo = 6, 6 + + // A forward-jump refold abandons everything (empty suffix). The restored + // debt is drained by this same refold's backfill, so the proof is the + // rebuilt content: the fresh journal must carry the abandoned heights + // again, not silently forget them. + p.refoldLocked(commitment.Head{0x99}, ^uint64(0), nil) + + rebuilt := map[uint64]bool{} + for _, it := range p.journal.items { + if it.kind == entrySeal { + rebuilt[it.height] = true + } + } + p.mu.Unlock() + + for _, h := range []uint64{4, 5, 6} { + if !rebuilt[h] { + t.Fatalf("height %d not rebuilt after its batch was abandoned: "+ + "the debt was forgotten (rebuilt: %v)", h, rebuilt) + } + } +} + +// A partial abandonment restores only what was actually abandoned: heights +// kept by the refold's suffix are still owed by the journal itself, and +// re-adding them to pending would double-deliver every surviving flush. +func TestPartialAbandonmentRestoresOnlyTheAbandonedHeights(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + parent := common.Hash{0xef} + for n := uint64(4); n <= 6; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, nil) + parent = header.Hash() + } + + p.mu.Lock() + cur := commitment.Seed(testChainID) + fresh := newJournal() + for n := uint64(4); n <= 6; n++ { + next, ok := p.appendBlockLocked(fresh, cur, chain.blocks[n]) + if !ok { + p.mu.Unlock() + t.Fatal("appendBlock failed") + } + cur = next + } + p.journal = fresh + p.ackedSeq = 0 + + // The refold keeps height 6 (a live flush) and abandons 4-5. + p.refoldLocked(commitment.Head{0x99}, 6, nil) + from, to := p.pendingFrom, p.pendingTo + + rebuilt := map[uint64]int{} + for _, it := range p.journal.items { + if it.kind == entrySeal { + rebuilt[it.height]++ + } + } + p.mu.Unlock() + + // 4 and 5 return via the restored pending range (drained by this same + // refold's backfill); 6 survives via the suffix — exactly once each. + for _, h := range []uint64{4, 5, 6} { + if rebuilt[h] != 1 { + t.Fatalf("height %d appears %d times, want exactly once "+ + "(pending was [%d,%d])", h, rebuilt[h], from, to) + } + } +} + +// A collapse during an active drain reaches heights below pendingFrom — the +// stranded batch suffix is the journal's oldest content — and must merge +// into the pending range. Clobbering pendingTo downward inverted the range, +// which the backfill read as "nothing owed" and zeroed: 25 heights of a +// 50-block outage evaporated as permanent holes exactly that way. +func TestCollapseMergesIntoThePendingRange(t *testing.T) { + p := barePublisher() + p.pendingFrom, p.pendingTo = 254, 270 + + parent := common.Hash{0xee} + + for n := uint64(228); n <= 231; n++ { + header := testHeader(n, parent) + p.OpenBlock(n, header.Time, parent, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + parent = header.Hash() + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.unackedSealsLocked() > journalHotSeals { + t.Fatalf("collapse did not run: %d unacked seals", p.unackedSealsLocked()) + } + + if p.pendingFrom != 228 { + t.Fatalf("pendingFrom = %d, want 228: the collapsed height must extend the range downward", p.pendingFrom) + } + + if p.pendingTo != 270 { + t.Fatalf("pendingTo = %d, want 270: the upper bound must never move down", p.pendingTo) + } + + if p.pendingEntries == 0 { + t.Fatal("collapsed entries were not accounted") + } +} diff --git a/eth/sequencer/degraded_test.go b/eth/sequencer/degraded_test.go new file mode 100644 index 0000000000..2abc850465 --- /dev/null +++ b/eth/sequencer/degraded_test.go @@ -0,0 +1,194 @@ +package sequencer + +import ( + "context" + "sync" + "testing" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ethereum/go-ethereum/common" +) + +// flakyConsumer drops a fixed share of tail reads, reproducing the devnet +// state that a clean in-process store cannot: "rung out of budget" fired 68 +// and 77 times in one five-minute window, and every decision that depends on +// a read was making it blind that often. +// +// Failures are counted rather than random so a failure here reproduces +// exactly. slow adds latency instead of an error, for the case where the read +// returns but too late to be useful. +type flakyConsumer struct { + pb.ConsumerServiceClient + + mu sync.Mutex + calls int + failEvery int + slow time.Duration + failures int +} + +func (c *flakyConsumer) Range(ctx context.Context, req *pb.RangeRequest, opts ...grpc.CallOption) (*pb.RangeResponse, error) { + c.mu.Lock() + c.calls++ + drop := c.failEvery > 0 && c.calls%c.failEvery == 0 + + if drop { + c.failures++ + } + + slow := c.slow + c.mu.Unlock() + + if slow > 0 { + time.Sleep(slow) + } + + if drop { + return nil, status.Error(codes.DeadlineExceeded, "tail read out of budget") + } + + return c.ConsumerServiceClient.Range(ctx, req, opts...) +} + +// degrade wraps a publisher's consumer client so its reads start failing. +func degrade(p *Publisher, failEvery int, slow time.Duration) *flakyConsumer { + f := &flakyConsumer{failEvery: failEvery, slow: slow} + + p.mu.Lock() + defer p.mu.Unlock() + + f.ConsumerServiceClient = p.read.cons + p.read.cons = f + + return f +} + +// blockableConsumer can be switched off and on, so a test can correlate a +// decision with whether the read behind it worked. A raw failure ratio does +// not discriminate: readTail issues several Range calls per check, so +// "seals <= successful reads" holds even when every decision is blind. +type blockableConsumer struct { + pb.ConsumerServiceClient + + mu sync.Mutex + blocked bool +} + +func (c *blockableConsumer) Range(ctx context.Context, req *pb.RangeRequest, opts ...grpc.CallOption) (*pb.RangeResponse, error) { + c.mu.Lock() + blocked := c.blocked + c.mu.Unlock() + + if blocked { + return nil, status.Error(codes.DeadlineExceeded, "tail read out of budget") + } + + return c.ConsumerServiceClient.Range(ctx, req, opts...) +} + +func (c *blockableConsumer) block(v bool) { + c.mu.Lock() + defer c.mu.Unlock() + + c.blocked = v +} + +// A contested height refuses without needing a read at all: the STALE that +// armed the hold is the store's own verdict, so degraded reads cannot turn +// contention into a blind seal — the failure that once sealed 10 of 10 +// contested heights while the store was unreadable. An uncontested drained +// producer with an unreadable store still seals: production never waits. +func TestContestedRefusalSurvivesUnreadableStore(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 1)) + waitDrained(t, p, 5*time.Second) + + gate := &blockableConsumer{} + + p.mu.Lock() + gate.ConsumerServiceClient = p.read.cons + p.read.cons = gate + p.mu.Unlock() + + gate.block(true) + + // Uncontested, drained, store unreadable: liveness seals. + if !awaitOurWindow(p, 300*time.Millisecond) { + t.Fatal("an unreadable store blocked an uncontested seal") + } + + // Contested: the hold refuses with no read required. + p.mu.Lock() + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + p.mu.Unlock() + + for height := uint64(2); height <= 11; height++ { + p.mu.Lock() + p.curHeight = height + p.mu.Unlock() + + if awaitOurWindow(p, 200*time.Millisecond) { + t.Fatalf("sealed contested height %d with the store unreadable", height) + } + } +} + +// Sustained contention across many heights, not one. The devnet had 36 +// contested heights in a single run; a per-height test cannot show damage +// that only accumulates. +func TestSustainedContentionAuditsBounded(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + degrade(p, 4, 0) + + const heights = 8 + + rivals := 0 + + for height := uint64(2); height <= heights; height++ { + // A rival opens its own generation and gets acked, then we contend. + foreignWindow(t, h, height, parent, testTx(t, 9)) + rivals++ + + p.mu.Lock() + p.curHeight = height + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + p.mu.Unlock() + + awaitOurWindow(p, 200*time.Millisecond) + + parent = common.Hash{byte(height)} // each rival window on its own tip + } + + // Nothing this node published can be stranded: it never sealed, so the + // only records at risk are the rivals' own. + audit := auditStore(t, h, map[uint64][]common.Hash{1: sealed1Txs(t, h)}) + + if len(audit.Displaced) > 0 { + t.Fatalf("sustained contention displaced %d records that were "+ + "promised at a specific height", len(audit.Displaced)) + } + + if len(audit.Revoked) > rivals { + t.Fatalf("revoked %d records but only %d rivals published: contention "+ + "is costing more than the losers' own promises", + len(audit.Revoked), rivals) + } +} diff --git a/eth/sequencer/derive_test.go b/eth/sequencer/derive_test.go new file mode 100644 index 0000000000..262b4b7f1d --- /dev/null +++ b/eth/sequencer/derive_test.go @@ -0,0 +1,126 @@ +package sequencer + +import ( + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "github.com/ethereum/go-ethereum/common" + "google.golang.org/grpc" +) + +// The head a walk lands on has to be derived from the entries the walk read. +// Taking the store's word for it makes a passing CAS mean only that we echoed +// back the value we were handed, which is not a check on shared history. +func TestFoldedHeadMustMatchTheReportedHead(t *testing.T) { + seed := commitment.Seed(1) + open := &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: 2, + BlockTimestamp: 1700000002, + ParentHash: common.Hash{0xaa}.Bytes(), + GasLimit: 30_000_000, + BaseFee: big25gwei(), + PrefixCommitment: seed.Bytes(), + }}} + + real, err := foldEntry(seed, open) + if err != nil { + t.Fatalf("fold: %v", err) + } + + tests := []struct { + name string + reported commitment.Head + want bool + }{ + {"head the entries produce", real, true}, + {"head the entries do not produce", commitment.Head{0x9e}, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + f := &folder{cur: seed, ok: true} + f.fold(open) + f.reached(tc.reported) + + if f.ok != tc.want { + t.Fatalf("explained = %v, want %v", f.ok, tc.want) + } + }) + } +} + +// A block-anchored walk that comes back empty derives no head — but the empty +// page is itself the answer a boundary read needs: nothing follows the block +// we started at, so opening there cannot land on another producer's window. +func TestEmptyPageStillExplainsABoundary(t *testing.T) { + f := &folder{ok: true, awaiting: true} + f.reached(commitment.Head{0x11}) + + if !f.ok { + t.Fatal("an empty page was treated as an unexplained head: every " + + "clean boundary would hold and the chain would stop opening blocks") + } +} + +// Entries that arrive without a base we can establish summarize into a head +// we cannot account for — the case that must not be trusted. +func TestUnbasedEntriesLeaveTheHeadUnexplained(t *testing.T) { + f := &folder{ok: true, awaiting: true} + f.fold(&pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{}}}) + f.reached(commitment.Head{0x11}) + + if f.ok { + t.Fatal("a mid-window start explained a head it never derived") + } +} + +// tamperingConsumer reports a head that its own entries do not produce. +type tamperingConsumer struct { + pb.ConsumerServiceClient +} + +func (c *tamperingConsumer) Range(ctx context.Context, req *pb.RangeRequest, + opts ...grpc.CallOption, +) (*pb.RangeResponse, error) { + resp, err := c.ConsumerServiceClient.Range(ctx, req, opts...) + if err != nil || len(resp.GetEntries()) == 0 { + return resp, err + } + + resp.Next = commitment.Head{0x9e, 0x9e}.Bytes() + + return resp, nil +} + +// End to end: a store head that the entries behind it do not produce must not +// become the position this publisher builds on. +func TestTamperedHeadIsNotAdopted(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 0)) + + p.mu.Lock() + p.read.cons = &tamperingConsumer{ConsumerServiceClient: p.read.cons} + p.mu.Unlock() + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatal("adopted a window whose head the store misreported") + } + + p.mu.Lock() + anchor := p.anchor + p.mu.Unlock() + + if anchor == (commitment.Head{0x9e, 0x9e}) { + t.Fatal("anchored on a head no entry chain produces") + } +} diff --git a/eth/sequencer/entry.go b/eth/sequencer/entry.go new file mode 100644 index 0000000000..38cb59533f --- /dev/null +++ b/eth/sequencer/entry.go @@ -0,0 +1,185 @@ +package sequencer + +import ( + "bytes" + "math/big" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "google.golang.org/protobuf/proto" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// contentEqual reports field-level equality of entry content, excluding the +// prefix commitment (design: byte-match). The outer protobuf encoding is +// never compared — the gateway re-marshals entries. +func contentEqual(a, b *pb.Entry) bool { + switch ak := a.GetKind().(type) { + case *pb.Entry_BlockOpen: + bo := b.GetBlockOpen() + if bo == nil { + return false + } + + ao := ak.BlockOpen + + return ao.GetBlockNumber() == bo.GetBlockNumber() && + ao.GetBlockTimestamp() == bo.GetBlockTimestamp() && + bytes.Equal(ao.GetParentHash(), bo.GetParentHash()) && + ao.GetGasLimit() == bo.GetGasLimit() && + bytes.Equal(ao.GetBaseFee(), bo.GetBaseFee()) + case *pb.Entry_Record: + br := b.GetRecord() + if br == nil || len(ak.Record.GetTransactions()) != len(br.GetTransactions()) { + return false + } + + for i, tx := range ak.Record.GetTransactions() { + if !bytes.Equal(tx, br.GetTransactions()[i]) { + return false + } + } + + return true + case *pb.Entry_BlockSeal: + bs := b.GetBlockSeal() + + return bs != nil && bytes.Equal(ak.BlockSeal.GetHeader(), bs.GetHeader()) + default: + return false + } +} + +// entryPrefix returns the prefix commitment carried by an entry. +func entryPrefix(e *pb.Entry) []byte { + switch k := e.GetKind().(type) { + case *pb.Entry_BlockOpen: + return k.BlockOpen.GetPrefixCommitment() + case *pb.Entry_Record: + return k.Record.GetPrefixCommitment() + case *pb.Entry_BlockSeal: + return k.BlockSeal.GetPrefixCommitment() + default: + return nil + } +} + +// foldEntry folds an entry carrying its existing prefix onto cur. +// openContext builds the fold input for an open entry. Both fold paths must +// construct it identically or a refold diverges from the original fold. +func openContext(bo *pb.BlockOpen) commitment.OpenContext { + return commitment.OpenContext{ + Number: bo.GetBlockNumber(), + Timestamp: bo.GetBlockTimestamp(), + ParentHash: [32]byte(bo.GetParentHash()), + GasLimit: bo.GetGasLimit(), + BaseFee: new(big.Int).SetBytes(bo.GetBaseFee()), + } +} + +// foldEntry advances a commitment head by one entry, dispatching on kind. +func foldEntry(cur commitment.Head, e *pb.Entry) (commitment.Head, error) { + switch k := e.GetKind().(type) { + case *pb.Entry_BlockOpen: + return commitment.FoldOpen(cur, openContext(k.BlockOpen)) + case *pb.Entry_Record: + return commitment.FoldTxs(cur, k.Record.GetTransactions()), nil + case *pb.Entry_BlockSeal: + return commitment.FoldSeal(cur, commitment.SealedHash(k.BlockSeal.GetHeader())), nil + default: + return commitment.Head{}, errRefold + } +} + +// setEntryPrefix rewrites the prefix commitment an entry carries. +func setEntryPrefix(e *pb.Entry, cur commitment.Head) bool { + switch k := e.GetKind().(type) { + case *pb.Entry_BlockOpen: + k.BlockOpen.PrefixCommitment = cur.Bytes() + case *pb.Entry_Record: + k.Record.PrefixCommitment = cur.Bytes() + case *pb.Entry_BlockSeal: + k.BlockSeal.PrefixCommitment = cur.Bytes() + default: + return false + } + + return true +} + +// refoldEntry clones a journal item's entry onto a new prefix, returning +// the rewritten entry with its post-fold head. Folding through foldEntry +// guarantees a refold computes exactly what the original fold did. +func refoldEntry(cur commitment.Head, item journalItem) (*pb.Entry, commitment.Head, error) { + entry, ok := proto.Clone(item.entry).(*pb.Entry) + if !ok || !setEntryPrefix(entry, cur) { + return nil, commitment.Head{}, errRefold + } + + next, err := foldEntry(cur, entry) + if err != nil { + return nil, commitment.Head{}, err + } + + return entry, next, nil +} + +// openEntry builds the wire entry for a block open. All open publishers +// (live build and window rebuild) must construct it identically or the +// mirror check and the consumer's context pinning would see drift. +func openEntry(oc commitment.OpenContext, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: oc.Number, + BlockTimestamp: oc.Timestamp, + ParentHash: oc.ParentHash[:], + GasLimit: oc.GasLimit, + BaseFee: baseFeeBytes(oc.BaseFee), + PrefixCommitment: prefix.Bytes(), + }}} +} + +// recordEntry builds the wire entry for one committed transaction. +func recordEntry(raw []byte, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: prefix.Bytes(), + }}} +} + +// sealEntry builds the wire entry for a sealed block header. +func sealEntry(raw []byte, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{ + Header: raw, + PrefixCommitment: prefix.Bytes(), + }}} +} + +// decodeSealHeader decodes a seal entry's RLP header. +func decodeSealHeader(raw []byte) (*types.Header, error) { + header := new(types.Header) + if err := rlp.DecodeBytes(raw, header); err != nil { + return nil, err + } + + return header, nil +} + +// entryHeight extracts a height from an open or seal entry; records carry +// none. +func entryHeight(e *pb.Entry) (uint64, bool) { + switch k := e.GetKind().(type) { + case *pb.Entry_BlockOpen: + return k.BlockOpen.GetBlockNumber(), true + case *pb.Entry_BlockSeal: + header, err := decodeSealHeader(k.BlockSeal.GetHeader()) + if err != nil { + return 0, false + } + + return header.Number.Uint64(), true + default: + return 0, false + } +} diff --git a/eth/sequencer/entry_test.go b/eth/sequencer/entry_test.go new file mode 100644 index 0000000000..537feff520 --- /dev/null +++ b/eth/sequencer/entry_test.go @@ -0,0 +1,145 @@ +package sequencer + +import ( + "testing" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" +) + +func testOpen(number uint64, parent common.Hash, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: number, + BlockTimestamp: 1700000000 + number, + ParentHash: parent.Bytes(), + GasLimit: 30_000_000, + BaseFee: []byte{0x01}, + PrefixCommitment: prefix.Bytes(), + }}} +} + +func testRecord(tx []byte, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{tx}, + PrefixCommitment: prefix.Bytes(), + }}} +} + +func testSeal(t *testing.T, number uint64, prefix commitment.Head) *pb.Entry { + t.Helper() + + raw, err := rlp.EncodeToBytes(testHeader(number, common.Hash{0x01})) + if err != nil { + t.Fatalf("rlp: %v", err) + } + + return &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{ + Header: raw, + PrefixCommitment: prefix.Bytes(), + }}} +} + +func TestContentEqual(t *testing.T) { + a, b := commitment.Head{0xaa}, commitment.Head{0xbb} + + cases := []struct { + name string + x, y *pb.Entry + want bool + }{ + {"open equal ignoring prefix", testOpen(1, common.Hash{0x01}, a), testOpen(1, common.Hash{0x01}, b), true}, + {"open different number", testOpen(1, common.Hash{0x01}, a), testOpen(2, common.Hash{0x01}, a), false}, + {"open different parent", testOpen(1, common.Hash{0x01}, a), testOpen(1, common.Hash{0x02}, a), false}, + {"record equal ignoring prefix", testRecord([]byte{0x01}, a), testRecord([]byte{0x01}, b), true}, + {"record different tx", testRecord([]byte{0x01}, a), testRecord([]byte{0x02}, a), false}, + {"record different length", testRecord([]byte{0x01}, a), &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{Transactions: [][]byte{{0x01}, {0x02}}}}}, false}, + {"kind mismatch", testOpen(1, common.Hash{0x01}, a), testRecord([]byte{0x01}, a), false}, + {"kind mismatch reversed", testSeal(t, 1, a), testOpen(1, common.Hash{0x01}, a), false}, + {"seal equal ignoring prefix", testSeal(t, 1, a), testSeal(t, 1, b), true}, + {"seal different header", testSeal(t, 1, a), testSeal(t, 2, a), false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := contentEqual(tc.x, tc.y); got != tc.want { + t.Fatalf("contentEqual = %v, want %v", got, tc.want) + } + }) + } +} + +func TestEntryHeight(t *testing.T) { + if h, ok := entryHeight(testOpen(7, common.Hash{0x01}, commitment.Head{})); !ok || h != 7 { + t.Fatalf("open height = %d/%v", h, ok) + } + + if h, ok := entryHeight(testSeal(t, 9, commitment.Head{})); !ok || h != 9 { + t.Fatalf("seal height = %d/%v", h, ok) + } + + if _, ok := entryHeight(testRecord([]byte{0x01}, commitment.Head{})); ok { + t.Fatal("record must carry no height") + } + + garbage := &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{Header: []byte{0xde}}}} + if _, ok := entryHeight(garbage); ok { + t.Fatal("undecodable seal must carry no height") + } +} + +// refoldEntry must reproduce the enqueue-side folds exactly, with only the +// prefix rewritten. +func TestRefoldEntryMatchesDirectFolds(t *testing.T) { + base := commitment.Head{0x11} + + open := testOpen(3, common.Hash{0x02}, commitment.Head{0xff}) + items := []journalItem{ + {entry: open, kind: entryOpen, height: 3}, + {entry: testRecord([]byte{0xbe, 0xef}, commitment.Head{0xff}), kind: entryRecord, height: 3}, + {entry: testSeal(t, 3, commitment.Head{0xff}), kind: entrySeal, height: 3}, + } + + cur := base + + for _, item := range items { + entry, next, err := refoldEntry(cur, item) + if err != nil { + t.Fatalf("refold: %v", err) + } + + if got := commitment.Head(entryPrefix(entry)); got != cur { + t.Fatalf("prefix %x, want %x", got, cur) + } + + if next == cur { + t.Fatal("fold did not advance") + } + + cur = next + } + + // The refolded open must fold identically to a direct FoldOpen. + wantOpen, err := commitment.FoldOpen(base, commitment.OpenContext{ + Number: 3, + Timestamp: open.GetBlockOpen().GetBlockTimestamp(), + ParentHash: common.Hash{0x02}, + GasLimit: open.GetBlockOpen().GetGasLimit(), + BaseFee: testHeader(0, common.Hash{}).BaseFee.SetBytes(open.GetBlockOpen().GetBaseFee()), + }) + if err != nil { + t.Fatalf("fold open: %v", err) + } + + if _, next, _ := refoldEntry(base, items[0]); next != wantOpen { + t.Fatalf("refolded open %x, want %x", next, wantOpen) + } +} + +func TestDecodeSealHeaderRejectsGarbage(t *testing.T) { + if _, err := decodeSealHeader([]byte{0xde, 0xad}); err == nil { + t.Fatal("garbage header decoded") + } +} diff --git a/eth/sequencer/equivocation_test.go b/eth/sequencer/equivocation_test.go new file mode 100644 index 0000000000..b49ed3d62a --- /dev/null +++ b/eth/sequencer/equivocation_test.go @@ -0,0 +1,514 @@ +package sequencer + +import ( + "testing" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/miner" +) + +// Two producers sharing a signing key both seal the same height. When the +// chain picks the other block, our flush must not overwrite the winner's +// content: the store's newest generation at a height has to agree with the +// canonical chain. +func TestForeignSealAtOwnHeightNonCanonical(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}, known: map[common.Hash]*types.Header{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // The twin publishes and seals ITS block 2 into the store first. + twin := testHeader(2, parent) + twin.Extra = []byte("twin") // distinct hash + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + // The chain accepts the TWIN's block 2 as canonical; ours lost. + chain.canonical[2] = twin.Hash() + + // Now we seal our own (losing) block 2 and flush. + ours := testHeader(2, parent) + p.OpenBlock(2, ours.Time, parent, ours.GasLimit, ours.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + + supersedes := reconcileSupersede.Snapshot().Count() + + p.SealBlock(blockFor(ours, []*types.Transaction{tx})) + time.Sleep(2 * time.Second) // let the flush + reconcile settle + + if got := reconcileSupersede.Snapshot().Count(); got > supersedes { + t.Fatalf("superseded a CANONICAL foreign seal at our own height "+ + "(supersede %d -> %d): the store's newest generation at height 2 "+ + "now holds non-canonical content", supersedes, got) + } +} + +// The mirror case: the chain chose OUR block, so the foreign seal at our +// height is the loser — supersede it, as before. +func TestForeignSealAtOwnHeightWeAreCanonical(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}, known: map[common.Hash]*types.Header{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + ours := testHeader(2, parent) + chain.canonical[2] = ours.Hash() // WE won + + p.OpenBlock(2, ours.Time, parent, ours.GasLimit, ours.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + + supersedes := reconcileSupersede.Snapshot().Count() + yields := reconcileYield.Snapshot().Count() + + p.SealBlock(blockFor(ours, []*types.Transaction{tx})) + + // The gate runs on every seal in production, and our block being + // canonical confirms it — which releases the flush to supersede the + // loser's content. Until then the pre-broadcast hold keeps our window + // off a height the store has already closed. + if v := p.ConfirmSeal(2 * time.Second); v != miner.SealConfirmed { + t.Fatalf("gate verdict = %v for our own canonical block", v) + } + + waitHead(t, h, p, 10*time.Second) + + if reconcileYield.Snapshot().Count() != yields { + t.Fatal("yielded despite being the canonical producer") + } + + if reconcileSupersede.Snapshot().Count() != supersedes+1 { + t.Fatal("canonical producer must supersede the losing seal") + } +} + +// The pre-seal barrier: a confirmed window clears, a contested one does +// not, and an unreachable store clears anyway so production never stalls. +func TestAwaitSequencedBarrier(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitHead(t, h, p, 5*time.Second) + + if !awaitOurWindow(p, 2*time.Second) { + t.Fatal("a fully confirmed window must clear the barrier") + } + + // A competitor takes the height with DIFFERENT content: its window is + // not a prefix of ours, so there is nothing to complete in place and + // our next write STALEs us into the contested hold. + foreignWindow(t, h, 2, parent, testTx(t, 9)) + p.PublishTx(testTx(t, 1)) + + waitFor(t, 5*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.hold.kind == holdSticky + }) + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("a contested window must NOT clear the barrier") + } +} + +// An unreachable store must not gate block production: the barrier clears +// on its deadline rather than stalling the chain. +func TestAwaitSequencedYieldsToLiveness(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() // store gone; entries pile up unacked + + p.OpenBlock(2, 1700000002, common.Hash{0x01}, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + + start := time.Now() + if !awaitOurWindow(p, 300*time.Millisecond) { + t.Fatal("an unreachable store must not block sealing") + } + + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("barrier waited %v; it must bound the stall", elapsed) + } +} + +// Takeover: the incumbent died, we adopt its window. We are the ONLY +// producer, so the barrier must let us seal. +func TestBarrierAllowsAdoptedTakeover(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // A dead producer's dangling window at height 2. + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + w := p.AdoptWindow(2, parent) + if w == nil { + t.Fatal("window not adopted") + } + + if !awaitOurWindow(p, 2*time.Second) { + t.Fatal("barrier blocked a takeover seal: the taker is the only " + + "producer, so refusing to seal would stall the chain") + } +} + +// A re-anchor must not re-post data the store already holds. With the +// store carrying a strict prefix of our window, completion absorbs that +// prefix as confirmed and leaves only the missing records to deliver — +// republishing the whole window would duplicate entries every reader has. +func TestReanchorPublishesOnlyTheDelta(t *testing.T) { + p := barePublisher() + + parent := common.Hash{0x01} + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + p.PublishTx(testTx(t, 1)) + p.PublishTx(testTx(t, 2)) + + items, _ := p.journal.after(0) + if len(items) != 4 { + t.Fatalf("expected open+3 records, got %d", len(items)) + } + + // The store holds a strict prefix: the open and the first record. + info := tailInfo{ + s: items[1].post, + tipOpen: true, + tipOpenHeight: 2, + tipOpenParent: parent, + window: []*pb.Entry{items[0].entry, items[1].entry}, + } + + p.mu.Lock() + p.curHeight = 2 + completed := p.completeExtendedWindowLocked(info, 2) + unacked := p.unackedLocked() + total := len(p.journal.items) + p.mu.Unlock() + + if !completed { + t.Fatal("a store prefix of our window must complete in place") + } + + // Only the two records the store lacks remain to send; the absorbed + // prefix is seated as already-confirmed, not queued for re-posting. + if unacked != 2 { + t.Fatalf("unacked = %d, want 2 (the delta only)", unacked) + } + + if total != 4 { + t.Fatalf("journal holds %d entries, want 4 — the window was duplicated", total) + } +} + +// waitDrained blocks until every journal entry is store-confirmed, which is +// the state the coverage branch of the barrier runs in. +func waitDrained(t *testing.T, p *Publisher, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + + for { + p.mu.Lock() + unacked := p.unackedLocked() + p.mu.Unlock() + + if unacked == 0 { + return + } + + if time.Now().After(deadline) { + t.Fatalf("journal still holds %d unacked entries", unacked) + } + + time.Sleep(5 * time.Millisecond) + } +} + +// appendForeignRecord writes one record onto the store's current chain, as a +// second producer building the same height would. +func appendForeignRecord(t *testing.T, h *harness, tx *types.Transaction) { + t.Helper() + + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + rec := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(rec); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign record rejected: %v", status) + } +} + +// Every entry of ours acking proves the store took them — not that they are +// all the store has. A second producer's records at the same height were +// acked too, so they carry the same preconfirmation; sealing a block that +// omits them strands the promise. The barrier asks for a rebuild instead. +func TestBarrierRebuildsWhenStoreHoldsMore(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + appendForeignRecord(t, h, testTx(t, 7)) + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("sealed a block omitting records the store already holds") + } + + if !p.ResyncNeeded() { + t.Fatal("coverage failure did not arm a rebuild") + } +} + +// The uncontested case must stay cheap and permissive: when the store holds +// exactly our window, the coverage read confirms it and the seal proceeds. +func TestBarrierSealsWhenStoreMatches(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + if !awaitOurWindow(p, 2*time.Second) { + t.Fatal("barrier blocked a seal whose window is exactly the store's") + } +} + +// A competing generation at our height refuses the seal and adopts. The +// old tie-break sealed here — "the later opener owns the height" — and two +// producers whose reads raced could each believe themselves later, which is +// where divergent double-seals came from. Under the adopt rule the refusal +// is unconditional and the rebuild converges on the store's window. +func TestForeignGenerationRefusedAndAdopted(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + p.PublishTx(testTx(t, 1)) + waitDrained(t, p, 5*time.Second) + + // A second producer opens its own generation at the same height. + foreignWindow(t, h, 2, parent, testTx(t, 9)) + + p.mu.Lock() + p.resync = false + p.mu.Unlock() + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("sealed beside a competing generation: two blocks holding " + + "different content at one height") + } + + if !p.ResyncNeeded() { + t.Fatal("the refusal must arm the rebuild that adopts the competing window") + } +} + +// A seal below the height being built is ordinary history, not evidence the +// height moved on. Bailing on it made the coverage check report "covered" +// for almost every block, because the walk from our anchor routinely crosses +// the previous block's seal. +func TestCoverageIgnoresSealBelowOurHeight(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Block 1's seal now sits between the anchor and anything appended next. + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + appendForeignRecord(t, h, testTx(t, 7)) + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("a seal at a lower height masked a real coverage gap") + } +} + +// Takeover must stay fast: the incumbent is gone, its window is not +// growing, so the adopter covers it by definition and seals at once. +// Refusing here would stall the chain on every producer failure. +func TestAdoptedDeadWindowStillSeals(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if p.AdoptWindow(2, parent) == nil { + t.Fatal("window not adopted") + } + + if !awaitOurWindow(p, 2*time.Second) { + t.Fatal("barrier blocked a takeover of a dead window: every " + + "producer failure would stall the chain") + } +} + +// The mirror: the incumbent is alive and its window grew past the prefix we +// adopted. Sealing now would strand every record it added. +func TestAdoptedLiveWindowDoesNotSeal(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if p.AdoptWindow(2, parent) == nil { + t.Fatal("window not adopted") + } + + // The incumbent turns out to be alive and keeps writing. + appendForeignRecord(t, h, testTx(t, 7)) + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("sealed a prefix of a window the incumbent is still growing") + } + + if !p.ResyncNeeded() { + t.Fatal("a grown window must arm the rebuild that covers it") + } +} + +// A read that falls off the anchor rung restarts at a block boundary and +// walks over our own open, so an open at our height is routinely ours. The +// check used to read that as a rival and pass every block it saw — a lone +// producer logged 36 "second generation" skips with no competitor running. +func TestCoverageRecognisesOurOwnWindow(t *testing.T) { + p := barePublisher() + + parent := common.Hash{0x01} + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + p.PublishTx(testTx(t, 1)) + + items, _ := p.journal.after(0) + ourWindow := []*pb.Entry{items[0].entry, items[1].entry, items[2].entry} + + p.mu.Lock() + defer p.mu.Unlock() + + // The whole window, as a fallback read would return it. + whole := tailInfo{tipOpen: true, tipOpenHeight: 2, window: ourWindow} + if got := p.relateWindowLocked(whole, 2); got != windowOurs { + t.Fatalf("our own window read as %v, want windowOurs", got) + } + + // A prefix of it — records still in flight. + prefix := tailInfo{tipOpen: true, tipOpenHeight: 2, window: ourWindow[:2]} + if got := p.relateWindowLocked(prefix, 2); got != windowOurs { + t.Fatalf("a prefix of our window read as %v, want windowOurs", got) + } + + // Our window with a record appended by someone else: the real gap. + grown := tailInfo{ + tipOpen: true, tipOpenHeight: 2, + window: append(append([]*pb.Entry{}, ourWindow...), txRecord(t, testTx(t, 9))), + } + if got := p.relateWindowLocked(grown, 2); got != windowExtendsOurs { + t.Fatalf("an extended window read as %v, want windowExtendsOurs", got) + } + + // A genuinely different lineage. + foreign := []*pb.Entry{items[0].entry, txRecord(t, testTx(t, 9))} + if got := p.relateWindowLocked(tailInfo{tipOpen: true, tipOpenHeight: 2, window: foreign}, 2); got != windowForeign { + t.Fatalf("a rival window read as %v, want windowForeign", got) + } +} + +// txRecord builds a bare record entry; contentEqual compares payloads, not +// prefix commitments, so no fold is needed. +func txRecord(t *testing.T, tx *types.Transaction) *pb.Entry { + t.Helper() + + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + return &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{Transactions: [][]byte{raw}}}} +} + +// Contention refuses, always: a sticky hold is the store's own verdict that +// another producer owns the height, and the only path to a seal is adopting +// their window. The refusal arms that rebuild. +func TestContestedRefusesAndArmsAdopt(t *testing.T) { + p := barePublisher() + + p.mu.Lock() + p.curHeight = 7 + p.hold = hold{after: 0, kind: holdSticky} + p.mu.Unlock() + + for i := 0; i < 3; i++ { + if awaitOurWindow(p, time.Second) { + t.Fatal("a contested height sealed: two producers sealing one " + + "height with different content is the revocation machine") + } + } + + if !p.ResyncNeeded() { + t.Fatal("the refusal must arm the rebuild that adopts the owner's window") + } +} diff --git a/eth/sequencer/exec.go b/eth/sequencer/exec.go new file mode 100644 index 0000000000..70ff825681 --- /dev/null +++ b/eth/sequencer/exec.go @@ -0,0 +1,170 @@ +package sequencer + +import ( + "errors" + "fmt" + "math/big" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/trie" +) + +// blockEnv re-executes one speculative block on top of a parent state, +// mirroring the producer's environment: same header context from the open +// record, author nil so the EVM coinbase resolves to the producer-independent +// CalculateCoinbase (post-Rio), difficulty constant 1 under VEBLOP, and the +// same pre-transaction system calls the producer runs (EIP-2935 post-Prague). +type blockEnv struct { + header *types.Header + statedb *state.StateDB + evm *vm.EVM + gasPool *core.GasPool + txs []*types.Transaction + receipts []*types.Receipt +} + +// newBlockEnv builds the execution environment. speculative maps heights of +// sealed-but-not-yet-imported ancestors to their sealed hashes, so BLOCKHASH +// resolves them exactly as the producer did — the canonical header walk +// returns zero for blocks the chain hasn't imported. +func newBlockEnv(chain *core.BlockChain, statedb *state.StateDB, open *pb.BlockOpen, speculative map[uint64]common.Hash) *blockEnv { + header := &types.Header{ + ParentHash: common.BytesToHash(open.GetParentHash()), + Number: new(big.Int).SetUint64(open.GetBlockNumber()), + GasLimit: open.GetGasLimit(), + Time: open.GetBlockTimestamp(), + BaseFee: new(big.Int).SetBytes(open.GetBaseFee()), + Difficulty: big.NewInt(1), + Coinbase: common.Address{}, + } + + blockCtx := core.NewEVMBlockContext(header, chain, nil) + + walk := blockCtx.GetHash + blockCtx.GetHash = func(n uint64) common.Hash { + if h, ok := speculative[n]; ok { + return h + } + + if h := walk(n); h != (common.Hash{}) { + return h + } + + // The default resolver walks parent headers and breaks at the first + // unimported speculative ancestor; anything at or below the + // canonical head is still resolvable directly. + return chain.GetCanonicalHash(n) + } + + env := &blockEnv{ + header: header, + statedb: statedb, + evm: vm.NewEVM(blockCtx, statedb, chain.Config(), vm.Config{}), + gasPool: new(core.GasPool).AddGas(header.GasLimit), + } + + if chain.Config().IsPrague(header.Number) { + core.ProcessParentBlockHash(header.ParentHash, env.evm) + } + + return env +} + +// applyRaw executes one streamed raw transaction. The producer only publishes +// transactions it committed, so any failure here is a determinism divergence, +// not a bad transaction. +func (env *blockEnv) applyRaw(raw []byte) (*types.Transaction, *types.Receipt, error) { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + return nil, nil, fmt.Errorf("decode streamed transaction: %w", err) + } + + env.statedb.SetTxContext(tx.Hash(), len(env.txs)) + + receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.statedb, env.header, tx, &env.header.GasUsed) + if err != nil { + return nil, nil, fmt.Errorf("re-execute tx %s: %w", tx.Hash(), err) + } + + // The block hash is unknown pre-seal (ApplyTransaction stamped the + // provisional unsealed header hash); zero it out until the seal record + // arrives. EffectiveGasPrice is not populated by execution — derive it + // the way DeriveFields would. + receipt.BlockHash = common.Hash{} + for _, l := range receipt.Logs { + l.BlockHash = common.Hash{} + } + + receipt.EffectiveGasPrice = effectiveGasPrice(tx, env.header.BaseFee) + + env.txs = append(env.txs, tx) + env.receipts = append(env.receipts, receipt) + + return tx, receipt, nil +} + +func effectiveGasPrice(tx *types.Transaction, baseFee *big.Int) *big.Int { + if baseFee == nil { + return tx.GasPrice() + } + + tip, err := tx.EffectiveGasTip(baseFee) + if err != nil { + // Streamed txs executed successfully, so the fee cap covers the + // base fee; this path is unreachable but must not panic. + tip = new(big.Int) + } + + return new(big.Int).Add(tip, baseFee) +} + +var errSealMismatch = errors.New("sealed header diverges from re-execution") + +// checkSeal cross-checks the sealed header against the open context this +// block was executed under and against the re-execution results, including +// the state root — the catch-all for anything execution missed. State-sync +// transactions are applied by the producer in Finalize and never enter the +// stream, and their gas and receipts live outside the header's GasUsed and +// ReceiptHash — so a sprint-start block with pending events passes the gas +// and receipts comparisons and is caught only by the state root differing. +func (env *blockEnv) checkSeal(sealed *types.Header) error { + switch { + case sealed.Number.Cmp(env.header.Number) != 0, + sealed.Time != env.header.Time, + sealed.ParentHash != env.header.ParentHash, + sealed.GasLimit != env.header.GasLimit, + !bigEqual(sealed.BaseFee, env.header.BaseFee): + return fmt.Errorf("%w: open context mismatch at block %s", errSealMismatch, sealed.Number) + case sealed.GasUsed != env.header.GasUsed: + return fmt.Errorf("%w: gas used %d != re-executed %d at block %s", + errSealMismatch, sealed.GasUsed, env.header.GasUsed, sealed.Number) + } + + receiptsRoot := types.DeriveSha(types.Receipts(env.receipts), trie.NewStackTrie(nil)) + if receiptsRoot != sealed.ReceiptHash { + return fmt.Errorf("%w: receipts root %s != re-executed %s at block %s", + errSealMismatch, sealed.ReceiptHash, receiptsRoot, sealed.Number) + } + + root := env.statedb.IntermediateRoot(env.evm.ChainConfig().IsEIP158(env.header.Number)) + if root != sealed.Root { + return fmt.Errorf("%w: state root %s != re-executed %s at block %s", + errSealMismatch, sealed.Root, root, sealed.Number) + } + + return nil +} + +func bigEqual(a, b *big.Int) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + + return a.Cmp(b) == 0 +} diff --git a/eth/sequencer/gate.go b/eth/sequencer/gate.go new file mode 100644 index 0000000000..4a77593a42 --- /dev/null +++ b/eth/sequencer/gate.go @@ -0,0 +1,376 @@ +package sequencer + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/miner" +) + +// gateVerdict is the broadcast gate's resolution state. gateLost outranks +// gateConfirmed by construction: a lost verdict is terminal, and the seal +// ack that would confirm can no longer arrive for a rejected seal. +type gateVerdict int + +const ( + gatePending gateVerdict = iota // awaiting the seal ack, the chain, or the deadline + gateConfirmed // the store acked the gated seal + gateLost // the store rejected it: another window owns the height +) + +// sealGate identifies the sealed block whose broadcast awaits the store's +// verdict. A zero height means no seal is gated. refuseOnTimeout inverts +// the liveness default: normally an undecided height broadcasts, but a +// block built over a seal the store already holds must not. +type sealGate struct { + height uint64 + hash common.Hash + verdict gateVerdict + refuseOnTimeout bool + + // contested records a STALE landing while the verdict was pending: the + // head moved under the flush, so our own reconcile loop is producing + // the verdict and the wait earns the longer budget. + contested bool + + // tolerateSealed carries a modeOverSealed decision to the gate: this + // block knowingly sealed a height the store closed because recovery + // declined, so a foreign store seal must not refuse it — the + // chain-canonical check still can. + tolerateSealed bool + + // published means a flush in the journal carries this block; a refusal + // must drop it (a refused block is never canonical, so its flush could + // never resolve) and roll the sealed tip back to prevTip so the rebuild + // at this height can adopt instead of muting itself. + published bool + prevTip uint64 + + // txs is the gated block's transaction sequence, kept for the last + // look a verdictless timeout takes before broadcasting. + txs []common.Hash +} + +// refusalStreak counts consecutive gate refusals at one height. The escape +// valve: refuse → rebuild is convergent when the rebuild can adopt what +// stands in the store, but a window it cannot adopt (dead producer, foreign +// parent) would refuse forever — past the cap, liveness wins and the block +// broadcasts without a verdict. +type refusalStreak struct { + height uint64 + count int +} + +// maxGateRefusals bounds the refuse → rebuild cycle per height. +const maxGateRefusals = 3 + +// ConfirmSeal waits for the store's verdict on the block just sealed. The +// store's head CAS admits exactly one seal at a lineage position, so the +// seal race has a single winner — this is where the loser finds out. +// +// Confirmation comes from our seal entry's ack. Refusal comes from our own +// chain: the winner's block importing at our height IS the rejection +// notice, needing no store read at all. No verdict inside the budget means +// the store is slow, unreachable, or the height genuinely unresolved — the +// caller broadcasts (production never waits on the store), which also +// covers the phantom case of a winner that sealed in the store and then +// died without broadcasting. +func (p *Publisher) ConfirmSeal(timeout time.Duration) miner.SealVerdict { + if p.unreachable.Load() { + return p.settle(miner.SealUnknown) // no verdict is coming; do not wait for one + } + + start := time.Now() + + var chainCheck time.Time + + for { + p.mu.Lock() + g := p.gate + p.mu.Unlock() + + if g.height == 0 { + return miner.SealUnknown // nothing gated (muted or failed build) + } + + if g.verdict == gateLost { + return p.refuseGated() + } + + if g.verdict == gateConfirmed { + return p.settle(miner.SealConfirmed) + } + + // The canonical lookup is an uncached database read, and this wait + // can now last a whole block period: take it on a coarse tick + // rather than every poll. + if now := time.Now(); p.chain != nil && now.After(chainCheck) { + chainCheck = now.Add(50 * time.Millisecond) + + switch canonical := p.chain.GetCanonicalHash(g.height); { + case canonical == g.hash: + // A same-key twin broadcast our exact block: it is already + // the chain. Broadcasting again is a harmless duplicate. + return p.settle(miner.SealConfirmed) + case canonical != (common.Hash{}): + return p.refuseGated() + } + } + + // A contested gate earns the longer budget: a STALE while pending + // means our own reconcile loop is producing the verdict, and cutting + // the wait short is what broadcast an empty block three seconds + // before its refusal would have arrived. + budget := timeout + if g.contested { + budget = max(budget, contestedGateTimeout) + } + + failed := p.failed.Load() + if !failed && time.Since(start) <= budget { + time.Sleep(2 * time.Millisecond) + + continue + } + + if g.refuseOnTimeout { + return p.refuseGated() + } + + if !failed { + switch p.gateRecheck(g) { + case miner.SealRefused: + return p.refuseGated() + case miner.SealConfirmed: + return p.settle(miner.SealConfirmed) + } + } + + return p.settle(miner.SealUnknown) + } +} + +// settle resolves the gate with a non-refusal verdict: clear, count, return. +// Refusals go through refuseGated, which also unwinds the refused flush. +func (p *Publisher) settle(v miner.SealVerdict) miner.SealVerdict { + p.clearGate() + + if v == miner.SealConfirmed { + gateConfirmedCount.Inc(1) + } else { + gateUnknownCount.Inc(1) + } + + return v +} + +// contestedGateTimeout is the verdict budget once the gate is contested: +// one block period, tied to recoverGrace so the two waits scale together — +// losing a slot is acceptable, losing acked records to a premature +// broadcast is not. +const contestedGateTimeout = recoverGrace + +// refuseGated resolves the gate as refused. The block will never broadcast, +// so the flush describing it is dropped — a refused block is never +// canonical, and a flush that can never resolve wedges every later build +// behind it — and the sealed tip it bumped rolls back so the rebuild at +// this height can adopt what stands in the store instead of muting itself. +// Past the per-height refusal cap, liveness wins: the rebuild evidently +// cannot adopt its way to convergence, and the block broadcasts without a +// verdict. +func (p *Publisher) refuseGated() miner.SealVerdict { + p.mu.Lock() + defer p.mu.Unlock() + + if p.gate.height != p.refusals.height { + p.refusals = refusalStreak{height: p.gate.height} + } + + p.refusals.count++ + + if p.refusals.count > maxGateRefusals { + log.Warn("Sequencer gate refused this height repeatedly, broadcasting for liveness", + "number", p.gate.height, "refusals", p.refusals.count) + p.gate = sealGate{} + gateUnknownCount.Inc(1) + + return miner.SealUnknown + } + + if p.gate.published { + p.dropRefusedFlushLocked(p.gate.height, p.gate.hash) + + if p.sealedTip == p.gate.height { + p.sealedTip = p.gate.prevTip + } + } + + p.gate = sealGate{} + gateRefusedCount.Inc(1) + + return miner.SealRefused +} + +// dropRefusedFlushLocked removes the refused block's undelivered trailing +// entries, and anything a concurrent work cycle chained above them — those +// describe blocks that re-flush from their own bodies at seal time. An +// acked prefix is store content and stays; older stacked flushes below the +// refused height are still owed and stay. The hold clears with the flush it +// gated, or the next barrier would resync against a lineage that no longer +// exists. +func (p *Publisher) dropRefusedFlushLocked(height uint64, hash common.Hash) { + if ours, ok := p.sealedHashAtLocked(height); !ok || ours != hash { + return // the trailing seal is not the refused block's + } + + cut := p.journal.cutFromHeight(p.ackedSeq, height) + if cut == len(p.journal.items) { + return + } + + publishDropMeter.Mark(int64(len(p.journal.items) - cut)) + p.rewindJournalLocked(cut) + p.syncWindowLocked() + p.hold = clearedHold() + publishQueueGauge.Update(int64(p.unackedLocked())) +} + +// gateRecheck is the last look before a verdictless broadcast: one +// height-anchored probe at the gated height, taken after the whole wait. +// The reads that precede it can each be individually fresh and still miss +// a rival's burst landing between them; what this read shows is exactly +// what the broadcast would bury. Anything unreadable keeps the timeout +// verdict — production never waits on a store it cannot see. +func (p *Publisher) gateRecheck(g sealGate) miner.SealVerdict { + if p.unreachable.Load() || p.read == nil || p.read.cons == nil { + return miner.SealUnknown + } + + ctx, cancel := context.WithTimeout(context.Background(), checkTailTimeout) + defer cancel() + + // One walk anchored at the gated height answers everything: an unknown + // height reads NOT_FOUND (nothing stands here), a live generation is + // served from its open, a sealed one from just past its seal. + info, out, done := p.tryWalk(ctx, blockReq(g.height), false) + if !done || out != recOK { + return miner.SealUnknown + } + + if info.tipOpen && info.tipOpenHeight == g.height { + if storeTxs, ok := windowTxHashes(info.window); ok && !windowLeadsHashes(storeTxs, g.txs) { + gateRecheckRefused.Inc(1) + log.Warn("Sequencer refusing broadcast: acked records stand at this height the block does not carry", + "number", g.height, "store", len(storeTxs), "block", len(g.txs)) + + return miner.SealRefused + } + + return miner.SealUnknown + } + + // A generation at the gated height with no live window is a sealed one. + // The walk is served from just past the seal, so the generation itself — + // seal included — comes from the block fetch. + return p.recheckSealedGeneration(ctx, g) +} + +// recheckSealedGeneration fetches the generation standing at the gated +// height and applies the seal policy (verdictForSeal) to its seal. +func (p *Publisher) recheckSealedGeneration(ctx context.Context, g sealGate) miner.SealVerdict { + entries, err := p.read.generation(ctx, g.height) + if err != nil { + return miner.SealUnknown + } + + for _, e := range entries { + seal := e.GetBlockSeal() + if seal == nil { + continue + } + + header, err := decodeSealHeader(seal.GetHeader()) + if err != nil { + return miner.SealUnknown + } + + v := g.verdictForSeal(header.Hash()) + if v == miner.SealRefused { + gateRecheckRefused.Inc(1) + log.Warn("Sequencer refusing broadcast: the store sealed this height with other content", + "number", g.height, "store", header.Hash()) + } + + return v + } + + return miner.SealUnknown // a live generation after all: nothing decisive +} + +func (p *Publisher) clearGate() { + p.mu.Lock() + defer p.mu.Unlock() + + p.gate = sealGate{} +} + +// gatePendingLocked reports whether an unbroadcast seal at this height is +// still awaiting its verdict. +func (p *Publisher) gatePendingLocked(height uint64) bool { + return p.gate.height != 0 && p.gate.height == height && p.gate.verdict != gateConfirmed +} + +// verdictForSeal is the gate's one seal policy: our own seal standing at +// the gated height confirms the broadcast (a twin delivered our copy), a +// foreign one refuses it — unless the build knowingly sealed over it (the +// liveness fallback), which leaves the verdict to the budget. +func (g sealGate) verdictForSeal(hash common.Hash) miner.SealVerdict { + switch { + case hash == g.hash: + return miner.SealConfirmed + case g.tolerateSealed: + return miner.SealUnknown + default: + return miner.SealRefused + } +} + +// resolveGateFromSealLocked applies the seal policy to a decoded store seal +// at the gated height. An undecoded seal proves nothing and leaves the +// budget to decide. +func (p *Publisher) resolveGateFromSealLocked(info tailInfo) { + if !info.sealDecoded || p.gate.verdict != gatePending { + return + } + + switch p.gate.verdictForSeal(info.lastSealHash) { + case miner.SealConfirmed: + p.gate.verdict = gateConfirmed + case miner.SealRefused: + p.gate.verdict = gateLost + } +} + +// markGateLost records what a STALE means for the gated block. Any STALE +// while the verdict is pending marks the gate contested — the head moved +// under our flush, reconciliation is now producing the verdict, and the +// wait earns the longer budget. A STALE on the gated seal itself is the +// verdict: another producer's window owns the height, and waiting out the +// liveness budget after that would broadcast a second block into a height +// the store has already given to someone else. +func (p *Publisher) markGateLost(item journalItem) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.gate.height == 0 || p.gate.verdict != gatePending { + return + } + + p.gate.contested = true + + if item.kind == entrySeal && p.gate.height == item.height { + p.gate.verdict = gateLost + } +} diff --git a/eth/sequencer/gate_test.go b/eth/sequencer/gate_test.go new file mode 100644 index 0000000000..6e8bd3078b --- /dev/null +++ b/eth/sequencer/gate_test.go @@ -0,0 +1,949 @@ +package sequencer + +import ( + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/miner" + "github.com/ethereum/go-ethereum/rlp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const sealGateBudget = contestedGateTimeout + +// gateBlock builds and seals a block through the publisher, returning it for +// ConfirmSeal. The window drains first, as the pre-seal barrier guarantees +// in production. +func gateBlock(t *testing.T, p *Publisher, h *harness, number uint64, parent common.Hash) *types.Block { + t.Helper() + + header := testHeader(number, parent) + p.OpenBlock(number, header.Time, parent, header.GasLimit, header.BaseFee) + + tx := testTx(t, 0) + p.PublishTx(tx) + waitDrained(t, p, 5*time.Second) + + block := blockFor(header, []*types.Transaction{tx}) + p.SealBlock(block) + + return block +} + +// The sole producer's seal acks and the gate confirms in milliseconds — the +// path every block takes when nothing is wrong. +func TestSealGateConfirmsSoleProducer(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + gateBlock(t, p, h, 2, sealHash(t, sealed)) + + if v := p.ConfirmSeal(2 * time.Second); v != miner.SealConfirmed { + t.Fatalf("sole producer's seal verdict = %v, want Confirmed", v) + } +} + +// The loser learns it lost from its own chain: the winner's block importing +// at our height is the rejection notice, no store read required. +func TestSealGateRefusesWhenRivalBlockImports(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Our window drains normally while we build... + header := testHeader(2, parent) + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + waitDrained(t, p, 5*time.Second) + + // ...the rival's seal lands in the store as we go to seal... + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + // ...and its block becomes our chain's block 2. + chain.canonical[2] = twin.Hash() + + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + + if v := p.ConfirmSeal(2 * time.Second); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: broadcasting would fork an "+ + "already-decided height", v) + } + + // The refused flush must not have stomped the winner's seal: one sealed + // generation at 2, the rival's. + sealedGens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("store holds %d sealed generations at height 2, want the "+ + "winner's alone", sealedGens) + } +} + +// A same-key twin that broadcast our exact block is a confirmation, not a +// refusal: the block on chain IS ours, and re-broadcasting is harmless. +func TestSealGateConfirmsWhenTwinShippedOurBlock(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + block := gateBlock(t, p, h, 2, sealHash(t, sealed)) + chain.canonical[2] = block.Hash() + + if v := p.ConfirmSeal(2 * time.Second); v != miner.SealConfirmed { + t.Fatalf("verdict = %v, want Confirmed for our own canonical block", v) + } +} + +// A foreign generation that sealed our height rejects our own seal, and the +// rejection is the answer: this height belongs to someone else, and the next +// build recovers its content. Broadcasting anyway put two blocks at one +// height on a devnet, and the milestone vote then displaced 38 preconfirmed +// transactions into the following block. The held flush must also not have +// superseded the winner meanwhile. +func TestSealGateRefusesWhenOurSealIsRejected(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + header := testHeader(2, parent) + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + waitDrained(t, p, 5*time.Second) + + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + // No canonical block at 2: the chain has not decided. + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + + start := time.Now() + if v := p.ConfirmSeal(sealGateBudget); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: the store rejected our seal "+ + "for this height", v) + } + + if time.Since(start) > 2*time.Second { + t.Fatal("the gate overstayed its budget") + } + + sealedGens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("the unbroadcast flush superseded a possible winner: %d "+ + "sealed generations at height 2", sealedGens) + } +} + +// A store that is down cannot delay production more than the budget. +func TestSealGateUnknownWhenStoreDown(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + h.stop() + + header := testHeader(2, parent) + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + p.PublishTx(testTx(t, 0)) + p.SealBlock(blockFor(header, []*types.Transaction{testTx(t, 0)})) + + start := time.Now() + if v := p.ConfirmSeal(300 * time.Millisecond); v != miner.SealUnknown { + t.Fatalf("verdict = %v, want Unknown with the store down", v) + } + + if time.Since(start) > 2*time.Second { + t.Fatal("a dead store gated production past the budget") + } +} + +// A height sealed in the store whose block the chain already has is an +// ordinary loss: the build mutes. Should the worker seal there anyway (a +// build already in flight when the winner landed), the flush must not stomp +// the store's standing seal. +func TestSealGateRefusesALostHeightAndStompsNothing(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + chain.canonical[2] = twin.Hash() + + if p.AdoptWindow(2, parent) != nil { + t.Fatal("a sealed height the chain already has offered an adoption") + } + + header := testHeader(2, parent) + header.Extra = []byte("ours") + p.SealBlock(blockFor(header, []*types.Transaction{testTx(t, 0)})) + + if v := p.ConfirmSeal(300 * time.Millisecond); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: the chain already holds "+ + "another block at this height", v) + } + + sealedGens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("the held flush superseded the standing seal: %d sealed "+ + "generations at height 2", sealedGens) + } +} + +// The upgraded twin property: with the gate, one height gets ONE broadcast. +// The winner confirms; the loser is refused the moment the winner's block +// is on its chain. +func TestTwinsNeverBothBroadcast(t *testing.T) { + h, a, b := twinPublishers(t) + chainA := &fakeChain{canonical: map[uint64]common.Hash{}} + chainB := &fakeChain{canonical: map[uint64]common.Hash{}} + + a.mu.Lock() + a.chain = chainA + a.mu.Unlock() + b.mu.Lock() + b.chain = chainB + b.mu.Unlock() + + parent := sealedParent(t, h, a, b) + + blockA := gateBlock(t, a, h, 2, parent) + + // B builds divergent content for the same height and seals it too. + headerB := testHeader(2, parent) + headerB.Extra = []byte("b") + b.OpenBlock(2, headerB.Time, parent, headerB.GasLimit, headerB.BaseFee) + b.PublishTx(testTx(t, 7)) + blockB := blockFor(headerB, []*types.Transaction{testTx(t, 7)}) + b.SealBlock(blockB) + + // Consensus: A's block wins on both chains. + chainA.canonical[2] = blockA.Hash() + chainB.canonical[2] = blockA.Hash() + + va := a.ConfirmSeal(2 * time.Second) + vb := b.ConfirmSeal(2 * time.Second) + + broadcasts := 0 + if va != miner.SealRefused { + broadcasts++ + } + + if vb != miner.SealRefused { + broadcasts++ + } + + if broadcasts != 1 { + t.Fatalf("verdicts A=%v B=%v: exactly one twin may broadcast", va, vb) + } +} + +// The commitment hash is how anyone learns exactly what the store holds: to +// publish at all, a producer must chain onto the current head, and that head +// encodes every seal before it. So a flush that would land after a foreign +// seal at its own height cannot claim ignorance of it — and must not publish +// a second sealed generation there while its own block is unbroadcast. +// +// This is the 764 defect: the losing twin reconciled onto a head that already +// carried the winner's seal, then republished its own window and seal on top, +// leaving the store's newest view of the height pointing at a block that +// never existed on any chain. +func TestFlushNeverRepublishesOverAKnownSeal(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // We build height 2 while it is genuinely clean, as the loser did. + header := testHeader(2, parent) + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + waitDrained(t, p, 5*time.Second) + + // The rival's complete generation lands first, closing the height. + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + // Our flush composes now, and the chain has not decided yet — the exact + // window in which the old canonicality-conditioned guard let a refold + // through. + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + + if v := p.ConfirmSeal(500 * time.Millisecond); v == miner.SealConfirmed { + t.Fatalf("verdict = %v: the store had already closed this height", v) + } + + // Give any reconcile a chance to act before auditing. + time.Sleep(500 * time.Millisecond) + + sealedGens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("store holds %d sealed generations at height 2: a producer "+ + "that could see the seal published over it anyway", sealedGens) + } +} + +// A producer that got its seal acked and then died leaves the height closed +// in the store and empty on the chain. Muting there strands the height +// forever; building fresh content there would orphan every record the dead +// producer already had acked. Recovery does neither: it rebuilds that exact +// prefix and publishes nothing, because the store already holds the whole +// generation. +func TestPhantomSealRecoversTheExactPrefix(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // A producer sealed height 2 in the store with three records... + t0, t1, t2 := testTx(t, 0), testTx(t, 1), testTx(t, 2) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{t0, t1, t2} { + appendForeignRecord(t, h, tx) + } + + dead := testHeader(2, parent) + appendForeignSeal(t, h, dead) + + // ...and died: the chain has no block at 2. + entriesBefore := len(readAllGenerations(t, h)) + + w := p.AdoptWindow(2, parent) + if w == nil { + t.Fatal("a height sealed in the store but absent from the chain was " + + "muted: nobody rebuilds it and it is stranded forever") + } + + if len(w.Txs) != 3 { + t.Fatalf("recovered %d txs, want the dead producer's 3", len(w.Txs)) + } + + for i, want := range []*types.Transaction{t0, t1, t2} { + if w.Txs[i].Hash() != want.Hash() { + t.Fatalf("recovered tx %d differs: rebuilding here would orphan "+ + "the records the dead producer already had acked", i) + } + } + + if w.Timestamp != dead.Time || w.ParentHash != parent { + t.Fatal("recovery did not inherit the sealed open context") + } + + // The store already holds the generation: the rebuild must add nothing. + p.OpenBlock(2, w.Timestamp, parent, w.GasLimit, w.BaseFee) + p.PublishTx(t0) + time.Sleep(300 * time.Millisecond) + + if got := len(readAllGenerations(t, h)); got != entriesBefore { + t.Fatalf("recovery republished into the store: %d generations, want %d", + got, entriesBefore) + } + + // Nor at the seal: the store already holds this generation's seal, so a + // second one would open a second generation over a sealed height. + p.SealBlock(types.NewBlockWithHeader(dead)) + time.Sleep(300 * time.Millisecond) + + if got := len(readAllGenerations(t, h)); got != entriesBefore { + t.Fatalf("recovery re-sealed into the store: %d generations, want %d", + got, entriesBefore) + } + + // And the block must reach the chain: a gate refusal here would strand + // the height a second time. + if v := p.ConfirmSeal(200 * time.Millisecond); v == miner.SealRefused { + t.Fatal("the recovery block was refused: the height stays stranded") + } +} + +// The ordinary loss must not be mistaken for a phantom: when the chain does +// hold a block at the height, there is nothing to recover and the build mutes. +func TestSealedHeightWithChainBlockStillMutes(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 0)) + + winner := testHeader(2, parent) + appendForeignSeal(t, h, winner) + + // The winner's block is on our chain: an ordinary loss. + chain.canonical[2] = winner.Hash() + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatalf("recovered a height the chain already has: %+v", w) + } +} + +// A seal only moments old may simply be in flight. Rebuilding for it that +// early races the real broadcast, so the grace has to elapse first — and +// until it does, a build carrying different content must not go out. +func TestFreshSealIsWaitedForNotRebuilt(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + now := uint64(time.Now().Unix()) + appendForeignOpenAt(t, h, 2, parent, now) + appendForeignRecord(t, h, testTx(t, 0)) + + fresh := testHeader(2, parent) + fresh.Time = now + appendForeignSeal(t, h, fresh) + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatal("rebuilt a seal that is still within its broadcast grace: " + + "this races the block already on its way") + } + + // The build proceeds anyway (mute does not stop the miner), so the gate + // is the last line: divergent content over a standing seal must not + // broadcast just because the height is undecided. + ours := testHeader(2, parent) + ours.Extra = []byte("ours") + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 9)})) + + if v := p.ConfirmSeal(300 * time.Millisecond); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: broadcasting here forks the "+ + "height away from the sequenced content", v) + } + + // And it published nothing: the store's generation stands alone. + sealedGens := 0 + + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("%d sealed generations at height 2, want the store's one", + sealedGens) + } +} + +// Once the grace has passed with no block, the same height is rebuilt from +// the store rather than left stranded. +func TestStaleSealIsRebuiltAfterGrace(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + past := uint64(time.Now().Add(-2 * recoverGrace).Unix()) + appendForeignOpenAt(t, h, 2, parent, past) + appendForeignRecord(t, h, testTx(t, 0)) + + dead := testHeader(2, parent) + dead.Time = past + appendForeignSeal(t, h, dead) + + if p.AdoptWindow(2, parent) == nil { + t.Fatal("a seal past its grace with no block was not rebuilt") + } +} + +// A consumer whose per-block fetch fails while the tail still reads: the +// store says the height is sealed, but its content cannot be recovered. +type unfetchableConsumer struct { + pb.ConsumerServiceClient +} + +func (c *unfetchableConsumer) GetBlock(ctx context.Context, req *pb.GetBlockRequest, + opts ...grpc.CallOption, +) (*pb.GetBlockResponse, error) { + return nil, status.Error(codes.Unavailable, "block fetch unavailable") +} + +// Refusing a height is only safe while the rebuild that resolves it is still +// coming. If recovery can never reconstruct the height, refusing every build +// there would halt the chain at that block forever — so an unrecoverable +// sealed height falls back to liveness. +func TestUnrecoverableSealedHeightStillBroadcasts(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + past := uint64(time.Now().Add(-2 * recoverGrace).Unix()) + appendForeignOpenAt(t, h, 2, parent, past) + appendForeignRecord(t, h, testTx(t, 0)) + + lost := testHeader(2, parent) + lost.Time = past + appendForeignSeal(t, h, lost) + + p.mu.Lock() + p.read.cons = &unfetchableConsumer{ConsumerServiceClient: p.read.cons} + p.mu.Unlock() + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatal("recovered a height whose content could not be fetched") + } + + ours := testHeader(2, parent) + ours.Extra = []byte("ours") + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 9)})) + + if v := p.ConfirmSeal(300 * time.Millisecond); v == miner.SealRefused { + t.Fatal("refused an unrecoverable height: every build here refuses " + + "the same way and the chain never gets past this block") + } +} + +// The store rejecting our seal is a verdict, not slowness: another +// producer's window owns the height. Riding the liveness timeout out to a +// broadcast after that is the double broadcast the gate exists to prevent. +func TestRejectedSealRefusesInsteadOfTimingOutToBroadcast(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + // Entries that are not this block's seal must not decide the gate: a + // refusal has to name our seal at our height, or an unrelated rejection + // would withhold a block nobody else is producing. + p.markGateLost(journalItem{kind: entryRecord, height: 2}) + p.markGateLost(journalItem{kind: entrySeal, height: 3}) + + p.mu.Lock() + spurious := p.gate.verdict == gateLost + p.mu.Unlock() + + if spurious { + t.Fatal("an unrelated rejection lost the gate") + } + + p.markGateLost(journalItem{kind: entrySeal, height: 2}) + + start := time.Now() + + if v := p.ConfirmSeal(sealGateBudget); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: the store rejected this "+ + "block's seal", v) + } + + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("took %v to refuse a seal the store already rejected: a "+ + "known verdict must not wait out the liveness budget", elapsed) + } +} + +// A refused block and its rebuild share a height, and the gate must confirm +// only on the gated block's own seal. Two layers enforce it: retire's +// lineage guard (a different block folds differently, so a stale ack fails +// the byte match) and the gate's hash key. Each is exercised here through a +// journal the item genuinely stands in. +func TestLateAckForAnotherSealDoesNotConfirmTheGate(t *testing.T) { + first := testHeader(2, common.Hash{0xaa}) + rebuild := testHeader(2, common.Hash{0xaa}) + rebuild.Extra = []byte("rebuild") + + seal := func(h *types.Header) ([]byte, *pb.Entry) { + raw, err := rlp.EncodeToBytes(h) + if err != nil { + t.Fatalf("encode: %v", err) + } + + return raw, sealEntry(raw, commitment.Head{}) + } + + ack := func(sealed *types.Header, gated common.Hash) gateVerdict { + p := barePublisher() + + _, entry := seal(sealed) + post := commitment.Head{0x99} + + p.mu.Lock() + p.journal.append(entry, commitment.Head{}, post, entrySeal, 2, 0, nil) + item := p.journal.items[len(p.journal.items)-1] + p.gate = sealGate{height: 2, hash: gated} + p.mu.Unlock() + + p.retire(item, time.Now()) + + p.mu.Lock() + defer p.mu.Unlock() + + return p.gate.verdict + } + + if v := ack(first, rebuild.Hash()); v == gateConfirmed { + t.Fatal("a late ack for a different block's seal confirmed the gate") + } + + if v := ack(rebuild, rebuild.Hash()); v != gateConfirmed { + t.Fatalf("the gated block's own seal ack did not confirm (verdict %d)", v) + } +} + +// A flush withheld while its gate is pending is the verdict, not a wait for +// one: the block is unbroadcast, so the canonical proof the withhold wants +// can never arrive. The refusal must also unwedge the rebuild — dead flush +// dropped, sealed tip rolled back, hold cleared — or every later build at +// this height orders itself behind a flush that cannot resolve. The +// liveness fallback's tolerance covers only the store's seal, so a foreign +// live window refuses a tolerant gate all the same. +func TestWithheldFlushRefusesThePendingGate(t *testing.T) { + for _, tolerate := range []bool{false, true} { + name := "plain gate" + if tolerate { + name = "tolerant gate" + } + + t.Run(name, func(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + h1 := testHeader(1, common.Hash{0xef}) + header := testHeader(2, h1.Hash()) + p.SealBlock(blockFor(header, []*types.Transaction{testTx(t, 1)})) + + p.mu.Lock() + p.gate.tolerateSealed = tolerate + p.mu.Unlock() + + foreign := openEntry(commitment.OpenContext{ + Number: 2, + Timestamp: header.Time + 7, + ParentHash: h1.Hash(), + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, commitment.Head{0x66}) + + info := tailInfo{ + s: commitment.Head{0x66}, + tipOpen: true, + tipOpenHeight: 2, + window: []*pb.Entry{foreign}, + } + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + if v := p.ConfirmSeal(sealGateBudget); v != miner.SealRefused { + t.Fatalf("verdict = %v, want refused", v) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if items, covered := p.journal.after(p.ackedSeq); !covered || len(items) != 0 { + t.Fatalf("the dead flush survived the refusal (covered=%v len=%d)", covered, len(items)) + } + + if p.sealedTip != 1 { + t.Fatalf("sealedTip = %d, want 1: the rebuild must adopt here, not mute", p.sealedTip) + } + + if p.hold.active() { + t.Fatal("the withheld flush's hold outlived the flush") + } + }) + } +} + +// A decoded store seal at the gated height settles the gate from +// classification: ours confirms (a twin delivered our copy), foreign +// refuses and drops the flush. +func TestStoreSealAtGateHeightSettlesTheGate(t *testing.T) { + h1 := testHeader(1, common.Hash{0xef}) + + cases := []struct { + name string + sealed func(ours common.Hash) common.Hash + want miner.SealVerdict + dropped bool + }{ + { + name: "foreign seal refuses", + sealed: func(common.Hash) common.Hash { return common.Hash{0xdd} }, + want: miner.SealRefused, + dropped: true, + }, + { + name: "our own seal confirms", + sealed: func(ours common.Hash) common.Hash { return ours }, + want: miner.SealConfirmed, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + header := testHeader(2, h1.Hash()) + p.SealBlock(blockFor(header, nil)) + + info := tailInfo{ + s: commitment.Head{0x55}, + haveSeal: true, + sealDecoded: true, + lastSealHeight: 2, + lastSealHash: tc.sealed(header.Hash()), + } + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + if v := p.ConfirmSeal(sealGateBudget); v != tc.want { + t.Fatalf("verdict = %v, want %v", v, tc.want) + } + + p.mu.Lock() + defer p.mu.Unlock() + + items, covered := p.journal.after(p.ackedSeq) + if gone := covered && len(items) == 0; gone != tc.dropped { + t.Fatalf("flush dropped = %v, want %v", gone, tc.dropped) + } + }) + } +} + +// A STALE while the gate is pending marks it contested, and a contested +// gate outlives the uncontested budget: the reconcile loop is producing the +// verdict, and cutting the wait short is what broadcast an empty block +// three seconds before its refusal would have arrived. +func TestContestedGateWaitsPastTheUncontestedBudget(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + h1 := testHeader(1, common.Hash{0xef}) + p.SealBlock(blockFor(testHeader(2, h1.Hash()), nil)) + + // A record of the flush STALEd: contest, not verdict. + p.markGateLost(journalItem{kind: entryRecord, height: 2}) + + verdicts := make(chan miner.SealVerdict, 1) + + go func() { verdicts <- p.ConfirmSeal(30 * time.Millisecond) }() + + select { + case v := <-verdicts: + t.Fatalf("gate resolved %v inside the uncontested budget; the contest must extend the wait", v) + case <-time.After(150 * time.Millisecond): + } + + // The seal's own STALE is the verdict. + p.markGateLost(journalItem{kind: entrySeal, height: 2}) + + select { + case v := <-verdicts: + if v != miner.SealRefused { + t.Fatalf("verdict = %v, want refused", v) + } + case <-time.After(2 * time.Second): + t.Fatal("contested gate never consumed the late verdict") + } +} + +// The last look before a verdictless broadcast: silence from the gate is +// not license to bury what the store accepted while we waited. +func TestGateTimeoutRecheck(t *testing.T) { + cases := []struct { + name string + tolerate bool + ownSeal bool + record bool + covered bool + want miner.SealVerdict + }{ + {name: "foreign seal refuses", want: miner.SealRefused}, + {name: "our own seal in the store confirms", ownSeal: true, want: miner.SealConfirmed}, + {name: "acked records the block lacks refuse", record: true, want: miner.SealRefused}, + {name: "a window the block carries broadcasts", record: true, covered: true, want: miner.SealUnknown}, + {name: "the liveness fallback tolerates the foreign seal", tolerate: true, want: miner.SealUnknown}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + gated := testHeader(2, parent) + tx := testTx(t, 3) + + appendForeignOpen(t, h, 2, parent) + + switch { + case tc.record: + appendForeignRecord(t, h, tx) + case tc.ownSeal: + appendForeignSeal(t, h, gated) + default: + lost := testHeader(2, parent) + lost.Extra = []byte("foreign") + appendForeignSeal(t, h, lost) + } + + var txs []common.Hash + if tc.covered { + txs = []common.Hash{tx.Hash()} + } + + p.mu.Lock() + p.gate = sealGate{height: 2, hash: gated.Hash(), txs: txs, tolerateSealed: tc.tolerate} + p.mu.Unlock() + + if v := p.ConfirmSeal(30 * time.Millisecond); v != tc.want { + t.Fatalf("verdict = %v, want %v", v, tc.want) + } + }) + } +} + +// A foreign live window above an unproven flush withholds the re-anchor: +// folding past it is how two full acked generations were buried under an +// empty sealed block. Only the chain ratifying our flush licenses the move, +// and while the gate is pending the withhold doubles as the refusal. +func TestFlushWithholdsUnderAForeignWindowAbove(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + h1 := testHeader(1, common.Hash{0xef}) + header := testHeader(2, h1.Hash()) + p.SealBlock(blockFor(header, []*types.Transaction{testTx(t, 1)})) + + foreign := openEntry(commitment.OpenContext{ + Number: 3, + Timestamp: header.Time + 9, + ParentHash: common.Hash{0x77}, + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, commitment.Head{0x88}) + + info := tailInfo{ + s: commitment.Head{0x88}, + tipOpen: true, + tipOpenHeight: 3, + window: []*pb.Entry{foreign}, + haveSeal: true, + lastSealHeight: 1, + lastSealHash: common.Hash{0xaa}, + } + + journalBefore := len(p.journal.items) + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.hold.kind != holdSticky { + t.Fatal("flush over a foreign window above was not withheld") + } + + if len(p.journal.items) != journalBefore { + t.Fatal("withhold must not touch the lineage") + } + + if p.gate.verdict != gateLost { + t.Fatalf("gate verdict = %d, want lost: the withhold is the verdict", p.gate.verdict) + } +} diff --git a/eth/sequencer/harness_test.go b/eth/sequencer/harness_test.go new file mode 100644 index 0000000000..bfa1939e1c --- /dev/null +++ b/eth/sequencer/harness_test.go @@ -0,0 +1,61 @@ +package sequencer + +import ( + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// awaitOurWindow runs the seal barrier for a block whose content is exactly +// what this publisher has published — the healthy case, and what every +// barrier test meant before the barrier compared content. Tests that need a +// block diverging from the journal call AwaitSequenced directly. +func awaitOurWindow(p *Publisher, timeout time.Duration) bool { + p.mu.Lock() + height := p.curHeight + txs := journalTxs(p) + p.mu.Unlock() + + return p.AwaitSequenced(timeout, height, txs) +} + +func journalTxs(p *Publisher) []*types.Transaction { + start := p.journal.openStart() + if start < 0 { + return nil + } + + var txs []*types.Transaction + + for _, it := range p.journal.items[start+1:] { + rec := it.entry.GetRecord() + if rec == nil { + continue + } + + for _, raw := range rec.GetTransactions() { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err == nil { + txs = append(txs, tx) + } + } + } + + return txs +} + +// sealOnChain seals a block and records it as this node's block at that +// height — what the miner does, since resultLoop writes the block before it +// announces. A flush only displaces a live foreign window on proof the chain +// kept our block, so a test producer that seals without a chain write is +// modelling a block nobody accepted. +func sealOnChain(p *Publisher, fc *fakeChain, header *types.Header, txs []*types.Transaction) { + p.SealBlock(blockFor(header, txs)) + + if fc.canonical == nil { + fc.canonical = map[uint64]common.Hash{} + } + + fc.canonical[header.Number.Uint64()] = header.Hash() +} diff --git a/eth/sequencer/journal.go b/eth/sequencer/journal.go new file mode 100644 index 0000000000..38164ffd62 --- /dev/null +++ b/eth/sequencer/journal.go @@ -0,0 +1,413 @@ +package sequencer + +import ( + "bytes" + "sort" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +func openMatchesHeader(open *pb.BlockOpen, header *types.Header) bool { + return open != nil && open.GetBlockTimestamp() == header.Time && + open.GetGasLimit() == header.GasLimit && + common.BytesToHash(open.GetParentHash()) == header.ParentHash && + bytes.Equal(open.GetBaseFee(), baseFeeBytes(header.BaseFee)) +} + +func recordsMirrorTxs(items []journalItem, txs types.Transactions) bool { + i := 0 + + for _, it := range items { + if it.kind != entryRecord { + return false + } + + for _, h := range it.txHashes { + if i >= len(txs) || h != txs[i].Hash() { + return false + } + + i++ + } + } + + return i == len(txs) +} + +// How a window standing in the store relates to the one being built. +type windowRelation int + +const ( + windowForeign windowRelation = iota // a different lineage: contention + windowOurs // ours, whole or a prefix of it + windowExtendsOurs // ours, plus records we do not hold +) + +// relateWindowLocked compares the window the read returned against the one +// this node holds, entry by entry. Length alone cannot separate the cases: +// a rival's window at the same height can be shorter, equal, or longer than +// ours, and only content says which lineage it belongs to. +func (p *Publisher) relateWindowLocked(info tailInfo, height uint64) windowRelation { + ours := p.journal.suffixFromHeight(height) + if len(info.window) == 0 || len(ours) == 0 || ours[0].kind != entryOpen { + return windowForeign + } + + for i := 0; i < min(len(info.window), len(ours)); i++ { + if !contentEqual(info.window[i], ours[i].entry) { + return windowForeign + } + } + + if len(info.window) > len(ours) { + return windowExtendsOurs + } + + return windowOurs +} + +const ( + entryOpen = iota + entryRecord + entrySeal +) + +// journalItem is one published (or provisionally folded, while the store is +// unreachable) entry with the fold heads around it. Items carry a monotonic +// seq so cursor positions survive eviction. +type journalItem struct { + seq uint64 + entry *pb.Entry + pre commitment.Head + post commitment.Head + kind int + height uint64 + // txHashes caches a record's transaction hashes at append time, so the + // per-block mirror checks compare 32-byte values instead of re-decoding + // and re-hashing the window's wire bytes under the publisher lock. + txHashes []common.Hash +} + +// journal is the publisher's window in flight: the open window, the +// sealed-but-undelivered flushes actively streaming (at most +// journalHotSeals — older ones collapse to a height range and are rebuilt +// from the chain database when the store can take them), and up to +// journalSealedBlocks delivered blocks kept for reconnect replay. The +// open window is always retained. The chain database is the archive; +// the journal never grows beyond the work in flight. +type journal struct { + items []journalItem + seals int + nextSeq uint64 +} + +const ( + journalSealedBlocks = 8 + // journalHotSeals bounds the undelivered flushes kept in memory; older + // ones collapse to a pending height range (rebuilt from the chain + // database on reconcile). + journalHotSeals = 2 + // backfillBatchBytes caps how much block rebuilding one drain cycle does + // while holding the publisher lock. Every miner call — PublishTx most of + // all — waits on that lock, so a large batch stalls production outright: + // a 32MB batch cost a devnet 44 seconds on the cycle after a store + // outage. Smaller batches drain over more cycles and never own the lock + // long enough to be felt. + backfillBatchBytes = 2 << 20 + + // journalMaxBytes caps the adoption-collection read and one backfill + // batch — not a retention bound. + journalMaxBytes = 32 << 20 +) + +func newJournal() *journal { + return &journal{nextSeq: 1} +} + +func (r *journal) append(entry *pb.Entry, pre, post commitment.Head, kind int, height uint64, ackedThrough uint64, txHashes []common.Hash) { + // Every record carries its hashes; decode here only for a caller that + // could not supply them cheaper. + if kind == entryRecord && txHashes == nil { + for _, raw := range entry.GetRecord().GetTransactions() { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + break + } + + txHashes = append(txHashes, tx.Hash()) + } + } + + item := journalItem{ + seq: r.nextSeq, + entry: entry, + pre: pre, + post: post, + kind: kind, + height: height, + txHashes: txHashes, + } + + r.nextSeq++ + r.items = append(r.items, item) + + if kind == entrySeal { + r.seals++ + } + + r.evict(ackedThrough) +} + +// openStart returns the index of the open entry starting the current +// (unsealed) window, or -1 when the journal's tail is sealed or empty. +func (r *journal) openStart() int { + for i := len(r.items) - 1; i >= 0; i-- { + switch r.items[i].kind { + case entrySeal: + return -1 + case entryOpen: + return i + } + } + + return -1 +} + +// evict drops whole sealed blocks from the front, never touching the +// current open window. Delivered blocks evict past journalSealedBlocks (the +// gap-fill replay depth); undelivered seals are what a flush still owes +// the store and ride out retry pacing — until the hard count or byte cap, +// whose overflow is the documented forward-jump. +func (r *journal) evict(ackedThrough uint64) { + for r.seals > journalSealedBlocks && r.oldestSealAcked(ackedThrough) && r.dropOldestSealed() { + } +} + +// collapseOldestUnacked drops the oldest sealed block whose seal is still +// undelivered, returning its height so the caller can record it for a +// chain-database rebuild. Delivered blocks in front of it drop too — the +// store already has them. +func (r *journal) collapseOldestUnacked(acked uint64) (uint64, int, bool) { + i := r.firstSeal() + for i >= 0 && r.items[i].seq <= acked { + r.dropOldestSealed() + i = r.firstSeal() + } + + if i < 0 { + return 0, 0, false + } + + h := r.items[i].height + removed := 0 + + for j := 0; j <= i; j++ { + if r.items[j].seq > acked { + removed++ + } + } + + r.dropOldestSealed() + + return h, removed, true +} + +// firstSeal returns the index of the oldest seal, or -1 when none. +func (r *journal) firstSeal() int { + for i := range r.items { + if r.items[i].kind == entrySeal { + return i + } + } + + return -1 +} + +// oldestSealAcked reports whether the front sealed block was delivered. +func (r *journal) oldestSealAcked(ackedThrough uint64) bool { + i := r.firstSeal() + + return i >= 0 && r.items[i].seq <= ackedThrough +} + +// dropOldestSealed removes items from the front through the first seal. +// Returns false when no sealed block remains to drop. +func (r *journal) dropOldestSealed() bool { + i := r.firstSeal() + if i < 0 { + return false + } + + r.items = append(r.items[:0], r.items[i+1:]...) + r.seals-- + + return true +} + +// truncate drops every item from index n on, keeping the byte and seal +// accounting exact. +func (r *journal) truncate(n int) { + for _, dropped := range r.items[n:] { + if dropped.kind == entrySeal { + r.seals-- + } + } + + r.items = r.items[:n] +} + +// cutFromHeight returns the index where the undelivered run at or above +// height begins — the truncation point that drops a refused flush and +// anything chained above it while keeping acked entries and older flushes. +func (r *journal) cutFromHeight(acked, height uint64) int { + cut := len(r.items) + + for cut > 0 { + it := r.items[cut-1] + if it.seq <= acked || it.height < height { + break + } + + cut-- + } + + return cut +} + +// rebuildCut returns the index just past the last seal or last acked item — +// the boundary a window rebuild must not rewind past (an undelivered flush +// is still owed to the store; confirmed entries never rewind). +func (r *journal) rebuildCut(acked uint64) int { + for i := len(r.items) - 1; i >= 0; i-- { + if it := r.items[i]; it.kind == entrySeal || it.seq <= acked { + return i + 1 + } + } + + return 0 +} + +// itemAt returns the journal item carrying seq, if it is still present — +// lineage swaps and rewinds replace or drop items, so a seq alone no +// longer identifies live content. +func (r *journal) itemAt(seq uint64) (journalItem, bool) { + i := sort.Search(len(r.items), func(i int) bool { return r.items[i].seq >= seq }) + if i < len(r.items) && r.items[i].seq == seq { + return r.items[i], true + } + + return journalItem{}, false +} + +// after returns the items strictly after seq, and whether that position is +// still covered by the journal (false means eviction created a gap). +func (r *journal) after(seq uint64) ([]journalItem, bool) { + if len(r.items) == 0 { + return nil, seq+1 >= r.nextSeq + } + + if seq+1 < r.items[0].seq { + return nil, false + } + + i := sort.Search(len(r.items), func(i int) bool { return r.items[i].seq > seq }) + + return r.items[i:], true +} + +// findPost returns the seq of the item whose post-fold head equals h. +func (r *journal) findPost(h commitment.Head) (uint64, bool) { + for i := range r.items { + if r.items[i].post == h { + return r.items[i].seq, true + } + } + + return 0, false +} + +// suffixFromHeight returns the retained items starting at the first open +// entry with height >= h (a window boundary), or nil when none exists. +func (r *journal) suffixFromHeight(h uint64) []journalItem { + for i := range r.items { + if r.items[i].kind == entryOpen && r.items[i].height >= h { + return r.items[i:] + } + } + + return nil +} + +// journalWindowLocked lists the transactions this publisher has already +// published for a height, in order. +func (p *Publisher) journalWindowLocked(height uint64) []common.Hash { + start := p.journal.openStart() + if start < 0 || p.journal.items[start].height != height { + return nil + } + + var out []common.Hash + + for _, it := range p.journal.items[start+1:] { + if it.kind == entryRecord { + out = append(out, it.txHashes...) + } + } + + return out +} + +// windowTxHashes lists the transactions a store window promises, in order. +func windowTxHashes(window []*pb.Entry) ([]common.Hash, bool) { + var out []common.Hash + + for _, e := range window { + rec := e.GetRecord() + if rec == nil { + continue + } + + for _, raw := range rec.GetTransactions() { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + return nil, false + } + + out = append(out, tx.Hash()) + } + } + + return out, true +} + +// windowLeadsHashes reports whether the block delivers the promised +// sequence: the window's transactions must be a leading run of the block's, +// so a block may still be filling past the window but may never drop or +// reorder what the store already acked. +func windowLeadsHashes(window, txs []common.Hash) bool { + if len(window) > len(txs) { + return false + } + + for i, h := range window { + if txs[i] != h { + return false + } + } + + return true +} + +// txHashes projects transactions onto their hashes. +func txHashes(txs types.Transactions) []common.Hash { + out := make([]common.Hash, len(txs)) + for i, tx := range txs { + out[i] = tx.Hash() + } + + return out +} diff --git a/eth/sequencer/metrics.go b/eth/sequencer/metrics.go new file mode 100644 index 0000000000..6244fbc0c4 --- /dev/null +++ b/eth/sequencer/metrics.go @@ -0,0 +1,56 @@ +package sequencer + +import "github.com/ethereum/go-ethereum/metrics" + +// Publisher state gauge values; 0 is reserved for "off" — +// a disabled node has no publisher and never reports. contending refines +// degraded: the store is healthy, we are just losing head races to another +// publisher. +const ( + gaugeLive = 1 + gaugeDegraded = 2 + gaugeResyncing = 3 + gaugeFailed = 4 + gaugeContending = 5 +) + +var ( + publishAckTimer = metrics.NewRegisteredTimer("sequencer/publish/ack", nil) + publishedCounter = metrics.NewRegisteredCounter("sequencer/publish/entries", nil) + publishDropMeter = metrics.NewRegisteredMeter("sequencer/publish/dropped", nil) + publishFailedGauge = metrics.NewRegisteredGauge("sequencer/publish/failed", nil) + publishStateGauge = metrics.NewRegisteredGauge("sequencer/publish/state", nil) + publishQueueGauge = metrics.NewRegisteredGauge("sequencer/publish/queue", nil) + publishStaleCount = metrics.NewRegisteredCounter("sequencer/publish/stale", nil) + publishMutedCount = metrics.NewRegisteredCounter("sequencer/publish/muted", nil) + publishRecoverCount = metrics.NewRegisteredCounter("sequencer/publish/recovered", nil) + readHeadMismatch = metrics.NewRegisteredCounter("sequencer/read/headmismatch", nil) + readUnexplained = metrics.NewRegisteredCounter("sequencer/read/unexplained", nil) + barrierDivergedCount = metrics.NewRegisteredCounter("sequencer/publish/barrierdiverged", nil) + publishCatchupSkip = metrics.NewRegisteredCounter("sequencer/publish/catchupskip", nil) + backfillBatchTimer = metrics.NewRegisteredTimer("sequencer/backfill/batch", nil) + windowDisplacedRecords = metrics.NewRegisteredCounter("sequencer/reconcile/displacedrecords", nil) + publishBarrierTimeout = metrics.NewRegisteredCounter("sequencer/publish/barriertimeout", nil) + publishRedialCount = metrics.NewRegisteredCounter("sequencer/publish/redial", nil) + + // The seal gate's verdicts: confirmed seals broadcast, refused ones are + // discarded (another producer won the height), unknown means the budget + // expired and liveness broadcast anyway. + gateConfirmedCount = metrics.NewRegisteredCounter("sequencer/gate/confirmed", nil) + gateRefusedCount = metrics.NewRegisteredCounter("sequencer/gate/refused", nil) + gateUnknownCount = metrics.NewRegisteredCounter("sequencer/gate/unknown", nil) + gateRecheckRefused = metrics.NewRegisteredCounter("sequencer/gate/recheckrefused", nil) + + // Consumer-side preconfirmation pipeline: per-tx re-execution latency + // and receipts served to RPC readers before canonical import. + preconfApplyTimer = metrics.NewRegisteredTimer("sequencer/preconf/apply", nil) + preconfServedMeter = metrics.NewRegisteredMeter("sequencer/preconf/served", nil) + + reconcileGapfill = metrics.NewRegisteredCounter("sequencer/reconcile/gapfill", nil) + reconcileAdopt = metrics.NewRegisteredCounter("sequencer/reconcile/adopt", nil) + reconcileSupersede = metrics.NewRegisteredCounter("sequencer/reconcile/supersede", nil) + reconcileForwardJump = metrics.NewRegisteredCounter("sequencer/reconcile/forwardjump", nil) + reconcileYield = metrics.NewRegisteredCounter("sequencer/reconcile/yield", nil) + reconcileResync = metrics.NewRegisteredCounter("sequencer/reconcile/resync", nil) + reconcileTimer = metrics.NewRegisteredTimer("sequencer/reconcile/duration", nil) +) diff --git a/eth/sequencer/mirror_test.go b/eth/sequencer/mirror_test.go new file mode 100644 index 0000000000..3af6a6200f --- /dev/null +++ b/eth/sequencer/mirror_test.go @@ -0,0 +1,615 @@ +package sequencer + +import ( + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/miner" +) + +// The barrier's job is to compare the block against the store, not the +// journal against the store. A block built before an adoption leaves the +// journal perfectly in sync while the block itself carries different +// transactions — position agreement says nothing about what we are about to +// broadcast, and on a devnet that gap put two blocks at one height and +// orphaned the loser's acked records into the next block. +func TestBarrierRefusesABlockMissingThePromisedSequence(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Another producer owns the window at height 2 and has three records + // acked there. + t0, t1, t2 := testTx(t, 0), testTx(t, 1), testTx(t, 2) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{t0, t1, t2} { + appendForeignRecord(t, h, tx) + } + + // Our block carries only two of them: t1 was promised at this height + // and this block does not deliver it. + block := []*types.Transaction{t0, t2} + + if p.AwaitSequenced(time.Second, 2, block) { + t.Fatal("barrier passed a block missing a transaction the store " + + "already acked at this height") + } + + if !p.ResyncNeeded() { + t.Fatal("barrier refused without asking for a rebuild: the slot is " + + "dropped instead of corrected") + } +} + +// The same read, with a block that does deliver the promised sequence, must +// pass — a barrier that refuses the correct block stalls the chain. +func TestBarrierPassesABlockCarryingThePromisedSequence(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + t0, t1 := testTx(t, 0), testTx(t, 1) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{t0, t1} { + appendForeignRecord(t, h, tx) + } + + if !p.AwaitSequenced(time.Second, 2, []*types.Transaction{t0, t1}) { + t.Fatal("barrier refused a block that is exactly the store's window") + } +} + +// A block may still be filling past what the store has acked; only dropping +// or reordering the acked prefix is a violation. +func TestBarrierPassesABlockExtendingThePromisedSequence(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + t0 := testTx(t, 0) + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, t0) + + if !p.AwaitSequenced(time.Second, 2, []*types.Transaction{t0, testTx(t, 5)}) { + t.Fatal("barrier refused a block that carries the promised prefix " + + "and one more transaction of its own") + } +} + +// Reordering the acked prefix breaks the promise as surely as dropping it: +// a preconfirmation names a position, not just membership. +func TestBarrierRefusesAReorderedPromisedSequence(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + t0, t1 := testTx(t, 0), testTx(t, 1) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{t0, t1} { + appendForeignRecord(t, h, tx) + } + + if p.AwaitSequenced(time.Second, 2, []*types.Transaction{t1, t0}) { + t.Fatal("barrier passed a block that reordered the acked prefix") + } +} + +// An unreadable store must not gate production: liveness outranks a check we +// cannot perform. +func TestBarrierPassesWhenTheStoreCannotBeRead(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 0)) + + gate := &blockableConsumer{} + + p.mu.Lock() + gate.ConsumerServiceClient = p.read.cons + p.read.cons = gate + p.mu.Unlock() + gate.block(true) + + if !p.AwaitSequenced(time.Second, 2, []*types.Transaction{testTx(t, 9)}) { + t.Fatal("an unreadable store blocked production") + } +} + +// A re-anchor rebuilds our own entries onto the store head without ever +// ingesting what stands behind it, so displacing a live foreign window +// silently drops every record it holds that our block lacks. That is only +// defensible when our block is the one the chain kept; when it lost, the +// flush must wait rather than destroy the winner's sequence. +func TestFlushWithholdsDisplacementWhenOurBlockLost(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // We build and seal height 2 locally. + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + // Another producer's window takes the head at the same height, with + // records of its own, before our seal goes out. + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 7)) + + before := len(readAllGenerations(t, h)) + + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + // The chain kept that producer's block, not ours. + chain.canonical[2] = common.Hash{0xbb} + + info, _ := p.readTail(t.Context()) + + p.mu.Lock() + out, handled := p.classifyPendingFlushLocked(info) + p.mu.Unlock() + + if !handled || out != recOK { + t.Fatalf("flush classification: out=%v handled=%v", out, handled) + } + + time.Sleep(300 * time.Millisecond) + + if got := len(readAllGenerations(t, h)); got != before { + t.Fatalf("flush displaced a live window for a block the chain did "+ + "not keep: %d generations, want %d", got, before) + } +} + +// The devnet case: our seal is rejected and our flush reaches the store in +// the same millisecond, 134ms before the winner's block imports. The chain +// has no opinion yet at that instant, and treating "undecided" as permission +// is what displaced a live window on behalf of a block that was never +// broadcast. +func TestFlushWithholdsDisplacementWhileChainUndecided(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 7)) + + before := len(readAllGenerations(t, h)) + + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + // The chain holds nothing at this height: the winner's block is still + // in flight. Our block is not known to have been kept. + info, _ := p.readTail(t.Context()) + + p.mu.Lock() + out, handled := p.classifyPendingFlushLocked(info) + p.mu.Unlock() + + if !handled || out != recOK { + t.Fatalf("flush classification: out=%v handled=%v", out, handled) + } + + time.Sleep(300 * time.Millisecond) + + if got := len(readAllGenerations(t, h)); got != before { + t.Fatalf("displaced a live window while the chain had not kept our "+ + "block: %d generations, want %d", got, before) + } +} + +func mustReadTail(t *testing.T, p *Publisher) tailInfo { + t.Helper() + + info, out := p.readTail(t.Context()) + if out != recOK { + t.Fatalf("tail read: %v", out) + } + + return info +} + +// The displacement counter has to name what was actually lost. Counting the +// whole displaced window reports thousands of orphaned records when the +// block delivers every one of them, which makes the metric useless for +// telling a harmless supersede from a damaging one. +func TestDisplacementCountsOnlyWhatTheBlockDropped(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Our window: two transactions, published and acked. + shared, alsoShared := testTx(t, 0), testTx(t, 1) + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(shared) + p.PublishTx(alsoShared) + waitDrained(t, p, 5*time.Second) + + // The window we would displace holds both of ours plus one we lack. + dropped := testTx(t, 7) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{shared, alsoShared, dropped} { + appendForeignRecord(t, h, tx) + } + + // Our block seals with our two, and the chain keeps it, so the flush is + // allowed to displace — the counter must then name the one record lost. + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{shared, alsoShared})) + chain.canonical[2] = ours.Hash() + chain.blocks = map[uint64]*types.Block{ + 2: blockFor(ours, []*types.Transaction{shared, alsoShared}), + } + + info := mustReadTail(t, p) + before := windowDisplacedRecords.Snapshot().Count() + + p.mu.Lock() + may := p.mayDisplaceWindowLocked(info, 2) + p.mu.Unlock() + + if !may { + t.Fatal("withheld a displacement for a block the chain kept") + } + + if got := windowDisplacedRecords.Snapshot().Count() - before; got != 1 { + t.Fatalf("counted %d orphaned records, want 1: only the transaction "+ + "this block does not carry is lost", got) + } +} + +// Under load the drain does not finish inside the barrier's budget, and the +// old deadline path surrendered — returning "seal it" without ever comparing +// content. That is how a block with none of a 9523-record window reached the +// chain while every one of those records was already acked. The check has to +// survive the deadline; a still-draining window is a prefix of our own block, +// so the comparison is valid mid-drain. +func TestBarrierChecksContentEvenWhenTheDrainOvershoots(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Another producer's window stands at our height with records acked. + promised := []*types.Transaction{testTx(t, 0), testTx(t, 1), testTx(t, 2)} + appendForeignOpen(t, h, 2, parent) + + for _, tx := range promised { + appendForeignRecord(t, h, tx) + } + + // Hold our own writes so the journal keeps entries in flight: the + // barrier can never reach unacked == 0 and must take the deadline path. + // A build hold, not a sticky one — sticky is the contended case and + // returns before the deadline is ever reached. + p.mu.Lock() + p.hold = hold{after: p.ackedSeq, kind: holdBuild} + p.mu.Unlock() + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 8)) + + p.mu.Lock() + draining, sticky := p.unackedLocked() > 0, p.hold.kind == holdSticky + p.mu.Unlock() + + if !draining || sticky { + t.Fatalf("setup did not reach the deadline path: draining=%v sticky=%v", + draining, sticky) + } + + // An empty block, exactly the shape that reached the chain at 915. + if p.AwaitSequenced(50*time.Millisecond, 2, nil) { + t.Fatal("barrier surrendered on its deadline and passed a block " + + "carrying none of the promised sequence") + } +} + +// The same deadline path must still pass a block that does carry the +// promised prefix, or a slow drain would stall production outright. +func TestBarrierDeadlineStillPassesACoveringBlock(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + t0 := testTx(t, 0) + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, t0) + + // Our block carries the promised record plus more still in flight. + if !p.AwaitSequenced(50*time.Millisecond, 2, + []*types.Transaction{t0, testTx(t, 1), testTx(t, 2)}) { + t.Fatal("barrier refused a block that carries the promised prefix " + + "and is still filling") + } +} + +// Every per-block wait on the store is pointless once the transport is +// failing, and paying them all pushes blocks past their slot — which is what +// arms bor's span-check path and turns a store outage into a chain +// slowdown. A devnet outage cost 22s per block this way. +func TestUnreachableStoreCostsNoWaiting(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.unreachable.Store(true) + + header := testHeader(2, parent) + p.SealBlock(blockFor(header, nil)) + + start := time.Now() + + if v := p.ConfirmSeal(4 * time.Second); v != miner.SealUnknown { + t.Fatalf("verdict = %v, want Unknown with the store unreachable", v) + } + + if !p.AwaitSequenced(4*time.Second, 3, nil) { + t.Fatal("the barrier gated production on an unreachable store") + } + + if elapsed := time.Since(start); elapsed > 200*time.Millisecond { + t.Fatalf("spent %v waiting on a store known to be down", elapsed) + } +} + +// And the moment a read succeeds the publisher stops treating the store as +// down: a latched flag would keep holding builds after recovery. +func TestSuccessfulReadClearsUnreachable(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + p.unreachable.Store(true) + p.AdoptWindow(2, sealHash(t, sealed)) + + if p.unreachable.Load() { + t.Fatal("a successful build-start read left the store marked down") + } +} + +// While the backfill drains, the store is behind us by construction: there +// is nothing worth comparing against and nothing worth waiting for. Making +// sealing wait through the drain is what stalled a devnet for 44 seconds on +// the cycle after a store outage. +func TestCatchUpDoesNotGateSealing(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // A foreign window stands at our height that our block does not carry — + // normally grounds for a rebuild. Draining a backfill outranks it. + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 0)) + + p.mu.Lock() + p.pendingFrom, p.pendingTo = 1, 1 + p.mu.Unlock() + + start := time.Now() + + if !p.AwaitSequenced(2*time.Second, 2, nil) { + t.Fatal("sealing was gated while the backfill was still draining") + } + + if elapsed := time.Since(start); elapsed > 100*time.Millisecond { + t.Fatalf("waited %v during a drain that makes the comparison "+ + "meaningless anyway", elapsed) + } +} + +// The unreachable flag must be set by the transport layer itself when the +// store goes away — the no-wait short-circuits are worthless if nothing +// arms them. +func TestTransportFailureMarksTheStoreUnreachable(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + p.OpenBlock(2, 1700000002, common.Hash{0xaa}, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + + waitFor(t, 10*time.Second, func() bool { return p.unreachable.Load() }) +} + +// A sealed foreign generation at the flush height is a stronger claim than a +// live window, and the flush must not chain past it on the head's bytes +// alone: with consensus undecided, superseding it is how a twin race minted +// generation after generation at one height. Only affirmative proof that +// the chain kept our block licenses the displacement — and our own seal +// already standing there is re-delivery, not displacement. +func TestFlushWithholdsOverAForeignSealUntilCanonical(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + // The twin's generation seals first; our own block for 2 exists too. + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 7)) + appendForeignSeal(t, h, twin) + + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + info := mustReadTail(t, p) + + if !info.sealDecoded || info.lastSealHeight != 2 { + t.Fatalf("setup: read did not decode the foreign seal (%+v)", info) + } + + p.mu.Lock() + undecided := p.mayDisplaceWindowLocked(info, 2) + chain.canonical[2] = ours.Hash() + won := p.mayDisplaceWindowLocked(info, 2) + chain.canonical[2] = twin.Hash() + lost := p.mayDisplaceWindowLocked(info, 2) + p.mu.Unlock() + + if undecided { + t.Fatal("flush chained past a foreign seal with consensus undecided") + } + + if !won { + t.Fatal("flush withheld even though the chain kept our block") + } + + if lost { + t.Fatal("flush displaced the seal of the block the chain kept") + } +} + +// The store's standing seal being our own block is the duplicate-delivery +// case, not a displacement: it must not be withheld, or a resumed flush +// could never finish. +func TestFlushProceedsOverOurOwnStandingSeal(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + ours := testHeader(2, parent) + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, ours) // the standing seal IS our block + + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + info := mustReadTail(t, p) + + p.mu.Lock() + may := p.mayDisplaceWindowLocked(info, 2) + p.mu.Unlock() + + if !may { + t.Fatal("our own standing seal was treated as a foreign displacement") + } +} + +// A displaced foreign seal has no window in the read to count, and the +// trailing window of a higher height must not be counted against this +// height's block: that mispairing once reported 1,960 phantom orphans for +// a displacement that destroyed nothing. +func TestSealDisplacementCountsNoPhantomOrphans(t *testing.T) { + fc := &fakeChain{canonical: map[uint64]common.Hash{}} + p, _ := lineagePublisher(t, fc) + + h1 := testHeader(1, common.Hash{0xef}) + header := testHeader(2, h1.Hash()) + tx := testTx(t, 1) + sealOnChain(p, fc, header, []*types.Transaction{tx}) + fc.blocks = map[uint64]*types.Block{2: blockFor(header, []*types.Transaction{tx})} + + stranger := testTx(t, 2) + raw, err := stranger.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + open3 := openEntry(commitment.OpenContext{ + Number: 3, + Timestamp: header.Time + 4, + ParentHash: header.Hash(), + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, commitment.Head{0x77}) + + info := tailInfo{ + s: commitment.Head{0x79}, + tipOpen: true, + tipOpenHeight: 3, + window: []*pb.Entry{open3, recordEntry(raw, commitment.Head{0x78})}, + haveSeal: true, + sealDecoded: true, + lastSealHeight: 2, + lastSealHash: common.Hash{0xdd}, + } + + before := windowDisplacedRecords.Snapshot().Count() + + p.mu.Lock() + may := p.mayDisplaceWindowLocked(info, 2) + p.mu.Unlock() + + if !may { + t.Fatal("withheld a canonical-proven displacement") + } + + if got := windowDisplacedRecords.Snapshot().Count() - before; got != 0 { + t.Fatalf("counted %d orphans from a higher height's window", got) + } +} diff --git a/eth/sequencer/outage_test.go b/eth/sequencer/outage_test.go new file mode 100644 index 0000000000..201654a2d9 --- /dev/null +++ b/eth/sequencer/outage_test.go @@ -0,0 +1,187 @@ +package sequencer + +import ( + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// outageChain builds a hash-linked chain the publisher can backfill from: the +// chain database is the archive during an outage, so the fake must serve +// complete blocks by number. +func outageChain(t *testing.T, txsPer int, through uint64) (*fakeChain, []*types.Block) { + t.Helper() + + chain := &fakeChain{ + canonical: map[uint64]common.Hash{}, + known: map[common.Hash]*types.Header{}, + blocks: map[uint64]*types.Block{}, + } + + parent := common.Hash{0xef} + blocks := make([]*types.Block, 0, through) + + for n := uint64(1); n <= through; n++ { + header := testHeader(n, parent) + + var txs []*types.Transaction + for i := 0; i < txsPer; i++ { + txs = append(txs, testTx(t, uint64(i))) + } + + block := blockFor(header, txs) + chain.blocks[n] = block + chain.canonical[n] = block.Hash() + chain.known[block.Hash()] = block.Header() + chain.current = block.Header() + blocks = append(blocks, block) + parent = block.Hash() + } + + return chain, blocks +} + +// sealThrough runs the producer's publishing lifecycle over the given blocks: +// open, records, seal — as the worker would during the outage. +func sealThrough(p *Publisher, blocks []*types.Block) { + for _, b := range blocks { + h := b.Header() + p.OpenBlock(h.Number.Uint64(), h.Time, h.ParentHash, h.GasLimit, h.BaseFee) + + for _, tx := range b.Transactions() { + p.PublishTx(tx) + } + + p.SealBlock(b) + } +} + +// The outage contract: the chain never stops for the store, and when the +// store returns, the producer backfills every missing block — oldest first, +// so a reader never sees the sealed tip jump a gap it would then refuse to +// fill. Nothing during the outage was acked, so nothing was promised, and +// the audit must come back clean. +func TestOutageBackfillsGaplessInOrder(t *testing.T) { + h := startHarness(t) + + const through = 12 + + chain, blocks := outageChain(t, 2, through) + p := newTestPublisher(t, h, chain) + + // Block 1 lands normally; the store then goes down. + sealThrough(p, blocks[:1]) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + // The chain keeps producing through the outage. No ack can arrive, so + // no preconfirmation is issued for any of this. + sealThrough(p, blocks[1:]) + + h.resume() + + // The store converges to the tip: every block sealed, oldest first. + waitFor(t, 20*time.Second, func() bool { + sealed := 0 + + for _, g := range readAllGenerations(t, h) { + if g.sealed { + sealed++ + } + } + + return sealed >= through + }) + + gens := readAllGenerations(t, h) + + last := uint64(0) + for _, g := range gens { + if g.height < last { + t.Fatalf("backfill wrote height %d after height %d: out of order, "+ + "the sealed tip jumped a gap", g.height, last) + } + + last = g.height + } + + canonical := map[uint64][]common.Hash{} + for _, b := range blocks { + hashes := []common.Hash{} + for _, tx := range b.Transactions() { + hashes = append(hashes, tx.Hash()) + } + + canonical[b.NumberU64()] = hashes + } + + audit := auditStore(t, h, canonical) + if !audit.clean() { + t.Fatalf("outage recovery broke promises: %+v", audit) + } +} + +// After catch-up the live stream resumes: a new block opened at the tip +// publishes normally, gated behind nothing. +func TestLiveStreamResumesAfterCatchUp(t *testing.T) { + h := startHarness(t) + + const through = 6 + + chain, blocks := outageChain(t, 1, through) + p := newTestPublisher(t, h, chain) + + sealThrough(p, blocks[:1]) + waitHead(t, h, p, 5*time.Second) + + h.stop() + sealThrough(p, blocks[1:]) + h.resume() + + waitFor(t, 20*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.pendingFrom == 0 && p.unackedLocked() == 0 + }) + + // The next build finds a clean boundary at the tip and streams live. + tip := blocks[len(blocks)-1] + if w := p.AdoptWindow(through+1, tip.Hash()); w != nil { + t.Fatalf("clean boundary after catch-up offered an adoption: %+v", w) + } + + p.OpenBlock(through+1, tip.Header().Time+2, tip.Hash(), tip.GasLimit(), fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 10*time.Second) +} + +// During the outage the barrier must not gate production: the store is +// unreachable, nothing acks, and the chain seals anyway. +func TestBarrierNeverGatesDuringOutage(t *testing.T) { + h := startHarness(t) + + chain, blocks := outageChain(t, 1, 3) + p := newTestPublisher(t, h, chain) + + sealThrough(p, blocks[:1]) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + head := blocks[0] + p.OpenBlock(2, head.Header().Time+2, head.Hash(), head.GasLimit(), fee25()) + p.PublishTx(testTx(t, 0)) + + start := time.Now() + if !awaitOurWindow(p, 300*time.Millisecond) { + t.Fatal("an unreachable store gated a seal") + } + + if time.Since(start) > 2*time.Second { + t.Fatal("the liveness override took too long") + } +} diff --git a/eth/sequencer/probes_test.go b/eth/sequencer/probes_test.go new file mode 100644 index 0000000000..4bafac8b40 --- /dev/null +++ b/eth/sequencer/probes_test.go @@ -0,0 +1,339 @@ +package sequencer + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ethereum/go-ethereum/common" +) + +// edgeConsumer answers Range block reads for heights 1..edge and NOT_FOUND +// above, isolating the probe arithmetic from transport. +type edgeConsumer struct { + pb.ConsumerServiceClient + edge uint64 +} + +func (c *edgeConsumer) Range(_ context.Context, req *pb.RangeRequest, _ ...grpc.CallOption) (*pb.RangeResponse, error) { + h := req.GetBlock() + if c.edge > 0 && h >= 1 && h <= c.edge { + return &pb.RangeResponse{}, nil + } + + return nil, status.Error(codes.NotFound, "unknown block") +} + +func probePublisher(edge uint64) *Publisher { + p := barePublisher() + p.read.cons = &edgeConsumer{edge: edge} + + return p +} + +func TestProbeDownTable(t *testing.T) { + cases := []struct { + name string + edge uint64 + from uint64 + want uint64 + wantFound bool + }{ + {name: "far above edge", edge: 3, from: 10, want: 3, wantFound: true}, + {name: "exact hit", edge: 3, from: 3, want: 3, wantFound: true}, + {name: "one above edge", edge: 3, from: 4, want: 3, wantFound: true}, + {name: "single block store", edge: 1, from: 1, want: 1, wantFound: true}, + {name: "descent lands inside store", edge: 2, from: 4, want: 2, wantFound: true}, + {name: "power of two descent", edge: 7, from: 8, want: 7, wantFound: true}, + // The descent gives up once its stride overshoots (step >= h): a low + // edge far below the start is the floor-read rung's job. + {name: "edge below descent reach", edge: 1, from: 5, wantFound: false}, + {name: "deep gap gives up", edge: 2, from: 100, wantFound: false}, + {name: "empty store", edge: 0, from: 6, wantFound: false}, + {name: "empty store from one", edge: 0, from: 1, wantFound: false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := probePublisher(tc.edge) + + got, found, err := p.read.probeDown(context.Background(), tc.from) + if err != nil { + t.Fatalf("probeDown err: %v", err) + } + + if found != tc.wantFound { + t.Fatalf("found = %v, want %v", found, tc.wantFound) + } + + if found && got != tc.want { + t.Fatalf("edge = %d, want %d", got, tc.want) + } + }) + } +} + +func TestProbeUpTable(t *testing.T) { + cases := []struct { + name string + edge uint64 + h0 uint64 + want uint64 + }{ + {name: "from floor", edge: 3, h0: 1, want: 3}, + {name: "already at edge", edge: 3, h0: 3, want: 3}, + {name: "doubling ascent", edge: 8, h0: 1, want: 8}, + {name: "mid start", edge: 5, h0: 2, want: 5}, + {name: "long ascent", edge: 21, h0: 1, want: 21}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := probePublisher(tc.edge) + + got, err := p.read.probeUp(context.Background(), tc.h0) + if err != nil { + t.Fatalf("probeUp err: %v", err) + } + + if got != tc.want { + t.Fatalf("edge = %d, want %d", got, tc.want) + } + }) + } +} + +type failingConsumer struct { + pb.ConsumerServiceClient +} + +func (c *failingConsumer) Range(context.Context, *pb.RangeRequest, ...grpc.CallOption) (*pb.RangeResponse, error) { + return nil, status.Error(codes.Unavailable, "store down") +} + +// Transport errors must propagate out of the probes — only NOT_FOUND means +// "unknown height"; anything else aborts the ladder rung. +func TestProbesPropagateTransportErrors(t *testing.T) { + p := barePublisher() + p.read.cons = &failingConsumer{} + + if _, _, err := p.read.probeDown(context.Background(), 10); err == nil { + t.Fatal("probeDown swallowed a transport error") + } + + if _, err := p.read.probeUp(context.Background(), 1); err == nil { + t.Fatal("probeUp swallowed a transport error") + } + + if _, _, err := p.read.binarySearchEdge(context.Background(), 1, 5); err == nil { + t.Fatal("binarySearchEdge swallowed a transport error") + } + + if _, err := p.read.blockKnown(context.Background(), 1); err == nil { + t.Fatal("blockKnown treated a transport error as an answer") + } +} + +// Unacked accounting is by sequence arithmetic: entries evicted from the +// journal while unconfirmed still count (they are what a forward jump drops). +func TestUnackedCountsEvicted(t *testing.T) { + p := barePublisher() + + parent := common.Hash{0xef} + for n := uint64(1); n <= 42; n++ { + header := testHeader(n, parent) + p.OpenBlock(n, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + parent = header.Hash() + } + + if p.failed.Load() { + t.Fatal("publisher failed during setup") + } + + total := 2 * 42 + + p.mu.Lock() + defer p.mu.Unlock() + + if got := p.unackedLocked(); got != total { + t.Fatalf("unacked = %d, want %d (evicted entries must count)", got, total) + } + + if int(p.journal.nextSeq-1) != total { + t.Fatalf("nextSeq = %d, want %d appends", p.journal.nextSeq-1, total) + } + + if items, _ := p.journal.after(0); len(items) >= total { + t.Fatal("eviction did not trim the journal; test premise broken") + } + + p.ackedSeq = 3 + + if got := p.unackedLocked(); got != total-3 { + t.Fatalf("unacked after acks = %d, want %d", got, total-3) + } +} + +// endlessConsumer returns a full page on every Range call and never reports +// live — a tail longer than the walk bound from the requested position. +type endlessConsumer struct { + pb.ConsumerServiceClient +} + +func (c *endlessConsumer) Range(_ context.Context, req *pb.RangeRequest, _ ...grpc.CallOption) (*pb.RangeResponse, error) { + n := int(req.GetLimit()) + if n == 0 || n > 64 { + n = 64 + } + + entries := make([]*pb.Entry, n) + for i := range entries { + entries[i] = &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{}}} + } + + return &pb.RangeResponse{Entries: entries, Next: make([]byte, 32)}, nil +} + +// A rung whose position is too far behind the tip to walk within bound must +// fall through the ladder (like NOT_FOUND), not spin on the same rung. +func TestTryWalkFallsThroughOnTooLongTail(t *testing.T) { + p := barePublisher() + p.read.cons = &endlessConsumer{} + + _, out, done := p.tryWalk(context.Background(), &pb.RangeRequest{}, false) + if done { + t.Fatal("too-long tail must fall through to the next ladder rung") + } + + if out != recRetry { + t.Fatalf("outcome = %v, want recRetry", out) + } +} + +// slowEndlessConsumer serves endless slow pages for tail walks from one +// specific position (a stale anchor over a huge history) and delegates +// everything else to the real store client. +type slowEndlessConsumer struct { + pb.ConsumerServiceClient + stale []byte +} + +func (c *slowEndlessConsumer) Range(ctx context.Context, req *pb.RangeRequest, opts ...grpc.CallOption) (*pb.RangeResponse, error) { + if h := req.GetHead(); h != nil && bytes.Equal(h, c.stale) { + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + // Like the real gRPC client: a status error, not the raw + // context sentinel. + return nil, status.Error(codes.DeadlineExceeded, ctx.Err().Error()) + } + + n := int(req.GetLimit()) + if n == 0 || n > 64 { + n = 64 + } + + entries := make([]*pb.Entry, n) + for i := range entries { + entries[i] = &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{}}} + } + + return &pb.RangeResponse{Entries: entries, Next: c.stale}, nil + } + + return c.ConsumerServiceClient.Range(ctx, req, opts...) +} + +// A takeover build whose publisher anchored far behind (an idle adopter) +// must still find and adopt the dangling window: the stale-anchor rung +// runs out of its budget slice and the ladder falls through to the +// tip-edge probe instead of consuming the whole read budget. +func TestStaleAnchorTakeoverStillAdopts(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx := testTx(t, 0) + foreignWindow(t, h, 2, parent, tx) + + // Pin the anchor (and the persisted head) to a position that walks + // forever — the stale-anchor shape after a long idle stretch. + stale := commitment.Head{0xaa, 0xbb} + p.mu.Lock() + p.read.cons = &slowEndlessConsumer{ConsumerServiceClient: p.read.cons, stale: stale.Bytes()} + p.anchor = stale + p.confirmed = true + p.mu.Unlock() + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 || w.Txs[0].Hash() != tx.Hash() { + t.Fatalf("stale-anchor read missed the window: %+v", w) + } +} + +// An idle publisher (a non-producer between spans) periodically re-anchors +// near the store tip, so its eventual takeover read starts close. +func TestIdleReconcileKeepsAnchorFresh(t *testing.T) { + restore := idleReconcileInterval + idleReconcileInterval = 80 * time.Millisecond + + t.Cleanup(func() { idleReconcileInterval = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // A foreign producer seals 2 and opens 3: the idle publisher should + // re-anchor at window 3's base without any build of its own. + parent := sealHash(t, sealed) + ts, gasLimit := foreignWindow(t, h, 2, parent, testTx(t, 0)) + appendForeignSeal(t, h, windowHeader(2, parent, ts, gasLimit)) + + base := h.store.Head() + foreignWindow(t, h, 3, common.Hash{0x33}, testTx(t, 1)) + + waitFor(t, 5*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.anchor == base + }) +} + +// emptyNotLiveConsumer models a trailing gateway: an empty page that is +// not yet at the tip. floorRead must fall through to the full floor walk +// instead of indexing the empty page (a crash found by adversarial review). +type emptyNotLiveConsumer struct{ pb.ConsumerServiceClient } + +func (c *emptyNotLiveConsumer) Range(_ context.Context, req *pb.RangeRequest, _ ...grpc.CallOption) (*pb.RangeResponse, error) { + if req.GetLimit() == 1 { + return &pb.RangeResponse{Next: make([]byte, 32), Live: false}, nil + } + + return &pb.RangeResponse{Next: make([]byte, 32), Live: true}, nil +} + +func TestFloorReadEmptyNotLivePage(t *testing.T) { + p := barePublisher() + p.read.cons = &emptyNotLiveConsumer{} + + info, out := p.read.floorRead(t.Context()) + if out == recTerminal { + t.Fatalf("empty not-live floor page must not be terminal: %v", out) + } + + _ = info +} diff --git a/eth/sequencer/publish.go b/eth/sequencer/publish.go new file mode 100644 index 0000000000..d35caba510 --- /dev/null +++ b/eth/sequencer/publish.go @@ -0,0 +1,314 @@ +package sequencer + +import ( + "math/big" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// OpenBlock publishes block context the moment the producer opens it. A +// re-open of the same height (work-cycle restart) or an open on a different +// parent (reorg mid-build) supersedes the in-progress window downstream. +func (p *Publisher) OpenBlock(number uint64, timestamp uint64, parent common.Hash, gasLimit uint64, baseFee *big.Int) { + if p.failed.Load() { + return + } + + open := commitment.OpenContext{ + Number: number, + Timestamp: timestamp, + ParentHash: parent, + GasLimit: gasLimit, + BaseFee: baseFee, + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.mode.kind != modeOpen || (number != 0 && number <= p.sealedTip) { + // Muted, or a seal result raced the mute clear between this + // build's check and its open: never open at/behind a sealed + // height. + return + } + + if p.adoptOpenLocked(number, timestamp, parent, gasLimit, baseFee) { + return // engaged an adopted window already in the store + } + + next, err := commitment.FoldOpen(p.head, open) + if err != nil { + p.fail("fold open context", "err", err) + + return + } + + p.awaitOpen = false + p.curHeight = number + p.appendLocked(openEntry(open, p.head), next, entryOpen, number, nil) +} + +// PublishTx publishes one committed transaction. +func (p *Publisher) PublishTx(tx *types.Transaction) { + if p.failed.Load() { + return + } + + raw, err := tx.MarshalBinary() + if err != nil { + p.fail("encode transaction", "hash", tx.Hash(), "err", err) + + return + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.mode.kind != modeOpen || p.awaitOpen { + return + } + + if p.adoptTxLocked(tx.Hash()) { + return // matched: the record is already in the store + } + + p.appendLocked(recordEntry(raw, p.head), commitment.FoldTx(p.head, raw), entryRecord, p.curHeight, []common.Hash{tx.Hash()}) +} + +// SealBlock is the seal flush: the block sealed and is being +// broadcast, so the store must come to match it. The lineage's current +// window either already mirrors the sealed content (the journal was built +// alongside the block) or is rebuilt from the block's body; the seal +// closes it and the hold lifts — buffered entries and the seal go out +// together. A seal is never dropped. STALEs on the released entries land +// in reconciliation, which completes (byte-match) or re-anchors +// (the only supersede). +func (p *Publisher) SealBlock(block *types.Block) { + if p.failed.Load() { + return + } + + header := block.Header() + + raw, err := rlp.EncodeToBytes(header) + if err != nil { + p.fail("encode sealed header", "number", header.Number, "err", err) + + return + } + + p.mu.Lock() + defer p.mu.Unlock() + + p.adopt = nil + + mode := p.mode + p.mode = buildMode{} + + n := header.Number.Uint64() + prevTip := p.sealedTip + + if n > p.sealedTip { + p.sealedTip = n + } + + recovered := mode.kind == modeRecover && mode.height == n + if recovered || mode.kind == modeSealedWait { + // This block is a rebuild of a generation the store already holds, + // seal included. Republishing it would open a second generation over + // a sealed height — the displacement this whole design exists to + // prevent. Nothing to publish and nothing to gate: the block only + // needs to reach the chain. + p.curHeight = 0 + p.awaitOpen = false + p.hold = clearedHold() + + if recovered { + p.gate = sealGate{} + + return + } + + // Not a recovery: the store closed this height with content this + // block does not carry, and the winner's block has not arrived yet. + // Broadcasting divergent content here is the displacement the gate + // exists to stop, so an undecided height refuses. The next build + // recovers the height properly once the grace has elapsed. + p.gate = sealGate{height: n, hash: block.Hash(), refuseOnTimeout: true} + + return + } + + if !p.windowMirrorsLocked(block) { + // The journal's window does not describe the sealed block (a adopt + // that raced a straggler seal, a purge, a divergent rebuild): + // rebuild it from the body so the flush carries the sealed truth. + if !p.rebuildWindowLocked(block) { + return // fail() already recorded the cause + } + } + + // appendLocked already signaled the send loop; the hold clears after so the + // released entries and the seal go out together on that wake. + p.appendLocked(sealEntry(raw, p.head), commitment.FoldSeal(p.head, commitment.SealedHash(raw)), entrySeal, n, nil) + + // Arm the broadcast gate: ConfirmSeal resolves it from the seal's ack, + // the chain, or its deadline. + p.gate = sealGate{ + height: n, hash: block.Hash(), published: true, prevTip: prevTip, + txs: txHashes(block.Transactions()), + tolerateSealed: mode.kind == modeOverSealed && mode.height == n, + } + + p.curHeight = 0 + p.awaitOpen = false + p.hold = clearedHold() + p.collapseColdLocked() +} + +// collapseColdLocked bounds the undelivered flushes held in memory: past +// journalHotSeals, the oldest collapse to the pending height range and are +// rebuilt from the chain database when a re-anchor can deliver them. The +// send cursor detects the gap and routes through reconciliation. +func (p *Publisher) collapseColdLocked() { + for p.unackedSealsLocked() > journalHotSeals { + h, removed, ok := p.journal.collapseOldestUnacked(p.ackedSeq) + if !ok { + return + } + + // Merge into the pending range, never clobber it: during an active + // drain the collapse reaches heights BELOW pendingFrom (a stranded + // batch suffix is the journal's oldest content), and overwriting + // pendingTo downward inverted the range — which the backfill then + // read as "nothing owed" and zeroed. That silent evaporation + // stranded 25 heights of a 50-block outage as permanent holes. + if p.pendingFrom == 0 || h < p.pendingFrom { + p.pendingFrom = h + } + + if h > p.pendingTo { + p.pendingTo = h + } + + p.pendingEntries += removed + } +} + +// unackedSealsLocked counts sealed-but-undelivered flushes in the journal. +func (p *Publisher) unackedSealsLocked() int { + n := 0 + + for _, it := range p.journal.items { + if it.kind == entrySeal && it.seq > p.ackedSeq { + n++ + } + } + + return n +} + +// windowMirrorsLocked reports whether the journal's trailing window is an +// open at the block's height whose records are exactly the block's +// transactions, in order. +func (p *Publisher) windowMirrorsLocked(block *types.Block) bool { + if p.awaitOpen { + return false + } + + start := p.journal.openStart() + if start < 0 || p.journal.items[start].height != block.NumberU64() { + return false + } + + // The window's open context must equal the sealed header: consumers + // pinned it at open and void the window on any mismatch, so a context + // drift means rebuild, not complete-in-place. + if !openMatchesHeader(p.journal.items[start].entry.GetBlockOpen(), block.Header()) { + return false + } + + return recordsMirrorTxs(p.journal.items[start+1:], block.Transactions()) +} + +// rebuildWindowLocked replaces the lineage's trailing window with one built +// from the sealed block's body, folded onto the confirmed prefix. +func (p *Publisher) rebuildWindowLocked(block *types.Block) bool { + // Drop the trailing undelivered open window; the sealed block replaces + // it (an adopted window is acked and stays; the rebuild folds on top). + if cut := p.journal.rebuildCut(p.ackedSeq); cut < len(p.journal.items) { + p.rewindJournalLocked(cut) + } + + header := block.Header() + open := openEntry(commitment.OpenContext{ + Number: header.Number.Uint64(), + Timestamp: header.Time, + ParentHash: header.ParentHash, + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, p.head) + + next, err := foldEntry(p.head, open) + if err != nil { + p.fail("fold flush open", "number", header.Number, "err", err) + + return false + } + + p.appendLocked(open, next, entryOpen, header.Number.Uint64(), nil) + + for _, tx := range block.Transactions() { + rawTx, err := tx.MarshalBinary() + if err != nil { + p.fail("encode flush transaction", "hash", tx.Hash(), "err", err) + + return false + } + + p.appendLocked(recordEntry(rawTx, p.head), commitment.FoldTx(p.head, rawTx), entryRecord, header.Number.Uint64(), []common.Hash{tx.Hash()}) + } + + return true +} + +// rewindJournalLocked truncates the journal to its first n items and returns the +// fold head to the truncation point. +func (p *Publisher) rewindJournalLocked(n int) { + p.journal.truncate(n) + + if n > 0 { + last := p.journal.items[n-1] + p.head = last.post + + if p.ackedSeq > last.seq { + p.ackedSeq = last.seq + } + } else { + p.head = p.anchor + } +} + +// appendLocked folds one entry into the lineage and wakes the transport. +func (p *Publisher) appendLocked(entry *pb.Entry, next commitment.Head, kind int, height uint64, txHashes []common.Hash) { + p.journal.append(entry, p.head, next, kind, height, p.ackedSeq, txHashes) + p.head = next + publishQueueGauge.Update(int64(p.unackedLocked())) + p.signalWake() +} + +// baseFeeBytes encodes a base fee for the wire: a nil or zero fee is the +// empty slice, which is what the commitment fold hashes — the two encodings +// must agree or a published open stops matching its fold head. +func baseFeeBytes(f *big.Int) []byte { + if f == nil { + return []byte{} + } + + return f.Bytes() +} diff --git a/eth/sequencer/publisher.go b/eth/sequencer/publisher.go new file mode 100644 index 0000000000..77346ba7e8 --- /dev/null +++ b/eth/sequencer/publisher.go @@ -0,0 +1,339 @@ +// Package sequencer connects Bor to the sequence store: the block producer +// publishes each block's lifecycle (open, transactions, seal) as it happens. +// The wire contract and commitment chain live in +// github.com/0xPolygon/sequence-store-proto; the design and terminology +// reference is docs/sequencer-bor.md. +package sequencer + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials/insecure" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +const ( + // Probe backoff while the store is unreachable. The cap stays small so a + // replayable gap cannot outgrow the journal while waiting. + probeBackoffMin = time.Second + probeBackoffMax = 5 * time.Second + + // Backoff between reconciles whose corrective publish immediately + // re-STALEs — losing head races to another publisher. + reconcileBackoffMin = 500 * time.Millisecond + reconcileBackoffMax = 8 * time.Second +) + +// ackStallTimeout bounds how long the send loop waits on an unacked entry before +// treating the stream as dead (a hung store stalls acks without +// erroring). Kept under the journal's coverage so the reconnect can still +// resend instead of jumping. Var for tests. +var ackStallTimeout = 5 * time.Second + +// chainReader is the chain access reconciliation needs: classifying +// sealed store content against the local chain, locating the store tail +// on a cold start from the last imported block, and rebuilding collapsed +// blocks for backfill — the chain database is the archive; the journal +// holds only the work in flight. +type chainReader interface { + GetCanonicalHash(number uint64) common.Hash + GetHeaderByHash(hash common.Hash) *types.Header + CurrentBlock() *types.Header + GetBlockByNumber(number uint64) *types.Block +} + +// Publisher streams block-production entries to the sequence store. Enqueue +// methods are called from the worker's goroutines and never block: folds are +// computed inline (sub-microsecond), transport happens on a background +// goroutine. When the store is unreachable the publisher keeps folding into +// its retention journal and recovers by journal replay or forward jump on +// reconnect; a STALE ack triggers tail-read reconciliation. +// Only MALFORMED acks and fold divergence are terminal. +type Publisher struct { + // mu makes fold-and-append atomic across the worker's goroutines and + // serializes lineage swaps against the transport goroutine. + mu sync.Mutex + head commitment.Head // local fold tip (end of the lineage) + journal *journal + ackedSeq uint64 // seq of the last store-confirmed journal item + anchor commitment.Head // store head confirming everything through ackedSeq + anchored bool // anchor established by a completed reconcile + confirmed bool // anchor was ever store-confirmed + curHeight uint64 // height of the open window being built (0 = none) + awaitOpen bool // drop records until the next OpenBlock (post purge) + adopt *adoption // store window being adopted + sealedTip uint64 // highest height we sealed-and-flushed + // mode is the build treatment for the current height, decided once per + // build-start classification; see buildMode. + mode buildMode + + // resync is set when the store shows another producer building the + // height this node is building: our block must not seal beside their + // sequence. Reading it clears it, ending the worker's current cycle — + // the next cycle's build-start read adopts the standing window, and a + // signal that survives to that read is cleared there as stale. + resync bool + + // pendingFrom..pendingTo (inclusive; 0 = none) are sealed blocks + // collapsed out of the journal while the store could not take them. + // They live in the chain database and are rebuilt at the next + // re-anchor (backfill). storeSealedTip is the highest height the + // store is known to have sealed — delivered by us or read from a + // tail — and floors the backfill: the store never needs a block at + // or below it. + pendingFrom uint64 + pendingTo uint64 + pendingEntries int // exact entry count of the collapsed range + storeSealedTip uint64 + + hold hold // the send loop's send ceiling; see the hold type + + // gate is the broadcast gate for the last sealed block; see sealGate. + gate sealGate + refusals refusalStreak + seed commitment.Head // an empty store's head, computable without the store + // unreachable is set while the transport is failing. Every per-block + // wait on the store is pointless in that state, and paying them all + // pushes blocks past their slot — which is what arms bor's span-check + // path and turns a store outage into a chain slowdown. + unreachable atomic.Bool + + failed atomic.Bool + + wake chan struct{} + + poll time.Duration + chain chainReader + + pubConn *grpc.ClientConn + consConn *grpc.ClientConn + pub pb.PublisherServiceClient + read *reader + + // Redial bookkeeping: the endpoints to dial again, and the time of the + // last successful store interaction of any kind. gRPC channels are + // supposed to heal on their own; a store container restart has wedged + // them in a permanent connect-retry loop while fresh dials worked, so + // after redialAfter of silence the publisher starts its connections + // over instead of trusting the channel state machine. + pubEndpoint string + consEndpoint string + lastContact atomic.Int64 // unix nanos + + cancel context.CancelFunc + done chan struct{} +} + +// NewPublisher dials the store and starts the transport goroutine. The +// publisher service (publish stream) and consumer service (reconcile tail +// reads) have their own endpoints. On a cold start the store +// tail is relocated from the chain's last imported block, so no +// local position is persisted. +func NewPublisher(publisherEndpoint, consumerEndpoint string, chainID uint64, poll time.Duration, chain chainReader) (*Publisher, error) { + pubConn, consConn, err := dialStore(publisherEndpoint, consumerEndpoint) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(context.Background()) + + p := &Publisher{ + head: commitment.Seed(chainID), + anchor: commitment.Seed(chainID), + seed: commitment.Seed(chainID), + hold: clearedHold(), + journal: newJournal(), + wake: make(chan struct{}, 1), + poll: poll, + chain: chain, + pubConn: pubConn, + consConn: consConn, + pub: pb.NewPublisherServiceClient(pubConn), + cancel: cancel, + done: make(chan struct{}), + } + + p.pubEndpoint, p.consEndpoint = publisherEndpoint, consumerEndpoint + p.lastContact.Store(time.Now().UnixNano()) + + p.read = newReader(pb.NewConsumerServiceClient(consConn), p.seed, p.markReachable) + + publishStateGauge.Update(gaugeDegraded) // until the startup reconcile anchors + + go p.run(ctx) + + return p, nil +} + +// markReachable notes a successful store round trip. The reader invokes it +// on every read and retire on every ack, so a recovered store stops being +// treated as down the moment anything hears from it — and the contact +// stamp is what holds the redial back. +func (p *Publisher) markReachable() { + p.unreachable.Store(false) + p.lastContact.Store(time.Now().UnixNano()) +} + +// dialStore opens the publisher and consumer connections. gRPC's own +// reconnect backoff is capped at the probe cap: after a long outage the +// default (up to 120s) would keep the connection down well past the +// store's return, growing the gap a forward jump abandons. +func dialStore(publisherEndpoint, consumerEndpoint string) (*grpc.ClientConn, *grpc.ClientConn, error) { + connBackoff := backoff.DefaultConfig + connBackoff.MaxDelay = probeBackoffMax + + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithConnectParams(grpc.ConnectParams{Backoff: connBackoff}), + } + + pubConn, err := grpc.NewClient(publisherEndpoint, opts...) + if err != nil { + return nil, nil, err + } + + consConn, err := grpc.NewClient(consumerEndpoint, opts...) + if err != nil { + _ = pubConn.Close() + + return nil, nil, err + } + + return pubConn, consConn, nil +} + +// redialAfter is how much store silence the publisher tolerates before +// rebuilding its connections, and the floor between rebuilds. Var for tests. +var redialAfter = time.Minute + +// silentTooLong reports whether every store interaction has failed for a +// whole redial interval. +func (p *Publisher) silentTooLong() bool { + return time.Since(time.Unix(0, p.lastContact.Load())) > redialAfter +} + +// redial replaces both store connections and swaps the clients in place. +// The transport goroutine owns p.pub; the reader hands out its client +// under its own lock, so in-flight calls finish on the old connections, +// which close in the background. +func (p *Publisher) redial() { + pubConn, consConn, err := dialStore(p.pubEndpoint, p.consEndpoint) + if err != nil { + log.Warn("Sequencer redial failed", "err", err) + + return + } + + p.mu.Lock() + oldPub, oldCons := p.pubConn, p.consConn + p.pubConn, p.consConn = pubConn, consConn + p.pub = pb.NewPublisherServiceClient(pubConn) + p.mu.Unlock() + + p.read.setClient(pb.NewConsumerServiceClient(consConn)) + + _ = oldPub.Close() + _ = oldCons.Close() + + publishRedialCount.Inc(1) + log.Warn("Sequencer redialed the store after prolonged silence") +} + +// signalWake nudges the transport goroutine without blocking: the wake +// channel is a depth-1 signal, so a pending wake absorbs this one. +func (p *Publisher) signalWake() { + select { + case p.wake <- struct{}{}: + default: + } +} + +// advanceStoreSealedTipLocked records the newest seal a read decoded. Only a +// decoded seal qualifies — an inferred boundary can be a live partial window, +// and booking one as sealed is how an outage hole got pinned as permanent. +func (p *Publisher) advanceStoreSealedTipLocked(info tailInfo, origin string) { + if info.sealDecoded && info.lastSealHeight > p.storeSealedTip { + log.Debug("Sequencer sealed tip advance", "origin", origin, + "from", p.storeSealedTip, "to", info.lastSealHeight) + p.storeSealedTip = info.lastSealHeight + } +} + +// unackedLocked counts entries past the acked frontier by sequence +// arithmetic, so unacked entries already evicted from the journal still count +// (they are exactly what a forward jump abandons). +func (p *Publisher) unackedLocked() int { + return int(p.journal.nextSeq - 1 - p.ackedSeq) +} + +// heldMidBuild reports a live build whose entries are gated by a sticky +// hold: we are not writing, so only a deliberate re-read can show us what +// the competing producer is doing. +func (p *Publisher) heldMidBuild() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.curHeight != 0 && p.hold.kind == holdSticky +} + +// ourOpenParentLocked is the parent hash of the window this node is +// building, read from its open entry in the journal. +func (p *Publisher) ourOpenParentLocked() (common.Hash, bool) { + idx := p.journal.openStart() + if idx < 0 { + return common.Hash{}, false + } + + open := p.journal.items[idx].entry.GetBlockOpen() + if open == nil { + return common.Hash{}, false + } + + return common.BytesToHash(open.GetParentHash()), true +} + +// RefreshInterval is the txpool poll cadence the worker uses while a block is +// open; zero keeps the one-shot fill. +func (p *Publisher) RefreshInterval() time.Duration { + return p.poll +} + +// Close stops the transport goroutine. No local state is persisted: a +// restart relocates the store tail from the chain's last imported block. +func (p *Publisher) Close() { + p.cancel() + <-p.done +} + +func (p *Publisher) fail(msg string, args ...any) { + if p.failed.CompareAndSwap(false, true) { + publishFailedGauge.Update(1) + publishStateGauge.Update(gaugeFailed) + log.Error("Sequencer publishing disabled: "+msg, args...) + } +} + +func (p *Publisher) isAnchored() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.anchored +} + +func (p *Publisher) setUnanchored() { + p.mu.Lock() + defer p.mu.Unlock() + + p.anchored = false +} diff --git a/eth/sequencer/publisher_test.go b/eth/sequencer/publisher_test.go new file mode 100644 index 0000000000..d20689d190 --- /dev/null +++ b/eth/sequencer/publisher_test.go @@ -0,0 +1,607 @@ +package sequencer + +import ( + "context" + "math/big" + "net" + "testing" + "time" + + "google.golang.org/grpc" + + "github.com/0xPolygon/sequence-store-proto/commitment" + "github.com/0xPolygon/sequence-store-proto/devstore" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" +) + +const testChainID = 1337 + +// harness serves one devstore over TCP and can stop and resume serving on +// the same address, keeping the store's state — an outage, from the +// publisher's point of view. +type harness struct { + t *testing.T + store *devstore.Store + addr string + srv *grpc.Server +} + +func startHarness(t *testing.T) *harness { + return startHarnessChain(t, testChainID) +} + +func startHarnessChain(t *testing.T, chainID uint64) *harness { + t.Helper() + + h := &harness{t: t, store: devstore.New(chainID)} + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + h.addr = lis.Addr().String() + h.serve(lis) + t.Cleanup(func() { h.srv.Stop() }) + + return h +} + +func (h *harness) serve(lis net.Listener) { + h.srv = grpc.NewServer() + pb.RegisterPublisherServiceServer(h.srv, h.store) + pb.RegisterConsumerServiceServer(h.srv, h.store) + + go func() { _ = h.srv.Serve(lis) }() +} + +func (h *harness) stop() { + h.srv.Stop() +} + +func (h *harness) resume() { + h.t.Helper() + + deadline := time.Now().Add(5 * time.Second) + + for { + lis, err := net.Listen("tcp", h.addr) + if err == nil { + h.serve(lis) + + return + } + + if time.Now().After(deadline) { + h.t.Fatalf("relisten %s: %v", h.addr, err) + } + + time.Sleep(50 * time.Millisecond) + } +} + +func newTestPublisher(t *testing.T, h *harness, chain chainReader) *Publisher { + t.Helper() + + p, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + + return p +} + +func testTx(t *testing.T, nonce uint64) *types.Transaction { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("key: %v", err) + } + + tx, err := types.SignNewTx(key, types.LatestSignerForChainID(big.NewInt(testChainID)), &types.DynamicFeeTx{ + ChainID: big.NewInt(testChainID), + Nonce: nonce, + GasTipCap: big.NewInt(1), + GasFeeCap: big.NewInt(30_000_000_000), + Gas: 21000, + To: &common.Address{0x01}, + }) + if err != nil { + t.Fatalf("sign: %v", err) + } + + return tx +} + +func testHeader(number uint64, parent common.Hash) *types.Header { + return &types.Header{ + ParentHash: parent, + Number: new(big.Int).SetUint64(number), + GasLimit: 30_000_000, + Time: 1700000000 + number, + BaseFee: big.NewInt(25_000_000_000), + Difficulty: big.NewInt(1), + } +} + +// blockFor assembles a sealed block from a header and its transactions. +func blockFor(header *types.Header, txs []*types.Transaction) *types.Block { + return types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: txs}) +} + +// publishBlock drives a full lifecycle through the publisher and returns the +// sealed header. +func publishBlock(t *testing.T, p *Publisher, number uint64, parent common.Hash, txs int) *types.Header { + t.Helper() + + header := testHeader(number, parent) + p.OpenBlock(number, header.Time, parent, header.GasLimit, header.BaseFee) + + var body []*types.Transaction + + for i := 0; i < txs; i++ { + tx := testTx(t, uint64(i)) + body = append(body, tx) + p.PublishTx(tx) + } + + p.SealBlock(blockFor(header, body)) + + return header +} + +func waitHead(t *testing.T, h *harness, p *Publisher, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + + for { + p.mu.Lock() + local := p.head + p.mu.Unlock() + + if h.store.Head() == local { + return + } + + if time.Now().After(deadline) { + p.mu.Lock() + defer p.mu.Unlock() + t.Fatalf("store head %x never reached local head %x", h.store.Head(), p.head) + } + + time.Sleep(20 * time.Millisecond) + } +} + +// The publisher's folds must match the store's across a full lifecycle. +func TestPublisherLifecycle(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + header := publishBlock(t, p, 1, common.Hash{0xef}, 3) + waitHead(t, h, p, 5*time.Second) + + raw, err := rlp.EncodeToBytes(header) + if err != nil { + t.Fatalf("rlp: %v", err) + } + + // Independently computed chain: seed → open → 3 txs → seal. + want := commitment.Seed(testChainID) + want, err = commitment.FoldOpen(want, commitment.OpenContext{ + Number: 1, + Timestamp: header.Time, + ParentHash: common.Hash{0xef}, + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }) + if err != nil { + t.Fatalf("fold open: %v", err) + } + + p.mu.Lock() + items, _ := p.journal.after(0) + p.mu.Unlock() + + for _, item := range items { + if item.kind == entryRecord { + want = commitment.FoldTxs(want, item.entry.GetRecord().GetTransactions()) + } + } + + want = commitment.FoldSeal(want, commitment.SealedHash(raw)) + + if h.store.Head() != want { + t.Fatalf("store head %x, want %x", h.store.Head(), want) + } +} + +// A restart against a non-empty store relocates the tail from the chain's +// last block and the next open extends it in place — no fresh topic +// required and no forward jump counted (nothing abandoned). +func TestRestartWarmResume(t *testing.T) { + h := startHarness(t) + + jumps := reconcileForwardJump.Snapshot().Count() + + first, err := NewPublisher(h.addr, h.addr, testChainID, 0, nil) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + sealed := publishBlock(t, first, 1, common.Hash{0xef}, 2) + waitHead(t, h, first, 5*time.Second) + first.Close() + + // The restart locates the store tail from the chain's last block. + chain := &fakeChain{current: &types.Header{Number: big.NewInt(1)}} + + second, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(second.Close) + + publishBlock(t, second, 2, sealHash(t, sealed), 1) + waitHead(t, h, second, 5*time.Second) + + if got := reconcileForwardJump.Snapshot().Count(); got != jumps { + t.Fatalf("clean warm resume counted %d forward jumps", got-jumps) + } +} + +// A cold publisher against a non-empty store anchors via the ladder's probe +// and floor rungs, located from the chain's last block. +func TestStartupColdNonEmptyStore(t *testing.T) { + h := startHarness(t) + + seed := newTestPublisher(t, h, nil) + sealed := publishBlock(t, seed, 1, common.Hash{0xef}, 1) + waitHead(t, h, seed, 5*time.Second) + seed.Close() + + chain := &fakeChain{current: &types.Header{Number: big.NewInt(1)}} + + p, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + + publishBlock(t, p, 2, sealHash(t, sealed), 1) + waitHead(t, h, p, 5*time.Second) +} + +// A store outage while blocks keep being produced recovers by journal replay: +// full continuity, every entry published. +func TestOutageJournalReplay(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + // Produced entirely during the outage; folded into the journal only. + sealed2 := publishBlock(t, p, 2, sealHash(t, sealed), 2) + publishBlock(t, p, 3, sealHash(t, sealed2), 1) + + h.resume() + waitHead(t, h, p, 15*time.Second) + + if got := h.store.Head(); got != localHead(p) { + t.Fatalf("replay did not converge: store %x local %x", got, localHead(p)) + } +} + +// A publisher whose chain names a height the store does not have (wiped or +// far behind) falls through the ladder to the floor and re-anchors on the +// seed. +func TestRestartAgainstUnknownStore(t *testing.T) { + h := startHarness(t) + + // Chain claims height 9, but the store is empty: probe finds nothing, + // the floor read seeds, and publishing resumes. + chain := &fakeChain{current: &types.Header{Number: big.NewInt(9)}} + + p, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + + waitFor(t, 5*time.Second, func() bool { return p.isAnchored() }) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) +} + +// A dial-time error must surface, not return a nil publisher. +func TestNewPublisherRejectsBadEndpoint(t *testing.T) { + if p, err := NewPublisher("bad\x7ftarget:99:99", "bad\x7ftarget:99:99", testChainID, 0, nil); err == nil { + p.Close() + t.Fatal("bad endpoint accepted") + } +} + +// A fold failure (nil base fee) is a publisher bug: terminal, and every +// later enqueue is a no-op. +func TestOpenBlockFoldFailureIsTerminal(t *testing.T) { + p := barePublisher() + + p.OpenBlock(1, 1700000001, common.Hash{0x01}, 30_000_000, nil) + + if !p.failed.Load() { + t.Fatal("fold failure not terminal") + } + + p.PublishTx(testTx(t, 0)) + + if items, _ := p.journal.after(0); len(items) != 0 { + t.Fatalf("enqueue after failure: %d items", len(items)) + } +} + +// Records and seals are dropped while awaiting the next open after a purge. +func TestAwaitOpenSuppressesRecords(t *testing.T) { + p := barePublisher() + p.awaitOpen = true + + p.PublishTx(testTx(t, 0)) + + if items, _ := p.journal.after(0); len(items) != 0 { + t.Fatalf("suppressed records reached the journal: %d", len(items)) + } + + // A seal is never suppressed: the flush rebuilds the window from + // the block body. + p.SealBlock(blockFor(testHeader(1, common.Hash{0x01}), nil)) + + if items, _ := p.journal.after(0); len(items) != 2 { + t.Fatalf("seal flush must rebuild open+seal: %d items", len(items)) + } + + header := testHeader(2, common.Hash{0x02}) + p.OpenBlock(2, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.PublishTx(testTx(t, 1)) + + if items, _ := p.journal.after(0); len(items) != 4 { + t.Fatalf("open must clear the suppression: %d items", len(items)) + } +} + +func sealHash(t *testing.T, header *types.Header) common.Hash { + t.Helper() + + return header.Hash() +} + +func localHead(p *Publisher) commitment.Head { + p.mu.Lock() + defer p.mu.Unlock() + + return p.head +} + +// Progress resets the contention streak; a repeat no-progress STALE backs +// off before the next reconcile, starting at the minimum. +func TestContentionSleep(t *testing.T) { + ctx := context.Background() + + if got := contentionSleep(ctx, true, 3); got != 0 { + t.Fatalf("progress must reset the streak, got %d", got) + } + + start := time.Now() + + if got := contentionSleep(ctx, false, 0); got != 1 { + t.Fatalf("first stale streak = %d, want 1", got) + } + + if elapsed := time.Since(start); elapsed > 200*time.Millisecond { + t.Fatalf("first stale must not sleep, took %v", elapsed) + } + + start = time.Now() + + if got := contentionSleep(ctx, false, 1); got != 2 { + t.Fatalf("second stale streak = %d, want 2", got) + } + + if elapsed := time.Since(start); elapsed < reconcileBackoffMin || elapsed > 4*reconcileBackoffMin { + t.Fatalf("second stale slept %v, want ~%v", elapsed, reconcileBackoffMin) + } +} + +// A publisher pointed at a store seeded for a different chain anchors on +// the foreign seed and publishes blindly — nothing alarms on the producer +// side (detection is the consumer's and the auditor's job). +func TestWrongChainStorePublishesBlindly(t *testing.T) { + h := startHarnessChain(t, testChainID+1) + p := newTestPublisher(t, h, nil) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + if p.failed.Load() { + t.Fatal("wrong-chain store must not fail the publisher (blind by design)") + } +} + +func journalEntry(kind int) *pb.Entry { + switch kind { + case entryOpen: + return &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{}}} + case entrySeal: + return &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{}}} + default: + return &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{Transactions: [][]byte{{0x01}}}}} + } +} + +// appendBlock adds an open+record+seal window at height h, delivered +// (acked), so the count bound may evict it. +func appendJournalBlock(r *journal, h uint64) { + r.append(journalEntry(entryOpen), commitment.Head{}, commitment.Head{}, entryOpen, h, r.nextSeq, nil) + r.append(journalEntry(entryRecord), commitment.Head{}, commitment.Head{}, entryRecord, h, r.nextSeq, nil) + r.append(journalEntry(entrySeal), commitment.Head{}, commitment.Head{}, entrySeal, h, r.nextSeq, nil) +} + +func TestJournalEvictsOldestSealedOnly(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= journalSealedBlocks+2; h++ { + appendJournalBlock(r, h) + } + + // Two oldest sealed blocks evicted, open-window invariant untouched. + if r.seals != journalSealedBlocks { + t.Fatalf("seals retained %d, want %d", r.seals, journalSealedBlocks) + } + + if first := r.items[0]; first.height != 3 || first.kind != entryOpen { + t.Fatalf("front is height %d kind %d, want open of height 3", first.height, first.kind) + } +} + +func TestJournalNeverEvictsOpenWindow(t *testing.T) { + r := newJournal() + + // One open window far over the byte cap: nothing sealed to drop. + r.append(journalEntry(entryOpen), commitment.Head{}, commitment.Head{}, entryOpen, 1, r.nextSeq, nil) + + huge := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{make([]byte, journalMaxBytes+1)}, + }}} + r.append(huge, commitment.Head{}, commitment.Head{}, entryRecord, 1, r.nextSeq, nil) + + if len(r.items) != 2 { + t.Fatalf("open window evicted: %d items", len(r.items)) + } +} + +func TestJournalAfterGapDetection(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= journalSealedBlocks+3; h++ { + appendJournalBlock(r, h) + } + + // seq 1 was evicted: the position is no longer covered. + if _, covered := r.after(0); covered { + t.Fatal("gap not detected after eviction") + } + + items, covered := r.after(r.items[0].seq) + if !covered || len(items) != len(r.items)-1 { + t.Fatalf("covered=%v len=%d", covered, len(items)) + } +} + +func TestJournalAfterEmptyBoundary(t *testing.T) { + r := newJournal() + + // Empty journal: only the position right before nextSeq is covered. + if _, covered := r.after(0); !covered { + t.Fatal("fresh empty journal must cover seq 0") + } + + appendJournalBlock(r, 1) + r.items, r.seals = nil, 0 // simulate a full drain + + if _, covered := r.after(r.nextSeq - 1); !covered { + t.Fatal("drained journal must cover its frontier") + } + + if _, covered := r.after(r.nextSeq - 2); covered { + t.Fatal("drained journal must not cover older positions") + } +} + +func TestJournalEvictOnlyBeyondSealedBound(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= journalSealedBlocks; h++ { + appendJournalBlock(r, h) + } + + if r.seals != journalSealedBlocks || r.items[0].height != 1 { + t.Fatalf("eviction fired at the bound: seals=%d front=%d", r.seals, r.items[0].height) + } +} + +// Undelivered seals are never evicted: eviction only trims delivered +// history. Older undelivered flushes leave via collapseOldestUnacked, +// which hands their heights to the chain-database backfill. +func TestJournalKeepsUndeliveredSeals(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= journalSealedBlocks+40; h++ { + r.append(journalEntry(entryOpen), commitment.Head{}, commitment.Head{}, entryOpen, h, 0, nil) + r.append(journalEntry(entrySeal), commitment.Head{}, commitment.Head{}, entrySeal, h, 0, nil) + } + + if r.seals != journalSealedBlocks+40 || r.items[0].height != 1 { + t.Fatalf("undelivered seal evicted: seals=%d front=%d", r.seals, r.items[0].height) + } +} + +func TestJournalCollapseOldestUnacked(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= 4; h++ { + r.append(journalEntry(entryOpen), commitment.Head{}, commitment.Head{}, entryOpen, h, 0, nil) + r.append(journalEntry(entrySeal), commitment.Head{}, commitment.Head{}, entrySeal, h, 0, nil) + } + + // Block 1 delivered (acked through its seal at seq 2); 2-4 undelivered. + // Each undelivered block is open+seal = 2 entries. + if h, n, ok := r.collapseOldestUnacked(2); !ok || h != 2 || n != 2 { + t.Fatalf("collapse = %d/%d/%v, want 2/2", h, n, ok) + } + + // The delivered block dropped with it; block 3 is now the front. + if r.items[0].height != 3 || r.seals != 2 { + t.Fatalf("front=%d seals=%d after collapse", r.items[0].height, r.seals) + } + + if _, _, ok := r.collapseOldestUnacked(2); !ok { + t.Fatal("second collapse must pop block 3") + } + + if h, n, ok := r.collapseOldestUnacked(2); !ok || h != 4 || n != 2 { + t.Fatalf("third collapse = %d/%d/%v, want 4/2", h, n, ok) + } + + if _, _, ok := r.collapseOldestUnacked(2); ok { + t.Fatal("no seals left to collapse") + } +} + +func TestJournalSuffixFromHeight(t *testing.T) { + r := newJournal() + appendJournalBlock(r, 5) + appendJournalBlock(r, 6) + + suffix := r.suffixFromHeight(6) + if len(suffix) != 3 || suffix[0].kind != entryOpen || suffix[0].height != 6 { + t.Fatalf("suffix wrong: len=%d", len(suffix)) + } + + if r.suffixFromHeight(7) != nil { + t.Fatal("suffix beyond retained heights should be nil") + } +} diff --git a/eth/sequencer/reader.go b/eth/sequencer/reader.go new file mode 100644 index 0000000000..d2c7af8269 --- /dev/null +++ b/eth/sequencer/reader.go @@ -0,0 +1,513 @@ +package sequencer + +import ( + "context" + "errors" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +const ( + // One Range page and the walk bound; anchored positions keep tails small. + walkLimit = 512 + maxWalkLoops = 128 + + // Per-read deadline (the worker never waits on this — the + // whole reconcile runs on the transport goroutine). + tailReadTimeout = time.Second +) + +var ( + errFoldDivergence = errors.New("byte-identical entry folds to a different head") + errTailTooLong = errors.New("tail walk exceeded bound") +) + +// reader is the store-facing read layer: bounded walks that derive the head +// they land on, generation probes over the block index, and per-height +// generation fetches. It holds no publisher state — the same machinery that +// serves the publisher's reconciliation serves any consumer of the store, +// which is what a subscribing RPC node will be. +type reader struct { + // mu guards cons, which a redial hot-swaps while the worker and + // transport goroutines read through it. + mu sync.RWMutex + cons pb.ConsumerServiceClient + + seed commitment.Head // the empty log's head, computed from the chain id + + // onRead runs after every successful store round trip. The publisher + // hangs its reachability bookkeeping on it: the build-start read runs + // every block, so it is what notices the store is back. + onRead func() +} + +func newReader(cons pb.ConsumerServiceClient, seed commitment.Head, onRead func()) *reader { + return &reader{cons: cons, seed: seed, onRead: onRead} +} + +func (r *reader) client() pb.ConsumerServiceClient { + r.mu.RLock() + defer r.mu.RUnlock() + + return r.cons +} + +func (r *reader) setClient(cons pb.ConsumerServiceClient) { + r.mu.Lock() + defer r.mu.Unlock() + + r.cons = cons +} + +// tailInfo summarizes one tail read: the store head S and what the tip +// looks like (classification inputs). +type tailInfo struct { + s commitment.Head + tipOpen bool + tipOpenHeight uint64 + tipOpenParent common.Hash + haveSeal bool + lastSealHeight uint64 + lastSealHash common.Hash + // sealDecoded separates a seal this read actually decoded from one a + // probe merely inferred. Only the former may drive sealed-tip + // bookkeeping: an inferred "boundary" can be a live partial window, + // and recording it as sealed told every backfill the height was owed + // nothing — pinning an outage's partial delivery as a permanent hole. + sealDecoded bool + + // window collects the trailing open window's entries for adoption, + // capped at the journal byte bound; an over-cap window is not + // collected and falls back to supersede. + window []*pb.Entry + windowBytes int + + // explained is set when s was folded from the walked entries rather + // than taken on the store's word. A head we cannot derive tells us + // nothing about what produced it — in particular, whether some other + // producer already has a live window at the height we are about to + // open. + explained bool +} + +// tryWalk runs one ladder rung; done=false falls through to the next rung: +// the position is unknown to the store (NOT_FOUND) or so far behind the tip +// that walking from it exceeds the bound — the probe and floor rungs anchor +// near the tip with a short walk instead. A recTerminal outcome is a fold +// divergence, which only a matched walk (an absorb hook that compares +// prefixes) can produce — the caller owns what terminal means. +func (r *reader) tryWalk(ctx context.Context, first *pb.RangeRequest, absorb func(*pb.Entry) error) (tailInfo, reconcileOutcome, bool) { + info, err := r.walk(ctx, first, absorb) + + switch { + case err == nil: + return info, recOK, true + case isNotFound(err): + return tailInfo{}, recRetry, false + case errors.Is(err, errTailTooLong): + log.Warn("Sequencer tail read position too far behind, probing near tip") + + return tailInfo{}, recRetry, false + case errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded: + // This rung's budget slice expired mid-walk — the position is too + // far behind to walk in time (gRPC surfaces the expiry as a status + // error that does not wrap the context sentinel). Fall through: + // the probe and floor rungs anchor near the tip with a short walk. + log.Warn("Sequencer tail read rung out of budget, probing near tip") + + return tailInfo{}, recRetry, false + case errors.Is(err, errFoldDivergence): + return tailInfo{}, recTerminal, true + default: + log.Warn("Sequencer tail read", "err", err) + + return tailInfo{}, recRetry, true + } +} + +func (r *reader) probedWalk(ctx context.Context, from uint64) (tailInfo, reconcileOutcome, bool) { + h, found, err := r.probeDown(ctx, from) + if err != nil { + return tailInfo{}, recRetry, true + } + + if !found { + return tailInfo{}, recRetry, false + } + + return r.tryWalk(ctx, blockReq(h), nil) +} + +// floorRead anchors on the earliest retained entry: an empty store yields +// the seed; a known floor height anchors an upward probe to the tip edge. +func (r *reader) floorRead(ctx context.Context) (tailInfo, reconcileOutcome) { + resp, err := r.rangeOnce(ctx, &pb.RangeRequest{Limit: 1}) + if err != nil { + return tailInfo{}, recRetry + } + + if len(resp.GetEntries()) == 0 && resp.GetLive() { + s, ok := headFrom(resp.GetNext()) + if !ok { + return tailInfo{}, recRetry + } + + // An empty store is the one head we can derive with no entries at + // all: it must be the seed. Anything else is the store telling us + // about history it did not show us. + return tailInfo{s: s, explained: s == r.seed}, recOK + } + + // An empty page that is not live is coherent (a trailing gateway not + // yet at its tip): nothing to anchor a probe on — fall through to the + // full floor walk, which loops pages to live. + if len(resp.GetEntries()) > 0 { + if h, ok := entryHeight(resp.GetEntries()[0]); ok { + if edge, err := r.probeUp(ctx, h); err == nil { + if info, out, done := r.tryWalk(ctx, blockReq(edge), nil); done { + return info, out + } + } + } + } + + // Records carry no height (or the probe raced retention): full floor walk. + info, err := r.walk(ctx, &pb.RangeRequest{}, nil) + if err != nil { + return tailInfo{}, recRetry + } + + return info, recOK +} + +// walk reads Range pages from first until live, tracking the tip. An absorb +// hook (the anchor rung's journal matcher) inspects every entry and may end +// the walk with its error — fold-integrity checks live behind it. +func (r *reader) walk(ctx context.Context, first *pb.RangeRequest, absorb func(*pb.Entry) error) (tailInfo, error) { + var info tailInfo + + f := r.newFolder(first) + + req := first + req.Limit = walkLimit + + for loops := 0; ; loops++ { + if loops >= maxWalkLoops { + return info, errTailTooLong + } + + resp, err := r.rangeOnce(ctx, req) + if err != nil { + return info, err + } + + for _, entry := range resp.GetEntries() { + if absorb != nil { + if err := absorb(entry); err != nil { + return info, err + } + } + + f.fold(entry) + trackTip(&info, entry) + } + + if s, ok := headFrom(resp.GetNext()); ok { + info.s = s + f.reached(s) + } + + if resp.GetLive() { + info.explained = f.ok + + return info, nil + } + + req = &pb.RangeRequest{ + After: &pb.RangeRequest_Head{Head: resp.GetNext()}, + Limit: walkLimit, + } + } +} + +// folder derives the head the walk lands on instead of accepting the one the +// store reports. Without it a CAS proves only that we echoed back the value +// we were handed — the commitment stops being a check on shared history and +// becomes a sequence token. +// +// A walk from our own anchor has a verified base. A walk that starts at a +// block takes the base its open declares, which is weaker but still chains +// every entry after it — and, crucially, means we have read the window. +// A walk that starts anywhere else has no base and cannot explain anything. +type folder struct { + cur commitment.Head + ok bool + awaiting bool // no base yet; the first entry must be an open that names one + sawAny bool +} + +func (r *reader) newFolder(first *pb.RangeRequest) *folder { + switch a := first.GetAfter().(type) { + case *pb.RangeRequest_Head: + if base, ok := headFrom(a.Head); ok { + return &folder{cur: base, ok: true} + } + case nil: + // A walk from the log's start: the base is the seed, which we + // compute from the chain id without asking the store. + return &folder{cur: r.seed, ok: true} + } + + return &folder{ok: true, awaiting: true} +} + +func (f *folder) fold(e *pb.Entry) { + f.sawAny = true + + if !f.ok { + return + } + + if f.awaiting { + open := e.GetBlockOpen() + if open == nil { + f.ok = false // mid-window start: nothing to fold from + + return + } + + f.cur = commitment.Head(open.GetPrefixCommitment()) + f.awaiting = false + } + + next, err := foldEntry(f.cur, e) + if err != nil { + f.ok = false + + return + } + + f.cur = next +} + +// reached compares the store's reported head for this page against the one +// the page's own entries produce. +func (f *folder) reached(s commitment.Head) { + switch { + case !f.ok: + return + case f.awaiting && !f.sawAny: + // A block-anchored walk that returned nothing: the head value stays + // underived, but the fact a boundary read needs — that no + // generation follows the block we started at — is what the empty + // page attests. Opening here cannot land on anyone's live window. + case f.awaiting: + // Entries came back, but not from a base we could establish, so + // they summarize into a head we cannot account for. + f.ok = false + case f.cur != s: + f.ok = false + + readHeadMismatch.Inc(1) + log.Warn("Store head does not match the entries it returned", + "reported", s, "folded", f.cur) + } +} + +func (r *reader) rangeOnce(ctx context.Context, req *pb.RangeRequest) (*pb.RangeResponse, error) { + cctx, cancel := context.WithTimeout(ctx, tailReadTimeout) + defer cancel() + + resp, err := r.client().Range(cctx, req) + if err == nil && r.onRead != nil { + r.onRead() + } + + return resp, err +} + +// generation fetches the entries of the newest generation standing at a +// height — open, records, and seal when one closed it. +func (r *reader) generation(ctx context.Context, height uint64) ([]*pb.Entry, error) { + resp, err := r.client().GetBlock(ctx, &pb.GetBlockRequest{BlockNumber: height}) + if err != nil { + return nil, err + } + + return resp.GetEntries(), nil +} + +func trackTip(info *tailInfo, entry *pb.Entry) { + switch k := entry.GetKind().(type) { + case *pb.Entry_BlockOpen: + info.tipOpen = true + info.tipOpenHeight = k.BlockOpen.GetBlockNumber() + info.tipOpenParent = common.BytesToHash(k.BlockOpen.GetParentHash()) + info.window = info.window[:0] + info.windowBytes = 0 + collectWindow(info, entry) + case *pb.Entry_Record: + collectWindow(info, entry) + case *pb.Entry_BlockSeal: + header, err := decodeSealHeader(k.BlockSeal.GetHeader()) + if err != nil { + log.Warn("Sequencer tail seal undecodable", "err", err) + + return + } + + info.tipOpen = false + info.haveSeal = true + info.sealDecoded = true + info.lastSealHeight = header.Number.Uint64() + info.lastSealHash = header.Hash() + info.window = nil + info.windowBytes = 0 + } +} + +// collectWindow accumulates the trailing window's entries up to the journal +// byte bound; past it the window is dropped for good (unadoptable). +func collectWindow(info *tailInfo, entry *pb.Entry) { + if info.windowBytes > journalMaxBytes { + return + } + + if info.windowBytes += proto.Size(entry); info.windowBytes > journalMaxBytes { + info.window = nil + + return + } + + if info.window != nil || entry.GetBlockOpen() != nil { + info.window = append(info.window, entry) + } +} + +// probeDown finds the highest store-known height at or below from: +// exponential descent, then binary search. +func (r *reader) probeDown(ctx context.Context, from uint64) (uint64, bool, error) { + h, step := from, uint64(1) + + // The lowest height the descent has proven unknown: the edge search + // need not re-probe it. In the common build-start case (from unknown, + // from-1 known) this leaves the binary search nothing to do at all — + // re-probing cost a wasted store round trip on every block. + unknown := from + 1 + + for { + known, err := r.blockKnown(ctx, h) + if err != nil { + return 0, false, err + } + + if known { + break + } + + unknown = h + + // step starts at 1 and doubles, so this also covers h <= 1. + if step >= h { + return 0, false, nil + } + + h -= step + step *= 2 + } + + return r.binarySearchEdge(ctx, h, unknown) +} + +// probeUp finds the highest store-known height starting from known floor +// height h0. +func (r *reader) probeUp(ctx context.Context, h0 uint64) (uint64, error) { + lo, step := h0, uint64(1) + + for { + known, err := r.blockKnown(ctx, lo+step) + if err != nil { + return 0, err + } + + if !known { + break + } + + lo += step + step *= 2 + } + + edge, _, err := r.binarySearchEdge(ctx, lo, lo+step) + + return edge, err +} + +// binarySearchEdge returns the highest known height in [lo, hi) given lo is +// known and hi is unknown (or the exclusive bound). +func (r *reader) binarySearchEdge(ctx context.Context, lo, hi uint64) (uint64, bool, error) { + for hi-lo > 1 { + mid := lo + (hi-lo)/2 + + known, err := r.blockKnown(ctx, mid) + if err != nil { + return 0, false, err + } + + if known { + lo = mid + } else { + hi = mid + } + } + + return lo, true, nil +} + +func (r *reader) blockKnown(ctx context.Context, h uint64) (bool, error) { + _, err := r.rangeOnce(ctx, &pb.RangeRequest{ + After: &pb.RangeRequest_Block{Block: h}, + Limit: 1, + }) + + switch { + case err == nil: + return true, nil + case isNotFound(err): + return false, nil + default: + return false, err + } +} + +func isNotFound(err error) bool { + return status.Code(err) == codes.NotFound +} + +// headFrom converts wire bytes to a Head when the length matches; short +// or oversized bytes are not a position. +func headFrom(b []byte) (commitment.Head, bool) { + if len(b) != len(commitment.Head{}) { + return commitment.Head{}, false + } + + return commitment.Head(b), true +} + +func headReq(h commitment.Head) *pb.RangeRequest { + return &pb.RangeRequest{After: &pb.RangeRequest_Head{Head: h.Bytes()}} +} + +func blockReq(h uint64) *pb.RangeRequest { + return &pb.RangeRequest{After: &pb.RangeRequest_Block{Block: h}} +} diff --git a/eth/sequencer/receipts.go b/eth/sequencer/receipts.go new file mode 100644 index 0000000000..6fad1a3f33 --- /dev/null +++ b/eth/sequencer/receipts.go @@ -0,0 +1,166 @@ +package sequencer + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// maxIndexedHeights bounds the index when canonical imports stall (or a +// misbehaving store streams far ahead): the lowest height is dropped to +// admit a new one. +const maxIndexedHeights = 256 + +// Index holds preconfirmation receipts built from re-executing the sequence +// stream, keyed by transaction hash. Receipts carry a zero BlockHash until +// the block's seal record arrives; the canonical receipt path always takes +// precedence, so entries here are only consulted for transactions the chain +// does not yet have. +type Index struct { + mu sync.RWMutex + byHash map[common.Hash]*indexed + byBlock map[uint64][]common.Hash +} + +type indexed struct { + receipt *types.Receipt + tx *types.Transaction +} + +// NewIndex returns an empty preconf receipt index. +func NewIndex() *Index { + return &Index{ + byHash: map[common.Hash]*indexed{}, + byBlock: map[uint64][]common.Hash{}, + } +} + +// Add records one preconf receipt for a speculative, not-yet-sealed block. +// The receipt must not be mutated after this call — Lookup hands the stored +// pointer to concurrent RPC readers. +func (ix *Index) Add(tx *types.Transaction, receipt *types.Receipt) { + ix.mu.Lock() + defer ix.mu.Unlock() + + number := receipt.BlockNumber.Uint64() + + if _, held := ix.byBlock[number]; !held && len(ix.byBlock) >= maxIndexedHeights { + ix.dropLowestLocked() + } + + ix.byHash[tx.Hash()] = &indexed{receipt: receipt, tx: tx} + ix.byBlock[number] = append(ix.byBlock[number], tx.Hash()) +} + +// Seal fills the sealed block hash into the height's receipts and their +// logs. Stored receipts are immutable once inserted (RPC readers hold the +// same pointers), so sealing swaps in copies. +func (ix *Index) Seal(number uint64, hash common.Hash) { + ix.mu.Lock() + defer ix.mu.Unlock() + + for _, txHash := range ix.byBlock[number] { + entry, ok := ix.byHash[txHash] + if !ok { + continue + } + + receipt := *entry.receipt + receipt.BlockHash = hash + receipt.Logs = make([]*types.Log, len(entry.receipt.Logs)) + + for i, old := range entry.receipt.Logs { + l := *old + l.BlockHash = hash + receipt.Logs[i] = &l + } + + ix.byHash[txHash] = &indexed{receipt: &receipt, tx: entry.tx} + } +} + +func (ix *Index) dropLowestLocked() { + lowest := uint64(0) + first := true + + for height := range ix.byBlock { + if first || height < lowest { + lowest = height + first = false + } + } + + if first { + return + } + + for _, h := range ix.byBlock[lowest] { + delete(ix.byHash, h) + } + + delete(ix.byBlock, lowest) +} + +// Lookup returns the preconf receipt and transaction for a hash, if held. +func (ix *Index) Lookup(hash common.Hash) (*types.Receipt, *types.Transaction, bool) { + ix.mu.RLock() + defer ix.mu.RUnlock() + + entry, ok := ix.byHash[hash] + if !ok { + return nil, nil, false + } + + preconfServedMeter.Mark(1) + + return entry.receipt, entry.tx, true +} + +// ClearFrom drops all entries at or above a height — a re-anchor voided them. +func (ix *Index) ClearFrom(number uint64) { + ix.mu.Lock() + defer ix.mu.Unlock() + + for height, hashes := range ix.byBlock { + if height < number { + continue + } + + for _, h := range hashes { + delete(ix.byHash, h) + } + + delete(ix.byBlock, height) + } +} + +// EvictThrough drops all entries at or below a height — the canonical chain +// now serves those receipts (or the transactions never landed and must not +// linger). +func (ix *Index) EvictThrough(number uint64) { + ix.mu.Lock() + defer ix.mu.Unlock() + + for height, hashes := range ix.byBlock { + if height > number { + continue + } + + for _, h := range hashes { + delete(ix.byHash, h) + } + + delete(ix.byBlock, height) + } +} + +// Reset drops everything — the consumer lost stream consistency and is +// re-anchoring from canonical state. +func (ix *Index) Reset() { + ix.mu.Lock() + defer ix.mu.Unlock() + + ix.byHash = map[common.Hash]*indexed{} + ix.byBlock = map[uint64][]common.Hash{} +} diff --git a/eth/sequencer/receipts_test.go b/eth/sequencer/receipts_test.go new file mode 100644 index 0000000000..d8165895a2 --- /dev/null +++ b/eth/sequencer/receipts_test.go @@ -0,0 +1,102 @@ +package sequencer + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +func indexedTx(nonce uint64) *types.Transaction { + return types.NewTx(&types.LegacyTx{Nonce: nonce}) +} + +func indexedReceipt(height uint64) *types.Receipt { + return &types.Receipt{ + BlockNumber: new(big.Int).SetUint64(height), + Logs: []*types.Log{{}}, + } +} + +// Sealing must hand new receipt copies to future readers while leaving the +// pointers earlier RPC readers hold untouched. +func TestIndexSealSwapsCopiesForReaders(t *testing.T) { + ix := NewIndex() + tx := indexedTx(1) + + ix.Add(tx, indexedReceipt(5)) + + before, _, ok := ix.Lookup(tx.Hash()) + if !ok { + t.Fatal("added receipt not found") + } + + sealedHash := common.Hash{0x5e} + ix.Seal(5, sealedHash) + + if before.BlockHash != (common.Hash{}) || before.Logs[0].BlockHash != (common.Hash{}) { + t.Fatal("sealing mutated a receipt an RPC reader already holds") + } + + after, _, _ := ix.Lookup(tx.Hash()) + if after.BlockHash != sealedHash || after.Logs[0].BlockHash != sealedHash { + t.Fatalf("sealed lookup carries %s, want %s on receipt and logs", after.BlockHash, sealedHash) + } +} + +// ClearFrom voids re-anchored heights upward; EvictThrough drops imported +// heights downward. Between them a height survives only while speculative. +func TestIndexClearAndEvictBounds(t *testing.T) { + ix := NewIndex() + txs := map[uint64]*types.Transaction{} + + for h := uint64(1); h <= 3; h++ { + txs[h] = indexedTx(h) + ix.Add(txs[h], indexedReceipt(h)) + } + + ix.ClearFrom(3) + ix.EvictThrough(1) + + for h, want := range map[uint64]bool{1: false, 2: true, 3: false} { + if _, _, ok := ix.Lookup(txs[h].Hash()); ok != want { + t.Fatalf("height %d held=%v, want %v", h, ok, want) + } + } +} + +// The index stays bounded when canonical imports stall: admitting a new +// height at the cap drops the lowest one. +func TestIndexCapDropsTheLowestHeight(t *testing.T) { + ix := NewIndex() + first := indexedTx(0) + ix.Add(first, indexedReceipt(1)) + + for h := uint64(2); h <= maxIndexedHeights; h++ { + ix.Add(indexedTx(h), indexedReceipt(h)) + } + + over := indexedTx(maxIndexedHeights + 1) + ix.Add(over, indexedReceipt(maxIndexedHeights+1)) + + if _, _, ok := ix.Lookup(first.Hash()); ok { + t.Fatal("lowest height survived past the cap") + } + + if _, _, ok := ix.Lookup(over.Hash()); !ok { + t.Fatal("newly admitted height missing") + } +} + +func TestIndexResetDropsEverything(t *testing.T) { + ix := NewIndex() + tx := indexedTx(9) + + ix.Add(tx, indexedReceipt(9)) + ix.Reset() + + if _, _, ok := ix.Lookup(tx.Hash()); ok { + t.Fatal("reset left a receipt behind") + } +} diff --git a/eth/sequencer/reconcile.go b/eth/sequencer/reconcile.go new file mode 100644 index 0000000000..2668af4667 --- /dev/null +++ b/eth/sequencer/reconcile.go @@ -0,0 +1,144 @@ +package sequencer + +import ( + "bytes" + "context" + "errors" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" +) + +var errRefold = errors.New("refold: unknown entry kind") + +type reconcileOutcome int + +const ( + recOK reconcileOutcome = iota + recRetry + recTerminal +) + +// reconcile reads the store tail from the best-known position and classifies +// it against the local lineage. It runs on the transport +// goroutine; the enqueue side keeps folding throughout. +func (p *Publisher) reconcile(ctx context.Context) reconcileOutcome { + start := time.Now() + defer func() { reconcileTimer.UpdateSince(start) }() + + publishStateGauge.Update(gaugeResyncing) + + info, out := p.readTail(ctx) + if out != recOK { + return out + } + + return p.applyTail(info) +} + +// readTail walks the position ladder: last acked head, then a +// block-anchor probe, then the floor read. The probe starts from the +// in-flight build height when there is one, else from the chain's last +// imported block — a cold start locates the store tail from the local +// database rather than any persisted hint. +func (p *Publisher) readTail(ctx context.Context) (tailInfo, reconcileOutcome) { + p.mu.Lock() + anchor, confirmed, probeFrom := p.anchor, p.confirmed, p.curHeight + p.mu.Unlock() + + if confirmed { + actx, cancel := sliceDeadline(ctx) + info, out, done := p.tryWalk(actx, headReq(anchor), true) + cancel() + + if done { + return info, out + } + } + + if probeFrom == 0 && p.chain != nil { + if head := p.chain.CurrentBlock(); head != nil { + probeFrom = head.Number.Uint64() + } + } + + if probeFrom > 0 { + if info, out, done := p.read.probedWalk(ctx, probeFrom); done { + return info, out + } + } + + return p.read.floorRead(ctx) +} + +// sliceDeadline halves the parent's remaining budget, so the anchor rung +// cannot starve the probe and floor rungs below it. +func sliceDeadline(ctx context.Context) (context.Context, context.CancelFunc) { + dl, ok := ctx.Deadline() + if !ok { + return context.WithCancel(ctx) + } + + return context.WithDeadline(ctx, time.Now().Add(time.Until(dl)/2)) +} + +// tryWalk runs one reader rung with the publisher's policy attached: a +// matched walk compares tail entries against the unconfirmed journal, and a +// fold divergence there is version skew — terminal for this publisher, not +// for the reader. +func (p *Publisher) tryWalk(ctx context.Context, first *pb.RangeRequest, match bool) (tailInfo, reconcileOutcome, bool) { + info, out, done := p.read.tryWalk(ctx, first, p.absorber(match)) + if done && out == recTerminal { + p.fail("fold divergence", "err", errFoldDivergence) + } + + return info, out, done +} + +// absorber arms the divergence check for the anchor rung; unmatched walks +// carry no hook. +func (p *Publisher) absorber(match bool) func(*pb.Entry) error { + if !match { + return nil + } + + p.mu.Lock() + items, _ := p.journal.after(p.ackedSeq) + snap := append([]journalItem(nil), items...) + p.mu.Unlock() + + m := &matcher{snap: snap, on: true} + + return m.absorb +} + +// matcher runs the divergence check on the anchor rung: tail entries are +// compared in lockstep with our unconfirmed journal items. +type matcher struct { + snap []journalItem + idx int + on bool +} + +func (m *matcher) absorb(entry *pb.Entry) error { + if !m.on || m.idx >= len(m.snap) { + m.on = false + + return nil + } + + item := m.snap[m.idx] + if !contentEqual(entry, item.entry) { + m.on = false + + return nil + } + + if !bytes.Equal(entryPrefix(entry), entryPrefix(item.entry)) { + return errFoldDivergence + } + + m.idx++ + + return nil +} diff --git a/eth/sequencer/reconcile_test.go b/eth/sequencer/reconcile_test.go new file mode 100644 index 0000000000..d27c67e831 --- /dev/null +++ b/eth/sequencer/reconcile_test.go @@ -0,0 +1,543 @@ +package sequencer + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" +) + +type fakeChain struct { + canonical map[uint64]common.Hash + known map[common.Hash]*types.Header + current *types.Header + blocks map[uint64]*types.Block +} + +func (f *fakeChain) GetCanonicalHash(number uint64) common.Hash { + return f.canonical[number] +} + +func (f *fakeChain) GetHeaderByHash(hash common.Hash) *types.Header { + return f.known[hash] +} + +func (f *fakeChain) CurrentBlock() *types.Header { + return f.current +} + +func (f *fakeChain) GetBlockByNumber(number uint64) *types.Block { + return f.blocks[number] +} + +// appendForeignOpen appends an open at the store head, as a competing +// publisher would. +func appendForeignOpen(t *testing.T, h *harness, number uint64, parent common.Hash) { + t.Helper() + appendForeignOpenAt(t, h, number, parent, 1700000000+number) +} + +func appendForeignOpenAt(t *testing.T, h *harness, number uint64, parent common.Hash, ts uint64) { + t.Helper() + + entry := &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: number, + BlockTimestamp: ts, + ParentHash: parent.Bytes(), + GasLimit: 30_000_000, + BaseFee: big25gwei(), + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(entry); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign open rejected: %v", status) + } +} + +func appendForeignSeal(t *testing.T, h *harness, header *types.Header) { + t.Helper() + + raw, err := rlp.EncodeToBytes(header) + if err != nil { + t.Fatalf("rlp: %v", err) + } + + entry := &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{ + Header: raw, + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(entry); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign seal rejected: %v", status) + } +} + +func big25gwei() []byte { + return testHeader(0, common.Hash{}).BaseFee.Bytes() +} + +// A foreign open landing mid-block holds our publishing (no re-anchor over +// unsealed work); our seal flush then overrides it — the only supersede. +func TestSealFlushOverridesForeignWindow(t *testing.T) { + h := startHarness(t) + fc := &fakeChain{} + p := newTestPublisher(t, h, fc) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Our window for block 2, confirmed. + header2 := testHeader(2, sealHash(t, sealed)) + p.OpenBlock(2, header2.Time, header2.ParentHash, header2.GasLimit, header2.BaseFee) + tx0 := testTx(t, 0) + p.PublishTx(tx0) + waitHead(t, h, p, 5*time.Second) + + // A competing publisher supersedes our window with its own block 2. + appendForeignOpen(t, h, 2, common.Hash{0xaa}) + foreignHead := h.store.Head() + + // Our next record STALEs; the publisher holds instead of re-anchoring. + tx1 := testTx(t, 1) + p.PublishTx(tx1) + + waitFor(t, 5*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.hold.after != noHold + }) + + if h.store.Head() != foreignHead { + t.Fatal("held publisher wrote to the store") + } + + // The seal flush re-anchors: the sealed block overrides the window. + sealOnChain(p, fc, header2, []*types.Transaction{tx0, tx1}) + waitHead(t, h, p, 10*time.Second) + + resp, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: 2}) + if err != nil { + t.Fatalf("GetBlock: %v", err) + } + + open := resp.GetEntries()[0].GetBlockOpen() + if got := common.BytesToHash(open.GetParentHash()); got != sealHash(t, sealed) { + t.Fatalf("latest generation parent %x, want ours %x", got, sealHash(t, sealed)) + } + + if last := resp.GetEntries()[len(resp.GetEntries())-1]; last.GetBlockSeal() == nil { + t.Fatal("flushed generation is not sealed") + } +} + +// A canonically sealed foreign block at our pending height holds our stale +// build; the next build-start check rebases onto the store head and +// publishing resumes cleanly. +func TestForeignSealedHoldsThenRebases(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}, known: map[common.Hash]*types.Header{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + p.OpenBlock(2, 1700000002, sealHash(t, sealed), 30_000_000, testHeader(2, common.Hash{}).BaseFee) + waitHead(t, h, p, 5*time.Second) + + // The real producer's block 2 lands in the store and on our chain. + foreign := testHeader(2, common.Hash{0xbb}) + appendForeignOpen(t, h, 2, common.Hash{0xbb}) + appendForeignSeal(t, h, foreign) + chain.canonical[2] = foreign.Hash() + + // Our stale-window record STALEs; the publisher holds. + p.PublishTx(testTx(t, 0)) + + waitFor(t, 10*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.hold.after != noHold + }) + + // Next build: the check finds a clean (sealed) tail and rebases. + if w := p.AdoptWindow(3, foreign.Hash()); w != nil { + t.Fatalf("clean tail returned a window: %+v", w) + } + + publishBlock(t, p, 3, foreign.Hash(), 1) + waitHead(t, h, p, 5*time.Second) +} + +// publishRecorded drives one block through the publisher and registers it +// with the fake chain, as block import would. +func publishRecorded(t *testing.T, p *Publisher, chain *fakeChain, number uint64, parent common.Hash, txs int) *types.Header { + t.Helper() + + header := testHeader(number, parent) + p.OpenBlock(number, header.Time, parent, header.GasLimit, header.BaseFee) + + var body []*types.Transaction + + for i := 0; i < txs; i++ { + tx := testTx(t, uint64(i)) + body = append(body, tx) + p.PublishTx(tx) + } + + block := blockFor(header, body) + chain.blocks[number] = block + p.SealBlock(block) + + return header +} + +// An outage that stacks past the hot-flush bound collapses older blocks out +// of the journal; on recovery they are rebuilt from the chain database and +// every height still delivers to the store. +func TestOutageBackfillsFromDB(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p := newTestPublisher(t, h, chain) + + sealed := publishRecorded(t, p, chain, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + last := uint64(journalHotSeals + 10) + parent := sealHash(t, sealed) + for n := uint64(2); n <= last; n++ { + parent = sealHash(t, publishRecorded(t, p, chain, n, parent, 2)) + } + + p.mu.Lock() + collapsed := p.pendingFrom != 0 + p.mu.Unlock() + + if !collapsed { + t.Fatal("outage past journalHotSeals must collapse to the pending range") + } + + h.resume() + waitHead(t, h, p, 15*time.Second) + + publishRecorded(t, p, chain, last+1, parent, 1) + waitHead(t, h, p, 15*time.Second) + + // Everything delivered: the collapsed blocks came back from the DB. + for n := uint64(2); n <= last+1; n++ { + if _, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: n}); err != nil { + t.Fatalf("block %d missing after backfill: %v", n, err) + } + } +} + +// The same outage without chain access (no DB to rebuild from) skips the +// collapsed range as a counted forward jump and resumes at the tip. +func TestOutageChainlessJumps(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + jumps := reconcileForwardJump.Snapshot().Count() + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + last := uint64(journalHotSeals + 6) + parent := sealHash(t, sealed) + for n := uint64(2); n <= last; n++ { + parent = sealHash(t, publishBlock(t, p, n, parent, 1)) + } + + h.resume() + waitHead(t, h, p, 15*time.Second) + + publishBlock(t, p, last+1, parent, 1) + waitHead(t, h, p, 15*time.Second) + + // The collapsed front is gone (no DB to rebuild from) and counted. + if _, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: 2}); err == nil { + t.Fatal("collapsed block 2 unexpectedly published without a chain") + } + + for _, kept := range []uint64{last, last + 1} { + if _, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: kept}); err != nil { + t.Fatalf("block %d missing after recovery: %v", kept, err) + } + } + + if reconcileForwardJump.Snapshot().Count() == jumps { + t.Fatal("chainless backfill skip must count a forward jump") + } +} + +// The matcher flags a byte-identical entry folding from a different prefix — +// version skew, which is terminal. +func TestMatcherFoldDivergence(t *testing.T) { + entry := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{{0x01}}, + PrefixCommitment: commitment.Head{0xaa}.Bytes(), + }}} + + other := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{{0x01}}, + PrefixCommitment: commitment.Head{0xbb}.Bytes(), + }}} + + m := &matcher{snap: []journalItem{{entry: entry}}, on: true} + if err := m.absorb(other); err == nil { + t.Fatal("divergence not detected") + } +} + +func waitFor(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + + deadline := time.Now().Add(timeout) + + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition never held") + } + + time.Sleep(20 * time.Millisecond) + } +} + +// A block rebuilt from the chain database folds to exactly the head the +// live build produced — the backfill's byte-fidelity contract. +func TestBackfillByteExact(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p := newTestPublisher(t, h, chain) + + sealed := publishRecorded(t, p, chain, 1, common.Hash{0xef}, 3) + waitHead(t, h, p, 5*time.Second) + + liveHead := localHead(p) + + // Rebuild block 1 from the DB onto the same base the live build used. + fresh := newJournal() + + p.mu.Lock() + cur, ok := p.appendBlockLocked(fresh, commitment.Seed(testChainID), chain.blocks[1]) + p.mu.Unlock() + + if !ok { + t.Fatal("rebuild failed") + } + + if cur != liveHead { + t.Fatalf("rebuilt fold %x, live fold %x", cur, liveHead) + } + + _ = sealed +} + +// The backfill drains oldest-first and completely: a store that was down +// owes its readers every block, in order, so nothing is abandoned for +// freshness — the sealed tip must never advance past a gap it would then +// refuse to fill. +func TestBackfillDrainsOldestFirstAndCompletely(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + parent := common.Hash{0xef} + for n := uint64(1); n <= 50; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, nil) + parent = header.Hash() + } + + p.pendingFrom, p.pendingTo = 1, 50 + p.pendingEntries = 100 // open+seal per empty block + + jumps := reconcileForwardJump.Snapshot().Count() + drops := publishDropMeter.Snapshot().Count() + + fresh := newJournal() + + p.mu.Lock() + p.backfillLocked(fresh, commitment.Seed(testChainID)) + pendingFrom := p.pendingFrom + p.mu.Unlock() + + if got := fresh.seals; got != 50 { + t.Fatalf("rebuilt %d blocks, want all 50", got) + } + + if fresh.items[0].height != 1 { + t.Fatalf("rebuild starts at %d, want 1 (oldest first)", fresh.items[0].height) + } + + if pendingFrom != 0 { + t.Fatalf("pending not cleared after a full drain: from=%d", pendingFrom) + } + + if reconcileForwardJump.Snapshot().Count() != jumps { + t.Fatal("a complete drain is not a forward jump") + } + + if got := publishDropMeter.Snapshot().Count() - drops; got != 0 { + t.Fatalf("a complete drain dropped %d entries", got) + } +} + +// A range wider than one journal budget drains in batches: the batch takes +// the oldest blocks that fit, the remainder stays pending, and the next +// call resumes where it stopped — nothing is abandoned in between. +func TestBackfillResumesAcrossBatches(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + // Each block carries ~12MiB of calldata, so a 32MiB budget takes two + // blocks and change per batch. + payload := make([]byte, 12<<20) + parent := common.Hash{0xef} + + for n := uint64(1); n <= 5; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, []*types.Transaction{bigTx(t, payload)}) + parent = header.Hash() + } + + p.pendingFrom, p.pendingTo = 1, 5 + p.pendingEntries = 15 + + fresh := newJournal() + + p.mu.Lock() + cur := p.backfillLocked(fresh, commitment.Seed(testChainID)) + from1, to1 := p.pendingFrom, p.pendingTo + p.mu.Unlock() + + if fresh.seals == 0 || fresh.seals >= 5 { + t.Fatalf("first batch rebuilt %d blocks, want a strict subset", fresh.seals) + } + + if fresh.items[0].height != 1 { + t.Fatalf("first batch starts at %d, want 1 (oldest first)", fresh.items[0].height) + } + + if from1 != uint64(fresh.seals)+1 || to1 != 5 { + t.Fatalf("remainder not preserved: pending=[%d,%d] after %d rebuilt", + from1, to1, fresh.seals) + } + + // The next call picks up exactly where the last stopped. + next := newJournal() + + p.mu.Lock() + p.backfillLocked(next, cur) + p.mu.Unlock() + + if next.items[0].height != from1 { + t.Fatalf("second batch starts at %d, want %d", next.items[0].height, from1) + } +} + +// The whole pending range drains, storeSealedTip notwithstanding. The tip +// is the newest seal, not proof of anything below it: live flushes seal +// heights above the gap while it waits, and a store restart can shed acked +// writes — pending heights skipped on the tip stayed holes in the store +// forever on a devnet. A duplicate generation for a height the store does +// have costs churn; a hole costs the height. +func TestBackfillDrainsBelowTheSealedTip(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + parent := common.Hash{0xef} + for n := uint64(1); n <= 6; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, nil) + parent = header.Hash() + } + + p.pendingFrom, p.pendingTo = 1, 6 + p.storeSealedTip = 4 + + fresh := newJournal() + + p.mu.Lock() + p.backfillLocked(fresh, commitment.Seed(testChainID)) + p.mu.Unlock() + + if fresh.seals != 6 || fresh.items[0].height != 1 { + t.Fatalf("rebuilt seals=%d from=%d, want all 6 from 1: heights "+ + "skipped on the tip are never revisited", fresh.seals, fresh.items[0].height) + } +} + +// Mining continues while the backfill recovers an outage: with the DB as +// the archive nothing is displaced — every height delivers. +func TestMiningDuringBackfillLosesNothing(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p := newTestPublisher(t, h, chain) + + sealed := publishRecorded(t, p, chain, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + parent := sealHash(t, sealed) + for n := uint64(2); n <= 20; n++ { + parent = sealHash(t, publishRecorded(t, p, chain, n, parent, 5)) + } + + h.resume() + + // New blocks keep sealing while the backfill drains. + for n := uint64(21); n <= 24; n++ { + parent = sealHash(t, publishRecorded(t, p, chain, n, parent, 5)) + time.Sleep(50 * time.Millisecond) + } + + waitHead(t, h, p, 30*time.Second) + + for n := uint64(2); n <= 24; n++ { + if _, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: n}); err != nil { + t.Fatalf("block %d missing: mining-during-backfill displaced it", n) + } + } +} + +// bigTx pads a transaction with calldata so a block's size is dominated by +// it — the batching tests size blocks against the journal byte budget. +func bigTx(t *testing.T, payload []byte) *types.Transaction { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("key: %v", err) + } + + tx, err := types.SignNewTx(key, types.LatestSignerForChainID(big.NewInt(testChainID)), &types.DynamicFeeTx{ + ChainID: big.NewInt(testChainID), + GasTipCap: big.NewInt(1), + GasFeeCap: big.NewInt(30_000_000_000), + Gas: 21000, + To: &common.Address{0x01}, + Data: payload, + }) + if err != nil { + t.Fatalf("sign: %v", err) + } + + return tx +} diff --git a/eth/sequencer/stall_test.go b/eth/sequencer/stall_test.go new file mode 100644 index 0000000000..a93c7c335b --- /dev/null +++ b/eth/sequencer/stall_test.go @@ -0,0 +1,164 @@ +package sequencer + +import ( + "net" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + + "github.com/0xPolygon/sequence-store-proto/commitment" + "github.com/0xPolygon/sequence-store-proto/devstore" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" +) + +// setForTest swaps a package variable for the test's duration. +func setForTest[T any](t *testing.T, p *T, v T) { + t.Helper() + + old := *p + *p = v + + t.Cleanup(func() { *p = old }) +} + +// stallPublisher accepts the stream and reads entries but never acks — a +// hung store from the send loop's point of view. +type stallPublisher struct { + pb.UnimplementedPublisherServiceServer + sessions atomic.Int64 + received atomic.Int64 +} + +func (s *stallPublisher) Publish(stream pb.PublisherService_PublishServer) error { + s.sessions.Add(1) + + for { + if _, err := stream.Recv(); err != nil { + return err + } + + s.received.Add(1) + } +} + +// startStallStore serves a store whose publish stream never acks, with reads +// answered from an empty devstore so the startup reconcile anchors. +func startStallStore(t *testing.T) (*stallPublisher, *Publisher) { + t.Helper() + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + stall := &stallPublisher{} + srv := grpc.NewServer() + pb.RegisterPublisherServiceServer(srv, stall) + pb.RegisterConsumerServiceServer(srv, devstore.New(testChainID)) + + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + p, err := NewPublisher(lis.Addr().String(), lis.Addr().String(), testChainID, 0, nil) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + + return stall, p +} + +// A store that stops acking must trip the stall deadline and reconnect +// (acks stall → degraded) rather than sitting live forever. +func TestStreamAckStallReconnects(t *testing.T) { + setForTest(t, &ackStallTimeout, 100*time.Millisecond) + + stall, p := startStallStore(t) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + + waitFor(t, 5*time.Second, func() bool { return stall.sessions.Load() >= 3 }) + + if p.failed.Load() { + t.Fatal("ack stall must degrade and retry, not fail terminally") + } +} + +// A store that is not acking gets at most the in-flight cap: an unbounded +// sender once buried the store under its own backlog and turned the stall +// watchdog into a false outage signal. +func TestSendPausesAtTheInflightCap(t *testing.T) { + setForTest(t, &maxInflightEntries, 4) + // Keep the watchdog out of the way: a reconnect resends and would count + // the same entries twice. + setForTest(t, &ackStallTimeout, time.Minute) + + stall, p := startStallStore(t) + + publishBlock(t, p, 1, common.Hash{0xef}, 10) // open + 10 records + seal + + waitFor(t, 5*time.Second, func() bool { return stall.received.Load() == 4 }) + time.Sleep(200 * time.Millisecond) + + if got := stall.received.Load(); got != 4 { + t.Fatalf("sender pushed %d entries into a non-acking store, cap is 4", got) + } +} + +// A cap smaller than the window must not wedge the drain: acks free slots +// and the post-ack send refills, so the whole journal still delivers even +// when the worker appends nothing further. +func TestCappedSenderStillDrainsCompletely(t *testing.T) { + setForTest(t, &maxInflightEntries, 2) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + publishBlock(t, p, 1, common.Hash{0xef}, 6) // open + 6 records + seal + waitHead(t, h, p, 5*time.Second) +} + +// An eviction gap must be reported even when a hold gates everything past +// the cursor: the gap is the drain cycle's reconcile trigger, and a devnet +// stranded a whole outage window when an early-out answered before the +// coverage check ran. +func TestGapDetectionOutranksTheHoldEarlyOut(t *testing.T) { + p := barePublisher() + p.journal.nextSeq = 5 + p.journal.append(nil, commitment.Head{}, commitment.Head{0x01}, entryRecord, 3, 0, nil) + p.hold = hold{after: 0, kind: holdBuild} + + if _, _, ok := p.sendableAfter(1, 100); ok { + t.Fatal("a gated suffix hid the eviction gap; the drain cycle never reconciles") + } +} + +// A store restart once wedged live gRPC channels in a permanent +// connect-retry loop while fresh dials worked. After a whole interval of +// silence the publisher rebuilds its connections rather than trusting the +// channel state machine — and keeps working through the swap. +func TestRedialAfterProlongedSilence(t *testing.T) { + setForTest(t, &redialAfter, 400*time.Millisecond) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + first := publishBlock(t, p, 1, common.Hash{0xef}, 2) + waitHead(t, h, p, 5*time.Second) + + before := publishRedialCount.Snapshot().Count() + + h.stop() + waitFor(t, 10*time.Second, func() bool { + return publishRedialCount.Snapshot().Count() > before + }) + + h.resume() + publishBlock(t, p, 2, first.Hash(), 2) + waitHead(t, h, p, 10*time.Second) +} diff --git a/eth/sequencer/stream.go b/eth/sequencer/stream.go new file mode 100644 index 0000000000..3acecff111 --- /dev/null +++ b/eth/sequencer/stream.go @@ -0,0 +1,581 @@ +package sequencer + +import ( + "context" + "sync/atomic" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/log" +) + +type streamEnd int + +const ( + endCtx streamEnd = iota + endTerminal + endTransport + endStale + endIdle + endWatch +) + +// idleReconcileInterval paces the idle catch-up read: a publisher with +// nothing in flight (a non-producer between spans) re-anchors near the +// store tip so a takeover build's bounded read starts close — from a +// stale anchor the read budget dies walking history and the adopt is +// missed. Var for tests. +var idleReconcileInterval = 30 * time.Second + +// heldWatchInterval paces the tail re-reads taken while a sticky hold gates +// a live build. Short enough to catch a competing producer inside one block +// interval; the reads only happen while held, which is rare. Var for tests. +var heldWatchInterval = 400 * time.Millisecond + +// maxInflightEntries bounds what one session keeps sent-but-unacked. Deep +// enough to saturate a group-committing ingress, shallow enough that the +// trailing entry's ack latency stays a queue drain, not a backlog: an +// unbounded sender once buried the store under 23k entries of its own +// republish churn, tripped the stall watchdog on the self-made backlog, and +// produced blind while flagged unreachable. With the cap, a stall means the +// store stopped — nobody is being promised anything — which is the only +// state where producing without a verdict is safe. Var for tests. +var maxInflightEntries = 2048 + +type streamResult struct { + reason streamEnd + // progressed reports whether any entry was confirmed this session — it + // resets the contention backoff streak. + progressed bool +} + +type ackResult struct { + status pb.AckStatus + err error +} + +type sent struct { + item journalItem + at time.Time +} + +// stallTracker watches ack progress from outside the send loop: a hung +// store fills the stream's flow-control window and blocks Send, so no +// in-loop timer can fire — the watchdog cancels the session context +// instead, which unblocks both Send and Recv. +type stallTracker struct { + inflight atomic.Int64 + lastAck atomic.Int64 // unix nanos of the last ack (or session start) +} + +func newStallTracker() *stallTracker { + t := &stallTracker{} + t.lastAck.Store(time.Now().UnixNano()) // session start counts as progress + + return t +} + +func (s *stallTracker) sent() { s.inflight.Add(1) } +func (s *stallTracker) acked() { s.inflight.Add(-1); s.lastAck.Store(time.Now().UnixNano()) } + +func (s *stallTracker) stalled(deadline time.Duration) bool { + return s.inflight.Load() > 0 && + time.Since(time.Unix(0, s.lastAck.Load())) > deadline +} + +// watch cancels the session once acks stall past the deadline; it exits +// with the session. The deadline arrives as a parameter so the goroutine +// never reads publisher state it could outlive. +func (s *stallTracker) watch(ctx context.Context, cancel context.CancelFunc, deadline time.Duration) { + tick := time.NewTicker(deadline / 4) + defer tick.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if s.stalled(deadline) { + log.Warn("Sequencer ack stall, reconnecting", + "inflight", s.inflight.Load(), + "waited", time.Since(time.Unix(0, s.lastAck.Load()))) + cancel() + + return + } + } + } +} + +// runStream runs one stream session: it resends unacked journal items, +// forwards new ones as the worker appends them, and retires items as acks +// arrive (the store acks in entry order, so the oldest in-flight item owns +// each ack). +func (p *Publisher) runStream(ctx context.Context) streamResult { + sctx, cancel := context.WithCancel(ctx) + defer cancel() + + stream, err := p.pub.Publish(sctx) + if err != nil { + return streamResult{reason: endTransport} + } + + acks := make(chan ackResult) + + go recvAcks(sctx, stream, acks) + + // A hung store stalls acks without erroring the stream; the watchdog + // forces a reconnect whose resend converges once the store wakes. + tracker := newStallTracker() + + go tracker.watch(sctx, cancel, ackStallTimeout) + + // A resent entry may already have been applied (the previous stream + // broke after the append, before the ack) — its STALE lands in + // reconciliation, whose anchored byte-match retires exactly the + // applied prefix. No local guessing: retiring on a STALE without + // store confirmation can mint a fictional frontier when the earlier + // send itself had STALEd. + var inflight []sent + + cursor, ok := p.sendAfter(stream, p.ackedSeqSnapshot(), &inflight, tracker) + if !ok { + return streamResult{reason: endStale} + } + + progressed := false + + idle := time.NewTicker(idleReconcileInterval) + defer idle.Stop() + + // A sticky hold stops our writes, which also stops the STALEs that + // would otherwise bring fresh tail reads — leaving us blind to the + // competing producer for the rest of the block. Keep looking on a + // short cadence while held: reads are harmless, and seeing the + // competitor is what lets the build follow their sequence. + watch := time.NewTicker(heldWatchInterval) + defer watch.Stop() + + for { + select { + case <-ctx.Done(): + return streamResult{reason: endCtx, progressed: progressed} + case <-sctx.Done(): + // Watchdog cancel; recvAcks may lose the race to deliver the + // recv error, so the session ends here. + return streamResult{reason: endTransport, progressed: progressed} + case <-idle.C: + if p.quiet() { + return streamResult{reason: endIdle, progressed: progressed} + } + case <-watch.C: + if p.heldMidBuild() { + return streamResult{reason: endWatch, progressed: progressed} + } + case <-p.wake: + case ack := <-acks: + res, done := p.handleAck(ack, &inflight) + if done { + res.progressed = progressed + + return res + } + + progressed = true // an OK ack retired an entry + + tracker.acked() + } + + // One send site serves every wake source: a worker append, an ack + // freeing a cap slot, or a tick that found nothing to end on. + if cursor, ok = p.sendAfter(stream, cursor, &inflight, tracker); !ok { + return streamResult{reason: endStale, progressed: progressed} + } + } +} + +// sendableAfter snapshots, under the lock, the journal items past cursor +// that may go out now: covered by the journal, at or below the send +// ceiling (a draining seal flush below the ceiling always finishes +// delivering), and at most limit items. acked rides along so the send loop +// can skip adoption-confirmed entries. ok is false when eviction has +// created a gap past cursor — reconciliation must decide. +func (p *Publisher) sendableAfter(cursor uint64, limit int) (items []journalItem, acked uint64, ok bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // The coverage check runs on every call, before any early-out: an + // eviction gap is the drain cycle's reconcile trigger, and a call that + // skips it (a full pipeline, a gated suffix) can wedge a backfill with + // nothing else scheduled to notice — a devnet stranded a whole outage + // window exactly that way. + view, covered := p.journal.after(cursor) + if !covered { + return nil, 0, false + } + + // A hold at or below the cursor gates everything past it — the common + // sticky-hold shape — and seqs ascend, so answer without walking. + if p.hold.active() && p.hold.after <= cursor { + return nil, p.ackedSeq, true + } + + // Truncate before trimming the held tail: gated items are always a + // suffix, so the result is the same and the walk is bounded by the + // cap instead of the window size. + if len(view) > limit { + view = view[:max(limit, 0)] + } + + for len(view) > 0 && p.hold.gates(view[len(view)-1].seq) { + view = view[:len(view)-1] + } + + // Copy under the lock: view aliases the journal's backing array, which + // worker-side evictions compact in place while Send blocks — iterating + // the alias unlocked is a data race. + return append([]journalItem(nil), view...), p.ackedSeq, true +} + +// sendAfter sends the sendable journal items past cursor, up to the +// in-flight cap, returning the new cursor. ok is false on an eviction gap. +// Acks free capacity, and the send after each ack refills — without that +// pairing a capped drain with an idle worker would send one window and +// stop, since nothing else wakes the loop. A full pipeline still calls +// down: the coverage check must run even when nothing can be sent. +func (p *Publisher) sendAfter(stream pb.PublisherService_PublishClient, cursor uint64, inflight *[]sent, tracker *stallTracker) (uint64, bool) { + items, acked, ok := p.sendableAfter(cursor, maxInflightEntries-len(*inflight)) + if !ok { + return cursor, false + } + + for _, item := range items { + // An adoption can mark entries acked between + // this session's sends: the store already holds them, and sending + // them again guarantees a STALE (the adopter is silent). + if item.seq <= acked { + cursor = item.seq + + continue + } + + if err := stream.Send(&pb.PublishRequest{Entry: item.entry}); err != nil { + // The recv side reports the definitive error; keep the cursor so + // nothing is skipped. + break + } + + *inflight = append(*inflight, sent{item: item, at: time.Now()}) + tracker.sent() + cursor = item.seq + + publishedCounter.Inc(1) + } + + return cursor, true +} + +// handleAck retires or rejects the oldest in-flight entry. done=true ends +// the session with the returned result; done=false means the ack was OK +// and an entry was retired — the session progressed. +func (p *Publisher) handleAck(ack ackResult, inflight *[]sent) (streamResult, bool) { + if ack.err != nil { + return streamResult{reason: endTransport}, true + } + + if len(*inflight) == 0 { + p.fail("ack without a pending entry") + + return streamResult{reason: endTerminal}, true + } + + first := (*inflight)[0] + *inflight = (*inflight)[1:] + + switch ack.status { + case pb.AckStatus_ACK_STATUS_OK: + if p.retire(first.item, first.at) { + return streamResult{reason: endWatch}, true + } + + return streamResult{}, false + case pb.AckStatus_ACK_STATUS_STALE_COMMITMENT: + p.markGateLost(first.item) + + // Even for a resend that may have been applied before its stream + // broke, reconciliation decides: the anchored byte-match retires + // exactly the applied prefix, without inventing a frontier. + return streamResult{reason: endStale}, true + case pb.AckStatus_ACK_STATUS_RATE_LIMITED: + // The rejected entry never advanced the store head and everything + // pipelined behind it failed the head check too — a fresh stream + // resending in order is exact. + return streamResult{reason: endTransport}, true + default: + p.fail("store rejected entry", "status", ack.status) + + return streamResult{reason: endTerminal}, true + } +} + +// quiet reports a lineage with nothing in flight: either fully +// drained, or every unacked entry gated behind the send ceiling (a parked +// adopter's dead buffer). Both states leave the idle catch-up read free — +// requiring a full drain would let a held buffer starve the catch-up all +// span, leaving a stale anchor exactly when a takeover needs a fresh one. +func (p *Publisher) quiet() bool { + p.mu.Lock() + defer p.mu.Unlock() + + if p.unackedLocked() == 0 { + return true + } + + if !p.hold.active() { + return false + } + + for _, it := range p.journal.items { + if it.seq > p.ackedSeq && !p.hold.gates(it.seq) { + return false // sendable in-flight work: not quiet + } + } + + return true +} + +func (p *Publisher) ackedSeqSnapshot() uint64 { + p.mu.Lock() + defer p.mu.Unlock() + + return p.ackedSeq +} + +func recvAcks(ctx context.Context, stream pb.PublisherService_PublishClient, acks chan<- ackResult) { + for { + resp, err := stream.Recv() + + result := ackResult{err: err} + if err == nil { + result.status = resp.GetStatus() + } + + select { + case acks <- result: + case <-ctx.Done(): + return + } + + if err != nil { + return + } + } +} + +// retire marks one item store-confirmed. It reports whether the retired +// item was the backfill batch's last entry with blocks still pending — the +// cue for the caller to reconcile the next batch in. +func (p *Publisher) retire(item journalItem, at time.Time) (drain bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // A lagging ack may describe an entry from a lineage that a swap + // (adoption, refold, rewind) has since replaced: advancing ackedSeq or + // the anchor from it would regress the frontier and fake an eviction + // gap, forcing a spurious forward-jump over a live window. The ack is + // only meaningful while the exact entry still stands in the journal. + if cur, ok := p.journal.itemAt(item.seq); !ok || cur.post != item.post || item.seq <= p.ackedSeq { + return false + } + + p.ackedSeq = item.seq + p.anchor = item.post + p.confirmed = true + p.markReachable() + + if item.kind == entrySeal && item.height > p.storeSealedTip { + log.Debug("Sequencer sealed tip advance", "origin", "seal-ack", + "from", p.storeSealedTip, "to", item.height) + p.storeSealedTip = item.height + } + + // Confirmation is keyed to the gated block's hash, not just its height: + // a refused block and its rebuild share a height, and a late ack for + // the first attempt's seal must not confirm the second's gate — that + // would broadcast content the store's seal does not describe. + if item.kind == entrySeal && p.gate.height == item.height && + p.gate.verdict == gatePending { + if header, err := decodeSealHeader(item.entry.GetBlockSeal().GetHeader()); err == nil && + header.Hash() == p.gate.hash { + p.gate.verdict = gateConfirmed + } + } + + // A build-start hold only orders the new window behind the draining + // flush: once the flush's seal is home, lift it so the window streams + // mid-block instead of batching until its own seal. During an outage + // drain the ceiling stays: the store is owed older blocks first, and + // the retired batch is the cue to reconcile the next one in. + if item.kind == entrySeal && p.hold.kind == holdBuild && !p.hold.gates(item.seq) { + if p.pendingFrom != 0 { + drain = true + } else { + p.hold = clearedHold() + p.signalWake() + } + } + + publishQueueGauge.Update(int64(p.unackedLocked())) + + publishAckTimer.UpdateSince(at) + + return drain +} + +// runState carries the run loop's backoff bookkeeping. +type runState struct { + probe time.Duration + contention int + lastRedial time.Time +} + +// run owns transport: anchor via reconciliation, then send loop the journal into the +// store, reconciling on STALE and probing with backoff while unreachable. +func (p *Publisher) run(ctx context.Context) { + defer close(p.done) + defer func() { + _ = p.pubConn.Close() + _ = p.consConn.Close() + }() + + state := runState{probe: probeBackoffMin} + + for ctx.Err() == nil && !p.failed.Load() { + if !p.step(ctx, &state) { + return + } + } +} + +// step runs one iteration: anchor when needed, otherwise one send loop session. +func (p *Publisher) step(ctx context.Context, state *runState) bool { + // A whole interval of silence means the connections themselves are + // suspect — a store restart has wedged the gRPC channels while fresh + // dials worked. Rate-limited to one rebuild per interval. + if p.silentTooLong() && time.Since(state.lastRedial) > redialAfter { + p.redial() + state.lastRedial = time.Now() + } + + if !p.isAnchored() { + return p.anchorStep(ctx, state) + } + + publishStateGauge.Update(gaugeLive) + + res := p.runStream(ctx) + + if res.reason != endTransport { + p.unreachable.Store(false) // the store answered, whatever it said + } + + switch res.reason { + case endCtx, endTerminal: + return false + case endIdle: + return p.idleReanchor(ctx) + case endWatch: + // A held build re-reading the store: reconcile only, keeping the + // build's height so the tail is classified against it. + return p.reconcile(ctx) != recTerminal + case endTransport: + p.unreachable.Store(true) + + return state.degradedSleep(ctx) + default: // endStale + publishStaleCount.Inc(1) + + state.contention = contentionSleep(ctx, res.progressed, state.contention) + if ctx.Err() != nil { + return false + } + + state.probe = probeBackoffMin + p.setUnanchored() + + return true + } +} + +func (p *Publisher) anchorStep(ctx context.Context, state *runState) bool { + if p.reconcile(ctx) == recOK { + p.unreachable.Store(false) // the read went through + state.probe = probeBackoffMin + + return true + } + + if p.failed.Load() { + return false + } + + return state.degradedSleep(ctx) +} + +func (s *runState) degradedSleep(ctx context.Context) bool { + publishStateGauge.Update(gaugeDegraded) + + if !sleepCtx(ctx, s.probe) { + return false + } + + s.probe = min(s.probe*2, probeBackoffMax) + + return true +} + +// idleReanchor re-anchors near the store tip after a quiet interval, so a +// later takeover build's bounded read starts close. A window still held +// after the quiet period is parked — its build died and its seal will +// never come — so drop its height, letting the reconcile refold the dead +// buffer instead of holding the anchor stale forever. A freely streaming +// window (no hold) is a healthy incumbent momentarily drained between +// txs, not a dead build: leaving its height intact keeps a coincident +// foreign write on the hold-and-let-our-seal-win path rather than +// superseding a window we may still seal. +func (p *Publisher) idleReanchor(ctx context.Context) bool { + p.mu.Lock() + if p.hold.active() { + p.curHeight = 0 + } + p.mu.Unlock() + + return p.reconcile(ctx) != recTerminal +} + +// contentionSleep applies the reconcile backoff when the previous +// reconcile's corrective publish immediately re-STALEd; the +// first STALE reconciles without delay, and progress resets the streak. +func contentionSleep(ctx context.Context, progressed bool, streak int) int { + if progressed { + return 0 + } + + if streak > 0 { + delay := min(reconcileBackoffMin<<(streak-1), reconcileBackoffMax) + publishStateGauge.Update(gaugeContending) + sleepCtx(ctx, delay) + } + + return streak + 1 +} + +func sleepCtx(ctx context.Context, d time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(d): + return true + } +} diff --git a/eth/sequencer/stream_test.go b/eth/sequencer/stream_test.go new file mode 100644 index 0000000000..3585016633 --- /dev/null +++ b/eth/sequencer/stream_test.go @@ -0,0 +1,136 @@ +package sequencer + +import ( + "errors" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" +) + +func barePublisher() *Publisher { + p := &Publisher{ + head: commitment.Seed(testChainID), + anchor: commitment.Seed(testChainID), + seed: commitment.Seed(testChainID), + journal: newJournal(), + wake: make(chan struct{}, 1), + } + p.read = newReader(nil, p.seed, p.markReachable) + + return p +} + +func TestHandleAck(t *testing.T) { + item := journalItem{seq: 7, post: commitment.Head{0x07}, kind: entrySeal} + + cases := []struct { + name string + ack ackResult + inflight []sent + wantReason streamEnd + wantDone bool + wantAcked uint64 + wantFailed bool + }{ + { + name: "ok retires", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_OK}, + inflight: []sent{{item: item, at: time.Now()}}, + wantDone: false, + wantAcked: 7, + wantReason: endCtx, // unused when done=false + }, + { + name: "stale reconciles even on a resend", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_STALE_COMMITMENT}, + inflight: []sent{{item: item}}, + wantDone: true, + wantReason: endStale, + }, + { + name: "rate limited retries transport", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_RATE_LIMITED}, + inflight: []sent{{item: item}}, + wantDone: true, + wantReason: endTransport, + }, + { + name: "malformed is terminal", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_MALFORMED}, + inflight: []sent{{item: item}}, + wantDone: true, + wantReason: endTerminal, + wantFailed: true, + }, + { + name: "transport error", + ack: ackResult{err: errors.New("recv")}, + inflight: []sent{{item: item}}, + wantDone: true, + wantReason: endTransport, + }, + { + name: "ack without pending is terminal", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_OK}, + wantDone: true, + wantReason: endTerminal, + wantFailed: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := barePublisher() + // retire only honors acks for entries still live in the journal; + // seat the fixture item at its seq. + p.journal.nextSeq = item.seq + p.journal.items = append(p.journal.items, item) + p.journal.nextSeq = item.seq + 1 + + inflight := append([]sent(nil), tc.inflight...) + + res, done := p.handleAck(tc.ack, &inflight) + + if done != tc.wantDone { + t.Fatalf("done = %v, want %v", done, tc.wantDone) + } + + if done && res.reason != tc.wantReason { + t.Fatalf("reason = %v, want %v", res.reason, tc.wantReason) + } + + if p.ackedSeq != tc.wantAcked { + t.Fatalf("ackedSeq = %d, want %d", p.ackedSeq, tc.wantAcked) + } + + if p.failed.Load() != tc.wantFailed { + t.Fatalf("failed = %v, want %v", p.failed.Load(), tc.wantFailed) + } + }) + } +} + +func TestHandleAckOKMarksProgress(t *testing.T) { + p := barePublisher() + live := journalItem{seq: 1, post: commitment.Head{0x01}} + p.journal.items = append(p.journal.items, live) + p.journal.nextSeq = 2 + + inflight := []sent{{item: live, at: time.Now()}} + + // done=false is the progress signal: the session counts it as an + // entry retired. + if _, done := p.handleAck(ackResult{status: pb.AckStatus_ACK_STATUS_OK}, &inflight); done { + t.Fatal("ok ack must not end the session") + } + + if p.anchor != (commitment.Head{0x01}) { + t.Fatalf("anchor = %x", p.anchor) + } + + if !p.confirmed { + t.Fatal("ok ack must confirm the anchor") + } +} diff --git a/eth/sequencer/twin_test.go b/eth/sequencer/twin_test.go new file mode 100644 index 0000000000..3bcae5b058 --- /dev/null +++ b/eth/sequencer/twin_test.go @@ -0,0 +1,391 @@ +package sequencer + +import ( + "testing" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" +) + +// Two publishers against one store: the unit-test stand-in for two block +// producers sharing a signing key. Every scenario below is one that cost a +// kurtosis run to find, so each is pinned here instead. +func twinPublishers(t *testing.T) (*harness, *Publisher, *Publisher) { + t.Helper() + + h := startHarness(t) + + return h, newTestPublisher(t, h, &fakeChain{}), newTestPublisher(t, h, &fakeChain{}) +} + +// sealedParent gets both publishers past a sealed block 1 so height 2 is a +// clean contested slot, and returns the parent hash they must both build on. +func sealedParent(t *testing.T, h *harness, incumbent, twin *Publisher) common.Hash { + t.Helper() + + sealed := publishBlock(t, incumbent, 1, common.Hash{0xef}, 1) + waitHead(t, h, incumbent, 5*time.Second) + + // The twin has to see block 1 too, or its own build starts from a stale + // anchor and the scenario tests the wrong thing. + waitFor(t, 5*time.Second, func() bool { return twin.isAnchored() }) + + return sealHash(t, sealed) +} + +// The height=243 failure: the twin adopts a snapshot of the incumbent's +// window, the incumbent keeps writing, and the twin seals the snapshot — +// stranding every record added since. Measured as a 362-record block +// displacing 1088 already-acked ones. +func TestTwinDoesNotSealStaleSnapshot(t *testing.T) { + h, incumbent, twin := twinPublishers(t) + parent := sealedParent(t, h, incumbent, twin) + + incumbent.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + incumbent.PublishTx(testTx(t, 0)) + incumbent.PublishTx(testTx(t, 1)) + waitDrained(t, incumbent, 5*time.Second) + + // The twin inherits the window as it stands now: two records. + w := twin.AdoptWindow(2, parent) + if w == nil { + t.Fatal("twin did not adopt the incumbent's window") + } + + if len(w.Txs) != 2 { + t.Fatalf("adopted %d txs, want the 2 published so far", len(w.Txs)) + } + + // The incumbent is alive and keeps going. The twin's snapshot is now + // stale by one record. + incumbent.PublishTx(testTx(t, 2)) + waitDrained(t, incumbent, 5*time.Second) + + // Mark the height contested, as a STALE would, and take the contested + // path twice: the first call rebuilds, the second must still refuse + // because the window moved on. + twin.mu.Lock() + twin.curHeight = 2 + twin.hold = hold{after: twin.ackedSeq, kind: holdSticky} + twin.mu.Unlock() + + if awaitOurWindow(twin, 2*time.Second) { + t.Fatal("first contested attempt sealed instead of rebuilding") + } + + if awaitOurWindow(twin, 2*time.Second) { + t.Fatal("twin sealed a stale snapshot: the records the incumbent " + + "added after adoption were acked, so they are preconfirmed and " + + "this block strands them") + } +} + +// The same path must still reach a seal once the incumbent stops, or +// contention has no liveness escape and the chain stalls — the failure that +// held a devnet at one height for five minutes. +func TestTwinSealsOnceWindowSettles(t *testing.T) { + h, incumbent, twin := twinPublishers(t) + parent := sealedParent(t, h, incumbent, twin) + + incumbent.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + incumbent.PublishTx(testTx(t, 0)) + waitDrained(t, incumbent, 5*time.Second) + + if twin.AdoptWindow(2, parent) == nil { + t.Fatal("twin did not adopt the incumbent's window") + } + + twin.mu.Lock() + twin.curHeight = 2 + twin.hold = hold{after: twin.ackedSeq, kind: holdSticky} + twin.mu.Unlock() + + if awaitOurWindow(twin, 2*time.Second) { + t.Fatal("a contested attempt sealed instead of rebuilding") + } + + twin.ResyncNeeded() // the worker's rebuild consumes the signal + + // The rebuild adopts: the incumbent wrote nothing more, so the adopted + // window is the whole of the store's content and the seal proceeds. + if twin.AdoptWindow(2, parent) == nil { + t.Fatal("rebuild did not adopt the settled window") + } + + if !awaitOurWindow(twin, 2*time.Second) { + t.Fatal("twin refused a settled, fully adopted window: nobody would " + + "ever close this height") + } +} + +// Adoption hands the build exactly the store's transactions: the adopted +// window is the content the block extends, in store order. +func TestContestedTwinSealsWindowExactly(t *testing.T) { + h, incumbent, twin := twinPublishers(t) + parent := sealedParent(t, h, incumbent, twin) + + incumbent.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + incumbent.PublishTx(testTx(t, 0)) + incumbent.PublishTx(testTx(t, 1)) + waitDrained(t, incumbent, 5*time.Second) + + w := twin.AdoptWindow(2, parent) + if w == nil { + t.Fatal("twin did not adopt the incumbent's window") + } + + if len(w.Txs) != 2 { + t.Fatalf("window carries %d txs, want the 2 in the store", len(w.Txs)) + } +} + +// The mid-window blind spot, closed by Rule 1: a producer whose anchor sits +// at the store head — past the live window's open — used to classify the +// tail as clean and open a second generation beside the incumbent's. The +// build-start read now probes down from the height itself, so the window is +// visible from anywhere. +func TestBuildStartSeesLiveWindowFromAMidWindowAnchor(t *testing.T) { + h, incumbent, twin := twinPublishers(t) + parent := sealedParent(t, h, incumbent, twin) + + incumbent.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + incumbent.PublishTx(testTx(t, 0)) + waitDrained(t, incumbent, 5*time.Second) + + // Strand the twin's anchor at the store head, mid-window: exactly the + // state a between-blocks re-anchor used to leave behind. + twin.mu.Lock() + twin.anchor = incumbent.head + twin.confirmed = true + twin.mu.Unlock() + + w := twin.AdoptWindow(2, parent) + if w == nil { + t.Fatal("a live window at this height was invisible from a " + + "mid-window anchor: the twin would open a second generation " + + "beside the incumbent's") + } + + if len(w.Txs) != 1 { + t.Fatalf("adopted %d txs, want the incumbent's 1", len(w.Txs)) + } +} + +// The open CAS is the election: when both twins open the same height, one +// open lands and the other STALEs. The loser must adopt — and the store must +// end up with exactly one generation at the height, which is the invariant +// every double-seal traced back to. +func TestLosingOpenAdoptsAndAddsNoGeneration(t *testing.T) { + h, a, b := twinPublishers(t) + parent := sealedParent(t, h, a, b) + + // A wins the height: its window is in the store. + a.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + a.PublishTx(testTx(t, 0)) + waitDrained(t, a, 5*time.Second) + + // B opens the same height blind (its build-start read raced A's open). + // Divergent content, so nothing absorbs: B's open must STALE. + b.OpenBlock(2, 1700000009, parent, 30_000_000, fee25()) + b.PublishTx(testTx(t, 7)) + + // B discovers the loss and asks for a rebuild. + waitFor(t, 5*time.Second, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + + return b.hold.kind == holdSticky || b.resync + }) + + if !awaitOurWindow(b, time.Second) { + b.ResyncNeeded() // the worker's rebuild consumes the signal + } + + // The rebuild's boundary read adopts A's window. + w := b.AdoptWindow(2, parent) + if w == nil { + t.Fatal("the losing twin did not adopt the winner's window") + } + + // The store holds exactly one generation at height 2: the CAS kept the + // loser's open out entirely. + gens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 { + gens++ + } + } + + if gens != 1 { + t.Fatalf("store holds %d generations at height 2, want exactly 1: "+ + "a second generation is where every double-seal came from", gens) + } +} + +// windowContent returns this publisher's entries for a height, so two +// publishers' views can be compared for genuine agreement rather than for +// having merely passed the same checks. +func windowContent(t *testing.T, p *Publisher, height uint64) []*pb.Entry { + t.Helper() + + p.mu.Lock() + defer p.mu.Unlock() + + var out []*pb.Entry + for _, it := range p.journal.suffixFromHeight(height) { + out = append(out, it.entry) + } + + return out +} + +func sameContent(a, b []*pb.Entry) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if !contentEqual(a[i], b[i]) { + return false + } + } + + return true +} + +// decide drives a publisher to a final seal answer. The contested path +// defers once before deciding, so one call is not the verdict. +func decide(p *Publisher) bool { + for i := 0; i < 3; i++ { + if ok := awaitOurWindow(p, time.Second); ok { + return true + } + + p.ResyncNeeded() // consume, as the worker's rebuild would + } + + return false +} + +// The invariant the whole mechanism exists for: two producers at one height +// must not both seal while holding different content. Every other test here +// checks a decision in isolation; this one checks the outcome, which is what +// a consumer actually experiences. +func TestTwinsNeverBothSealDivergentContent(t *testing.T) { + h, a, b := twinPublishers(t) + parent := sealedParent(t, h, a, b) + + // A gets its window in first. + a.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + a.PublishTx(testTx(t, 0)) + waitDrained(t, a, 5*time.Second) + + // B builds the same height on the same context with different content. + b.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + b.PublishTx(testTx(t, 7)) + + waitFor(t, 5*time.Second, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + + return b.hold.kind == holdSticky || b.unackedLocked() == 0 + }) + + sealA, sealB := decide(a), decide(b) + + if sealA && sealB && !sameContent(windowContent(t, a, 2), windowContent(t, b, 2)) { + t.Fatal("both producers sealed height 2 holding different content: " + + "consensus discards one block and every record only in it loses " + + "its preconfirmation") + } +} + +// A contested height must not seal on an unreadable tail. This is the state +// that held for 68 of one node's reads in a five-minute devnet window +// ("rung out of budget"), and treating it as permission to seal is what +// turned every missed read into a divergent block. +func TestContestedSealRefusedWhenTailUnreadable(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + p.mu.Lock() + p.curHeight = 2 + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + p.read.cons = &failingConsumer{} + p.mu.Unlock() + + if awaitOurWindow(p, time.Second) { + t.Fatal("sealed a contested height without being able to read the " + + "store: with no way to know what the other producer wrote, this " + + "block can only diverge") + } +} + +// The same requirement for the coverage check on its own: an unreadable tail +// is not evidence of coverage. Failing open here is what made every dropped +// read a licence to seal. +func TestCoverageFailsClosedOnUnreadableTail(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + // Uncontested but unreadable: an ordinary build must still seal, because + // production never waits on the store. The distinction is contention. + p.mu.Lock() + p.read.cons = &failingConsumer{} + p.mu.Unlock() + + if !awaitOurWindow(p, time.Second) { + t.Fatal("an unreachable store blocked an uncontested seal: " + + "production must never wait on the store") + } +} + +// The counterpart invariant: under contention someone must still seal. Two +// producers politely refusing each other is what held a devnet at one height +// for five minutes, and no safety property is worth a stopped chain. +// +// Together with TestTwinsNeverBothSealDivergentContent this pins the whole +// trade: that test forbids two divergent seals, this one forbids zero seals. +// Any change that satisfies one by breaking the other has not solved +// anything. +func TestTwinsAlwaysProduceASeal(t *testing.T) { + h, a, b := twinPublishers(t) + parent := sealedParent(t, h, a, b) + + a.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + a.PublishTx(testTx(t, 0)) + waitDrained(t, a, 5*time.Second) + + b.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + b.PublishTx(testTx(t, 7)) + + waitFor(t, 5*time.Second, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + + return b.hold.kind == holdSticky || b.unackedLocked() == 0 + }) + + if !decide(a) && !decide(b) { + t.Fatal("neither producer sealed height 2: the height never closes " + + "and the chain stops advancing") + } +} diff --git a/go.mod b/go.mod index e9c852a6e9..de13de5270 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/0xPolygon/crand v1.0.3 github.com/0xPolygon/heimdall-v2 v0.7.1 github.com/0xPolygon/polyproto v0.0.8 + github.com/0xPolygon/sequence-store-proto v0.0.0-20260719224427-276104d12aff github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 github.com/BurntSushi/toml v1.4.0 github.com/JekaMas/go-grpc-net-conn v0.0.0-20220708155319-6aff21f2d13d diff --git a/go.sum b/go.sum index f0230f2cee..c936ec90e7 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,8 @@ github.com/0xPolygon/heimdall-v2 v0.7.1 h1:L50HuFky97OvSF7uHlRXoClujKTFDeMP059GN github.com/0xPolygon/heimdall-v2 v0.7.1/go.mod h1:YrGakfr3jRlcXzGrBiJPxkfOfkqodzDZHNhf+2mYG5U= github.com/0xPolygon/polyproto v0.0.8 h1:69IQ6V8CwhF9UFoJtDn7Viy6PjyRLwiZw1m5kVDdykg= github.com/0xPolygon/polyproto v0.0.8/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= +github.com/0xPolygon/sequence-store-proto v0.0.0-20260719224427-276104d12aff h1:t/QGvpSPb2OMOKiO+BE+ph9Q6dpsk2nLYtjGJJBdRaY= +github.com/0xPolygon/sequence-store-proto v0.0.0-20260719224427-276104d12aff/go.mod h1:cuEfi/jhoNrbEZ/SHOBfPT3MBlqyzbsvcivwJIOTH00= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= diff --git a/internal/cli/dumpconfig.go b/internal/cli/dumpconfig.go index 53089bb45e..665a4ca7ba 100644 --- a/internal/cli/dumpconfig.go +++ b/internal/cli/dumpconfig.go @@ -75,6 +75,7 @@ func (c *DumpconfigCommand) Run(args []string) int { userConfig.Gpo.IgnorePriceRaw = userConfig.Gpo.IgnorePrice.String() userConfig.Cache.TrieTimeoutRaw = userConfig.Cache.TrieTimeout.String() userConfig.P2P.TxArrivalWaitRaw = userConfig.P2P.TxArrivalWait.String() + userConfig.Sequencer.PollRaw = userConfig.Sequencer.Poll.String() if err := toml.NewEncoder(os.Stdout).Encode(userConfig); err != nil { c.UI.Error(err.Error()) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 99d60680c1..d96c3dfa63 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -3,7 +3,6 @@ package server import ( "crypto/ecdsa" "fmt" - "math" "math/big" "os" @@ -175,6 +174,8 @@ type Config struct { // Relay has transaction relay related settings Relay *RelayConfig `hcl:"relay,block" toml:"relay,block"` + + Sequencer *SequencerConfig `hcl:"sequencer,block" toml:"sequencer,block"` } type HistoryConfig struct { @@ -822,6 +823,53 @@ type RelayConfig struct { BlockProducerRpcEndpoints []string `hcl:"bp-rpc-endpoints,optional" toml:"bp-rpc-endpoints,optional"` } +// sequencerSettings derives the ethconfig sequencer fields: a mining node +// publishes, a non-mining node consumes the stream for preconf receipts. +// Enabling requires both service endpoints (the publisher also reads the +// tail through the consumer service when it reconciles). +func (c *Config) sequencerSettings() (string, string, string, time.Duration, error) { + if c.Sequencer == nil || !c.Sequencer.Enabled { + return "", "", "", 0, nil + } + + if c.Sequencer.PublisherEndpoint == "" { + return "", "", "", 0, fmt.Errorf("sequencer.enabled requires sequencer.publisher-endpoint") + } + + if c.Sequencer.ConsumerEndpoint == "" { + return "", "", "", 0, fmt.Errorf("sequencer.enabled requires sequencer.consumer-endpoint") + } + + role := "consumer" + if c.Sealer.Enabled { + role = "producer" + } + + return role, c.Sequencer.PublisherEndpoint, c.Sequencer.ConsumerEndpoint, c.Sequencer.Poll, nil +} + +// SequencerConfig configures the sequence store integration. Role is +// derived, not configured: a mining node publishes the block lifecycle; +// a non-mining node consumes the stream for preconf receipts. +type SequencerConfig struct { + // Enabled turns the sequence store integration on. + Enabled bool `hcl:"enabled,optional" toml:"enabled,optional"` + + // PublisherEndpoint is the gRPC address of the store's publisher + // service (the publish stream). + PublisherEndpoint string `hcl:"publisher-endpoint,optional" toml:"publisher-endpoint,optional"` + + // ConsumerEndpoint is the gRPC address of the store's consumer + // service (tail reads during reconciliation; consumers in a future + // phase). + ConsumerEndpoint string `hcl:"consumer-endpoint,optional" toml:"consumer-endpoint,optional"` + + // Poll is the producer's txpool poll cadence while a block is open + // (continuous building); zero keeps the one-shot fill at slot start. + Poll time.Duration `hcl:"-,optional" toml:"-"` + PollRaw string `hcl:"poll,optional" toml:"poll,optional"` +} + func DefaultConfig() *Config { return &Config{ Chain: "mainnet", @@ -1086,6 +1134,12 @@ func DefaultConfig() *Config { EnablePrivateTx: false, BlockProducerRpcEndpoints: []string{}, }, + Sequencer: &SequencerConfig{ + Enabled: false, + PublisherEndpoint: "", + ConsumerEndpoint: "", + Poll: 200 * time.Millisecond, + }, } } @@ -1149,6 +1203,14 @@ func (c *Config) fillTimeDurations() error { {"rpc.txsync.maxtimeout", &c.JsonRPC.TxSyncMaxTimeout, &c.JsonRPC.TxSyncMaxTimeoutRaw}, } + if c.Sequencer != nil { + tds = append(tds, struct { + path string + td *time.Duration + str *string + }{"sequencer.poll", &c.Sequencer.Poll, &c.Sequencer.PollRaw}) + } + for _, x := range tds { if x.td != nil && x.str != nil && *x.str != "" { d, err := time.ParseDuration(*x.str) @@ -1716,7 +1778,18 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* n.DisableBlindForkValidation = c.DisableBlindForkValidation n.MaxBlindForkValidationLimit = c.MaxBlindForkValidationLimit - // Set preconf / private transaction flags for relay + // Sequence store: role is derived — a mining node publishes, a + // non-mining node consumes the stream for preconf receipts. + seqRole, seqPubEndpoint, seqConsEndpoint, seqPoll, err := c.sequencerSettings() + if err != nil { + return nil, err + } + + n.SequencerRole = seqRole + n.SequencerPublisherEndpoint = seqPubEndpoint + n.SequencerConsumerEndpoint = seqConsEndpoint + n.SequencerPoll = seqPoll + n.EnablePreconfs = c.Relay.EnablePreconfs n.EnablePrivateTx = c.Relay.EnablePrivateTx n.BlockProducerRpcEndpoints = c.Relay.BlockProducerRpcEndpoints @@ -1822,7 +1895,6 @@ var ( // tries unlocking the specified account a few times. func unlockAccount(ks *keystore.KeyStore, address string, i int, passwords []string) (accounts.Account, string) { account, err := utils.MakeAddress(ks, address) - if err != nil { utils.Fatalf("Could not list accounts: %v", err) } diff --git a/internal/cli/server/config_test.go b/internal/cli/server/config_test.go index 45c9e86faa..7f4e55236c 100644 --- a/internal/cli/server/config_test.go +++ b/internal/cli/server/config_test.go @@ -626,3 +626,72 @@ func TestDeveloperModeGasParameters(t *testing.T) { assert.Contains(t, err.Error(), "miner.targetGasPercentage must be between 1-100") }) } + +func TestSequencerConfigValidation(t *testing.T) { + cases := []struct { + name string + mutate func(*Config) + wantErr bool + role string + }{ + { + name: "disabled sequencer builds", + mutate: func(c *Config) {}, + }, + // Per-field endpoint validation is the sequencerSettings table's + // job (sequencer_flags_test.go); this case pins that a settings + // error fails the whole build. + { + name: "enabled without publisher endpoint fails the build", + mutate: func(c *Config) { + c.Sequencer = &SequencerConfig{Enabled: true, ConsumerEndpoint: "localhost:9550"} + }, + wantErr: true, + }, + { + name: "enabled on a sealing node publishes", + mutate: func(c *Config) { + c.Sealer.Enabled = true + c.Sequencer = &SequencerConfig{ + Enabled: true, + PublisherEndpoint: "localhost:9550", + ConsumerEndpoint: "localhost:9551", + Poll: time.Second, + } + }, + role: "producer", + }, + { + name: "enabled on a non-sealing node consumes", + mutate: func(c *Config) { + c.Sequencer = &SequencerConfig{ + Enabled: true, + PublisherEndpoint: "localhost:9550", + ConsumerEndpoint: "localhost:9551", + Poll: time.Second, + } + }, + role: "consumer", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + config := DefaultConfig() + tc.mutate(config) + + assert.NoError(t, config.loadChain()) + + ethConfig, err := config.buildEth(nil, nil) + if tc.wantErr { + assert.Error(t, err) + assert.Nil(t, ethConfig) + + return + } + + assert.NoError(t, err) + assert.Equal(t, tc.role, ethConfig.SequencerRole) + }) + } +} diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index 07f0c01b50..15b8f222cc 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -1452,5 +1452,34 @@ func (c *Command) Flags(config *Config) *flagset.Flagset { Group: "P2P", }) + c.registerSequencerFlags(f) + return f } + +func (c *Command) registerSequencerFlags(f *flagset.Flagset) { + f.BoolFlag(&flagset.BoolFlag{ + Name: "sequencer.enabled", + Usage: "Enable the sequence store integration (a mining node publishes the block lifecycle)", + Value: &c.cliConfig.Sequencer.Enabled, + Default: c.cliConfig.Sequencer.Enabled, + }) + f.StringFlag(&flagset.StringFlag{ + Name: "sequencer.publisher-endpoint", + Usage: "Sequence store publisher service gRPC endpoint (publish stream)", + Value: &c.cliConfig.Sequencer.PublisherEndpoint, + Default: c.cliConfig.Sequencer.PublisherEndpoint, + }) + f.StringFlag(&flagset.StringFlag{ + Name: "sequencer.consumer-endpoint", + Usage: "Sequence store consumer service gRPC endpoint (tail reads during reconciliation)", + Value: &c.cliConfig.Sequencer.ConsumerEndpoint, + Default: c.cliConfig.Sequencer.ConsumerEndpoint, + }) + f.DurationFlag(&flagset.DurationFlag{ + Name: "sequencer.poll", + Usage: "Producer txpool poll cadence while a block is open (continuous building); 0 keeps the one-shot fill", + Value: &c.cliConfig.Sequencer.Poll, + Default: c.cliConfig.Sequencer.Poll, + }) +} diff --git a/internal/cli/server/sequencer_flags_test.go b/internal/cli/server/sequencer_flags_test.go new file mode 100644 index 0000000000..2d4cbe6ad2 --- /dev/null +++ b/internal/cli/server/sequencer_flags_test.go @@ -0,0 +1,104 @@ +package server + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSequencerFlags(t *testing.T) { + t.Parallel() + + var c Command + + args := []string{ + "--sequencer.enabled", + "--sequencer.publisher-endpoint", "127.0.0.1:9550", + "--sequencer.consumer-endpoint", "127.0.0.1:9551", + "--sequencer.poll", "150ms", + } + + require.NoError(t, c.extractFlags(args)) + require.True(t, c.config.Sequencer.Enabled) + require.Equal(t, "127.0.0.1:9550", c.config.Sequencer.PublisherEndpoint) + require.Equal(t, "127.0.0.1:9551", c.config.Sequencer.ConsumerEndpoint) + require.Equal(t, 150*time.Millisecond, c.config.Sequencer.Poll) +} + +func TestSequencerDefaults(t *testing.T) { + t.Parallel() + + def := DefaultConfig() + require.False(t, def.Sequencer.Enabled) + require.Equal(t, "", def.Sequencer.PublisherEndpoint) + require.Equal(t, "", def.Sequencer.ConsumerEndpoint) + require.Equal(t, 200*time.Millisecond, def.Sequencer.Poll) +} + +func TestSequencerSettingsDerivation(t *testing.T) { + t.Parallel() + + base := func(enabled, sealer bool, pubEndpoint, consEndpoint string) *Config { + c := DefaultConfig() + c.Sequencer.Enabled = enabled + c.Sequencer.PublisherEndpoint = pubEndpoint + c.Sequencer.ConsumerEndpoint = consEndpoint + c.Sealer.Enabled = sealer + + return c + } + + cases := []struct { + name string + config *Config + wantRole string + wantErr bool + }{ + {"disabled", base(false, true, "h:1", "h:2"), "", false}, + {"nil block", &Config{Sealer: DefaultConfig().Sealer}, "", false}, + {"enabled without publisher endpoint", base(true, true, "", "h:2"), "", true}, + {"enabled without consumer endpoint", base(true, true, "h:1", ""), "", true}, + {"enabled mining node", base(true, true, "h:1", "h:2"), "producer", false}, + {"enabled non-mining node", base(true, false, "h:1", "h:2"), "consumer", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + role, pubEndpoint, consEndpoint, poll, err := tc.config.sequencerSettings() + + if tc.wantErr { + require.Error(t, err) + require.Empty(t, role, "error return must not report a sequencer role") + + return + } + + require.NoError(t, err) + require.Equal(t, tc.wantRole, role) + + if role != "" { + require.Equal(t, "h:1", pubEndpoint) + require.Equal(t, "h:2", consEndpoint) + require.Equal(t, 200*time.Millisecond, poll) + } + }) + } +} + +func TestSequencerPollFromConfigFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "seq.toml") + require.NoError(t, os.WriteFile(path, []byte("[sequencer]\nenabled = true\npublisher-endpoint = \"h:1\"\nconsumer-endpoint = \"h:2\"\npoll = \"75ms\"\n"), 0o600)) + + var c Command + + require.NoError(t, c.extractFlags([]string{"--config", path})) + require.True(t, c.config.Sequencer.Enabled) + require.Equal(t, "h:1", c.config.Sequencer.PublisherEndpoint) + require.Equal(t, "h:2", c.config.Sequencer.ConsumerEndpoint) + require.Equal(t, 75*time.Millisecond, c.config.Sequencer.Poll) +} diff --git a/miner/fake_miner.go b/miner/fake_miner.go index 4db0be668b..58ffbf0460 100644 --- a/miner/fake_miner.go +++ b/miner/fake_miner.go @@ -200,10 +200,8 @@ func createMockSpanForTest(address common.Address, chainId string) borTypes.Span return span0 } -var ( - // Test chain configurations - testTxPoolConfigBor legacypool.Config -) +// Test chain configurations +var testTxPoolConfigBor legacypool.Config // TODO - Arpit, Duplicate Functions type mockBackendBor struct { @@ -222,6 +220,12 @@ func (m *mockBackendBor) BlockChain() *core.BlockChain { return m.bc } +// WhitelistedMilestone implements Backend. The mock reports no milestone, +// which leaves the finality gate to its startup grace. +func (*mockBackendBor) WhitelistedMilestone() (bool, uint64, common.Hash) { + return false, 0, common.Hash{} +} + // PeerCount implements Backend. Returns a constant; tests using // mockBackendBor don't drive the peer count. func (*mockBackendBor) PeerCount() int { diff --git a/miner/miner.go b/miner/miner.go index dbd4607542..f05aca0c9a 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -42,6 +42,90 @@ type Backend interface { BlockChain() *core.BlockChain TxPool() *txpool.TxPool PeerCount() int + + // WhitelistedMilestone is finality's view of the chain: the newest + // Heimdall milestone the node has whitelisted, or false when none has + // arrived yet. The worker's finality gate compares it against the local + // chain before a producer builds. + WhitelistedMilestone() (bool, uint64, common.Hash) +} + +// AdoptedWindow is a previous producer's unsealed block handed back by the +// sequence store: the open context the block must inherit and the ordered +// transactions to commit before consulting the txpool. +type AdoptedWindow struct { + Number uint64 + Timestamp uint64 + ParentHash common.Hash + GasLimit uint64 + BaseFee *big.Int + Txs []*types.Transaction +} + +// SealVerdict is the store's answer to "may this sealed block be broadcast". +// The zero value is SealUnknown so an implementation that never answers +// defaults to broadcasting — production must not gate on the store. +type SealVerdict int + +const ( + // SealUnknown: no verdict in budget (store slow, unreachable, or catching + // up). Broadcast anyway — the liveness override. + SealUnknown SealVerdict = iota + + // SealConfirmed: the store took our seal (or the chain already holds our + // exact block). Broadcast. + SealConfirmed + + // SealRefused: another producer's block owns this height — its seal beat + // ours in the store and its block is on our chain. Discard ours; + // broadcasting it would fork the chain against an already-decided height. + SealRefused +) + +// BlockSequencer receives block-production progress for the sequence store: +// the block context when a build starts, each transaction as it commits, and +// the sealed header the moment sealing completes. Implementations must never +// block — they are called on the worker's hot paths. +type BlockSequencer interface { + OpenBlock(number uint64, timestamp uint64, parent common.Hash, gasLimit uint64, baseFee *big.Int) + PublishTx(tx *types.Transaction) + + // SealBlock delivers the complete sealed block: the seal flush needs + // the body to complete or re-anchor the store window (design §3.5). + SealBlock(block *types.Block) + + // AdoptWindow reads the store tail for the block about to be built + // (bounded; design §3.4). A returned window is an unsealed incumbent + // window on this tip that the build must follow: the header inherits + // its context and its transactions are committed first. nil means + // build normally. + AdoptWindow(number uint64, parent common.Hash) *AdoptedWindow + + // AwaitSequenced blocks until the window being built is confirmed by + // the store, so the block about to be sealed is provably the store's + // sequence at this height. false means another producer holds the + // height and this block must not be sealed. An unreachable store + // returns true — production never waits on the store. + AwaitSequenced(timeout time.Duration, number uint64, txs []*types.Transaction) bool + + // ResyncNeeded reports that another producer holds the height this + // node is building and reached the store first. The build stops rather + // than seal beside their sequence; the next work cycle adopts it. + // Reading consumes the signal. + ResyncNeeded() bool + + // ConfirmSeal reports whether the block just handed to SealBlock may be + // broadcast, waiting up to timeout for the store's verdict. The store's + // head CAS elects exactly one seal per height; the loser learns it lost + // and withholds its block, so one height gets one broadcast. + ConfirmSeal(timeout time.Duration) SealVerdict + + // RefreshInterval is the mempool re-snapshot cadence while a block is + // open: when non-zero the worker keeps the block open until just before + // its announce time, refilling from the pool on this cadence so + // transactions are executed (and preconfirmed) as they arrive. Zero + // keeps the default one-shot fill. + RefreshInterval() time.Duration } // Config is the configuration parameters of mining. @@ -133,6 +217,12 @@ func (miner *Miner) GetWorker() *worker { return miner.worker } +// SetSequencer attaches a sequence-store publisher to the worker. Call before +// the miner starts; the worker reads the field without synchronization. +func (miner *Miner) SetSequencer(s BlockSequencer) { + miner.worker.sequencer = s +} + // update keeps track of the downloader events. Please be aware that this is a one shot type of update loop. // It's entered once and as soon as `Done` or `Failed` has been broadcasted the events are unregistered and // the loop is exited. This to prevent a major security vuln where external parties can DOS you with blocks diff --git a/miner/worker.go b/miner/worker.go index 48f7d1322a..66ca968730 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common/tracing" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/bor" + "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" @@ -111,6 +112,7 @@ const ( var ( errBlockInterruptedByNewHead = errors.New("new head arrived while building block") errBlockInterruptedByRecommit = errors.New("recommit interrupt while building block") + errRebuildForSequence = errors.New("competing producer holds this height; rebuild on the store's sequence") errBlockInterruptedByTimeout = errors.New("timeout while building block") // metrics gauge to track total and empty blocks sealed by a miner @@ -351,6 +353,11 @@ func (env *environment) txFitsSize(tx *types.Transaction) bool { return env.size+tx.Size() < params.MaxBlockSize-maxBlockSizeBufferZone-env.stateSyncReserve } +// sequenceBarrierTimeout bounds the pre-seal wait for store confirmation. +// Past it the block seals regardless: a slow or unreachable store must not +// stall block production. +const sequenceBarrierTimeout = 120 * time.Millisecond + const ( commitInterruptNone int32 = iota commitInterruptNewHead @@ -380,7 +387,6 @@ type newPayloadResult struct { receipts []*types.Receipt // Receipts collected during construction requests [][]byte // Consensus layer requests collected during block construction witness *stateless.Witness // Witness is an optional stateless proof - } // getWorkReq represents a request for getting a new sealing work with provided parameters. @@ -406,6 +412,11 @@ type worker struct { eth Backend chain *core.BlockChain + // sequencer, when set, receives block-production progress (open, per-tx, + // seal) for the sequence store. Set before the worker starts; nil when + // sequencing is disabled. All its methods are non-blocking. + sequencer BlockSequencer + prio []common.Address // A list of senders to prioritize // Feeds @@ -456,6 +467,11 @@ type worker struct { newTxs atomic.Int32 // New arrival transaction count since last sealing work submitting. syncing atomic.Bool // The indicator whether the node is still syncing. + // finalityLatched is set once a whitelisted milestone confirms this + // node's chain; startedAt anchors the startup grace before that. + finalityLatched atomic.Bool + startedAt time.Time + // newpayloadTimeout is the maximum timeout allowance for creating payload. // The default value is 2 seconds but node operator can set it to arbitrary // large value. A large timeout allowance may cause Geth to fail creating @@ -500,6 +516,7 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus worker := &worker{ config: config, chainConfig: chainConfig, + startedAt: time.Now(), engine: engine, eth: eth, chain: eth.BlockChain(), @@ -1281,6 +1298,11 @@ func (w *worker) resultLoop() { log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", hash) continue } + + if !w.sealAndGate(block) { + continue + } + // Different block could share same sealhash, deep copy here to prevent write-write conflict. var ( receipts = make([]*types.Receipt, len(task.receipts)) @@ -1490,6 +1512,13 @@ func (w *worker) commitTransaction(env *environment, tx *types.Transaction) ([]* env.tcount++ env.size += tx.Size() + // Only actual block production is sequenced: the pending-block snapshot + // and payload-building paths also commit transactions, but those never + // seal, and publishing them would poison the store chain. + if w.sequencingActive(env.header.Number) { + w.sequencer.PublishTx(tx) + } + return receipt.Logs, nil } @@ -1932,6 +1961,8 @@ type generateParams struct { builderPlanCh chan *types.Transaction // Builder sends each validated tx here before execution; prefetcher reads and warms state concurrently builderGasFreedCh chan uint64 // Builder sends (declared−actual) gas after each successful tx; prefetcher uses it to predict overflow txs planWg sync.WaitGroup // Tracks sendPlan goroutines; must reach zero before builderPlanCh is closed + adoption *AdoptedWindow // Dangling store window this build inherits and seeds + production bool // Set only by commitWork: payload-building (generateWork) must never touch the sequencer } // makeHeader creates a new block header for sealing. The caller must hold w.mu @@ -2031,6 +2062,13 @@ func (w *worker) prepareWork(genParams *generateParams, witness bool) (*environm if err != nil { return nil, err } + // Only an authorized build reads the store (Prepare rejected everyone + // else): a dangling window for this exact header is adopted rather + // than superseded — fetched at real build time so the snapshot cannot + // go stale against a still-streaming incumbent. + w.fetchAdoption(genParams, header) + // Before makeEnv: executed transactions must see the adopted context. + w.applyAdoption(genParams, header) makeHeaderDuration := time.Since(makeHeaderStart) // Could potentially happen if starting to mine in an odd state. @@ -2354,7 +2392,6 @@ func (w *worker) generateWork(params *generateParams, witness bool) *newPayloadR var block *types.Block block, work.receipts, _, err = w.engine.FinalizeAndAssemble(w.chain, work.header, work.state, &body, work.receipts) - if err != nil { return &newPayloadResult{err: err} } @@ -2397,6 +2434,33 @@ func (w *worker) commitWork(interrupt *atomic.Int32, noempty bool, timestamp int // Find the parent block for sealing task parent := w.chain.CurrentBlock() + // A producer whose chain finality has not confirmed must not build: a + // restarting node that mines before its milestone view catches up + // produces a private fork every peer will refuse — four doomed blocks + // on a devnet, reorged away with all their preconfirmations. + next := new(big.Int).Add(parent.Number, common.Big1) + if w.sequencer != nil && sequencerActive(w.chainConfig.Bor, next) && !w.finalityConfirmed() { + log.Warn("Not building: finality has not confirmed this chain", + "number", next) + + return + } + + // Contention discovered mid-build — a competing producer's window + // landing after the build-start read — ends the work cycle rather than + // retrying inside it. The outer cycle machinery is the retry: the + // winner's imported block triggers a fresh cycle immediately (the + // recommit tick covers the rest), its build-start read adopts the + // standing window, and the seal gate referees anything that persists. + w.buildAttempt(interrupt, noempty, timestamp, coinbase, parent, buildStart) +} + +// buildAttempt runs one full build cycle on a fresh parent state, returning +// true when the build was abandoned in favour of another producer's sequence +// and should be retried. +func (w *worker) buildAttempt(interrupt *atomic.Int32, noempty bool, timestamp int64, + coinbase common.Address, parent *types.Header, buildStart time.Time, +) { // Retrieve the parent state to execute on top, with separate readers for stats tracking. state, throwaway, prefetchReader, processReader, err := w.chain.StateAtWithReaders(parent.Root) if err != nil { @@ -2412,6 +2476,7 @@ func (w *worker) commitWork(interrupt *atomic.Int32, noempty bool, timestamp int processReader: processReader, prefetchedTxHashes: &sync.Map{}, preBuildDuration: time.Since(buildStart), + production: true, } var interruptPrefetch atomic.Bool @@ -2442,7 +2507,10 @@ func (w *worker) commitWork(interrupt *atomic.Int32, noempty bool, timestamp int w.buildAndCommitBlock(interrupt, noempty, &genParams, &interruptPrefetch) } -// buildAndCommitBlock prepares work, fills transactions, and commits the block for sealing. +// buildAndCommitBlock prepares work, fills transactions, and commits the block +// for sealing. A build abandoned because another producer holds this height +// simply ends the work cycle: the next one's build-start read adopts their +// window. func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genParams *generateParams, interruptPrefetch *atomic.Bool) { // Must be the first defer so the prefetcher goroutine is signaled to exit // on every return path — including the early return below when prepareWork @@ -2458,6 +2526,10 @@ func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genP prepareWorkDuration := time.Since(prepareWorkStart) prepareWorkTimer.Update(prepareWorkDuration) + // The header context is final here (engine.Prepare included): publish + // the block-open record before any transaction commits. + w.sequencerOpen(work) + // Starts accounting time after prepareWork, since it includes the wait we have on Prepare phase of Bor start := time.Now() @@ -2521,8 +2593,9 @@ func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genP // Mark the start of full-block building. Set after the optional empty pre-seal commit so that // productionElapsed for the full block does not include empty-block overhead. genParams.productionStart = time.Now() - // Fill pending transactions from the txpool into the block. - err = w.fillTransactions(interrupt, work, genParams) + // Fill pending transactions from the txpool into the block: a single + // snapshot, or repeated ones until announce time when sequencing. + err = w.fillBlock(interrupt, work, genParams) // Wait for any sendPlan goroutines to finish before closing the channel. // These goroutines do only non-blocking sends so they complete in microseconds. // Waiting here ensures no goroutine sends to a closed channel. @@ -2565,7 +2638,24 @@ func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genP // which could result in higher uncle rate. work.discard() return + + case errors.Is(err, errRebuildForSequence): + // A competing producer owns this height in the store. Committing + // this block would seal content that diverges from the sequence + // consumers already saw. Discard; the next work cycle's build-start + // read adopts their window. + log.Warn("Discarding build to follow the store's sequence", "number", work.header.Number) + work.discard() + + return + } + + if !w.sealBarrier(work) { + work.discard() + + return } + // Submit the generated block for consensus sealing. _ = w.commit(work.copy(), w.fullTaskHook, true, start, genParams) @@ -2579,6 +2669,352 @@ func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genP w.currentMu.Unlock() } +// sequencerOpen publishes the block-open record when a production build +// starts. A rebuild of the same height or a build on a new parent +// republishes — downstream re-anchoring handles both. Gated on IsRunning: +// pending-block maintenance also builds work cycles, but those never seal. +func (w *worker) sequencerOpen(work *environment) { + if !w.sequencingActive(work.header.Number) { + return + } + + w.sequencer.OpenBlock(work.header.Number.Uint64(), work.header.Time, + work.header.ParentHash, work.header.GasLimit, work.header.BaseFee) +} + +// fillBlock fills the pending block from the txpool: one snapshot on the +// stock path, or — with a sequencer attached and producing — repeated +// snapshots until announce time, so transactions are executed and streamed +// to the sequence store as they arrive instead of waiting for the next +// slot (continuous building). An adopted window seeds the block first. +func (w *worker) fillBlock(interrupt *atomic.Int32, work *environment, genParams *generateParams) error { + if genParams.adoption != nil { + w.seedAdopted(work, genParams.adoption) + + // Past the announce time already — a rebuild that inherited a + // window late. The sequenced transactions are committed, which is + // everything consumers were promised, so close here. Filling from + // the pool would push the seal further past its deadline to add + // content nobody is waiting on, and each added transaction is one + // more preconfirmation riding a block that is already late. + if time.Until(work.header.GetActualTime()) <= sealMargin { + return nil + } + } + + poll := w.sequencerPoll(work.header.Number) + if poll <= 0 { + return w.fillTransactions(interrupt, work, genParams) + } + + return w.fillUntilAnnounce(interrupt, work, genParams, poll) +} + +// adoptionSeedBudget is the minimum build time an adopted block gets: its +// inherited announce time is typically already past (the stall fallback +// fires at least a block time after the stalled open), so without a floor +// there is no room left to fill beyond the seeded window. +const adoptionSeedBudget = 500 * time.Millisecond + +// sealMargin is the tail of the block interval reserved for sealing, kept +// clear of transaction execution. +const sealMargin = 150 * time.Millisecond + +// finalityGrace is how long a starting producer waits for finality to say +// anything at all before building anyway. The grace is for ambiguity — no +// milestone yet, Heimdall still connecting — not for contradiction: a +// milestone that conflicts with the local chain refuses production for as +// long as the conflict holds. Var for tests. +var finalityGrace = 10 * time.Second + +// sealGateTimeout bounds the wait for an uncontested seal verdict before +// the block is broadcast regardless. Measured ack latency on a loaded +// devnet: p50 4.6ms, p99 50ms, p99.9 341ms — so this covers the tail with +// margin while costing a block period only when the store is genuinely +// gone. A contested seal is different: there a verdict is provably in +// progress, and the publisher extends the wait itself rather than +// broadcasting a block whose refusal is seconds away. +const sealGateTimeout = 500 * time.Millisecond + +// fetchAdoption asks the sequencer for a dangling store window matching the +// prepared header. It runs after engine.Prepare, so only a build the +// engine authorized ever reads or adopts from the store — a doomed backup +// build taking a snapshot of a still-streaming window would leave a stale +// armed offer behind for the rotation taker to seal short. +func (w *worker) fetchAdoption(genParams *generateParams, header *types.Header) { + genParams.adoption = nil + + if !genParams.production || !w.sequencingActive(header.Number) { + return + } + + genParams.adoption = w.sequencer.AdoptWindow(header.Number.Uint64(), header.ParentHash) +} + +// finalityConfirmed reports whether finality has ratified the chain this +// node holds. It latches on the first confirmation: steady-state milestones +// always trail the head, so re-checking would stall every producer forever. +// +// no milestone yet -> wait out the startup grace, then proceed +// (a Heimdall outage costs a pause, not the chain) +// milestone on chain -> confirmed, latch open +// milestone conflicts -> refuse while the conflict holds: the local +// chain is provably a fork finality rejected +func (w *worker) finalityConfirmed() bool { + if w.finalityLatched.Load() { + return true + } + + exists, number, hash := w.eth.WhitelistedMilestone() + if !exists { + return time.Since(w.startedAt) > finalityGrace + } + + if w.chain.GetCanonicalHash(number) == hash { + w.finalityLatched.Store(true) + + return true + } + + return false +} + +// sequencerActive gates the whole sequence-store integration on Rio for +// bor chains (post-Rio everywhere). Pre-Rio, every validator +// builds every height with per-succession timing rules; publishing, +// continuous filling, and adoption all interact with that regime — adopted +// timestamps are invalid for other signers, and the extra build churn +// starves out-of-turn seal delays. Below Rio the worker behaves stock. +func sequencerActive(bor *params.BorConfig, number *big.Int) bool { + return bor == nil || bor.IsRio(number) +} + +// sealBarrier reports whether the built block may be sealed, and when it may +// not, whether the worker should rebuild rather than drop the slot. +// +// The store is the arbiter of who owns a height: seal only once our window is +// confirmed there. Two blocks at one height is precisely what leaves +// consumers holding revoked preconfirmations. +func (w *worker) sealBarrier(work *environment) bool { + if !w.sequencingActive(work.header.Number) || + w.sequencer.AwaitSequenced(sequenceBarrierTimeout, + work.header.Number.Uint64(), work.txs) { + return true + } + + // Consuming the resync signal here keeps it from leaking into the next + // cycle and tells the two refusal shapes apart in the log. Either way + // the cycle ends; the next one's build-start read follows the store. + if w.sequencer.ResyncNeeded() { + log.Warn("Not sealing: the store holds records this block does not cover", + "number", work.header.Number) + } else { + log.Warn("Not sealing: another producer holds this height in the store", + "number", work.header.Number) + } + + return false +} + +// sequencingActive reports whether a mining build at this height feeds the +// sequence store: a sequencer is attached, this node is actually producing +// (IsRunning gates out the pending-block/payload snapshot builds, which +// never seal), and the height is post-Rio. The result-loop SealBlock hook +// gates without IsRunning — it only ever sees blocks this node sealed. +func (w *worker) sequencingActive(number *big.Int) bool { + return w.sequencer != nil && w.IsRunning() && sequencerActive(w.chainConfig.Bor, number) +} + +// applyAdoption rewrites the prepared header with the adopted window's open +// context — consumers pinned those fields at open and cross-check them +// against the sealed header, so a block sealed under a different context +// voids the window. Runs after engine.Prepare (which owns the +// timestamp otherwise) and before makeEnv builds the EVM context. The +// announce deadline is synthesized onto the header's local actual-time +// hint; the adopted announce time is typically already past. +// adoptionReject names the bound an offered window failed, or "" when the +// window is adoptable. The reason is worth naming: a rejected window is what +// turns one producer's height into two competing generations. +func (w *worker) adoptionReject(a *AdoptedWindow, header *types.Header) string { + parent := w.chain.GetHeaderByHash(header.ParentHash) + + switch { + case parent == nil: + return "parent unknown" + case a.ParentHash != header.ParentHash: + return "parent mismatch" + case a.Number != header.Number.Uint64(): + return "height mismatch" + } + + minTime := parent.Time + 1 + if w.chainConfig.Bor != nil { + minTime = parent.Time + w.chainConfig.Bor.CalculatePeriod(a.Number) + } + + switch { + case a.Timestamp < minTime: + return "timestamp below parent period" + case misc.VerifyGaslimit(parent.GasLimit, a.GasLimit) != nil: + return "gas limit out of bounds" + // Both producers derive the base fee from the same parent with the same + // rules; a differing value marks a window this node cannot legally seal. + case header.BaseFee == nil || a.BaseFee == nil || header.BaseFee.Cmp(a.BaseFee) != 0: + return "base fee mismatch" + } + + return "" +} + +func (w *worker) applyAdoption(genParams *generateParams, header *types.Header) { + a := genParams.adoption + if a == nil { + return + } + + genParams.adoption = nil // reinstated only when every bound holds + + if reason := w.adoptionReject(a, header); reason != "" { + // Declining is not a no-op: the build goes on to publish its own + // open at a height the store already holds a window for, which + // starts a second generation there. Both producers then preconfirm + // and only one block can seal, so every line here is a mismatch or + // a displacement waiting to happen. + log.Warn("Declined the store's window, opening our own", + "number", a.Number, "txs", len(a.Txs), "reason", reason) + + return + } + + header.Time = a.Timestamp + header.GasLimit = a.GasLimit + + deadline := time.Now().Add(adoptionSeedBudget) + if adopted := time.Unix(int64(a.Timestamp), 0); adopted.After(deadline) { + deadline = adopted + } + + header.ActualTime = deadline + genParams.adoption = a + + log.Info("Adopting sequenced window", "number", a.Number, + "txs", len(a.Txs), "deadline", deadline.Format(time.RFC3339Nano)) +} + +// seedAdopted commits the adopted window's transactions in order before any +// pool fill. The seed ignores the interrupt and runs to completion: these +// transactions are already published and preconfirmed, so cutting it short +// seals a block missing content consumers were promised, and the remainder +// resurfaces at the next height as a displaced preconfirmation. An +// inapplicable transaction (nonce consumed, balance moved) is skipped — the +// publisher's expectation matching turns that divergence into a +// partial-adoption supersede. +func (w *worker) seedAdopted(work *environment, adoption *AdoptedWindow) { + if work.gasPool == nil { + work.gasPool = new(core.GasPool).AddGas(work.header.GasLimit) + } + + applied := 0 + + for _, tx := range adoption.Txs { + work.state.SetTxContext(tx.Hash(), work.tcount) + + if _, err := w.commitTransaction(work, tx); err != nil { + log.Debug("Adopted transaction dropped", "hash", tx.Hash(), "err", err) + + continue + } + + applied++ + } + + log.Info("Seeded adopted window", "number", adoption.Number, + "applied", applied, "of", len(adoption.Txs)) +} + +// sequencerPoll returns the sequencing poll cadence, or zero when the block +// being built is not sequenced (no sequencer, not producing, or one-shot +// fill configured). +func (w *worker) sequencerPoll(number *big.Int) time.Duration { + if !w.sequencingActive(number) { + return 0 + } + + return w.sequencer.RefreshInterval() +} + +// fillUntilAnnounce fills the block immediately, then keeps re-snapshotting +// the txpool until sealMargin before the block's announce time. +// Already-committed transactions are skipped by the nonce checks inside +// commitTransactions; an interrupt aborts exactly as it does on the stock +// path. +func (w *worker) fillUntilAnnounce(interrupt *atomic.Int32, work *environment, genParams *generateParams, poll time.Duration) error { + if err := w.fillTransactions(interrupt, work, genParams); err != nil { + return err + } + + for { + remaining := time.Until(work.header.GetActualTime()) - sealMargin + if remaining <= 0 { + return nil + } + + time.Sleep(min(remaining, poll)) + + if err := w.haltFill(interrupt, work.header.Number); err != nil { + return err + } + + if err := w.fillTransactions(interrupt, work, genParams); err != nil { + return err + } + } +} + +// haltFill reports why the fill loop should stop short of the announce +// deadline, or nil to keep filling. +func (w *worker) haltFill(interrupt *atomic.Int32, number *big.Int) error { + if interrupt != nil { + if signal := interrupt.Load(); signal != commitInterruptNone { + return signalToErr(signal) + } + } + + // Another producer reached the store first for this height: abandon this + // build so the next one adopts their window and follows their ordering, + // instead of sealing a block that diverges from it. + if w.sequencingActive(number) && w.sequencer.ResyncNeeded() { + return errRebuildForSequence + } + + return nil +} + +// sealAndGate publishes the seal record the moment the sealed block exists — +// ahead of the chain write and the announcement — so stream consumers can +// close the block, and reports whether the block may be broadcast. +// +// The store's head CAS elects one seal per height; the gate is where the +// loser learns it lost and withholds its block, so a height gets one +// broadcast instead of a fork. No verdict inside the budget broadcasts +// anyway — production never waits on the store. +func (w *worker) sealAndGate(block *types.Block) bool { + if w.sequencer == nil || !sequencerActive(w.chainConfig.Bor, block.Number()) { + return true + } + + w.sequencer.SealBlock(block) + + if w.sequencer.ConfirmSeal(sealGateTimeout) == SealRefused { + log.Warn("Discarding sealed block: another producer's block owns this height", + "number", block.Number(), "hash", block.Hash()) + + return false + } + + return true +} + // runPrefetcher owns the lifecycle of the unified prefetcher stream for one block. // It starts a single long-lived worker pool (via PrefetchStream), runs the idle tx // provider until the builder flips, executes the idle→builder handoff, and then diff --git a/miner/worker_finality_test.go b/miner/worker_finality_test.go new file mode 100644 index 0000000000..b3e8163eec --- /dev/null +++ b/miner/worker_finality_test.go @@ -0,0 +1,135 @@ +package miner + +import ( + "testing" + "time" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/ethereum/go-ethereum/common" +) + +// These share the package-level finalityGrace, so they must not run in +// parallel with each other. + +// A restarting producer must not build until finality has spoken about the +// chain it holds. The failure this prevents: a same-key node came back from +// a restart, synced a head, and mined a four-block private fork before its +// milestone view caught up — every peer refused the whole lineage with a +// whitelist mismatch, and the blocks were reorged away. +func TestFinalityGateWaitsForAMilestoneAfterRestart(t *testing.T) { + w, b, _ := newSequencerTestWorker(t) + + if w.finalityConfirmed() { + t.Fatal("a fresh producer with no milestone built immediately: this " + + "is the window where a restart mines onto a fork finality has " + + "already rejected") + } + + // A milestone naming our own chain confirms the fork. + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + + if !w.finalityConfirmed() { + t.Fatal("a milestone matching our canonical chain must release the gate") + } +} + +// Confirmation latches: steady-state production must not re-check, or a +// producer would stall every time the milestone view lagged its own head. +func TestFinalityGateLatchesOnceConfirmed(t *testing.T) { + w, b, _ := newSequencerTestWorker(t) + + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + + if !w.finalityConfirmed() { + t.Fatal("gate did not open on a matching milestone") + } + + // A milestone that no longer matches (our head has moved on, as it + // always does) must not re-close the gate. + b.setMilestone(head.Number.Uint64()+5, common.Hash{0x99}) + + if !w.finalityConfirmed() { + t.Fatal("gate re-closed after confirming: a producer would stall " + + "whenever finality lagged its own head, which is always") + } +} + +// A milestone that names a block we do not have is proof we are on a +// rejected fork. Refuse for as long as that holds — the grace window is for +// ambiguity, not for contradiction. +func TestFinalityGateRefusesAConflictingChainPastGrace(t *testing.T) { + w, _, _ := newSequencerTestWorker(t) + + restore := finalityGrace + finalityGrace = 0 // grace already expired + t.Cleanup(func() { finalityGrace = restore }) + + w.eth.(*testWorkerBackend).setMilestone(1, common.Hash{0xbe, 0xef}) + + if w.finalityConfirmed() { + t.Fatal("built on a chain that provably conflicts with the " + + "whitelisted milestone: every block extends ground that is " + + "already reorged away") + } +} + +// Liveness: with no milestone at all — a fresh chain, or Heimdall down — the +// grace expires and production proceeds. A producer that never hears from +// Heimdall must pause, not halt. +func TestFinalityGateOpensAfterGraceWithoutMilestone(t *testing.T) { + w, _, _ := newSequencerTestWorker(t) + + restore := finalityGrace + finalityGrace = 0 + t.Cleanup(func() { finalityGrace = restore }) + + if !w.finalityConfirmed() { + t.Fatal("a producer with no milestone never started: a Heimdall " + + "outage must cost a pause, not the chain") + } +} + +// The gate must actually stop production, not merely report. Driven through +// the running worker's own loops — commitWork is the single path every build +// takes, and this asserts the wiring, not the predicate. +func TestFinalityGateBlocksProduction(t *testing.T) { + w, b, rec := newSequencerTestWorker(t) + + b.txPool.Add([]*types.Transaction{b.newRandomTxWithNonce(false, 0)}, true) + + // No milestone, grace running: the gate is shut. + w.start() + defer w.stop() + + time.Sleep(1500 * time.Millisecond) + + if opens, _, _, _ := rec.snapshot(); len(opens) != 0 { + t.Fatalf("produced %d blocks while finality had not confirmed our "+ + "chain: this is the restart window that mined a doomed fork", + len(opens)) + } + + // Confirm our chain, then retrigger. In production the veblop fallback + // does this every block period for a stalled producer; the clique test + // engine skips that path, so poke startCh directly. + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + w.start() + + deadline := time.Now().Add(10 * time.Second) + for { + if opens, _, _, _ := rec.snapshot(); len(opens) > 0 { + return + } + + if time.Now().After(deadline) { + t.Fatal("no production after finality confirmed the chain: the " + + "gate never reopens and the producer is stuck") + } + + time.Sleep(50 * time.Millisecond) + } +} diff --git a/miner/worker_sequencer_test.go b/miner/worker_sequencer_test.go new file mode 100644 index 0000000000..c87f785ec2 --- /dev/null +++ b/miner/worker_sequencer_test.go @@ -0,0 +1,814 @@ +package miner + +import ( + "errors" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/clique" + "github.com/ethereum/go-ethereum/consensus/misc/eip1559" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" +) + +// recordingSequencer captures the worker's sequencer callbacks. +type recordingSequencer struct { + mu sync.Mutex + refresh time.Duration + opens []uint64 + seals []uint64 + txs int + order []byte + + adoptable *AdoptedWindow // handed out once when the queried block matches + adopted int + verdict SealVerdict + resyncN int // number of rebuild signals a test wants served + contested bool // when set, AwaitSequenced reports the window contested +} + +func (r *recordingSequencer) OpenBlock(number uint64, _ uint64, _ common.Hash, _ uint64, _ *big.Int) { + r.mu.Lock() + defer r.mu.Unlock() + + r.opens = append(r.opens, number) + r.order = append(r.order, 'o') +} + +func (r *recordingSequencer) PublishTx(*types.Transaction) { + r.mu.Lock() + defer r.mu.Unlock() + + r.txs++ + r.order = append(r.order, 't') +} + +func (r *recordingSequencer) SealBlock(block *types.Block) { + r.mu.Lock() + defer r.mu.Unlock() + + r.seals = append(r.seals, block.NumberU64()) + r.order = append(r.order, 's') +} + +func (r *recordingSequencer) AdoptWindow(number uint64, parent common.Hash) *AdoptedWindow { + r.mu.Lock() + defer r.mu.Unlock() + + a := r.adoptable + if a == nil || a.Number != number || a.ParentHash != parent { + return nil + } + + r.adoptable = nil + r.adopted++ + + return a +} + +func (r *recordingSequencer) AwaitSequenced(time.Duration, uint64, []*types.Transaction) bool { + r.mu.Lock() + defer r.mu.Unlock() + + return !r.contested +} + +// verdict is what ConfirmSeal reports; the zero value (SealUnknown) lets +// every existing test proceed to broadcast unchanged. +func (r *recordingSequencer) ConfirmSeal(time.Duration) SealVerdict { + r.mu.Lock() + defer r.mu.Unlock() + + return r.verdict +} + +func (r *recordingSequencer) ResyncNeeded() bool { + r.mu.Lock() + defer r.mu.Unlock() + + if r.resyncN <= 0 { + return false + } + + r.resyncN-- + + return true +} + +func (r *recordingSequencer) RefreshInterval() time.Duration { + return r.refresh +} + +func (r *recordingSequencer) snapshot() (opens, seals []uint64, txs int, order []byte) { + r.mu.Lock() + defer r.mu.Unlock() + + return append([]uint64(nil), r.opens...), append([]uint64(nil), r.seals...), r.txs, append([]byte(nil), r.order...) +} + +func newSequencerTestWorker(t *testing.T) (*worker, *testWorkerBackend, *recordingSequencer) { + t.Helper() + + db := rawdb.NewMemoryDatabase() + config := *params.AllCliqueProtocolChanges + config.Clique = ¶ms.CliqueConfig{Period: 1, Epoch: 30000} + + // Rio from genesis: adoption is Rio-gated (sequencerActive), and the + // template config carries a Bor section with no Rio block scheduled. + borCfg := *config.Bor + borCfg.RioBlock = big.NewInt(0) + config.Bor = &borCfg + + engine := clique.New(config.Clique, db) + + w, b, _ := newTestWorker(t, DefaultTestConfig(), &config, engine, db, false, 0) + t.Cleanup(w.close) + + rec := &recordingSequencer{} + w.sequencer = rec + + return w, b, rec +} + +// The worker publishes the full lifecycle — open before transactions, +// transactions before the seal — for produced blocks. +func TestSequencerLifecycleHooks(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + + // Finality has ratified this chain, as it has in any steady state. + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + + w.start() + defer w.stop() + + b.txPool.Add([]*types.Transaction{b.newRandomTx(false)}, false) + + // The first sealed block may predate the transaction; poll until every + // hook kind has fired. + deadline := time.Now().Add(10 * time.Second) + + var ( + opens, seals []uint64 + txs int + order []byte + ) + + for { + opens, seals, txs, order = rec.snapshot() + if len(opens) > 0 && len(seals) > 0 && txs > 0 { + break + } + + if time.Now().After(deadline) { + t.Fatalf("hooks missed: opens=%d seals=%d txs=%d", len(opens), len(seals), txs) + } + + time.Sleep(50 * time.Millisecond) + } + + firstOpen, firstSeal := -1, -1 + + for i, k := range order { + if k == 'o' && firstOpen < 0 { + firstOpen = i + } + + if k == 's' && firstSeal < 0 { + firstSeal = i + } + } + + if firstSeal < firstOpen { + t.Fatalf("seal published before any open: order %q", order) + } +} + +// fillUntilAnnounce fills immediately, then keeps polling the pool until +// the announce margin, so a transaction arriving mid-block is committed +// without waiting for the next slot. +func TestFillUntilAnnounceCommitsLateTx(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + // The harness's preloaded txs are signed for another chain id and never + // enter this pool; drive the test with the backend's own tx builder, + // with explicit nonces. (Committed txs stay "pending" in the pool until + // import, so tcount is the observable throughout.) + b.txPool.Add([]*types.Transaction{b.newRandomTxWithNonce(false, 0)}, true) + waitFor(t, 5*time.Second, func() bool { + return countPendingTransactions(b) >= 1 + }) + + // The loop runs until ~announce time; give it a window. + work.header.Time = uint64(time.Now().Unix()) + 3 + + go func() { + time.Sleep(300 * time.Millisecond) + b.txPool.Add([]*types.Transaction{b.newRandomTxWithNonce(false, 1)}, true) + }() + + if err := w.fillUntilAnnounce(nil, work, genParams, 100*time.Millisecond); err != nil { + t.Fatalf("fill until announce: %v", err) + } + + // The initial fill commits the first tx; a later poll must pick up the + // late one on top. + if work.tcount < 2 { + t.Fatalf("late transaction not committed: tcount %d, want >= 2", work.tcount) + } +} + +func waitFor(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + + deadline := time.Now().Add(timeout) + + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition never held") + } + + time.Sleep(20 * time.Millisecond) + } +} + +// With the announce time already past, the poll loop exits right after the +// initial fill — no lingering until a stale deadline. +func TestFillUntilAnnounceLateBlockExitsImmediately(t *testing.T) { + t.Parallel() + + w, _, _ := newSequencerTestWorker(t) + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + work.header.Time = uint64(time.Now().Unix()) - 10 + + start := time.Now() + + if err := w.fillUntilAnnounce(nil, work, genParams, 100*time.Millisecond); err != nil { + t.Fatalf("fill until announce: %v", err) + } + + if time.Since(start) > time.Second { + t.Fatal("loop did not exit immediately for a late block") + } +} + +// A mid-loop interrupt aborts exactly as it does on the stock path. +func TestFillUntilAnnounceInterrupt(t *testing.T) { + t.Parallel() + + w, _, _ := newSequencerTestWorker(t) + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + work.header.Time = uint64(time.Now().Unix()) + 5 + + interrupt := new(atomic.Int32) + + go func() { + time.Sleep(200 * time.Millisecond) + interrupt.Store(commitInterruptNewHead) + }() + + if err := w.fillUntilAnnounce(interrupt, work, genParams, 50*time.Millisecond); !errors.Is(err, errBlockInterruptedByNewHead) { + t.Fatalf("err = %v, want interrupt", err) + } +} + +// The poll gate: zero unless a sequencer is attached, the worker is +// producing, and a poll cadence is configured. +func TestSequencerPollGate(t *testing.T) { + t.Parallel() + + w, _, rec := newSequencerTestWorker(t) + + // Attached but not producing. + rec.refresh = 100 * time.Millisecond + if got := w.sequencerPoll(big.NewInt(1)); got != 0 { + t.Fatalf("poll while not producing = %v", got) + } + + // No sequencer at all. + w.sequencer = nil + if got := w.sequencerPoll(big.NewInt(1)); got != 0 { + t.Fatalf("poll without sequencer = %v", got) + } +} + +// An adoptable window is applied verbatim: the header inherits the adopted +// context and the window's transactions are committed before the pool's. +func TestAdoptedWindowSeedsBlock(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + adopted := &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 2, + ParentHash: parent.Hash(), + GasLimit: parent.GasLimit, + BaseFee: eip1559.CalcBaseFee(b.chain.Config(), parent), + Txs: []*types.Transaction{ + b.newRandomTxWithNonce(false, 0), + b.newRandomTxWithNonce(false, 1), + }, + } + + rec.mu.Lock() + rec.adoptable = adopted + rec.mu.Unlock() + + // Finality has ratified this chain, as it has in any steady state. + b.setMilestone(parent.Number.Uint64(), parent.Hash()) + + // The empty pre-seal shortcut would race the seeded block to the same + // height; production sequencing targets post-Rio, where it is skipped. + w.noempty.Store(true) + + w.start() + defer w.stop() + + var block *types.Block + + waitFor(t, 10*time.Second, func() bool { + rec.mu.Lock() + adoptions := rec.adopted + rec.mu.Unlock() + + block = b.chain.GetBlockByNumber(adopted.Number) + + return adoptions == 1 && block != nil && len(block.Transactions()) >= 2 + }) + + if block.Time() != adopted.Timestamp { + t.Fatalf("sealed time %d, want adopted %d", block.Time(), adopted.Timestamp) + } + + if block.GasLimit() != adopted.GasLimit { + t.Fatalf("sealed gas limit %d, want adopted %d", block.GasLimit(), adopted.GasLimit) + } + + for i, tx := range adopted.Txs { + if block.Transactions()[i].Hash() != tx.Hash() { + t.Fatalf("tx %d = %s, want adopted %s", i, block.Transactions()[i].Hash(), tx.Hash()) + } + } +} + +// A window for a block that is not the one being built is left alone. +func TestAdoptionSkipsMismatchedWindow(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + rec.mu.Lock() + rec.adoptable = &AdoptedWindow{ + Number: parent.Number.Uint64() + 5, // not the next block + Timestamp: parent.Time + 2, + ParentHash: parent.Hash(), + GasLimit: parent.GasLimit, + BaseFee: eip1559.CalcBaseFee(b.chain.Config(), parent), + } + rec.mu.Unlock() + + // Finality has ratified this chain, as it has in any steady state. + b.setMilestone(parent.Number.Uint64(), parent.Hash()) + + w.start() + defer w.stop() + + waitFor(t, 10*time.Second, func() bool { + return b.chain.GetBlockByNumber(parent.Number.Uint64()+1) != nil + }) + + rec.mu.Lock() + defer rec.mu.Unlock() + + if rec.adopted != 0 { + t.Fatal("mismatched window must not be adopted") + } +} + +// applyAdoption applies only a window that matches the build exactly and +// passes every consensus bound; anything else drops the adoption. +func TestApplyAdoptionValidation(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + base := func() *AdoptedWindow { + return &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 2, + ParentHash: parent.Hash(), + GasLimit: parent.GasLimit, + BaseFee: eip1559.CalcBaseFee(b.chain.Config(), parent), + Txs: []*types.Transaction{b.newRandomTxWithNonce(false, 0)}, + } + } + + cases := []struct { + name string + mutate func(*AdoptedWindow) + wantAdopt bool + }{ + {name: "valid", mutate: func(*AdoptedWindow) {}, wantAdopt: true}, + {name: "wrong number", mutate: func(a *AdoptedWindow) { a.Number += 3 }}, + {name: "wrong parent", mutate: func(a *AdoptedWindow) { a.ParentHash = common.Hash{0x99} }}, + {name: "timestamp at parent", mutate: func(a *AdoptedWindow) { a.Timestamp = parent.Time }}, + {name: "gas limit out of bound", mutate: func(a *AdoptedWindow) { a.GasLimit = parent.GasLimit * 3 }}, + {name: "base fee mismatch", mutate: func(a *AdoptedWindow) { a.BaseFee = big.NewInt(1) }}, + {name: "nil base fee", mutate: func(a *AdoptedWindow) { a.BaseFee = nil }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + genParams := &generateParams{coinbase: testBankAddress} + + header, _, err := w.makeHeader(genParams, false) + if err != nil { + t.Fatalf("makeHeader: %v", err) + } + + a := base() + tc.mutate(a) + genParams.adoption = a + + w.applyAdoption(genParams, header) + + if got := genParams.adoption != nil; got != tc.wantAdopt { + t.Fatalf("adoption kept = %v, want %v", got, tc.wantAdopt) + } + + if tc.wantAdopt { + if header.Time != a.Timestamp || header.GasLimit != a.GasLimit { + t.Fatalf("header not rewritten: time %d gas %d", header.Time, header.GasLimit) + } + + if header.ActualTime.Before(time.Now().Add(adoptionSeedBudget / 2)) { + t.Fatal("announce deadline missing the seed budget") + } + } + }) + } +} + +// fetchAdoption queries for exactly the prepared header's height and +// parent, and only while running — it sits after engine.Prepare, so only +// an authorized build ever reads the store. +func TestFetchAdoptionUsesPreparedHeader(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + header := &types.Header{ + Number: new(big.Int).Add(parent.Number, common.Big1), + ParentHash: parent.Hash(), + } + + rec.mu.Lock() + rec.adoptable = &AdoptedWindow{ + Number: header.Number.Uint64(), + ParentHash: header.ParentHash, + } + rec.mu.Unlock() + + // Not running: no query at all. + genParams := &generateParams{production: true} + w.fetchAdoption(genParams, header) + + if genParams.adoption != nil { + t.Fatal("fetch while not producing") + } + + w.start() + defer w.stop() + + // Payload building (production unset) must never touch the sequencer. + genParams = &generateParams{} + w.fetchAdoption(genParams, header) + + if genParams.adoption != nil { + t.Fatal("payload build queried the sequencer") + } + + genParams = &generateParams{production: true} + w.fetchAdoption(genParams, header) + + if genParams.adoption == nil { + t.Fatal("prepared header did not resolve the window") + } +} + +// An adopted window is already published and preconfirmed, so the seed +// commits all of it. A partial seed would leave the block short of what +// consumers were promised and displace the remainder to the next height. +func TestSeedAdoptedCommitsWholeWindow(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + window := &AdoptedWindow{Txs: []*types.Transaction{ + b.newRandomTxWithNonce(false, 0), + b.newRandomTxWithNonce(false, 1), + b.newRandomTxWithNonce(false, 2), + }} + + w.seedAdopted(work, window) + + if work.tcount != len(window.Txs) { + t.Fatalf("seed committed %d of %d adopted txs", work.tcount, len(window.Txs)) + } +} + +// Adoption is Rio-gated on bor chains: pre-Rio, every validator builds +// every height and an inherited timestamp is invalid for any other signer. +func TestSequencerRioGate(t *testing.T) { + t.Parallel() + + if sequencerActive(¶ms.BorConfig{RioBlock: big.NewInt(1_000_000)}, big.NewInt(1)) { + t.Fatal("pre-Rio bor chain must not sequence") + } + + if !sequencerActive(¶ms.BorConfig{RioBlock: big.NewInt(0)}, big.NewInt(1)) { + t.Fatal("post-Rio bor chain must sequence") + } + + if !sequencerActive(nil, big.NewInt(1)) { + t.Fatal("non-bor chain (tests) must allow sequencing") + } +} + +// Without a Bor config the adoption timestamp floor falls back to exactly +// parent.Time+1: a window on the floor is accepted (a mutated floor — +// off by one in either direction, or underflowed — rejects it). +func TestApplyAdoptionMinTimeWithoutBor(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + genParams := &generateParams{coinbase: testBankAddress} + + header, _, err := w.makeHeader(genParams, false) + if err != nil { + t.Fatalf("makeHeader: %v", err) + } + + saved := w.chainConfig + cfg := *saved + cfg.Bor = nil + w.chainConfig = &cfg + + defer func() { w.chainConfig = saved }() + + // Base fee copied from the prepared header (the value applyAdoption + // compares against): the timestamp floor is then the only bound that + // can reject this window. + genParams.adoption = &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 1, // exactly on the fallback floor + ParentHash: parent.Hash(), + GasLimit: header.GasLimit, + BaseFee: new(big.Int).Set(header.BaseFee), + } + + w.applyAdoption(genParams, header) + + if genParams.adoption == nil { + t.Fatal("window exactly on the parent.Time+1 floor must be accepted") + } +} + +// sequencerPoll gates on the full sequencing predicate and forwards the +// publisher's cadence: zero when the node is not producing, the configured +// interval when it is. +func TestSequencerPollGating(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + rec.refresh = 75 * time.Millisecond + + n := new(big.Int).Add(b.chain.CurrentBlock().Number, big.NewInt(1)) + + if got := w.sequencerPoll(n); got != 0 { + t.Fatalf("poll while stopped = %v, want 0", got) + } + + w.start() + + if got := w.sequencerPoll(n); got != 75*time.Millisecond { + t.Fatalf("poll while running = %v, want 75ms", got) + } +} + +// A zero poll is the one-shot fill: fillBlock must return immediately, not +// hold the block open until the announce deadline. +func TestFillBlockZeroPollIsOneShot(t *testing.T) { + t.Parallel() + + w, _, rec := newSequencerTestWorker(t) + rec.refresh = 0 + w.start() + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + work.header.ActualTime = time.Now().Add(2 * time.Second) + + start := time.Now() + + if err := w.fillBlock(nil, work, genParams); err != nil { + t.Fatalf("fillBlock: %v", err) + } + + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("zero poll held the block open %v — one-shot fill expected", elapsed) + } +} + +// With a Bor config the adoption timestamp floor is parent.Time plus the +// configured block period, not the +1 fallback: a window between the two +// floors is rejected. +func TestApplyAdoptionMinTimeBorPeriod(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + genParams := &generateParams{coinbase: testBankAddress} + + header, _, err := w.makeHeader(genParams, false) + if err != nil { + t.Fatalf("makeHeader: %v", err) + } + + saved := w.chainConfig + cfg := *saved + borCfg := *saved.Bor + borCfg.Period = map[string]uint64{"0": 2} + cfg.Bor = &borCfg + w.chainConfig = &cfg + + defer func() { w.chainConfig = saved }() + + genParams.adoption = &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 1, // above +1 fallback, below the period-2 floor + ParentHash: parent.Hash(), + GasLimit: header.GasLimit, + BaseFee: new(big.Int).Set(header.BaseFee), + } + + w.applyAdoption(genParams, header) + + if genParams.adoption != nil { + t.Fatal("window below the Bor period floor must be rejected") + } +} + +// A competing producer at our height aborts the fill so the next work cycle +// adopts their window: the build must not seal content that diverges from +// the sequence consumers already saw. +func TestFillBlockRestartsOnResync(t *testing.T) { + t.Parallel() + + w, _, rec := newSequencerTestWorker(t) + rec.refresh = 20 * time.Millisecond + w.start() + + // The worker's own production loop also consumes the signal; arm + // enough for this explicit build to observe one. + rec.mu.Lock() + rec.resyncN = 64 + rec.mu.Unlock() + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + work.header.ActualTime = time.Now().Add(3 * time.Second) + + if err := w.fillBlock(nil, work, genParams); !errors.Is(err, errRebuildForSequence) { + t.Fatalf("fillBlock err = %v, want errRebuildForSequence", err) + } + + // Each read consumes one signal: with none armed the build proceeds. + rec.mu.Lock() + rec.resyncN = 0 + rec.mu.Unlock() + + if rec.ResyncNeeded() { + t.Fatal("resync signal must be consumed, not sticky") + } +} + +// A contested window must not clear the barrier, so the seal path declines +// to commit it. The barrier's own semantics are covered in the sequencer +// package; this pins the worker-side gate. +func TestContestedWindowBlocksSeal(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + w.start() + + n := new(big.Int).Add(b.chain.CurrentBlock().Number, big.NewInt(1)) + + if !w.sequencingActive(n) { + t.Fatal("sequencing must be active for the gate to apply") + } + + if !w.sequencer.AwaitSequenced(time.Second, 1, nil) { + t.Fatal("an uncontested window should clear") + } + + rec.mu.Lock() + rec.contested = true + rec.mu.Unlock() + + if w.sequencer.AwaitSequenced(time.Second, 1, nil) { + t.Fatal("a contested window must fail the gate that precedes commit") + } +} + +// A refused seal never reaches the chain: the store elected another +// producer's block for the height, and broadcasting ours would fork an +// already-decided height. The worker keeps sealing (each attempt is +// discarded), so refusal costs blocks, never liveness machinery. +func TestSealGateRefusalStopsBroadcast(t *testing.T) { + w, b, rec := newSequencerTestWorker(t) + + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + + rec.mu.Lock() + rec.verdict = SealRefused + rec.mu.Unlock() + + w.start() + defer w.stop() + + deadline := time.Now().Add(5 * time.Second) + for { + _, seals, _, _ := rec.snapshot() + if len(seals) > 0 { + break + } + + if time.Now().After(deadline) { + t.Fatal("worker never sealed") + } + + time.Sleep(20 * time.Millisecond) + } + + if got := b.chain.CurrentBlock().Number.Uint64(); got != 0 { + t.Fatalf("chain advanced to %d on refused seals: a refused block "+ + "was written and broadcast", got) + } +} diff --git a/miner/worker_test.go b/miner/worker_test.go index d37f769fe6..71d084b33c 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -61,12 +61,12 @@ import ( borSpan "github.com/ethereum/go-ethereum/consensus/bor/heimdall/span" ) -// TestPendingStateNotStaleForNonValidator verifies that a Bor node whose signer -// is NOT in the active validator set still keeps its pending snapshot fresh. -// Regression test: previously, Prepare() returned UnauthorizedSignerError for -// non-validators, which caused prepareWork to fail and the snapshot to never -// update, leading to stale trie errors on "pending" RPC queries. -func TestPendingStateNotStaleForNonValidator(t *testing.T) { +// TestNonValidatorDoesNotBuild verifies that a Bor node whose signer is NOT +// in the active validator set (post-Rio: not the span's producer) builds no +// candidate at all: Prepare rejects the signer, so nothing is mined and — +// with a sequencer attached — nothing is ever published. Nodes without a +// signer (RPC) are unaffected by this check. +func TestNonValidatorDoesNotBuild(t *testing.T) { chainConfig := *params.BorUnittestChainConfig engine, ctrl := getFakeBorFromConfig(t, &chainConfig) @@ -90,26 +90,14 @@ func TestPendingStateNotStaleForNonValidator(t *testing.T) { }) w.setEtherbase(nonValidatorAddr) - // Start the worker. It will call commitWork which calls Prepare. - // Before the fix: Prepare fails with UnauthorizedSignerError, snapshot - // is never set, pending() returns nil. - // After the fix: Prepare succeeds (defaults succession to 0), snapshot - // is updated, pending() returns valid state. + // Start the worker: commitWork calls Prepare, which rejects the + // unauthorized signer, so no pending snapshot is ever produced. w.start() - // Give the worker time to process the start event and run commitWork. time.Sleep(1 * time.Second) - pendingBlock, _, pendingState := w.pending() - require.NotNil(t, pendingBlock, "pending block should not be nil for non-validator node") - require.NotNil(t, pendingState, "pending state should not be nil for non-validator node") - - // The pending state must be readable without errors (not a stale trie). - balance := pendingState.GetBalance(testBankAddress) - require.False(t, balance.IsZero(), - "pending state balance for funded account should not be zero (would indicate stale trie)") - require.NoError(t, pendingState.Error(), - "pending state should have no database errors") + pendingBlock, _, _ := w.pending() + require.Nil(t, pendingBlock, "a signer outside the validator set must not build") } // nolint : paralleltest @@ -164,9 +152,7 @@ func testGenerateBlockAndImport(t *testing.T, isClique bool, isBor bool) { // Start mining! w.start() - var ( - err error - ) + var err error // []*types.Transaction{tx} var i uint64 for i = 0; i < 5; i++ { @@ -279,6 +265,11 @@ func DefaultTestConfig() *Config { // testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. type testWorkerBackend struct { + msMu sync.Mutex + msExists bool + msNumber uint64 + msHash common.Hash + db ethdb.Database txPool *txpool.TxPool chain *core.BlockChain @@ -286,7 +277,7 @@ type testWorkerBackend struct { } func newTestWorkerBackend(t TensingObject, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database) *testWorkerBackend { - var gspec = &core.Genesis{ + gspec := &core.Genesis{ Config: chainConfig, Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, } @@ -324,6 +315,22 @@ func newTestWorkerBackend(t TensingObject, chainConfig *params.ChainConfig, engi return b } +// setMilestone drives the finality gate in tests: the whitelisted milestone +// the backend reports to the worker. +func (b *testWorkerBackend) setMilestone(number uint64, hash common.Hash) { + b.msMu.Lock() + defer b.msMu.Unlock() + + b.msExists, b.msNumber, b.msHash = true, number, hash +} + +func (b *testWorkerBackend) WhitelistedMilestone() (bool, uint64, common.Hash) { + b.msMu.Lock() + defer b.msMu.Unlock() + + return b.msExists, b.msNumber, b.msHash +} + func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } func (b *testWorkerBackend) PeerCount() int { @@ -546,6 +553,7 @@ func TestEmptyWorkEthash(t *testing.T) { t.Skip() testEmptyWork(t, ethashChainConfig, ethash.NewFaker()) } + func TestEmptyWorkClique(t *testing.T) { t.Skip() testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) @@ -750,7 +758,7 @@ func testGetSealingWork(t *testing.T, chainConfig *params.ChainConfig, engine co t.Errorf("Mismatched block number, want %d got %d", number, block.NumberU64()) } } - var cases = []struct { + cases := []struct { parent common.Hash coinbase common.Address random common.Hash diff --git a/miner/worker_twin_test.go b/miner/worker_twin_test.go new file mode 100644 index 0000000000..4404a9f11b --- /dev/null +++ b/miner/worker_twin_test.go @@ -0,0 +1,113 @@ +package miner + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/consensus/clique" + "github.com/ethereum/go-ethereum/consensus/misc/eip1559" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +// The whole convergence argument rests on this: two producers holding one key +// and sealing the same header must emit the same signature, or "they derive +// the same block" is false and contention always produces two blocks. +// secp256k1 signing uses RFC 6979 deterministic nonces, on both the cgo and +// nocgo paths — asserted here rather than assumed. +func TestSigningIsDeterministic(t *testing.T) { + t.Parallel() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("key: %v", err) + } + + digest := crypto.Keccak256([]byte("the same header, signed twice")) + + first, err := crypto.Sign(digest, key) + if err != nil { + t.Fatalf("sign: %v", err) + } + + second, err := crypto.Sign(digest, key) + if err != nil { + t.Fatalf("sign again: %v", err) + } + + if !bytes.Equal(first, second) { + t.Fatal("the same key signing the same digest produced two different " + + "signatures: two producers can never derive one block, so " + + "contention must be resolved by refusal rather than agreement") + } +} + +// Header identity: two independent workers on the same config, given the same +// adopted window, must prepare byte-identical headers. This is the other half +// of the convergence claim — agreeing on transactions is worthless if the +// headers differ, because the block hashes then differ anyway. +// +// The seal hash is the exact thing consensus signs, so comparing it is +// stricter than comparing fields one by one. +func TestTwinWorkersPrepareIdenticalHeaders(t *testing.T) { + t.Parallel() + + w1, _, _ := newSequencerTestWorker(t) + w2, _, _ := newSequencerTestWorker(t) + + parent := w1.chain.CurrentBlock() + if got := w2.chain.CurrentBlock().Hash(); got != parent.Hash() { + t.Fatalf("the two workers start from different genesis blocks (%s vs %s)", + parent.Hash(), got) + } + + // The same inherited open context on both sides, as adoption supplies it. + // The base fee is derived from the parent by rules both producers share. + window := func() *AdoptedWindow { + return &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 2, + ParentHash: parent.Hash(), + GasLimit: parent.GasLimit, + BaseFee: eip1559.CalcBaseFee(w1.chain.Config(), parent), + } + } + + h1 := prepareAdoptedHeader(t, w1, window()) + h2 := prepareAdoptedHeader(t, w2, window()) + + if clique.SealHash(h1) != clique.SealHash(h2) { + t.Fatalf("two producers derived different headers from one open "+ + "context: seal hashes %s vs %s. Agreeing on content cannot make "+ + "them one block if the headers differ.", + clique.SealHash(h1), clique.SealHash(h2)) + } +} + +// prepareAdoptedHeader drives the real adoption path — makeHeader, then +// applyAdoption — and fails if the window is rejected, so this cannot quietly +// degrade into comparing two ordinary headers. +func prepareAdoptedHeader(t *testing.T, w *worker, window *AdoptedWindow) *types.Header { + t.Helper() + + genParams := &generateParams{coinbase: testBankAddress, adoption: window} + + header, _, err := w.makeHeader(genParams, false) + if err != nil { + t.Fatalf("makeHeader: %v", err) + } + + w.applyAdoption(genParams, header) + + if genParams.adoption == nil { + t.Fatal("the window was rejected, so this header never inherited the " + + "open context and proves nothing about two producers agreeing") + } + + if header.Time != window.Timestamp { + t.Fatalf("header did not inherit the adopted timestamp: %d vs %d", + header.Time, window.Timestamp) + } + + return header +}