diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 52d8f53839..553ce3e78d 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -1129,12 +1129,14 @@ func (c *Bor) Prepare(chain consensus.ChainHeaderReader, header *types.Header, w var succession int // if signer is not empty if currentSigner.signer != (common.Address{}) { + // A signer outside the active set (post-Rio: outside the span's + // producer set) does not build at all — its candidate could never + // seal, and its sequence must never reach the store. Nodes without + // a signer (RPC) skip this check and keep their pending snapshot + // fresh. succession, err = snap.GetSignerSuccessionNumber(currentSigner.signer) if err != nil { - // If the signer is not in the active validator set, use succession 0 - // so that the pending block header is still valid for RPC queries. - // Seal() will independently reject the block if unauthorized. - succession = 0 + return err } } diff --git a/docs/cli/default_config.toml b/docs/cli/default_config.toml index 5ec5b8c01e..1f2d00b74b 100644 --- a/docs/cli/default_config.toml +++ b/docs/cli/default_config.toml @@ -267,3 +267,9 @@ devfakeauthor = false enable-preconfs = false enable-private-tx = false bp-rpc-endpoints = [] + +[sequencer] + enabled = false + publisher-endpoint = "" + consumer-endpoint = "" + poll = "200ms" diff --git a/docs/cli/server.md b/docs/cli/server.md index 37139cb150..72e59f1e8e 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -104,6 +104,14 @@ The ```bor server``` command runs the Bor client. - ```rpc.returndatalimit```: Maximum size (in bytes) a result of an rpc request could have (use 0 for no limits) (default: 100000) +- ```sequencer.consumer-endpoint```: Sequence store consumer service gRPC endpoint (tail reads during reconciliation) + +- ```sequencer.enabled```: Enable the sequence store integration (a mining node publishes the block lifecycle) (default: false) + +- ```sequencer.poll```: Producer txpool poll cadence while a block is open (continuous building); 0 keeps the one-shot fill (default: 200ms) + +- ```sequencer.publisher-endpoint```: Sequence store publisher service gRPC endpoint (publish stream) + - ```snapshot```: Enables the snapshot-database mode (default: true) - ```state.scheme```: Scheme to use for storing ethereum state ('hash' or 'path') (default: path) diff --git a/eth/backend.go b/eth/backend.go index d6414dd0b1..2f7da4655f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -55,6 +55,7 @@ import ( "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/eth/protocols/wit" "github.com/ethereum/go-ethereum/eth/relay" + "github.com/ethereum/go-ethereum/eth/sequencer" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -130,6 +131,11 @@ type Ethereum struct { gasPrice *big.Int etherbase common.Address + // Sequence store integration (design docs/sequencer-bor.md): at most + // one is set, by role. + seqPublisher *sequencer.Publisher // mining node: publishes the block lifecycle + seqConsumer *sequencer.Consumer // RPC node: re-executes the stream for preconf receipts + networkID uint64 netRPCService *ethapi.NetAPI @@ -142,6 +148,45 @@ type Ethereum struct { shutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully } +// attachSequencer wires the sequence store integration by role (design +// docs/sequencer-bor.md): a mining node publishes the block lifecycle, a +// non-mining node follows the stream for preconf receipts. +func (s *Ethereum) attachSequencer(config *ethconfig.Config) error { + switch config.SequencerRole { + case "producer": + if s.miner == nil { + return nil + } + + publisher, err := sequencer.NewPublisher(config.SequencerPublisherEndpoint, + config.SequencerConsumerEndpoint, s.blockchain.Config().ChainID.Uint64(), + config.SequencerPoll, s.blockchain) + if err != nil { + return fmt.Errorf("sequencer publisher: %w", err) + } + + s.seqPublisher = publisher + s.miner.SetSequencer(publisher) + case "consumer": + // A consumer that cannot start (not a bor chain) must not block the + // node: preconfs stay off, everything else runs. + consumer, err := sequencer.NewConsumer(config.SequencerConsumerEndpoint, s.blockchain) + if err != nil { + log.Error("Sequencer consumer disabled", "err", err) + + return nil + } + + s.seqConsumer = consumer + consumer.Start() + case "": + default: + return fmt.Errorf("unknown sequencer role %q", config.SequencerRole) + } + + return nil +} + // New creates a new Ethereum object (including the initialisation of the common Ethereum object), // whose lifecycle will be managed by the provided node. func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { @@ -305,32 +350,30 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { trieJournalDirectory = stack.ResolvePath("triedb") } - var ( - options = &core.BlockChainConfig{ - TrieCleanLimit: config.TrieCleanCache, - NoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - ArchiveMode: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - StateScheme: scheme, - TriesInMemory: config.TriesInMemory, - ChainHistoryMode: config.HistoryMode, - TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), - AddressCacheSizes: config.AddressCacheSizes, - PreloadRateLimit: config.PreloadRateLimit, - VmConfig: vmCfg, - Stateless: config.SyncMode == downloader.StatelessSync, - // Enables file journaling for the trie database. The journal files will be stored - // within the data directory. The corresponding paths will be either: - // - DATADIR/triedb/merkle.journal - // - DATADIR/triedb/verkle.journal - TrieJournalDirectory: trieJournalDirectory, - StateSizeTracking: config.EnableStateSizeTracking, - } - ) + options := &core.BlockChainConfig{ + TrieCleanLimit: config.TrieCleanCache, + NoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + ArchiveMode: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + StateScheme: scheme, + TriesInMemory: config.TriesInMemory, + ChainHistoryMode: config.HistoryMode, + TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), + AddressCacheSizes: config.AddressCacheSizes, + PreloadRateLimit: config.PreloadRateLimit, + VmConfig: vmCfg, + Stateless: config.SyncMode == downloader.StatelessSync, + // Enables file journaling for the trie database. The journal files will be stored + // within the data directory. The corresponding paths will be either: + // - DATADIR/triedb/merkle.journal + // - DATADIR/triedb/verkle.journal + TrieJournalDirectory: trieJournalDirectory, + StateSizeTracking: config.EnableStateSizeTracking, + } checker := whitelist.NewService(chainDb, config.DisableBlindForkValidation, config.MaxBlindForkValidationLimit) @@ -474,6 +517,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.miner.SetPrioAddresses(config.TxPool.Locals) } + if err := eth.attachSequencer(config); err != nil { + return nil, err + } + // 1.14.8: NewOracle function definition was changed to accept (startPrice *big.Int) param. eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, config.GPO, config.Miner.GasPrice) eth.APIBackend.gpo.ProcessCache() @@ -696,13 +743,21 @@ func (s *Ethereum) StopMining() { func (s *Ethereum) IsMining() bool { return s.miner.Mining() } func (s *Ethereum) Miner() *miner.Miner { return s.miner } -func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } -func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } -func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool } -func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool } -func (s *Ethereum) Engine() consensus.Engine { return s.engine } -func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } -func (s *Ethereum) IsListening() bool { return true } // Always listening +func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } +func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } +func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool } +func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool } +func (s *Ethereum) Engine() consensus.Engine { return s.engine } +func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } +func (s *Ethereum) IsListening() bool { return true } // Always listening +// WhitelistedMilestone implements miner.Backend: the newest Heimdall +// milestone the downloader has whitelisted, or false when none has arrived. +// The miner's finality gate compares it against the local chain before a +// producer builds. +func (s *Ethereum) WhitelistedMilestone() (bool, uint64, common.Hash) { + return s.Downloader().ChainValidator.GetWhitelistedMilestone() +} + func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downloader } func (s *Ethereum) Synced() bool { return s.handler.synced.Load() } func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() } @@ -1083,6 +1138,14 @@ func (s *Ethereum) Stop() error { <-ch s.filterMaps.Stop() s.txPool.Close() + if s.seqPublisher != nil { + s.seqPublisher.Close() + } + + if s.seqConsumer != nil { + s.seqConsumer.Close() + } + if s.miner != nil { s.miner.Close() } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index c7643976f3..7fd1cfd358 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -329,7 +329,20 @@ type Config struct { TxSyncDefaultTimeout time.Duration `toml:",omitempty"` TxSyncMaxTimeout time.Duration `toml:",omitempty"` + // Sequence store integration (design docs/sequencer-bor.md). Role is + // derived, not configured: "producer" on a mining node (publishes the + // block lifecycle), "consumer" on a non-mining node (re-executes the + // stream for preconf receipts), empty when disabled. The publisher and + // consumer gRPC services have their own endpoints (the publisher reads + // the tail through the consumer service when it reconciles); Poll is + // the txpool poll cadence while a block is open. + SequencerRole string + SequencerPublisherEndpoint string + SequencerConsumerEndpoint string + SequencerPoll time.Duration + // Preconf / Private transaction relay related settings + EnablePreconfs bool EnablePrivateTx bool BlockProducerRpcEndpoints []string diff --git a/eth/sequencer/adoption.go b/eth/sequencer/adoption.go new file mode 100644 index 0000000000..a330dd9b67 --- /dev/null +++ b/eth/sequencer/adoption.go @@ -0,0 +1,848 @@ +package sequencer + +import ( + "context" + "math/big" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/miner" +) + +// checkTailTimeout bounds the build-start tail read: past it +// the block is built from the pool with the publisher buffering silently. +var checkTailTimeout = 250 * time.Millisecond + +// noHold disables the send loop's send ceiling. +const noHold = ^uint64(0) + +// buildMode is the build treatment for the current height, decided once by +// the build-start classification and consumed by OpenBlock/PublishTx (mute) +// and SealBlock (gate arming). It replaces three formerly loose flags whose +// legal combinations were enforced only by discipline. +type buildMode struct { + kind int + height uint64 // the height the treatment was decided for (modeOpen: unused) +} + +const ( + modeOpen = iota // publish normally + modeMuted // build at/behind a sealed height: publish nothing + modeSealedWait // muted over a store-closed height whose block is still owed + modeRecover // rebuilding a sealed generation the chain never received + // modeOverSealed is muted at a height the store sealed but recovery + // declined: the liveness fallback, whose broadcast the store's standing + // seal must not refuse — refusing every build here would halt the chain + // at this block forever. + modeOverSealed +) + +// keepNone is the keepFrom value for a refold that keeps no suffix: every +// unacked entry is abandoned to supersession convergence. +const keepNone = ^uint64(0) + +// hold is the send loop's send ceiling: entries with seq above after stay +// buffered in the journal. A build-start hold gates only the new build's +// entries (after = last existing seq), so a prior seal flush always +// finishes draining; a mid-block STALE hold gates everything unacked +// (after = ackedSeq). kind decides the release: holdBuild lifts the moment +// the flush it ordered behind is home — batching a window until its own +// seal would defeat mid-block streaming — while holdSticky persists until +// a seal or reconcile resolves the lineage. The cleared state pairs +// noHold with holdNone; use clearedHold(), never the zero value. +type hold struct { + after uint64 + kind int +} + +func clearedHold() hold { return hold{after: noHold, kind: holdNone} } + +func (h hold) active() bool { return h.kind != holdNone } + +func (h hold) gates(seq uint64) bool { return seq > h.after } + +// Send-ceiling kinds. Only the self-release behavior is branched on: +// a build-start hold lifts when the flush it ordered behind drains, while a +// sticky hold (foreign tail, adopt, or mid-block STALE) persists until a +// seal or reconciliation resolves the lineage. +const ( + holdNone = iota + holdBuild + holdSticky +) + +// adoption tracks a store window being adopted — a three-state machine: +// p.adopt == nil (none) -> armed (returned by AdoptWindow, awaiting the +// engaging open; awaitOpen is true exactly then) -> engaged (idx tracks +// the next PublishTx match). It ends by full match, divergence rewind, +// unadopt, or a seal flush. +type adoption struct { + number uint64 + timestamp uint64 + parent common.Hash + gasLimit uint64 + baseFee *big.Int + base commitment.Head // store position the window folds on (its open's prefix) + + txs []*types.Transaction // the window's transactions, decoded from store bytes + idx int // next expected PublishTx match + engaged bool +} + +// buildStartState is the store's relationship to the height about to be +// built, resolved from the parent's seal boundary. +type buildStartState int + +const ( + buildAtBoundary buildStartState = iota // info holds this height's live state + buildSealedPast // the store sealed at or past this height + buildBehind // the store lacks the parent: owed a backfill + buildUnknown // the store could not be read in budget + buildRecover // sealed in the store, absent from the chain + buildSealedWait // sealed in the store, its block still owed to us +) + +// recoverState is how a height sealed in the store but missing from the chain +// should be handled. +type recoverState int + +const ( + recoverNone recoverState = iota // not recoverable: treat as an ordinary sealed height + recoverWait // the block may still arrive; do not rebuild yet + recoverReady // the sealer is presumed dead; rebuild its prefix +) + +// buildStartRead resolves what stands at the height about to be built. It +// anchors at the height itself — probing down for the nearest generation — +// rather than at our own anchor: the anchor-based walk only sees entries +// written after our last confirmed write, and goes blind to a live window +// after a mid-window rebase. This read cannot. +// +// The probe finds generations, sealed or live (the store's block index +// serves both), so the walk disambiguates: a live generation is served from +// its open, and a sealed one from just past its seal. +func (p *Publisher) buildStartRead(ctx context.Context, number uint64) (tailInfo, buildStartState) { + h, found, err := p.read.probeDown(ctx, number) + if err != nil { + return tailInfo{}, buildUnknown + } + + if !found { + // No generation at or below this height. An empty store is a clean + // boundary — a fresh chain, or an operator wipe, where the open + // forward-jumps from the seed by design. A non-empty store without + // our parent is behind. + info, out := p.read.floorRead(ctx) + if out != recOK { + return tailInfo{}, buildUnknown + } + + if number <= 1 || (!info.haveSeal && !info.tipOpen && len(info.window) == 0) { + return info, boundaryOrHold(info) + } + + return info, buildBehind + } + + if h > number { + return tailInfo{}, buildSealedPast + } + + if h < number-1 { + return tailInfo{haveSeal: true, lastSealHeight: h}, buildBehind + } + + // h is our height or our parent's. Walk from its boundary: a live + // generation at h is served from its open, so the window is in view. + info, out, done := p.tryWalk(ctx, blockReq(h), false) + if !done || out != recOK { + return tailInfo{}, buildUnknown + } + + switch { + case info.tipOpen && info.tipOpenHeight == number: + // A live window at our height: the adopt case. + return info, buildAtBoundary + + case info.tipOpen && info.tipOpenHeight < number: + // A dangling unsealed window at our parent's height: its producer + // died before the seal flushed. The store is owed the canonical + // parent — the backfill supersedes the dangling window with it. + return info, buildBehind + + case h == number || info.tipOpen || + (info.haveSeal && info.lastSealHeight >= number): + // Our height is sealed, or the store's newest window is past us: + // the height is closed. Unless the chain never received that block — + // then its producer sealed and died before broadcasting, and this + // build recovers the height instead of muting it away forever. + switch rec, st := p.recoverSealed(ctx, number); st { + case recoverReady: + return rec, buildRecover + case recoverWait: + return info, buildSealedWait + } + + return info, buildSealedPast + } + + // h == number-1, sealed, nothing after it: a clean boundary. Carry the + // probe's knowledge — the parent is sealed — for the dead-build discard + // and the sealed-tip bookkeeping. + if !info.haveSeal { + info.haveSeal, info.lastSealHeight = true, number-1 + } + + return info, boundaryOrHold(info) +} + +// boundaryOrHold refuses to mint a generation on a head we cannot derive. +// +// A boundary read says "nothing stands at this height" — but a head taken on +// the store's word says nothing of the sort, because we never saw what +// produced it. Opening there is how a second generation lands on top of +// another producer's live window: the CAS passes, since passing only means +// we echoed the value we were handed. Holding costs this block's records; +// opening blind costs the other producer's. +func boundaryOrHold(info tailInfo) buildStartState { + if !info.explained { + readUnexplained.Inc(1) + log.Warn("Sequencer holding build: store head could not be derived from entries read") + + return buildUnknown + } + + return buildAtBoundary +} + +// recoverGrace is how long a height sealed in the store may stay missing from +// the chain before another build rebuilds it. One block period: long enough +// that an in-flight broadcast wins the race, short enough that a dead sealer +// costs one slot rather than the chain. +const recoverGrace = 4 * time.Second + +// recoverSealed reconstructs a sealed generation the chain never received. +// +// A producer that got its seal acked and then died leaves the height closed +// in the store and empty on the chain: muting there strands it forever, and +// building fresh content there would orphan every record the dead producer +// already had acked — those are preconfirmations, and the store is the proof +// they were issued. So the recovery build must carry exactly that prefix, +// which means reading the generation back out of the store. +// +// Only the content is recovered. The store already holds the complete +// generation, seal included, so the rebuild publishes nothing — the caller +// mutes it. +func (p *Publisher) recoverSealed(ctx context.Context, number uint64) (tailInfo, recoverState) { + if p.chain == nil || p.chain.GetCanonicalHash(number) != (common.Hash{}) { + return tailInfo{}, recoverNone // the chain has the block: an ordinary loss + } + + entries, err := p.read.generation(ctx, number) + if err != nil { + return tailInfo{}, recoverNone + } + + if len(entries) == 0 || entries[0].GetBlockOpen() == nil { + return tailInfo{}, recoverNone + } + + // Give the sealer its block period to broadcast before rebuilding on its + // behalf: the block may simply still be in flight. Measuring from the + // block's own timestamp rather than from now means a build that starts + // late does not add another period of waiting. + if ts := entries[0].GetBlockOpen().GetBlockTimestamp(); ts != 0 && + time.Now().Before(time.Unix(int64(ts), 0).Add(recoverGrace)) { + return tailInfo{}, recoverWait + } + + // Everything up to (not including) the seal is the window; the seal + // itself stays the store's, not ours to reissue. A generation with no + // seal is a live window, not a phantom — the probe resolves generations + // rather than sealed blocks, so this is the disambiguation. + window := make([]*pb.Entry, 0, len(entries)) + sealed := false + + for _, e := range entries { + if e.GetBlockSeal() != nil { + sealed = true + + break + } + + window = append(window, e) + } + + if !sealed { + return tailInfo{}, recoverNone + } + + cur := commitment.Head(window[0].GetBlockOpen().GetPrefixCommitment()) + + for _, e := range window { + next, err := foldEntry(cur, e) + if err != nil { + return tailInfo{}, recoverNone + } + + cur = next + } + + open := window[0].GetBlockOpen() + + publishRecoverCount.Inc(1) + log.Warn("Recovering a height sealed in the store but absent from the chain", + "number", number, "records", len(window)-1) + + return tailInfo{ + s: cur, + tipOpen: true, + tipOpenHeight: number, + tipOpenParent: common.BytesToHash(open.GetParentHash()), + window: window, + }, recoverReady +} + +// AdoptWindow is the worker's build-start check, and the whole of Rule 1: +// an open may be published only at a seal boundary. The store elects the +// owner of every height — writes are a CAS on its head — and this read is +// how a build respects the election before its open is even attempted: +// +// sealed at/past this height -> mute (the height is closed) +// live window on our parent -> adopt (the store has an owner; extend it) +// clean boundary -> open (we are the owner-elect; the CAS +// settles the simultaneous-open race) +// store behind our parent -> hold + prime the backfill (outage) +// store unreadable -> hold (build and buffer; the flush repairs) +func (p *Publisher) AdoptWindow(number uint64, parent common.Hash) *miner.AdoptedWindow { + if p.failed.Load() { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), checkTailTimeout) + defer cancel() + + info, state := p.buildStartRead(ctx, number) + + // An armed offer the previous work cycle never consumed (its build died + // before opening) stays servable — without it this read would mistake + // our own absorbed window for a clean boundary. + if w, handled := p.reoffer(ctx, number, parent, info); handled { + return w + } + + p.mu.Lock() + defer p.mu.Unlock() + + // Every non-reoffer classification starts disarmed and unmuted, and a + // resync signal armed during the previous cycle dies here: this fresh + // read supersedes whatever that signal saw, so it must not abort the + // build it just classified. + p.adopt = nil + p.mode = buildMode{} + p.resync = false + + p.advanceStoreSealedTipLocked(info, "build-start read") + + // Lost head race, or a reorg rebuilding sealed ground — no phantom open + // on our own sealed tip. If it seals after all, the flush rebuild + // publishes the complete window. + if number <= p.sealedTip { + return p.muteLocked(number) + } + + // A draining flush is not a foreign tail — gate only the new build's + // entries so the flush finishes delivering first. + if _, flushing := p.pendingFlushLocked(); flushing { + p.holdNewLocked(holdBuild) + + return nil + } + + switch state { + case buildUnknown: + // Blind at build start: build and buffer; the transport goroutine + // keeps probing and the seal flush repairs. + log.Warn("Sequencer could not read the store at build start", "number", number) + p.holdNewLocked(holdBuild) + + return nil + + case buildSealedPast: + w := p.muteLocked(number) + // Recovery declined this store-sealed height (the chain already has + // the block, or its generation cannot be fetched). A build that + // still seals here is the liveness fallback the gate must let + // through. + p.mode.kind = modeOverSealed + + return w + + case buildSealedWait: + // The store closed this height and its block is still owed to us. + // Refusing the broadcast is safe only while the rebuild is still + // coming: once the grace elapses this height recovers properly, so + // the refusal cannot outlive it and strand the chain here. + w := p.muteLocked(number) + p.mode.kind = modeSealedWait + + return w + + case buildRecover: + // Rebuild the dead producer's block exactly: its transactions were + // acked, so they are promised at this height. The store already + // holds the whole generation, so publish nothing — mute — and hand + // the content to the worker to seal and broadcast. + a, _, ok := parseWindow(info) + if !ok || a.number != number || a.parent != parent { + return p.muteLocked(number) + } + + p.mode = buildMode{kind: modeRecover, height: number} + + return offerLocked(a) + + case buildBehind: + // The store is owed every block from its seal edge to our parent. + // Build and buffer: the backfill drains oldest-first, and the live + // window follows only once the store reaches our boundary. + p.primeBackfillLocked(info, number) + p.holdNewLocked(holdBuild) + + return nil + } + + // buildAtBoundary: info is exactly this height's live state. + p.discardLostLocked(info) + + if len(info.window) == 0 { + // The parent's seal is the store head: a clean boundary, and the + // open publishes. Sync our lineage to the head when nothing of ours + // is pending, so the open folds onto the true tip. + if p.head == p.anchor && info.s != p.anchor { + p.rebaseLocked(info.s) + } + + p.hold = clearedHold() + + return nil + } + + // A live window at this height: the store already has an owner. + a, items, ok := parseWindow(info) + if ok && a.number == number && a.parent == parent { + return p.adoptWindowLocked(info, a, items) + } + + // Unparseable, or on a different parent (reorg territory): build + // locally, publish nothing; whoever seals resolves the height. + p.holdNewLocked(holdSticky) + + return nil +} + +// primeBackfillLocked registers the store's gap [edge+1, parent] as owed +// from the chain database. The collapse machinery already tracks blocks +// sealed while the store was down; this covers a process that restarted +// mid-outage, whose collapsed range died with its journal. +func (p *Publisher) primeBackfillLocked(info tailInfo, number uint64) { + if number < 2 || p.chain == nil { + return + } + + // The range is based on what this read decoded, never the accumulated + // storeSealedTip: the tip has carried inferred values, and even a real + // tip proves nothing about heights below it. An undecoded boundary + // (probe inference) includes the boundary height itself — it may be a + // partial delivery, and a duplicate generation is cheaper than a hole. + var from uint64 + + switch { + case info.sealDecoded: + from = info.lastSealHeight + 1 + case info.haveSeal: + from = info.lastSealHeight + } + + if from <= 1 { + // The store's seal edge is unknown; priming from genesis would + // republish the world. Leave it to the collapse machinery. + return + } + + to := number - 1 + + if p.pendingFrom != 0 { + if p.pendingFrom < from { + from = p.pendingFrom + } + + if p.pendingTo > to { + to = p.pendingTo + } + } + + if from > to { + return + } + + p.pendingFrom, p.pendingTo = from, to +} + +// holdNewLocked gates entries appended from here on while letting +// everything already in the journal (a draining flush included) deliver. +func (p *Publisher) holdNewLocked(kind int) { + p.hold = hold{after: p.journal.nextSeq - 1, kind: kind} +} + +// muteLocked silences the coming build: OpenBlock and PublishTx append +// nothing, so the doomed lineage never reaches the journal or the store. +func (p *Publisher) muteLocked(number uint64) *miner.AdoptedWindow { + p.mode = buildMode{kind: modeMuted, height: number} + + publishMutedCount.Inc(1) + log.Debug("Sequencer muting build at sealed height", "number", number, "sealedTip", p.sealedTip) + + return nil +} + +// adoptWindowLocked absorbs an unsealed tail window as confirmed lineage: its +// entries enter the journal as published-and-acked, the head moves to S, and +// the window is handed to the miner. Continuations then publish normally, +// so their acks (or STALEs) settle who owns the height. +func (p *Publisher) adoptWindowLocked(info tailInfo, a *adoption, items []journalItem) *miner.AdoptedWindow { + if abandoned := p.unackedLocked(); abandoned > 0 { + publishDropMeter.Mark(int64(abandoned)) + } + + fresh := newJournal() + fresh.nextSeq = p.journal.nextSeq + + for _, it := range items { + fresh.append(it.entry, it.pre, it.post, it.kind, it.height, fresh.nextSeq, it.txHashes) + } + + p.journal = fresh + p.ackedSeq = fresh.nextSeq - 1 + p.head = info.s + p.anchor = info.s + p.anchored, p.confirmed = true, true + p.curHeight = a.number + p.awaitOpen = true + p.adopt = a + + // No hold: continuations publish immediately. Extending the adopted + // window is how ownership is discovered — an ack means we own this + // height and may seal, a STALE means another producer is still + // writing it and the pre-seal barrier must stop us. Holding here + // would hide both answers and strand a legitimate takeover. + p.hold = clearedHold() + + publishQueueGauge.Update(int64(p.unackedLocked())) + reconcileAdopt.Inc(1) + log.Info("Sequencer adopting store window", "number", a.number, "txs", len(a.txs)) + + return offerLocked(a) +} + +// reoffer re-serves or refreshes an armed, unengaged offer for the same +// build. handled=false falls through to normal classification, disarmed. +func (p *Publisher) reoffer(ctx context.Context, number uint64, parent common.Hash, info tailInfo) (*miner.AdoptedWindow, bool) { + p.mu.Lock() + + a := p.adopt + if a == nil || a.engaged || a.number != number || a.parent != parent { + p.mu.Unlock() + + return nil, false + } + + if info.s == p.head { + defer p.mu.Unlock() + + // The offer is re-served exactly as a fresh adoption would be: any + // hold armed since (a STALE from the death throes of the previous + // cycle) is residue of a lineage this adoption replaced, and keeping + // it would gate the continuations that discover ownership. + p.hold = clearedHold() + + return offerLocked(a), true // window unchanged since absorption + } + + p.mu.Unlock() + + return p.readoptGrown(ctx, a, number, parent) +} + +// readoptGrown re-reads a window that kept growing after our snapshot from +// its base and re-adopts it in full — sealing the stale snapshot would +// supersede the extra records at the flush. Any failure (read miss, the +// lineage moved while unlocked, the window is no longer ours) falls +// through to normal classification, disarmed. Reading a.base unlocked is +// race-free: it is written once, before p.adopt publishes it. +func (p *Publisher) readoptGrown(ctx context.Context, a *adoption, number uint64, parent common.Hash) (*miner.AdoptedWindow, bool) { + fresh, out, done := p.tryWalk(ctx, headReq(a.base), false) + if !done || out != recOK { + return nil, false + } + + p.mu.Lock() + defer p.mu.Unlock() + + // Bail if the lineage moved while unlocked (a seal flush landing). + if p.adopt != a || a.engaged { + return nil, false + } + + if a2, items, ok := parseWindow(fresh); ok && a2.number == number && a2.parent == parent { + return p.adoptWindowLocked(fresh, a2, items), true + } + + return nil, false +} + +// offerLocked builds the miner's view of an adopted window. +func offerLocked(a *adoption) *miner.AdoptedWindow { + return &miner.AdoptedWindow{ + Number: a.number, + Timestamp: a.timestamp, + ParentHash: a.parent, + GasLimit: a.gasLimit, + BaseFee: new(big.Int).Set(a.baseFee), + Txs: a.txs, + } +} + +// discardLostLocked drops the unacked suffix when a foreign seal has +// covered all of it, re-anchoring on the store head. Entries above the +// sealed tip (an outage buffer awaiting journal replay) are never touched. +func (p *Publisher) discardLostLocked(info tailInfo) { + lost := p.unackedLocked() + if lost == 0 || !info.haveSeal { + return + } + + for _, it := range p.journal.items { + if it.seq > p.ackedSeq && it.height > info.lastSealHeight { + return + } + } + + publishDropMeter.Mark(int64(lost)) + p.rebaseLocked(info.s) + publishQueueGauge.Update(0) + log.Debug("Sequencer discarded superseded buffer", "entries", lost, "sealedTip", info.lastSealHeight) +} + +// unadoptLocked drops an absorbed-but-abandoned adopt back to the +// window's base: the dead window leaves the journal and the next open folds +// onto the base, so a divergent local build supersedes the incumbent +// window through the normal STALE→reconcile path. +func (p *Publisher) unadoptLocked(a *adoption) { + p.rebaseLocked(a.base) + p.adopt = nil + p.awaitOpen = false + p.curHeight = 0 + p.hold = clearedHold() +} + +// rebaseLocked re-anchors an idle lineage (nothing unconfirmed) onto a +// fresh store head, so the next open extends the true tip. It is the +// primitive under three resets of increasing scope — callers own what it +// deliberately does not touch (adopt, hold, curHeight/awaitOpen): +// +// rebaseLocked journal/head/anchor only (clean tail; between-blocks anchor) +// discardLostLocked + drop metric; onto the store head; hold set by the caller after +// unadoptLocked + clears adopt, awaitOpen, curHeight, hold; onto the WINDOW BASE, +// so the divergent open STALEs onto the counted reconcile path +// installSwapLocked (classify.go) full swap with refolded content; re-derives the window +// dropRefusedFlushLocked (publisher.go) the rewind-side sibling: truncates a refused +// flush off the tail instead of re-anchoring under it +func (p *Publisher) rebaseLocked(s commitment.Head) { + fresh := newJournal() + fresh.nextSeq = p.journal.nextSeq + + p.journal = fresh + p.ackedSeq = fresh.nextSeq - 1 + p.head = s + p.anchor = s + p.anchored, p.confirmed = true, true +} + +// parseWindow decodes a collected tail window into an adoption candidate and +// the journal items representing its entries, verifying the fold chain ends +// at the store head. +func parseWindow(info tailInfo) (*adoption, []journalItem, bool) { + if len(info.window) == 0 { + return nil, nil, false + } + + open := info.window[0].GetBlockOpen() + if open == nil { + return nil, nil, false + } + + a := &adoption{ + number: open.GetBlockNumber(), + timestamp: open.GetBlockTimestamp(), + parent: common.BytesToHash(open.GetParentHash()), + gasLimit: open.GetGasLimit(), + baseFee: new(big.Int).SetBytes(open.GetBaseFee()), + base: commitment.Head(open.GetPrefixCommitment()), + } + + items := make([]journalItem, 0, len(info.window)) + cur := commitment.Head(info.window[0].GetBlockOpen().GetPrefixCommitment()) + + for _, entry := range info.window { + next, err := foldEntry(cur, entry) + if err != nil { + return nil, nil, false + } + + kind := entryOpen + + var hashes []common.Hash + + if rec := entry.GetRecord(); rec != nil { + kind = entryRecord + + for _, raw := range rec.GetTransactions() { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + return nil, nil, false + } + + a.txs = append(a.txs, tx) + hashes = append(hashes, tx.Hash()) + } + } + + items = append(items, journalItem{entry: entry, pre: cur, post: next, kind: kind, height: a.number, txHashes: hashes}) + cur = next + } + + if cur != info.s { + return nil, nil, false + } + + return a, items, true +} + +// matchesOpen reports whether the worker's open reproduces the adopted +// window's context exactly — any mismatch means the worker rejected the +// inherited context. +func (a *adoption) matchesOpen(number, timestamp uint64, parent common.Hash, gasLimit uint64, baseFee *big.Int) bool { + return a.number == number && a.timestamp == timestamp && + a.parent == parent && a.gasLimit == gasLimit && + baseFee != nil && a.baseFee.Cmp(baseFee) == 0 +} + +// adoptOpenLocked resolves an adopted window against the worker's +// OpenBlock. swallow=true means the open engaged the window and must not +// be appended; false drops the adopt and the open proceeds (buffered: +// hold stays until the seal flush). +func (p *Publisher) adoptOpenLocked(number uint64, timestamp uint64, parent common.Hash, gasLimit uint64, baseFee *big.Int) (swallow bool) { + a := p.adopt + if a == nil { + return false + } + + if a.engaged || !a.matchesOpen(number, timestamp, parent, gasLimit, baseFee) { + // The build diverged from the adopted window — the worker + // rejected the inherited context (a bogus or version-skewed + // incumbent window; honest producers derive the same context). + // Undo the absorption back to the window's base so the local open + // folds there and STALEs against the store's incumbent window: + // the supersede then lands on the counted reconcile path + // instead of appending a silent second generation on the store + // head that no reconcile ever sees. + p.unadoptLocked(a) + + return false + } + + a.engaged = true + p.awaitOpen = false + p.curHeight = number + + return true +} + +// adoptTxLocked matches one committed transaction against the adopted +// window. swallow=true: the transaction is already in the store. A +// mismatch (the worker dropped or replaced a window transaction) rewinds +// the lineage to the matched prefix — the dead window tail leaves the +// journal, the head returns to the prefix, and subsequent commits fold onto +// it, buffered; the seal flush resolves the divergence. Nothing is +// published here. +func (p *Publisher) adoptTxLocked(hash common.Hash) (swallow bool) { + a := p.adopt + if a == nil || !a.engaged { + return false + } + + // Only reachable for an empty adopted window: a full match clears + // p.adopt eagerly below. + if a.idx >= len(a.txs) { + p.adopt = nil + + return false + } + + if a.txs[a.idx].Hash() == hash { + a.idx++ + if a.idx == len(a.txs) { + p.adopt = nil // fully adopted; continuations buffer normally + } + + return true + } + + log.Warn("Sequencer adopted window diverged, deferring to the seal flush", + "number", a.number, "matched", a.idx, "of", len(a.txs)) + + p.rewindToPrefixLocked(a.idx) + p.adopt = nil + + return false // the divergent tx folds onto the rewound prefix, buffered +} + +// rewindToPrefixLocked drops the adopted window's unmatched tail from the +// journal and returns the fold head to the end of the matched prefix. At +// call time the journal is exactly the absorbed window (matches append +// nothing; a re-open while engaged unadopts first), so the scan finds the +// window's open first. +func (p *Publisher) rewindToPrefixLocked(matched int) { + cut := -1 + seen := 0 + + for i, it := range p.journal.items { + if it.kind == entryOpen && it.height == p.curHeight { + cut = i // at minimum, keep the window's open + } + + if it.kind != entryRecord { + continue + } + + seen += len(it.entry.GetRecord().GetTransactions()) + if seen >= matched && matched > 0 { + cut = i + + break + } + } + + if cut < 0 { + return + } + + p.rewindJournalLocked(cut + 1) + p.ackedSeq = p.journal.items[cut].seq +} diff --git a/eth/sequencer/adoption_test.go b/eth/sequencer/adoption_test.go new file mode 100644 index 0000000000..6e3635d99b --- /dev/null +++ b/eth/sequencer/adoption_test.go @@ -0,0 +1,901 @@ +package sequencer + +import ( + "math/big" + "testing" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// foreignWindow appends a dangling open window (open + txs records) to the +// store, as an incumbent producer's unsealed work. +func foreignWindow(t *testing.T, h *harness, number uint64, parent common.Hash, txs ...*types.Transaction) (uint64, uint64) { + t.Helper() + + ts := 1700000000 + number + gasLimit := uint64(30_000_000) + + entry := &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: number, + BlockTimestamp: ts, + ParentHash: parent.Bytes(), + GasLimit: gasLimit, + BaseFee: big25gwei(), + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(entry); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign open rejected: %v", status) + } + + for _, tx := range txs { + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + rec := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(rec); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign record rejected: %v", status) + } + } + + return ts, gasLimit +} + +func fee25() *big.Int { + return new(big.Int).SetBytes(big25gwei()) +} + +func windowHeader(number uint64, parent common.Hash, ts uint64, gasLimit uint64) *types.Header { + return &types.Header{ + ParentHash: parent, + Number: new(big.Int).SetUint64(number), + GasLimit: gasLimit, + Time: ts, + BaseFee: fee25(), + Difficulty: big.NewInt(1), + } +} + +// The flagship adopt path: the build-start check finds the incumbent's +// window and adopts it — engage swallows the open, matched transactions +// publish nothing, continuations stay buffered — and the seal flush +// completes the window in place. No supersession anywhere. +func TestFollowCompletesWindowAtSeal(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1, tx2 := testTx(t, 0), testTx(t, 1) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1, tx2) + + adopts := reconcileAdopt.Snapshot().Count() + supersedes := reconcileSupersede.Snapshot().Count() + headBefore := h.store.Head() + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 2 || w.Timestamp != ts || w.GasLimit != gasLimit { + t.Fatalf("AdoptWindow = %+v", w) + } + + if got := reconcileAdopt.Snapshot().Count(); got != adopts+1 { + t.Fatalf("adopt counter = %d, want %d", got, adopts+1) + } + + // The worker builds under the adopted context, re-committing the + // window's transactions: those are matched and swallowed, so the + // store does not move for them. + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + p.PublishTx(w.Txs[1]) + + time.Sleep(100 * time.Millisecond) + + if h.store.Head() != headBefore { + t.Fatal("re-committed window transactions must publish nothing") + } + + // A continuation extends the adopted window immediately — its ack is + // what proves this node owns the height and may seal it. + tx3 := testTx(t, 2) + p.PublishTx(tx3) + waitHead(t, h, p, 5*time.Second) + + // The seal completes the window in place. + header := windowHeader(2, parent, ts, gasLimit) + p.SealBlock(blockFor(header, []*types.Transaction{w.Txs[0], w.Txs[1], tx3})) + waitHead(t, h, p, 5*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("adopted block missing: %v", err) + } + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("adopt completion superseded: counter %d -> %d", supersedes, got) + } +} + +// A worker that drops a window transaction (divergence) publishes nothing +// at the moment of divergence; the seal flush re-anchors — the only +// supersession — and the store converges on the sealed content. +func TestFollowDivergenceResolvesAtSeal(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1, tx2 := testTx(t, 0), testTx(t, 1) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1, tx2) + + headBefore := h.store.Head() + + w := p.AdoptWindow(2, parent) + if w == nil { + t.Fatal("no window to adopt") + } + + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + + other := testTx(t, 7) // tx2 failed to apply; the worker committed another + p.PublishTx(other) + + time.Sleep(100 * time.Millisecond) + + if h.store.Head() != headBefore { + t.Fatal("divergence published before sealing") + } + + header := windowHeader(2, parent, ts, gasLimit) + p.SealBlock(blockFor(header, []*types.Transaction{w.Txs[0], other})) + waitHead(t, h, p, 10*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("sealed block missing after divergence flush: %v", err) + } +} + +// A build that raced the window under its own context (the check returned +// the window but the worker opened differently) buffers silently and the +// seal flush re-anchors to the sealed truth. +func TestFollowMismatchedOpenResolvesAtSeal(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if w := p.AdoptWindow(2, parent); w == nil { + t.Fatal("no window offered") + } + + supersedes := reconcileSupersede.Snapshot().Count() + + // Own context: the adopt disengages, un-adopts back to the window's + // base, and the divergent local build supersedes the incumbent window. + ownTs := uint64(1600000099) + p.OpenBlock(2, ownTs, parent, 30_000_000, fee25()) + tx := testTx(t, 3) + p.PublishTx(tx) + + header := windowHeader(2, parent, ownTs, 30_000_000) + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + waitHead(t, h, p, 10*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("sealed block missing after flush: %v", err) + } + + // The supersede of the incumbent's window is counted, not silent + // (finding: divergent takeover on the clean-append path escaped the + // metric). + if got := reconcileSupersede.Snapshot().Count(); got != supersedes+1 { + t.Fatalf("divergent takeover supersede uncounted: %d -> %d", supersedes, got) + } +} + +// A window at a height the build is not at (chain moved) is not adopted: +// the build buffers and the flush repairs. +func TestCheckTailForeignHeightHolds(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if w := p.AdoptWindow(3, common.Hash{0x99}); w != nil { + t.Fatal("mismatched build must not receive the window") + } + + p.mu.Lock() + ceiling := p.hold.after + next := p.journal.nextSeq + p.mu.Unlock() + + if ceiling == noHold || ceiling != next-1 { + t.Fatalf("foreign window must gate new entries (ceiling=%d next=%d)", ceiling, next) + } +} + +// A record-less window (open only) is adopted with an empty seed; the +// first pool transaction buffers as a continuation and the flush completes +// the window. +func TestFollowEmptyWindow(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + ts, gasLimit := foreignWindow(t, h, 2, parent) // zero records + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 0 { + t.Fatalf("AdoptWindow = %+v, want empty window", w) + } + + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + tx := testTx(t, 0) + p.PublishTx(tx) + + header := windowHeader(2, parent, ts, gasLimit) + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + waitHead(t, h, p, 5*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("adopted block missing: %v", err) + } +} + +// A clean tail (no window) publishes normally — the incumbent path. +func TestCheckTailCleanPublishes(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatalf("clean tail returned a window: %+v", w) + } + + publishBlock(t, p, 2, parent, 1) + waitHead(t, h, p, 5*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("incumbent block missing: %v", err) + } +} + +// A seal arriving while an adopted window stands but was never engaged +// (the build sealed under its own steam) is never dropped: the flush +// rebuilds the window from the block and the store converges. +func TestSealNeverDropped(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if w := p.AdoptWindow(2, parent); w == nil { + t.Fatal("no window offered") + } + + // No OpenBlock at all — the straggler seal arrives first. + header := testHeader(2, parent) + tx := testTx(t, 5) + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + waitHead(t, h, p, 10*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("straggler seal never landed: %v", err) + } +} + +// A build-start check must never strangle the previous block's draining +// seal flush: the send ceiling gates only the new build's entries, so a +// flush buffered during an outage still delivers once the store returns — +// even though the next build's check ran (and held) in between. +func TestBuildStartHoldDoesNotBlockFlush(t *testing.T) { + restore := checkTailTimeout + checkTailTimeout = 100 * time.Millisecond + + t.Cleanup(func() { checkTailTimeout = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Store goes down; block 2 is built and sealed entirely offline — the + // flush entries sit in the journal, undeliverable. + h.stop() + + parent := sealHash(t, sealed) + sealed2 := publishBlock(t, p, 2, parent, 2) + + // The next build's check runs while the flush is still pending (store + // unreachable): it must gate only future entries. + if w := p.AdoptWindow(3, sealed2.Hash()); w != nil { + t.Fatalf("unreachable store returned a window: %+v", w) + } + + // Store returns: the pending flush must drain without any further + // publisher calls. + h.resume() + waitHead(t, h, p, 15*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("flush strangled by build-start hold: %v", err) + } +} + +// An offer whose build dies before opening (rotation churn) is re-served +// at the next work cycle: the absorbed window must be resumed, never +// mistaken for a clean tail and replaced by a fresh generation. +func TestUnconsumedOfferReoffered(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx := testTx(t, 0) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx) + + // First work cycle takes the offer and dies before OpenBlock. + if w := p.AdoptWindow(2, parent); w == nil { + t.Fatal("no window offered") + } + + supersedes := reconcileSupersede.Snapshot().Count() + + // The replacement build for the same height must get the offer again. + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 || w.Timestamp != ts { + t.Fatalf("unconsumed offer not re-served: %+v", w) + } + + // This time the build engages and completes the window in place. + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + p.SealBlock(blockFor(windowHeader(2, parent, ts, gasLimit), []*types.Transaction{w.Txs[0]})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("resumed window superseded: %d -> %d", supersedes, got) + } +} + +// An armed offer whose window kept growing after the snapshot (the +// incumbent streamed more records before dying) is re-read from its base +// and re-adopted in full — sealing the stale snapshot would supersede +// the extra records at the flush. +func TestGrownWindowReadoptedNotStale(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1 := testTx(t, 0) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1) + + // Snapshot taken at one record; the build dies before opening. + if w := p.AdoptWindow(2, parent); w == nil || len(w.Txs) != 1 { + t.Fatalf("first offer wrong: %+v", w) + } + + // The incumbent streams one more record before dying. + tx2 := testTx(t, 1) + raw, err := tx2.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + rec := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: h.store.Head().Bytes(), + }}} + if status := h.store.Append(rec); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("grown record rejected: %v", status) + } + + supersedes := reconcileSupersede.Snapshot().Count() + + // The replacement build must receive the FULL grown window. + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 2 { + t.Fatalf("grown window not re-adopted: %+v", w) + } + + // Engage and complete in place: both records already in the store. + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + p.PublishTx(w.Txs[1]) + p.SealBlock(blockFor(windowHeader(2, parent, ts, gasLimit), []*types.Transaction{w.Txs[0], w.Txs[1]})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("grown-window resume superseded: %d -> %d", supersedes, got) + } +} + +// A producer restarted mid-window resumes it (self-adoption): with no +// persisted position, the fresh publisher relocates the store tail from +// the chain's last imported block, the startup reconcile anchors at the +// window's base, the first build-start check collects and adopts it, and +// the seal completes the same generation — its own preconfs never revoked. +func TestRestartResumesOwnWindow(t *testing.T) { + h := startHarness(t) + + first, err := NewPublisher(h.addr, h.addr, testChainID, 0, nil) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + sealed := publishBlock(t, first, 1, common.Hash{0xef}, 1) + waitHead(t, h, first, 5*time.Second) + + // Mid-window death: open 2 and stream one record, then die unsealed. + parent := sealHash(t, sealed) + tx := testTx(t, 0) + first.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + first.PublishTx(tx) + waitHead(t, h, first, 5*time.Second) + first.Close() + + // A fresh publisher with block 1 as its chain head: the restart probe + // locates the store tail from that height — no local state carried over. + chain := &fakeChain{current: &types.Header{Number: big.NewInt(1)}} + + p, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + waitFor(t, 5*time.Second, func() bool { return p.isAnchored() }) + + supersedes := reconcileSupersede.Snapshot().Count() + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 || w.Txs[0].Hash() != tx.Hash() { + t.Fatalf("restart did not resume own window: %+v", w) + } + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(w.Txs[0]) + p.SealBlock(blockFor(windowHeader(2, parent, 1700000002, 30_000_000), []*types.Transaction{w.Txs[0]})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("self-resume superseded own window: %d -> %d", supersedes, got) + } +} + +// A build-start hold lifts the moment the flush it ordered behind is +// drained: the gated window then streams mid-block instead of batching +// until its own seal — under continuous load a seal-persistent hold +// would ratchet (each flush still draining at the next build's check) +// and turn preconf streaming into at-seal batching. +func TestBuildStartHoldReleasesOnDrain(t *testing.T) { + restore := checkTailTimeout + checkTailTimeout = 100 * time.Millisecond + + t.Cleanup(func() { checkTailTimeout = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Block 2 seals while the store is down: its flush sits in the journal. + h.stop() + + parent := sealHash(t, sealed) + sealed2 := publishBlock(t, p, 2, parent, 1) + + // Build 3 starts against the unreachable store: its window is gated + // behind the pending flush. + if w := p.AdoptWindow(3, sealed2.Hash()); w != nil { + t.Fatalf("unreachable store returned a window: %+v", w) + } + + p.OpenBlock(3, sealed2.Time+2, sealed2.Hash(), sealed2.GasLimit, fee25()) + p.PublishTx(testTx(t, 3)) + + // The store returns: the flush drains, the hold lifts, and block 3's + // window streams to the store with no SealBlock in sight. + h.resume() + waitHead(t, h, p, 15*time.Second) + + p.mu.Lock() + tail := p.journal.items[len(p.journal.items)-1] + held := p.hold.after != noHold + p.mu.Unlock() + + if tail.kind != entryRecord || tail.height != 3 || held { + t.Fatalf("mid-block window not streaming: tail kind=%d height=%d held=%v", tail.kind, tail.height, held) + } +} + +// A build racing its own chain-head update (its height already sealed and +// flushed) is muted: no phantom open dangles on the sealed tip, and the +// next legitimate build publishes normally. +func TestStaleBuildPublishesNothing(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + headBefore := h.store.Head() + muted := publishMutedCount.Snapshot().Count() + + // The worker re-prepares height 1 before the chain head catches up. + if w := p.AdoptWindow(1, common.Hash{0xef}); w != nil { + t.Fatalf("stale build received a window: %+v", w) + } + + if got := publishMutedCount.Snapshot().Count(); got != muted+1 { + t.Fatalf("muted counter = %d, want %d", got, muted+1) + } + + p.OpenBlock(1, sealed.Time, common.Hash{0xef}, sealed.GasLimit, fee25()) + p.PublishTx(testTx(t, 7)) + + p.mu.Lock() + pending := p.journal.nextSeq - 1 - p.ackedSeq + p.mu.Unlock() + + if pending != 0 { + t.Fatalf("muted build appended %d entries", pending) + } + + if h.store.Head() != headBefore { + t.Fatal("muted build reached the store") + } + + // The interrupted build gives way to the real next height. + parent := sealHash(t, sealed) + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatalf("clean tail returned a window: %+v", w) + } + + publishBlock(t, p, 2, parent, 1) + waitHead(t, h, p, 5*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("post-mute block missing: %v", err) + } +} + +// A builder whose published window lost to a foreign seal discards the +// dead buffer at the next work cycle instead of re-STALEing it +// forever, and its next open folds onto the foreign store head. +func TestBuildStartDiscardsSupersededBuffer(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Our window for 2 folds locally but the store never sees it (the + // foreign producer owns the height); its content stays unacked. + parent := sealHash(t, sealed) + h.stop() + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + + // The foreign producer's block 2 lands in the store, sealed. + h.resume() + ts, gasLimit := foreignWindow(t, h, 2, parent, testTx(t, 1)) + appendForeignSeal(t, h, windowHeader(2, parent, ts, gasLimit)) + + p.mu.Lock() + buffered := p.journal.nextSeq - 1 - p.ackedSeq + p.mu.Unlock() + + if buffered == 0 { + t.Fatal("test setup: nothing buffered") + } + + // Next work cycle: the read shows the foreign sealed tip covering our + // whole buffer — it is discarded and the lineage re-anchors. + if w := p.AdoptWindow(3, common.Hash{0x33}); w != nil { + t.Fatalf("AdoptWindow = %+v", w) + } + + p.mu.Lock() + left := p.journal.nextSeq - 1 - p.ackedSeq + rebased := p.head == h.store.Head() && p.anchor == p.head + p.mu.Unlock() + + if left != 0 || !rebased { + t.Fatalf("buffer not discarded: unacked=%d rebased=%v", left, rebased) + } + + // The new build publishes cleanly onto the foreign head. + publishBlock(t, p, 3, common.Hash{0x33}, 1) + waitHead(t, h, p, 5*time.Second) +} + +// A muted build that seals after all (a reorg win) loses nothing: the +// flush rebuilds the complete window from the block — the designed +// supersede-by-seal. +func TestStaleBuildSealFlushRebuilds(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + if w := p.AdoptWindow(1, common.Hash{0xef}); w != nil { + t.Fatalf("stale build received a window: %+v", w) + } + + tx := testTx(t, 9) + p.OpenBlock(1, sealed.Time+2, common.Hash{0xef}, sealed.GasLimit, fee25()) + p.PublishTx(tx) + + header := testHeader(1, common.Hash{0xef}) + header.Time = sealed.Time + 2 + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + waitHead(t, h, p, 10*time.Second) + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 1}); err != nil { + t.Fatalf("reorg seal never landed: %v", err) + } +} + +// A adopt snapshot raced by the dying producer's draining stream: the +// store window gains records after our absorption, so the flush STALEs — +// but the store's copy is a strict prefix of the sealed content, and the +// flush completes it in place. Same generation, nothing revoked. +func TestFlushCompletesExtendedWindowInPlace(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1, tx2 := testTx(t, 0), testTx(t, 1) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1) + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 { + t.Fatalf("offer: %+v", w) + } + + // The dying producer's last in-flight record lands after our snapshot. + raw, err := tx2.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + rec := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: h.store.Head().Bytes(), + }}} + if st := h.store.Append(rec); st != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("extension record rejected: %v", st) + } + + supersedes := reconcileSupersede.Snapshot().Count() + + // The build seeds the snapshot, then commits the extension tx (it is + // in our pool too) and a continuation, and seals all three. + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + p.PublishTx(tx2) + + tx3 := testTx(t, 2) + p.PublishTx(tx3) + p.SealBlock(blockFor(windowHeader(2, parent, ts, gasLimit), []*types.Transaction{w.Txs[0], tx2, tx3})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("extended-window completion superseded: %d -> %d", supersedes, got) + } + + if _, err := h.store.GetBlock(t.Context(), &pb.GetBlockRequest{BlockNumber: 2}); err != nil { + t.Fatalf("completed block missing: %v", err) + } +} + +// A adopt inside a live send loop session must not re-send the absorbed +// window: those entries are already in the store, and re-sending them +// guarantees a STALE plus a spurious gap-fill on every rotation adopt. +func TestFollowDoesNotResendAbsorbed(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx1 := testTx(t, 0) + ts, gasLimit := foreignWindow(t, h, 2, parent, tx1) + + stales := publishStaleCount.Snapshot().Count() + gapfills := reconcileGapfill.Snapshot().Count() + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 { + t.Fatalf("offer: %+v", w) + } + + p.OpenBlock(2, ts, parent, gasLimit, fee25()) + p.PublishTx(w.Txs[0]) + + tx2 := testTx(t, 1) + p.PublishTx(tx2) + p.SealBlock(blockFor(windowHeader(2, parent, ts, gasLimit), []*types.Transaction{w.Txs[0], tx2})) + waitHead(t, h, p, 10*time.Second) + + if got := publishStaleCount.Snapshot().Count(); got != stales { + t.Fatalf("adopt re-sent absorbed entries: stale %d -> %d", stales, got) + } + + if got := reconcileGapfill.Snapshot().Count(); got != gapfills { + t.Fatalf("adopt caused spurious gapfill: %d -> %d", gapfills, got) + } +} + +// A parked adopter (its mid-block build died; no seal ever comes) must +// not starve the idle catch-up: the held dead buffer is refolded away and +// the anchor tracks the store tip. +func TestParkedAdopterIdleRecovers(t *testing.T) { + restore := idleReconcileInterval + idleReconcileInterval = 80 * time.Millisecond + + t.Cleanup(func() { idleReconcileInterval = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Foreign content wins the height mid-build: our window STALEs into a + // stale-hold with no seal ever coming (the parked-adopter shape). + parent := sealHash(t, sealed) + ts, gasLimit := foreignWindow(t, h, 2, parent, testTx(t, 0)) + appendForeignSeal(t, h, windowHeader(2, parent, ts, gasLimit)) + + p.OpenBlock(2, ts+9, parent, gasLimit, fee25()) + p.PublishTx(testTx(t, 5)) + + base := h.store.Head() + + waitFor(t, 10*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.anchor == base && p.unackedLocked() == 0 + }) +} + +// A seal result racing the mute-clear between a build's check and its +// open must not publish an open at or behind the sealed tip. +func TestOpenBlockRefusesSealedHeight(t *testing.T) { + p := barePublisher() + + header := testHeader(1, common.Hash{0xef}) + p.OpenBlock(1, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + + before := len(p.journal.items) + p.OpenBlock(1, header.Time+4, header.ParentHash, header.GasLimit, header.BaseFee) + + if len(p.journal.items) != before { + t.Fatal("open published at a sealed height") + } +} + +// The 30 s idle ticker landing on a healthy producer's drained-open window +// (all records acked, seal pending — unacked==0 with a hold of holdNone) +// must NOT zero curHeight: doing so tagged later records height=0 and +// misclassified the live window as between-blocks. Only a held/parked +// window is cleaned. +func TestIdleTickPreservesLiveWindow(t *testing.T) { + restore := idleReconcileInterval + idleReconcileInterval = 40 * time.Millisecond + + t.Cleanup(func() { idleReconcileInterval = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Open the next window and stream a record; wait until it drains + // (open+record acked) — a healthy incumbent between txs. + parent := sealHash(t, sealed) + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + tx := testTx(t, 0) + p.PublishTx(tx) + waitHead(t, h, p, 5*time.Second) + + p.mu.Lock() + drained := p.unackedLocked() == 0 && p.journal.openStart() >= 0 && p.hold.kind == holdNone + p.mu.Unlock() + + if !drained { + t.Fatal("test setup: window not in the drained-open state") + } + + // Let several idle ticks fire across the drained-open window. + time.Sleep(200 * time.Millisecond) + + p.mu.Lock() + curHeight := p.curHeight + var zeroHeightRecords int + for _, it := range p.journal.items { + if it.kind == entryRecord && it.height == 0 { + zeroHeightRecords++ + } + } + p.mu.Unlock() + + if curHeight != 2 { + t.Fatalf("idle tick zeroed a live window: curHeight=%d, want 2", curHeight) + } + + // A continuation after the ticks must still tag the live height. + p.PublishTx(testTx(t, 1)) + + p.mu.Lock() + for _, it := range p.journal.items { + if it.kind == entryRecord && it.height == 0 { + zeroHeightRecords++ + } + } + p.mu.Unlock() + + if zeroHeightRecords != 0 { + t.Fatalf("records tagged height=0 after idle tick: %d", zeroHeightRecords) + } + + // The window still seals in place — no supersession. + supersedes := reconcileSupersede.Snapshot().Count() + p.SealBlock(blockFor(windowHeader(2, parent, 1700000002, 30_000_000), []*types.Transaction{tx, testTx(t, 1)})) + waitHead(t, h, p, 10*time.Second) + + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("live window superseded after idle tick: %d -> %d", supersedes, got) + } +} diff --git a/eth/sequencer/audit_test.go b/eth/sequencer/audit_test.go new file mode 100644 index 0000000000..3051484d39 --- /dev/null +++ b/eth/sequencer/audit_test.go @@ -0,0 +1,419 @@ +package sequencer + +import ( + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// The preconfirmation failures storeprobe measures on a devnet. Asserting +// them here closes the gap that let a 40x regression pass a green suite: the +// other tests check what the publisher decides, these check what a consumer +// of the store would actually experience. +type storeAudit struct { + // Revoked: a record the store acked — so it was preconfirmed — that + // never landed in any canonical block. The worst outcome; the promise + // was simply broken. + Revoked []common.Hash + + // Displaced: acked at one height, landed at another. The transaction + // executes, but not where it was promised. + Displaced []common.Hash + + // Reordered: landed at the promised height in a different order. + // Position within a block is part of the promise for anything + // order-sensitive. + Reordered int + + // Mismatch: the store's newest generation at a height disagrees with the + // canonical block. A consumer reading the store's latest view of that + // height gets the wrong answer. + Mismatch []uint64 +} + +func (a storeAudit) clean() bool { + return len(a.Revoked) == 0 && len(a.Displaced) == 0 && + a.Reordered == 0 && len(a.Mismatch) == 0 +} + +// generation is one open..seal span at a height, in store order. +type generation struct { + height uint64 + txs []common.Hash + sealed bool +} + +// readAllGenerations replays the whole log and groups it the way the store's +// readers see it: an open starts a generation at its height, records extend +// the current one, and later generations at a height supersede earlier ones. +func readAllGenerations(t *testing.T, h *harness) []generation { + t.Helper() + + seed := commitment.Seed(testChainID) + + var ( + gens []generation + cur *generation + next = &pb.RangeRequest{ + After: &pb.RangeRequest_Head{Head: seed.Bytes()}, + Limit: 512, + } + ) + + for { + resp, err := h.store.Range(context.Background(), next) + if err != nil { + t.Fatalf("range: %v", err) + } + + for _, e := range resp.GetEntries() { + switch { + case e.GetBlockOpen() != nil: + gens = append(gens, generation{height: e.GetBlockOpen().GetBlockNumber()}) + cur = &gens[len(gens)-1] + case e.GetRecord() != nil && cur != nil: + for _, raw := range e.GetRecord().GetTransactions() { + var tx types.Transaction + if err := tx.UnmarshalBinary(raw); err != nil { + t.Fatalf("decode record tx: %v", err) + } + + cur.txs = append(cur.txs, tx.Hash()) + } + case e.GetBlockSeal() != nil && cur != nil: + cur.sealed = true + } + } + + if resp.GetLive() { + return gens + } + + next = &pb.RangeRequest{ + After: &pb.RangeRequest_Head{Head: resp.GetNext()}, + Limit: 512, + } + } +} + +// auditStore compares everything the store acked against the canonical +// chain the test declares (height -> ordered transaction hashes). +func auditStore(t *testing.T, h *harness, canonical map[uint64][]common.Hash) storeAudit { + t.Helper() + + gens := readAllGenerations(t, h) + + landedAt := map[common.Hash]uint64{} + indexIn := map[common.Hash]int{} + + for height, txs := range canonical { + for i, tx := range txs { + landedAt[tx] = height + indexIn[tx] = i + } + } + + var audit storeAudit + + for _, g := range gens { + for _, tx := range g.txs { + at, ok := landedAt[tx] + switch { + case !ok: + audit.Revoked = append(audit.Revoked, tx) + case at != g.height: + audit.Displaced = append(audit.Displaced, tx) + } + } + } + + // The newest generation at a height is what a reader takes as the truth + // for that height, so that is what has to agree with the block. + newest := map[uint64]generation{} + for _, g := range gens { + newest[g.height] = g + } + + for height, g := range newest { + block, ok := canonical[height] + if !ok { + continue // no block at this height: nothing to disagree with + } + + if len(g.txs) != len(block) { + audit.Mismatch = append(audit.Mismatch, height) + + continue + } + + for i := range g.txs { + if g.txs[i] != block[i] { + audit.Mismatch = append(audit.Mismatch, height) + + break + } + } + + // Order within the promised height, counted pairwise as storeprobe + // does: two records promised in one order must not land reversed. + for i := 0; i < len(g.txs); i++ { + for j := i + 1; j < len(g.txs); j++ { + a, b := g.txs[i], g.txs[j] + if landedAt[a] == height && landedAt[b] == height && + indexIn[a] > indexIn[b] { + audit.Reordered++ + } + } + } + } + + return audit +} + +// The auditor has to fail before it is worth trusting: each failure class is +// constructed deliberately and must be reported. An auditor that only ever +// says "clean" is what a green suite looked like while a devnet burned. +func TestAuditorDetectsEachFailureClass(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Three records acked at height 2, in this order. + t0, t1, t2 := testTx(t, 0), testTx(t, 1), testTx(t, 2) + foreignWindow(t, h, 2, parent, t0, t1, t2) + + block1 := []common.Hash{} + for _, tx := range sealed1Txs(t, h) { + block1 = append(block1, tx) + } + + cases := []struct { + name string + canonical map[uint64][]common.Hash + want func(storeAudit) bool + reason string + }{ + { + name: "clean", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash(), t1.Hash(), t2.Hash()}}, + want: func(a storeAudit) bool { return a.clean() }, + reason: "an exact match must audit clean", + }, + { + name: "revoked", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash(), t1.Hash()}}, + want: func(a storeAudit) bool { return len(a.Revoked) == 1 }, + reason: "a preconfirmed record in no block is a revocation", + }, + { + name: "displaced", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash(), t1.Hash()}, 3: {t2.Hash()}}, + want: func(a storeAudit) bool { return len(a.Displaced) == 1 }, + reason: "acked at 2 but landed at 3 is a displacement", + }, + { + name: "reordered", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash(), t2.Hash(), t1.Hash()}}, + want: func(a storeAudit) bool { return a.Reordered > 0 }, + reason: "same height, swapped order, is a reorder", + }, + { + name: "mismatch", + canonical: map[uint64][]common.Hash{1: block1, 2: {t0.Hash()}}, + want: func(a storeAudit) bool { return len(a.Mismatch) == 1 }, + reason: "the newest generation must equal the block at that height", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := auditStore(t, h, tc.canonical) + if !tc.want(got) { + t.Fatalf("%s: audit=%+v", tc.reason, got) + } + }) + } +} + +// sealed1Txs returns the transactions the store holds at height 1, so a test +// can declare a canonical chain that agrees with it. +func sealed1Txs(t *testing.T, h *harness) []common.Hash { + t.Helper() + + for _, g := range readAllGenerations(t, h) { + if g.height == 1 { + return g.txs + } + } + + return nil +} + +// canonicalFrom builds the canonical view a test declares by treating the +// blocks a publisher actually sealed as the chain, which is what happens when +// only one producer closes a height. +func canonicalFrom(blocks map[uint64][]*types.Transaction) map[uint64][]common.Hash { + out := map[uint64][]common.Hash{} + + for height, txs := range blocks { + hashes := make([]common.Hash, 0, len(txs)) + for _, tx := range txs { + hashes = append(hashes, tx.Hash()) + } + + out[height] = hashes + } + + return out +} + +// The end-to-end property, stated in the terms a consumer cares about: a +// single producer's published window must audit clean against the block it +// seals. No revocation, no displacement, no reorder, no mismatch. +func TestSoleProducerAuditsClean(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + txs := []*types.Transaction{testTx(t, 0), testTx(t, 1), testTx(t, 2)} + + header := testHeader(1, common.Hash{0xef}) + p.OpenBlock(1, header.Time, common.Hash{0xef}, header.GasLimit, header.BaseFee) + + for _, tx := range txs { + p.PublishTx(tx) + } + + p.SealBlock(blockFor(header, txs)) + waitHead(t, h, p, 5*time.Second) + + audit := auditStore(t, h, canonicalFrom(map[uint64][]*types.Transaction{1: txs})) + if !audit.clean() { + t.Fatalf("a sole producer broke its own preconfirmations: %+v", audit) + } +} + +// Refusal buys agreement: whichever producer closes the height, the store's +// newest generation there must equal the block. That is the mismatch class, +// and it is what a consumer reading the store's latest view depends on. +func TestRefusalPreventsMismatch(t *testing.T) { + h, a, b := twinPublishers(t) + parent := sealedParent(t, h, a, b) + + header := testHeader(2, parent) + + // Distinct nonces throughout: height 1 already holds nonce 0. + aTxs := []*types.Transaction{testTx(t, 1), testTx(t, 2)} + bTxs := []*types.Transaction{testTx(t, 7)} + + a.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + + for _, tx := range aTxs { + a.PublishTx(tx) + } + + waitDrained(t, a, 5*time.Second) + + b.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + + for _, tx := range bTxs { + b.PublishTx(tx) + } + + waitFor(t, 5*time.Second, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + + return b.hold.kind == holdSticky || b.unackedLocked() == 0 + }) + + sealA, sealB := decide(a), decide(b) + if sealA && sealB { + t.Fatal("both sealed: divergent blocks at one height") + } + + if !sealA && !sealB { + t.Fatal("neither sealed: the height never closes") + } + + winner := bTxs + if sealA { + winner = aTxs + } + + canonical := canonicalFrom(map[uint64][]*types.Transaction{2: winner}) + canonical[1] = sealed1Txs(t, h) + + audit := auditStore(t, h, canonical) + + if len(audit.Mismatch) > 0 { + t.Fatalf("the store's newest generation disagrees with the sealed "+ + "block at %v: a consumer reading the store gets the wrong "+ + "answer for that height", audit.Mismatch) + } +} + +// What refusal does NOT buy, stated plainly so nobody expects it to. Once a +// rival's records are acked they are preconfirmed, and they cannot also be in +// a block built from different content. They land later or not at all. +// +// Note the escape the publisher gets for free in the common case: a second +// producer appending to the same generation STALEs and never acks, so it +// never promises anything. This test uses a rival that opens its own +// generation, which the store accepts by design — the one shape where both +// sides really do hold promises. Nothing available to bor prevents it; only +// the store declining a second open at a live height removes the class. +func TestRivalGenerationStrandsItsOwnRecords(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + header := testHeader(2, parent) + ours := []*types.Transaction{testTx(t, 1), testTx(t, 2)} + + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + + for _, tx := range ours { + p.PublishTx(tx) + } + + waitDrained(t, p, 5*time.Second) + + // A rival opens its own generation on the current head, so its record is + // accepted and therefore preconfirmed. + rival := testTx(t, 9) + foreignWindow(t, h, 2, parent, rival) + + // Only one block exists at height 2, and it holds our content. + canonical := canonicalFrom(map[uint64][]*types.Transaction{2: ours}) + canonical[1] = sealed1Txs(t, h) + + audit := auditStore(t, h, canonical) + + stranded := len(audit.Revoked) + len(audit.Displaced) + if stranded != 1 { + t.Fatalf("expected exactly the rival's record stranded, got %d "+ + "(revoked=%d displaced=%d): more than that means contention is "+ + "costing the winner's promises too", + stranded, len(audit.Revoked), len(audit.Displaced)) + } + + // And the mismatch class is the one bor can still control: the store's + // newest generation at this height is the rival's, so a reader gets the + // wrong answer until the winner's flush corrects it. + if len(audit.Mismatch) != 1 || audit.Mismatch[0] != 2 { + t.Fatalf("expected the rival generation to leave height 2 mismatched, got %v", + audit.Mismatch) + } +} diff --git a/eth/sequencer/barrier.go b/eth/sequencer/barrier.go new file mode 100644 index 0000000000..8e8b285b2f --- /dev/null +++ b/eth/sequencer/barrier.go @@ -0,0 +1,203 @@ +package sequencer + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +// coverageSkip records a seal allowed through without a proven-complete +// window. Each reason is a different way this block can be sealed short of +// what the store holds, so they are worth telling apart rather than +// collapsing into one silent "true". +func coverageSkip(height uint64, reason string, ctx ...any) { + log.Info("Coverage check skipped", + append([]any{"number", height, "reason", reason}, ctx...)...) +} + +// AwaitSequenced blocks until the block about to be sealed is provably the +// store's sequence at this height. The rule is a mirror: seal only a block +// whose content is exactly the store's live window. +// +// The store elects the owner of every height: writes are a compare-and-swap +// on its head, so exactly one producer's open lands and everyone else +// STALEs. The owner's window is the store, its mirror check passes, and it +// seals — liveness by construction, no tie-break needed. A producer that +// finds itself behind adopts the store's window and rebuilds; false from +// here means exactly that (the resync signal is armed). +// +// A store that is merely unreachable, slow, or catching up after an outage +// returns true on the deadline: block production never waits on the store. +func (p *Publisher) AwaitSequenced(timeout time.Duration, number uint64, txs []*types.Transaction) bool { + deadline := time.Now().Add(timeout) + + for { + if p.failed.Load() { + return true // publishing is off; it must not gate production + } + + p.mu.Lock() + unacked := p.unackedLocked() + sticky := p.hold.kind == holdSticky + catchingUp := p.pendingFrom != 0 + p.mu.Unlock() + + switch { + case sticky: + // Another producer owns this height (our writes STALEd). Adopt + // their window instead of sealing beside it: the rebuild's + // boundary read collects it. + p.armResync() + + return false + case catchingUp: + // The store is behind us by construction while the backfill + // drains, so there is nothing here worth comparing against and + // nothing worth waiting for. Seal now; the mirror check resumes + // once the drain finishes. + publishCatchupSkip.Inc(1) + + return true + case unacked == 0: + return p.sealMirror(number, txs) + case time.Now().After(deadline): + publishBarrierTimeout.Inc(1) + + // The drain did not finish in budget, but that says nothing + // about whether this block carries what the store promised — + // and surrendering here drops the content check exactly when + // load makes it matter. A window still draining is a prefix of + // our own block, so the comparison holds mid-drain; it was a + // block with none of a 9523-record window that rode this exit + // out to a broadcast. + return p.sealMirror(number, txs) + } + + time.Sleep(2 * time.Millisecond) + } +} + +// armResync requests a rebuild that follows the store's sequence. Idempotent; +// the worker consumes it via ResyncNeeded. +func (p *Publisher) armResync() { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.resync { + p.resync = true + reconcileResync.Inc(1) + } +} + +// sealMirror reports whether the block about to be sealed is the store's +// sequence at this height. +// +// The comparison is against the block's own transactions, not against our +// journal position. A block built before an adoption leaves the journal +// perfectly in sync with the store while the block itself carries different +// content — position agreement says nothing about the transactions we are +// about to broadcast, and those are what the store promised. +func (p *Publisher) sealMirror(height uint64, txs []*types.Transaction) bool { + if p.unreachable.Load() { + coverageSkip(height, "store unreachable") + + return true // production never waits on a store we cannot reach + } + + ctx, cancel := context.WithTimeout(context.Background(), tailReadTimeout) + defer cancel() + + info, out := p.readTail(ctx) + if out != recOK { + // A window we cannot read is not a window we can prove divergent, + // and production never waits on the store. + coverageSkip(height, "tail unreadable", "outcome", int(out)) + + return true + } + + if info.haveSeal && info.lastSealHeight >= height { + coverageSkip(height, "height already sealed in the store") + + return true // the height moved on without us: the next build mutes + } + + // The store's live window at our height is the sequence this block owes + // its consumers. It must appear in the block, in order, from the front: + // anything else means transactions were promised at this height that + // the block does not deliver there. + if info.tipOpen && info.tipOpenHeight == height { + if storeTxs, ok := windowTxHashes(info.window); ok { + return p.mirrorVerdict(height, storeTxs, txs, len(storeTxs)) + } + } + + // The read shows nothing past the entries we published, so the store's + // window at this height is the one in our journal — already acked, so + // comparing the block against it compares it against the store. + p.mu.Lock() + ourTxs, atHead := p.journalWindowLocked(height), info.s == p.anchor + p.mu.Unlock() + + if atHead { + return p.mirrorVerdict(height, ourTxs, txs, len(ourTxs)) + } + + // The store holds content this block does not — a record extension of + // our window, or a generation we have not absorbed. Either way the + // answer is the same: adopt the store's sequence and rebuild on it. + // Never seal beside it, never republish over it. + p.mu.Lock() + p.resync = true + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + reconcileResync.Inc(1) + ours := len(p.journal.suffixFromHeight(height)) + p.mu.Unlock() + + log.Warn("Store holds content this block does not cover, adopting", + "number", height, "ours", ours, "storeWindow", len(info.window)) + + return false +} + +// ResyncNeeded reports whether a competing producer holds the height this +// node is building, meaning this build must stop — the next work cycle's +// build-start read follows the store's sequence. Reading consumes the +// signal; it fires at most once per height. +func (p *Publisher) ResyncNeeded() bool { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.resync { + return false + } + + p.resync = false + + return true +} + +// mirrorVerdict passes a block that delivers the promised sequence and arms +// a rebuild for one that does not. +func (p *Publisher) mirrorVerdict(height uint64, promised []common.Hash, + txs []*types.Transaction, window int, +) bool { + if windowLeadsHashes(promised, txHashes(txs)) { + return true + } + + barrierDivergedCount.Inc(1) + log.Warn("Sequencer block does not carry the sequence promised at this height, rebuilding", + "number", height, "promised", window, "block", len(txs)) + + p.mu.Lock() + p.resync = true + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + reconcileResync.Inc(1) + p.mu.Unlock() + + return false +} diff --git a/eth/sequencer/classify.go b/eth/sequencer/classify.go new file mode 100644 index 0000000000..c30ceb3d1b --- /dev/null +++ b/eth/sequencer/classify.go @@ -0,0 +1,848 @@ +package sequencer + +import ( + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/rlp" +) + +// applyTail classifies one tail read and applies the corrective action. +// Writes are reserved for replay of our own lineage, gap +// crossings, and the seal flush: a foreign tail with our window unsealed +// is held (the flush resolves it), never superseded. +func (p *Publisher) applyTail(info tailInfo) reconcileOutcome { + p.mu.Lock() + defer p.mu.Unlock() + + p.advanceStoreSealedTipLocked(info, "reconcile read") + + // Row 1: the store head is on our lineage — retire through it and let + // the send loop replay the rest. A diverged flush branches *below* the + // anchor (the adopt rewound to a prefix): its suffix does not extend + // the store head and must re-anchor onto it instead. + if info.s == p.anchor { + if fh, ok := p.pendingFlushLocked(); ok && !p.suffixExtendsLocked(info.s) { + return p.refoldLocked(info.s, fh, reconcileSupersede) + } + + return p.finishRow1Locked() + } + + if seq, ok := p.journal.findPost(info.s); ok { + p.ackedSeq = seq + p.anchor = info.s + p.confirmed = true + + return p.finishRow1Locked() + } + + return p.classifyForeignLocked(info) +} + +func (p *Publisher) finishRow1Locked() reconcileOutcome { + if items, covered := p.journal.after(p.ackedSeq); covered && p.pendingFrom == 0 { + if len(items) > 0 { + reconcileGapfill.Inc(1) + } + + p.anchored, p.confirmed = true, true + + return recOK + } + + // Eviction opened a gap behind the replayable suffix: jump, keeping + // everything a flush still owes the store (the oldest undelivered + // seal onward), else the current window (always retained). + keep := p.curHeight + if h := p.oldestPendingSealLocked(); h != 0 { + keep = h + } + + if keep == 0 { + keep = keepNone + } + + return p.refoldLocked(p.anchor, keep, reconcileForwardJump) +} + +// classifyForeignLocked resolves a foreign tail against what this publisher +// holds, in rank order: a flush in flight first, then the between-blocks +// re-anchor, else hold until our seal resolves the height. +func (p *Publisher) classifyForeignLocked(info tailInfo) reconcileOutcome { + if out, handled := p.classifyPendingFlushLocked(info); handled { + return out + } + + if p.curHeight == 0 { + return p.classifyBetweenBlocksLocked(info) + } + + // The store's window at our height may be a prefix of what we hold — + // our own earlier generation, or an identical build. Absorb it and + // deliver only the records it lacks. Republishing the whole window + // instead would write a fresh generation of data the store already + // has, which is pure duplication: every reader already has the prefix. + if p.completeExtendedWindowLocked(info, p.curHeight) { + return recOK + } + + // A window standing at our own height on our own parent means another + // producer is building this slot and reached the store first. Adopt it: + // rebuilding onto their sequence is what keeps the two blocks from + // diverging, and adoption extends their window rather than replacing + // it, so nothing already published is revoked. Only this exact shape + // arms the signal: a transient STALE, a foreign window at another + // height, or one on a different parent must never restart a + // legitimate build. + if ourParent, ok := p.ourOpenParentLocked(); ok && + info.tipOpen && info.tipOpenHeight == p.curHeight && + info.tipOpenParent == ourParent { + p.resync = true + + reconcileResync.Inc(1) + log.Warn("Sequencer found a competing producer at our height, requesting rebuild", + "number", p.curHeight, "parent", ourParent) + } + + // Our window is unsealed: nothing we hold outranks the store yet. + // Buffer everything unacked until the seal flush resolves the height; + // a build the chain moved past heals at the next build-start check. + return p.holdStaleLocked() +} + +// classifyPendingFlushLocked handles a flush in flight: a sealed trailing +// window with undelivered entries — the sealed, broadcast block overrides +// whatever the store shows at its height. handled=false means either no +// flush is pending, or a canonical seal past the flush outranks it and +// classification continues as if no flush were pending. +func (p *Publisher) classifyPendingFlushLocked(info tailInfo) (reconcileOutcome, bool) { + flushHeight, ok := p.pendingFlushLocked() + if !ok { + return recOK, false + } + + // The store's standing window may be OUR window extended by records + // that were still in flight when we read it (a dying producer's drain, + // an adoption snapshot racing the stream). When it is a strict prefix + // of what this flush carries, complete it in place — absorbing the + // store's copy and delivering only the remainder — instead of + // superseding content that matches ours. + if p.completeExtendedWindowLocked(info, flushHeight) { + return recOK, true + } + + // A seal standing at our own flush height is another producer claiming + // the same slot (duplicate signing keys), or our own copy already + // delivered. Superseding it is only right when the chain chose OUR + // block: if the store's seal is the canonical one, overwriting it + // would leave the store's newest generation disagreeing with the + // chain. We lost — yield instead. + if info.haveSeal && info.lastSealHeight == flushHeight && + p.chain != nil && p.foreignSealCanonicalLocked(info) { + return p.yieldFlushLocked(info), true + } + + // A foreign seal already stands at our flush height and our own block + // is still unbroadcast: the store says this height is closed, and the + // head we would chain onto encodes that seal — we cannot claim not to + // know. Publishing our window and seal over it would put a second + // sealed generation at a height the store already closed, for a block + // that may never exist on any chain. + // + // Unconditional while the gate is pending: the chain's opinion is not + // required, and waiting for it is what let a refold slip through in + // the ~140ms before the winner's block imported. Once the gate clears + // (broadcast, or the liveness timeout) this height becomes ordinary + // repair, where canonicality decides supersede versus yield. A decoded + // seal also settles the gate itself: this hold is the verdict, not a + // wait for one. + if info.haveSeal && info.lastSealHeight == flushHeight && + p.gatePendingLocked(flushHeight) { + p.resolveGateFromSealLocked(info) + + return p.holdStaleLocked(), true + } + + // Only a sealed height past the flush outranks it. A foreign unsealed + // open above the flush height is a contender that already lost to this + // seal — stranding the flush on it would leave the height sealed on + // chain but forever open in the store. + if !info.haveSeal || info.lastSealHeight <= flushHeight { + if !p.mayDisplaceWindowLocked(info, flushHeight) { + return p.holdStaleLocked(), true + } + + return p.reanchorFlushLocked(info), true + } + + // A foreign seal stands above our pending flush. Dropping our seal for + // it is right only when that lineage is the canonical chain — we were + // genuinely reorged past. If the chain has not accepted it (a reorg + // loser, or a lineage not yet imported), the store tail is not + // authoritative: hold our seal and let our next flush re-anchor onto + // the canonical chain. Never drop a canonical seal for a non-canonical + // store tail. + if !p.foreignSealCanonicalLocked(info) { + return p.holdStaleLocked(), true + } + + return recOK, false // canonical seal past the flush outranks it +} + +// restoreAbandonedDebtLocked folds the sealed heights a refold is about to +// abandon back into the pending range. The backfill advances pendingFrom at +// build time, before delivery is confirmed; when the batch then loses its +// head race and the next classification abandons it, nothing else remembers +// those heights are owed — a devnet pinned an outage height as a permanent +// hole exactly this way. Restoring the debt costs at most a duplicate +// delivery. +func (p *Publisher) restoreAbandonedDebtLocked(suffix []journalItem) { + kept := uint64(0) + if len(suffix) > 0 { + kept = suffix[0].seq + } + + lo, hi := uint64(0), uint64(0) + + for _, it := range p.journal.items { + if it.seq <= p.ackedSeq || it.kind != entrySeal { + continue + } + + if kept != 0 && it.seq >= kept { + break + } + + if lo == 0 || it.height < lo { + lo = it.height + } + + if it.height > hi { + hi = it.height + } + } + + if lo == 0 { + return + } + + if p.pendingFrom == 0 || lo < p.pendingFrom { + p.pendingFrom = lo + } + + if hi > p.pendingTo { + p.pendingTo = hi + } + + log.Warn("Sequencer restoring abandoned sealed heights to the backfill", + "from", lo, "to", hi) +} + +// mayDisplaceWindowLocked decides whether this flush may overwrite a live +// foreign window, having looked at what that window holds. +// +// A re-anchor rebuilds our own entries onto the store head; it never ingests +// what stands behind that head. So displacing a live window silently drops +// every record it holds that our block does not carry — records the store +// already acked, which is to say preconfirmations. That is only defensible +// when our block is the one the chain kept: then the store must end up +// holding our content or it disagrees with the chain. When our block did not +// win, displacing is pure destruction, and holding lets the winner's flush +// resolve the height. +func (p *Publisher) mayDisplaceWindowLocked(info tailInfo, flushHeight uint64) bool { + // A foreign live window at the flush height is content this flush would + // overwrite; one above it is re-anchored past just as surely — a silent + // fold-past exactly there is how two full acked generations ended up + // buried under an empty sealed block. Either shape needs proof, and a + // flush the chain ratified still proceeds: the re-anchor only moves our + // lineage, so a window above stays the newest generation at its own + // height, where the next build's boundary read adopts it. + foreignWindow := info.tipOpen && info.tipOpenHeight >= flushHeight && + p.relateWindowLocked(info, info.tipOpenHeight) == windowForeign + + sealCandidate := info.sealDecoded && info.lastSealHeight == flushHeight + + if !foreignWindow && !sealCandidate { + return true // nothing foreign standing here to displace + } + + // A sealed foreign generation at our height is a stronger claim than a + // live window, and it gets the same treatment: chaining our flush past + // it reads the head's bytes without understanding what they encode — a + // closed height. Only a decoded seal counts, and our own seal already + // standing there is re-delivery, not displacement. + ours, sealedHere := p.sealedHashAtLocked(flushHeight) + foreignSealed := sealCandidate && !(sealedHere && info.lastSealHash == ours) + + if !foreignWindow && !foreignSealed { + return true // the standing seal is our own re-delivery + } + + if p.chain == nil { + return true + } + + // Only affirmative proof that the chain kept our block licenses a + // displacement. "Not decided yet" is not proof — on a devnet the + // winner's block imported 134ms after a displacement made on exactly + // that reasoning, and the records it destroyed were already acked. + if !sealedHere || p.chain.GetCanonicalHash(flushHeight) != ours { + // With the gate still pending this withhold is the verdict: an + // unbroadcast block can never become canonical, so waiting for the + // proof would deadlock against producing it. Refuse the broadcast + // and the rebuild adopts what stands here instead. The liveness + // fallback's tolerance covers exactly the store's seal — a foreign + // window still refuses it. + if p.gatePendingLocked(flushHeight) && + (foreignWindow || !p.gate.tolerateSealed) { + p.gate.verdict = gateLost + } + + log.Warn("Sequencer withholding flush: the chain has not kept this block", + "number", flushHeight, "ours", ours) + + return false + } + + // Only what this block does not carry is actually orphaned, and only a + // window standing at the flush height itself can be counted truthfully: + // a displaced foreign seal's generation is not in this read, and the + // trailing window of a higher height compared against this height's + // block once reported 1,960 phantom orphans for a displacement that + // destroyed nothing. + if foreignWindow && info.tipOpenHeight == flushHeight { + if orphans := p.orphanedByDisplacementLocked(info, flushHeight); orphans > 0 { + windowDisplacedRecords.Inc(int64(orphans)) + log.Warn("Sequencer flush displacing acked records this block does not carry", + "number", flushHeight, "orphaned", orphans) + } + } + + return true +} + +// orphanedByDisplacementLocked counts the displaced window's transactions +// that our block does not deliver at this height. The caller has already +// established that the canonical block at this height is ours, so the chain +// holds the authoritative copy of what we delivered — the journal has moved +// on by seal time and no longer describes the window. +func (p *Publisher) orphanedByDisplacementLocked(info tailInfo, height uint64) int { + theirs, ok := windowTxHashes(info.window) + if !ok || p.chain == nil { + return 0 + } + + block := p.chain.GetBlockByNumber(height) + if block == nil { + return 0 + } + + ours := make(map[common.Hash]struct{}, block.Transactions().Len()) + for _, tx := range block.Transactions() { + ours[tx.Hash()] = struct{}{} + } + + orphans := 0 + + for _, h := range theirs { + if _, carried := ours[h]; !carried { + orphans++ + } + } + + return orphans +} + +// sealedHashAtLocked returns the block hash our undelivered seal carries for +// a height. +func (p *Publisher) sealedHashAtLocked(height uint64) (common.Hash, bool) { + for i := len(p.journal.items) - 1; i >= 0; i-- { + it := p.journal.items[i] + if it.kind != entrySeal || it.height != height { + continue + } + + header, err := decodeSealHeader(it.entry.GetBlockSeal().GetHeader()) + if err != nil { + return common.Hash{}, false + } + + return header.Hash(), true + } + + return common.Hash{}, false +} + +// yieldFlushLocked abandons our sealed window because the chain chose +// another producer's block at that height. Our content is non-canonical; +// republishing it would overwrite the winner's correct content. Counted +// apart from supersede — here we are the loser, not the winner. +func (p *Publisher) yieldFlushLocked(info tailInfo) reconcileOutcome { + log.Warn("Sequencer yielding to canonical seal at our own height", + "height", info.lastSealHeight, "canonical", info.lastSealHash) + reconcileYield.Inc(1) + + return p.refoldLocked(info.s, keepNone, nil) +} + +// reanchorFlushLocked re-anchors every undelivered seal onto the store +// head, oldest window first — flushes stack behind a reconcile backoff, +// and a skipped one would leave its height unsealed in the store forever. +func (p *Publisher) reanchorFlushLocked(info tailInfo) reconcileOutcome { + keepFrom := p.oldestPendingSealLocked() + + var counter *metrics.Counter + + switch { + case (info.tipOpen && info.tipOpenHeight >= keepFrom) || + (info.haveSeal && info.lastSealHeight >= keepFrom): + counter = reconcileSupersede // foreign content on re-anchored ground is overridden + case info.haveSeal && info.lastSealHeight+1 == keepFrom: + counter = nil // contiguous re-delivery: nothing overridden or abandoned + default: + counter = reconcileForwardJump // sealed ground missing below the flush + } + + return p.refoldLocked(info.s, keepFrom, counter) +} + +// classifyBetweenBlocksLocked resolves a foreign tail with no build in +// progress. A tail ending in an adoptable unsealed window is not history +// to rebase past — it is what the next build-start check adopts (a +// restarted producer resumes its own window this way): anchor at the +// window's base, keeping the window ahead of the anchor where the +// build-start read can collect it. Otherwise rebase onto the store head +// and let the next open extend it; abandoning unconfirmed history here is +// the forward-jump case (counted only when entries are actually left +// behind). +func (p *Publisher) classifyBetweenBlocksLocked(info tailInfo) reconcileOutcome { + if len(info.window) > 0 && p.unackedLocked() == 0 { + base := commitment.Head(info.window[0].GetBlockOpen().GetPrefixCommitment()) + if base != p.anchor { + p.rebaseLocked(base) + } + + p.anchored = true + + return recOK + } + + log.Info("Sequencer rebasing onto store head", "head", info.s) + + return p.refoldLocked(info.s, keepNone, reconcileForwardJump) +} + +// holdStaleLocked buffers everything unacked until a seal flush resolves +// the height. +func (p *Publisher) holdStaleLocked() reconcileOutcome { + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + p.anchored = true + + return recOK +} + +// foreignSealCanonicalLocked reports whether the store's trailing seal is +// the block the local chain considers canonical at that height. A nil +// chain (test publishers) preserves the pre-check behavior. +func (p *Publisher) foreignSealCanonicalLocked(info tailInfo) bool { + if !info.haveSeal { + return false + } + + if p.chain == nil { + return true + } + + return p.chain.GetCanonicalHash(info.lastSealHeight) == info.lastSealHash +} + +// pendingFlushLocked reports whether the lineage carries any undelivered +// seal — a flush in flight, possibly stacked behind retry pacing and +// possibly with the next build's window already opened on top — and the +// height of the newest one. Back-to-back builds leave only a sliver of +// time where a seal is the trailing item, so keying on the trailing item +// alone would hide the stack from nearly every reconcile. +func (p *Publisher) pendingFlushLocked() (uint64, bool) { + for i := len(p.journal.items) - 1; i >= 0; i-- { + it := p.journal.items[i] + if it.seq <= p.ackedSeq { + return 0, false + } + + if it.kind == entrySeal { + return it.height, true + } + } + + return 0, false +} + +// oldestPendingSealLocked returns the height of the first undelivered +// seal — the start of a possibly stacked flush suffix. +func (p *Publisher) oldestPendingSealLocked() uint64 { + for _, it := range p.journal.items { + if it.seq > p.ackedSeq && it.kind == entrySeal { + return it.height + } + } + + return 0 +} + +// suffixExtendsLocked reports whether the first undelivered journal item +// folds directly onto s — the send loop can replay it verbatim. +func (p *Publisher) suffixExtendsLocked(s commitment.Head) bool { + items, covered := p.journal.after(p.ackedSeq) + if !covered || len(items) == 0 { + return true + } + + return items[0].pre == s +} + +// completeExtendedWindowLocked completes a window in place when the store's +// standing window is a prefix of what we hold: the store's copy is absorbed +// as acked and only the records it lacks (plus the seal, once sealed) fold +// onto its head. Nothing already published is re-sent and no new generation +// is written — readers already have the prefix. +func (p *Publisher) completeExtendedWindowLocked(info tailInfo, flushHeight uint64) bool { + ours, items, ok := p.matchExtendedWindowLocked(info, flushHeight) + if !ok { + return false + } + + p.absorbExtendedWindowLocked(info, ours, items, flushHeight) + + return true +} + +// matchExtendedWindowLocked reports whether the store's standing window is +// a strict prefix of the flush at flushHeight (same open, records a +// leading byte-subsequence of ours, same order), returning our flush +// suffix and the store's window parsed as journal items. +func (p *Publisher) matchExtendedWindowLocked(info tailInfo, flushHeight uint64) (ours, items []journalItem, ok bool) { + if !info.tipOpen || info.tipOpenHeight != flushHeight || len(info.window) == 0 { + return nil, nil, false + } + + // The window may still be building (no seal yet): a mid-build + // re-anchor completes in place just as a flush does. + ours = p.journal.suffixFromHeight(flushHeight) + if len(ours) == 0 || ours[0].kind != entryOpen { + return nil, nil, false + } + + if len(info.window) > len(ours)-1 { // window can cover at most our records + return nil, nil, false + } + + for i, entry := range info.window { + if !contentEqual(entry, ours[i].entry) { + return nil, nil, false + } + } + + stored, items, ok := parseWindow(info) + if !ok || stored.number != flushHeight { + return nil, nil, false + } + + return ours, items, true +} + +// absorbExtendedWindowLocked absorbs the store's copy as confirmed lineage +// and re-folds the remainder (records past the stored prefix, and the +// seal) onto its head for the send loop to deliver. A refold failure +// latches fail(); nothing else to classify. +func (p *Publisher) absorbExtendedWindowLocked(info tailInfo, ours, items []journalItem, flushHeight uint64) { + // Older stacked flushes below this window are abandoned by the swap: + // their heights are already represented byte-identically in the store + // lineage this window folds on (the open's parent hash pins it), but + // the entries themselves never deliver — count them. + if abandoned := p.unackedLocked() - len(ours); abandoned > 0 { + publishDropMeter.Mark(int64(abandoned)) + } + + fresh := newJournal() + fresh.nextSeq = p.journal.nextSeq + + for _, it := range items { + fresh.append(it.entry, it.pre, it.post, it.kind, it.height, fresh.nextSeq, it.txHashes) + } + + p.ackedSeq = fresh.nextSeq - 1 + cur := info.s + + for _, item := range ours[len(info.window):] { + entry, next, err := refoldEntry(cur, item) + if err != nil { + p.fail("refold flush remainder", "err", err) + + return + } + + fresh.append(entry, cur, next, item.kind, item.height, 0, item.txHashes) + cur = next + } + + log.Info("Sequencer completing extended window in place", "number", flushHeight, + "stored", len(info.window), "delivering", len(ours)-len(info.window)) + p.installSwapLocked(fresh, cur, info.s) +} + +// backfillLocked drains the oldest owed blocks from the chain database onto +// cur, appending to fresh as undelivered entries, and returns the new fold +// head. One call rebuilds at most a journal byte budget's worth; the +// remainder stays pending, and the drain resumes on the next re-anchor — +// oldest first, so the store's sealed tip never advances past a gap it +// would then refuse to fill. Blocks the store already sealed are not owed; +// blocks missing from the database (pruned) are the one gap backfill cannot +// close, skipped as a counted forward jump. +func (p *Publisher) backfillLocked(fresh *journal, cur commitment.Head) commitment.Head { + if p.pendingFrom == 0 { + return cur + } + + // The whole pending range drains, storeSealedTip notwithstanding. The + // tip is the newest seal, not proof of anything below it: live flushes + // seal heights above the gap while it waits, and a store restart can + // shed acked writes, so "the store has these" was twice false on a + // devnet — pending heights skipped on the tip stayed holes forever. + // Re-delivering a height the store does have only adds an identical + // duplicate generation, which costs churn, not correctness. + lo, hi := p.pendingFrom, p.pendingTo + + log.Info("Sequencer backfill starting", "from", lo, "to", hi, + "storeSealedTip", p.storeSealedTip) + + if lo > hi || p.chain == nil { + if p.chain == nil && hi >= lo { + reconcileForwardJump.Inc(1) + } + + // An inverted range cannot arise anymore — the collapse merges + // rather than clobbers — so hitting one means new accounting is + // broken somewhere. Dropping the debt is still the only safe move + // (a wedged drain is worse), but never a silent one again. + if lo > hi { + log.Warn("Sequencer backfill pending range inverted, dropping the debt", + "from", lo, "to", hi) + } + + p.pendingFrom, p.pendingTo, p.pendingEntries = 0, 0, 0 + + return cur + } + + var ( + budget, rebuilt int + jumped bool + n = lo + started = time.Now() + ) + + for ; n <= hi; n++ { + block := p.chain.GetBlockByNumber(n) + if block == nil { + jumped = true // pruned: unfillable + + continue + } + + // Always take at least one block, or a block larger than the + // budget would wedge the drain forever. + if budget += int(block.Size()); budget > backfillBatchBytes && rebuilt > 0 { + break + } + + before := len(fresh.items) + + next, ok := p.appendBlockLocked(fresh, cur, block) + if !ok { + return cur // fail() latched + } + + rebuilt += len(fresh.items) - before + cur = next + } + + if n > hi { + p.pendingFrom, p.pendingTo, p.pendingEntries = 0, 0, 0 + } else { + p.pendingFrom = n + if p.pendingEntries > rebuilt { + p.pendingEntries -= rebuilt + } else { + p.pendingEntries = 0 + } + + log.Info("Sequencer backfill batch, remainder pending", + "rebuilt-through", n-1, "pending", n, "to", hi, + "batch", time.Since(started)) + + backfillBatchTimer.UpdateSince(started) + } + + if jumped { + reconcileForwardJump.Inc(1) + log.Warn("Sequencer backfill skipped pruned blocks", "pending", lo, "through", hi) + } + + return cur +} + +// appendBlockLocked rebuilds one chain block as journal entries folded onto +// cur — the same encodings the live build publishes, so a rebuilt block is +// byte-identical to the original. +func (p *Publisher) appendBlockLocked(fresh *journal, cur commitment.Head, block *types.Block) (commitment.Head, bool) { + header := block.Header() + n := header.Number.Uint64() + + open := openEntry(commitment.OpenContext{ + Number: n, + Timestamp: header.Time, + ParentHash: header.ParentHash, + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, cur) + + next, err := foldEntry(cur, open) + if err != nil { + p.fail("backfill fold open", "number", n, "err", err) + + return cur, false + } + + fresh.append(open, cur, next, entryOpen, n, 0, nil) + cur = next + + for _, tx := range block.Transactions() { + raw, err := tx.MarshalBinary() + if err != nil { + p.fail("backfill encode transaction", "hash", tx.Hash(), "err", err) + + return cur, false + } + + rec := recordEntry(raw, cur) + next = commitment.FoldTxs(cur, [][]byte{raw}) + fresh.append(rec, cur, next, entryRecord, n, 0, []common.Hash{tx.Hash()}) + cur = next + } + + raw, err := rlp.EncodeToBytes(header) + if err != nil { + p.fail("backfill encode header", "number", n, "err", err) + + return cur, false + } + + seal := sealEntry(raw, cur) + next = commitment.FoldSeal(cur, commitment.SealedHash(raw)) + fresh.append(seal, cur, next, entrySeal, n, 0, nil) + + return next, true +} + +// refoldLocked swaps the lineage: journal items from the first window at or +// above keepFrom are re-prefixed onto s (mid-window republish); +// everything earlier is abandoned to supersession convergence. +func (p *Publisher) refoldLocked(s commitment.Head, keepFrom uint64, counter *metrics.Counter) reconcileOutcome { + suffix := p.journal.suffixFromHeight(keepFrom) + + // Entries behind the kept suffix are abandoned to supersession + // convergence; count the unconfirmed ones as dropped. Collapsed + // entries are excluded — the backfill accounts for them (rebuilt or + // skipped) itself. A re-anchor that abandons nothing (a fresh process + // resuming at the store tail) is not a forward jump — only count the + // jump when entries are left behind. + abandoned := p.unackedLocked() - len(suffix) - p.pendingEntries + if abandoned > 0 { + publishDropMeter.Mark(int64(abandoned)) + p.restoreAbandonedDebtLocked(suffix) + } else if counter == reconcileForwardJump { + counter = nil + } + + fresh := newJournal() + fresh.nextSeq = p.journal.nextSeq + p.ackedSeq = fresh.nextSeq - 1 + + cur := p.backfillLocked(fresh, s) + + // Everything appended so far is the backfill batch; the suffix follows. + batchCeiling := fresh.nextSeq - 1 + + for _, item := range suffix { + entry, next, err := refoldEntry(cur, item) + if err != nil { + p.fail("refold entry", "err", err) + + return recTerminal + } + + // Refolded entries are all undelivered: watermark 0 keeps a + // stacked re-anchor from evicting its own seals. + fresh.append(entry, cur, next, item.kind, item.height, 0, item.txHashes) + cur = next + } + + p.installSwapLocked(fresh, cur, s) + + // An unfinished drain gates the suffix behind the batch: a suffix seal + // landing out of order would advance the store's sealed tip past the + // gap, and the floor above would then skip the middle forever. The + // drain cycle re-arms this ceiling each batch and lifts it with the + // last one. + if p.pendingFrom != 0 { + p.hold = hold{after: batchCeiling, kind: holdBuild} + } + + if counter != nil { + counter.Inc(1) + } + + return recOK +} + +// installSwapLocked commits a rebuilt lineage: it points the journal, head and +// anchor at the swap result and resets every field a lineage swap must clear +// together — the confirmed-anchor latches, any adopted window (folded on +// state that no longer exists), and the send hold — then re-derives the open +// window and wakes the transport. Centralizing this keeps the "reset as a +// unit" invariant in one place across the refold and complete-in-place paths. +func (p *Publisher) installSwapLocked(fresh *journal, head, anchor commitment.Head) { + p.journal = fresh + p.head = head + p.anchor = anchor + p.anchored, p.confirmed = true, true + p.adopt = nil + p.hold = clearedHold() + p.syncWindowLocked() + publishQueueGauge.Update(int64(p.unackedLocked())) + p.signalWake() +} + +// syncWindowLocked recomputes the open-window bookkeeping after a lineage +// swap. A worker mid-window whose entries were all dropped must not have its +// remaining records published without their open (awaitOpen). +func (p *Publisher) syncWindowLocked() { + if idx := p.journal.openStart(); idx >= 0 { + p.curHeight = p.journal.items[idx].height + p.awaitOpen = false + + return + } + + if p.curHeight != 0 { + p.awaitOpen = true + } + + p.curHeight = 0 +} diff --git a/eth/sequencer/classify_test.go b/eth/sequencer/classify_test.go new file mode 100644 index 0000000000..42cb7e26c8 --- /dev/null +++ b/eth/sequencer/classify_test.go @@ -0,0 +1,474 @@ +package sequencer + +import ( + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// lineagePublisher builds a bare publisher holding a confirmed one-block +// lineage plus an open window at height 2 with one record. +func lineagePublisher(t *testing.T, chain chainReader) (*Publisher, commitment.Head) { + t.Helper() + + p := barePublisher() + p.chain = chain + + header := testHeader(1, common.Hash{0xef}) + p.OpenBlock(1, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + + p.OpenBlock(2, header.Time+1, header.Hash(), header.GasLimit, header.BaseFee) + p.PublishTx(testTx(t, 0)) + + // Everything through the seal of block 1 is store-confirmed. + items, _ := p.journal.after(0) + sealItem := items[1] + p.ackedSeq = sealItem.seq + p.anchor = sealItem.post + p.confirmed = true + + return p, sealItem.post +} + +func TestApplyTailRow1Replay(t *testing.T) { + p, anchor := lineagePublisher(t, &fakeChain{}) + + if out := p.applyTail(tailInfo{s: anchor}); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if !p.anchored || p.head == anchor { + t.Fatalf("must anchor and keep the unconfirmed suffix (anchored=%v)", p.anchored) + } +} + +func TestApplyTailRow1FindPost(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) + + // The store confirmed one entry further than we knew: the open of 2. + items, _ := p.journal.after(p.ackedSeq) + openItem := items[0] + + if out := p.applyTail(tailInfo{s: openItem.post}); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.ackedSeq != openItem.seq || p.anchor != openItem.post { + t.Fatalf("frontier not advanced: acked=%d", p.ackedSeq) + } +} + +// A foreign tail while our window is unsealed never writes: the publisher +// holds and the seal flush resolves the height. +func TestApplyTailForeignUnsealedHolds(t *testing.T) { + cases := []struct { + name string + info tailInfo + }{ + {name: "open window at pending", info: tailInfo{s: commitment.Head{0x99}, tipOpen: true, tipOpenHeight: 2}}, + {name: "open window above pending", info: tailInfo{s: commitment.Head{0x33}, tipOpen: true, tipOpenHeight: 5}}, + {name: "sealed past pending", info: tailInfo{s: commitment.Head{0x55}, haveSeal: true, lastSealHeight: 2, lastSealHash: common.Hash{0xcc}}}, + {name: "store behind", info: tailInfo{s: commitment.Head{0x77}, haveSeal: true, lastSealHeight: 1, lastSealHash: common.Hash{0xaa}}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + journalBefore := len(p.journal.items) + + if out := p.applyTail(tc.info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.hold.after != p.ackedSeq { + t.Fatalf("unsealed window against a foreign tail must gate unacked entries (ceiling=%d acked=%d)", p.hold.after, p.ackedSeq) + } + + if len(p.journal.items) != journalBefore { + t.Fatal("hold must not touch the lineage") + } + }) + } +} + +// The same foreign tails with the window sealed (flush in flight) re-anchor +// the sealed window onto the store head — supersede for contention shapes, +// forward jump for a store still behind. +func TestApplyTailSealedFlushReanchors(t *testing.T) { + cases := []struct { + name string + info tailInfo + wantNone bool + }{ + {name: "open window at pending", info: tailInfo{s: commitment.Head{0x99}, tipOpen: true, tipOpenHeight: 2}}, + {name: "sealed divergent at pending", info: tailInfo{s: commitment.Head{0x55}, haveSeal: true, lastSealHeight: 2, lastSealHash: common.Hash{0xcc}}}, + {name: "parent sealed, clean tail", info: tailInfo{s: commitment.Head{0x77}, haveSeal: true, lastSealHeight: 1, lastSealHash: common.Hash{0xaa}}, wantNone: true}, + {name: "unsealed contender above flush", info: tailInfo{s: commitment.Head{0x44}, haveSeal: true, lastSealHeight: 1, lastSealHash: common.Hash{0xaa}, tipOpen: true, tipOpenHeight: 3}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + // Seal the pending window locally: the flush owns the height now. + header2 := testHeader(2, common.Hash{0x01}) + sealOnChain(p, fc, header2, []*types.Transaction{testTx(t, 0)}) + + // The block went out on the gate's liveness path (Unknown): + // from here the flush is post-broadcast repair, which is the + // behavior under test. + p.ConfirmSeal(time.Millisecond) + + supersedes := reconcileSupersede.Snapshot().Count() + jumps := reconcileForwardJump.Snapshot().Count() + + if out := p.applyTail(tc.info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + items, covered := p.journal.after(p.ackedSeq) + if !covered || len(items) == 0 { + t.Fatalf("re-anchored suffix missing: covered=%v len=%d", covered, len(items)) + } + + if items[0].kind != entryOpen || items[0].height != 2 { + t.Fatalf("suffix starts with kind %d height %d", items[0].kind, items[0].height) + } + + if got := commitment.Head(entryPrefix(items[0].entry)); got != tc.info.s { + t.Fatalf("re-prefixed onto %x, want %x", got, tc.info.s) + } + + if items[len(items)-1].kind != entrySeal { + t.Fatal("the flush suffix must end with the seal") + } + + if tc.wantNone { + if got := reconcileSupersede.Snapshot().Count(); got != supersedes { + t.Fatalf("re-delivery counted as supersede: %d -> %d", supersedes, got) + } + + if got := reconcileForwardJump.Snapshot().Count(); got != jumps { + t.Fatalf("re-delivery counted as jump: %d -> %d", jumps, got) + } + } else { + if got := reconcileSupersede.Snapshot().Count(); got != supersedes+1 { + t.Fatalf("supersede counter = %d, want %d", got, supersedes+1) + } + } + }) + } +} + +// Flushes stacked behind a reconcile backoff all re-anchor — oldest +// window first — so no sealed-on-chain height is left unsealed in the +// store (the quad-strand shape). +func TestStackedFlushesAllDeliver(t *testing.T) { + fc2 := &fakeChain{} + p, _ := lineagePublisher(t, fc2) + + // Two blocks seal while nothing delivers: the journal carries two + // complete flush windows beyond the acked prefix. + header2 := testHeader(2, common.Hash{0x01}) + sealOnChain(p, fc2, header2, []*types.Transaction{testTx(t, 0)}) + p.OpenBlock(3, header2.Time+1, header2.Hash(), header2.GasLimit, header2.BaseFee) + tx := testTx(t, 1) + p.PublishTx(tx) + sealOnChain(p, fc2, testHeader(3, header2.Hash()), []*types.Transaction{tx}) + + // The next build has already opened on top of the stack — the flush + // must stay visible through the trailing open (back-to-back builds + // leave almost no trailing-seal moments for a reconcile to land in). + header3 := testHeader(3, common.Hash{0x02}) + p.OpenBlock(4, header3.Time+1, header3.Hash(), header3.GasLimit, header3.BaseFee) + + // A foreign contender window stands above both flushes. + foreign := commitment.Head{0x44} + info := tailInfo{s: foreign, haveSeal: true, lastSealHeight: 1, lastSealHash: common.Hash{0xaa}, tipOpen: true, tipOpenHeight: 3} + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + items, covered := p.journal.after(p.ackedSeq) + if !covered || len(items) == 0 { + t.Fatalf("re-anchored suffix missing: covered=%v len=%d", covered, len(items)) + } + + if items[0].kind != entryOpen || items[0].height != 2 { + t.Fatalf("suffix starts with kind %d height %d, want the oldest flush window", items[0].kind, items[0].height) + } + + if got := commitment.Head(entryPrefix(items[0].entry)); got != foreign { + t.Fatalf("re-prefixed onto %x, want %x", got, foreign) + } + + seals := 0 + + for _, it := range items { + if it.kind == entrySeal { + seals++ + } + } + + if seals != 2 { + t.Fatalf("re-anchored suffix carries %d seals, want both", seals) + } +} + +// A seal flush whose window does not mirror the sealed block rebuilds the +// window — without erasing older flushes still undelivered behind it. +func TestSealRebuildPreservesStackedFlushes(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) + + // First flush: seals the standing window for height 2, undelivered. + header2 := testHeader(2, common.Hash{0x01}) + p.SealBlock(blockFor(header2, []*types.Transaction{testTx(t, 0)})) + + // Second build diverges from what seals (its window will not mirror + // the block), forcing the rebuild path. + p.OpenBlock(3, header2.Time+1, header2.Hash(), header2.GasLimit, header2.BaseFee) + p.PublishTx(testTx(t, 1)) + + sealedTx := testTx(t, 2) + p.SealBlock(blockFor(testHeader(3, header2.Hash()), []*types.Transaction{sealedTx})) + + p.mu.Lock() + defer p.mu.Unlock() + + heights := map[uint64]int{} + + for _, it := range p.journal.items { + if it.kind == entrySeal && it.seq > p.ackedSeq { + heights[it.height]++ + } + } + + if heights[2] != 1 || heights[3] != 1 { + t.Fatalf("stacked flushes lost in rebuild: undelivered seals per height = %v", heights) + } +} + +func TestApplyTailBetweenBlocksRebases(t *testing.T) { + p := barePublisher() + + // The store's sealed lineage is far past our stale flush and is the + // canonical chain locally: the chain moved on, so a rebase is right + // (a canonical sealed height past the flush outranks it). + p.chain = &fakeChain{canonical: map[uint64]common.Hash{8: {0xbb}}} + + header := testHeader(1, common.Hash{0xef}) + p.OpenBlock(1, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + p.curHeight = 0 // between blocks + + foreign := commitment.Head{0x22} + info := tailInfo{s: foreign, haveSeal: true, lastSealHeight: 8, lastSealHash: common.Hash{0xbb}, tipOpen: true, tipOpenHeight: 9} + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.head != foreign || p.anchor != foreign { + t.Fatalf("head/anchor not rebased: %x/%x", p.head, p.anchor) + } + + if items, _ := p.journal.after(p.ackedSeq); len(items) != 0 { + t.Fatal("rebase must purge the lineage") + } +} + +func TestProbeFindsStoreEdge(t *testing.T) { + h := startHarness(t) + + seed := newTestPublisher(t, h, nil) + sealed := publishBlock(t, seed, 1, common.Hash{0xef}, 1) + sealed2 := publishBlock(t, seed, 2, sealed.Hash(), 1) + publishBlock(t, seed, 3, sealed2.Hash(), 1) + waitHead(t, h, seed, 5*time.Second) + seed.Close() + + p := newTestPublisher(t, h, nil) + waitFor(t, 5*time.Second, func() bool { return p.isAnchored() }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + edge, found, err := p.read.probeDown(ctx, 10) + if err != nil || !found || edge != 3 { + t.Fatalf("probeDown = %d/%v/%v, want 3", edge, found, err) + } + + up, err := p.read.probeUp(ctx, 1) + if err != nil || up != 3 { + t.Fatalf("probeUp = %d/%v, want 3", up, err) + } +} + +// A lagging ack from a lineage an adoption has since replaced must not +// regress the frontier — the phantom eviction gap it faked used to force +// a spurious forward-jump superseding a window under silent adopt. +func TestLaggingAckAfterLineageSwapIgnored(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) + + inflight, _ := p.journal.after(p.ackedSeq) + openItem := inflight[0] + + var win []*pb.Entry + for _, it := range inflight { + win = append(win, it.entry) + } + + p.mu.Lock() + info := tailInfo{s: p.head, tipOpen: true, tipOpenHeight: 2, window: win} + + a, items, ok := parseWindow(info) + if !ok { + p.mu.Unlock() + t.Fatal("window unparseable") + } + + p.adoptWindowLocked(info, a, items) + acked := p.ackedSeq + p.mu.Unlock() + + jumps := reconcileForwardJump.Snapshot().Count() + + // The transport's lagging ack for the pre-swap open arrives. + p.retire(openItem, time.Now()) + + p.mu.Lock() + after, anchor := p.ackedSeq, p.anchor + p.mu.Unlock() + + if after != acked { + t.Fatalf("stale ack regressed ackedSeq: %d -> %d", acked, after) + } + + if out := p.applyTail(tailInfo{s: anchor}); out != recOK { + t.Fatalf("row-1 outcome: %v", out) + } + + if got := reconcileForwardJump.Snapshot().Count(); got != jumps { + t.Fatalf("spurious forward-jump: %d -> %d", jumps, got) + } +} + +// completeExtendedWindowLocked abandons older stacked flushes — their +// content is pinned byte-identically in the store lineage the window +// folds on — but must count them as drops. +func TestExtendedCompletionCountsAbandonedStack(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) + + header2 := testHeader(2, common.Hash{0x01}) + p.SealBlock(blockFor(header2, []*types.Transaction{testTx(t, 0)})) + + p.OpenBlock(3, header2.Time+1, header2.Hash(), header2.GasLimit, header2.BaseFee) + tx := testTx(t, 1) + p.PublishTx(tx) + p.SealBlock(blockFor(testHeader(3, header2.Hash()), []*types.Transaction{tx})) + + p.mu.Lock() + defer p.mu.Unlock() + + ours := p.journal.suffixFromHeight(3) + win := []*pb.Entry{ours[0].entry, ours[1].entry} + info := tailInfo{s: ours[1].post, tipOpen: true, tipOpenHeight: 3, window: win} + + drops := publishDropMeter.Snapshot().Count() + + if !p.completeExtendedWindowLocked(info, 3) { + t.Fatal("completion did not fire") + } + + if got := publishDropMeter.Snapshot().Count(); got <= drops { + t.Fatalf("abandoned stack uncounted: %d -> %d", drops, got) + } +} + +// A foreign seal above our pending flush that the local chain does NOT +// consider canonical (a reorg loser, or a lineage not yet imported) must +// not drop our canonical seal: hold and let our next flush re-anchor. +func TestForeignSealAboveNonCanonicalHolds(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{}) // nil canonical map: seal not canonical + + header2 := testHeader(2, common.Hash{0x01}) + p.SealBlock(blockFor(header2, []*types.Transaction{testTx(t, 0)})) + + forwardBefore := reconcileForwardJump.Snapshot().Count() + + // A foreign seal at height 3 stands above our pending flush for 2, but + // GetCanonicalHash(3) is zero (not this hash) → not canonical. + info := tailInfo{s: commitment.Head{0x22}, haveSeal: true, lastSealHeight: 3, lastSealHash: common.Hash{0xbb}} + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + // Our pending seal must survive (held, not dropped) and no jump counted. + items, covered := p.journal.after(p.ackedSeq) + if !covered || len(items) == 0 || items[len(items)-1].kind != entrySeal { + t.Fatalf("pending seal dropped for a non-canonical foreign tail: covered=%v len=%d", covered, len(items)) + } + + if p.hold.kind != holdSticky { + t.Fatalf("expected holdSticky, got holdKind=%d", p.hold.kind) + } + + if got := reconcileForwardJump.Snapshot().Count(); got != forwardBefore { + t.Fatalf("dropped a canonical seal (forward jump %d -> %d)", forwardBefore, got) + } +} + +// A foreign seal above our pending flush that IS canonical locally means +// we were genuinely reorged past: drop the stale seal and rebase. +func TestForeignSealAboveCanonicalRebases(t *testing.T) { + p, _ := lineagePublisher(t, &fakeChain{canonical: map[uint64]common.Hash{3: {0xbb}}}) + + header2 := testHeader(2, common.Hash{0x01}) + p.SealBlock(blockFor(header2, []*types.Transaction{testTx(t, 0)})) + + info := tailInfo{s: commitment.Head{0x22}, haveSeal: true, lastSealHeight: 3, lastSealHash: common.Hash{0xbb}} + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.anchor != (commitment.Head{0x22}) { + t.Fatalf("canonical reorg-past must rebase: anchor=%x", p.anchor) + } +} diff --git a/eth/sequencer/consumer.go b/eth/sequencer/consumer.go new file mode 100644 index 0000000000..8d0cd539b5 --- /dev/null +++ b/eth/sequencer/consumer.go @@ -0,0 +1,428 @@ +package sequencer + +import ( + "context" + "errors" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/log" +) + +const consumerRetryDelay = 2 * time.Second + +// Consumer follows the sequence store stream on an RPC node, re-executes it +// on top of canonical state, and fills the preconf receipt Index that +// eth_getTransactionReceipt consults for not-yet-imported transactions. +// +// Position and application are handled separately, per the design's +// chain-everything-apply-selectively rule: only commitment gaps and transport +// errors abandon the stream position (warm resume by running head, falling +// back to a cold block anchor, falling back to the earliest retained entry); +// application problems — unknown parents, unavailable state, execution or +// seal divergence — void the speculative work and skip forward until an open +// record re-anchors on a canonical block. +type Consumer struct { + chain *core.BlockChain + endpoint string + index *Index + + cancel context.CancelFunc + done chan struct{} +} + +// NewConsumer returns a stopped consumer. Determinism preconditions (Rio +// active, coinbase map present) are re-checked per session, not here — a +// node still syncing pre-Rio history becomes eligible once it catches up. +func NewConsumer(endpoint string, chain *core.BlockChain) (*Consumer, error) { + if chain.Config().Bor == nil { + return nil, errors.New("sequencer consumer requires a bor chain") + } + + return &Consumer{ + chain: chain, + endpoint: endpoint, + index: NewIndex(), + }, nil +} + +// Index exposes the preconf receipts for the RPC layer. +func (c *Consumer) Index() *Index { + return c.index +} + +// Start launches the stream-follow loop. +func (c *Consumer) Start() { + ctx, cancel := context.WithCancel(context.Background()) + c.cancel = cancel + c.done = make(chan struct{}) + + go c.run(ctx) + go c.evictLoop(ctx) +} + +// Close stops the consumer and waits for the follow loop to exit. +func (c *Consumer) Close() { + if c.cancel != nil { + c.cancel() + <-c.done + } +} + +// deterministic reports whether the producer's execution context is +// reproducible at the current head: pre-Rio the EVM coinbase is the +// producer's own address, unknowable pre-seal; post-Rio it is +// CalculateCoinbase from the chain config — reproducible only when the +// coinbase map is set. +func (c *Consumer) deterministic() error { + config := c.chain.Config().Bor + + head := c.chain.CurrentBlock().Number + if !config.IsRio(head) { + return errors.New("rio fork not active at current head") + } + + if common.HexToAddress(config.CalculateCoinbase(head.Uint64())) == (common.Address{}) { + return errors.New("chain config has no coinbase map") + } + + return nil +} + +func (c *Consumer) run(ctx context.Context) { + defer close(c.done) + + var sess *session + + for { + var err error + if derr := c.deterministic(); derr != nil { + err = fmt.Errorf("preconf re-execution not deterministic yet: %w", derr) + } else { + sess, err = c.follow(ctx, sess) + } + + if ctx.Err() != nil { + return + } + + log.Warn("Sequence stream session ended", "err", err) + + select { + case <-ctx.Done(): + return + case <-time.After(consumerRetryDelay): + } + } +} + +// evictLoop drops preconf receipts for heights the canonical chain has +// imported — the normal receipt path serves them from there on. +func (c *Consumer) evictLoop(ctx context.Context) { + heads := make(chan core.ChainHeadEvent, 16) + sub := c.chain.SubscribeChainHeadEvent(heads) + + defer sub.Unsubscribe() + + for { + select { + case <-ctx.Done(): + return + case head := <-heads: + c.index.EvictThrough(head.Header.Number.Uint64()) + case <-sub.Err(): + return + } + } +} + +// resumeRequest picks the stream position: warm (the session's running head), +// cold (a block anchor at the canonical head), or from the earliest retained +// entry. attempt counts NOT_FOUND fallbacks within one session start. +func (c *Consumer) resumeRequest(sess *session, attempt int) *pb.StreamRequest { + if sess != nil && sess.seeded && attempt == 0 { + return &pb.StreamRequest{After: &pb.StreamRequest_Head{Head: sess.head.Bytes()}} + } + + if attempt <= 1 { + return &pb.StreamRequest{After: &pb.StreamRequest_Block{Block: c.chain.CurrentBlock().Number.Uint64()}} + } + + return &pb.StreamRequest{} +} + +// follow runs one streaming session. It returns the session for a warm +// resume when the stream position is still valid, or nil when position was +// lost (commitment gap, malformed entry) and the next attempt must re-anchor. +func (c *Consumer) follow(ctx context.Context, sess *session) (*session, error) { + conn, err := grpc.NewClient(c.endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return sess, fmt.Errorf("dial sequence store: %w", err) + } + + defer func() { + if cerr := conn.Close(); cerr != nil { + log.Warn("Sequence store connection close", "err", cerr) + } + }() + + client := pb.NewConsumerServiceClient(conn) + + for attempt := 0; ; attempt++ { + stream, serr := client.Stream(ctx, c.resumeRequest(sess, attempt)) + if serr != nil { + return sess, fmt.Errorf("open stream: %w", serr) + } + + if attempt > 0 || sess == nil { + // Any non-warm position invalidates the old fold state. + sess = &session{consumer: c} + c.index.Reset() + } + + sess, err = c.consume(stream, sess) + if status.Code(err) == codes.NotFound && attempt < 2 { + continue + } + + return sess, err + } +} + +func (c *Consumer) consume(stream pb.ConsumerService_StreamClient, sess *session) (*session, error) { + for { + frame, err := stream.Recv() + if err != nil { + return sess, fmt.Errorf("stream recv: %w", err) + } + + entry := frame.GetEntry() + if entry == nil { + log.Info("Sequence stream live", "head", fmt.Sprintf("%x", sess.head[:8])) + + continue + } + + if err := sess.handle(entry); err != nil { + // Position lost: the caller must re-anchor, not resume. + return nil, err + } + } +} + +// session is one consistent stretch of the stream: a running commitment +// head, the speculative execution state, and the speculative seal hashes for +// BLOCKHASH resolution. env == nil between blocks and while skipping. +type session struct { + consumer *Consumer + head commitment.Head + seeded bool + env *blockEnv // block currently being applied + parked *state.StateDB // post-state of the last speculative seal + tip common.Hash // last speculatively sealed hash + sealed map[uint64]common.Hash +} + +// handle verifies one entry against the commitment chain, folds it, and +// applies it best-effort. An error means the stream position itself is +// invalid; application problems skip instead (void speculative work, wait +// for a re-anchoring open). +func (s *session) handle(entry *pb.Entry) error { + prefix, next, err := s.fold(entry) + if err != nil { + return err + } + + if !s.seeded { + // Cold start: adopt the stream's prefix as the seed; integrity from + // here comes from the chain and the self-authenticating seals. + s.head = prefix + s.seeded = true + + var refold error + if _, next, refold = s.fold(entry); refold != nil { + return refold + } + } else if prefix != s.head { + return fmt.Errorf("commitment gap: entry prefix %x != running head %x", prefix[:8], s.head[:8]) + } + + s.apply(entry) + s.head = next + + return nil +} + +// fold checks an entry's wire shape, computes its post-fold head over the +// running head, and returns the entry's claimed prefix alongside it. +func (s *session) fold(entry *pb.Entry) (commitment.Head, commitment.Head, error) { + prefix := entryPrefix(entry) + if len(prefix) != 32 { + return commitment.Head{}, commitment.Head{}, errors.New("malformed entry prefix") + } + + if open := entry.GetBlockOpen(); open != nil && len(open.GetParentHash()) != 32 { + return commitment.Head{}, commitment.Head{}, errors.New("malformed open parent hash") + } + + next, err := foldEntry(s.head, entry) + if err != nil { + return commitment.Head{}, commitment.Head{}, err + } + + return commitment.Head(prefix), next, nil +} + +func (s *session) apply(entry *pb.Entry) { + switch kind := entry.GetKind().(type) { + case *pb.Entry_BlockOpen: + s.applyOpen(kind.BlockOpen) + case *pb.Entry_Record: + s.applyRecord(kind.Record) + case *pb.Entry_BlockSeal: + s.applySeal(kind.BlockSeal) + } +} + +// skip voids the speculative work from a height upward and waits for the +// next re-anchoring open. +func (s *session) skip(from uint64, reason string, args ...any) { + log.Warn("Preconf application skipped: "+reason, args...) + s.consumer.index.ClearFrom(from) + s.env = nil + s.parked = nil +} + +// applyOpen starts a speculative block. A canonical parent is preferred as +// the base even on the happy path — it bounds how long one speculative +// StateDB lives; the parked post-seal state covers parents the chain hasn't +// imported yet. +func (s *session) applyOpen(open *pb.BlockOpen) { + parent := common.BytesToHash(open.GetParentHash()) + number := open.GetBlockNumber() + + if s.env != nil { + // A new open while a block is still open is a producer rebuild of + // the in-progress height; its state is unusable. + s.skip(s.env.header.Number.Uint64(), "producer rebuilt in-progress block", "number", number) + } + + if header := s.consumer.chain.GetHeaderByHash(parent); header != nil && + s.consumer.chain.GetCanonicalHash(header.Number.Uint64()) == parent { + if number != header.Number.Uint64()+1 { + s.skip(number, "open height is not parent height+1", "number", number, "parent", header.Number) + + return + } + + statedb, err := s.consumer.chain.StateAt(header.Root) + if err != nil { + s.skip(number, "parent state unavailable", "parent", parent, "err", err) + + return + } + + s.consumer.index.ClearFrom(number) + s.pruneSealed(number) + + s.tip = parent + s.parked = nil + s.env = newBlockEnv(s.consumer.chain, statedb, open, s.sealed) + + return + } + + if parent == s.tip && s.parked != nil { + s.env = newBlockEnv(s.consumer.chain, s.parked, open, s.sealed) + s.parked = nil + + return + } + + s.skip(number, "open parent neither canonical nor speculative tip", "parent", parent) +} + +func (s *session) applyRecord(record *pb.Record) { + if s.env == nil { + return // skipping until the next re-anchoring open + } + + for _, raw := range record.GetTransactions() { + start := time.Now() + + tx, receipt, err := s.env.applyRaw(raw) + if err != nil { + s.skip(s.env.header.Number.Uint64(), "re-execution diverged", "err", err) + + return + } + + s.consumer.index.Add(tx, receipt) + preconfApplyTimer.UpdateSince(start) + } +} + +func (s *session) applySeal(seal *pb.BlockSeal) { + if s.env == nil { + return // skipping until the next re-anchoring open + } + + sealed, err := decodeSealHeader(seal.GetHeader()) + if err != nil { + s.skip(s.env.header.Number.Uint64(), "undecodable sealed header", "err", err) + + return + } + + if err := s.env.checkSeal(sealed); err != nil { + s.skip(s.env.header.Number.Uint64(), "seal cross-check failed", "err", err) + + return + } + + sealedHash := common.Hash(commitment.SealedHash(seal.GetHeader())) + number := s.env.header.Number.Uint64() + + s.consumer.index.Seal(number, sealedHash) + + if s.sealed == nil { + s.sealed = map[uint64]common.Hash{} + } + + s.sealed[number] = sealedHash + + // BLOCKHASH reaches at most 256 back; older speculative hashes would + // otherwise accumulate for the whole parked stretch. + for height := range s.sealed { + if height+256 < number { + delete(s.sealed, height) + } + } + + // Park the post-block state as the base for the next open. + s.tip = sealedHash + s.parked = s.env.statedb + s.env = nil +} + +// pruneSealed drops speculative seal hashes that a canonical re-anchor +// superseded (at or above the new height) or that BLOCKHASH can no longer +// reach (more than 256 below it). +func (s *session) pruneSealed(number uint64) { + for height := range s.sealed { + if height >= number || height+256 < number { + delete(s.sealed, height) + } + } +} diff --git a/eth/sequencer/consumer_test.go b/eth/sequencer/consumer_test.go new file mode 100644 index 0000000000..bfd182adbe --- /dev/null +++ b/eth/sequencer/consumer_test.go @@ -0,0 +1,114 @@ +package sequencer + +import ( + "math/big" + "strings" + "testing" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" +) + +func testSession() *session { + return &session{consumer: &Consumer{index: NewIndex()}} +} + +// A cold-started session adopts the first entry's prefix as its seed and +// advances past it, so a mid-stream resume verifies from that point on. +func TestSessionColdSeedAdoptsTheFirstPrefix(t *testing.T) { + seed := commitment.Head{0xaa, 0xbb} + raw := []byte{0x01, 0x02} + sess := testSession() + + if err := sess.handle(recordEntry(raw, seed)); err != nil { + t.Fatalf("cold-seed handle: %v", err) + } + + if want := commitment.FoldTxs(seed, [][]byte{raw}); sess.head != want { + t.Fatalf("head %x, want the seed folded past the entry %x", sess.head[:8], want[:8]) + } +} + +// A seeded session accepts only entries whose prefix extends its running +// head; anything else is a gap and invalidates the stream position. +func TestSessionRejectsACommitmentGap(t *testing.T) { + seed := commitment.Seed(testChainID) + sess := testSession() + + if err := sess.handle(recordEntry([]byte{0x01}, seed)); err != nil { + t.Fatalf("seeding handle: %v", err) + } + + next := recordEntry([]byte{0x02}, sess.head) + if err := sess.handle(next); err != nil { + t.Fatalf("chained handle: %v", err) + } + + gapped := recordEntry([]byte{0x03}, commitment.Head{0xde, 0xad}) + if err := sess.handle(gapped); err == nil || !strings.Contains(err.Error(), "commitment gap") { + t.Fatalf("gapped entry must fail the position, got %v", err) + } +} + +// Wire entries with short commitment or hash fields must fail the fold, not +// panic the fixed-width conversions. +func TestSessionFoldRejectsMalformedEntries(t *testing.T) { + sess := testSession() + + short := recordEntry([]byte{0x01}, commitment.Head{}) + short.GetRecord().PrefixCommitment = []byte{0x01} + + if _, _, err := sess.fold(short); err == nil || !strings.Contains(err.Error(), "malformed entry prefix") { + t.Fatalf("short prefix must fail the fold, got %v", err) + } + + open := openEntry(commitment.OpenContext{Number: 7, BaseFee: big.NewInt(1)}, commitment.Head{}) + open.GetBlockOpen().ParentHash = []byte{0x01, 0x02} + + if _, _, err := sess.fold(open); err == nil || !strings.Contains(err.Error(), "malformed open parent hash") { + t.Fatalf("short parent hash must fail the fold, got %v", err) + } +} + +// The consumer's fold must agree with the publisher-side foldEntry on every +// entry kind — both ends of the wire chain the same bytes. +func TestSessionFoldMatchesFoldEntry(t *testing.T) { + head := commitment.Seed(testChainID) + oc := commitment.OpenContext{ + Number: 3, + Timestamp: 100, + ParentHash: [32]byte{0x0f}, + GasLimit: 30_000_000, + BaseFee: big.NewInt(32), + } + + entries := []*pb.Entry{ + openEntry(oc, head), + recordEntry([]byte{0x01, 0x02}, head), + sealEntry([]byte{0x0a, 0x0b}, head), + } + + for _, entry := range entries { + sess := testSession() + sess.head = head + sess.seeded = true + + prefix, next, err := sess.fold(entry) + if err != nil { + t.Fatalf("fold: %v", err) + } + + if prefix != head { + t.Fatalf("claimed prefix %x, entry was built on %x", prefix[:8], head[:8]) + } + + want, err := foldEntry(head, entry) + if err != nil { + t.Fatalf("foldEntry: %v", err) + } + + if next != want { + t.Fatalf("fold %x diverges from foldEntry %x", next[:8], want[:8]) + } + } +} diff --git a/eth/sequencer/debt_test.go b/eth/sequencer/debt_test.go new file mode 100644 index 0000000000..be310d7049 --- /dev/null +++ b/eth/sequencer/debt_test.go @@ -0,0 +1,206 @@ +package sequencer + +import ( + "testing" + + "github.com/0xPolygon/sequence-store-proto/commitment" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// A probe-inferred boundary is not a decoded seal: recording it as the +// sealed tip told every backfill the height was owed nothing, and an +// outage's partial delivery at exactly that height became a permanent hole +// (devnet height 364: partial 340/1399, skipped by every repairer). +func TestInferredBoundaryDoesNotAdvanceTheSealedTip(t *testing.T) { + p := barePublisher() + p.chain = &fakeChain{} + + p.applyTail(tailInfo{haveSeal: true, lastSealHeight: 9}) // no sealDecoded + + p.mu.Lock() + tip := p.storeSealedTip + p.mu.Unlock() + + if tip != 0 { + t.Fatalf("storeSealedTip = %d from an inferred boundary, want 0: "+ + "only a decoded seal proves anything is sealed", tip) + } +} + +// Priming from an undecoded boundary includes the boundary height itself: it +// may be a partial delivery, and a duplicate generation is cheaper than a +// hole. +func TestPrimeIncludesAnUndecodedBoundary(t *testing.T) { + p := barePublisher() + p.chain = &fakeChain{} + + p.mu.Lock() + p.primeBackfillLocked(tailInfo{haveSeal: true, lastSealHeight: 7}, 10) + from, to := p.pendingFrom, p.pendingTo + p.mu.Unlock() + + if from != 7 || to != 9 { + t.Fatalf("pending = [%d,%d], want [7,9]: an inferred boundary at 7 "+ + "may be a partial window and must be re-delivered", from, to) + } +} + +// And a decoded seal at the boundary starts the range above it, as before. +func TestPrimeExcludesADecodedSeal(t *testing.T) { + p := barePublisher() + p.chain = &fakeChain{} + + p.mu.Lock() + p.primeBackfillLocked(tailInfo{haveSeal: true, sealDecoded: true, lastSealHeight: 7}, 10) + from, to := p.pendingFrom, p.pendingTo + p.mu.Unlock() + + if from != 8 || to != 9 { + t.Fatalf("pending = [%d,%d], want [8,9]", from, to) + } +} + +// A refold that abandons unacked sealed heights must put them back in the +// pending range: the backfill advanced past them at build time, and without +// restoration nothing remembers the debt. +func TestAbandonedBackfillHeightsReturnToPending(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + parent := common.Hash{0xef} + for n := uint64(4); n <= 6; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, nil) + parent = header.Hash() + } + + // The journal holds rebuilt-but-unacked seals for 4..5, as after a + // backfill batch whose delivery lost its head race. pendingFrom has + // already advanced past them. + p.mu.Lock() + cur := commitment.Seed(testChainID) + fresh := newJournal() + for n := uint64(4); n <= 5; n++ { + next, ok := p.appendBlockLocked(fresh, cur, chain.blocks[n]) + if !ok { + p.mu.Unlock() + t.Fatal("appendBlock failed") + } + cur = next + } + p.journal = fresh + p.ackedSeq = 0 + p.pendingFrom, p.pendingTo = 6, 6 + + // A forward-jump refold abandons everything (empty suffix). The restored + // debt is drained by this same refold's backfill, so the proof is the + // rebuilt content: the fresh journal must carry the abandoned heights + // again, not silently forget them. + p.refoldLocked(commitment.Head{0x99}, ^uint64(0), nil) + + rebuilt := map[uint64]bool{} + for _, it := range p.journal.items { + if it.kind == entrySeal { + rebuilt[it.height] = true + } + } + p.mu.Unlock() + + for _, h := range []uint64{4, 5, 6} { + if !rebuilt[h] { + t.Fatalf("height %d not rebuilt after its batch was abandoned: "+ + "the debt was forgotten (rebuilt: %v)", h, rebuilt) + } + } +} + +// A partial abandonment restores only what was actually abandoned: heights +// kept by the refold's suffix are still owed by the journal itself, and +// re-adding them to pending would double-deliver every surviving flush. +func TestPartialAbandonmentRestoresOnlyTheAbandonedHeights(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + parent := common.Hash{0xef} + for n := uint64(4); n <= 6; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, nil) + parent = header.Hash() + } + + p.mu.Lock() + cur := commitment.Seed(testChainID) + fresh := newJournal() + for n := uint64(4); n <= 6; n++ { + next, ok := p.appendBlockLocked(fresh, cur, chain.blocks[n]) + if !ok { + p.mu.Unlock() + t.Fatal("appendBlock failed") + } + cur = next + } + p.journal = fresh + p.ackedSeq = 0 + + // The refold keeps height 6 (a live flush) and abandons 4-5. + p.refoldLocked(commitment.Head{0x99}, 6, nil) + from, to := p.pendingFrom, p.pendingTo + + rebuilt := map[uint64]int{} + for _, it := range p.journal.items { + if it.kind == entrySeal { + rebuilt[it.height]++ + } + } + p.mu.Unlock() + + // 4 and 5 return via the restored pending range (drained by this same + // refold's backfill); 6 survives via the suffix — exactly once each. + for _, h := range []uint64{4, 5, 6} { + if rebuilt[h] != 1 { + t.Fatalf("height %d appears %d times, want exactly once "+ + "(pending was [%d,%d])", h, rebuilt[h], from, to) + } + } +} + +// A collapse during an active drain reaches heights below pendingFrom — the +// stranded batch suffix is the journal's oldest content — and must merge +// into the pending range. Clobbering pendingTo downward inverted the range, +// which the backfill read as "nothing owed" and zeroed: 25 heights of a +// 50-block outage evaporated as permanent holes exactly that way. +func TestCollapseMergesIntoThePendingRange(t *testing.T) { + p := barePublisher() + p.pendingFrom, p.pendingTo = 254, 270 + + parent := common.Hash{0xee} + + for n := uint64(228); n <= 231; n++ { + header := testHeader(n, parent) + p.OpenBlock(n, header.Time, parent, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + parent = header.Hash() + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.unackedSealsLocked() > journalHotSeals { + t.Fatalf("collapse did not run: %d unacked seals", p.unackedSealsLocked()) + } + + if p.pendingFrom != 228 { + t.Fatalf("pendingFrom = %d, want 228: the collapsed height must extend the range downward", p.pendingFrom) + } + + if p.pendingTo != 270 { + t.Fatalf("pendingTo = %d, want 270: the upper bound must never move down", p.pendingTo) + } + + if p.pendingEntries == 0 { + t.Fatal("collapsed entries were not accounted") + } +} diff --git a/eth/sequencer/degraded_test.go b/eth/sequencer/degraded_test.go new file mode 100644 index 0000000000..2abc850465 --- /dev/null +++ b/eth/sequencer/degraded_test.go @@ -0,0 +1,194 @@ +package sequencer + +import ( + "context" + "sync" + "testing" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ethereum/go-ethereum/common" +) + +// flakyConsumer drops a fixed share of tail reads, reproducing the devnet +// state that a clean in-process store cannot: "rung out of budget" fired 68 +// and 77 times in one five-minute window, and every decision that depends on +// a read was making it blind that often. +// +// Failures are counted rather than random so a failure here reproduces +// exactly. slow adds latency instead of an error, for the case where the read +// returns but too late to be useful. +type flakyConsumer struct { + pb.ConsumerServiceClient + + mu sync.Mutex + calls int + failEvery int + slow time.Duration + failures int +} + +func (c *flakyConsumer) Range(ctx context.Context, req *pb.RangeRequest, opts ...grpc.CallOption) (*pb.RangeResponse, error) { + c.mu.Lock() + c.calls++ + drop := c.failEvery > 0 && c.calls%c.failEvery == 0 + + if drop { + c.failures++ + } + + slow := c.slow + c.mu.Unlock() + + if slow > 0 { + time.Sleep(slow) + } + + if drop { + return nil, status.Error(codes.DeadlineExceeded, "tail read out of budget") + } + + return c.ConsumerServiceClient.Range(ctx, req, opts...) +} + +// degrade wraps a publisher's consumer client so its reads start failing. +func degrade(p *Publisher, failEvery int, slow time.Duration) *flakyConsumer { + f := &flakyConsumer{failEvery: failEvery, slow: slow} + + p.mu.Lock() + defer p.mu.Unlock() + + f.ConsumerServiceClient = p.read.cons + p.read.cons = f + + return f +} + +// blockableConsumer can be switched off and on, so a test can correlate a +// decision with whether the read behind it worked. A raw failure ratio does +// not discriminate: readTail issues several Range calls per check, so +// "seals <= successful reads" holds even when every decision is blind. +type blockableConsumer struct { + pb.ConsumerServiceClient + + mu sync.Mutex + blocked bool +} + +func (c *blockableConsumer) Range(ctx context.Context, req *pb.RangeRequest, opts ...grpc.CallOption) (*pb.RangeResponse, error) { + c.mu.Lock() + blocked := c.blocked + c.mu.Unlock() + + if blocked { + return nil, status.Error(codes.DeadlineExceeded, "tail read out of budget") + } + + return c.ConsumerServiceClient.Range(ctx, req, opts...) +} + +func (c *blockableConsumer) block(v bool) { + c.mu.Lock() + defer c.mu.Unlock() + + c.blocked = v +} + +// A contested height refuses without needing a read at all: the STALE that +// armed the hold is the store's own verdict, so degraded reads cannot turn +// contention into a blind seal — the failure that once sealed 10 of 10 +// contested heights while the store was unreadable. An uncontested drained +// producer with an unreadable store still seals: production never waits. +func TestContestedRefusalSurvivesUnreadableStore(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 1)) + waitDrained(t, p, 5*time.Second) + + gate := &blockableConsumer{} + + p.mu.Lock() + gate.ConsumerServiceClient = p.read.cons + p.read.cons = gate + p.mu.Unlock() + + gate.block(true) + + // Uncontested, drained, store unreadable: liveness seals. + if !awaitOurWindow(p, 300*time.Millisecond) { + t.Fatal("an unreadable store blocked an uncontested seal") + } + + // Contested: the hold refuses with no read required. + p.mu.Lock() + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + p.mu.Unlock() + + for height := uint64(2); height <= 11; height++ { + p.mu.Lock() + p.curHeight = height + p.mu.Unlock() + + if awaitOurWindow(p, 200*time.Millisecond) { + t.Fatalf("sealed contested height %d with the store unreadable", height) + } + } +} + +// Sustained contention across many heights, not one. The devnet had 36 +// contested heights in a single run; a per-height test cannot show damage +// that only accumulates. +func TestSustainedContentionAuditsBounded(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + degrade(p, 4, 0) + + const heights = 8 + + rivals := 0 + + for height := uint64(2); height <= heights; height++ { + // A rival opens its own generation and gets acked, then we contend. + foreignWindow(t, h, height, parent, testTx(t, 9)) + rivals++ + + p.mu.Lock() + p.curHeight = height + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + p.mu.Unlock() + + awaitOurWindow(p, 200*time.Millisecond) + + parent = common.Hash{byte(height)} // each rival window on its own tip + } + + // Nothing this node published can be stranded: it never sealed, so the + // only records at risk are the rivals' own. + audit := auditStore(t, h, map[uint64][]common.Hash{1: sealed1Txs(t, h)}) + + if len(audit.Displaced) > 0 { + t.Fatalf("sustained contention displaced %d records that were "+ + "promised at a specific height", len(audit.Displaced)) + } + + if len(audit.Revoked) > rivals { + t.Fatalf("revoked %d records but only %d rivals published: contention "+ + "is costing more than the losers' own promises", + len(audit.Revoked), rivals) + } +} diff --git a/eth/sequencer/derive_test.go b/eth/sequencer/derive_test.go new file mode 100644 index 0000000000..262b4b7f1d --- /dev/null +++ b/eth/sequencer/derive_test.go @@ -0,0 +1,126 @@ +package sequencer + +import ( + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "github.com/ethereum/go-ethereum/common" + "google.golang.org/grpc" +) + +// The head a walk lands on has to be derived from the entries the walk read. +// Taking the store's word for it makes a passing CAS mean only that we echoed +// back the value we were handed, which is not a check on shared history. +func TestFoldedHeadMustMatchTheReportedHead(t *testing.T) { + seed := commitment.Seed(1) + open := &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: 2, + BlockTimestamp: 1700000002, + ParentHash: common.Hash{0xaa}.Bytes(), + GasLimit: 30_000_000, + BaseFee: big25gwei(), + PrefixCommitment: seed.Bytes(), + }}} + + real, err := foldEntry(seed, open) + if err != nil { + t.Fatalf("fold: %v", err) + } + + tests := []struct { + name string + reported commitment.Head + want bool + }{ + {"head the entries produce", real, true}, + {"head the entries do not produce", commitment.Head{0x9e}, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + f := &folder{cur: seed, ok: true} + f.fold(open) + f.reached(tc.reported) + + if f.ok != tc.want { + t.Fatalf("explained = %v, want %v", f.ok, tc.want) + } + }) + } +} + +// A block-anchored walk that comes back empty derives no head — but the empty +// page is itself the answer a boundary read needs: nothing follows the block +// we started at, so opening there cannot land on another producer's window. +func TestEmptyPageStillExplainsABoundary(t *testing.T) { + f := &folder{ok: true, awaiting: true} + f.reached(commitment.Head{0x11}) + + if !f.ok { + t.Fatal("an empty page was treated as an unexplained head: every " + + "clean boundary would hold and the chain would stop opening blocks") + } +} + +// Entries that arrive without a base we can establish summarize into a head +// we cannot account for — the case that must not be trusted. +func TestUnbasedEntriesLeaveTheHeadUnexplained(t *testing.T) { + f := &folder{ok: true, awaiting: true} + f.fold(&pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{}}}) + f.reached(commitment.Head{0x11}) + + if f.ok { + t.Fatal("a mid-window start explained a head it never derived") + } +} + +// tamperingConsumer reports a head that its own entries do not produce. +type tamperingConsumer struct { + pb.ConsumerServiceClient +} + +func (c *tamperingConsumer) Range(ctx context.Context, req *pb.RangeRequest, + opts ...grpc.CallOption, +) (*pb.RangeResponse, error) { + resp, err := c.ConsumerServiceClient.Range(ctx, req, opts...) + if err != nil || len(resp.GetEntries()) == 0 { + return resp, err + } + + resp.Next = commitment.Head{0x9e, 0x9e}.Bytes() + + return resp, nil +} + +// End to end: a store head that the entries behind it do not produce must not +// become the position this publisher builds on. +func TestTamperedHeadIsNotAdopted(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 0)) + + p.mu.Lock() + p.read.cons = &tamperingConsumer{ConsumerServiceClient: p.read.cons} + p.mu.Unlock() + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatal("adopted a window whose head the store misreported") + } + + p.mu.Lock() + anchor := p.anchor + p.mu.Unlock() + + if anchor == (commitment.Head{0x9e, 0x9e}) { + t.Fatal("anchored on a head no entry chain produces") + } +} diff --git a/eth/sequencer/entry.go b/eth/sequencer/entry.go new file mode 100644 index 0000000000..38cb59533f --- /dev/null +++ b/eth/sequencer/entry.go @@ -0,0 +1,185 @@ +package sequencer + +import ( + "bytes" + "math/big" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "google.golang.org/protobuf/proto" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// contentEqual reports field-level equality of entry content, excluding the +// prefix commitment (design: byte-match). The outer protobuf encoding is +// never compared — the gateway re-marshals entries. +func contentEqual(a, b *pb.Entry) bool { + switch ak := a.GetKind().(type) { + case *pb.Entry_BlockOpen: + bo := b.GetBlockOpen() + if bo == nil { + return false + } + + ao := ak.BlockOpen + + return ao.GetBlockNumber() == bo.GetBlockNumber() && + ao.GetBlockTimestamp() == bo.GetBlockTimestamp() && + bytes.Equal(ao.GetParentHash(), bo.GetParentHash()) && + ao.GetGasLimit() == bo.GetGasLimit() && + bytes.Equal(ao.GetBaseFee(), bo.GetBaseFee()) + case *pb.Entry_Record: + br := b.GetRecord() + if br == nil || len(ak.Record.GetTransactions()) != len(br.GetTransactions()) { + return false + } + + for i, tx := range ak.Record.GetTransactions() { + if !bytes.Equal(tx, br.GetTransactions()[i]) { + return false + } + } + + return true + case *pb.Entry_BlockSeal: + bs := b.GetBlockSeal() + + return bs != nil && bytes.Equal(ak.BlockSeal.GetHeader(), bs.GetHeader()) + default: + return false + } +} + +// entryPrefix returns the prefix commitment carried by an entry. +func entryPrefix(e *pb.Entry) []byte { + switch k := e.GetKind().(type) { + case *pb.Entry_BlockOpen: + return k.BlockOpen.GetPrefixCommitment() + case *pb.Entry_Record: + return k.Record.GetPrefixCommitment() + case *pb.Entry_BlockSeal: + return k.BlockSeal.GetPrefixCommitment() + default: + return nil + } +} + +// foldEntry folds an entry carrying its existing prefix onto cur. +// openContext builds the fold input for an open entry. Both fold paths must +// construct it identically or a refold diverges from the original fold. +func openContext(bo *pb.BlockOpen) commitment.OpenContext { + return commitment.OpenContext{ + Number: bo.GetBlockNumber(), + Timestamp: bo.GetBlockTimestamp(), + ParentHash: [32]byte(bo.GetParentHash()), + GasLimit: bo.GetGasLimit(), + BaseFee: new(big.Int).SetBytes(bo.GetBaseFee()), + } +} + +// foldEntry advances a commitment head by one entry, dispatching on kind. +func foldEntry(cur commitment.Head, e *pb.Entry) (commitment.Head, error) { + switch k := e.GetKind().(type) { + case *pb.Entry_BlockOpen: + return commitment.FoldOpen(cur, openContext(k.BlockOpen)) + case *pb.Entry_Record: + return commitment.FoldTxs(cur, k.Record.GetTransactions()), nil + case *pb.Entry_BlockSeal: + return commitment.FoldSeal(cur, commitment.SealedHash(k.BlockSeal.GetHeader())), nil + default: + return commitment.Head{}, errRefold + } +} + +// setEntryPrefix rewrites the prefix commitment an entry carries. +func setEntryPrefix(e *pb.Entry, cur commitment.Head) bool { + switch k := e.GetKind().(type) { + case *pb.Entry_BlockOpen: + k.BlockOpen.PrefixCommitment = cur.Bytes() + case *pb.Entry_Record: + k.Record.PrefixCommitment = cur.Bytes() + case *pb.Entry_BlockSeal: + k.BlockSeal.PrefixCommitment = cur.Bytes() + default: + return false + } + + return true +} + +// refoldEntry clones a journal item's entry onto a new prefix, returning +// the rewritten entry with its post-fold head. Folding through foldEntry +// guarantees a refold computes exactly what the original fold did. +func refoldEntry(cur commitment.Head, item journalItem) (*pb.Entry, commitment.Head, error) { + entry, ok := proto.Clone(item.entry).(*pb.Entry) + if !ok || !setEntryPrefix(entry, cur) { + return nil, commitment.Head{}, errRefold + } + + next, err := foldEntry(cur, entry) + if err != nil { + return nil, commitment.Head{}, err + } + + return entry, next, nil +} + +// openEntry builds the wire entry for a block open. All open publishers +// (live build and window rebuild) must construct it identically or the +// mirror check and the consumer's context pinning would see drift. +func openEntry(oc commitment.OpenContext, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: oc.Number, + BlockTimestamp: oc.Timestamp, + ParentHash: oc.ParentHash[:], + GasLimit: oc.GasLimit, + BaseFee: baseFeeBytes(oc.BaseFee), + PrefixCommitment: prefix.Bytes(), + }}} +} + +// recordEntry builds the wire entry for one committed transaction. +func recordEntry(raw []byte, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: prefix.Bytes(), + }}} +} + +// sealEntry builds the wire entry for a sealed block header. +func sealEntry(raw []byte, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{ + Header: raw, + PrefixCommitment: prefix.Bytes(), + }}} +} + +// decodeSealHeader decodes a seal entry's RLP header. +func decodeSealHeader(raw []byte) (*types.Header, error) { + header := new(types.Header) + if err := rlp.DecodeBytes(raw, header); err != nil { + return nil, err + } + + return header, nil +} + +// entryHeight extracts a height from an open or seal entry; records carry +// none. +func entryHeight(e *pb.Entry) (uint64, bool) { + switch k := e.GetKind().(type) { + case *pb.Entry_BlockOpen: + return k.BlockOpen.GetBlockNumber(), true + case *pb.Entry_BlockSeal: + header, err := decodeSealHeader(k.BlockSeal.GetHeader()) + if err != nil { + return 0, false + } + + return header.Number.Uint64(), true + default: + return 0, false + } +} diff --git a/eth/sequencer/entry_test.go b/eth/sequencer/entry_test.go new file mode 100644 index 0000000000..537feff520 --- /dev/null +++ b/eth/sequencer/entry_test.go @@ -0,0 +1,145 @@ +package sequencer + +import ( + "testing" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" +) + +func testOpen(number uint64, parent common.Hash, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: number, + BlockTimestamp: 1700000000 + number, + ParentHash: parent.Bytes(), + GasLimit: 30_000_000, + BaseFee: []byte{0x01}, + PrefixCommitment: prefix.Bytes(), + }}} +} + +func testRecord(tx []byte, prefix commitment.Head) *pb.Entry { + return &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{tx}, + PrefixCommitment: prefix.Bytes(), + }}} +} + +func testSeal(t *testing.T, number uint64, prefix commitment.Head) *pb.Entry { + t.Helper() + + raw, err := rlp.EncodeToBytes(testHeader(number, common.Hash{0x01})) + if err != nil { + t.Fatalf("rlp: %v", err) + } + + return &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{ + Header: raw, + PrefixCommitment: prefix.Bytes(), + }}} +} + +func TestContentEqual(t *testing.T) { + a, b := commitment.Head{0xaa}, commitment.Head{0xbb} + + cases := []struct { + name string + x, y *pb.Entry + want bool + }{ + {"open equal ignoring prefix", testOpen(1, common.Hash{0x01}, a), testOpen(1, common.Hash{0x01}, b), true}, + {"open different number", testOpen(1, common.Hash{0x01}, a), testOpen(2, common.Hash{0x01}, a), false}, + {"open different parent", testOpen(1, common.Hash{0x01}, a), testOpen(1, common.Hash{0x02}, a), false}, + {"record equal ignoring prefix", testRecord([]byte{0x01}, a), testRecord([]byte{0x01}, b), true}, + {"record different tx", testRecord([]byte{0x01}, a), testRecord([]byte{0x02}, a), false}, + {"record different length", testRecord([]byte{0x01}, a), &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{Transactions: [][]byte{{0x01}, {0x02}}}}}, false}, + {"kind mismatch", testOpen(1, common.Hash{0x01}, a), testRecord([]byte{0x01}, a), false}, + {"kind mismatch reversed", testSeal(t, 1, a), testOpen(1, common.Hash{0x01}, a), false}, + {"seal equal ignoring prefix", testSeal(t, 1, a), testSeal(t, 1, b), true}, + {"seal different header", testSeal(t, 1, a), testSeal(t, 2, a), false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := contentEqual(tc.x, tc.y); got != tc.want { + t.Fatalf("contentEqual = %v, want %v", got, tc.want) + } + }) + } +} + +func TestEntryHeight(t *testing.T) { + if h, ok := entryHeight(testOpen(7, common.Hash{0x01}, commitment.Head{})); !ok || h != 7 { + t.Fatalf("open height = %d/%v", h, ok) + } + + if h, ok := entryHeight(testSeal(t, 9, commitment.Head{})); !ok || h != 9 { + t.Fatalf("seal height = %d/%v", h, ok) + } + + if _, ok := entryHeight(testRecord([]byte{0x01}, commitment.Head{})); ok { + t.Fatal("record must carry no height") + } + + garbage := &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{Header: []byte{0xde}}}} + if _, ok := entryHeight(garbage); ok { + t.Fatal("undecodable seal must carry no height") + } +} + +// refoldEntry must reproduce the enqueue-side folds exactly, with only the +// prefix rewritten. +func TestRefoldEntryMatchesDirectFolds(t *testing.T) { + base := commitment.Head{0x11} + + open := testOpen(3, common.Hash{0x02}, commitment.Head{0xff}) + items := []journalItem{ + {entry: open, kind: entryOpen, height: 3}, + {entry: testRecord([]byte{0xbe, 0xef}, commitment.Head{0xff}), kind: entryRecord, height: 3}, + {entry: testSeal(t, 3, commitment.Head{0xff}), kind: entrySeal, height: 3}, + } + + cur := base + + for _, item := range items { + entry, next, err := refoldEntry(cur, item) + if err != nil { + t.Fatalf("refold: %v", err) + } + + if got := commitment.Head(entryPrefix(entry)); got != cur { + t.Fatalf("prefix %x, want %x", got, cur) + } + + if next == cur { + t.Fatal("fold did not advance") + } + + cur = next + } + + // The refolded open must fold identically to a direct FoldOpen. + wantOpen, err := commitment.FoldOpen(base, commitment.OpenContext{ + Number: 3, + Timestamp: open.GetBlockOpen().GetBlockTimestamp(), + ParentHash: common.Hash{0x02}, + GasLimit: open.GetBlockOpen().GetGasLimit(), + BaseFee: testHeader(0, common.Hash{}).BaseFee.SetBytes(open.GetBlockOpen().GetBaseFee()), + }) + if err != nil { + t.Fatalf("fold open: %v", err) + } + + if _, next, _ := refoldEntry(base, items[0]); next != wantOpen { + t.Fatalf("refolded open %x, want %x", next, wantOpen) + } +} + +func TestDecodeSealHeaderRejectsGarbage(t *testing.T) { + if _, err := decodeSealHeader([]byte{0xde, 0xad}); err == nil { + t.Fatal("garbage header decoded") + } +} diff --git a/eth/sequencer/equivocation_test.go b/eth/sequencer/equivocation_test.go new file mode 100644 index 0000000000..b49ed3d62a --- /dev/null +++ b/eth/sequencer/equivocation_test.go @@ -0,0 +1,514 @@ +package sequencer + +import ( + "testing" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/miner" +) + +// Two producers sharing a signing key both seal the same height. When the +// chain picks the other block, our flush must not overwrite the winner's +// content: the store's newest generation at a height has to agree with the +// canonical chain. +func TestForeignSealAtOwnHeightNonCanonical(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}, known: map[common.Hash]*types.Header{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // The twin publishes and seals ITS block 2 into the store first. + twin := testHeader(2, parent) + twin.Extra = []byte("twin") // distinct hash + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + // The chain accepts the TWIN's block 2 as canonical; ours lost. + chain.canonical[2] = twin.Hash() + + // Now we seal our own (losing) block 2 and flush. + ours := testHeader(2, parent) + p.OpenBlock(2, ours.Time, parent, ours.GasLimit, ours.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + + supersedes := reconcileSupersede.Snapshot().Count() + + p.SealBlock(blockFor(ours, []*types.Transaction{tx})) + time.Sleep(2 * time.Second) // let the flush + reconcile settle + + if got := reconcileSupersede.Snapshot().Count(); got > supersedes { + t.Fatalf("superseded a CANONICAL foreign seal at our own height "+ + "(supersede %d -> %d): the store's newest generation at height 2 "+ + "now holds non-canonical content", supersedes, got) + } +} + +// The mirror case: the chain chose OUR block, so the foreign seal at our +// height is the loser — supersede it, as before. +func TestForeignSealAtOwnHeightWeAreCanonical(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}, known: map[common.Hash]*types.Header{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + ours := testHeader(2, parent) + chain.canonical[2] = ours.Hash() // WE won + + p.OpenBlock(2, ours.Time, parent, ours.GasLimit, ours.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + + supersedes := reconcileSupersede.Snapshot().Count() + yields := reconcileYield.Snapshot().Count() + + p.SealBlock(blockFor(ours, []*types.Transaction{tx})) + + // The gate runs on every seal in production, and our block being + // canonical confirms it — which releases the flush to supersede the + // loser's content. Until then the pre-broadcast hold keeps our window + // off a height the store has already closed. + if v := p.ConfirmSeal(2 * time.Second); v != miner.SealConfirmed { + t.Fatalf("gate verdict = %v for our own canonical block", v) + } + + waitHead(t, h, p, 10*time.Second) + + if reconcileYield.Snapshot().Count() != yields { + t.Fatal("yielded despite being the canonical producer") + } + + if reconcileSupersede.Snapshot().Count() != supersedes+1 { + t.Fatal("canonical producer must supersede the losing seal") + } +} + +// The pre-seal barrier: a confirmed window clears, a contested one does +// not, and an unreachable store clears anyway so production never stalls. +func TestAwaitSequencedBarrier(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitHead(t, h, p, 5*time.Second) + + if !awaitOurWindow(p, 2*time.Second) { + t.Fatal("a fully confirmed window must clear the barrier") + } + + // A competitor takes the height with DIFFERENT content: its window is + // not a prefix of ours, so there is nothing to complete in place and + // our next write STALEs us into the contested hold. + foreignWindow(t, h, 2, parent, testTx(t, 9)) + p.PublishTx(testTx(t, 1)) + + waitFor(t, 5*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.hold.kind == holdSticky + }) + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("a contested window must NOT clear the barrier") + } +} + +// An unreachable store must not gate block production: the barrier clears +// on its deadline rather than stalling the chain. +func TestAwaitSequencedYieldsToLiveness(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() // store gone; entries pile up unacked + + p.OpenBlock(2, 1700000002, common.Hash{0x01}, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + + start := time.Now() + if !awaitOurWindow(p, 300*time.Millisecond) { + t.Fatal("an unreachable store must not block sealing") + } + + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("barrier waited %v; it must bound the stall", elapsed) + } +} + +// Takeover: the incumbent died, we adopt its window. We are the ONLY +// producer, so the barrier must let us seal. +func TestBarrierAllowsAdoptedTakeover(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // A dead producer's dangling window at height 2. + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + w := p.AdoptWindow(2, parent) + if w == nil { + t.Fatal("window not adopted") + } + + if !awaitOurWindow(p, 2*time.Second) { + t.Fatal("barrier blocked a takeover seal: the taker is the only " + + "producer, so refusing to seal would stall the chain") + } +} + +// A re-anchor must not re-post data the store already holds. With the +// store carrying a strict prefix of our window, completion absorbs that +// prefix as confirmed and leaves only the missing records to deliver — +// republishing the whole window would duplicate entries every reader has. +func TestReanchorPublishesOnlyTheDelta(t *testing.T) { + p := barePublisher() + + parent := common.Hash{0x01} + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + p.PublishTx(testTx(t, 1)) + p.PublishTx(testTx(t, 2)) + + items, _ := p.journal.after(0) + if len(items) != 4 { + t.Fatalf("expected open+3 records, got %d", len(items)) + } + + // The store holds a strict prefix: the open and the first record. + info := tailInfo{ + s: items[1].post, + tipOpen: true, + tipOpenHeight: 2, + tipOpenParent: parent, + window: []*pb.Entry{items[0].entry, items[1].entry}, + } + + p.mu.Lock() + p.curHeight = 2 + completed := p.completeExtendedWindowLocked(info, 2) + unacked := p.unackedLocked() + total := len(p.journal.items) + p.mu.Unlock() + + if !completed { + t.Fatal("a store prefix of our window must complete in place") + } + + // Only the two records the store lacks remain to send; the absorbed + // prefix is seated as already-confirmed, not queued for re-posting. + if unacked != 2 { + t.Fatalf("unacked = %d, want 2 (the delta only)", unacked) + } + + if total != 4 { + t.Fatalf("journal holds %d entries, want 4 — the window was duplicated", total) + } +} + +// waitDrained blocks until every journal entry is store-confirmed, which is +// the state the coverage branch of the barrier runs in. +func waitDrained(t *testing.T, p *Publisher, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + + for { + p.mu.Lock() + unacked := p.unackedLocked() + p.mu.Unlock() + + if unacked == 0 { + return + } + + if time.Now().After(deadline) { + t.Fatalf("journal still holds %d unacked entries", unacked) + } + + time.Sleep(5 * time.Millisecond) + } +} + +// appendForeignRecord writes one record onto the store's current chain, as a +// second producer building the same height would. +func appendForeignRecord(t *testing.T, h *harness, tx *types.Transaction) { + t.Helper() + + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + rec := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{raw}, + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(rec); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign record rejected: %v", status) + } +} + +// Every entry of ours acking proves the store took them — not that they are +// all the store has. A second producer's records at the same height were +// acked too, so they carry the same preconfirmation; sealing a block that +// omits them strands the promise. The barrier asks for a rebuild instead. +func TestBarrierRebuildsWhenStoreHoldsMore(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + appendForeignRecord(t, h, testTx(t, 7)) + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("sealed a block omitting records the store already holds") + } + + if !p.ResyncNeeded() { + t.Fatal("coverage failure did not arm a rebuild") + } +} + +// The uncontested case must stay cheap and permissive: when the store holds +// exactly our window, the coverage read confirms it and the seal proceeds. +func TestBarrierSealsWhenStoreMatches(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + if !awaitOurWindow(p, 2*time.Second) { + t.Fatal("barrier blocked a seal whose window is exactly the store's") + } +} + +// A competing generation at our height refuses the seal and adopts. The +// old tie-break sealed here — "the later opener owns the height" — and two +// producers whose reads raced could each believe themselves later, which is +// where divergent double-seals came from. Under the adopt rule the refusal +// is unconditional and the rebuild converges on the store's window. +func TestForeignGenerationRefusedAndAdopted(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + p.PublishTx(testTx(t, 1)) + waitDrained(t, p, 5*time.Second) + + // A second producer opens its own generation at the same height. + foreignWindow(t, h, 2, parent, testTx(t, 9)) + + p.mu.Lock() + p.resync = false + p.mu.Unlock() + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("sealed beside a competing generation: two blocks holding " + + "different content at one height") + } + + if !p.ResyncNeeded() { + t.Fatal("the refusal must arm the rebuild that adopts the competing window") + } +} + +// A seal below the height being built is ordinary history, not evidence the +// height moved on. Bailing on it made the coverage check report "covered" +// for almost every block, because the walk from our anchor routinely crosses +// the previous block's seal. +func TestCoverageIgnoresSealBelowOurHeight(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Block 1's seal now sits between the anchor and anything appended next. + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + appendForeignRecord(t, h, testTx(t, 7)) + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("a seal at a lower height masked a real coverage gap") + } +} + +// Takeover must stay fast: the incumbent is gone, its window is not +// growing, so the adopter covers it by definition and seals at once. +// Refusing here would stall the chain on every producer failure. +func TestAdoptedDeadWindowStillSeals(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if p.AdoptWindow(2, parent) == nil { + t.Fatal("window not adopted") + } + + if !awaitOurWindow(p, 2*time.Second) { + t.Fatal("barrier blocked a takeover of a dead window: every " + + "producer failure would stall the chain") + } +} + +// The mirror: the incumbent is alive and its window grew past the prefix we +// adopted. Sealing now would strand every record it added. +func TestAdoptedLiveWindowDoesNotSeal(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + foreignWindow(t, h, 2, parent, testTx(t, 0)) + + if p.AdoptWindow(2, parent) == nil { + t.Fatal("window not adopted") + } + + // The incumbent turns out to be alive and keeps writing. + appendForeignRecord(t, h, testTx(t, 7)) + + if awaitOurWindow(p, 2*time.Second) { + t.Fatal("sealed a prefix of a window the incumbent is still growing") + } + + if !p.ResyncNeeded() { + t.Fatal("a grown window must arm the rebuild that covers it") + } +} + +// A read that falls off the anchor rung restarts at a block boundary and +// walks over our own open, so an open at our height is routinely ours. The +// check used to read that as a rival and pass every block it saw — a lone +// producer logged 36 "second generation" skips with no competitor running. +func TestCoverageRecognisesOurOwnWindow(t *testing.T) { + p := barePublisher() + + parent := common.Hash{0x01} + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + p.PublishTx(testTx(t, 1)) + + items, _ := p.journal.after(0) + ourWindow := []*pb.Entry{items[0].entry, items[1].entry, items[2].entry} + + p.mu.Lock() + defer p.mu.Unlock() + + // The whole window, as a fallback read would return it. + whole := tailInfo{tipOpen: true, tipOpenHeight: 2, window: ourWindow} + if got := p.relateWindowLocked(whole, 2); got != windowOurs { + t.Fatalf("our own window read as %v, want windowOurs", got) + } + + // A prefix of it — records still in flight. + prefix := tailInfo{tipOpen: true, tipOpenHeight: 2, window: ourWindow[:2]} + if got := p.relateWindowLocked(prefix, 2); got != windowOurs { + t.Fatalf("a prefix of our window read as %v, want windowOurs", got) + } + + // Our window with a record appended by someone else: the real gap. + grown := tailInfo{ + tipOpen: true, tipOpenHeight: 2, + window: append(append([]*pb.Entry{}, ourWindow...), txRecord(t, testTx(t, 9))), + } + if got := p.relateWindowLocked(grown, 2); got != windowExtendsOurs { + t.Fatalf("an extended window read as %v, want windowExtendsOurs", got) + } + + // A genuinely different lineage. + foreign := []*pb.Entry{items[0].entry, txRecord(t, testTx(t, 9))} + if got := p.relateWindowLocked(tailInfo{tipOpen: true, tipOpenHeight: 2, window: foreign}, 2); got != windowForeign { + t.Fatalf("a rival window read as %v, want windowForeign", got) + } +} + +// txRecord builds a bare record entry; contentEqual compares payloads, not +// prefix commitments, so no fold is needed. +func txRecord(t *testing.T, tx *types.Transaction) *pb.Entry { + t.Helper() + + raw, err := tx.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + return &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{Transactions: [][]byte{raw}}}} +} + +// Contention refuses, always: a sticky hold is the store's own verdict that +// another producer owns the height, and the only path to a seal is adopting +// their window. The refusal arms that rebuild. +func TestContestedRefusesAndArmsAdopt(t *testing.T) { + p := barePublisher() + + p.mu.Lock() + p.curHeight = 7 + p.hold = hold{after: 0, kind: holdSticky} + p.mu.Unlock() + + for i := 0; i < 3; i++ { + if awaitOurWindow(p, time.Second) { + t.Fatal("a contested height sealed: two producers sealing one " + + "height with different content is the revocation machine") + } + } + + if !p.ResyncNeeded() { + t.Fatal("the refusal must arm the rebuild that adopts the owner's window") + } +} diff --git a/eth/sequencer/exec.go b/eth/sequencer/exec.go new file mode 100644 index 0000000000..70ff825681 --- /dev/null +++ b/eth/sequencer/exec.go @@ -0,0 +1,170 @@ +package sequencer + +import ( + "errors" + "fmt" + "math/big" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/trie" +) + +// blockEnv re-executes one speculative block on top of a parent state, +// mirroring the producer's environment: same header context from the open +// record, author nil so the EVM coinbase resolves to the producer-independent +// CalculateCoinbase (post-Rio), difficulty constant 1 under VEBLOP, and the +// same pre-transaction system calls the producer runs (EIP-2935 post-Prague). +type blockEnv struct { + header *types.Header + statedb *state.StateDB + evm *vm.EVM + gasPool *core.GasPool + txs []*types.Transaction + receipts []*types.Receipt +} + +// newBlockEnv builds the execution environment. speculative maps heights of +// sealed-but-not-yet-imported ancestors to their sealed hashes, so BLOCKHASH +// resolves them exactly as the producer did — the canonical header walk +// returns zero for blocks the chain hasn't imported. +func newBlockEnv(chain *core.BlockChain, statedb *state.StateDB, open *pb.BlockOpen, speculative map[uint64]common.Hash) *blockEnv { + header := &types.Header{ + ParentHash: common.BytesToHash(open.GetParentHash()), + Number: new(big.Int).SetUint64(open.GetBlockNumber()), + GasLimit: open.GetGasLimit(), + Time: open.GetBlockTimestamp(), + BaseFee: new(big.Int).SetBytes(open.GetBaseFee()), + Difficulty: big.NewInt(1), + Coinbase: common.Address{}, + } + + blockCtx := core.NewEVMBlockContext(header, chain, nil) + + walk := blockCtx.GetHash + blockCtx.GetHash = func(n uint64) common.Hash { + if h, ok := speculative[n]; ok { + return h + } + + if h := walk(n); h != (common.Hash{}) { + return h + } + + // The default resolver walks parent headers and breaks at the first + // unimported speculative ancestor; anything at or below the + // canonical head is still resolvable directly. + return chain.GetCanonicalHash(n) + } + + env := &blockEnv{ + header: header, + statedb: statedb, + evm: vm.NewEVM(blockCtx, statedb, chain.Config(), vm.Config{}), + gasPool: new(core.GasPool).AddGas(header.GasLimit), + } + + if chain.Config().IsPrague(header.Number) { + core.ProcessParentBlockHash(header.ParentHash, env.evm) + } + + return env +} + +// applyRaw executes one streamed raw transaction. The producer only publishes +// transactions it committed, so any failure here is a determinism divergence, +// not a bad transaction. +func (env *blockEnv) applyRaw(raw []byte) (*types.Transaction, *types.Receipt, error) { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + return nil, nil, fmt.Errorf("decode streamed transaction: %w", err) + } + + env.statedb.SetTxContext(tx.Hash(), len(env.txs)) + + receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.statedb, env.header, tx, &env.header.GasUsed) + if err != nil { + return nil, nil, fmt.Errorf("re-execute tx %s: %w", tx.Hash(), err) + } + + // The block hash is unknown pre-seal (ApplyTransaction stamped the + // provisional unsealed header hash); zero it out until the seal record + // arrives. EffectiveGasPrice is not populated by execution — derive it + // the way DeriveFields would. + receipt.BlockHash = common.Hash{} + for _, l := range receipt.Logs { + l.BlockHash = common.Hash{} + } + + receipt.EffectiveGasPrice = effectiveGasPrice(tx, env.header.BaseFee) + + env.txs = append(env.txs, tx) + env.receipts = append(env.receipts, receipt) + + return tx, receipt, nil +} + +func effectiveGasPrice(tx *types.Transaction, baseFee *big.Int) *big.Int { + if baseFee == nil { + return tx.GasPrice() + } + + tip, err := tx.EffectiveGasTip(baseFee) + if err != nil { + // Streamed txs executed successfully, so the fee cap covers the + // base fee; this path is unreachable but must not panic. + tip = new(big.Int) + } + + return new(big.Int).Add(tip, baseFee) +} + +var errSealMismatch = errors.New("sealed header diverges from re-execution") + +// checkSeal cross-checks the sealed header against the open context this +// block was executed under and against the re-execution results, including +// the state root — the catch-all for anything execution missed. State-sync +// transactions are applied by the producer in Finalize and never enter the +// stream, and their gas and receipts live outside the header's GasUsed and +// ReceiptHash — so a sprint-start block with pending events passes the gas +// and receipts comparisons and is caught only by the state root differing. +func (env *blockEnv) checkSeal(sealed *types.Header) error { + switch { + case sealed.Number.Cmp(env.header.Number) != 0, + sealed.Time != env.header.Time, + sealed.ParentHash != env.header.ParentHash, + sealed.GasLimit != env.header.GasLimit, + !bigEqual(sealed.BaseFee, env.header.BaseFee): + return fmt.Errorf("%w: open context mismatch at block %s", errSealMismatch, sealed.Number) + case sealed.GasUsed != env.header.GasUsed: + return fmt.Errorf("%w: gas used %d != re-executed %d at block %s", + errSealMismatch, sealed.GasUsed, env.header.GasUsed, sealed.Number) + } + + receiptsRoot := types.DeriveSha(types.Receipts(env.receipts), trie.NewStackTrie(nil)) + if receiptsRoot != sealed.ReceiptHash { + return fmt.Errorf("%w: receipts root %s != re-executed %s at block %s", + errSealMismatch, sealed.ReceiptHash, receiptsRoot, sealed.Number) + } + + root := env.statedb.IntermediateRoot(env.evm.ChainConfig().IsEIP158(env.header.Number)) + if root != sealed.Root { + return fmt.Errorf("%w: state root %s != re-executed %s at block %s", + errSealMismatch, sealed.Root, root, sealed.Number) + } + + return nil +} + +func bigEqual(a, b *big.Int) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + + return a.Cmp(b) == 0 +} diff --git a/eth/sequencer/gate.go b/eth/sequencer/gate.go new file mode 100644 index 0000000000..4a77593a42 --- /dev/null +++ b/eth/sequencer/gate.go @@ -0,0 +1,376 @@ +package sequencer + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/miner" +) + +// gateVerdict is the broadcast gate's resolution state. gateLost outranks +// gateConfirmed by construction: a lost verdict is terminal, and the seal +// ack that would confirm can no longer arrive for a rejected seal. +type gateVerdict int + +const ( + gatePending gateVerdict = iota // awaiting the seal ack, the chain, or the deadline + gateConfirmed // the store acked the gated seal + gateLost // the store rejected it: another window owns the height +) + +// sealGate identifies the sealed block whose broadcast awaits the store's +// verdict. A zero height means no seal is gated. refuseOnTimeout inverts +// the liveness default: normally an undecided height broadcasts, but a +// block built over a seal the store already holds must not. +type sealGate struct { + height uint64 + hash common.Hash + verdict gateVerdict + refuseOnTimeout bool + + // contested records a STALE landing while the verdict was pending: the + // head moved under the flush, so our own reconcile loop is producing + // the verdict and the wait earns the longer budget. + contested bool + + // tolerateSealed carries a modeOverSealed decision to the gate: this + // block knowingly sealed a height the store closed because recovery + // declined, so a foreign store seal must not refuse it — the + // chain-canonical check still can. + tolerateSealed bool + + // published means a flush in the journal carries this block; a refusal + // must drop it (a refused block is never canonical, so its flush could + // never resolve) and roll the sealed tip back to prevTip so the rebuild + // at this height can adopt instead of muting itself. + published bool + prevTip uint64 + + // txs is the gated block's transaction sequence, kept for the last + // look a verdictless timeout takes before broadcasting. + txs []common.Hash +} + +// refusalStreak counts consecutive gate refusals at one height. The escape +// valve: refuse → rebuild is convergent when the rebuild can adopt what +// stands in the store, but a window it cannot adopt (dead producer, foreign +// parent) would refuse forever — past the cap, liveness wins and the block +// broadcasts without a verdict. +type refusalStreak struct { + height uint64 + count int +} + +// maxGateRefusals bounds the refuse → rebuild cycle per height. +const maxGateRefusals = 3 + +// ConfirmSeal waits for the store's verdict on the block just sealed. The +// store's head CAS admits exactly one seal at a lineage position, so the +// seal race has a single winner — this is where the loser finds out. +// +// Confirmation comes from our seal entry's ack. Refusal comes from our own +// chain: the winner's block importing at our height IS the rejection +// notice, needing no store read at all. No verdict inside the budget means +// the store is slow, unreachable, or the height genuinely unresolved — the +// caller broadcasts (production never waits on the store), which also +// covers the phantom case of a winner that sealed in the store and then +// died without broadcasting. +func (p *Publisher) ConfirmSeal(timeout time.Duration) miner.SealVerdict { + if p.unreachable.Load() { + return p.settle(miner.SealUnknown) // no verdict is coming; do not wait for one + } + + start := time.Now() + + var chainCheck time.Time + + for { + p.mu.Lock() + g := p.gate + p.mu.Unlock() + + if g.height == 0 { + return miner.SealUnknown // nothing gated (muted or failed build) + } + + if g.verdict == gateLost { + return p.refuseGated() + } + + if g.verdict == gateConfirmed { + return p.settle(miner.SealConfirmed) + } + + // The canonical lookup is an uncached database read, and this wait + // can now last a whole block period: take it on a coarse tick + // rather than every poll. + if now := time.Now(); p.chain != nil && now.After(chainCheck) { + chainCheck = now.Add(50 * time.Millisecond) + + switch canonical := p.chain.GetCanonicalHash(g.height); { + case canonical == g.hash: + // A same-key twin broadcast our exact block: it is already + // the chain. Broadcasting again is a harmless duplicate. + return p.settle(miner.SealConfirmed) + case canonical != (common.Hash{}): + return p.refuseGated() + } + } + + // A contested gate earns the longer budget: a STALE while pending + // means our own reconcile loop is producing the verdict, and cutting + // the wait short is what broadcast an empty block three seconds + // before its refusal would have arrived. + budget := timeout + if g.contested { + budget = max(budget, contestedGateTimeout) + } + + failed := p.failed.Load() + if !failed && time.Since(start) <= budget { + time.Sleep(2 * time.Millisecond) + + continue + } + + if g.refuseOnTimeout { + return p.refuseGated() + } + + if !failed { + switch p.gateRecheck(g) { + case miner.SealRefused: + return p.refuseGated() + case miner.SealConfirmed: + return p.settle(miner.SealConfirmed) + } + } + + return p.settle(miner.SealUnknown) + } +} + +// settle resolves the gate with a non-refusal verdict: clear, count, return. +// Refusals go through refuseGated, which also unwinds the refused flush. +func (p *Publisher) settle(v miner.SealVerdict) miner.SealVerdict { + p.clearGate() + + if v == miner.SealConfirmed { + gateConfirmedCount.Inc(1) + } else { + gateUnknownCount.Inc(1) + } + + return v +} + +// contestedGateTimeout is the verdict budget once the gate is contested: +// one block period, tied to recoverGrace so the two waits scale together — +// losing a slot is acceptable, losing acked records to a premature +// broadcast is not. +const contestedGateTimeout = recoverGrace + +// refuseGated resolves the gate as refused. The block will never broadcast, +// so the flush describing it is dropped — a refused block is never +// canonical, and a flush that can never resolve wedges every later build +// behind it — and the sealed tip it bumped rolls back so the rebuild at +// this height can adopt what stands in the store instead of muting itself. +// Past the per-height refusal cap, liveness wins: the rebuild evidently +// cannot adopt its way to convergence, and the block broadcasts without a +// verdict. +func (p *Publisher) refuseGated() miner.SealVerdict { + p.mu.Lock() + defer p.mu.Unlock() + + if p.gate.height != p.refusals.height { + p.refusals = refusalStreak{height: p.gate.height} + } + + p.refusals.count++ + + if p.refusals.count > maxGateRefusals { + log.Warn("Sequencer gate refused this height repeatedly, broadcasting for liveness", + "number", p.gate.height, "refusals", p.refusals.count) + p.gate = sealGate{} + gateUnknownCount.Inc(1) + + return miner.SealUnknown + } + + if p.gate.published { + p.dropRefusedFlushLocked(p.gate.height, p.gate.hash) + + if p.sealedTip == p.gate.height { + p.sealedTip = p.gate.prevTip + } + } + + p.gate = sealGate{} + gateRefusedCount.Inc(1) + + return miner.SealRefused +} + +// dropRefusedFlushLocked removes the refused block's undelivered trailing +// entries, and anything a concurrent work cycle chained above them — those +// describe blocks that re-flush from their own bodies at seal time. An +// acked prefix is store content and stays; older stacked flushes below the +// refused height are still owed and stay. The hold clears with the flush it +// gated, or the next barrier would resync against a lineage that no longer +// exists. +func (p *Publisher) dropRefusedFlushLocked(height uint64, hash common.Hash) { + if ours, ok := p.sealedHashAtLocked(height); !ok || ours != hash { + return // the trailing seal is not the refused block's + } + + cut := p.journal.cutFromHeight(p.ackedSeq, height) + if cut == len(p.journal.items) { + return + } + + publishDropMeter.Mark(int64(len(p.journal.items) - cut)) + p.rewindJournalLocked(cut) + p.syncWindowLocked() + p.hold = clearedHold() + publishQueueGauge.Update(int64(p.unackedLocked())) +} + +// gateRecheck is the last look before a verdictless broadcast: one +// height-anchored probe at the gated height, taken after the whole wait. +// The reads that precede it can each be individually fresh and still miss +// a rival's burst landing between them; what this read shows is exactly +// what the broadcast would bury. Anything unreadable keeps the timeout +// verdict — production never waits on a store it cannot see. +func (p *Publisher) gateRecheck(g sealGate) miner.SealVerdict { + if p.unreachable.Load() || p.read == nil || p.read.cons == nil { + return miner.SealUnknown + } + + ctx, cancel := context.WithTimeout(context.Background(), checkTailTimeout) + defer cancel() + + // One walk anchored at the gated height answers everything: an unknown + // height reads NOT_FOUND (nothing stands here), a live generation is + // served from its open, a sealed one from just past its seal. + info, out, done := p.tryWalk(ctx, blockReq(g.height), false) + if !done || out != recOK { + return miner.SealUnknown + } + + if info.tipOpen && info.tipOpenHeight == g.height { + if storeTxs, ok := windowTxHashes(info.window); ok && !windowLeadsHashes(storeTxs, g.txs) { + gateRecheckRefused.Inc(1) + log.Warn("Sequencer refusing broadcast: acked records stand at this height the block does not carry", + "number", g.height, "store", len(storeTxs), "block", len(g.txs)) + + return miner.SealRefused + } + + return miner.SealUnknown + } + + // A generation at the gated height with no live window is a sealed one. + // The walk is served from just past the seal, so the generation itself — + // seal included — comes from the block fetch. + return p.recheckSealedGeneration(ctx, g) +} + +// recheckSealedGeneration fetches the generation standing at the gated +// height and applies the seal policy (verdictForSeal) to its seal. +func (p *Publisher) recheckSealedGeneration(ctx context.Context, g sealGate) miner.SealVerdict { + entries, err := p.read.generation(ctx, g.height) + if err != nil { + return miner.SealUnknown + } + + for _, e := range entries { + seal := e.GetBlockSeal() + if seal == nil { + continue + } + + header, err := decodeSealHeader(seal.GetHeader()) + if err != nil { + return miner.SealUnknown + } + + v := g.verdictForSeal(header.Hash()) + if v == miner.SealRefused { + gateRecheckRefused.Inc(1) + log.Warn("Sequencer refusing broadcast: the store sealed this height with other content", + "number", g.height, "store", header.Hash()) + } + + return v + } + + return miner.SealUnknown // a live generation after all: nothing decisive +} + +func (p *Publisher) clearGate() { + p.mu.Lock() + defer p.mu.Unlock() + + p.gate = sealGate{} +} + +// gatePendingLocked reports whether an unbroadcast seal at this height is +// still awaiting its verdict. +func (p *Publisher) gatePendingLocked(height uint64) bool { + return p.gate.height != 0 && p.gate.height == height && p.gate.verdict != gateConfirmed +} + +// verdictForSeal is the gate's one seal policy: our own seal standing at +// the gated height confirms the broadcast (a twin delivered our copy), a +// foreign one refuses it — unless the build knowingly sealed over it (the +// liveness fallback), which leaves the verdict to the budget. +func (g sealGate) verdictForSeal(hash common.Hash) miner.SealVerdict { + switch { + case hash == g.hash: + return miner.SealConfirmed + case g.tolerateSealed: + return miner.SealUnknown + default: + return miner.SealRefused + } +} + +// resolveGateFromSealLocked applies the seal policy to a decoded store seal +// at the gated height. An undecoded seal proves nothing and leaves the +// budget to decide. +func (p *Publisher) resolveGateFromSealLocked(info tailInfo) { + if !info.sealDecoded || p.gate.verdict != gatePending { + return + } + + switch p.gate.verdictForSeal(info.lastSealHash) { + case miner.SealConfirmed: + p.gate.verdict = gateConfirmed + case miner.SealRefused: + p.gate.verdict = gateLost + } +} + +// markGateLost records what a STALE means for the gated block. Any STALE +// while the verdict is pending marks the gate contested — the head moved +// under our flush, reconciliation is now producing the verdict, and the +// wait earns the longer budget. A STALE on the gated seal itself is the +// verdict: another producer's window owns the height, and waiting out the +// liveness budget after that would broadcast a second block into a height +// the store has already given to someone else. +func (p *Publisher) markGateLost(item journalItem) { + p.mu.Lock() + defer p.mu.Unlock() + + if p.gate.height == 0 || p.gate.verdict != gatePending { + return + } + + p.gate.contested = true + + if item.kind == entrySeal && p.gate.height == item.height { + p.gate.verdict = gateLost + } +} diff --git a/eth/sequencer/gate_test.go b/eth/sequencer/gate_test.go new file mode 100644 index 0000000000..6e8bd3078b --- /dev/null +++ b/eth/sequencer/gate_test.go @@ -0,0 +1,949 @@ +package sequencer + +import ( + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/miner" + "github.com/ethereum/go-ethereum/rlp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const sealGateBudget = contestedGateTimeout + +// gateBlock builds and seals a block through the publisher, returning it for +// ConfirmSeal. The window drains first, as the pre-seal barrier guarantees +// in production. +func gateBlock(t *testing.T, p *Publisher, h *harness, number uint64, parent common.Hash) *types.Block { + t.Helper() + + header := testHeader(number, parent) + p.OpenBlock(number, header.Time, parent, header.GasLimit, header.BaseFee) + + tx := testTx(t, 0) + p.PublishTx(tx) + waitDrained(t, p, 5*time.Second) + + block := blockFor(header, []*types.Transaction{tx}) + p.SealBlock(block) + + return block +} + +// The sole producer's seal acks and the gate confirms in milliseconds — the +// path every block takes when nothing is wrong. +func TestSealGateConfirmsSoleProducer(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + gateBlock(t, p, h, 2, sealHash(t, sealed)) + + if v := p.ConfirmSeal(2 * time.Second); v != miner.SealConfirmed { + t.Fatalf("sole producer's seal verdict = %v, want Confirmed", v) + } +} + +// The loser learns it lost from its own chain: the winner's block importing +// at our height is the rejection notice, no store read required. +func TestSealGateRefusesWhenRivalBlockImports(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Our window drains normally while we build... + header := testHeader(2, parent) + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + waitDrained(t, p, 5*time.Second) + + // ...the rival's seal lands in the store as we go to seal... + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + // ...and its block becomes our chain's block 2. + chain.canonical[2] = twin.Hash() + + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + + if v := p.ConfirmSeal(2 * time.Second); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: broadcasting would fork an "+ + "already-decided height", v) + } + + // The refused flush must not have stomped the winner's seal: one sealed + // generation at 2, the rival's. + sealedGens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("store holds %d sealed generations at height 2, want the "+ + "winner's alone", sealedGens) + } +} + +// A same-key twin that broadcast our exact block is a confirmation, not a +// refusal: the block on chain IS ours, and re-broadcasting is harmless. +func TestSealGateConfirmsWhenTwinShippedOurBlock(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + block := gateBlock(t, p, h, 2, sealHash(t, sealed)) + chain.canonical[2] = block.Hash() + + if v := p.ConfirmSeal(2 * time.Second); v != miner.SealConfirmed { + t.Fatalf("verdict = %v, want Confirmed for our own canonical block", v) + } +} + +// A foreign generation that sealed our height rejects our own seal, and the +// rejection is the answer: this height belongs to someone else, and the next +// build recovers its content. Broadcasting anyway put two blocks at one +// height on a devnet, and the milestone vote then displaced 38 preconfirmed +// transactions into the following block. The held flush must also not have +// superseded the winner meanwhile. +func TestSealGateRefusesWhenOurSealIsRejected(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + header := testHeader(2, parent) + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + waitDrained(t, p, 5*time.Second) + + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + // No canonical block at 2: the chain has not decided. + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + + start := time.Now() + if v := p.ConfirmSeal(sealGateBudget); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: the store rejected our seal "+ + "for this height", v) + } + + if time.Since(start) > 2*time.Second { + t.Fatal("the gate overstayed its budget") + } + + sealedGens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("the unbroadcast flush superseded a possible winner: %d "+ + "sealed generations at height 2", sealedGens) + } +} + +// A store that is down cannot delay production more than the budget. +func TestSealGateUnknownWhenStoreDown(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + h.stop() + + header := testHeader(2, parent) + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + p.PublishTx(testTx(t, 0)) + p.SealBlock(blockFor(header, []*types.Transaction{testTx(t, 0)})) + + start := time.Now() + if v := p.ConfirmSeal(300 * time.Millisecond); v != miner.SealUnknown { + t.Fatalf("verdict = %v, want Unknown with the store down", v) + } + + if time.Since(start) > 2*time.Second { + t.Fatal("a dead store gated production past the budget") + } +} + +// A height sealed in the store whose block the chain already has is an +// ordinary loss: the build mutes. Should the worker seal there anyway (a +// build already in flight when the winner landed), the flush must not stomp +// the store's standing seal. +func TestSealGateRefusesALostHeightAndStompsNothing(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + chain.canonical[2] = twin.Hash() + + if p.AdoptWindow(2, parent) != nil { + t.Fatal("a sealed height the chain already has offered an adoption") + } + + header := testHeader(2, parent) + header.Extra = []byte("ours") + p.SealBlock(blockFor(header, []*types.Transaction{testTx(t, 0)})) + + if v := p.ConfirmSeal(300 * time.Millisecond); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: the chain already holds "+ + "another block at this height", v) + } + + sealedGens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("the held flush superseded the standing seal: %d sealed "+ + "generations at height 2", sealedGens) + } +} + +// The upgraded twin property: with the gate, one height gets ONE broadcast. +// The winner confirms; the loser is refused the moment the winner's block +// is on its chain. +func TestTwinsNeverBothBroadcast(t *testing.T) { + h, a, b := twinPublishers(t) + chainA := &fakeChain{canonical: map[uint64]common.Hash{}} + chainB := &fakeChain{canonical: map[uint64]common.Hash{}} + + a.mu.Lock() + a.chain = chainA + a.mu.Unlock() + b.mu.Lock() + b.chain = chainB + b.mu.Unlock() + + parent := sealedParent(t, h, a, b) + + blockA := gateBlock(t, a, h, 2, parent) + + // B builds divergent content for the same height and seals it too. + headerB := testHeader(2, parent) + headerB.Extra = []byte("b") + b.OpenBlock(2, headerB.Time, parent, headerB.GasLimit, headerB.BaseFee) + b.PublishTx(testTx(t, 7)) + blockB := blockFor(headerB, []*types.Transaction{testTx(t, 7)}) + b.SealBlock(blockB) + + // Consensus: A's block wins on both chains. + chainA.canonical[2] = blockA.Hash() + chainB.canonical[2] = blockA.Hash() + + va := a.ConfirmSeal(2 * time.Second) + vb := b.ConfirmSeal(2 * time.Second) + + broadcasts := 0 + if va != miner.SealRefused { + broadcasts++ + } + + if vb != miner.SealRefused { + broadcasts++ + } + + if broadcasts != 1 { + t.Fatalf("verdicts A=%v B=%v: exactly one twin may broadcast", va, vb) + } +} + +// The commitment hash is how anyone learns exactly what the store holds: to +// publish at all, a producer must chain onto the current head, and that head +// encodes every seal before it. So a flush that would land after a foreign +// seal at its own height cannot claim ignorance of it — and must not publish +// a second sealed generation there while its own block is unbroadcast. +// +// This is the 764 defect: the losing twin reconciled onto a head that already +// carried the winner's seal, then republished its own window and seal on top, +// leaving the store's newest view of the height pointing at a block that +// never existed on any chain. +func TestFlushNeverRepublishesOverAKnownSeal(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // We build height 2 while it is genuinely clean, as the loser did. + header := testHeader(2, parent) + p.OpenBlock(2, header.Time, parent, header.GasLimit, header.BaseFee) + tx := testTx(t, 0) + p.PublishTx(tx) + waitDrained(t, p, 5*time.Second) + + // The rival's complete generation lands first, closing the height. + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, twin) + + // Our flush composes now, and the chain has not decided yet — the exact + // window in which the old canonicality-conditioned guard let a refold + // through. + p.SealBlock(blockFor(header, []*types.Transaction{tx})) + + if v := p.ConfirmSeal(500 * time.Millisecond); v == miner.SealConfirmed { + t.Fatalf("verdict = %v: the store had already closed this height", v) + } + + // Give any reconcile a chance to act before auditing. + time.Sleep(500 * time.Millisecond) + + sealedGens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("store holds %d sealed generations at height 2: a producer "+ + "that could see the seal published over it anyway", sealedGens) + } +} + +// A producer that got its seal acked and then died leaves the height closed +// in the store and empty on the chain. Muting there strands the height +// forever; building fresh content there would orphan every record the dead +// producer already had acked. Recovery does neither: it rebuilds that exact +// prefix and publishes nothing, because the store already holds the whole +// generation. +func TestPhantomSealRecoversTheExactPrefix(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // A producer sealed height 2 in the store with three records... + t0, t1, t2 := testTx(t, 0), testTx(t, 1), testTx(t, 2) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{t0, t1, t2} { + appendForeignRecord(t, h, tx) + } + + dead := testHeader(2, parent) + appendForeignSeal(t, h, dead) + + // ...and died: the chain has no block at 2. + entriesBefore := len(readAllGenerations(t, h)) + + w := p.AdoptWindow(2, parent) + if w == nil { + t.Fatal("a height sealed in the store but absent from the chain was " + + "muted: nobody rebuilds it and it is stranded forever") + } + + if len(w.Txs) != 3 { + t.Fatalf("recovered %d txs, want the dead producer's 3", len(w.Txs)) + } + + for i, want := range []*types.Transaction{t0, t1, t2} { + if w.Txs[i].Hash() != want.Hash() { + t.Fatalf("recovered tx %d differs: rebuilding here would orphan "+ + "the records the dead producer already had acked", i) + } + } + + if w.Timestamp != dead.Time || w.ParentHash != parent { + t.Fatal("recovery did not inherit the sealed open context") + } + + // The store already holds the generation: the rebuild must add nothing. + p.OpenBlock(2, w.Timestamp, parent, w.GasLimit, w.BaseFee) + p.PublishTx(t0) + time.Sleep(300 * time.Millisecond) + + if got := len(readAllGenerations(t, h)); got != entriesBefore { + t.Fatalf("recovery republished into the store: %d generations, want %d", + got, entriesBefore) + } + + // Nor at the seal: the store already holds this generation's seal, so a + // second one would open a second generation over a sealed height. + p.SealBlock(types.NewBlockWithHeader(dead)) + time.Sleep(300 * time.Millisecond) + + if got := len(readAllGenerations(t, h)); got != entriesBefore { + t.Fatalf("recovery re-sealed into the store: %d generations, want %d", + got, entriesBefore) + } + + // And the block must reach the chain: a gate refusal here would strand + // the height a second time. + if v := p.ConfirmSeal(200 * time.Millisecond); v == miner.SealRefused { + t.Fatal("the recovery block was refused: the height stays stranded") + } +} + +// The ordinary loss must not be mistaken for a phantom: when the chain does +// hold a block at the height, there is nothing to recover and the build mutes. +func TestSealedHeightWithChainBlockStillMutes(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 0)) + + winner := testHeader(2, parent) + appendForeignSeal(t, h, winner) + + // The winner's block is on our chain: an ordinary loss. + chain.canonical[2] = winner.Hash() + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatalf("recovered a height the chain already has: %+v", w) + } +} + +// A seal only moments old may simply be in flight. Rebuilding for it that +// early races the real broadcast, so the grace has to elapse first — and +// until it does, a build carrying different content must not go out. +func TestFreshSealIsWaitedForNotRebuilt(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + now := uint64(time.Now().Unix()) + appendForeignOpenAt(t, h, 2, parent, now) + appendForeignRecord(t, h, testTx(t, 0)) + + fresh := testHeader(2, parent) + fresh.Time = now + appendForeignSeal(t, h, fresh) + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatal("rebuilt a seal that is still within its broadcast grace: " + + "this races the block already on its way") + } + + // The build proceeds anyway (mute does not stop the miner), so the gate + // is the last line: divergent content over a standing seal must not + // broadcast just because the height is undecided. + ours := testHeader(2, parent) + ours.Extra = []byte("ours") + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 9)})) + + if v := p.ConfirmSeal(300 * time.Millisecond); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: broadcasting here forks the "+ + "height away from the sequenced content", v) + } + + // And it published nothing: the store's generation stands alone. + sealedGens := 0 + + for _, g := range readAllGenerations(t, h) { + if g.height == 2 && g.sealed { + sealedGens++ + } + } + + if sealedGens != 1 { + t.Fatalf("%d sealed generations at height 2, want the store's one", + sealedGens) + } +} + +// Once the grace has passed with no block, the same height is rebuilt from +// the store rather than left stranded. +func TestStaleSealIsRebuiltAfterGrace(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + past := uint64(time.Now().Add(-2 * recoverGrace).Unix()) + appendForeignOpenAt(t, h, 2, parent, past) + appendForeignRecord(t, h, testTx(t, 0)) + + dead := testHeader(2, parent) + dead.Time = past + appendForeignSeal(t, h, dead) + + if p.AdoptWindow(2, parent) == nil { + t.Fatal("a seal past its grace with no block was not rebuilt") + } +} + +// A consumer whose per-block fetch fails while the tail still reads: the +// store says the height is sealed, but its content cannot be recovered. +type unfetchableConsumer struct { + pb.ConsumerServiceClient +} + +func (c *unfetchableConsumer) GetBlock(ctx context.Context, req *pb.GetBlockRequest, + opts ...grpc.CallOption, +) (*pb.GetBlockResponse, error) { + return nil, status.Error(codes.Unavailable, "block fetch unavailable") +} + +// Refusing a height is only safe while the rebuild that resolves it is still +// coming. If recovery can never reconstruct the height, refusing every build +// there would halt the chain at that block forever — so an unrecoverable +// sealed height falls back to liveness. +func TestUnrecoverableSealedHeightStillBroadcasts(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + past := uint64(time.Now().Add(-2 * recoverGrace).Unix()) + appendForeignOpenAt(t, h, 2, parent, past) + appendForeignRecord(t, h, testTx(t, 0)) + + lost := testHeader(2, parent) + lost.Time = past + appendForeignSeal(t, h, lost) + + p.mu.Lock() + p.read.cons = &unfetchableConsumer{ConsumerServiceClient: p.read.cons} + p.mu.Unlock() + + if w := p.AdoptWindow(2, parent); w != nil { + t.Fatal("recovered a height whose content could not be fetched") + } + + ours := testHeader(2, parent) + ours.Extra = []byte("ours") + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 9)})) + + if v := p.ConfirmSeal(300 * time.Millisecond); v == miner.SealRefused { + t.Fatal("refused an unrecoverable height: every build here refuses " + + "the same way and the chain never gets past this block") + } +} + +// The store rejecting our seal is a verdict, not slowness: another +// producer's window owns the height. Riding the liveness timeout out to a +// broadcast after that is the double broadcast the gate exists to prevent. +func TestRejectedSealRefusesInsteadOfTimingOutToBroadcast(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + // Entries that are not this block's seal must not decide the gate: a + // refusal has to name our seal at our height, or an unrelated rejection + // would withhold a block nobody else is producing. + p.markGateLost(journalItem{kind: entryRecord, height: 2}) + p.markGateLost(journalItem{kind: entrySeal, height: 3}) + + p.mu.Lock() + spurious := p.gate.verdict == gateLost + p.mu.Unlock() + + if spurious { + t.Fatal("an unrelated rejection lost the gate") + } + + p.markGateLost(journalItem{kind: entrySeal, height: 2}) + + start := time.Now() + + if v := p.ConfirmSeal(sealGateBudget); v != miner.SealRefused { + t.Fatalf("verdict = %v, want Refused: the store rejected this "+ + "block's seal", v) + } + + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("took %v to refuse a seal the store already rejected: a "+ + "known verdict must not wait out the liveness budget", elapsed) + } +} + +// A refused block and its rebuild share a height, and the gate must confirm +// only on the gated block's own seal. Two layers enforce it: retire's +// lineage guard (a different block folds differently, so a stale ack fails +// the byte match) and the gate's hash key. Each is exercised here through a +// journal the item genuinely stands in. +func TestLateAckForAnotherSealDoesNotConfirmTheGate(t *testing.T) { + first := testHeader(2, common.Hash{0xaa}) + rebuild := testHeader(2, common.Hash{0xaa}) + rebuild.Extra = []byte("rebuild") + + seal := func(h *types.Header) ([]byte, *pb.Entry) { + raw, err := rlp.EncodeToBytes(h) + if err != nil { + t.Fatalf("encode: %v", err) + } + + return raw, sealEntry(raw, commitment.Head{}) + } + + ack := func(sealed *types.Header, gated common.Hash) gateVerdict { + p := barePublisher() + + _, entry := seal(sealed) + post := commitment.Head{0x99} + + p.mu.Lock() + p.journal.append(entry, commitment.Head{}, post, entrySeal, 2, 0, nil) + item := p.journal.items[len(p.journal.items)-1] + p.gate = sealGate{height: 2, hash: gated} + p.mu.Unlock() + + p.retire(item, time.Now()) + + p.mu.Lock() + defer p.mu.Unlock() + + return p.gate.verdict + } + + if v := ack(first, rebuild.Hash()); v == gateConfirmed { + t.Fatal("a late ack for a different block's seal confirmed the gate") + } + + if v := ack(rebuild, rebuild.Hash()); v != gateConfirmed { + t.Fatalf("the gated block's own seal ack did not confirm (verdict %d)", v) + } +} + +// A flush withheld while its gate is pending is the verdict, not a wait for +// one: the block is unbroadcast, so the canonical proof the withhold wants +// can never arrive. The refusal must also unwedge the rebuild — dead flush +// dropped, sealed tip rolled back, hold cleared — or every later build at +// this height orders itself behind a flush that cannot resolve. The +// liveness fallback's tolerance covers only the store's seal, so a foreign +// live window refuses a tolerant gate all the same. +func TestWithheldFlushRefusesThePendingGate(t *testing.T) { + for _, tolerate := range []bool{false, true} { + name := "plain gate" + if tolerate { + name = "tolerant gate" + } + + t.Run(name, func(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + h1 := testHeader(1, common.Hash{0xef}) + header := testHeader(2, h1.Hash()) + p.SealBlock(blockFor(header, []*types.Transaction{testTx(t, 1)})) + + p.mu.Lock() + p.gate.tolerateSealed = tolerate + p.mu.Unlock() + + foreign := openEntry(commitment.OpenContext{ + Number: 2, + Timestamp: header.Time + 7, + ParentHash: h1.Hash(), + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, commitment.Head{0x66}) + + info := tailInfo{ + s: commitment.Head{0x66}, + tipOpen: true, + tipOpenHeight: 2, + window: []*pb.Entry{foreign}, + } + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + if v := p.ConfirmSeal(sealGateBudget); v != miner.SealRefused { + t.Fatalf("verdict = %v, want refused", v) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if items, covered := p.journal.after(p.ackedSeq); !covered || len(items) != 0 { + t.Fatalf("the dead flush survived the refusal (covered=%v len=%d)", covered, len(items)) + } + + if p.sealedTip != 1 { + t.Fatalf("sealedTip = %d, want 1: the rebuild must adopt here, not mute", p.sealedTip) + } + + if p.hold.active() { + t.Fatal("the withheld flush's hold outlived the flush") + } + }) + } +} + +// A decoded store seal at the gated height settles the gate from +// classification: ours confirms (a twin delivered our copy), foreign +// refuses and drops the flush. +func TestStoreSealAtGateHeightSettlesTheGate(t *testing.T) { + h1 := testHeader(1, common.Hash{0xef}) + + cases := []struct { + name string + sealed func(ours common.Hash) common.Hash + want miner.SealVerdict + dropped bool + }{ + { + name: "foreign seal refuses", + sealed: func(common.Hash) common.Hash { return common.Hash{0xdd} }, + want: miner.SealRefused, + dropped: true, + }, + { + name: "our own seal confirms", + sealed: func(ours common.Hash) common.Hash { return ours }, + want: miner.SealConfirmed, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + header := testHeader(2, h1.Hash()) + p.SealBlock(blockFor(header, nil)) + + info := tailInfo{ + s: commitment.Head{0x55}, + haveSeal: true, + sealDecoded: true, + lastSealHeight: 2, + lastSealHash: tc.sealed(header.Hash()), + } + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + if v := p.ConfirmSeal(sealGateBudget); v != tc.want { + t.Fatalf("verdict = %v, want %v", v, tc.want) + } + + p.mu.Lock() + defer p.mu.Unlock() + + items, covered := p.journal.after(p.ackedSeq) + if gone := covered && len(items) == 0; gone != tc.dropped { + t.Fatalf("flush dropped = %v, want %v", gone, tc.dropped) + } + }) + } +} + +// A STALE while the gate is pending marks it contested, and a contested +// gate outlives the uncontested budget: the reconcile loop is producing the +// verdict, and cutting the wait short is what broadcast an empty block +// three seconds before its refusal would have arrived. +func TestContestedGateWaitsPastTheUncontestedBudget(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + h1 := testHeader(1, common.Hash{0xef}) + p.SealBlock(blockFor(testHeader(2, h1.Hash()), nil)) + + // A record of the flush STALEd: contest, not verdict. + p.markGateLost(journalItem{kind: entryRecord, height: 2}) + + verdicts := make(chan miner.SealVerdict, 1) + + go func() { verdicts <- p.ConfirmSeal(30 * time.Millisecond) }() + + select { + case v := <-verdicts: + t.Fatalf("gate resolved %v inside the uncontested budget; the contest must extend the wait", v) + case <-time.After(150 * time.Millisecond): + } + + // The seal's own STALE is the verdict. + p.markGateLost(journalItem{kind: entrySeal, height: 2}) + + select { + case v := <-verdicts: + if v != miner.SealRefused { + t.Fatalf("verdict = %v, want refused", v) + } + case <-time.After(2 * time.Second): + t.Fatal("contested gate never consumed the late verdict") + } +} + +// The last look before a verdictless broadcast: silence from the gate is +// not license to bury what the store accepted while we waited. +func TestGateTimeoutRecheck(t *testing.T) { + cases := []struct { + name string + tolerate bool + ownSeal bool + record bool + covered bool + want miner.SealVerdict + }{ + {name: "foreign seal refuses", want: miner.SealRefused}, + {name: "our own seal in the store confirms", ownSeal: true, want: miner.SealConfirmed}, + {name: "acked records the block lacks refuse", record: true, want: miner.SealRefused}, + {name: "a window the block carries broadcasts", record: true, covered: true, want: miner.SealUnknown}, + {name: "the liveness fallback tolerates the foreign seal", tolerate: true, want: miner.SealUnknown}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + gated := testHeader(2, parent) + tx := testTx(t, 3) + + appendForeignOpen(t, h, 2, parent) + + switch { + case tc.record: + appendForeignRecord(t, h, tx) + case tc.ownSeal: + appendForeignSeal(t, h, gated) + default: + lost := testHeader(2, parent) + lost.Extra = []byte("foreign") + appendForeignSeal(t, h, lost) + } + + var txs []common.Hash + if tc.covered { + txs = []common.Hash{tx.Hash()} + } + + p.mu.Lock() + p.gate = sealGate{height: 2, hash: gated.Hash(), txs: txs, tolerateSealed: tc.tolerate} + p.mu.Unlock() + + if v := p.ConfirmSeal(30 * time.Millisecond); v != tc.want { + t.Fatalf("verdict = %v, want %v", v, tc.want) + } + }) + } +} + +// A foreign live window above an unproven flush withholds the re-anchor: +// folding past it is how two full acked generations were buried under an +// empty sealed block. Only the chain ratifying our flush licenses the move, +// and while the gate is pending the withhold doubles as the refusal. +func TestFlushWithholdsUnderAForeignWindowAbove(t *testing.T) { + fc := &fakeChain{} + p, _ := lineagePublisher(t, fc) + + h1 := testHeader(1, common.Hash{0xef}) + header := testHeader(2, h1.Hash()) + p.SealBlock(blockFor(header, []*types.Transaction{testTx(t, 1)})) + + foreign := openEntry(commitment.OpenContext{ + Number: 3, + Timestamp: header.Time + 9, + ParentHash: common.Hash{0x77}, + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, commitment.Head{0x88}) + + info := tailInfo{ + s: commitment.Head{0x88}, + tipOpen: true, + tipOpenHeight: 3, + window: []*pb.Entry{foreign}, + haveSeal: true, + lastSealHeight: 1, + lastSealHash: common.Hash{0xaa}, + } + + journalBefore := len(p.journal.items) + + if out := p.applyTail(info); out != recOK { + t.Fatalf("outcome = %v", out) + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.hold.kind != holdSticky { + t.Fatal("flush over a foreign window above was not withheld") + } + + if len(p.journal.items) != journalBefore { + t.Fatal("withhold must not touch the lineage") + } + + if p.gate.verdict != gateLost { + t.Fatalf("gate verdict = %d, want lost: the withhold is the verdict", p.gate.verdict) + } +} diff --git a/eth/sequencer/harness_test.go b/eth/sequencer/harness_test.go new file mode 100644 index 0000000000..bfa1939e1c --- /dev/null +++ b/eth/sequencer/harness_test.go @@ -0,0 +1,61 @@ +package sequencer + +import ( + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// awaitOurWindow runs the seal barrier for a block whose content is exactly +// what this publisher has published — the healthy case, and what every +// barrier test meant before the barrier compared content. Tests that need a +// block diverging from the journal call AwaitSequenced directly. +func awaitOurWindow(p *Publisher, timeout time.Duration) bool { + p.mu.Lock() + height := p.curHeight + txs := journalTxs(p) + p.mu.Unlock() + + return p.AwaitSequenced(timeout, height, txs) +} + +func journalTxs(p *Publisher) []*types.Transaction { + start := p.journal.openStart() + if start < 0 { + return nil + } + + var txs []*types.Transaction + + for _, it := range p.journal.items[start+1:] { + rec := it.entry.GetRecord() + if rec == nil { + continue + } + + for _, raw := range rec.GetTransactions() { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err == nil { + txs = append(txs, tx) + } + } + } + + return txs +} + +// sealOnChain seals a block and records it as this node's block at that +// height — what the miner does, since resultLoop writes the block before it +// announces. A flush only displaces a live foreign window on proof the chain +// kept our block, so a test producer that seals without a chain write is +// modelling a block nobody accepted. +func sealOnChain(p *Publisher, fc *fakeChain, header *types.Header, txs []*types.Transaction) { + p.SealBlock(blockFor(header, txs)) + + if fc.canonical == nil { + fc.canonical = map[uint64]common.Hash{} + } + + fc.canonical[header.Number.Uint64()] = header.Hash() +} diff --git a/eth/sequencer/journal.go b/eth/sequencer/journal.go new file mode 100644 index 0000000000..38164ffd62 --- /dev/null +++ b/eth/sequencer/journal.go @@ -0,0 +1,413 @@ +package sequencer + +import ( + "bytes" + "sort" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +func openMatchesHeader(open *pb.BlockOpen, header *types.Header) bool { + return open != nil && open.GetBlockTimestamp() == header.Time && + open.GetGasLimit() == header.GasLimit && + common.BytesToHash(open.GetParentHash()) == header.ParentHash && + bytes.Equal(open.GetBaseFee(), baseFeeBytes(header.BaseFee)) +} + +func recordsMirrorTxs(items []journalItem, txs types.Transactions) bool { + i := 0 + + for _, it := range items { + if it.kind != entryRecord { + return false + } + + for _, h := range it.txHashes { + if i >= len(txs) || h != txs[i].Hash() { + return false + } + + i++ + } + } + + return i == len(txs) +} + +// How a window standing in the store relates to the one being built. +type windowRelation int + +const ( + windowForeign windowRelation = iota // a different lineage: contention + windowOurs // ours, whole or a prefix of it + windowExtendsOurs // ours, plus records we do not hold +) + +// relateWindowLocked compares the window the read returned against the one +// this node holds, entry by entry. Length alone cannot separate the cases: +// a rival's window at the same height can be shorter, equal, or longer than +// ours, and only content says which lineage it belongs to. +func (p *Publisher) relateWindowLocked(info tailInfo, height uint64) windowRelation { + ours := p.journal.suffixFromHeight(height) + if len(info.window) == 0 || len(ours) == 0 || ours[0].kind != entryOpen { + return windowForeign + } + + for i := 0; i < min(len(info.window), len(ours)); i++ { + if !contentEqual(info.window[i], ours[i].entry) { + return windowForeign + } + } + + if len(info.window) > len(ours) { + return windowExtendsOurs + } + + return windowOurs +} + +const ( + entryOpen = iota + entryRecord + entrySeal +) + +// journalItem is one published (or provisionally folded, while the store is +// unreachable) entry with the fold heads around it. Items carry a monotonic +// seq so cursor positions survive eviction. +type journalItem struct { + seq uint64 + entry *pb.Entry + pre commitment.Head + post commitment.Head + kind int + height uint64 + // txHashes caches a record's transaction hashes at append time, so the + // per-block mirror checks compare 32-byte values instead of re-decoding + // and re-hashing the window's wire bytes under the publisher lock. + txHashes []common.Hash +} + +// journal is the publisher's window in flight: the open window, the +// sealed-but-undelivered flushes actively streaming (at most +// journalHotSeals — older ones collapse to a height range and are rebuilt +// from the chain database when the store can take them), and up to +// journalSealedBlocks delivered blocks kept for reconnect replay. The +// open window is always retained. The chain database is the archive; +// the journal never grows beyond the work in flight. +type journal struct { + items []journalItem + seals int + nextSeq uint64 +} + +const ( + journalSealedBlocks = 8 + // journalHotSeals bounds the undelivered flushes kept in memory; older + // ones collapse to a pending height range (rebuilt from the chain + // database on reconcile). + journalHotSeals = 2 + // backfillBatchBytes caps how much block rebuilding one drain cycle does + // while holding the publisher lock. Every miner call — PublishTx most of + // all — waits on that lock, so a large batch stalls production outright: + // a 32MB batch cost a devnet 44 seconds on the cycle after a store + // outage. Smaller batches drain over more cycles and never own the lock + // long enough to be felt. + backfillBatchBytes = 2 << 20 + + // journalMaxBytes caps the adoption-collection read and one backfill + // batch — not a retention bound. + journalMaxBytes = 32 << 20 +) + +func newJournal() *journal { + return &journal{nextSeq: 1} +} + +func (r *journal) append(entry *pb.Entry, pre, post commitment.Head, kind int, height uint64, ackedThrough uint64, txHashes []common.Hash) { + // Every record carries its hashes; decode here only for a caller that + // could not supply them cheaper. + if kind == entryRecord && txHashes == nil { + for _, raw := range entry.GetRecord().GetTransactions() { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + break + } + + txHashes = append(txHashes, tx.Hash()) + } + } + + item := journalItem{ + seq: r.nextSeq, + entry: entry, + pre: pre, + post: post, + kind: kind, + height: height, + txHashes: txHashes, + } + + r.nextSeq++ + r.items = append(r.items, item) + + if kind == entrySeal { + r.seals++ + } + + r.evict(ackedThrough) +} + +// openStart returns the index of the open entry starting the current +// (unsealed) window, or -1 when the journal's tail is sealed or empty. +func (r *journal) openStart() int { + for i := len(r.items) - 1; i >= 0; i-- { + switch r.items[i].kind { + case entrySeal: + return -1 + case entryOpen: + return i + } + } + + return -1 +} + +// evict drops whole sealed blocks from the front, never touching the +// current open window. Delivered blocks evict past journalSealedBlocks (the +// gap-fill replay depth); undelivered seals are what a flush still owes +// the store and ride out retry pacing — until the hard count or byte cap, +// whose overflow is the documented forward-jump. +func (r *journal) evict(ackedThrough uint64) { + for r.seals > journalSealedBlocks && r.oldestSealAcked(ackedThrough) && r.dropOldestSealed() { + } +} + +// collapseOldestUnacked drops the oldest sealed block whose seal is still +// undelivered, returning its height so the caller can record it for a +// chain-database rebuild. Delivered blocks in front of it drop too — the +// store already has them. +func (r *journal) collapseOldestUnacked(acked uint64) (uint64, int, bool) { + i := r.firstSeal() + for i >= 0 && r.items[i].seq <= acked { + r.dropOldestSealed() + i = r.firstSeal() + } + + if i < 0 { + return 0, 0, false + } + + h := r.items[i].height + removed := 0 + + for j := 0; j <= i; j++ { + if r.items[j].seq > acked { + removed++ + } + } + + r.dropOldestSealed() + + return h, removed, true +} + +// firstSeal returns the index of the oldest seal, or -1 when none. +func (r *journal) firstSeal() int { + for i := range r.items { + if r.items[i].kind == entrySeal { + return i + } + } + + return -1 +} + +// oldestSealAcked reports whether the front sealed block was delivered. +func (r *journal) oldestSealAcked(ackedThrough uint64) bool { + i := r.firstSeal() + + return i >= 0 && r.items[i].seq <= ackedThrough +} + +// dropOldestSealed removes items from the front through the first seal. +// Returns false when no sealed block remains to drop. +func (r *journal) dropOldestSealed() bool { + i := r.firstSeal() + if i < 0 { + return false + } + + r.items = append(r.items[:0], r.items[i+1:]...) + r.seals-- + + return true +} + +// truncate drops every item from index n on, keeping the byte and seal +// accounting exact. +func (r *journal) truncate(n int) { + for _, dropped := range r.items[n:] { + if dropped.kind == entrySeal { + r.seals-- + } + } + + r.items = r.items[:n] +} + +// cutFromHeight returns the index where the undelivered run at or above +// height begins — the truncation point that drops a refused flush and +// anything chained above it while keeping acked entries and older flushes. +func (r *journal) cutFromHeight(acked, height uint64) int { + cut := len(r.items) + + for cut > 0 { + it := r.items[cut-1] + if it.seq <= acked || it.height < height { + break + } + + cut-- + } + + return cut +} + +// rebuildCut returns the index just past the last seal or last acked item — +// the boundary a window rebuild must not rewind past (an undelivered flush +// is still owed to the store; confirmed entries never rewind). +func (r *journal) rebuildCut(acked uint64) int { + for i := len(r.items) - 1; i >= 0; i-- { + if it := r.items[i]; it.kind == entrySeal || it.seq <= acked { + return i + 1 + } + } + + return 0 +} + +// itemAt returns the journal item carrying seq, if it is still present — +// lineage swaps and rewinds replace or drop items, so a seq alone no +// longer identifies live content. +func (r *journal) itemAt(seq uint64) (journalItem, bool) { + i := sort.Search(len(r.items), func(i int) bool { return r.items[i].seq >= seq }) + if i < len(r.items) && r.items[i].seq == seq { + return r.items[i], true + } + + return journalItem{}, false +} + +// after returns the items strictly after seq, and whether that position is +// still covered by the journal (false means eviction created a gap). +func (r *journal) after(seq uint64) ([]journalItem, bool) { + if len(r.items) == 0 { + return nil, seq+1 >= r.nextSeq + } + + if seq+1 < r.items[0].seq { + return nil, false + } + + i := sort.Search(len(r.items), func(i int) bool { return r.items[i].seq > seq }) + + return r.items[i:], true +} + +// findPost returns the seq of the item whose post-fold head equals h. +func (r *journal) findPost(h commitment.Head) (uint64, bool) { + for i := range r.items { + if r.items[i].post == h { + return r.items[i].seq, true + } + } + + return 0, false +} + +// suffixFromHeight returns the retained items starting at the first open +// entry with height >= h (a window boundary), or nil when none exists. +func (r *journal) suffixFromHeight(h uint64) []journalItem { + for i := range r.items { + if r.items[i].kind == entryOpen && r.items[i].height >= h { + return r.items[i:] + } + } + + return nil +} + +// journalWindowLocked lists the transactions this publisher has already +// published for a height, in order. +func (p *Publisher) journalWindowLocked(height uint64) []common.Hash { + start := p.journal.openStart() + if start < 0 || p.journal.items[start].height != height { + return nil + } + + var out []common.Hash + + for _, it := range p.journal.items[start+1:] { + if it.kind == entryRecord { + out = append(out, it.txHashes...) + } + } + + return out +} + +// windowTxHashes lists the transactions a store window promises, in order. +func windowTxHashes(window []*pb.Entry) ([]common.Hash, bool) { + var out []common.Hash + + for _, e := range window { + rec := e.GetRecord() + if rec == nil { + continue + } + + for _, raw := range rec.GetTransactions() { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + return nil, false + } + + out = append(out, tx.Hash()) + } + } + + return out, true +} + +// windowLeadsHashes reports whether the block delivers the promised +// sequence: the window's transactions must be a leading run of the block's, +// so a block may still be filling past the window but may never drop or +// reorder what the store already acked. +func windowLeadsHashes(window, txs []common.Hash) bool { + if len(window) > len(txs) { + return false + } + + for i, h := range window { + if txs[i] != h { + return false + } + } + + return true +} + +// txHashes projects transactions onto their hashes. +func txHashes(txs types.Transactions) []common.Hash { + out := make([]common.Hash, len(txs)) + for i, tx := range txs { + out[i] = tx.Hash() + } + + return out +} diff --git a/eth/sequencer/metrics.go b/eth/sequencer/metrics.go new file mode 100644 index 0000000000..6244fbc0c4 --- /dev/null +++ b/eth/sequencer/metrics.go @@ -0,0 +1,56 @@ +package sequencer + +import "github.com/ethereum/go-ethereum/metrics" + +// Publisher state gauge values; 0 is reserved for "off" — +// a disabled node has no publisher and never reports. contending refines +// degraded: the store is healthy, we are just losing head races to another +// publisher. +const ( + gaugeLive = 1 + gaugeDegraded = 2 + gaugeResyncing = 3 + gaugeFailed = 4 + gaugeContending = 5 +) + +var ( + publishAckTimer = metrics.NewRegisteredTimer("sequencer/publish/ack", nil) + publishedCounter = metrics.NewRegisteredCounter("sequencer/publish/entries", nil) + publishDropMeter = metrics.NewRegisteredMeter("sequencer/publish/dropped", nil) + publishFailedGauge = metrics.NewRegisteredGauge("sequencer/publish/failed", nil) + publishStateGauge = metrics.NewRegisteredGauge("sequencer/publish/state", nil) + publishQueueGauge = metrics.NewRegisteredGauge("sequencer/publish/queue", nil) + publishStaleCount = metrics.NewRegisteredCounter("sequencer/publish/stale", nil) + publishMutedCount = metrics.NewRegisteredCounter("sequencer/publish/muted", nil) + publishRecoverCount = metrics.NewRegisteredCounter("sequencer/publish/recovered", nil) + readHeadMismatch = metrics.NewRegisteredCounter("sequencer/read/headmismatch", nil) + readUnexplained = metrics.NewRegisteredCounter("sequencer/read/unexplained", nil) + barrierDivergedCount = metrics.NewRegisteredCounter("sequencer/publish/barrierdiverged", nil) + publishCatchupSkip = metrics.NewRegisteredCounter("sequencer/publish/catchupskip", nil) + backfillBatchTimer = metrics.NewRegisteredTimer("sequencer/backfill/batch", nil) + windowDisplacedRecords = metrics.NewRegisteredCounter("sequencer/reconcile/displacedrecords", nil) + publishBarrierTimeout = metrics.NewRegisteredCounter("sequencer/publish/barriertimeout", nil) + publishRedialCount = metrics.NewRegisteredCounter("sequencer/publish/redial", nil) + + // The seal gate's verdicts: confirmed seals broadcast, refused ones are + // discarded (another producer won the height), unknown means the budget + // expired and liveness broadcast anyway. + gateConfirmedCount = metrics.NewRegisteredCounter("sequencer/gate/confirmed", nil) + gateRefusedCount = metrics.NewRegisteredCounter("sequencer/gate/refused", nil) + gateUnknownCount = metrics.NewRegisteredCounter("sequencer/gate/unknown", nil) + gateRecheckRefused = metrics.NewRegisteredCounter("sequencer/gate/recheckrefused", nil) + + // Consumer-side preconfirmation pipeline: per-tx re-execution latency + // and receipts served to RPC readers before canonical import. + preconfApplyTimer = metrics.NewRegisteredTimer("sequencer/preconf/apply", nil) + preconfServedMeter = metrics.NewRegisteredMeter("sequencer/preconf/served", nil) + + reconcileGapfill = metrics.NewRegisteredCounter("sequencer/reconcile/gapfill", nil) + reconcileAdopt = metrics.NewRegisteredCounter("sequencer/reconcile/adopt", nil) + reconcileSupersede = metrics.NewRegisteredCounter("sequencer/reconcile/supersede", nil) + reconcileForwardJump = metrics.NewRegisteredCounter("sequencer/reconcile/forwardjump", nil) + reconcileYield = metrics.NewRegisteredCounter("sequencer/reconcile/yield", nil) + reconcileResync = metrics.NewRegisteredCounter("sequencer/reconcile/resync", nil) + reconcileTimer = metrics.NewRegisteredTimer("sequencer/reconcile/duration", nil) +) diff --git a/eth/sequencer/mirror_test.go b/eth/sequencer/mirror_test.go new file mode 100644 index 0000000000..3af6a6200f --- /dev/null +++ b/eth/sequencer/mirror_test.go @@ -0,0 +1,615 @@ +package sequencer + +import ( + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/miner" +) + +// The barrier's job is to compare the block against the store, not the +// journal against the store. A block built before an adoption leaves the +// journal perfectly in sync while the block itself carries different +// transactions — position agreement says nothing about what we are about to +// broadcast, and on a devnet that gap put two blocks at one height and +// orphaned the loser's acked records into the next block. +func TestBarrierRefusesABlockMissingThePromisedSequence(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Another producer owns the window at height 2 and has three records + // acked there. + t0, t1, t2 := testTx(t, 0), testTx(t, 1), testTx(t, 2) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{t0, t1, t2} { + appendForeignRecord(t, h, tx) + } + + // Our block carries only two of them: t1 was promised at this height + // and this block does not deliver it. + block := []*types.Transaction{t0, t2} + + if p.AwaitSequenced(time.Second, 2, block) { + t.Fatal("barrier passed a block missing a transaction the store " + + "already acked at this height") + } + + if !p.ResyncNeeded() { + t.Fatal("barrier refused without asking for a rebuild: the slot is " + + "dropped instead of corrected") + } +} + +// The same read, with a block that does deliver the promised sequence, must +// pass — a barrier that refuses the correct block stalls the chain. +func TestBarrierPassesABlockCarryingThePromisedSequence(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + t0, t1 := testTx(t, 0), testTx(t, 1) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{t0, t1} { + appendForeignRecord(t, h, tx) + } + + if !p.AwaitSequenced(time.Second, 2, []*types.Transaction{t0, t1}) { + t.Fatal("barrier refused a block that is exactly the store's window") + } +} + +// A block may still be filling past what the store has acked; only dropping +// or reordering the acked prefix is a violation. +func TestBarrierPassesABlockExtendingThePromisedSequence(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + t0 := testTx(t, 0) + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, t0) + + if !p.AwaitSequenced(time.Second, 2, []*types.Transaction{t0, testTx(t, 5)}) { + t.Fatal("barrier refused a block that carries the promised prefix " + + "and one more transaction of its own") + } +} + +// Reordering the acked prefix breaks the promise as surely as dropping it: +// a preconfirmation names a position, not just membership. +func TestBarrierRefusesAReorderedPromisedSequence(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + t0, t1 := testTx(t, 0), testTx(t, 1) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{t0, t1} { + appendForeignRecord(t, h, tx) + } + + if p.AwaitSequenced(time.Second, 2, []*types.Transaction{t1, t0}) { + t.Fatal("barrier passed a block that reordered the acked prefix") + } +} + +// An unreadable store must not gate production: liveness outranks a check we +// cannot perform. +func TestBarrierPassesWhenTheStoreCannotBeRead(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 0)) + + gate := &blockableConsumer{} + + p.mu.Lock() + gate.ConsumerServiceClient = p.read.cons + p.read.cons = gate + p.mu.Unlock() + gate.block(true) + + if !p.AwaitSequenced(time.Second, 2, []*types.Transaction{testTx(t, 9)}) { + t.Fatal("an unreadable store blocked production") + } +} + +// A re-anchor rebuilds our own entries onto the store head without ever +// ingesting what stands behind it, so displacing a live foreign window +// silently drops every record it holds that our block lacks. That is only +// defensible when our block is the one the chain kept; when it lost, the +// flush must wait rather than destroy the winner's sequence. +func TestFlushWithholdsDisplacementWhenOurBlockLost(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // We build and seal height 2 locally. + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + // Another producer's window takes the head at the same height, with + // records of its own, before our seal goes out. + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 7)) + + before := len(readAllGenerations(t, h)) + + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + // The chain kept that producer's block, not ours. + chain.canonical[2] = common.Hash{0xbb} + + info, _ := p.readTail(t.Context()) + + p.mu.Lock() + out, handled := p.classifyPendingFlushLocked(info) + p.mu.Unlock() + + if !handled || out != recOK { + t.Fatalf("flush classification: out=%v handled=%v", out, handled) + } + + time.Sleep(300 * time.Millisecond) + + if got := len(readAllGenerations(t, h)); got != before { + t.Fatalf("flush displaced a live window for a block the chain did "+ + "not keep: %d generations, want %d", got, before) + } +} + +// The devnet case: our seal is rejected and our flush reaches the store in +// the same millisecond, 134ms before the winner's block imports. The chain +// has no opinion yet at that instant, and treating "undecided" as permission +// is what displaced a live window on behalf of a block that was never +// broadcast. +func TestFlushWithholdsDisplacementWhileChainUndecided(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 7)) + + before := len(readAllGenerations(t, h)) + + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + // The chain holds nothing at this height: the winner's block is still + // in flight. Our block is not known to have been kept. + info, _ := p.readTail(t.Context()) + + p.mu.Lock() + out, handled := p.classifyPendingFlushLocked(info) + p.mu.Unlock() + + if !handled || out != recOK { + t.Fatalf("flush classification: out=%v handled=%v", out, handled) + } + + time.Sleep(300 * time.Millisecond) + + if got := len(readAllGenerations(t, h)); got != before { + t.Fatalf("displaced a live window while the chain had not kept our "+ + "block: %d generations, want %d", got, before) + } +} + +func mustReadTail(t *testing.T, p *Publisher) tailInfo { + t.Helper() + + info, out := p.readTail(t.Context()) + if out != recOK { + t.Fatalf("tail read: %v", out) + } + + return info +} + +// The displacement counter has to name what was actually lost. Counting the +// whole displaced window reports thousands of orphaned records when the +// block delivers every one of them, which makes the metric useless for +// telling a harmless supersede from a damaging one. +func TestDisplacementCountsOnlyWhatTheBlockDropped(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Our window: two transactions, published and acked. + shared, alsoShared := testTx(t, 0), testTx(t, 1) + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(shared) + p.PublishTx(alsoShared) + waitDrained(t, p, 5*time.Second) + + // The window we would displace holds both of ours plus one we lack. + dropped := testTx(t, 7) + appendForeignOpen(t, h, 2, parent) + + for _, tx := range []*types.Transaction{shared, alsoShared, dropped} { + appendForeignRecord(t, h, tx) + } + + // Our block seals with our two, and the chain keeps it, so the flush is + // allowed to displace — the counter must then name the one record lost. + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{shared, alsoShared})) + chain.canonical[2] = ours.Hash() + chain.blocks = map[uint64]*types.Block{ + 2: blockFor(ours, []*types.Transaction{shared, alsoShared}), + } + + info := mustReadTail(t, p) + before := windowDisplacedRecords.Snapshot().Count() + + p.mu.Lock() + may := p.mayDisplaceWindowLocked(info, 2) + p.mu.Unlock() + + if !may { + t.Fatal("withheld a displacement for a block the chain kept") + } + + if got := windowDisplacedRecords.Snapshot().Count() - before; got != 1 { + t.Fatalf("counted %d orphaned records, want 1: only the transaction "+ + "this block does not carry is lost", got) + } +} + +// Under load the drain does not finish inside the barrier's budget, and the +// old deadline path surrendered — returning "seal it" without ever comparing +// content. That is how a block with none of a 9523-record window reached the +// chain while every one of those records was already acked. The check has to +// survive the deadline; a still-draining window is a prefix of our own block, +// so the comparison is valid mid-drain. +func TestBarrierChecksContentEvenWhenTheDrainOvershoots(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // Another producer's window stands at our height with records acked. + promised := []*types.Transaction{testTx(t, 0), testTx(t, 1), testTx(t, 2)} + appendForeignOpen(t, h, 2, parent) + + for _, tx := range promised { + appendForeignRecord(t, h, tx) + } + + // Hold our own writes so the journal keeps entries in flight: the + // barrier can never reach unacked == 0 and must take the deadline path. + // A build hold, not a sticky one — sticky is the contended case and + // returns before the deadline is ever reached. + p.mu.Lock() + p.hold = hold{after: p.ackedSeq, kind: holdBuild} + p.mu.Unlock() + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 8)) + + p.mu.Lock() + draining, sticky := p.unackedLocked() > 0, p.hold.kind == holdSticky + p.mu.Unlock() + + if !draining || sticky { + t.Fatalf("setup did not reach the deadline path: draining=%v sticky=%v", + draining, sticky) + } + + // An empty block, exactly the shape that reached the chain at 915. + if p.AwaitSequenced(50*time.Millisecond, 2, nil) { + t.Fatal("barrier surrendered on its deadline and passed a block " + + "carrying none of the promised sequence") + } +} + +// The same deadline path must still pass a block that does carry the +// promised prefix, or a slow drain would stall production outright. +func TestBarrierDeadlineStillPassesACoveringBlock(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + t0 := testTx(t, 0) + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, t0) + + // Our block carries the promised record plus more still in flight. + if !p.AwaitSequenced(50*time.Millisecond, 2, + []*types.Transaction{t0, testTx(t, 1), testTx(t, 2)}) { + t.Fatal("barrier refused a block that carries the promised prefix " + + "and is still filling") + } +} + +// Every per-block wait on the store is pointless once the transport is +// failing, and paying them all pushes blocks past their slot — which is what +// arms bor's span-check path and turns a store outage into a chain +// slowdown. A devnet outage cost 22s per block this way. +func TestUnreachableStoreCostsNoWaiting(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.unreachable.Store(true) + + header := testHeader(2, parent) + p.SealBlock(blockFor(header, nil)) + + start := time.Now() + + if v := p.ConfirmSeal(4 * time.Second); v != miner.SealUnknown { + t.Fatalf("verdict = %v, want Unknown with the store unreachable", v) + } + + if !p.AwaitSequenced(4*time.Second, 3, nil) { + t.Fatal("the barrier gated production on an unreachable store") + } + + if elapsed := time.Since(start); elapsed > 200*time.Millisecond { + t.Fatalf("spent %v waiting on a store known to be down", elapsed) + } +} + +// And the moment a read succeeds the publisher stops treating the store as +// down: a latched flag would keep holding builds after recovery. +func TestSuccessfulReadClearsUnreachable(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + p.unreachable.Store(true) + p.AdoptWindow(2, sealHash(t, sealed)) + + if p.unreachable.Load() { + t.Fatal("a successful build-start read left the store marked down") + } +} + +// While the backfill drains, the store is behind us by construction: there +// is nothing worth comparing against and nothing worth waiting for. Making +// sealing wait through the drain is what stalled a devnet for 44 seconds on +// the cycle after a store outage. +func TestCatchUpDoesNotGateSealing(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + // A foreign window stands at our height that our block does not carry — + // normally grounds for a rebuild. Draining a backfill outranks it. + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 0)) + + p.mu.Lock() + p.pendingFrom, p.pendingTo = 1, 1 + p.mu.Unlock() + + start := time.Now() + + if !p.AwaitSequenced(2*time.Second, 2, nil) { + t.Fatal("sealing was gated while the backfill was still draining") + } + + if elapsed := time.Since(start); elapsed > 100*time.Millisecond { + t.Fatalf("waited %v during a drain that makes the comparison "+ + "meaningless anyway", elapsed) + } +} + +// The unreachable flag must be set by the transport layer itself when the +// store goes away — the no-wait short-circuits are worthless if nothing +// arms them. +func TestTransportFailureMarksTheStoreUnreachable(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{canonical: map[uint64]common.Hash{}}) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + p.OpenBlock(2, 1700000002, common.Hash{0xaa}, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + + waitFor(t, 10*time.Second, func() bool { return p.unreachable.Load() }) +} + +// A sealed foreign generation at the flush height is a stronger claim than a +// live window, and the flush must not chain past it on the head's bytes +// alone: with consensus undecided, superseding it is how a twin race minted +// generation after generation at one height. Only affirmative proof that +// the chain kept our block licenses the displacement — and our own seal +// already standing there is re-delivery, not displacement. +func TestFlushWithholdsOverAForeignSealUntilCanonical(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + // The twin's generation seals first; our own block for 2 exists too. + twin := testHeader(2, parent) + twin.Extra = []byte("twin") + appendForeignOpen(t, h, 2, parent) + appendForeignRecord(t, h, testTx(t, 7)) + appendForeignSeal(t, h, twin) + + ours := testHeader(2, parent) + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + info := mustReadTail(t, p) + + if !info.sealDecoded || info.lastSealHeight != 2 { + t.Fatalf("setup: read did not decode the foreign seal (%+v)", info) + } + + p.mu.Lock() + undecided := p.mayDisplaceWindowLocked(info, 2) + chain.canonical[2] = ours.Hash() + won := p.mayDisplaceWindowLocked(info, 2) + chain.canonical[2] = twin.Hash() + lost := p.mayDisplaceWindowLocked(info, 2) + p.mu.Unlock() + + if undecided { + t.Fatal("flush chained past a foreign seal with consensus undecided") + } + + if !won { + t.Fatal("flush withheld even though the chain kept our block") + } + + if lost { + t.Fatal("flush displaced the seal of the block the chain kept") + } +} + +// The store's standing seal being our own block is the duplicate-delivery +// case, not a displacement: it must not be withheld, or a resumed flush +// could never finish. +func TestFlushProceedsOverOurOwnStandingSeal(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + ours := testHeader(2, parent) + appendForeignOpen(t, h, 2, parent) + appendForeignSeal(t, h, ours) // the standing seal IS our block + + p.SealBlock(blockFor(ours, []*types.Transaction{testTx(t, 0)})) + + info := mustReadTail(t, p) + + p.mu.Lock() + may := p.mayDisplaceWindowLocked(info, 2) + p.mu.Unlock() + + if !may { + t.Fatal("our own standing seal was treated as a foreign displacement") + } +} + +// A displaced foreign seal has no window in the read to count, and the +// trailing window of a higher height must not be counted against this +// height's block: that mispairing once reported 1,960 phantom orphans for +// a displacement that destroyed nothing. +func TestSealDisplacementCountsNoPhantomOrphans(t *testing.T) { + fc := &fakeChain{canonical: map[uint64]common.Hash{}} + p, _ := lineagePublisher(t, fc) + + h1 := testHeader(1, common.Hash{0xef}) + header := testHeader(2, h1.Hash()) + tx := testTx(t, 1) + sealOnChain(p, fc, header, []*types.Transaction{tx}) + fc.blocks = map[uint64]*types.Block{2: blockFor(header, []*types.Transaction{tx})} + + stranger := testTx(t, 2) + raw, err := stranger.MarshalBinary() + if err != nil { + t.Fatalf("marshal: %v", err) + } + + open3 := openEntry(commitment.OpenContext{ + Number: 3, + Timestamp: header.Time + 4, + ParentHash: header.Hash(), + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, commitment.Head{0x77}) + + info := tailInfo{ + s: commitment.Head{0x79}, + tipOpen: true, + tipOpenHeight: 3, + window: []*pb.Entry{open3, recordEntry(raw, commitment.Head{0x78})}, + haveSeal: true, + sealDecoded: true, + lastSealHeight: 2, + lastSealHash: common.Hash{0xdd}, + } + + before := windowDisplacedRecords.Snapshot().Count() + + p.mu.Lock() + may := p.mayDisplaceWindowLocked(info, 2) + p.mu.Unlock() + + if !may { + t.Fatal("withheld a canonical-proven displacement") + } + + if got := windowDisplacedRecords.Snapshot().Count() - before; got != 0 { + t.Fatalf("counted %d orphans from a higher height's window", got) + } +} diff --git a/eth/sequencer/outage_test.go b/eth/sequencer/outage_test.go new file mode 100644 index 0000000000..201654a2d9 --- /dev/null +++ b/eth/sequencer/outage_test.go @@ -0,0 +1,187 @@ +package sequencer + +import ( + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// outageChain builds a hash-linked chain the publisher can backfill from: the +// chain database is the archive during an outage, so the fake must serve +// complete blocks by number. +func outageChain(t *testing.T, txsPer int, through uint64) (*fakeChain, []*types.Block) { + t.Helper() + + chain := &fakeChain{ + canonical: map[uint64]common.Hash{}, + known: map[common.Hash]*types.Header{}, + blocks: map[uint64]*types.Block{}, + } + + parent := common.Hash{0xef} + blocks := make([]*types.Block, 0, through) + + for n := uint64(1); n <= through; n++ { + header := testHeader(n, parent) + + var txs []*types.Transaction + for i := 0; i < txsPer; i++ { + txs = append(txs, testTx(t, uint64(i))) + } + + block := blockFor(header, txs) + chain.blocks[n] = block + chain.canonical[n] = block.Hash() + chain.known[block.Hash()] = block.Header() + chain.current = block.Header() + blocks = append(blocks, block) + parent = block.Hash() + } + + return chain, blocks +} + +// sealThrough runs the producer's publishing lifecycle over the given blocks: +// open, records, seal — as the worker would during the outage. +func sealThrough(p *Publisher, blocks []*types.Block) { + for _, b := range blocks { + h := b.Header() + p.OpenBlock(h.Number.Uint64(), h.Time, h.ParentHash, h.GasLimit, h.BaseFee) + + for _, tx := range b.Transactions() { + p.PublishTx(tx) + } + + p.SealBlock(b) + } +} + +// The outage contract: the chain never stops for the store, and when the +// store returns, the producer backfills every missing block — oldest first, +// so a reader never sees the sealed tip jump a gap it would then refuse to +// fill. Nothing during the outage was acked, so nothing was promised, and +// the audit must come back clean. +func TestOutageBackfillsGaplessInOrder(t *testing.T) { + h := startHarness(t) + + const through = 12 + + chain, blocks := outageChain(t, 2, through) + p := newTestPublisher(t, h, chain) + + // Block 1 lands normally; the store then goes down. + sealThrough(p, blocks[:1]) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + // The chain keeps producing through the outage. No ack can arrive, so + // no preconfirmation is issued for any of this. + sealThrough(p, blocks[1:]) + + h.resume() + + // The store converges to the tip: every block sealed, oldest first. + waitFor(t, 20*time.Second, func() bool { + sealed := 0 + + for _, g := range readAllGenerations(t, h) { + if g.sealed { + sealed++ + } + } + + return sealed >= through + }) + + gens := readAllGenerations(t, h) + + last := uint64(0) + for _, g := range gens { + if g.height < last { + t.Fatalf("backfill wrote height %d after height %d: out of order, "+ + "the sealed tip jumped a gap", g.height, last) + } + + last = g.height + } + + canonical := map[uint64][]common.Hash{} + for _, b := range blocks { + hashes := []common.Hash{} + for _, tx := range b.Transactions() { + hashes = append(hashes, tx.Hash()) + } + + canonical[b.NumberU64()] = hashes + } + + audit := auditStore(t, h, canonical) + if !audit.clean() { + t.Fatalf("outage recovery broke promises: %+v", audit) + } +} + +// After catch-up the live stream resumes: a new block opened at the tip +// publishes normally, gated behind nothing. +func TestLiveStreamResumesAfterCatchUp(t *testing.T) { + h := startHarness(t) + + const through = 6 + + chain, blocks := outageChain(t, 1, through) + p := newTestPublisher(t, h, chain) + + sealThrough(p, blocks[:1]) + waitHead(t, h, p, 5*time.Second) + + h.stop() + sealThrough(p, blocks[1:]) + h.resume() + + waitFor(t, 20*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.pendingFrom == 0 && p.unackedLocked() == 0 + }) + + // The next build finds a clean boundary at the tip and streams live. + tip := blocks[len(blocks)-1] + if w := p.AdoptWindow(through+1, tip.Hash()); w != nil { + t.Fatalf("clean boundary after catch-up offered an adoption: %+v", w) + } + + p.OpenBlock(through+1, tip.Header().Time+2, tip.Hash(), tip.GasLimit(), fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 10*time.Second) +} + +// During the outage the barrier must not gate production: the store is +// unreachable, nothing acks, and the chain seals anyway. +func TestBarrierNeverGatesDuringOutage(t *testing.T) { + h := startHarness(t) + + chain, blocks := outageChain(t, 1, 3) + p := newTestPublisher(t, h, chain) + + sealThrough(p, blocks[:1]) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + head := blocks[0] + p.OpenBlock(2, head.Header().Time+2, head.Hash(), head.GasLimit(), fee25()) + p.PublishTx(testTx(t, 0)) + + start := time.Now() + if !awaitOurWindow(p, 300*time.Millisecond) { + t.Fatal("an unreachable store gated a seal") + } + + if time.Since(start) > 2*time.Second { + t.Fatal("the liveness override took too long") + } +} diff --git a/eth/sequencer/probes_test.go b/eth/sequencer/probes_test.go new file mode 100644 index 0000000000..4bafac8b40 --- /dev/null +++ b/eth/sequencer/probes_test.go @@ -0,0 +1,339 @@ +package sequencer + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/ethereum/go-ethereum/common" +) + +// edgeConsumer answers Range block reads for heights 1..edge and NOT_FOUND +// above, isolating the probe arithmetic from transport. +type edgeConsumer struct { + pb.ConsumerServiceClient + edge uint64 +} + +func (c *edgeConsumer) Range(_ context.Context, req *pb.RangeRequest, _ ...grpc.CallOption) (*pb.RangeResponse, error) { + h := req.GetBlock() + if c.edge > 0 && h >= 1 && h <= c.edge { + return &pb.RangeResponse{}, nil + } + + return nil, status.Error(codes.NotFound, "unknown block") +} + +func probePublisher(edge uint64) *Publisher { + p := barePublisher() + p.read.cons = &edgeConsumer{edge: edge} + + return p +} + +func TestProbeDownTable(t *testing.T) { + cases := []struct { + name string + edge uint64 + from uint64 + want uint64 + wantFound bool + }{ + {name: "far above edge", edge: 3, from: 10, want: 3, wantFound: true}, + {name: "exact hit", edge: 3, from: 3, want: 3, wantFound: true}, + {name: "one above edge", edge: 3, from: 4, want: 3, wantFound: true}, + {name: "single block store", edge: 1, from: 1, want: 1, wantFound: true}, + {name: "descent lands inside store", edge: 2, from: 4, want: 2, wantFound: true}, + {name: "power of two descent", edge: 7, from: 8, want: 7, wantFound: true}, + // The descent gives up once its stride overshoots (step >= h): a low + // edge far below the start is the floor-read rung's job. + {name: "edge below descent reach", edge: 1, from: 5, wantFound: false}, + {name: "deep gap gives up", edge: 2, from: 100, wantFound: false}, + {name: "empty store", edge: 0, from: 6, wantFound: false}, + {name: "empty store from one", edge: 0, from: 1, wantFound: false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := probePublisher(tc.edge) + + got, found, err := p.read.probeDown(context.Background(), tc.from) + if err != nil { + t.Fatalf("probeDown err: %v", err) + } + + if found != tc.wantFound { + t.Fatalf("found = %v, want %v", found, tc.wantFound) + } + + if found && got != tc.want { + t.Fatalf("edge = %d, want %d", got, tc.want) + } + }) + } +} + +func TestProbeUpTable(t *testing.T) { + cases := []struct { + name string + edge uint64 + h0 uint64 + want uint64 + }{ + {name: "from floor", edge: 3, h0: 1, want: 3}, + {name: "already at edge", edge: 3, h0: 3, want: 3}, + {name: "doubling ascent", edge: 8, h0: 1, want: 8}, + {name: "mid start", edge: 5, h0: 2, want: 5}, + {name: "long ascent", edge: 21, h0: 1, want: 21}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := probePublisher(tc.edge) + + got, err := p.read.probeUp(context.Background(), tc.h0) + if err != nil { + t.Fatalf("probeUp err: %v", err) + } + + if got != tc.want { + t.Fatalf("edge = %d, want %d", got, tc.want) + } + }) + } +} + +type failingConsumer struct { + pb.ConsumerServiceClient +} + +func (c *failingConsumer) Range(context.Context, *pb.RangeRequest, ...grpc.CallOption) (*pb.RangeResponse, error) { + return nil, status.Error(codes.Unavailable, "store down") +} + +// Transport errors must propagate out of the probes — only NOT_FOUND means +// "unknown height"; anything else aborts the ladder rung. +func TestProbesPropagateTransportErrors(t *testing.T) { + p := barePublisher() + p.read.cons = &failingConsumer{} + + if _, _, err := p.read.probeDown(context.Background(), 10); err == nil { + t.Fatal("probeDown swallowed a transport error") + } + + if _, err := p.read.probeUp(context.Background(), 1); err == nil { + t.Fatal("probeUp swallowed a transport error") + } + + if _, _, err := p.read.binarySearchEdge(context.Background(), 1, 5); err == nil { + t.Fatal("binarySearchEdge swallowed a transport error") + } + + if _, err := p.read.blockKnown(context.Background(), 1); err == nil { + t.Fatal("blockKnown treated a transport error as an answer") + } +} + +// Unacked accounting is by sequence arithmetic: entries evicted from the +// journal while unconfirmed still count (they are what a forward jump drops). +func TestUnackedCountsEvicted(t *testing.T) { + p := barePublisher() + + parent := common.Hash{0xef} + for n := uint64(1); n <= 42; n++ { + header := testHeader(n, parent) + p.OpenBlock(n, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.SealBlock(blockFor(header, nil)) + parent = header.Hash() + } + + if p.failed.Load() { + t.Fatal("publisher failed during setup") + } + + total := 2 * 42 + + p.mu.Lock() + defer p.mu.Unlock() + + if got := p.unackedLocked(); got != total { + t.Fatalf("unacked = %d, want %d (evicted entries must count)", got, total) + } + + if int(p.journal.nextSeq-1) != total { + t.Fatalf("nextSeq = %d, want %d appends", p.journal.nextSeq-1, total) + } + + if items, _ := p.journal.after(0); len(items) >= total { + t.Fatal("eviction did not trim the journal; test premise broken") + } + + p.ackedSeq = 3 + + if got := p.unackedLocked(); got != total-3 { + t.Fatalf("unacked after acks = %d, want %d", got, total-3) + } +} + +// endlessConsumer returns a full page on every Range call and never reports +// live — a tail longer than the walk bound from the requested position. +type endlessConsumer struct { + pb.ConsumerServiceClient +} + +func (c *endlessConsumer) Range(_ context.Context, req *pb.RangeRequest, _ ...grpc.CallOption) (*pb.RangeResponse, error) { + n := int(req.GetLimit()) + if n == 0 || n > 64 { + n = 64 + } + + entries := make([]*pb.Entry, n) + for i := range entries { + entries[i] = &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{}}} + } + + return &pb.RangeResponse{Entries: entries, Next: make([]byte, 32)}, nil +} + +// A rung whose position is too far behind the tip to walk within bound must +// fall through the ladder (like NOT_FOUND), not spin on the same rung. +func TestTryWalkFallsThroughOnTooLongTail(t *testing.T) { + p := barePublisher() + p.read.cons = &endlessConsumer{} + + _, out, done := p.tryWalk(context.Background(), &pb.RangeRequest{}, false) + if done { + t.Fatal("too-long tail must fall through to the next ladder rung") + } + + if out != recRetry { + t.Fatalf("outcome = %v, want recRetry", out) + } +} + +// slowEndlessConsumer serves endless slow pages for tail walks from one +// specific position (a stale anchor over a huge history) and delegates +// everything else to the real store client. +type slowEndlessConsumer struct { + pb.ConsumerServiceClient + stale []byte +} + +func (c *slowEndlessConsumer) Range(ctx context.Context, req *pb.RangeRequest, opts ...grpc.CallOption) (*pb.RangeResponse, error) { + if h := req.GetHead(); h != nil && bytes.Equal(h, c.stale) { + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + // Like the real gRPC client: a status error, not the raw + // context sentinel. + return nil, status.Error(codes.DeadlineExceeded, ctx.Err().Error()) + } + + n := int(req.GetLimit()) + if n == 0 || n > 64 { + n = 64 + } + + entries := make([]*pb.Entry, n) + for i := range entries { + entries[i] = &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{}}} + } + + return &pb.RangeResponse{Entries: entries, Next: c.stale}, nil + } + + return c.ConsumerServiceClient.Range(ctx, req, opts...) +} + +// A takeover build whose publisher anchored far behind (an idle adopter) +// must still find and adopt the dangling window: the stale-anchor rung +// runs out of its budget slice and the ladder falls through to the +// tip-edge probe instead of consuming the whole read budget. +func TestStaleAnchorTakeoverStillAdopts(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + parent := sealHash(t, sealed) + tx := testTx(t, 0) + foreignWindow(t, h, 2, parent, tx) + + // Pin the anchor (and the persisted head) to a position that walks + // forever — the stale-anchor shape after a long idle stretch. + stale := commitment.Head{0xaa, 0xbb} + p.mu.Lock() + p.read.cons = &slowEndlessConsumer{ConsumerServiceClient: p.read.cons, stale: stale.Bytes()} + p.anchor = stale + p.confirmed = true + p.mu.Unlock() + + w := p.AdoptWindow(2, parent) + if w == nil || len(w.Txs) != 1 || w.Txs[0].Hash() != tx.Hash() { + t.Fatalf("stale-anchor read missed the window: %+v", w) + } +} + +// An idle publisher (a non-producer between spans) periodically re-anchors +// near the store tip, so its eventual takeover read starts close. +func TestIdleReconcileKeepsAnchorFresh(t *testing.T) { + restore := idleReconcileInterval + idleReconcileInterval = 80 * time.Millisecond + + t.Cleanup(func() { idleReconcileInterval = restore }) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // A foreign producer seals 2 and opens 3: the idle publisher should + // re-anchor at window 3's base without any build of its own. + parent := sealHash(t, sealed) + ts, gasLimit := foreignWindow(t, h, 2, parent, testTx(t, 0)) + appendForeignSeal(t, h, windowHeader(2, parent, ts, gasLimit)) + + base := h.store.Head() + foreignWindow(t, h, 3, common.Hash{0x33}, testTx(t, 1)) + + waitFor(t, 5*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.anchor == base + }) +} + +// emptyNotLiveConsumer models a trailing gateway: an empty page that is +// not yet at the tip. floorRead must fall through to the full floor walk +// instead of indexing the empty page (a crash found by adversarial review). +type emptyNotLiveConsumer struct{ pb.ConsumerServiceClient } + +func (c *emptyNotLiveConsumer) Range(_ context.Context, req *pb.RangeRequest, _ ...grpc.CallOption) (*pb.RangeResponse, error) { + if req.GetLimit() == 1 { + return &pb.RangeResponse{Next: make([]byte, 32), Live: false}, nil + } + + return &pb.RangeResponse{Next: make([]byte, 32), Live: true}, nil +} + +func TestFloorReadEmptyNotLivePage(t *testing.T) { + p := barePublisher() + p.read.cons = &emptyNotLiveConsumer{} + + info, out := p.read.floorRead(t.Context()) + if out == recTerminal { + t.Fatalf("empty not-live floor page must not be terminal: %v", out) + } + + _ = info +} diff --git a/eth/sequencer/publish.go b/eth/sequencer/publish.go new file mode 100644 index 0000000000..d35caba510 --- /dev/null +++ b/eth/sequencer/publish.go @@ -0,0 +1,314 @@ +package sequencer + +import ( + "math/big" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// OpenBlock publishes block context the moment the producer opens it. A +// re-open of the same height (work-cycle restart) or an open on a different +// parent (reorg mid-build) supersedes the in-progress window downstream. +func (p *Publisher) OpenBlock(number uint64, timestamp uint64, parent common.Hash, gasLimit uint64, baseFee *big.Int) { + if p.failed.Load() { + return + } + + open := commitment.OpenContext{ + Number: number, + Timestamp: timestamp, + ParentHash: parent, + GasLimit: gasLimit, + BaseFee: baseFee, + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.mode.kind != modeOpen || (number != 0 && number <= p.sealedTip) { + // Muted, or a seal result raced the mute clear between this + // build's check and its open: never open at/behind a sealed + // height. + return + } + + if p.adoptOpenLocked(number, timestamp, parent, gasLimit, baseFee) { + return // engaged an adopted window already in the store + } + + next, err := commitment.FoldOpen(p.head, open) + if err != nil { + p.fail("fold open context", "err", err) + + return + } + + p.awaitOpen = false + p.curHeight = number + p.appendLocked(openEntry(open, p.head), next, entryOpen, number, nil) +} + +// PublishTx publishes one committed transaction. +func (p *Publisher) PublishTx(tx *types.Transaction) { + if p.failed.Load() { + return + } + + raw, err := tx.MarshalBinary() + if err != nil { + p.fail("encode transaction", "hash", tx.Hash(), "err", err) + + return + } + + p.mu.Lock() + defer p.mu.Unlock() + + if p.mode.kind != modeOpen || p.awaitOpen { + return + } + + if p.adoptTxLocked(tx.Hash()) { + return // matched: the record is already in the store + } + + p.appendLocked(recordEntry(raw, p.head), commitment.FoldTx(p.head, raw), entryRecord, p.curHeight, []common.Hash{tx.Hash()}) +} + +// SealBlock is the seal flush: the block sealed and is being +// broadcast, so the store must come to match it. The lineage's current +// window either already mirrors the sealed content (the journal was built +// alongside the block) or is rebuilt from the block's body; the seal +// closes it and the hold lifts — buffered entries and the seal go out +// together. A seal is never dropped. STALEs on the released entries land +// in reconciliation, which completes (byte-match) or re-anchors +// (the only supersede). +func (p *Publisher) SealBlock(block *types.Block) { + if p.failed.Load() { + return + } + + header := block.Header() + + raw, err := rlp.EncodeToBytes(header) + if err != nil { + p.fail("encode sealed header", "number", header.Number, "err", err) + + return + } + + p.mu.Lock() + defer p.mu.Unlock() + + p.adopt = nil + + mode := p.mode + p.mode = buildMode{} + + n := header.Number.Uint64() + prevTip := p.sealedTip + + if n > p.sealedTip { + p.sealedTip = n + } + + recovered := mode.kind == modeRecover && mode.height == n + if recovered || mode.kind == modeSealedWait { + // This block is a rebuild of a generation the store already holds, + // seal included. Republishing it would open a second generation over + // a sealed height — the displacement this whole design exists to + // prevent. Nothing to publish and nothing to gate: the block only + // needs to reach the chain. + p.curHeight = 0 + p.awaitOpen = false + p.hold = clearedHold() + + if recovered { + p.gate = sealGate{} + + return + } + + // Not a recovery: the store closed this height with content this + // block does not carry, and the winner's block has not arrived yet. + // Broadcasting divergent content here is the displacement the gate + // exists to stop, so an undecided height refuses. The next build + // recovers the height properly once the grace has elapsed. + p.gate = sealGate{height: n, hash: block.Hash(), refuseOnTimeout: true} + + return + } + + if !p.windowMirrorsLocked(block) { + // The journal's window does not describe the sealed block (a adopt + // that raced a straggler seal, a purge, a divergent rebuild): + // rebuild it from the body so the flush carries the sealed truth. + if !p.rebuildWindowLocked(block) { + return // fail() already recorded the cause + } + } + + // appendLocked already signaled the send loop; the hold clears after so the + // released entries and the seal go out together on that wake. + p.appendLocked(sealEntry(raw, p.head), commitment.FoldSeal(p.head, commitment.SealedHash(raw)), entrySeal, n, nil) + + // Arm the broadcast gate: ConfirmSeal resolves it from the seal's ack, + // the chain, or its deadline. + p.gate = sealGate{ + height: n, hash: block.Hash(), published: true, prevTip: prevTip, + txs: txHashes(block.Transactions()), + tolerateSealed: mode.kind == modeOverSealed && mode.height == n, + } + + p.curHeight = 0 + p.awaitOpen = false + p.hold = clearedHold() + p.collapseColdLocked() +} + +// collapseColdLocked bounds the undelivered flushes held in memory: past +// journalHotSeals, the oldest collapse to the pending height range and are +// rebuilt from the chain database when a re-anchor can deliver them. The +// send cursor detects the gap and routes through reconciliation. +func (p *Publisher) collapseColdLocked() { + for p.unackedSealsLocked() > journalHotSeals { + h, removed, ok := p.journal.collapseOldestUnacked(p.ackedSeq) + if !ok { + return + } + + // Merge into the pending range, never clobber it: during an active + // drain the collapse reaches heights BELOW pendingFrom (a stranded + // batch suffix is the journal's oldest content), and overwriting + // pendingTo downward inverted the range — which the backfill then + // read as "nothing owed" and zeroed. That silent evaporation + // stranded 25 heights of a 50-block outage as permanent holes. + if p.pendingFrom == 0 || h < p.pendingFrom { + p.pendingFrom = h + } + + if h > p.pendingTo { + p.pendingTo = h + } + + p.pendingEntries += removed + } +} + +// unackedSealsLocked counts sealed-but-undelivered flushes in the journal. +func (p *Publisher) unackedSealsLocked() int { + n := 0 + + for _, it := range p.journal.items { + if it.kind == entrySeal && it.seq > p.ackedSeq { + n++ + } + } + + return n +} + +// windowMirrorsLocked reports whether the journal's trailing window is an +// open at the block's height whose records are exactly the block's +// transactions, in order. +func (p *Publisher) windowMirrorsLocked(block *types.Block) bool { + if p.awaitOpen { + return false + } + + start := p.journal.openStart() + if start < 0 || p.journal.items[start].height != block.NumberU64() { + return false + } + + // The window's open context must equal the sealed header: consumers + // pinned it at open and void the window on any mismatch, so a context + // drift means rebuild, not complete-in-place. + if !openMatchesHeader(p.journal.items[start].entry.GetBlockOpen(), block.Header()) { + return false + } + + return recordsMirrorTxs(p.journal.items[start+1:], block.Transactions()) +} + +// rebuildWindowLocked replaces the lineage's trailing window with one built +// from the sealed block's body, folded onto the confirmed prefix. +func (p *Publisher) rebuildWindowLocked(block *types.Block) bool { + // Drop the trailing undelivered open window; the sealed block replaces + // it (an adopted window is acked and stays; the rebuild folds on top). + if cut := p.journal.rebuildCut(p.ackedSeq); cut < len(p.journal.items) { + p.rewindJournalLocked(cut) + } + + header := block.Header() + open := openEntry(commitment.OpenContext{ + Number: header.Number.Uint64(), + Timestamp: header.Time, + ParentHash: header.ParentHash, + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }, p.head) + + next, err := foldEntry(p.head, open) + if err != nil { + p.fail("fold flush open", "number", header.Number, "err", err) + + return false + } + + p.appendLocked(open, next, entryOpen, header.Number.Uint64(), nil) + + for _, tx := range block.Transactions() { + rawTx, err := tx.MarshalBinary() + if err != nil { + p.fail("encode flush transaction", "hash", tx.Hash(), "err", err) + + return false + } + + p.appendLocked(recordEntry(rawTx, p.head), commitment.FoldTx(p.head, rawTx), entryRecord, header.Number.Uint64(), []common.Hash{tx.Hash()}) + } + + return true +} + +// rewindJournalLocked truncates the journal to its first n items and returns the +// fold head to the truncation point. +func (p *Publisher) rewindJournalLocked(n int) { + p.journal.truncate(n) + + if n > 0 { + last := p.journal.items[n-1] + p.head = last.post + + if p.ackedSeq > last.seq { + p.ackedSeq = last.seq + } + } else { + p.head = p.anchor + } +} + +// appendLocked folds one entry into the lineage and wakes the transport. +func (p *Publisher) appendLocked(entry *pb.Entry, next commitment.Head, kind int, height uint64, txHashes []common.Hash) { + p.journal.append(entry, p.head, next, kind, height, p.ackedSeq, txHashes) + p.head = next + publishQueueGauge.Update(int64(p.unackedLocked())) + p.signalWake() +} + +// baseFeeBytes encodes a base fee for the wire: a nil or zero fee is the +// empty slice, which is what the commitment fold hashes — the two encodings +// must agree or a published open stops matching its fold head. +func baseFeeBytes(f *big.Int) []byte { + if f == nil { + return []byte{} + } + + return f.Bytes() +} diff --git a/eth/sequencer/publisher.go b/eth/sequencer/publisher.go new file mode 100644 index 0000000000..77346ba7e8 --- /dev/null +++ b/eth/sequencer/publisher.go @@ -0,0 +1,339 @@ +// Package sequencer connects Bor to the sequence store: the block producer +// publishes each block's lifecycle (open, transactions, seal) as it happens. +// The wire contract and commitment chain live in +// github.com/0xPolygon/sequence-store-proto; the design and terminology +// reference is docs/sequencer-bor.md. +package sequencer + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials/insecure" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +const ( + // Probe backoff while the store is unreachable. The cap stays small so a + // replayable gap cannot outgrow the journal while waiting. + probeBackoffMin = time.Second + probeBackoffMax = 5 * time.Second + + // Backoff between reconciles whose corrective publish immediately + // re-STALEs — losing head races to another publisher. + reconcileBackoffMin = 500 * time.Millisecond + reconcileBackoffMax = 8 * time.Second +) + +// ackStallTimeout bounds how long the send loop waits on an unacked entry before +// treating the stream as dead (a hung store stalls acks without +// erroring). Kept under the journal's coverage so the reconnect can still +// resend instead of jumping. Var for tests. +var ackStallTimeout = 5 * time.Second + +// chainReader is the chain access reconciliation needs: classifying +// sealed store content against the local chain, locating the store tail +// on a cold start from the last imported block, and rebuilding collapsed +// blocks for backfill — the chain database is the archive; the journal +// holds only the work in flight. +type chainReader interface { + GetCanonicalHash(number uint64) common.Hash + GetHeaderByHash(hash common.Hash) *types.Header + CurrentBlock() *types.Header + GetBlockByNumber(number uint64) *types.Block +} + +// Publisher streams block-production entries to the sequence store. Enqueue +// methods are called from the worker's goroutines and never block: folds are +// computed inline (sub-microsecond), transport happens on a background +// goroutine. When the store is unreachable the publisher keeps folding into +// its retention journal and recovers by journal replay or forward jump on +// reconnect; a STALE ack triggers tail-read reconciliation. +// Only MALFORMED acks and fold divergence are terminal. +type Publisher struct { + // mu makes fold-and-append atomic across the worker's goroutines and + // serializes lineage swaps against the transport goroutine. + mu sync.Mutex + head commitment.Head // local fold tip (end of the lineage) + journal *journal + ackedSeq uint64 // seq of the last store-confirmed journal item + anchor commitment.Head // store head confirming everything through ackedSeq + anchored bool // anchor established by a completed reconcile + confirmed bool // anchor was ever store-confirmed + curHeight uint64 // height of the open window being built (0 = none) + awaitOpen bool // drop records until the next OpenBlock (post purge) + adopt *adoption // store window being adopted + sealedTip uint64 // highest height we sealed-and-flushed + // mode is the build treatment for the current height, decided once per + // build-start classification; see buildMode. + mode buildMode + + // resync is set when the store shows another producer building the + // height this node is building: our block must not seal beside their + // sequence. Reading it clears it, ending the worker's current cycle — + // the next cycle's build-start read adopts the standing window, and a + // signal that survives to that read is cleared there as stale. + resync bool + + // pendingFrom..pendingTo (inclusive; 0 = none) are sealed blocks + // collapsed out of the journal while the store could not take them. + // They live in the chain database and are rebuilt at the next + // re-anchor (backfill). storeSealedTip is the highest height the + // store is known to have sealed — delivered by us or read from a + // tail — and floors the backfill: the store never needs a block at + // or below it. + pendingFrom uint64 + pendingTo uint64 + pendingEntries int // exact entry count of the collapsed range + storeSealedTip uint64 + + hold hold // the send loop's send ceiling; see the hold type + + // gate is the broadcast gate for the last sealed block; see sealGate. + gate sealGate + refusals refusalStreak + seed commitment.Head // an empty store's head, computable without the store + // unreachable is set while the transport is failing. Every per-block + // wait on the store is pointless in that state, and paying them all + // pushes blocks past their slot — which is what arms bor's span-check + // path and turns a store outage into a chain slowdown. + unreachable atomic.Bool + + failed atomic.Bool + + wake chan struct{} + + poll time.Duration + chain chainReader + + pubConn *grpc.ClientConn + consConn *grpc.ClientConn + pub pb.PublisherServiceClient + read *reader + + // Redial bookkeeping: the endpoints to dial again, and the time of the + // last successful store interaction of any kind. gRPC channels are + // supposed to heal on their own; a store container restart has wedged + // them in a permanent connect-retry loop while fresh dials worked, so + // after redialAfter of silence the publisher starts its connections + // over instead of trusting the channel state machine. + pubEndpoint string + consEndpoint string + lastContact atomic.Int64 // unix nanos + + cancel context.CancelFunc + done chan struct{} +} + +// NewPublisher dials the store and starts the transport goroutine. The +// publisher service (publish stream) and consumer service (reconcile tail +// reads) have their own endpoints. On a cold start the store +// tail is relocated from the chain's last imported block, so no +// local position is persisted. +func NewPublisher(publisherEndpoint, consumerEndpoint string, chainID uint64, poll time.Duration, chain chainReader) (*Publisher, error) { + pubConn, consConn, err := dialStore(publisherEndpoint, consumerEndpoint) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(context.Background()) + + p := &Publisher{ + head: commitment.Seed(chainID), + anchor: commitment.Seed(chainID), + seed: commitment.Seed(chainID), + hold: clearedHold(), + journal: newJournal(), + wake: make(chan struct{}, 1), + poll: poll, + chain: chain, + pubConn: pubConn, + consConn: consConn, + pub: pb.NewPublisherServiceClient(pubConn), + cancel: cancel, + done: make(chan struct{}), + } + + p.pubEndpoint, p.consEndpoint = publisherEndpoint, consumerEndpoint + p.lastContact.Store(time.Now().UnixNano()) + + p.read = newReader(pb.NewConsumerServiceClient(consConn), p.seed, p.markReachable) + + publishStateGauge.Update(gaugeDegraded) // until the startup reconcile anchors + + go p.run(ctx) + + return p, nil +} + +// markReachable notes a successful store round trip. The reader invokes it +// on every read and retire on every ack, so a recovered store stops being +// treated as down the moment anything hears from it — and the contact +// stamp is what holds the redial back. +func (p *Publisher) markReachable() { + p.unreachable.Store(false) + p.lastContact.Store(time.Now().UnixNano()) +} + +// dialStore opens the publisher and consumer connections. gRPC's own +// reconnect backoff is capped at the probe cap: after a long outage the +// default (up to 120s) would keep the connection down well past the +// store's return, growing the gap a forward jump abandons. +func dialStore(publisherEndpoint, consumerEndpoint string) (*grpc.ClientConn, *grpc.ClientConn, error) { + connBackoff := backoff.DefaultConfig + connBackoff.MaxDelay = probeBackoffMax + + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithConnectParams(grpc.ConnectParams{Backoff: connBackoff}), + } + + pubConn, err := grpc.NewClient(publisherEndpoint, opts...) + if err != nil { + return nil, nil, err + } + + consConn, err := grpc.NewClient(consumerEndpoint, opts...) + if err != nil { + _ = pubConn.Close() + + return nil, nil, err + } + + return pubConn, consConn, nil +} + +// redialAfter is how much store silence the publisher tolerates before +// rebuilding its connections, and the floor between rebuilds. Var for tests. +var redialAfter = time.Minute + +// silentTooLong reports whether every store interaction has failed for a +// whole redial interval. +func (p *Publisher) silentTooLong() bool { + return time.Since(time.Unix(0, p.lastContact.Load())) > redialAfter +} + +// redial replaces both store connections and swaps the clients in place. +// The transport goroutine owns p.pub; the reader hands out its client +// under its own lock, so in-flight calls finish on the old connections, +// which close in the background. +func (p *Publisher) redial() { + pubConn, consConn, err := dialStore(p.pubEndpoint, p.consEndpoint) + if err != nil { + log.Warn("Sequencer redial failed", "err", err) + + return + } + + p.mu.Lock() + oldPub, oldCons := p.pubConn, p.consConn + p.pubConn, p.consConn = pubConn, consConn + p.pub = pb.NewPublisherServiceClient(pubConn) + p.mu.Unlock() + + p.read.setClient(pb.NewConsumerServiceClient(consConn)) + + _ = oldPub.Close() + _ = oldCons.Close() + + publishRedialCount.Inc(1) + log.Warn("Sequencer redialed the store after prolonged silence") +} + +// signalWake nudges the transport goroutine without blocking: the wake +// channel is a depth-1 signal, so a pending wake absorbs this one. +func (p *Publisher) signalWake() { + select { + case p.wake <- struct{}{}: + default: + } +} + +// advanceStoreSealedTipLocked records the newest seal a read decoded. Only a +// decoded seal qualifies — an inferred boundary can be a live partial window, +// and booking one as sealed is how an outage hole got pinned as permanent. +func (p *Publisher) advanceStoreSealedTipLocked(info tailInfo, origin string) { + if info.sealDecoded && info.lastSealHeight > p.storeSealedTip { + log.Debug("Sequencer sealed tip advance", "origin", origin, + "from", p.storeSealedTip, "to", info.lastSealHeight) + p.storeSealedTip = info.lastSealHeight + } +} + +// unackedLocked counts entries past the acked frontier by sequence +// arithmetic, so unacked entries already evicted from the journal still count +// (they are exactly what a forward jump abandons). +func (p *Publisher) unackedLocked() int { + return int(p.journal.nextSeq - 1 - p.ackedSeq) +} + +// heldMidBuild reports a live build whose entries are gated by a sticky +// hold: we are not writing, so only a deliberate re-read can show us what +// the competing producer is doing. +func (p *Publisher) heldMidBuild() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.curHeight != 0 && p.hold.kind == holdSticky +} + +// ourOpenParentLocked is the parent hash of the window this node is +// building, read from its open entry in the journal. +func (p *Publisher) ourOpenParentLocked() (common.Hash, bool) { + idx := p.journal.openStart() + if idx < 0 { + return common.Hash{}, false + } + + open := p.journal.items[idx].entry.GetBlockOpen() + if open == nil { + return common.Hash{}, false + } + + return common.BytesToHash(open.GetParentHash()), true +} + +// RefreshInterval is the txpool poll cadence the worker uses while a block is +// open; zero keeps the one-shot fill. +func (p *Publisher) RefreshInterval() time.Duration { + return p.poll +} + +// Close stops the transport goroutine. No local state is persisted: a +// restart relocates the store tail from the chain's last imported block. +func (p *Publisher) Close() { + p.cancel() + <-p.done +} + +func (p *Publisher) fail(msg string, args ...any) { + if p.failed.CompareAndSwap(false, true) { + publishFailedGauge.Update(1) + publishStateGauge.Update(gaugeFailed) + log.Error("Sequencer publishing disabled: "+msg, args...) + } +} + +func (p *Publisher) isAnchored() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.anchored +} + +func (p *Publisher) setUnanchored() { + p.mu.Lock() + defer p.mu.Unlock() + + p.anchored = false +} diff --git a/eth/sequencer/publisher_test.go b/eth/sequencer/publisher_test.go new file mode 100644 index 0000000000..d20689d190 --- /dev/null +++ b/eth/sequencer/publisher_test.go @@ -0,0 +1,607 @@ +package sequencer + +import ( + "context" + "math/big" + "net" + "testing" + "time" + + "google.golang.org/grpc" + + "github.com/0xPolygon/sequence-store-proto/commitment" + "github.com/0xPolygon/sequence-store-proto/devstore" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" +) + +const testChainID = 1337 + +// harness serves one devstore over TCP and can stop and resume serving on +// the same address, keeping the store's state — an outage, from the +// publisher's point of view. +type harness struct { + t *testing.T + store *devstore.Store + addr string + srv *grpc.Server +} + +func startHarness(t *testing.T) *harness { + return startHarnessChain(t, testChainID) +} + +func startHarnessChain(t *testing.T, chainID uint64) *harness { + t.Helper() + + h := &harness{t: t, store: devstore.New(chainID)} + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + h.addr = lis.Addr().String() + h.serve(lis) + t.Cleanup(func() { h.srv.Stop() }) + + return h +} + +func (h *harness) serve(lis net.Listener) { + h.srv = grpc.NewServer() + pb.RegisterPublisherServiceServer(h.srv, h.store) + pb.RegisterConsumerServiceServer(h.srv, h.store) + + go func() { _ = h.srv.Serve(lis) }() +} + +func (h *harness) stop() { + h.srv.Stop() +} + +func (h *harness) resume() { + h.t.Helper() + + deadline := time.Now().Add(5 * time.Second) + + for { + lis, err := net.Listen("tcp", h.addr) + if err == nil { + h.serve(lis) + + return + } + + if time.Now().After(deadline) { + h.t.Fatalf("relisten %s: %v", h.addr, err) + } + + time.Sleep(50 * time.Millisecond) + } +} + +func newTestPublisher(t *testing.T, h *harness, chain chainReader) *Publisher { + t.Helper() + + p, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + + return p +} + +func testTx(t *testing.T, nonce uint64) *types.Transaction { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("key: %v", err) + } + + tx, err := types.SignNewTx(key, types.LatestSignerForChainID(big.NewInt(testChainID)), &types.DynamicFeeTx{ + ChainID: big.NewInt(testChainID), + Nonce: nonce, + GasTipCap: big.NewInt(1), + GasFeeCap: big.NewInt(30_000_000_000), + Gas: 21000, + To: &common.Address{0x01}, + }) + if err != nil { + t.Fatalf("sign: %v", err) + } + + return tx +} + +func testHeader(number uint64, parent common.Hash) *types.Header { + return &types.Header{ + ParentHash: parent, + Number: new(big.Int).SetUint64(number), + GasLimit: 30_000_000, + Time: 1700000000 + number, + BaseFee: big.NewInt(25_000_000_000), + Difficulty: big.NewInt(1), + } +} + +// blockFor assembles a sealed block from a header and its transactions. +func blockFor(header *types.Header, txs []*types.Transaction) *types.Block { + return types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: txs}) +} + +// publishBlock drives a full lifecycle through the publisher and returns the +// sealed header. +func publishBlock(t *testing.T, p *Publisher, number uint64, parent common.Hash, txs int) *types.Header { + t.Helper() + + header := testHeader(number, parent) + p.OpenBlock(number, header.Time, parent, header.GasLimit, header.BaseFee) + + var body []*types.Transaction + + for i := 0; i < txs; i++ { + tx := testTx(t, uint64(i)) + body = append(body, tx) + p.PublishTx(tx) + } + + p.SealBlock(blockFor(header, body)) + + return header +} + +func waitHead(t *testing.T, h *harness, p *Publisher, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + + for { + p.mu.Lock() + local := p.head + p.mu.Unlock() + + if h.store.Head() == local { + return + } + + if time.Now().After(deadline) { + p.mu.Lock() + defer p.mu.Unlock() + t.Fatalf("store head %x never reached local head %x", h.store.Head(), p.head) + } + + time.Sleep(20 * time.Millisecond) + } +} + +// The publisher's folds must match the store's across a full lifecycle. +func TestPublisherLifecycle(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + header := publishBlock(t, p, 1, common.Hash{0xef}, 3) + waitHead(t, h, p, 5*time.Second) + + raw, err := rlp.EncodeToBytes(header) + if err != nil { + t.Fatalf("rlp: %v", err) + } + + // Independently computed chain: seed → open → 3 txs → seal. + want := commitment.Seed(testChainID) + want, err = commitment.FoldOpen(want, commitment.OpenContext{ + Number: 1, + Timestamp: header.Time, + ParentHash: common.Hash{0xef}, + GasLimit: header.GasLimit, + BaseFee: header.BaseFee, + }) + if err != nil { + t.Fatalf("fold open: %v", err) + } + + p.mu.Lock() + items, _ := p.journal.after(0) + p.mu.Unlock() + + for _, item := range items { + if item.kind == entryRecord { + want = commitment.FoldTxs(want, item.entry.GetRecord().GetTransactions()) + } + } + + want = commitment.FoldSeal(want, commitment.SealedHash(raw)) + + if h.store.Head() != want { + t.Fatalf("store head %x, want %x", h.store.Head(), want) + } +} + +// A restart against a non-empty store relocates the tail from the chain's +// last block and the next open extends it in place — no fresh topic +// required and no forward jump counted (nothing abandoned). +func TestRestartWarmResume(t *testing.T) { + h := startHarness(t) + + jumps := reconcileForwardJump.Snapshot().Count() + + first, err := NewPublisher(h.addr, h.addr, testChainID, 0, nil) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + sealed := publishBlock(t, first, 1, common.Hash{0xef}, 2) + waitHead(t, h, first, 5*time.Second) + first.Close() + + // The restart locates the store tail from the chain's last block. + chain := &fakeChain{current: &types.Header{Number: big.NewInt(1)}} + + second, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(second.Close) + + publishBlock(t, second, 2, sealHash(t, sealed), 1) + waitHead(t, h, second, 5*time.Second) + + if got := reconcileForwardJump.Snapshot().Count(); got != jumps { + t.Fatalf("clean warm resume counted %d forward jumps", got-jumps) + } +} + +// A cold publisher against a non-empty store anchors via the ladder's probe +// and floor rungs, located from the chain's last block. +func TestStartupColdNonEmptyStore(t *testing.T) { + h := startHarness(t) + + seed := newTestPublisher(t, h, nil) + sealed := publishBlock(t, seed, 1, common.Hash{0xef}, 1) + waitHead(t, h, seed, 5*time.Second) + seed.Close() + + chain := &fakeChain{current: &types.Header{Number: big.NewInt(1)}} + + p, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + + publishBlock(t, p, 2, sealHash(t, sealed), 1) + waitHead(t, h, p, 5*time.Second) +} + +// A store outage while blocks keep being produced recovers by journal replay: +// full continuity, every entry published. +func TestOutageJournalReplay(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + // Produced entirely during the outage; folded into the journal only. + sealed2 := publishBlock(t, p, 2, sealHash(t, sealed), 2) + publishBlock(t, p, 3, sealHash(t, sealed2), 1) + + h.resume() + waitHead(t, h, p, 15*time.Second) + + if got := h.store.Head(); got != localHead(p) { + t.Fatalf("replay did not converge: store %x local %x", got, localHead(p)) + } +} + +// A publisher whose chain names a height the store does not have (wiped or +// far behind) falls through the ladder to the floor and re-anchors on the +// seed. +func TestRestartAgainstUnknownStore(t *testing.T) { + h := startHarness(t) + + // Chain claims height 9, but the store is empty: probe finds nothing, + // the floor read seeds, and publishing resumes. + chain := &fakeChain{current: &types.Header{Number: big.NewInt(9)}} + + p, err := NewPublisher(h.addr, h.addr, testChainID, 0, chain) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + + waitFor(t, 5*time.Second, func() bool { return p.isAnchored() }) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) +} + +// A dial-time error must surface, not return a nil publisher. +func TestNewPublisherRejectsBadEndpoint(t *testing.T) { + if p, err := NewPublisher("bad\x7ftarget:99:99", "bad\x7ftarget:99:99", testChainID, 0, nil); err == nil { + p.Close() + t.Fatal("bad endpoint accepted") + } +} + +// A fold failure (nil base fee) is a publisher bug: terminal, and every +// later enqueue is a no-op. +func TestOpenBlockFoldFailureIsTerminal(t *testing.T) { + p := barePublisher() + + p.OpenBlock(1, 1700000001, common.Hash{0x01}, 30_000_000, nil) + + if !p.failed.Load() { + t.Fatal("fold failure not terminal") + } + + p.PublishTx(testTx(t, 0)) + + if items, _ := p.journal.after(0); len(items) != 0 { + t.Fatalf("enqueue after failure: %d items", len(items)) + } +} + +// Records and seals are dropped while awaiting the next open after a purge. +func TestAwaitOpenSuppressesRecords(t *testing.T) { + p := barePublisher() + p.awaitOpen = true + + p.PublishTx(testTx(t, 0)) + + if items, _ := p.journal.after(0); len(items) != 0 { + t.Fatalf("suppressed records reached the journal: %d", len(items)) + } + + // A seal is never suppressed: the flush rebuilds the window from + // the block body. + p.SealBlock(blockFor(testHeader(1, common.Hash{0x01}), nil)) + + if items, _ := p.journal.after(0); len(items) != 2 { + t.Fatalf("seal flush must rebuild open+seal: %d items", len(items)) + } + + header := testHeader(2, common.Hash{0x02}) + p.OpenBlock(2, header.Time, header.ParentHash, header.GasLimit, header.BaseFee) + p.PublishTx(testTx(t, 1)) + + if items, _ := p.journal.after(0); len(items) != 4 { + t.Fatalf("open must clear the suppression: %d items", len(items)) + } +} + +func sealHash(t *testing.T, header *types.Header) common.Hash { + t.Helper() + + return header.Hash() +} + +func localHead(p *Publisher) commitment.Head { + p.mu.Lock() + defer p.mu.Unlock() + + return p.head +} + +// Progress resets the contention streak; a repeat no-progress STALE backs +// off before the next reconcile, starting at the minimum. +func TestContentionSleep(t *testing.T) { + ctx := context.Background() + + if got := contentionSleep(ctx, true, 3); got != 0 { + t.Fatalf("progress must reset the streak, got %d", got) + } + + start := time.Now() + + if got := contentionSleep(ctx, false, 0); got != 1 { + t.Fatalf("first stale streak = %d, want 1", got) + } + + if elapsed := time.Since(start); elapsed > 200*time.Millisecond { + t.Fatalf("first stale must not sleep, took %v", elapsed) + } + + start = time.Now() + + if got := contentionSleep(ctx, false, 1); got != 2 { + t.Fatalf("second stale streak = %d, want 2", got) + } + + if elapsed := time.Since(start); elapsed < reconcileBackoffMin || elapsed > 4*reconcileBackoffMin { + t.Fatalf("second stale slept %v, want ~%v", elapsed, reconcileBackoffMin) + } +} + +// A publisher pointed at a store seeded for a different chain anchors on +// the foreign seed and publishes blindly — nothing alarms on the producer +// side (detection is the consumer's and the auditor's job). +func TestWrongChainStorePublishesBlindly(t *testing.T) { + h := startHarnessChain(t, testChainID+1) + p := newTestPublisher(t, h, nil) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + if p.failed.Load() { + t.Fatal("wrong-chain store must not fail the publisher (blind by design)") + } +} + +func journalEntry(kind int) *pb.Entry { + switch kind { + case entryOpen: + return &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{}}} + case entrySeal: + return &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{}}} + default: + return &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{Transactions: [][]byte{{0x01}}}}} + } +} + +// appendBlock adds an open+record+seal window at height h, delivered +// (acked), so the count bound may evict it. +func appendJournalBlock(r *journal, h uint64) { + r.append(journalEntry(entryOpen), commitment.Head{}, commitment.Head{}, entryOpen, h, r.nextSeq, nil) + r.append(journalEntry(entryRecord), commitment.Head{}, commitment.Head{}, entryRecord, h, r.nextSeq, nil) + r.append(journalEntry(entrySeal), commitment.Head{}, commitment.Head{}, entrySeal, h, r.nextSeq, nil) +} + +func TestJournalEvictsOldestSealedOnly(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= journalSealedBlocks+2; h++ { + appendJournalBlock(r, h) + } + + // Two oldest sealed blocks evicted, open-window invariant untouched. + if r.seals != journalSealedBlocks { + t.Fatalf("seals retained %d, want %d", r.seals, journalSealedBlocks) + } + + if first := r.items[0]; first.height != 3 || first.kind != entryOpen { + t.Fatalf("front is height %d kind %d, want open of height 3", first.height, first.kind) + } +} + +func TestJournalNeverEvictsOpenWindow(t *testing.T) { + r := newJournal() + + // One open window far over the byte cap: nothing sealed to drop. + r.append(journalEntry(entryOpen), commitment.Head{}, commitment.Head{}, entryOpen, 1, r.nextSeq, nil) + + huge := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{make([]byte, journalMaxBytes+1)}, + }}} + r.append(huge, commitment.Head{}, commitment.Head{}, entryRecord, 1, r.nextSeq, nil) + + if len(r.items) != 2 { + t.Fatalf("open window evicted: %d items", len(r.items)) + } +} + +func TestJournalAfterGapDetection(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= journalSealedBlocks+3; h++ { + appendJournalBlock(r, h) + } + + // seq 1 was evicted: the position is no longer covered. + if _, covered := r.after(0); covered { + t.Fatal("gap not detected after eviction") + } + + items, covered := r.after(r.items[0].seq) + if !covered || len(items) != len(r.items)-1 { + t.Fatalf("covered=%v len=%d", covered, len(items)) + } +} + +func TestJournalAfterEmptyBoundary(t *testing.T) { + r := newJournal() + + // Empty journal: only the position right before nextSeq is covered. + if _, covered := r.after(0); !covered { + t.Fatal("fresh empty journal must cover seq 0") + } + + appendJournalBlock(r, 1) + r.items, r.seals = nil, 0 // simulate a full drain + + if _, covered := r.after(r.nextSeq - 1); !covered { + t.Fatal("drained journal must cover its frontier") + } + + if _, covered := r.after(r.nextSeq - 2); covered { + t.Fatal("drained journal must not cover older positions") + } +} + +func TestJournalEvictOnlyBeyondSealedBound(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= journalSealedBlocks; h++ { + appendJournalBlock(r, h) + } + + if r.seals != journalSealedBlocks || r.items[0].height != 1 { + t.Fatalf("eviction fired at the bound: seals=%d front=%d", r.seals, r.items[0].height) + } +} + +// Undelivered seals are never evicted: eviction only trims delivered +// history. Older undelivered flushes leave via collapseOldestUnacked, +// which hands their heights to the chain-database backfill. +func TestJournalKeepsUndeliveredSeals(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= journalSealedBlocks+40; h++ { + r.append(journalEntry(entryOpen), commitment.Head{}, commitment.Head{}, entryOpen, h, 0, nil) + r.append(journalEntry(entrySeal), commitment.Head{}, commitment.Head{}, entrySeal, h, 0, nil) + } + + if r.seals != journalSealedBlocks+40 || r.items[0].height != 1 { + t.Fatalf("undelivered seal evicted: seals=%d front=%d", r.seals, r.items[0].height) + } +} + +func TestJournalCollapseOldestUnacked(t *testing.T) { + r := newJournal() + + for h := uint64(1); h <= 4; h++ { + r.append(journalEntry(entryOpen), commitment.Head{}, commitment.Head{}, entryOpen, h, 0, nil) + r.append(journalEntry(entrySeal), commitment.Head{}, commitment.Head{}, entrySeal, h, 0, nil) + } + + // Block 1 delivered (acked through its seal at seq 2); 2-4 undelivered. + // Each undelivered block is open+seal = 2 entries. + if h, n, ok := r.collapseOldestUnacked(2); !ok || h != 2 || n != 2 { + t.Fatalf("collapse = %d/%d/%v, want 2/2", h, n, ok) + } + + // The delivered block dropped with it; block 3 is now the front. + if r.items[0].height != 3 || r.seals != 2 { + t.Fatalf("front=%d seals=%d after collapse", r.items[0].height, r.seals) + } + + if _, _, ok := r.collapseOldestUnacked(2); !ok { + t.Fatal("second collapse must pop block 3") + } + + if h, n, ok := r.collapseOldestUnacked(2); !ok || h != 4 || n != 2 { + t.Fatalf("third collapse = %d/%d/%v, want 4/2", h, n, ok) + } + + if _, _, ok := r.collapseOldestUnacked(2); ok { + t.Fatal("no seals left to collapse") + } +} + +func TestJournalSuffixFromHeight(t *testing.T) { + r := newJournal() + appendJournalBlock(r, 5) + appendJournalBlock(r, 6) + + suffix := r.suffixFromHeight(6) + if len(suffix) != 3 || suffix[0].kind != entryOpen || suffix[0].height != 6 { + t.Fatalf("suffix wrong: len=%d", len(suffix)) + } + + if r.suffixFromHeight(7) != nil { + t.Fatal("suffix beyond retained heights should be nil") + } +} diff --git a/eth/sequencer/reader.go b/eth/sequencer/reader.go new file mode 100644 index 0000000000..d2c7af8269 --- /dev/null +++ b/eth/sequencer/reader.go @@ -0,0 +1,513 @@ +package sequencer + +import ( + "context" + "errors" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +const ( + // One Range page and the walk bound; anchored positions keep tails small. + walkLimit = 512 + maxWalkLoops = 128 + + // Per-read deadline (the worker never waits on this — the + // whole reconcile runs on the transport goroutine). + tailReadTimeout = time.Second +) + +var ( + errFoldDivergence = errors.New("byte-identical entry folds to a different head") + errTailTooLong = errors.New("tail walk exceeded bound") +) + +// reader is the store-facing read layer: bounded walks that derive the head +// they land on, generation probes over the block index, and per-height +// generation fetches. It holds no publisher state — the same machinery that +// serves the publisher's reconciliation serves any consumer of the store, +// which is what a subscribing RPC node will be. +type reader struct { + // mu guards cons, which a redial hot-swaps while the worker and + // transport goroutines read through it. + mu sync.RWMutex + cons pb.ConsumerServiceClient + + seed commitment.Head // the empty log's head, computed from the chain id + + // onRead runs after every successful store round trip. The publisher + // hangs its reachability bookkeeping on it: the build-start read runs + // every block, so it is what notices the store is back. + onRead func() +} + +func newReader(cons pb.ConsumerServiceClient, seed commitment.Head, onRead func()) *reader { + return &reader{cons: cons, seed: seed, onRead: onRead} +} + +func (r *reader) client() pb.ConsumerServiceClient { + r.mu.RLock() + defer r.mu.RUnlock() + + return r.cons +} + +func (r *reader) setClient(cons pb.ConsumerServiceClient) { + r.mu.Lock() + defer r.mu.Unlock() + + r.cons = cons +} + +// tailInfo summarizes one tail read: the store head S and what the tip +// looks like (classification inputs). +type tailInfo struct { + s commitment.Head + tipOpen bool + tipOpenHeight uint64 + tipOpenParent common.Hash + haveSeal bool + lastSealHeight uint64 + lastSealHash common.Hash + // sealDecoded separates a seal this read actually decoded from one a + // probe merely inferred. Only the former may drive sealed-tip + // bookkeeping: an inferred "boundary" can be a live partial window, + // and recording it as sealed told every backfill the height was owed + // nothing — pinning an outage's partial delivery as a permanent hole. + sealDecoded bool + + // window collects the trailing open window's entries for adoption, + // capped at the journal byte bound; an over-cap window is not + // collected and falls back to supersede. + window []*pb.Entry + windowBytes int + + // explained is set when s was folded from the walked entries rather + // than taken on the store's word. A head we cannot derive tells us + // nothing about what produced it — in particular, whether some other + // producer already has a live window at the height we are about to + // open. + explained bool +} + +// tryWalk runs one ladder rung; done=false falls through to the next rung: +// the position is unknown to the store (NOT_FOUND) or so far behind the tip +// that walking from it exceeds the bound — the probe and floor rungs anchor +// near the tip with a short walk instead. A recTerminal outcome is a fold +// divergence, which only a matched walk (an absorb hook that compares +// prefixes) can produce — the caller owns what terminal means. +func (r *reader) tryWalk(ctx context.Context, first *pb.RangeRequest, absorb func(*pb.Entry) error) (tailInfo, reconcileOutcome, bool) { + info, err := r.walk(ctx, first, absorb) + + switch { + case err == nil: + return info, recOK, true + case isNotFound(err): + return tailInfo{}, recRetry, false + case errors.Is(err, errTailTooLong): + log.Warn("Sequencer tail read position too far behind, probing near tip") + + return tailInfo{}, recRetry, false + case errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded: + // This rung's budget slice expired mid-walk — the position is too + // far behind to walk in time (gRPC surfaces the expiry as a status + // error that does not wrap the context sentinel). Fall through: + // the probe and floor rungs anchor near the tip with a short walk. + log.Warn("Sequencer tail read rung out of budget, probing near tip") + + return tailInfo{}, recRetry, false + case errors.Is(err, errFoldDivergence): + return tailInfo{}, recTerminal, true + default: + log.Warn("Sequencer tail read", "err", err) + + return tailInfo{}, recRetry, true + } +} + +func (r *reader) probedWalk(ctx context.Context, from uint64) (tailInfo, reconcileOutcome, bool) { + h, found, err := r.probeDown(ctx, from) + if err != nil { + return tailInfo{}, recRetry, true + } + + if !found { + return tailInfo{}, recRetry, false + } + + return r.tryWalk(ctx, blockReq(h), nil) +} + +// floorRead anchors on the earliest retained entry: an empty store yields +// the seed; a known floor height anchors an upward probe to the tip edge. +func (r *reader) floorRead(ctx context.Context) (tailInfo, reconcileOutcome) { + resp, err := r.rangeOnce(ctx, &pb.RangeRequest{Limit: 1}) + if err != nil { + return tailInfo{}, recRetry + } + + if len(resp.GetEntries()) == 0 && resp.GetLive() { + s, ok := headFrom(resp.GetNext()) + if !ok { + return tailInfo{}, recRetry + } + + // An empty store is the one head we can derive with no entries at + // all: it must be the seed. Anything else is the store telling us + // about history it did not show us. + return tailInfo{s: s, explained: s == r.seed}, recOK + } + + // An empty page that is not live is coherent (a trailing gateway not + // yet at its tip): nothing to anchor a probe on — fall through to the + // full floor walk, which loops pages to live. + if len(resp.GetEntries()) > 0 { + if h, ok := entryHeight(resp.GetEntries()[0]); ok { + if edge, err := r.probeUp(ctx, h); err == nil { + if info, out, done := r.tryWalk(ctx, blockReq(edge), nil); done { + return info, out + } + } + } + } + + // Records carry no height (or the probe raced retention): full floor walk. + info, err := r.walk(ctx, &pb.RangeRequest{}, nil) + if err != nil { + return tailInfo{}, recRetry + } + + return info, recOK +} + +// walk reads Range pages from first until live, tracking the tip. An absorb +// hook (the anchor rung's journal matcher) inspects every entry and may end +// the walk with its error — fold-integrity checks live behind it. +func (r *reader) walk(ctx context.Context, first *pb.RangeRequest, absorb func(*pb.Entry) error) (tailInfo, error) { + var info tailInfo + + f := r.newFolder(first) + + req := first + req.Limit = walkLimit + + for loops := 0; ; loops++ { + if loops >= maxWalkLoops { + return info, errTailTooLong + } + + resp, err := r.rangeOnce(ctx, req) + if err != nil { + return info, err + } + + for _, entry := range resp.GetEntries() { + if absorb != nil { + if err := absorb(entry); err != nil { + return info, err + } + } + + f.fold(entry) + trackTip(&info, entry) + } + + if s, ok := headFrom(resp.GetNext()); ok { + info.s = s + f.reached(s) + } + + if resp.GetLive() { + info.explained = f.ok + + return info, nil + } + + req = &pb.RangeRequest{ + After: &pb.RangeRequest_Head{Head: resp.GetNext()}, + Limit: walkLimit, + } + } +} + +// folder derives the head the walk lands on instead of accepting the one the +// store reports. Without it a CAS proves only that we echoed back the value +// we were handed — the commitment stops being a check on shared history and +// becomes a sequence token. +// +// A walk from our own anchor has a verified base. A walk that starts at a +// block takes the base its open declares, which is weaker but still chains +// every entry after it — and, crucially, means we have read the window. +// A walk that starts anywhere else has no base and cannot explain anything. +type folder struct { + cur commitment.Head + ok bool + awaiting bool // no base yet; the first entry must be an open that names one + sawAny bool +} + +func (r *reader) newFolder(first *pb.RangeRequest) *folder { + switch a := first.GetAfter().(type) { + case *pb.RangeRequest_Head: + if base, ok := headFrom(a.Head); ok { + return &folder{cur: base, ok: true} + } + case nil: + // A walk from the log's start: the base is the seed, which we + // compute from the chain id without asking the store. + return &folder{cur: r.seed, ok: true} + } + + return &folder{ok: true, awaiting: true} +} + +func (f *folder) fold(e *pb.Entry) { + f.sawAny = true + + if !f.ok { + return + } + + if f.awaiting { + open := e.GetBlockOpen() + if open == nil { + f.ok = false // mid-window start: nothing to fold from + + return + } + + f.cur = commitment.Head(open.GetPrefixCommitment()) + f.awaiting = false + } + + next, err := foldEntry(f.cur, e) + if err != nil { + f.ok = false + + return + } + + f.cur = next +} + +// reached compares the store's reported head for this page against the one +// the page's own entries produce. +func (f *folder) reached(s commitment.Head) { + switch { + case !f.ok: + return + case f.awaiting && !f.sawAny: + // A block-anchored walk that returned nothing: the head value stays + // underived, but the fact a boundary read needs — that no + // generation follows the block we started at — is what the empty + // page attests. Opening here cannot land on anyone's live window. + case f.awaiting: + // Entries came back, but not from a base we could establish, so + // they summarize into a head we cannot account for. + f.ok = false + case f.cur != s: + f.ok = false + + readHeadMismatch.Inc(1) + log.Warn("Store head does not match the entries it returned", + "reported", s, "folded", f.cur) + } +} + +func (r *reader) rangeOnce(ctx context.Context, req *pb.RangeRequest) (*pb.RangeResponse, error) { + cctx, cancel := context.WithTimeout(ctx, tailReadTimeout) + defer cancel() + + resp, err := r.client().Range(cctx, req) + if err == nil && r.onRead != nil { + r.onRead() + } + + return resp, err +} + +// generation fetches the entries of the newest generation standing at a +// height — open, records, and seal when one closed it. +func (r *reader) generation(ctx context.Context, height uint64) ([]*pb.Entry, error) { + resp, err := r.client().GetBlock(ctx, &pb.GetBlockRequest{BlockNumber: height}) + if err != nil { + return nil, err + } + + return resp.GetEntries(), nil +} + +func trackTip(info *tailInfo, entry *pb.Entry) { + switch k := entry.GetKind().(type) { + case *pb.Entry_BlockOpen: + info.tipOpen = true + info.tipOpenHeight = k.BlockOpen.GetBlockNumber() + info.tipOpenParent = common.BytesToHash(k.BlockOpen.GetParentHash()) + info.window = info.window[:0] + info.windowBytes = 0 + collectWindow(info, entry) + case *pb.Entry_Record: + collectWindow(info, entry) + case *pb.Entry_BlockSeal: + header, err := decodeSealHeader(k.BlockSeal.GetHeader()) + if err != nil { + log.Warn("Sequencer tail seal undecodable", "err", err) + + return + } + + info.tipOpen = false + info.haveSeal = true + info.sealDecoded = true + info.lastSealHeight = header.Number.Uint64() + info.lastSealHash = header.Hash() + info.window = nil + info.windowBytes = 0 + } +} + +// collectWindow accumulates the trailing window's entries up to the journal +// byte bound; past it the window is dropped for good (unadoptable). +func collectWindow(info *tailInfo, entry *pb.Entry) { + if info.windowBytes > journalMaxBytes { + return + } + + if info.windowBytes += proto.Size(entry); info.windowBytes > journalMaxBytes { + info.window = nil + + return + } + + if info.window != nil || entry.GetBlockOpen() != nil { + info.window = append(info.window, entry) + } +} + +// probeDown finds the highest store-known height at or below from: +// exponential descent, then binary search. +func (r *reader) probeDown(ctx context.Context, from uint64) (uint64, bool, error) { + h, step := from, uint64(1) + + // The lowest height the descent has proven unknown: the edge search + // need not re-probe it. In the common build-start case (from unknown, + // from-1 known) this leaves the binary search nothing to do at all — + // re-probing cost a wasted store round trip on every block. + unknown := from + 1 + + for { + known, err := r.blockKnown(ctx, h) + if err != nil { + return 0, false, err + } + + if known { + break + } + + unknown = h + + // step starts at 1 and doubles, so this also covers h <= 1. + if step >= h { + return 0, false, nil + } + + h -= step + step *= 2 + } + + return r.binarySearchEdge(ctx, h, unknown) +} + +// probeUp finds the highest store-known height starting from known floor +// height h0. +func (r *reader) probeUp(ctx context.Context, h0 uint64) (uint64, error) { + lo, step := h0, uint64(1) + + for { + known, err := r.blockKnown(ctx, lo+step) + if err != nil { + return 0, err + } + + if !known { + break + } + + lo += step + step *= 2 + } + + edge, _, err := r.binarySearchEdge(ctx, lo, lo+step) + + return edge, err +} + +// binarySearchEdge returns the highest known height in [lo, hi) given lo is +// known and hi is unknown (or the exclusive bound). +func (r *reader) binarySearchEdge(ctx context.Context, lo, hi uint64) (uint64, bool, error) { + for hi-lo > 1 { + mid := lo + (hi-lo)/2 + + known, err := r.blockKnown(ctx, mid) + if err != nil { + return 0, false, err + } + + if known { + lo = mid + } else { + hi = mid + } + } + + return lo, true, nil +} + +func (r *reader) blockKnown(ctx context.Context, h uint64) (bool, error) { + _, err := r.rangeOnce(ctx, &pb.RangeRequest{ + After: &pb.RangeRequest_Block{Block: h}, + Limit: 1, + }) + + switch { + case err == nil: + return true, nil + case isNotFound(err): + return false, nil + default: + return false, err + } +} + +func isNotFound(err error) bool { + return status.Code(err) == codes.NotFound +} + +// headFrom converts wire bytes to a Head when the length matches; short +// or oversized bytes are not a position. +func headFrom(b []byte) (commitment.Head, bool) { + if len(b) != len(commitment.Head{}) { + return commitment.Head{}, false + } + + return commitment.Head(b), true +} + +func headReq(h commitment.Head) *pb.RangeRequest { + return &pb.RangeRequest{After: &pb.RangeRequest_Head{Head: h.Bytes()}} +} + +func blockReq(h uint64) *pb.RangeRequest { + return &pb.RangeRequest{After: &pb.RangeRequest_Block{Block: h}} +} diff --git a/eth/sequencer/receipts.go b/eth/sequencer/receipts.go new file mode 100644 index 0000000000..6fad1a3f33 --- /dev/null +++ b/eth/sequencer/receipts.go @@ -0,0 +1,166 @@ +package sequencer + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// maxIndexedHeights bounds the index when canonical imports stall (or a +// misbehaving store streams far ahead): the lowest height is dropped to +// admit a new one. +const maxIndexedHeights = 256 + +// Index holds preconfirmation receipts built from re-executing the sequence +// stream, keyed by transaction hash. Receipts carry a zero BlockHash until +// the block's seal record arrives; the canonical receipt path always takes +// precedence, so entries here are only consulted for transactions the chain +// does not yet have. +type Index struct { + mu sync.RWMutex + byHash map[common.Hash]*indexed + byBlock map[uint64][]common.Hash +} + +type indexed struct { + receipt *types.Receipt + tx *types.Transaction +} + +// NewIndex returns an empty preconf receipt index. +func NewIndex() *Index { + return &Index{ + byHash: map[common.Hash]*indexed{}, + byBlock: map[uint64][]common.Hash{}, + } +} + +// Add records one preconf receipt for a speculative, not-yet-sealed block. +// The receipt must not be mutated after this call — Lookup hands the stored +// pointer to concurrent RPC readers. +func (ix *Index) Add(tx *types.Transaction, receipt *types.Receipt) { + ix.mu.Lock() + defer ix.mu.Unlock() + + number := receipt.BlockNumber.Uint64() + + if _, held := ix.byBlock[number]; !held && len(ix.byBlock) >= maxIndexedHeights { + ix.dropLowestLocked() + } + + ix.byHash[tx.Hash()] = &indexed{receipt: receipt, tx: tx} + ix.byBlock[number] = append(ix.byBlock[number], tx.Hash()) +} + +// Seal fills the sealed block hash into the height's receipts and their +// logs. Stored receipts are immutable once inserted (RPC readers hold the +// same pointers), so sealing swaps in copies. +func (ix *Index) Seal(number uint64, hash common.Hash) { + ix.mu.Lock() + defer ix.mu.Unlock() + + for _, txHash := range ix.byBlock[number] { + entry, ok := ix.byHash[txHash] + if !ok { + continue + } + + receipt := *entry.receipt + receipt.BlockHash = hash + receipt.Logs = make([]*types.Log, len(entry.receipt.Logs)) + + for i, old := range entry.receipt.Logs { + l := *old + l.BlockHash = hash + receipt.Logs[i] = &l + } + + ix.byHash[txHash] = &indexed{receipt: &receipt, tx: entry.tx} + } +} + +func (ix *Index) dropLowestLocked() { + lowest := uint64(0) + first := true + + for height := range ix.byBlock { + if first || height < lowest { + lowest = height + first = false + } + } + + if first { + return + } + + for _, h := range ix.byBlock[lowest] { + delete(ix.byHash, h) + } + + delete(ix.byBlock, lowest) +} + +// Lookup returns the preconf receipt and transaction for a hash, if held. +func (ix *Index) Lookup(hash common.Hash) (*types.Receipt, *types.Transaction, bool) { + ix.mu.RLock() + defer ix.mu.RUnlock() + + entry, ok := ix.byHash[hash] + if !ok { + return nil, nil, false + } + + preconfServedMeter.Mark(1) + + return entry.receipt, entry.tx, true +} + +// ClearFrom drops all entries at or above a height — a re-anchor voided them. +func (ix *Index) ClearFrom(number uint64) { + ix.mu.Lock() + defer ix.mu.Unlock() + + for height, hashes := range ix.byBlock { + if height < number { + continue + } + + for _, h := range hashes { + delete(ix.byHash, h) + } + + delete(ix.byBlock, height) + } +} + +// EvictThrough drops all entries at or below a height — the canonical chain +// now serves those receipts (or the transactions never landed and must not +// linger). +func (ix *Index) EvictThrough(number uint64) { + ix.mu.Lock() + defer ix.mu.Unlock() + + for height, hashes := range ix.byBlock { + if height > number { + continue + } + + for _, h := range hashes { + delete(ix.byHash, h) + } + + delete(ix.byBlock, height) + } +} + +// Reset drops everything — the consumer lost stream consistency and is +// re-anchoring from canonical state. +func (ix *Index) Reset() { + ix.mu.Lock() + defer ix.mu.Unlock() + + ix.byHash = map[common.Hash]*indexed{} + ix.byBlock = map[uint64][]common.Hash{} +} diff --git a/eth/sequencer/receipts_test.go b/eth/sequencer/receipts_test.go new file mode 100644 index 0000000000..d8165895a2 --- /dev/null +++ b/eth/sequencer/receipts_test.go @@ -0,0 +1,102 @@ +package sequencer + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +func indexedTx(nonce uint64) *types.Transaction { + return types.NewTx(&types.LegacyTx{Nonce: nonce}) +} + +func indexedReceipt(height uint64) *types.Receipt { + return &types.Receipt{ + BlockNumber: new(big.Int).SetUint64(height), + Logs: []*types.Log{{}}, + } +} + +// Sealing must hand new receipt copies to future readers while leaving the +// pointers earlier RPC readers hold untouched. +func TestIndexSealSwapsCopiesForReaders(t *testing.T) { + ix := NewIndex() + tx := indexedTx(1) + + ix.Add(tx, indexedReceipt(5)) + + before, _, ok := ix.Lookup(tx.Hash()) + if !ok { + t.Fatal("added receipt not found") + } + + sealedHash := common.Hash{0x5e} + ix.Seal(5, sealedHash) + + if before.BlockHash != (common.Hash{}) || before.Logs[0].BlockHash != (common.Hash{}) { + t.Fatal("sealing mutated a receipt an RPC reader already holds") + } + + after, _, _ := ix.Lookup(tx.Hash()) + if after.BlockHash != sealedHash || after.Logs[0].BlockHash != sealedHash { + t.Fatalf("sealed lookup carries %s, want %s on receipt and logs", after.BlockHash, sealedHash) + } +} + +// ClearFrom voids re-anchored heights upward; EvictThrough drops imported +// heights downward. Between them a height survives only while speculative. +func TestIndexClearAndEvictBounds(t *testing.T) { + ix := NewIndex() + txs := map[uint64]*types.Transaction{} + + for h := uint64(1); h <= 3; h++ { + txs[h] = indexedTx(h) + ix.Add(txs[h], indexedReceipt(h)) + } + + ix.ClearFrom(3) + ix.EvictThrough(1) + + for h, want := range map[uint64]bool{1: false, 2: true, 3: false} { + if _, _, ok := ix.Lookup(txs[h].Hash()); ok != want { + t.Fatalf("height %d held=%v, want %v", h, ok, want) + } + } +} + +// The index stays bounded when canonical imports stall: admitting a new +// height at the cap drops the lowest one. +func TestIndexCapDropsTheLowestHeight(t *testing.T) { + ix := NewIndex() + first := indexedTx(0) + ix.Add(first, indexedReceipt(1)) + + for h := uint64(2); h <= maxIndexedHeights; h++ { + ix.Add(indexedTx(h), indexedReceipt(h)) + } + + over := indexedTx(maxIndexedHeights + 1) + ix.Add(over, indexedReceipt(maxIndexedHeights+1)) + + if _, _, ok := ix.Lookup(first.Hash()); ok { + t.Fatal("lowest height survived past the cap") + } + + if _, _, ok := ix.Lookup(over.Hash()); !ok { + t.Fatal("newly admitted height missing") + } +} + +func TestIndexResetDropsEverything(t *testing.T) { + ix := NewIndex() + tx := indexedTx(9) + + ix.Add(tx, indexedReceipt(9)) + ix.Reset() + + if _, _, ok := ix.Lookup(tx.Hash()); ok { + t.Fatal("reset left a receipt behind") + } +} diff --git a/eth/sequencer/reconcile.go b/eth/sequencer/reconcile.go new file mode 100644 index 0000000000..2668af4667 --- /dev/null +++ b/eth/sequencer/reconcile.go @@ -0,0 +1,144 @@ +package sequencer + +import ( + "bytes" + "context" + "errors" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" +) + +var errRefold = errors.New("refold: unknown entry kind") + +type reconcileOutcome int + +const ( + recOK reconcileOutcome = iota + recRetry + recTerminal +) + +// reconcile reads the store tail from the best-known position and classifies +// it against the local lineage. It runs on the transport +// goroutine; the enqueue side keeps folding throughout. +func (p *Publisher) reconcile(ctx context.Context) reconcileOutcome { + start := time.Now() + defer func() { reconcileTimer.UpdateSince(start) }() + + publishStateGauge.Update(gaugeResyncing) + + info, out := p.readTail(ctx) + if out != recOK { + return out + } + + return p.applyTail(info) +} + +// readTail walks the position ladder: last acked head, then a +// block-anchor probe, then the floor read. The probe starts from the +// in-flight build height when there is one, else from the chain's last +// imported block — a cold start locates the store tail from the local +// database rather than any persisted hint. +func (p *Publisher) readTail(ctx context.Context) (tailInfo, reconcileOutcome) { + p.mu.Lock() + anchor, confirmed, probeFrom := p.anchor, p.confirmed, p.curHeight + p.mu.Unlock() + + if confirmed { + actx, cancel := sliceDeadline(ctx) + info, out, done := p.tryWalk(actx, headReq(anchor), true) + cancel() + + if done { + return info, out + } + } + + if probeFrom == 0 && p.chain != nil { + if head := p.chain.CurrentBlock(); head != nil { + probeFrom = head.Number.Uint64() + } + } + + if probeFrom > 0 { + if info, out, done := p.read.probedWalk(ctx, probeFrom); done { + return info, out + } + } + + return p.read.floorRead(ctx) +} + +// sliceDeadline halves the parent's remaining budget, so the anchor rung +// cannot starve the probe and floor rungs below it. +func sliceDeadline(ctx context.Context) (context.Context, context.CancelFunc) { + dl, ok := ctx.Deadline() + if !ok { + return context.WithCancel(ctx) + } + + return context.WithDeadline(ctx, time.Now().Add(time.Until(dl)/2)) +} + +// tryWalk runs one reader rung with the publisher's policy attached: a +// matched walk compares tail entries against the unconfirmed journal, and a +// fold divergence there is version skew — terminal for this publisher, not +// for the reader. +func (p *Publisher) tryWalk(ctx context.Context, first *pb.RangeRequest, match bool) (tailInfo, reconcileOutcome, bool) { + info, out, done := p.read.tryWalk(ctx, first, p.absorber(match)) + if done && out == recTerminal { + p.fail("fold divergence", "err", errFoldDivergence) + } + + return info, out, done +} + +// absorber arms the divergence check for the anchor rung; unmatched walks +// carry no hook. +func (p *Publisher) absorber(match bool) func(*pb.Entry) error { + if !match { + return nil + } + + p.mu.Lock() + items, _ := p.journal.after(p.ackedSeq) + snap := append([]journalItem(nil), items...) + p.mu.Unlock() + + m := &matcher{snap: snap, on: true} + + return m.absorb +} + +// matcher runs the divergence check on the anchor rung: tail entries are +// compared in lockstep with our unconfirmed journal items. +type matcher struct { + snap []journalItem + idx int + on bool +} + +func (m *matcher) absorb(entry *pb.Entry) error { + if !m.on || m.idx >= len(m.snap) { + m.on = false + + return nil + } + + item := m.snap[m.idx] + if !contentEqual(entry, item.entry) { + m.on = false + + return nil + } + + if !bytes.Equal(entryPrefix(entry), entryPrefix(item.entry)) { + return errFoldDivergence + } + + m.idx++ + + return nil +} diff --git a/eth/sequencer/reconcile_test.go b/eth/sequencer/reconcile_test.go new file mode 100644 index 0000000000..d27c67e831 --- /dev/null +++ b/eth/sequencer/reconcile_test.go @@ -0,0 +1,543 @@ +package sequencer + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" +) + +type fakeChain struct { + canonical map[uint64]common.Hash + known map[common.Hash]*types.Header + current *types.Header + blocks map[uint64]*types.Block +} + +func (f *fakeChain) GetCanonicalHash(number uint64) common.Hash { + return f.canonical[number] +} + +func (f *fakeChain) GetHeaderByHash(hash common.Hash) *types.Header { + return f.known[hash] +} + +func (f *fakeChain) CurrentBlock() *types.Header { + return f.current +} + +func (f *fakeChain) GetBlockByNumber(number uint64) *types.Block { + return f.blocks[number] +} + +// appendForeignOpen appends an open at the store head, as a competing +// publisher would. +func appendForeignOpen(t *testing.T, h *harness, number uint64, parent common.Hash) { + t.Helper() + appendForeignOpenAt(t, h, number, parent, 1700000000+number) +} + +func appendForeignOpenAt(t *testing.T, h *harness, number uint64, parent common.Hash, ts uint64) { + t.Helper() + + entry := &pb.Entry{Kind: &pb.Entry_BlockOpen{BlockOpen: &pb.BlockOpen{ + BlockNumber: number, + BlockTimestamp: ts, + ParentHash: parent.Bytes(), + GasLimit: 30_000_000, + BaseFee: big25gwei(), + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(entry); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign open rejected: %v", status) + } +} + +func appendForeignSeal(t *testing.T, h *harness, header *types.Header) { + t.Helper() + + raw, err := rlp.EncodeToBytes(header) + if err != nil { + t.Fatalf("rlp: %v", err) + } + + entry := &pb.Entry{Kind: &pb.Entry_BlockSeal{BlockSeal: &pb.BlockSeal{ + Header: raw, + PrefixCommitment: h.store.Head().Bytes(), + }}} + + if status := h.store.Append(entry); status != pb.AckStatus_ACK_STATUS_OK { + t.Fatalf("foreign seal rejected: %v", status) + } +} + +func big25gwei() []byte { + return testHeader(0, common.Hash{}).BaseFee.Bytes() +} + +// A foreign open landing mid-block holds our publishing (no re-anchor over +// unsealed work); our seal flush then overrides it — the only supersede. +func TestSealFlushOverridesForeignWindow(t *testing.T) { + h := startHarness(t) + fc := &fakeChain{} + p := newTestPublisher(t, h, fc) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + // Our window for block 2, confirmed. + header2 := testHeader(2, sealHash(t, sealed)) + p.OpenBlock(2, header2.Time, header2.ParentHash, header2.GasLimit, header2.BaseFee) + tx0 := testTx(t, 0) + p.PublishTx(tx0) + waitHead(t, h, p, 5*time.Second) + + // A competing publisher supersedes our window with its own block 2. + appendForeignOpen(t, h, 2, common.Hash{0xaa}) + foreignHead := h.store.Head() + + // Our next record STALEs; the publisher holds instead of re-anchoring. + tx1 := testTx(t, 1) + p.PublishTx(tx1) + + waitFor(t, 5*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.hold.after != noHold + }) + + if h.store.Head() != foreignHead { + t.Fatal("held publisher wrote to the store") + } + + // The seal flush re-anchors: the sealed block overrides the window. + sealOnChain(p, fc, header2, []*types.Transaction{tx0, tx1}) + waitHead(t, h, p, 10*time.Second) + + resp, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: 2}) + if err != nil { + t.Fatalf("GetBlock: %v", err) + } + + open := resp.GetEntries()[0].GetBlockOpen() + if got := common.BytesToHash(open.GetParentHash()); got != sealHash(t, sealed) { + t.Fatalf("latest generation parent %x, want ours %x", got, sealHash(t, sealed)) + } + + if last := resp.GetEntries()[len(resp.GetEntries())-1]; last.GetBlockSeal() == nil { + t.Fatal("flushed generation is not sealed") + } +} + +// A canonically sealed foreign block at our pending height holds our stale +// build; the next build-start check rebases onto the store head and +// publishing resumes cleanly. +func TestForeignSealedHoldsThenRebases(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{canonical: map[uint64]common.Hash{}, known: map[common.Hash]*types.Header{}} + p := newTestPublisher(t, h, chain) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + p.OpenBlock(2, 1700000002, sealHash(t, sealed), 30_000_000, testHeader(2, common.Hash{}).BaseFee) + waitHead(t, h, p, 5*time.Second) + + // The real producer's block 2 lands in the store and on our chain. + foreign := testHeader(2, common.Hash{0xbb}) + appendForeignOpen(t, h, 2, common.Hash{0xbb}) + appendForeignSeal(t, h, foreign) + chain.canonical[2] = foreign.Hash() + + // Our stale-window record STALEs; the publisher holds. + p.PublishTx(testTx(t, 0)) + + waitFor(t, 10*time.Second, func() bool { + p.mu.Lock() + defer p.mu.Unlock() + + return p.hold.after != noHold + }) + + // Next build: the check finds a clean (sealed) tail and rebases. + if w := p.AdoptWindow(3, foreign.Hash()); w != nil { + t.Fatalf("clean tail returned a window: %+v", w) + } + + publishBlock(t, p, 3, foreign.Hash(), 1) + waitHead(t, h, p, 5*time.Second) +} + +// publishRecorded drives one block through the publisher and registers it +// with the fake chain, as block import would. +func publishRecorded(t *testing.T, p *Publisher, chain *fakeChain, number uint64, parent common.Hash, txs int) *types.Header { + t.Helper() + + header := testHeader(number, parent) + p.OpenBlock(number, header.Time, parent, header.GasLimit, header.BaseFee) + + var body []*types.Transaction + + for i := 0; i < txs; i++ { + tx := testTx(t, uint64(i)) + body = append(body, tx) + p.PublishTx(tx) + } + + block := blockFor(header, body) + chain.blocks[number] = block + p.SealBlock(block) + + return header +} + +// An outage that stacks past the hot-flush bound collapses older blocks out +// of the journal; on recovery they are rebuilt from the chain database and +// every height still delivers to the store. +func TestOutageBackfillsFromDB(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p := newTestPublisher(t, h, chain) + + sealed := publishRecorded(t, p, chain, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + last := uint64(journalHotSeals + 10) + parent := sealHash(t, sealed) + for n := uint64(2); n <= last; n++ { + parent = sealHash(t, publishRecorded(t, p, chain, n, parent, 2)) + } + + p.mu.Lock() + collapsed := p.pendingFrom != 0 + p.mu.Unlock() + + if !collapsed { + t.Fatal("outage past journalHotSeals must collapse to the pending range") + } + + h.resume() + waitHead(t, h, p, 15*time.Second) + + publishRecorded(t, p, chain, last+1, parent, 1) + waitHead(t, h, p, 15*time.Second) + + // Everything delivered: the collapsed blocks came back from the DB. + for n := uint64(2); n <= last+1; n++ { + if _, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: n}); err != nil { + t.Fatalf("block %d missing after backfill: %v", n, err) + } + } +} + +// The same outage without chain access (no DB to rebuild from) skips the +// collapsed range as a counted forward jump and resumes at the tip. +func TestOutageChainlessJumps(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + jumps := reconcileForwardJump.Snapshot().Count() + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + last := uint64(journalHotSeals + 6) + parent := sealHash(t, sealed) + for n := uint64(2); n <= last; n++ { + parent = sealHash(t, publishBlock(t, p, n, parent, 1)) + } + + h.resume() + waitHead(t, h, p, 15*time.Second) + + publishBlock(t, p, last+1, parent, 1) + waitHead(t, h, p, 15*time.Second) + + // The collapsed front is gone (no DB to rebuild from) and counted. + if _, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: 2}); err == nil { + t.Fatal("collapsed block 2 unexpectedly published without a chain") + } + + for _, kept := range []uint64{last, last + 1} { + if _, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: kept}); err != nil { + t.Fatalf("block %d missing after recovery: %v", kept, err) + } + } + + if reconcileForwardJump.Snapshot().Count() == jumps { + t.Fatal("chainless backfill skip must count a forward jump") + } +} + +// The matcher flags a byte-identical entry folding from a different prefix — +// version skew, which is terminal. +func TestMatcherFoldDivergence(t *testing.T) { + entry := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{{0x01}}, + PrefixCommitment: commitment.Head{0xaa}.Bytes(), + }}} + + other := &pb.Entry{Kind: &pb.Entry_Record{Record: &pb.Record{ + Transactions: [][]byte{{0x01}}, + PrefixCommitment: commitment.Head{0xbb}.Bytes(), + }}} + + m := &matcher{snap: []journalItem{{entry: entry}}, on: true} + if err := m.absorb(other); err == nil { + t.Fatal("divergence not detected") + } +} + +func waitFor(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + + deadline := time.Now().Add(timeout) + + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition never held") + } + + time.Sleep(20 * time.Millisecond) + } +} + +// A block rebuilt from the chain database folds to exactly the head the +// live build produced — the backfill's byte-fidelity contract. +func TestBackfillByteExact(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p := newTestPublisher(t, h, chain) + + sealed := publishRecorded(t, p, chain, 1, common.Hash{0xef}, 3) + waitHead(t, h, p, 5*time.Second) + + liveHead := localHead(p) + + // Rebuild block 1 from the DB onto the same base the live build used. + fresh := newJournal() + + p.mu.Lock() + cur, ok := p.appendBlockLocked(fresh, commitment.Seed(testChainID), chain.blocks[1]) + p.mu.Unlock() + + if !ok { + t.Fatal("rebuild failed") + } + + if cur != liveHead { + t.Fatalf("rebuilt fold %x, live fold %x", cur, liveHead) + } + + _ = sealed +} + +// The backfill drains oldest-first and completely: a store that was down +// owes its readers every block, in order, so nothing is abandoned for +// freshness — the sealed tip must never advance past a gap it would then +// refuse to fill. +func TestBackfillDrainsOldestFirstAndCompletely(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + parent := common.Hash{0xef} + for n := uint64(1); n <= 50; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, nil) + parent = header.Hash() + } + + p.pendingFrom, p.pendingTo = 1, 50 + p.pendingEntries = 100 // open+seal per empty block + + jumps := reconcileForwardJump.Snapshot().Count() + drops := publishDropMeter.Snapshot().Count() + + fresh := newJournal() + + p.mu.Lock() + p.backfillLocked(fresh, commitment.Seed(testChainID)) + pendingFrom := p.pendingFrom + p.mu.Unlock() + + if got := fresh.seals; got != 50 { + t.Fatalf("rebuilt %d blocks, want all 50", got) + } + + if fresh.items[0].height != 1 { + t.Fatalf("rebuild starts at %d, want 1 (oldest first)", fresh.items[0].height) + } + + if pendingFrom != 0 { + t.Fatalf("pending not cleared after a full drain: from=%d", pendingFrom) + } + + if reconcileForwardJump.Snapshot().Count() != jumps { + t.Fatal("a complete drain is not a forward jump") + } + + if got := publishDropMeter.Snapshot().Count() - drops; got != 0 { + t.Fatalf("a complete drain dropped %d entries", got) + } +} + +// A range wider than one journal budget drains in batches: the batch takes +// the oldest blocks that fit, the remainder stays pending, and the next +// call resumes where it stopped — nothing is abandoned in between. +func TestBackfillResumesAcrossBatches(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + // Each block carries ~12MiB of calldata, so a 32MiB budget takes two + // blocks and change per batch. + payload := make([]byte, 12<<20) + parent := common.Hash{0xef} + + for n := uint64(1); n <= 5; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, []*types.Transaction{bigTx(t, payload)}) + parent = header.Hash() + } + + p.pendingFrom, p.pendingTo = 1, 5 + p.pendingEntries = 15 + + fresh := newJournal() + + p.mu.Lock() + cur := p.backfillLocked(fresh, commitment.Seed(testChainID)) + from1, to1 := p.pendingFrom, p.pendingTo + p.mu.Unlock() + + if fresh.seals == 0 || fresh.seals >= 5 { + t.Fatalf("first batch rebuilt %d blocks, want a strict subset", fresh.seals) + } + + if fresh.items[0].height != 1 { + t.Fatalf("first batch starts at %d, want 1 (oldest first)", fresh.items[0].height) + } + + if from1 != uint64(fresh.seals)+1 || to1 != 5 { + t.Fatalf("remainder not preserved: pending=[%d,%d] after %d rebuilt", + from1, to1, fresh.seals) + } + + // The next call picks up exactly where the last stopped. + next := newJournal() + + p.mu.Lock() + p.backfillLocked(next, cur) + p.mu.Unlock() + + if next.items[0].height != from1 { + t.Fatalf("second batch starts at %d, want %d", next.items[0].height, from1) + } +} + +// The whole pending range drains, storeSealedTip notwithstanding. The tip +// is the newest seal, not proof of anything below it: live flushes seal +// heights above the gap while it waits, and a store restart can shed acked +// writes — pending heights skipped on the tip stayed holes in the store +// forever on a devnet. A duplicate generation for a height the store does +// have costs churn; a hole costs the height. +func TestBackfillDrainsBelowTheSealedTip(t *testing.T) { + p := barePublisher() + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p.chain = chain + + parent := common.Hash{0xef} + for n := uint64(1); n <= 6; n++ { + header := testHeader(n, parent) + chain.blocks[n] = blockFor(header, nil) + parent = header.Hash() + } + + p.pendingFrom, p.pendingTo = 1, 6 + p.storeSealedTip = 4 + + fresh := newJournal() + + p.mu.Lock() + p.backfillLocked(fresh, commitment.Seed(testChainID)) + p.mu.Unlock() + + if fresh.seals != 6 || fresh.items[0].height != 1 { + t.Fatalf("rebuilt seals=%d from=%d, want all 6 from 1: heights "+ + "skipped on the tip are never revisited", fresh.seals, fresh.items[0].height) + } +} + +// Mining continues while the backfill recovers an outage: with the DB as +// the archive nothing is displaced — every height delivers. +func TestMiningDuringBackfillLosesNothing(t *testing.T) { + h := startHarness(t) + chain := &fakeChain{blocks: map[uint64]*types.Block{}} + p := newTestPublisher(t, h, chain) + + sealed := publishRecorded(t, p, chain, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + + h.stop() + + parent := sealHash(t, sealed) + for n := uint64(2); n <= 20; n++ { + parent = sealHash(t, publishRecorded(t, p, chain, n, parent, 5)) + } + + h.resume() + + // New blocks keep sealing while the backfill drains. + for n := uint64(21); n <= 24; n++ { + parent = sealHash(t, publishRecorded(t, p, chain, n, parent, 5)) + time.Sleep(50 * time.Millisecond) + } + + waitHead(t, h, p, 30*time.Second) + + for n := uint64(2); n <= 24; n++ { + if _, err := h.store.GetBlock(context.Background(), &pb.GetBlockRequest{BlockNumber: n}); err != nil { + t.Fatalf("block %d missing: mining-during-backfill displaced it", n) + } + } +} + +// bigTx pads a transaction with calldata so a block's size is dominated by +// it — the batching tests size blocks against the journal byte budget. +func bigTx(t *testing.T, payload []byte) *types.Transaction { + t.Helper() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("key: %v", err) + } + + tx, err := types.SignNewTx(key, types.LatestSignerForChainID(big.NewInt(testChainID)), &types.DynamicFeeTx{ + ChainID: big.NewInt(testChainID), + GasTipCap: big.NewInt(1), + GasFeeCap: big.NewInt(30_000_000_000), + Gas: 21000, + To: &common.Address{0x01}, + Data: payload, + }) + if err != nil { + t.Fatalf("sign: %v", err) + } + + return tx +} diff --git a/eth/sequencer/stall_test.go b/eth/sequencer/stall_test.go new file mode 100644 index 0000000000..a93c7c335b --- /dev/null +++ b/eth/sequencer/stall_test.go @@ -0,0 +1,164 @@ +package sequencer + +import ( + "net" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + + "github.com/0xPolygon/sequence-store-proto/commitment" + "github.com/0xPolygon/sequence-store-proto/devstore" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" +) + +// setForTest swaps a package variable for the test's duration. +func setForTest[T any](t *testing.T, p *T, v T) { + t.Helper() + + old := *p + *p = v + + t.Cleanup(func() { *p = old }) +} + +// stallPublisher accepts the stream and reads entries but never acks — a +// hung store from the send loop's point of view. +type stallPublisher struct { + pb.UnimplementedPublisherServiceServer + sessions atomic.Int64 + received atomic.Int64 +} + +func (s *stallPublisher) Publish(stream pb.PublisherService_PublishServer) error { + s.sessions.Add(1) + + for { + if _, err := stream.Recv(); err != nil { + return err + } + + s.received.Add(1) + } +} + +// startStallStore serves a store whose publish stream never acks, with reads +// answered from an empty devstore so the startup reconcile anchors. +func startStallStore(t *testing.T) (*stallPublisher, *Publisher) { + t.Helper() + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + stall := &stallPublisher{} + srv := grpc.NewServer() + pb.RegisterPublisherServiceServer(srv, stall) + pb.RegisterConsumerServiceServer(srv, devstore.New(testChainID)) + + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + p, err := NewPublisher(lis.Addr().String(), lis.Addr().String(), testChainID, 0, nil) + if err != nil { + t.Fatalf("NewPublisher: %v", err) + } + + t.Cleanup(p.Close) + + return stall, p +} + +// A store that stops acking must trip the stall deadline and reconnect +// (acks stall → degraded) rather than sitting live forever. +func TestStreamAckStallReconnects(t *testing.T) { + setForTest(t, &ackStallTimeout, 100*time.Millisecond) + + stall, p := startStallStore(t) + + publishBlock(t, p, 1, common.Hash{0xef}, 1) + + waitFor(t, 5*time.Second, func() bool { return stall.sessions.Load() >= 3 }) + + if p.failed.Load() { + t.Fatal("ack stall must degrade and retry, not fail terminally") + } +} + +// A store that is not acking gets at most the in-flight cap: an unbounded +// sender once buried the store under its own backlog and turned the stall +// watchdog into a false outage signal. +func TestSendPausesAtTheInflightCap(t *testing.T) { + setForTest(t, &maxInflightEntries, 4) + // Keep the watchdog out of the way: a reconnect resends and would count + // the same entries twice. + setForTest(t, &ackStallTimeout, time.Minute) + + stall, p := startStallStore(t) + + publishBlock(t, p, 1, common.Hash{0xef}, 10) // open + 10 records + seal + + waitFor(t, 5*time.Second, func() bool { return stall.received.Load() == 4 }) + time.Sleep(200 * time.Millisecond) + + if got := stall.received.Load(); got != 4 { + t.Fatalf("sender pushed %d entries into a non-acking store, cap is 4", got) + } +} + +// A cap smaller than the window must not wedge the drain: acks free slots +// and the post-ack send refills, so the whole journal still delivers even +// when the worker appends nothing further. +func TestCappedSenderStillDrainsCompletely(t *testing.T) { + setForTest(t, &maxInflightEntries, 2) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + publishBlock(t, p, 1, common.Hash{0xef}, 6) // open + 6 records + seal + waitHead(t, h, p, 5*time.Second) +} + +// An eviction gap must be reported even when a hold gates everything past +// the cursor: the gap is the drain cycle's reconcile trigger, and a devnet +// stranded a whole outage window when an early-out answered before the +// coverage check ran. +func TestGapDetectionOutranksTheHoldEarlyOut(t *testing.T) { + p := barePublisher() + p.journal.nextSeq = 5 + p.journal.append(nil, commitment.Head{}, commitment.Head{0x01}, entryRecord, 3, 0, nil) + p.hold = hold{after: 0, kind: holdBuild} + + if _, _, ok := p.sendableAfter(1, 100); ok { + t.Fatal("a gated suffix hid the eviction gap; the drain cycle never reconciles") + } +} + +// A store restart once wedged live gRPC channels in a permanent +// connect-retry loop while fresh dials worked. After a whole interval of +// silence the publisher rebuilds its connections rather than trusting the +// channel state machine — and keeps working through the swap. +func TestRedialAfterProlongedSilence(t *testing.T) { + setForTest(t, &redialAfter, 400*time.Millisecond) + + h := startHarness(t) + p := newTestPublisher(t, h, nil) + + first := publishBlock(t, p, 1, common.Hash{0xef}, 2) + waitHead(t, h, p, 5*time.Second) + + before := publishRedialCount.Snapshot().Count() + + h.stop() + waitFor(t, 10*time.Second, func() bool { + return publishRedialCount.Snapshot().Count() > before + }) + + h.resume() + publishBlock(t, p, 2, first.Hash(), 2) + waitHead(t, h, p, 10*time.Second) +} diff --git a/eth/sequencer/stream.go b/eth/sequencer/stream.go new file mode 100644 index 0000000000..3acecff111 --- /dev/null +++ b/eth/sequencer/stream.go @@ -0,0 +1,581 @@ +package sequencer + +import ( + "context" + "sync/atomic" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/log" +) + +type streamEnd int + +const ( + endCtx streamEnd = iota + endTerminal + endTransport + endStale + endIdle + endWatch +) + +// idleReconcileInterval paces the idle catch-up read: a publisher with +// nothing in flight (a non-producer between spans) re-anchors near the +// store tip so a takeover build's bounded read starts close — from a +// stale anchor the read budget dies walking history and the adopt is +// missed. Var for tests. +var idleReconcileInterval = 30 * time.Second + +// heldWatchInterval paces the tail re-reads taken while a sticky hold gates +// a live build. Short enough to catch a competing producer inside one block +// interval; the reads only happen while held, which is rare. Var for tests. +var heldWatchInterval = 400 * time.Millisecond + +// maxInflightEntries bounds what one session keeps sent-but-unacked. Deep +// enough to saturate a group-committing ingress, shallow enough that the +// trailing entry's ack latency stays a queue drain, not a backlog: an +// unbounded sender once buried the store under 23k entries of its own +// republish churn, tripped the stall watchdog on the self-made backlog, and +// produced blind while flagged unreachable. With the cap, a stall means the +// store stopped — nobody is being promised anything — which is the only +// state where producing without a verdict is safe. Var for tests. +var maxInflightEntries = 2048 + +type streamResult struct { + reason streamEnd + // progressed reports whether any entry was confirmed this session — it + // resets the contention backoff streak. + progressed bool +} + +type ackResult struct { + status pb.AckStatus + err error +} + +type sent struct { + item journalItem + at time.Time +} + +// stallTracker watches ack progress from outside the send loop: a hung +// store fills the stream's flow-control window and blocks Send, so no +// in-loop timer can fire — the watchdog cancels the session context +// instead, which unblocks both Send and Recv. +type stallTracker struct { + inflight atomic.Int64 + lastAck atomic.Int64 // unix nanos of the last ack (or session start) +} + +func newStallTracker() *stallTracker { + t := &stallTracker{} + t.lastAck.Store(time.Now().UnixNano()) // session start counts as progress + + return t +} + +func (s *stallTracker) sent() { s.inflight.Add(1) } +func (s *stallTracker) acked() { s.inflight.Add(-1); s.lastAck.Store(time.Now().UnixNano()) } + +func (s *stallTracker) stalled(deadline time.Duration) bool { + return s.inflight.Load() > 0 && + time.Since(time.Unix(0, s.lastAck.Load())) > deadline +} + +// watch cancels the session once acks stall past the deadline; it exits +// with the session. The deadline arrives as a parameter so the goroutine +// never reads publisher state it could outlive. +func (s *stallTracker) watch(ctx context.Context, cancel context.CancelFunc, deadline time.Duration) { + tick := time.NewTicker(deadline / 4) + defer tick.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if s.stalled(deadline) { + log.Warn("Sequencer ack stall, reconnecting", + "inflight", s.inflight.Load(), + "waited", time.Since(time.Unix(0, s.lastAck.Load()))) + cancel() + + return + } + } + } +} + +// runStream runs one stream session: it resends unacked journal items, +// forwards new ones as the worker appends them, and retires items as acks +// arrive (the store acks in entry order, so the oldest in-flight item owns +// each ack). +func (p *Publisher) runStream(ctx context.Context) streamResult { + sctx, cancel := context.WithCancel(ctx) + defer cancel() + + stream, err := p.pub.Publish(sctx) + if err != nil { + return streamResult{reason: endTransport} + } + + acks := make(chan ackResult) + + go recvAcks(sctx, stream, acks) + + // A hung store stalls acks without erroring the stream; the watchdog + // forces a reconnect whose resend converges once the store wakes. + tracker := newStallTracker() + + go tracker.watch(sctx, cancel, ackStallTimeout) + + // A resent entry may already have been applied (the previous stream + // broke after the append, before the ack) — its STALE lands in + // reconciliation, whose anchored byte-match retires exactly the + // applied prefix. No local guessing: retiring on a STALE without + // store confirmation can mint a fictional frontier when the earlier + // send itself had STALEd. + var inflight []sent + + cursor, ok := p.sendAfter(stream, p.ackedSeqSnapshot(), &inflight, tracker) + if !ok { + return streamResult{reason: endStale} + } + + progressed := false + + idle := time.NewTicker(idleReconcileInterval) + defer idle.Stop() + + // A sticky hold stops our writes, which also stops the STALEs that + // would otherwise bring fresh tail reads — leaving us blind to the + // competing producer for the rest of the block. Keep looking on a + // short cadence while held: reads are harmless, and seeing the + // competitor is what lets the build follow their sequence. + watch := time.NewTicker(heldWatchInterval) + defer watch.Stop() + + for { + select { + case <-ctx.Done(): + return streamResult{reason: endCtx, progressed: progressed} + case <-sctx.Done(): + // Watchdog cancel; recvAcks may lose the race to deliver the + // recv error, so the session ends here. + return streamResult{reason: endTransport, progressed: progressed} + case <-idle.C: + if p.quiet() { + return streamResult{reason: endIdle, progressed: progressed} + } + case <-watch.C: + if p.heldMidBuild() { + return streamResult{reason: endWatch, progressed: progressed} + } + case <-p.wake: + case ack := <-acks: + res, done := p.handleAck(ack, &inflight) + if done { + res.progressed = progressed + + return res + } + + progressed = true // an OK ack retired an entry + + tracker.acked() + } + + // One send site serves every wake source: a worker append, an ack + // freeing a cap slot, or a tick that found nothing to end on. + if cursor, ok = p.sendAfter(stream, cursor, &inflight, tracker); !ok { + return streamResult{reason: endStale, progressed: progressed} + } + } +} + +// sendableAfter snapshots, under the lock, the journal items past cursor +// that may go out now: covered by the journal, at or below the send +// ceiling (a draining seal flush below the ceiling always finishes +// delivering), and at most limit items. acked rides along so the send loop +// can skip adoption-confirmed entries. ok is false when eviction has +// created a gap past cursor — reconciliation must decide. +func (p *Publisher) sendableAfter(cursor uint64, limit int) (items []journalItem, acked uint64, ok bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // The coverage check runs on every call, before any early-out: an + // eviction gap is the drain cycle's reconcile trigger, and a call that + // skips it (a full pipeline, a gated suffix) can wedge a backfill with + // nothing else scheduled to notice — a devnet stranded a whole outage + // window exactly that way. + view, covered := p.journal.after(cursor) + if !covered { + return nil, 0, false + } + + // A hold at or below the cursor gates everything past it — the common + // sticky-hold shape — and seqs ascend, so answer without walking. + if p.hold.active() && p.hold.after <= cursor { + return nil, p.ackedSeq, true + } + + // Truncate before trimming the held tail: gated items are always a + // suffix, so the result is the same and the walk is bounded by the + // cap instead of the window size. + if len(view) > limit { + view = view[:max(limit, 0)] + } + + for len(view) > 0 && p.hold.gates(view[len(view)-1].seq) { + view = view[:len(view)-1] + } + + // Copy under the lock: view aliases the journal's backing array, which + // worker-side evictions compact in place while Send blocks — iterating + // the alias unlocked is a data race. + return append([]journalItem(nil), view...), p.ackedSeq, true +} + +// sendAfter sends the sendable journal items past cursor, up to the +// in-flight cap, returning the new cursor. ok is false on an eviction gap. +// Acks free capacity, and the send after each ack refills — without that +// pairing a capped drain with an idle worker would send one window and +// stop, since nothing else wakes the loop. A full pipeline still calls +// down: the coverage check must run even when nothing can be sent. +func (p *Publisher) sendAfter(stream pb.PublisherService_PublishClient, cursor uint64, inflight *[]sent, tracker *stallTracker) (uint64, bool) { + items, acked, ok := p.sendableAfter(cursor, maxInflightEntries-len(*inflight)) + if !ok { + return cursor, false + } + + for _, item := range items { + // An adoption can mark entries acked between + // this session's sends: the store already holds them, and sending + // them again guarantees a STALE (the adopter is silent). + if item.seq <= acked { + cursor = item.seq + + continue + } + + if err := stream.Send(&pb.PublishRequest{Entry: item.entry}); err != nil { + // The recv side reports the definitive error; keep the cursor so + // nothing is skipped. + break + } + + *inflight = append(*inflight, sent{item: item, at: time.Now()}) + tracker.sent() + cursor = item.seq + + publishedCounter.Inc(1) + } + + return cursor, true +} + +// handleAck retires or rejects the oldest in-flight entry. done=true ends +// the session with the returned result; done=false means the ack was OK +// and an entry was retired — the session progressed. +func (p *Publisher) handleAck(ack ackResult, inflight *[]sent) (streamResult, bool) { + if ack.err != nil { + return streamResult{reason: endTransport}, true + } + + if len(*inflight) == 0 { + p.fail("ack without a pending entry") + + return streamResult{reason: endTerminal}, true + } + + first := (*inflight)[0] + *inflight = (*inflight)[1:] + + switch ack.status { + case pb.AckStatus_ACK_STATUS_OK: + if p.retire(first.item, first.at) { + return streamResult{reason: endWatch}, true + } + + return streamResult{}, false + case pb.AckStatus_ACK_STATUS_STALE_COMMITMENT: + p.markGateLost(first.item) + + // Even for a resend that may have been applied before its stream + // broke, reconciliation decides: the anchored byte-match retires + // exactly the applied prefix, without inventing a frontier. + return streamResult{reason: endStale}, true + case pb.AckStatus_ACK_STATUS_RATE_LIMITED: + // The rejected entry never advanced the store head and everything + // pipelined behind it failed the head check too — a fresh stream + // resending in order is exact. + return streamResult{reason: endTransport}, true + default: + p.fail("store rejected entry", "status", ack.status) + + return streamResult{reason: endTerminal}, true + } +} + +// quiet reports a lineage with nothing in flight: either fully +// drained, or every unacked entry gated behind the send ceiling (a parked +// adopter's dead buffer). Both states leave the idle catch-up read free — +// requiring a full drain would let a held buffer starve the catch-up all +// span, leaving a stale anchor exactly when a takeover needs a fresh one. +func (p *Publisher) quiet() bool { + p.mu.Lock() + defer p.mu.Unlock() + + if p.unackedLocked() == 0 { + return true + } + + if !p.hold.active() { + return false + } + + for _, it := range p.journal.items { + if it.seq > p.ackedSeq && !p.hold.gates(it.seq) { + return false // sendable in-flight work: not quiet + } + } + + return true +} + +func (p *Publisher) ackedSeqSnapshot() uint64 { + p.mu.Lock() + defer p.mu.Unlock() + + return p.ackedSeq +} + +func recvAcks(ctx context.Context, stream pb.PublisherService_PublishClient, acks chan<- ackResult) { + for { + resp, err := stream.Recv() + + result := ackResult{err: err} + if err == nil { + result.status = resp.GetStatus() + } + + select { + case acks <- result: + case <-ctx.Done(): + return + } + + if err != nil { + return + } + } +} + +// retire marks one item store-confirmed. It reports whether the retired +// item was the backfill batch's last entry with blocks still pending — the +// cue for the caller to reconcile the next batch in. +func (p *Publisher) retire(item journalItem, at time.Time) (drain bool) { + p.mu.Lock() + defer p.mu.Unlock() + + // A lagging ack may describe an entry from a lineage that a swap + // (adoption, refold, rewind) has since replaced: advancing ackedSeq or + // the anchor from it would regress the frontier and fake an eviction + // gap, forcing a spurious forward-jump over a live window. The ack is + // only meaningful while the exact entry still stands in the journal. + if cur, ok := p.journal.itemAt(item.seq); !ok || cur.post != item.post || item.seq <= p.ackedSeq { + return false + } + + p.ackedSeq = item.seq + p.anchor = item.post + p.confirmed = true + p.markReachable() + + if item.kind == entrySeal && item.height > p.storeSealedTip { + log.Debug("Sequencer sealed tip advance", "origin", "seal-ack", + "from", p.storeSealedTip, "to", item.height) + p.storeSealedTip = item.height + } + + // Confirmation is keyed to the gated block's hash, not just its height: + // a refused block and its rebuild share a height, and a late ack for + // the first attempt's seal must not confirm the second's gate — that + // would broadcast content the store's seal does not describe. + if item.kind == entrySeal && p.gate.height == item.height && + p.gate.verdict == gatePending { + if header, err := decodeSealHeader(item.entry.GetBlockSeal().GetHeader()); err == nil && + header.Hash() == p.gate.hash { + p.gate.verdict = gateConfirmed + } + } + + // A build-start hold only orders the new window behind the draining + // flush: once the flush's seal is home, lift it so the window streams + // mid-block instead of batching until its own seal. During an outage + // drain the ceiling stays: the store is owed older blocks first, and + // the retired batch is the cue to reconcile the next one in. + if item.kind == entrySeal && p.hold.kind == holdBuild && !p.hold.gates(item.seq) { + if p.pendingFrom != 0 { + drain = true + } else { + p.hold = clearedHold() + p.signalWake() + } + } + + publishQueueGauge.Update(int64(p.unackedLocked())) + + publishAckTimer.UpdateSince(at) + + return drain +} + +// runState carries the run loop's backoff bookkeeping. +type runState struct { + probe time.Duration + contention int + lastRedial time.Time +} + +// run owns transport: anchor via reconciliation, then send loop the journal into the +// store, reconciling on STALE and probing with backoff while unreachable. +func (p *Publisher) run(ctx context.Context) { + defer close(p.done) + defer func() { + _ = p.pubConn.Close() + _ = p.consConn.Close() + }() + + state := runState{probe: probeBackoffMin} + + for ctx.Err() == nil && !p.failed.Load() { + if !p.step(ctx, &state) { + return + } + } +} + +// step runs one iteration: anchor when needed, otherwise one send loop session. +func (p *Publisher) step(ctx context.Context, state *runState) bool { + // A whole interval of silence means the connections themselves are + // suspect — a store restart has wedged the gRPC channels while fresh + // dials worked. Rate-limited to one rebuild per interval. + if p.silentTooLong() && time.Since(state.lastRedial) > redialAfter { + p.redial() + state.lastRedial = time.Now() + } + + if !p.isAnchored() { + return p.anchorStep(ctx, state) + } + + publishStateGauge.Update(gaugeLive) + + res := p.runStream(ctx) + + if res.reason != endTransport { + p.unreachable.Store(false) // the store answered, whatever it said + } + + switch res.reason { + case endCtx, endTerminal: + return false + case endIdle: + return p.idleReanchor(ctx) + case endWatch: + // A held build re-reading the store: reconcile only, keeping the + // build's height so the tail is classified against it. + return p.reconcile(ctx) != recTerminal + case endTransport: + p.unreachable.Store(true) + + return state.degradedSleep(ctx) + default: // endStale + publishStaleCount.Inc(1) + + state.contention = contentionSleep(ctx, res.progressed, state.contention) + if ctx.Err() != nil { + return false + } + + state.probe = probeBackoffMin + p.setUnanchored() + + return true + } +} + +func (p *Publisher) anchorStep(ctx context.Context, state *runState) bool { + if p.reconcile(ctx) == recOK { + p.unreachable.Store(false) // the read went through + state.probe = probeBackoffMin + + return true + } + + if p.failed.Load() { + return false + } + + return state.degradedSleep(ctx) +} + +func (s *runState) degradedSleep(ctx context.Context) bool { + publishStateGauge.Update(gaugeDegraded) + + if !sleepCtx(ctx, s.probe) { + return false + } + + s.probe = min(s.probe*2, probeBackoffMax) + + return true +} + +// idleReanchor re-anchors near the store tip after a quiet interval, so a +// later takeover build's bounded read starts close. A window still held +// after the quiet period is parked — its build died and its seal will +// never come — so drop its height, letting the reconcile refold the dead +// buffer instead of holding the anchor stale forever. A freely streaming +// window (no hold) is a healthy incumbent momentarily drained between +// txs, not a dead build: leaving its height intact keeps a coincident +// foreign write on the hold-and-let-our-seal-win path rather than +// superseding a window we may still seal. +func (p *Publisher) idleReanchor(ctx context.Context) bool { + p.mu.Lock() + if p.hold.active() { + p.curHeight = 0 + } + p.mu.Unlock() + + return p.reconcile(ctx) != recTerminal +} + +// contentionSleep applies the reconcile backoff when the previous +// reconcile's corrective publish immediately re-STALEd; the +// first STALE reconciles without delay, and progress resets the streak. +func contentionSleep(ctx context.Context, progressed bool, streak int) int { + if progressed { + return 0 + } + + if streak > 0 { + delay := min(reconcileBackoffMin<<(streak-1), reconcileBackoffMax) + publishStateGauge.Update(gaugeContending) + sleepCtx(ctx, delay) + } + + return streak + 1 +} + +func sleepCtx(ctx context.Context, d time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(d): + return true + } +} diff --git a/eth/sequencer/stream_test.go b/eth/sequencer/stream_test.go new file mode 100644 index 0000000000..3585016633 --- /dev/null +++ b/eth/sequencer/stream_test.go @@ -0,0 +1,136 @@ +package sequencer + +import ( + "errors" + "testing" + "time" + + "github.com/0xPolygon/sequence-store-proto/commitment" + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" +) + +func barePublisher() *Publisher { + p := &Publisher{ + head: commitment.Seed(testChainID), + anchor: commitment.Seed(testChainID), + seed: commitment.Seed(testChainID), + journal: newJournal(), + wake: make(chan struct{}, 1), + } + p.read = newReader(nil, p.seed, p.markReachable) + + return p +} + +func TestHandleAck(t *testing.T) { + item := journalItem{seq: 7, post: commitment.Head{0x07}, kind: entrySeal} + + cases := []struct { + name string + ack ackResult + inflight []sent + wantReason streamEnd + wantDone bool + wantAcked uint64 + wantFailed bool + }{ + { + name: "ok retires", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_OK}, + inflight: []sent{{item: item, at: time.Now()}}, + wantDone: false, + wantAcked: 7, + wantReason: endCtx, // unused when done=false + }, + { + name: "stale reconciles even on a resend", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_STALE_COMMITMENT}, + inflight: []sent{{item: item}}, + wantDone: true, + wantReason: endStale, + }, + { + name: "rate limited retries transport", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_RATE_LIMITED}, + inflight: []sent{{item: item}}, + wantDone: true, + wantReason: endTransport, + }, + { + name: "malformed is terminal", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_MALFORMED}, + inflight: []sent{{item: item}}, + wantDone: true, + wantReason: endTerminal, + wantFailed: true, + }, + { + name: "transport error", + ack: ackResult{err: errors.New("recv")}, + inflight: []sent{{item: item}}, + wantDone: true, + wantReason: endTransport, + }, + { + name: "ack without pending is terminal", + ack: ackResult{status: pb.AckStatus_ACK_STATUS_OK}, + wantDone: true, + wantReason: endTerminal, + wantFailed: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := barePublisher() + // retire only honors acks for entries still live in the journal; + // seat the fixture item at its seq. + p.journal.nextSeq = item.seq + p.journal.items = append(p.journal.items, item) + p.journal.nextSeq = item.seq + 1 + + inflight := append([]sent(nil), tc.inflight...) + + res, done := p.handleAck(tc.ack, &inflight) + + if done != tc.wantDone { + t.Fatalf("done = %v, want %v", done, tc.wantDone) + } + + if done && res.reason != tc.wantReason { + t.Fatalf("reason = %v, want %v", res.reason, tc.wantReason) + } + + if p.ackedSeq != tc.wantAcked { + t.Fatalf("ackedSeq = %d, want %d", p.ackedSeq, tc.wantAcked) + } + + if p.failed.Load() != tc.wantFailed { + t.Fatalf("failed = %v, want %v", p.failed.Load(), tc.wantFailed) + } + }) + } +} + +func TestHandleAckOKMarksProgress(t *testing.T) { + p := barePublisher() + live := journalItem{seq: 1, post: commitment.Head{0x01}} + p.journal.items = append(p.journal.items, live) + p.journal.nextSeq = 2 + + inflight := []sent{{item: live, at: time.Now()}} + + // done=false is the progress signal: the session counts it as an + // entry retired. + if _, done := p.handleAck(ackResult{status: pb.AckStatus_ACK_STATUS_OK}, &inflight); done { + t.Fatal("ok ack must not end the session") + } + + if p.anchor != (commitment.Head{0x01}) { + t.Fatalf("anchor = %x", p.anchor) + } + + if !p.confirmed { + t.Fatal("ok ack must confirm the anchor") + } +} diff --git a/eth/sequencer/twin_test.go b/eth/sequencer/twin_test.go new file mode 100644 index 0000000000..3bcae5b058 --- /dev/null +++ b/eth/sequencer/twin_test.go @@ -0,0 +1,391 @@ +package sequencer + +import ( + "testing" + "time" + + pb "github.com/0xPolygon/sequence-store-proto/sequencestore/v1" + + "github.com/ethereum/go-ethereum/common" +) + +// Two publishers against one store: the unit-test stand-in for two block +// producers sharing a signing key. Every scenario below is one that cost a +// kurtosis run to find, so each is pinned here instead. +func twinPublishers(t *testing.T) (*harness, *Publisher, *Publisher) { + t.Helper() + + h := startHarness(t) + + return h, newTestPublisher(t, h, &fakeChain{}), newTestPublisher(t, h, &fakeChain{}) +} + +// sealedParent gets both publishers past a sealed block 1 so height 2 is a +// clean contested slot, and returns the parent hash they must both build on. +func sealedParent(t *testing.T, h *harness, incumbent, twin *Publisher) common.Hash { + t.Helper() + + sealed := publishBlock(t, incumbent, 1, common.Hash{0xef}, 1) + waitHead(t, h, incumbent, 5*time.Second) + + // The twin has to see block 1 too, or its own build starts from a stale + // anchor and the scenario tests the wrong thing. + waitFor(t, 5*time.Second, func() bool { return twin.isAnchored() }) + + return sealHash(t, sealed) +} + +// The height=243 failure: the twin adopts a snapshot of the incumbent's +// window, the incumbent keeps writing, and the twin seals the snapshot — +// stranding every record added since. Measured as a 362-record block +// displacing 1088 already-acked ones. +func TestTwinDoesNotSealStaleSnapshot(t *testing.T) { + h, incumbent, twin := twinPublishers(t) + parent := sealedParent(t, h, incumbent, twin) + + incumbent.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + incumbent.PublishTx(testTx(t, 0)) + incumbent.PublishTx(testTx(t, 1)) + waitDrained(t, incumbent, 5*time.Second) + + // The twin inherits the window as it stands now: two records. + w := twin.AdoptWindow(2, parent) + if w == nil { + t.Fatal("twin did not adopt the incumbent's window") + } + + if len(w.Txs) != 2 { + t.Fatalf("adopted %d txs, want the 2 published so far", len(w.Txs)) + } + + // The incumbent is alive and keeps going. The twin's snapshot is now + // stale by one record. + incumbent.PublishTx(testTx(t, 2)) + waitDrained(t, incumbent, 5*time.Second) + + // Mark the height contested, as a STALE would, and take the contested + // path twice: the first call rebuilds, the second must still refuse + // because the window moved on. + twin.mu.Lock() + twin.curHeight = 2 + twin.hold = hold{after: twin.ackedSeq, kind: holdSticky} + twin.mu.Unlock() + + if awaitOurWindow(twin, 2*time.Second) { + t.Fatal("first contested attempt sealed instead of rebuilding") + } + + if awaitOurWindow(twin, 2*time.Second) { + t.Fatal("twin sealed a stale snapshot: the records the incumbent " + + "added after adoption were acked, so they are preconfirmed and " + + "this block strands them") + } +} + +// The same path must still reach a seal once the incumbent stops, or +// contention has no liveness escape and the chain stalls — the failure that +// held a devnet at one height for five minutes. +func TestTwinSealsOnceWindowSettles(t *testing.T) { + h, incumbent, twin := twinPublishers(t) + parent := sealedParent(t, h, incumbent, twin) + + incumbent.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + incumbent.PublishTx(testTx(t, 0)) + waitDrained(t, incumbent, 5*time.Second) + + if twin.AdoptWindow(2, parent) == nil { + t.Fatal("twin did not adopt the incumbent's window") + } + + twin.mu.Lock() + twin.curHeight = 2 + twin.hold = hold{after: twin.ackedSeq, kind: holdSticky} + twin.mu.Unlock() + + if awaitOurWindow(twin, 2*time.Second) { + t.Fatal("a contested attempt sealed instead of rebuilding") + } + + twin.ResyncNeeded() // the worker's rebuild consumes the signal + + // The rebuild adopts: the incumbent wrote nothing more, so the adopted + // window is the whole of the store's content and the seal proceeds. + if twin.AdoptWindow(2, parent) == nil { + t.Fatal("rebuild did not adopt the settled window") + } + + if !awaitOurWindow(twin, 2*time.Second) { + t.Fatal("twin refused a settled, fully adopted window: nobody would " + + "ever close this height") + } +} + +// Adoption hands the build exactly the store's transactions: the adopted +// window is the content the block extends, in store order. +func TestContestedTwinSealsWindowExactly(t *testing.T) { + h, incumbent, twin := twinPublishers(t) + parent := sealedParent(t, h, incumbent, twin) + + incumbent.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + incumbent.PublishTx(testTx(t, 0)) + incumbent.PublishTx(testTx(t, 1)) + waitDrained(t, incumbent, 5*time.Second) + + w := twin.AdoptWindow(2, parent) + if w == nil { + t.Fatal("twin did not adopt the incumbent's window") + } + + if len(w.Txs) != 2 { + t.Fatalf("window carries %d txs, want the 2 in the store", len(w.Txs)) + } +} + +// The mid-window blind spot, closed by Rule 1: a producer whose anchor sits +// at the store head — past the live window's open — used to classify the +// tail as clean and open a second generation beside the incumbent's. The +// build-start read now probes down from the height itself, so the window is +// visible from anywhere. +func TestBuildStartSeesLiveWindowFromAMidWindowAnchor(t *testing.T) { + h, incumbent, twin := twinPublishers(t) + parent := sealedParent(t, h, incumbent, twin) + + incumbent.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + incumbent.PublishTx(testTx(t, 0)) + waitDrained(t, incumbent, 5*time.Second) + + // Strand the twin's anchor at the store head, mid-window: exactly the + // state a between-blocks re-anchor used to leave behind. + twin.mu.Lock() + twin.anchor = incumbent.head + twin.confirmed = true + twin.mu.Unlock() + + w := twin.AdoptWindow(2, parent) + if w == nil { + t.Fatal("a live window at this height was invisible from a " + + "mid-window anchor: the twin would open a second generation " + + "beside the incumbent's") + } + + if len(w.Txs) != 1 { + t.Fatalf("adopted %d txs, want the incumbent's 1", len(w.Txs)) + } +} + +// The open CAS is the election: when both twins open the same height, one +// open lands and the other STALEs. The loser must adopt — and the store must +// end up with exactly one generation at the height, which is the invariant +// every double-seal traced back to. +func TestLosingOpenAdoptsAndAddsNoGeneration(t *testing.T) { + h, a, b := twinPublishers(t) + parent := sealedParent(t, h, a, b) + + // A wins the height: its window is in the store. + a.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + a.PublishTx(testTx(t, 0)) + waitDrained(t, a, 5*time.Second) + + // B opens the same height blind (its build-start read raced A's open). + // Divergent content, so nothing absorbs: B's open must STALE. + b.OpenBlock(2, 1700000009, parent, 30_000_000, fee25()) + b.PublishTx(testTx(t, 7)) + + // B discovers the loss and asks for a rebuild. + waitFor(t, 5*time.Second, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + + return b.hold.kind == holdSticky || b.resync + }) + + if !awaitOurWindow(b, time.Second) { + b.ResyncNeeded() // the worker's rebuild consumes the signal + } + + // The rebuild's boundary read adopts A's window. + w := b.AdoptWindow(2, parent) + if w == nil { + t.Fatal("the losing twin did not adopt the winner's window") + } + + // The store holds exactly one generation at height 2: the CAS kept the + // loser's open out entirely. + gens := 0 + for _, g := range readAllGenerations(t, h) { + if g.height == 2 { + gens++ + } + } + + if gens != 1 { + t.Fatalf("store holds %d generations at height 2, want exactly 1: "+ + "a second generation is where every double-seal came from", gens) + } +} + +// windowContent returns this publisher's entries for a height, so two +// publishers' views can be compared for genuine agreement rather than for +// having merely passed the same checks. +func windowContent(t *testing.T, p *Publisher, height uint64) []*pb.Entry { + t.Helper() + + p.mu.Lock() + defer p.mu.Unlock() + + var out []*pb.Entry + for _, it := range p.journal.suffixFromHeight(height) { + out = append(out, it.entry) + } + + return out +} + +func sameContent(a, b []*pb.Entry) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if !contentEqual(a[i], b[i]) { + return false + } + } + + return true +} + +// decide drives a publisher to a final seal answer. The contested path +// defers once before deciding, so one call is not the verdict. +func decide(p *Publisher) bool { + for i := 0; i < 3; i++ { + if ok := awaitOurWindow(p, time.Second); ok { + return true + } + + p.ResyncNeeded() // consume, as the worker's rebuild would + } + + return false +} + +// The invariant the whole mechanism exists for: two producers at one height +// must not both seal while holding different content. Every other test here +// checks a decision in isolation; this one checks the outcome, which is what +// a consumer actually experiences. +func TestTwinsNeverBothSealDivergentContent(t *testing.T) { + h, a, b := twinPublishers(t) + parent := sealedParent(t, h, a, b) + + // A gets its window in first. + a.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + a.PublishTx(testTx(t, 0)) + waitDrained(t, a, 5*time.Second) + + // B builds the same height on the same context with different content. + b.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + b.PublishTx(testTx(t, 7)) + + waitFor(t, 5*time.Second, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + + return b.hold.kind == holdSticky || b.unackedLocked() == 0 + }) + + sealA, sealB := decide(a), decide(b) + + if sealA && sealB && !sameContent(windowContent(t, a, 2), windowContent(t, b, 2)) { + t.Fatal("both producers sealed height 2 holding different content: " + + "consensus discards one block and every record only in it loses " + + "its preconfirmation") + } +} + +// A contested height must not seal on an unreadable tail. This is the state +// that held for 68 of one node's reads in a five-minute devnet window +// ("rung out of budget"), and treating it as permission to seal is what +// turned every missed read into a divergent block. +func TestContestedSealRefusedWhenTailUnreadable(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + p.mu.Lock() + p.curHeight = 2 + p.hold = hold{after: p.ackedSeq, kind: holdSticky} + p.read.cons = &failingConsumer{} + p.mu.Unlock() + + if awaitOurWindow(p, time.Second) { + t.Fatal("sealed a contested height without being able to read the " + + "store: with no way to know what the other producer wrote, this " + + "block can only diverge") + } +} + +// The same requirement for the coverage check on its own: an unreadable tail +// is not evidence of coverage. Failing open here is what made every dropped +// read a licence to seal. +func TestCoverageFailsClosedOnUnreadableTail(t *testing.T) { + h := startHarness(t) + p := newTestPublisher(t, h, &fakeChain{}) + + sealed := publishBlock(t, p, 1, common.Hash{0xef}, 1) + waitHead(t, h, p, 5*time.Second) + parent := sealHash(t, sealed) + + p.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + p.PublishTx(testTx(t, 0)) + waitDrained(t, p, 5*time.Second) + + // Uncontested but unreadable: an ordinary build must still seal, because + // production never waits on the store. The distinction is contention. + p.mu.Lock() + p.read.cons = &failingConsumer{} + p.mu.Unlock() + + if !awaitOurWindow(p, time.Second) { + t.Fatal("an unreachable store blocked an uncontested seal: " + + "production must never wait on the store") + } +} + +// The counterpart invariant: under contention someone must still seal. Two +// producers politely refusing each other is what held a devnet at one height +// for five minutes, and no safety property is worth a stopped chain. +// +// Together with TestTwinsNeverBothSealDivergentContent this pins the whole +// trade: that test forbids two divergent seals, this one forbids zero seals. +// Any change that satisfies one by breaking the other has not solved +// anything. +func TestTwinsAlwaysProduceASeal(t *testing.T) { + h, a, b := twinPublishers(t) + parent := sealedParent(t, h, a, b) + + a.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + a.PublishTx(testTx(t, 0)) + waitDrained(t, a, 5*time.Second) + + b.OpenBlock(2, 1700000002, parent, 30_000_000, fee25()) + b.PublishTx(testTx(t, 7)) + + waitFor(t, 5*time.Second, func() bool { + b.mu.Lock() + defer b.mu.Unlock() + + return b.hold.kind == holdSticky || b.unackedLocked() == 0 + }) + + if !decide(a) && !decide(b) { + t.Fatal("neither producer sealed height 2: the height never closes " + + "and the chain stops advancing") + } +} diff --git a/go.mod b/go.mod index e9c852a6e9..de13de5270 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/0xPolygon/crand v1.0.3 github.com/0xPolygon/heimdall-v2 v0.7.1 github.com/0xPolygon/polyproto v0.0.8 + github.com/0xPolygon/sequence-store-proto v0.0.0-20260719224427-276104d12aff github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 github.com/BurntSushi/toml v1.4.0 github.com/JekaMas/go-grpc-net-conn v0.0.0-20220708155319-6aff21f2d13d diff --git a/go.sum b/go.sum index f0230f2cee..c936ec90e7 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,8 @@ github.com/0xPolygon/heimdall-v2 v0.7.1 h1:L50HuFky97OvSF7uHlRXoClujKTFDeMP059GN github.com/0xPolygon/heimdall-v2 v0.7.1/go.mod h1:YrGakfr3jRlcXzGrBiJPxkfOfkqodzDZHNhf+2mYG5U= github.com/0xPolygon/polyproto v0.0.8 h1:69IQ6V8CwhF9UFoJtDn7Viy6PjyRLwiZw1m5kVDdykg= github.com/0xPolygon/polyproto v0.0.8/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= +github.com/0xPolygon/sequence-store-proto v0.0.0-20260719224427-276104d12aff h1:t/QGvpSPb2OMOKiO+BE+ph9Q6dpsk2nLYtjGJJBdRaY= +github.com/0xPolygon/sequence-store-proto v0.0.0-20260719224427-276104d12aff/go.mod h1:cuEfi/jhoNrbEZ/SHOBfPT3MBlqyzbsvcivwJIOTH00= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= diff --git a/internal/cli/dumpconfig.go b/internal/cli/dumpconfig.go index 53089bb45e..665a4ca7ba 100644 --- a/internal/cli/dumpconfig.go +++ b/internal/cli/dumpconfig.go @@ -75,6 +75,7 @@ func (c *DumpconfigCommand) Run(args []string) int { userConfig.Gpo.IgnorePriceRaw = userConfig.Gpo.IgnorePrice.String() userConfig.Cache.TrieTimeoutRaw = userConfig.Cache.TrieTimeout.String() userConfig.P2P.TxArrivalWaitRaw = userConfig.P2P.TxArrivalWait.String() + userConfig.Sequencer.PollRaw = userConfig.Sequencer.Poll.String() if err := toml.NewEncoder(os.Stdout).Encode(userConfig); err != nil { c.UI.Error(err.Error()) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 99d60680c1..d96c3dfa63 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -3,7 +3,6 @@ package server import ( "crypto/ecdsa" "fmt" - "math" "math/big" "os" @@ -175,6 +174,8 @@ type Config struct { // Relay has transaction relay related settings Relay *RelayConfig `hcl:"relay,block" toml:"relay,block"` + + Sequencer *SequencerConfig `hcl:"sequencer,block" toml:"sequencer,block"` } type HistoryConfig struct { @@ -822,6 +823,53 @@ type RelayConfig struct { BlockProducerRpcEndpoints []string `hcl:"bp-rpc-endpoints,optional" toml:"bp-rpc-endpoints,optional"` } +// sequencerSettings derives the ethconfig sequencer fields: a mining node +// publishes, a non-mining node consumes the stream for preconf receipts. +// Enabling requires both service endpoints (the publisher also reads the +// tail through the consumer service when it reconciles). +func (c *Config) sequencerSettings() (string, string, string, time.Duration, error) { + if c.Sequencer == nil || !c.Sequencer.Enabled { + return "", "", "", 0, nil + } + + if c.Sequencer.PublisherEndpoint == "" { + return "", "", "", 0, fmt.Errorf("sequencer.enabled requires sequencer.publisher-endpoint") + } + + if c.Sequencer.ConsumerEndpoint == "" { + return "", "", "", 0, fmt.Errorf("sequencer.enabled requires sequencer.consumer-endpoint") + } + + role := "consumer" + if c.Sealer.Enabled { + role = "producer" + } + + return role, c.Sequencer.PublisherEndpoint, c.Sequencer.ConsumerEndpoint, c.Sequencer.Poll, nil +} + +// SequencerConfig configures the sequence store integration. Role is +// derived, not configured: a mining node publishes the block lifecycle; +// a non-mining node consumes the stream for preconf receipts. +type SequencerConfig struct { + // Enabled turns the sequence store integration on. + Enabled bool `hcl:"enabled,optional" toml:"enabled,optional"` + + // PublisherEndpoint is the gRPC address of the store's publisher + // service (the publish stream). + PublisherEndpoint string `hcl:"publisher-endpoint,optional" toml:"publisher-endpoint,optional"` + + // ConsumerEndpoint is the gRPC address of the store's consumer + // service (tail reads during reconciliation; consumers in a future + // phase). + ConsumerEndpoint string `hcl:"consumer-endpoint,optional" toml:"consumer-endpoint,optional"` + + // Poll is the producer's txpool poll cadence while a block is open + // (continuous building); zero keeps the one-shot fill at slot start. + Poll time.Duration `hcl:"-,optional" toml:"-"` + PollRaw string `hcl:"poll,optional" toml:"poll,optional"` +} + func DefaultConfig() *Config { return &Config{ Chain: "mainnet", @@ -1086,6 +1134,12 @@ func DefaultConfig() *Config { EnablePrivateTx: false, BlockProducerRpcEndpoints: []string{}, }, + Sequencer: &SequencerConfig{ + Enabled: false, + PublisherEndpoint: "", + ConsumerEndpoint: "", + Poll: 200 * time.Millisecond, + }, } } @@ -1149,6 +1203,14 @@ func (c *Config) fillTimeDurations() error { {"rpc.txsync.maxtimeout", &c.JsonRPC.TxSyncMaxTimeout, &c.JsonRPC.TxSyncMaxTimeoutRaw}, } + if c.Sequencer != nil { + tds = append(tds, struct { + path string + td *time.Duration + str *string + }{"sequencer.poll", &c.Sequencer.Poll, &c.Sequencer.PollRaw}) + } + for _, x := range tds { if x.td != nil && x.str != nil && *x.str != "" { d, err := time.ParseDuration(*x.str) @@ -1716,7 +1778,18 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* n.DisableBlindForkValidation = c.DisableBlindForkValidation n.MaxBlindForkValidationLimit = c.MaxBlindForkValidationLimit - // Set preconf / private transaction flags for relay + // Sequence store: role is derived — a mining node publishes, a + // non-mining node consumes the stream for preconf receipts. + seqRole, seqPubEndpoint, seqConsEndpoint, seqPoll, err := c.sequencerSettings() + if err != nil { + return nil, err + } + + n.SequencerRole = seqRole + n.SequencerPublisherEndpoint = seqPubEndpoint + n.SequencerConsumerEndpoint = seqConsEndpoint + n.SequencerPoll = seqPoll + n.EnablePreconfs = c.Relay.EnablePreconfs n.EnablePrivateTx = c.Relay.EnablePrivateTx n.BlockProducerRpcEndpoints = c.Relay.BlockProducerRpcEndpoints @@ -1822,7 +1895,6 @@ var ( // tries unlocking the specified account a few times. func unlockAccount(ks *keystore.KeyStore, address string, i int, passwords []string) (accounts.Account, string) { account, err := utils.MakeAddress(ks, address) - if err != nil { utils.Fatalf("Could not list accounts: %v", err) } diff --git a/internal/cli/server/config_test.go b/internal/cli/server/config_test.go index 45c9e86faa..7f4e55236c 100644 --- a/internal/cli/server/config_test.go +++ b/internal/cli/server/config_test.go @@ -626,3 +626,72 @@ func TestDeveloperModeGasParameters(t *testing.T) { assert.Contains(t, err.Error(), "miner.targetGasPercentage must be between 1-100") }) } + +func TestSequencerConfigValidation(t *testing.T) { + cases := []struct { + name string + mutate func(*Config) + wantErr bool + role string + }{ + { + name: "disabled sequencer builds", + mutate: func(c *Config) {}, + }, + // Per-field endpoint validation is the sequencerSettings table's + // job (sequencer_flags_test.go); this case pins that a settings + // error fails the whole build. + { + name: "enabled without publisher endpoint fails the build", + mutate: func(c *Config) { + c.Sequencer = &SequencerConfig{Enabled: true, ConsumerEndpoint: "localhost:9550"} + }, + wantErr: true, + }, + { + name: "enabled on a sealing node publishes", + mutate: func(c *Config) { + c.Sealer.Enabled = true + c.Sequencer = &SequencerConfig{ + Enabled: true, + PublisherEndpoint: "localhost:9550", + ConsumerEndpoint: "localhost:9551", + Poll: time.Second, + } + }, + role: "producer", + }, + { + name: "enabled on a non-sealing node consumes", + mutate: func(c *Config) { + c.Sequencer = &SequencerConfig{ + Enabled: true, + PublisherEndpoint: "localhost:9550", + ConsumerEndpoint: "localhost:9551", + Poll: time.Second, + } + }, + role: "consumer", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + config := DefaultConfig() + tc.mutate(config) + + assert.NoError(t, config.loadChain()) + + ethConfig, err := config.buildEth(nil, nil) + if tc.wantErr { + assert.Error(t, err) + assert.Nil(t, ethConfig) + + return + } + + assert.NoError(t, err) + assert.Equal(t, tc.role, ethConfig.SequencerRole) + }) + } +} diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index 07f0c01b50..15b8f222cc 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -1452,5 +1452,34 @@ func (c *Command) Flags(config *Config) *flagset.Flagset { Group: "P2P", }) + c.registerSequencerFlags(f) + return f } + +func (c *Command) registerSequencerFlags(f *flagset.Flagset) { + f.BoolFlag(&flagset.BoolFlag{ + Name: "sequencer.enabled", + Usage: "Enable the sequence store integration (a mining node publishes the block lifecycle)", + Value: &c.cliConfig.Sequencer.Enabled, + Default: c.cliConfig.Sequencer.Enabled, + }) + f.StringFlag(&flagset.StringFlag{ + Name: "sequencer.publisher-endpoint", + Usage: "Sequence store publisher service gRPC endpoint (publish stream)", + Value: &c.cliConfig.Sequencer.PublisherEndpoint, + Default: c.cliConfig.Sequencer.PublisherEndpoint, + }) + f.StringFlag(&flagset.StringFlag{ + Name: "sequencer.consumer-endpoint", + Usage: "Sequence store consumer service gRPC endpoint (tail reads during reconciliation)", + Value: &c.cliConfig.Sequencer.ConsumerEndpoint, + Default: c.cliConfig.Sequencer.ConsumerEndpoint, + }) + f.DurationFlag(&flagset.DurationFlag{ + Name: "sequencer.poll", + Usage: "Producer txpool poll cadence while a block is open (continuous building); 0 keeps the one-shot fill", + Value: &c.cliConfig.Sequencer.Poll, + Default: c.cliConfig.Sequencer.Poll, + }) +} diff --git a/internal/cli/server/sequencer_flags_test.go b/internal/cli/server/sequencer_flags_test.go new file mode 100644 index 0000000000..2d4cbe6ad2 --- /dev/null +++ b/internal/cli/server/sequencer_flags_test.go @@ -0,0 +1,104 @@ +package server + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSequencerFlags(t *testing.T) { + t.Parallel() + + var c Command + + args := []string{ + "--sequencer.enabled", + "--sequencer.publisher-endpoint", "127.0.0.1:9550", + "--sequencer.consumer-endpoint", "127.0.0.1:9551", + "--sequencer.poll", "150ms", + } + + require.NoError(t, c.extractFlags(args)) + require.True(t, c.config.Sequencer.Enabled) + require.Equal(t, "127.0.0.1:9550", c.config.Sequencer.PublisherEndpoint) + require.Equal(t, "127.0.0.1:9551", c.config.Sequencer.ConsumerEndpoint) + require.Equal(t, 150*time.Millisecond, c.config.Sequencer.Poll) +} + +func TestSequencerDefaults(t *testing.T) { + t.Parallel() + + def := DefaultConfig() + require.False(t, def.Sequencer.Enabled) + require.Equal(t, "", def.Sequencer.PublisherEndpoint) + require.Equal(t, "", def.Sequencer.ConsumerEndpoint) + require.Equal(t, 200*time.Millisecond, def.Sequencer.Poll) +} + +func TestSequencerSettingsDerivation(t *testing.T) { + t.Parallel() + + base := func(enabled, sealer bool, pubEndpoint, consEndpoint string) *Config { + c := DefaultConfig() + c.Sequencer.Enabled = enabled + c.Sequencer.PublisherEndpoint = pubEndpoint + c.Sequencer.ConsumerEndpoint = consEndpoint + c.Sealer.Enabled = sealer + + return c + } + + cases := []struct { + name string + config *Config + wantRole string + wantErr bool + }{ + {"disabled", base(false, true, "h:1", "h:2"), "", false}, + {"nil block", &Config{Sealer: DefaultConfig().Sealer}, "", false}, + {"enabled without publisher endpoint", base(true, true, "", "h:2"), "", true}, + {"enabled without consumer endpoint", base(true, true, "h:1", ""), "", true}, + {"enabled mining node", base(true, true, "h:1", "h:2"), "producer", false}, + {"enabled non-mining node", base(true, false, "h:1", "h:2"), "consumer", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + role, pubEndpoint, consEndpoint, poll, err := tc.config.sequencerSettings() + + if tc.wantErr { + require.Error(t, err) + require.Empty(t, role, "error return must not report a sequencer role") + + return + } + + require.NoError(t, err) + require.Equal(t, tc.wantRole, role) + + if role != "" { + require.Equal(t, "h:1", pubEndpoint) + require.Equal(t, "h:2", consEndpoint) + require.Equal(t, 200*time.Millisecond, poll) + } + }) + } +} + +func TestSequencerPollFromConfigFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "seq.toml") + require.NoError(t, os.WriteFile(path, []byte("[sequencer]\nenabled = true\npublisher-endpoint = \"h:1\"\nconsumer-endpoint = \"h:2\"\npoll = \"75ms\"\n"), 0o600)) + + var c Command + + require.NoError(t, c.extractFlags([]string{"--config", path})) + require.True(t, c.config.Sequencer.Enabled) + require.Equal(t, "h:1", c.config.Sequencer.PublisherEndpoint) + require.Equal(t, "h:2", c.config.Sequencer.ConsumerEndpoint) + require.Equal(t, 75*time.Millisecond, c.config.Sequencer.Poll) +} diff --git a/miner/fake_miner.go b/miner/fake_miner.go index 4db0be668b..58ffbf0460 100644 --- a/miner/fake_miner.go +++ b/miner/fake_miner.go @@ -200,10 +200,8 @@ func createMockSpanForTest(address common.Address, chainId string) borTypes.Span return span0 } -var ( - // Test chain configurations - testTxPoolConfigBor legacypool.Config -) +// Test chain configurations +var testTxPoolConfigBor legacypool.Config // TODO - Arpit, Duplicate Functions type mockBackendBor struct { @@ -222,6 +220,12 @@ func (m *mockBackendBor) BlockChain() *core.BlockChain { return m.bc } +// WhitelistedMilestone implements Backend. The mock reports no milestone, +// which leaves the finality gate to its startup grace. +func (*mockBackendBor) WhitelistedMilestone() (bool, uint64, common.Hash) { + return false, 0, common.Hash{} +} + // PeerCount implements Backend. Returns a constant; tests using // mockBackendBor don't drive the peer count. func (*mockBackendBor) PeerCount() int { diff --git a/miner/miner.go b/miner/miner.go index dbd4607542..f05aca0c9a 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -42,6 +42,90 @@ type Backend interface { BlockChain() *core.BlockChain TxPool() *txpool.TxPool PeerCount() int + + // WhitelistedMilestone is finality's view of the chain: the newest + // Heimdall milestone the node has whitelisted, or false when none has + // arrived yet. The worker's finality gate compares it against the local + // chain before a producer builds. + WhitelistedMilestone() (bool, uint64, common.Hash) +} + +// AdoptedWindow is a previous producer's unsealed block handed back by the +// sequence store: the open context the block must inherit and the ordered +// transactions to commit before consulting the txpool. +type AdoptedWindow struct { + Number uint64 + Timestamp uint64 + ParentHash common.Hash + GasLimit uint64 + BaseFee *big.Int + Txs []*types.Transaction +} + +// SealVerdict is the store's answer to "may this sealed block be broadcast". +// The zero value is SealUnknown so an implementation that never answers +// defaults to broadcasting — production must not gate on the store. +type SealVerdict int + +const ( + // SealUnknown: no verdict in budget (store slow, unreachable, or catching + // up). Broadcast anyway — the liveness override. + SealUnknown SealVerdict = iota + + // SealConfirmed: the store took our seal (or the chain already holds our + // exact block). Broadcast. + SealConfirmed + + // SealRefused: another producer's block owns this height — its seal beat + // ours in the store and its block is on our chain. Discard ours; + // broadcasting it would fork the chain against an already-decided height. + SealRefused +) + +// BlockSequencer receives block-production progress for the sequence store: +// the block context when a build starts, each transaction as it commits, and +// the sealed header the moment sealing completes. Implementations must never +// block — they are called on the worker's hot paths. +type BlockSequencer interface { + OpenBlock(number uint64, timestamp uint64, parent common.Hash, gasLimit uint64, baseFee *big.Int) + PublishTx(tx *types.Transaction) + + // SealBlock delivers the complete sealed block: the seal flush needs + // the body to complete or re-anchor the store window (design §3.5). + SealBlock(block *types.Block) + + // AdoptWindow reads the store tail for the block about to be built + // (bounded; design §3.4). A returned window is an unsealed incumbent + // window on this tip that the build must follow: the header inherits + // its context and its transactions are committed first. nil means + // build normally. + AdoptWindow(number uint64, parent common.Hash) *AdoptedWindow + + // AwaitSequenced blocks until the window being built is confirmed by + // the store, so the block about to be sealed is provably the store's + // sequence at this height. false means another producer holds the + // height and this block must not be sealed. An unreachable store + // returns true — production never waits on the store. + AwaitSequenced(timeout time.Duration, number uint64, txs []*types.Transaction) bool + + // ResyncNeeded reports that another producer holds the height this + // node is building and reached the store first. The build stops rather + // than seal beside their sequence; the next work cycle adopts it. + // Reading consumes the signal. + ResyncNeeded() bool + + // ConfirmSeal reports whether the block just handed to SealBlock may be + // broadcast, waiting up to timeout for the store's verdict. The store's + // head CAS elects exactly one seal per height; the loser learns it lost + // and withholds its block, so one height gets one broadcast. + ConfirmSeal(timeout time.Duration) SealVerdict + + // RefreshInterval is the mempool re-snapshot cadence while a block is + // open: when non-zero the worker keeps the block open until just before + // its announce time, refilling from the pool on this cadence so + // transactions are executed (and preconfirmed) as they arrive. Zero + // keeps the default one-shot fill. + RefreshInterval() time.Duration } // Config is the configuration parameters of mining. @@ -133,6 +217,12 @@ func (miner *Miner) GetWorker() *worker { return miner.worker } +// SetSequencer attaches a sequence-store publisher to the worker. Call before +// the miner starts; the worker reads the field without synchronization. +func (miner *Miner) SetSequencer(s BlockSequencer) { + miner.worker.sequencer = s +} + // update keeps track of the downloader events. Please be aware that this is a one shot type of update loop. // It's entered once and as soon as `Done` or `Failed` has been broadcasted the events are unregistered and // the loop is exited. This to prevent a major security vuln where external parties can DOS you with blocks diff --git a/miner/worker.go b/miner/worker.go index 48f7d1322a..66ca968730 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common/tracing" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/bor" + "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" @@ -111,6 +112,7 @@ const ( var ( errBlockInterruptedByNewHead = errors.New("new head arrived while building block") errBlockInterruptedByRecommit = errors.New("recommit interrupt while building block") + errRebuildForSequence = errors.New("competing producer holds this height; rebuild on the store's sequence") errBlockInterruptedByTimeout = errors.New("timeout while building block") // metrics gauge to track total and empty blocks sealed by a miner @@ -351,6 +353,11 @@ func (env *environment) txFitsSize(tx *types.Transaction) bool { return env.size+tx.Size() < params.MaxBlockSize-maxBlockSizeBufferZone-env.stateSyncReserve } +// sequenceBarrierTimeout bounds the pre-seal wait for store confirmation. +// Past it the block seals regardless: a slow or unreachable store must not +// stall block production. +const sequenceBarrierTimeout = 120 * time.Millisecond + const ( commitInterruptNone int32 = iota commitInterruptNewHead @@ -380,7 +387,6 @@ type newPayloadResult struct { receipts []*types.Receipt // Receipts collected during construction requests [][]byte // Consensus layer requests collected during block construction witness *stateless.Witness // Witness is an optional stateless proof - } // getWorkReq represents a request for getting a new sealing work with provided parameters. @@ -406,6 +412,11 @@ type worker struct { eth Backend chain *core.BlockChain + // sequencer, when set, receives block-production progress (open, per-tx, + // seal) for the sequence store. Set before the worker starts; nil when + // sequencing is disabled. All its methods are non-blocking. + sequencer BlockSequencer + prio []common.Address // A list of senders to prioritize // Feeds @@ -456,6 +467,11 @@ type worker struct { newTxs atomic.Int32 // New arrival transaction count since last sealing work submitting. syncing atomic.Bool // The indicator whether the node is still syncing. + // finalityLatched is set once a whitelisted milestone confirms this + // node's chain; startedAt anchors the startup grace before that. + finalityLatched atomic.Bool + startedAt time.Time + // newpayloadTimeout is the maximum timeout allowance for creating payload. // The default value is 2 seconds but node operator can set it to arbitrary // large value. A large timeout allowance may cause Geth to fail creating @@ -500,6 +516,7 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus worker := &worker{ config: config, chainConfig: chainConfig, + startedAt: time.Now(), engine: engine, eth: eth, chain: eth.BlockChain(), @@ -1281,6 +1298,11 @@ func (w *worker) resultLoop() { log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", hash) continue } + + if !w.sealAndGate(block) { + continue + } + // Different block could share same sealhash, deep copy here to prevent write-write conflict. var ( receipts = make([]*types.Receipt, len(task.receipts)) @@ -1490,6 +1512,13 @@ func (w *worker) commitTransaction(env *environment, tx *types.Transaction) ([]* env.tcount++ env.size += tx.Size() + // Only actual block production is sequenced: the pending-block snapshot + // and payload-building paths also commit transactions, but those never + // seal, and publishing them would poison the store chain. + if w.sequencingActive(env.header.Number) { + w.sequencer.PublishTx(tx) + } + return receipt.Logs, nil } @@ -1932,6 +1961,8 @@ type generateParams struct { builderPlanCh chan *types.Transaction // Builder sends each validated tx here before execution; prefetcher reads and warms state concurrently builderGasFreedCh chan uint64 // Builder sends (declared−actual) gas after each successful tx; prefetcher uses it to predict overflow txs planWg sync.WaitGroup // Tracks sendPlan goroutines; must reach zero before builderPlanCh is closed + adoption *AdoptedWindow // Dangling store window this build inherits and seeds + production bool // Set only by commitWork: payload-building (generateWork) must never touch the sequencer } // makeHeader creates a new block header for sealing. The caller must hold w.mu @@ -2031,6 +2062,13 @@ func (w *worker) prepareWork(genParams *generateParams, witness bool) (*environm if err != nil { return nil, err } + // Only an authorized build reads the store (Prepare rejected everyone + // else): a dangling window for this exact header is adopted rather + // than superseded — fetched at real build time so the snapshot cannot + // go stale against a still-streaming incumbent. + w.fetchAdoption(genParams, header) + // Before makeEnv: executed transactions must see the adopted context. + w.applyAdoption(genParams, header) makeHeaderDuration := time.Since(makeHeaderStart) // Could potentially happen if starting to mine in an odd state. @@ -2354,7 +2392,6 @@ func (w *worker) generateWork(params *generateParams, witness bool) *newPayloadR var block *types.Block block, work.receipts, _, err = w.engine.FinalizeAndAssemble(w.chain, work.header, work.state, &body, work.receipts) - if err != nil { return &newPayloadResult{err: err} } @@ -2397,6 +2434,33 @@ func (w *worker) commitWork(interrupt *atomic.Int32, noempty bool, timestamp int // Find the parent block for sealing task parent := w.chain.CurrentBlock() + // A producer whose chain finality has not confirmed must not build: a + // restarting node that mines before its milestone view catches up + // produces a private fork every peer will refuse — four doomed blocks + // on a devnet, reorged away with all their preconfirmations. + next := new(big.Int).Add(parent.Number, common.Big1) + if w.sequencer != nil && sequencerActive(w.chainConfig.Bor, next) && !w.finalityConfirmed() { + log.Warn("Not building: finality has not confirmed this chain", + "number", next) + + return + } + + // Contention discovered mid-build — a competing producer's window + // landing after the build-start read — ends the work cycle rather than + // retrying inside it. The outer cycle machinery is the retry: the + // winner's imported block triggers a fresh cycle immediately (the + // recommit tick covers the rest), its build-start read adopts the + // standing window, and the seal gate referees anything that persists. + w.buildAttempt(interrupt, noempty, timestamp, coinbase, parent, buildStart) +} + +// buildAttempt runs one full build cycle on a fresh parent state, returning +// true when the build was abandoned in favour of another producer's sequence +// and should be retried. +func (w *worker) buildAttempt(interrupt *atomic.Int32, noempty bool, timestamp int64, + coinbase common.Address, parent *types.Header, buildStart time.Time, +) { // Retrieve the parent state to execute on top, with separate readers for stats tracking. state, throwaway, prefetchReader, processReader, err := w.chain.StateAtWithReaders(parent.Root) if err != nil { @@ -2412,6 +2476,7 @@ func (w *worker) commitWork(interrupt *atomic.Int32, noempty bool, timestamp int processReader: processReader, prefetchedTxHashes: &sync.Map{}, preBuildDuration: time.Since(buildStart), + production: true, } var interruptPrefetch atomic.Bool @@ -2442,7 +2507,10 @@ func (w *worker) commitWork(interrupt *atomic.Int32, noempty bool, timestamp int w.buildAndCommitBlock(interrupt, noempty, &genParams, &interruptPrefetch) } -// buildAndCommitBlock prepares work, fills transactions, and commits the block for sealing. +// buildAndCommitBlock prepares work, fills transactions, and commits the block +// for sealing. A build abandoned because another producer holds this height +// simply ends the work cycle: the next one's build-start read adopts their +// window. func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genParams *generateParams, interruptPrefetch *atomic.Bool) { // Must be the first defer so the prefetcher goroutine is signaled to exit // on every return path — including the early return below when prepareWork @@ -2458,6 +2526,10 @@ func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genP prepareWorkDuration := time.Since(prepareWorkStart) prepareWorkTimer.Update(prepareWorkDuration) + // The header context is final here (engine.Prepare included): publish + // the block-open record before any transaction commits. + w.sequencerOpen(work) + // Starts accounting time after prepareWork, since it includes the wait we have on Prepare phase of Bor start := time.Now() @@ -2521,8 +2593,9 @@ func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genP // Mark the start of full-block building. Set after the optional empty pre-seal commit so that // productionElapsed for the full block does not include empty-block overhead. genParams.productionStart = time.Now() - // Fill pending transactions from the txpool into the block. - err = w.fillTransactions(interrupt, work, genParams) + // Fill pending transactions from the txpool into the block: a single + // snapshot, or repeated ones until announce time when sequencing. + err = w.fillBlock(interrupt, work, genParams) // Wait for any sendPlan goroutines to finish before closing the channel. // These goroutines do only non-blocking sends so they complete in microseconds. // Waiting here ensures no goroutine sends to a closed channel. @@ -2565,7 +2638,24 @@ func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genP // which could result in higher uncle rate. work.discard() return + + case errors.Is(err, errRebuildForSequence): + // A competing producer owns this height in the store. Committing + // this block would seal content that diverges from the sequence + // consumers already saw. Discard; the next work cycle's build-start + // read adopts their window. + log.Warn("Discarding build to follow the store's sequence", "number", work.header.Number) + work.discard() + + return + } + + if !w.sealBarrier(work) { + work.discard() + + return } + // Submit the generated block for consensus sealing. _ = w.commit(work.copy(), w.fullTaskHook, true, start, genParams) @@ -2579,6 +2669,352 @@ func (w *worker) buildAndCommitBlock(interrupt *atomic.Int32, noempty bool, genP w.currentMu.Unlock() } +// sequencerOpen publishes the block-open record when a production build +// starts. A rebuild of the same height or a build on a new parent +// republishes — downstream re-anchoring handles both. Gated on IsRunning: +// pending-block maintenance also builds work cycles, but those never seal. +func (w *worker) sequencerOpen(work *environment) { + if !w.sequencingActive(work.header.Number) { + return + } + + w.sequencer.OpenBlock(work.header.Number.Uint64(), work.header.Time, + work.header.ParentHash, work.header.GasLimit, work.header.BaseFee) +} + +// fillBlock fills the pending block from the txpool: one snapshot on the +// stock path, or — with a sequencer attached and producing — repeated +// snapshots until announce time, so transactions are executed and streamed +// to the sequence store as they arrive instead of waiting for the next +// slot (continuous building). An adopted window seeds the block first. +func (w *worker) fillBlock(interrupt *atomic.Int32, work *environment, genParams *generateParams) error { + if genParams.adoption != nil { + w.seedAdopted(work, genParams.adoption) + + // Past the announce time already — a rebuild that inherited a + // window late. The sequenced transactions are committed, which is + // everything consumers were promised, so close here. Filling from + // the pool would push the seal further past its deadline to add + // content nobody is waiting on, and each added transaction is one + // more preconfirmation riding a block that is already late. + if time.Until(work.header.GetActualTime()) <= sealMargin { + return nil + } + } + + poll := w.sequencerPoll(work.header.Number) + if poll <= 0 { + return w.fillTransactions(interrupt, work, genParams) + } + + return w.fillUntilAnnounce(interrupt, work, genParams, poll) +} + +// adoptionSeedBudget is the minimum build time an adopted block gets: its +// inherited announce time is typically already past (the stall fallback +// fires at least a block time after the stalled open), so without a floor +// there is no room left to fill beyond the seeded window. +const adoptionSeedBudget = 500 * time.Millisecond + +// sealMargin is the tail of the block interval reserved for sealing, kept +// clear of transaction execution. +const sealMargin = 150 * time.Millisecond + +// finalityGrace is how long a starting producer waits for finality to say +// anything at all before building anyway. The grace is for ambiguity — no +// milestone yet, Heimdall still connecting — not for contradiction: a +// milestone that conflicts with the local chain refuses production for as +// long as the conflict holds. Var for tests. +var finalityGrace = 10 * time.Second + +// sealGateTimeout bounds the wait for an uncontested seal verdict before +// the block is broadcast regardless. Measured ack latency on a loaded +// devnet: p50 4.6ms, p99 50ms, p99.9 341ms — so this covers the tail with +// margin while costing a block period only when the store is genuinely +// gone. A contested seal is different: there a verdict is provably in +// progress, and the publisher extends the wait itself rather than +// broadcasting a block whose refusal is seconds away. +const sealGateTimeout = 500 * time.Millisecond + +// fetchAdoption asks the sequencer for a dangling store window matching the +// prepared header. It runs after engine.Prepare, so only a build the +// engine authorized ever reads or adopts from the store — a doomed backup +// build taking a snapshot of a still-streaming window would leave a stale +// armed offer behind for the rotation taker to seal short. +func (w *worker) fetchAdoption(genParams *generateParams, header *types.Header) { + genParams.adoption = nil + + if !genParams.production || !w.sequencingActive(header.Number) { + return + } + + genParams.adoption = w.sequencer.AdoptWindow(header.Number.Uint64(), header.ParentHash) +} + +// finalityConfirmed reports whether finality has ratified the chain this +// node holds. It latches on the first confirmation: steady-state milestones +// always trail the head, so re-checking would stall every producer forever. +// +// no milestone yet -> wait out the startup grace, then proceed +// (a Heimdall outage costs a pause, not the chain) +// milestone on chain -> confirmed, latch open +// milestone conflicts -> refuse while the conflict holds: the local +// chain is provably a fork finality rejected +func (w *worker) finalityConfirmed() bool { + if w.finalityLatched.Load() { + return true + } + + exists, number, hash := w.eth.WhitelistedMilestone() + if !exists { + return time.Since(w.startedAt) > finalityGrace + } + + if w.chain.GetCanonicalHash(number) == hash { + w.finalityLatched.Store(true) + + return true + } + + return false +} + +// sequencerActive gates the whole sequence-store integration on Rio for +// bor chains (post-Rio everywhere). Pre-Rio, every validator +// builds every height with per-succession timing rules; publishing, +// continuous filling, and adoption all interact with that regime — adopted +// timestamps are invalid for other signers, and the extra build churn +// starves out-of-turn seal delays. Below Rio the worker behaves stock. +func sequencerActive(bor *params.BorConfig, number *big.Int) bool { + return bor == nil || bor.IsRio(number) +} + +// sealBarrier reports whether the built block may be sealed, and when it may +// not, whether the worker should rebuild rather than drop the slot. +// +// The store is the arbiter of who owns a height: seal only once our window is +// confirmed there. Two blocks at one height is precisely what leaves +// consumers holding revoked preconfirmations. +func (w *worker) sealBarrier(work *environment) bool { + if !w.sequencingActive(work.header.Number) || + w.sequencer.AwaitSequenced(sequenceBarrierTimeout, + work.header.Number.Uint64(), work.txs) { + return true + } + + // Consuming the resync signal here keeps it from leaking into the next + // cycle and tells the two refusal shapes apart in the log. Either way + // the cycle ends; the next one's build-start read follows the store. + if w.sequencer.ResyncNeeded() { + log.Warn("Not sealing: the store holds records this block does not cover", + "number", work.header.Number) + } else { + log.Warn("Not sealing: another producer holds this height in the store", + "number", work.header.Number) + } + + return false +} + +// sequencingActive reports whether a mining build at this height feeds the +// sequence store: a sequencer is attached, this node is actually producing +// (IsRunning gates out the pending-block/payload snapshot builds, which +// never seal), and the height is post-Rio. The result-loop SealBlock hook +// gates without IsRunning — it only ever sees blocks this node sealed. +func (w *worker) sequencingActive(number *big.Int) bool { + return w.sequencer != nil && w.IsRunning() && sequencerActive(w.chainConfig.Bor, number) +} + +// applyAdoption rewrites the prepared header with the adopted window's open +// context — consumers pinned those fields at open and cross-check them +// against the sealed header, so a block sealed under a different context +// voids the window. Runs after engine.Prepare (which owns the +// timestamp otherwise) and before makeEnv builds the EVM context. The +// announce deadline is synthesized onto the header's local actual-time +// hint; the adopted announce time is typically already past. +// adoptionReject names the bound an offered window failed, or "" when the +// window is adoptable. The reason is worth naming: a rejected window is what +// turns one producer's height into two competing generations. +func (w *worker) adoptionReject(a *AdoptedWindow, header *types.Header) string { + parent := w.chain.GetHeaderByHash(header.ParentHash) + + switch { + case parent == nil: + return "parent unknown" + case a.ParentHash != header.ParentHash: + return "parent mismatch" + case a.Number != header.Number.Uint64(): + return "height mismatch" + } + + minTime := parent.Time + 1 + if w.chainConfig.Bor != nil { + minTime = parent.Time + w.chainConfig.Bor.CalculatePeriod(a.Number) + } + + switch { + case a.Timestamp < minTime: + return "timestamp below parent period" + case misc.VerifyGaslimit(parent.GasLimit, a.GasLimit) != nil: + return "gas limit out of bounds" + // Both producers derive the base fee from the same parent with the same + // rules; a differing value marks a window this node cannot legally seal. + case header.BaseFee == nil || a.BaseFee == nil || header.BaseFee.Cmp(a.BaseFee) != 0: + return "base fee mismatch" + } + + return "" +} + +func (w *worker) applyAdoption(genParams *generateParams, header *types.Header) { + a := genParams.adoption + if a == nil { + return + } + + genParams.adoption = nil // reinstated only when every bound holds + + if reason := w.adoptionReject(a, header); reason != "" { + // Declining is not a no-op: the build goes on to publish its own + // open at a height the store already holds a window for, which + // starts a second generation there. Both producers then preconfirm + // and only one block can seal, so every line here is a mismatch or + // a displacement waiting to happen. + log.Warn("Declined the store's window, opening our own", + "number", a.Number, "txs", len(a.Txs), "reason", reason) + + return + } + + header.Time = a.Timestamp + header.GasLimit = a.GasLimit + + deadline := time.Now().Add(adoptionSeedBudget) + if adopted := time.Unix(int64(a.Timestamp), 0); adopted.After(deadline) { + deadline = adopted + } + + header.ActualTime = deadline + genParams.adoption = a + + log.Info("Adopting sequenced window", "number", a.Number, + "txs", len(a.Txs), "deadline", deadline.Format(time.RFC3339Nano)) +} + +// seedAdopted commits the adopted window's transactions in order before any +// pool fill. The seed ignores the interrupt and runs to completion: these +// transactions are already published and preconfirmed, so cutting it short +// seals a block missing content consumers were promised, and the remainder +// resurfaces at the next height as a displaced preconfirmation. An +// inapplicable transaction (nonce consumed, balance moved) is skipped — the +// publisher's expectation matching turns that divergence into a +// partial-adoption supersede. +func (w *worker) seedAdopted(work *environment, adoption *AdoptedWindow) { + if work.gasPool == nil { + work.gasPool = new(core.GasPool).AddGas(work.header.GasLimit) + } + + applied := 0 + + for _, tx := range adoption.Txs { + work.state.SetTxContext(tx.Hash(), work.tcount) + + if _, err := w.commitTransaction(work, tx); err != nil { + log.Debug("Adopted transaction dropped", "hash", tx.Hash(), "err", err) + + continue + } + + applied++ + } + + log.Info("Seeded adopted window", "number", adoption.Number, + "applied", applied, "of", len(adoption.Txs)) +} + +// sequencerPoll returns the sequencing poll cadence, or zero when the block +// being built is not sequenced (no sequencer, not producing, or one-shot +// fill configured). +func (w *worker) sequencerPoll(number *big.Int) time.Duration { + if !w.sequencingActive(number) { + return 0 + } + + return w.sequencer.RefreshInterval() +} + +// fillUntilAnnounce fills the block immediately, then keeps re-snapshotting +// the txpool until sealMargin before the block's announce time. +// Already-committed transactions are skipped by the nonce checks inside +// commitTransactions; an interrupt aborts exactly as it does on the stock +// path. +func (w *worker) fillUntilAnnounce(interrupt *atomic.Int32, work *environment, genParams *generateParams, poll time.Duration) error { + if err := w.fillTransactions(interrupt, work, genParams); err != nil { + return err + } + + for { + remaining := time.Until(work.header.GetActualTime()) - sealMargin + if remaining <= 0 { + return nil + } + + time.Sleep(min(remaining, poll)) + + if err := w.haltFill(interrupt, work.header.Number); err != nil { + return err + } + + if err := w.fillTransactions(interrupt, work, genParams); err != nil { + return err + } + } +} + +// haltFill reports why the fill loop should stop short of the announce +// deadline, or nil to keep filling. +func (w *worker) haltFill(interrupt *atomic.Int32, number *big.Int) error { + if interrupt != nil { + if signal := interrupt.Load(); signal != commitInterruptNone { + return signalToErr(signal) + } + } + + // Another producer reached the store first for this height: abandon this + // build so the next one adopts their window and follows their ordering, + // instead of sealing a block that diverges from it. + if w.sequencingActive(number) && w.sequencer.ResyncNeeded() { + return errRebuildForSequence + } + + return nil +} + +// sealAndGate publishes the seal record the moment the sealed block exists — +// ahead of the chain write and the announcement — so stream consumers can +// close the block, and reports whether the block may be broadcast. +// +// The store's head CAS elects one seal per height; the gate is where the +// loser learns it lost and withholds its block, so a height gets one +// broadcast instead of a fork. No verdict inside the budget broadcasts +// anyway — production never waits on the store. +func (w *worker) sealAndGate(block *types.Block) bool { + if w.sequencer == nil || !sequencerActive(w.chainConfig.Bor, block.Number()) { + return true + } + + w.sequencer.SealBlock(block) + + if w.sequencer.ConfirmSeal(sealGateTimeout) == SealRefused { + log.Warn("Discarding sealed block: another producer's block owns this height", + "number", block.Number(), "hash", block.Hash()) + + return false + } + + return true +} + // runPrefetcher owns the lifecycle of the unified prefetcher stream for one block. // It starts a single long-lived worker pool (via PrefetchStream), runs the idle tx // provider until the builder flips, executes the idle→builder handoff, and then diff --git a/miner/worker_finality_test.go b/miner/worker_finality_test.go new file mode 100644 index 0000000000..b3e8163eec --- /dev/null +++ b/miner/worker_finality_test.go @@ -0,0 +1,135 @@ +package miner + +import ( + "testing" + "time" + + "github.com/ethereum/go-ethereum/core/types" + + "github.com/ethereum/go-ethereum/common" +) + +// These share the package-level finalityGrace, so they must not run in +// parallel with each other. + +// A restarting producer must not build until finality has spoken about the +// chain it holds. The failure this prevents: a same-key node came back from +// a restart, synced a head, and mined a four-block private fork before its +// milestone view caught up — every peer refused the whole lineage with a +// whitelist mismatch, and the blocks were reorged away. +func TestFinalityGateWaitsForAMilestoneAfterRestart(t *testing.T) { + w, b, _ := newSequencerTestWorker(t) + + if w.finalityConfirmed() { + t.Fatal("a fresh producer with no milestone built immediately: this " + + "is the window where a restart mines onto a fork finality has " + + "already rejected") + } + + // A milestone naming our own chain confirms the fork. + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + + if !w.finalityConfirmed() { + t.Fatal("a milestone matching our canonical chain must release the gate") + } +} + +// Confirmation latches: steady-state production must not re-check, or a +// producer would stall every time the milestone view lagged its own head. +func TestFinalityGateLatchesOnceConfirmed(t *testing.T) { + w, b, _ := newSequencerTestWorker(t) + + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + + if !w.finalityConfirmed() { + t.Fatal("gate did not open on a matching milestone") + } + + // A milestone that no longer matches (our head has moved on, as it + // always does) must not re-close the gate. + b.setMilestone(head.Number.Uint64()+5, common.Hash{0x99}) + + if !w.finalityConfirmed() { + t.Fatal("gate re-closed after confirming: a producer would stall " + + "whenever finality lagged its own head, which is always") + } +} + +// A milestone that names a block we do not have is proof we are on a +// rejected fork. Refuse for as long as that holds — the grace window is for +// ambiguity, not for contradiction. +func TestFinalityGateRefusesAConflictingChainPastGrace(t *testing.T) { + w, _, _ := newSequencerTestWorker(t) + + restore := finalityGrace + finalityGrace = 0 // grace already expired + t.Cleanup(func() { finalityGrace = restore }) + + w.eth.(*testWorkerBackend).setMilestone(1, common.Hash{0xbe, 0xef}) + + if w.finalityConfirmed() { + t.Fatal("built on a chain that provably conflicts with the " + + "whitelisted milestone: every block extends ground that is " + + "already reorged away") + } +} + +// Liveness: with no milestone at all — a fresh chain, or Heimdall down — the +// grace expires and production proceeds. A producer that never hears from +// Heimdall must pause, not halt. +func TestFinalityGateOpensAfterGraceWithoutMilestone(t *testing.T) { + w, _, _ := newSequencerTestWorker(t) + + restore := finalityGrace + finalityGrace = 0 + t.Cleanup(func() { finalityGrace = restore }) + + if !w.finalityConfirmed() { + t.Fatal("a producer with no milestone never started: a Heimdall " + + "outage must cost a pause, not the chain") + } +} + +// The gate must actually stop production, not merely report. Driven through +// the running worker's own loops — commitWork is the single path every build +// takes, and this asserts the wiring, not the predicate. +func TestFinalityGateBlocksProduction(t *testing.T) { + w, b, rec := newSequencerTestWorker(t) + + b.txPool.Add([]*types.Transaction{b.newRandomTxWithNonce(false, 0)}, true) + + // No milestone, grace running: the gate is shut. + w.start() + defer w.stop() + + time.Sleep(1500 * time.Millisecond) + + if opens, _, _, _ := rec.snapshot(); len(opens) != 0 { + t.Fatalf("produced %d blocks while finality had not confirmed our "+ + "chain: this is the restart window that mined a doomed fork", + len(opens)) + } + + // Confirm our chain, then retrigger. In production the veblop fallback + // does this every block period for a stalled producer; the clique test + // engine skips that path, so poke startCh directly. + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + w.start() + + deadline := time.Now().Add(10 * time.Second) + for { + if opens, _, _, _ := rec.snapshot(); len(opens) > 0 { + return + } + + if time.Now().After(deadline) { + t.Fatal("no production after finality confirmed the chain: the " + + "gate never reopens and the producer is stuck") + } + + time.Sleep(50 * time.Millisecond) + } +} diff --git a/miner/worker_sequencer_test.go b/miner/worker_sequencer_test.go new file mode 100644 index 0000000000..c87f785ec2 --- /dev/null +++ b/miner/worker_sequencer_test.go @@ -0,0 +1,814 @@ +package miner + +import ( + "errors" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/clique" + "github.com/ethereum/go-ethereum/consensus/misc/eip1559" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" +) + +// recordingSequencer captures the worker's sequencer callbacks. +type recordingSequencer struct { + mu sync.Mutex + refresh time.Duration + opens []uint64 + seals []uint64 + txs int + order []byte + + adoptable *AdoptedWindow // handed out once when the queried block matches + adopted int + verdict SealVerdict + resyncN int // number of rebuild signals a test wants served + contested bool // when set, AwaitSequenced reports the window contested +} + +func (r *recordingSequencer) OpenBlock(number uint64, _ uint64, _ common.Hash, _ uint64, _ *big.Int) { + r.mu.Lock() + defer r.mu.Unlock() + + r.opens = append(r.opens, number) + r.order = append(r.order, 'o') +} + +func (r *recordingSequencer) PublishTx(*types.Transaction) { + r.mu.Lock() + defer r.mu.Unlock() + + r.txs++ + r.order = append(r.order, 't') +} + +func (r *recordingSequencer) SealBlock(block *types.Block) { + r.mu.Lock() + defer r.mu.Unlock() + + r.seals = append(r.seals, block.NumberU64()) + r.order = append(r.order, 's') +} + +func (r *recordingSequencer) AdoptWindow(number uint64, parent common.Hash) *AdoptedWindow { + r.mu.Lock() + defer r.mu.Unlock() + + a := r.adoptable + if a == nil || a.Number != number || a.ParentHash != parent { + return nil + } + + r.adoptable = nil + r.adopted++ + + return a +} + +func (r *recordingSequencer) AwaitSequenced(time.Duration, uint64, []*types.Transaction) bool { + r.mu.Lock() + defer r.mu.Unlock() + + return !r.contested +} + +// verdict is what ConfirmSeal reports; the zero value (SealUnknown) lets +// every existing test proceed to broadcast unchanged. +func (r *recordingSequencer) ConfirmSeal(time.Duration) SealVerdict { + r.mu.Lock() + defer r.mu.Unlock() + + return r.verdict +} + +func (r *recordingSequencer) ResyncNeeded() bool { + r.mu.Lock() + defer r.mu.Unlock() + + if r.resyncN <= 0 { + return false + } + + r.resyncN-- + + return true +} + +func (r *recordingSequencer) RefreshInterval() time.Duration { + return r.refresh +} + +func (r *recordingSequencer) snapshot() (opens, seals []uint64, txs int, order []byte) { + r.mu.Lock() + defer r.mu.Unlock() + + return append([]uint64(nil), r.opens...), append([]uint64(nil), r.seals...), r.txs, append([]byte(nil), r.order...) +} + +func newSequencerTestWorker(t *testing.T) (*worker, *testWorkerBackend, *recordingSequencer) { + t.Helper() + + db := rawdb.NewMemoryDatabase() + config := *params.AllCliqueProtocolChanges + config.Clique = ¶ms.CliqueConfig{Period: 1, Epoch: 30000} + + // Rio from genesis: adoption is Rio-gated (sequencerActive), and the + // template config carries a Bor section with no Rio block scheduled. + borCfg := *config.Bor + borCfg.RioBlock = big.NewInt(0) + config.Bor = &borCfg + + engine := clique.New(config.Clique, db) + + w, b, _ := newTestWorker(t, DefaultTestConfig(), &config, engine, db, false, 0) + t.Cleanup(w.close) + + rec := &recordingSequencer{} + w.sequencer = rec + + return w, b, rec +} + +// The worker publishes the full lifecycle — open before transactions, +// transactions before the seal — for produced blocks. +func TestSequencerLifecycleHooks(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + + // Finality has ratified this chain, as it has in any steady state. + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + + w.start() + defer w.stop() + + b.txPool.Add([]*types.Transaction{b.newRandomTx(false)}, false) + + // The first sealed block may predate the transaction; poll until every + // hook kind has fired. + deadline := time.Now().Add(10 * time.Second) + + var ( + opens, seals []uint64 + txs int + order []byte + ) + + for { + opens, seals, txs, order = rec.snapshot() + if len(opens) > 0 && len(seals) > 0 && txs > 0 { + break + } + + if time.Now().After(deadline) { + t.Fatalf("hooks missed: opens=%d seals=%d txs=%d", len(opens), len(seals), txs) + } + + time.Sleep(50 * time.Millisecond) + } + + firstOpen, firstSeal := -1, -1 + + for i, k := range order { + if k == 'o' && firstOpen < 0 { + firstOpen = i + } + + if k == 's' && firstSeal < 0 { + firstSeal = i + } + } + + if firstSeal < firstOpen { + t.Fatalf("seal published before any open: order %q", order) + } +} + +// fillUntilAnnounce fills immediately, then keeps polling the pool until +// the announce margin, so a transaction arriving mid-block is committed +// without waiting for the next slot. +func TestFillUntilAnnounceCommitsLateTx(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + // The harness's preloaded txs are signed for another chain id and never + // enter this pool; drive the test with the backend's own tx builder, + // with explicit nonces. (Committed txs stay "pending" in the pool until + // import, so tcount is the observable throughout.) + b.txPool.Add([]*types.Transaction{b.newRandomTxWithNonce(false, 0)}, true) + waitFor(t, 5*time.Second, func() bool { + return countPendingTransactions(b) >= 1 + }) + + // The loop runs until ~announce time; give it a window. + work.header.Time = uint64(time.Now().Unix()) + 3 + + go func() { + time.Sleep(300 * time.Millisecond) + b.txPool.Add([]*types.Transaction{b.newRandomTxWithNonce(false, 1)}, true) + }() + + if err := w.fillUntilAnnounce(nil, work, genParams, 100*time.Millisecond); err != nil { + t.Fatalf("fill until announce: %v", err) + } + + // The initial fill commits the first tx; a later poll must pick up the + // late one on top. + if work.tcount < 2 { + t.Fatalf("late transaction not committed: tcount %d, want >= 2", work.tcount) + } +} + +func waitFor(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + + deadline := time.Now().Add(timeout) + + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition never held") + } + + time.Sleep(20 * time.Millisecond) + } +} + +// With the announce time already past, the poll loop exits right after the +// initial fill — no lingering until a stale deadline. +func TestFillUntilAnnounceLateBlockExitsImmediately(t *testing.T) { + t.Parallel() + + w, _, _ := newSequencerTestWorker(t) + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + work.header.Time = uint64(time.Now().Unix()) - 10 + + start := time.Now() + + if err := w.fillUntilAnnounce(nil, work, genParams, 100*time.Millisecond); err != nil { + t.Fatalf("fill until announce: %v", err) + } + + if time.Since(start) > time.Second { + t.Fatal("loop did not exit immediately for a late block") + } +} + +// A mid-loop interrupt aborts exactly as it does on the stock path. +func TestFillUntilAnnounceInterrupt(t *testing.T) { + t.Parallel() + + w, _, _ := newSequencerTestWorker(t) + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + work.header.Time = uint64(time.Now().Unix()) + 5 + + interrupt := new(atomic.Int32) + + go func() { + time.Sleep(200 * time.Millisecond) + interrupt.Store(commitInterruptNewHead) + }() + + if err := w.fillUntilAnnounce(interrupt, work, genParams, 50*time.Millisecond); !errors.Is(err, errBlockInterruptedByNewHead) { + t.Fatalf("err = %v, want interrupt", err) + } +} + +// The poll gate: zero unless a sequencer is attached, the worker is +// producing, and a poll cadence is configured. +func TestSequencerPollGate(t *testing.T) { + t.Parallel() + + w, _, rec := newSequencerTestWorker(t) + + // Attached but not producing. + rec.refresh = 100 * time.Millisecond + if got := w.sequencerPoll(big.NewInt(1)); got != 0 { + t.Fatalf("poll while not producing = %v", got) + } + + // No sequencer at all. + w.sequencer = nil + if got := w.sequencerPoll(big.NewInt(1)); got != 0 { + t.Fatalf("poll without sequencer = %v", got) + } +} + +// An adoptable window is applied verbatim: the header inherits the adopted +// context and the window's transactions are committed before the pool's. +func TestAdoptedWindowSeedsBlock(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + adopted := &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 2, + ParentHash: parent.Hash(), + GasLimit: parent.GasLimit, + BaseFee: eip1559.CalcBaseFee(b.chain.Config(), parent), + Txs: []*types.Transaction{ + b.newRandomTxWithNonce(false, 0), + b.newRandomTxWithNonce(false, 1), + }, + } + + rec.mu.Lock() + rec.adoptable = adopted + rec.mu.Unlock() + + // Finality has ratified this chain, as it has in any steady state. + b.setMilestone(parent.Number.Uint64(), parent.Hash()) + + // The empty pre-seal shortcut would race the seeded block to the same + // height; production sequencing targets post-Rio, where it is skipped. + w.noempty.Store(true) + + w.start() + defer w.stop() + + var block *types.Block + + waitFor(t, 10*time.Second, func() bool { + rec.mu.Lock() + adoptions := rec.adopted + rec.mu.Unlock() + + block = b.chain.GetBlockByNumber(adopted.Number) + + return adoptions == 1 && block != nil && len(block.Transactions()) >= 2 + }) + + if block.Time() != adopted.Timestamp { + t.Fatalf("sealed time %d, want adopted %d", block.Time(), adopted.Timestamp) + } + + if block.GasLimit() != adopted.GasLimit { + t.Fatalf("sealed gas limit %d, want adopted %d", block.GasLimit(), adopted.GasLimit) + } + + for i, tx := range adopted.Txs { + if block.Transactions()[i].Hash() != tx.Hash() { + t.Fatalf("tx %d = %s, want adopted %s", i, block.Transactions()[i].Hash(), tx.Hash()) + } + } +} + +// A window for a block that is not the one being built is left alone. +func TestAdoptionSkipsMismatchedWindow(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + rec.mu.Lock() + rec.adoptable = &AdoptedWindow{ + Number: parent.Number.Uint64() + 5, // not the next block + Timestamp: parent.Time + 2, + ParentHash: parent.Hash(), + GasLimit: parent.GasLimit, + BaseFee: eip1559.CalcBaseFee(b.chain.Config(), parent), + } + rec.mu.Unlock() + + // Finality has ratified this chain, as it has in any steady state. + b.setMilestone(parent.Number.Uint64(), parent.Hash()) + + w.start() + defer w.stop() + + waitFor(t, 10*time.Second, func() bool { + return b.chain.GetBlockByNumber(parent.Number.Uint64()+1) != nil + }) + + rec.mu.Lock() + defer rec.mu.Unlock() + + if rec.adopted != 0 { + t.Fatal("mismatched window must not be adopted") + } +} + +// applyAdoption applies only a window that matches the build exactly and +// passes every consensus bound; anything else drops the adoption. +func TestApplyAdoptionValidation(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + base := func() *AdoptedWindow { + return &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 2, + ParentHash: parent.Hash(), + GasLimit: parent.GasLimit, + BaseFee: eip1559.CalcBaseFee(b.chain.Config(), parent), + Txs: []*types.Transaction{b.newRandomTxWithNonce(false, 0)}, + } + } + + cases := []struct { + name string + mutate func(*AdoptedWindow) + wantAdopt bool + }{ + {name: "valid", mutate: func(*AdoptedWindow) {}, wantAdopt: true}, + {name: "wrong number", mutate: func(a *AdoptedWindow) { a.Number += 3 }}, + {name: "wrong parent", mutate: func(a *AdoptedWindow) { a.ParentHash = common.Hash{0x99} }}, + {name: "timestamp at parent", mutate: func(a *AdoptedWindow) { a.Timestamp = parent.Time }}, + {name: "gas limit out of bound", mutate: func(a *AdoptedWindow) { a.GasLimit = parent.GasLimit * 3 }}, + {name: "base fee mismatch", mutate: func(a *AdoptedWindow) { a.BaseFee = big.NewInt(1) }}, + {name: "nil base fee", mutate: func(a *AdoptedWindow) { a.BaseFee = nil }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + genParams := &generateParams{coinbase: testBankAddress} + + header, _, err := w.makeHeader(genParams, false) + if err != nil { + t.Fatalf("makeHeader: %v", err) + } + + a := base() + tc.mutate(a) + genParams.adoption = a + + w.applyAdoption(genParams, header) + + if got := genParams.adoption != nil; got != tc.wantAdopt { + t.Fatalf("adoption kept = %v, want %v", got, tc.wantAdopt) + } + + if tc.wantAdopt { + if header.Time != a.Timestamp || header.GasLimit != a.GasLimit { + t.Fatalf("header not rewritten: time %d gas %d", header.Time, header.GasLimit) + } + + if header.ActualTime.Before(time.Now().Add(adoptionSeedBudget / 2)) { + t.Fatal("announce deadline missing the seed budget") + } + } + }) + } +} + +// fetchAdoption queries for exactly the prepared header's height and +// parent, and only while running — it sits after engine.Prepare, so only +// an authorized build ever reads the store. +func TestFetchAdoptionUsesPreparedHeader(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + header := &types.Header{ + Number: new(big.Int).Add(parent.Number, common.Big1), + ParentHash: parent.Hash(), + } + + rec.mu.Lock() + rec.adoptable = &AdoptedWindow{ + Number: header.Number.Uint64(), + ParentHash: header.ParentHash, + } + rec.mu.Unlock() + + // Not running: no query at all. + genParams := &generateParams{production: true} + w.fetchAdoption(genParams, header) + + if genParams.adoption != nil { + t.Fatal("fetch while not producing") + } + + w.start() + defer w.stop() + + // Payload building (production unset) must never touch the sequencer. + genParams = &generateParams{} + w.fetchAdoption(genParams, header) + + if genParams.adoption != nil { + t.Fatal("payload build queried the sequencer") + } + + genParams = &generateParams{production: true} + w.fetchAdoption(genParams, header) + + if genParams.adoption == nil { + t.Fatal("prepared header did not resolve the window") + } +} + +// An adopted window is already published and preconfirmed, so the seed +// commits all of it. A partial seed would leave the block short of what +// consumers were promised and displace the remainder to the next height. +func TestSeedAdoptedCommitsWholeWindow(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + window := &AdoptedWindow{Txs: []*types.Transaction{ + b.newRandomTxWithNonce(false, 0), + b.newRandomTxWithNonce(false, 1), + b.newRandomTxWithNonce(false, 2), + }} + + w.seedAdopted(work, window) + + if work.tcount != len(window.Txs) { + t.Fatalf("seed committed %d of %d adopted txs", work.tcount, len(window.Txs)) + } +} + +// Adoption is Rio-gated on bor chains: pre-Rio, every validator builds +// every height and an inherited timestamp is invalid for any other signer. +func TestSequencerRioGate(t *testing.T) { + t.Parallel() + + if sequencerActive(¶ms.BorConfig{RioBlock: big.NewInt(1_000_000)}, big.NewInt(1)) { + t.Fatal("pre-Rio bor chain must not sequence") + } + + if !sequencerActive(¶ms.BorConfig{RioBlock: big.NewInt(0)}, big.NewInt(1)) { + t.Fatal("post-Rio bor chain must sequence") + } + + if !sequencerActive(nil, big.NewInt(1)) { + t.Fatal("non-bor chain (tests) must allow sequencing") + } +} + +// Without a Bor config the adoption timestamp floor falls back to exactly +// parent.Time+1: a window on the floor is accepted (a mutated floor — +// off by one in either direction, or underflowed — rejects it). +func TestApplyAdoptionMinTimeWithoutBor(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + genParams := &generateParams{coinbase: testBankAddress} + + header, _, err := w.makeHeader(genParams, false) + if err != nil { + t.Fatalf("makeHeader: %v", err) + } + + saved := w.chainConfig + cfg := *saved + cfg.Bor = nil + w.chainConfig = &cfg + + defer func() { w.chainConfig = saved }() + + // Base fee copied from the prepared header (the value applyAdoption + // compares against): the timestamp floor is then the only bound that + // can reject this window. + genParams.adoption = &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 1, // exactly on the fallback floor + ParentHash: parent.Hash(), + GasLimit: header.GasLimit, + BaseFee: new(big.Int).Set(header.BaseFee), + } + + w.applyAdoption(genParams, header) + + if genParams.adoption == nil { + t.Fatal("window exactly on the parent.Time+1 floor must be accepted") + } +} + +// sequencerPoll gates on the full sequencing predicate and forwards the +// publisher's cadence: zero when the node is not producing, the configured +// interval when it is. +func TestSequencerPollGating(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + rec.refresh = 75 * time.Millisecond + + n := new(big.Int).Add(b.chain.CurrentBlock().Number, big.NewInt(1)) + + if got := w.sequencerPoll(n); got != 0 { + t.Fatalf("poll while stopped = %v, want 0", got) + } + + w.start() + + if got := w.sequencerPoll(n); got != 75*time.Millisecond { + t.Fatalf("poll while running = %v, want 75ms", got) + } +} + +// A zero poll is the one-shot fill: fillBlock must return immediately, not +// hold the block open until the announce deadline. +func TestFillBlockZeroPollIsOneShot(t *testing.T) { + t.Parallel() + + w, _, rec := newSequencerTestWorker(t) + rec.refresh = 0 + w.start() + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + work.header.ActualTime = time.Now().Add(2 * time.Second) + + start := time.Now() + + if err := w.fillBlock(nil, work, genParams); err != nil { + t.Fatalf("fillBlock: %v", err) + } + + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("zero poll held the block open %v — one-shot fill expected", elapsed) + } +} + +// With a Bor config the adoption timestamp floor is parent.Time plus the +// configured block period, not the +1 fallback: a window between the two +// floors is rejected. +func TestApplyAdoptionMinTimeBorPeriod(t *testing.T) { + t.Parallel() + + w, b, _ := newSequencerTestWorker(t) + + parent := b.chain.CurrentBlock() + genParams := &generateParams{coinbase: testBankAddress} + + header, _, err := w.makeHeader(genParams, false) + if err != nil { + t.Fatalf("makeHeader: %v", err) + } + + saved := w.chainConfig + cfg := *saved + borCfg := *saved.Bor + borCfg.Period = map[string]uint64{"0": 2} + cfg.Bor = &borCfg + w.chainConfig = &cfg + + defer func() { w.chainConfig = saved }() + + genParams.adoption = &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 1, // above +1 fallback, below the period-2 floor + ParentHash: parent.Hash(), + GasLimit: header.GasLimit, + BaseFee: new(big.Int).Set(header.BaseFee), + } + + w.applyAdoption(genParams, header) + + if genParams.adoption != nil { + t.Fatal("window below the Bor period floor must be rejected") + } +} + +// A competing producer at our height aborts the fill so the next work cycle +// adopts their window: the build must not seal content that diverges from +// the sequence consumers already saw. +func TestFillBlockRestartsOnResync(t *testing.T) { + t.Parallel() + + w, _, rec := newSequencerTestWorker(t) + rec.refresh = 20 * time.Millisecond + w.start() + + // The worker's own production loop also consumes the signal; arm + // enough for this explicit build to observe one. + rec.mu.Lock() + rec.resyncN = 64 + rec.mu.Unlock() + + genParams := &generateParams{coinbase: testBankAddress} + + work, err := w.prepareWork(genParams, false) + if err != nil { + t.Fatalf("prepareWork: %v", err) + } + + work.header.ActualTime = time.Now().Add(3 * time.Second) + + if err := w.fillBlock(nil, work, genParams); !errors.Is(err, errRebuildForSequence) { + t.Fatalf("fillBlock err = %v, want errRebuildForSequence", err) + } + + // Each read consumes one signal: with none armed the build proceeds. + rec.mu.Lock() + rec.resyncN = 0 + rec.mu.Unlock() + + if rec.ResyncNeeded() { + t.Fatal("resync signal must be consumed, not sticky") + } +} + +// A contested window must not clear the barrier, so the seal path declines +// to commit it. The barrier's own semantics are covered in the sequencer +// package; this pins the worker-side gate. +func TestContestedWindowBlocksSeal(t *testing.T) { + t.Parallel() + + w, b, rec := newSequencerTestWorker(t) + w.start() + + n := new(big.Int).Add(b.chain.CurrentBlock().Number, big.NewInt(1)) + + if !w.sequencingActive(n) { + t.Fatal("sequencing must be active for the gate to apply") + } + + if !w.sequencer.AwaitSequenced(time.Second, 1, nil) { + t.Fatal("an uncontested window should clear") + } + + rec.mu.Lock() + rec.contested = true + rec.mu.Unlock() + + if w.sequencer.AwaitSequenced(time.Second, 1, nil) { + t.Fatal("a contested window must fail the gate that precedes commit") + } +} + +// A refused seal never reaches the chain: the store elected another +// producer's block for the height, and broadcasting ours would fork an +// already-decided height. The worker keeps sealing (each attempt is +// discarded), so refusal costs blocks, never liveness machinery. +func TestSealGateRefusalStopsBroadcast(t *testing.T) { + w, b, rec := newSequencerTestWorker(t) + + head := b.chain.CurrentBlock() + b.setMilestone(head.Number.Uint64(), head.Hash()) + + rec.mu.Lock() + rec.verdict = SealRefused + rec.mu.Unlock() + + w.start() + defer w.stop() + + deadline := time.Now().Add(5 * time.Second) + for { + _, seals, _, _ := rec.snapshot() + if len(seals) > 0 { + break + } + + if time.Now().After(deadline) { + t.Fatal("worker never sealed") + } + + time.Sleep(20 * time.Millisecond) + } + + if got := b.chain.CurrentBlock().Number.Uint64(); got != 0 { + t.Fatalf("chain advanced to %d on refused seals: a refused block "+ + "was written and broadcast", got) + } +} diff --git a/miner/worker_test.go b/miner/worker_test.go index d37f769fe6..71d084b33c 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -61,12 +61,12 @@ import ( borSpan "github.com/ethereum/go-ethereum/consensus/bor/heimdall/span" ) -// TestPendingStateNotStaleForNonValidator verifies that a Bor node whose signer -// is NOT in the active validator set still keeps its pending snapshot fresh. -// Regression test: previously, Prepare() returned UnauthorizedSignerError for -// non-validators, which caused prepareWork to fail and the snapshot to never -// update, leading to stale trie errors on "pending" RPC queries. -func TestPendingStateNotStaleForNonValidator(t *testing.T) { +// TestNonValidatorDoesNotBuild verifies that a Bor node whose signer is NOT +// in the active validator set (post-Rio: not the span's producer) builds no +// candidate at all: Prepare rejects the signer, so nothing is mined and — +// with a sequencer attached — nothing is ever published. Nodes without a +// signer (RPC) are unaffected by this check. +func TestNonValidatorDoesNotBuild(t *testing.T) { chainConfig := *params.BorUnittestChainConfig engine, ctrl := getFakeBorFromConfig(t, &chainConfig) @@ -90,26 +90,14 @@ func TestPendingStateNotStaleForNonValidator(t *testing.T) { }) w.setEtherbase(nonValidatorAddr) - // Start the worker. It will call commitWork which calls Prepare. - // Before the fix: Prepare fails with UnauthorizedSignerError, snapshot - // is never set, pending() returns nil. - // After the fix: Prepare succeeds (defaults succession to 0), snapshot - // is updated, pending() returns valid state. + // Start the worker: commitWork calls Prepare, which rejects the + // unauthorized signer, so no pending snapshot is ever produced. w.start() - // Give the worker time to process the start event and run commitWork. time.Sleep(1 * time.Second) - pendingBlock, _, pendingState := w.pending() - require.NotNil(t, pendingBlock, "pending block should not be nil for non-validator node") - require.NotNil(t, pendingState, "pending state should not be nil for non-validator node") - - // The pending state must be readable without errors (not a stale trie). - balance := pendingState.GetBalance(testBankAddress) - require.False(t, balance.IsZero(), - "pending state balance for funded account should not be zero (would indicate stale trie)") - require.NoError(t, pendingState.Error(), - "pending state should have no database errors") + pendingBlock, _, _ := w.pending() + require.Nil(t, pendingBlock, "a signer outside the validator set must not build") } // nolint : paralleltest @@ -164,9 +152,7 @@ func testGenerateBlockAndImport(t *testing.T, isClique bool, isBor bool) { // Start mining! w.start() - var ( - err error - ) + var err error // []*types.Transaction{tx} var i uint64 for i = 0; i < 5; i++ { @@ -279,6 +265,11 @@ func DefaultTestConfig() *Config { // testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. type testWorkerBackend struct { + msMu sync.Mutex + msExists bool + msNumber uint64 + msHash common.Hash + db ethdb.Database txPool *txpool.TxPool chain *core.BlockChain @@ -286,7 +277,7 @@ type testWorkerBackend struct { } func newTestWorkerBackend(t TensingObject, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database) *testWorkerBackend { - var gspec = &core.Genesis{ + gspec := &core.Genesis{ Config: chainConfig, Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, } @@ -324,6 +315,22 @@ func newTestWorkerBackend(t TensingObject, chainConfig *params.ChainConfig, engi return b } +// setMilestone drives the finality gate in tests: the whitelisted milestone +// the backend reports to the worker. +func (b *testWorkerBackend) setMilestone(number uint64, hash common.Hash) { + b.msMu.Lock() + defer b.msMu.Unlock() + + b.msExists, b.msNumber, b.msHash = true, number, hash +} + +func (b *testWorkerBackend) WhitelistedMilestone() (bool, uint64, common.Hash) { + b.msMu.Lock() + defer b.msMu.Unlock() + + return b.msExists, b.msNumber, b.msHash +} + func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } func (b *testWorkerBackend) PeerCount() int { @@ -546,6 +553,7 @@ func TestEmptyWorkEthash(t *testing.T) { t.Skip() testEmptyWork(t, ethashChainConfig, ethash.NewFaker()) } + func TestEmptyWorkClique(t *testing.T) { t.Skip() testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, rawdb.NewMemoryDatabase())) @@ -750,7 +758,7 @@ func testGetSealingWork(t *testing.T, chainConfig *params.ChainConfig, engine co t.Errorf("Mismatched block number, want %d got %d", number, block.NumberU64()) } } - var cases = []struct { + cases := []struct { parent common.Hash coinbase common.Address random common.Hash diff --git a/miner/worker_twin_test.go b/miner/worker_twin_test.go new file mode 100644 index 0000000000..4404a9f11b --- /dev/null +++ b/miner/worker_twin_test.go @@ -0,0 +1,113 @@ +package miner + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/consensus/clique" + "github.com/ethereum/go-ethereum/consensus/misc/eip1559" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +// The whole convergence argument rests on this: two producers holding one key +// and sealing the same header must emit the same signature, or "they derive +// the same block" is false and contention always produces two blocks. +// secp256k1 signing uses RFC 6979 deterministic nonces, on both the cgo and +// nocgo paths — asserted here rather than assumed. +func TestSigningIsDeterministic(t *testing.T) { + t.Parallel() + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("key: %v", err) + } + + digest := crypto.Keccak256([]byte("the same header, signed twice")) + + first, err := crypto.Sign(digest, key) + if err != nil { + t.Fatalf("sign: %v", err) + } + + second, err := crypto.Sign(digest, key) + if err != nil { + t.Fatalf("sign again: %v", err) + } + + if !bytes.Equal(first, second) { + t.Fatal("the same key signing the same digest produced two different " + + "signatures: two producers can never derive one block, so " + + "contention must be resolved by refusal rather than agreement") + } +} + +// Header identity: two independent workers on the same config, given the same +// adopted window, must prepare byte-identical headers. This is the other half +// of the convergence claim — agreeing on transactions is worthless if the +// headers differ, because the block hashes then differ anyway. +// +// The seal hash is the exact thing consensus signs, so comparing it is +// stricter than comparing fields one by one. +func TestTwinWorkersPrepareIdenticalHeaders(t *testing.T) { + t.Parallel() + + w1, _, _ := newSequencerTestWorker(t) + w2, _, _ := newSequencerTestWorker(t) + + parent := w1.chain.CurrentBlock() + if got := w2.chain.CurrentBlock().Hash(); got != parent.Hash() { + t.Fatalf("the two workers start from different genesis blocks (%s vs %s)", + parent.Hash(), got) + } + + // The same inherited open context on both sides, as adoption supplies it. + // The base fee is derived from the parent by rules both producers share. + window := func() *AdoptedWindow { + return &AdoptedWindow{ + Number: parent.Number.Uint64() + 1, + Timestamp: parent.Time + 2, + ParentHash: parent.Hash(), + GasLimit: parent.GasLimit, + BaseFee: eip1559.CalcBaseFee(w1.chain.Config(), parent), + } + } + + h1 := prepareAdoptedHeader(t, w1, window()) + h2 := prepareAdoptedHeader(t, w2, window()) + + if clique.SealHash(h1) != clique.SealHash(h2) { + t.Fatalf("two producers derived different headers from one open "+ + "context: seal hashes %s vs %s. Agreeing on content cannot make "+ + "them one block if the headers differ.", + clique.SealHash(h1), clique.SealHash(h2)) + } +} + +// prepareAdoptedHeader drives the real adoption path — makeHeader, then +// applyAdoption — and fails if the window is rejected, so this cannot quietly +// degrade into comparing two ordinary headers. +func prepareAdoptedHeader(t *testing.T, w *worker, window *AdoptedWindow) *types.Header { + t.Helper() + + genParams := &generateParams{coinbase: testBankAddress, adoption: window} + + header, _, err := w.makeHeader(genParams, false) + if err != nil { + t.Fatalf("makeHeader: %v", err) + } + + w.applyAdoption(genParams, header) + + if genParams.adoption == nil { + t.Fatal("the window was rejected, so this header never inherited the " + + "open context and proves nothing about two producers agreeing") + } + + if header.Time != window.Timestamp { + t.Fatalf("header did not inherit the adopted timestamp: %d vs %d", + header.Time, window.Timestamp) + } + + return header +}