Let cells where something moves take one-sub-pixel steps sooner

dev/check.ts

@@ -3,7 +3,7 @@
 // and, only when given, on frame cost above --max-ms: bun dev/check.ts [--quiet] [--max-ms=N] [--max-flicker=N] [--verify-dirty] [wallpaper...] [season] [weather]
 // --verify-dirty plays each combination twice, refitting every cell every frame (WALLPAPER_DIRTY=0) and then only the
 // cells that changed, with text edits, redraws between frames and agent events mixed in, and fails on any cell that
-// differs between the two.
+// differs between the two, and on any pixel a scene draws outside its touched boxes.
 import { mkdirSync, rmSync } from "node:fs"
 import { parseArgs } from "node:util"
 import { ACTIVITIES, SEASONS, TIMES, WEATHERS, type Activity, type Engine, type Time, type Wallpaper } from "../src/engine"
@@ -47,6 +47,7 @@ for (const { wallpaper, activity, time } of runs) {
     result.flicker > maxFlicker && `flicker over ${maxFlicker}`,
     result.ms > maxMs && `slower than ${maxMs} ms`,
     result.mismatches && `${result.mismatches} cells differ from a full refit, first ${result.first}`,
+    result.uncovered && `${result.uncovered} pixels drawn outside the touched boxes, first ${result.where}`,
   ].filter(Boolean)
   if (problems.length) failed++
   if (args.values.quiet && !problems.length) continue
@@ -67,15 +68,18 @@ function play(wallpaper: Wallpaper, activity: Activity, time: Time, dirty: strin
   performance.now = () => (hold ? held : (held = tick()))
   screen.renderer.getCursorState = () => cursor
   seed(1)
-  screen.engine.start(wallpaper, { activity, time, season, weather })
+  let uncovered = 0
+  let where = ""
+  const watched = verify && dirty === "1" ? watch(wallpaper, (n, at) => ((uncovered += n), (where ||= at))) : wallpaper
+  screen.engine.start(watched, { activity, time, season, weather })
   let previous = new Uint32Array(W * H)
   let changes = 0
   let total = 0
   let mismatches = 0
   let first = ""
   const recording: Uint16Array[] = []
   for (let f = 0; f < frames; f++) {
-    if (verify) script(f, screen.engine, wallpaper, activity, time)
+    if (verify) script(f, screen.engine, watched, activity, time)
     const start = Bun.nanoseconds()
     screen.frame(verify && f >= 100 && f < 160 ? (x, y) => text(x + 3, y) : text)
     hold = false
@@ -106,7 +110,7 @@ function play(wallpaper: Wallpaper, activity: Activity, time: Time, dirty: strin
     }
   }
   screen.engine.stop()
-  return { ms: total / (frames - 60), flicker: changes / (frames - 60), mismatches, first, recording }
+  return { ms: total / (frames - 60), flicker: changes / (frames - 60), mismatches, first, recording, uncovered, where }
 }
 
 // Text edits, redraws between frames, a moving cursor, agent events, a brightness change and a crossfade, to exercise every reason
@@ -121,3 +125,35 @@ function script(f: number, engine: Engine, wallpaper: Wallpaper, activity: Activ
   if (f === 180) engine.start(wallpaper, { activity, time, season, weather, brightness: "vivid" })
   if (f === 205) engine.start(wallpaper, { activity, time: time === "night" ? "day" : "night", season, weather, brightness: "vivid" })
 }
+
+// A wallpaper whose scenes check, after every render, that each pixel that differs from the cached background lies in
+// a touched box: finish() only works out what's under them, so anything drawn elsewhere would never show.
+function watch(wallpaper: Wallpaper, report: (pixels: number, at: string) => void): Wallpaper {
+  return {
+    ...wallpaper,
+    create(settings) {
+      const scene = wallpaper.create(settings)
+      const canvas = scene as unknown as { hdr: Float32Array; background?: Float32Array; touched?: number[] }
+      const render = scene.render.bind(scene)
+      scene.render = () => {
+        render()
+        const bg = canvas.background
+        const t = canvas.touched
+        if (!bg || !t || bg.length !== canvas.hdr.length) return
+        const inside = new Uint8Array(scene.W * scene.H)
+        for (let b = 0; b < t.length; b += 4)
+          for (let y = Math.max(0, Math.floor(t[b + 1])); y <= Math.min(scene.H - 1, Math.floor(t[b + 3])); y++)
+            for (let x = Math.max(0, Math.floor(t[b])); x <= Math.min(scene.W - 1, Math.floor(t[b + 2])); x++) inside[y * scene.W + x] = 1
+        let n = 0
+        let at = ""
+        for (let i = 0; i < inside.length; i++) {
+          if (inside[i] || (canvas.hdr[i * 3] === bg[i * 3] && canvas.hdr[i * 3 + 1] === bg[i * 3 + 1] && canvas.hdr[i * 3 + 2] === bg[i * 3 + 2])) continue
+          n++
+          at ||= `${i % scene.W},${(i / scene.W) | 0}`
+        }
+        if (n) report(n, `${settings.time} at ${at}`)
+      }
+      return scene
+    },
+  }
+}

src/engine.tsx

@@ -161,9 +161,9 @@ function evalMask(q: Float32Array, mask: number) {
 }
 
 // Splits a cell's eight sub-pixels into two colours by brightness threshold, choosing the split with the least
-// squared error. A cell keeps last frame's split unless the new one is clearly better, so slow motion doesn't
-// make the glyph pattern flicker between near-equal choices. Returns the mask used.
-function writeOctant(q: Float32Array, order: Uint8Array, previous: number, char: Uint32Array, fg: Uint16Array, bg: Uint16Array, i: number, o: number) {
+// squared error. A cell keeps last frame's split unless the new one is clearly better (see tolerance), so slow
+// motion doesn't make the glyph pattern flicker between near-equal choices. Returns the mask used.
+function writeOctant(q: Float32Array, order: Uint8Array, previous: number, older: number, moving: number, char: Uint32Array, fg: Uint16Array, bg: Uint16Array, i: number, o: number) {
   for (let s = 0; s < 8; s++) {
     const l = q[s * 3] * 0.2126 + q[s * 3 + 1] * 0.7152 + q[s * 3 + 2] * 0.0722
     let j = s
@@ -210,7 +210,7 @@ function writeOctant(q: Float32Array, order: Uint8Array, previous: number, char:
   let chosen = bestMask
   if (previous >= 0 && previous !== bestMask) {
     evalMask(q, previous)
-    if (split.err <= bestErr * 1.35 + HYSTERESIS) chosen = previous
+    if (split.err <= tolerance(bestErr, bestMask, previous, older, moving, HYSTERESIS)) chosen = previous
   }
   evalMask(q, chosen)
   if (chosen === 0 || chosen === 255) {
@@ -236,6 +236,14 @@ function writeOctant(q: Float32Array, order: Uint8Array, previous: number, char:
 const luma8 = new Float32Array(8)
 // Extra squared error (in 0-255 channel units) a cell tolerates before abandoning last frame's split.
 const HYSTERESIS = 150
+// Pixels around a touched box that still count as moving: anti-aliased edges and the trail PERSISTENCE leaves. Bloom
+// reaches further, but it only shifts colors smoothly, so those cells keep the steady fit.
+const MOVING_REACH = 4
+// Touched boxes under this many pixels (stars, snowflakes, motes) are points rather than edges and keep the steady fit.
+const MOVING_MIN = 30
+// Boxes over this share of the screen are layers redrawn every frame (water, an aurora, light shafts, fog), not
+// something moving across a still background, so they keep the steady fit too.
+const MOVING_MAX = 0.1
 // Share of the previous frame blended into each new one, which smooths sub-pixel motion.
 const PERSISTENCE = 0.45
 // How often the scene advances and redraws; wallpapers move slowly, so this stays low to save CPU and output.
@@ -266,7 +274,7 @@ const QUADRANTS = [
 
 // Picks the two-colour split of a cell's four sub-pixels with the least error and writes it as a quadrant block. Like
 // writeOctant it keeps last frame's split unless the new one is clearly better. Returns the mask used.
-function writeQuadrant(q8: Float32Array, previous: number, char: Uint32Array, fg: Uint16Array, bg: Uint16Array, i: number, o: number) {
+function writeQuadrant(q8: Float32Array, previous: number, older: number, moving: number, char: Uint32Array, fg: Uint16Array, bg: Uint16Array, i: number, o: number) {
   const q = quad4
   for (let s = 0; s < 4; s++) {
     const top = ((s >> 1) * 4 + (s & 1)) * 3
@@ -284,7 +292,7 @@ function writeQuadrant(q8: Float32Array, previous: number, char: Uint32Array, fg
   let chosen = bestMask
   if (previous >= 0 && previous !== bestMask) {
     evalQuad(q, previous)
-    if (split.err <= bestErr * 1.35 + HYSTERESIS / 2) chosen = previous
+    if (split.err <= tolerance(bestErr, bestMask, previous, older, moving, HYSTERESIS / 2)) chosen = previous
   }
   evalQuad(q, chosen)
   bg[o] = split.br
@@ -345,6 +353,32 @@ function evalQuad(q: Float32Array, mask: number) {
   split.err = err
 }
 
+// Marks the cells under the scene's touched boxes (see Canvas.touched), widened by MOVING_REACH pixels.
+function markMoving(moving: Uint8Array, touched: readonly number[] | undefined, PW: number, PH: number, W: number, H: number) {
+  moving.fill(0)
+  if (!touched) return
+  const sx = W / PW
+  const sy = H / PH
+  for (let b = 0; b < touched.length; b += 4) {
+    const area = (touched[b + 2] - touched[b] + 1) * (touched[b + 3] - touched[b + 1] + 1)
+    if (area < MOVING_MIN || area > PW * PH * MOVING_MAX) continue
+    const x0 = Math.max(0, Math.floor((touched[b] - MOVING_REACH) * sx))
+    const x1 = Math.min(W, Math.ceil((touched[b + 2] + 1 + MOVING_REACH) * sx))
+    const y0 = Math.max(0, Math.floor((touched[b + 1] - MOVING_REACH) * sy))
+    const y1 = Math.min(H, Math.ceil((touched[b + 3] + 1 + MOVING_REACH) * sy))
+    for (let y = y0; y < y1; y++) if (x0 < x1) moving.fill(1, y * W + x0, y * W + x1)
+  }
+}
+
+// The most error a cell's previous split may have and still be kept, given the best split's. Where something moves,
+// a step of one sub-pixel (not back to the split before) is taken as soon as it fits a little better, so slow motion
+// glides a sub-pixel at a time instead of jumping several at once. Bigger changes, steps back, and every static cell
+// keep the full margin, so noise and swaying stay steady.
+function tolerance(bestErr: number, best: number, previous: number, older: number, moving: number, margin: number) {
+  const d = best ^ previous
+  return moving && (d & (d - 1)) === 0 && best !== older ? bestErr * 1.05 + margin * 0.3 : bestErr * 1.35 + margin
+}
+
 // The terminal is sent every cell whose character or colors differ from the last frame, and slow light (aurora, twinkle,
 // bloom around moving things) nudges hundreds of cells by a level or two of 255 each frame. A color the engine drew
 // that is within QUIET levels in every channel of what the cell showed last frame keeps last frame's value, so drift
@@ -612,6 +646,9 @@ export function createEngine(
   let surface = new Uint8Array(0)
   let cellTint = new Int32Array(0)
   let previousMask = new Int16Array(0)
+  // The split each cell showed before its current one, and the cells where something moves this frame (see tolerance).
+  let olderMask = new Int16Array(0)
+  let moving = new Uint8Array(0)
   let history = new Float32Array(0)
   // What each cell showed last frame, and the colors OpenCode drew this frame before the scene went in, for quieten.
   let shownChar = new Uint32Array(0)
@@ -868,7 +905,7 @@ export function createEngine(
     const desaturate = lerp(DESATURATE[was.settings.time], DESATURATE[settings!.time], t)
     // Blending in the previous frame fades at the same speed per second whatever the frame rate.
     const persistence = interval > 1000 / FPS ? PERSISTENCE ** (interval / (1000 / FPS)) : PERSISTENCE
-    paint(buf, from ? mixed : scene.pixels, scene.W, scene.H, advanced, pixels, desaturate, shade, persistence, scrim, from ? undefined : scene.changed)
+    paint(buf, from ? mixed : scene.pixels, scene.W, scene.H, advanced, pixels, desaturate, shade, persistence, scrim, from ? undefined : scene.changed, from ? undefined : scene.touched)
     if (mapFresh) {
       mapFresh = false
       scene.textMap?.(textMap)
@@ -894,7 +931,7 @@ export function createEngine(
 
   // Everything after the scene has drawn its frame. It never touches the scene object: each wallpaper's scene is a
   // different class, and the optimizer would otherwise fall back to slow code for this whole function after a switch.
-  const paint = (buf: OptimizedBuffer, px: Uint8Array, PW: number, PH: number, advanced: boolean, pixels: boolean, desaturate: number, shade: number, persistence: number, scrim: Scrim, changed: readonly number[] | undefined) => {
+  const paint = (buf: OptimizedBuffer, px: Uint8Array, PW: number, PH: number, advanced: boolean, pixels: boolean, desaturate: number, shade: number, persistence: number, scrim: Scrim, changed: readonly number[] | undefined, touched: readonly number[] | undefined) => {
     const W = buf.width
     const H = buf.height
     const GW = W * 2
@@ -924,8 +961,10 @@ export function createEngine(
       cellY = new Int32Array(PH)
       for (let x = 0; x < W; x++) cellX.fill(x, colStart[x], colStart[x + 1])
       for (let y = 0; y < H; y++) cellY.fill(y, rowStart[y], rowStart[y + 1])
+      moving = new Uint8Array(cells)
     }
     if (advanced) {
+      markMoving(moving, touched, PW, PH, W, H)
       // Anything that changes every pixel's blend, or a frame without a list of what changed, blends every cell again.
       const blend = `${desaturate} ${shade} ${persistence}`
       const everything = reset || !dirty || !changed || px !== lastSource || blend !== lastBlend
@@ -970,6 +1009,7 @@ export function createEngine(
       cellTint = new Int32Array(W * H)
     }
     if (previousMask.length !== cells) previousMask = new Int16Array(cells).fill(-1)
+    if (olderMask.length !== cells) olderMask = new Int16Array(cells).fill(-1)
     mask.fill(0, 0, W * MH)
     for (let i = 0; i < cells; i++) {
       const tint = tintOf(key(bg, i * 4))
@@ -1039,7 +1079,8 @@ export function createEngine(
       const [deepR, deepG, deepB] = scrimAt(scrim, y / Math.max(1, H - 1), light)
       for (let x = 0; x < W; x++) {
         const i = y * W + x
-        const reusable = fitted[i] === 1 && !refitAll && refit[i] === 0
+        // Cells where something moves fit differently (see tolerance), so they are never reused.
+        const reusable = fitted[i] === 1 && !refitAll && refit[i] === 0 && moving[i] === 0
         fitted[i] = 0
         if (imageCell[i]) continue
         const o = i * 4
@@ -1086,12 +1127,15 @@ export function createEngine(
           if (!surface[i]) continue
         }
         if (fits) {
-          if (octants) previousMask[i] = writeOctant(q, order, previousMask[i], char, fg, bg, i, o)
-          else previousMask[i] = writeQuadrant(q, previousMask[i], char, fg, bg, i, o)
+          const was = previousMask[i]
+          if (octants) previousMask[i] = writeOctant(q, order, was, olderMask[i], moving[i], char, fg, bg, i, o)
+          else previousMask[i] = writeQuadrant(q, was, olderMask[i], moving[i], char, fg, bg, i, o)
+          if (previousMask[i] !== was) olderMask[i] = was
           rememberFit(i, o, char, fg, bg, fitChar, fitFg, fitBg)
           fitTop[i] = st
           fitBottom[i] = sb
-          fitted[i] = 1
+          // A fit made while something moved (see tolerance) isn't reused next frame.
+          fitted[i] = moving[i] ? 0 : 1
           applyTint(bg, o, cellTint[i])
           if (char[i] !== SPACE) applyTint(fg, o, cellTint[i])
           continue
@@ -1181,6 +1225,7 @@ export function createEngine(
       lastTarget = ""
       history = new Float32Array(0)
       previousMask = new Int16Array(0)
+      olderMask = new Int16Array(0)
       shownChar = new Uint32Array(0)
       dirty = process.env.WALLPAPER_DIRTY !== "0"
       lastActivity = Date.now()

src/wallpaper.ts

@@ -11,6 +11,9 @@ export interface Scene {
   // Optional: boxes [x0, y0, x1, y1, ...] in pixels, inclusive, outside which this frame's pixels match the last
   // frame's (Canvas provides it). Without it the engine refits every cell every frame.
   readonly changed?: readonly number[]
+  // Optional: boxes like changed's around what this frame drew over the still background, so the engine can let cells
+  // where something moves follow it more closely (Canvas provides it).
+  readonly touched?: readonly number[]
   resize(W: number, H: number, cells?: boolean): void
   step(dt: number): void
   render(): void