From 4a67df54cc07aff119b55ed45231a09f3de3ad3e Mon Sep 17 00:00:00 2001 From: Peter Hedenskog Date: Sat, 8 Aug 2026 17:36:21 +0200 Subject: [PATCH 1/2] Let heartbeats re-register testrunners the server has forgotten MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The index page showed "Waiting for test runners to come online" while the API kept accepting and running jobs. The online state comes from an in-memory registry fed only by a runner's start broadcast, while API queue routing lives in separate add-only maps. The registry prune only needs the local clock, but heartbeat delivery needs Redis — so a Redis outage longer than the stale window got a perfectly healthy runner pruned. Once pruned, its heartbeats were deliberately ignored and nothing ever asked it to announce itself again, so the state stuck until a process restart. Heartbeats now carry the runner's serverConfig so the server can re-register an unknown-but-alive runner on the next beat, making the registry self-healing within one heartbeat interval. Old runners without the payload keep the previous behavior. Heartbeat jobs are also cleaned up on completion now — they fire every 30s and were accumulating in Redis forever. Co-authored-by: Claude Fable 5 noreply@anthropic.com --- server/src/server.js | 48 +++++++++++++++--------- server/src/testrunners.js | 9 +++-- testrunner/src/sitespeedio-testrunner.js | 14 ++++++- 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/server/src/server.js b/server/src/server.js index 3f086c75..3679cdfd 100644 --- a/server/src/server.js +++ b/server/src/server.js @@ -161,6 +161,22 @@ async function setupResultQueue() { }); } +function registerTestRunner(serverConfig) { + addTestRunner(serverConfig); + for (let setup of serverConfig.setup) { + let queueName = setup.queue; + + const queue = getExistingQueue(queueName); + // We only add queue that do not exist + if (!queue) { + onMessage(queueName, 'global:active', setActiveStatus); + onMessage(queueName, 'global:failed', setFailedStatus); + onMessage(queueName, 'global:stalled', setStalledStatus); + addDeviceToQueue(setup.deviceId, serverConfig.name, queueName); + } + } +} + async function setupTestRunnerQueue() { // Create the queue that handle testrunners processJob('testrunners', async job => { @@ -169,7 +185,19 @@ async function setupTestRunnerQueue() { // up), stop (graceful shutdown) and heartbeat (still here). A runner // that misses heartbeats long enough gets pruned server-side. if (job.data.type === 'heartbeat') { - touchTestRunner(job.data.hostname); + const known = touchTestRunner(job.data.hostname); + // A heartbeat from an unknown hostname means the runner is alive + // but fell out of the registry — pruned during a Redis blip, or + // its start broadcast was lost across a server restart. The + // runner sends its serverConfig with each heartbeat so we can + // heal by re-registering instead of ignoring it forever. + if (!known && job.data.serverConfig) { + logger.info( + 'Re-registering testrunner %s from heartbeat', + job.data.hostname + ); + registerTestRunner(job.data.serverConfig); + } return resolve(); } if (job.data.type === 'start') { @@ -179,23 +207,7 @@ async function setupTestRunnerQueue() { job.data.serverConfig ); - addTestRunner(job.data.serverConfig); - for (let setup of job.data.serverConfig.setup) { - let queueName = setup.queue; - - const queue = getExistingQueue(queueName); - // We only add queue that do not exist - if (!queue) { - onMessage(queueName, 'global:active', setActiveStatus); - onMessage(queueName, 'global:failed', setFailedStatus); - onMessage(queueName, 'global:stalled', setStalledStatus); - addDeviceToQueue( - setup.deviceId, - job.data.serverConfig.name, - queueName - ); - } - } + registerTestRunner(job.data.serverConfig); return resolve(); } else { logger.info('TestRunner %s is shutting down', job.data.name); diff --git a/server/src/testrunners.js b/server/src/testrunners.js index 140f6c55..b32559c0 100755 --- a/server/src/testrunners.js +++ b/server/src/testrunners.js @@ -107,14 +107,17 @@ export function removeTestRunner(config) { updateTestRunnerMetrics(); } -// Heartbeat handler: a known runner says "still here". Unknown runners -// (heartbeat before the start message lands, or after a server-side prune) -// are ignored — the next start broadcast will register them properly. +// Heartbeat handler: a known runner says "still here". Returns whether the +// hostname was known, so the caller can re-register an unknown runner from +// the heartbeat's serverConfig (heartbeat before the start message lands, +// after a server-side prune, or a start broadcast lost across a restart). export function touchTestRunner(hostname) { const runner = testRunners[hostname]; if (runner) { runner.lastSeenAt = Date.now(); + return true; } + return false; } export function pruneStaleTestRunners(now = Date.now()) { diff --git a/testrunner/src/sitespeedio-testrunner.js b/testrunner/src/sitespeedio-testrunner.js index 7db0b1ff..c7c40ac8 100644 --- a/testrunner/src/sitespeedio-testrunner.js +++ b/testrunner/src/sitespeedio-testrunner.js @@ -75,10 +75,22 @@ export class SitespeedioTestRunner { // Heartbeat. Reuses the existing `testrunners` queue used by start/stop; // the server treats a missing heartbeat as a dead runner and prunes us. + // The serverConfig rides along so a server that no longer knows us + // (pruned during a Redis blip, or restarted while our start broadcast + // was lost) can re-register us instead of ignoring the heartbeat. const testRunnerQueue = await queueHandler.getQueue('testrunners'); heartbeatTimer = setInterval(() => { testRunnerQueue - .add({ type: 'heartbeat', hostname: serverConfig.hostname }) + .add( + { + type: 'heartbeat', + hostname: serverConfig.hostname, + serverConfig: serverConfig + }, + // Heartbeats fire every 30s and carry the full serverConfig — + // without cleanup the completed jobs pile up in Redis forever. + { removeOnComplete: true, removeOnFail: true } + ) .catch(error => logger.error('Failed to publish heartbeat: %s', error.message) ); From 7289bfcee2a95afb2eeb1d34886b9465fbe0adac Mon Sep 17 00:00:00 2001 From: Peter Hedenskog Date: Sun, 9 Aug 2026 09:14:03 +0200 Subject: [PATCH 2/2] fix --- .github/workflows/docker-api-local-linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-api-local-linux.yml b/.github/workflows/docker-api-local-linux.yml index afdaf83f..ba2b3c32 100644 --- a/.github/workflows/docker-api-local-linux.yml +++ b/.github/workflows/docker-api-local-linux.yml @@ -28,6 +28,7 @@ jobs: mtalk.google.com:5228 nodejs.org:443 production.cloudflare.docker.com:443 + production.cloudfront.docker.com:443 registry-1.docker.io:443 registry.npmjs.org:443 release-assets.githubusercontent.com:443