Let heartbeats re-register testrunners the server has forgotten

.github/workflows/docker-api-local-linux.yml

@@ -28,6 +28,7 @@ jobs:
           mtalk.google.com:5228
           nodejs.org:443
           production.cloudflare.docker.com:443
+          production.cloudfront.docker.com:443
           registry-1.docker.io:443
           registry.npmjs.org:443
           release-assets.githubusercontent.com:443

server/src/server.js

@@ -161,6 +161,22 @@ async function setupResultQueue() {
   });
 }
 
+function registerTestRunner(serverConfig) {
+  addTestRunner(serverConfig);
+  for (let setup of serverConfig.setup) {
+    let queueName = setup.queue;
+
+    const queue = getExistingQueue(queueName);
+    // We only add queue that do not exist
+    if (!queue) {
+      onMessage(queueName, 'global:active', setActiveStatus);
+      onMessage(queueName, 'global:failed', setFailedStatus);
+      onMessage(queueName, 'global:stalled', setStalledStatus);
+      addDeviceToQueue(setup.deviceId, serverConfig.name, queueName);
+    }
+  }
+}
+
 async function setupTestRunnerQueue() {
   // Create the queue that handle testrunners
   processJob('testrunners', async job => {
@@ -169,7 +185,19 @@ async function setupTestRunnerQueue() {
       // up), stop (graceful shutdown) and heartbeat (still here). A runner
       // that misses heartbeats long enough gets pruned server-side.
       if (job.data.type === 'heartbeat') {
-        touchTestRunner(job.data.hostname);
+        const known = touchTestRunner(job.data.hostname);
+        // A heartbeat from an unknown hostname means the runner is alive
+        // but fell out of the registry — pruned during a Redis blip, or
+        // its start broadcast was lost across a server restart. The
+        // runner sends its serverConfig with each heartbeat so we can
+        // heal by re-registering instead of ignoring it forever.
+        if (!known && job.data.serverConfig) {
+          logger.info(
+            'Re-registering testrunner %s from heartbeat',
+            job.data.hostname
+          );
+          registerTestRunner(job.data.serverConfig);
+        }
         return resolve();
       }
       if (job.data.type === 'start') {
@@ -179,23 +207,7 @@ async function setupTestRunnerQueue() {
           job.data.serverConfig
         );
 
-        addTestRunner(job.data.serverConfig);
-        for (let setup of job.data.serverConfig.setup) {
-          let queueName = setup.queue;
-
-          const queue = getExistingQueue(queueName);
-          // We only add queue that do not exist
-          if (!queue) {
-            onMessage(queueName, 'global:active', setActiveStatus);
-            onMessage(queueName, 'global:failed', setFailedStatus);
-            onMessage(queueName, 'global:stalled', setStalledStatus);
-            addDeviceToQueue(
-              setup.deviceId,
-              job.data.serverConfig.name,
-              queueName
-            );
-          }
-        }
+        registerTestRunner(job.data.serverConfig);
         return resolve();
       } else {
         logger.info('TestRunner %s is shutting down', job.data.name);

server/src/testrunners.js

@@ -107,14 +107,17 @@ export function removeTestRunner(config) {
   updateTestRunnerMetrics();
 }
 
-// Heartbeat handler: a known runner says "still here". Unknown runners
-// (heartbeat before the start message lands, or after a server-side prune)
-// are ignored — the next start broadcast will register them properly.
+// Heartbeat handler: a known runner says "still here". Returns whether the
+// hostname was known, so the caller can re-register an unknown runner from
+// the heartbeat's serverConfig (heartbeat before the start message lands,
+// after a server-side prune, or a start broadcast lost across a restart).
 export function touchTestRunner(hostname) {
   const runner = testRunners[hostname];
   if (runner) {
     runner.lastSeenAt = Date.now();
+    return true;
   }
+  return false;
 }
 
 export function pruneStaleTestRunners(now = Date.now()) {

testrunner/src/sitespeedio-testrunner.js

@@ -75,10 +75,22 @@ export class SitespeedioTestRunner {
 
     // Heartbeat. Reuses the existing `testrunners` queue used by start/stop;
     // the server treats a missing heartbeat as a dead runner and prunes us.
+    // The serverConfig rides along so a server that no longer knows us
+    // (pruned during a Redis blip, or restarted while our start broadcast
+    // was lost) can re-register us instead of ignoring the heartbeat.
     const testRunnerQueue = await queueHandler.getQueue('testrunners');
     heartbeatTimer = setInterval(() => {
       testRunnerQueue
-        .add({ type: 'heartbeat', hostname: serverConfig.hostname })
+        .add(
+          {
+            type: 'heartbeat',
+            hostname: serverConfig.hostname,
+            serverConfig: serverConfig
+          },
+          // Heartbeats fire every 30s and carry the full serverConfig —
+          // without cleanup the completed jobs pile up in Redis forever.
+          { removeOnComplete: true, removeOnFail: true }
+        )
         .catch(error =>
           logger.error('Failed to publish heartbeat: %s', error.message)
         );