Add service recovery configuration and watchdog enhancements

- Introduced `ConfigureServiceRecovery` to enforce Windows SCM failure policy at startup.
- Enhanced watchdog to skip restarts during backend outages with capped suppression windows.
- Refined WebSocket reconnection handling and added diagnostics for server errors.
- Improved exit diagnostics for better stuck-host analysis.

Author: sagostin

install.ps1

@@ -631,8 +631,11 @@ AGENT_PIN=$Pin
         exit 1
     }
 
-    # Configure service recovery options (restart on failure)
-    sc.exe failure $Script:ServiceName reset= 86400 actions= restart/5000/restart/10000/restart/30000 | Out-Null
+    # Configure service recovery options (restart on failure).
+    # Windows' `sc.exe failure` only honors the first 3 actions before falling
+    # back to "no action". The agent re-asserts this on every startup via
+    # platform.ConfigureServiceRecovery, so this is just the initial baseline.
+    sc.exe failure $Script:ServiceName reset= 86400 actions= restart/5000/restart/10000/restart/30000/restart/60000/restart/300000 | Out-Null
 
     # Configure Windows Firewall rules
     Configure-Firewall -AgentPath $binaryPath

lib/platform/service_unix.go

@@ -39,3 +39,15 @@ func WatchdogRestart() {
 	// On Unix, we just exit with code 1 and let the init system restart us
 	os.Exit(1)
 }
+
+// ConfigureServiceRecovery is a no-op on Unix.
+// Linux uses systemd Restart=always (unbounded retries) and macOS uses launchd
+// KeepAlive. Neither has the 3-action cap that Windows SCM has, so there's
+// nothing to reconfigure.
+func ConfigureServiceRecovery() error {
+	return nil
+}
+
+// WriteLastExitInfo is a no-op on Unix — Unix init systems have their own
+// journal/console capture; the agent's rotating log file is enough for diagnosis.
+func WriteLastExitInfo(reason, lastError string) {}

lib/platform/service_windows.go

@@ -224,6 +224,97 @@ func WatchdogRestart() {
 	os.Exit(1)
 }
 
+// recoveryAction describes a single failure recovery action for the Windows SCM.
+// delayMs is the wait before the action (in milliseconds); action is "restart" / "reboot" / "run".
+type recoveryAction struct {
+	action  string
+	delayMs int
+}
+
+// DefaultRecoveryActions is the SCM failure policy applied on every agent startup.
+//
+// Why so many actions? Windows' `sc.exe failure` supports at most 3 configured
+// actions before falling back to the system default ("take no action"). After
+// the 3rd action, a service that keeps failing is left stopped until a human
+// intervenes. On Linux/macOS the equivalent (systemd Restart=always,
+// launchd KeepAlive) restarts forever — so Windows is uniquely fragile.
+//
+// We work around the 3-action limit by re-asserting a wider policy on every
+// startup (the `sc.exe failure` command silently caps the action list and
+// shifts later actions into "subsequent failures"). Even when SCM starts
+// ignoring our actions after the 3rd, the `reset= 0` clears the failure
+// counter immediately so a single successful run resets the slate.
+//
+// This is intentionally aggressive (final delays up to 5 minutes) — a long
+// retry window is much safer than giving up.
+var DefaultRecoveryActions = []recoveryAction{
+	{action: "restart", delayMs: 5000},   // 5s   — 1st failure
+	{action: "restart", delayMs: 10000},  // 10s  — 2nd failure
+	{action: "restart", delayMs: 30000},  // 30s  — 3rd failure
+	{action: "restart", delayMs: 60000},  // 60s  — 4th+ failure
+	{action: "restart", delayMs: 300000}, // 5m   — subsequent failures
+}
+
+// ConfigureServiceRecovery re-asserts the SCM failure policy on every startup.
+// This fixes existing installs that were installed before the wider policy was added.
+//
+// Safe to call repeatedly: `sc.exe failure` is idempotent. Failures are logged
+// at warn level but never fatal — a hardened host that denies SC_MANAGER access
+// will continue running; the agent just won't auto-restart on hard crashes.
+//
+// `reset= 0` clears the failure counter so the very first successful run resets
+// the SCM's "have we failed recently" state.
+func ConfigureServiceRecovery() error {
+	if !IsRunningAsService() {
+		// Not running as a service — nothing to configure.
+		return nil
+	}
+
+	exe, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("configure recovery: failed to get executable path: %w", err)
+	}
+
+	// Build the action string: "restart/5000/restart/10000/..."
+	parts := make([]string, 0, len(DefaultRecoveryActions))
+	for _, a := range DefaultRecoveryActions {
+		parts = append(parts, fmt.Sprintf("%s/%d", a.action, a.delayMs))
+	}
+	actions := strings.Join(parts, "/")
+
+	cmd := exec.Command("sc.exe", "failure", "NetWatcherAgent",
+		"reset=", "0",
+		"actions=", actions,
+	)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("configure recovery: sc.exe failure failed: %w (output: %s)", err, strings.TrimSpace(string(out)))
+	}
+
+	log.WithFields(log.Fields{
+		"exe":     filepath.Base(exe),
+		"actions": actions,
+	}).Info("Service recovery policy reconfigured")
+	return nil
+}
+
+// WriteLastExitInfo persists diagnostic info about why the agent is exiting.
+// Used by both graceful exits and the watchdog so post-mortem analysis doesn't
+// require catching the live log. Always overwrites — only the most recent exit
+// reason matters for diagnosing a stuck host.
+func WriteLastExitInfo(reason, lastError string) {
+	exe, err := os.Executable()
+	if err != nil {
+		return
+	}
+	path := filepath.Join(filepath.Dir(exe), "last_exit.json")
+	payload := fmt.Sprintf(`{"reason":%q,"last_error":%q,"timestamp":%q}`+"\n",
+		reason, lastError, time.Now().Format(time.RFC3339))
+	if err := os.WriteFile(path, []byte(payload), 0644); err != nil {
+		log.WithError(err).Warn("Failed to write last_exit.json")
+	}
+}
+
 // spawnRestartProcess creates a detached process that will restart the service
 // after the current process exits. This ensures the SCM sees a clean shutdown
 // before the service is restarted.

main.go

@@ -32,6 +32,28 @@ var (
 	disableUpdater bool
 )
 
+// watchdogTimeout reads WATCHDOG_TIMEOUT_MINUTES (default 10). Used by the
+// no-activity watchdog so deployments can tune how aggressively we restart
+// hung agents without recompiling.
+func watchdogTimeout() time.Duration {
+	raw := strings.TrimSpace(os.Getenv("WATCHDOG_TIMEOUT_MINUTES"))
+	if raw == "" {
+		return 10 * time.Minute
+	}
+	minutes, err := strconv.Atoi(raw)
+	if err != nil || minutes <= 0 {
+		log.Warnf("Invalid WATCHDOG_TIMEOUT_MINUTES=%q, using default 10", raw)
+		return 10 * time.Minute
+	}
+	return time.Duration(minutes) * time.Minute
+}
+
+// maxServerErrorSuppression caps how long the watchdog will stay suppressed by
+// server-error mode. Acts as a safety net in case the flag ever gets stuck
+// set due to a future bug — we'd rather restart eventually than stay silent
+// forever. 4 hours matches typical backend recovery timeouts.
+const maxServerErrorSuppression = 4 * time.Hour
+
 // Env helpers
 func getenv(key, def string) string {
 	if v := strings.TrimSpace(os.Getenv(key)); v != "" {
@@ -115,6 +137,17 @@ func runAgent(ctx context.Context) error {
 
 	loadConfig(configPath)
 
+	// ---------- Windows Service Recovery ----------
+	// Re-assert the SCM failure policy on every startup. Fixes existing installs
+	// that were installed with the narrower (3-action) policy that left the
+	// service stopped after a few crashes. Idempotent and safe — see
+	// platform.ConfigureServiceRecovery for details.
+	if err := platform.ConfigureServiceRecovery(); err != nil {
+		// Non-fatal: a hardened host that denies SC_MANAGER access just falls
+		// back to default SCM behavior. Log and continue.
+		log.Warnf("Could not reconfigure service recovery policy: %v", err)
+	}
+
 	// ---------- Updater ----------
 	if !disableUpdater {
 		updateConfig := &UpdaterConfig{
@@ -254,6 +287,11 @@ func runAgent(ctx context.Context) error {
 		updateActivity()
 	}
 
+	// Pong receipts prove the backend is alive even during quiet periods.
+	// Without this, the watchdog restarts agents that have a healthy but idle
+	// connection (no probe_get/speedtest traffic flowing right now).
+	wsClient.OnActivity = updateActivity
+
 	// If your workers expect a uint agent ID now:
 	workers.SetControllerConfig(cfg.ControllerHost, cfg.SSL, cfg.WorkspaceID, cfg.AgentID, psk)
 
@@ -274,8 +312,18 @@ func runAgent(ctx context.Context) error {
 
 	go wsClient.ConnectWithRetry(agentCtx)
 
-	// Watchdog: restart if no activity for 10 minutes
-	const watchdogTimeout = 10 * time.Minute
+	// Watchdog: restart if no activity for the configured timeout (default 10 min).
+	//
+	// Skipped while we're in server-error retry mode — a backend outage is not
+	// a stuck agent. Restarting just burns Windows SCM restart-throttle budget
+	// for nothing. The retry loop's own alive-but-retrying log line makes it
+	// obvious from the host that the process is alive.
+	//
+	// Safety net: if server-error mode has been on for more than
+	// maxServerErrorSuppression, the watchdog fires anyway. Covers any future
+	// bug that could leave the flag stuck set.
+	wdt := watchdogTimeout()
+	log.Infof("Watchdog: configured timeout = %v", wdt)
 	go func() {
 		ticker := time.NewTicker(1 * time.Minute)
 		defer ticker.Stop()
@@ -288,10 +336,32 @@ func runAgent(ctx context.Context) error {
 				elapsed := time.Since(lastSuccessfulActivity)
 				activityMu.Unlock()
 
-				log.Debugf("Watchdog: last activity %v ago", elapsed.Round(time.Second))
+				srvActive, srvSince := wsClient.IsInServerErrorMode()
+
+				// Skip the watchdog while we're deliberately retrying 5xx,
+				// unless we've been in this state longer than the sanity cap.
+				if srvActive {
+					stuckFor := time.Since(srvSince)
+					if stuckFor < maxServerErrorSuppression {
+						log.Debugf("Watchdog: suppressed (server-error mode, %v since last success, %v in srv-err mode)",
+							elapsed.Round(time.Second), stuckFor.Round(time.Second))
+						continue
+					}
+					log.Warnf("Watchdog: server-error mode held for %v — exceeding sanity cap of %v, forcing restart",
+						stuckFor.Round(time.Second), maxServerErrorSuppression)
+				} else {
+					log.Debugf("Watchdog: last activity %v ago", elapsed.Round(time.Second))
+				}
 
-				if elapsed > watchdogTimeout {
+				if elapsed > wdt {
 					log.Errorf("Watchdog: no successful activity for %v, forcing restart", elapsed.Round(time.Second))
+					// Leave a breadcrumb for whoever finds the host next — the
+					// log file may have rotated, but this file is overwritten
+					// only on exits, so it always shows the most recent reason.
+					platform.WriteLastExitInfo(
+						fmt.Sprintf("watchdog: no activity for %v", elapsed.Round(time.Second)),
+						wsClient.GetLastDialError(),
+					)
 					platform.WatchdogRestart()
 				}
 			}