From 5d9ebe1930acf254fca583c855680c214b003e46 Mon Sep 17 00:00:00 2001 From: Mathih13 Date: Wed, 7 Oct 2026 07:05:27 +0200 Subject: [PATCH] fix(ops): retain cleanup success while pruning runs --- deploy/lyracore-disk-guard.py | 28 ++++++++++--- deploy/test_disk_policy.py | 72 ++++++++++++++++++++++++++++++++++ docs/operations/disk-policy.md | 4 ++ 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/deploy/lyracore-disk-guard.py b/deploy/lyracore-disk-guard.py index 5acbe7eb8..c3fc09dea 100644 --- a/deploy/lyracore-disk-guard.py +++ b/deploy/lyracore-disk-guard.py @@ -98,6 +98,22 @@ def require_recent_prune(status, result, age): return healthy +def check_prune(status, previous, properties, now, boot_id): + result = properties.get('Result', 'unknown') + finished = int(properties.get('ExecMainExitTimestampMonotonic', '0')) / 1_000_000 + if not finished and result == 'success' and properties.get('ActiveState') == 'activating': + # A new oneshot clears its exit timestamp. Retain the last observed success, + # with its original deadline, only within the same boot. + if previous.get('boot_id') == boot_id: + finished = previous.get('prune_success_monotonic', 0) + if (isinstance(finished, bool) or not isinstance(finished, (int, float)) + or not math.isfinite(finished) or not 0 < finished <= now): + finished = 0 + status['boot_id'] = boot_id + status['prune_success_monotonic'] = finished if result == 'success' else 0 + return require_recent_prune(status, result, now - finished if finished else None) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('config', type=Path) @@ -112,13 +128,13 @@ def main(): previous = json.loads(path.read_text()) if path.exists() else {} status = sample(config, previous, time.time(), shutil.disk_usage(config['data_directory']).free, args.resume) prune = subprocess.run(['systemctl', 'show', 'spacetimedb-prune.service', - '--property=Result', '--property=ExecMainExitTimestampMonotonic'], + '--property=Result', '--property=ActiveState', + '--property=ExecMainExitTimestampMonotonic'], capture_output=True, text=True, timeout=5) - properties = dict(line.split('=', 1) for line in prune.stdout.splitlines() if '=' in line) - finished = int(properties.get('ExecMainExitTimestampMonotonic', '0')) / 1_000_000 - prune_healthy = require_recent_prune(status, - properties.get('Result', 'unknown') if prune.returncode == 0 else 'unknown', - time.monotonic() - finished if finished else None) + properties = (dict(line.split('=', 1) for line in prune.stdout.splitlines() if '=' in line) + if prune.returncode == 0 else {}) + prune_healthy = check_prune(status, previous, properties, time.monotonic(), + Path('/proc/sys/kernel/random/boot_id').read_text().strip()) # Persist the latch before network calls, so a partial update cannot resume bots later. atomic_json(path, status) failures = renew(config, status) diff --git a/deploy/test_disk_policy.py b/deploy/test_disk_policy.py index 657220981..3aa189c11 100644 --- a/deploy/test_disk_policy.py +++ b/deploy/test_disk_policy.py @@ -1,12 +1,16 @@ """Exercise the disk policy at its filesystem and SpacetimeDB process boundaries.""" import importlib.util +import contextlib +import io import json from pathlib import Path import subprocess import sys import tempfile import time +from types import SimpleNamespace import unittest +from unittest.mock import patch def module(name): @@ -21,6 +25,74 @@ def module(name): class DiskPolicyTests(unittest.TestCase): + def test_cleanup_start_does_not_revoke_a_recently_verified_lease(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + config = root / 'config.json' + config.write_text(json.dumps({ + 'data_directory': directory, 'state_directory': directory, + 'spacetime': '/pinned/cli', 'databases': ['world'], + })) + (root / 'status.json').write_text(json.dumps({'state': 'healthy'})) + cleanup = {'result': 'success', 'state': 'inactive', 'finished': 900_000_000} + leases = [] + + def run(args, **kwargs): + if args[0] == 'systemctl': + output = (f"Result={cleanup['result']}\nActiveState={cleanup['state']}\n" + f"ExecMainExitTimestampMonotonic={cleanup['finished']}\n") + return subprocess.CompletedProcess(args, 0, stdout=output) + leases.append(json.loads(args[-2])) + return subprocess.CompletedProcess(args, 0) + + # systemd clears the current exit timestamp while its next oneshot is running. + renew = guard.renew + with patch.object(sys, 'argv', ['guard', str(config)]), \ + patch.object(guard.subprocess, 'run', side_effect=run), \ + patch.object(guard, 'renew', side_effect=lambda config, status: renew(config, status, run)), \ + patch.object(guard.shutil, 'disk_usage', return_value=SimpleNamespace(free=50 * guard.GIB)), \ + patch.object(guard.time, 'monotonic', return_value=1000), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(guard.main(), 0) + cleanup.update(state='activating', finished=0) + self.assertEqual(guard.main(), 0) + self.assertTrue(all(int(value) > 0 for value in leases), leases) + status = json.loads((root / 'status.json').read_text()) + self.assertEqual(status['state'], 'healthy') + self.assertEqual(status['prune_age_seconds'], 100) + + def test_running_cleanup_cannot_extend_the_last_success_deadline(self): + previous = {'boot_id': 'boot-one', 'prune_success_monotonic': 100} + properties = {'Result': 'success', 'ActiveState': 'activating', + 'ExecMainExitTimestampMonotonic': '0'} + for now in [1000, 1800, 1900]: + status = {'state': 'healthy', 'lease_until_micros': 300_000_000} + self.assertTrue(guard.check_prune(status, previous, properties, now, 'boot-one')) + previous = status + self.assertFalse(guard.check_prune(status, previous, properties, 1901, 'boot-one')) + self.assertEqual(status['lease_until_micros'], 0) + + def test_cleanup_failure_or_untrusted_history_cannot_renew_capacity(self): + cases = [ + ('exit-code', 'failed', 'boot-one', 100), + ('unknown', 'activating', 'boot-one', 100), + ('success', 'inactive', 'boot-one', 100), + ('success', 'activating', 'previous-boot', 100), + ('success', 'activating', 'boot-one', 1001), + ('success', 'activating', 'boot-one', 'broken'), + ('success', 'activating', 'boot-one', True), + ('success', 'activating', 'boot-one', float('nan')), + ('success', 'activating', 'boot-one', 0), + ] + for result, active, boot, finished in cases: + with self.subTest(result=result, active=active, boot=boot, finished=finished): + previous = {'boot_id': boot, 'prune_success_monotonic': finished} + properties = {'Result': result, 'ActiveState': active, + 'ExecMainExitTimestampMonotonic': '0'} + status = {'state': 'healthy', 'lease_until_micros': 300_000_000} + self.assertFalse(guard.check_prune(status, previous, properties, 1000, 'boot-one')) + self.assertEqual(status['lease_until_micros'], 0) + def test_reserve_suspends_and_recovery_requires_explicit_resume(self): self.assertEqual(guard.decide(19 * guard.GIB, False), 'suspended') self.assertEqual(guard.decide(50 * guard.GIB, True), 'suspended') diff --git a/docs/operations/disk-policy.md b/docs/operations/disk-policy.md index 417d6a81f..91e4df605 100644 --- a/docs/operations/disk-policy.md +++ b/docs/operations/disk-policy.md @@ -10,6 +10,10 @@ failures in `/var/lib/lyracore-disk-guard/status.json`. Failed or overdue prunin capacity and produces a failed monitor service. Journal priorities make warnings and errors visible to host monitoring; these units do not send email or chat notifications. +While cleanup runs, the monitor uses its last observed successful completion from the same boot. +Starting cleanup does not extend that completion's thirty-minute deadline. Missing history, a +failed cleanup, or an overdue completion revokes capacity even if cleanup is still running. + Each successful sample renews a three-minute Bot Capacity Lease on every configured Shard through `set_package_config`. Playerbots must include the Package capacity check before enabling this monitor. Expiry refuses new bots and controller activation; the Runner freezes existing bots