fix(ops): keep bot capacity during successful cleanup

deploy/lyracore-disk-guard.py

@@ -98,6 +98,22 @@ def require_recent_prune(status, result, age):
     return healthy
 
 
+def check_prune(status, previous, properties, now, boot_id):
+    result = properties.get('Result', 'unknown')
+    finished = int(properties.get('ExecMainExitTimestampMonotonic', '0')) / 1_000_000
+    if not finished and result == 'success' and properties.get('ActiveState') == 'activating':
+        # A new oneshot clears its exit timestamp. Retain the last observed success,
+        # with its original deadline, only within the same boot.
+        if previous.get('boot_id') == boot_id:
+            finished = previous.get('prune_success_monotonic', 0)
+    if (isinstance(finished, bool) or not isinstance(finished, (int, float))
+            or not math.isfinite(finished) or not 0 < finished <= now):
+        finished = 0
+    status['boot_id'] = boot_id
+    status['prune_success_monotonic'] = finished if result == 'success' else 0
+    return require_recent_prune(status, result, now - finished if finished else None)
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('config', type=Path)
@@ -112,13 +128,13 @@ def main():
         previous = json.loads(path.read_text()) if path.exists() else {}
         status = sample(config, previous, time.time(), shutil.disk_usage(config['data_directory']).free, args.resume)
         prune = subprocess.run(['systemctl', 'show', 'spacetimedb-prune.service',
-                                '--property=Result', '--property=ExecMainExitTimestampMonotonic'],
+                                '--property=Result', '--property=ActiveState',
+                                '--property=ExecMainExitTimestampMonotonic'],
                                capture_output=True, text=True, timeout=5)
-        properties = dict(line.split('=', 1) for line in prune.stdout.splitlines() if '=' in line)
-        finished = int(properties.get('ExecMainExitTimestampMonotonic', '0')) / 1_000_000
-        prune_healthy = require_recent_prune(status,
-            properties.get('Result', 'unknown') if prune.returncode == 0 else 'unknown',
-            time.monotonic() - finished if finished else None)
+        properties = (dict(line.split('=', 1) for line in prune.stdout.splitlines() if '=' in line)
+                      if prune.returncode == 0 else {})
+        prune_healthy = check_prune(status, previous, properties, time.monotonic(),
+                                   Path('/proc/sys/kernel/random/boot_id').read_text().strip())
         # Persist the latch before network calls, so a partial update cannot resume bots later.
         atomic_json(path, status)
         failures = renew(config, status)

deploy/test_disk_policy.py

@@ -1,12 +1,16 @@
 """Exercise the disk policy at its filesystem and SpacetimeDB process boundaries."""
 import importlib.util
+import contextlib
+import io
 import json
 from pathlib import Path
 import subprocess
 import sys
 import tempfile
 import time
+from types import SimpleNamespace
 import unittest
+from unittest.mock import patch
 
 
 def module(name):
@@ -21,6 +25,74 @@ def module(name):
 
 
 class DiskPolicyTests(unittest.TestCase):
+    def test_cleanup_start_does_not_revoke_a_recently_verified_lease(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            config = root / 'config.json'
+            config.write_text(json.dumps({
+                'data_directory': directory, 'state_directory': directory,
+                'spacetime': '/pinned/cli', 'databases': ['world'],
+            }))
+            (root / 'status.json').write_text(json.dumps({'state': 'healthy'}))
+            cleanup = {'result': 'success', 'state': 'inactive', 'finished': 900_000_000}
+            leases = []
+
+            def run(args, **kwargs):
+                if args[0] == 'systemctl':
+                    output = (f"Result={cleanup['result']}\nActiveState={cleanup['state']}\n"
+                              f"ExecMainExitTimestampMonotonic={cleanup['finished']}\n")
+                    return subprocess.CompletedProcess(args, 0, stdout=output)
+                leases.append(json.loads(args[-2]))
+                return subprocess.CompletedProcess(args, 0)
+
+            # systemd clears the current exit timestamp while its next oneshot is running.
+            renew = guard.renew
+            with patch.object(sys, 'argv', ['guard', str(config)]), \
+                    patch.object(guard.subprocess, 'run', side_effect=run), \
+                    patch.object(guard, 'renew', side_effect=lambda config, status: renew(config, status, run)), \
+                    patch.object(guard.shutil, 'disk_usage', return_value=SimpleNamespace(free=50 * guard.GIB)), \
+                    patch.object(guard.time, 'monotonic', return_value=1000), \
+                    contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(guard.main(), 0)
+                cleanup.update(state='activating', finished=0)
+                self.assertEqual(guard.main(), 0)
+                self.assertTrue(all(int(value) > 0 for value in leases), leases)
+                status = json.loads((root / 'status.json').read_text())
+                self.assertEqual(status['state'], 'healthy')
+                self.assertEqual(status['prune_age_seconds'], 100)
+
+    def test_running_cleanup_cannot_extend_the_last_success_deadline(self):
+        previous = {'boot_id': 'boot-one', 'prune_success_monotonic': 100}
+        properties = {'Result': 'success', 'ActiveState': 'activating',
+                      'ExecMainExitTimestampMonotonic': '0'}
+        for now in [1000, 1800, 1900]:
+            status = {'state': 'healthy', 'lease_until_micros': 300_000_000}
+            self.assertTrue(guard.check_prune(status, previous, properties, now, 'boot-one'))
+            previous = status
+        self.assertFalse(guard.check_prune(status, previous, properties, 1901, 'boot-one'))
+        self.assertEqual(status['lease_until_micros'], 0)
+
+    def test_cleanup_failure_or_untrusted_history_cannot_renew_capacity(self):
+        cases = [
+            ('exit-code', 'failed', 'boot-one', 100),
+            ('unknown', 'activating', 'boot-one', 100),
+            ('success', 'inactive', 'boot-one', 100),
+            ('success', 'activating', 'previous-boot', 100),
+            ('success', 'activating', 'boot-one', 1001),
+            ('success', 'activating', 'boot-one', 'broken'),
+            ('success', 'activating', 'boot-one', True),
+            ('success', 'activating', 'boot-one', float('nan')),
+            ('success', 'activating', 'boot-one', 0),
+        ]
+        for result, active, boot, finished in cases:
+            with self.subTest(result=result, active=active, boot=boot, finished=finished):
+                previous = {'boot_id': boot, 'prune_success_monotonic': finished}
+                properties = {'Result': result, 'ActiveState': active,
+                              'ExecMainExitTimestampMonotonic': '0'}
+                status = {'state': 'healthy', 'lease_until_micros': 300_000_000}
+                self.assertFalse(guard.check_prune(status, previous, properties, 1000, 'boot-one'))
+                self.assertEqual(status['lease_until_micros'], 0)
+
     def test_reserve_suspends_and_recovery_requires_explicit_resume(self):
         self.assertEqual(guard.decide(19 * guard.GIB, False), 'suspended')
         self.assertEqual(guard.decide(50 * guard.GIB, True), 'suspended')

docs/operations/disk-policy.md

@@ -10,6 +10,10 @@ failures in `/var/lib/lyracore-disk-guard/status.json`. Failed or overdue prunin
 capacity and produces a failed monitor service. Journal priorities make warnings and errors visible to host
 monitoring; these units do not send email or chat notifications.
 
+While cleanup runs, the monitor uses its last observed successful completion from the same boot.
+Starting cleanup does not extend that completion's thirty-minute deadline. Missing history, a
+failed cleanup, or an overdue completion revokes capacity even if cleanup is still running.
+
 Each successful sample renews a three-minute Bot Capacity Lease on every configured Shard through
 `set_package_config`. Playerbots must include the Package capacity check before enabling this
 monitor. Expiry refuses new bots and controller activation; the Runner freezes existing bots