|
| 1 | +"""Per-FRID accounting for the two fix loops, and detection of a loop that is stuck. |
| 2 | +
|
| 3 | +Both loops — unit tests during implementation, conformance tests afterwards — patch, |
| 4 | +re-run the script, and repeat until a budget runs out. Neither noticed when an attempt |
| 5 | +changed nothing: benchmark renders spent twenty attempts rewriting one file against one |
| 6 | +unchanging assertion before abandoning the render. Two things were missing, and this |
| 7 | +module supplies both. |
| 8 | +
|
| 9 | +*Detection*: a failure is fingerprinted, and consecutive identical fingerprints for the |
| 10 | +same loop and FRID are counted. A streak means the loop is re-patching without effect, |
| 11 | +which is the moment worth reporting — not the exhaustion twenty attempts later. |
| 12 | +
|
| 13 | +*Measurement*: attempts and failures are counted per (module, FRID, loop), so a render |
| 14 | +reports how many iterations convergence took rather than only whether it eventually gave |
| 15 | +up. Exhaustion is a rare binary event and a poor basis for comparing configurations; |
| 16 | +iterations-to-convergence is close to continuous and says something after a single run. |
| 17 | +
|
| 18 | +Recording never affects rendering. These are observations. |
| 19 | +""" |
| 20 | + |
| 21 | +import hashlib |
| 22 | +import re |
| 23 | +from dataclasses import dataclass, field |
| 24 | +from typing import Dict, List, Optional, Tuple |
| 25 | + |
| 26 | +from plain2code_console import console |
| 27 | + |
| 28 | +UNIT_LOOP = "unit" |
| 29 | +CONFORMANCE_LOOP = "conformance" |
| 30 | + |
| 31 | +# Parts of a test script's output that differ between two runs of the very same failure. |
| 32 | +# Left in place, any one of them would make every attempt look novel and hide a stuck |
| 33 | +# loop; over-normalising would do the reverse and merge failures that differ for real, so |
| 34 | +# only demonstrably volatile tokens are erased. |
| 35 | +_VOLATILE_PATTERNS = ( |
| 36 | + re.compile(r"/tmp/[^\s'\"]+"), # renderer scratch paths: /tmp/tmpk8flk7f1.script_output |
| 37 | + re.compile(r"\b0x[0-9a-fA-F]+\b"), # memory addresses |
| 38 | + re.compile(r"\b[0-9a-fA-F]{8,}\b"), # hashes, uuids, run ids |
| 39 | + re.compile(r"\b\d+(?:\.\d+)?\s*m?s\b"), # durations: "1335.821531 ms", "22.5s" |
| 40 | + re.compile(r"duration_ms\s+[\d.]+"), |
| 41 | +) |
| 42 | + |
| 43 | + |
| 44 | +def failure_fingerprint(output: str) -> str: |
| 45 | + """A stable identity for one failure, insensitive to run-to-run noise.""" |
| 46 | + normalized = output or "" |
| 47 | + for pattern in _VOLATILE_PATTERNS: |
| 48 | + normalized = pattern.sub("", normalized) |
| 49 | + normalized = " ".join(normalized.split()) |
| 50 | + return hashlib.sha1(normalized.encode("utf-8", errors="replace")).hexdigest()[:12] |
| 51 | + |
| 52 | + |
| 53 | +@dataclass |
| 54 | +class _LoopCounters: |
| 55 | + attempts: int = 0 |
| 56 | + failures: int = 0 |
| 57 | + max_repeat: int = 1 |
| 58 | + last_fingerprint: Optional[str] = None |
| 59 | + current_repeat: int = 0 |
| 60 | + |
| 61 | + |
| 62 | +@dataclass |
| 63 | +class FixLoopMetrics: |
| 64 | + """One per render. Keyed by (module, frid) so a re-rendered FRID keeps accumulating.""" |
| 65 | + |
| 66 | + _counters: Dict[Tuple[str, str], Dict[str, _LoopCounters]] = field(default_factory=dict) |
| 67 | + _order: List[Tuple[str, str]] = field(default_factory=list) |
| 68 | + |
| 69 | + def record(self, loop: str, module: str, frid: str, passed: bool, output: str) -> Optional[int]: |
| 70 | + """Records one script run. Returns the streak length when this failure is a |
| 71 | + repeat of the one before it in the same loop, otherwise None.""" |
| 72 | + key = (module, str(frid)) |
| 73 | + if key not in self._counters: |
| 74 | + self._counters[key] = {} |
| 75 | + self._order.append(key) |
| 76 | + counters = self._counters[key].setdefault(loop, _LoopCounters()) |
| 77 | + |
| 78 | + counters.attempts += 1 |
| 79 | + if passed: |
| 80 | + counters.last_fingerprint = None |
| 81 | + counters.current_repeat = 0 |
| 82 | + return None |
| 83 | + |
| 84 | + counters.failures += 1 |
| 85 | + fingerprint = failure_fingerprint(output) |
| 86 | + if fingerprint == counters.last_fingerprint: |
| 87 | + counters.current_repeat += 1 |
| 88 | + counters.max_repeat = max(counters.max_repeat, counters.current_repeat) |
| 89 | + return counters.current_repeat |
| 90 | + |
| 91 | + counters.last_fingerprint = fingerprint |
| 92 | + counters.current_repeat = 1 |
| 93 | + return None |
| 94 | + |
| 95 | + def frid_summary(self, module: str, frid: str) -> Optional[str]: |
| 96 | + """One greppable line per FRID, or None if no script ran for it.""" |
| 97 | + counters = self._counters.get((module, str(frid))) |
| 98 | + if not counters: |
| 99 | + return None |
| 100 | + |
| 101 | + parts = [f"[fix-loop] module={module} frid={frid}"] |
| 102 | + for loop in (UNIT_LOOP, CONFORMANCE_LOOP): |
| 103 | + if loop in counters: |
| 104 | + parts.append(f"{loop}={counters[loop].attempts} {loop}_failed={counters[loop].failures}") |
| 105 | + parts.append(f"max_repeat={max(loop.max_repeat for loop in counters.values())}") |
| 106 | + return " ".join(parts) |
| 107 | + |
| 108 | + def render_summary(self) -> List[str]: |
| 109 | + """Every FRID that ran a test script, in the order it was first reached.""" |
| 110 | + summaries = (self.frid_summary(module, frid) for module, frid in self._order) |
| 111 | + return [summary for summary in summaries if summary] |
| 112 | + |
| 113 | + |
| 114 | +# How many identical failures in a row before the loop is called out. Two can happen when |
| 115 | +# a patch legitimately addresses something else first; by three the loop is re-patching |
| 116 | +# against a failure it is not moving. |
| 117 | +REPEATED_FAILURE_WARNING_THRESHOLD = 3 |
| 118 | + |
| 119 | + |
| 120 | +def report_fix_loop_attempt(render_context, loop: str, frid: Optional[str], passed: bool, output: str) -> None: |
| 121 | + """Records one script run and tells the user when the loop stops making progress.""" |
| 122 | + if frid is None: |
| 123 | + return |
| 124 | + |
| 125 | + streak = render_context.fix_loop_metrics.record( |
| 126 | + loop, module=render_context.module_name, frid=frid, passed=passed, output=output |
| 127 | + ) |
| 128 | + |
| 129 | + if streak is not None and streak >= REPEATED_FAILURE_WARNING_THRESHOLD: |
| 130 | + console.warning( |
| 131 | + f"The {loop} tests for functionality {frid} have failed the same way {streak} times in a row. " |
| 132 | + f"The last {streak - 1} fix attempts changed nothing that the tests can see." |
| 133 | + ) |
| 134 | + |
| 135 | + |
| 136 | +def report_frid_fix_loop_summary(render_context, frid: Optional[str]) -> None: |
| 137 | + """Emits the per-FRID counts once the FRID is done, successfully or not.""" |
| 138 | + if frid is None: |
| 139 | + return |
| 140 | + |
| 141 | + summary = render_context.fix_loop_metrics.frid_summary(render_context.module_name, frid) |
| 142 | + if summary: |
| 143 | + console.info(summary) |
0 commit comments