feat(task): add structured validation reports

CLAUDE.md

@@ -18,6 +18,7 @@ Key concepts:
 |-----------|---------|
 | `coral/types.py` | Core types: `Task`, `Score`, `ScoreBundle`, `Attempt` |
 | `coral/config.py` | OmegaConf-backed YAML configuration (`CoralConfig`, `GraderConfig`, `AgentConfig`, `GatewayConfig`, `WarmStartConfig`, `HeartbeatActionConfig`, ...) |
+| `coral/task/` | Frontend-independent task validation with structured reports and diagnostics |
 | `coral/agent/` | Agent lifecycle: `manager.py` (multi-agent supervisor), `runtime.py` (abstract), `state.py`, `heartbeat.py`, `exit_classifier.py`, `warmstart.py`, `process.py`, `registry.py` |
 | `coral/sandbox/` | Pluggable agent sandboxing (`agents.sandbox`): `protocol.py` (`SandboxProvider` + spec/context types), `registry.py` (name or `module:Class` entrypoint resolution), `srt.py` (built-in srt provider: OS-level FS/network enforcement + allow-all proxy) |
 | `coral/agent/builtin/` | Concrete runtimes: `claude_code`, `codex`, `cursor_agent`, `kiro`, `opencode` |

coral/cli/validation.py

@@ -7,67 +7,9 @@
 
 from pathlib import Path
 
-from coral.config import CoralConfig
+from coral.task.validation import validate_task as validate_task_report
 
 
 def validate_task(task_dir: Path) -> list[str]:
     """Validate a task directory. Returns a list of error strings (empty = valid)."""
-    errors: list[str] = []
-
-    # 1. task.yaml exists and parses
-    task_yaml = task_dir / "task.yaml"
-    if not task_yaml.exists():
-        errors.append(f"task.yaml not found in {task_dir}")
-        return errors  # Can't continue without config
-
-    try:
-        config = CoralConfig.from_yaml(task_yaml)
-    except Exception as e:
-        errors.append(f"task.yaml parse error: {e}")
-        return errors
-
-    # 2. grader.entrypoint is set and well-formed.
-    if not config.grader.entrypoint:
-        errors.append(
-            "No grader configured. Set grader.entrypoint = "
-            "'your_pkg.module:Grader' in task.yaml and grader.setup to "
-            "install the package."
-        )
-    elif ":" not in config.grader.entrypoint:
-        errors.append(
-            f"grader.entrypoint must be 'module.path:ClassName', got {config.grader.entrypoint!r}"
-        )
-
-    # 3. direction is valid
-    if config.grader.direction not in ("maximize", "minimize"):
-        errors.append(
-            f"grader.direction must be 'maximize' or 'minimize', got '{config.grader.direction}'"
-        )
-
-    # 4. Extra private files exist, and are NOT inside the grader package.
-    # The grader package (task_dir/grader) is surfaced read-only to agents at
-    # <shared_dir>/grader/, so a grader.private path living inside it would be
-    # both copied to .coral/private/ AND exposed via the surfaced source — a
-    # leak. Hidden inputs must sit outside the grader dir (e.g. a sibling
-    # ``taskdata/`` declared as ``taskdata``).
-    grader_dir = (task_dir / "grader").resolve()
-    for private_path in config.grader.private:
-        p = Path(private_path)
-        if not p.is_absolute():
-            p = task_dir / p
-        if not p.exists():
-            errors.append(f"Private file not found: {private_path}")
-            continue
-        try:
-            p.resolve().relative_to(grader_dir)
-        except ValueError:
-            pass  # outside the grader package — good
-        else:
-            errors.append(
-                f"grader.private path '{private_path}' is inside the grader package "
-                f"(grader/), which is surfaced read-only to agents at "
-                f"<shared_dir>/grader/ — this would leak it. Move it outside grader/ "
-                f"(e.g. a sibling 'taskdata/')."
-            )
-
-    return errors
+    return validate_task_report(task_dir).error_messages

coral/task/__init__.py

@@ -0,0 +1,5 @@
+"""Task inspection and validation APIs."""
+
+from coral.task.validation import ValidationDiagnostic, ValidationReport, validate_task
+
+__all__ = ["ValidationDiagnostic", "ValidationReport", "validate_task"]

coral/task/validation.py

@@ -0,0 +1,156 @@
+"""Structured validation for CORAL task directories."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from coral.config import CoralConfig
+
+
+@dataclass(frozen=True)
+class ValidationDiagnostic:
+    """A machine-readable problem found in a task directory."""
+
+    code: str
+    message: str
+    path: str | None = None
+    severity: Literal["error", "warning"] = "error"
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "code": self.code,
+            "message": self.message,
+            "severity": self.severity,
+        }
+        if self.path is not None:
+            data["path"] = self.path
+        return data
+
+
+@dataclass(frozen=True)
+class ValidationReport:
+    """Structured result of validating one task directory."""
+
+    task_dir: Path
+    diagnostics: tuple[ValidationDiagnostic, ...]
+
+    @property
+    def valid(self) -> bool:
+        return all(diagnostic.severity != "error" for diagnostic in self.diagnostics)
+
+    @property
+    def error_messages(self) -> list[str]:
+        """Return the legacy error representation used by the CLI."""
+        return [
+            diagnostic.message for diagnostic in self.diagnostics if diagnostic.severity == "error"
+        ]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "task_dir": str(self.task_dir),
+            "valid": self.valid,
+            "diagnostics": [diagnostic.to_dict() for diagnostic in self.diagnostics],
+        }
+
+
+def validate_task(task_dir: Path) -> ValidationReport:
+    """Validate a task directory and return structured diagnostics."""
+    diagnostics: list[ValidationDiagnostic] = []
+
+    task_yaml = task_dir / "task.yaml"
+    if not task_yaml.exists():
+        diagnostics.append(
+            ValidationDiagnostic(
+                code="task.config.missing",
+                message=f"task.yaml not found in {task_dir}",
+                path="task.yaml",
+            )
+        )
+        return ValidationReport(task_dir, tuple(diagnostics))
+
+    try:
+        config = CoralConfig.from_yaml(task_yaml)
+    except Exception as exc:
+        diagnostics.append(
+            ValidationDiagnostic(
+                code="task.config.invalid",
+                message=f"task.yaml parse error: {exc}",
+                path="task.yaml",
+            )
+        )
+        return ValidationReport(task_dir, tuple(diagnostics))
+
+    if not config.grader.entrypoint:
+        diagnostics.append(
+            ValidationDiagnostic(
+                code="grader.entrypoint.missing",
+                message=(
+                    "No grader configured. Set grader.entrypoint = "
+                    "'your_pkg.module:Grader' in task.yaml and grader.setup to "
+                    "install the package."
+                ),
+                path="task.yaml",
+            )
+        )
+    elif ":" not in config.grader.entrypoint:
+        diagnostics.append(
+            ValidationDiagnostic(
+                code="grader.entrypoint.invalid",
+                message=(
+                    "grader.entrypoint must be 'module.path:ClassName', "
+                    f"got {config.grader.entrypoint!r}"
+                ),
+                path="task.yaml",
+            )
+        )
+
+    if config.grader.direction not in ("maximize", "minimize"):
+        diagnostics.append(
+            ValidationDiagnostic(
+                code="grader.direction.invalid",
+                message=(
+                    "grader.direction must be 'maximize' or 'minimize', "
+                    f"got '{config.grader.direction}'"
+                ),
+                path="task.yaml",
+            )
+        )
+
+    # The grader package is surfaced read-only to agents at <shared_dir>/grader/.
+    # Private paths inside it would therefore be copied into .coral/private/ and
+    # exposed through the surfaced source at the same time.
+    grader_dir = (task_dir / "grader").resolve()
+    for private_path in config.grader.private:
+        path = Path(private_path)
+        if not path.is_absolute():
+            path = task_dir / path
+        if not path.exists():
+            diagnostics.append(
+                ValidationDiagnostic(
+                    code="grader.private.missing",
+                    message=f"Private file not found: {private_path}",
+                    path=str(private_path),
+                )
+            )
+            continue
+        try:
+            path.resolve().relative_to(grader_dir)
+        except ValueError:
+            pass
+        else:
+            diagnostics.append(
+                ValidationDiagnostic(
+                    code="grader.private.exposed",
+                    message=(
+                        f"grader.private path '{private_path}' is inside the grader package "
+                        "(grader/), which is surfaced read-only to agents at "
+                        "<shared_dir>/grader/ — this would leak it. Move it outside grader/ "
+                        "(e.g. a sibling 'taskdata/')."
+                    ),
+                    path=str(private_path),
+                )
+            )
+
+    return ValidationReport(task_dir, tuple(diagnostics))

tests/test_validation.py

@@ -6,6 +6,8 @@
 from pathlib import Path
 
 from coral.cli.validation import validate_task
+from coral.task import validation as task_validation
+from coral.task.validation import ValidationDiagnostic, ValidationReport
 
 _TASK_YAML = """\
 task:
@@ -38,6 +40,44 @@ def test_validate_rejects_missing_entrypoint():
         assert any("No grader configured" in e for e in errors)
 
 
+def test_structured_validation_report_is_serializable():
+    with tempfile.TemporaryDirectory() as d:
+        task_dir = _make_task(Path(d), "  timeout: 60")
+
+        report = task_validation.validate_task(task_dir)
+
+        assert not report.valid
+        assert report.error_messages == validate_task(task_dir)
+        assert report.to_dict() == {
+            "task_dir": str(task_dir),
+            "valid": False,
+            "diagnostics": [
+                {
+                    "code": "grader.entrypoint.missing",
+                    "message": report.error_messages[0],
+                    "path": "task.yaml",
+                    "severity": "error",
+                }
+            ],
+        }
+
+
+def test_warning_diagnostic_does_not_fail_report():
+    report = ValidationReport(
+        task_dir=Path("task"),
+        diagnostics=(
+            ValidationDiagnostic(
+                code="task.example.warning",
+                message="Example warning",
+                severity="warning",
+            ),
+        ),
+    )
+
+    assert report.valid
+    assert report.error_messages == []
+
+
 def test_validate_rejects_malformed_entrypoint():
     with tempfile.TemporaryDirectory() as d:
         task_dir = _make_task(Path(d), "  entrypoint: my_pkg.grader.Grader")
@@ -55,6 +95,17 @@ def _make_task_with_dirs(base: Path, grader_body: str, dirs: list[str]) -> Path:
     return task_dir
 
 
+def test_structured_validation_reports_private_path():
+    body = '  entrypoint: "p.g:G"\n  private:\n    - "missing-data"'
+    with tempfile.TemporaryDirectory() as d:
+        task_dir = _make_task_with_dirs(Path(d), body, [])
+
+        report = task_validation.validate_task(task_dir)
+
+        assert [diagnostic.code for diagnostic in report.diagnostics] == ["grader.private.missing"]
+        assert report.diagnostics[0].path == "missing-data"
+
+
 def test_validate_accepts_private_sibling_of_grader():
     """The common, safe layout: hidden data beside grader/ (e.g. taskdata/)."""
     body = '  entrypoint: "p.g:G"\n  private:\n    - "taskdata"'