From f28d722dd5376b87ece5ef4390496a62024c4174 Mon Sep 17 00:00:00 2001 From: BoggersTheFish Date: Wed, 22 Jul 2026 01:13:16 +0100 Subject: [PATCH 1/2] feat: import adversarial topology benchmark --- .../ADVERSARIAL_TOPOLOGY_GENERALIZATION.md | 65 +++ docs/migration/import-ledger.md | 2 +- docs/migration/sources/TS-Benchmarks.md | 24 ++ .../adversarial_topology_generalization.py | 391 ++++++++++++++++++ .../tasks/adversarial_topology.py | 266 ++++++++++++ .../test_adversarial_topology.py | 91 ++++ 6 files changed, 838 insertions(+), 1 deletion(-) create mode 100644 docs/benchmarks/graph_scaling/ADVERSARIAL_TOPOLOGY_GENERALIZATION.md create mode 100644 src/thinking_system/benchmarks/graph_scaling/runners/adversarial_topology_generalization.py create mode 100644 src/thinking_system/benchmarks/graph_scaling/tasks/adversarial_topology.py create mode 100644 tests/benchmarks/graph_scaling/test_adversarial_topology.py diff --git a/docs/benchmarks/graph_scaling/ADVERSARIAL_TOPOLOGY_GENERALIZATION.md b/docs/benchmarks/graph_scaling/ADVERSARIAL_TOPOLOGY_GENERALIZATION.md new file mode 100644 index 0000000..a43162c --- /dev/null +++ b/docs/benchmarks/graph_scaling/ADVERSARIAL_TOPOLOGY_GENERALIZATION.md @@ -0,0 +1,65 @@ +# Adversarial Topology Generalization + +This is the v0.5 experimental task for TS-Benchmarks. + +## Framing + +v0.4 passed on known graph families. v0.5 tries to falsify that result by testing mixed, near-boundary, and contradiction-placement-stressed graphs. The goal is not to prove universal scaling; the goal is to measure where topology-aware policy selection generalizes and where it fails. + +## Graph Families + +- `mixed_core_periphery`: scale-free hub core, small-world/random periphery, sparse bridge edges. +- `hub_threshold_sweep`: gradually varied hub concentration near selector boundary. +- `topology_noise_sweep`: known topology perturbed by edge rewiring/noise. + +## Contradiction Placements + +- `hub_hub` +- `hub_leaf` +- `leaf_leaf` +- `random` + +## Oracle Comparison + +- `selector_policy`: chosen from pre-run topology diagnostics only. +- `oracle_best_policy`: chosen after seeing policy outcomes. + +The receipt reports both, but the selector must never use oracle information. + +## Minimum Metrics + +- `selector_policy` +- `selector_reason` +- `oracle_best_policy` +- `selector_matches_oracle` +- `selector_regret_final_tension` +- `selector_regret_f1` +- `catastrophic_regression` +- `boundary_case` +- `contradiction_placement` +- `topology_family` + +## Command + +```bash +python -m thinking_system.benchmarks.graph_scaling.runners.adversarial_topology_generalization \ + --sizes 100,1000 \ + --families mixed_core_periphery,hub_threshold_sweep,topology_noise_sweep \ + --placements hub_hub,hub_leaf,leaf_leaf,random \ + --hub-strengths 0.25,0.35,0.45,0.60,0.75 \ + --seed 42 \ + --out-dir artifacts/adversarial-topology +``` + +## Strict Success Criteria + +- selector beats or matches always-reference on aggregate +- selector beats or matches always-degree-normalized on aggregate +- no catastrophic family-level regression +- receipts include oracle-best-policy comparison +- selector uses topology diagnostics only, not oracle labels or final outcome metrics +- boundary cases are documented honestly + +## Claim Boundary + +> v0.5 is an adversarial synthetic selector stress test. It does not prove TS-Core scales to real knowledge/provenance graphs. diff --git a/docs/migration/import-ledger.md b/docs/migration/import-ledger.md index 5894cc6..d129719 100644 --- a/docs/migration/import-ledger.md +++ b/docs/migration/import-ledger.md @@ -53,7 +53,7 @@ Empty placeholder directories (no tracked source): `packages/ts-*`, top-level `e |------------|-------------------------------------|----------------------------------|--------|-------| | TS-Core | `3ef48ad00efef8659ad0981d71de509a9827f584` | `src/thinking_system/core/typed_tension/` | **PARTIALLY_IMPORTED** | Domain-neutral typed tension kernel imported with its focused test and example; legacy Python application layers, Rust acceleration, CLI, Z3, Grok, UI, and Kernel Wave 12 surfaces remain unimported | | cig-ts-engine | `71d91a6a0cdc9e7a5439df972130596f0bae5d2f` | `src/thinking_system/graph/cig/` | **PARTIALLY_IMPORTED** | Core graph models, deterministic propagation, edge tension, derivative observer, representational radius, context-split proposal, YAML IO, seed graph and focused tests imported; CLI, plotting, reports, Obsidian export and generated artifacts remain outside | -| TS-Benchmarks | `7c0611f7ef4dc2d150b12dc4197a8d48a462207f` | `src/thinking_system/benchmarks/graph_scaling/` | **CONSOLIDATED** | Complete implemented Workstream A imported with graph generators, relaxation policies, baselines, receipts, schemas, runners, reports, failure decomposition, topology selection, focused tests, technical documentation and retained evidence; only repository scaffolding and a public-post draft were excluded | +| TS-Benchmarks | `7c0611f7ef4dc2d150b12dc4197a8d48a462207f` | `src/thinking_system/benchmarks/graph_scaling/` | **CONSOLIDATED** | Complete implemented Workstream A imported with graph generators, relaxation policies, baselines, receipts, schemas, runners, reports, failure decomposition, topology selection, focused tests, technical documentation and retained evidence; only repository scaffolding and a public-post draft were excluded; supplemental v0.5 adversarial-topology stress test imported from `ea6a944f5f4170a3ae02cfa8f93ce1bd87d31392` | | ts-chat-language | Not verified here | `packages/ts-language` | **INSPECTED** / **PLANNED** | TSLC-like code in `core/language/tslc.py` is in-tree monorepo code; full satellite port not evidenced | | TensionForge | Not verified here | `engines/tension-forge` | **PLANNED** (satellite); in-tree `inference/tension_forge/` is separate | Empty `engines/tension-forge/` | | Ten-SON-LM | Not verified here | `engines/tension-lm` | **PLANNED** | Empty destination | diff --git a/docs/migration/sources/TS-Benchmarks.md b/docs/migration/sources/TS-Benchmarks.md index 013e445..52d2adf 100644 --- a/docs/migration/sources/TS-Benchmarks.md +++ b/docs/migration/sources/TS-Benchmarks.md @@ -79,3 +79,27 @@ monorepo. Archiving the satellite remains a separate maintainer action after this import is merged and verified by CI. + +## Supplemental v0.5 adversarial-topology import + +- Source pull request: `BoggersTheFish/TS-Benchmarks#7` +- Source branch: `experiment/v0.5-adversarial-topology-generalization` +- Source commit: `ea6a944f5f4170a3ae02cfa8f93ce1bd87d31392` +- Canonical issue: `#15` + +Imported: + +- mixed core/periphery graph generation +- hub-threshold topology sweeps +- topology-noise sweeps +- hub-hub, hub-leaf, leaf-leaf, and random contradiction placement +- selector-versus-oracle comparisons +- selector regret metrics +- catastrophic-regression detection +- adversarial benchmark receipts and reports +- focused tests + +The recorded source result failed its strict success criteria. The import +preserves that negative result as a falsification finding, not a capability +claim. + diff --git a/src/thinking_system/benchmarks/graph_scaling/runners/adversarial_topology_generalization.py b/src/thinking_system/benchmarks/graph_scaling/runners/adversarial_topology_generalization.py new file mode 100644 index 0000000..96e9bb6 --- /dev/null +++ b/src/thinking_system/benchmarks/graph_scaling/runners/adversarial_topology_generalization.py @@ -0,0 +1,391 @@ +"""Run v0.5 adversarial topology generalization.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +from thinking_system.benchmarks.graph_scaling.receipts import ( + build_receipt, + stable_hash, + write_json, +) +from thinking_system.benchmarks.graph_scaling.tasks.adversarial_topology import ( + FAMILIES, + PLACEMENTS, + AdversarialSpec, + generate_adversarial_graph, +) +from thinking_system.benchmarks.graph_scaling.tasks.scaling import ( + RelaxationConfig, + run_relaxation, +) +from thinking_system.benchmarks.graph_scaling.tasks.topology_policy import ( + select_policy, + selection_payload, +) + +POLICIES = ["reference", "degree_normalized"] + + +def _split_ints(value: str) -> list[int]: + return [int(item.strip()) for item in value.split(",") if item.strip()] + + +def _split_strings(value: str) -> list[str]: + return [item.strip() for item in value.split(",") if item.strip()] + + +def _split_floats(value: str) -> list[float]: + return [float(item.strip()) for item in value.split(",") if item.strip()] + + +def run_generalization(args: argparse.Namespace) -> dict[str, Any]: + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + rows: list[dict[str, Any]] = [] + for family in _split_strings(args.families): + for nodes in _split_ints(args.sizes): + for hub_strength in _split_floats(args.hub_strengths): + for placement in _split_strings(args.placements): + spec = AdversarialSpec( + family=family, + placement=placement, + nodes=nodes, + seed=args.seed, + hub_strength=hub_strength, + noise_rate=args.noise_rate, + ) + graph = generate_adversarial_graph(spec) + selection = select_policy(graph) + policy_results = { + policy: run_policy(graph, args, policy) for policy in POLICIES + } + oracle = choose_oracle(policy_results) + selected = selection.selected_policy + selected_result = policy_results[selected] + reference_result = policy_results["reference"] + degree_result = policy_results["degree_normalized"] + rows.append( + { + "topology_family": family, + "nodes": nodes, + "seed": args.seed, + "hub_strength": hub_strength, + "boundary_case": is_boundary_case(selection), + "contradiction_placement": placement, + "selector_policy": selected, + "selector_reason": selection.reason, + "selector_diagnostics": selection_payload(selection), + "oracle_best_policy": oracle["policy"], + "selector_matches_oracle": selected == oracle["policy"], + "selector_regret_final_tension": ( + float(selected_result.metrics["final_global_tension"]) + - float(oracle["metrics"]["final_global_tension"]) + ), + "selector_regret_f1": ( + float( + oracle["metrics"]["contradiction_localization_f1"] + ) + - float( + selected_result.metrics[ + "contradiction_localization_f1" + ] + ) + ), + "catastrophic_regression": catastrophic_regression( + selected_result.metrics, oracle["metrics"] + ), + "selector_metrics": selected_result.metrics, + "reference_metrics": reference_result.metrics, + "degree_normalized_metrics": degree_result.metrics, + "oracle_metrics": oracle["metrics"], + } + ) + + aggregate = aggregate_rows(rows) + payload = { + "task": "Adversarial Topology Generalization", + "version": "v0.5-experimental", + "run_id": f"adversarial-topology-generalization-seed{args.seed}", + "framing": ( + "v0.4 passed on known graph families. v0.5 tries to falsify that result " + "by testing mixed, near-boundary, and contradiction-placement-stressed graphs." + ), + "selector_boundary": "selector uses pre-run topology diagnostics only; oracle labels are post-run audit fields", + "rows": rows, + "aggregate": aggregate, + } + json_path = out_dir / "adversarial_topology_generalization.json" + report_path = out_dir / "ADVERSARIAL_TOPOLOGY_GENERALIZATION.md" + write_json(json_path, payload) + write_report(payload, report_path) + + repo_root = Path(__file__).resolve().parents[5] + receipt = build_receipt( + run_id=str(payload["run_id"]), + repo_root=repo_root, + command=" ".join(sys.argv), + dataset={ + "name": "synthetic-adversarial-topology-generalization", + "version": "v0.5-experimental", + "hash": stable_hash( + { + "families": _split_strings(args.families), + "placements": _split_strings(args.placements), + "sizes": _split_ints(args.sizes), + "hub_strengths": _split_floats(args.hub_strengths), + "seed": args.seed, + "noise_rate": args.noise_rate, + } + ), + }, + system={ + "name": "ts-core-reference-relaxation", + "variant": "adversarial-topology-generalization", + }, + config={ + "seed": args.seed, + "families": _split_strings(args.families), + "placements": _split_strings(args.placements), + "sizes": _split_ints(args.sizes), + "hub_strengths": _split_floats(args.hub_strengths), + "noise_rate": args.noise_rate, + "steps": args.steps, + "learning_rate": args.learning_rate, + "damping": args.damping, + "tolerance": args.tolerance, + }, + metrics=aggregate, + graph_family="adversarial_mixed", + known_caveats=[ + "Synthetic adversarial topology stress test only.", + "Oracle-best-policy is computed after outcomes and is not available to the selector.", + "A pass supports robustness on this adversarial harness only, not real knowledge graph scalability.", + ], + artifacts=[json_path, report_path], + ) + receipt_path = out_dir / "adversarial_topology_generalization.receipt.json" + write_json(receipt_path, receipt) + payload["receipt_path"] = str(receipt_path) + write_json(json_path, payload) + return payload + + +def run_policy(graph: Any, args: argparse.Namespace, policy: str) -> Any: + return run_relaxation( + graph, + RelaxationConfig( + steps=args.steps, + learning_rate=args.learning_rate, + damping=args.damping, + tolerance=args.tolerance, + update_policy=policy, + ), + ) + + +def choose_oracle(policy_results: dict[str, Any]) -> dict[str, Any]: + def score(item: tuple[str, Any]) -> tuple[float, float]: + _policy, result = item + return ( + float(result.metrics["contradiction_localization_f1"]), + -float(result.metrics["final_global_tension"]), + ) + + policy, result = max(policy_results.items(), key=score) + return {"policy": policy, "metrics": result.metrics} + + +def is_boundary_case(selection: Any) -> bool: + diagnostics = selection.diagnostics + hub_touch = float(diagnostics["hub_edge_touch_share"]) + gini = float(diagnostics["degree_gini"]) + max_to_mean = float(diagnostics["max_to_mean_degree"]) + return ( + abs(hub_touch - 0.35) <= 0.08 + or abs(gini - 0.30) <= 0.06 + or abs(max_to_mean - 6.0) <= 1.5 + ) + + +def catastrophic_regression( + selected_metrics: dict[str, Any], oracle_metrics: dict[str, Any] +) -> bool: + f1_regret = float(oracle_metrics["contradiction_localization_f1"]) - float( + selected_metrics["contradiction_localization_f1"] + ) + tension_regret = float(selected_metrics["final_global_tension"]) - float( + oracle_metrics["final_global_tension"] + ) + return f1_regret > 0.50 or tension_regret > 0.10 + + +def aggregate_rows(rows: list[dict[str, Any]]) -> dict[str, Any]: + if not rows: + return {} + selector_wins = sum(1 for row in rows if row["selector_matches_oracle"]) + catastrophic = [row for row in rows if row["catastrophic_regression"]] + boundary_failures = [ + row + for row in rows + if row["boundary_case"] and not row["selector_matches_oracle"] + ] + selector_f1 = sum( + float(row["selector_metrics"]["contradiction_localization_f1"]) for row in rows + ) + reference_f1 = sum( + float(row["reference_metrics"]["contradiction_localization_f1"]) for row in rows + ) + degree_f1 = sum( + float(row["degree_normalized_metrics"]["contradiction_localization_f1"]) + for row in rows + ) + selector_tension = sum( + float(row["selector_metrics"]["final_global_tension"]) for row in rows + ) + reference_tension = sum( + float(row["reference_metrics"]["final_global_tension"]) for row in rows + ) + degree_tension = sum( + float(row["degree_normalized_metrics"]["final_global_tension"]) for row in rows + ) + family_summary: dict[str, dict[str, Any]] = {} + for row in rows: + summary = family_summary.setdefault( + row["topology_family"], + {"count": 0, "catastrophic": 0, "selector_oracle_matches": 0}, + ) + summary["count"] += 1 + summary["catastrophic"] += int(row["catastrophic_regression"]) + summary["selector_oracle_matches"] += int(row["selector_matches_oracle"]) + + success = { + "beats_or_matches_always_reference_on_f1": selector_f1 >= reference_f1, + "beats_or_matches_always_degree_normalized_on_f1": selector_f1 >= degree_f1, + "beats_or_matches_always_reference_on_tension": selector_tension + <= reference_tension, + "beats_or_matches_always_degree_normalized_on_tension": selector_tension + <= degree_tension, + "no_catastrophic_family_regression": not catastrophic, + "boundary_cases_documented": True, + } + success["passed"] = all(success.values()) + return { + "row_count": len(rows), + "selector_oracle_match_rate": selector_wins / len(rows), + "catastrophic_regression_count": len(catastrophic), + "boundary_failure_count": len(boundary_failures), + "selector_total_f1": selector_f1, + "always_reference_total_f1": reference_f1, + "always_degree_normalized_total_f1": degree_f1, + "selector_total_final_tension": selector_tension, + "always_reference_total_final_tension": reference_tension, + "always_degree_normalized_total_final_tension": degree_tension, + "family_summary": family_summary, + "success_criteria": success, + } + + +def write_report(payload: dict[str, Any], out_path: Path) -> None: + aggregate = payload["aggregate"] + lines = [ + "# Adversarial Topology Generalization", + "", + payload["framing"], + "", + "The goal is not to prove universal scaling; the goal is to measure where topology-aware policy selection generalizes and where it fails.", + "", + "## Success Criteria", + "", + ] + for key, value in aggregate["success_criteria"].items(): + lines.append(f"- `{key}`: `{value}`") + lines.extend( + [ + "", + "## Aggregate Metrics", + "", + f"- Rows: `{aggregate['row_count']}`", + f"- Selector/oracle match rate: `{aggregate['selector_oracle_match_rate']:.3f}`", + f"- Catastrophic regressions: `{aggregate['catastrophic_regression_count']}`", + f"- Boundary failures: `{aggregate['boundary_failure_count']}`", + f"- Selector total F1: `{aggregate['selector_total_f1']:.3f}`", + f"- Always-reference total F1: `{aggregate['always_reference_total_f1']:.3f}`", + f"- Always-degree-normalized total F1: `{aggregate['always_degree_normalized_total_f1']:.3f}`", + f"- Selector total final tension: `{aggregate['selector_total_final_tension']:.6f}`", + f"- Always-reference total final tension: `{aggregate['always_reference_total_final_tension']:.6f}`", + f"- Always-degree-normalized total final tension: `{aggregate['always_degree_normalized_total_final_tension']:.6f}`", + "", + "## Selector Confusion Rows", + "", + "| Family | Nodes | Hub strength | Placement | Boundary | Selector | Oracle | F1 regret | Tension regret | Catastrophic |", + "| --- | ---: | ---: | --- | --- | --- | --- | ---: | ---: | --- |", + ] + ) + for row in payload["rows"]: + if row["selector_matches_oracle"] and not row["boundary_case"]: + continue + lines.append( + "| {family} | {nodes} | {hub:.2f} | {placement} | {boundary} | {selector} | {oracle} | {f1:.3f} | {tension:.6f} | {cat} |".format( + family=row["topology_family"], + nodes=row["nodes"], + hub=row["hub_strength"], + placement=row["contradiction_placement"], + boundary=row["boundary_case"], + selector=row["selector_policy"], + oracle=row["oracle_best_policy"], + f1=row["selector_regret_f1"], + tension=row["selector_regret_final_tension"], + cat=row["catastrophic_regression"], + ) + ) + lines.extend( + [ + "", + "## Claim Boundary", + "", + "> v0.5 is an adversarial synthetic selector stress test. It does not prove TS-Core scales to real knowledge/provenance graphs.", + ] + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sizes", default="100,1000") + parser.add_argument("--families", default=",".join(FAMILIES)) + parser.add_argument("--placements", default=",".join(PLACEMENTS)) + parser.add_argument("--hub-strengths", default="0.25,0.35,0.45,0.60,0.75") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--noise-rate", type=float, default=0.10) + parser.add_argument("--steps", type=int, default=64) + parser.add_argument("--learning-rate", type=float, default=0.12) + parser.add_argument("--damping", type=float, default=0.85) + parser.add_argument("--tolerance", type=float, default=1e-4) + parser.add_argument("--out-dir", required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + payload = run_generalization(args) + print( + json.dumps( + { + "run_id": payload["run_id"], + "passed": payload["aggregate"]["success_criteria"]["passed"], + "receipt_path": payload["receipt_path"], + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/thinking_system/benchmarks/graph_scaling/tasks/adversarial_topology.py b/src/thinking_system/benchmarks/graph_scaling/tasks/adversarial_topology.py new file mode 100644 index 0000000..324fd8b --- /dev/null +++ b/src/thinking_system/benchmarks/graph_scaling/tasks/adversarial_topology.py @@ -0,0 +1,266 @@ +"""Adversarial topology generators for v0.5 selector stress tests.""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from typing import Iterable + +from thinking_system.benchmarks.graph_scaling.tasks.scaling import ( + CONTRADICTS, + SUPPORT, + Edge, + GraphSpec, + SyntheticGraph, + build_adjacency, + node_degrees, +) + +PLACEMENTS = ["hub_hub", "hub_leaf", "leaf_leaf", "random"] +FAMILIES = ["mixed_core_periphery", "hub_threshold_sweep", "topology_noise_sweep"] + + +@dataclass(frozen=True) +class AdversarialSpec: + family: str + placement: str + nodes: int + seed: int + hub_strength: float = 0.5 + noise_rate: float = 0.1 + + +def generate_adversarial_graph(spec: AdversarialSpec) -> SyntheticGraph: + if spec.family == "mixed_core_periphery": + return mixed_core_periphery(spec) + if spec.family == "hub_threshold_sweep": + return hub_threshold_sweep(spec) + if spec.family == "topology_noise_sweep": + return topology_noise_sweep(spec) + raise ValueError(f"unknown adversarial family: {spec.family}") + + +def mixed_core_periphery(spec: AdversarialSpec) -> SyntheticGraph: + rng = random.Random(spec.seed) + nodes = spec.nodes + core_size = max(8, int(nodes * (0.12 + 0.18 * spec.hub_strength))) + periphery_start = core_size + values = [rng.uniform(-0.25, 0.25) for _ in range(nodes)] + edges: list[Edge] = [] + + edges.extend(preferential_edges(rng, start=0, end=core_size, links_per_node=3)) + edges.extend( + ring_lattice_edges(rng, start=periphery_start, end=nodes, radius=2, rewire=0.06) + ) + bridge_count = max(1, int(nodes * 0.02)) + for _ in range(bridge_count): + src = rng.randrange(0, core_size) + dst = rng.randrange(periphery_start, nodes) + edges.append(support_edge(src, dst, rng)) + + edges, contradiction_pairs = inject_placed_contradiction( + rng, nodes, edges, values, spec.placement + ) + return make_graph(spec, values, edges, contradiction_pairs) + + +def hub_threshold_sweep(spec: AdversarialSpec) -> SyntheticGraph: + rng = random.Random(spec.seed) + nodes = spec.nodes + values = [rng.uniform(-0.25, 0.25) for _ in range(nodes)] + edges: list[Edge] = [] + hub_count = max(2, int(nodes * (0.01 + 0.06 * spec.hub_strength))) + target_edges = max(nodes * 3, nodes - 1) + + for node in range(hub_count, nodes): + if rng.random() < spec.hub_strength: + dst = rng.randrange(0, hub_count) + else: + dst = rng.randrange(0, node) + if node != dst: + edges.append(support_edge(node, dst, rng)) + while len(edges) < target_edges: + src = rng.randrange(nodes) + if rng.random() < spec.hub_strength: + dst = rng.randrange(0, hub_count) + else: + dst = rng.randrange(nodes) + if src != dst: + edges.append(support_edge(src, dst, rng)) + + edges = dedupe_edges(edges) + edges, contradiction_pairs = inject_placed_contradiction( + rng, nodes, edges, values, spec.placement + ) + return make_graph(spec, values, edges, contradiction_pairs) + + +def topology_noise_sweep(spec: AdversarialSpec) -> SyntheticGraph: + rng = random.Random(spec.seed) + nodes = spec.nodes + values = [rng.uniform(-0.25, 0.25) for _ in range(nodes)] + if spec.hub_strength >= 0.5: + edges = preferential_edges(rng, start=0, end=nodes, links_per_node=3) + else: + edges = ring_lattice_edges(rng, start=0, end=nodes, radius=3, rewire=0.08) + + replace_count = int(len(edges) * spec.noise_rate) + for idx in rng.sample(range(len(edges)), k=min(replace_count, len(edges))): + src = rng.randrange(nodes) + dst = rng.randrange(nodes) + if src != dst: + edges[idx] = support_edge(src, dst, rng) + + edges = dedupe_edges(edges) + edges, contradiction_pairs = inject_placed_contradiction( + rng, nodes, edges, values, spec.placement + ) + return make_graph(spec, values, edges, contradiction_pairs) + + +def preferential_edges( + rng: random.Random, *, start: int, end: int, links_per_node: int +) -> list[Edge]: + if end - start < 2: + return [] + edges = [support_edge(start, start + 1, rng)] + degree_bag = [start, start + 1] + for node in range(start + 2, end): + targets: set[int] = set() + while len(targets) < min(links_per_node, node - start): + targets.add(rng.choice(degree_bag)) + for target in targets: + edges.append(support_edge(node, target, rng)) + degree_bag.extend([node, target]) + return edges + + +def ring_lattice_edges( + rng: random.Random, + *, + start: int, + end: int, + radius: int, + rewire: float, +) -> list[Edge]: + edges: list[Edge] = [] + count = end - start + if count < 2: + return edges + for offset_node in range(count): + src = start + offset_node + for offset in range(1, radius + 1): + dst = start + ((offset_node + offset) % count) + if rng.random() < rewire: + dst = rng.randrange(start, end) + if src != dst: + edges.append(support_edge(src, dst, rng)) + return dedupe_edges(edges) + + +def inject_placed_contradiction( + rng: random.Random, + nodes: int, + edges: list[Edge], + values: list[float], + placement: str, +) -> tuple[list[Edge], list[tuple[int, int]]]: + degrees = degree_from_edges(nodes, edges) + hubs = ranked_nodes(degrees, reverse=True) + leaves = ranked_nodes(degrees, reverse=False) + if placement == "hub_hub": + src, dst = pick_pair(hubs[: max(2, len(hubs) // 10)], rng) + elif placement == "hub_leaf": + src = rng.choice(hubs[: max(1, len(hubs) // 10)]) + dst = rng.choice(leaves[: max(1, len(leaves) // 10)]) + elif placement == "leaf_leaf": + src, dst = pick_pair(leaves[: max(2, len(leaves) // 10)], rng) + elif placement == "random": + src, dst = pick_pair(list(range(nodes)), rng) + else: + raise ValueError(f"unknown contradiction placement: {placement}") + + sign = 1.0 if rng.random() >= 0.5 else -1.0 + values[src] = sign * rng.uniform(0.65, 0.95) + values[dst] = sign * rng.uniform(0.65, 0.95) + updated = list(edges) + updated.append( + Edge( + src=src, + dst=dst, + relation=CONTRADICTS, + weight=rng.uniform(1.5, 2.5), + provenance=f"synthetic_{placement}_contradiction", + ) + ) + return updated, [(src, dst)] + + +def make_graph( + spec: AdversarialSpec, + values: list[float], + edges: list[Edge], + contradiction_pairs: list[tuple[int, int]], +) -> SyntheticGraph: + edges = dedupe_edges(edges, keep_contradictions=True) + graph_spec = GraphSpec( + graph_type=spec.family, + nodes=spec.nodes, + edges=len(edges), + seed=spec.seed, + contradiction_pairs=contradiction_pairs, + ) + return SyntheticGraph( + spec=graph_spec, + values=values, + edges=edges, + adjacency=build_adjacency(spec.nodes, edges), + ) + + +def support_edge(src: int, dst: int, rng: random.Random) -> Edge: + return Edge(src=src, dst=dst, relation=SUPPORT, weight=rng.uniform(0.5, 1.0)) + + +def dedupe_edges( + edges: Iterable[Edge], keep_contradictions: bool = False +) -> list[Edge]: + seen: set[tuple[int, int, str]] = set() + result: list[Edge] = [] + for edge in edges: + if edge.src == edge.dst: + continue + relation_key = edge.relation if keep_contradictions else SUPPORT + key = (min(edge.src, edge.dst), max(edge.src, edge.dst), relation_key) + if key in seen: + continue + seen.add(key) + result.append(edge) + return result + + +def degree_from_edges(nodes: int, edges: Iterable[Edge]) -> list[int]: + degrees = [0 for _ in range(nodes)] + for edge in edges: + degrees[edge.src] += 1 + degrees[edge.dst] += 1 + return degrees + + +def ranked_nodes(degrees: list[int], *, reverse: bool) -> list[int]: + return [ + idx + for idx, _degree in sorted( + enumerate(degrees), key=lambda row: row[1], reverse=reverse + ) + ] + + +def pick_pair(candidates: list[int], rng: random.Random) -> tuple[int, int]: + if len(candidates) < 2: + raise ValueError("need at least two candidates") + src = rng.choice(candidates) + dst = rng.choice(candidates) + while dst == src: + dst = rng.choice(candidates) + return src, dst diff --git a/tests/benchmarks/graph_scaling/test_adversarial_topology.py b/tests/benchmarks/graph_scaling/test_adversarial_topology.py new file mode 100644 index 0000000..f613054 --- /dev/null +++ b/tests/benchmarks/graph_scaling/test_adversarial_topology.py @@ -0,0 +1,91 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from thinking_system.benchmarks.graph_scaling.runners.adversarial_topology_generalization import ( + build_parser, + run_generalization, +) +from thinking_system.benchmarks.graph_scaling.tasks.adversarial_topology import ( + AdversarialSpec, + generate_adversarial_graph, +) + + +class AdversarialTopologyTests(unittest.TestCase): + def test_generators_emit_placed_contradictions(self): + for family in [ + "mixed_core_periphery", + "hub_threshold_sweep", + "topology_noise_sweep", + ]: + graph = generate_adversarial_graph( + AdversarialSpec( + family=family, + placement="hub_leaf", + nodes=100, + seed=42, + hub_strength=0.45, + ) + ) + self.assertEqual(graph.spec.graph_type, family) + self.assertEqual(len(graph.spec.contradiction_pairs), 1) + self.assertGreater(graph.spec.edges, 0) + + def test_runner_writes_oracle_receipt(self): + with tempfile.TemporaryDirectory() as tmp: + args = build_parser().parse_args( + [ + "--sizes", + "50", + "--families", + "mixed_core_periphery", + "--placements", + "hub_hub,leaf_leaf", + "--hub-strengths", + "0.35", + "--seed", + "42", + "--steps", + "4", + "--out-dir", + tmp, + ] + ) + payload = run_generalization(args) + out_dir = Path(tmp) + self.assertEqual(payload["task"], "Adversarial Topology Generalization") + self.assertTrue( + (out_dir / "adversarial_topology_generalization.json").exists() + ) + self.assertTrue( + (out_dir / "ADVERSARIAL_TOPOLOGY_GENERALIZATION.md").exists() + ) + self.assertTrue( + (out_dir / "adversarial_topology_generalization.receipt.json").exists() + ) + row = payload["rows"][0] + for key in [ + "selector_policy", + "oracle_best_policy", + "selector_matches_oracle", + "selector_regret_final_tension", + "selector_regret_f1", + "catastrophic_regression", + "boundary_case", + "contradiction_placement", + "topology_family", + ]: + self.assertIn(key, row) + receipt = json.loads( + ( + out_dir / "adversarial_topology_generalization.receipt.json" + ).read_text() + ) + self.assertEqual(receipt["dataset"]["version"], "v0.5-experimental") + self.assertIn("selector_oracle_match_rate", receipt["metrics"]) + + +if __name__ == "__main__": + unittest.main() From 69c22e6d71f1ea817cddf6633c1af3b607a79cb8 Mon Sep 17 00:00:00 2001 From: BoggersTheFish Date: Wed, 22 Jul 2026 01:21:56 +0100 Subject: [PATCH 2/2] fix: clean adversarial benchmark import --- docs/migration/sources/TS-Benchmarks.md | 1 - .../benchmarks/graph_scaling/tasks/adversarial_topology.py | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/migration/sources/TS-Benchmarks.md b/docs/migration/sources/TS-Benchmarks.md index 52d2adf..8769c24 100644 --- a/docs/migration/sources/TS-Benchmarks.md +++ b/docs/migration/sources/TS-Benchmarks.md @@ -102,4 +102,3 @@ Imported: The recorded source result failed its strict success criteria. The import preserves that negative result as a falsification finding, not a capability claim. - diff --git a/src/thinking_system/benchmarks/graph_scaling/tasks/adversarial_topology.py b/src/thinking_system/benchmarks/graph_scaling/tasks/adversarial_topology.py index 324fd8b..31cd1f7 100644 --- a/src/thinking_system/benchmarks/graph_scaling/tasks/adversarial_topology.py +++ b/src/thinking_system/benchmarks/graph_scaling/tasks/adversarial_topology.py @@ -13,7 +13,6 @@ GraphSpec, SyntheticGraph, build_adjacency, - node_degrees, ) PLACEMENTS = ["hub_hub", "hub_leaf", "leaf_leaf", "random"]