From 00d5c811193086c01902cefbecd88867f20be720 Mon Sep 17 00:00:00 2001 From: BAF1927 Date: Wed, 26 Aug 2026 16:08:16 -0400 Subject: [PATCH 1/3] Update in information --- benchmarks/pokec/README.md | 4 ++- benchmarks/pokec/run_bench.py | 55 +++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/benchmarks/pokec/README.md b/benchmarks/pokec/README.md index 527c5f1a..ce5daf15 100644 --- a/benchmarks/pokec/README.md +++ b/benchmarks/pokec/README.md @@ -74,7 +74,9 @@ Timed windows: `import_s`, `setup_s`, `reason_s`. Thresholds apply to `reason_s` only. After `reason()`, `relevance_rows` at the final timestep must **exactly** match `baselines.json` (semantics tripwire). First child is an untimed warmup so Numba compilation stays out of measured windows. -`PYTHONHASHSEED=0` on every child. +`PYTHONHASHSEED=0` on every child. Each timed child prints its own +`reason_s` / `setup_s` / `import_s` / `relevance_rows` line; the PASS line +also reports min / max / mean / stdev of `reason_s` for that job. Banked on `ubuntu-latest` (3 Actions medians per type; cap = 2× worst median): diff --git a/benchmarks/pokec/run_bench.py b/benchmarks/pokec/run_bench.py index 71acb2e6..e8796495 100644 --- a/benchmarks/pokec/run_bench.py +++ b/benchmarks/pokec/run_bench.py @@ -294,6 +294,26 @@ def _median(xs): return float(statistics.median(xs)) if xs else None +def _metric_stats(xs): + """Return min / max / mean / stdev / n for a list of floats.""" + if not xs: + return { + "min": None, + "max": None, + "mean": None, + "stdev": None, + "n": 0, + } + return { + "min": float(min(xs)), + "max": float(max(xs)), + "mean": float(statistics.mean(xs)), + # Sample stdev needs ≥2 points; a single child has no spread. + "stdev": float(statistics.stdev(xs)) if len(xs) >= 2 else 0.0, + "n": len(xs), + } + + def _load_baselines(path: Path) -> dict: """Read baselines.json and return it as a dictionary.""" if not path.exists(): @@ -374,7 +394,21 @@ def _parent_main(args) -> int: print(f" timed run {i + 1}/{args.repeats}", flush=True) # Run one benchmark measurement in a fresh child process # and save its returned timing/correctness data. - runs.append(_spawn_child(args, warmup=False)) + child = _spawn_child(args, warmup=False) + runs.append(child) + # Print this child's numbers so a CI log (or 10× refire) can be + # scraped for max / stdev without opening the JSON artifact. + print( + f" child {i + 1}/{args.repeats}: " + f"reason_s={child['reason_s']:.4f}s " + f"setup_s={child['setup_s']:.4f}s " + f"import_s={child['import_s']:.4f}s " + f"relevance_rows={child['relevance_rows']}", + flush=True, + ) + reason_vals = [r["reason_s"] for r in runs] + setup_vals = [r["setup_s"] for r in runs] + import_vals = [r["import_s"] for r in runs] # Build the final benchmark report using all measured runs. report = { # Record which benchmark type was executed. @@ -390,12 +424,12 @@ def _parent_main(args) -> int: # Calculate one representative median value for each metric. "medians": { # Median PyReason import time across all measured runs. - "import_s": _median([r["import_s"] for r in runs]), + "import_s": _median(import_vals), # Median graph/rule/fact setup time. - "setup_s": _median([r["setup_s"] for r in runs]), + "setup_s": _median(setup_vals), # Median actual reasoning time. # This is the main performance metric. - "reason_s": _median([r["reason_s"] for r in runs]), + "reason_s": _median(reason_vals), # Median final relevance-row count. # Normally every run should already have exactly the same count. "relevance_rows": int( @@ -404,6 +438,12 @@ def _parent_main(args) -> int: ) ), }, + # Per-job spread for banking a tighter cap (e.g. max × 1.05). + "stats": { + "reason_s": _metric_stats(reason_vals), + "setup_s": _metric_stats(setup_vals), + "import_s": _metric_stats(import_vals), + }, # Record the Python version used by the child processes. # If no runs somehow exist, fall back to the parent Python version. "python": ( @@ -460,12 +500,17 @@ def _parent_main(args) -> int: ) # Create a short alias to the median metrics for easier printing. med = report["medians"] + reason_stats = report["stats"]["reason_s"] # Convert the boolean pass result into a readable status string. status = "PASS" if report["pass"] else "FAIL" - # Print a one-line benchmark summary. + # Print a one-line benchmark summary plus this job's reason_s spread. print( f"{status} {args.type}: " f"reason_s_med={med['reason_s']:.4f}s " + f"reason_s_min={reason_stats['min']:.4f}s " + f"reason_s_max={reason_stats['max']:.4f}s " + f"reason_s_mean={reason_stats['mean']:.4f}s " + f"reason_s_stdev={reason_stats['stdev']:.4f}s " f"setup_s_med={med['setup_s']:.4f}s " f"import_s_med={med['import_s']:.4f}s " f"relevance_rows={med['relevance_rows']} " From fc1f2d9af03a92c81b8dd00e7ca68d0ea09f0d94 Mon Sep 17 00:00:00 2001 From: BAF1927 Date: Thu, 27 Aug 2026 09:58:44 -0400 Subject: [PATCH 2/3] Runs Experiment Results --- benchmarks/pokec/README.md | 14 +- benchmarks/pokec/baselines.json | 4 +- benchmarks/pokec/time_tests.json | 314 +++++++++++++++++++++++++++++++ 3 files changed, 323 insertions(+), 9 deletions(-) create mode 100644 benchmarks/pokec/time_tests.json diff --git a/benchmarks/pokec/README.md b/benchmarks/pokec/README.md index ce5daf15..f2edc093 100644 --- a/benchmarks/pokec/README.md +++ b/benchmarks/pokec/README.md @@ -78,12 +78,12 @@ untimed warmup so Numba compilation stays out of measured windows. `reason_s` / `setup_s` / `import_s` / `relevance_rows` line; the PASS line also reports min / max / mean / stdev of `reason_s` for that job. -Banked on `ubuntu-latest` (3 Actions medians per type; cap = 2× worst median): +Banked on `ubuntu-latest` (10 CI jobs × 3 children per size; cap = 1.05 × max observed child `reason_s`): | Type | `relevance_rows` | `reason_s_max` | |------|------------------|----------------| -| 2k | **1573** | **20.6428** s | -| 10k | **8006** | **154.8286** s | +| 2k | **1573** | **10.8126** s | +| 10k | **8006** | **72.0894** s | ## CI (`.github/workflows/perf.yml`) @@ -132,11 +132,11 @@ Existing `python-package-version-test.yml` / `python-publish.yml` are untouched. Caps above are already banked in `baselines.json`. Re-banking is a **deliberate, reviewed** edit — never automatic. -1. Run the type 3× on Actions, collect median `reason_s` from each report. -2. Take the **worst** median, multiply by **2.0**, set `reason_s_max`. -3. `relevance_rows` must stay identical across those runs; change it only if +1. Run the size 10× on Actions (3 children per run = 30 child measurements). +2. Take the **maximum observed child** `reason_s`, multiply by **1.05** (5% headroom), set `reason_s_max`. +3. `relevance_rows` must stay identical across all runs; change it only if the count is stable and the semantics change is intentional. -4. Cite the three report artifacts in the PR that edits `baselines.json`. +4. Cite the campaign artifact / report data in the PR that edits `baselines.json`. `relevance_rows` mismatches are hard failures. `reason_s` over threshold fails with a message to re-run once before investigating (hosted-runner noise diff --git a/benchmarks/pokec/baselines.json b/benchmarks/pokec/baselines.json index c0c25288..7cdad66c 100644 --- a/benchmarks/pokec/baselines.json +++ b/benchmarks/pokec/baselines.json @@ -1,11 +1,11 @@ { "runner": "ubuntu-latest", "pokec-2k": { - "reason_s_max": 20.6428, + "reason_s_max": 10.8126, "relevance_rows": 1573 }, "pokec-10k": { - "reason_s_max": 154.8286, + "reason_s_max": 72.0894, "relevance_rows": 8006 } } diff --git a/benchmarks/pokec/time_tests.json b/benchmarks/pokec/time_tests.json new file mode 100644 index 00000000..c3a9546f --- /dev/null +++ b/benchmarks/pokec/time_tests.json @@ -0,0 +1,314 @@ +{ + "goal": "Collect 10 CI jobs × 3 children per type, then set reason_s_max = max(all child reason_s) × 1.05", + "runner": "ubuntu-latest", + "jobs_target": 10, + "children_per_job": 3, + "pokec-2k": { + "jobs": [ + { + "job": 1, + "children": [ + {"child": 1, "reason_s": 7.6855, "setup_s": 3.8062, "import_s": 1.9445, "relevance_rows": 1573}, + {"child": 2, "reason_s": 7.7142, "setup_s": 3.8406, "import_s": 1.8453, "relevance_rows": 1573}, + {"child": 3, "reason_s": 7.5947, "setup_s": 4.0568, "import_s": 1.8477, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 7.6855, + "reason_s_min": 7.5947, + "reason_s_max": 7.7142, + "reason_s_mean": 7.6648, + "reason_s_stdev": 0.0624 + } + }, + { + "job": 2, + "children": [ + {"child": 1, "reason_s": 9.3472, "setup_s": 5.6154, "import_s": 2.6527, "relevance_rows": 1573}, + {"child": 2, "reason_s": 9.3922, "setup_s": 5.6230, "import_s": 2.6494, "relevance_rows": 1573}, + {"child": 3, "reason_s": 9.5583, "setup_s": 5.6886, "import_s": 2.6497, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 9.3922, + "reason_s_min": 9.3472, + "reason_s_max": 9.5583, + "reason_s_mean": 9.4326, + "reason_s_stdev": 0.1112 + } + }, + { + "job": 3, + "children": [ + {"child": 1, "reason_s": 10.1664, "setup_s": 6.1032, "import_s": 2.9361, "relevance_rows": 1573}, + {"child": 2, "reason_s": 10.2977, "setup_s": 6.1209, "import_s": 2.8601, "relevance_rows": 1573}, + {"child": 3, "reason_s": 10.2672, "setup_s": 6.1633, "import_s": 2.9023, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 10.2672, + "reason_s_min": 10.1664, + "reason_s_max": 10.2977, + "reason_s_mean": 10.2438, + "reason_s_stdev": 0.0687 + } + }, + { + "job": 4, + "children": [ + {"child": 1, "reason_s": 9.2964, "setup_s": 5.9366, "import_s": 2.7192, "relevance_rows": 1573}, + {"child": 2, "reason_s": 9.2649, "setup_s": 5.9168, "import_s": 2.6901, "relevance_rows": 1573}, + {"child": 3, "reason_s": 9.3189, "setup_s": 5.9039, "import_s": 2.6971, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 9.2964, + "reason_s_min": 9.2649, + "reason_s_max": 9.3189, + "reason_s_mean": 9.2934, + "reason_s_stdev": 0.0271 + } + }, + { + "job": 5, + "children": [ + {"child": 1, "reason_s": 9.6295, "setup_s": 6.2064, "import_s": 2.8286, "relevance_rows": 1573}, + {"child": 2, "reason_s": 9.7216, "setup_s": 6.1650, "import_s": 2.8088, "relevance_rows": 1573}, + {"child": 3, "reason_s": 9.5436, "setup_s": 6.0924, "import_s": 2.7823, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 9.6295, + "reason_s_min": 9.5436, + "reason_s_max": 9.7216, + "reason_s_mean": 9.6316, + "reason_s_stdev": 0.0890 + } + }, + { + "job": 6, + "children": [ + {"child": 1, "reason_s": 9.3659, "setup_s": 5.9451, "import_s": 2.7442, "relevance_rows": 1573}, + {"child": 2, "reason_s": 9.4344, "setup_s": 5.9832, "import_s": 2.7496, "relevance_rows": 1573}, + {"child": 3, "reason_s": 9.4073, "setup_s": 5.9858, "import_s": 2.7453, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 9.4073, + "reason_s_min": 9.3659, + "reason_s_max": 9.4344, + "reason_s_mean": 9.4026, + "reason_s_stdev": 0.0345 + } + }, + { + "job": 7, + "children": [ + {"child": 1, "reason_s": 9.8594, "setup_s": 5.8226, "import_s": 2.8182, "relevance_rows": 1573}, + {"child": 2, "reason_s": 9.9775, "setup_s": 6.0764, "import_s": 2.8914, "relevance_rows": 1573}, + {"child": 3, "reason_s": 10.1897, "setup_s": 6.0478, "import_s": 2.8427, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 9.9775, + "reason_s_min": 9.8594, + "reason_s_max": 10.1897, + "reason_s_mean": 10.0089, + "reason_s_stdev": 0.1674 + } + }, + { + "job": 8, + "children": [ + {"child": 1, "reason_s": 7.7029, "setup_s": 4.6202, "import_s": 2.2212, "relevance_rows": 1573}, + {"child": 2, "reason_s": 7.6472, "setup_s": 4.5873, "import_s": 2.2173, "relevance_rows": 1573}, + {"child": 3, "reason_s": 7.7295, "setup_s": 4.6060, "import_s": 2.2271, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 7.7029, + "reason_s_min": 7.6472, + "reason_s_max": 7.7295, + "reason_s_mean": 7.6932, + "reason_s_stdev": 0.0420 + } + }, + { + "job": 9, + "children": [ + {"child": 1, "reason_s": 9.2472, "setup_s": 5.9282, "import_s": 2.7092, "relevance_rows": 1573}, + {"child": 2, "reason_s": 9.2137, "setup_s": 5.8851, "import_s": 2.7034, "relevance_rows": 1573}, + {"child": 3, "reason_s": 9.2525, "setup_s": 5.8881, "import_s": 2.7146, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 9.2472, + "reason_s_min": 9.2137, + "reason_s_max": 9.2525, + "reason_s_mean": 9.2378, + "reason_s_stdev": 0.0210 + } + }, + { + "job": 10, + "children": [ + {"child": 1, "reason_s": 9.5723, "setup_s": 6.0360, "import_s": 2.7702, "relevance_rows": 1573}, + {"child": 2, "reason_s": 9.6950, "setup_s": 6.1249, "import_s": 2.8001, "relevance_rows": 1573}, + {"child": 3, "reason_s": 9.6418, "setup_s": 6.1457, "import_s": 2.8046, "relevance_rows": 1573} + ], + "job_summary": { + "reason_s_med": 9.6418, + "reason_s_min": 9.5723, + "reason_s_max": 9.6950, + "reason_s_mean": 9.6364, + "reason_s_stdev": 0.0615 + } + } + ] + }, + "pokec-10k": { + "jobs": [ + { + "job": 1, + "children": [ + {"child": 1, "reason_s": 61.3486, "setup_s": 13.3036, "import_s": 2.8297, "relevance_rows": 8006}, + {"child": 2, "reason_s": 60.5919, "setup_s": 13.4724, "import_s": 2.8008, "relevance_rows": 8006}, + {"child": 3, "reason_s": 61.0907, "setup_s": 13.3693, "import_s": 2.7490, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 61.0907, + "reason_s_min": 60.5919, + "reason_s_max": 61.3486, + "reason_s_mean": 61.0104, + "reason_s_stdev": 0.3847 + } + }, + { + "job": 2, + "children": [ + {"child": 1, "reason_s": 56.4989, "setup_s": 13.5754, "import_s": 2.7762, "relevance_rows": 8006}, + {"child": 2, "reason_s": 61.6788, "setup_s": 13.4386, "import_s": 2.7973, "relevance_rows": 8006}, + {"child": 3, "reason_s": 62.2084, "setup_s": 13.7842, "import_s": 2.8540, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 61.6788, + "reason_s_min": 56.4989, + "reason_s_max": 62.2084, + "reason_s_mean": 60.1287, + "reason_s_stdev": 3.1546 + } + }, + { + "job": 3, + "children": [ + {"child": 1, "reason_s": 67.3815, "setup_s": 9.0419, "import_s": 2.0642, "relevance_rows": 8006}, + {"child": 2, "reason_s": 66.9899, "setup_s": 9.1442, "import_s": 2.0869, "relevance_rows": 8006}, + {"child": 3, "reason_s": 68.6566, "setup_s": 9.0728, "import_s": 2.1219, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 67.3815, + "reason_s_min": 66.9899, + "reason_s_max": 68.6566, + "reason_s_mean": 67.6760, + "reason_s_stdev": 0.8715 + } + }, + { + "job": 4, + "children": [ + {"child": 1, "reason_s": 54.7664, "setup_s": 12.3158, "import_s": 2.8277, "relevance_rows": 8006}, + {"child": 2, "reason_s": 54.7528, "setup_s": 12.2771, "import_s": 2.8037, "relevance_rows": 8006}, + {"child": 3, "reason_s": 54.5292, "setup_s": 12.4110, "import_s": 2.8328, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 54.7528, + "reason_s_min": 54.5292, + "reason_s_max": 54.7664, + "reason_s_mean": 54.6828, + "reason_s_stdev": 0.1332 + } + }, + { + "job": 5, + "children": [ + {"child": 1, "reason_s": 54.7477, "setup_s": 12.2123, "import_s": 2.7857, "relevance_rows": 8006}, + {"child": 2, "reason_s": 54.5496, "setup_s": 12.4485, "import_s": 2.7793, "relevance_rows": 8006}, + {"child": 3, "reason_s": 54.8248, "setup_s": 12.1688, "import_s": 2.7803, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 54.7477, + "reason_s_min": 54.5496, + "reason_s_max": 54.8248, + "reason_s_mean": 54.7074, + "reason_s_stdev": 0.1420 + } + }, + { + "job": 6, + "children": [ + {"child": 1, "reason_s": 53.5537, "setup_s": 11.8891, "import_s": 2.6605, "relevance_rows": 8006}, + {"child": 2, "reason_s": 53.7185, "setup_s": 12.0079, "import_s": 2.6535, "relevance_rows": 8006}, + {"child": 3, "reason_s": 54.0737, "setup_s": 12.0236, "import_s": 2.6744, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 53.7185, + "reason_s_min": 53.5537, + "reason_s_max": 54.0737, + "reason_s_mean": 53.7820, + "reason_s_stdev": 0.2657 + } + }, + { + "job": 7, + "children": [ + {"child": 1, "reason_s": 53.7327, "setup_s": 11.9361, "import_s": 2.6700, "relevance_rows": 8006}, + {"child": 2, "reason_s": 53.7334, "setup_s": 11.9261, "import_s": 2.6430, "relevance_rows": 8006}, + {"child": 3, "reason_s": 53.8189, "setup_s": 12.0504, "import_s": 2.6735, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 53.7334, + "reason_s_min": 53.7327, + "reason_s_max": 53.8189, + "reason_s_mean": 53.7617, + "reason_s_stdev": 0.0496 + } + }, + { + "job": 8, + "children": [ + {"child": 1, "reason_s": 52.6410, "setup_s": 12.5460, "import_s": 2.5177, "relevance_rows": 8006}, + {"child": 2, "reason_s": 57.4740, "setup_s": 12.4863, "import_s": 2.5360, "relevance_rows": 8006}, + {"child": 3, "reason_s": 57.6541, "setup_s": 12.5568, "import_s": 2.5209, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 57.4740, + "reason_s_min": 52.6410, + "reason_s_max": 57.6541, + "reason_s_mean": 55.9231, + "reason_s_stdev": 2.8437 + } + }, + { + "job": 9, + "children": [ + {"child": 1, "reason_s": 61.8219, "setup_s": 13.7398, "import_s": 2.8610, "relevance_rows": 8006}, + {"child": 2, "reason_s": 62.2665, "setup_s": 13.6310, "import_s": 2.8898, "relevance_rows": 8006}, + {"child": 3, "reason_s": 56.5094, "setup_s": 13.5053, "import_s": 2.8397, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 61.8219, + "reason_s_min": 56.5094, + "reason_s_max": 62.2665, + "reason_s_mean": 60.1993, + "reason_s_stdev": 3.2032 + } + }, + { + "job": 10, + "children": [ + {"child": 1, "reason_s": 55.6749, "setup_s": 7.7769, "import_s": 1.7840, "relevance_rows": 8006}, + {"child": 2, "reason_s": 55.3314, "setup_s": 8.1678, "import_s": 1.7904, "relevance_rows": 8006}, + {"child": 3, "reason_s": 55.3241, "setup_s": 7.7732, "import_s": 1.8116, "relevance_rows": 8006} + ], + "job_summary": { + "reason_s_med": 55.3314, + "reason_s_min": 55.3241, + "reason_s_max": 55.6749, + "reason_s_mean": 55.4435, + "reason_s_stdev": 0.2005 + } + } + ] + } +} From 61e6569c7169eaa55bb71a9b63c1b6714263b4b0 Mon Sep 17 00:00:00 2001 From: BAF1927 Date: Thu, 27 Aug 2026 14:17:47 -0400 Subject: [PATCH 3/3] Pyreason 3.7 Version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 54f08ef2..35d71a08 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='pyreason', - version='3.6.0', + version='3.7.0', author='Dyuman Aditya', author_email='dyuman.aditya@gmail.com', description='An explainable inference software supporting annotated, real valued, graph based and temporal logic',