|
8 | 8 | one README, a DOI cited nowhere else, a root module left a version behind. |
9 | 9 | This script clones the org's repos and checks the invariants that span them. |
10 | 10 |
|
| 11 | +The deployed pages listed in SITES are fetched and flattened to text so that |
| 12 | +the same checks reach them. Without this, the program's most-read claim |
| 13 | +surface would be the only one nothing audits. |
| 14 | +
|
11 | 15 | Source of truth: program/LEDGER.json (see EMBEDDED_LEDGER for the schema). |
12 | 16 | If that file is absent the embedded fallback is used and a warning is emitted. |
13 | 17 |
|
|
23 | 27 | from __future__ import annotations |
24 | 28 |
|
25 | 29 | import argparse |
| 30 | +import html |
26 | 31 | import json |
27 | 32 | import os |
28 | 33 | import re |
29 | 34 | import shutil |
30 | 35 | import subprocess |
31 | 36 | import sys |
32 | 37 | import tempfile |
| 38 | +import urllib.request |
33 | 39 | from collections import defaultdict |
34 | 40 | from dataclasses import dataclass, field, asdict |
35 | 41 |
|
36 | 42 | ORG = "arithmon" |
37 | 43 | REPOS = [".github", "program", "atlas", "sieve", "lean", "k7", "k7-lean"] |
38 | 44 |
|
| 45 | +# The program's front door is uploaded, not committed, so cloning cannot reach |
| 46 | +# it. Each entry is fetched and flattened to text under a pseudo-repo, which |
| 47 | +# lets every claim check below apply to the served page unchanged. Skipped |
| 48 | +# under --no-network. The rendered filename is what LEDGER.claim_surfaces |
| 49 | +# must list for the page to count as a claim surface. |
| 50 | +SITES = {"site": ("https://arithmon.com/", "arithmon-com.md")} |
| 51 | + |
39 | 52 | # Paths whose content is historical by design and must never trigger drift errors. |
40 | 53 | HISTORICAL = re.compile(r"(^|/)(legacy|archive)/|CHANGELOG|/\.git/|/\.lake/|/build/") |
41 | 54 |
|
|
74 | 87 | # Files where a number is a *headline claim* rather than a local subsection |
75 | 88 | # count. Drift checks run here only: this is what keeps the report readable. |
76 | 89 | "claim_surfaces": [ |
77 | | - "README.md", "profile/README.md", "CITATION.md", "CITATION.cff", |
| 90 | + "README.md", "arithmon-com.md", "profile/README.md", "CITATION.md", "CITATION.cff", |
78 | 91 | "STRUCTURE.md", "INDEX.md", "CONFRONTATIONS.md", |
79 | 92 | "docs/wiki/Home.md", "docs/wiki/Home.fr.md", |
80 | 93 | "docs/GIFT_EXEC_SUMMARY.md", "docs/GIFT_EXEC_SUMMARY.fr.md", |
@@ -151,6 +164,48 @@ def clone_all(workdir, quiet=True): |
151 | 164 | return paths |
152 | 165 |
|
153 | 166 |
|
| 167 | +BLOCK_TAGS = "p|div|li|tr|h[1-6]|section|article|dt|dd|blockquote|br" |
| 168 | + |
| 169 | + |
| 170 | +def render_page_text(src): |
| 171 | + """Flatten a served HTML page to one line per block element. |
| 172 | +
|
| 173 | + Link targets are kept inline so that the link checks see them; everything |
| 174 | + else becomes plain text, because the claim checks are line-based and a |
| 175 | + number wrapped in markup would otherwise never match. |
| 176 | + """ |
| 177 | + src = re.sub(r"(?is)<(script|style)\b.*?</\1>", " ", src) |
| 178 | + src = re.sub(r'(?i)<a\b[^>]*href="([^"]+)"[^>]*>', r" \1 ", src) |
| 179 | + src = re.sub(r"(?i)<(%s)\b[^>]*>" % BLOCK_TAGS, "\n", src) |
| 180 | + src = re.sub(r"(?i)</(%s)>" % BLOCK_TAGS, "\n", src) |
| 181 | + src = re.sub(r"<[^>]+>", " ", src) |
| 182 | + src = html.unescape(src) |
| 183 | + lines = (re.sub(r"[ \t ]+", " ", ln).strip() for ln in src.splitlines()) |
| 184 | + return "\n".join(ln for ln in lines if ln) |
| 185 | + |
| 186 | + |
| 187 | +def materialize_sites(workdir, quiet=True): |
| 188 | + """Fetch each deployed page into a pseudo-repo. Returns {key: path}.""" |
| 189 | + paths = {} |
| 190 | + for key, (url, fname) in SITES.items(): |
| 191 | + dest = os.path.join(workdir, key) |
| 192 | + try: |
| 193 | + req = urllib.request.Request( |
| 194 | + url, headers={"User-Agent": "arithmon-consistency-check"}) |
| 195 | + with urllib.request.urlopen(req, timeout=30) as r: |
| 196 | + raw = r.read().decode("utf-8", errors="replace") |
| 197 | + except Exception as exc: |
| 198 | + print(f" ! could not fetch {url}: {exc}", file=sys.stderr) |
| 199 | + continue |
| 200 | + os.makedirs(dest, exist_ok=True) |
| 201 | + with open(os.path.join(dest, fname), "w", encoding="utf-8", newline="\n") as fh: |
| 202 | + fh.write(render_page_text(raw)) |
| 203 | + paths[key] = dest |
| 204 | + if not quiet: |
| 205 | + print(f" fetched {url}") |
| 206 | + return paths |
| 207 | + |
| 208 | + |
154 | 209 | def resolve_local(workdir): |
155 | 210 | """Map repo names onto an existing directory of clones.""" |
156 | 211 | paths = {} |
@@ -594,16 +649,25 @@ def main(): |
594 | 649 | tmp = None |
595 | 650 | try: |
596 | 651 | if args.local: |
597 | | - paths = resolve_local(args.local) |
| 652 | + workdir = args.local |
| 653 | + paths = resolve_local(workdir) |
598 | 654 | if not paths: |
599 | | - print(f"no clones found under {args.local}", file=sys.stderr) |
| 655 | + print(f"no clones found under {workdir}", file=sys.stderr) |
600 | 656 | return 2 |
601 | 657 | else: |
602 | 658 | tmp = tempfile.mkdtemp(prefix="arithmon-audit-") |
| 659 | + workdir = tmp |
603 | 660 | if not args.quiet: |
604 | 661 | print("cloning org repos...") |
605 | 662 | paths = clone_all(tmp, quiet=args.quiet) |
606 | 663 |
|
| 664 | + if args.no_network: |
| 665 | + report.warn("coverage", "-", |
| 666 | + "deployed pages not audited (--no-network): " |
| 667 | + + ", ".join(url for url, _ in SITES.values())) |
| 668 | + else: |
| 669 | + paths.update(materialize_sites(workdir, quiet=args.quiet)) |
| 670 | + |
607 | 671 | missing = [r for r in REPOS if r not in paths] |
608 | 672 | if missing: |
609 | 673 | report.warn("coverage", "-", f"repos not available for audit: {', '.join(missing)}") |
|
0 commit comments