[WIP] Temporary Austrian source validation

.github/workflows/tmp-austria-register-validation.yml

@@ -0,0 +1,32 @@
+name: Temporary Austrian register source validation
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'tools/tmp_austria_register/**'
+      - '.github/workflows/tmp-austria-register-validation.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -m pip install requests beautifulsoup4 lxml
+      - run: python tools/tmp_austria_register/validate_municipality.py
+      - run: python tools/tmp_austria_register/validate_municipality_result.py
+      - if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: municipality-validation
+          path: validation-out/**
+          if-no-files-found: error
+          retention-days: 1

tools/tmp_austria_register/validate_municipality.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import re
+import urllib.parse
+from pathlib import Path
+
+import requests
+from bs4 import BeautifulSoup
+
+OUT = Path("validation-out/municipality")
+OUT.mkdir(parents=True, exist_ok=True)
+URL = "https://www.oesterreich.gv.at/de/orgsearch/gemeindeauswahl/orgtypegroup/2"
+
+session = requests.Session()
+session.headers.update({
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/126 Safari/537.36",
+    "Accept-Language": "de-AT,de;q=0.9,en;q=0.5",
+})
+response = session.get(URL, timeout=60)
+response.raise_for_status()
+(OUT / "picker.html").write_text(response.text, encoding="utf-8")
+soup = BeautifulSoup(response.text, "lxml")
+summary: dict[str, object] = {
+    "status": response.status_code,
+    "url": response.url,
+    "forms": [],
+    "controls": [],
+    "links": [],
+    "scripts": [],
+    "html_snippets": {},
+}
+for form in soup.find_all("form"):
+    summary["forms"].append({
+        "action": form.get("action"),
+        "method": form.get("method"),
+        "text": " ".join(form.stripped_strings)[:1000],
+    })
+for control in soup.find_all(["input", "button", "select"]):
+    summary["controls"].append({
+        "tag": control.name,
+        "name": control.get("name"),
+        "id": control.get("id"),
+        "type": control.get("type"),
+        "value": control.get("value"),
+        "aria": control.get("aria-label"),
+        "text": " ".join(control.stripped_strings)[:200],
+    })
+for anchor in soup.find_all("a", href=True):
+    summary["links"].append({
+        "text": " ".join(anchor.stripped_strings),
+        "href": anchor.get("href"),
+    })
+
+terms = [
+    "gemeinde", "region", "autocomplete", "suggest", "orgtypegroup",
+    "regionSelection", "api/", "search", "postal", "municipality",
+]
+for term in terms:
+    matches = []
+    for match in list(re.finditer(term, response.text, re.I))[:40]:
+        matches.append(
+            response.text[max(0, match.start() - 300):match.start() + 900].replace("\n", " ")
+        )
+    summary["html_snippets"][term] = matches
+
+for index, script in enumerate(soup.find_all("script", src=True)):
+    src = urllib.parse.urljoin(URL, script.get("src"))
+    item: dict[str, object] = {
+        "src": src,
+        "status": None,
+        "size": 0,
+        "matched_terms": [],
+        "snippets": {},
+    }
+    try:
+        script_response = session.get(src, timeout=60)
+        item["status"] = script_response.status_code
+        item["size"] = len(script_response.content)
+        if script_response.ok:
+            text = script_response.text
+            matched = [term for term in terms if term.casefold() in text.casefold()]
+            item["matched_terms"] = matched
+            if matched:
+                safe_name = re.sub(
+                    r"[^A-Za-z0-9._-]", "_", urllib.parse.urlsplit(src).path
+                )[-140:]
+                (OUT / f"script-{index:02d}-{safe_name}.js").write_text(text, encoding="utf-8")
+                snippets: dict[str, list[str]] = {}
+                for term in matched:
+                    snippets[term] = [
+                        text[max(0, match.start() - 400):match.start() + 1200]
+                        for match in list(re.finditer(term, text, re.I))[:30]
+                    ]
+                item["snippets"] = snippets
+    except Exception as exc:  # noqa: BLE001
+        item["error"] = repr(exc)
+    summary["scripts"].append(item)
+
+(OUT / "summary.json").write_text(
+    json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8"
+)
+print(json.dumps({
+    "status": summary["status"],
+    "forms": summary["forms"],
+    "controls": summary["controls"],
+    "matching_scripts": [
+        {
+            "src": item["src"],
+            "size": item["size"],
+            "matched_terms": item["matched_terms"],
+        }
+        for item in summary["scripts"]
+        if item.get("matched_terms")
+    ],
+}, ensure_ascii=False, indent=2))

tools/tmp_austria_register/validate_municipality_result.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import requests
+from bs4 import BeautifulSoup
+
+OUT = Path("validation-out/municipality-results")
+OUT.mkdir(parents=True, exist_ok=True)
+BASE = "https://www.oesterreich.gv.at"
+URLS = {
+    "picker-eisenstadt": f"{BASE}/de/orgsearch/gemeindeauswahl/orgtypegroup/2?q=Eisenstadt",
+    "picker-gkz": f"{BASE}/de/orgsearch/gemeindeauswahl/orgtypegroup/2?q=10101",
+    "group": f"{BASE}/de/orgsearch/orgtypegroup/2?gkz=10101",
+    "type": f"{BASE}/de/orgsearch/orgtyp/10?gkz=10101",
+    "group-node": f"{BASE}/orgsearch/orgtypegroup/2?gkz=10101",
+    "type-node": f"{BASE}/orgsearch/orgtyp/10?gkz=10101",
+}
+
+session = requests.Session()
+session.headers.update({
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/126 Safari/537.36",
+    "Accept-Language": "de-AT,de;q=0.9,en;q=0.5",
+})
+summary: dict[str, object] = {}
+for key, url in URLS.items():
+    item: dict[str, object] = {"requested_url": url}
+    try:
+        response = session.get(url, timeout=60, allow_redirects=True)
+        item.update({
+            "status": response.status_code,
+            "final_url": response.url,
+            "size": len(response.content),
+            "content_type": response.headers.get("content-type"),
+        })
+        (OUT / f"{key}.html").write_text(response.text, encoding="utf-8")
+        soup = BeautifulSoup(response.text, "lxml")
+        item["title"] = soup.title.get_text(" ", strip=True) if soup.title else ""
+        item["h1"] = [h.get_text(" ", strip=True) for h in soup.find_all("h1")]
+        item["h2"] = [h.get_text(" ", strip=True) for h in soup.find_all("h2")]
+        item["h3"] = [h.get_text(" ", strip=True) for h in soup.find_all("h3")]
+        item["external_links"] = [
+            {"text": a.get_text(" ", strip=True), "href": a.get("href")}
+            for a in soup.find_all("a", href=True)
+            if str(a.get("href")).startswith(("http://", "https://"))
+        ]
+        item["gkz_objects"] = []
+        for match in re.finditer(r'\\?"gkz\\?"\s*:\s*\\?"?([0-9]{5})', response.text):
+            start = max(0, match.start() - 300)
+            item["gkz_objects"].append(response.text[start:match.start() + 900])
+            if len(item["gkz_objects"]) >= 20:
+                break
+        item["homepage_snippets"] = []
+        for term in ["Homepage", "Internet", "Eisenstadt", "vs-eisenstadt", "rathaus"]:
+            matches = []
+            for match in list(re.finditer(term, response.text, re.I))[:10]:
+                matches.append(response.text[max(0, match.start() - 250):match.start() + 700])
+            item["homepage_snippets"].append({"term": term, "matches": matches})
+    except Exception as exc:  # noqa: BLE001
+        item["error"] = repr(exc)
+    summary[key] = item
+
+(OUT / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
+print(json.dumps({key: {
+    "status": value.get("status"),
+    "final_url": value.get("final_url"),
+    "h1": value.get("h1"),
+    "h2": value.get("h2"),
+    "h3": value.get("h3"),
+    "external_links": value.get("external_links"),
+    "gkz_count": len(value.get("gkz_objects", [])),
+} for key, value in summary.items()}, ensure_ascii=False, indent=2))