From 90ff398e47079f835b18026a15e359ca0466fdc5 Mon Sep 17 00:00:00 2001 From: Kurt Dirnbauer <16100986+dirnbauer@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:11:49 +0200 Subject: [PATCH 1/4] [TASK] Add temporary Austrian municipality source validation --- .../validate_municipality.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tools/tmp_austria_register/validate_municipality.py diff --git a/tools/tmp_austria_register/validate_municipality.py b/tools/tmp_austria_register/validate_municipality.py new file mode 100644 index 00000000..7d384f01 --- /dev/null +++ b/tools/tmp_austria_register/validate_municipality.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import re +import urllib.parse +from pathlib import Path + +import requests +from bs4 import BeautifulSoup + +OUT = Path("validation-out/municipality") +OUT.mkdir(parents=True, exist_ok=True) +URL = "https://www.oesterreich.gv.at/de/orgsearch/gemeindeauswahl/orgtypegroup/2" + +session = requests.Session() +session.headers.update({ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/126 Safari/537.36", + "Accept-Language": "de-AT,de;q=0.9,en;q=0.5", +}) +response = session.get(URL, timeout=60) +response.raise_for_status() +(OUT / "picker.html").write_text(response.text, encoding="utf-8") +soup = BeautifulSoup(response.text, "lxml") +summary: dict[str, object] = { + "status": response.status_code, + "url": response.url, + "forms": [], + "controls": [], + "links": [], + "scripts": [], + "html_snippets": {}, +} +for form in soup.find_all("form"): + summary["forms"].append({ + "action": form.get("action"), + "method": form.get("method"), + "text": " ".join(form.stripped_strings)[:1000], + }) +for control in soup.find_all(["input", "button", "select"]): + summary["controls"].append({ + "tag": control.name, + "name": control.get("name"), + "id": control.get("id"), + "type": control.get("type"), + "value": control.get("value"), + "aria": control.get("aria-label"), + "text": " ".join(control.stripped_strings)[:200], + }) +for anchor in soup.find_all("a", href=True): + summary["links"].append({ + "text": " ".join(anchor.stripped_strings), + "href": anchor.get("href"), + }) + +terms = [ + "gemeinde", "region", "autocomplete", "suggest", "orgtypegroup", + "regionSelection", "api/", "search", "postal", "municipality", +] +for term in terms: + matches = [] + for match in list(re.finditer(term, response.text, re.I))[:40]: + matches.append( + response.text[max(0, match.start() - 300):match.start() + 900].replace("\n", " ") + ) + summary["html_snippets"][term] = matches + +for index, script in enumerate(soup.find_all("script", src=True)): + src = urllib.parse.urljoin(URL, script.get("src")) + item: dict[str, object] = { + "src": src, + "status": None, + "size": 0, + "matched_terms": [], + "snippets": {}, + } + try: + script_response = session.get(src, timeout=60) + item["status"] = script_response.status_code + item["size"] = len(script_response.content) + if script_response.ok: + text = script_response.text + matched = [term for term in terms if term.casefold() in text.casefold()] + item["matched_terms"] = matched + if matched: + safe_name = re.sub( + r"[^A-Za-z0-9._-]", "_", urllib.parse.urlsplit(src).path + )[-140:] + (OUT / f"script-{index:02d}-{safe_name}.js").write_text(text, encoding="utf-8") + snippets: dict[str, list[str]] = {} + for term in matched: + snippets[term] = [ + text[max(0, match.start() - 400):match.start() + 1200] + for match in list(re.finditer(term, text, re.I))[:30] + ] + item["snippets"] = snippets + except Exception as exc: # noqa: BLE001 + item["error"] = repr(exc) + summary["scripts"].append(item) + +(OUT / "summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8" +) +print(json.dumps({ + "status": summary["status"], + "forms": summary["forms"], + "controls": summary["controls"], + "matching_scripts": [ + { + "src": item["src"], + "size": item["size"], + "matched_terms": item["matched_terms"], + } + for item in summary["scripts"] + if item.get("matched_terms") + ], +}, ensure_ascii=False, indent=2)) From 0fd2fcf81915b7d1fe1f98f0ec85309630274d1d Mon Sep 17 00:00:00 2001 From: Kurt Dirnbauer <16100986+dirnbauer@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:12:20 +0200 Subject: [PATCH 2/4] [TASK] Run temporary Austrian source validation --- .../tmp-austria-register-validation.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/tmp-austria-register-validation.yml diff --git a/.github/workflows/tmp-austria-register-validation.yml b/.github/workflows/tmp-austria-register-validation.yml new file mode 100644 index 00000000..f47823ed --- /dev/null +++ b/.github/workflows/tmp-austria-register-validation.yml @@ -0,0 +1,31 @@ +name: Temporary Austrian register source validation + +on: + pull_request: + branches: + - master + paths: + - 'tools/tmp_austria_register/**' + - '.github/workflows/tmp-austria-register-validation.yml' + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install requests beautifulsoup4 lxml + - run: python tools/tmp_austria_register/validate_municipality.py + - if: always() + uses: actions/upload-artifact@v4 + with: + name: municipality-validation + path: validation-out/** + if-no-files-found: error + retention-days: 1 From 8b2ff158bf33884bb65607f70f4aec7dbbbb375d Mon Sep 17 00:00:00 2001 From: Kurt Dirnbauer <16100986+dirnbauer@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:15:03 +0200 Subject: [PATCH 3/4] [TASK] Validate Austrian municipality result routes --- .../validate_municipality_result.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tools/tmp_austria_register/validate_municipality_result.py diff --git a/tools/tmp_austria_register/validate_municipality_result.py b/tools/tmp_austria_register/validate_municipality_result.py new file mode 100644 index 00000000..726322c0 --- /dev/null +++ b/tools/tmp_austria_register/validate_municipality_result.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import re +from pathlib import Path + +import requests +from bs4 import BeautifulSoup + +OUT = Path("validation-out/municipality-results") +OUT.mkdir(parents=True, exist_ok=True) +BASE = "https://www.oesterreich.gv.at" +URLS = { + "picker-eisenstadt": f"{BASE}/de/orgsearch/gemeindeauswahl/orgtypegroup/2?q=Eisenstadt", + "picker-gkz": f"{BASE}/de/orgsearch/gemeindeauswahl/orgtypegroup/2?q=10101", + "group": f"{BASE}/de/orgsearch/orgtypegroup/2?gkz=10101", + "type": f"{BASE}/de/orgsearch/orgtyp/10?gkz=10101", + "group-node": f"{BASE}/orgsearch/orgtypegroup/2?gkz=10101", + "type-node": f"{BASE}/orgsearch/orgtyp/10?gkz=10101", +} + +session = requests.Session() +session.headers.update({ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/126 Safari/537.36", + "Accept-Language": "de-AT,de;q=0.9,en;q=0.5", +}) +summary: dict[str, object] = {} +for key, url in URLS.items(): + item: dict[str, object] = {"requested_url": url} + try: + response = session.get(url, timeout=60, allow_redirects=True) + item.update({ + "status": response.status_code, + "final_url": response.url, + "size": len(response.content), + "content_type": response.headers.get("content-type"), + }) + (OUT / f"{key}.html").write_text(response.text, encoding="utf-8") + soup = BeautifulSoup(response.text, "lxml") + item["title"] = soup.title.get_text(" ", strip=True) if soup.title else "" + item["h1"] = [h.get_text(" ", strip=True) for h in soup.find_all("h1")] + item["h2"] = [h.get_text(" ", strip=True) for h in soup.find_all("h2")] + item["h3"] = [h.get_text(" ", strip=True) for h in soup.find_all("h3")] + item["external_links"] = [ + {"text": a.get_text(" ", strip=True), "href": a.get("href")} + for a in soup.find_all("a", href=True) + if str(a.get("href")).startswith(("http://", "https://")) + ] + item["gkz_objects"] = [] + for match in re.finditer(r'\\?"gkz\\?"\s*:\s*\\?"?([0-9]{5})', response.text): + start = max(0, match.start() - 300) + item["gkz_objects"].append(response.text[start:match.start() + 900]) + if len(item["gkz_objects"]) >= 20: + break + item["homepage_snippets"] = [] + for term in ["Homepage", "Internet", "Eisenstadt", "vs-eisenstadt", "rathaus"]: + matches = [] + for match in list(re.finditer(term, response.text, re.I))[:10]: + matches.append(response.text[max(0, match.start() - 250):match.start() + 700]) + item["homepage_snippets"].append({"term": term, "matches": matches}) + except Exception as exc: # noqa: BLE001 + item["error"] = repr(exc) + summary[key] = item + +(OUT / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") +print(json.dumps({key: { + "status": value.get("status"), + "final_url": value.get("final_url"), + "h1": value.get("h1"), + "h2": value.get("h2"), + "h3": value.get("h3"), + "external_links": value.get("external_links"), + "gkz_count": len(value.get("gkz_objects", [])), +} for key, value in summary.items()}, ensure_ascii=False, indent=2)) From 48ae333d5ca9710e790b3a53e64e7b9cfc02d204 Mon Sep 17 00:00:00 2001 From: Kurt Dirnbauer <16100986+dirnbauer@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:15:48 +0200 Subject: [PATCH 4/4] [TASK] Validate municipality result routes --- .github/workflows/tmp-austria-register-validation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tmp-austria-register-validation.yml b/.github/workflows/tmp-austria-register-validation.yml index f47823ed..b722eee9 100644 --- a/.github/workflows/tmp-austria-register-validation.yml +++ b/.github/workflows/tmp-austria-register-validation.yml @@ -22,6 +22,7 @@ jobs: python-version: '3.12' - run: python -m pip install requests beautifulsoup4 lxml - run: python tools/tmp_austria_register/validate_municipality.py + - run: python tools/tmp_austria_register/validate_municipality_result.py - if: always() uses: actions/upload-artifact@v4 with: