From 752539b462c58f9012d719b7e235ffb1337520e5 Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Fri, 2 Oct 2026 09:31:35 -0400 Subject: [PATCH 1/8] Add SSH-config awareness layer and multi-forge providers gendia only knew the forges it had a hand-written provider for, and the accounts it could reach had to be typed into gendia.json by hand. Most users already describe their forges in ~/.ssh/config, so reading that file removes the double bookkeeping and lets gendia absorb the standalone git-helpers script. This adds the ssh/ package (config parser, JSON forge registry, key auditor, account synthesiser), Gitea, Azure DevOps and Bitbucket Server providers, GitHub Enterprise host support, the generate / scan / convert operations, the `gendia ssh` command group, `setup --from-ssh`, an ssh keys section in `doctor`, and the manual_repos / forges_file config keys. --- examples/forges.json | 51 ++ examples/ssh-bootstrap.json | 65 ++ pyproject.toml | 3 + src/gendia/cli/arguments.py | 143 ++++- src/gendia/cli/commands/doctor.py | 20 + src/gendia/cli/commands/setup.py | 68 ++ src/gendia/cli/commands/ssh.py | 277 ++++++++ src/gendia/config/__init__.py | 2 + src/gendia/config/loader.py | 6 + src/gendia/config/schema.py | 46 +- src/gendia/data/forges.json | 48 ++ src/gendia/operations/__init__.py | 6 + src/gendia/operations/convert.py | 146 +++++ src/gendia/operations/generate.py | 598 ++++++++++++++++++ src/gendia/operations/scan.py | 136 ++++ src/gendia/providers/azure.py | 128 ++++ src/gendia/providers/bitbucket_server.py | 122 ++++ src/gendia/providers/factory.py | 6 + src/gendia/providers/gitea.py | 117 ++++ src/gendia/providers/github.py | 31 +- src/gendia/ssh/__init__.py | 29 + src/gendia/ssh/bootstrap.py | 147 +++++ src/gendia/ssh/config.py | 316 +++++++++ src/gendia/ssh/forges.py | 273 ++++++++ src/gendia/ssh/inspector.py | 298 +++++++++ src/gendia/util/repo_scan.py | 173 +++++ src/gendia/util/url.py | 243 +++++++ tests/unit/test_forge_registry.py | 90 +++ tests/unit/test_forges_file.py | 51 ++ tests/unit/test_manual_repos.py | 50 ++ tests/unit/test_op_convert.py | 55 ++ tests/unit/test_op_generate.py | 117 ++++ .../unit/test_op_generate_account_mapping.py | 113 ++++ tests/unit/test_op_scan.py | 53 ++ tests/unit/test_provider_azure.py | 46 ++ tests/unit/test_provider_gitea.py | 64 ++ tests/unit/test_provider_github_enterprise.py | 30 + tests/unit/test_repo_scan.py | 63 ++ tests/unit/test_ssh_config.py | 142 +++++ tests/unit/test_url_convert.py | 89 +++ 40 files changed, 4455 insertions(+), 6 deletions(-) create mode 100644 examples/forges.json create mode 100644 examples/ssh-bootstrap.json create mode 100644 src/gendia/cli/commands/ssh.py create mode 100644 src/gendia/data/forges.json create mode 100644 src/gendia/operations/convert.py create mode 100644 src/gendia/operations/generate.py create mode 100644 src/gendia/operations/scan.py create mode 100644 src/gendia/providers/azure.py create mode 100644 src/gendia/providers/bitbucket_server.py create mode 100644 src/gendia/providers/gitea.py create mode 100644 src/gendia/ssh/__init__.py create mode 100644 src/gendia/ssh/bootstrap.py create mode 100644 src/gendia/ssh/config.py create mode 100644 src/gendia/ssh/forges.py create mode 100644 src/gendia/ssh/inspector.py create mode 100644 src/gendia/util/repo_scan.py create mode 100644 src/gendia/util/url.py create mode 100644 tests/unit/test_forge_registry.py create mode 100644 tests/unit/test_forges_file.py create mode 100644 tests/unit/test_manual_repos.py create mode 100644 tests/unit/test_op_convert.py create mode 100644 tests/unit/test_op_generate.py create mode 100644 tests/unit/test_op_generate_account_mapping.py create mode 100644 tests/unit/test_op_scan.py create mode 100644 tests/unit/test_provider_azure.py create mode 100644 tests/unit/test_provider_gitea.py create mode 100644 tests/unit/test_provider_github_enterprise.py create mode 100644 tests/unit/test_repo_scan.py create mode 100644 tests/unit/test_ssh_config.py create mode 100644 tests/unit/test_url_convert.py diff --git a/examples/forges.json b/examples/forges.json new file mode 100644 index 0000000..cc6cba8 --- /dev/null +++ b/examples/forges.json @@ -0,0 +1,51 @@ +{ + "_comment": "Example custom forge registry. Point gendia.json's `forges_file` at this file to extend the bundled catalogue.", + + "known_forges_exact": { + "github.com": "github", + "gitlab.com": "gitlab", + "bitbucket.org": "bitbucket", + "codeberg.org": "codeberg", + "gitea.com": "gitea", + "ssh.dev.azure.com": "azure-devops", + "vs-ssh.visualstudio.com": "azure-devops", + + "git.acme-corp.example": "github-enterprise", + "code.client-x.example": "gitlab" + }, + + "known_forges_patterns": [ + ["github", "github-enterprise"], + ["gitlab", "gitlab"], + ["forgejo", "forgejo"], + ["gitea", "gitea"], + ["bitbucket", "bitbucket-server"], + ["gogs", "gitea"] + ], + + "public_forges": [ + "github.com", + "gitlab.com", + "bitbucket.org", + "codeberg.org", + "gitea.com" + ], + + "comment_keywords": [ + ["github enterprise", "github-enterprise"], + ["github", "github"], + ["gitlab", "gitlab"], + ["bitbucket server", "bitbucket-server"], + ["bitbucket", "bitbucket"], + ["forgejo", "forgejo"], + ["codeberg", "codeberg"], + ["gitea", "gitea"], + ["azure devops", "azure-devops"], + ["gogs", "gitea"] + ], + + "tlds": [ + "com", "org", "net", "edu", "io", "dev", + "co", "uk", "us", "ca", "de", "fr" + ] +} diff --git a/examples/ssh-bootstrap.json b/examples/ssh-bootstrap.json new file mode 100644 index 0000000..ebf0596 --- /dev/null +++ b/examples/ssh-bootstrap.json @@ -0,0 +1,65 @@ +{ + "_comment": "Example showing the new SSH-aware fields: forges_file, manual_repos, and a Gitea + Azure DevOps account alongside GitHub/GitLab/Bitbucket.", + + "forges_file": "~/.config/gendia/forges.json", + + "accounts": { + "personal": { + "platform": "github", + "username": "yourhandle", + "credential_ref": "GITHUB_TOKEN_PERSONAL", + "ssh_key_path": "~/.ssh/keys/personal/id_ed25519", + "git_auth": "ssh", + "manual_repos": [ + { + "host": "github-personal", + "org": "yourhandle", + "repos": ["dotfiles", "scratch"] + } + ] + }, + + "ghe-work": { + "platform": "github", + "host": "ghe.acme-corp.example", + "org": "platform", + "credential_ref": "GITHUB_TOKEN_GHE_WORK", + "git_auth": "https" + }, + + "self-hosted-gitlab": { + "platform": "gitlab", + "host": "gitlab.acme-corp.example", + "group": "tools", + "credential_ref": "GITLAB_TOKEN_INTERNAL" + }, + + "azure": { + "platform": "azure", + "org": "myorg", + "workspace": "Platform", + "credential_ref": "AZURE_DEVOPS_PAT", + "git_auth": "https" + }, + + "codeberg": { + "platform": "gitea", + "username": "yourhandle", + "credential_ref": "CODEBERG_TOKEN", + "git_auth": "ssh" + } + }, + + "registries": { + "packagist": { + "kind": "packagist", + "username": "your-packagist-handle", + "credential_ref": "PACKAGIST_API_TOKEN" + } + }, + + "defaults": { + "concurrency": 4, + "log_format": "human" + } +} diff --git a/pyproject.toml b/pyproject.toml index d58fcbe..971bec1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,9 @@ gendia = "gendia.__main__:main" [tool.hatch.build.targets.wheel] packages = ["src/gendia"] +[tool.hatch.build.targets.wheel.force-include] +"src/gendia/data/forges.json" = "gendia/data/forges.json" + [tool.ruff] line-length = 100 target-version = "py312" diff --git a/src/gendia/cli/arguments.py b/src/gendia/cli/arguments.py index 3b44eb1..0828d0b 100644 --- a/src/gendia/cli/arguments.py +++ b/src/gendia/cli/arguments.py @@ -39,6 +39,8 @@ from gendia.operations import ( AuditOperation, CleanupOperation, + ConvertOperation, + GenerateOperation, InitOperation, InventoryOperation, MirrorOperation, @@ -46,11 +48,23 @@ OperationContext, OperationResult, ReleaseOperation, + ScanOperation, StatusOperation, SyncOperation, VerifyOperation, ) -from gendia.operations import conventions as conventions_op +from gendia.operations import ( + conventions as conventions_op, +) +from gendia.operations import ( + convert as convert_op, +) +from gendia.operations import ( + generate as generate_op, +) +from gendia.operations import ( + scan as scan_op, +) from gendia.providers import build_provider from gendia.registries import build_registry from gendia.util.paths import env_file_path @@ -145,6 +159,20 @@ def build_parser() -> argparse.ArgumentParser: # conventions [PATH] [--rules FILE] [--strict] [--json] conventions_op.add_subparser(sub) + # generate [--format ...] [--output FILE] [--execute] ... + generate_op.add_subparser(sub) + + # scan [PATH] [--check-dirty] [--json] + scan_op.add_subparser(sub) + + # convert URL... [--to ssh|https] + convert_op.add_subparser(sub) + + # ssh {list, test, inspect, bootstrap} + from gendia.cli.commands import ssh as ssh_cmd # noqa: PLC0415 + + ssh_cmd.add_subparser(sub) + return parser @@ -189,6 +217,12 @@ def _csv(value: str) -> frozenset[str]: "Lint repo hygiene: GitHub-special files, naming, ban-list glyphs, " "spec filename rules, sub-folder readmes, shell-script shebangs." ), + "generate": ( + "Emit copy-paste git commands per detected SSH host " + "(text/script/json/markdown; --execute runs the bash form)." + ), + "scan": "Find git repos on disk and classify each (clean / dirty / orphan / https).", + "convert": "Flip git URLs between SSH and HTTPS (alias + insteadOf aware).", } @@ -224,6 +258,13 @@ def dispatch(args: argparse.Namespace) -> int: # noqa: PLR0911 — many sidecar return identity_cmd.dispatch(args) if args.command == "conventions": return conventions_op.dispatch(args) + if args.command == "ssh": + from gendia.cli.commands import ssh as ssh_cmd # noqa: PLC0415 + + return ssh_cmd.dispatch(args) + + if args.command in {"generate", "scan", "convert"}: + return _run_project_optional(args) try: config = load_config(project_path=args.config) @@ -297,6 +338,106 @@ def _build_credentials(args: argparse.Namespace) -> CredentialResolver: return CredentialResolver(*backends) +def _run_project_optional(args: argparse.Namespace) -> int: + """Run `generate` / `scan` / `convert`, which don't require a project block. + + These ops still want a config (for accounts + forges_file). When loading + fails or no project exists, we fall back to a stub project so the existing + OperationContext shape stays usable. + """ + config: GendiaConfig | None = None + try: + config = load_config(project_path=args.config) + except ConfigError as exc: + _log.warning("config load failed; continuing without it", extra={"error": str(exc)}) + + if config is None or config.project is None: + # Build a stub config that still preserves any accounts the user + # already configured (so generate can discover repos via providers). + accounts = config.accounts if config else {} + registries = config.registries if config else {} + defaults = config.defaults if config else Defaults() + forges_file = config.forges_file if config else None + stub_project = Project( + name="", + account=next(iter(accounts), "") or "", + root=Path.cwd(), + repos=(RepoSpec(dir="."),), + ) + if not accounts: + accounts = { + "": Account( + name="", + platform="github", + credential_ref="STUB", + org="", + ) + } + config = GendiaConfig( + accounts=accounts, + registries=registries, + project=stub_project, + defaults=defaults, + forges_file=forges_file, + ) + + credentials = _build_credentials(args) + project = config.project + if project is None: + # Defensive: the rebuild above guarantees a stub project, but mypy + # can't see that across the conditional reassignment. + raise RuntimeError("internal: project should not be None at this point") + + account_name = project.account + if account_name not in config.accounts: + # Pick any available account as the provider seed; ops that don't + # need provider calls will still function. + account_name = next(iter(config.accounts)) + account = config.account(account_name) + provider = build_provider(account, credentials) + registry = ( + build_registry(config.registry(project.registry), credentials) if project.registry else None + ) + + ctx = OperationContext( + config=config, + project=project, + provider=provider, + registry=registry, + dry_run=getattr(args, "dry_run", False), + workers=(getattr(args, "concurrency", None) or config.defaults.concurrency), + only=getattr(args, "only", frozenset()) or frozenset(), + skip=getattr(args, "skip", frozenset()) or frozenset(), + ) + + operation: Operation + suppress_summary = False + if args.command == "generate": + forges_path = Path(config.forges_file).expanduser() if config.forges_file else None + operation = GenerateOperation( + ctx, + request=generate_op.request_from_args(args, forges_file=forges_path), + ) + # The operation writes its own structured output; the summary line + # would only confuse json/script consumers. + suppress_summary = getattr(args, "fmt", "text") in {"json", "script"} or bool( + getattr(args, "output", None) + ) + elif args.command == "scan": + operation = ScanOperation(ctx, request=scan_op.request_from_args(args)) + suppress_summary = bool(getattr(args, "json_out", False)) + elif args.command == "convert": + operation = ConvertOperation(ctx, request=convert_op.request_from_args(args)) + suppress_summary = bool(getattr(args, "json_out", False)) + else: + raise ValueError(f"unexpected command: {args.command}") + + result = operation.run() + if not suppress_summary: + _print_result(result, args) + return 0 if result.all_ok else 1 + + def _run_init(args: argparse.Namespace) -> int: """Init doesn't need a config; build a minimal stub context.""" stub_account = Account(name="stub", platform="github", credential_ref="STUB", org="stub") diff --git a/src/gendia/cli/commands/doctor.py b/src/gendia/cli/commands/doctor.py index 363e3a6..6560d4c 100644 --- a/src/gendia/cli/commands/doctor.py +++ b/src/gendia/cli/commands/doctor.py @@ -110,6 +110,7 @@ def dispatch(args: argparse.Namespace) -> int: _check_env_file(report, args.env_file or env_file_path()) _check_binaries(report) _check_ssh(report, args.config) + _check_ssh_keys(report) _check_credentials(report, args.config, args.env_file or env_file_path()) report.render() @@ -263,6 +264,25 @@ def _check_ssh(report: Report, project_config: Path | None) -> None: report.add("ok", section, f"account {account.name!r}: key {key} mode {mode:o}") +def _check_ssh_keys(report: Report) -> None: + """Audit IdentityFiles referenced in ~/.ssh/config (mode + strength + reuse).""" + section = "ssh keys" + from gendia.ssh.config import SSHConfig # noqa: PLC0415 + from gendia.ssh.inspector import audit_keys # noqa: PLC0415 + + cfg = SSHConfig.load() + findings = audit_keys(cfg.hosts().exclude_wildcards()) + if not findings: + return # silent when there are no IdentityFile entries to audit + for finding in findings: + report.add( + severity=finding.severity, + section=section, + message=f"{finding.path}: {finding.message}", + fix=finding.fix, + ) + + def _check_credentials(report: Report, project_config: Path | None, env_path: Path) -> None: section = "credentials" try: diff --git a/src/gendia/cli/commands/setup.py b/src/gendia/cli/commands/setup.py index 63837d9..42a93c5 100644 --- a/src/gendia/cli/commands/setup.py +++ b/src/gendia/cli/commands/setup.py @@ -22,6 +22,7 @@ from dataclasses import dataclass from importlib import resources from pathlib import Path +from typing import Any from gendia.auth.env_file import secure_create_dir from gendia.observability.logger import get_logger @@ -55,6 +56,14 @@ def add_subparser(subparsers: argparse._SubParsersAction[argparse.ArgumentParser action="store_true", help="Print what would be written, don't actually create files.", ) + p.add_argument( + "--from-ssh", + action="store_true", + help=( + "Synthesize accounts from ~/.ssh/config and merge them into " + "~/.config/gendia/gendia.json (use --dry-run to preview)." + ), + ) _HANDLERS_WITH_FLAGS = {"local", "vps", "project", "docker"} @@ -65,6 +74,9 @@ def dispatch(args: argparse.Namespace) -> int: if args.detect: return _print_detection() + if args.from_ssh: + return _setup_from_ssh(force=args.force, dry_run=args.dry_run) + shape = args.shape or _resolve_default_shape(force=args.force) if shape in _HANDLERS_WITH_FLAGS: @@ -417,6 +429,62 @@ def _setup_ci() -> int: # --- helpers ----------------------------------------------------------------- +def _setup_from_ssh(*, force: bool, dry_run: bool) -> int: + """Synthesize accounts from ~/.ssh/config and merge into gendia.json.""" + import json # noqa: PLC0415 — keep stdlib imports local to the rare path + + from gendia.ssh.bootstrap import propose_accounts, render_proposed_config # noqa: PLC0415 + from gendia.ssh.config import SSHConfig # noqa: PLC0415 + from gendia.ssh.forges import load_registry # noqa: PLC0415 + + cfg = SSHConfig.load() + if not cfg.hosts(): + print("no Host entries in ~/.ssh/config; nothing to bootstrap") + return 1 + + stubs = propose_accounts(config=cfg, registry=load_registry()) + if not stubs: + print( + "no recognized forge hosts in ~/.ssh/config " + "(use `gendia ssh list` to inspect the parser's view)." + ) + return 1 + + proposal = render_proposed_config(stubs) + if dry_run: + sys.stdout.write(proposal) + print(f"\n(dry-run) would merge {len(stubs)} account(s) into ~/.config/gendia/gendia.json") + return 0 + + target = Path.home() / ".config" / "gendia" / "gendia.json" + target.parent.mkdir(parents=True, exist_ok=True) + + existing: dict[str, Any] = {} + if target.is_file(): + try: + existing = json.loads(target.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + if not force: + print(f" ✗ {target} is not valid JSON: {exc}", file=sys.stderr) + print(" pass --force to overwrite the unparseable file", file=sys.stderr) + return 2 + print(f" ⚠ {target} is invalid JSON; --force given, overwriting") + + accounts = dict(existing.get("accounts") or {}) + proposed = json.loads(proposal) + accounts.update(proposed["accounts"]) + existing["accounts"] = accounts + + target.write_text(json.dumps(existing, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f" ✓ wrote {len(stubs)} account(s) to {target}") + print() + print("Next steps:") + print(f" 1. $EDITOR {target} # review the proposal, fill in real scopes") + print(" 2. gendia config set GIT_TOKEN_