diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 00000000..2da6b952 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "name": "auplc-skills", + "owner": { + "name": "AMD Research" + }, + "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", + "metadata": { + "version": "0.1.1" + }, + "plugins": [ + { + "name": "auplc", + "source": "./", + "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs." + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 00000000..49a42888 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,21 @@ +{ + "name": "auplc", + "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs.", + "version": "0.1.1", + "author": { + "name": "AMD Research" + }, + "homepage": "https://github.com/AMDResearch/aup-learning-cloud", + "repository": "https://github.com/AMDResearch/aup-learning-cloud", + "keywords": [ + "aup-learning-cloud", + "auplc", + "jupyterhub", + "k3s", + "rocm", + "pxe", + "ansible", + "helm", + "deployment" + ] +} diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json new file mode 100644 index 00000000..2da6b952 --- /dev/null +++ b/.cursor-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "name": "auplc-skills", + "owner": { + "name": "AMD Research" + }, + "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", + "metadata": { + "version": "0.1.1" + }, + "plugins": [ + { + "name": "auplc", + "source": "./", + "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs." + } + ] +} diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json new file mode 100644 index 00000000..cfac9ec2 --- /dev/null +++ b/.cursor-plugin/plugin.json @@ -0,0 +1,19 @@ +{ + "name": "auplc", + "version": "0.1.1", + "description": "Skills for deploying and maintaining AUP Learning Cloud: install, deploy, configure courses, build images, upgrade, troubleshoot, configure auth, manage users and quota, monitor, expose with TLS/storage, configure repo cloning, and author courses for the multi-node JupyterHub-on-k3s platform for AMD GPUs.", + "author": { + "name": "AMD Research" + }, + "keywords": [ + "aup-learning-cloud", + "auplc", + "jupyterhub", + "k3s", + "rocm", + "pxe", + "ansible", + "helm", + "deployment" + ] +} diff --git a/.github/scripts/check.sh b/.github/scripts/check.sh new file mode 100755 index 00000000..7d0cc3f8 --- /dev/null +++ b/.github/scripts/check.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Validate skills, version metadata, skill tests, and generated plugin manifests. +# +# Usage: +# ./.github/scripts/check.sh Run every skill-package validation. +# ./.github/scripts/check.sh -h|--help Print this help. +# +# Requires `uv` (https://github.com/astral-sh/uv). + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +usage() { + sed -n 's/^# \{0,1\}//p' "${BASH_SOURCE[0]}" | sed -n '/^Usage:/,/^Requires/p' +} + +case "${1:-}" in + "") + uv run .github/scripts/validate_skills.py + uv run python scripts/check_skills_version.py + uv run --extra test pytest tests/skills + uv run .github/scripts/generate_cursor_marketplace.py --check + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 + exit 2 + ;; +esac diff --git a/.github/scripts/generate_cursor_marketplace.py b/.github/scripts/generate_cursor_marketplace.py new file mode 100755 index 00000000..a2721b0d --- /dev/null +++ b/.github/scripts/generate_cursor_marketplace.py @@ -0,0 +1,195 @@ +#!/usr/bin/env -S uv run --quiet +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Generate the Cursor plugin manifests from the canonical sources. + +`auplc-skills` ships as a single bundled plugin: the whole repository is one +plugin whose `skills/` folder every supported agent discovers automatically +(this mirrors how `cloudflare/skills` is published). To avoid drift, the Cursor +manifests are generated from the Claude manifests rather than hand-maintained. + +Sources of truth: +- `plugin-metadata.json` (repo root): shared identity and discovery metadata + (name, description, version, author, homepage, repository, + keywords). This is the vendor-neutral metadata file, reused by every + marketplace/manifest target. It is NOT a plugin manifest. +- `.claude-plugin/marketplace.json`: the marketplace catalog with the single + bundled plugin entry and its human-readable description (hand-maintained, + since the catalog blurb intentionally differs from the SKILL.md routing + descriptions). +- `.claude-plugin/plugin.json`: the bundled plugin manifest (hand-maintained). + +Outputs: +- `.cursor-plugin/marketplace.json`: a mirror of the Claude marketplace so + Cursor exposes exactly the same plugin as Claude. +- `.cursor-plugin/plugin.json`: the Cursor plugin manifest derived from the + Claude plugin manifest + `plugin-metadata.json`. + +Usage: + uv run .github/scripts/generate_cursor_marketplace.py # write + uv run .github/scripts/generate_cursor_marketplace.py --check # validate only + +`--check` fails if any generated file is stale or if the Claude manifests' +top-level identity has drifted from `plugin-metadata.json`. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent.parent +PLUGIN_METADATA = ROOT / "plugin-metadata.json" +CLAUDE_MARKETPLACE = ROOT / ".claude-plugin" / "marketplace.json" +CLAUDE_PLUGIN = ROOT / ".claude-plugin" / "plugin.json" +CURSOR_MARKETPLACE = ROOT / ".cursor-plugin" / "marketplace.json" +CURSOR_PLUGIN = ROOT / ".cursor-plugin" / "plugin.json" + + +def load_json(path: Path) -> dict: + if not path.exists(): + raise FileNotFoundError(f"Missing required file: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def check_identity_consistency(metadata: dict, claude: dict, claude_plugin: dict) -> list[str]: + """Return error strings if the Claude manifests' identity has drifted from + the canonical `plugin-metadata.json`.""" + errors: list[str] = [] + + name = metadata.get("name") + description = metadata.get("description") + version = metadata.get("version") + + if claude.get("name") != name: + errors.append( + f".claude-plugin/marketplace.json `name` ({claude.get('name')!r}) " + f"must match plugin-metadata.json `name` ({name!r})." + ) + claude_description = claude.get("description") + if claude_description != description: + errors.append(".claude-plugin/marketplace.json `description` must match plugin-metadata.json `description`.") + claude_version = (claude.get("metadata") or {}).get("version") + if claude_version != version: + errors.append( + f".claude-plugin/marketplace.json metadata.version " + f"({claude_version!r}) must match plugin-metadata.json `version` " + f"({version!r})." + ) + + # The single bundled plugin entry's name must match the plugin manifest. + plugins = claude.get("plugins") + if not isinstance(plugins, list) or len(plugins) != 1: + errors.append(".claude-plugin/marketplace.json must list exactly one bundled plugin (source `./`).") + else: + entry_name = plugins[0].get("name") + if entry_name != claude_plugin.get("name"): + errors.append( + f".claude-plugin/marketplace.json plugin `name` ({entry_name!r}) " + f"must match .claude-plugin/plugin.json `name` " + f"({claude_plugin.get('name')!r})." + ) + + if claude_plugin.get("version") != version: + errors.append( + f".claude-plugin/plugin.json `version` " + f"({claude_plugin.get('version')!r}) must match plugin-metadata.json " + f"`version` ({version!r})." + ) + return errors + + +def build_cursor_marketplace(metadata: dict, claude: dict) -> dict: + author = metadata.get("author") or {} + owner_name = author.get("name") if isinstance(author, dict) else None + + return { + "name": metadata["name"], + "owner": {"name": owner_name} if owner_name else {}, + "description": metadata["description"], + "metadata": { + "version": metadata["version"], + }, + "plugins": claude.get("plugins", []), + } + + +def build_cursor_plugin(metadata: dict, claude_plugin: dict) -> dict: + return { + "name": claude_plugin["name"], + "version": metadata["version"], + "description": claude_plugin.get("description", metadata["description"]), + "author": metadata.get("author") or {}, + "keywords": metadata.get("keywords", []), + } + + +def render_json(data: dict) -> str: + return json.dumps(data, indent=2, ensure_ascii=False) + "\n" + + +def write_or_check(path: Path, content: str, check: bool) -> bool: + """Return True when the file is already up to date.""" + current = path.read_text(encoding="utf-8") if path.exists() else None + if current == content: + return True + if check: + return False + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return True + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Generate the .cursor-plugin/ manifests from the canonical " + "Claude manifests and plugin-metadata.json." + ) + parser.add_argument( + "--check", + action="store_true", + help="Validate the generated manifests are up to date without writing.", + ) + args = parser.parse_args(argv) + + metadata = load_json(PLUGIN_METADATA) + claude = load_json(CLAUDE_MARKETPLACE) + claude_plugin = load_json(CLAUDE_PLUGIN) + + identity_errors = check_identity_consistency(metadata, claude, claude_plugin) + if identity_errors: + print("Plugin manifest identity is inconsistent:", file=sys.stderr) + for err in identity_errors: + print(f" - {err}", file=sys.stderr) + return 1 + + targets = { + CURSOR_MARKETPLACE: render_json(build_cursor_marketplace(metadata, claude)), + CURSOR_PLUGIN: render_json(build_cursor_plugin(metadata, claude_plugin)), + } + + stale = [path for path, content in targets.items() if not write_or_check(path, content, check=args.check)] + + if args.check: + if stale: + for path in stale: + print(f"{path.relative_to(ROOT)} is out of date.", file=sys.stderr) + print( + "Run: uv run .github/scripts/generate_cursor_marketplace.py", + file=sys.stderr, + ) + return 1 + print("Cursor plugin manifests are up to date.") + return 0 + + for path in targets: + print(f"Wrote {path.relative_to(ROOT)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/publish.sh b/.github/scripts/publish.sh new file mode 100755 index 00000000..0732dce5 --- /dev/null +++ b/.github/scripts/publish.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Regenerate every committed artifact derived from skills/ and the +# canonical marketplace + metadata sources. +# +# Usage: +# ./.github/scripts/publish.sh Regenerate all derived artifacts. +# ./.github/scripts/publish.sh --check Verify derived artifacts are up to date. +# ./.github/scripts/publish.sh -h|--help Print this help. +# +# Currently regenerates: +# - .cursor-plugin/marketplace.json (mirror of .claude-plugin/marketplace.json) +# - .cursor-plugin/plugin.json (derived from .claude-plugin/plugin.json +# + plugin-metadata.json) +# +# The `.claude-plugin/` manifests are hand-maintained because the human-facing +# plugin description intentionally differs from the SKILL.md routing +# descriptions; ./.github/scripts/check.sh enforces that they stay consistent +# with plugin-metadata.json. +# +# Requires `uv` (https://github.com/astral-sh/uv). + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +usage() { + sed -n 's/^# \{0,1\}//p' "${BASH_SOURCE[0]}" | sed -n '/^Usage:/,/^Requires/p' +} + +case "${1:-}" in + "") + uv run .github/scripts/generate_cursor_marketplace.py + echo "Publish artifacts generated successfully." + ;; + --check) + uv run .github/scripts/generate_cursor_marketplace.py --check + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 + exit 2 + ;; +esac diff --git a/.github/scripts/validate_skills.py b/.github/scripts/validate_skills.py new file mode 100755 index 00000000..11a953cb --- /dev/null +++ b/.github/scripts/validate_skills.py @@ -0,0 +1,373 @@ +#!/usr/bin/env -S uv run --quiet +# /// script +# requires-python = ">=3.10" +# dependencies = ["pyyaml>=6.0"] +# /// +"""Validate auplc-skills against the standardized Agent Skills format. + +Enforces the repository's skill format and governance requirements: + + - SKILL.md exists at the skill root + - YAML frontmatter is parseable + - `name` is lowercase-with-hyphens, <=64 chars, no `anthropic`/`claude` + substrings, and matches the directory name + - `description` is a non-empty string <=1024 chars + - SKILL.md body is <=500 lines + - skill-card.md exists at the skill root and has non-empty + `## Description` and `## Owner` sections + +Also validates the bundled-plugin manifests: `.claude-plugin/marketplace.json` +must list exactly one plugin whose `source` is `./` (the whole repo is one +plugin, mirroring how `cloudflare/skills` is published), and +`.claude-plugin/plugin.json` must exist with a matching `name`. + +Run from the repo root: + + ./.github/scripts/check.sh # used locally; thin wrapper + uv run .github/scripts/validate_skills.py # validate every skill + manifest + uv run .github/scripts/validate_skills.py --skills-dir skills + uv run .github/scripts/validate_skills.py --list # print skill names as JSON + uv run .github/scripts/validate_skills.py --skill deploy-aup-learning-cloud + uv run .github/scripts/validate_skills.py --marketplace-only # manifest only + +The `--list` / `--skill` options let CI validate each skill in its own job +(see .github/workflows/validate-skills.yml) so a single bad skill doesn't mask the +status of the others. + +Exits non-zero if any validated skill (or the marketplace check) fails. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +DEFAULT_SKILLS_DIR = REPO_ROOT / "skills" +CLAUDE_MARKETPLACE = REPO_ROOT / ".claude-plugin" / "marketplace.json" +CLAUDE_PLUGIN = REPO_ROOT / ".claude-plugin" / "plugin.json" + +# Limits from the standardized Agent Skills format and repository policy. +MAX_NAME_LEN = 64 +MAX_DESCRIPTION_LEN = 1024 +MAX_BODY_LINES = 500 + +NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +FRONTMATTER_RE = re.compile( + r"\A---\r?\n(?P.*?)\r?\n---\r?\n?(?P.*)\Z", + re.DOTALL, +) +RESERVED_NAME_SUBSTRINGS = ("anthropic", "claude") + +# Per-skill governance card (see plugin-docs/skill-cards.md). Each section must be a +# top-level `##` heading followed by some non-empty body text. +CARD_FILENAME = "skill-card.md" +REQUIRED_CARD_SECTIONS = ("Description", "Owner") + + +@dataclass +class SkillReport: + skill: str + errors: list[str] = field(default_factory=list) + + +def validate_skill(skill_dir: Path) -> SkillReport: + """Run every validation rule against `skill_dir` and return a report.""" + report = SkillReport(skill=skill_dir.name) + skill_md = skill_dir / "SKILL.md" + + if not skill_md.exists(): + report.errors.append("Missing SKILL.md.") + return report + + text = skill_md.read_text(encoding="utf-8") + match = FRONTMATTER_RE.match(text) + if match is None: + report.errors.append( + "SKILL.md must start with a `---` YAML frontmatter block followed by `---` on its own line." + ) + return report + + try: + frontmatter = yaml.safe_load(match.group("frontmatter")) + except yaml.YAMLError as exc: + report.errors.append(f"YAML frontmatter is invalid: {exc}") + return report + + if not isinstance(frontmatter, dict): + report.errors.append("YAML frontmatter must be a mapping with at least `name` and `description`.") + return report + + _validate_name(frontmatter.get("name"), skill_dir.name, report) + _validate_description(frontmatter.get("description"), report) + _validate_body(match.group("body"), report) + _validate_card(skill_dir, report) + return report + + +def _validate_name(name: object, dir_name: str, report: SkillReport) -> None: + if not isinstance(name, str) or not name: + report.errors.append("Frontmatter `name` is missing or not a non-empty string.") + return + + if len(name) > MAX_NAME_LEN: + report.errors.append(f"`name` length {len(name)} exceeds {MAX_NAME_LEN} characters.") + if not NAME_RE.match(name): + report.errors.append( + f"`name` `{name}` must be lowercase-with-hyphens (letters, digits, single hyphens between segments)." + ) + for sub in RESERVED_NAME_SUBSTRINGS: + if sub in name.lower(): + report.errors.append(f"`name` may not contain `{sub}`.") + if name != dir_name: + report.errors.append(f"`name` `{name}` must match the skill directory name `{dir_name}`.") + + +def _validate_description(description: object, report: SkillReport) -> None: + if not isinstance(description, str) or not description: + report.errors.append("Frontmatter `description` is missing or not a non-empty string.") + return + if len(description) > MAX_DESCRIPTION_LEN: + report.errors.append(f"`description` length {len(description)} exceeds {MAX_DESCRIPTION_LEN} characters.") + + +def _validate_body(body: str, report: SkillReport) -> None: + # Skip surrounding blank lines so the blank line after `---` doesn't + # inflate the count. + lines = body.splitlines() + while lines and not lines[0].strip(): + lines.pop(0) + while lines and not lines[-1].strip(): + lines.pop() + if len(lines) > MAX_BODY_LINES: + report.errors.append( + f"SKILL.md body is {len(lines)} lines; max is {MAX_BODY_LINES}. " + "Move reference material into sibling files (reference.md, " + "examples.md, ...) and link to them from SKILL.md." + ) + + +def _validate_card(skill_dir: Path, report: SkillReport) -> None: + """Require a skill-card.md with non-empty Description, Owner.""" + card = skill_dir / CARD_FILENAME + if not card.exists(): + report.errors.append( + f"Missing {CARD_FILENAME} (governance card). See plugin-docs/skill-cards.md; " + "it needs `## Description` and `## Owner` sections." + ) + return + + sections = _parse_card_sections(card.read_text(encoding="utf-8")) + for name in REQUIRED_CARD_SECTIONS: + body = sections.get(name.lower()) + if body is None: + report.errors.append(f"{CARD_FILENAME} is missing a `## {name}` section.") + elif not body.strip(): + report.errors.append(f"{CARD_FILENAME} `## {name}` section is empty.") + + +def _parse_card_sections(text: str) -> dict[str, str]: + """Map each `##` heading (lowercased) to the text until the next heading.""" + sections: dict[str, str] = {} + current: str | None = None + buffer: list[str] = [] + + def flush() -> None: + if current is not None: + sections[current] = "\n".join(buffer).strip() + + for line in text.splitlines(): + heading = re.match(r"^##\s+(?P.+?)\s*$", line) + if heading: + flush() + current = heading.group("title").lower() + buffer = [] + elif current is not None: + buffer.append(line) + flush() + return sections + + +def discover_skills(root: Path) -> list[Path]: + """List skill directories under `root`, ignoring dotfiles.""" + if not root.exists(): + return [] + return sorted(p for p in root.iterdir() if p.is_dir() and not p.name.startswith(".")) + + +def validate_claude_marketplace() -> list[str]: + """Validate the single bundled-plugin manifests. + + `auplc-skills` is published as one plugin whose `source` is `./` (the whole + repo), so the marketplace must list exactly one plugin and a matching + `.claude-plugin/plugin.json` must exist. The marketplace's human-readable + `description` is intentionally allowed to differ from the SKILL.md + descriptions, so its text is not cross-checked. + """ + errors: list[str] = [] + + if not CLAUDE_MARKETPLACE.exists(): + return [ + f"Missing {CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}; expected a " + "single bundled-plugin entry (source `./`)." + ] + + try: + data = json.loads(CLAUDE_MARKETPLACE.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return [f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: invalid JSON: {exc}"] + + plugins = data.get("plugins") if isinstance(data, dict) else None + if not isinstance(plugins, list): + return [f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: top-level `plugins` array is missing."] + if len(plugins) != 1: + return [ + f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: expected exactly one " + f"bundled plugin (source `./`), found {len(plugins)}." + ] + + entry = plugins[0] + if not isinstance(entry, dict): + return [f"{CLAUDE_MARKETPLACE.relative_to(REPO_ROOT)}: plugins[0] must be an object."] + + name = entry.get("name") + source = entry.get("source") + description = entry.get("description") + + if not isinstance(name, str) or not name: + errors.append("plugins[0] is missing a non-empty `name`.") + if source != "./": + errors.append(f"plugins[0]: `source` must be `./`, got `{source}`.") + if not isinstance(description, str) or not description.strip(): + errors.append("plugins[0] is missing a non-empty `description`.") + + if not CLAUDE_PLUGIN.exists(): + errors.append( + f"Missing {CLAUDE_PLUGIN.relative_to(REPO_ROOT)}; the bundled plugin " + "needs a `.claude-plugin/plugin.json` manifest." + ) + else: + try: + plugin = json.loads(CLAUDE_PLUGIN.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + errors.append(f"{CLAUDE_PLUGIN.relative_to(REPO_ROOT)}: invalid JSON: {exc}") + else: + plugin_name = plugin.get("name") if isinstance(plugin, dict) else None + if plugin_name != name: + errors.append( + f"{CLAUDE_PLUGIN.relative_to(REPO_ROOT)} `name` " + f"({plugin_name!r}) must match the marketplace plugin " + f"`name` ({name!r})." + ) + + return errors + + +def _print_report(report: SkillReport) -> int: + """Print a single skill report and return its error count.""" + status = "OK " if not report.errors else "FAIL" + print(f"[{status}] {report.skill}") + for err in report.errors: + print(f" {err}") + return len(report.errors) + + +def list_skills(skills_dir: Path) -> int: + """Print discovered skill names as a compact JSON array (for CI matrices).""" + skills = discover_skills(skills_dir) + if not skills: + print(f"No skills found under {skills_dir}", file=sys.stderr) + return 1 + print(json.dumps([p.name for p in skills], separators=(",", ":"))) + return 0 + + +def run_single(skills_dir: Path, name: str) -> int: + """Validate a single skill directory by name (no marketplace cross-check).""" + skill_dir = skills_dir / name + if not skill_dir.is_dir(): + print(f"No such skill directory: {skill_dir}", file=sys.stderr) + return 1 + + errors = _print_report(validate_skill(skill_dir)) + print(f"\nSummary: {errors} error(s) in skill `{name}`") + return 0 if errors == 0 else 1 + + +def run_marketplace(skills_dir: Path) -> int: + """Validate only the bundled-plugin manifests.""" + marketplace_errors = validate_claude_marketplace() + status = "OK " if not marketplace_errors else "FAIL" + print(f"[{status}] .claude-plugin/marketplace.json") + for err in marketplace_errors: + print(f" {err}") + print(f"\nSummary: {len(marketplace_errors)} error(s) in marketplace manifest") + return 0 if not marketplace_errors else 1 + + +def run(skills_dir: Path) -> int: + skills = discover_skills(skills_dir) + if not skills: + print(f"No skills found under {skills_dir}", file=sys.stderr) + return 1 + + print(f"Validating {len(skills)} skill(s) in {skills_dir}\n") + total_errors = 0 + for skill_dir in skills: + total_errors += _print_report(validate_skill(skill_dir)) + + marketplace_errors = validate_claude_marketplace() + marketplace_status = "OK " if not marketplace_errors else "FAIL" + print(f"\n[{marketplace_status}] .claude-plugin/marketplace.json") + for err in marketplace_errors: + print(f" {err}") + total_errors += len(marketplace_errors) + + print(f"\nSummary: {total_errors} error(s) across {len(skills)} skill(s)") + return 0 if total_errors == 0 else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "--skills-dir", + type=Path, + default=DEFAULT_SKILLS_DIR, + help=f"Directory containing skill folders (default: {DEFAULT_SKILLS_DIR}).", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--list", + action="store_true", + help="Print discovered skill names as a JSON array and exit.", + ) + group.add_argument( + "--skill", + metavar="NAME", + help="Validate only the named skill directory (skips the marketplace cross-check, which is repo-wide).", + ) + group.add_argument( + "--marketplace-only", + action="store_true", + help="Only validate that marketplace.json is in sync with skills/.", + ) + args = parser.parse_args(argv) + skills_dir = args.skills_dir.resolve() + + if args.list: + return list_skills(skills_dir) + if args.skill: + return run_single(skills_dir, args.skill) + if args.marketplace_only: + return run_marketplace(skills_dir) + return run(skills_dir) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml new file mode 100644 index 00000000..70ba5487 --- /dev/null +++ b/.github/workflows/validate-skills.yml @@ -0,0 +1,124 @@ +name: validate-skills + +on: + push: + branches: [main] + paths: + - "skills/**" + - "templates/**" + - ".claude-plugin/**" + - ".cursor-plugin/**" + - ".github/scripts/**" + - ".github/workflows/validate-skills.yml" + - "pyproject.toml" + - "plugin-metadata.json" + - "scripts/check_skills_version.py" + - "tests/skills/**" + - "README-SKILL.md" + - "plugin-docs/**" + pull_request: + paths: + - "skills/**" + - "templates/**" + - ".claude-plugin/**" + - ".cursor-plugin/**" + - ".github/scripts/**" + - ".github/workflows/validate-skills.yml" + - "pyproject.toml" + - "plugin-metadata.json" + - "scripts/check_skills_version.py" + - "tests/skills/**" + - "README-SKILL.md" + - "plugin-docs/**" + workflow_dispatch: + +# Least privilege: these jobs only read the repo to validate skills/manifests. +permissions: + contents: read + +jobs: + # Enumerate the skills so the validation job can fan out over them with a + # matrix. Running each skill in its own job (with fail-fast disabled) means + # one broken skill shows up as a single red check instead of failing the + # whole suite and hiding the status of every other skill. + discover-skills: + name: Discover skills + runs-on: ubuntu-latest + outputs: + skills: ${{ steps.discover.outputs.skills }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: List skills + id: discover + run: echo "skills=$(uv run .github/scripts/validate_skills.py --list)" >> "$GITHUB_OUTPUT" + + validate-skill: + name: Validate skill + needs: discover-skills + runs-on: ubuntu-latest + strategy: + # Don't cancel the other skills when one fails; we want to see every + # skill's status in a single run. + fail-fast: false + matrix: + skill: ${{ fromJson(needs.discover-skills.outputs.skills) }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Validate skill + run: uv run .github/scripts/validate_skills.py --skill "${{ matrix.skill }}" + + # Repo-wide checks that aren't tied to a single skill: version sync, public + # skill CLI tests, and generated plugin manifests. + validate-manifests: + name: Validate plugin metadata and manifests + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Validate marketplace manifest + run: uv run .github/scripts/validate_skills.py --marketplace-only + + - name: Validate skill version sync + run: uv run python scripts/check_skills_version.py + + - name: Run skill tests + run: uv run --extra test pytest tests/skills + + - name: Validate generated Cursor manifest + run: uv run .github/scripts/generate_cursor_marketplace.py --check + + # Single gate that aggregates the per-skill matrix and the repo-wide manifest + # checks. Branch protection can require just this one check: it only passes + # when every skill validated and the manifest job succeeded. Because matrix + # jobs always succeed individually under `fail-fast: false`, we inspect the + # job results explicitly rather than relying on `needs` short-circuiting. + validate: + name: Validate skills and plugin manifests + needs: [validate-skill, validate-manifests] + if: always() + runs-on: ubuntu-latest + steps: + - name: Verify all validation jobs passed + run: | + echo "validate-skill result: ${{ needs.validate-skill.result }}" + echo "validate-manifests result: ${{ needs.validate-manifests.result }}" + if [ "${{ needs.validate-skill.result }}" != "success" ] || \ + [ "${{ needs.validate-manifests.result }}" != "success" ]; then + echo "One or more validation jobs failed." >&2 + exit 1 + fi + echo "All skill and manifest validations passed." diff --git a/README-SKILL.md b/README-SKILL.md new file mode 100644 index 00000000..a854d736 --- /dev/null +++ b/README-SKILL.md @@ -0,0 +1,220 @@ +# AUP Learning Cloud Skills (`auplc-skills`) + +Agent Skills that help any coding agent deploy and maintain +[AUP Learning Cloud](https://github.com/AMDResearch/aup-learning-cloud) — the +multi-node JupyterHub-on-k3s teaching platform for AMD GPUs. + +Skills follow the standardized [Agent Skills](https://github.com/anthropics/skills) +format and interoperate with the major coding agents: Cursor, Claude Code, +OpenAI Codex, and Gemini CLI. + +> **Tech preview.** The catalog spans install → deploy → configure → build → +> upgrade → troubleshoot. Expect frequent changes while the foundations settle; +> the skills are a first draft to review with the operators who own each area. + +## The catalog + +The skills are organized into three groups so an agent can start in the right +place for a task. It is still **one bundled plugin** — installing it brings +every skill at once; the groups are a routing aid (each skill's `description` +also carries its `Group:` tag). See +[plugin-docs/skill-categories.md](plugin-docs/skill-categories.md) for the full taxonomy and +routing guidance. + +### Plan and deploy AUP Learning Cloud + +Bring the platform into existence: size it, install or deploy it, and build its +images. + +| Skill | What it does | Status | +| --- | --- | --- | +| [`plan-aup-learning-cloud-deployment`](skills/plan-aup-learning-cloud-deployment/SKILL.md) | Size a new deployment for a prospective adopter: interview course/headcount needs and the network, research current AMD silicon, then recommend how many AIPCs/workstations/servers and routers/switches to buy, the topology, an IP plan, and a buyer-facing bill of materials. | in-repo | +| [`install-aup-learning-cloud-single-node`](skills/install-aup-learning-cloud-single-node/SKILL.md) | Install on a single AMD GPU/APU box with the `./auplc-installer` flow: prerequisites, GPU/courses/image flags, gated install, verify at `localhost:30890`. | in-repo | +| [`deploy-aup-learning-cloud`](skills/deploy-aup-learning-cloud/SKILL.md) | Deploy end to end on a multi-AIPC PXE-diskless or SSH-preinstalled k3s cluster: interview the operator, generate the Ansible inventory + PXE vars + Helm values (helper scripts), then drive the install with confirmation gates at risky steps. | in-repo | +| [`build-aup-learning-cloud-images`](skills/build-aup-learning-cloud-images/SKILL.md) | Build and publish the Hub and notebook/course Docker images with `img build`, incl. GPU-target tagging and registry push. | in-repo | + +### Maintain AUP Learning Cloud + +Operate and keep a running deployment healthy: upgrade, debug, observe, secure +logins, manage users, and control network/storage exposure. + +| Skill | What it does | Status | +| --- | --- | --- | +| [`upgrade-aup-learning-cloud`](skills/upgrade-aup-learning-cloud/SKILL.md) | Upgrade the JupyterHub chart/values/images and the k3s cluster on a running deployment, in a safe order with rollback. | in-repo | +| [`troubleshoot-aup-learning-cloud`](skills/troubleshoot-aup-learning-cloud/SKILL.md) | Diagnose netboot, node-join, GPU scheduling, storage, and auth failures from runtime evidence, then hand off the fix. | in-repo | +| [`monitor-aup-learning-cloud`](skills/monitor-aup-learning-cloud/SKILL.md) | Wire the Hub into Prometheus + Grafana: ServiceMonitor, authenticated metrics, dashboards, alert rules, and the metrics NetworkPolicy. | in-repo | +| [`configure-aup-learning-cloud-auth`](skills/configure-aup-learning-cloud-auth/SKILL.md) | Configure the auth mode (auto-login/dummy/github/multi), the GitHub App / OAuth + team sync, native accounts, and admin bootstrap. | in-repo | +| [`manage-aup-learning-cloud-users`](skills/manage-aup-learning-cloud-users/SKILL.md) | Day-2 user/group/quota operations via the admin console and `manage_users.py`: bulk onboarding, passwords, admins, and quota grants/refresh. | in-repo | +| [`expose-aup-learning-cloud`](skills/expose-aup-learning-cloud/SKILL.md) | Take a deployment past the local defaults: NodePort/LoadBalancer/ingress + TLS, CORS origins, externally-terminated TLS, and shared NFS storage. | in-repo | + +### Course and other editor + +Edit what lives inside the platform: the course catalog, new course content, and +per-user repository cloning. + +| Skill | What it does | Status | +| --- | --- | --- | +| [`configure-aup-learning-cloud-courses`](skills/configure-aup-learning-cloud-courses/SKILL.md) | Edit the course catalog, spawn-UI metadata, GPU accelerator selectors, team mappings, and quota in `values.yaml`, then re-apply. | in-repo | +| [`develop-aup-learning-cloud-courses`](skills/develop-aup-learning-cloud-courses/SKILL.md) | Author a new course end to end: notebooks under `projects/`, a course image, and catalog registration, then build + wire it in. | in-repo | +| [`configure-aup-learning-cloud-repos`](skills/configure-aup-learning-cloud-repos/SKILL.md) | Configure per-user Git repo cloning: the spawn-form repo field/picker, private-repo tokens, provider allowlist, and clone persistence. | in-repo | + +## What is a skill? + +A skill is a self-contained folder that bundles everything an agent needs to +perform a focused task: instructions, helper scripts, and references. At its +core is a `SKILL.md` file with YAML frontmatter — a `name` and a short +`description` that tells the agent *when* the skill should activate — followed +by the guidance the agent reads while the skill is in use. + +``` +skills/ + deploy-aup-learning-cloud/ + SKILL.md # routing frontmatter + workflow + skill-card.md # governance card (Description, Owner) + reference.md # full step-by-step commands + troubleshooting + scripts/ # executable helpers +``` + +When an agent decides a skill is relevant (or you invoke it explicitly), it +loads `SKILL.md` and follows the instructions inside. Descriptions stay in +context cheaply; the full body loads only when the task actually matches. + +## Installation + +The whole catalog ships as a single bundled plugin (`auplc`), so any of the +methods below installs every skill at once. Pick the one that matches your +agent. + +### Claude Code + +Install with the [plugin marketplace](https://code.claude.com/docs/en/plugin-marketplaces): + +``` +/plugin marketplace add AMDResearch/aup-learning-cloud +/plugin install auplc@auplc-skills +``` + +### Cursor + +Install from the Cursor Marketplace, or add manually via **Settings → Rules → +Add Rule → Remote Rule (Github)** with `AMDResearch/aup-learning-cloud`. Cursor scans +the repo and copies the skills into `.cursor/skills/`. + +### npx skills + +Install with the [`npx skills`](https://skills.sh) CLI (works with any agent +that follows the Agent Skills standard): + +``` +npx skills add https://github.com/AMDResearch/aup-learning-cloud +``` + +### Clone / Copy + +Clone this repo and copy (or symlink) the skill folders you want from `skills/` +into your agent's skills directory. Each agent discovers `SKILL.md` +automatically. + +```bash +git clone https://github.com/AMDResearch/aup-learning-cloud.git +cp -r aup-learning-cloud/skills/deploy-aup-learning-cloud <agent-skills-dir>/ +``` + +| Agent | Skills directory (personal / project) | +| --- | --- | +| Cursor | `~/.cursor/skills/` / `.cursor/skills/` | +| Claude Code | `~/.claude/skills/` / `.claude/skills/` | +| Codex | `$HOME/.agents/skills` / `$REPO_ROOT/.agents/skills` | + +## Recommended models + +These skills drive long, gated workflows — Ansible runs, `kubectl`/`helm` +rollouts, netboot setup — where the agent has to hold a plan across many phases +and stop at each confirmation gate. They work best on a frontier reasoning model +with the reasoning effort turned up. + +| Agent | Model | Reasoning effort | +| --- | --- | --- | +| Claude Code | Opus 4.8 | high | +| Codex | GPT-5.6-Sol | high | +| OpenCode | DeepSeek V4 Flash | high | + +Any agent that follows the Agent Skills standard can load the catalog. If yours +isn't listed, pick its strongest reasoning model and raise the effort/thinking +setting to high. + +## Using a skill + +Once installed, reference it in plain language while talking to your agent. In +most cases the agent picks the right skill on its own from the description. + +### Example prompts — the three ways to stand up a deployment + +There are three deployment paths. Pick the prompt that matches your hardware; +the agent routes to the right skill and interviews you for the rest. + +- **Single node** (one AMD GPU/APU box → `install-aup-learning-cloud-single-node`): + + > *"Install AUP Learning Cloud on this single AMD GPU workstation with the + > `./auplc-installer` flow and verify it at `localhost:30890`."* + +- **Multi-node, PXE diskless netboot** (one service machine netboots diskless + agents → `deploy-aup-learning-cloud`, `topology: pxe-diskless`): + + > *"Deploy AUP Learning Cloud across my 3 AIPCs over PXE — the machine I'm on + > right now is the head/service node, and the other two are diskless agents + > that should netboot and auto-join k3s."* + +- **Multi-node, SSH pre-installed** (every node already runs Ubuntu, reachable + over SSH → `deploy-aup-learning-cloud`, `topology: ssh-preinstalled`): + + > *"Deploy AUP Learning Cloud on my 4 nodes that already run Ubuntu 24.04 and + > are reachable over SSH — the machine I'm on right now is the head/server + > node, install k3s and ROCm on all of them with Ansible, no PXE."* + +The two multi-node prompts both drive `deploy-aup-learning-cloud`; its Phase 1a +gate asks you to confirm the topology (`pxe-diskless` vs `ssh-preinstalled`) +before touching any machine. + +> **Tip — watch every command live in your own tmux.** These deploy/install +> skills run a lot of shell commands (Ansible, `kubectl`, `helm`, netboot +> setup). To see exactly what an agent runs, have it drive a tmux session you +> keep open instead of its hidden shell. First, open the session: +> +> ```bash +> tmux new -s auplc +> ``` +> +> Then tell the agent to send commands to it, e.g.: +> +> > *"Run every shell command by sending it to my tmux session `auplc` with +> > `tmux send-keys -t auplc '<command>' Enter`, then read the pane with +> > `tmux capture-pane -t auplc -p` to check the result — don't use your own +> > shell."* +> +> You watch the commands and their output scroll in the `auplc` pane in real +> time, and can hit `Ctrl-C` there to stop anything that looks wrong. This works +> in any agent that has terminal access (Claude Code, Cursor, Codex). + +## Repository layout + +``` +skills/ # All skills the agent can load +templates/skill-template # Starting point for a new skill +plugin-docs/ # Plugin authoring + governance docs +.claude-plugin/ # Claude marketplace + bundled-plugin manifest (hand-maintained) +.cursor-plugin/ # Cursor marketplace + plugin manifest (generated) +plugin-metadata.json # Vendor-neutral identity/discovery metadata +.github/scripts/ # Validation + publish scripts +.github/workflows/ # CI that validates skills and manifests +``` + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for authoring conventions and +[plugin-docs/adding-a-skill.md](plugin-docs/adding-a-skill.md) for the step-by-step procedure +to add a new skill. Run the same checks CI runs before opening a PR: + +```bash +./.github/scripts/check.sh +``` diff --git a/README.md b/README.md index 37eb6219..efcc2ae7 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ Full documentation is available at: **https://amdresearch.github.io/aup-learning - [Authentication Guide](https://amdresearch.github.io/aup-learning-cloud/jupyterhub/authentication-guide.html) - GitHub App and native authentication - [User Management Guide](https://amdresearch.github.io/aup-learning-cloud/jupyterhub/user-management.html) - Batch user operations with scripts - [User Quota System](https://amdresearch.github.io/aup-learning-cloud/jupyterhub/quota-system.html) - Resource usage tracking and quota management +- [AUP Learning Cloud Skills](README-SKILL.md) - Agent Skills for deploying and maintaining AUP Learning Cloud ## Contributing diff --git a/plugin-docs/adding-a-skill.md b/plugin-docs/adding-a-skill.md new file mode 100644 index 00000000..8b7854ce --- /dev/null +++ b/plugin-docs/adding-a-skill.md @@ -0,0 +1,73 @@ +# Adding a skill + +This catalog is designed to grow. Each capability for working with AUP Learning +Cloud (deploying, course configuration, image building, upgrades, troubleshooting) +is its own self-contained skill folder under `skills/`. Adding one is a fixed, +five-step procedure. + +## 1. Copy the template + +```bash +cp -r templates/skill-template skills/<your-skill-name> +``` + +Use a `lowercase-with-hyphens` name tied to the outcome (e.g. +`configure-aup-learning-cloud-courses`, `build-aup-learning-cloud-images`, +`upgrade-aup-learning-cloud`). Avoid generic names like `helper` or `utils`. + +## 2. Write `SKILL.md` + +- Set `name:` in the frontmatter to **exactly** the directory name. +- Write a `description:` in the third person that states **what** the skill + produces and **when** an agent should reach for it, including the trigger + words a user is likely to say. Keep it under 1024 characters. +- **Assign a category.** Pick exactly one group from + [skill-categories.md](skill-categories.md) and **prepend its `Group:` tag** to + the start of the `description` (e.g. `Group: Maintain AUP Learning Cloud.`). + This is what lets an agent route to the right group first. +- Keep the body under 500 lines. Push long reference material into sibling + files (`reference.md`, `examples.md`, ...) linked one level deep. + +See [CONTRIBUTING.md](../CONTRIBUTING.md) for the full authoring conventions. + +## 3. Write `skill-card.md` + +Fill in the `## Description` and `## Owner` sections. See +[skill-cards.md](skill-cards.md). + +## 4. List the skill in the catalog + +The repo ships as a single bundled plugin (`source: "./"`), so the plugin +manifests do **not** need a per-skill entry — dropping the folder under +`skills/` is enough for every install method to pick it up. Add a row to the +catalog table in the [skills README](../README-SKILL.md) **under the skill's group section**, +list it under that group in [skill-categories.md](skill-categories.md) so people +can discover it, then keep the Cursor manifests in sync: + +```bash +./.github/scripts/publish.sh # regenerates .cursor-plugin/ from the canonical sources +``` + +## 5. Validate + +```bash +./.github/scripts/check.sh # same command CI runs +``` + +CI runs the same validation on every pull request via +`.github/workflows/validate-skills.yml`, fanning out one job per skill so a single +broken skill is easy to spot. + +## Ideas for future skills + +The catalog now covers install, deploy, configure (courses), build, upgrade, +and troubleshoot, plus auth, user/quota management, monitoring, network/storage +exposure, per-user repo cloning, and course authoring (see the +[skills README](../README-SKILL.md)). Natural next additions, each following the same +procedure: + +| Skill | Outcome | +| --- | --- | +| `backup-aup-learning-cloud` | Back up and restore the Hub DB PVC and user home data (snapshot, off-cluster copy, restore drill). | +| `offline-aup-learning-cloud` | Drive the air-gapped `pack`/`pack --local` bundle workflow end to end, including registry/PyPI/npm mirrors. | +| `tune-aup-learning-cloud-resources` | Right-size per-course CPU/memory/GPU requirements, prePuller, and node scheduling for a given fleet. | diff --git a/plugin-docs/skill-cards.md b/plugin-docs/skill-cards.md new file mode 100644 index 00000000..3690a292 --- /dev/null +++ b/plugin-docs/skill-cards.md @@ -0,0 +1,41 @@ +# Skill cards + +Every skill in this catalog ships a `skill-card.md` next to its `SKILL.md`. The card is a short, human-facing governance record: it tells a reviewer *what* the skill is and *who* owns it, without making them read the source first. + +A `SKILL.md` is written for the agent (routing and instructions). A skill card is written for the people deciding whether to trust, install, or maintain the skill. + +## Required sections + +The card is intentionally minimal. Two sections are required, each a top-level `##` heading with non-empty body text: + +| Section | Question it answers | +| --- | --- | +| Description | What does this skill do, in one sentence? | +| Owner | Who is accountable for maintaining it? | + +The validator (`.github/scripts/validate_skills.py`) fails any skill whose card is missing or whose required sections are absent or empty. + +## Template + +Copy this into `skills/<your-skill>/skill-card.md`: + +```markdown +# Skill Card + +## Description + +<one sentence: what the skill does, for whom> + +## Owner + +<team or org accountable for maintenance, e.g. AMD Research> +``` + +## Writing a good Description + +Keep it to one sentence that states the outcome, matching the marketplace blurb. Avoid restating internal mechanics (that belongs in `SKILL.md`). + +``` +Good: Deploy AUP Learning Cloud onto a multi-AIPC PXE/k3s cluster end to end. +Bad: Runs a series of Ansible playbooks and Helm commands in order. +``` diff --git a/plugin-docs/skill-categories.md b/plugin-docs/skill-categories.md new file mode 100644 index 00000000..73755d5f --- /dev/null +++ b/plugin-docs/skill-categories.md @@ -0,0 +1,80 @@ +# Skill categories + +The catalog is one bundled plugin, but its skills are organized into three +groups so an agent (or a person) can start in the right place for a task. The +grouping is a routing aid, not a packaging boundary: installing the plugin +brings every skill, and each skill's `SKILL.md` `description` begins with a +`Group:` tag so the category travels with the routing signal the agent loads. + +When you get a task, identify the group first, then pick the skill within it. + +## Plan and deploy AUP Learning Cloud + +Bring the platform into existence: size it, install or deploy it, and build the +images it runs. Reach for this group when nothing is running yet (or you are +adding/replacing infrastructure) and the goal is to stand the platform up. + +- `plan-aup-learning-cloud-deployment` — pre-purchase sizing, topology, network + plan, and bill of materials. +- `install-aup-learning-cloud-single-node` — single-box `./auplc-installer` + install. +- `deploy-aup-learning-cloud` — multi-node PXE / SSH + Ansible + Helm cluster + deploy. +- `build-aup-learning-cloud-images` — build/publish the Hub and notebook/course + images. + +Tag: `Group: Plan & deploy AUP Learning Cloud.` + +## Maintain AUP Learning Cloud + +Operate and keep a running deployment healthy. Reach for this group when the +platform already exists and the goal is day-2 operations: upgrades, debugging, +observability, login security, user/quota administration, and how the Hub is +exposed and stored. + +- `upgrade-aup-learning-cloud` — chart/values/image and k3s upgrades with + rollback. +- `troubleshoot-aup-learning-cloud` — evidence-first diagnosis of a broken + deployment. +- `monitor-aup-learning-cloud` — Prometheus/Grafana, ServiceMonitor, alerts. +- `configure-aup-learning-cloud-auth` — auth mode, GitHub App/OAuth, team sync, + native accounts, admin bootstrap. +- `manage-aup-learning-cloud-users` — users/groups/quota operations and class + onboarding. +- `expose-aup-learning-cloud` — NodePort/LoadBalancer/ingress + TLS, CORS, and + NFS storage. + +Tag: `Group: Maintain AUP Learning Cloud.` + +## Course and other editor + +Edit what lives inside the platform. Reach for this group when the cluster is +fine and the goal is content: the spawnable course catalog, authoring new course +material, or the per-user repositories learners pull into their workspaces. + +- `configure-aup-learning-cloud-courses` — edit the course catalog, spawn-UI + metadata, accelerators, team mapping, and quota knobs in `values.yaml`. +- `develop-aup-learning-cloud-courses` — author a new course end to end + (notebooks → image → catalog registration). +- `configure-aup-learning-cloud-repos` — per-user Git repo cloning (spawn-form + field/picker, private-repo tokens, persistence). + +Tag: `Group: Course & other editor.` + +## Cross-group handoffs + +Tasks often cross a boundary; hand off rather than stretch a skill: + +- Sizing/planning (plan) hands off to install or deploy once a plan is agreed. +- Authoring a course (develop, editor group) hands off to + `build-aup-learning-cloud-images` (deploy group) to build the image, then to + `configure-aup-learning-cloud-courses` (editor group) to wire it in. +- `troubleshoot` (maintain) diagnoses, then hands the fix to the matching + deploy/install/configure/upgrade skill. + +## Adding a new skill + +Assign exactly one group, prepend the group's `Group:` tag to the new skill's +`description`, add its row under that group's table in the +[skills README](../README-SKILL.md), and list it here. See +[adding-a-skill.md](adding-a-skill.md) for the full procedure. diff --git a/plugin-metadata.json b/plugin-metadata.json new file mode 100644 index 00000000..28935c85 --- /dev/null +++ b/plugin-metadata.json @@ -0,0 +1,22 @@ +{ + "name": "auplc-skills", + "description": "Agent Skills for deploying and maintaining AUP Learning Cloud.", + "version": "0.1.1", + "author": { + "name": "AMD Research" + }, + "homepage": "https://github.com/AMDResearch/aup-learning-cloud", + "repository": "https://github.com/AMDResearch/aup-learning-cloud", + "license": "MIT", + "keywords": [ + "aup-learning-cloud", + "auplc", + "jupyterhub", + "k3s", + "rocm", + "pxe", + "ansible", + "helm", + "deployment" + ] +} diff --git a/pyproject.toml b/pyproject.toml index 190d05ee..fe43c64e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ [project] name = "aup-learning-cloud" -version = "0.1.0" +version = "0.1.1" description = "AUP Learning Cloud - JupyterHub deployment for AI education" requires-python = ">=3.10" diff --git a/scripts/check_skills_version.py b/scripts/check_skills_version.py new file mode 100644 index 00000000..1e682333 --- /dev/null +++ b/scripts/check_skills_version.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +""" +Check that the bundled auplc-skills version fields stay in sync. + +The project version in pyproject.toml is the single source of truth (strategy +A + C: skills share the main project version and are pinned at install time via +git tags/refs). This script is READ-ONLY: it only compares the version strings +declared across the plugin manifests against pyproject.toml and reports any +mismatch. It never edits skills or any other file. + +Checked version fields: + - pyproject.toml -> [project].version (source of truth) + - .claude-plugin/marketplace.json -> metadata.version + - .cursor-plugin/marketplace.json -> metadata.version + - .claude-plugin/plugin.json -> version + - .cursor-plugin/plugin.json -> version + - plugin-metadata.json -> version + +Usage: + python scripts/check_skills_version.py + +Exits non-zero if any version field does not match pyproject.toml. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def read_pyproject_version(path: Path) -> str: + text = path.read_text(encoding="utf-8") + # Match the version key inside the [project] table without adding a TOML dep. + match = re.search(r'(?m)^\s*version\s*=\s*"([^"]+)"', text) + if not match: + raise ValueError(f"could not find a version in {path}") + return match.group(1) + + +def read_json_field(path: Path, *keys: str) -> str: + data = json.loads(path.read_text(encoding="utf-8")) + node = data + for key in keys: + node = node[key] + return node + + +def main() -> int: + source_version = read_pyproject_version(REPO_ROOT / "pyproject.toml") + + # (relative path, (nested json keys ...)) + targets = [ + (".claude-plugin/marketplace.json", ("metadata", "version")), + (".cursor-plugin/marketplace.json", ("metadata", "version")), + (".claude-plugin/plugin.json", ("version",)), + (".cursor-plugin/plugin.json", ("version",)), + ("plugin-metadata.json", ("version",)), + ] + + mismatches: list[str] = [] + print(f"source of truth: pyproject.toml version = {source_version}") + for rel_path, keys in targets: + path = REPO_ROOT / rel_path + if not path.exists(): + mismatches.append(f"missing file: {rel_path}") + continue + try: + value = read_json_field(path, *keys) + except (KeyError, TypeError): + mismatches.append(f"missing field {'.'.join(keys)} in {rel_path}") + continue + status = "ok" if value == source_version else "MISMATCH" + print(f" [{status}] {rel_path} ({'.'.join(keys)}) = {value}") + if value != source_version: + mismatches.append(f"{rel_path}: {'.'.join(keys)} = {value}, expected {source_version}") + + if mismatches: + print("\nversion check failed:", file=sys.stderr) + for item in mismatches: + print(f" - {item}", file=sys.stderr) + return 1 + + print("\nall skill version fields match pyproject.toml") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/build-aup-learning-cloud-images/SKILL.md b/skills/build-aup-learning-cloud-images/SKILL.md new file mode 100644 index 00000000..132506bb --- /dev/null +++ b/skills/build-aup-learning-cloud-images/SKILL.md @@ -0,0 +1,106 @@ +--- +name: build-aup-learning-cloud-images +description: >- + Group: Plan & deploy AUP Learning Cloud. Builds and publishes the AUP Learning + Cloud Docker images — the Hub image and + the CPU/GPU notebook and course images — with ./auplc-installer img build. + Use when the user wants to build, rebuild, tag, or push AUPLC images, mentions + img build / img pull, the dockerfiles/ directory, auplc-hub / auplc-base / + auplc-default / auplc-cv / auplc-dl / auplc-llm / auplc-physim / code-cpu / + code-gpu, a gfx-specific image tag, the GHCR registry, code-server VS Code + extensions, or preparing images for an offline/registry deployment. Covers + GPU-target tagging and pushing to a registry. Do not use to install or deploy + a cluster (install-/deploy-aup-learning-cloud) or to edit the course catalog + in values.yaml (configure-aup-learning-cloud-courses). +--- + +# Build AUP Learning Cloud images + +Produce the container images the platform runs: the Hub image plus the notebook +and course images, GPU-tagged per accelerator family, and (optionally) pushed +to a registry for a multi-node or offline deployment. + +`./auplc-installer img build` is the source of truth and wraps +`dockerfiles/`. Your job is to pick the right targets + GPU tag, run the build, +and (if asked) push. Target list, tag scheme, and the push flow are in +**[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud`; Docker with enough disk (GPU images are + large) and, for course images, network access to base layers. +- For pushing: `docker login` to the target registry (default + `ghcr.io/amdresearch`). +- Know the **GPU target** for GPU images (`phx`, `strix`, `strix-halo`, + `9070xt`, `r9700`, …) — GPU images are tagged `:<tag>-<gpu_target>`. + +## Targets at a glance + +| Target | Image | GPU-tagged? | +| --- | --- | --- | +| `hub` | `auplc-hub` | no (infra image) | +| `base-cpu` | `auplc-default` | no | +| `base-rocm` | `auplc-base` | yes | +| `code-cpu` / `code-gpu` | `auplc-code-cpu` / `auplc-code-gpu` | gpu only | +| `cv` / `dl` / `llm` / `physim` | `auplc-cv` / `-dl` / `-llm` / `-physim` | yes | +| `all` | hub + selected courses | mixed | + +## Workflow + +1. **Decide scope.** Which targets, and the GPU target for ROCm images. For a + demo rebuild of one course, build just that target; avoid `all` unless + needed. +2. **Build.** + + ```bash + ./auplc-installer img build hub + ./auplc-installer img build base-rocm --gpu=strix + ./auplc-installer img build cv dl --gpu=strix-halo + ./auplc-installer img build --image-tag=develop base-rocm --gpu=strix-halo + ``` + +3. **(Optional) Push** to the registry referenced by `custom.resources.images`: + + ```bash + docker push ghcr.io/amdresearch/auplc-hub:latest + docker push ghcr.io/amdresearch/auplc-base:latest-gfx1151 # GPU-tagged example + ``` + +4. **Wire the tag in.** If you changed the tag, update + `custom.resources.images` (and `prePuller.extraImages` if used) — that's the + configure-aup-learning-cloud-courses skill — then `rt upgrade` / `helm + upgrade`. + +## Editing the Hub image — preserve attribution + +If a change touches Hub source, **all four attribution layers from the project +`AGENTS.md` must stay intact** (do not remove/rename any): + +1. `X-Powered-By: AUP Learning Cloud` header in + `runtime/hub/core/jupyterhub_config.py`. +2. `PlatformInfoHandler` (`/api/platform`, unauthenticated) in + `runtime/hub/core/handlers.py`. +3. The `<footer id="auplc-powered-by-footer">` in + `runtime/hub/frontend/templates/page.html` (kept outside all Jinja blocks). +4. `PLATFORM_NAME` / `PLATFORM_VENDOR` / `PLATFORM_WEBSITE` in + `runtime/hub/frontend/packages/shared/src/branding.ts` (import, never + hardcode the platform string). + +Also keep the `Copyright (C) … Advanced Micro Devices, Inc.` header on every +source file (MIT requirement). + +## Safety + +- **Disk + time.** GPU/course image builds are large and slow — confirm before + `all` or `--image-source=build` on a small box. +- **Pushing is publishing.** Confirm the registry, repo, and tag before any + `docker push`; never push secrets baked into a layer. +- **code-server safety.** The code images run `code-server --auth none` on port + 8888; this is safe only behind the Hub proxy. Never expose that port via + NodePort/LoadBalancer/ingress. Confirm VS Code/OpenVSX extension licenses + before adding to `dockerfiles/Code/extensions.txt`. + +## Reference + +Full target list, the gfx tag scheme, `img pull` for offline, registry/mirror +flags, and troubleshooting: [reference.md](reference.md). diff --git a/skills/build-aup-learning-cloud-images/reference.md b/skills/build-aup-learning-cloud-images/reference.md new file mode 100644 index 00000000..e7ce3c90 --- /dev/null +++ b/skills/build-aup-learning-cloud-images/reference.md @@ -0,0 +1,95 @@ +# Build AUP Learning Cloud images — Reference + +Target list, tag scheme, push/pull flows, and troubleshooting for +`./auplc-installer img build`. Workflow and the attribution rules are in +[SKILL.md](SKILL.md). + +## Source + +- Repo README "Available Notebook and Coding Environments" + `./auplc-installer help`. +- `auplc_installer/catalog.py` (course → image basename + make target). +- `dockerfiles/` (the actual build context, incl. `dockerfiles/Code/extensions.txt`). + +## Target → image map + +| `img build` target | Image basename | GPU-tagged | Make target | +| --- | --- | --- | --- | +| `hub` | `auplc-hub` | no | (hub) | +| `base-cpu` | `auplc-default` | no | `base-cpu` | +| `base-rocm` | `auplc-base` | yes | `base-rocm` | +| `code-cpu` | `auplc-code-cpu` | no | `code-cpu` | +| `code-gpu` | `auplc-code-gpu` | yes | `code-gpu` | +| `cv` | `auplc-cv` | yes | `cv` | +| `dl` | `auplc-dl` | yes | `dl` | +| `llm` | `auplc-llm` | yes | `llm` | +| `physim` | `auplc-physim` | yes | `physim` | +| `all` | hub + selected courses | mixed | — | +| `code` | both code-server images | — | — | + +## Tag scheme + +- Plain (non-GPU) images: `:<IMAGE_TAG>` (default `IMAGE_TAG=latest`). +- GPU images: `:<IMAGE_TAG>-<gpu_target>` — the GPU suffix is appended + automatically from `--gpu` (e.g. `auplc-base:latest-gfx1151` for strix-halo). +- Registry prefix: `--image-registry` / `IMAGE_REGISTRY` + (default `ghcr.io/amdresearch`). + +## Build examples + +```bash +./auplc-installer img build hub +./auplc-installer img build base-rocm --gpu=strix +./auplc-installer img build cv dl llm physim --gpu=strix-halo +./auplc-installer img build --image-tag=develop base-rocm --gpu=strix-halo +./auplc-installer img build all --gpu=strix-halo # hub + all courses +``` + +Relevant global flags (see install skill for the full table): `--gpu`, +`--image-tag`, `--image-registry`, `--mirror=`, `--mirror-pip=`, `--mirror-npm=`, +`-v/--verbose`. + +## Push to a registry + +```bash +docker login ghcr.io +docker push ghcr.io/amdresearch/auplc-hub:latest +docker push ghcr.io/amdresearch/auplc-default:latest +docker push ghcr.io/amdresearch/auplc-base:latest-gfx1151 +docker push ghcr.io/amdresearch/auplc-cv:latest-gfx1151 +``` + +Then point `custom.resources.images` (and `prePuller.extraImages` if used) at +the pushed tags — see configure-aup-learning-cloud-courses. + +## Offline: pull external images + +```bash +./auplc-installer img pull # fetch external (non-custom) images for offline use +``` + +For a full air-gapped bundle (custom + external + installer), use +`./auplc-installer pack` (see install-aup-learning-cloud-single-node). + +## code-server images + +The `code-cpu` / `code-gpu` images launch `code-server --auth none` on port +**8888**, safe only behind the JupyterHub proxy auth boundary — never expose +that port directly. Built-in extensions come from +`dockerfiles/Code/extensions.txt` plus local `.vsix` packages (e.g. the AUPLC +Back-to-Hub extension). Confirm extension licenses / marketplace terms before +adding any. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Build fails pulling base layers | Network / mirror | `--mirror=`, `--mirror-pip=`, retry; check Docker daemon proxy | +| `no space left on device` | GPU/course images are large | Free disk, build fewer targets, prune `docker image prune` | +| Wrong gfx kernels at runtime | Built for the wrong `--gpu` target | Rebuild with the correct `--gpu`; for Phoenix note `HSA_OVERRIDE_GFX_VERSION` | +| Pushed image not used by Hub | `custom.resources.images` tag not updated | Update the overlay + `rt upgrade`/`helm upgrade` | +| Attribution check fails in review | A Hub-source edit dropped a layer | Restore all four `AGENTS.md` layers + file copyright headers | + +## Out of scope + +Installing/deploying a cluster, editing the values course catalog, and authoring +new course curricula (notebooks). This skill builds and publishes the images. diff --git a/skills/build-aup-learning-cloud-images/skill-card.md b/skills/build-aup-learning-cloud-images/skill-card.md new file mode 100644 index 00000000..520c81ae --- /dev/null +++ b/skills/build-aup-learning-cloud-images/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Build and publish the AUP Learning Cloud Hub and notebook/course Docker images with ./auplc-installer img build, for maintainers. + +## Owner + +AMD Research diff --git a/skills/configure-aup-learning-cloud-auth/SKILL.md b/skills/configure-aup-learning-cloud-auth/SKILL.md new file mode 100644 index 00000000..12003da8 --- /dev/null +++ b/skills/configure-aup-learning-cloud-auth/SKILL.md @@ -0,0 +1,104 @@ +--- +name: configure-aup-learning-cloud-auth +description: >- + Group: Maintain AUP Learning Cloud. Configures authentication for AUP Learning + Cloud: auth modes (auto-login/dummy/github/multi), GitHub App / OAuth, GitHub + team-to-group sync, native local accounts, password policy and forced + first-login change, and admin bootstrap. Use when the user wants to set or + switch custom.authMode, enable GitHub login, create or migrate a GitHub + App, set oauth_callback_url / client_id / client_secret / app_id / + private_key_file, sync GitHub teams into JupyterHub groups, enable native + accounts, bootstrap the initial admin (custom.adminUser), or debug "Resource + not accessible by integration", a login 404, or OAuth callback errors. + Triggers include custom.authMode, GitHubOAuthenticator, custom.githubOrgName, + allowed_organizations, jupyterhub-admin-credentials. Do not use to map which + resources a group sees (configure-aup-learning-cloud-courses), to + bulk-manage users (manage-aup-learning-cloud-users), or to configure + private-repo cloning (configure-aup-learning-cloud-repos). +--- + +# Configure AUP Learning Cloud authentication + +Choose and wire the Hub's login path: pick the `custom.authMode`, set up the +GitHub App (OAuth + server-to-server team sync) and/or native local accounts, +and bootstrap the initial admin — then re-apply with the installer or Helm. + +Edit a **values overlay** (`runtime/values.yaml`, `values-multi-nodes.yaml`, or +`values.local.yaml`), never hardcode secrets into tracked files. The full +GitHub App walkthrough, value blocks, and troubleshooting are in +**[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud`; a running (or about-to-deploy) Hub. +- `helm` + `kubectl` against the cluster, or `./auplc-installer` on a + single-node box. +- For `github` / `multi`: a GitHub **organization** you own (the App is created + under the org, not a personal account) and admin access to its settings. + +## Pick the auth mode + +| Mode | When to use | Notes | +| --- | --- | --- | +| `auto-login` | Local demo / single dev box | No credentials; quota auto-disabled unless forced. The checked-in default. | +| `dummy` | Throwaway testing only | Accepts any user/password; not for real use; its login can 404 in normal setups. | +| `github` | Org-backed SSO | GitHub App only; team membership syncs into Hub groups. | +| `multi` | GitHub + local accounts | Combined login page; native accounts for users without GitHub. | + +`custom.authMode` is the single switch. Confirm the target mode with the user +before changing a live Hub (a `helm upgrade` restarts the Hub pod, a brief +login blip). + +## Workflow + +1. **Read current state.** Check `custom.authMode`, `custom.adminUser.enabled`, + `custom.githubOrgName`, and `hub.config.GitHubOAuthenticator` in the active + overlay. +2. **Set the mode** in the overlay. For `auto-login`/`dummy` you are done with + credentials; skip to step 6. +3. **GitHub App (github/multi).** Create the App under the org with the exact + callback URL for the mode and `Members: Read-only` + `Contents: Read-only` + permissions, then fill `hub.config.GitHubOAuthenticator` (`app_id`, + `client_id`, `client_secret`, `private_key_file`, `allowed_organizations`, + `scope: []`) and `custom.githubOrgName`. Step-by-step in + [reference.md](reference.md). + - **Callback URL must match the mode exactly:** `multi` uses + `…/hub/github/oauth_callback`; single `github` uses `…/hub/oauth_callback`. +4. **Team sync.** Team-to-group sync uses the App installation token; the org + teams are intersected with `custom.teams.mapping`. Mapping *which resource* a + group sees stays in the configure-courses skill — this skill only makes the + groups exist. +5. **Native accounts (multi).** The first-use authenticator has + `create_users = False`, so accounts must be created by an admin before login + (see manage-users skill). Password policy: ≥8 chars with upper, lower, digit, + and special; users can be forced to change on first login. +6. **Admin bootstrap (optional).** Set `custom.adminUser.enabled: true` to have + the chart mint the `jupyterhub-admin-credentials` secret and the `admin` + user. +7. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f + runtime/values.yaml -f <overlay>` must succeed. +8. **Apply.** Single-node: `./auplc-installer rt upgrade`. Multi/manual: + `helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub -f + runtime/values.yaml -f <overlay>`. +9. **Verify.** Load the Hub: the expected login page appears, a GitHub user + lands in the right groups, and (if bootstrapped) the admin can log in. Read + the secret with the commands in [reference.md](reference.md). + +## Safety + +- **Secrets never go in tracked files.** `client_secret`, the App private key, + and `jupyterhub-admin-credentials` must come from a mounted K8s secret or an + untracked overlay. Never commit them. +- **Avoid `dummy` outside isolated testing** — it accepts any credentials. +- **Switching modes is disruptive.** `auto-login` → `github`/`multi` forces + every user through login and changes who can spawn; confirm timing for a live + class. +- A `helm upgrade` / `rt upgrade` restarts the Hub pod (brief auth blip). +- If Hub source is touched, preserve the four attribution layers and per-file + copyright headers (see the project `AGENTS.md`). + +## Reference + +GitHub App creation walkthrough, every `GitHubOAuthenticator` field, the +OAuth-App→GitHub-App migration, native-account/password details, admin secret +retrieval, and the troubleshooting table: [reference.md](reference.md). diff --git a/skills/configure-aup-learning-cloud-auth/reference.md b/skills/configure-aup-learning-cloud-auth/reference.md new file mode 100644 index 00000000..8c659dab --- /dev/null +++ b/skills/configure-aup-learning-cloud-auth/reference.md @@ -0,0 +1,156 @@ +# Configure AUP Learning Cloud authentication — Reference + +Full GitHub App setup, every `GitHubOAuthenticator` field, the OAuth-App → +GitHub-App migration, native accounts, admin bootstrap, and troubleshooting. +Workflow and gates are in [SKILL.md](SKILL.md). + +## Source guides + +- Authentication Guide: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/authentication-guide.html> +- GitHub App Setup: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/github-app-setup.html> +- Configuration Reference: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/configuration-reference.html> + +The live `runtime/values.yaml` and `runtime/chart/values.schema.yaml` are the +source of truth; verify keys against them. + +## 1. Auth modes (`custom.authMode`) + +```yaml +custom: + authMode: "auto-login" # auto-login | dummy | github | multi +``` + +- `auto-login` — shared, no credentials. Quota auto-disables unless explicitly + enabled. Checked-in single-node default. +- `dummy` — accepts any username/password. Testing only. +- `github` — GitHub App only. `oauth_callback_url` ends in `/hub/oauth_callback`. +- `multi` — GitHub App + native accounts on one page. `oauth_callback_url` ends + in `/hub/github/oauth_callback`. + +## 2. Admin bootstrap (`custom.adminUser`) + +```yaml +custom: + adminUser: + enabled: true +``` + +The chart creates the `jupyterhub-admin-credentials` secret and bootstraps the +`admin` user. Retrieve: + +```bash +kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ + -o jsonpath='{.data.admin-password}' | base64 -d && echo +kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ + -o jsonpath='{.data.api-token}' | base64 -d && echo +``` + +## 3. GitHub App — create it (github/multi) + +1. **Create the App under the organization** (not a personal account): + `https://github.com/organizations/<ORG>/settings/apps/new`. +2. **Basic info:** name (e.g. `auplc-hub`), Homepage = Hub URL, **Callback URL** + matching the mode: + - `multi`: `https://<domain>/hub/github/oauth_callback` + - single `github`: `https://<domain>/hub/oauth_callback` +3. Check **Expire user authorization tokens** and **Request user authorization + (OAuth) during installation**. Uncheck **Webhook → Active**. +4. **Permissions:** + - Repository → `Contents`: Read-only (private-repo cloning), `Metadata`: + Read-only (default). + - Organization → `Members`: **Read-only** (required for team sync/group + mapping — without it the Hub logs `Resource not accessible by + integration`). +5. **Installation scope:** Any account. Create the App. +6. Record **App ID**, **Client ID** (`Iv23li…`, different from App ID), + generate a **Client secret**, and generate a **private key** (`.pem`). Mount + the `.pem` into the Hub pod and record the path. +7. **Install the App on the org** configured as `custom.githubOrgName`; pick the + repos users may access if private cloning is used. + +## 4. GitHub App — configure the Hub + +```yaml +custom: + githubOrgName: "<YOUR-ORG-NAME>" + + gitClone: + githubAppName: "your-app-slug" # only if private-repo cloning is wanted (see repos skill) + +hub: + config: + GitHubOAuthenticator: + oauth_callback_url: "https://<domain>/hub/github/oauth_callback" + app_id: "<GitHub App App ID>" + installation_id: "" # blank = auto-discover from the org installation + private_key_file: "/path/to/mounted/github-app-private-key.pem" + # private_key: "" # alternative; prefer a mounted secret + team_sync_ttl_seconds: 3600 + client_id: "<GitHub App Client ID>" + client_secret: "<GitHub App Client Secret>" + allowed_organizations: + - <YOUR-ORG-NAME> + scope: [] # GitHub App uses App permissions, not OAuth scopes +``` + +`scope: []` is correct for a GitHub App. `installation_id` can stay blank when +the App is installed on the org (auto-discovered via `GET /orgs/{org}/installation`). + +## 5. Team-to-group sync + +The Hub lists actual org teams, intersects them with `custom.teams.mapping`, +and batches member lookups through GitHub GraphQL using the App installation +token. Team keys correspond to GitHub team slugs (e.g. `AUP` is queried as +`aup`, but the JupyterHub group stays `AUP`). Missing teams are logged and +skipped rather than failing the whole sync. Assigning *resources* to those +groups is the configure-courses skill. + +GitHub users without a matched team fall into a `github-users` fallback group; +native users can be assigned `native-users`. + +## 6. Native accounts (multi) + +- The first-use authenticator sets `create_users = False` — accounts must exist + before login (create them via the manage-users skill or `/hub/admin`). +- **Password policy:** ≥8 chars, ≥1 uppercase, ≥1 lowercase, ≥1 digit, ≥1 + special. Applies to admin-set and user-changed passwords. +- **Forced first-login change** uses `/auth/check-force-password-change` and + `/auth/change-password`. + +## 7. Migrating OAuth App → GitHub App + +Keep `oauth_callback_url` and `allowed_organizations`. Change `client_id` / +`client_secret` to the App's, add `app_id`, `installation_id` (blank ok), +`private_key_file`, `team_sync_ttl_seconds`, set `scope: []`, and set +`gitClone.githubAppName`. Existing sessions keep working; new logins use the +App. Delete the old OAuth App after everyone has re-logged. + +## 8. Apply and verify + +```bash +# render check +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null + +# single-node +sudo ./auplc-installer rt upgrade +# multi-node / manual +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + +kubectl rollout status -n jupyterhub deploy/hub +kubectl logs -n jupyterhub deployment/hub | grep -i -E 'admin|github|oauth' +``` + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Login 404 / no login page | `authMode: dummy`, or wrong mode for the deploy | Set `github`/`multi`/`auto-login`; re-apply | +| OAuth callback error | `oauth_callback_url` mismatch (mode or http/https) | Match the App's Callback URL exactly to the mode | +| `Resource not accessible by integration` | App missing `Members: Read-only` | Add the org permission; an org owner must approve the updated install | +| GitHub users see no/wrong resources | `githubOrgName`, `allowed_organizations`, `teams.mapping`, or team membership | Verify all four; confirm the user's GitHub teams | +| Configured team skipped in sync | Team doesn't exist on GitHub | The Hub only syncs teams that exist; create it or fix the key | +| Installation token unavailable | `app_id`/`private_key_file` wrong or App not installed on org | Verify both and the org installation | +| No admin user created | `custom.adminUser.enabled` not true | Set it, re-apply, `kubectl logs … | grep -i admin` | +| Native user can't log in | Not `multi`, user not pre-created, or no local password | Confirm mode + that an admin created the account | +| Password change keeps failing | New password fails the strength policy | Re-check length + upper/lower/digit/special | diff --git a/skills/configure-aup-learning-cloud-auth/skill-card.md b/skills/configure-aup-learning-cloud-auth/skill-card.md new file mode 100644 index 00000000..5dde230e --- /dev/null +++ b/skills/configure-aup-learning-cloud-auth/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Configure AUP Learning Cloud authentication — auth mode, GitHub App / OAuth, team-to-group sync, native accounts, and admin bootstrap — for operators standing up or securing a Hub. + +## Owner + +AMD Research diff --git a/skills/configure-aup-learning-cloud-courses/SKILL.md b/skills/configure-aup-learning-cloud-courses/SKILL.md new file mode 100644 index 00000000..ff8a079f --- /dev/null +++ b/skills/configure-aup-learning-cloud-courses/SKILL.md @@ -0,0 +1,85 @@ +--- +name: configure-aup-learning-cloud-courses +description: >- + Group: Course & other editor. Edits the AUP Learning Cloud course catalog and + access control in the + JupyterHub values.yaml: course images, resource requirements, spawn-UI + metadata, group ordering, GPU accelerator selectors, team-to-course mappings, + and the quota knobs. Use when the user wants to add/remove a course or + notebook environment, show/hide an option in the spawn picker, map a GitHub + team or group to courses, set per-course CPU/memory/amd.com/gpu requirements, + add or retune an accelerator (custom.accelerators), or configure quota + (cpuRate, quotaRate, minimumToStart, refresh rules). Triggers include + values.yaml, custom.resources.images, custom.teams.mapping, + custom.accelerators, custom.quota, acceleratorKeys, launchMode. Do not use to + build the images themselves (build-aup-learning-cloud-images) or to install a + cluster (install-/deploy-aup-learning-cloud). +--- + +# Configure AUP Learning Cloud courses + +Change what users can spawn and who can see it, by editing the `custom:` block +of the JupyterHub values and re-applying with Helm. One coherent surface: +course images, their resource requirements, the spawn-UI metadata, accelerator +selectors, team mappings, and quota. + +Edit a **values overlay** (e.g. `runtime/values-basic-example.yaml` or +`values.local.yaml`), never the chart defaults blindly. The key map and the +full field guide are in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud`; a running Hub (single- or multi-node). +- `helm` + `kubectl` against the cluster, or `./auplc-installer` on a + single-node box. +- Know which keys already exist: `custom.resources.images` is the catalog; + course keys are `cpu`, `gpu`, `code-cpu`, `code-gpu`, and `Course-CV`, + `Course-DL`, `Course-LLM`, `Course-PhySim`. + +## The four places a course lives + +A course key must be consistent across **all** of these or the spawn UI breaks: + +1. `custom.resources.images.<key>` — the container image. +2. `custom.resources.requirements.<key>` — `cpu`, `memory`, and `amd.com/gpu`. +3. `custom.resources.metadata.<key>` — spawn-UI `group`, `description`, + `accelerator`, `acceleratorKeys`, `allowGitClone`, `launchMode`, + `resourceType`. +4. `custom.teams.mapping.<team>` — the teams allowed to launch it. + +## Workflow + +1. **Read the current state.** Open `runtime/values.yaml` for the canonical + shape, and the active overlay for what is deployed. Confirm the exact key + you are changing. +2. **Make the edit in the overlay.** Add/modify the key in all four places + above (or, for accelerators/quota, the relevant block). Keep `acceleratorKeys` + pointing at real `custom.accelerators` keys (`phx`, `strix`, `strix-halo`, + `9070xt`, `r9700`). +3. **Keep accelerator selectors honest.** Each `custom.accelerators.<key>.nodeSelector` + must equal a real node label — confirm with + `kubectl describe node <node> | grep amd.com/gpu.product-name`. +4. **Validate the render before applying.** `helm template jupyterhub + ./runtime/chart -f runtime/values.yaml -f <overlay>` must succeed; the repo + also ships `runtime/chart/values.schema.json`. +5. **Apply.** Single-node: `./auplc-installer rt upgrade`. Multi/manual: + `helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub -f + runtime/values.yaml -f <overlay>`. +6. **Verify.** Reload the spawn page; the course appears in its `group` for the + mapped teams only, and a launched pod gets the expected resources/node. + +## Safety + +- **Edit overlays, not secrets.** Never put OAuth secrets or tokens in tracked + files. Never commit a `values.local.yaml` that carries site config. +- **Removing a course** hides it and can strand running servers on that image — + confirm with the user and check for active spawns first. +- **Quota changes apply cluster-wide.** Lowering `minimumToStart` / `cpuRate` + or editing `refreshRules` affects every user; confirm before applying. +- A `helm upgrade` restarts the Hub pod (brief auth blip). Confirm timing for a + live class. + +## Reference + +Course-key map, every `metadata`/`requirements` field, the accelerator block, +team-mapping semantics, and the quota knobs: [reference.md](reference.md). diff --git a/skills/configure-aup-learning-cloud-courses/reference.md b/skills/configure-aup-learning-cloud-courses/reference.md new file mode 100644 index 00000000..eb773a9e --- /dev/null +++ b/skills/configure-aup-learning-cloud-courses/reference.md @@ -0,0 +1,149 @@ +# Configure AUP Learning Cloud courses — Reference + +The course-key map, every field under `custom.resources`, the accelerator and +team blocks, and the quota knobs, as they appear in `runtime/values.yaml`. +Workflow and gates are in [SKILL.md](SKILL.md). + +## Source guides + +- Configuration Reference (`runtime/values.yaml`): <https://amdresearch.github.io/aup-learning-cloud/> +- Overview (resource selection, teams, quota): <https://amdresearch.github.io/aup-learning-cloud/introduction/overview.html> + +The live `runtime/values.yaml` is the source of truth; verify keys against it. + +## Course catalog (default keys) + +| Key | Default image | HW | Notes | +| --- | --- | --- | --- | +| `cpu` | `ghcr.io/amdresearch/auplc-default:latest` | CPU | Basic Python notebook | +| `gpu` | `ghcr.io/amdresearch/auplc-base:latest` | GPU | Basic GPU notebook | +| `code-cpu` | `ghcr.io/amdresearch/auplc-code-cpu:latest` | CPU | code-server (`launchMode: code-server`) | +| `code-gpu` | `ghcr.io/amdresearch/auplc-code-gpu:latest` | GPU | code-server | +| `Course-CV` | `ghcr.io/amdresearch/auplc-cv:latest` | GPU | Computer Vision | +| `Course-DL` | `ghcr.io/amdresearch/auplc-dl:latest` | GPU | Deep Learning | +| `Course-LLM` | `ghcr.io/amdresearch/auplc-llm:latest` | GPU | LLM from scratch | +| `Course-PhySim` | `ghcr.io/amdresearch/auplc-physim:latest` | GPU | Genesis physics sim | + +These keys must match across `custom.resources.{images,requirements,metadata}` +and be referenced by `custom.teams.mapping`. The installer mirrors this in +`auplc_installer/catalog.py`; keep both consistent if you add a course used by +`./auplc-installer --courses`. + +## custom.resources.requirements.<key> + +```yaml +gpu: + cpu: "0" # "0" = no explicit request/limit (best-effort) + memory: "0Gi" + amd.com/gpu: "1" # present only for GPU courses +``` + +## custom.resources.metadata.<key> + +```yaml +Course-CV: + group: "TEACHING LABS" # spawn-UI grouping (see groupOrder) + description: "Computer Vision Course" + subDescription: "Suitable for CV experiments with GPU" + accelerator: "GPU" # "" for CPU courses + acceleratorKeys: # which custom.accelerators entries apply + - strix-halo + allowGitClone: true + launchMode: "code-server" # only for browser-IDE resources; omit for notebooks + resourceType: "notebook" # or "browser-ide" + # acceleratorOverrides: # optional per-accelerator image/env override + # 9070xt: + # image: "ghcr.io/your-org/auplc-cv:<tag-for-9070xt>" +``` + +`custom.resources.groupOrder` is a list controlling spawn/Home group order +(e.g. `TEACHING LABS`, `DEVELOPMENT ENVIRONMENT`, `CUSTOM REPOS`). Unlisted +groups follow alphabetically. + +## custom.accelerators.<key> + +```yaml +strix-halo: + displayName: "AMD Radeon™ 8060S (Strix Halo iGPU)" + description: "RDNA 3.5 (gfx1151) | Compute Units 40 | 64GB LPDDR5X" + nodeSelector: + amd.com/gpu.product-name: "AMD_Radeon_8060S_Graphics" # MUST match a real node label + env: {} # e.g. HSA_OVERRIDE_GFX_VERSION for Phoenix (phx) + quotaRate: 3 # quota consumed per hour when this accelerator is used +``` + +Default accelerator keys → product label: + +| Key | `amd.com/gpu.product-name` | +| --- | --- | +| `phx` | `AMD_Radeon_780M_Graphics` (sets `HSA_OVERRIDE_GFX_VERSION: 11.0.0`) | +| `strix` | `AMD_Radeon_890M_Graphics` | +| `strix-halo` | `AMD_Radeon_8060S_Graphics` | +| `9070xt` | `AMD_Radeon_RX_9070_XT` | +| `r9700` | `AMD_Radeon_AI_PRO_R9700` | + +If your fleet normalizes a product name differently, change the `nodeSelector` +to the exact string from `kubectl describe node`. + +## custom.teams.mapping.<team> + +A team name maps to the list of course keys its members can launch. Built-in +teams seen in defaults include `cpu`, `gpu`, `official`, `AUP`, `native-users`, +`github-users`. In GitHub auth, GitHub team membership syncs into these groups. + +```yaml +teams: + mapping: + gpu: + - code-gpu + - Course-CV + - Course-DL + - Course-LLM + - Course-PhySim +``` + +When the installer is run with `--courses=<subset>`, each team's list is +rewritten as the intersection with the selection, so unselected courses +disappear from the UI. + +## custom.quota + +```yaml +quota: + enabled: null # null = auto (disabled for auto-login/dummy unless set true) + cpuRate: 1 # quota/hour for CPU-only sessions + minimumToStart: 10 # min balance required to spawn anything + defaultQuota: 0 # initial allocation for new users (0 = none) + refreshRules: {} # each rule becomes a K8s CronJob that tops up balances +``` + +Per-accelerator consumption is `custom.accelerators.<key>.quotaRate`. + +## Apply and verify + +```bash +# render check +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null + +# single-node +./auplc-installer rt upgrade +# multi-node / manual +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + +kubectl rollout status -n jupyterhub deploy/hub +``` + +Reload the spawn page: the course shows in its `group` for mapped teams only; +a launched pod gets the declared `requirements` and lands on a node matching +the accelerator `nodeSelector`. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Course missing from spawn UI | Key absent from `metadata`/`images`, or team mapping | Confirm the key in all four places + `teams.mapping` | +| GPU course Pending | `acceleratorKeys` → `nodeSelector` label mismatch | `kubectl describe node | grep amd.com/gpu.product-name` | +| code-server resource opens as a notebook | `launchMode`/`resourceType` not set | `launchMode: code-server`, `resourceType: browser-ide` | +| Quota blocks all spawns | `minimumToStart` too high or `defaultQuota: 0` | Review `custom.quota`, grant balance via Admin console | +| `helm upgrade` schema error | Value violates `values.schema.json` | Read the error; fix the offending key's type | diff --git a/skills/configure-aup-learning-cloud-courses/skill-card.md b/skills/configure-aup-learning-cloud-courses/skill-card.md new file mode 100644 index 00000000..cd5985ee --- /dev/null +++ b/skills/configure-aup-learning-cloud-courses/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Edit the AUP Learning Cloud course catalog, team mappings, accelerator selectors, and quota in the JupyterHub values.yaml, for platform admins. + +## Owner + +AMD Research diff --git a/skills/configure-aup-learning-cloud-repos/SKILL.md b/skills/configure-aup-learning-cloud-repos/SKILL.md new file mode 100644 index 00000000..4eaceb0b --- /dev/null +++ b/skills/configure-aup-learning-cloud-repos/SKILL.md @@ -0,0 +1,104 @@ +--- +name: configure-aup-learning-cloud-repos +description: >- + Group: Course & other editor. Configures per-user Git repository cloning: the + custom.gitClone block (githubAppName repo picker, defaultAccessToken for + private repos, allowedProviders, maxCloneTimeout, defaultPersistence, + allowPersistenceChoice) and the per-resource metadata.allowGitClone gate that + clones a repo into a user's workspace at spawn time. Use when the user wants + to let learners clone a Git repo on startup, enable the spawn-form repo + URL/branch field or GitHub repo picker, give access to a private repo (bot PAT + or GitHub App token), choose whether cloned repos persist, allow + GitLab/Bitbucket, or debug "Repository URL ignored" or a failed clone init + container. Triggers include custom.gitClone, allowGitClone, githubAppName, + defaultAccessToken, allowedProviders, init-clone-repo. Do not use to set up + GitHub login itself (configure-aup-learning-cloud-auth), to publish a course + to the catalog (configure-/develop-aup-learning-cloud-courses), or to build + images (build-aup-learning-cloud-images). +--- + +# Configure AUP Learning Cloud repository cloning + +Enable the runtime, per-user feature where a learner pastes a Git URL on the +spawn form (or picks a private repo) and the Hub clones it into their home PVC +via an init container. This is **not** how you publish a course to the catalog +(that is develop-/configure-courses); it brings *each user's own* repo into +*their own* workspace. + +Edit a **values overlay** and re-apply. The token model, the persistence rules, +and the GitHub App requirement are subtle and partly silent — read the gates +below. Full details and troubleshooting are in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud` and a running (or about-to-deploy) Hub; + `helm` + `kubectl` or `./auplc-installer`. +- For private repos via GitHub App: the App configured in the auth skill + (`hub.config.GitHubOAuthenticator` + `custom.githubOrgName`). +- For private repos via a shared token: a read-only bot/service-account PAT. + +## The two gates (both required, one is silent) + +A repo URL is only cloned when **both** are true: + +1. `custom.gitClone` is configured (at minimum the feature is on; private repos + need a token source). +2. The **selected resource** has `custom.resources.metadata.<key>.allowGitClone: + true`. + +If `allowGitClone` is false for the chosen resource, the Hub **silently drops** +the repo URL (it logs a warning but shows no user error). Always set both. + +## Token priority (private repos) + +`OAuth token (GitHub App) > defaultAccessToken > none (public only)` + +- `githubAppName` — enables the repo picker + automatic per-repo OAuth token, + but **only for GitHub-App users**. No effect for auto-login/native users. +- `defaultAccessToken` — a bot PAT applied transparently to **all** users + (including auto-login); right for single-node/classroom shared private repos. + Helm base64s it into the `jupyterhub-git-default-token` secret. + +## Workflow + +1. **Read current state.** Inspect `custom.gitClone` and which + `metadata.<key>.allowGitClone` are already true. +2. **Turn on cloning** in the overlay; set `allowedProviders` (defaults + `github.com`, `gitlab.com`, `bitbucket.org`) and `maxCloneTimeout` as needed. +3. **Open the gate per resource.** Set `allowGitClone: true` on each course/env + that should accept a user repo (configure-courses owns the rest of that + metadata block). +4. **Private repos (optional).** Pick a token source: + - GitHub App: ensure the auth skill's App is set, then + `custom.gitClone.githubAppName: "<app-slug>"`. + - Shared PAT: `custom.gitClone.defaultAccessToken: "<read-only PAT>"` (keep + it out of tracked files — see Safety). +5. **Persistence policy.** Decide `defaultPersistence` (default `true`; cloned + repos survive server stop, no auto-pull after first clone) and whether to let + users choose with `allowPersistenceChoice`. +6. **Pre-flight + apply.** `helm template …` must succeed; then + `./auplc-installer rt upgrade` (single) or `helm upgrade --install …` + (multi). +7. **Verify.** On the spawn page for an allowed resource, the repo URL/branch + field (and picker, if `githubAppName`) appears; launch with a repo and + confirm `init-clone-repo` succeeds and the repo lands under + `/home/jovyan/<repo>`. + +## Safety + +- **`defaultAccessToken` is a secret.** It is base64'd into a K8s secret — never + commit it in a tracked values file. Scope the PAT **read-only** to the + specific repos to limit blast radius. +- **Persistence has destructive edges.** Ephemeral mode deletes the clone via a + `preStop` hook; the script refuses to touch a directory it didn't create and + refuses to replace a persistent clone for an ephemeral request. Don't flip + `defaultPersistence` casually on a class with in-progress work. +- **Provider allowlist is a security control.** Only add providers you trust; + cloning runs inside the user's pod. +- A `helm upgrade` restarts the Hub pod (brief login blip). + +## Reference + +Every `custom.gitClone` field, the init-container/token mechanics, the +persistence state machine, the `allowGitClone` gate, and troubleshooting: +[reference.md](reference.md). diff --git a/skills/configure-aup-learning-cloud-repos/reference.md b/skills/configure-aup-learning-cloud-repos/reference.md new file mode 100644 index 00000000..6198bbc0 --- /dev/null +++ b/skills/configure-aup-learning-cloud-repos/reference.md @@ -0,0 +1,112 @@ +# Configure AUP Learning Cloud repository cloning — Reference + +Every `custom.gitClone` field, the init-container/token mechanics, the +persistence state machine, and troubleshooting. Workflow and gates are in +[SKILL.md](SKILL.md). + +## Source guides + +- Configuration Reference (section 4, custom.gitClone): <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/configuration-reference.html> +- Authentication Guide (GitHub App for repos): <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/authentication-guide.html> + +The live `runtime/values.yaml` (`custom.gitClone`) and +`runtime/hub/core/scripts/git-clone.sh` are the source of truth. + +## custom.gitClone fields + +```yaml +custom: + gitClone: + # -- Private repo access -- + githubAppName: "" # GitHub App slug; enables repo picker + OAuth token. + # Only effective for GitHub-App users. + defaultAccessToken: "" # Bot/service-account PAT for ALL users (incl. auto-login). + # Helm creates secret jupyterhub-git-default-token from it. + # -- Clone behavior -- + allowedProviders: # subdomains of these are also accepted + - github.com + - gitlab.com + - bitbucket.org + maxCloneTimeout: 300 # seconds per clone/fetch + initContainerImage: "alpine/git:2.47.2" # must contain git + sh + # -- Persistence -- + defaultPersistence: true # keep clones after the server stops + allowPersistenceChoice: false # expose a per-user persist toggle on the spawn form +``` + +## The per-resource gate + +```yaml +custom: + resources: + metadata: + gpu: + allowGitClone: true # REQUIRED for this resource to accept a repo URL +``` + +If the selected resource's `allowGitClone` is false, the spawner discards the +submitted `repo_url` and logs `Repository URL ignored … does not allow git +cloning` — no user-visible error. This metadata block otherwise belongs to the +configure-courses skill; this skill only flips the clone gate. + +## Token model + +Priority: **OAuth (GitHub App) > defaultAccessToken > none (public only)**. + +- The spawner injects the chosen token as `GIT_ACCESS_TOKEN` into the + `init-clone-repo` container via a `secretKeyRef`. +- `git-clone.sh` rewrites the HTTPS remote to + `https://x-access-token:<token>@<host>/…`, so any provider/token type works. +- `githubAppName` users authorize specific private repos through the GitHub App + UI on the spawn page; the token comes from their OAuth session. +- `defaultAccessToken` is applied transparently to everyone — ideal for a shared + classroom private repo with no GitHub login. + +## Persistence state machine + +`git-clone.sh` writes repo-external metadata under `~/.auplc/git-clones` and: + +- **persistent** (default): reuses a compatible existing clone; **does not + auto-pull/reset/sync** after the first successful clone. +- **ephemeral**: a `preStop` hook `rm -rf`s the clone when the session ends. +- Refuses to modify a directory lacking compatible AUPLC metadata (won't clobber + a user's own folder). +- Refuses to replace a persistent managed clone for an ephemeral request. + +`allowPersistenceChoice: true` exposes the choice to users; otherwise +`defaultPersistence` is enforced. + +## Branch selection + +Users can pass a branch, or paste a `/tree/<branch>` URL — the spawner extracts +the branch from `https://host/owner/repo/tree/<branch>`. `git-clone.sh` does a +`--depth 1` clone of that branch (or the default branch). + +## Apply and verify + +```bash +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null +sudo ./auplc-installer rt upgrade # single-node +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> # multi-node + +# after a user spawns with a repo: +kubectl get pods -n jupyterhub -o wide +kubectl logs -n jupyterhub <user-pod> -c init-clone-repo +``` + +The repo URL/branch field (and picker if `githubAppName`) shows on the spawn +page for allowed resources; a successful spawn has the repo under +`/home/jovyan/<repo>`. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Repo field absent on spawn | `allowGitClone` not true for that resource | Set `metadata.<key>.allowGitClone: true`, re-apply | +| "Repository URL ignored" in Hub logs | Same gate — resource disallows cloning | Same as above | +| Private clone fails (auth) | No usable token for that user | GitHub-App user must authorize the repo; or set `defaultAccessToken` | +| Clone fails ("could not be cloned") | Bad URL/branch, provider not allowed, timeout | Check URL, `allowedProviders`, raise `maxCloneTimeout`; read `init-clone-repo` logs | +| Server fails to start, `repo_clone_failed` | Init container clone error | `kubectl logs … -c init-clone-repo`; verify repo access/network | +| "Refusing to modify existing directory" | Target dir exists without AUPLC metadata | User has a same-named folder; choose another path or remove it | +| Changes to persistence not taking | Switched mode under a managed clone | Persistent↔ephemeral has refusal rules; clear the clone or keep the mode | diff --git a/skills/configure-aup-learning-cloud-repos/skill-card.md b/skills/configure-aup-learning-cloud-repos/skill-card.md new file mode 100644 index 00000000..74a69f5b --- /dev/null +++ b/skills/configure-aup-learning-cloud-repos/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Configure AUP Learning Cloud's per-user Git repository cloning — the spawn-form repo field, private-repo tokens, provider allowlist, and persistence — for operators enabling bring-your-own-repo workspaces. + +## Owner + +AMD Research diff --git a/skills/deploy-aup-learning-cloud/SKILL.md b/skills/deploy-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..6946dca6 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/SKILL.md @@ -0,0 +1,253 @@ +--- +name: deploy-aup-learning-cloud +description: >- + Group: Plan & deploy AUP Learning Cloud. Deploys AUP Learning Cloud (a + multi-node JupyterHub-on-k3s platform for AMD + GPUs) onto physical hardware end to end. Use when the user wants to install, + deploy, set up, or stand up AUP Learning Cloud, AUPLC, or "the learning + cloud" on a cluster; mentions a multi-AIPC or 3-node mini-cluster, PXE / + netboot / diskless agents, the Ansible inventory.yml, pb-pxe-controller, + pb-k3s-site, the ROCm GPU device plugin/labeller, an NFS provisioner, or a + JupyterHub values.yaml / Helm chart for this project. Covers both the + PXE-diskless topology and the SSH-preinstalled multi-node topology. Do not + use for the single-node "./auplc-installer install" flow, for building + notebook images, or for non-AUPLC JupyterHub or k3s installs. +--- + +# Deploy AUP Learning Cloud + +Stand up AUP Learning Cloud on a multi-node k3s cluster: build the cluster with +Ansible, expose AMD GPUs, provide shared storage, and deploy the JupyterHub +chart with Helm so users can log in and spawn GPU notebooks. + +This skill is written for any coding agent. Run the commands and edit the files +as described; the full, copy-runnable command sequence and the troubleshooting +table live in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud` on the operator/service machine. +- The service machine runs Ubuntu 24.04 with a reserved/static IP and internet + access. +- `ansible` on the operator machine; `kubectl` and `helm` for the cluster + (reference.md has the Helm install command). +- For GPU scheduling: AMD GPU nodes with a working in-kernel NIC driver. +- The user supplies the physical hardware. **No site values (IPs, subnet, SSH + keys, tokens) ship in the repo** — this skill generates them. + +## Helper script paths + +Resolve the deploy helpers before running the commands below. From any directory +in an AUP Learning Cloud checkout: + +```bash +REPO_ROOT="$(git rev-parse --show-toplevel)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +``` + +When this skill is installed as a plugin rather than used from a checkout, set +`DEPLOY_SKILL_DIR` to the absolute directory containing the loaded `SKILL.md`, +then derive the helpers from that directory: + +```bash +DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" +DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" +``` + +## Phase 1 — Interview + +Work through this in order. **The deployment-method choice (1a) is a hard gate: +ask it first and get an explicit answer before collecting anything else or +touching the machines.** + +### Phase 1a — Choose the deployment method (ask first, always) + +Ask the user to pick one. **Never assume or auto-select** — even when the +machines "look like" one case, present both options and let the user decide (you +may recommend, but you still need an explicit choice before continuing): + +| Choose | When | +| --- | --- | +| **PXE Diskless Netboot** (`topology: pxe-diskless`) — one service machine netboots diskless agents | Agents have no OS installed; you want zero per-machine install; small teaching lab. This is the [3-node mini-cluster guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html). | +| **Multi Node SSH Installation** (`topology: ssh-preinstalled`) — every node already runs Ubuntu | Each node has an OS and is reachable over SSH; closer to a long-running lab. This is the [multi-node guide](https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html). | + +The value in parentheses is the `topology` field for `gen_configs.py` (Phase 3) +and selects the matching section in [reference.md](reference.md). + +### Phase 1b — Collect the rest (some items branch on the choice above) + +Collect, and confirm back to the user, before touching anything: + +1. **Courses** wanted — drives the `values.yaml` course keys + team mappings + (full catalog setup lives in `configure-aup-learning-cloud-courses`). +2. **Node count** and which node is the controller/server, plus its static IP. + - *SSH path only:* also the hostname + IP of every agent node, and confirm + passwordless root SSH already reaches each one. +3. **GPU — do not ask the user to name the model.** Let the tooling find it: the + detectors report the GPUs (`$DEPLOY_SCRIPTS/detect_hardware.sh` in Phase 2) + and the real ROCm `amd.com/gpu.product-name` label + (`$DEPLOY_SCRIPTS/detect_cluster.sh` in Phase 5). Then + **confirm the detected GPU → accelerator-key mapping with the user** before it + goes into the values file. +4. *PXE path only:* service-machine NIC, subnet (CIDR), gateway, and DNS servers + (also auto-detected in Phase 2 and cross-checked), plus at least one SSH + public key for the rootfs and the apache web port. + +Login mode (`custom.authMode`) is unchanged — it stays at its `auto-login` +default; switch it later with `configure-aup-learning-cloud-auth` if needed. The +detailed steps for both paths are in [reference.md](reference.md). + +## Phase 2 — Discover + +On the service machine, run the bundled detector and cross-check its JSON +against the Phase 1 answers: + +```bash +"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns_servers, gpus[] +``` + +It reports the default-route NIC, the service-machine IP + subnet CIDR, the +gateway, DNS servers, and each AMD GPU (`lspci`, vendor `1002`) with the bound +`kernel_driver`. If a GPU's `kernel_driver` is empty, note its module for +`pxe_initramfs_modules` (PXE path only). Empty fields come back in `warnings` +so you know exactly what to ask the operator for. The detected GPUs are the +source of truth for the accelerator mapping — Phase 1 does not ask the user to +name them, so surface the detected list and confirm it with the user. + +## Phase 3 — Generate config + +Drive `$DEPLOY_SCRIPTS/gen_configs.py` rather than hand-writing YAML — it keeps the +three artifacts consistent, mints the k3s token locally with a CSPRNG (never +printed), `chmod 600`s the inventory, and pins `pxe_k3s_version == k3s_version`. + +```bash +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # fill from Phase 1 + 2 +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" +``` + +It writes, into `--out-dir`: + +1. `inventory.yml` — `server` host + `token` + `k3s_version` (agents empty for + PXE; listed for SSH) plus the `pxe_controller` group for PXE. +2. `pb-pxe-controller.vars.yml` — PXE path only: extra vars passed to + `deploy/ansible/playbooks/pb-pxe-controller.yml` with `-e @<absolute-path>` (`pxe_network_interface`, + `pxe_subnet`, `pxe_gateway`, `pxe_dns_servers`, `pxe_controller_ip`, + `pxe_k3s_server_ips`, `pxe_k3s_version`, `pxe_web_port`, + `pxe_rootfs_password`, `pxe_rootfs_authorized_keys`). +3. `values-basic-example.yaml` — `custom.accelerators.*.nodeSelector` (matched + to real GPU labels in Phase 5), `custom.resources.images`, the storage class + (`nfs-client`), `custom.authMode`, and the proxy `NodePort` (e.g. 30890). + +Review the artifacts, install the inventory and runtime overlay into the +checkout, and keep the PXE vars in the generated directory. +**Never commit `inventory.yml` — it holds the token.** Field-by-field guidance +is in [reference.md](reference.md). + +Map the generated artifacts into the checkout before Phase 5 validation: + +```bash +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" + +# PXE only: keep this generated secret in place and use its absolute path. +PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" +chmod 0600 "$PXE_VARS" +``` + +The generated `gpu.acceleratorKeys` activates the selected accelerators only +for the generic GPU resource. Wire selected accelerators into course resources +separately with `configure-aup-learning-cloud-courses`. + +## Phase 4 — Execute (with confirmation gates) + +Run the install in order. **Pause for explicit user confirmation before each +risky/irreversible step** (see Safety). The PXE path is, in brief: + +1. Install host packages on the service machine. +2. Run `pb-pxe-controller.yml -e @"$PXE_VARS"` to build the PXE/NFS rootfs, + then verify the + controller (dnsmasq, NFS, apache2, TFTP boot files). +3. `pb-base.yml` + `pb-k3s-site.yml` to install the single-node k3s server. +4. Publish the k3s token + kubeconfig for agents over the apache `/k3s/` endpoint. +5. Netboot the agents; watch them auto-join with `kubectl get nodes -o wide`. + +Run the PXE controller step with the generated vars file: + +```bash +cd "$REPO_ROOT/deploy/ansible" +ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" +``` + +The SSH path runs `pb-base.yml`, `pb-k3s-site.yml`, and `pb-rocm.yml` against +the inventory instead. Full commands for both paths are in [reference.md](reference.md). + +## Phase 5 — GPU, storage, and chart + +1. Install the AMD GPU device plugin + ROCm labeller, then read the **real** + cluster state: + + ```bash +"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # nodes[], gpu_product_names[], storage_classes[] + ``` + + Confirm the detected GPU → accelerator-key mapping with the user, then patch + `custom.accelerators.*.nodeSelector` so each `amd.com/gpu.product-name` + matches a value in `gpu_product_names`. Gate the install on a clean + pre-flight (exits non-zero on any mismatch): + + ```bash +# Set this to the topology selected in Phase 1a. +DEPLOY_TOPOLOGY=pxe-diskless +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml \ + --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run +``` + +For the PXE path, the validator and Ansible receive the same generated vars +file. Omit `--pxe-vars "$PXE_VARS"` for the SSH path. + +2. Create the notebook-PVC NFS export and install the `nfs-subdir-external-provisioner` + (storage class `nfs-client`). +3. Deploy the chart: + +```bash +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml +``` + +## Phase 6 — Validate end to end + +```bash +kubectl get nodes -o wide # server + agents Ready +kubectl get pods -A # nothing CrashLoopBackOff/Pending/ImagePullBackOff +kubectl get storageclass # nfs-client present +``` + +Then open the Hub (NodePort example: `http://<SERVICE_IP>:30890`), log in, +spawn a CPU notebook, confirm file persistence across a restart, then spawn a +GPU notebook and confirm its pod lands on a GPU node +(`kubectl get pods -n jupyterhub -o wide`). + +## Safety + +These steps are destructive or hard to reverse — **stop and get explicit user +confirmation before each one**, and never run them silently: + +- Building/rebuilding the PXE rootfs (`pxe_rootfs_force_rebuild: true`). +- Editing `/etc/exports` and restarting `nfs-kernel-server`. +- `kubectl delete node <name>` (debugging only). +- `helm uninstall` or a cluster reset (`pb-k3s-reset.yml`). +- Changing firmware boot order / disabling Secure Boot on agents. + +Never commit or push. Never write the k3s token, OAuth secrets, or SSH private +keys into tracked files. Preserve the four AUP Learning Cloud attribution +layers (see the project `AGENTS.md`) if any chart/Hub source is touched. + +## Reference + +Full step-by-step commands for both topologies, the GPU-label-to-accelerator +mapping, the `values.yaml` field guide, and the troubleshooting table: +[reference.md](reference.md). diff --git a/skills/deploy-aup-learning-cloud/reference.md b/skills/deploy-aup-learning-cloud/reference.md new file mode 100644 index 00000000..e67faf02 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/reference.md @@ -0,0 +1,442 @@ +# Deploy AUP Learning Cloud — Reference + +Full, copy-runnable commands for both deployment topologies, the GPU label +mapping, the `values.yaml` field guide, and the troubleshooting table. The +workflow and confirmation gates are in [SKILL.md](SKILL.md). + +## Contents + +- [Source guides](#source-guides) +- [PXE-diskless topology (3-node mini-cluster)](#pxe-diskless-topology-3-node-mini-cluster) +- [SSH-preinstalled topology (standard multi-node)](#ssh-preinstalled-topology-standard-multi-node) +- [GPU label to accelerator key](#gpu-label-to-accelerator-key) +- [values.yaml field guide](#valuesyaml-field-guide) +- [Troubleshooting](#troubleshooting) + +## Source guides + +- 3-node mini-cluster (PXE diskless): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html> +- Standard multi-node (SSH): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> + +Treat the live docs as the source of truth for version pins; this file +condenses the opinionated path. + +The helper commands are resolved through `DEPLOY_SCRIPTS` as defined in +[SKILL.md](SKILL.md#helper-script-paths), not through a checkout-root +`scripts/` directory. + +The two topology sections below are the two branches of the Phase 1a gate in +[SKILL.md](SKILL.md): **PXE Diskless Netboot** (`topology: pxe-diskless`) → +[PXE-diskless topology](#pxe-diskless-topology-3-node-mini-cluster); **Multi Node +SSH Installation** (`topology: ssh-preinstalled`) → +[SSH-preinstalled topology](#ssh-preinstalled-topology-standard-multi-node). + +## PXE-diskless topology (3-node mini-cluster) + +One service machine (AIPC 1) runs the PXE controller, the single-node k3s +server, NFS, and the apache k3s-credential endpoint. The other machines are +diskless agents that netboot and auto-join. Only AIPC 1 is Ansible-managed. + +### Step 1 — Prepare the service machine + +```bash +sudo apt update +sudo apt install -y git ansible curl ca-certificates jq \ + dnsmasq pxelinux syslinux-common apache2 \ + nfs-kernel-server debootstrap \ + grub-efi-amd64-signed shim-signed + +ip -br addr # record the NIC and IP +ip route # record the gateway +``` + +Give the local `root` a passwordless SSH login (or add `ansible_connection: +local` to the host vars to skip SSH entirely): + +```bash +sudo install -d -m 0700 /root/.ssh +sudo tee -a /root/.ssh/authorized_keys < ~/.ssh/id_ed25519.pub >/dev/null +sudo chmod 0600 /root/.ssh/authorized_keys +ssh root@<SERVICE_IP> true && echo root-ssh-ok +``` + +### Step 2 — Configure the inventory + +Edit `deploy/ansible/inventory.yml`. AIPC 1 is the only host; the `agent` group +stays empty (netboot agents are not Ansible-managed). Generate the token with +`openssl rand -base64 64` and keep it out of chat/VCS. + +```yaml +k3s_cluster: + children: + server: + hosts: + aipc1: + ansible_host: <SERVICE_IP> + agent: + hosts: {} # diskless netboot agents auto-join; do NOT list them here + vars: + ansible_user: root + k3s_version: v1.32.3+k3s1 + token: "<paste-a-strong-random-token>" # openssl rand -base64 64 + api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" + +pxe_controller: + hosts: + aipc1: + ansible_host: <SERVICE_IP> + vars: + ansible_port: 22 + ansible_user: root +``` + +### Step 3 — Prepare the generated PXE controller vars + +The network, controller, server-IP, and SSH-key values are empty by default and +the role asserts on them. `$DEPLOY_SCRIPTS/gen_configs.py` writes these values +to `generated/pb-pxe-controller.vars.yml`. Keep that file at mode `0600`; it can +contain `pxe_rootfs_password`. Resolve its absolute path for the Ansible and +validator commands instead of copying or merging it into the playbook: + +```bash +PXE_VARS="$(realpath ./generated/pb-pxe-controller.vars.yml)" +chmod 0600 "$PXE_VARS" +test "$(stat -c '%a' "$PXE_VARS")" = 600 +``` + +Review the generated values before the first run: + +```yaml +pxe_rootfs_force_rebuild: true # true for the first build (RISKY: rebuilds rootfs) +pxe_network_interface: "enp1s0" # service-machine NIC (Step 1) +pxe_subnet: "192.168.1.0/24" # node subnet, CIDR +pxe_gateway: "192.168.1.1" # default gateway (informational) +pxe_dns_servers: "8.8.8.8,8.8.4.4" +pxe_controller_ip: "192.168.1.10" # this service machine's IP +pxe_k3s_server_ips: + - "192.168.1.10" +pxe_k3s_version: "v1.32.3+k3s1" # MUST match inventory k3s_version +pxe_web_port: 8080 # apache port for the k3s token/kubeconfig (not 80) +pxe_rootfs_password: "" # optional; empty disables password login (use ansible-vault if set) +pxe_rootfs_authorized_keys: + - "ssh-ed25519 AAAA... you@host" # at least one key required +``` + +Set `pxe_rootfs_force_rebuild: false` after the first stable build so you do +not rebuild the rootfs under running agents. The playbook also exposes +`pxe_apt_mirror`, `pxe_rootfs_packages`, and `pxe_initramfs_modules` (add your +NIC module here if it lacks an in-kernel driver) — leave these at their defaults +unless discovery flagged a need. + +### Step 4 — Run the PXE controller playbook + +```bash +cd ~/aup-learning-cloud +REPO_ROOT="$(pwd)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +PXE_VARS="$(realpath "$REPO_ROOT/generated/pb-pxe-controller.vars.yml")" +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" \ + --topology pxe-diskless --pxe-vars "$PXE_VARS" \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml +cd "$REPO_ROOT/deploy/ansible" +ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" +``` + +### Step 5 — Verify the controller + +```bash +systemctl is-active dnsmasq nfs-kernel-server apache2 +showmount -e localhost +ls -l /srv/tftp/pxelinux.0 /srv/tftp/grubnetx64.efi /srv/tftp/vmlinuz /srv/tftp/initrd.img +curl -I http://127.0.0.1:8080/k3s/ # 403 expected (dir exists, empty) +``` + +The `/k3s/` endpoint is served on port 8080 (k3s owns 80/443 for ingress). + +### Step 6 — Install the single-node k3s server + +Run **without** `sudo` (key-based root SSH already connects as root): + +```bash +cd ~/aup-learning-cloud/deploy/ansible +ansible-playbook -i inventory.yml playbooks/pb-base.yml +ansible-playbook -i inventory.yml playbooks/pb-k3s-site.yml +export KUBECONFIG=~/.kube/config # add to ~/.bashrc to persist +kubectl get nodes -o wide +``` + +### Step 7 — Publish k3s credentials for agents + +```bash +sudo install -d -m 0755 /var/www/html/k3s +sudo install -m 0644 /var/lib/rancher/k3s/server/token /var/www/html/k3s/token +sudo sed "s#https://127.0.0.1:6443#https://<SERVICE_IP>:6443#g" \ + /etc/rancher/k3s/k3s.yaml | sudo tee /var/www/html/k3s/kubeconfig >/dev/null +sudo chmod 0644 /var/www/html/k3s/token /var/www/html/k3s/kubeconfig +sudo systemctl reload apache2 + +curl -fsS http://127.0.0.1:8080/k3s/token >/dev/null && echo token-ok +curl -fsS http://127.0.0.1:8080/k3s/kubeconfig >/dev/null && echo kubeconfig-ok +``` + +### Step 8 — Netboot the agents + +On each agent: disable Secure Boot, enable network boot, and put PXE before the +local disk in the firmware boot order. Boot, then watch them register: + +```bash +watch kubectl get nodes -o wide +``` + +Agents appear as `agent-<mac>` nodes and become `Ready`. + +### Step 9 — Validate agent persistence + +Reboot one agent; confirm it rejoins with the same identity. On the agent: + +```bash +mount | grep /var/lib/rancher/k3s +test -f /var/lib/rancher/k3s/node-password && echo node-password-ok +systemctl status mount-local-disk k3s-agent --no-pager +``` + +`kubectl delete node <name>` clears a stale node object — **debugging only**, +confirm with the user first. + +Continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). + +## SSH-preinstalled topology (standard multi-node) + +Every node already runs Ubuntu 24.04 and is reachable over passwordless SSH. + +### Prepare SSH and inventory + +Helper scripts in `deploy/scripts/` enable root SSH and distribute kubeconfig: + +```bash +./deploy/scripts/edit_sshd.sh +./deploy/scripts/setup_ssh_root_access.sh +./deploy/scripts/deploy-kubeconfig.sh +``` + +Edit `deploy/ansible/inventory.yml` — list every node under `server`/`agent`: + +```yaml +k3s_cluster: + children: + server: + hosts: + <SERVER-HOSTNAME>: + agent: + hosts: + <AGENT-HOSTNAME-1>: + <AGENT-HOSTNAME-2>: + vars: + ansible_port: 22 + ansible_user: root + k3s_version: v1.32.3+k3s1 + token: "<strong-random-token>" # openssl rand -base64 64 + api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" +``` + +### Build the cluster + +```bash +cd deploy/ansible +sudo ansible-playbook playbooks/pb-base.yml # base OS / packages +sudo ansible-playbook playbooks/pb-k3s-site.yml # deploy k3s +sudo ansible-playbook playbooks/pb-rocm.yml # ROCm on GPU nodes +``` + +Related: `pb-k3s-upgrade.yml` (upgrade), `pb-k3s-reset.yml` (reset — RISKY). +Then install `kubectl`/`helm` on the operator machine (see Helm command below) +and continue with [Step 10 (GPU)](#step-10--amd-gpu-device-plugin-and-labeller). + +### Install Helm + +```bash +wget https://get.helm.sh/helm-v3.17.2-linux-amd64.tar.gz -O /tmp/helm.tar.gz +cd /tmp && tar -zxvf helm.tar.gz +sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm +``` + +## Step 10 — AMD GPU device plugin and labeller + +```bash +kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-dp.yaml +kubectl create -f https://raw.githubusercontent.com/ROCm/k8s-device-plugin/master/k8s-ds-amdgpu-labeller.yaml + +kubectl get pods -A | grep -i amd +kubectl describe node <AGENT_NODE_NAME> | grep amd.com/gpu +``` + +Use the labels that actually appear. Common keys: +`amd.com/gpu.product-name`, `amd.com/gpu.family`, `amd.com/gpu.device-id`. + +## Step 11 — Shared NFS storage for notebook PVCs + +This is separate from the PXE rootfs export. Append the export directly to +`/etc/exports` (on Ubuntu 24.04 `/etc/exports.d/*.conf` is ignored): + +```bash +sudo mkdir -p <NFS_EXPORT> +sudo chown -R nobody:nogroup <NFS_EXPORT> +sudo chmod 0777 <NFS_EXPORT> +echo "<NFS_EXPORT> <CLUSTER_SUBNET>(rw,sync,no_subtree_check,no_root_squash,insecure)" | sudo tee -a /etc/exports +sudo exportfs -ra +sudo systemctl restart nfs-kernel-server +showmount -e localhost +``` + +Install the provisioner (storage class `nfs-client`): + +```bash +cd ~/aup-learning-cloud +cp deploy/k8s/nfs-provisioner/values.yaml deploy/k8s/nfs-provisioner/values.local.yaml +# edit values.local.yaml: nfs.server, nfs.path, storageClass.name = nfs-client +helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ +helm repo update +helm upgrade --install nfs-subdir-external-provisioner \ + nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ + --namespace nfs-provisioner --create-namespace \ + -f deploy/k8s/nfs-provisioner/values.local.yaml +kubectl get storageclass +``` + +## Step 12 — Configure JupyterHub values + +The generated `runtime/values-basic-example.yaml` is the canonical deployment +overlay. Review and keep it when Phase 3 generated one. Only when no generated +overlay exists, start a manual overlay from the example: + +```bash +cd ~/aup-learning-cloud/runtime +if [ ! -e values-basic-example.yaml ]; then + cp values-multi-nodes.yaml.example values-basic-example.yaml +fi +``` + +Minimum edits (see the [field guide](#valuesyaml-field-guide)): + +```yaml +custom: + authMode: "auto-login" # single-machine default; avoid "dummy" (login 404s) + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: "<GPU_PRODUCT_LABEL>" # from Step 10 + quotaRate: 3 + resources: + images: + cpu: "<CPU_NOTEBOOK_IMAGE>" + gpu: "<GPU_NOTEBOOK_IMAGE>" +hub: + db: + pvc: + storageClassName: nfs-client +singleuser: + storage: + dynamic: + storageClass: nfs-client +proxy: + service: + type: NodePort + nodePorts: + http: 30890 +``` + +## Step 13 — Deploy AUP Learning Cloud + +```bash +cd ~/aup-learning-cloud +helm upgrade --install jupyterhub ./runtime/chart \ + --namespace jupyterhub --create-namespace \ + -f runtime/values.yaml \ + -f runtime/values-basic-example.yaml + +kubectl get pods -n jupyterhub -o wide +kubectl get svc -n jupyterhub +``` + +For later config changes, re-run the same `helm upgrade --install`. + +## Step 14 — End-to-end validation + +```bash +kubectl get nodes -o wide +kubectl get pods -A +kubectl get storageclass +kubectl describe node <AGENT_NODE_NAME> | grep amd.com/gpu +``` + +Then browse to `http://<SERVICE_IP>:30890` (or your ingress host), log in, +spawn a CPU notebook, create a file, restart and confirm it persists, then +spawn a GPU notebook and confirm its pod lands on a GPU node. + +## GPU label to accelerator key + +The chart's accelerator catalog (`runtime/values.yaml`) is keyed by accelerator +names; map the ROCm labeller's `amd.com/gpu.product-name` to the right key. The +GPU is auto-detected (Phase 2 and Phase 5), not named by the user in the +interview — use this table to confirm the detected product-name → key mapping +with the user. Verify against the live values file — product names can normalize +differently per fleet. + +| `amd.com/gpu.product-name` (example) | Accelerator key | +| --- | --- | +| `AMD_Radeon_780M_Graphics` | `phx` | +| `AMD_Radeon_890M_Graphics` | `strix` | +| `AMD_Radeon_8060S_Graphics` | `strix-halo` | +| `AMD_Radeon_RX_9070_XT` | `9070xt` | +| `AMD_Radeon_AI_PRO_R9700` | `r9700` | +| `AMD_Radeon_RX_9600_GRE` | `9600gre` | + +If your labeller reports a different product name, update the matching +`custom.accelerators.*.nodeSelector` entry to that exact string. + +## values.yaml field guide + +Sections to review in the generated `values-basic-example.yaml`, or in the +manual `values-multi-nodes.yaml.example` copy when generation was not used: + +| Field | Purpose | +| --- | --- | +| `custom.authMode` | `auto-login` for the single-machine example; OAuth modes for real auth | +| `custom.githubOrgName`, `hub.config.GitHubOAuthenticator` | GitHub OAuth (when not auto-login) | +| `custom.adminUser` | Hub admin | +| `custom.accelerators.*.nodeSelector` | Must match real `amd.com/gpu.*` labels | +| `custom.resources.images` | CPU/GPU/course notebook images | +| `custom.resources.requirements`, `custom.teams.mapping`, `custom.quota` | Per-team resources and quotas | +| `hub.db.pvc.storageClassName`, `singleuser.storage.dynamic.storageClass` | `nfs-client` for multi-node | +| `proxy.service`, `ingress` | NodePort (e.g. 30890) or ingress host | + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Playbook fails immediately on an assert | A required PXE var is empty | Re-check `pxe_controller_ip`, `pxe_subnet`, `pxe_network_interface`, `pxe_dns_servers`, `pxe_k3s_server_ips`, and at least one SSH key | +| Agent never shows the PXE menu | Firmware boot order, network boot disabled, or Proxy-DHCP not reaching the client | Firmware, switch port, `systemctl status dnsmasq`, `journalctl -u dnsmasq` | +| Agent gets an IP but cannot load boot files | TFTP blocked, missing files, or Secure Boot still on | `/srv/tftp`, firewall, Secure Boot disabled, `dnsmasq` logs | +| Agent has no network during netboot | NIC has no in-kernel driver in the initramfs | `lspci -nnk`, add the module to `pxe_initramfs_modules`, rebuild rootfs | +| Agent kernel boots but cannot mount rootfs | NFS export, subnet ACL, or wrong `pxe_controller_ip` | `showmount -e <SERVICE_IP>`, `/etc/exports`, rootfs kernel args | +| Agent waits for the k3s token | Token not published or apache ACL blocks the subnet | `curl http://<SERVICE_IP>:8080/k3s/token`, apache config | +| Agent joins once but fails after reboot | Missing local k3s persistence or lost node password | `mount-local-disk`, `/var/lib/rancher/k3s/node-password`, `k3s-agent` logs | +| Agent fails to join with a version error | Agent rootfs k3s newer than the server | Align `pxe_k3s_version` with `k3s_version`, rebuild rootfs | +| Agent node does not join (SSH path) | Hostname resolution, token, or `api_endpoint` mismatch | `systemctl status k3s-agent`, `journalctl -u k3s-agent`, `/etc/hosts` | +| GPU notebook stays Pending | Chart `nodeSelector` mismatch or GPUs exhausted | `kubectl describe pod -n jupyterhub`, node labels | +| PVC stays Pending | StorageClass name mismatch or NFS provisioner cannot mount | `kubectl get storageclass`, provisioner logs, NFS export | +| `kubectl` permission denied on `k3s.yaml` | kubeconfig not readable | `export KUBECONFIG=~/.kube/config`, or `--write-kubeconfig-mode=644` in inventory `extra_server_args` | + +For a complete reset (RISKY — confirm with the user): + +```bash +cd deploy/ansible +sudo ansible-playbook playbooks/pb-k3s-reset.yml # whole cluster +sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit <node> # single node +``` + +## Out of scope + +Zot registry mirror, Cloudflare Tunnel ingress, monitoring/Grafana, HA k3s, +external databases, and NPU setup. Add them only after the minimal deployment +boots agents, schedules GPU notebooks, and persists notebook storage. diff --git a/skills/deploy-aup-learning-cloud/scripts/README.md b/skills/deploy-aup-learning-cloud/scripts/README.md new file mode 100644 index 00000000..4e212dcc --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/README.md @@ -0,0 +1,77 @@ +# Helper scripts + +Deterministic helpers the deploy skill runs instead of generating commands ad +hoc. They are dependency-light (`bash` + `python3`, plus the obvious system +tools) and agent-agnostic, and follow the script conventions in +[../../../CONTRIBUTING.md](../../../CONTRIBUTING.md). Each emits JSON or a clear +report and uses exit codes the agent can branch on. + +| Script | Run when | What it does | +| --- | --- | --- | +| `detect_hardware.sh` | Phase 2, on the service machine | Detects the default-route NIC, IPv4 + subnet CIDR, gateway, DNS servers, and AMD GPUs (`lspci`, vendor `1002`) with their kernel driver. Emits JSON for filling PXE / network vars. Read-only. | +| `detect_cluster.sh` | After k3s + the device plugin are up | `kubectl get` of nodes, real `amd.com/gpu.*` labels, storage classes, and whether the ROCm device plugin + labeller DaemonSets are running. Emits JSON. Read-only. | +| `gen_configs.py` | Phase 3 | From a small cluster-spec (`--print-schema`), writes `inventory.yml`, `pb-pxe-controller.vars.yml` (PXE only), and `values-basic-example.yaml`. Generates the k3s token locally with `secrets` (never printed), `chmod 600` on the inventory, and pins `pxe_k3s_version == k3s_version`. | +| `validate.py` | Before each `ansible-playbook` / `helm` run | For `pxe-diskless`, checks required PXE vars and `k3s_version == pxe_k3s_version`; for both topologies, checks GPU labels only for active resource `acceleratorKeys` (when given `detect_cluster.sh` output), and optionally runs a `helm template` dry-run. Exit 1 on any failure. | + +## Quick reference + +From any directory in a checkout, resolve helpers with: + +```bash +REPO_ROOT="$(git rev-parse --show-toplevel)" +DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts" +``` + +For an installed plugin, set `DEPLOY_SKILL_DIR` to the absolute directory +containing the loaded `SKILL.md`, then use: + +```bash +DEPLOY_SKILL_DIR="/absolute/path/to/deploy-aup-learning-cloud" +DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts" +``` + +```bash +# Phase 2 — discover the host +"$DEPLOY_SCRIPTS/detect_hardware.sh" # JSON: nic, ip, subnet_cidr, gateway, dns, gpus[] + +# Phase 3 — generate config from a spec +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --print-schema > spec.json # then edit spec.json +GENERATED_DIR="$REPO_ROOT/generated" +python3 "$DEPLOY_SCRIPTS/gen_configs.py" --spec spec.json --out-dir "$GENERATED_DIR" +install -m 0600 "$GENERATED_DIR/inventory.yml" "$REPO_ROOT/deploy/ansible/inventory.yml" +install -m 0644 "$GENERATED_DIR/values-basic-example.yaml" "$REPO_ROOT/runtime/values-basic-example.yaml" +# PXE only: keep the generated secret in place and resolve its absolute path. +PXE_VARS="$(realpath "$GENERATED_DIR/pb-pxe-controller.vars.yml")" +chmod 0600 "$PXE_VARS" +cd "$REPO_ROOT/deploy/ansible" +ansible-playbook -i inventory.yml playbooks/pb-pxe-controller.yml -e @"$PXE_VARS" + +# Phase 5 — after k3s + device plugin are up +"$DEPLOY_SCRIPTS/detect_cluster.sh" > cluster.json # JSON: nodes[], gpu_product_names[], storage_classes[] + +# Before running playbooks / helm (set to the selected topology) +DEPLOY_TOPOLOGY=pxe-diskless +python3 "$DEPLOY_SCRIPTS/validate.py" --repo "$REPO_ROOT" --topology "$DEPLOY_TOPOLOGY" \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml \ + --pxe-vars "$PXE_VARS" --cluster cluster.json --helm-dry-run +``` + +Omit `--pxe-vars "$PXE_VARS"` for `ssh-preinstalled`. For `pxe-diskless`, the +validator and Ansible must receive the same generated file. + +Generated `gpu.acceleratorKeys` wires the selected accelerators to the generic +GPU resource. Use `configure-aup-learning-cloud-courses` to wire course +resources separately. + +## Conventions + +- **JSON to stdout, diagnostics to stderr.** `detect_*.sh` always print a JSON + object; partial detection is reported via empty fields + a `warnings` array + rather than failing, so the agent can decide what to ask the operator. +- **Exit codes mean something.** `0` success (warnings allowed), `1` a real + validation failure, `2` a usage / missing-tooling error. +- **Secrets never touch stdout or VCS.** `gen_configs.py` mints the k3s token + with a CSPRNG, writes it only into `inventory.yml`, and `chmod 600`s it. +- **No third-party Python.** `gen_configs.py` / `validate.py` use the stdlib + only (no PyYAML), so they run on a bare operator machine. YAML is emitted + from templates and parsed with targeted scanning. diff --git a/skills/deploy-aup-learning-cloud/scripts/detect_cluster.sh b/skills/deploy-aup-learning-cloud/scripts/detect_cluster.sh new file mode 100755 index 00000000..eda1cc65 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/detect_cluster.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# detect_cluster.sh -- after k3s is up, emit a JSON snapshot of the cluster the +# deploy skill needs to align custom.accelerators.*.nodeSelector with the REAL +# amd.com/gpu.* labels, confirm storage, and check the ROCm device plugin + +# labeller are running. Read-only: it only runs `kubectl get`. +# +# Usage: +# ./detect_cluster.sh # uses current KUBECONFIG +# KUBECONFIG=~/.kube/config ./detect_cluster.sh +# ./detect_cluster.sh --kubeconfig /path/to/k3s.yaml +# ./detect_cluster.sh -h | --help +# +# Output (stdout) is a single JSON object: +# { +# "nodes": [ +# {"name":"aipc1","ready":true,"roles":["control-plane"], +# "internal_ip":"192.168.0.140","gpu_product_names":["AMD_Radeon_8060S_Graphics"], +# "gpu_allocatable":"1","gpu_labels":{...}} +# ], +# "gpu_product_names": ["AMD_Radeon_8060S_Graphics"], +# "storage_classes": [{"name":"local-path","default":true}], +# "amdgpu_device_plugin": true, +# "amdgpu_labeller": true, +# "warnings": ["..."] +# } +# +# Exit codes: 0 on success (including "cluster reachable but nothing labelled +# yet"); 2 if kubectl/python3 missing or the API server is unreachable. +# +# Dependencies: bash, kubectl, python3 (stdlib only -- parses `kubectl -o json`). + +set -uo pipefail + +KCFG="" +while [[ $# -gt 0 ]]; do + case "$1" in + --kubeconfig) KCFG="${2:-}"; shift 2 ;; + -h|--help) sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "detect_cluster: unknown arg $1" >&2; exit 2 ;; + esac +done + +command -v kubectl >/dev/null 2>&1 || { echo "detect_cluster: kubectl is required" >&2; exit 2; } +command -v python3 >/dev/null 2>&1 || { echo "detect_cluster: python3 is required" >&2; exit 2; } +[[ -n "$KCFG" ]] && export KUBECONFIG="$KCFG" + +kc() { kubectl "$@" 2>/dev/null; } + +# Fail fast (exit 2) if we cannot reach the API server at all -- this is the +# single most common "ran too early / wrong kubeconfig" case. +if ! kc version --request-timeout=10s >/dev/null; then + echo "detect_cluster: cannot reach the Kubernetes API server. Check KUBECONFIG / that k3s is up." >&2 + exit 2 +fi + +NODES_JSON="$(kc get nodes -o json || echo '{}')" +SC_JSON="$(kc get storageclass -o json || echo '{}')" +# The device plugin + labeller are DaemonSets; their names/namespaces can vary, +# so we scan all daemonsets and match on the amdgpu substring. +DS_JSON="$(kc get ds -A -o json || echo '{}')" + +export DC_NODES="$NODES_JSON" DC_SC="$SC_JSON" DC_DS="$DS_JSON" + +python3 <<'PY' +import json, os + +def load(name): + try: + return json.loads(os.environ.get(name, "") or "{}") + except json.JSONDecodeError: + return {} + +nodes_raw = load("DC_NODES").get("items", []) +sc_raw = load("DC_SC").get("items", []) +ds_raw = load("DC_DS").get("items", []) + +warnings = [] +nodes = [] +all_products = set() +for n in nodes_raw: + meta = n.get("metadata", {}) + name = meta.get("name", "") + labels = meta.get("labels", {}) or {} + status = n.get("status", {}) + ready = False + for c in status.get("conditions", []) or []: + if c.get("type") == "Ready": + ready = (c.get("status") == "True") + roles = sorted( + k.split("/", 1)[1] or "node" + for k in labels + if k.startswith("node-role.kubernetes.io/") + ) + internal_ip = "" + for a in status.get("addresses", []) or []: + if a.get("type") == "InternalIP": + internal_ip = a.get("address", "") + gpu_labels = {k: v for k, v in labels.items() if k.startswith("amd.com/gpu")} + products = [v for k, v in gpu_labels.items() if k == "amd.com/gpu.product-name"] + all_products.update(products) + alloc = (status.get("allocatable", {}) or {}).get("amd.com/gpu", "0") + nodes.append({ + "name": name, + "ready": ready, + "roles": roles, + "internal_ip": internal_ip, + "gpu_product_names": products, + "gpu_allocatable": alloc, + "gpu_labels": gpu_labels, + }) + +storage_classes = [] +for sc in sc_raw: + meta = sc.get("metadata", {}) + ann = meta.get("annotations", {}) or {} + is_default = ann.get("storageclass.kubernetes.io/is-default-class") == "true" + storage_classes.append({"name": meta.get("name", ""), "default": is_default}) + +def has_ds(substr): + for ds in ds_raw: + if substr in ds.get("metadata", {}).get("name", "").lower(): + return True + return False + +device_plugin = has_ds("device-plugin") or has_ds("amdgpu-dp") or ( + any("amdgpu" in ds.get("metadata", {}).get("name", "").lower() + and "label" not in ds.get("metadata", {}).get("name", "").lower() + for ds in ds_raw) +) +labeller = has_ds("labeller") or has_ds("labeler") or has_ds("amdgpu-labeller") + +if not nodes: + warnings.append("no nodes returned; cluster may still be initialising") +if not all_products: + warnings.append("no amd.com/gpu.product-name labels yet; install the ROCm device plugin + labeller, then re-run") +if not device_plugin: + warnings.append("AMD GPU device plugin DaemonSet not detected") +if not labeller: + warnings.append("ROCm node labeller DaemonSet not detected") + +print(json.dumps({ + "nodes": nodes, + "gpu_product_names": sorted(all_products), + "storage_classes": storage_classes, + "amdgpu_device_plugin": bool(device_plugin), + "amdgpu_labeller": bool(labeller), + "warnings": warnings, +}, indent=2)) +PY diff --git a/skills/deploy-aup-learning-cloud/scripts/detect_hardware.sh b/skills/deploy-aup-learning-cloud/scripts/detect_hardware.sh new file mode 100755 index 00000000..67d5a87c --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/detect_hardware.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# detect_hardware.sh -- inspect the service machine and emit a JSON snapshot of +# the network and AMD GPU facts the deploy skill needs to fill in PXE / inventory +# variables. Read-only: it never changes the host. +# +# Usage: +# ./detect_hardware.sh # auto-detect the default-route NIC +# ./detect_hardware.sh --nic enp1s0 # force a specific NIC +# ./detect_hardware.sh -h | --help +# +# Output (stdout) is a single JSON object: +# { +# "nic": "enp1s0", +# "ip": "192.168.0.140", +# "subnet_cidr": "192.168.0.0/24", +# "gateway": "192.168.0.1", +# "dns_servers": "8.8.8.8,8.8.4.4", +# "gpus": [ {"pci":"c5:00.0","vendor":"1002","description":"...","kernel_driver":"amdgpu"} ], +# "warnings": [ "..." ] +# } +# +# Exit codes: 0 always (partial detection is reported via empty fields + +# warnings so the agent can decide what to ask the operator). Hard tooling +# failures (no python3) exit 2. +# +# Dependencies: bash, iproute2 (ip), pciutils (lspci), python3 (stdlib only). +# python3 is used purely to serialise JSON safely (lspci descriptions contain +# brackets, quotes, commas). No third-party packages. + +set -uo pipefail + +NIC="" +while [[ $# -gt 0 ]]; do + case "$1" in + --nic) NIC="${2:-}"; shift 2 ;; + -h|--help) sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "detect_hardware: unknown arg $1" >&2; exit 2 ;; + esac +done + +command -v python3 >/dev/null 2>&1 || { echo "detect_hardware: python3 is required" >&2; exit 2; } + +warnings=() + +# --- NIC: default to the interface owning the default route --------------- +if [[ -z "$NIC" ]]; then + NIC="$(ip -o route show default 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}')" +fi +[[ -z "$NIC" ]] && warnings+=("no default-route NIC found; pass --nic explicitly") + +# --- IPv4 address + CIDR on that NIC -------------------------------------- +IP=""; CIDR="" +if [[ -n "$NIC" ]]; then + # e.g. "192.168.0.140/24" + addr="$(ip -o -f inet addr show "$NIC" 2>/dev/null | awk '{print $4; exit}')" + if [[ -n "$addr" ]]; then + IP="${addr%/*}" + prefix="${addr#*/}" + # Network address for the CIDR (zero the host bits) via python ipaddress. + CIDR="$(python3 - "$addr" <<'PY' 2>/dev/null +import ipaddress, sys +net = ipaddress.ip_interface(sys.argv[1]).network +print(net.with_prefixlen) +PY +)" + fi +fi +[[ -z "$IP" ]] && warnings+=("no IPv4 address on NIC '$NIC'") + +# --- Default gateway ------------------------------------------------------ +GATEWAY="$(ip -o route show default 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="via"){print $(i+1); exit}}')" +[[ -z "$GATEWAY" ]] && warnings+=("no default gateway found") + +# --- DNS servers ---------------------------------------------------------- +# Prefer systemd-resolved when present; fall back to /etc/resolv.conf. +DNS="" +if command -v resolvectl >/dev/null 2>&1; then + DNS="$(resolvectl dns 2>/dev/null | grep -oE '([0-9]{1,3}\.){3}[0-9]{1,3}' | sort -u | paste -sd, -)" +fi +if [[ -z "$DNS" && -r /etc/resolv.conf ]]; then + DNS="$(awk '/^nameserver/{print $2}' /etc/resolv.conf | grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}$' | paste -sd, -)" +fi +[[ -z "$DNS" ]] && warnings+=("no DNS servers detected; defaulting suggestion is 8.8.8.8,8.8.4.4") + +# --- AMD GPUs via lspci --------------------------------------------------- +# AMD/ATI PCI vendor id is 1002. We hand the full `lspci -D -nnk` dump to +# python3 (below) and parse device blocks there: mawk (Ubuntu's default awk) +# does not support {n} interval regexes, so block parsing in python is far more +# portable. We record the bound kernel driver (amdgpu = the in-kernel driver is +# loaded, which the PXE rootfs needs for GPU scheduling). +LSPCI_RAW="" +if command -v lspci >/dev/null 2>&1; then + LSPCI_RAW="$(lspci -D -nnk 2>/dev/null)" +else + warnings+=("lspci not found (install pciutils); GPU detection skipped") +fi + +# Hand everything to python3 for safe JSON assembly. +export DH_NIC="$NIC" DH_IP="$IP" DH_CIDR="$CIDR" DH_GW="$GATEWAY" DH_DNS="$DNS" +export DH_LSPCI="$LSPCI_RAW" +DH_WARNINGS="$(printf '%s\n' "${warnings[@]:-}")" +export DH_WARNINGS + +python3 <<'PY' +import json, os, re + +# PCI classes we treat as a GPU/accelerator: VGA (0300), 3D (0302), +# Display (0380), Processing accelerator (1200). +GPU_CLASSES = ("0300", "0302", "0380", "1200") + +def amd_gpus(raw): + out = [] + cur = None + for line in (raw or "").splitlines(): + # Device header lines start at column 0 with a PCI address. + if re.match(r"^[0-9a-fA-F]{4}:", line): + if cur: + out.append(cur) + cur = None + m = re.match( + r"^(\S+)\s+.*?\[(?P<cls>[0-9a-f]{4})\]:\s+(?P<desc>.*?)\s*" + r"\[(?P<vendor>[0-9a-f]{4}):(?P<dev>[0-9a-f]{4})\]", + line) + if not m: + continue + if m.group("vendor") != "1002" or m.group("cls") not in GPU_CLASSES: + continue + cur = { + "pci": m.group(1), + "vendor": "1002", + "device_id": m.group("dev"), + "description": m.group("desc").strip(), + "kernel_driver": "", + } + elif cur is not None: + dm = re.search(r"Kernel driver in use:\s*(\S+)", line) + if dm: + cur["kernel_driver"] = dm.group(1) + if cur: + out.append(cur) + return out + +warnings = [w for w in (os.environ.get("DH_WARNINGS", "").splitlines()) if w.strip()] +print(json.dumps({ + "nic": os.environ.get("DH_NIC", ""), + "ip": os.environ.get("DH_IP", ""), + "subnet_cidr": os.environ.get("DH_CIDR", ""), + "gateway": os.environ.get("DH_GW", ""), + "dns_servers": os.environ.get("DH_DNS", ""), + "gpus": amd_gpus(os.environ.get("DH_LSPCI", "")), + "warnings": warnings, +}, indent=2)) +PY diff --git a/skills/deploy-aup-learning-cloud/scripts/gen_configs.py b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py new file mode 100755 index 00000000..e0d91ee8 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/gen_configs.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Generate AUP Learning Cloud deploy artifacts from a small cluster-spec. + +Given a JSON cluster-spec (see ``--print-schema``), write the three files the +deploy skill needs, keeping them mutually consistent: + + 1. ``inventory.yml`` -- Ansible inventory (server + token + + k3s_version; agents listed for the + SSH topology, empty for PXE). + 2. ``pb-pxe-controller.vars.yml`` -- PXE topology only: extra vars passed to + pb-pxe-controller.yml with + ``-e @<absolute-path>``. + 3. ``values-basic-example.yaml`` -- Helm overlay: accelerator nodeSelectors, + storage class, proxy NodePort, authMode. + +Design choices (deliberate): + + * stdlib only (json, argparse, secrets, base64, pathlib). No PyYAML, so this + runs on a bare operator machine. YAML is emitted from templates, not a + serialiser -- the output is small, fixed-shape, and carries the copyright header. + * The k3s token is generated locally with ``secrets`` (CSPRNG) and written + ONLY into inventory.yml. It is never printed to stdout/stderr. Pass + ``--token-file`` to reuse an existing token instead of minting one. + * ``pxe_k3s_version`` is forced equal to ``k3s_version`` so agents can never + be newer than the server (k3s refuses that). + * Existing files are not overwritten unless ``--force`` is given. + +Usage: + gen_configs.py --print-schema + gen_configs.py --spec spec.json --out-dir ./generated + cat spec.json | gen_configs.py --spec - --out-dir ./generated --force + +Exit codes: 0 on success; 1 on a spec/validation error; 2 on a usage error. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import secrets +import shutil +import sys +import tempfile +from contextlib import suppress +from pathlib import Path + +HEADER_HASH = ( + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.\n" + "# Generated by auplc-skills gen_configs.py -- review before use.\n" +) + +# Default GPU product-name labels, keyed by the accelerator key used in +# runtime/values.yaml (custom.accelerators.<key>). Verified against the chart's +# values.yaml; override per fleet via spec["accelerators"][key]["product_name"]. +DEFAULT_ACCEL_LABELS = { + "phx": "AMD_Radeon_780M_Graphics", + "strix": "AMD_Radeon_890M_Graphics", + "strix-halo": "AMD_Radeon_8060S_Graphics", + "9070xt": "AMD_Radeon_RX_9070_XT", + "r9700": "AMD_Radeon_AI_PRO_R9700", + "9600gre": "AMD_Radeon_RX_9600_GRE", +} + +SCHEMA = { + "topology": "pxe-diskless | ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "aipc1", "ip": "192.168.0.140"}, + "agents": [{"name": "aipc2", "ip": "192.168.0.141"}], + "network": { + "interface": "enp1s0", + "subnet": "192.168.0.0/24", + "gateway": "192.168.0.1", + "dns_servers": "8.8.8.8,8.8.4.4", + }, + "pxe": { + "authorized_keys": ["ssh-ed25519 AAAA... you@host"], + "rootfs_password": "", + "web_port": 8080, + }, + "accelerators": {"strix-halo": {"product_name": "AMD_Radeon_8060S_Graphics"}}, + "storage": {"class": "nfs-client"}, + "proxy": {"node_port": 30890}, + "auth_mode": "auto-login", + "images": {"cpu": "ghcr.io/amdresearch/auplc-default:latest", "gpu": "ghcr.io/amdresearch/auplc-base:latest"}, +} + + +def die(msg: str, code: int = 1) -> None: + print(f"gen_configs: {msg}", file=sys.stderr) + raise SystemExit(code) + + +def gen_token() -> str: + # Mirror `openssl rand -base64 64`: 64 random bytes, base64-encoded. + return base64.b64encode(secrets.token_bytes(64)).decode("ascii") + + +def require(spec: dict, path: str): + cur = spec + for part in path.split("."): + if not isinstance(cur, dict) or part not in cur or cur[part] in (None, "", []): + die(f"spec is missing required field '{path}'") + cur = cur[part] + return cur + + +def yaml_quote(s: str) -> str: + return '"' + str(s).replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def validate_accelerators(spec: dict) -> None: + if "accelerators" not in spec: + return + accelerators = spec["accelerators"] + if not isinstance(accelerators, dict): + die("spec.accelerators must be a mapping") + unsupported = sorted(set(accelerators) - set(DEFAULT_ACCEL_LABELS)) + if len(unsupported) == 1: + die(f"unsupported accelerator key '{unsupported[0]}'") + if unsupported: + die(f"unsupported accelerator keys: {', '.join(unsupported)}") + for key, config in accelerators.items(): + if not isinstance(config, dict): + die(f"accelerators.{key} must be a mapping") + + +def validate_config_shapes(spec: dict) -> None: + if not isinstance(spec, dict): + die("spec must be a mapping") + validate_accelerators(spec) + for key in ("network", "pxe", "storage", "proxy", "images"): + if key in spec and not isinstance(spec[key], dict): + die(f"spec.{key} must be a mapping") + + +def render_inventory(spec: dict, token: str) -> str: + topo = spec["topology"] + server = spec["server"] + k3s_version = spec["k3s_version"] + lines = [ + HEADER_HASH, + "k3s_cluster:", + " children:", + " server:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {server['ip']}", + " agent:", + ] + if topo == "ssh-preinstalled" and spec.get("agents"): + lines.append(" hosts:") + for a in spec["agents"]: + lines.append(f" {a['name']}:") + lines.append(f" ansible_host: {a['ip']}") + else: + # PXE diskless agents auto-join by netboot; do NOT list them here. + lines.append(" hosts: {}") + lines += [ + " vars:", + " ansible_port: 22", + " ansible_user: root", + f" k3s_version: {k3s_version}", + f" token: {yaml_quote(token)}", + " api_endpoint: \"{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}\"", + ] + if topo == "pxe-diskless": + lines += [ + "", + "pxe_controller:", + " hosts:", + f" {server['name']}:", + f" ansible_host: {server['ip']}", + " vars:", + " ansible_port: 22", + " ansible_user: root", + ] + return "\n".join(lines) + "\n" + + +def render_pxe_vars(spec: dict) -> str: + net = require(spec, "network") + pxe = spec.get("pxe", {}) + keys = pxe.get("authorized_keys", []) + if not keys: + die("pxe.authorized_keys must contain at least one SSH public key") + server_ip = spec["server"]["ip"] + k3s_version = spec["k3s_version"] + lines = [ + HEADER_HASH, + "# Pass this file to pb-pxe-controller.yml with", + "# ansible-playbook ... -e @<absolute-path-to-this-file>", + "# pxe_k3s_version is pinned to k3s_version so agents are never newer", + "# than the server.", + "pxe_rootfs_force_rebuild: true # first build only; set false afterwards", + f"pxe_network_interface: {yaml_quote(net['interface'])}", + f"pxe_subnet: {yaml_quote(net['subnet'])}", + f"pxe_gateway: {yaml_quote(net.get('gateway', ''))}", + f"pxe_dns_servers: {yaml_quote(net.get('dns_servers', '8.8.8.8,8.8.4.4'))}", + f"pxe_controller_ip: {yaml_quote(server_ip)}", + "pxe_k3s_server_ips:", + f" - {yaml_quote(server_ip)}", + f"pxe_k3s_version: {yaml_quote(k3s_version)}", + f"pxe_web_port: {int(pxe.get('web_port', 8080))}", + f"pxe_rootfs_password: {yaml_quote(pxe.get('rootfs_password', ''))}", + "pxe_rootfs_authorized_keys:", + ] + for k in keys: + lines.append(f" - {yaml_quote(k)}") + return "\n".join(lines) + "\n" + + +def render_values(spec: dict) -> str: + accel = spec.get("accelerators") or {} + storage_class = (spec.get("storage") or {}).get("class", "nfs-client") + node_port = (spec.get("proxy") or {}).get("node_port", 30890) + auth_mode = spec.get("auth_mode", "auto-login") + images = spec.get("images") or {} + + lines = [ + "# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved.", + "# Helm overlay generated by auplc-skills gen_configs.py.", + "# Layer this on top of runtime/values.yaml:", + "# helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \\", + "# --create-namespace -f runtime/values.yaml -f <this file>", + "custom:", + f" authMode: {yaml_quote(auth_mode)}", + ] + if accel: + lines.append(" accelerators:") + for key, cfg in accel.items(): + product = (cfg or {}).get("product_name") or DEFAULT_ACCEL_LABELS.get(key) + if not product: + die( + f"accelerator '{key}' has no product_name and no known default; " + "add accelerators.<key>.product_name from `kubectl describe node`" + ) + lines += [ + f" {key}:", + " nodeSelector:", + f" amd.com/gpu.product-name: {yaml_quote(product)}", + ] + if accel or images: + lines.append(" resources:") + if accel: + lines += [" metadata:", " gpu:", " acceleratorKeys:"] + lines.extend(f" - {yaml_quote(key)}" for key in accel) + if images: + lines.append(" images:") + for k, v in images.items(): + lines.append(f" {k}: {yaml_quote(v)}") + lines += [ + "hub:", + " db:", + " pvc:", + f" storageClassName: {yaml_quote(storage_class)}", + "singleuser:", + " storage:", + " dynamic:", + f" storageClass: {yaml_quote(storage_class)}", + "proxy:", + " service:", + " type: NodePort", + " nodePorts:", + f" http: {int(node_port)}", + ] + return "\n".join(lines) + "\n" + + +def preflight_destinations(paths: list[Path], force: bool) -> None: + if force: + return + for path in paths: + if os.path.lexists(path): + die(f"refusing to overwrite existing {path} (use --force)", 1) + + +def stage_file(path: Path, content: str, mode: int) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + fd, staged_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "w", encoding="utf-8") as staged_file: + staged_file.write(content) + staged_file.flush() + os.fsync(staged_file.fileno()) + except OSError: + with suppress(OSError): + os.close(fd) + Path(staged_path).unlink(missing_ok=True) + raise + return Path(staged_path) + + +def remove_destination(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + +def backup_destination(path: Path) -> tuple[Path, Path]: + backup_dir = Path(tempfile.mkdtemp(prefix=f".{path.name}.backup.", dir=path.parent)) + backup_path = backup_dir / path.name + os.replace(path, backup_path) + return backup_dir, backup_path + + +def publish_artifacts(artifacts: list[tuple[Path, str, int, bool]], force: bool) -> None: + staged: list[tuple[Path, Path, bool]] = [] + published: list[Path] = [] + backups: list[tuple[Path, Path, Path]] = [] + try: + for path, content, mode, secret in artifacts: + staged.append((path, stage_file(path, content, mode), secret)) + for path, staged_path, secret in staged: + if force and os.path.lexists(path): + backup_dir, backup_path = backup_destination(path) + backups.append((path, backup_dir, backup_path)) + if force: + os.replace(staged_path, path) + else: + os.link(staged_path, path) + os.unlink(staged_path) + published.append(path) + print(f"wrote {path}" + (" (chmod 600 -- contains the k3s token)" if secret else "")) + except OSError as exc: + for path in reversed(published): + remove_destination(path) + for path, backup_dir, backup_path in reversed(backups): + remove_destination(path) + os.replace(backup_path, path) + backup_dir.rmdir() + die(f"could not publish generated artifacts: {exc}") + else: + for _, backup_dir, _ in backups: + shutil.rmtree(backup_dir) + finally: + for _, staged_path, _ in staged: + staged_path.unlink(missing_ok=True) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--spec", help="path to the cluster-spec JSON, or - for stdin") + ap.add_argument("--out-dir", default="generated", help="directory to write artifacts into (default: ./generated)") + ap.add_argument("--token-file", help="read the k3s token from this file instead of generating one") + ap.add_argument("--force", action="store_true", help="overwrite existing files") + ap.add_argument("--print-schema", action="store_true", help="print an example cluster-spec and exit") + args = ap.parse_args(argv) + + if args.print_schema: + print(json.dumps(SCHEMA, indent=2)) + return 0 + if not args.spec: + die("--spec is required (or use --print-schema)", 2) + + raw = sys.stdin.read() if args.spec == "-" else Path(args.spec).read_text(encoding="utf-8") + try: + spec = json.loads(raw) + except json.JSONDecodeError as exc: + die(f"spec is not valid JSON: {exc}") + + if not isinstance(spec, dict): + die("spec must be a mapping") + topo = spec.get("topology") + if topo not in ("pxe-diskless", "ssh-preinstalled"): + die("spec.topology must be 'pxe-diskless' or 'ssh-preinstalled'") + require(spec, "k3s_version") + require(spec, "server.name") + require(spec, "server.ip") + validate_config_shapes(spec) + + if args.token_file: + token = Path(args.token_file).read_text(encoding="utf-8").strip() + if not token: + die("--token-file is empty") + else: + token = gen_token() + + out = Path(args.out_dir) + artifacts = [(out / "inventory.yml", render_inventory(spec, token), 0o600, True)] + if topo == "pxe-diskless": + artifacts.append((out / "pb-pxe-controller.vars.yml", render_pxe_vars(spec), 0o600, False)) + artifacts.append((out / "values-basic-example.yaml", render_values(spec), 0o644, False)) + preflight_destinations([path for path, _, _, _ in artifacts], args.force) + publish_artifacts(artifacts, args.force) + + print( + "\nNext: review the files, then copy them into your aup-learning-cloud " + "checkout. Never commit inventory.yml -- it holds the k3s token." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/deploy-aup-learning-cloud/scripts/validate.py b/skills/deploy-aup-learning-cloud/scripts/validate.py new file mode 100755 index 00000000..8408f020 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/scripts/validate.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +"""Pre-flight validation for an AUP Learning Cloud deploy. + +Catches the mistakes that otherwise surface only after a long playbook or a +failed spawn: + + * required PXE vars empty (PXE topology only: interface / subnet / + controller_ip / dns / k3s_server_ips / at least one authorized key); + * the k3s server version and the PXE agent rootfs version disagree (PXE + topology only; agents must not be newer than the server); + * nodeSelectors for the accelerators actually referenced by effective + custom.resources.metadata.*.acceleratorKeys, checked against + detect_cluster.sh output when supplied; + * (optional) the chart does not render: a `helm template` dry-run. + +This intentionally uses regex/line scanning rather than a YAML parser so it +runs on a bare operator machine with stdlib only. It is a linter, not a schema +validator: it reports what it can prove wrong, and says so when it cannot +inspect something. + +Usage: + validate.py --repo ~/aup-learning-cloud --topology pxe-diskless + validate.py --repo ~/aup-learning-cloud \ + --topology ssh-preinstalled \ + --values runtime/values.yaml --values runtime/values-basic-example.yaml \ + --cluster cluster.json --helm-dry-run + +Exit codes: 0 if every check passed (warnings allowed); 1 if any check failed; +2 on a usage error. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path + +PXE_PLAYBOOK = "deploy/ansible/playbooks/pb-pxe-controller.yml" +INVENTORY = "deploy/ansible/inventory.yml" +CHART = "runtime/chart" + +errors: list[str] = [] +warnings: list[str] = [] +passed: list[str] = [] + + +def ok(msg: str) -> None: + passed.append(msg) + + +def warn(msg: str) -> None: + warnings.append(msg) + + +def fail(msg: str) -> None: + errors.append(msg) + + +def scalar(text: str, key: str) -> str | None: + """First `key: value` scalar in `text` (ignores list/empty values).""" + m = re.search(rf"^\s*{re.escape(key)}\s*:\s*(.+?)\s*$", text, re.MULTILINE) + if not m: + return None + val = m.group(1).strip().strip('"').strip("'") + return val or None + + +def key_occurrences(text: str, key: str) -> int: + return len(re.findall(rf"^\s*{re.escape(key)}\s*:", text, re.MULTILINE)) + + +def list_nonempty(text: str, key: str) -> bool: + """True if `key:` is a YAML list with at least one item, or an inline + non-empty flow list (``[...]`` with content).""" + # Inline flow list: key: ["a", "b"] or key: [] + m = re.search(rf"^\s*{re.escape(key)}\s*:\s*\[(.*?)\]\s*$", text, re.MULTILINE) + if m: + return bool(m.group(1).strip()) + # Block list: key:\n - item + m = re.search(rf"^(\s*){re.escape(key)}\s*:\s*$", text, re.MULTILINE) + if not m: + return False + indent = len(m.group(1)) + tail = text[m.end() :].splitlines() + for line in tail: + if not line.strip(): + continue + cur_indent = len(line) - len(line.lstrip()) + if cur_indent <= indent: + break + if line.lstrip().startswith("- "): + return True + return False + + +def pxe_vars_path(repo: Path, configured_path: str | None) -> Path: + return Path(configured_path).expanduser() if configured_path else repo / PXE_PLAYBOOK + + +def check_pxe_vars(repo: Path, configured_path: str | None = None) -> None: + pb = pxe_vars_path(repo, configured_path) + if not pb.exists(): + fail(f"PXE vars file not found: {pb}") + return + text = pb.read_text(encoding="utf-8") + required_scalars = { + "pxe_network_interface": "service-machine NIC", + "pxe_subnet": "node subnet CIDR", + "pxe_controller_ip": "service host IP", + "pxe_dns_servers": "rootfs DNS servers", + } + safety_keys = [*required_scalars, "pxe_k3s_server_ips", "pxe_rootfs_authorized_keys", "pxe_k3s_version"] + for key in safety_keys: + if key_occurrences(text, key) > 1: + fail(f"duplicate PXE key '{key}' in {pb}") + for key, what in required_scalars.items(): + if scalar(text, key): + ok(f"PXE var {key} is set") + else: + fail(f"PXE var {key} ({what}) is empty -- the playbook asserts on this") + if list_nonempty(text, "pxe_k3s_server_ips"): + ok("PXE var pxe_k3s_server_ips has at least one IP") + else: + fail("PXE var pxe_k3s_server_ips is empty") + if list_nonempty(text, "pxe_rootfs_authorized_keys"): + ok("PXE var pxe_rootfs_authorized_keys has at least one key") + else: + fail("PXE var pxe_rootfs_authorized_keys is empty (rootfs would be unreachable)") + + +def check_version_sync(repo: Path, configured_path: str | None = None) -> None: + inv = repo / INVENTORY + pb = pxe_vars_path(repo, configured_path) + if not inv.exists(): + warn(f"{INVENTORY} not found; skipping k3s version sync check") + return + inventory_text = inv.read_text(encoding="utf-8") + if key_occurrences(inventory_text, "k3s_version") > 1: + fail(f"duplicate inventory key 'k3s_version' in {inv}") + return + server_ver = scalar(inventory_text, "k3s_version") + if not server_ver: + warn("k3s_version not found in inventory.yml") + return + if not pb.exists(): + ok(f"k3s server version is {server_ver} (no PXE playbook to cross-check)") + return + agent_ver = scalar(pb.read_text(encoding="utf-8"), "pxe_k3s_version") + if not agent_ver: + warn("pxe_k3s_version not found in the PXE playbook") + return + if agent_ver == server_ver: + ok(f"k3s_version == pxe_k3s_version ({server_ver})") + else: + fail( + f"version mismatch: inventory k3s_version={server_ver} but " + f"pxe_k3s_version={agent_ver}. Agents must not be newer than the server." + ) + + +def yaml_scalar(value: str) -> str: + return value.strip().strip('"').strip("'") + + +def yaml_optional_scalar(value: str) -> str: + scalar_value = yaml_scalar(value) + return "" if scalar_value in {"", "null", "~"} else scalar_value + + +def yaml_indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def parse_inline_list(value: str) -> list[str]: + items = value.strip()[1:-1].strip() + if not items: + return [] + return [yaml_scalar(item) for item in items.split(",") if yaml_scalar(item)] + + +def is_relevant_flow_path(path: tuple[str, ...]) -> bool: + return path == ("custom",) or path[:2] in {("custom", "accelerators"), ("custom", "resources")} + + +def unsupported_yaml_syntax(value: str) -> bool: + return value.startswith(("&", "*", "!", "|", ">")) + + +def parse_values_file(text: str) -> tuple[dict[str, str | None], dict[str, list[str]], list[str]]: + """Extract the deploy-relevant mappings from a fixed-shape values YAML file. + + The helpers deliberately remain stdlib-only. This scanner handles the + mapping/list shapes used by values overlays, rather than pretending to be a + general YAML parser. + """ + accelerators: dict[str, str | None] = {} + metadata: dict[str, list[str]] = {} + parse_errors: list[str] = [] + stack: list[tuple[int, str]] = [] + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].rstrip() + if not line.strip(): + continue + indent = yaml_indent(line) + stripped = line.strip() + + while stack and indent <= stack[-1][0]: + stack.pop() + path = tuple(key for _, key in stack) + + if stripped.startswith("- "): + if len(path) == 5 and path[:3] == ("custom", "resources", "metadata") and path[-1] == "acceleratorKeys": + metadata.setdefault(path[3], []).append(yaml_scalar(stripped[2:])) + continue + + product_label_match = re.fullmatch( + r"(?:[\"']amd\.com/gpu\.product-name[\"']|amd\.com/gpu\.product-name):\s*(.*)", stripped + ) + if product_label_match: + if len(path) == 4 and path[:2] == ("custom", "accelerators") and path[-1] == "nodeSelector": + value = product_label_match.group(1).strip() + if unsupported_yaml_syntax(value): + parse_errors.append( + f"unsupported YAML syntax at custom.accelerators.{path[2]}.nodeSelector.amd.com/gpu.product-name" + ) + else: + accelerators[path[2]] = yaml_optional_scalar(value) + continue + + mapping_match = re.fullmatch(r"(.+?):(?:\s*(.*))?", stripped) + if not mapping_match: + continue + key = mapping_match.group(1).strip("\"'") + value = (mapping_match.group(2) or "").strip() + candidate_path = path + (key,) + if value.startswith("{") and value != "{}" and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported non-empty flow-style mapping at {'.'.join(candidate_path)}") + if unsupported_yaml_syntax(value) and is_relevant_flow_path(candidate_path): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + if path == ("custom", "accelerators"): + accelerators.setdefault(key, None) + if len(path) == 4 and path[:3] == ("custom", "resources", "metadata") and key == "acceleratorKeys": + resource_key = path[3] + if unsupported_yaml_syntax(value): + parse_errors.append(f"unsupported YAML syntax at {'.'.join(candidate_path)}") + elif value.startswith("[") and value.endswith("]"): + metadata[resource_key] = parse_inline_list(value) + elif not value or value in {"null", "~"}: + metadata[resource_key] = [] + else: + parse_errors.append(f"acceleratorKeys must be a list at {'.'.join(candidate_path)}") + stack.append((indent, key)) + return accelerators, metadata, parse_errors + + +def collect_effective_values(repo: Path, values: list[str]) -> tuple[dict[str, str], dict[str, list[str]], list[str]]: + paths = values or ["runtime/values.yaml"] + accelerators: dict[str, str] = {} + metadata: dict[str, list[str]] = {} + parse_errors: list[str] = [] + for rel in paths: + p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if p.exists(): + parsed_accelerators, parsed_metadata, file_errors = parse_values_file(p.read_text(encoding="utf-8")) + for key, selector in parsed_accelerators.items(): + if selector is not None or key not in accelerators: + accelerators[key] = selector + metadata.update(parsed_metadata) + parse_errors.extend(file_errors) + else: + fail(f"values file not found: {rel}") + return accelerators, metadata, parse_errors + + +def check_accelerator_labels( + accelerators: dict[str, str], metadata: dict[str, list[str]], cluster: dict | None +) -> None: + active_keys = sorted({key for keys in metadata.values() for key in keys}) + if not active_keys: + warn("no acceleratorKeys found in effective custom.resources.metadata") + return + declared: list[str] = [] + for key in active_keys: + if key not in accelerators: + fail(f"active accelerator '{key}' is not defined under custom.accelerators") + elif not accelerators[key]: + fail(f"active accelerator '{key}' has no amd.com/gpu.product-name nodeSelector") + else: + declared.append(accelerators[key]) + if not declared: + return + if cluster is None: + warn( + "no --cluster snapshot; cannot confirm nodeSelector labels match real " + f"nodes. Declared: {', '.join(declared)}" + ) + return + real = set(cluster.get("gpu_product_names", [])) + if not real: + fail("cluster snapshot has no GPU product labels for active accelerators") + return + for d in declared: + if d in real: + ok(f"nodeSelector '{d}' matches a real node label") + else: + fail(f"nodeSelector '{d}' matches no node label. Real labels: {', '.join(sorted(real))}") + + +def check_helm(repo: Path, values: list[str]) -> None: + if not shutil.which("helm"): + warn("helm not on PATH; skipped chart dry-run") + return + chart = repo / CHART + if not chart.exists(): + warn(f"chart not found at {CHART}; skipped dry-run") + return + cmd = ["helm", "template", "jupyterhub", str(chart)] + for rel in values or ["runtime/values.yaml"]: + p = (repo / rel) if not Path(rel).is_absolute() else Path(rel) + if p.exists(): + cmd += ["-f", str(p)] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode == 0: + ok("helm template rendered the chart successfully") + else: + tail = (proc.stderr or proc.stdout).strip().splitlines()[-5:] + fail("helm template failed:\n " + "\n ".join(tail)) + + +def main(argv=None) -> int: + global errors, passed, warnings + errors = [] + warnings = [] + passed = [] + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--repo", required=True, help="path to the aup-learning-cloud checkout") + ap.add_argument( + "--topology", + choices=("pxe-diskless", "ssh-preinstalled"), + default="pxe-diskless", + help="deployment topology (default: pxe-diskless)", + ) + ap.add_argument( + "--values", action="append", default=[], help="values file (repeatable); defaults to runtime/values.yaml" + ) + ap.add_argument( + "--pxe-vars", + help="PXE vars file to validate instead of deploy/ansible/playbooks/pb-pxe-controller.yml", + ) + ap.add_argument("--cluster", help="detect_cluster.sh JSON output to match labels against") + ap.add_argument("--helm-dry-run", action="store_true", help="also run `helm template`") + ap.add_argument("--json", action="store_true", help="emit a JSON report instead of text") + args = ap.parse_args(argv) + + repo = Path(args.repo).expanduser() + if not repo.exists(): + print(f"validate: repo not found: {repo}", file=sys.stderr) + return 2 + + cluster = None + if args.cluster: + try: + cluster = json.loads(Path(args.cluster).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"validate: cannot read --cluster: {exc}", file=sys.stderr) + return 2 + + if args.topology == "pxe-diskless": + check_pxe_vars(repo, args.pxe_vars) + check_version_sync(repo, args.pxe_vars) + else: + ok("skipped PXE checks for ssh-preinstalled topology") + accelerators, metadata, parse_errors = collect_effective_values(repo, args.values) + for message in parse_errors: + fail(message) + check_accelerator_labels(accelerators, metadata, cluster) + if args.helm_dry_run: + check_helm(repo, args.values) + + if args.json: + print( + json.dumps( + {"passed": passed, "warnings": warnings, "errors": errors, "status": "ok" if not errors else "error"}, + indent=2, + ) + ) + else: + for m in passed: + print(f"[ OK ] {m}") + for m in warnings: + print(f"[WARN] {m}") + for m in errors: + print(f"[FAIL] {m}") + print(f"\n{len(passed)} ok, {len(warnings)} warning(s), {len(errors)} error(s)") + return 0 if not errors else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/deploy-aup-learning-cloud/skill-card.md b/skills/deploy-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..640e1441 --- /dev/null +++ b/skills/deploy-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Deploy AUP Learning Cloud end to end on a multi-node k3s cluster — either PXE-diskless netboot or SSH-preinstalled nodes — for operators standing up a K3s cluster with AUP Learning Cloud Service. + +## Owner + +AMD Research diff --git a/skills/develop-aup-learning-cloud-courses/SKILL.md b/skills/develop-aup-learning-cloud-courses/SKILL.md new file mode 100644 index 00000000..52101dad --- /dev/null +++ b/skills/develop-aup-learning-cloud-courses/SKILL.md @@ -0,0 +1,98 @@ +--- +name: develop-aup-learning-cloud-courses +description: >- + Group: Course & other editor. Authors a new learning toolkit for AUP Learning + Cloud end to end: write the + course notebooks under projects/<NAME>/, package them into a course Docker + image (dockerfiles/Courses/<NAME>/ + a Makefile target on the ROCm GPU base), + register the course in auplc_installer/catalog.py (COURSE_CATALOG + team + mapping), then hand off to build + values wiring. Use when an educator wants + to add a new course or lab set, create a toolkit like CV/DL/LLM/PhySim, turn a + notebook folder into a spawnable course, add a Dockerfile/build.sh for a + course, or add a course key to the catalog. Triggers include projects/CV, + projects/DL, projects/LLM, projects/PhySim, dockerfiles/Courses, + COURSE_CATALOG, "add a course", "new toolkit". Do not use to only build an + existing image (build-aup-learning-cloud-images), to only edit the values + catalog for an existing image (configure-aup-learning-cloud-courses), or to + clone a user's repo at runtime (configure-aup-learning-cloud-repos). +--- + +# Develop AUP Learning Cloud courses + +Create a brand-new course (a set of hands-on notebooks) and make it a spawnable +environment: author the curriculum, bake it into a course image, register the +course key, then build and wire it into the spawn UI. This is the +author/educator workflow that *produces* what configure-courses later tunes. + +The notebooks and the image build context are the source of truth; the catalog +keeps keys consistent. Per-file conventions, the new-course checklist, and the +directory map are in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud`; Docker with enough disk (course images are + large); the ROCm GPU base image available (`auplc-base`, built by + build-images or pulled). +- Familiarity with the existing toolkits under `projects/{CV,DL,LLM,PhySim}` as + patterns. +- For the build + deploy hand-off: the build-images and configure-courses skills. + +## Where a course lives (four coordinated places) + +A new course `Course-<NAME>` must be consistent across: + +1. **Curriculum** — `projects/<NAME>/` (the `.ipynb` labs, README, assets). +2. **Image build context** — `dockerfiles/Courses/<NAME>/` (Dockerfile + + `build.sh`) layered on the GPU base, plus a `Makefile` target. +3. **Catalog** — a `Course(...)` entry in `auplc_installer/catalog.py` + `COURSE_CATALOG` (key, image basename, `gpu_required`, make target, display + name) and the mirrored bash `COURSE_CATALOG`, plus `BASE_TEAM_MAPPING`. +4. **Values** — `custom.resources.{images,requirements,metadata}.<key>` and + `custom.teams.mapping` (this is the configure-courses skill). + +## Workflow + +1. **Author the curriculum.** Add the notebooks under `projects/<NAME>/` + following the existing numbering/README pattern (e.g. `LLM01-…`). Keep the + per-file `Copyright (C) … Advanced Micro Devices, Inc.` header (MIT). +2. **Create the image build context.** Add `dockerfiles/Courses/<NAME>/` with a + `Dockerfile` + `build.sh` modeled on an existing course, `FROM` the GPU base + (`BASE_IMAGE=ghcr.io/amdresearch/auplc-base:latest`), and `COPY` the + `projects/<NAME>/` content into the image. Pin pip deps for reproducibility. +3. **Add the Makefile target.** Add a `<name>` target in `dockerfiles/Makefile` + that builds, GPU-tags (`:latest-$(GPU_TARGET)`), and `save-image`s — mirror + the `cv`/`dl` targets. Add it to the `courses` aggregate. +4. **Register the course key.** Add a `Course("Course-<NAME>", "auplc-<name>", + True, "<name>", "<Display Name>")` to `COURSE_CATALOG` in `catalog.py`, keep + the bash table byte-for-byte identical, and add the key to the relevant + `BASE_TEAM_MAPPING` groups. +5. **Build the image** (hand off to build-images): + + ```bash + ./auplc-installer img build <name> --gpu=<target> + ``` + +6. **Wire it into values** (hand off to configure-courses): add the key under + `custom.resources.images/requirements/metadata` and `custom.teams.mapping`, + then `rt upgrade` / `helm upgrade`. +7. **Verify.** The course appears in its spawn-UI `group` for mapped teams, and + a launched pod runs the new image with the notebooks present under the home + tree. + +## Safety + +- **Large/slow builds.** Course images are big; confirm disk and time before a + full build, and prefer building just the new `<name>` target. +- **Keep the catalog in sync.** `catalog.py` and the mirrored bash table must + match exactly, or `--courses` selection/overlay generation breaks. +- **Licensing.** Only bundle datasets, models, and third-party code whose + licenses permit redistribution; keep AMD copyright headers on new source. +- **Attribution.** If any change touches Hub source (not typical for a course), + preserve the four attribution layers from the project `AGENTS.md`. +- Never commit secrets or large binary blobs that belong in object storage. + +## Reference + +The new-course checklist, the `projects/`/`dockerfiles/Courses/` layout, the +`catalog.py` entry shape, GPU-tag rules, and troubleshooting: +[reference.md](reference.md). diff --git a/skills/develop-aup-learning-cloud-courses/reference.md b/skills/develop-aup-learning-cloud-courses/reference.md new file mode 100644 index 00000000..c907f2d3 --- /dev/null +++ b/skills/develop-aup-learning-cloud-courses/reference.md @@ -0,0 +1,106 @@ +# Develop AUP Learning Cloud courses — Reference + +The new-course checklist, the directory layout, the `catalog.py` entry shape, +and troubleshooting. Workflow and gates are in [SKILL.md](SKILL.md). + +## Source + +- Repo README "Learning Solution" + `projects/{CV,DL,LLM,PhySim}/README.md`. +- `dockerfiles/Makefile` (course targets) and `dockerfiles/Courses/<NAME>/`. +- `auplc_installer/catalog.py` (the course catalog source of truth) and its + mirrored bash `COURSE_CATALOG` / `BASE_TEAM_MAPPING`. +- Build details: build-aup-learning-cloud-images. Values wiring: + configure-aup-learning-cloud-courses. + +## Directory layout + +``` +projects/<NAME>/ # curriculum: NN_*.ipynb labs, README.md, assets/ +dockerfiles/Courses/<NAME>/ # Dockerfile + build.sh (FROM the GPU base) +dockerfiles/Makefile # add a <name> target; add it to `courses` +auplc_installer/catalog.py # add a Course(...) to COURSE_CATALOG + team map +runtime/values.yaml # custom.resources.{images,requirements,metadata} +``` + +Existing toolkits to copy from: `projects/CV` (10 labs), `projects/DL` (12), +`projects/LLM` (9), `projects/PhySim` (Genesis robotics). + +## Makefile target (mirror cv/dl) + +```make +courses: cv dl llm physim <name> + +<name>: + cd Courses/<NAME> && BASE_IMAGE=$(GPU_BASE_IMAGE) bash ./build.sh + docker tag ghcr.io/amdresearch/auplc-<name>:latest ghcr.io/amdresearch/auplc-<name>:latest-$(GPU_TARGET) + $(MAKE) save-image IMAGE=ghcr.io/amdresearch/auplc-<name>:latest +``` + +GPU course images are tagged `:<IMAGE_TAG>-<gpu_target>` (e.g. `latest-gfx1151`). +`GPU_BASE_IMAGE` defaults to `ghcr.io/amdresearch/auplc-base:latest`. + +## catalog.py entry + +```python +COURSE_CATALOG: tuple[Course, ...] = ( + # ...existing entries... + Course("Course-<NAME>", "auplc-<name>", True, "<name>", "<Display Name> Course"), +) +``` + +`Course(key, image_basename, gpu_required, make_target, display_name)`: + +- `key` — matches `custom.resources.{images,requirements,metadata}` and + `custom.teams.mapping` (convention: `Course-<NAME>`). +- `image_basename` — `auplc-<name>` (no registry/tag). +- `gpu_required` — `True` → GPU-tagged build; `False` → plain `:<tag>`. +- `make_target` — the `dockerfiles/Makefile` target. + +Add the same row to the mirrored **bash** `COURSE_CATALOG` (byte-for-byte) and +add the key to the appropriate `BASE_TEAM_MAPPING` groups (e.g. `gpu`, +`official`, `AUP`, `native-users`, `github-users`). `COURSE_PRESET_BASIC` is +only `cpu, gpu, code-cpu, code-gpu`; new courses join `all`, not `basic`. + +## Build and wire (hand-offs) + +```bash +# build the new course image (build-images skill) +./auplc-installer img build <name> --gpu=<target> +# optional push for multi-node / offline +docker push ghcr.io/amdresearch/auplc-<name>:latest-<gpu_target> +``` + +Then, with configure-courses, add to the values overlay: + +```yaml +custom: + resources: + images: + Course-<NAME>: "ghcr.io/amdresearch/auplc-<name>:latest" + requirements: + Course-<NAME>: { cpu: "0", memory: "0Gi", amd.com/gpu: "1" } + metadata: + Course-<NAME>: + group: "TEACHING LABS" + description: "<Display Name> Course" + accelerator: "GPU" + acceleratorKeys: [strix-halo] + allowGitClone: true + resourceType: "notebook" + teams: + mapping: + gpu: [..., Course-<NAME>] +``` + +Apply with `./auplc-installer rt upgrade` (single) or `helm upgrade` (multi). + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| `unknown course key` from installer | `catalog.py`/bash table out of sync, or key typo | Make both tables identical; use the exact `Course-<NAME>` key | +| Course image build fails | Missing base image or bad Dockerfile context | Build `base-rocm` first; verify `dockerfiles/Courses/<NAME>` paths | +| Notebooks missing in the pod | `COPY` path wrong in the course Dockerfile | Confirm `projects/<NAME>/` is copied into the image home tree | +| Course not in spawn UI | Values catalog/team mapping incomplete | Add the key in all of images/requirements/metadata + `teams.mapping` | +| GPU course Pending | `acceleratorKeys` → node label mismatch | `kubectl describe node | grep amd.com/gpu.product-name` | +| Wrong gfx kernels at runtime | Built for the wrong `--gpu` | Rebuild with the correct target | diff --git a/skills/develop-aup-learning-cloud-courses/skill-card.md b/skills/develop-aup-learning-cloud-courses/skill-card.md new file mode 100644 index 00000000..94ea9437 --- /dev/null +++ b/skills/develop-aup-learning-cloud-courses/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Author a new AUP Learning Cloud course end to end — notebooks, course image, and catalog registration — for educators and curriculum authors adding a learning toolkit. + +## Owner + +AMD Research diff --git a/skills/expose-aup-learning-cloud/SKILL.md b/skills/expose-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..9135bece --- /dev/null +++ b/skills/expose-aup-learning-cloud/SKILL.md @@ -0,0 +1,109 @@ +--- +name: expose-aup-learning-cloud +description: >- + Group: Maintain AUP Learning Cloud. Configures how AUP Learning Cloud is + exposed and stored for a real + deployment: the proxy service type (NodePort vs LoadBalancer/ingress), ingress + hostname and TLS, external-TLS handling (custom.security.publicScheme), CORS + origins (custom.hub/notebook.allowedOrigins), and + the shared NFS storage class for the Hub DB and user PVCs. Use when the user + wants to put the Hub behind a domain, enable HTTPS/TLS/certificates, set up + ingress, change the NodePort, move storage from local-path to NFS + (nfs-client / nfs-subdir-external-provisioner), fix mixed-content / _xsrf + cookie issues behind a reverse proxy, or allow embedding/CORS. Triggers + include ingress.enabled, proxy.service.type, nodePorts.http, publicScheme, + allowedOrigins, storageClassName, nfs-client, TLS, cert-manager. Do not use + for the first cluster build (deploy-/install-aup-learning-cloud), the GitHub + OAuth callback URL (configure-aup-learning-cloud-auth), or course/quota config + (configure-aup-learning-cloud-courses). +--- + +# Expose AUP Learning Cloud + +Take a deployment from the local NodePort/`local-path` defaults to a real +network and storage posture: choose how the proxy is reached (NodePort, +LoadBalancer, or ingress + TLS), tell the Hub about externally-terminated TLS, +set CORS origins, and move persistent data onto shared NFS. + +Edit a **values overlay** and re-apply with Helm / the installer. NFS, ingress, +and TLS are opt-in — the checked-in defaults are a plain HTTP NodePort. The full +value blocks, the NFS provisioner setup, and troubleshooting are in +**[reference.md](reference.md)**. + +## Prerequisites + +- A running AUP Learning Cloud and `helm` + `kubectl` (or `./auplc-installer`). +- For ingress/TLS: an ingress controller in the cluster, a DNS record for the + hostname, and a certificate source (cert-manager issuer or a TLS secret). +- For NFS storage: an NFS server/export reachable from every node. + +## The defaults you are changing + +The checked-in `runtime/values.yaml` is local-oriented: `proxy.service.type: +NodePort` on `30890`, `ingress.enabled: false`, `hub.db.pvc.storageClassName: +local-path`, `singleuser.storage.dynamic.storageClass: local-path`. Treat NFS, +ingress, and TLS as deliberate additions. + +## Pick the exposure path + +| Path | When | Key values | +| --- | --- | --- | +| **NodePort** (default) | Lab on a known node IP | `proxy.service.type: NodePort`, `nodePorts.http` | +| **LoadBalancer** | Cloud / MetalLB | `proxy.service.type: LoadBalancer` | +| **Ingress + TLS** | Real domain, HTTPS | `ingress.enabled: true`, host, TLS secret/issuer | + +## Workflow + +1. **Read current state.** Note `proxy.service`, `ingress`, the two + `storageClassName`s, and whether TLS is terminated by the chart or upstream. +2. **Set exposure** in the overlay (one path above). For ingress, set the host + and the TLS config; point DNS at the controller. +3. **Handle TLS termination.** If TLS terminates **outside** the chart (LB or + external proxy), set `custom.security.publicScheme: "https"` so the Hub marks + `_xsrf` cookies secure and builds correct https URLs. +4. **CORS / embedding (only if needed).** Add origins to + `custom.hub.allowedOrigins` (Hub CORS) and/or `custom.notebook.allowedOrigins` + (single-user server args). Leave empty unless something embeds the Hub. +5. **Storage (multi-node / production).** Move the Hub DB and user PVCs to + `nfs-client`: install `nfs-subdir-external-provisioner` against your NFS + export, then set both `storageClassName`s. Provisioner setup is in + [reference.md](reference.md). +6. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f + runtime/values.yaml -f <overlay>` must succeed. +7. **Apply** with `helm upgrade --install … -n jupyterhub` (or `rt upgrade` + single-node) and **verify**: + + ```bash + kubectl get svc,ingress -n jupyterhub + kubectl get storageclass + kubectl get pvc -A + ``` + + Then load the public URL over HTTPS, log in, and confirm a spawned pod's PVC + binds on the new storage class. + +## code-server exposure safety + +code-server resources run `code-server --auth none` on port `8888` and are safe +**only** behind the JupyterHub proxy's auth boundary. Never expose that pod port +directly via NodePort, LoadBalancer, or ingress. Only the JupyterHub proxy +service should be public. + +## Safety + +- **Changing storage class does not migrate existing data.** Switching + `storageClassName` affects new PVCs; the Hub DB PVC and user homes do not move + automatically. Plan a migration/backup before changing it on a live Hub — + confirm with the user. +- **Editing `/etc/exports` + restarting `nfs-kernel-server`** is disruptive; + gate it (see deploy/troubleshoot skills for the NFS host side). +- **Exposing to the internet raises the stakes** — pair with HTTPS, a real auth + mode (configure-auth), and never expose code-server's raw port. +- A `helm upgrade` restarts the Hub pod (brief login blip). +- Never commit TLS private keys or put them in tracked values; use a K8s secret. + +## Reference + +NodePort/LoadBalancer/ingress value blocks, TLS + cert-manager options, +`publicScheme`/`allowedOrigins`, the NFS provisioner install and default-class +patch, and troubleshooting: [reference.md](reference.md). diff --git a/skills/expose-aup-learning-cloud/reference.md b/skills/expose-aup-learning-cloud/reference.md new file mode 100644 index 00000000..5937beb9 --- /dev/null +++ b/skills/expose-aup-learning-cloud/reference.md @@ -0,0 +1,174 @@ +# Expose AUP Learning Cloud — Reference + +Exposure value blocks (NodePort / LoadBalancer / ingress + TLS), externally +terminated TLS, CORS origins, and the NFS storage setup. Workflow and gates are +in [SKILL.md](SKILL.md). + +## Source guides + +- Configuration Reference (sections 9, 10, 13): <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/configuration-reference.html> +- Multi-Node Cluster Deployment (storage, ingress): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> +- Single-Node Deployment (defaults): <https://amdresearch.github.io/aup-learning-cloud/installation/single-node.html> + +The chart follows zero-to-jupyterhub conventions; the live +`runtime/chart/values.schema.yaml` is the source of truth. + +## Local defaults (what you change) + +```yaml +proxy: + service: + type: NodePort + nodePorts: + http: 30890 +ingress: + enabled: false +hub: + db: + pvc: + storageClassName: local-path +singleuser: + storage: + dynamic: + storageClass: local-path +``` + +## Exposure option A — NodePort + +```yaml +proxy: + service: + type: NodePort + nodePorts: + http: 30890 # reach the Hub at http://<node-ip>:30890 +``` + +## Exposure option B — LoadBalancer + +```yaml +proxy: + service: + type: LoadBalancer # cloud LB or MetalLB + nodePorts: + http: null +``` + +## Exposure option C — Ingress + TLS (production) + +```yaml +proxy: + service: + type: ClusterIP # ingress fronts the proxy + nodePorts: + http: null + +ingress: + enabled: true + ingressClassName: traefik # or nginx + hosts: + - your.domain.com + tls: + - hosts: + - your.domain.com + secretName: jupyter-tls-cert # a K8s TLS secret, or one cert-manager creates + # annotations: # e.g. cert-manager issuer + # cert-manager.io/cluster-issuer: letsencrypt-prod +``` + +Point a DNS record for `your.domain.com` at the ingress controller. Provide the +TLS secret directly, or let cert-manager mint it via the annotation + an Issuer +you manage. + +## Externally terminated TLS + +If TLS is terminated by something outside the chart (cloud LB, external ingress, +Cloudflare tunnel) rather than the chart's `proxy.https`, tell the Hub the +public scheme is https so `_xsrf` cookies are marked Secure and URLs are https: + +```yaml +custom: + security: + publicScheme: "https" +``` + +## CORS / allowed origins + +Defaults are permissive (`["*"]`); tighten them for a public deployment. + +```yaml +custom: + hub: + allowedOrigins: ["https://portal.example.com"] # Access-Control-Allow-Origin on Hub responses + notebook: + allowedOrigins: ["https://portal.example.com"] # --ServerApp.allow_origin_pat (kernel WebSocket) +``` + +## Shared NFS storage + +### 1. NFS server/export (on a storage/controller node) + +```bash +sudo apt install nfs-kernel-server +sudo mkdir -p /nfs && sudo chown -R nobody:nogroup /nfs && sudo chmod 777 /nfs +echo "/nfs <subnet>/24(rw,sync,no_subtree_check,no_root_squash,insecure)" | sudo tee -a /etc/exports +sudo systemctl restart nfs-kernel-server +# worker nodes: +sudo apt install nfs-common +``` + +### 2. Provisioner (creates the `nfs-client` storage class) + +```bash +helm repo add nfs-subdir-external-provisioner \ + https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/ +helm repo update +helm install nfs-subdir-external-provisioner \ + nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ + --namespace nfs-provisioner --create-namespace \ + -f deploy/k8s/nfs-provisioner/values.yaml +# optional: make it default +kubectl patch storageclass nfs-client \ + -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +``` + +### 3. Point the chart at it + +```yaml +hub: + db: + pvc: + storageClassName: nfs-client +singleuser: + storage: + dynamic: + storageClass: nfs-client +``` + +Changing the class affects **new** PVCs only; existing Hub DB / user homes are +not migrated automatically. + +## Apply and verify + +```bash +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + +kubectl get svc,ingress -n jupyterhub +kubectl get storageclass +kubectl get pvc -A +``` + +Load the public URL over HTTPS, log in, and confirm a spawned pod's PVC binds on +the intended storage class. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Ingress 404 / no route | Controller/class/host mismatch | `kubectl get ingress -n jupyterhub`; confirm `ingressClassName` + DNS | +| TLS cert not issued | cert-manager annotation/Issuer wrong, or secret missing | Describe the ingress + the Certificate; check the issuer | +| Login loops / `_xsrf` errors behind a proxy | External TLS without `publicScheme: https` | Set `custom.security.publicScheme: "https"`, re-apply | +| Mixed-content / blocked embed | `allowedOrigins` too strict/loose | Adjust `custom.hub`/`notebook.allowedOrigins` | +| PVC Pending | Storage class missing / NFS export wrong | `kubectl get storageclass`; provisioner logs; `showmount -e <nfs>` | +| code-server reachable without login | Pod port exposed directly | Only expose the JupyterHub proxy; never NodePort/ingress port `8888` | diff --git a/skills/expose-aup-learning-cloud/skill-card.md b/skills/expose-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..1490c4ed --- /dev/null +++ b/skills/expose-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Configure AUP Learning Cloud network exposure and storage — NodePort/LoadBalancer/ingress, TLS, CORS, and shared NFS — for operators taking a deployment beyond the local demo defaults. + +## Owner + +AMD Research diff --git a/skills/install-aup-learning-cloud-single-node/SKILL.md b/skills/install-aup-learning-cloud-single-node/SKILL.md new file mode 100644 index 00000000..575141db --- /dev/null +++ b/skills/install-aup-learning-cloud-single-node/SKILL.md @@ -0,0 +1,106 @@ +--- +name: install-aup-learning-cloud-single-node +description: >- + Group: Plan & deploy AUP Learning Cloud. Installs AUP Learning Cloud on a + single machine with the ./auplc-installer + flow (single-node k3s + JupyterHub for an AMD GPU/APU workstation). Use when + the user wants to install, set up, try, or demo AUP Learning Cloud / AUPLC on + one box, mentions ./auplc-installer, the installer TUI, "install" / "quick + start" / "single-node", --gpu / --courses / --image-source flags, a Ryzen AI + APU or Radeon dGPU dev box, localhost:30890, or uninstalling it. Also covers + the OEM kernel + Docker prerequisites and offline (pack) bundles. Do not use + for multi-node or PXE/netboot clusters (use deploy-aup-learning-cloud), for + building images (build-aup-learning-cloud-images), or for editing courses + (configure-aup-learning-cloud-courses). +--- + +# Install AUP Learning Cloud (single node) + +Stand up AUP Learning Cloud on one machine using the project's own installer: +detect the GPU, install single-node k3s, pull images, deploy the ROCm device +plugin, and `helm install` the Hub so the user can open `localhost:30890` and +spawn notebooks. This is the "quick start / dev / demo" path. + +The installer is the source of truth. Your job is to confirm prerequisites, +pick the right flags, run it (gating the risky steps), and verify. Full flag +table, offline flow, and troubleshooting are in **[reference.md](reference.md)**. + +## Prerequisites + +- A checkout of `aup-learning-cloud` (run from its root). +- Hardware: a supported **Ryzen AI 300-series+ APU** or **Radeon 9000-series** + GPU; 32 GB+ RAM (64 GB recommended); 500 GB+ SSD. +- **Ubuntu 24.04**. Docker installed and usable without `sudo` + (`docker run hello-world` as the user). +- **Ryzen AI APU only:** the ROCm OEM kernel + (`sudo apt install linux-oem-6.14`) and a reboot. Radeon dGPU + boxes typically use the stock kernel — confirm against ROCm docs. +- For the interactive TUI: `python3-questionary` + `python3-prompt-toolkit` + (apt), or `pip install questionary prompt_toolkit` in a venv. The + non-interactive `./auplc-installer install` does not need these. + +## Phase 1 — Interview (keep it short) + +1. **GPU**: let the installer auto-detect, or have the user name it so you can + pass `--gpu` (`phx`, `strix`, `strix-halo`, `9070xt`, `r9700`, `9600gre`, + `rdna4`). Confirm with `./auplc-installer detect-gpu`. +2. **Courses**: `all` (default), `basic` (cpu/gpu + code-server), `none` + (Hub only), or an explicit list (`cpu,gpu,Course-CV`). +3. **Image source**: `pull` (default, from `ghcr.io/amdresearch`) or `build` + (local from `dockerfiles/`). For a quick demo prefer `pull`. +4. **Online or offline**: a normal machine with internet, or an air-gapped one + that needs a `pack` bundle (see reference). + +## Phase 2 — Verify the environment + +```bash +docker run --rm hello-world # docker works rootless +uname -r # OEM kernel on Ryzen AI APU +./auplc-installer detect-gpu # installer agrees with the hardware +./auplc-installer install --dry-run # prints the Configuration summary, no changes +``` + +Read the `--dry-run` summary back to the user and **get confirmation before the +real install** — it installs k3s system-wide and needs sudo. + +## Phase 3 — Install (confirmation gate) + +Default, opinionated path: + +```bash +./auplc-installer install # auto GPU, all courses, pull images +# or pin choices: +./auplc-installer install --gpu=strix-halo --courses=basic --image-tag=develop +``` + +The installer runs 8 stages (detect GPU → values overlay → helm+k9s → k3s → +pull images → ROCm device plugin + labeller → refresh overlay from node labels +→ deploy Hub). It prompts for sudo once. Use `-y` only for scripted/CI runs. + +## Phase 4 — Verify + +```bash +kubectl get nodes # the node is Ready +kubectl get pods -n jupyterhub # hub + proxy Running, no CrashLoop/ImagePull +``` + +Open `http://localhost:30890` — the default values auto-log-in as `student` +(NodePort 30890, `local-path` storage, ingress disabled). Spawn a CPU notebook, +then a GPU notebook, and confirm the GPU pod schedules. + +## Safety + +Stop and get explicit confirmation before: + +- The real `install` (installs k3s + a containerd/Docker runtime, needs sudo). +- `./auplc-installer uninstall` (removes k3s **and** the runtime; data loss). +- Switching `--runtime` (docker ↔ containerd) on an existing install. +- Any `--image-source=build` run on a slow/low-disk box (large local builds). + +Never commit changes to the checkout. The installer writes a local values +overlay (e.g. `values.local.yaml`); do not commit it. + +## Reference + +Flag-by-flag table, the offline `pack`/air-gapped flow, `dev`/`rt` +subcommands, default-values facts, and troubleshooting: [reference.md](reference.md). diff --git a/skills/install-aup-learning-cloud-single-node/reference.md b/skills/install-aup-learning-cloud-single-node/reference.md new file mode 100644 index 00000000..514da9f9 --- /dev/null +++ b/skills/install-aup-learning-cloud-single-node/reference.md @@ -0,0 +1,128 @@ +# Install AUP Learning Cloud (single node) — Reference + +Full flag table, offline flow, subcommands, and troubleshooting for the +`./auplc-installer` single-node path. Workflow and gates are in +[SKILL.md](SKILL.md). + +## Source guides + +- Quick Start / Single-Node: <https://amdresearch.github.io/aup-learning-cloud/installation/> +- Repo README "Quick Start" section. + +Treat the installer's `--help` and the live docs as the source of truth for +flags and version pins; this file condenses the opinionated path. + +## Prerequisite commands + +```bash +# Ryzen AI APU only: ROCm OEM kernel (reboot afterwards) +sudo apt update && sudo apt install linux-oem-6.14 + +# Docker (rootless usage) +curl -fsSL https://get.docker.com | sh +sudo usermod -aG docker "$USER" && newgrp docker +sudo apt install build-essential + +# Interactive TUI deps (system Python) +sudo apt install python3-questionary python3-prompt-toolkit +``` + +## Commands + +| Command | What it does | +| --- | --- | +| `./auplc-installer` | Launch the interactive TUI (when a real terminal is attached). | +| `./auplc-installer install [--pull]` | Full install: k3s + images + runtime. Default pulls pre-built images. | +| `./auplc-installer install --dry-run` | Print the Configuration summary and exit. No sudo, no changes. | +| `./auplc-installer uninstall` | Remove everything (k3s + runtime). **Destructive.** | +| `./auplc-installer install-tools` | Install `helm` + `k9s` only. | +| `./auplc-installer detect-gpu` | Show the detected GPU configuration. | +| `./auplc-installer img build [target...]` | Build images (see build-aup-learning-cloud-images). | +| `./auplc-installer img pull` | Pull external images for offline use. | +| `./auplc-installer pack [--local]` | Create an offline deployment bundle. | +| `./auplc-installer rt install\|reinstall\|upgrade\|remove` | Runtime (Hub) only — for image/values changes without touching k3s. | +| `./auplc-installer dev [deploy\|upgrade\|reinstall]` | Dev cycle: rebuild hub image + restart, with a dev overlay (student=admin, pullPolicy=Never). | + +## Flags + +| Flag | Values / default | Notes | +| --- | --- | --- | +| `--gpu=TYPE` | `auto` (default), `phx`, `strix`, `strix-halo`, `9070xt`, `r9700`, `9600gre`, `rdna4`/`dgpu`, `gfxNNNN` | Auto-detect via rocminfo/KFD. Env `GPU_TYPE`. | +| `--courses=SPEC` | `all` (default), `basic`, `none`, or `cpu,gpu,Course-CV,...` | Restricts image build/pull **and** hides unselected courses in the spawn UI. Env `AUPLC_COURSES`. | +| `--image-source=SRC` | `pull` (default) or `build` | `pull` = registry; `build` = local from `dockerfiles/`. | +| `--image-registry=PREFIX` | default `ghcr.io/amdresearch` | Env `IMAGE_REGISTRY`. | +| `--image-tag=TAG` | default `latest` | GPU suffix appended automatically. Env `IMAGE_TAG`. Use `develop` for the preview UI. | +| `--runtime=MODE` | `docker` (default) or `containerd` | `docker` makes images visible to k3s immediately; `containerd` exports for offline. | +| `--courses`, `--mirror=`, `--mirror-pip=`, `--mirror-npm=` | — | Registry / PyPI / npm mirrors for restricted networks. | +| `-y`, `--yes` | — | Assume yes (scripted/CI). Env `AUPLC_YES=1`. | +| `--dry-run` (`--try-run`) | — | Preview only. | +| `-v`, `--verbose` | — | Stream every subprocess line. Env `AUPLC_VERBOSE=1`. | + +### Examples + +```bash +./auplc-installer install --dry-run +./auplc-installer install --image-source=pull --image-tag=develop +./auplc-installer install --gpu=strix-halo --courses=basic +./auplc-installer install --runtime=containerd --image-source=build +./auplc-installer install --mirror=mirror.example.com +``` + +## What a successful install looks like + +``` + ✓ [1/8] Detecting GPU + ✓ [2/8] Generating values overlay (initial) + ✓ [3/8] Installing helm + k9s + ✓ [4/8] Installing K3s (single-node) + ✓ [5/8] Pulling custom + external images + ✓ [6/8] Deploying ROCm GPU device plugin + node labeller + ✓ [7/8] Refreshing values overlay from node labels + ✓ [8/8] Deploying JupyterHub runtime (helm install + wait) + + Open in your browser: http://localhost:30890 + (auto-logged-in as 'student' — no login needed) +``` + +## Default deployment facts + +The checked-in defaults describe a local deployment: NodePort **30890**, +`local-path` storage, ingress **disabled**, prePuller **disabled**, and +`custom.authMode: auto-login`. To change auth, courses, or accelerators, layer +a values overlay (see configure-aup-learning-cloud-courses) and +`./auplc-installer rt upgrade`. + +## Offline / air-gapped (pack) + +On a machine with Docker + internet: + +```bash +./auplc-installer pack --gpu=strix-halo # pull pre-built images into a bundle +./auplc-installer pack --gpu=strix-halo --local # or build locally first +``` + +Transfer the bundle, then on the air-gapped box: + +```bash +tar xzf auplc-bundle-gfx1151-*.tar.gz +cd auplc-bundle-gfx1151-* +sudo ./auplc-installer install +``` + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| `detect-gpu` shows the wrong/no GPU | ROCm not seeing the device, wrong kernel | OEM kernel installed + rebooted (`uname -r`), `rocminfo`, pass `--gpu=` explicitly | +| Install fails pulling images | Registry/network or wrong tag | `--image-tag`, `--mirror=`, or `--image-source=build` | +| Hub pod `ImagePullBackOff` | Tag mismatch between overlay and registry | `kubectl describe pod -n jupyterhub`, align `--image-tag` | +| GPU notebook stays Pending | Device plugin/labeller not ready or label mismatch | `kubectl get ds -A | grep amd`, `kubectl describe node | grep amd.com/gpu` | +| `localhost:30890` refused | Proxy not up or NodePort changed | `kubectl get svc -n jupyterhub`, `kubectl get pods -n jupyterhub` | +| `docker` permission denied | User not in docker group | re-run `usermod -aG docker $USER` then re-login / `newgrp docker` | +| Need to re-apply values only | Changed the overlay, not images | `./auplc-installer rt upgrade` (don't reinstall k3s) | + +## Out of scope + +Multi-node / PXE clusters (use deploy-aup-learning-cloud), GitHub OAuth and +production TLS/ingress hardening, image authoring, and course-catalog edits +(those are their own skills). This skill targets the one-box install. diff --git a/skills/install-aup-learning-cloud-single-node/skill-card.md b/skills/install-aup-learning-cloud-single-node/skill-card.md new file mode 100644 index 00000000..fcb03048 --- /dev/null +++ b/skills/install-aup-learning-cloud-single-node/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Install AUP Learning Cloud on a single AMD GPU/APU machine with the ./auplc-installer flow, for developers and demos. + +## Owner + +AMD Research diff --git a/skills/manage-aup-learning-cloud-users/SKILL.md b/skills/manage-aup-learning-cloud-users/SKILL.md new file mode 100644 index 00000000..6de97b3d --- /dev/null +++ b/skills/manage-aup-learning-cloud-users/SKILL.md @@ -0,0 +1,149 @@ +--- +name: manage-aup-learning-cloud-users +description: >- + Group: Maintain AUP Learning Cloud. Manages users, groups, passwords, admins, + and quota balances day to day with the built-in AUP Learning Cloud scripts + (scripts/generate_users_template.py and scripts/manage_users.py) plus the web + admin console (/hub/admin). Use when the user wants to onboard a class, + generate a roster CSV/Excel, bulk-create native users, export or back up users, + generate/reset passwords, force or skip first-login password changes, grant or + revoke admins, delete users, create or edit groups, run GitHub group sync, or + set/add/list quota balances and scheduled quota refresh rules. Triggers include + manage_users.py, generate_users_template.py, users.csv, passwords_output.csv, + /hub/admin, jupyterhub-admin-credentials, JUPYTERHUB_URL, JUPYTERHUB_TOKEN, + set-admin, set-passwords, set-quota, add-quota, list-quota, refreshRules, + "onboard a class", and "bulk users". Do not use to choose auth mode, configure + course visibility/quota rates, or install/deploy a cluster. +--- + +# Manage AUP Learning Cloud users + +Run the day-2 people operations: create and onboard users (including a whole +class), set/reset passwords, manage admins and groups, and grant or refresh +quota balances. Prefer the repository's deterministic scripts for bulk work and +use the web console for interactive inspection or one-off admin edits. + +The two built-in scripts are the primary automation surface: + +- `scripts/generate_users_template.py` creates CSV/Excel rosters with the + columns `manage_users.py` expects. +- `scripts/manage_users.py` performs API-backed user/admin/password work and + quota commands. + +Exact command variants, file formats, env setup, and the quota field guide are +in **[reference.md](reference.md)**. + +## Prerequisites + +- A running Hub and an **admin** account (or `custom.adminUser.enabled: true` + and the bootstrapped `admin`). +- For CLI work: run from the `aup-learning-cloud` checkout and install + `pandas`, `openpyxl`, and `requests` in the Python environment that runs the + scripts. +- `manage_users.py` requires `JUPYTERHUB_URL` and `JUPYTERHUB_TOKEN` for every + subcommand. The bundled `scripts/hub-api-env.sh` derives both from the + `jupyterhub-admin-credentials` secret and checks reachability. +- Quota subcommands use the Hub admin API. `kubectl` is only needed to bootstrap + an API token from `jupyterhub-admin-credentials` or inspect scheduled quota + refresh CronJobs. +- Native-user creation/password reset requires `authMode: multi` (or another + mode with native accounts). Password actions never apply to GitHub identities. + +## Two surfaces + +| Task | Best surface | Command | +| --- | --- | --- | +| Generate roster | CLI | `generate_users_template.py --prefix student --count 50 -o users.csv` | +| Create users | CLI for bulk, web for one-off | `manage_users.py create users.csv` | +| Passwords | CLI for bulk native-user resets | `manage_users.py set-passwords users.csv --generate -o passwords_output.csv` | +| Admins | CLI or web | `manage_users.py set-admin [--file admins.csv] [--revoke]` | +| Groups | Web console | `/hub/admin` Groups view, including Sync Now | +| Quota | CLI for repeatable grants, web for inspection | `set-quota` / `add-quota` / `list-quota` | +| Export/backup | CLI | `manage_users.py export backup.xlsx` | + +Unlimited quota is entered as `-1`, `∞`, or `unlimited`. Admin users and the +current admin are protected from deletion. + +## Workflow — onboard a class (most common) + +1. **Confirm the live script surface.** The project can evolve; quickly check + help before composing a large batch command: + + ```bash + python scripts/generate_users_template.py --help + python scripts/manage_users.py --help + ``` +2. **Set env** so `manage_users.py` can reach the Hub API: + + ```bash + source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh + ``` + + (Or export `JUPYTERHUB_URL`/`JUPYTERHUB_TOKEN` yourself — see reference.) + Use `HUB_URL="https://hub.example.com"` and `HUB_NAMESPACE=<namespace>` when + the Hub is not the default local NodePort in namespace `jupyterhub`. +3. **Generate a roster template**: + + ```bash + python scripts/generate_users_template.py --prefix student --count 50 --output users.csv + ``` +4. **Inspect the roster.** Confirm the `username` and optional `admin` columns, + and remember usernames are normalized to lowercase by `manage_users.py`. +5. **Create the users**: + + ```bash + python scripts/manage_users.py create users.csv + ``` +6. **Issue passwords** (generated, forced change on first login by default): + + ```bash + python scripts/manage_users.py set-passwords users.csv --generate -o passwords_output.csv + ``` +7. **Promote teaching staff** as needed: + + ```bash + python scripts/manage_users.py set-admin teacher01 teacher02 + ``` +8. **Grant starting quota** (if quota is enabled): + + ```bash + python scripts/manage_users.py set-quota student01 student02 --amount 1000 + ``` +9. **Deliver credentials securely** from `passwords_output.csv`, then verify in + `/hub/admin` (users appear, groups correct, balances set). + +## Quota operations + +This skill owns quota **operations** (granting/refreshing balances, scheduled +refresh). Quota **rates and enable/disable knobs** (`custom.quota.*`, +`accelerators.*.quotaRate`) live in the configure-courses skill. + +- One-off: `set-quota` (absolute) / `add-quota` (delta) / `list-quota`, or the + inline/batch editors and global "Refresh Quota" in `/hub/admin`. +- File-driven: `set-quota --file quotas.csv` expects `username,quota` columns; + `add-quota --file users.csv --amount 100` expects at least `username`. +- Scheduled: `custom.quota.refreshRules` become Kubernetes CronJobs. Verify with + `kubectl -n jupyterhub get cronjobs -l app.kubernetes.io/component=quota-refresh`. + The rule schema is in [reference.md](reference.md). + +## Safety + +- **Credentials are sensitive.** Generated passwords and `passwords_output.csv` + must be delivered securely and never committed. +- **Check rosters before writes.** Generated users are easy to create in bulk; + inspect the CSV/Excel and confirm count, prefix, admin flags, and target Hub + before running `create`, `set-passwords`, `set-admin`, or quota commands. +- **Bulk delete is destructive.** `manage_users.py delete … --yes` removes + accounts; confirm the list with the user first. Admins/current admin are + protected, but data on user PVCs can still be orphaned. +- **`set-admin` grants full platform control** — confirm the target list. +- **Quota refresh rules apply broadly.** A global Refresh Quota or a broad + `refreshRules` filter touches many users; confirm before applying. +- CLI quota commands call the Hub admin API; they need a valid API token and a + reachable Hub, not `kubectl` access. Use `kubectl` only for the secret + bootstrap or scheduled-refresh CronJob inspection described above. + +## Reference + +Env setup, every `manage_users.py` subcommand, the admin console views, +`refreshRules` schema, and troubleshooting: [reference.md](reference.md). diff --git a/skills/manage-aup-learning-cloud-users/reference.md b/skills/manage-aup-learning-cloud-users/reference.md new file mode 100644 index 00000000..ea12f92f --- /dev/null +++ b/skills/manage-aup-learning-cloud-users/reference.md @@ -0,0 +1,237 @@ +# Manage AUP Learning Cloud users — Reference + +Env setup, the built-in user-management script surface, roster file formats, +the admin console views, the `refreshRules` schema, and troubleshooting. +Workflow and gates are in [SKILL.md](SKILL.md). + +## Source guides + +- User Management Guide: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/user-management.html> +- User Quota System: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/quota-system.html> + +The live `scripts/generate_users_template.py` and `scripts/manage_users.py` in +`aup-learning-cloud` are the source of truth; verify subcommands/flags against +`--help` before large batches. + +```bash +python scripts/generate_users_template.py --help +python scripts/manage_users.py --help +python scripts/manage_users.py set-passwords --help +python scripts/manage_users.py set-quota --help +``` + +## API environment + +`manage_users.py` checks the Hub API before executing any subcommand. Set +`JUPYTERHUB_URL` and `JUPYTERHUB_TOKEN`; the token comes from the +admin-credentials secret (requires `custom.adminUser.enabled`): + +```bash +export JUPYTERHUB_URL="http://localhost:30890" +export JUPYTERHUB_TOKEN=$(kubectl -n jupyterhub get secret jupyterhub-admin-credentials \ + -o jsonpath='{.data.api-token}' | base64 -d) +``` + +The bundled `scripts/hub-api-env.sh` does this and probes `/hub/api/`. Source +it (don't execute) so the exports land in your shell: + +```bash +source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +# override the URL if not localhost:30890: +HUB_URL="https://hub.example.com" source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +``` + +CLI **quota** commands call the Hub admin API, so they need a valid API token +and a reachable Hub. `kubectl` is only needed to bootstrap the token from the +secret above or inspect scheduled quota refresh CronJobs. + +## Python dependencies + +```bash +pip install pandas openpyxl requests +``` + +## Generate roster templates + +Use `generate_users_template.py` to create the input files that +`manage_users.py` consumes. It supports numbered users or explicit names, CSV or +Excel output, optional admin flags, custom starting numbers, and digit padding. + +```bash +python scripts/generate_users_template.py --prefix student --count 50 --output users.csv +python scripts/generate_users_template.py --prefix AUP --count 30 --start 1 --output aup_users.xlsx +python scripts/generate_users_template.py --prefix student --count 100 --digits 3 --output users.csv +python scripts/generate_users_template.py --prefix admin --count 5 --admin --output admins.csv +python scripts/generate_users_template.py --names alice bob charlie --output custom_users.csv +``` + +Generated files contain at least: + +```csv +username,admin +student01,false +student02,false +``` + +You can add a `password` column before `set-passwords`, and `set-quota --file` +can read a `quota` column. + +## manage_users.py subcommands + +```bash +# Users +python scripts/manage_users.py create users.csv +python scripts/manage_users.py list +python scripts/manage_users.py export backup.xlsx +python scripts/manage_users.py delete remove_list.csv --yes + +# Admins +python scripts/manage_users.py set-admin teacher01 teacher02 +python scripts/manage_users.py set-admin --file admins.csv +python scripts/manage_users.py set-admin --revoke student01 + +# Passwords (native users only) +python scripts/manage_users.py set-passwords users.csv --generate -o passwords_output.csv +python scripts/manage_users.py set-passwords users.csv --generate --default-password "Welcome123" +python scripts/manage_users.py set-passwords users.csv --no-force-change + +# Quota +python scripts/manage_users.py set-quota user1 user2 --amount 1000 # absolute +python scripts/manage_users.py set-quota --file quotas.csv # username,quota columns +python scripts/manage_users.py add-quota user1 user2 --amount 100 # delta +python scripts/manage_users.py add-quota --file users.csv --amount 100 +python scripts/manage_users.py list-quota +``` + +Every command accepts `--url` and `--token`, but export the environment instead +so tokens do not appear in shell history or process arguments. Use a read-only +CLI command to confirm reachability: + +```bash +python scripts/manage_users.py list +``` + +### Command behavior notes + +- Usernames are normalized to lowercase before API writes, matching JupyterHub's + default behavior. Avoid rosters that depend on case-sensitive usernames. +- `create` reads `username` and optional `admin`; it does not set passwords. + Run `set-passwords` after creating native users. +- `set-passwords` requires either a `password` column or `--generate`. Generated + passwords can be saved with `--output`; that file is sensitive. +- `set-passwords` forces first-login password change unless + `--no-force-change` is passed. +- `set-quota` with positional users requires `--amount`; with `--file`, the file + can provide per-user `quota` values. +- `delete --yes` skips the interactive confirmation and should only be used + after the exact roster has been reviewed. + +## Web admin console (`/hub/admin`) + +- **Users view:** search/page, filter to active servers, create native users + (single or many, random or shared password, force change, optional admin), + edit details, reset password (native), batch password reset, inline quota + edit, batch quota update, start/stop servers, batch delete, per-user usage. + Admins and the current admin are protected from deletion. +- **Groups view:** distinguishes GitHub-synced, system-managed, and manual + groups; create manual groups, edit membership of editable groups, review + group-to-resource mappings, and **Sync Now** (manual GitHub sync when + `custom.githubOrgName` is set). System-managed groups are read-only; + GitHub-synced groups are protected from deletion. +- **Dashboard view:** total users, active sessions, usage minutes, weekly active + users, usage trends, resource distribution, top users, live sessions, pending + spawns. + +Admin quota API endpoints used by the UI: `GET/POST /hub/admin/api/quota/`, +`POST /hub/admin/api/quota/batch`, `POST /hub/admin/api/quota/refresh`, +`GET /hub/api/quota/rates`, `GET /hub/api/quota/me`. + +## Scheduled quota refresh (`refreshRules`) + +Configured under `custom.quota.refreshRules`; each rule becomes a CronJob. + +```yaml +custom: + quota: + refreshRules: + daily-topup: + enabled: true + schedule: "0 0 * * *" # cron + action: add # add | set + amount: 100 + maxBalance: 500 # also: minBalance + targets: + includeUnlimited: false + balanceBelow: 400 # also: balanceAbove, includeUsers, + # excludeUsers, usernamePattern +``` + +Verify: + +```bash +kubectl -n jupyterhub get cronjobs -l app.kubernetes.io/component=quota-refresh +kubectl -n jupyterhub get jobs -l app.kubernetes.io/component=quota-refresh +kubectl -n jupyterhub logs -l app.kubernetes.io/component=quota-refresh --tail=50 +``` + +Changing rate/enablement knobs (`custom.quota.enabled`, `cpuRate`, +`minimumToStart`, `defaultQuota`, `accelerators.*.quotaRate`) is the +configure-courses skill; re-apply with `rt upgrade` / `helm upgrade`. + +## Common runbooks + +### Onboard 50 native students + +```bash +pip install pandas openpyxl requests +source skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh +python scripts/generate_users_template.py --prefix student --count 50 --output users.csv +python scripts/manage_users.py create users.csv +python scripts/manage_users.py set-passwords users.csv --generate --output passwords_output.csv +python scripts/manage_users.py list +``` + +Review `passwords_output.csv`, distribute it through a secure channel, then +delete it when no longer needed. + +### Add teaching assistants as admins + +```bash +python scripts/generate_users_template.py --names ta01 ta02 --admin --output tas.csv +python scripts/manage_users.py create tas.csv +python scripts/manage_users.py set-passwords tas.csv --generate --output ta_passwords.csv +python scripts/manage_users.py set-admin --file tas.csv +``` + +### Grant class quota + +```bash +python scripts/manage_users.py set-quota --file quotas.csv +python scripts/manage_users.py add-quota --file users.csv --amount 100 +python scripts/manage_users.py list-quota +``` + +`quotas.csv` should contain `username,quota` when using `set-quota --file`. +`users.csv` only needs `username` for `add-quota --file`. + +## Apply config changes + +```bash +# single-node +sudo ./auplc-installer rt upgrade +# multi-node / manual +cd runtime && helm upgrade --install jupyterhub ./chart \ + -n jupyterhub --create-namespace -f values-multi-nodes.yaml +``` + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Script cannot connect to the Hub | `JUPYTERHUB_URL`/`JUPYTERHUB_TOKEN` wrong | Re-source `hub-api-env.sh`, then run `python scripts/manage_users.py list` | +| Password reset fails | Target is a GitHub user, weak password, or session lacks perms | Native users only; meet the strength policy | +| Quota command fails | Hub admin API rejects the token or is unreachable | Re-source the API environment and run `python scripts/manage_users.py list` before retrying quota work | +| No api-token secret | `custom.adminUser.enabled: false` | Enable admin bootstrap, re-apply | +| Group membership can't be edited | System-managed or GitHub-synced group | Only manual/editable groups accept edits | +| Refresh rule didn't run | Rule disabled or absent from the applied values | `kubectl … get cronjobs -l …quota-refresh`; re-apply | +| Users log in with lowercase names | Script and JupyterHub normalize usernames | Keep rosters lowercase or communicate normalized usernames | diff --git a/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh new file mode 100644 index 00000000..49b8f3a7 --- /dev/null +++ b/skills/manage-aup-learning-cloud-users/scripts/hub-api-env.sh @@ -0,0 +1,48 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Derive the JupyterHub API environment for AUP Learning Cloud user-management +# scripts and probe reachability. SOURCE this file (do not execute) so the +# exports persist in your shell: +# +# source scripts/hub-api-env.sh +# HUB_URL="https://hub.example.com" source scripts/hub-api-env.sh +# +# Environment inputs (all optional): +# HUB_URL Hub base URL (default: http://localhost:30890) +# HUB_NAMESPACE Kubernetes namespace (default: jupyterhub) +# +# Exports on success: JUPYTERHUB_URL, JUPYTERHUB_TOKEN + +_auplc_ns="${HUB_NAMESPACE:-jupyterhub}" +_auplc_url="${HUB_URL:-http://localhost:30890}" + +_auplc_token="$(kubectl -n "$_auplc_ns" get secret jupyterhub-admin-credentials \ + -o jsonpath='{.data.api-token}' 2>/dev/null | base64 -d 2>/dev/null)" + +if [ -z "$_auplc_token" ]; then + echo "hub-api-env: could not read api-token from secret 'jupyterhub-admin-credentials'" >&2 + echo " - is custom.adminUser.enabled: true and the Hub deployed?" >&2 + echo " - is your kube context/namespace ('$_auplc_ns') correct?" >&2 + # This file is meant to be sourced; `return` exits the caller's shell. The + # `exit 1` fallback only runs if the file is executed directly. + # shellcheck disable=SC2317 + return 1 2>/dev/null || exit 1 +fi + +export JUPYTERHUB_URL="$_auplc_url" +export JUPYTERHUB_TOKEN="$_auplc_token" + +# Probe the API (non-fatal: token may still be valid behind an auth proxy). +if command -v curl >/dev/null 2>&1; then + _auplc_code="$(printf 'header = "Authorization: token %s"\n' "$JUPYTERHUB_TOKEN" | \ + curl --config - -s -o /dev/null -w '%{http_code}' \ + "${JUPYTERHUB_URL%/}/hub/api/" 2>/dev/null)" + case "$_auplc_code" in + 200) echo "hub-api-env: OK — $JUPYTERHUB_URL/hub/api/ reachable (200)" ;; + *) echo "hub-api-env: WARNING — $JUPYTERHUB_URL/hub/api/ returned '$_auplc_code'; check HUB_URL/network" >&2 ;; + esac +fi + +echo "hub-api-env: exported JUPYTERHUB_URL=$JUPYTERHUB_URL and JUPYTERHUB_TOKEN (hidden)" + +unset _auplc_ns _auplc_url _auplc_token _auplc_code diff --git a/skills/manage-aup-learning-cloud-users/skill-card.md b/skills/manage-aup-learning-cloud-users/skill-card.md new file mode 100644 index 00000000..39e91ca5 --- /dev/null +++ b/skills/manage-aup-learning-cloud-users/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Manage AUP Learning Cloud users, groups, passwords, admins, and quota balances day to day with the built-in roster/template and user-management scripts, for operators and teaching staff running classes. + +## Owner + +AMD Research diff --git a/skills/monitor-aup-learning-cloud/SKILL.md b/skills/monitor-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..ae3bdc75 --- /dev/null +++ b/skills/monitor-aup-learning-cloud/SKILL.md @@ -0,0 +1,129 @@ +--- +name: monitor-aup-learning-cloud +description: >- + Group: Maintain AUP Learning Cloud. Wires AUP Learning Cloud into a + Prometheus + Grafana monitoring stack: enables + the chart's monitoring resources (ServiceMonitor, authenticated metrics token, + Grafana dashboard ConfigMaps, PrometheusRule alerts, and the metrics + NetworkPolicy) and connects them to kube-prometheus-stack or an existing + Prometheus Operator. Use when the user wants to monitor the Hub, scrape + /hub/metrics, set up Prometheus/Grafana/alerts, install kube-prometheus-stack, + enable a ServiceMonitor, see the AUP Hub Grafana dashboards, or debug a hub + target that is DOWN / Unauthorized / not scraped. Triggers include + monitoring.enabled, serviceMonitor, releaseLabel, hubMetrics, + allowUnauthenticatedScrape, prometheusRule, grafana.dashboard, + kube-prometheus-stack, hub-metrics, hub_spawn_failed_total, + hub-metrics-token. Do not use to install/deploy the platform itself + (install-/deploy-aup-learning-cloud) or to edit courses/quota + (configure-aup-learning-cloud-courses). +--- + +# Monitor AUP Learning Cloud + +Turn on Hub observability: have the chart create the monitoring objects +(`ServiceMonitor`, authenticated token secret, Grafana dashboard ConfigMaps, +alert rules, metrics `NetworkPolicy`) and make a Prometheus Operator stack +scrape `/hub/metrics` so dashboards and alerts light up. + +Enable the `monitoring.*` block in a values overlay and re-apply with Helm. The +full value reference, the kube-prometheus-stack install, and troubleshooting are +in **[reference.md](reference.md)**. + +## Prerequisites + +- A running (or about-to-deploy) AUP Learning Cloud, plus `helm` + `kubectl`. +- Either install `kube-prometheus-stack` (reference) **or** an existing + Prometheus Operator + Grafana you can point at the `jupyterhub` namespace. +- Know the Prometheus Operator's selector label — the chart stamps `release: + <monitoring.releaseLabel>` on `ServiceMonitor`/`PrometheusRule`, and it must + match what the operator selects. + +## Decide the integration + +| Situation | Action | +| --- | --- | +| No monitoring stack yet | Install `kube-prometheus-stack` as release `monitoring` in namespace `monitoring`; keep `releaseLabel: monitoring` | +| Existing Prometheus Operator + Grafana | Set `monitoring.releaseLabel` to the operator's selector; confirm it watches `monitoring` ns and can scrape `jupyterhub` | + +## Workflow + +1. **Ensure a stack exists.** Confirm Prometheus Operator + Grafana are running + (install kube-prometheus-stack if not — see reference). +2. **Enable monitoring values** in the overlay. Recommended production shape: + + ```yaml + monitoring: + enabled: true + namespace: monitoring + releaseLabel: monitoring + hubMetrics: + enabled: true + allowUnauthenticatedScrape: false + serviceMonitor: + enabled: true + interval: 15s + authorization: + enabled: true + type: Bearer + hubServiceName: prometheus-metrics + secret: { create: true, name: "", key: token } + grafana: + dashboard: { enabled: true } + prometheusRule: + enabled: true + ``` + +3. **Keep `releaseLabel` honest.** It must equal the operator's rule/monitor + selector or nothing gets scraped. +4. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f + runtime/values.yaml -f <overlay>` must succeed (the chart validates that + `hubServiceName` exists under `hub.services` with a matching `read:metrics` + role). +5. **Apply.** + + ```bash + helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + ``` + +6. **Verify** the objects and the live target: + + ```bash + skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh + ``` + + It checks the `ServiceMonitor`, token secret, dashboard ConfigMap, and + metrics `NetworkPolicy`, then port-forwards Prometheus and confirms the + `hub` target is `UP`. Manual checks are in [reference.md](reference.md). + +## Authenticated scraping (default, recommended) + +`/hub/metrics` requires a JupyterHub token. The `ServiceMonitor` authorization +block makes the chart create a token secret (`<release>-metrics-token`) in the +monitoring namespace and scrape with a Bearer token. Annotation-based scraping +cannot attach the token — leave `serviceAnnotations` off in production. + +## Useful Hub metrics + +`hub_spawn_gpu_total`, `hub_spawn_failed_total`, `hub_active_sessions`, +`hub_session_runtime_minutes`, `hub_spawn_duration_seconds`, +`hub_quota_denied_total`, `hub_quota_deducted_total`, `hub_pod_failure_total`, +`hub_repo_clone_failed_total`. Alert rules cover `hub_spawn_failed_total` and +`hub_pod_failure_total`. + +## Safety + +- **Do not set `allowUnauthenticatedScrape: true` in production.** It exposes + `/hub/metrics` without a token; only safe in an isolated dev cluster where the + endpoint is never reachable via proxy/NodePort/LoadBalancer/Ingress. +- A `helm upgrade` restarts the Hub pod (brief login blip) — schedule around a + live class. +- Don't commit any real metrics token; the chart manages the secret. +- Read-only verification (`scripts/verify_monitoring.sh`) only port-forwards; + it makes no cluster changes. + +## Reference + +The full `monitoring.*` value reference, kube-prometheus-stack install, +existing-stack reuse, manual verification commands, and troubleshooting: +[reference.md](reference.md). diff --git a/skills/monitor-aup-learning-cloud/reference.md b/skills/monitor-aup-learning-cloud/reference.md new file mode 100644 index 00000000..cacaf968 --- /dev/null +++ b/skills/monitor-aup-learning-cloud/reference.md @@ -0,0 +1,103 @@ +# Monitor AUP Learning Cloud — Reference + +The kube-prometheus-stack install, the full `monitoring.*` value reference, +existing-stack reuse, manual verification, and troubleshooting. Workflow and +gates are in [SKILL.md](SKILL.md). + +## Source guide + +- Monitoring Deployment Guide: <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/monitoring.html> +- Configuration Reference (section 12): <https://amdresearch.github.io/aup-learning-cloud/jupyterhub/configuration-reference.html> + +The chart ships the dashboards under `runtime/chart/dashboards/`; the live +`runtime/chart/values.schema.yaml` is the source of truth for the schema. + +## Install kube-prometheus-stack (reference stack) + +```bash +kubectl create namespace monitoring # AlreadyExists is safe to ignore + +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update + +helm upgrade --install monitoring prometheus-community/kube-prometheus-stack \ + --namespace monitoring + +kubectl -n monitoring get pods +``` + +The release name `monitoring` makes the operator select `release: monitoring`, +matching the default `monitoring.releaseLabel`. If you use a different release +name or selector, set `monitoring.releaseLabel` to match. + +## Reuse an existing Prometheus + Grafana + +Confirm with the monitoring owner that: + +- the Operator watches `ServiceMonitor` in the `monitoring` namespace, +- Prometheus may scrape services in `jupyterhub`, +- the operator's selector matches `release: <monitoring.releaseLabel>`, +- the Grafana sidecar reads dashboard ConfigMaps labelled `grafana_dashboard: "1"` + from `monitoring`. + +Example: if the stack selects `release: platform-monitoring`, set +`monitoring.releaseLabel: platform-monitoring`. + +## monitoring.* value reference + +| Value | Meaning | +| --- | --- | +| `monitoring.enabled` | Master switch for all monitoring objects | +| `monitoring.namespace` | Namespace the objects are created in (`monitoring`) | +| `monitoring.releaseLabel` | `release` label on ServiceMonitor/PrometheusRule; must match the operator selector | +| `monitoring.hubMetrics.enabled` | Hub metrics integration; also creates a metrics NetworkPolicy allowing the monitoring ns to reach the Hub on `8081` | +| `monitoring.hubMetrics.allowUnauthenticatedScrape` | Allow `/hub/metrics` without a token — dev only | +| `monitoring.hubMetrics.serviceAnnotations.enabled` | Adds `prometheus.io/*` annotations; cannot carry the token — prefer the ServiceMonitor path | +| `monitoring.serviceMonitor.enabled` | Creates `ServiceMonitor` `hub-metrics` selecting `component: hub`, port `8081`, path `<hub.baseUrl>/hub/metrics` | +| `monitoring.serviceMonitor.interval` | Scrape interval, e.g. `15s` | +| `monitoring.serviceMonitor.authorization.enabled` | Authenticated scraping (keep on) | +| `monitoring.serviceMonitor.authorization.type` | Default `Bearer` | +| `monitoring.serviceMonitor.authorization.hubServiceName` | Hub service account for the token; default `prometheus-metrics` must match `hub.services` + `hub.loadRoles` (`read:metrics`) | +| `monitoring.serviceMonitor.authorization.secret.create` | Create the token secret in the monitoring ns | +| `monitoring.serviceMonitor.authorization.secret.name` | Custom/existing secret name; blank = `<release>-metrics-token` | +| `monitoring.serviceMonitor.authorization.secret.key` | Secret key; default `token` | +| `monitoring.grafana.dashboard.enabled` | Creates dashboard ConfigMaps labelled `grafana_dashboard: "1"` | +| `monitoring.prometheusRule.enabled` | Creates alert rules for `hub_spawn_failed_total`, `hub_pod_failure_total` | + +## Apply + +```bash +helm upgrade --install jupyterhub ./runtime/chart -n jupyterhub \ + -f runtime/values.yaml -f <overlay> +# include any local overlay too, e.g. -f runtime/values.local.yaml +``` + +## Manual verification + +```bash +kubectl -n monitoring get servicemonitor hub-metrics +kubectl -n monitoring get secret | grep metrics-token +kubectl -n monitoring get configmap grafana-dashboard-aup-hub +kubectl -n jupyterhub get networkpolicy hub-metrics +# alerts, if enabled: +kubectl -n monitoring get prometheusrule hub-alerts + +# Is the target UP? +kubectl -n monitoring port-forward svc/monitoring-kube-prometheus-prometheus 9090:9090 & +curl -fsSL 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22hub%22%7D' +# open http://127.0.0.1:9090/targets and look for hub-metrics = UP +``` + +A healthy query returns `"job":"hub"`, `"namespace":"jupyterhub"`, value `"1"`. +The dashboard ConfigMap should contain `aup-hub-operations.json` and +`aup-hub-notebook-resources.json`. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| ServiceMonitor exists but no scraping | `release` label mismatch | `kubectl -n monitoring get servicemonitor hub-metrics --show-labels`; fix `releaseLabel`, re-apply | +| Target DOWN / Unauthorized | Annotation scraping or auth disabled | Use `serviceMonitor.authorization.enabled: true`, `serviceAnnotations` off | +| Token secret missing | Auth/secret create not enabled, or `hubServiceName` invalid | Enable `secret.create`; ensure `hubServiceName` exists under `hub.services` with `read:metrics` | +| Grafana dashboards absent | Sidecar not watching ns/label | ConfigMap label `grafana_dashboard: "1"`; sidecar must watch `monitoring` | +| Alerts absent | Rule ns/label not watched | `kubectl -n monitoring get prometheusrule hub-alerts --show-labels`; match the operator's rule selector | diff --git a/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh b/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh new file mode 100755 index 00000000..5d426ecf --- /dev/null +++ b/skills/monitor-aup-learning-cloud/scripts/verify_monitoring.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Read-only verification that AUP Learning Cloud monitoring is wired up: +# checks the ServiceMonitor, metrics token secret, Grafana dashboard ConfigMap, +# and metrics NetworkPolicy, then port-forwards Prometheus and confirms the +# Hub target is UP. Makes no cluster changes. +# +# Usage: +# scripts/verify_monitoring.sh +# +# Environment (optional): +# MON_NS monitoring namespace (default: monitoring) +# HUB_NS jupyterhub namespace (default: jupyterhub) +# PROM_SVC Prometheus service (default: monitoring-kube-prometheus-prometheus) + +set -uo pipefail + +MON_NS="${MON_NS:-monitoring}" +HUB_NS="${HUB_NS:-jupyterhub}" +PROM_SVC="${PROM_SVC:-monitoring-kube-prometheus-prometheus}" + +rc=0 +pass() { printf ' [OK] %s\n' "$1"; } +warn() { printf ' [WARN] %s\n' "$1"; rc=1; } + +echo "Checking monitoring objects (mon ns=$MON_NS, hub ns=$HUB_NS)..." + +if kubectl -n "$MON_NS" get servicemonitor hub-metrics >/dev/null 2>&1; then + pass "ServiceMonitor hub-metrics present" +else + warn "ServiceMonitor hub-metrics missing (serviceMonitor.enabled?)" +fi + +if kubectl -n "$MON_NS" get secret 2>/dev/null | grep -q 'metrics-token'; then + pass "metrics token secret present" +else + warn "metrics token secret missing (authorization.secret.create?)" +fi + +if kubectl -n "$MON_NS" get configmap grafana-dashboard-aup-hub >/dev/null 2>&1; then + pass "Grafana dashboard ConfigMap present" +else + warn "Grafana dashboard ConfigMap missing (grafana.dashboard.enabled?)" +fi + +if kubectl -n "$HUB_NS" get networkpolicy hub-metrics >/dev/null 2>&1; then + pass "metrics NetworkPolicy present" +else + warn "metrics NetworkPolicy missing (hubMetrics.enabled?)" +fi + +echo "Checking the live Prometheus target..." +if ! kubectl -n "$MON_NS" get svc "$PROM_SVC" >/dev/null 2>&1; then + warn "Prometheus service '$PROM_SVC' not found; set PROM_SVC to your service name" + echo "Done (with warnings)."; exit "$rc" +fi + +kubectl -n "$MON_NS" port-forward "svc/$PROM_SVC" 9090:9090 >/dev/null 2>&1 & +pf_pid=$! +trap 'kill "$pf_pid" 2>/dev/null' EXIT +sleep 3 + +result="$(curl -fsS 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22hub%22%7D' 2>/dev/null)" +case "$result" in + *'"job":"hub"'*'"1"'*) pass "Prometheus reports hub target UP" ;; + *'"job":"hub"'*) warn "hub target found but not UP (value != 1)" ;; + *) warn "hub target not found in Prometheus (label/selector mismatch?)" ;; +esac + +echo "Done." +exit "$rc" diff --git a/skills/monitor-aup-learning-cloud/skill-card.md b/skills/monitor-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..d8e45314 --- /dev/null +++ b/skills/monitor-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Connect AUP Learning Cloud to Prometheus and Grafana — ServiceMonitor, authenticated metrics, dashboards, and alert rules — for operators who need Hub observability. + +## Owner + +AMD Research diff --git a/skills/plan-aup-learning-cloud-deployment/SKILL.md b/skills/plan-aup-learning-cloud-deployment/SKILL.md new file mode 100644 index 00000000..1989cc54 --- /dev/null +++ b/skills/plan-aup-learning-cloud-deployment/SKILL.md @@ -0,0 +1,150 @@ +--- +name: plan-aup-learning-cloud-deployment +description: >- + Group: Plan & deploy AUP Learning Cloud. Recommends the hardware sizing, + cluster topology, network plan, and a + buyer-facing bill of materials (BOM) for someone who saw an AUP Learning + Cloud / AUPLC demo and wants to stand up their own local deployment. Use + when the user asks how many AIPCs / machines / GPUs / routers they need, + wants sizing or a hardware recommendation, says "I saw the demo and want to + deploy this myself", "what should I buy", "bill of materials", "spec out a + lab", or describes a class headcount and a network (routers, subnets, static + IP vs DHCP) and wants a configuration. It interviews requirements + network, + researches current AMD silicon, and sizes the cluster. Do not use to actually + run the install (install-aup-learning-cloud-single-node), build a multi-node + cluster (deploy-aup-learning-cloud), or edit the course catalog + (configure-aup-learning-cloud-courses) — hand off to those once the plan is + agreed. +--- + +# Plan an AUP Learning Cloud deployment + +Turn a prospective adopter's needs into a concrete recommendation: how many +AMD machines (Ryzen AI AIPC, Radeon workstation, or server) and how much +networking gear to buy, which chips to pick, what cluster topology to use, and +an IP/network plan — ending in a sizing table and a buyer-facing bill of +materials (BOM). This is the pre-purchase advisory step that precedes the +install/deploy skills. + +The single measurable outcome: a defensible BOM + sizing/topology/network plan +the user can act on. Full sizing math, the hardware-research method, the +network decision table, worked examples, and the BOM template are in +**[reference.md](reference.md)**. + +## Prerequisites + +- Web access (to look up the latest AMD silicon and confirm ROCm support). +- No cluster or checkout is required — this skill produces a plan, not a + running system. +- Helpful context: the AUP Learning Cloud + [overview](https://amdresearch.github.io/aup-learning-cloud/introduction/overview.html) + and [quick start](https://amdresearch.github.io/aup-learning-cloud/installation/quick-start.html). + +## Phase 1 — Interview the requirements + +Ask, and confirm back, before sizing anything: + +1. **Courses/toolkits** wanted: Computer Vision, Deep Learning, LLM-from-scratch, + Physics Sim, and/or generic CPU/GPU + code-server. This drives both the GPU + VRAM tier and which images to enable later. +2. **Total headcount** and the **session pattern**: a whole class on at the same + time (scheduled lab) vs self-paced/錯峰 usage. +3. **Peak concurrent users**, split into **GPU sessions vs CPU-only sessions**. + If the user only knows the total, estimate peak (see reference) and confirm. +4. **Persistence/storage** expectations (do notebooks need to survive reboots; + rough per-user disk). +5. **Budget band** and **online vs air-gapped**. + +## Phase 2 — Interview the network environment + +1. How many **routers**, and how many **subnets / CIDRs** with which IP ranges. +2. **Static IP vs DHCP**; can a stable/reserved IP be given to one machine. +3. A **managed switch** and how many **free ports** (PoE not needed). +4. **Internet access** from the would-be service machine; any VLANs/firewalls. +5. Whether the machines are **bare (can netboot)** or will each get an OS. + +## Phase 3 — Research current AMD hardware + +Do not rely on memory — **web-search the latest AMD silicon** and match it to +the requirements: + +1. Search current AMD options across form factors: **Ryzen AI APUs** (mini-PC / + laptop AIPC), **Radeon workstation dGPUs**, and **multi-GPU workstations or + servers**. Compare by **compute (CU/TFLOPs) and VRAM**, not marketing tier. +2. **Gate every candidate on ROCm support** — if a chip is not ROCm-supported it + cannot run the GPU notebooks. +3. **Map the chip to an existing chart accelerator key** (`phx`, `strix`, + `strix-halo`, `9070xt`, `r9700`, or `9600gre`) and the expected + `amd.com/gpu.product-name` node label. `rdna4` is an installer detection + fallback, not an existing chart accelerator key accepted by + `gen_configs.py`. A new chart key requires + `configure-aup-learning-cloud-courses` work before it can be generated. +4. Prefer **multi-GPU chassis** (workstation/server) when peak concurrent GPU + users is high enough that many single-GPU AIPCs become impractical to cable, + power, and manage. Keep AIPCs for small labs and the demo-like experience. + +## Phase 4 — Size the cluster + +The full formulas and per-notebook config table are in +[reference.md](reference.md). The shape of it: + +1. **Concurrency, not headcount.** Convert total users to **peak concurrent** + (~40-60% of total for self-paced; ~100% for a whole-class scheduled lab). +2. **GPU drives machine count (whole-GPU, no sharing).** Each GPU notebook in + AUPLC claims a **whole, exclusive** `amd.com/gpu: "1"` (request == limit); + there is no time-slicing/MIG, and this is the same for every GPU course. So + `GPUs needed = peak concurrent GPU users`. An APU box = 1 GPU; a + workstation/server = N cards. +3. **RAM/CPU sets the per-machine spec.** CPU notebooks are best-effort and pack + densely (RAM-bound): `RAM ≈ (concurrent users on the node × max mem/user) + + overhead`. Pick per-user memory from the course type (reference table). +4. **VRAM picks the chip tier.** Exclude 4GB iGPUs (780M/890M) for LLM/large + models; steer to Strix Halo (64GB) or R9700 (32GB) for those. +5. **Add a control/service node.** PXE/NFS/k3s-server overhead; small labs may + co-locate it on a GPU node (state the single-point-of-failure trade-off). + +## Phase 5 — Plan topology and network + +1. **Choose the topology** (decision table in reference): + - **Single-node** (`./auplc-installer`) for one box / demo replica. + - **PXE-diskless cluster** for bare AIPCs on **one flat L2 subnet** that can + netboot (relies on the user's existing DHCP/router; the service machine + needs a static IP). + - **SSH-preinstalled cluster** when nodes already have an OS or the network is + routed/multi-subnet. +2. Derive the **switch-port count** (≈ nodes + uplink) and whether the existing + router(s) suffice or a managed switch is needed. +3. Produce an **IP plan**: the static service-machine IP, the node subnet/CIDR, + gateway, and DNS — consistent with the topology you chose. + +## Phase 6 — Deliver the recommendation + +Produce, for the user: + +- A **sizing table** (peak concurrency → GPU count → machine count + the chosen + chip/VRAM, with the assumptions spelled out). +- A **topology choice** and an **IP/network plan**. +- A **bill of materials**: machine model + quantity + GPU, plus switch/router and + cabling, framed so the user can purchase (this is what leads to AMD hardware + sales). Offer at least an AIPC-based option and a denser workstation/server + option when concurrency is non-trivial. +- A **handoff**: point to `install-aup-learning-cloud-single-node` (one box) or + `deploy-aup-learning-cloud` (cluster) to execute, and + `configure-aup-learning-cloud-courses` to enable the chosen courses. + +## Safety + +- **Advisory only.** This skill plans; it does not install, buy, or change any + system. Never run installer/deploy commands from here. +- **State every assumption** (concurrency ratio, per-user memory, GPUs per + chassis) so the user can correct them before spending money. +- **Always confirm ROCm support** for any recommended silicon; never recommend a + chip you could not verify is supported. +- **Flag single-point-of-failure** trade-offs of all-in-one small labs, and + storage durability (local-path/NFS-on-one-box is disposable without backups). + +## Reference + +Sizing formulas + per-notebook config table, the whole-GPU evidence, the +hardware-research method, the network/topology decision table, worked examples, +the BOM template, and the interview question bank: [reference.md](reference.md). diff --git a/skills/plan-aup-learning-cloud-deployment/reference.md b/skills/plan-aup-learning-cloud-deployment/reference.md new file mode 100644 index 00000000..f0e840cf --- /dev/null +++ b/skills/plan-aup-learning-cloud-deployment/reference.md @@ -0,0 +1,290 @@ +# Plan an AUP Learning Cloud deployment — Reference + +Sizing math, the whole-GPU evidence, the hardware-research method, the +network/topology decision table, worked examples, the BOM template, and the +interview question bank. The workflow and gates are in [SKILL.md](SKILL.md). + +## Contents + +- [Source guides](#source-guides) +- [Sizing model](#sizing-model) +- [Per-notebook resource config (typical)](#per-notebook-resource-config-typical) +- [Accelerator catalog and VRAM tiers](#accelerator-catalog-and-vram-tiers) +- [Researching current AMD hardware](#researching-current-amd-hardware) +- [Topology and network decision](#topology-and-network-decision) +- [Sizing procedure](#sizing-procedure) +- [Worked examples](#worked-examples) +- [BOM template](#bom-template) +- [Interview question bank](#interview-question-bank) +- [Handoff](#handoff) + +## Source guides + +- Overview: <https://amdresearch.github.io/aup-learning-cloud/introduction/overview.html> +- Quick Start (single-node): <https://amdresearch.github.io/aup-learning-cloud/installation/quick-start.html> +- 3-node mini-cluster (PXE diskless): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node/multi-aipc-hardware-deployment.html> +- Standard multi-node (SSH): <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> + +Treat the live `aup-learning-cloud` repo (`runtime/values.yaml`, the spawner) +and AMD's current product pages as the sources of truth; this file condenses +the opinionated sizing path. + +## Sizing model + +The model is validated against industry JupyterHub capacity-planning practice. + +### Concurrency, not headcount + +Size on **peak concurrent users**, not total registrations — the always-on Hub +overhead is tiny and costs scale with simultaneously active users +([JupyterHub capacity planning](https://jupyterhub.readthedocs.io/en/stable/explanation/capacity-planning.html)). +Rule of thumb: peak concurrent ≈ **40-60% of total** for self-paced cohorts +([TLJH](https://tljh.jupyter.org/en/latest/howto/admin/resource-estimation.html), +[UC Berkeley CDSS](https://cdss.berkeley.edu/choosing-right-jupyterhub-infrastructure)). +Use **~100%** when a whole class is scheduled on at the same time. + +### GPU dimension = machine count (whole-GPU, exclusive, no sharing) + +In AUP Learning Cloud every GPU notebook claims a **whole, exclusive GPU**. +The spawner sets both the guarantee (request) and the limit to the same +integer, in +[`runtime/hub/core/spawner/kubernetes.py`](https://github.com/AMDResearch/aup-learning-cloud/blob/main/runtime/hub/core/spawner/kubernetes.py) +(around lines 740-743): + +```python +if "amd.com/gpu" in requirements: + self.extra_resource_guarantees = {"amd.com/gpu": str(requirements["amd.com/gpu"])} + self.extra_resource_limits = {"amd.com/gpu": str(requirements["amd.com/gpu"])} +``` + +`amd.com/gpu` is a Kubernetes **integer extended resource** with request == +limit, so a pod takes whole cards only. There is **no fractional / time-slicing +/ MIG / MPS sharing** in this chart (those are NVIDIA-only: +[NVIDIA time-slicing](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/25.10/gpu-sharing.html), +[MIG/MPS](https://kubedojo.com/gpu-sharing-mig-time-slicing-k8s)); the AMD ROCm +k8s device plugin allocates whole devices. This is **universal across every GPU +course** — `gpu`, `code-gpu`, `Course-CV`, `Course-DL`, `Course-LLM`, and +`Course-PhySim` all set `amd.com/gpu: "1"` in `custom.resources.requirements` +and share the same `_configure_spawner()` path; `cpu`, `code-cpu`, and `none` +request no GPU. The count is admin-configurable but only as an integer number +of whole cards. + +Consequence: + +``` +concurrent GPU notebooks = total physical GPUs in the cluster +GPUs needed = peak concurrent GPU users +``` + +- An APU AIPC (e.g. Strix Halo 8060S) = **1 iGPU = 1 concurrent GPU user**. +- A workstation/server holds **N dGPUs = N concurrent GPU users**. +- When all GPUs are busy, extra GPU spawns stay `Pending` until one frees. + +### RAM/CPU dimension = per-machine spec + +CPU notebooks are **best-effort** in the default chart (`cpu: "0"`, +`memory: "0Gi"`), so they pack densely and the binding constraint is **RAM**. +Standard formulas +([TLJH](https://tljh.jupyter.org/en/latest/howto/admin/resource-estimation.html), +[CDSS](https://cdss.berkeley.edu/choosing-right-jupyterhub-infrastructure)): + +``` +RAM per machine = (concurrent users on that machine × max memory per user) + overhead +vCPU per machine = (concurrent users on that machine × CPU per user) + 20% +``` + +Note: the spawner derives a CPU limit of `cpu × 1.25` and a memory limit of +`memory × 1.5` when not explicitly set, so if you raise the per-course +`requirements` the effective ceiling is a bit higher than the request. + +## Per-notebook resource config (typical) + +Web-sourced typical per-user values; tune with Prometheus once running. z2jh's +default guarantee is 1G RAM, and a conservative classroom starting point is +0.5 CPU + 2GB +([z2jh user resources](https://z2jh.jupyter.org/en/stable/jupyterhub/customizing/user-resources.html)). + +| Course / use | Memory per user | CPU per user | GPU | VRAM note | +| --- | --- | --- | --- | --- | +| Entry / light Python (generic `cpu`, code-server) | 2 GB (limit higher) | 0.5 vCPU | none | — | +| Computer Vision (`Course-CV`) | 8-16 GB | 1-2 vCPU | 1 whole GPU | mid VRAM ok | +| Deep Learning (`Course-DL`) | 8-16 GB | 1-2 vCPU | 1 whole GPU | needs decent VRAM; enlarge `/dev/shm` for PyTorch DataLoader | +| LLM from scratch (`Course-LLM`) | 16 GB+ | 2+ vCPU | 1 whole GPU | **large VRAM** — exclude 4GB iGPUs | +| Physics Sim / Genesis (`Course-PhySim`) | 8-16 GB | 1-2 vCPU | 1 whole GPU | mid/large VRAM | + +DL frameworks try to grab most VRAM; with whole-GPU allocation that is fine +(one user per card), but it also means you cannot pack two GPU users onto one +card. + +## Accelerator catalog and VRAM tiers + +From `runtime/values.yaml` (`custom.accelerators`). The VRAM column is the key +chip-selection driver: + +| Accelerator key | Chip | VRAM | CU | `amd.com/gpu.product-name` | Good for | +| --- | --- | --- | --- | --- | --- | +| `phx` | Radeon 780M (Phoenix iGPU) | 4 GB shared | 12 | `AMD_Radeon_780M_Graphics` | light CPU/GPU only; NOT LLM | +| `strix` | Radeon 890M (Strix iGPU) | 4 GB shared | 16 | `AMD_Radeon_890M_Graphics` | light CPU/GPU only; NOT LLM | +| `strix-halo` | Radeon 8060S (Strix Halo iGPU) | 64 GB unified | 40 | `AMD_Radeon_8060S_Graphics` | CV/DL/LLM/PhySim | +| `9070xt` | Radeon RX 9070 XT | 16 GB GDDR6 | 64 | `AMD_Radeon_RX_9070_XT` | CV/DL; mid LLM | +| `r9700` | Radeon AI PRO R9700 | 32 GB GDDR6 | 64 | `AMD_Radeon_AI_PRO_R9700` | CV/DL/LLM; multi-card workstation/server | +| `9600gre` | Radeon RX 9600 GRE | 12 GB GDDR6 | 32 | `AMD_Radeon_RX_9600_GRE` | CV/DL; light to mid LLM | + +`phx` also sets `HSA_OVERRIDE_GFX_VERSION: 11.0.0`. If a fleet normalizes a +product name differently, the `nodeSelector` string must be changed to match +the real node label. + +## Researching current AMD hardware + +Always confirm against current AMD product pages; silicon refreshes often. + +1. **Search by form factor and capability**, not tier name: + - Ryzen AI APU mini-PCs / laptops (the AIPC, demo-like experience). + - Radeon workstation dGPUs (e.g. AI PRO class) for single- or multi-card boxes. + - Multi-GPU workstations / rack servers when concurrency is high. +2. **ROCm gate.** Only recommend chips with confirmed ROCm support; otherwise + the GPU notebooks will not run. +3. **Map to a chart key.** Fit the chip to an existing accelerator key + (`phx`/`strix`/`strix-halo`/`9070xt`/`r9700`/`9600gre`) and the expected + `amd.com/gpu.product-name`. If it is a brand-new product with no key yet, + tell the user it needs a `configure-aup-learning-cloud-courses` accelerator + entry (and possibly a new image) before deployment. +4. **AIPC vs workstation vs server:** prefer many single-GPU AIPCs for small + labs and the closest match to the demo; switch to multi-GPU chassis when the + GPU count makes cabling/power/management of many boxes impractical. + +## Topology and network decision + +| Topology | When | Network needs | +| --- | --- | --- | +| **Single-node** (`./auplc-installer`) | One box; replicate the demo; ≤ a handful of users sharing one GPU sequentially | Any network; `localhost:30890` | +| **PXE-diskless cluster** | Bare AIPCs that can netboot; small teaching lab; zero per-machine install | **One flat L2 subnet**; the user's existing DHCP/router stays (dnsmasq runs Proxy-DHCP and does NOT hand out leases); service machine needs a **static/reserved IP**; Secure Boot off; netboot in firmware | +| **SSH-preinstalled cluster** | Nodes already run Ubuntu, or the network is routed/multi-subnet, or netboot is not possible | Each node reachable over SSH; tolerates multiple subnets/routers | + +Networking gear rules of thumb: + +- **One flat subnet** is strongly preferred for PXE-diskless (Proxy-DHCP is + broadcast/L2-bound). Multiple routers/subnets break it unless they share a + broadcast domain or you add DHCP relay — in that case prefer SSH-preinstalled. +- **Switch ports ≈ number of nodes + 1 uplink.** A typical consumer router has + ~4 LAN ports; beyond that, add a managed switch (1GbE is fine for a teaching + lab; NFS traffic benefits from 2.5/10GbE on larger clusters). +- **Static IP:** reserve one for the service/control machine (PXE/NFS/k3s + server / API endpoint all use it). Other nodes can be DHCP. +- Keep `k3s_version` and `pxe_k3s_version` in sync (agents must not be newer + than the server) — relevant when handing off to `deploy-aup-learning-cloud`. + +### Sample IP plan (single flat subnet) + +| Item | Value (example) | +| --- | --- | +| Subnet / CIDR | `192.168.1.0/24` | +| Gateway (existing router) | `192.168.1.1` | +| DHCP pool (existing) | `192.168.1.100-199` | +| Service machine (static) | `192.168.1.10` | +| Agents | DHCP from the existing pool (PXE) or static outside it (SSH) | +| Hub access | `http://192.168.1.10:30890` (NodePort) | + +## Sizing procedure + +1. Total users → **peak concurrent** (×0.4-0.6, or ×1.0 for a scheduled class). +2. Split peak into **GPU sessions** and **CPU-only sessions**. +3. **GPU count = peak concurrent GPU users.** Convert to machines by chassis: + AIPC = 1 GPU/box; workstation/server = N GPUs/box. +4. **RAM check** each machine against the CPU/GPU sessions it will host using + the RAM formula; bump per-machine memory or add a box if short. +5. **Chip tier** from per-course VRAM needs (LLM → 64GB Strix Halo or 32GB + R9700; light → smaller is fine). +6. **+1 control/service node** (or co-locate on a GPU node for a tiny lab, with + a stated SPOF caveat). +7. **Research current models** that satisfy 3-5 and are ROCm-supported; produce + the BOM. + +## Worked examples + +### Example A — 30 students, LLM course, one scheduled class slot + +- Concurrency: whole class on together → peak ≈ **30**, all GPU, all need large + VRAM. +- GPUs needed = 30. LLM ⇒ Strix Halo (64GB) or R9700 (32GB). +- **Option 1 (AIPC):** 30× Strix Halo AIPC (1 GPU each) + 1 control node ≈ + **31 machines**, one flat subnet, a 48-port switch. +- **Option 2 (dense):** workstations/servers with 4× R9700 each → ~8 GPU boxes + + 1 control node ≈ **9 machines**; fewer boxes to cable/power/manage, higher + per-box cost. +- Present both; let the user trade box count vs per-box cost. + +### Example B — 60 students, mixed CV/DL, self-paced + +- Concurrency ≈ 50% → peak ≈ **30** active; assume ~20 GPU + ~10 CPU at peak. +- GPUs needed = 20 (CV/DL ⇒ 16-32GB VRAM ok: 9070xt/R9700, or Strix Halo). +- CPU-only 10 sessions pack onto a few nodes; RAM = 10 × ~4GB + overhead ≈ a + single 64GB node handles them, or fold onto GPU nodes. +- ~20 GPU boxes (AIPC) **or** ~5 boxes × 4 cards + 1 control node. + +### Example C — small demo replica + +- 1 box, sequential single-GPU use. Use **single-node** `./auplc-installer` on + one Strix Halo AIPC. No switch/router changes. Hand off to + `install-aup-learning-cloud-single-node`. + +## BOM template + +``` +AUP Learning Cloud — recommended bill of materials + +Requirements assumed: + Courses : <e.g. LLM, DL> + Total students : <N> Peak concurrent: <M> (assumption: <ratio/scheduled>) + Peak GPU sessions : <G> Peak CPU sessions: <C> + +Compute: + <qty> × <AMD machine model> (<chip>, <VRAM>, <GPUs/box>) → <total GPUs> + 1 × control/service node (<model or "co-located">) + +Networking: + 1 × <managed switch, port count> (≈ nodes + uplink) + reuse existing router/DHCP; reserve 1 static IP for the service node + <cabling> + +Topology : <single-node | PXE-diskless | SSH-preinstalled> +Storage : <local-path (single box) | NFS on service node | dedicated NFS> + +Notes / assumptions: + - GPU is whole-card per user (no sharing): concurrent GPU users = total GPUs. + - <SPOF / backup caveats> +Next step : <install-aup-learning-cloud-single-node | deploy-aup-learning-cloud> +``` + +## Interview question bank + +Requirements: + +- Which courses/toolkits (CV / DL / LLM / PhySim / generic)? +- Total students; one scheduled class at a time, or self-paced? +- Best guess at peak concurrent users; how many of those need a GPU? +- Do notebooks need to persist across reboots? Rough per-user disk? +- Budget band? Internet access or air-gapped? + +Network: + +- How many routers? How many subnets/CIDRs and what IP ranges? +- Static IP available for one machine, or DHCP only? +- Managed switch? How many free ports? +- Can the machines network-boot (PXE), or will each get an OS install? +- Any VLANs/firewalls between the machines? + +## Handoff + +| After the plan is agreed | Use skill | +| --- | --- | +| Install on one box / demo replica | `install-aup-learning-cloud-single-node` | +| Build the multi-node cluster (PXE or SSH) | `deploy-aup-learning-cloud` | +| Enable the chosen courses / add an accelerator entry for a new chip | `configure-aup-learning-cloud-courses` | +| Build/publish custom course images | `build-aup-learning-cloud-images` | + +## Out of scope + +Running any install/deploy command, buying hardware, production HA/TLS/ingress +hardening, monitoring, and authoring images or course catalogs — this skill +stops at the recommendation/BOM and hands off. diff --git a/skills/plan-aup-learning-cloud-deployment/skill-card.md b/skills/plan-aup-learning-cloud-deployment/skill-card.md new file mode 100644 index 00000000..e35e1e4c --- /dev/null +++ b/skills/plan-aup-learning-cloud-deployment/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Recommends hardware sizing, cluster topology, a network/IP plan, and a buyer-facing bill of materials for a prospective AUP Learning Cloud adopter who saw the demo and wants to deploy locally. + +## Owner + +AMD Research diff --git a/skills/troubleshoot-aup-learning-cloud/SKILL.md b/skills/troubleshoot-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..8bc51964 --- /dev/null +++ b/skills/troubleshoot-aup-learning-cloud/SKILL.md @@ -0,0 +1,100 @@ +--- +name: troubleshoot-aup-learning-cloud +description: >- + Group: Maintain AUP Learning Cloud. Diagnoses a broken AUP Learning Cloud + deployment against a known list of + causes: PXE/netboot failures, agent nodes not joining, GPU notebooks stuck + Pending or ROCm labels missing, NFS/PVC storage provisioning failures, and + login/authentication problems. Use when the user reports that AUPLC is broken, + a node won't join, a pod is Pending / CrashLoopBackOff / ImagePullBackOff, the + GPU isn't scheduling, storage won't bind, PXE agents won't boot, the Hub login + 404s, or asks to debug/diagnose/figure out why something failed. Evidence-first + and read-only: gather state, identify the cause, then hand off the fix to the + matching deploy/install/configure/upgrade skill. Do not use to perform a fresh + install or a routine config change when nothing is actually failing. +--- + +# Troubleshoot AUP Learning Cloud + +Find the root cause of a failing deployment from runtime evidence, name it, and +point at the fix — without thrashing. Gather state first, match the symptom to +a known cause, change one thing, re-check. The full symptom → cause → checks +matrices live in **[reference.md](reference.md)**. + +## Prerequisites + +- Access to the cluster (`kubectl`, the right `KUBECONFIG`) and/or the service + machine (for PXE/host issues). +- A checkout of `aup-learning-cloud` for config cross-checks. +- The deploy skill's `$DEPLOY_SCRIPTS/detect_cluster.sh` is a fast way to snapshot + nodes, GPU labels, storage classes, and the device plugin/labeller state. + +From any checkout directory, define +`REPO_ROOT="$(git rev-parse --show-toplevel)"` and +`DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts"`. For an +installed plugin, define `DEPLOY_SKILL_DIR` as the absolute directory containing +the loaded deploy skill's `SKILL.md`, then set +`DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts"`. + +## Method (don't thrash) + +1. **Scope it.** Which layer is failing — netboot, node join, GPU scheduling, + storage, or auth? One layer at a time. +2. **Gather evidence before acting.** + + ```bash + kubectl get nodes -o wide + kubectl get pods -A | grep -Ev 'Running|Completed' + kubectl describe pod -n jupyterhub <pod> # Events explain Pending/ImagePull + "$DEPLOY_SCRIPTS/detect_cluster.sh" # from the deploy skill + ``` + +3. **Match to a cause** using the [reference.md](reference.md) matrices. +4. **Change one thing**, then re-check the same evidence. Do not stack + speculative changes. After ~4 failed attempts with no new evidence, stop and + report what you observed and the most likely next step. +5. **Hand off the fix** to the right skill (below) rather than improvising. + +## Where each fix lives + +| Failing layer | Fix with | +| --- | --- | +| PXE rootfs vars / rebuild, agent netboot, NFS rootfs, k3s token publish | deploy-aup-learning-cloud | +| Single-node install / GPU detect / `localhost:30890` | install-aup-learning-cloud-single-node | +| `nodeSelector` ↔ GPU label, course/team/quota, auth mode | configure-aup-learning-cloud-courses | +| Image tag / `ImagePullBackOff` from a missing build | build-aup-learning-cloud-images | +| Version mismatch after a bump, chart rollback | upgrade-aup-learning-cloud | + +## First checks by layer + +- **Netboot:** `systemctl status dnsmasq nfs-kernel-server apache2`, + `journalctl -u dnsmasq`, firmware boot order + Secure Boot, TFTP files in + `/srv/tftp`. +- **Node join:** `systemctl status k3s-agent`, `journalctl -u k3s-agent`, + hostname/`api_endpoint`/token, `curl http://<SERVICE_IP>:8080/k3s/token`. +- **GPU:** `kubectl get ds -A | grep amd`, + `kubectl describe node <n> | grep amd.com/gpu`, then compare to + `custom.accelerators.*.nodeSelector`. +- **Storage:** `kubectl get pvc -A`, provisioner logs, `showmount -e <NFS>`, + `/etc/exports`. +- **Auth:** Hub logs (`kubectl logs -n jupyterhub deploy/hub`), `custom.authMode` + (avoid `dummy`, whose login 404s), GitHub OAuth callback URL. + +## Safety + +Evidence-first and read-only by default. Stop and get explicit confirmation +before any state change, especially: + +- `kubectl delete node <name>` (clears a stale node object — debugging only). +- `helm uninstall`, `helm rollback`, or recreating any PVC (data loss). +- `pb-k3s-reset.yml` (whole cluster or `--limit <node>`). +- Rebuilding the PXE rootfs under running agents (`pxe_rootfs_force_rebuild`). + +Never commit changes or write secrets (k3s token, OAuth secrets, SSH keys) into +tracked files while debugging. + +## Reference + +Full symptom → cause → first-checks matrices for netboot, node join, GPU, +storage, auth, and kubeconfig, plus the reset/escape hatches: +[reference.md](reference.md). diff --git a/skills/troubleshoot-aup-learning-cloud/reference.md b/skills/troubleshoot-aup-learning-cloud/reference.md new file mode 100644 index 00000000..cbdee482 --- /dev/null +++ b/skills/troubleshoot-aup-learning-cloud/reference.md @@ -0,0 +1,75 @@ +# Troubleshoot AUP Learning Cloud — Reference + +Symptom → cause → first-checks matrices by layer, plus the escape hatches. +Method and safety gates are in [SKILL.md](SKILL.md). + +## Source guides + +- Multi-Node + 3-node mini-cluster troubleshooting sections: + <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> +- The deploy skill's reference troubleshooting table (PXE/agent detail). + +## PXE / netboot + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Playbook fails immediately on an assert | A required PXE var is empty | `pxe_controller_ip`, `pxe_subnet`, `pxe_network_interface`, `pxe_dns_servers`, `pxe_k3s_server_ips`, ≥1 SSH key | +| Agent never shows the PXE menu | Firmware boot order, netboot disabled, Proxy-DHCP not reaching client | Firmware, switch port, `systemctl status dnsmasq`, `journalctl -u dnsmasq` | +| Agent gets an IP but can't load boot files | TFTP blocked, missing files, Secure Boot on | `/srv/tftp`, firewall, Secure Boot disabled, dnsmasq logs | +| Agent has no network during netboot | NIC lacks an in-kernel driver in the initramfs | `lspci -nnk`, add the module to `pxe_initramfs_modules`, rebuild rootfs | +| Agent kernel boots but can't mount rootfs | NFS export / subnet ACL / wrong `pxe_controller_ip` | `showmount -e <SERVICE_IP>`, `/etc/exports`, rootfs kernel args | +| Agent waits for the k3s token | Token not published / apache ACL blocks subnet | `curl http://<SERVICE_IP>:8080/k3s/token`, apache config | +| Agent joins once but fails after reboot | Missing local k3s persistence / lost node password | `mount-local-disk`, `/var/lib/rancher/k3s/node-password`, `k3s-agent` logs | + +## Node join (SSH topology) + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Agent node does not join | Hostname resolution, token, or `api_endpoint` mismatch | `systemctl status k3s-agent`, `journalctl -u k3s-agent -n 100`, `/etc/hosts`, `ping <server>` | +| Agent fails to join with a version error | Agent k3s newer than server | Align `pxe_k3s_version`/agent version with server `k3s_version` | + +## GPU scheduling + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| GPU notebook stays Pending | `nodeSelector` mismatch or GPUs exhausted | `kubectl describe pod -n jupyterhub <pod>` (Events), node labels | +| `amd.com/gpu` labels missing | Device plugin / labeller not running | `kubectl get ds -A | grep amdgpu`, `kubectl describe node | grep amd.com/gpu` | +| Label exists but selector doesn't match | Product-name normalized differently per fleet | Compare real `amd.com/gpu.product-name` to `custom.accelerators.*.nodeSelector` | +| GPU pod runs but ROCm errors | Wrong gfx image or missing `HSA_OVERRIDE_GFX_VERSION` (Phoenix) | Image gfx target, accelerator `env` | + +## Storage + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| PVC stays Pending | StorageClass name mismatch or provisioner can't mount | `kubectl get storageclass`, `kubectl get pvc -A`, provisioner logs | +| NFS provisioner crashing | Wrong `nfs.server`/`nfs.path` or export ACL | `kubectl logs -n nfs-provisioner deploy/nfs-subdir-external-provisioner`, `showmount -e <NFS>`, `/etc/exports` | +| Notebook data not persisting | Using `local-path` on multi-node, or wrong storageClass | `hub.db.pvc.storageClassName`, `singleuser.storage.dynamic.storageClass` = `nfs-client` | + +## Authentication / login + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Login page 404s | `custom.authMode: dummy` | Use `auto-login` (single machine) or a real OAuth mode | +| GitHub login loops/fails | OAuth callback URL or org/team config | `hub.config.GitHubOAuthenticator`, `custom.githubOrgName`, callback URL matches host | +| User sees no courses | Team mapping empty for their group | `custom.teams.mapping`, group membership in Admin console | +| Can't reach admin console | Wrong admin user | `custom.adminUser`, `/hub/admin` | + +## kubeconfig / access + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| `permission denied` on `k3s.yaml` | kubeconfig not readable | `export KUBECONFIG=~/.kube/config`, or `--write-kubeconfig-mode=644` in inventory `extra_server_args` | +| `localhost:30890` refused (single-node) | Proxy down / NodePort changed | `kubectl get svc -n jupyterhub`, `kubectl get pods -n jupyterhub` | + +## Escape hatches (gated — confirm with the user) + +```bash +kubectl delete node <name> # clear a stale node object (debug only) +helm history jupyterhub -n jupyterhub # then: helm rollback jupyterhub <rev> +cd deploy/ansible +sudo ansible-playbook playbooks/pb-k3s-reset.yml # whole cluster (DESTRUCTIVE) +sudo ansible-playbook playbooks/pb-k3s-reset.yml --limit <node> # single node +``` + +After a reset, redeploy with deploy-aup-learning-cloud (multi-node) or +install-aup-learning-cloud-single-node. diff --git a/skills/troubleshoot-aup-learning-cloud/skill-card.md b/skills/troubleshoot-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..9d2d71a9 --- /dev/null +++ b/skills/troubleshoot-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Diagnose AUP Learning Cloud failures — netboot, node join, GPU scheduling, storage, and auth — from runtime evidence, for operators. + +## Owner + +AMD Research diff --git a/skills/upgrade-aup-learning-cloud/SKILL.md b/skills/upgrade-aup-learning-cloud/SKILL.md new file mode 100644 index 00000000..b83b6f4a --- /dev/null +++ b/skills/upgrade-aup-learning-cloud/SKILL.md @@ -0,0 +1,81 @@ +--- +name: upgrade-aup-learning-cloud +description: >- + Group: Maintain AUP Learning Cloud. Upgrades a running AUP Learning Cloud + deployment: the JupyterHub Helm + release/chart and values, and the underlying k3s cluster. Use when the user + wants to upgrade, update, bump, or roll out a new version of AUPLC, the Hub + image, the chart, or k3s on an already-installed cluster; mentions helm + upgrade, ./auplc-installer rt upgrade / rt reinstall, pb-k3s-upgrade, + bumping k3s_version / pxe_k3s_version, or applying a values change to a live + Hub. Covers both single-node (installer) and multi-node (Ansible + Helm) + paths, and the safe ordering of cluster vs chart upgrades. Do not use for the + first install (install-/deploy-aup-learning-cloud), for building images + (build-aup-learning-cloud-images), or for routine course edits + (configure-aup-learning-cloud-courses) unless a version bump is involved. +--- + +# Upgrade AUP Learning Cloud + +Move a live deployment to new versions without losing user data: apply chart / +values / image changes, and (separately, more carefully) upgrade k3s. Two +independent axes — **the Hub (Helm)** and **the cluster (k3s)** — upgraded in a +safe order. Commands per topology and the rollback notes are in +**[reference.md](reference.md)**. + +## Prerequisites + +- A running cluster and a checkout of `aup-learning-cloud` matching (or ahead + of) what is deployed. +- `helm` + `kubectl` (multi-node) or `./auplc-installer` (single-node). +- Know what is changing: values only, Hub image tag, chart version, and/or k3s + version. Each has a different, least-disruptive path. + +## Decide the smallest sufficient action + +| Change | Path | +| --- | --- | +| values.yaml / overlay only | `helm upgrade` (multi) or `./auplc-installer rt upgrade` (single) | +| New Hub/notebook image tag | bump `custom.resources.images`, then the same upgrade; single-node image swap: `rt reinstall` | +| Chart bump | `helm upgrade --install` with the new chart | +| k3s version | Ansible `pb-k3s-upgrade.yml` (multi) — separate, gated step | + +Prefer the narrowest path. A values/image change does **not** require a k3s +upgrade. + +## Workflow + +1. **Snapshot state.** `kubectl get nodes -o wide`, `helm list -n jupyterhub`, + `kubectl get pods -n jupyterhub`. Note the current chart + k3s versions and + that nothing is already broken. +2. **Pre-flight the render.** `helm template jupyterhub ./runtime/chart -f + runtime/values.yaml -f <overlay>` must succeed before any apply. +3. **Upgrade the Hub (Helm).** Apply the chart/values change; watch the + rollout. This restarts the Hub pod (brief login blip); running user servers + are generally unaffected. +4. **Upgrade k3s only if needed** (gated — see Safety). Multi-node uses + `pb-k3s-upgrade.yml`. **Keep `pxe_k3s_version` (PXE rootfs) in sync with the + server `k3s_version`** — agents must not be newer than the server. +5. **Verify end to end.** Nodes `Ready`, no `CrashLoopBackOff`/`ImagePullBackOff`, + the Hub loads, an existing user can log in, and a fresh spawn (CPU then GPU) + works. + +## Safety + +Stop and get explicit confirmation before: + +- **A k3s upgrade** — it restarts the kubelet/control plane and can disrupt + running pods; do it in a maintenance window, server before agents. +- **`pb-k3s-reset.yml`** (whole cluster or `--limit <node>`) — destructive. +- **`helm uninstall`** or any change that recreates the Hub DB PVC — data loss. +- **A Hub image tag bump during a live class** — schedule the restart. + +Never commit changes, and never bump `pxe_k3s_version` above the server +`k3s_version`. If a chart upgrade misbehaves, `helm rollback jupyterhub <rev>` +(see reference) before experimenting further. + +## Reference + +Per-topology commands (single-node installer, multi-node Helm, k3s playbooks), +version-pin locations, `helm history`/`rollback`, and troubleshooting: +[reference.md](reference.md). diff --git a/skills/upgrade-aup-learning-cloud/reference.md b/skills/upgrade-aup-learning-cloud/reference.md new file mode 100644 index 00000000..09ae4eb0 --- /dev/null +++ b/skills/upgrade-aup-learning-cloud/reference.md @@ -0,0 +1,102 @@ +# Upgrade AUP Learning Cloud — Reference + +Per-topology upgrade commands, version-pin locations, rollback, and +troubleshooting. Workflow and gates are in [SKILL.md](SKILL.md). + +## Source guides + +- Multi-Node "Apply Later Configuration Changes" + upgrade playbooks: + <https://amdresearch.github.io/aup-learning-cloud/installation/multi-node.html> +- `scripts/helm_upgrade.bash` and `./auplc-installer help` (`rt`, `dev`). + +## Version-pin locations + +| Pin | File | +| --- | --- | +| k3s server version | `deploy/ansible/inventory.yml` → `k3s_version` | +| PXE agent rootfs k3s version | `deploy/ansible/playbooks/pb-pxe-controller.yml` → `pxe_k3s_version` | +| Hub image tag | `custom.resources.images` (values overlay) + `hub.image.tag` | +| Chart | `runtime/chart/Chart.yaml` | + +Keep `pxe_k3s_version == k3s_version`. The deploy skill's +`$DEPLOY_SCRIPTS/validate.py` cross-checks this when invoked with +`--topology pxe-diskless`. From a checkout, resolve that helper with +`REPO_ROOT="$(git rev-parse --show-toplevel)"` and +`DEPLOY_SCRIPTS="$REPO_ROOT/skills/deploy-aup-learning-cloud/scripts"`; from +an installed plugin, define `DEPLOY_SKILL_DIR` as the absolute directory +containing the loaded deploy skill's `SKILL.md`, then set +`DEPLOY_SCRIPTS="$DEPLOY_SKILL_DIR/scripts"`. + +## Hub (Helm) upgrade — values / image / chart + +Single-node (installer): + +```bash +./auplc-installer rt upgrade # values change on a running runtime +./auplc-installer rt reinstall # container image change +./auplc-installer dev upgrade # dev overlay (student=admin, pullPolicy=Never) +``` + +Multi-node / manual: + +```bash +# pre-flight render +helm template jupyterhub ./runtime/chart -f runtime/values.yaml -f <overlay> >/dev/null + +helm upgrade --install jupyterhub ./runtime/chart \ + -n jupyterhub \ + -f runtime/values.yaml -f <overlay> + +kubectl rollout status -n jupyterhub deploy/hub +``` + +(`scripts/helm_upgrade.bash` runs the bare +`helm upgrade jupyterhub runtime/chart -n jupyterhub --values runtime/values.yaml`.) + +## k3s upgrade (multi-node, gated) + +```bash +cd deploy/ansible +# bump k3s_version in inventory.yml first (and pxe_k3s_version to match) +sudo ansible-playbook playbooks/pb-k3s-upgrade.yml +kubectl get nodes -o wide # versions advance, nodes stay Ready +``` + +Upgrade the server first, then agents. For PXE diskless agents, bump +`pxe_k3s_version` and rebuild the rootfs (deploy skill) so netbooted agents +match. + +## Install / refresh Helm itself + +```bash +wget https://get.helm.sh/helm-v3.17.2-linux-amd64.tar.gz -O /tmp/helm.tar.gz +cd /tmp && tar -zxvf helm.tar.gz && sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm +# or: ./auplc-installer install-tools # helm + k9s +``` + +## Rollback + +```bash +helm history jupyterhub -n jupyterhub +helm rollback jupyterhub <REVISION> -n jupyterhub +kubectl rollout status -n jupyterhub deploy/hub +``` + +k3s has no one-command rollback; pin back the version in inventory and re-run +the upgrade playbook, or restore from a node/etcd snapshot if you keep one. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| Hub pod `CrashLoopBackOff` after upgrade | Bad values / incompatible chart | `kubectl logs -n jupyterhub deploy/hub`, `helm rollback` | +| `ImagePullBackOff` after image bump | Tag not pushed or wrong registry | `kubectl describe pod -n jupyterhub`, confirm the pushed tag | +| Agent fails to rejoin after k3s bump | Agent newer than server / rootfs not rebuilt | Align `pxe_k3s_version`, rebuild rootfs, `journalctl -u k3s-agent` | +| Quota CronJobs missing after upgrade | `custom.quota.refreshRules` changed | `kubectl get cronjob -n jupyterhub` | +| PVC lost / Hub DB reset | PVC recreated by an upgrade | Never delete the Hub DB PVC; restore from backup | + +## Out of scope + +First-time install/deploy, image authoring, and HA/external-DB migrations +(treat those as explicit operator projects). This skill upgrades an existing +deployment in place. diff --git a/skills/upgrade-aup-learning-cloud/skill-card.md b/skills/upgrade-aup-learning-cloud/skill-card.md new file mode 100644 index 00000000..dba47fb7 --- /dev/null +++ b/skills/upgrade-aup-learning-cloud/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +Upgrade a running AUP Learning Cloud deployment — the JupyterHub chart/values and the k3s cluster — safely, for operators. + +## Owner + +AMD Research diff --git a/templates/skill-template/SKILL.md b/templates/skill-template/SKILL.md new file mode 100644 index 00000000..60d3b2ce --- /dev/null +++ b/templates/skill-template/SKILL.md @@ -0,0 +1,41 @@ +--- +name: skill-template +description: >- + One- to three-sentence routing description in the third person. State WHAT + this skill produces and WHEN an agent should use it, and list the trigger + words a user is likely to say (product names, file names, commands, error + messages). Keep under 1024 characters. Add negative triggers if the + boundary is easily crossed (e.g. "Do not use for the single-node installer + flow"). Replace this entire block when you copy the template. +--- + +# Skill title + +One paragraph: what this skill does and the single, measurable outcome it +drives toward. + +## Prerequisites + +- List the tools, access, and state the agent must have before starting + (e.g. `kubectl` + `helm` on the operator machine, SSH access, a checkout of + `aup-learning-cloud`). + +## Workflow + +Describe the ordered steps. Use exact commands for fragile operations and +plain instructions for steps with acceptable variation. Keep the body under +500 lines; move long reference material into a sibling `reference.md` and link +to it one level deep. + +1. Step one. +2. Step two. + +## Safety + +Enumerate the risky or irreversible actions that REQUIRE explicit user +confirmation before running. Never commit, push, or write real secrets into +tracked files. + +## Reference + +Link to sibling files such as [reference.md](reference.md). diff --git a/templates/skill-template/reference.md b/templates/skill-template/reference.md new file mode 100644 index 00000000..a2352954 --- /dev/null +++ b/templates/skill-template/reference.md @@ -0,0 +1,19 @@ +# <Skill title> — Reference + +Long-form material that does not belong in `SKILL.md`: full command sequences, +field-by-field config guides, lookup tables, and a troubleshooting table. The +agent loads this only when `SKILL.md` links to it, so keep `SKILL.md` lean and +push the detail here. + +Add a table of contents once this file grows past ~100 lines so the agent can +see the full scope when it previews the top. + +## Section one + +Replace this with real reference content. + +## Troubleshooting + +| Symptom | Likely cause | First checks | +| --- | --- | --- | +| ... | ... | ... | diff --git a/templates/skill-template/skill-card.md b/templates/skill-template/skill-card.md new file mode 100644 index 00000000..36e53cb2 --- /dev/null +++ b/templates/skill-template/skill-card.md @@ -0,0 +1,9 @@ +# Skill Card + +## Description + +<one sentence: what the skill does, for whom> + +## Owner + +AMD Research diff --git a/tests/skills/test_check_skills_version.py b/tests/skills/test_check_skills_version.py new file mode 100644 index 00000000..05aec6f9 --- /dev/null +++ b/tests/skills/test_check_skills_version.py @@ -0,0 +1,69 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Public CLI regression tests for the skill-version checker.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / "scripts" / "check_skills_version.py" +CURSOR_GENERATOR = ROOT / ".github" / "scripts" / "generate_cursor_marketplace.py" + + +def write_json(path: Path, data: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_version_checker_fails_for_a_mismatched_manifest_version(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + checker = repo / "scripts" / CHECKER.name + checker.parent.mkdir(parents=True) + shutil.copy2(CHECKER, checker) + + (repo / "pyproject.toml").write_text('[project]\nname = "fixture"\nversion = "1.2.3"\n', encoding="utf-8") + for relative_path, data in { + ".claude-plugin/marketplace.json": {"metadata": {"version": "1.2.3"}}, + ".cursor-plugin/marketplace.json": {"metadata": {"version": "1.2.3"}}, + ".claude-plugin/plugin.json": {"version": "1.2.3"}, + ".cursor-plugin/plugin.json": {"version": "1.2.3"}, + "plugin-metadata.json": {"version": "0.0.0"}, + }.items(): + write_json(repo / relative_path, data) + + result = subprocess.run( + [sys.executable, str(checker)], + cwd=repo, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 1 + assert "version check failed:" in result.stderr + assert "plugin-metadata.json: version = 0.0.0, expected 1.2.3" in result.stderr + + +def test_marketplace_uses_root_description_and_metadata_version() -> None: + marketplace = json.loads((ROOT / ".claude-plugin" / "marketplace.json").read_text(encoding="utf-8")) + metadata = json.loads((ROOT / "plugin-metadata.json").read_text(encoding="utf-8")) + + assert marketplace["description"] == metadata["description"] + assert "description" not in marketplace["metadata"] + assert marketplace["metadata"]["version"] == metadata["version"] + assert marketplace["plugins"][0]["description"] + + result = subprocess.run( + [sys.executable, str(CURSOR_GENERATOR), "--check"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/skills/test_deploy_scripts.py b/tests/skills/test_deploy_scripts.py new file mode 100644 index 00000000..7e5b6f2c --- /dev/null +++ b/tests/skills/test_deploy_scripts.py @@ -0,0 +1,1009 @@ +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. + +"""Public CLI regression tests for deploy-skill helper scripts.""" + +from __future__ import annotations + +import importlib.util +import io +import json +import os +import subprocess +import sys +from contextlib import redirect_stdout +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +DEPLOY_SCRIPTS = ROOT / "skills" / "deploy-aup-learning-cloud" / "scripts" +VALIDATE = DEPLOY_SCRIPTS / "validate.py" +GEN_CONFIGS = DEPLOY_SCRIPTS / "gen_configs.py" + + +def run_script(script: Path, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(script), *args], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def write_file(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def write_cluster(repo: Path, labels: list[str]) -> Path: + return write_file(repo / "cluster.json", json.dumps({"gpu_product_names": labels})) + + +def load_validate_module(): + spec = importlib.util.spec_from_file_location("deploy_validate", VALIDATE) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def load_generator_module(): + spec = importlib.util.spec_from_file_location("deploy_generator", GEN_CONFIGS) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_ssh_topology_skips_pxe_checks_and_version_sync(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + write_file(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\n") + write_file( + repo / "deploy/ansible/playbooks/pb-pxe-controller.yml", + """pxe_network_interface: "" +pxe_subnet: "" +pxe_controller_ip: "" +pxe_dns_servers: "" +pxe_k3s_server_ips: [] +pxe_rootfs_authorized_keys: [] +pxe_k3s_version: v1.33.0+k3s1 +""", + ) + write_file(repo / "runtime/values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled") + + assert result.returncode == 0, result.stdout + result.stderr + assert "skipped PXE checks for ssh-preinstalled topology" in result.stdout + assert "[FAIL] PXE var" not in result.stdout + assert "version mismatch" not in result.stdout + + +def test_validator_checks_only_effective_active_accelerators_in_values_order(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + phx: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_780M_Graphics + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: + - phx +""", + ) + overlay = write_file( + repo / "runtime/values-strix-halo.yaml", + """custom: + resources: + metadata: + gpu: + acceleratorKeys: + - strix-halo +""", + ) + cluster = write_cluster(repo, ["AMD_Radeon_8060S_Graphics"]) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(overlay), + "--cluster", + str(cluster), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "AMD_Radeon_8060S_Graphics" in result.stdout + assert "AMD_Radeon_780M_Graphics" not in result.stdout + + +def test_validator_retains_selectors_from_partial_accelerator_overlays(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + overlay = write_file( + repo / "runtime/values-overlay.yaml", + """custom: + accelerators: + strix-halo: + displayName: "Renamed Strix Halo" +""", + ) + cluster = write_cluster(repo, ["AMD_Radeon_8060S_Graphics"]) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(overlay), + "--cluster", + str(cluster), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "AMD_Radeon_8060S_Graphics" in result.stdout + + +def test_validator_accepts_quoted_product_label_keys(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + 9070xt: + nodeSelector: + "amd.com/gpu.product-name": "AMD_Radeon_RX_9070_XT" + resources: + metadata: + gpu: + acceleratorKeys: [9070xt] +""", + ) + cluster = write_cluster(repo, ["AMD_Radeon_RX_9070_XT"]) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + "--cluster", + str(cluster), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "AMD_Radeon_RX_9070_XT" in result.stdout + + +def test_validator_rejects_relevant_non_empty_flow_mappings(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: {9070xt: {nodeSelector: {amd.com/gpu.product-name: AMD_Radeon_RX_9070_XT}}} + resources: + metadata: + gpu: {acceleratorKeys: [9070xt]} +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "unsupported non-empty flow-style mapping" in result.stdout + + +def test_validator_rejects_flow_style_custom_resources_wrapper(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: {metadata: {gpu: {acceleratorKeys: [strix-halo]}}} +""", + ) + + result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(values)) + + assert result.returncode == 1 + assert "unsupported non-empty flow-style mapping at custom.resources" in result.stdout + + +def test_validator_rejects_fully_flow_style_custom_wrapper(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: {accelerators: {strix-halo: {nodeSelector: {amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics}}}, resources: {metadata: {gpu: {acceleratorKeys: [strix-halo]}}}} +""", + ) + + result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(values)) + + assert result.returncode == 1 + assert "unsupported non-empty flow-style mapping at custom" in result.stdout + + +def test_validator_rejects_parent_aliases_and_scalar_accelerator_keys(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + alias_values = write_file(repo / "alias.yaml", "defaults: {}\ncustom: *defaults\n") + scalar_keys = write_file( + repo / "scalar-keys.yaml", + """custom: + resources: + metadata: + gpu: + acceleratorKeys: strix-halo +""", + ) + + alias_result = run_script( + VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(alias_values) + ) + scalar_result = run_script( + VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(scalar_keys) + ) + + assert alias_result.returncode == 1 + assert "unsupported YAML syntax at custom" in alias_result.stdout + assert scalar_result.returncode == 1 + assert "acceleratorKeys must be a list" in scalar_result.stdout + + +def test_validator_fails_for_missing_explicit_and_default_values_files(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + repo.mkdir() + explicit_result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(repo / "missing.yaml"), + ) + default_result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled") + + assert explicit_result.returncode == 1 + assert default_result.returncode == 1 + assert "values file not found" in explicit_result.stdout + assert "values file not found" in default_result.stdout + + +def test_validator_rejects_duplicate_pxe_and_inventory_safety_keys(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + write_file(repo / "runtime/values.yaml", "custom:\n resources:\n metadata: {}\n") + write_file(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\nk3s_version: v1.33.0+k3s1\n") + vars_file = write_file( + repo / "pxe-vars.yml", + """pxe_network_interface: enp1s0 +pxe_network_interface: "" +pxe_subnet: 192.168.1.0/24 +pxe_controller_ip: 192.168.1.10 +pxe_dns_servers: 8.8.8.8 +pxe_k3s_server_ips: + - 192.168.1.10 +pxe_rootfs_authorized_keys: + - ssh-ed25519 AAAA test@example +pxe_k3s_version: v1.32.3+k3s1 +pxe_k3s_version: v1.33.0+k3s1 +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--pxe-vars", + str(vars_file), + ) + + assert result.returncode == 1 + assert "duplicate PXE key 'pxe_network_interface'" in result.stdout + assert "duplicate PXE key 'pxe_k3s_version'" in result.stdout + assert "duplicate inventory key 'k3s_version'" in result.stdout + + +def test_validator_fails_empty_supplied_cluster_for_active_accelerators(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + cluster = write_file(repo / "cluster.json", "{}") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + "--cluster", + str(cluster), + ) + + assert result.returncode == 1 + assert "cluster snapshot has no GPU product labels" in result.stdout + + +def test_validator_rejects_unsupported_yaml_syntax_at_relevant_values(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + for index, value in enumerate(("&keys [strix-halo]", "*keys", "!list [strix-halo]", "|")): + overlay = write_file( + repo / f"unsupported-keys-{index}.yaml", + f"""custom: + resources: + metadata: + gpu: + acceleratorKeys: {value} +""", + ) + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(overlay), + ) + assert result.returncode == 1 + assert "unsupported YAML syntax" in result.stdout + + +def test_validator_rejects_unsupported_yaml_syntax_at_product_selector(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: &label AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + + result = run_script(VALIDATE, "--repo", str(repo), "--topology", "ssh-preinstalled", "--values", str(values)) + + assert result.returncode == 1 + assert "unsupported YAML syntax at custom.accelerators.strix-halo.nodeSelector" in result.stdout + + +def test_validator_uses_generated_pxe_vars_file_when_requested(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + spec_path = write_file(repo / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + generated = repo / "generated" + generation = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(generated)) + write_file(repo / "deploy/ansible/inventory.yml", "k3s_version: v1.32.3+k3s1\n") + write_file(repo / "deploy/ansible/playbooks/pb-pxe-controller.yml", "pxe_k3s_version: v1.33.0+k3s1\n") + write_file(repo / "runtime/values.yaml", "custom:\n resources:\n metadata: {}\n") + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "pxe-diskless", + "--pxe-vars", + str(generated / "pb-pxe-controller.vars.yml"), + ) + + assert generation.returncode == 0, generation.stdout + generation.stderr + assert result.returncode == 0, result.stdout + result.stderr + assert "k3s_version == pxe_k3s_version" in result.stdout + + +def test_validator_preserves_explicit_selector_and_accelerator_key_clears(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + selector_clear = write_file( + repo / "selector-clear.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: null +""", + ) + keys_clear = write_file( + repo / "keys-clear.yaml", + """custom: + resources: + metadata: + gpu: + acceleratorKeys: ~ +""", + ) + + selector_result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(selector_clear), + ) + keys_result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(keys_clear), + ) + + assert selector_result.returncode == 1 + assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in selector_result.stdout + assert keys_result.returncode == 0, keys_result.stdout + keys_result.stderr + assert "no acceleratorKeys found" in keys_result.stdout + + +def test_validator_honors_every_supported_explicit_clear_syntax(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + + for index, clear_value in enumerate(('""', "null", "~")): + selector_overlay = write_file( + repo / f"selector-clear-{index}.yaml", + f"""custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: {clear_value} +""", + ) + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(selector_overlay), + ) + assert result.returncode == 1 + assert "has no amd.com/gpu.product-name nodeSelector" in result.stdout + + for index, clear_value in enumerate(("null", "~", "[]")): + keys_overlay = write_file( + repo / f"keys-clear-{index}.yaml", + f"""custom: + resources: + metadata: + gpu: + acceleratorKeys: {clear_value} +""", + ) + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base), + "--values", + str(keys_overlay), + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "no acceleratorKeys found" in result.stdout + + +def test_validator_main_resets_report_state_between_invocations(tmp_path: Path) -> None: + module = load_validate_module() + failed_repo = tmp_path / "failed" + success_repo = tmp_path / "success" + failed_values = write_file( + failed_repo / "runtime/values.yaml", + """custom: + accelerators: {} + resources: + metadata: + gpu: + acceleratorKeys: [missing] +""", + ) + success_values = write_file(success_repo / "runtime/values.yaml", "custom:\n resources:\n metadata: {}\n") + + with redirect_stdout(io.StringIO()): + first = module.main( + ["--repo", str(failed_repo), "--topology", "ssh-preinstalled", "--values", str(failed_values)] + ) + second = module.main( + ["--repo", str(success_repo), "--topology", "ssh-preinstalled", "--values", str(success_values)] + ) + + assert first == 1 + assert second == 0 + + +def test_validator_requires_product_labels_under_active_accelerator_node_selectors(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + env: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout + + +def test_validator_ignores_accelerators_and_metadata_outside_custom_resources(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +other: + accelerators: + typo-gpu: + nodeSelector: + amd.com/gpu.product-name: AMD_Typo_GPU + metadata: + gpu: + acceleratorKeys: [typo-gpu] +""", + ) + cluster = write_cluster(repo, ["AMD_Radeon_8060S_Graphics"]) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + "--cluster", + str(cluster), + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "typo-gpu" not in result.stdout + + +def test_validator_fails_when_an_active_accelerator_key_is_missing(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: {} + resources: + metadata: + gpu: + acceleratorKeys: + - typo-gpu +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "active accelerator 'typo-gpu' is not defined under custom.accelerators" in result.stdout + + +def test_validator_fails_when_an_active_accelerator_has_no_product_selector(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: {} + resources: + metadata: + gpu: + acceleratorKeys: + - strix-halo +""", + ) + + result = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(values), + ) + + assert result.returncode == 1 + assert "active accelerator 'strix-halo' has no amd.com/gpu.product-name nodeSelector" in result.stdout + + +def test_generator_rejects_unknown_accelerator_keys_before_writing_artifacts(tmp_path: Path) -> None: + spec = write_file( + tmp_path / "spec.json", + json.dumps( + { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "accelerators": {"typo-gpu": {"product_name": "AMD_Typo_GPU"}}, + } + ), + ) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "unsupported accelerator key 'typo-gpu'" in result.stderr + assert not out_dir.exists() + + +def test_generator_retains_known_accelerator_product_name_overrides(tmp_path: Path) -> None: + spec = write_file( + tmp_path / "spec.json", + json.dumps( + { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "accelerators": {"strix-halo": {"product_name": "AMD_Custom_8060S"}}, + } + ), + ) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stdout + result.stderr + values = (out_dir / "values-basic-example.yaml").read_text(encoding="utf-8") + assert 'amd.com/gpu.product-name: "AMD_Custom_8060S"' in values + + +def test_generator_rejects_a_non_mapping_accelerators_field_before_writing_artifacts(tmp_path: Path) -> None: + spec = write_file( + tmp_path / "spec.json", + json.dumps( + { + "topology": "ssh-preinstalled", + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + "accelerators": [], + } + ), + ) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "spec.accelerators must be a mapping" in result.stderr + assert not out_dir.exists() + + +def generator_spec(topology: str = "ssh-preinstalled", accelerators: object | None = None) -> dict[str, object]: + spec: dict[str, object] = { + "topology": topology, + "k3s_version": "v1.32.3+k3s1", + "server": {"name": "server", "ip": "192.168.1.10"}, + } + if accelerators is not None: + spec["accelerators"] = accelerators + if topology == "pxe-diskless": + spec["network"] = {"interface": "enp1s0", "subnet": "192.168.1.0/24"} + spec["pxe"] = {"authorized_keys": ["ssh-ed25519 AAAA test@example"]} + return spec + + +def test_generator_validates_all_pxe_requirements_before_writing(tmp_path: Path) -> None: + spec = generator_spec("pxe-diskless") + spec["pxe"] = {"authorized_keys": []} + spec_path = write_file(tmp_path / "spec.json", json.dumps(spec)) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "pxe.authorized_keys must contain at least one SSH public key" in result.stderr + assert not out_dir.exists() + + +def test_generator_rejects_non_mapping_known_accelerator_config_before_writing(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec(accelerators={"9070xt": []}))) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "accelerators.9070xt must be a mapping" in result.stderr + assert not out_dir.exists() + + +def test_generator_preflights_second_destination_collisions_before_writing(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + out_dir = tmp_path / "generated" + write_file(out_dir / "pb-pxe-controller.vars.yml", "existing\n") + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "refusing to overwrite existing" in result.stderr + assert not (out_dir / "inventory.yml").exists() + + +def test_generator_preflights_third_destination_collisions_before_writing(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + out_dir = tmp_path / "generated" + write_file(out_dir / "values-basic-example.yaml", "existing\n") + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "refusing to overwrite existing" in result.stderr + assert not (out_dir / "inventory.yml").exists() + assert not (out_dir / "pb-pxe-controller.vars.yml").exists() + + +def test_generator_refuses_dangling_symlink_destinations_without_partial_artifacts(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec())) + out_dir = tmp_path / "generated" + dangling_target = tmp_path / "missing-target" + out_dir.mkdir() + (out_dir / "inventory.yml").symlink_to(dangling_target) + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 1 + assert "refusing to overwrite existing" in result.stderr + assert (out_dir / "inventory.yml").is_symlink() + assert not dangling_target.exists() + assert not (out_dir / "values-basic-example.yaml").exists() + + +def test_generator_publishes_secret_and_public_artifacts_with_expected_modes(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec("pxe-diskless"))) + out_dir = tmp_path / "generated" + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir)) + + assert result.returncode == 0, result.stdout + result.stderr + assert os.stat(out_dir / "inventory.yml").st_mode & 0o777 == 0o600 + assert os.stat(out_dir / "pb-pxe-controller.vars.yml").st_mode & 0o777 == 0o600 + assert os.stat(out_dir / "values-basic-example.yaml").st_mode & 0o777 == 0o644 + + +def test_generator_force_replaces_symlink_entry_without_following_target(tmp_path: Path) -> None: + spec_path = write_file(tmp_path / "spec.json", json.dumps(generator_spec())) + out_dir = tmp_path / "generated" + target = write_file(tmp_path / "target-values.yaml", "keep-this-target\n") + out_dir.mkdir() + (out_dir / "values-basic-example.yaml").symlink_to(target) + + result = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(out_dir), "--force") + + published = out_dir / "values-basic-example.yaml" + assert result.returncode == 0, result.stdout + result.stderr + assert not published.is_symlink() + assert target.read_text(encoding="utf-8") == "keep-this-target\n" + assert "Helm overlay generated" in published.read_text(encoding="utf-8") + + +def test_generator_force_failure_restores_all_original_destination_types( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = load_generator_module() + inventory = write_file(tmp_path / "inventory.yml", "old inventory\n") + pxe_vars = tmp_path / "pb-pxe-controller.vars.yml" + pxe_vars.mkdir() + write_file(pxe_vars / "legacy", "old directory\n") + values_target = write_file(tmp_path / "values-target.yml", "old symlink target\n") + values = tmp_path / "values-basic-example.yaml" + values.symlink_to(values_target) + artifacts = [ + (inventory, "new inventory\n", 0o600, True), + (pxe_vars, "new pxe vars\n", 0o600, False), + (values, "new values\n", 0o644, False), + ] + original_replace = module.os.replace + + def fail_late_replace(source, destination): + if Path(destination).name == "values-basic-example.yaml" and ".backup." not in Path(source).name: + raise OSError("injected late publish failure") + return original_replace(source, destination) + + monkeypatch.setattr(module.os, "replace", fail_late_replace) + + with pytest.raises(SystemExit): + module.publish_artifacts(artifacts, force=True) + + assert inventory.read_text(encoding="utf-8") == "old inventory\n" + assert pxe_vars.is_dir() + assert (pxe_vars / "legacy").read_text(encoding="utf-8") == "old directory\n" + assert values.is_symlink() + assert values_target.read_text(encoding="utf-8") == "old symlink target\n" + + +def test_generated_overlay_activates_selected_accelerators_for_validation(tmp_path: Path) -> None: + repo = tmp_path / "checkout" + base_values = write_file( + repo / "runtime/values.yaml", + """custom: + accelerators: + strix-halo: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_8060S_Graphics + 9070xt: + nodeSelector: + amd.com/gpu.product-name: AMD_Radeon_RX_9070_XT + resources: + metadata: + gpu: + acceleratorKeys: [strix-halo] +""", + ) + spec_path = write_file( + repo / "spec.json", + json.dumps(generator_spec(accelerators={"9070xt": {"product_name": "AMD_Radeon_RX_9070_XT"}})), + ) + generated = repo / "generated" + generation = run_script(GEN_CONFIGS, "--spec", str(spec_path), "--out-dir", str(generated)) + cluster = write_cluster(repo, ["AMD_Radeon_RX_9070_XT"]) + + validation = run_script( + VALIDATE, + "--repo", + str(repo), + "--topology", + "ssh-preinstalled", + "--values", + str(base_values), + "--values", + str(generated / "values-basic-example.yaml"), + "--cluster", + str(cluster), + ) + + assert generation.returncode == 0, generation.stdout + generation.stderr + assert validation.returncode == 0, validation.stdout + validation.stderr + assert "AMD_Radeon_RX_9070_XT" in validation.stdout + assert "AMD_Radeon_8060S_Graphics" not in validation.stdout + + +def test_checkout_root_helper_path_is_a_runnable_public_cli() -> None: + result = run_script(GEN_CONFIGS, "--print-schema", cwd=ROOT) + + assert result.returncode == 0, result.stdout + result.stderr + assert '"topology": "pxe-diskless | ssh-preinstalled"' in result.stdout