diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bf279d8..5afd360d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,7 @@ name: Continous Integration on: pull_request: - branches: [ main ] - paths-ignore: - - 'docs/**' + branches: [ main, feat/interfaces ] push: branches: [ main ] paths-ignore: @@ -41,84 +39,8 @@ jobs: tests: needs: pre-commit-checks - strategy: - fail-fast: false - matrix: - python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - timeout-minutes: 10 - env: - CANFAR_BASEURL: ${{ secrets.CANFAR_BASEURL }} - CANFAR_USERNAME: ${{ secrets.CANFAR_USERNAME }} - CANFAR_PASSWORD: ${{ secrets.CANFAR_PASSWORD }} + uses: ./.github/workflows/reusable-tests.yml + with: + full-suite: false + secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - steps: - - name: Harden Runner - timeout-minutes: 10 - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 - with: - egress-policy: audit - - - - name: Setup code repository - timeout-minutes: 10 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 1 - - - name: Setup uv - timeout-minutes: 10 - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - enable-cache: true - - - name: Setup Python ${{ matrix.python-version }} - timeout-minutes: 10 - run: | - uv python install ${{ matrix.python-version }} - uv venv --python ${{ matrix.python-version }} - uv sync --all-extras --dev - - - name: Login to CANFAR - timeout-minutes: 10 - if: ${{ env.CANFAR_BASEURL != '' && env.CANFAR_USERNAME != '' && env.CANFAR_PASSWORD != '' }} - run: | - set -euo pipefail - printf "machine %s\n login %s\n password %s\n" "${CANFAR_BASEURL}" "${CANFAR_USERNAME}" "${CANFAR_PASSWORD}" > ~/.netrc - uv run cadc-get-cert --days-valid 1 --netrc-file ~/.netrc - rm ~/.netrc - test -f "${HOME}/.ssl/cadcproxy.pem" - - - name: Run tests - timeout-minutes: 10 - run: | - set -euo pipefail - if [ -n "${CANFAR_BASEURL}" ] && [ -n "${CANFAR_USERNAME}" ] && [ -n "${CANFAR_PASSWORD}" ]; then - uv run pytest tests -m "not slow" --cov --cov-report=xml --junitxml=junit.xml -o junit_family=legacy - else - uv run pytest tests -m "not slow" --cov --cov-report=xml --junitxml=junit.xml -o junit_family=legacy - fi - - - name: Remove CANFAR Certificate - timeout-minutes: 10 - if: always() - run: | - rm -rf ~/.ssl/ - - - name: Upload coverage to Codecov - timeout-minutes: 10 - if: ${{ !cancelled() && env.CODECOV_TOKEN != '' }} - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - with: - fail_ci_if_error: true # Fail the CI if an error occurs during the upload - token: ${{ env.CODECOV_TOKEN }} - flags: ${{ matrix.python-version }} - verbose: true # optional (default = false) - - - name: Upload test results to Codecov - timeout-minutes: 10 - if: ${{ !cancelled() && env.CODECOV_TOKEN != '' }} - uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1.2.1 - with: - token: ${{ env.CODECOV_TOKEN }} diff --git a/.github/workflows/edge.yml b/.github/workflows/edge.yml index 6ef8be4e..ce9aa748 100644 --- a/.github/workflows/edge.yml +++ b/.github/workflows/edge.yml @@ -4,80 +4,18 @@ on: repository_dispatch: types: [edge-build] -env: - GHCR_REGISTRY: ghcr.io - IMAGE_NAME: opencadc/canfar - IMAGE_TAG: edge - permissions: contents: read jobs: edge-build: - runs-on: ubuntu-latest - timeout-minutes: 10 + uses: ./.github/workflows/reusable-container.yml permissions: packages: write attestations: write id-token: write - steps: - - name: Harden Runner - timeout-minutes: 10 - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 - with: - egress-policy: audit - - - - name: Client Payload - timeout-minutes: 10 - run: | - echo "Client Payload: ${{ toJson(github.event.client_payload) }}" - - - name: Checkout Code - timeout-minutes: 10 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Setup Docker Buildx - timeout-minutes: 10 - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - with: - install: true - - - name: Perform GHCR Login - timeout-minutes: 10 - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build & Push Docker Image - timeout-minutes: 10 - id: build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - target: production - file: Dockerfile - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - provenance: mode=max - sbom: true - push: true - labels: | - org.opencontainers.image.title=canfar - org.opencontainers.image.version=edge - org.opencontainers.image.description='Python Client for CANFAR Science Platform' - org.opencontainers.image.licenses=AGPL-3.0 - org.opencontainers.image.source=https://github.com/opencadc/canfar - tags: | - ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - - - name: Attest GHCR Container Image - timeout-minutes: 10 - uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 - with: - subject-name: ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }} - subject-digest: ${{ steps.build.outputs.digest }} - push-to-registry: true + with: + client-payload: ${{ toJson(github.event.client_payload) }} + image-description: Python Client for CANFAR Science Platform + image-tags: ghcr.io/opencadc/canfar:edge + image-version: edge diff --git a/.github/workflows/full-tests.yml b/.github/workflows/full-tests.yml new file mode 100644 index 00000000..c35457fa --- /dev/null +++ b/.github/workflows/full-tests.yml @@ -0,0 +1,23 @@ +name: Full Tests + +on: + pull_request_target: + types: [closed] + branches: [ main ] + paths: + - 'canfar/**' + - 'tests/**' + +permissions: read-all + +jobs: + tests: + if: ${{ github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' }} + uses: ./.github/workflows/reusable-tests.yml + with: + full-suite: true + secrets: + CANFAR_BASEURL: ${{ secrets.CANFAR_BASEURL }} + CANFAR_USERNAME: ${{ secrets.CANFAR_USERNAME }} + CANFAR_PASSWORD: ${{ secrets.CANFAR_PASSWORD }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2146330e..c503c308 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,84 +4,21 @@ on: repository_dispatch: types: [release-build] -env: - GHCR_REGISTRY: ghcr.io - IMAGE_NAME: opencadc/canfar - IMAGE_TAG_LATEST: latest - IMAGE_TAG_RELEASE: ${{ github.event.client_payload.tag_name }} - permissions: contents: read jobs: release-build: - runs-on: ubuntu-latest - timeout-minutes: 10 + uses: ./.github/workflows/reusable-container.yml permissions: packages: write # Upload package to ghcr.io attestations: write # Attest the build provenance id-token: write # Genereate OIDC token for attestations - steps: - - name: Harden Runner - timeout-minutes: 10 - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 - with: - egress-policy: audit - - - - name: Client Payload - timeout-minutes: 10 - run: | - echo "Client Payload: ${{ toJson(github.event.client_payload) }}" - - - name: Checkout Code - timeout-minutes: 10 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.client_payload.tag_name }} - - - name: Setup Docker Buildx - timeout-minutes: 10 - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - with: - install: true - - - name: Perform GHCR Login - timeout-minutes: 10 - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build & Push Docker Image - timeout-minutes: 10 - id: build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - target: production - file: Dockerfile - platforms: linux/amd64,linux/arm64 - cache-from: type=gha - cache-to: type=gha,mode=max - provenance: mode=max - sbom: true - push: true - labels: | - org.opencontainers.image.title=canfar - org.opencontainers.image.version=${{ env.IMAGE_TAG_RELEASE }} - org.opencontainers.image.description='Python Client for CANFAR Science Portal' - org.opencontainers.image.licenses=AGPL-3.0 - org.opencontainers.image.source=https://github.com/opencadc/canfar - tags: | - ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG_RELEASE }} - ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG_LATEST }} - - - name: Attest GHCR Container Image - timeout-minutes: 10 - uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 - with: - subject-name: ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }} - subject-digest: ${{ steps.build.outputs.digest }} - push-to-registry: true + with: + client-payload: ${{ toJson(github.event.client_payload) }} + checkout-ref: ${{ github.event.client_payload.tag_name }} + image-description: Python Client for CANFAR Science Portal + image-tags: | + ghcr.io/opencadc/canfar:${{ github.event.client_payload.tag_name }} + ghcr.io/opencadc/canfar:latest + image-version: ${{ github.event.client_payload.tag_name }} diff --git a/.github/workflows/reusable-container.yml b/.github/workflows/reusable-container.yml new file mode 100644 index 00000000..18b31505 --- /dev/null +++ b/.github/workflows/reusable-container.yml @@ -0,0 +1,101 @@ +name: Reusable Container Build + +on: + workflow_call: + inputs: + checkout-ref: + type: string + required: false + default: '' + client-payload: + type: string + required: true + image-description: + type: string + required: true + image-tags: + type: string + required: true + image-version: + type: string + required: true + +permissions: + contents: read + +env: + GHCR_REGISTRY: ghcr.io + IMAGE_NAME: opencadc/canfar + +jobs: + container-build: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + packages: write + attestations: write + id-token: write + steps: + - name: Harden Runner + timeout-minutes: 10 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - + name: Client Payload + timeout-minutes: 10 + env: + CLIENT_PAYLOAD: ${{ inputs.client-payload }} + run: | + echo "Client Payload: ${CLIENT_PAYLOAD}" + - + name: Checkout Code + timeout-minutes: 10 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-ref }} + - + name: Setup Docker Buildx + timeout-minutes: 10 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + with: + install: true + - + name: Perform GHCR Login + timeout-minutes: 10 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Build & Push Docker Image + timeout-minutes: 10 + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + target: production + file: Dockerfile + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true + push: true + labels: | + org.opencontainers.image.title=canfar + org.opencontainers.image.version=${{ inputs.image-version }} + org.opencontainers.image.description='${{ inputs.image-description }}' + org.opencontainers.image.licenses=AGPL-3.0 + org.opencontainers.image.source=https://github.com/opencadc/canfar + tags: ${{ inputs.image-tags }} + - + name: Attest GHCR Container Image + timeout-minutes: 10 + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-name: ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml new file mode 100644 index 00000000..88870e14 --- /dev/null +++ b/.github/workflows/reusable-tests.yml @@ -0,0 +1,117 @@ +name: Reusable Python Tests + +on: + workflow_call: + inputs: + full-suite: + type: boolean + required: false + default: false + secrets: + CANFAR_BASEURL: + required: false + CANFAR_USERNAME: + required: false + CANFAR_PASSWORD: + required: false + CODECOV_TOKEN: + required: false + +permissions: read-all + +jobs: + tests: + strategy: + fail-fast: false + matrix: + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + CANFAR_BASEURL: ${{ secrets.CANFAR_BASEURL }} + CANFAR_USERNAME: ${{ secrets.CANFAR_USERNAME }} + CANFAR_PASSWORD: ${{ secrets.CANFAR_PASSWORD }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + steps: + - name: Harden Runner + timeout-minutes: 10 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - + name: Setup code repository + timeout-minutes: 10 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + - + name: Setup uv + timeout-minutes: 10 + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + - + name: Setup Python ${{ matrix.python-version }} + timeout-minutes: 10 + run: | + uv python install ${{ matrix.python-version }} + uv venv --python ${{ matrix.python-version }} + uv sync --all-extras --dev + - + name: Verify CANFAR credentials + timeout-minutes: 10 + if: ${{ inputs.full-suite }} + run: | + set -euo pipefail + if [ -z "${CANFAR_BASEURL}" ] || [ -z "${CANFAR_USERNAME}" ] || [ -z "${CANFAR_PASSWORD}" ]; then + echo "CANFAR credentials are required for the full test suite" >&2 + exit 1 + fi + - + name: Login to CANFAR + timeout-minutes: 10 + if: ${{ inputs.full-suite }} + run: | + set -euo pipefail + printf "machine %s\n login %s\n password %s\n" "${CANFAR_BASEURL}" "${CANFAR_USERNAME}" "${CANFAR_PASSWORD}" > ~/.netrc + uv run cadc-get-cert --days-valid 1 --netrc-file ~/.netrc + rm ~/.netrc + test -f "${HOME}/.ssl/cadcproxy.pem" + - + name: Run fast test suite + timeout-minutes: 10 + if: ${{ !inputs.full-suite }} + run: | + set -euo pipefail + uv run pytest -m "not slow" tests --cov --cov-report=xml --junitxml=junit.xml -o junit_family=legacy + - + name: Run full test suite + timeout-minutes: 10 + if: ${{ inputs.full-suite }} + run: | + set -euo pipefail + CANFAR_TEST_HOME="$HOME" uv run pytest --cov --cov-report=xml --junitxml=junit.xml -o junit_family=legacy + - + name: Remove CANFAR Certificate + timeout-minutes: 10 + if: ${{ always() && inputs.full-suite }} + run: | + rm -rf ~/.ssl/ + - + name: Upload coverage to Codecov + timeout-minutes: 10 + if: ${{ !cancelled() && env.CODECOV_TOKEN != '' }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + fail_ci_if_error: true # Fail CI if an error occurs during the upload + token: ${{ env.CODECOV_TOKEN }} + flags: ${{ matrix.python-version }} + verbose: true # optional (default = false) + - + name: Upload test results to Codecov + timeout-minutes: 10 + if: ${{ !cancelled() && env.CODECOV_TOKEN != '' }} + uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1.2.1 + with: + token: ${{ env.CODECOV_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 842122c3..4e9c3bb4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,7 @@ repos: exclude: (^|/)uv\.lock$|^tests/ # Python code formatting and linting with Ruff - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.16 + rev: v0.16.0 hooks: # Linter - id: ruff diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 564bf265..59a832f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,16 @@ uv run pytest Some tests in the Skaha test suite are marked as "slow" because they involve network operations, waiting for session states, or other time-consuming operations. These tests can take several minutes to complete. -**Run all tests (including slow ones):** +**Run the deterministic pull-request suite:** +```bash +uv run pytest -m "not slow" tests +``` + +Pull requests targeting `main` or `feat/interfaces` use this non-slow suite in CI. +The unfiltered suite runs in CI only after a code or test change is merged into +`main`; documentation- and workflow-only merges do not trigger it. + +**Run all tests (including slow ones) locally:** ```bash uv run pytest ``` @@ -101,7 +110,7 @@ The slow tests are primarily integration tests that interact with the CANFAR Sci - Authentication timeout tests - Session statistics tests -For rapid development and testing, it's recommended to use `-m "not slow"` to skip these time-consuming tests during your development cycle, and run the full test suite before submitting your pull request. +For rapid development and testing, use `-m "not slow"` to skip these time-consuming tests during your development cycle. Run the full suite locally when you have valid CANFAR credentials and a certificate. ### 6. Commit Your Changes diff --git a/README.md b/README.md index f3bbc034..b2f42dda 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ pip install canfar --upgrade canfar login cadc canfar create notebook skaha/astroml:26.04 -canfar ps --json +canfar ps -o json # assumes jq is installed -canfar open $(canfar ps --json | jq -r ".[0].id") +canfar open $(canfar ps -o json | jq -r ".[0].id") ``` ```python diff --git a/canfar/__init__.py b/canfar/__init__.py index e18be464..7aa56feb 100644 --- a/canfar/__init__.py +++ b/canfar/__init__.py @@ -14,7 +14,7 @@ CERT_PATH: Path = Path.home() / ".ssl" / "cadcproxy.pem" from . import authentication, server # noqa: E402 -from .authentication import login # noqa: E402 +from .authentication import alogin, login # noqa: E402 # Kept in sync with pyproject.toml by release-please # DO NOT EDIT MANUALLY @@ -24,6 +24,7 @@ "CONFIG_DIR", "CONFIG_PATH", "__version__", + "alogin", "authentication", "configure_logging", "get_logger", diff --git a/canfar/_server_discovery.py b/canfar/_server_discovery.py new file mode 100644 index 00000000..2343e167 --- /dev/null +++ b/canfar/_server_discovery.py @@ -0,0 +1,674 @@ +"""Private registry, transport, and VOSI implementation for Server discovery.""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING +from xml.etree.ElementTree import ParseError + +import httpx +from defusedxml.common import DefusedXmlException +from pydantic import AnyHttpUrl, AnyUrl, ValidationError + +from canfar.auth.x509 import CertificateError +from canfar.errors import ErrorCode, StructuredError +from canfar.exceptions.context import AuthContextError, AuthExpiredError +from canfar.hooks.httpx.auth import AuthenticationError as HTTPAuthenticationError +from canfar.idp import get_idp +from canfar.models.config import Configuration +from canfar.models.http import ( + DEFAULT_SERVER_CORES, + DEFAULT_SERVER_GPUS, + DEFAULT_SERVER_RAM_GB, + Server, + VOSpaceService, +) +from canfar.models.registry import Server as RegistryResource +from canfar.utils import registry, vosi +from canfar.utils.registry import RegistryEvidenceError + +if TYPE_CHECKING: + from pathlib import Path + +log = logging.getLogger(__name__) + +_STORAGE_RESOURCE_UNSET = object() + + +class ServerDiscoveryError(RuntimeError): + """Raised when server discovery fails for an Identity Provider.""" + + def __init__( + self, + message: str, + *, + code: ErrorCode = ErrorCode.SERVER_DISCOVERY_FAILED, + ) -> None: + super().__init__(message) + self.code = code + self.structured = StructuredError(code=code, message=message) + + +class ServerFetchError(RuntimeError): + """Raised when server fetch or validation fails.""" + + def __init__( + self, + message: str, + *, + code: ErrorCode = ErrorCode.TRANSPORT_FAILURE, + ) -> None: + super().__init__(message) + self.code = code + self.structured = StructuredError(code=code, message=message) + + +def _merge_storage( + known: dict[str, VOSpaceService], + found: dict[str, VOSpaceService], +) -> dict[str, VOSpaceService]: + """Merge discovered VOSpace Services into known ones, keyed by IVOA URI. + + Discovery names a new Service after its Server Name, so an existing Service + configured under its registry leaf (``arc``) is refreshed in place instead + of being duplicated under the Server Name on every rediscovery. + """ + merged = dict(known) + names = {str(service.uri): name for name, service in known.items()} + for name, service in found.items(): + merged[names.get(str(service.uri), name)] = service + return merged + + +async def _discover_for_idp( + idp: str, + *, + config: Configuration | None = None, + dev: bool = False, + timeout: int = 2, +) -> list[Server]: + """Discover active servers for a single Identity Provider.""" + evidence = await registry.evidence( + idp, + dev=dev, + timeout=timeout, + check_platforms=True, + ) + if not evidence.available: + errors = "; ".join(evidence.errors) + msg = f"Failed to discover servers for IDP '{idp}': {errors}" + raise ServerDiscoveryError(msg) + + endpoints = [ + resource + for resource in evidence.resources + if resource.uri.endswith("/skaha") and resource.status == 200 + ] + if not endpoints: + return [] + + storage_resources = [ + resource + for resource in evidence.resources + if resource.uri.endswith(f"/{evidence.leaf}") + ] + workers = await registry.workers( + config, + idp, + endpoint=endpoints[0], + count=len(endpoints), + ) + if workers is None: + return [_registry_resource_to_server(endpoint, idp) for endpoint in endpoints] + + return list( + await asyncio.gather( + *( + asyncio.to_thread( + _discovered_to_server, + endpoint, + idp, + config=worker_config, + token=workers.token, + certificate=workers.certificate, + timeout=timeout, + storage_resource=_select_storage( + endpoint, + storage_resources, + strict=False, + ), + ) + for endpoint, worker_config in zip( + endpoints, + workers.configs, + strict=True, + ) + ) + ) + ) + + +def _host_slug(uri: AnyUrl) -> str | None: + """Return a Server Name slug derived from a URI host (dots -> hyphens).""" + if uri.host is None: + return None + return uri.host.replace(".", "-") + + +def _select_storage( + endpoint: RegistryResource, + resources: list[RegistryResource], + *, + strict: bool, +) -> RegistryResource | None: + """Map private registry ambiguity to the public server fetch error.""" + try: + return registry.select_storage(endpoint, resources, strict=strict) + except RegistryEvidenceError as exc: + raise ServerFetchError(str(exc)) from exc + + +async def _discover_storage( + server: Server, + idp: str, + *, + dev: bool, + timeout: int, +) -> RegistryResource | None: + """Return fresh registry evidence for a server's primary VOSpace service.""" + try: + return await registry.discover_storage( + str(server.uri) if server.uri is not None else None, + str(server.url) if server.url is not None else None, + server.name, + idp, + dev=dev, + timeout=timeout, + ) + except RegistryEvidenceError as exc: + raise ServerFetchError(str(exc)) from exc + + +def _configured_storage_resource(server: Server) -> RegistryResource | None: + """Convert the persisted primary VOSpace service to inspection evidence.""" + if server.name is None: + return None + service = server.storage.get(server.name) + if service is None: + return None + return RegistryResource( + registry="configuration", + uri=str(service.uri), + url=str(service.url), + ) + + +def _registry_resource_to_server(endpoint: RegistryResource, idp: str) -> Server: + """Convert registry endpoint identity without performing capability I/O.""" + uri = AnyUrl(endpoint.uri) + return Server( + idp=idp, + name=endpoint.name or _host_slug(uri), + uri=uri, + url=AnyHttpUrl(endpoint.url), + ) + + +def _discovered_to_server( + endpoint: RegistryResource, + idp: str, + *, + config: Configuration | None = None, + token: str | None = None, + certificate: Path | None = None, + timeout: int = 2, + storage_resource: RegistryResource | None = None, +) -> Server: + """Convert a registry discovery record to a persisted HTTP server model.""" + server = _registry_resource_to_server(endpoint, idp) + return enrich( + server, + config=config, + token=token, + certificate=certificate, + strict=False, + timeout=timeout, + storage_resource=storage_resource, + ) + + +def enrich( + server: Server, + *, + config: Configuration | None = None, + authentication_idp: str | None = None, + token: str | None = None, + certificate: Path | None = None, + strict: bool = True, + timeout: int = 2, + storage_resource: RegistryResource | None | object = _STORAGE_RESOURCE_UNSET, # noqa: RUF036 +) -> Server: + """Return a validated Server enriched from its VOSI capabilities. + + Args: + server: Server record to enrich. + config: Configuration whose Authentication Record should authorize the + capability request. The transient selector does not change or persist + Authentication or Server Selection. + authentication_idp: Optional Authentication Record selector. Defaults to + the Server IDP, then the active Authentication. + token: Optional runtime bearer token for capability requests. + certificate: Optional runtime certificate for capability requests. + strict: When ``False``, keep usable registry and existing storage data + when session or storage capabilities cannot be retrieved or parsed. + Other successful enrichment may still be returned, so the result can + be partial. Discovery uses non-strict mode so one malformed endpoint + does not abort listing for an IDP. + timeout: HTTP timeout in seconds for VOSI capabilities requests. + storage_resource: Retained same-namespace VOSpace registry record. Passing + ``None`` records that the preferred resource was absent; omitting the + argument leaves storage outside this inspection. + + Returns: + Server: Copy with version and auth modes populated when discoverable. + + Raises: + ServerFetchError: If ``strict`` is ``True`` and capabilities cannot + be retrieved, parsed, or contain no session capabilities. + """ + base_config = config or Configuration() # ty: ignore[missing-argument] + active_idp = authentication_idp or server.idp or base_config.active.authentication + if storage_resource is not _STORAGE_RESOURCE_UNSET: + server = _enrich_storage( + server, + storage_resource=( + storage_resource + if isinstance(storage_resource, RegistryResource) + else None + ), + config=base_config, + authentication_idp=active_idp, + token=token, + certificate=certificate, + strict=strict, + timeout=timeout, + ) + if server.url is None: + msg = "Server URL is required to inspect capabilities." + raise ServerFetchError(msg) + try: + capabilities = vosi.capabilities( + xml=_fetch_capabilities( + server.url, + config=base_config, + authentication_idp=active_idp, + token=token, + certificate=certificate, + timeout=timeout, + ) + ) + except ( + httpx.HTTPError, + OSError, + AuthContextError, + AuthExpiredError, + CertificateError, + HTTPAuthenticationError, + ParseError, + DefusedXmlException, + ) as exc: + return _keep_or_raise( + server, + strict=strict, + error=f"Failed to fetch capabilities for {server.url}: {exc}", + cause=exc, + debug="Skipping capability enrichment for %s during discovery: %s", + args=(server.url, exc), + ) + + primary = next( + ( + capability + for capability in capabilities + if capability.get("version") and capability.get("auth_modes") + ), + None, + ) + if primary is None: + return _keep_or_raise( + server, + strict=strict, + error=f"No complete session capabilities found for {server.url}.", + debug=( + "No complete session capabilities found for %s during discovery; " + "keeping registry metadata only." + ), + args=(server.url,), + ) + + try: + return Server.model_validate( + { + **server.model_dump(mode="python"), + "url": primary["baseurl"], + "version": primary["version"], + "auths": primary["auth_modes"], + } + ) + except ValidationError as exc: + return _keep_or_raise( + server, + strict=strict, + error=f"Invalid capabilities for {server.url}: {exc}", + cause=exc, + debug=( + "Ignoring invalid capability enrichment for %s during discovery: %s" + ), + args=(server.url, exc), + ) + + +def _enrich_storage( + server: Server, + *, + storage_resource: RegistryResource | None, + config: Configuration, + authentication_idp: str, + token: str | None, + certificate: Path | None, + strict: bool, + timeout: int, +) -> Server: + """Validate and attach one retained primary VOSpace registry resource.""" + error: BaseException | None = None + if storage_resource is None: + leaf = get_idp(authentication_idp).leaf + subject = f"same-namespace '{leaf}' registry record" + error = ValueError( + f"No {subject} found for Science Platform Server '{server.name}'." + ) + else: + subject = storage_resource.uri + try: + xml = _fetch_capabilities( + AnyHttpUrl(storage_resource.url), + config=config, + authentication_idp=authentication_idp, + token=token, + certificate=certificate, + timeout=timeout, + ) + valid = vosi.is_vospace_service(xml) + except ( + httpx.HTTPError, + OSError, + AuthContextError, + AuthExpiredError, + CertificateError, + HTTPAuthenticationError, + ParseError, + DefusedXmlException, + ValueError, + ) as exc: + error = exc + else: + if not valid: + error = ValueError( + "required VOSpace node capability is missing or malformed" + ) + elif server.name is None: + error = ValueError("Science Platform Server has no Server Name") + + if error is not None: + message = ( + f"Failed to inspect VOSpace Service '{subject}' for Science " + f"Platform Server '{server.name}': {error}" + ) + return _keep_or_raise( + server, + strict=strict, + error=message, + cause=error, + debug="Skipping VOSpace Service %s during discovery: %s", + args=(subject, error), + ) + assert storage_resource is not None + assert server.name is not None + service = VOSpaceService.model_validate( + {"uri": storage_resource.uri, "url": storage_resource.url} + ) + + return server.model_copy( + update={"storage": {**server.storage, server.name: service}}, + deep=True, + ) + + +def _fetch_capabilities( + url: AnyHttpUrl, + *, + config: Configuration, + authentication_idp: str, + token: str | None = None, + certificate: Path | None = None, + timeout: int, +) -> str: + """Fetch one VOSI capabilities document through the existing HTTP seam.""" + from canfar.client import HTTPClient # noqa: PLC0415 + + with HTTPClient.build( + config=config, + authentication_idp=authentication_idp, + url=url, + token=token, + certificate=certificate, + timeout=timeout, + raise_http_errors=False, + ) as client: + request_client = client.client + request_client.headers["Accept"] = "application/xml" + request_client.headers.pop("Content-Type", None) + request_client.headers.pop("X-Skaha-Registry-Auth", None) + response = request_client.get("capabilities") + response.raise_for_status() + return response.text + + +def _keep_or_raise( + server: Server, + *, + strict: bool, + error: str, + debug: str, + args: tuple[object, ...] = (), + cause: BaseException | None = None, +) -> Server: + """Raise on strict enrich failures; otherwise keep the original server.""" + if strict: + raise ServerFetchError(error) from cause + log.debug(debug, *args) + return server.model_copy(deep=True) + + +def _validate_server( + server: Server, + *, + config: Configuration | None = None, + idp: str | None = None, + dev: bool = False, + timeout: int = 2, +) -> Server: + """Fetch and validate a server before persisting it as active.""" + base_config = config or Configuration() # ty: ignore[missing-argument] + active_idp = idp or server.idp or base_config.active.authentication + storage_resource = _configured_storage_resource(server) + if storage_resource is None: + storage_resource = asyncio.run( + _discover_storage( + server, + active_idp, + dev=dev, + timeout=timeout, + ) + ) + enriched = enrich( + server, + config=base_config, + authentication_idp=active_idp, + strict=True, + timeout=timeout, + storage_resource=storage_resource, + ) + if enriched.url is None or enriched.version is None: + msg = "Server URL and version are required before activation." + raise ServerFetchError(msg) + + return _fetch_resources( + enriched, + timeout=timeout, + config=base_config, + authentication_idp=active_idp, + ) + + +def _fetch_resources( + server: Server, + *, + config: Configuration, + authentication_idp: str, + timeout: int, +) -> Server: + """Return a Server populated from its authenticated context endpoint.""" + from canfar.client import HTTPClient # noqa: PLC0415 + + if server.url is None or server.version is None: + msg = "Server URL and version are required for resource enrichment." + raise ValueError(msg) + client = HTTPClient( + config=config, + authentication_idp=authentication_idp, + url=AnyHttpUrl(f"{server.url}/{server.version}"), + timeout=timeout, + raise_http_errors=False, + ) + try: + with client: + response = client.client.get("context") + response.raise_for_status() + data = dict(response.json()) + + cores_data = data.get("cores") or {} + ram_data = data.get("memoryGB") or {} + gpus_data = data.get("gpus") or {} + cores = cores_data.get("defaultLimit") + ram = ram_data.get("defaultLimit") + gpu_options = gpus_data.get("options") or [] + return Server.model_validate( + { + **server.model_dump(mode="python"), + "cores": cores if cores is not None else DEFAULT_SERVER_CORES, + "ram": ram if ram is not None else DEFAULT_SERVER_RAM_GB, + "gpus": max(gpu_options) if gpu_options else DEFAULT_SERVER_GPUS, + } + ) + except (httpx.HTTPError, OSError, ValueError, TypeError): + return server.model_copy( + update={ + "cores": DEFAULT_SERVER_CORES, + "ram": DEFAULT_SERVER_RAM_GB, + "gpus": DEFAULT_SERVER_GPUS, + }, + deep=True, + ) + + +def _store_discovered_servers(config: Configuration, servers: list[Server]) -> None: + """Merge discovered Science Platform Servers through the editor boundary.""" + updated = dict(config.servers) + for server in servers: + if server.name is not None: + updated[server.name] = server + config.editor._set_top_level(servers=updated) # noqa: SLF001 + + +def discover( + idp: str, + *, + config: Configuration | None = None, + dev: bool = False, + timeout: int = 2, + save: bool = True, +) -> list[Server]: + """Discover, merge, and optionally persist servers for ``idp``. + + Args: + idp: Canonical Identity Provider key. + config: Configuration to update in place. Defaults to loading config. + dev: Include development registries and endpoints during discovery. + timeout: HTTP timeout in seconds for discovery requests. + save: Persist the configuration after merging discovered servers. + + Returns: + list[Server]: Newly discovered server records. + + Raises: + ServerDiscoveryError: If discovery fails or finds no usable servers. + """ + target_config = config or Configuration() # ty: ignore[missing-argument] + discovered = asyncio.run( + _discover_for_idp( + idp, + config=target_config, + dev=dev, + timeout=timeout, + ) + ) + known_servers = dict(target_config.servers) + canonical: dict[str, Server] = {} + for server in sorted( + discovered, + key=lambda item: ( + item.name is None, + (item.name or "").casefold(), + str(item.uri or ""), + str(item.url or ""), + ), + ): + name = server.name + if name is None: + continue + known = canonical.get(name, known_servers.get(name)) + if server.version is None or not server.auths: + if known is not None and known.version is not None and known.auths: + if server.storage: + known = known.model_copy( + update={ + "storage": _merge_storage(known.storage, server.storage) + }, + deep=True, + ) + canonical[name] = known + continue + merged_server = server + if known is not None: + updates = server.model_dump( + include={"idp", "name", "uri", "url", "version", "auths"}, + exclude_none=True, + ) + if server.storage: + updates["storage"] = _merge_storage(known.storage, server.storage) + merged_server = known.model_copy(update=updates, deep=True) + canonical[name] = merged_server + + if not canonical: + msg = f"No servers discovered for IDP '{idp}'." + raise ServerDiscoveryError(msg, code=ErrorCode.SERVER_NONE_AVAILABLE) + merged = [ + canonical[name] + for name in sorted(canonical, key=lambda value: (value.casefold(), value)) + ] + _store_discovered_servers(target_config, merged) + if save: + target_config.editor.save() + return merged diff --git a/canfar/auth/oidc.py b/canfar/auth/oidc.py index 3c87ca72..170737d8 100644 --- a/canfar/auth/oidc.py +++ b/canfar/auth/oidc.py @@ -3,32 +3,38 @@ from __future__ import annotations import asyncio +import logging import socket import time from collections.abc import Awaitable, Callable, Generator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NoReturn, cast import httpx from authlib.integrations.base_client.errors import OAuthError from authlib.oauth2.rfc8628 import DEVICE_CODE_GRANT_TYPE from pydantic import SecretStr, ValidationError -from canfar import get_logger from canfar.models.auth import DeviceAuthorization, Expiry, OIDCCredential, Token if TYPE_CHECKING: - from authlib.integrations.httpx_client import AsyncOAuth2Client + from authlib.integrations.httpx_client import AsyncOAuth2Client, OAuth2Client from canfar.models.config import Configuration -log = get_logger(__name__) +log = logging.getLogger(__name__) _BASIC_AUTH_METHOD = "client_secret_basic" +_OIDC_SCOPE = "openid profile email offline_access" DeviceFlow = Callable[ [str, str, str, str, "AsyncOAuth2Client"], Awaitable[dict[str, Any]], ] +SyncDeviceFlow = Callable[ + [str, str, str, str, "OAuth2Client"], + dict[str, Any], +] +ChallengePresenter = Callable[[DeviceAuthorization], None] class AuthPendingError(Exception): @@ -39,13 +45,166 @@ class SlowDownError(Exception): """Exception raised when the client should slow down its requests.""" +def _validate_discovery_data(data: Any, expected_issuer: str) -> dict[str, Any]: + """Validate OIDC Identity Provider metadata shared by discovery paths.""" + if not isinstance(data, dict) or data.get("issuer") != expected_issuer: + msg = "OIDC discovery issuer mismatch" + raise ValueError(msg) + + required = { + "device_authorization_endpoint", + "registration_endpoint", + "token_endpoint", + "userinfo_endpoint", + } + missing = sorted(field for field in required if not data.get(field)) + if missing: + msg = f"OIDC discovery missing required metadata: {', '.join(missing)}" + raise ValueError(msg) + return data + + +def _registration_payload() -> dict[str, Any]: + """Build the dynamic registration request shared by both transports.""" + hostname = socket.gethostname() + date = time.strftime("%Y-%m-%d %H:%M", time.gmtime()) + return { + "client_name": f"Science Platform CLI @ {hostname} {date}", + "grant_types": [ + "urn:ietf:params:oauth:grant-type:device_code", + "refresh_token", + ], + "response_types": ["token"], + "token_endpoint_auth_method": _BASIC_AUTH_METHOD, + "scope": _OIDC_SCOPE, + } + + +def _device_authorization_payload(identity: str) -> dict[str, str]: + """Build the device authorization request shared by both transports.""" + return {"client_id": identity, "scope": _OIDC_SCOPE} + + +def _parse_device_authorization(response: httpx.Response) -> DeviceAuthorization: + """Parse one OIDC Identity Provider challenge without exposing secrets.""" + try: + return DeviceAuthorization.model_validate(response.json()) + except ValidationError: + msg = "Invalid OIDC device authorization response" + raise ValueError(msg) from None + + +def _raise_poll_oauth_error(error: OAuthError) -> NoReturn: + """Translate an OAuth device-poll error into the public protocol errors.""" + if error.error == "authorization_pending": + raise AuthPendingError from None + if error.error == "slow_down": + raise SlowDownError from None + if error.error == "access_denied": + msg = "OIDC device authorization was denied" + raise PermissionError(msg) from None + if error.error == "expired_token": + msg = "OIDC device authorization expired" + raise TimeoutError(msg) from None + msg = "OIDC device authorization failed" + raise ValueError(msg) from None + + +def _validate_poll_token(token: Any) -> dict[str, Any]: + """Validate one successful device token response.""" + if not isinstance(token, dict) or "access_token" not in token: + msg = "OIDC device authorization failed: malformed token response" + raise ValueError(msg) + return token + + +def _install_tokens(credential: OIDCCredential, tokens: dict[str, Any]) -> None: + """Install validated token data on a credential for both auth paths.""" + try: + token = Token( + access=tokens["access_token"], + refresh=tokens.get("refresh_token"), + token_type=tokens.get("token_type"), + scope=tokens.get("scope"), + ) + expiry = Expiry(access=tokens.get("expires_at"), refresh=None) + except (KeyError, TypeError, ValidationError): + msg = "OIDC device authorization failed: malformed token response" + raise ValueError(msg) from None + credential.token, credential.expiry = token, expiry + + +def _client_credentials(device: Any) -> tuple[str, str]: + """Extract a dynamic client pair without exposing Identity Provider data.""" + if not isinstance(device, dict): + msg = "OIDC device authorization failed: malformed client response" + raise TypeError(msg) + try: + identity = device["client_id"] + secret = device["client_secret"] + except KeyError: + msg = "OIDC device authorization failed: malformed client response" + raise ValueError(msg) from None + if not isinstance(identity, str) or not isinstance(secret, str): + msg = "OIDC device authorization failed: malformed client response" + raise TypeError(msg) + return identity, secret + + +def _userinfo_headers(credential: OIDCCredential) -> dict[str, str]: + """Build the bearer header for the post-login UserInfo request.""" + access = credential.token.access + return {"Authorization": f"Bearer {access.get_secret_value() if access else ''}"} + + +def _userinfo_username(response: httpx.Response) -> str | None: + """Validate UserInfo and return its optional display identifier.""" + response.raise_for_status() + return response.json().get("preferred_username") + + +def _apply_discovery( + credential: OIDCCredential, + discovery: dict[str, Any], +) -> None: + """Apply discovered Identity Provider endpoints and log their values.""" + credential.endpoints.device = discovery["device_authorization_endpoint"] + credential.endpoints.registration = discovery["registration_endpoint"] + credential.endpoints.token = discovery["token_endpoint"] + log.debug("Discovered OIDC configuration:") + log.debug("Device Registration Endpoint: %s", credential.endpoints.registration) + log.debug("Device Authorization Endpoint: %s", credential.endpoints.device) + log.debug("Token Endpoint: %s", credential.endpoints.token) + + +def _install_client_credentials( + credential: OIDCCredential, + device: Any, +) -> tuple[str, str]: + """Install registered client credentials and return their values.""" + identity, secret = _client_credentials(device) + credential.client.identity = identity + credential.client.secret = SecretStr(secret) + return identity, secret + + +def _finalize_authentication( + response: httpx.Response, + on_authenticated: Callable[[str | None], None] | None, +) -> None: + """Validate UserInfo and notify observers of the authenticated username.""" + username = _userinfo_username(response) + if on_authenticated is not None: + on_authenticated(username) + + async def discover( url: str, client: httpx.AsyncClient | None = None, *, expected_issuer: str, ) -> dict[str, Any]: - """Discover OIDC provider configuration. + """Discover OIDC Identity Provider configuration. Args: url (str): OIDC Discovery URL. @@ -54,7 +213,7 @@ async def discover( expected_issuer: Exact issuer configured for the Identity Provider. Returns: - dict[str, Any]: OIDC provider configuration. + dict[str, Any]: OIDC Identity Provider configuration. """ if client is None: async with httpx.AsyncClient() as http_client: @@ -66,27 +225,13 @@ async def discover( response.raise_for_status() data = response.json() - if data.get("issuer") != expected_issuer: - msg = "OIDC discovery issuer mismatch" - raise ValueError(msg) - - required = { - "device_authorization_endpoint", - "registration_endpoint", - "token_endpoint", - "userinfo_endpoint", - } - missing = sorted(field for field in required if not data.get(field)) - if missing: - msg = f"OIDC discovery missing required metadata: {', '.join(missing)}" - raise ValueError(msg) - + data = _validate_discovery_data(data, expected_issuer) log.debug("OIDC Discovery Data: %s", data) return data async def register(url: str, client: httpx.AsyncClient | None = None) -> dict[str, Any]: - """Register a new client with the OIDC provider. + """Register a new client with the OIDC Identity Provider. Args: url (str): OIDC Registration URL. @@ -96,18 +241,7 @@ async def register(url: str, client: httpx.AsyncClient | None = None) -> dict[st Returns: dict[str, Any]: Client registration details. """ - hostname = socket.gethostname() - date = time.strftime("%Y-%m-%d %H:%M", time.gmtime()) - payload: dict[str, Any] = { - "client_name": f"Science Platform CLI @ {hostname} {date}", - "grant_types": [ - "urn:ietf:params:oauth:grant-type:device_code", - "refresh_token", - ], - "response_types": ["token"], - "token_endpoint_auth_method": "client_secret_basic", - "scope": "openid profile email offline_access", - } + payload = _registration_payload() if client is None: async with httpx.AsyncClient() as http: @@ -146,28 +280,14 @@ async def _poll_token(url: str, code: str, client: AsyncOAuth2Client) -> dict[st device_code=code, ) except OAuthError as error: - if error.error == "authorization_pending": - raise AuthPendingError from None - if error.error == "slow_down": - raise SlowDownError from None - if error.error == "access_denied": - msg = "OIDC device authorization was denied" - raise PermissionError(msg) from None - if error.error == "expired_token": - msg = "OIDC device authorization expired" - raise TimeoutError(msg) from None - msg = "OIDC device authorization failed" - raise ValueError(msg) from None + _raise_poll_oauth_error(error) except httpx.HTTPStatusError: msg = "OIDC device authorization failed" raise ValueError(msg) from None except ValueError: msg = "OIDC device authorization failed: malformed token response" raise ValueError(msg) from None - if not isinstance(token, dict) or "access_token" not in token: - msg = "OIDC device authorization failed: malformed token response" - raise ValueError(msg) - return token + return _validate_poll_token(token) @contextmanager @@ -249,10 +369,13 @@ def _persist( raise ValueError(msg) from None updated = credential.model_copy(update={"token": token, "expiry": expiry}) + if credential.idp not in config.authentication: + msg = f"Authentication record for IDP '{credential.idp}' not found." + raise KeyError(msg) candidate = config.model_copy(deep=True) - candidate.update_credential(updated) - candidate.save() - config.update_credential(updated) + candidate.editor.set(f"authentication.{updated.idp}", updated) + candidate.editor.save() + config.editor.set(f"authentication.{updated.idp}", updated) return updated @@ -330,6 +453,8 @@ async def authflow( identity: str, secret: str, client: AsyncOAuth2Client | None = None, + *, + on_challenge: ChallengePresenter | None = None, ) -> dict[str, Any]: """OIDC Authorization Flow. @@ -340,6 +465,7 @@ async def authflow( secret (str): Client secret. client: Optional Authlib async OAuth client. If None, creates a new one. Defaults to None. + on_challenge: Optional callback invoked after device authorization. Returns: dict[str, Any]: OIDC tokens including access and refresh tokens. @@ -355,7 +481,12 @@ async def authflow( token_endpoint_auth_method=_BASIC_AUTH_METHOD, ) as http_client: return await _authflow_impl( - device_auth_url, token_url, identity, secret, http_client + device_auth_url, + token_url, + identity, + secret, + http_client, + on_challenge=on_challenge, ) else: return await _authflow_impl( @@ -364,6 +495,7 @@ async def authflow( identity, secret, client, + on_challenge=on_challenge, ) @@ -386,18 +518,11 @@ async def start_device_authorization( """ response = await client.post( url, - data={ - "client_id": identity, - "scope": "openid profile email offline_access", - }, + data=_device_authorization_payload(identity), auth=(identity, secret), ) response.raise_for_status() - try: - challenge = DeviceAuthorization.model_validate(response.json()) - except ValidationError: - msg = "Invalid OIDC device authorization response" - raise ValueError(msg) from None + challenge = _parse_device_authorization(response) log.debug("OIDC device authorization challenge received") return challenge @@ -474,6 +599,8 @@ async def _authflow_impl( identity: str, secret: str, client: AsyncOAuth2Client, + *, + on_challenge: ChallengePresenter | None = None, ) -> dict[str, Any]: """Implementation of the auth flow with an existing client. @@ -483,6 +610,7 @@ async def _authflow_impl( identity (str): Client identity. secret (str): Client secret. client: Authlib async OAuth client. + on_challenge: Optional callback invoked after device authorization. Returns: dict[str, Any]: OIDC tokens including access and refresh tokens. @@ -496,6 +624,8 @@ async def _authflow_impl( secret, client, ) + if on_challenge is not None: + on_challenge(challenge) return await poll_device_token(token_url, challenge, client) @@ -506,6 +636,7 @@ async def authenticate_credential( expected_issuer: str, timeout: int | None = None, device_flow: DeviceFlow | None = None, + on_challenge: ChallengePresenter | None = None, on_authenticated: Callable[[str | None], None] | None = None, ) -> OIDCCredential: """Authenticate using OIDC Device Authorization Flow. @@ -516,6 +647,8 @@ async def authenticate_credential( timeout: HTTP timeout in seconds for OIDC HTTP requests. device_flow: Device authorization coordinator. Defaults to the presentation-free protocol flow. + on_challenge: Optional callback invoked by the default protocol flow + after device authorization. on_authenticated: Optional observer for the authenticated username. Returns: @@ -533,21 +666,12 @@ async def authenticate_credential( client, expected_issuer=expected_issuer, ) - credential.endpoints.device = response["device_authorization_endpoint"] - credential.endpoints.registration = response["registration_endpoint"] - credential.endpoints.token = response["token_endpoint"] - - log.debug("Discovered OIDC configuration:") - log.debug("Device Registration Endpoint: %s", credential.endpoints.registration) - log.debug("Device Authorization Endpoint: %s", credential.endpoints.device) - log.debug("Token Endpoint: %s", credential.endpoints.token) + _apply_discovery(credential, response) device: dict[str, Any] = await register( str(credential.endpoints.registration), client ) - credential.client.identity = device["client_id"] - client_secret = device["client_secret"] - credential.client.secret = SecretStr(client_secret) + identity, client_secret = _install_client_credentials(credential, device) from authlib.integrations.httpx_client import ( # noqa: PLC0415 AsyncOAuth2Client, @@ -555,52 +679,243 @@ async def authenticate_credential( if request_timeout is None: oauth_context = AsyncOAuth2Client( - credential.client.identity, + identity, client_secret, token_endpoint_auth_method=_BASIC_AUTH_METHOD, ) else: oauth_context = AsyncOAuth2Client( - credential.client.identity, + identity, client_secret, token_endpoint_auth_method=_BASIC_AUTH_METHOD, timeout=request_timeout, ) async with oauth_context as oauth_client: - authorize = device_flow or authflow - tokens = await authorize( - str(credential.endpoints.device), - str(credential.endpoints.token), - str(credential.client.identity), - client_secret, - oauth_client, - ) + if device_flow is None: + tokens = await authflow( + str(credential.endpoints.device), + str(credential.endpoints.token), + identity, + client_secret, + oauth_client, + on_challenge=on_challenge, + ) + else: + tokens = await device_flow( + str(credential.endpoints.device), + str(credential.endpoints.token), + identity, + client_secret, + oauth_client, + ) + + _install_tokens(credential, tokens) + + url: str = response["userinfo_endpoint"] + user = await client.get(url, headers=_userinfo_headers(credential)) + _finalize_authentication(user, on_authenticated) + return credential + + +def sync_discover( + url: str, + client: httpx.Client, + *, + expected_issuer: str, +) -> dict[str, Any]: + """Discover OIDC Identity Provider configuration with sync HTTP.""" + response = client.get(url) + response.raise_for_status() + data: dict[str, Any] = response.json() + data = _validate_discovery_data(data, expected_issuer) + log.debug("OIDC Discovery Data: %s", data) + return data + + +def sync_register( + url: str, + client: httpx.Client, +) -> dict[str, Any]: + """Register a device-flow client with a synchronous HTTP client.""" + payload = _registration_payload() + response = client.post(url, json=payload) + response.raise_for_status() + data: dict[str, Any] = response.json() + log.debug("OIDC dynamic client registration succeeded.") + return data + +def sync_start_device_authorization( + url: str, + identity: str, + secret: str, + client: httpx.Client, +) -> DeviceAuthorization: + """Request an OIDC device authorization challenge synchronously.""" + response = client.post( + url, + data=_device_authorization_payload(identity), + auth=(identity, secret), + ) + response.raise_for_status() + challenge = _parse_device_authorization(response) + log.debug("OIDC device authorization challenge received") + return challenge + + +def _sync_poll_token( + url: str, + code: str, + client: OAuth2Client, +) -> dict[str, Any]: + """Exchange a device code for tokens with a synchronous OAuth client.""" + try: + token: dict[str, Any] = client.fetch_token( + url, + grant_type=DEVICE_CODE_GRANT_TYPE, + device_code=code, + ) + except OAuthError as error: + _raise_poll_oauth_error(error) + except httpx.HTTPStatusError: + msg = "OIDC device authorization failed" + raise ValueError(msg) from None + except ValueError: + msg = "OIDC device authorization failed: malformed token response" + raise ValueError(msg) from None + return _validate_poll_token(token) + + +def sync_poll_device_token( + token_url: str, + challenge: DeviceAuthorization, + client: OAuth2Client, +) -> dict[str, Any]: + """Poll for OIDC tokens with the RFC 8628 interval using sync I/O.""" + interval = challenge.interval + deadline = time.monotonic() + challenge.expires_in + code = challenge.device_code.get_secret_value() + + while time.monotonic() < deadline: try: - token = Token( - access=tokens["access_token"], - refresh=tokens.get("refresh_token"), - token_type=tokens.get("token_type"), - scope=tokens.get("scope"), + return _sync_poll_token(token_url, code, client) + except AuthPendingError: + pass + except SlowDownError: + interval += 5 + except httpx.TransportError: + interval *= 2 + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(min(interval, remaining)) + + msg = "Device flow timed out" + raise TimeoutError(msg) + + +def sync_authflow( + device_auth_url: str, + token_url: str, + identity: str, + secret: str, + client: OAuth2Client | None = None, + *, + on_challenge: ChallengePresenter | None = None, +) -> dict[str, Any]: + """Run the presentation-free OIDC device flow with native sync I/O.""" + if client is None: + from authlib.integrations.httpx_client import OAuth2Client # noqa: PLC0415 + + with OAuth2Client( + identity, + secret, + token_endpoint_auth_method=_BASIC_AUTH_METHOD, + ) as http_client: + return sync_authflow( + device_auth_url, + token_url, + identity, + secret, + http_client, + on_challenge=on_challenge, + ) + challenge = sync_start_device_authorization( + device_auth_url, + identity, + secret, + client, + ) + if on_challenge is not None: + on_challenge(challenge) + return sync_poll_device_token(token_url, challenge, client) + + +def sync_authenticate_credential( + credential: OIDCCredential, + *, + expected_issuer: str, + timeout: int | None = None, + device_flow: SyncDeviceFlow | None = None, + on_challenge: ChallengePresenter | None = None, + on_authenticated: Callable[[str | None], None] | None = None, +) -> OIDCCredential: + """Authenticate an OIDC record with native synchronous HTTP operations.""" + request_timeout = None if timeout is None else httpx.Timeout(timeout) + + from authlib.integrations.httpx_client import OAuth2Client # noqa: PLC0415 + + if request_timeout is None: + client_context = httpx.Client() + else: + client_context = httpx.Client(timeout=request_timeout) + + with client_context as client: + response = sync_discover( + str(credential.endpoints.discovery), + client, + expected_issuer=expected_issuer, + ) + _apply_discovery(credential, response) + + device = sync_register(str(credential.endpoints.registration), client) + identity, client_secret = _install_client_credentials(credential, device) + + if request_timeout is None: + oauth_context = OAuth2Client( + identity, + client_secret, + token_endpoint_auth_method=_BASIC_AUTH_METHOD, ) - expiry = Expiry(access=tokens.get("expires_at"), refresh=None) - except (KeyError, TypeError, ValidationError): - msg = "OIDC device authorization failed: malformed token response" - raise ValueError(msg) from None - credential.token, credential.expiry = token, expiry + else: + oauth_context = OAuth2Client( + identity, + client_secret, + token_endpoint_auth_method=_BASIC_AUTH_METHOD, + timeout=request_timeout, + ) + with oauth_context as oauth_client: + if device_flow is None: + tokens = sync_authflow( + str(credential.endpoints.device), + str(credential.endpoints.token), + identity, + client_secret, + oauth_client, + on_challenge=on_challenge, + ) + else: + tokens = device_flow( + str(credential.endpoints.device), + str(credential.endpoints.token), + identity, + client_secret, + oauth_client, + ) + + _install_tokens(credential, tokens) url: str = response["userinfo_endpoint"] - headers = { - "Authorization": ( - f"Bearer {credential.token.access.get_secret_value()}" - if credential.token.access - else "" - ), - } - user = await client.get(url, headers=headers) - user.raise_for_status() - username = user.json().get("preferred_username") - if on_authenticated is not None: - on_authenticated(username) + user = client.get(url, headers=_userinfo_headers(credential)) + _finalize_authentication(user, on_authenticated) return credential diff --git a/canfar/auth/x509.py b/canfar/auth/x509.py index 8a45637f..96437b58 100644 --- a/canfar/auth/x509.py +++ b/canfar/auth/x509.py @@ -6,20 +6,20 @@ from __future__ import annotations +import logging from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Any from cadcutils.net.auth import Subject, get_cert from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from canfar import CERT_PATH, get_logger +from canfar import CERT_PATH if TYPE_CHECKING: from canfar.models.auth import X509Credential -log = get_logger(__name__) +log = logging.getLogger(__name__) class CertificateError(ValueError): @@ -36,42 +36,6 @@ def __init__( self.expired_at = expired_at -def _to_utc(value: datetime) -> datetime: - """Return timezone aware datetime. - - Args: - value (datetime): Input datetime. - - Returns: - datetime: Timezone aware datetime. - """ - if value.tzinfo is None: - return value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc) - - -def _validity_window(cert: x509.Certificate) -> tuple[datetime, datetime]: - """Return validity start/end datetimes in UTC. - - Args: - cert (x509.Certificate): Certificate to inspect. - - Raises: - CertificateError: If certificate is expired or not yet valid. - - Returns: - tuple[datetime, datetime]: Validity start and end datetimes in UTC. - """ - try: - start = getattr(cert, "not_valid_before_utc", None) or cert.not_valid_before - end = getattr(cert, "not_valid_after_utc", None) or cert.not_valid_after - except AttributeError as err: # pragma: no cover - defensive path - msg = "Certificate is missing validity information." - raise CertificateError(msg) from err - - return _to_utc(start), _to_utc(end) - - def assert_valid_dates( destination: Path, valid_from: datetime, valid_until: datetime ) -> None: @@ -240,8 +204,9 @@ def expiry(path: Path = CERT_PATH) -> float: try: destination = path.resolve(strict=True) data = destination.read_bytes() - cert = x509.load_pem_x509_certificate(data, default_backend()) - valid_from, valid_until = _validity_window(cert) + cert = x509.load_pem_x509_certificate(data) + valid_from = cert.not_valid_before_utc + valid_until = cert.not_valid_after_utc assert_valid_dates(destination, valid_from, valid_until) return valid_until.timestamp() except FileNotFoundError as err: diff --git a/canfar/authentication.py b/canfar/authentication.py index 6cc438f7..9d0cdc12 100644 --- a/canfar/authentication.py +++ b/canfar/authentication.py @@ -2,18 +2,34 @@ from __future__ import annotations +import asyncio +import sys from typing import TYPE_CHECKING, NoReturn +import httpx + import canfar.server as server_service +from canfar.auth import oidc from canfar.errors import ErrorCode, StructuredError from canfar.idp import IdpInfo, get_idp from canfar.models.auth import ( Authentication, AuthenticationCredential, AuthMode, + Client, + DeviceAuthorization, + Endpoint, + Expiry, + OIDCCredential, + Token, X509Credential, ) -from canfar.models.config import Configuration +from canfar.models.config import ( + Configuration, + default_active, + default_authentication, + default_servers, +) if TYPE_CHECKING: import builtins @@ -35,6 +51,15 @@ def __init__(self, error: StructuredError | Mapping[str, object]) -> None: super().__init__(self.error.message) +_OIDC_DEVICE_LOGIN_ERRORS = ( + PermissionError, + TimeoutError, + TypeError, + ValueError, + httpx.HTTPError, +) + + def _authentication_error( *, code: ErrorCode, @@ -78,9 +103,42 @@ def login(idp: str, force: bool = False) -> None: return credential = _authenticate(idp_info) - config.upsert_credential(credential) + config.editor.set(f"authentication.{credential.idp}", credential) server_service.discover(idp, config=config, save=False) - config.save() + config.editor.save() + + +async def alogin(idp: str, force: bool = False) -> None: + """Authenticate an IDP from an existing asynchronous event loop. + + The OIDC protocol uses native asynchronous HTTP and polling. Synchronous + X.509 inspection and Science Platform discovery run in worker threads so + this API does not block the caller's event loop. + + Args: + idp: Canonical Identity Provider key. + force: Re-authenticate and rediscover when true. + + Raises: + KeyError: Unknown IDP key. + AuthenticationError: Credential or discovery failure. + """ + idp_info = get_idp(idp) + config = Configuration() # ty: ignore[missing-argument] + + if _has_authentication(config, idp) and not force: + return + + credential = await _authenticate_async(idp_info) + editor = config.editor + editor.set(f"authentication.{credential.idp}", credential) + await asyncio.to_thread( + server_service.discover, + idp, + config=config, + save=False, + ) + await asyncio.to_thread(editor.save) def use(idp: str) -> None: @@ -99,7 +157,7 @@ def use(idp: str) -> None: config = Configuration() # ty: ignore[missing-argument] try: - config.get_credential(idp) + _authentication_record(config, idp) except KeyError as exc: raise _authentication_error( code=ErrorCode.AUTHENTICATION_REQUIRED, @@ -107,8 +165,7 @@ def use(idp: str) -> None: hint="Run canfar.login() for this IDP before selecting it.", ) from exc - config.set_active_authentication(idp) - config.save() + server_service.activate_authentication(idp, config=config) def list() -> builtins.list[Authentication]: # noqa: A001 @@ -152,8 +209,7 @@ def remove(idp: str, *, force: bool = False) -> None: hint="Use --force or switch authentication before removing.", ) - config.remove_authentication(idp) - config.save() + _remove_authentication(config, idp) def purge(*, force: bool = False) -> None: @@ -175,8 +231,7 @@ def purge(*, force: bool = False) -> None: ) config = Configuration() # ty: ignore[missing-argument] - config.purge_authentication() - config.save() + _purge_authentication(config) def show() -> Authentication: @@ -190,7 +245,7 @@ def show() -> Authentication: """ config = Configuration() # ty: ignore[missing-argument] try: - credential = config.get_credential(config.active.authentication) + credential = _authentication_record(config, config.active.authentication) except KeyError as exc: raise _authentication_error( code=ErrorCode.AUTHENTICATION_REQUIRED, @@ -208,6 +263,71 @@ def _has_authentication(config: Configuration, idp: str) -> bool: return idp in config.authentication +def _authentication_record( + config: Configuration, + idp: str, +) -> AuthenticationCredential: + """Return a saved Authentication Record by its IDP key.""" + try: + return config.authentication[idp] + except KeyError as exc: + msg = f"Authentication record for IDP '{idp}' not found." + raise KeyError(msg) from exc + + +def _remove_authentication(config: Configuration, idp: str) -> None: + """Remove one Authentication Record and its associated Server state.""" + authentication = dict(config.authentication) + authentication.pop(idp, None) + if not authentication: + _purge_authentication(config) + return + + servers = { + name: server for name, server in config.servers.items() if server.idp != idp + } + selections = { + selected_idp: name + for selected_idp, name in config.active.servers.items() + if selected_idp != idp and name in servers + } + active = config.active.model_copy(update={"servers": selections}) + if active.authentication == idp: + active = active.model_copy( + update={ + "authentication": next(iter(authentication)), + "server": None, + }, + ) + elif active.server not in servers: + active = active.model_copy(update={"server": None}) + + editor = config.editor + editor._set_top_level( # noqa: SLF001 + active=active, + authentication=authentication, + servers=servers, + ) + editor.save() + + +def _purge_authentication(config: Configuration) -> None: + """Reset Authentication and Server state while preserving other settings.""" + editor = config.editor + editor._set_top_level( # noqa: SLF001 + active=default_active.model_copy(deep=True), + authentication={ + key: credential.model_copy(deep=True) + for key, credential in default_authentication.items() + }, + servers={ + name: server.model_copy(deep=True) + for name, server in default_servers.items() + }, + ) + editor.save() + + def _authentication_for_credential( config: Configuration, credential: AuthenticationCredential, @@ -218,7 +338,7 @@ def _authentication_for_credential( if active and config.active.server is not None: try: - server = config.get_active_server() + server = config.servers[config.active.server] except KeyError: server_ref = config.active.server else: @@ -245,10 +365,15 @@ def _credential_expiry(credential: AuthenticationCredential) -> float | None: def _authenticate(idp_info: IdpInfo) -> AuthenticationCredential: if idp_info.auth_mode == "x509": - credential = _authenticate_x509(idp_info.key) - else: - _authenticate_oidc(idp_info.key) - return credential + return _authenticate_x509(idp_info.key) + return _authenticate_oidc(idp_info) + + +async def _authenticate_async(idp_info: IdpInfo) -> AuthenticationCredential: + """Acquire one Authentication Record without blocking an event loop.""" + if idp_info.auth_mode == "x509": + return await asyncio.to_thread(_authenticate_x509, idp_info.key) + return await _authenticate_oidc_async(idp_info) def _authenticate_x509(idp: str) -> X509Credential: @@ -270,18 +395,77 @@ def _authenticate_x509(idp: str) -> X509Credential: ) -def _authenticate_oidc(idp: str) -> NoReturn: - _fail( - code=ErrorCode.AUTHENTICATION_CREDENTIAL_MISSING, - message=f"OIDC authentication for IDP '{idp}' requires interactive login.", - hint="Use the CLI login flow for first-time OIDC authentication.", +def _print_device_challenge(challenge: DeviceAuthorization) -> None: + """Print only user-facing device authorization data to the terminal.""" + lines = [f"Verification URL: {challenge.verification_uri}"] + if challenge.verification_uri_complete is not None: + lines.append( + f"Verification URL (complete): {challenge.verification_uri_complete}" + ) + lines.append(f"Device code: {challenge.user_code.get_secret_value()}") + sys.stdout.write( + "\n".join(lines) + "\n", + ) + sys.stdout.flush() + + +def _oidc_credential(idp: str, info: IdpInfo) -> OIDCCredential: + """Build an empty OIDC Authentication Record from IDP metadata.""" + if info.oidc_discovery_url is None: + msg = f"OIDC discovery URL is not configured for IDP '{idp}'." + raise RuntimeError(msg) + if info.oidc_issuer is None: + msg = f"OIDC issuer is not configured for IDP '{idp}'." + raise RuntimeError(msg) + return OIDCCredential( + idp=idp, + endpoints=Endpoint(discovery=str(info.oidc_discovery_url)), + client=Client(), + token=Token(), + expiry=Expiry(), ) +def _raise_oidc_authentication_error(exc: Exception) -> NoReturn: + """Translate an OIDC device-login failure into a structured error.""" + raise _authentication_error( + code=ErrorCode.AUTHENTICATION_CREDENTIAL_MISSING, + message=f"OIDC authentication failed: {exc}", + hint="Complete the device authorization before it expires.", + ) from exc + + +def _authenticate_oidc(info: IdpInfo) -> OIDCCredential: + """Acquire one OIDC Authentication Record with native sync I/O.""" + credential = _oidc_credential(info.key, info) + try: + return oidc.sync_authenticate_credential( + credential, + expected_issuer=str(info.oidc_issuer), + on_challenge=_print_device_challenge, + ) + except _OIDC_DEVICE_LOGIN_ERRORS as exc: + return _raise_oidc_authentication_error(exc) + + +async def _authenticate_oidc_async(info: IdpInfo) -> OIDCCredential: + """Acquire one OIDC Authentication Record with native async I/O.""" + credential = _oidc_credential(info.key, info) + try: + return await oidc.authenticate_credential( + credential, + expected_issuer=str(info.oidc_issuer), + on_challenge=_print_device_challenge, + ) + except _OIDC_DEVICE_LOGIN_ERRORS as exc: + return _raise_oidc_authentication_error(exc) + + __all__ = [ "AuthMode", "Authentication", "AuthenticationError", + "alogin", "list", "login", "purge", diff --git a/canfar/cli/auth.py b/canfar/cli/auth.py index 1ee7cdb6..7d36752a 100644 --- a/canfar/cli/auth.py +++ b/canfar/cli/auth.py @@ -27,10 +27,9 @@ show as auth_show, ) from canfar.cli import output -from canfar.cli.machine import JsonOption, YamlOption, resolve_mode +from canfar.cli.machine import OutputOption, resolve_mode from canfar.config.migration import ConfigResetRequiredError from canfar.errors import StructuredError -from canfar.hooks.typer.aliases import AliasGroup from canfar.idp import get_idp from canfar.models.config import Configuration from canfar.server import ( @@ -42,7 +41,7 @@ from canfar.server import ( activate as activate_server, ) -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console if TYPE_CHECKING: from typing import NoReturn @@ -55,7 +54,6 @@ no_args_is_help=False, invoke_without_command=True, rich_markup_mode="rich", - cls=AliasGroup, ) @@ -202,40 +200,43 @@ def _auth_show(mode: output.OutputMode) -> None: @auth.callback(invoke_without_command=True) def auth_default( ctx: typer.Context, - json_output: JsonOption = False, - yaml_output: YamlOption = False, + output_format: OutputOption = None, ) -> None: """Active authentication state.""" if ctx.invoked_subcommand is not None: - if json_output or yaml_output: + if output_format is not None: typer.echo( - "Place --json or --yaml after the subcommand.", + "Place --output json or --output yaml after the subcommand.", err=True, ) raise typer.Exit(output.OUTPUT_CONFLICT_EXIT_CODE) return - mode = resolve_mode(json_output, yaml_output) + mode = resolve_mode(output_format) + if mode is output.OutputMode.HUMAN: + emit_cli_active_server_banner() _auth_show(mode) @auth.command("show") def auth_show_command( - json_output: JsonOption = False, - yaml_output: YamlOption = False, + output_format: OutputOption = None, ) -> None: """Active authentication state.""" - mode = resolve_mode(json_output, yaml_output) + mode = resolve_mode(output_format) + if mode is output.OutputMode.HUMAN: + emit_cli_active_server_banner() _auth_show(mode) @auth.command("ls") def auth_list_command( - json_output: JsonOption = False, - yaml_output: YamlOption = False, + output_format: OutputOption = None, ) -> None: """Available auth providers.""" - mode = resolve_mode(json_output, yaml_output) + mode = resolve_mode(output_format) + if mode is output.OutputMode.HUMAN: + emit_cli_active_server_banner() try: summaries = auth_list() except ConfigResetRequiredError as exc: @@ -246,57 +247,23 @@ def auth_list_command( _render_auth_list_table() -@auth.command("login") -def auth_login_command( - idp: Annotated[ - str | None, - typer.Argument(help="Canonical Identity Provider key."), - ] = None, - force: Annotated[ - bool, - typer.Option("-f", "--force", help="Force re-authentication."), - ] = False, - dev: Annotated[ - bool, - typer.Option("--dev", help="Include dev servers in discovery."), - ] = False, - timeout: Annotated[ - int, - typer.Option( - "-t", - "--timeout", - help="Timeout for HTTP requests during login.", - min=1, - ), - ] = 10, -) -> None: - """Alias for canfar login.""" - from canfar.cli.login import _login_flow # noqa: PLC0415 - from canfar.cli.prompts import select_idp # noqa: PLC0415 - from canfar.idp import list_idps # noqa: PLC0415 - - get_console(stderr=True).print( - "\n[red]Deprecation Notice:[/red]" - "\n[yellow]canfar auth login[/yellow] will be removed soon." - " Use [green][bold]canfar login[/bold][/green] instead.\n" - ) - selected_idp = idp or select_idp(list_idps()) - _login_flow(selected_idp, force=force, dev=dev, timeout=timeout) - - @auth.command("use") def auth_use_command( idp: Annotated[str, typer.Argument(help="Canonical Identity Provider key.")], ) -> None: """Switch auth provider.""" + emit_cli_active_server_banner() config = Configuration() # ty: ignore[missing-argument] try: get_idp(idp) - config.get_credential(idp) except KeyError as exc: get_console(stderr=True).print(f"[bold red]{exc}[/bold red]") raise typer.Exit(1) from exc + if idp not in config.authentication: + msg = f"Authentication record for IDP '{idp}' not found." + get_console(stderr=True).print(f"[bold red]{msg}[/bold red]") + raise typer.Exit(1) try: activation = activate_server(idp, config=config) @@ -329,6 +296,7 @@ def auth_remove_command( ] = False, ) -> None: """Remove auth and associated servers.""" + emit_cli_active_server_banner() config = Configuration() # ty: ignore[missing-argument] if config.active.authentication == idp and not force: should_remove = Confirm.ask( @@ -364,6 +332,7 @@ def auth_purge_command( ] = False, ) -> None: """Remove all auths and servers.""" + emit_cli_active_server_banner() if not force: get_console(stderr=True).print( "[bold red]Authentication purge requires --force.[/bold red]" diff --git a/canfar/cli/config.py b/canfar/cli/config.py index 71963361..f101202e 100644 --- a/canfar/cli/config.py +++ b/canfar/cli/config.py @@ -10,16 +10,13 @@ from canfar import CONFIG_PATH from canfar.cli import output -from canfar.cli.machine import JsonOption, YamlOption, resolve_mode +from canfar.cli.machine import OutputOption, resolve_mode from canfar.config.migration import ConfigResetRequiredError from canfar.errors import ErrorCode, StructuredError -from canfar.hooks.typer.aliases import AliasGroup from canfar.models.config import Configuration -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console -config: typer.Typer = typer.Typer( - cls=AliasGroup, -) +config: typer.Typer = typer.Typer() def _configuration_failure(error: Exception) -> StructuredError: @@ -39,11 +36,12 @@ def _configuration_failure(error: Exception) -> StructuredError: @config.command("show", help="Display client configuration") def show( - json_output: JsonOption = False, - yaml_output: YamlOption = False, + output_format: OutputOption = None, ) -> None: """Display client configuration.""" - mode = resolve_mode(json_output, yaml_output) + mode = resolve_mode(output_format) + if mode is output.OutputMode.HUMAN: + emit_cli_active_server_banner() try: cfg = Configuration() # ty: ignore[missing-argument] except ( @@ -95,8 +93,7 @@ def get( ..., help="Config key to get in dot notation.", ), - json_output: JsonOption = False, - yaml_output: YamlOption = False, + output_format: OutputOption = None, ) -> None: """Retrieve a config value. @@ -104,7 +101,9 @@ def get( canfar config get active.server canfar config get servers.canfar.url """ - mode = resolve_mode(json_output, yaml_output) + mode = resolve_mode(output_format) + if mode is output.OutputMode.HUMAN: + emit_cli_active_server_banner() try: cfg = Configuration() # ty: ignore[missing-argument] except ( @@ -124,7 +123,7 @@ def get( raise typer.Exit(1) from err try: - value = cfg.get_value(key) + value = cfg.editor.get(key) except (AttributeError, KeyError, IndexError, TypeError, ValueError) as err: failure = StructuredError( code=ErrorCode.COMMAND_VALIDATION_FAILED, @@ -156,11 +155,12 @@ def set_value( canfar config set active.authentication cadc canfar config set servers.canfar.url https://ws-uv.canfar.net/skaha """ + emit_cli_active_server_banner() cfg = Configuration() # ty: ignore[missing-argument] try: parsed = yaml.safe_load(value) - updated = cfg.set_value(key, parsed) - updated.save() + cfg.editor.set(key, parsed) + cfg.editor.save() except (AttributeError, KeyError, IndexError, TypeError, ValueError) as err: get_console(stderr=True).print(f"[bold red]Error:[/bold red] {err}") raise typer.Exit(1) from err @@ -169,4 +169,5 @@ def set_value( @config.command("path", help="Local path of config") def path() -> None: """Local path of config.""" + emit_cli_active_server_banner() get_console().print(f"[green]{CONFIG_PATH}[/green]") diff --git a/canfar/cli/create.py b/canfar/cli/create.py index ab514d78..aa04a2b6 100644 --- a/canfar/cli/create.py +++ b/canfar/cli/create.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated, Any, get_args +from typing import Annotated, Any, get_args import click import httpx @@ -11,23 +11,15 @@ from canfar.cli import output from canfar.cli._run import run -from canfar.cli.machine import ( - JsonOption, - YamlOption, - resolve_mode, -) +from canfar.cli.machine import OutputOption, resolve_mode from canfar.config.migration import ConfigResetRequiredError from canfar.errors import ErrorCode, StructuredError from canfar.exceptions.context import AuthContextError, AuthExpiredError -from canfar.hooks.typer.aliases import AliasGroup from canfar.models.session import CreateRequest from canfar.models.types import Kind from canfar.sessions import AsyncSession from canfar.utils import funny -from canfar.utils.console import get_console - -if TYPE_CHECKING: - from typer._click.core import Context +from canfar.utils.console import emit_cli_active_server_banner, get_console kinds: list[str] = list(get_args(Kind)) # Remove desktop-app from the list of kinds for usage message since, @@ -35,32 +27,6 @@ kinds.remove("desktop-app") -class CreateUsageMessage(AliasGroup): - """Custom usage message for prune command. - - Args: - typer (TyperGroup): Base class for grouping commands in Typer. - """ - - def get_usage(self, ctx: Context) -> str: # noqa: ARG002 - """Get the usage message for the prune command. - - Args: - ctx (typer.Context): The Typer context. - - Returns: - str: The usage message. - """ - return "Usage: canfar create [OPTIONS] KIND IMAGE [-- CMD [ARGS]...]" - - -create = typer.Typer( - name="create", - no_args_is_help=True, - cls=CreateUsageMessage, -) - - def _parse_environment(env: list[str] | None) -> dict[str, Any]: """Parse repeated ``KEY=VALUE`` options for the session request.""" environment: dict[str, Any] = {} @@ -136,14 +102,7 @@ def _render_create_result( raise typer.Exit(1) -@create.callback( - invoke_without_command=True, - context_settings={ - "help_option_names": ["-h", "--help"], - "allow_interspersed_args": True, - }, -) -def creation( +def creation( # noqa: PLR0917 kind: Annotated[ Kind, typer.Argument( @@ -208,8 +167,7 @@ def creation( help="Dry run. Parse parameters and exit.", ), ] = False, - json_output: JsonOption = False, - yaml_output: YamlOption = False, + output_format: OutputOption = None, ) -> None: """Launch a new session. @@ -218,10 +176,13 @@ def creation( canfar create notebook images.canfar.net/skaha/base-notebook:latest canfar create headless skaha/base-notebook:latest -- python3 /path/to/script.py """ - mode = resolve_mode(json_output, yaml_output) + mode = resolve_mode(output_format) + if mode is output.OutputMode.HUMAN: + emit_cli_active_server_banner() if dry and mode is not output.OutputMode.HUMAN: typer.echo( - "Incompatible flags: --dry-run cannot be used with --json or --yaml.", + "Incompatible flags: --dry-run cannot be used with --output json or " + "--output yaml.", err=True, ) raise typer.Exit(output.OUTPUT_CONFLICT_EXIT_CODE) diff --git a/canfar/cli/data.py b/canfar/cli/data.py index e3f3d3d9..1352dfa3 100644 --- a/canfar/cli/data.py +++ b/canfar/cli/data.py @@ -9,7 +9,7 @@ from typer.core import TyperGroup from typer.main import get_group -from canfar.storage import sources +from canfar.storage import _sources if TYPE_CHECKING: from typer._click.core import Command, Context @@ -25,7 +25,7 @@ def group() -> TyperGroup: """ return get_group( App( - sources(), + _sources(), capabilities={"recursion": {"copy": True, "remove": False}}, ).typer_app ) diff --git a/canfar/cli/delete.py b/canfar/cli/delete.py index 5a7c64fe..457e5235 100644 --- a/canfar/cli/delete.py +++ b/canfar/cli/delete.py @@ -8,18 +8,10 @@ from rich.prompt import Confirm from canfar.cli._run import run -from canfar.hooks.typer.aliases import AliasGroup from canfar.sessions import AsyncSession -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console -delete = typer.Typer( - name="delete", - no_args_is_help=True, - cls=AliasGroup, -) - -@delete.callback(invoke_without_command=True) def delete_sessions( session_ids: Annotated[ list[str], @@ -40,6 +32,7 @@ def delete_sessions( canfar delete abc123 canfar delete abc123 def456 """ + emit_cli_active_server_banner() if force: proceed: bool = True else: diff --git a/canfar/cli/events.py b/canfar/cli/events.py index 43e412c7..54e7a4cb 100644 --- a/canfar/cli/events.py +++ b/canfar/cli/events.py @@ -11,16 +11,9 @@ from canfar.cli._run import run from canfar.sessions import AsyncSession -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console -events = typer.Typer( - name="events", - help="List events for sessions.", - no_args_is_help=False, -) - -@events.callback(invoke_without_command=True) def get_events( session_ids: Annotated[ list[str], @@ -28,6 +21,7 @@ def get_events( ], ) -> None: """Get events from the science platform server.""" + emit_cli_active_server_banner() async def _get_events() -> None: """Fetch events for the requested sessions and render them.""" diff --git a/canfar/cli/image.py b/canfar/cli/image.py index c67af7a8..f27e0882 100644 --- a/canfar/cli/image.py +++ b/canfar/cli/image.py @@ -12,7 +12,7 @@ from canfar.images import Images from canfar.models.types import Kind -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console if TYPE_CHECKING: from canfar.models.containers import Image @@ -80,6 +80,7 @@ def ls( ] = None, ) -> None: """List available images.""" + emit_cli_active_server_banner() payload: list[Image] = Images().details() images = [image for image in payload if not kind or kind in image.types] if not images: diff --git a/canfar/cli/info.py b/canfar/cli/info.py index c764c245..99609654 100644 --- a/canfar/cli/info.py +++ b/canfar/cli/info.py @@ -14,14 +14,7 @@ from canfar.cli._run import run from canfar.models.session import FetchResponse from canfar.sessions import AsyncSession -from canfar.utils.console import get_console - -info = typer.Typer( - name="info", - help="Get detailed information about sessions.", - no_args_is_help=True, - context_settings={"allow_interspersed_args": True}, -) +from canfar.utils.console import emit_cli_active_server_banner, get_console ALL_FIELDS: dict[str, str] = { "id": "Session ID", @@ -168,7 +161,6 @@ async def _get_info( _display(response, debug=debug) -@info.callback(invoke_without_command=True) def get_info( session_ids: Annotated[ list[str], @@ -183,4 +175,5 @@ def get_info( ] = False, ) -> None: """Get detailed information about one or more sessions.""" + emit_cli_active_server_banner() run(_get_info(session_ids, debug)) diff --git a/canfar/cli/login.py b/canfar/cli/login.py index c1b47fa1..e3dbaac6 100644 --- a/canfar/cli/login.py +++ b/canfar/cli/login.py @@ -4,6 +4,7 @@ from typing import Annotated +import httpx import typer from canfar import CONFIG_PATH @@ -18,7 +19,7 @@ activate, discover, ) -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console def _authentication_exists_on_disk(idp: str) -> bool: @@ -67,12 +68,18 @@ def _login_flow( idp_info = get_idp(idp) try: credential = authenticate_for_cli(idp_info, timeout=timeout, force=force) - except (ValueError, RuntimeError) as exc: + except ( + PermissionError, + TimeoutError, + ValueError, + RuntimeError, + httpx.HTTPError, + ) as exc: get_console(stderr=True).print(f"[bold red]{exc}[/bold red]") raise typer.Exit(1) from exc config = Configuration() # ty: ignore[missing-argument] - config.upsert_credential(credential) + config.editor.set(f"authentication.{credential.idp}", credential) try: servers = discover( @@ -142,6 +149,7 @@ def login_command( ] = 10, ) -> None: """Login to CANFAR Science Platform.""" + emit_cli_active_server_banner() selected_idp = idp or select_idp(list_idps()) try: get_idp(selected_idp) diff --git a/canfar/cli/logs.py b/canfar/cli/logs.py index f0d411a4..896c9fb1 100644 --- a/canfar/cli/logs.py +++ b/canfar/cli/logs.py @@ -8,16 +8,9 @@ from canfar.cli._run import run from canfar.sessions import AsyncSession -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console -logs = typer.Typer( - name="logs", - help="Get logs for sessions.", - no_args_is_help=True, -) - -@logs.callback(invoke_without_command=True) def get_logs( session_ids: Annotated[ list[str], @@ -25,6 +18,7 @@ def get_logs( ], ) -> None: """Get logs from the science platform server.""" + emit_cli_active_server_banner() async def _get_logs() -> None: """Fetch logs for the requested sessions and render them.""" diff --git a/canfar/cli/machine.py b/canfar/cli/machine.py index f433197c..60c91729 100644 --- a/canfar/cli/machine.py +++ b/canfar/cli/machine.py @@ -2,46 +2,36 @@ from __future__ import annotations -from typing import Annotated +from typing import Annotated, Literal import typer from canfar.cli import output -JsonOption = Annotated[ - bool, - typer.Option("--json", help="Emit machine-readable JSON on stdout."), +OutputOption = Annotated[ + Literal["json", "yaml"] | None, + typer.Option( + "-o", + "--output", + metavar="json|yaml", + help="Emit machine-readable JSON or YAML on stdout.", + ), ] -"""Leaf command flag for JSON machine output.""" - -YamlOption = Annotated[ - bool, - typer.Option("--yaml", help="Emit machine-readable YAML on stdout."), -] -"""Leaf command flag for YAML machine output.""" +"""Leaf command option for selecting machine output format.""" -def resolve_mode(json_output: bool, yaml_output: bool) -> output.OutputMode: - """Resolve machine output mode from leaf command flags. +def resolve_mode(output_format: str | None) -> output.OutputMode: + """Resolve machine output mode from the leaf output option. Args: - json_output: Whether ``--json`` was supplied. - yaml_output: Whether ``--yaml`` was supplied. + output_format: Requested machine output format, if any. Returns: Effective output mode for the invocation. Raises: - typer.Exit: Exit code 2 when both machine flags are supplied. + ValueError: If the format is not supported. """ - if json_output and yaml_output: - typer.echo( - "Conflicting machine output flags: use only one of --json or --yaml.", - err=True, - ) - raise typer.Exit(output.OUTPUT_CONFLICT_EXIT_CODE) - if json_output: - return output.OutputMode.JSON - if yaml_output: - return output.OutputMode.YAML - return output.OutputMode.HUMAN + if output_format is None: + return output.OutputMode.HUMAN + return output.OutputMode(output_format) diff --git a/canfar/cli/main.py b/canfar/cli/main.py index a7c6f0c0..9f6d699d 100644 --- a/canfar/cli/main.py +++ b/canfar/cli/main.py @@ -6,32 +6,28 @@ from typing import TYPE_CHECKING, Annotated import typer +from typer.core import TyperCommand, TyperGroup from canfar.cli import output from canfar.cli.auth import auth from canfar.cli.config import config -from canfar.cli.create import create +from canfar.cli.create import creation from canfar.cli.data import data -from canfar.cli.delete import delete -from canfar.cli.events import events +from canfar.cli.delete import delete_sessions +from canfar.cli.events import get_events from canfar.cli.image import image -from canfar.cli.info import info +from canfar.cli.info import get_info from canfar.cli.login import register_login_command -from canfar.cli.logs import logs -from canfar.cli.open import open_command -from canfar.cli.prune import prune -from canfar.cli.ps import ps +from canfar.cli.logs import get_logs +from canfar.cli.open import open_sessions +from canfar.cli.prune import prune_sessions +from canfar.cli.ps import show as show_sessions from canfar.cli.server import server -from canfar.cli.stats import stats -from canfar.cli.version import version +from canfar.cli.stats import get_stats +from canfar.cli.version import callback as version_callback from canfar.config.migration import ConfigResetRequiredError from canfar.exceptions.context import AuthContextError, AuthExpiredError -from canfar.hooks.typer.aliases import ( - ROOT_CHILD_ARGS_META_KEY, - AliasGroup, - set_before_command, -) -from canfar.utils.console import emit_active_server_banner, get_console +from canfar.utils.console import activate_cli_root, get_console from canfar.utils.logging import ( InvalidLogFilePathError, InvalidLoggingEnvironmentError, @@ -40,25 +36,61 @@ ) if TYPE_CHECKING: - from collections.abc import Mapping + from typer._click.core import Context as ClickContext from canfar.errors import StructuredError +_ROOT_CHILD_ARGS_META_KEY = "canfar.root_child_args" + + def _leaf_output_mode(args: list[str]) -> output.OutputMode: - """Infer an already-parsed leaf machine flag for root setup failures.""" - if "--json" in args: - return output.OutputMode.JSON - if "--yaml" in args: - return output.OutputMode.YAML + """Infer a leaf output option before root setup has completed.""" + for index, arg in enumerate(args): + if arg == "--": + break + if arg in {"-o", "--output"} and index + 1 < len(args): + value = args[index + 1] + if value in {"json", "yaml"}: + return output.OutputMode(value) + if arg.startswith("--output=") and arg.removeprefix("--output=") in { + "json", + "yaml", + }: + return output.OutputMode(arg.removeprefix("--output=")) + if arg.startswith("-o") and arg not in {"-o", "--output"}: + value = arg.removeprefix("-o") + if value in {"json", "yaml"}: + return output.OutputMode(value) return output.OutputMode.HUMAN -def _emit_banner_for_command(params: Mapping[str, object]) -> None: - """Emit the active-server banner for a parsed human-output command.""" - if params.get("json_output") or params.get("yaml_output"): - return - emit_active_server_banner() +class _RootTyperGroup(TyperGroup): + """Capture child argv so root setup can infer a leaf output mode.""" + + def parse_args(self, ctx: ClickContext, args: list[str]) -> list[str]: + """Record unconsumed child arguments without changing dispatch.""" + child_args = super().parse_args(ctx, args) + if ctx.parent is None: + ctx.meta[_ROOT_CHILD_ARGS_META_KEY] = list(child_args) + return child_args + + +_LEAF_USAGE = { + "create": "Usage: canfar create [OPTIONS] KIND IMAGE [-- CMD [ARGS]...]", + "prune": "Usage: canfar prune [OPTIONS] PREFIX KIND STATUS COMMAND [ARGS]...", +} + + +class _LeafUsageCommand(TyperCommand): + """Keep root leaf usage text aligned with delimiter-bearing commands.""" + + def get_usage(self, ctx: ClickContext) -> str: + """Return the canonical usage line for a leaf with custom syntax.""" + name = self.name or "" + if name in _LEAF_USAGE: + return _LEAF_USAGE[name] + return super().get_usage(ctx) def callback( @@ -88,9 +120,8 @@ def callback( ] = None, ) -> None: """Main callback that handles no subcommand case.""" - child_args: list[str] = ctx.meta.get(ROOT_CHILD_ARGS_META_KEY, []) - if "--" in child_args: - child_args = child_args[: child_args.index("--")] + activate_cli_root(ctx) + child_args: list[str] = ctx.meta.get(_ROOT_CHILD_ARGS_META_KEY, []) setup_mode = _leaf_output_mode(child_args) def warning_writer(error: StructuredError) -> None: @@ -116,8 +147,6 @@ def warning_writer(error: StructuredError) -> None: if ctx.invoked_subcommand is None: get_console().print(ctx.get_help()) raise typer.Exit(0) - if ctx.invoked_subcommand != "data": - set_before_command(ctx, _emit_banner_for_command) cli: typer.Typer = typer.Typer( @@ -133,7 +162,7 @@ def warning_writer(error: StructuredError) -> None: rich_help_panel="CANFAR CLI Commands", callback=callback, invoke_without_command=True, - cls=AliasGroup, + cls=_RootTyperGroup, ) register_login_command(cli) @@ -146,15 +175,6 @@ def warning_writer(error: StructuredError) -> None: rich_help_panel="Auth Management", ) -cli.add_typer( - auth, - name="authentication", - help="Alias for auth.", - no_args_is_help=False, - rich_help_panel="Aliases", - hidden=True, -) - cli.add_typer( server, name="server", @@ -169,79 +189,64 @@ def warning_writer(error: StructuredError) -> None: rich_help_panel="Data Management", ) -cli.add_typer( - create, +cli.command( + "create", + cls=_LeafUsageCommand, + context_settings={ + "help_option_names": ["-h", "--help"], + "allow_interspersed_args": True, + }, + help="Launch a new session.", no_args_is_help=True, rich_help_panel="Session Management", -) +)(creation) -cli.add_typer( - ps, - no_args_is_help=False, +cli.command( + "ps", + help="Show sessions.", rich_help_panel="Session Management", -) -cli.add_typer( - events, - no_args_is_help=False, +)(show_sessions) +cli.command( + "events", + help="List events for sessions.", rich_help_panel="Session Management", -) - -cli.add_typer( - info, +)(get_events) +cli.command( + "info", help="Show session info", - no_args_is_help=False, rich_help_panel="Session Management", -) - -cli.add_typer( - open_command, - name="open", +)(get_info) +cli.command( + "open", help="Open sessions in a browser", + context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True, rich_help_panel="Session Management", -) - -cli.add_typer( - logs, +)(open_sessions) +cli.command( + "logs", help="Show session logs", - no_args_is_help=False, rich_help_panel="Session Management", -) - -cli.add_typer( - delete, +)(get_logs) +cli.command( + "delete", + help="Delete sessions by ID.", no_args_is_help=True, rich_help_panel="Session Management", -) - -cli.add_typer( - prune, +)(delete_sessions) +cli.command( + "prune", + cls=_LeafUsageCommand, + context_settings={"help_option_names": ["-h", "--help"]}, + help="Delete sessions by criteria.", no_args_is_help=True, rich_help_panel="Session Management", -) - -cli.add_typer( - create, - name="run | launch", - help="Aliases for create.", - no_args_is_help=True, - rich_help_panel="Aliases", -) - -cli.add_typer( - delete, - name="del", - help="Aliases for delete.", - no_args_is_help=True, - rich_help_panel="Aliases", -) - -cli.add_typer( - stats, +)(prune_sessions) +cli.command( + "stats", help="Show cluster stats", - no_args_is_help=False, rich_help_panel="Cluster Information", -) +)(get_stats) cli.add_typer( image, @@ -258,13 +263,11 @@ def warning_writer(error: StructuredError) -> None: no_args_is_help=True, rich_help_panel="Client Info", ) -cli.add_typer( - version, - name="version", +cli.command( + "version", help="View client info", - no_args_is_help=False, rich_help_panel="Client Info", -) +)(version_callback) def main() -> None: diff --git a/canfar/cli/open.py b/canfar/cli/open.py index fec14e0a..83d51a87 100644 --- a/canfar/cli/open.py +++ b/canfar/cli/open.py @@ -9,15 +9,9 @@ from canfar.cli._run import run from canfar.sessions import AsyncSession, connection_url +from canfar.utils.console import emit_cli_active_server_banner -open_command = typer.Typer( - name="open", - help="Open sessions in a browser.", - no_args_is_help=True, -) - -@open_command.callback(invoke_without_command=True) def open_sessions( session_ids: Annotated[ list[str], @@ -25,6 +19,7 @@ def open_sessions( ], ) -> None: """Open one or more sessions in a web browser.""" + emit_cli_active_server_banner() async def _open_sessions() -> None: """Look up the requested sessions and open their connect URLs.""" diff --git a/canfar/cli/prune.py b/canfar/cli/prune.py index 1cb91d4a..d771d7f2 100644 --- a/canfar/cli/prune.py +++ b/canfar/cli/prune.py @@ -2,51 +2,17 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated, get_args +from typing import Annotated, get_args import click import typer -import typer.core from canfar.cli._run import run from canfar.models.types import Pruneable, Status from canfar.sessions import AsyncSession -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console -if TYPE_CHECKING: - from typer._click.core import Context - -class PruneUsageMessage(typer.core.TyperGroup): - """Custom usage message for prune command. - - Args: - typer (TyperGroup): Base class for grouping commands in Typer. - """ - - def get_usage(self, ctx: Context) -> str: # noqa: ARG002 - """Get the usage message for the prune command. - - Args: - ctx (typer.Context): The Typer context. - - Returns: - str: The usage message. - """ - return "Usage: canfar prune [OPTIONS] PREFIX KIND STATUS COMMAND [ARGS]..." - - -prune = typer.Typer( - name="prune", - no_args_is_help=True, -) - - -@prune.callback( - invoke_without_command=True, - context_settings={"help_option_names": ["-h", "--help"]}, - cls=PruneUsageMessage, -) def prune_sessions( prefix: Annotated[ str, @@ -82,6 +48,7 @@ def prune_sessions( canfar prune session-name headless Succeeded canfar prune 'session.*' notebook Running """ + emit_cli_active_server_banner() async def _prune() -> None: """Delete matching sessions from the science platform server.""" diff --git a/canfar/cli/ps.py b/canfar/cli/ps.py index 682160f8..0d1dd8af 100644 --- a/canfar/cli/ps.py +++ b/canfar/cli/ps.py @@ -15,26 +15,19 @@ from canfar.cli import output from canfar.cli._run import run -from canfar.cli.machine import JsonOption, YamlOption, resolve_mode +from canfar.cli.machine import OutputOption, resolve_mode from canfar.config.migration import ConfigResetRequiredError from canfar.exceptions.context import AuthContextError, AuthExpiredError -from canfar.hooks.typer.aliases import AliasGroup from canfar.models.session import FetchResponse from canfar.models.types import Kind, Status from canfar.sessions import AsyncSession -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console if TYPE_CHECKING: from typing import NoReturn from canfar.errors import StructuredError -ps = typer.Typer( - name="ps", - no_args_is_help=False, - cls=AliasGroup, -) - async def _fetch_sessions( kind: Kind | None, @@ -156,12 +149,11 @@ def _render_human_sessions( get_console(stderr=True).print(f"[dim]- {message}[/dim]") -@ps.callback(invoke_without_command=True) def show( everything: Annotated[ bool, typer.Option( - "--all", "-a", help="Show all sessions (default shows just running)." + "--all", "-a", help="Show all sessions (default shows Pending and Running)." ), ] = False, quiet: Annotated[ @@ -195,16 +187,17 @@ def show( help="Show Session response warnings.", ), ] = False, - json_output: JsonOption = False, - yaml_output: YamlOption = False, + output_format: OutputOption = None, ) -> None: """Show sessions.""" - mode = resolve_mode(json_output, yaml_output) + mode = resolve_mode(output_format) + if mode is output.OutputMode.HUMAN: + emit_cli_active_server_banner() if quiet and mode is not output.OutputMode.HUMAN: typer.echo( "Incompatible flags: --quiet is human-only and cannot be used with " - "--json or --yaml.", + "--output json or --output yaml.", err=True, ) raise typer.Exit(output.OUTPUT_CONFLICT_EXIT_CODE) diff --git a/canfar/cli/server.py b/canfar/cli/server.py index a344e512..b5474374 100644 --- a/canfar/cli/server.py +++ b/canfar/cli/server.py @@ -11,10 +11,9 @@ from canfar.authentication import AuthenticationError from canfar.authentication import show as auth_show from canfar.cli import output -from canfar.cli.machine import JsonOption, YamlOption, resolve_mode +from canfar.cli.machine import OutputOption, resolve_mode from canfar.config.migration import ConfigResetRequiredError from canfar.errors import ErrorCode, StructuredError -from canfar.hooks.typer.aliases import AliasGroup from canfar.server import ( ServerDiscoveryError, ServerFetchError, @@ -26,7 +25,7 @@ from canfar.server import ( use as server_use, ) -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console if TYPE_CHECKING: from canfar.models.http import Server @@ -35,7 +34,6 @@ name="server", help="Manage science platform servers.", no_args_is_help=True, - cls=AliasGroup, ) @@ -63,13 +61,12 @@ def _render_server_list_table(servers: list[Server]) -> None: get_console().print(table) -@server.command("list, ls") +@server.command("ls") def server_list_command( - json_output: JsonOption = False, - yaml_output: YamlOption = False, + output_format: OutputOption = None, ) -> None: """List servers for the active Identity Provider.""" - mode = resolve_mode(json_output, yaml_output) + mode = resolve_mode(output_format) try: auth_show() @@ -119,6 +116,7 @@ def server_list_command( output.to_stdout(servers, mode) return + emit_cli_active_server_banner() _render_server_list_table(servers) @@ -127,6 +125,7 @@ def server_use_command( selector: Annotated[str, typer.Argument(help="Server name or URI.")], ) -> None: """Select the active server by name or URI.""" + emit_cli_active_server_banner() try: server_use(selector) except ServerSelectorError as exc: diff --git a/canfar/cli/stats.py b/canfar/cli/stats.py index cce75874..2d4c2ed4 100644 --- a/canfar/cli/stats.py +++ b/canfar/cli/stats.py @@ -2,24 +2,17 @@ from __future__ import annotations -import typer from rich import box from rich.table import Table from canfar.cli._run import run from canfar.sessions import AsyncSession -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console -stats = typer.Typer( - name="stats", - help="Display cluster-wide statistics.", - no_args_is_help=False, -) - -@stats.callback(invoke_without_command=True) def get_stats() -> None: """Display cluster-wide usage and status statistics.""" + emit_cli_active_server_banner() async def _get_stats() -> None: """Fetch cluster-wide statistics and render them.""" diff --git a/canfar/cli/version.py b/canfar/cli/version.py index cd4fc217..443f7dd6 100644 --- a/canfar/cli/version.py +++ b/canfar/cli/version.py @@ -10,7 +10,7 @@ from rich.table import Table from canfar import __version__ -from canfar.utils.console import get_console +from canfar.utils.console import emit_cli_active_server_banner, get_console def callback( @@ -22,6 +22,7 @@ def callback( ), ) -> None: """CANFAR Python Client version information.""" + emit_cli_active_server_banner() if not debug: # Simple version output get_console().print(f"CANFAR Python Client {__version__}") @@ -89,16 +90,6 @@ def callback( raise typer.Exit(0) -version = typer.Typer( - name="version", - help="Show canfar client version information", - no_args_is_help=False, - rich_help_panel="Information Commands", - callback=callback, - invoke_without_command=True, -) - - def _get_package_version(name: str) -> str: """Get version of an installed package. diff --git a/canfar/client.py b/canfar/client.py index 5ff1bd53..e25f2ab5 100644 --- a/canfar/client.py +++ b/canfar/client.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio +import logging import ssl from datetime import datetime, timezone from email.utils import formatdate @@ -19,7 +21,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from typing_extensions import Self -from canfar import __version__, get_logger +from canfar import __version__ from canfar.auth import oidc, x509 from canfar.exceptions.context import AuthContextError from canfar.hooks.httpx import auth, debug, errors, expiry @@ -34,7 +36,12 @@ if TYPE_CHECKING: from types import TracebackType -log = get_logger(__name__) +log = logging.getLogger(__name__) + + +def _has_runtime_token(token: SecretStr | None) -> bool: + """Return whether a runtime token contains a non-empty value.""" + return token is not None and bool(token.get_secret_value()) class HTTPClient(BaseSettings): @@ -123,6 +130,7 @@ class HTTPClient(BaseSettings): # Private attributes _client: Client | None = PrivateAttr(default=None) _asynclient: AsyncClient | None = PrivateAttr(default=None) + _refresh_lock: asyncio.Lock | None = PrivateAttr(default=None) # Client Properties @property @@ -148,7 +156,7 @@ def asynclient(self) -> AsyncClient: @property def uses_runtime_credentials(self) -> bool: """Return whether runtime token or certificate credentials are active.""" - return bool(self.token or self.certificate) + return _has_runtime_token(self.token) or self.certificate is not None @property def authentication_record(self) -> AuthenticationCredential | None: @@ -159,7 +167,7 @@ def authentication_record(self) -> AuthenticationCredential | None: else self.authentication_idp ) try: - credential = self.config.get_credential(idp) + credential = self.config.authentication[idp] except KeyError: return None if isinstance(credential, X509Credential) and credential.path is None: @@ -176,17 +184,17 @@ def _validate(self) -> Self: Returns: Self: The configured client. """ - if self.token and self.certificate: + if _has_runtime_token(self.token) and self.certificate is not None: log.warning("Both runtime token and certificate values provided.") log.warning("Runtime token takes precedence over certificate.") log.warning("Certificate will be ignored in favor of the token.") self.certificate = None # Nullify certificate to ensure token is used - if (self.token or self.certificate) and not self.url: + if self.uses_runtime_credentials and not self.url: msg = "Server URL must be provided when using runtime credentials." raise ValueError(msg) - if self.certificate: + if self.certificate is not None: info = x509.inspect(self.certificate) expiry = datetime.fromtimestamp(info["expiry"], tz=timezone.utc).isoformat() msg = f"{self.certificate} valid till {expiry}" @@ -232,6 +240,28 @@ def _resolved_authentication_record(self) -> AuthenticationCredential | None: ) return credential + def _get_refresh_lock(self) -> asyncio.Lock: + """Return the one async refresh lock shared by this client.""" + if self._refresh_lock is None: + self._refresh_lock = asyncio.Lock() + return self._refresh_lock + + async def _refresh_oidc(self) -> OIDCCredential | None: + """Resolve and refresh the canonical saved OIDC record under one lock.""" + async with self._get_refresh_lock(): + prepared = auth._refresh(self) # noqa: SLF001 + if prepared is None: + return None + credential, parameters = prepared + if parameters is None: + return credential + refreshed = await oidc.refresh(*parameters) + return oidc._persist( # noqa: SLF001 + self.config, + credential, + refreshed, + ) + @classmethod def build( cls, @@ -274,11 +304,9 @@ async def _materialize_credentials(self) -> RuntimeCredential: Returns: RuntimeCredential: Exactly one of a bearer token or a certificate. """ - if self.token is not None: - token = self.token.get_secret_value() - if token: - return RuntimeCredential(token=token) - raise ValueError + if _has_runtime_token(self.token): + assert self.token is not None + return RuntimeCredential(token=self.token.get_secret_value()) if self.certificate is not None: return RuntimeCredential(certificate=x509.valid(self.certificate)) @@ -292,16 +320,9 @@ async def _materialize_credentials(self) -> RuntimeCredential: if not isinstance(credential, OIDCCredential): raise TypeError - if credential.expired: - parameters = oidc._refresh(credential) # noqa: SLF001 - if parameters is None: - raise ValueError - refreshed = await oidc.refresh(*parameters) - credential = oidc._persist( # noqa: SLF001 - self.config, - credential, - refreshed, - ) + credential = await self._refresh_oidc() + if credential is None: + raise ValueError if credential.token.access is None: raise ValueError token = credential.token.access.get_secret_value() @@ -317,8 +338,14 @@ def _get_base_url(self) -> URL: """ if self.url: return URL(str(self.url)) + if self.config.active.server is None: + msg = ( + "Server not found for Authentication Record: " + f"{self.config.active.authentication}" + ) + raise ValueError(msg) try: - server = self.config.get_active_server() + server = self.config.servers[self.config.active.server] except KeyError as exc: msg = ( "Server not found for Authentication Record: " @@ -345,20 +372,12 @@ def _get_client_kwargs( Returns: dict[str, Any]: Keyword arguments for creating an HTTPx client. """ - catcher = errors.acatch if asynchronous else errors.catch - req_logger = debug.arequest if asynchronous else debug.request - resp_logger = debug.aresponse if asynchronous else debug.response - response_hooks = [resp_logger] - if self.raise_http_errors: - response_hooks.append(catcher) - request_hooks: list[Any] = [] - if credential is not None: - checker = expiry.acheck(self) if asynchronous else expiry.check(self) - request_hooks.append(checker) - request_hooks.append(req_logger) kwargs: dict[str, Any] = { "timeout": Timeout(self.timeout), - "event_hooks": {"request": request_hooks, "response": response_hooks}, + "event_hooks": self._get_event_hooks( + asynchronous=asynchronous, + credential=credential, + ), "base_url": self._get_base_url(), } if asynchronous: @@ -366,20 +385,14 @@ def _get_client_kwargs( max_connections=self.concurrency, max_keepalive_connections=self.concurrency // 4, ) - if self.token: + if _has_runtime_token(self.token): return kwargs - if self.certificate: - msg = "creating runtime ssl context with: {self.certificate}" - log.debug(msg) + if self.certificate is not None: + log.debug("Creating runtime SSL context with: %s", self.certificate) kwargs["verify"] = self._get_ssl_context(self.certificate) return kwargs - if isinstance(credential, OIDCCredential): - refresher = auth.arefresh(self) if asynchronous else auth.refresh(self) - kwargs["event_hooks"]["request"].insert(0, refresher) - return kwargs - if isinstance(credential, X509Credential): if credential.path is None: raise AuthContextError( @@ -397,6 +410,31 @@ def _get_client_kwargs( return kwargs return kwargs + def _get_event_hooks( + self, + *, + asynchronous: bool, + credential: AuthenticationCredential | None, + ) -> dict[str, list[Any]]: + """Build native HTTPX hooks from the shared transport policy.""" + catcher = errors.acatch if asynchronous else errors.catch + request_logger = debug.arequest if asynchronous else debug.request + response_logger = debug.aresponse if asynchronous else debug.response + request_hooks: list[Any] = [] + if credential is not None: + if isinstance(credential, OIDCCredential): + request_hooks.append( + auth.arefresh(self) if asynchronous else auth.refresh(self) + ) + request_hooks.append( + expiry.acheck(self) if asynchronous else expiry.check(self) + ) + request_hooks.append(request_logger) + response_hooks = [response_logger] + if self.raise_http_errors: + response_hooks.append(catcher) + return {"request": request_hooks, "response": response_hooks} + def _get_ssl_context(self, source: Path) -> ssl.SSLContext: """Get SSL context from certificate file. @@ -432,10 +470,11 @@ def _get_http_headers( "User-Agent": f"python-canfar/{__version__}", } - if self.token: + if _has_runtime_token(self.token): + assert self.token is not None headers["Authorization"] = f"Bearer {self.token.get_secret_value()}" headers["X-Skaha-Authentication-Type"] = "RUNTIME-TOKEN" - elif self.certificate: + elif self.certificate is not None: headers["X-Skaha-Authentication-Type"] = "RUNTIME-X509" elif isinstance(credential, OIDCCredential): if credential.token.access is not None: @@ -459,7 +498,6 @@ def _get_http_headers( # Context Manager Methods def __enter__(self) -> Self: """Sync context manager entry.""" - log.debug("Entering synchronous context manager") return self def __exit__( @@ -469,22 +507,16 @@ def __exit__( exc_tb: TracebackType | None, ) -> None: """Sync context manager exit.""" - log.debug("Exiting synchronous context manager") self._close() def _close(self) -> None: """Close sync client.""" - if self._client: - log.debug("Closing synchronous HTTPx client") - self._client.close() - self._client = None - log.debug("Synchronous HTTPx client closed") - else: - log.debug("No synchronous client to close") + client, self._client = self._client, None + if client is not None: + client.close() async def __aenter__(self) -> Self: """Async context manager entry.""" - log.debug("Entering asynchronous context manager") return self async def __aexit__( @@ -494,15 +526,10 @@ async def __aexit__( exc_tb: TracebackType | None, ) -> None: """Async context manager exit.""" - log.debug("Exiting asynchronous context manager") await self._aclose() async def _aclose(self) -> None: """Close async client.""" - if self._asynclient: - log.debug("Closing asynchronous HTTPx client") - await self._asynclient.aclose() - self._asynclient = None - log.debug("Asynchronous HTTPx client closed") - else: - log.debug("No asynchronous client to close") + client, self._asynclient = self._asynclient, None + if client is not None: + await client.aclose() diff --git a/canfar/config/__init__.py b/canfar/config/__init__.py index e1370d70..00453ab3 100644 --- a/canfar/config/__init__.py +++ b/canfar/config/__init__.py @@ -1 +1 @@ -"""Configuration loading, storage, and compatibility helpers.""" +"""Configuration editing and compatibility helpers.""" diff --git a/canfar/config/editor.py b/canfar/config/editor.py index 27ccc14d..5e013d1b 100644 --- a/canfar/config/editor.py +++ b/canfar/config/editor.py @@ -2,29 +2,36 @@ from __future__ import annotations +import os +import tempfile +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, Any +import yaml +from pydantic import ValidationError + +from canfar.models.auth import OIDCCredential + if TYPE_CHECKING: from canfar.models.config import Configuration -def _parse_dotted_path(path: str) -> list[str | int]: - segments: list[str | int] = [] +def _parse_dotted_path(path: str) -> list[str]: + segments: list[str] = [] for raw in path.split("."): if not raw: msg = f"Invalid path {path!r}: empty segment" raise ValueError(msg) - segments.append(int(raw) if raw.isdigit() else raw) + if raw.isdigit(): + msg = "List indices are not supported in configuration paths" + raise ValueError(msg) + segments.append(raw) return segments -def _get_from_container(container: Any, key: str | int) -> Any: - if isinstance(key, int): - if not isinstance(container, list): - msg = f"Expected list for index {key}" - raise TypeError(msg) - return container[key] - +def _get_from_container(container: Any, key: str) -> Any: if isinstance(container, dict): return container[key] @@ -32,14 +39,7 @@ def _get_from_container(container: Any, key: str | int) -> Any: raise KeyError(msg) -def _set_in_container(container: Any, key: str | int, value: Any) -> None: - if isinstance(key, int): - if not isinstance(container, list): - msg = f"Expected list for index {key}" - raise TypeError(msg) - container[key] = value - return - +def _set_in_container(container: Any, key: str, value: Any) -> None: if isinstance(container, dict): container[key] = value return @@ -48,11 +48,7 @@ def _set_in_container(container: Any, key: str | int, value: Any) -> None: raise TypeError(msg) -def _ensure_child_container(parent: Any, key: str | int) -> Any: - if isinstance(key, int): - msg = "List indices are not supported for intermediate path segments" - raise TypeError(msg) - +def _ensure_child_container(parent: Any, key: str) -> Any: if not isinstance(parent, dict): msg = f"Expected mapping for key {key!r}" raise TypeError(msg) @@ -70,13 +66,21 @@ def get_value(config: Configuration, path: str) -> Any: return value -def set_value(config: Configuration, path: str, value: Any) -> Configuration: - """Return a new validated configuration with a dotted-path value updated.""" - segments = _parse_dotted_path(path) - if not segments: - msg = "Path cannot be empty" - raise ValueError(msg) +def _validated_copy(config: Configuration, **updates: Any) -> Configuration: + """Validate a source-isolated copy of a complete Configuration.""" + data = {**config.model_dump(mode="python"), **updates} + candidate = config.__class__.model_construct() + # Full validation must not re-enter BaseSettings persisted sources. + config.__class__.__pydantic_validator__.validate_python( + data, + self_instance=candidate, + ) + return candidate + +def _updated_data(config: Configuration, path: str, value: Any) -> dict[str, Any]: + """Return serialized top-level data with one dotted-path value updated.""" + segments = _parse_dotted_path(path) data = config.model_dump(mode="python") cursor: Any = data @@ -84,4 +88,98 @@ def set_value(config: Configuration, path: str, value: Any) -> Configuration: cursor = _ensure_child_container(cursor, segment) _set_in_container(cursor, segments[-1], value) - return config.__class__.model_validate(data) + return data + + +def set_value(config: Configuration, path: str, value: Any) -> Configuration: + """Return a new validated configuration with a dotted-path value updated.""" + return _validated_copy(config, **_updated_data(config, path, value)) + + +def _restore_oidc_secrets(config: Configuration, data: dict[str, Any]) -> None: + """Replace masked ``SecretStr`` placeholders with values for YAML persistence.""" + authentication = data.get("authentication") + if not isinstance(authentication, dict): + return + + for idp, credential_data in authentication.items(): + credential = config.authentication.get(idp) + if not isinstance(credential, OIDCCredential) or not isinstance( + credential_data, dict + ): + continue + + client = credential_data.get("client") + if isinstance(client, dict) and credential.client.secret is not None: + client["secret"] = credential.client.secret.get_secret_value() + + token = credential_data.get("token") + if not isinstance(token, dict): + continue + if credential.token.access is not None: + token["access"] = credential.token.access.get_secret_value() + if credential.token.refresh is not None: + token["refresh"] = credential.token.refresh.get_secret_value() + + +def _default_config_path() -> Path: + """Resolve the configured YAML path lazily to preserve test isolation.""" + from canfar.models.config import CONFIG_PATH # noqa: PLC0415 + + return CONFIG_PATH + + +def _save_config(config: Configuration, path: Path | None = None) -> None: + """Atomically save a validated Configuration to YAML.""" + target = path or _default_config_path() + target.parent.mkdir(parents=True, exist_ok=True) + temporary: Path | None = None + try: + candidate = _validated_copy(config) + data = candidate.model_dump(mode="json", exclude_none=True) + _restore_oidc_secrets(candidate, data) + serialized = yaml.dump(data, default_flow_style=False, sort_keys=True, indent=2) + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=target.parent, + prefix=f".{target.name}.", + delete=False, + ) as handle: + temporary = Path(handle.name) + handle.write(serialized) + handle.flush() + os.fsync(handle.fileno()) + temporary.replace(target) + except (OSError, TypeError, ValidationError) as exc: + if temporary is not None: + with suppress(OSError): + temporary.unlink(missing_ok=True) + msg = f"Failed to save configuration to {target}: {exc}" + raise OSError(msg) from exc + + +@dataclass(slots=True) +class ConfigurationEditor: + """Bound editing and persistence boundary for a Configuration.""" + + _config: Configuration + + def _set_top_level(self, **updates: Any) -> Configuration: + """Validate and install one complete top-level configuration update.""" + updated = _validated_copy(self._config, **updates) + self._config.__dict__ = updated.__dict__.copy() + self._config.__pydantic_fields_set__ = updated.__pydantic_fields_set__.copy() + return self._config + + def get(self, key: str) -> Any: + """Read a scalar, mapping, or whole-list value by dotted path.""" + return get_value(self._config, key) + + def set(self, key: str, value: Any) -> Configuration: + """Validate and install a dotted-path update on the bound config.""" + return self._set_top_level(**_updated_data(self._config, key, value)) + + def save(self) -> None: + """Atomically persist the bound Configuration.""" + _save_config(self._config) diff --git a/canfar/config/store.py b/canfar/config/store.py deleted file mode 100644 index 475085c2..00000000 --- a/canfar/config/store.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Persistence helpers for configuration objects.""" - -from __future__ import annotations - -import os -import tempfile -from contextlib import suppress -from pathlib import Path -from typing import TYPE_CHECKING, Any - -import yaml -from pydantic import ValidationError - -from canfar.models.auth import OIDCCredential - -if TYPE_CHECKING: - from canfar.models.config import Configuration - - -def _default_config_path() -> Path: - from canfar.models.config import CONFIG_PATH # noqa: PLC0415 - - return CONFIG_PATH - - -def _restore_oidc_secrets(config: Configuration, data: dict[str, Any]) -> None: - """Replace masked ``SecretStr`` placeholders with values for YAML persistence.""" - authentication = data.get("authentication") - if not isinstance(authentication, dict): - return - - for idp, credential_data in authentication.items(): - credential = config.authentication.get(idp) - if not isinstance(credential, OIDCCredential) or not isinstance( - credential_data, dict - ): - continue - - client = credential_data.get("client") - if isinstance(client, dict) and credential.client.secret is not None: - client["secret"] = credential.client.secret.get_secret_value() - - token = credential_data.get("token") - if not isinstance(token, dict): - continue - if credential.token.access is not None: - token["access"] = credential.token.access.get_secret_value() - if credential.token.refresh is not None: - token["refresh"] = credential.token.refresh.get_secret_value() - - -def save_config(config: Configuration, path: Path | None = None) -> None: - """Save ``config`` to YAML.""" - target = path or _default_config_path() - target.parent.mkdir(parents=True, exist_ok=True) - temporary: Path | None = None - try: - candidate = config._validated_copy() # noqa: SLF001 - data = candidate.model_dump(mode="json", exclude_none=True) - _restore_oidc_secrets(candidate, data) - serialized = yaml.dump(data, default_flow_style=False, sort_keys=True, indent=2) - with tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - dir=target.parent, - prefix=f".{target.name}.", - delete=False, - ) as handle: - temporary = Path(handle.name) - handle.write(serialized) - handle.flush() - os.fsync(handle.fileno()) - temporary.replace(target) - except (OSError, TypeError, ValidationError) as exc: - if temporary is not None: - with suppress(OSError): - temporary.unlink(missing_ok=True) - msg = f"Failed to save configuration to {target}: {exc}" - raise OSError(msg) from exc diff --git a/canfar/hooks/httpx/auth.py b/canfar/hooks/httpx/auth.py index 03e2e4f5..20888453 100644 --- a/canfar/hooks/httpx/auth.py +++ b/canfar/hooks/httpx/auth.py @@ -29,10 +29,9 @@ from __future__ import annotations -import asyncio -from typing import TYPE_CHECKING, Any, Callable +import logging +from typing import TYPE_CHECKING, Callable -from canfar import get_logger from canfar.auth import oidc from canfar.models.auth import OIDCCredential @@ -44,13 +43,16 @@ from canfar.client import HTTPClient -log = get_logger(__name__) +log = logging.getLogger(__name__) class AuthenticationError(Exception): """Exception raised when authentication refresh fails.""" +RefreshParameters = tuple[str, str, str, str] + + def _get_oidc_credential(client: HTTPClient) -> OIDCCredential | None: """Return the selected canonical OIDC record unless runtime auth wins.""" if client.uses_runtime_credentials: @@ -59,6 +61,23 @@ def _get_oidc_credential(client: HTTPClient) -> OIDCCredential | None: return credential if isinstance(credential, OIDCCredential) else None +def _refresh( + client: HTTPClient, +) -> tuple[OIDCCredential, RefreshParameters | None] | None: + """Resolve one OIDC record and prepare its refresh inputs.""" + credential = _get_oidc_credential(client) + if credential is None: + log.debug("Skipping auth refresh without a saved OIDC record.") + return None + if not credential.expired: + return credential, None + parameters = oidc._refresh(credential) # noqa: SLF001 + if parameters is None: + log.warning("OIDC Authentication Record cannot be refreshed.") + return None + return credential, parameters + + def _apply_access_header( token: SecretStr, httpx_client_headers: MutableMapping[str, str], @@ -70,25 +89,6 @@ def _apply_access_header( request.headers["Authorization"] = header -def _apply_refreshed_token( - client: HTTPClient, - credential: OIDCCredential, - refreshed: dict[str, Any], - httpx_client_headers: MutableMapping[str, str], - request: httpx.Request, -) -> None: - """Atomically persist refreshed OIDC state, then update active headers.""" - updated = oidc._persist( # noqa: SLF001 - client.config, credential, refreshed - ) - log.debug("Authentication refreshed and configuration saved.") - - assert updated.token.access is not None - _apply_access_header(updated.token.access, httpx_client_headers, request) - log.debug("HTTP request headers updated with new token.") - log.info("OIDC Access Token Refreshed.") - - def refresh(client: HTTPClient) -> Callable[[httpx.Request], None]: """Create an authentication refresh hook for httpx clients. @@ -105,12 +105,11 @@ def hook(request: httpx.Request) -> None: Args: request (httpx.Request): The outgoing HTTP request. """ - credential = _get_oidc_credential(client) - if credential is None: - log.debug("Skipping auth refresh without a saved OIDC record.") + prepared = _refresh(client) + if prepared is None: return - - if not credential.expired: + credential, parameters = prepared + if parameters is None: if credential.token.access is not None: _apply_access_header( credential.token.access, @@ -119,11 +118,6 @@ def hook(request: httpx.Request) -> None: ) log.debug("Skipping auth refresh, access token is not expired.") return - - parameters = oidc._refresh(credential) # noqa: SLF001 - if parameters is None: - log.warning("OIDC Authentication Record cannot be refreshed.") - return token_url, identity, client_secret, refresh_token = parameters try: @@ -135,13 +129,15 @@ def hook(request: httpx.Request) -> None: token=refresh_token, ) log.debug("Synchronous OIDC token refresh successful.") - _apply_refreshed_token( - client, - credential, - token, - client.client.headers, - request, + updated = oidc._persist( # noqa: SLF001 + client.config, credential, token ) + log.debug("Authentication refreshed and configuration saved.") + + assert updated.token.access is not None + _apply_access_header(updated.token.access, client.client.headers, request) + log.debug("HTTP request headers updated with new token.") + log.info("OIDC Access Token Refreshed.") except (ValueError, OSError): msg = "Failed to refresh OIDC token" @@ -159,7 +155,6 @@ def arefresh(client: HTTPClient) -> Callable[[httpx.Request], Awaitable[None]]: Returns: Callable[[httpx.Request], Awaitable[None]]: The async auth hook. """ - lock = asyncio.Lock() async def ahook(request: httpx.Request) -> None: """Asynchronous refresh hook for httpx clients. @@ -167,46 +162,33 @@ async def ahook(request: httpx.Request) -> None: Args: request (httpx.Request): The outgoing HTTP request. """ - async with lock: - credential = _get_oidc_credential(client) - if credential is None: - log.debug("Skipping auth refresh without a saved OIDC record.") - return - - if not credential.expired: - if credential.token.access is not None: - _apply_access_header( - credential.token.access, - client.asynclient.headers, - request, - ) - return - - parameters = oidc._refresh(credential) # noqa: SLF001 - if parameters is None: - log.warning("OIDC Authentication Record cannot be refreshed.") - return - token_url, identity, client_secret, refresh_token = parameters - - try: - log.debug("Starting asynchronous OIDC token refresh.") - token = await oidc.refresh( - url=token_url, - identity=identity, - secret=client_secret, - token=refresh_token, - ) - log.debug("Asynchronous OIDC token refresh successful.") - _apply_refreshed_token( - client, - credential, - token, + previous = client.authentication_record + if isinstance(previous, OIDCCredential) and previous.expired: + log.debug("Starting asynchronous OIDC token refresh.") + try: + credential = await client._refresh_oidc() # noqa: SLF001 + except (ValueError, OSError): + msg = "Failed to refresh OIDC token" + raise AuthenticationError(msg) from None + if credential is None: + return + if credential == previous: + if credential.token.access is not None: + _apply_access_header( + credential.token.access, client.asynclient.headers, request, ) - - except (ValueError, OSError): - msg = "Failed to refresh OIDC token" - raise AuthenticationError(msg) from None + log.debug("Skipping auth refresh, access token is not expired.") + return + log.debug("Asynchronous OIDC token refresh successful.") + if credential.token.access is not None: + _apply_access_header( + credential.token.access, + client.asynclient.headers, + request, + ) + log.debug("HTTP request headers updated with new token.") + log.info("OIDC Access Token Refreshed.") return ahook diff --git a/canfar/hooks/httpx/debug.py b/canfar/hooks/httpx/debug.py index cb4963d9..399e9f30 100644 --- a/canfar/hooks/httpx/debug.py +++ b/canfar/hooks/httpx/debug.py @@ -5,12 +5,10 @@ import logging from typing import TYPE_CHECKING -from canfar import get_logger - if TYPE_CHECKING: import httpx -log = get_logger(__name__) +log = logging.getLogger(__name__) def request(req: httpx.Request) -> None: diff --git a/canfar/hooks/httpx/errors.py b/canfar/hooks/httpx/errors.py index e21330f1..f31a8ced 100644 --- a/canfar/hooks/httpx/errors.py +++ b/canfar/hooks/httpx/errors.py @@ -6,14 +6,14 @@ """ import contextlib +import logging from collections.abc import Generator import httpx -from canfar import get_logger from canfar.utils.logging import safe_url -log = get_logger(__name__) +log = logging.getLogger(__name__) CONN_ERR_MSG = ( "Failed to establish connection within the timeout period. " diff --git a/canfar/hooks/httpx/expiry.py b/canfar/hooks/httpx/expiry.py index 30c84e2e..9ebc7a68 100644 --- a/canfar/hooks/httpx/expiry.py +++ b/canfar/hooks/httpx/expiry.py @@ -2,13 +2,13 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING, Callable -from canfar import get_logger from canfar.auth import x509 from canfar.exceptions.context import AuthExpiredError -log = get_logger(__name__) +log = logging.getLogger(__name__) if TYPE_CHECKING: from collections.abc import Awaitable diff --git a/canfar/hooks/typer/__init__.py b/canfar/hooks/typer/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/canfar/hooks/typer/aliases.py b/canfar/hooks/typer/aliases.py deleted file mode 100644 index 03decf9e..00000000 --- a/canfar/hooks/typer/aliases.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Typer group extensions for aliases and terminal command dispatch.""" - -from __future__ import annotations - -import re -from typing import TYPE_CHECKING, Any, TypeVar - -from typer import Context -from typer.core import TyperGroup - -if TYPE_CHECKING: - from collections.abc import Callable, Mapping - - from typer._click.core import Command - from typer._click.core import Context as ClickContext - -ROOT_CHILD_ARGS_META_KEY = "canfar.root_child_args" -_BEFORE_COMMAND_META_KEY = "canfar.before_command" -_Value = TypeVar("_Value") - - -def set_before_command( - ctx: Context, - callback: Callable[[Mapping[str, object]], None], -) -> None: - """Run a callback after terminal command parsing and before its handler.""" - ctx.meta[_BEFORE_COMMAND_META_KEY] = callback - - -def _run_before_command(ctx: Context) -> None: - callback = ctx.meta.pop(_BEFORE_COMMAND_META_KEY, None) - if callback is not None: - callback(ctx.params) - - -class _BeforeCommandContext(Context): - """Run the pending hook immediately before a terminal command callback.""" - - def invoke( - self, - callback: Callable[..., _Value], - /, - *args: Any, - **kwargs: Any, - ) -> _Value: - if not isinstance(self.command, TyperGroup) or self.invoked_subcommand is None: - _run_before_command(self) - return super().invoke(callback, *args, **kwargs) - - -def _install_before_command_context(command: Command) -> None: - command.context_class = _BeforeCommandContext - if isinstance(command, TyperGroup): - for child in command.commands.values(): - _install_before_command_context(child) - - -class AliasGroup(TyperGroup): - """Typer group with command aliases and one-shot terminal hooks.""" - - _CMD_SPLIT_P = re.compile(r" ?[,|] ?") - context_class = _BeforeCommandContext - - def parse_args(self, ctx: ClickContext, args: list[str]) -> list[str]: - """Preserve the root command's parsed child arguments for setup errors.""" - child_args = super().parse_args(ctx, args) - if ctx.parent is None: - ctx.meta[ROOT_CHILD_ARGS_META_KEY] = list(child_args) - return child_args - - def get_command(self, ctx: ClickContext, cmd_name: str) -> Command | None: - """Retrieve a command by name, supporting aliases. - - Args: - ctx (Context): The Click context. - cmd_name (str): The command name or alias. - - Returns: - Command | None: The matched command or None if not found. - """ - cmd_name = self._group_cmd_name(cmd_name) - command = super().get_command(ctx, cmd_name) - if command is not None: - _install_before_command_context(command) - return command - - def _group_cmd_name(self, default: str) -> str: - for cmd in self.commands.values(): - name: str = getattr(cmd, "name", "") - if name and default in self._CMD_SPLIT_P.split(name): - return name - return default diff --git a/canfar/images.py b/canfar/images.py index 2b65b427..a28e5fe5 100644 --- a/canfar/images.py +++ b/canfar/images.py @@ -2,16 +2,16 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING -from canfar import get_logger from canfar.client import HTTPClient from canfar.models.containers import Image if TYPE_CHECKING: from httpx import Response -log = get_logger(__name__) +log = logging.getLogger(__name__) class Images(HTTPClient): diff --git a/canfar/models/auth.py b/canfar/models/auth.py index 40a1de42..01d49c91 100644 --- a/canfar/models/auth.py +++ b/canfar/models/auth.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import math import time from pathlib import Path # noqa: TC003 @@ -9,10 +10,9 @@ from pydantic import BaseModel, ConfigDict, Field, SecretStr -from canfar import get_logger from canfar.auth import x509 -log = get_logger(__name__) +log = logging.getLogger(__name__) def _secret_present(value: SecretStr | str | None) -> bool: diff --git a/canfar/models/config.py b/canfar/models/config.py index 7b7ed71a..3e4c37cf 100644 --- a/canfar/models/config.py +++ b/canfar/models/config.py @@ -16,8 +16,6 @@ ) if TYPE_CHECKING: - from collections.abc import Iterable - from pydantic.fields import FieldInfo from pydantic_settings import ( @@ -28,19 +26,16 @@ ) from pydantic_settings.sources import EnvSettingsSource -from canfar import CONFIG_PATH, get_logger -from canfar.config.editor import get_value as _get_value -from canfar.config.editor import set_value as _set_value +from canfar import CONFIG_PATH +from canfar.config.editor import ConfigurationEditor as _ConfigurationEditor from canfar.models.active import ActiveConfig from canfar.models.auth import ( AuthenticationCredential, X509Credential, ) -from canfar.models.http import LOCAL, Server, VOSpaceService +from canfar.models.http import Server, VOSpaceService from canfar.models.registry import ContainerRegistry -log = get_logger(__name__) - _CADC_URI = AnyUrl("ivo://cadc.nrc.ca/skaha") _CONFIG_KEY_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9_-]*$") _SERVER_NAME_PATTERN = _CONFIG_KEY_PATTERN @@ -188,41 +183,9 @@ def settings_customise_sources( file_secret_settings, ) - def _heal_default_storage(self) -> None: - """Restore default Storage Identifiers on Servers saved by an older client. - - Older clients keyed a discovered VOSpace Service by its Server Name, so - an existing configuration holds ``canfar`` instead of ``arc`` and never - gained later defaults such as ``vault``. Healing is scoped to the - default Servers so federated Servers keep their Server Name keys. - - This normalizes the loaded Configuration in memory only; the file on - disk is rewritten when something independently calls ``save()``. - """ - for name, default in default_servers.items(): - server = self.servers.get(name) - if server is None or not default.storage or server.idp != default.idp: - continue - storage = dict(server.storage) - legacy = storage.pop(name, None) - if legacy is None and storage: - # Deliberate Storage Identifiers are configuration, not stale defaults. - continue - if legacy is not None: - leaf = str(legacy.uri).rpartition("/")[2] or name - storage.setdefault(leaf, legacy) - for identifier, service in default.storage.items(): - storage.setdefault(identifier, service.model_copy(deep=True)) - if storage != server.storage: - self.servers[name] = server.model_copy( - update={"storage": storage}, - deep=True, - ) - @model_validator(mode="after") def _normalize_and_validate_servers(self) -> Configuration: """Inject Server Names and validate Server and Storage Identifier keys.""" - self._heal_default_storage() updated: dict[str, Server] = {} server_by_identifier: dict[str, str] = {} for name, server in self.servers.items(): @@ -312,317 +275,10 @@ def _validate_active_references(self) -> Configuration: return self - def save(self) -> None: - """Save the current configuration to the default YAML file.""" - from canfar.config.store import save_config # noqa: PLC0415 - - save_config(self) - - def _validated_copy(self, **updates: Any) -> Configuration: - """Validate a source-isolated copy of this complete Configuration.""" - data = {**self.model_dump(mode="python"), **updates} - candidate = self.__class__.model_construct() - self.__class__.__pydantic_validator__.validate_python( - data, - self_instance=candidate, - ) - return candidate - - def _replace_state( - self, - *, - active: ActiveConfig | None = None, - authentication: dict[str, AuthenticationCredential] | None = None, - servers: dict[str, Server] | None = None, - ) -> None: - """Validate and install a complete Authentication and Server state.""" - # Validate only this candidate; BaseSettings construction would reload and - # merge persisted/environment sources, resurrecting keys being removed. - candidate = self._validated_copy( - active=self.active if active is None else active, - authentication=( - self.authentication if authentication is None else authentication - ), - servers=self.servers if servers is None else servers, - ) - self.active = candidate.active - self.authentication = candidate.authentication - self.servers = candidate.servers - - def get_value(self, path: str) -> Any: - """Get a nested configuration value via dotted path (e.g. 'console.width').""" - return _get_value(self, path) - - def set_value(self, path: str, value: Any) -> Configuration: - """Return a new validated Configuration with a dotted-path value updated.""" - return _set_value(self, path, value) - - def get_credential(self, idp: str) -> AuthenticationCredential: - """Return the saved authentication credential for an IDP key. - - Args: - idp: Canonical identity provider key. - - Returns: - Matching authentication credential. - - Raises: - KeyError: If no credential exists for ``idp``. - """ - if idp not in self.authentication: - msg = f"Authentication record for IDP '{idp}' not found." - raise KeyError(msg) - return self.authentication[idp] - - def _get_server_by_name(self, name: str) -> Server: - """Return a known server by Server Name.""" - if name not in self.servers: - msg = f"Server '{name}' not found." - raise KeyError(msg) - return self.servers[name] - - def storage_identifiers(self) -> list[str]: - """Return every addressable Storage Identifier, ``local`` last. - - Returns: - list[str]: Configured Storage Identifiers plus reserved ``local``. - """ - configured = { - identifier - for server in self.servers.values() - for identifier in server.storage - } - return [*sorted(configured), LOCAL] - - def _resolve_storage(self, identifier: str) -> tuple[str, str]: - """Resolve a Storage Identifier to its endpoint and parent server IDP.""" - for server in self.servers.values(): - service = server.storage.get(identifier) - if service is not None: - if server.idp is None: - msg = ( - f"Storage Identifier '{identifier}' belongs to a " - "Science Platform " - "Server without an IDP." - ) - raise ValueError(msg) - return str(service.url), server.idp - msg = f"Storage Identifier '{identifier}' is not configured." - raise KeyError(msg) - - def upsert_credential(self, credential: AuthenticationCredential) -> None: - """Insert or replace a validated Authentication Record. - - Args: - credential: Authentication Record to store by its IDP key. - """ - self._replace_state( - authentication={**self.authentication, credential.idp: credential}, - ) - - def update_credential(self, credential: AuthenticationCredential) -> None: - """Replace an existing validated Authentication Record. - - Raises: - KeyError: If no Authentication Record exists for the credential IDP. - """ - self.get_credential(credential.idp) - self.upsert_credential(credential) - - def set_active_authentication(self, idp: str) -> None: - """Select an Authentication Record and its remembered Server, if any.""" - self.get_credential(idp) - remembered = self.get_remembered_server_for_idp(idp) - if remembered is not None: - self.set_active_selection(idp, remembered) - return - - selections = self._server_selection_history() - server_name = self.active.server - if server_name is not None: - try: - active_server = self.get_active_server() - except KeyError: - server_name = None - else: - if active_server.idp != idp: - server_name = None - self._replace_state( - active=self.active.model_copy( - update={ - "authentication": idp, - "server": server_name, - "servers": selections, - }, - ), - ) - - def remove_authentication(self, idp: str) -> None: - """Remove an Authentication Record and its Science Platform Servers.""" - authentication = dict(self.authentication) - authentication.pop(idp, None) - servers = { - name: server for name, server in self.servers.items() if server.idp != idp - } - selections = { - selected_idp: name - for selected_idp, name in self.active.servers.items() - if selected_idp != idp - } - - if not authentication: - self.purge_authentication() - return - - active = self.active.model_copy(update={"servers": selections}) - if active.authentication == idp: - active = active.model_copy( - update={ - "authentication": next(iter(authentication)), - "server": None, - }, - ) - self._replace_state( - active=active, - authentication=authentication, - servers=servers, - ) - - def purge_authentication(self) -> None: - """Reset Authentication and Server state while preserving other settings.""" - self._replace_state( - active=default_active.model_copy(deep=True), - authentication={ - key: credential.model_copy(deep=True) - for key, credential in default_authentication.items() - }, - servers={ - name: server.model_copy(deep=True) - for name, server in default_servers.items() - }, - ) - - def get_server_by_uri(self, uri: str | AnyUrl) -> Server: - """Return a known server by IVOA URI. - - Args: - uri: Server URI to resolve. - - Returns: - Matching server record. - - Raises: - KeyError: If no server exists for ``uri``. - """ - target = str(uri) - for server in self.servers.values(): - if server.uri is not None and str(server.uri) == target: - return server - msg = f"Server '{target}' not found." - raise KeyError(msg) - - def get_active_server(self) -> Server: - """Return the active science platform server record. - - Raises: - KeyError: If no active server is selected. - """ - if self.active.server is None: - msg = "No active server selected." - raise KeyError(msg) - return self._get_server_by_name(self.active.server) - - def get_server_for_idp(self, idp: str) -> Server: - """Return the best-known server for an IDP. - - Uses the active server when it matches ``idp``; otherwise returns the - first saved server for the IDP. - - Args: - idp: Canonical identity provider key. - - Returns: - Server record for the IDP. - - Raises: - KeyError: If no server exists for ``idp``. - """ - if self.active.authentication == idp and self.active.server is not None: - return self.get_active_server() - - for server in self.servers.values(): - if server.idp == idp: - return server - msg = f"No server found for IDP '{idp}'." - raise KeyError(msg) - - def get_remembered_server_for_idp(self, idp: str) -> Server | None: - """Return the last selected server for ``idp`` when still valid. - - Args: - idp: Canonical identity provider key. - - Returns: - Matching server record, or ``None`` when no remembered selection is - available for ``idp``. - """ - name = self._server_selection_history().get(idp) - if name is None or name not in self.servers: - return None - server = self.servers[name] - if server.idp != idp: - return None - return server - - def _server_selection_history(self) -> dict[str, str]: - """Return remembered selections seeded with the current active pair.""" - selections = dict(self.active.servers) - active_name = self.active.server - if active_name is None or active_name not in self.servers: - return selections - server = self.servers[active_name] - if server.idp == self.active.authentication and server.name is not None: - selections[self.active.authentication] = server.name - return selections - - def set_active_selection(self, idp: str, server: Server) -> None: - """Persist ``idp`` and ``server`` as the active pair. - - Args: - idp: Canonical identity provider key. - server: Server record to activate. - - Raises: - ValueError: If the server has no Server Name. - """ - if server.name is None: - msg = "Server name is required for active selection." - raise ValueError(msg) - - selected = server.model_copy(update={"idp": idp}, deep=True) - servers = {**self.servers, server.name: selected} - selections = self._server_selection_history() - selections[idp] = server.name - active = self.active.model_copy( - update={ - "authentication": idp, - "server": server.name, - "servers": selections, - }, - ) - self._replace_state(active=active, servers=servers) - - def upsert_server(self, server: Server) -> None: - """Insert or replace a validated server record keyed by Server Name.""" - self.upsert_servers((server,)) - - def upsert_servers(self, servers: Iterable[Server]) -> None: - """Insert or replace validated server records in one state change.""" - updated = dict(self.servers) - for server in servers: - if server.name is not None: - updated[server.name] = server - self._replace_state(servers=updated) + @property + def editor(self) -> _ConfigurationEditor: + """Return the bound editing and persistence boundary.""" + return _ConfigurationEditor(self) __all__ = ["CONFIG_PATH", "Configuration", "ConsoleConfig"] diff --git a/canfar/models/http.py b/canfar/models/http.py index 3a08b71f..813517ca 100644 --- a/canfar/models/http.py +++ b/canfar/models/http.py @@ -19,11 +19,6 @@ LOCAL = "local" """Reserved Storage Identifier for the machine where the code runs.""" -RESERVED_IDENTIFIERS = frozenset( - {LOCAL, "filesystem", "identifiers", "sources"}, -) -"""Storage Identifiers that would shadow the ``canfar.storage`` module surface.""" - class VOSpaceService(BaseModel): """VOSpace Service discovered through an IVOA registry.""" @@ -150,14 +145,11 @@ def _validate_storage_identifiers(cls, value: Any) -> Any: name = None else: name = original_name.strip() - if name is None or ( - not name or name in RESERVED_IDENTIFIERS or name.startswith("-") - ): - reserved = ", ".join(sorted(RESERVED_IDENTIFIERS)) + if name is None or (not name or name == LOCAL or name.startswith("-")): msg = ( f"Invalid Storage Identifier {original_name!r}: after whitespace " - f"normalization it must be non-empty, avoid the reserved names " - f"({reserved}), contain no colon, NUL, or newline, and not start " + "normalization it must be non-empty, differ from reserved " + f"'{LOCAL}', contain no colon, NUL, or newline, and not start " "with '-'." ) raise ValueError(msg) diff --git a/canfar/overview.py b/canfar/overview.py index 743eea15..366802db 100644 --- a/canfar/overview.py +++ b/canfar/overview.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING from defusedxml import ElementTree @@ -9,13 +10,12 @@ from pydantic import model_validator from typing_extensions import Self -from canfar import get_logger from canfar.client import HTTPClient if TYPE_CHECKING: from httpx import Response -log = get_logger(__name__) +log = logging.getLogger(__name__) class Overview(HTTPClient): diff --git a/canfar/server.py b/canfar/server.py index ac77c592..d1c3540f 100644 --- a/canfar/server.py +++ b/canfar/server.py @@ -2,38 +2,33 @@ from __future__ import annotations -import asyncio -from typing import TYPE_CHECKING, Literal -from xml.etree.ElementTree import ParseError - -import httpx -from defusedxml.common import DefusedXmlException -from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, ValidationError - -from canfar import get_logger -from canfar.auth.x509 import CertificateError -from canfar.errors import ErrorCode, StructuredError -from canfar.exceptions.context import AuthContextError, AuthExpiredError -from canfar.hooks.httpx.auth import AuthenticationError as HTTPAuthenticationError -from canfar.idp import get_idp -from canfar.models.config import Configuration -from canfar.models.http import ( - DEFAULT_SERVER_CORES, - DEFAULT_SERVER_GPUS, - DEFAULT_SERVER_RAM_GB, - Server, - VOSpaceService, -) -from canfar.models.registry import Server as RegistryResource -from canfar.utils import registry, vosi -from canfar.utils.registry import RegistryEvidenceError +from typing import Literal -if TYPE_CHECKING: - from pathlib import Path +from pydantic import BaseModel, ConfigDict -log = get_logger(__name__) - -_STORAGE_RESOURCE_UNSET = object() +from canfar._server_discovery import ( + ServerDiscoveryError, + ServerFetchError, + _validate_server, + discover, + enrich, +) +from canfar.models.config import Configuration +from canfar.models.http import Server # noqa: TC001 + +__all__ = [ + "ServerActivation", + "ServerDiscoveryError", + "ServerFetchError", + "ServerSelectionRequiredError", + "ServerSelectorError", + "activate", + "activate_authentication", + "discover", + "enrich", + "list_servers", + "use", +] class ServerSelectorError(ValueError): @@ -44,34 +39,6 @@ def __init__(self, message: str, *, hint: str | None = None) -> None: self.hint = hint -class ServerDiscoveryError(RuntimeError): - """Raised when server discovery fails for an Identity Provider.""" - - def __init__( - self, - message: str, - *, - code: ErrorCode = ErrorCode.SERVER_DISCOVERY_FAILED, - ) -> None: - super().__init__(message) - self.code = code - self.structured = StructuredError(code=code, message=message) - - -class ServerFetchError(RuntimeError): - """Raised when server fetch or validation fails.""" - - def __init__( - self, - message: str, - *, - code: ErrorCode = ErrorCode.TRANSPORT_FAILURE, - ) -> None: - super().__init__(message) - self.code = code - self.structured = StructuredError(code=code, message=message) - - class ServerSelectionRequiredError(RuntimeError): """Raised when activation needs the caller to choose a server.""" @@ -95,115 +62,6 @@ class ServerActivation(BaseModel): reason: Literal["active", "remembered", "single", "selected"] -def _merge_storage( - known: dict[str, VOSpaceService], - found: dict[str, VOSpaceService], -) -> dict[str, VOSpaceService]: - """Merge discovered VOSpace Services into known ones, keyed by IVOA URI. - - Discovery names a new Service after its Server Name, so an existing Service - configured under its registry leaf (``arc``) is refreshed in place instead - of being duplicated under the Server Name on every rediscovery. - - Args: - known: Configured VOSpace Services keyed by Storage Identifier. - found: Newly discovered VOSpace Services keyed by Storage Identifier. - - Returns: - dict[str, VOSpaceService]: Merged Services keyed by Storage Identifier. - """ - merged = dict(known) - names = {str(service.uri): name for name, service in known.items()} - for name, service in found.items(): - merged[names.get(str(service.uri), name)] = service - return merged - - -def discover( - idp: str, - *, - config: Configuration | None = None, - dev: bool = False, - timeout: int = 2, - save: bool = True, -) -> list[Server]: - """Discover, merge, and optionally persist servers for ``idp``. - - Args: - idp: Canonical Identity Provider key. - config: Configuration to update in place. Defaults to loading config. - dev: Include development registries and endpoints during discovery. - timeout: HTTP timeout in seconds for discovery requests. - save: Persist the configuration after merging discovered servers. - - Returns: - list[Server]: Newly discovered server records. - - Raises: - ServerDiscoveryError: If discovery fails or finds no usable servers. - """ - target_config = config or Configuration() # ty: ignore[missing-argument] - discovered = asyncio.run( - _discover_for_idp( - idp, - config=target_config, - dev=dev, - timeout=timeout, - ) - ) - known_servers = dict(target_config.servers) - canonical: dict[str, Server] = {} - for server in sorted( - discovered, - key=lambda item: ( - item.name is None, - (item.name or "").casefold(), - str(item.uri or ""), - str(item.url or ""), - ), - ): - name = server.name - if name is None: - continue - known = canonical.get(name, known_servers.get(name)) - if server.version is None or not server.auths: - if known is not None and known.version is not None and known.auths: - if server.storage: - known = known.model_copy( - update={ - "storage": _merge_storage(known.storage, server.storage) - }, - deep=True, - ) - canonical[name] = known - continue - merged_server = server - if known is not None: - updates = server.model_dump( - include={"idp", "name", "uri", "url", "version", "auths"}, - exclude_none=True, - ) - if server.storage: - updates["storage"] = _merge_storage(known.storage, server.storage) - merged_server = known.model_copy(update=updates, deep=True) - canonical[name] = merged_server - - if not canonical: - msg = f"No servers discovered for IDP '{idp}'." - raise ServerDiscoveryError( - msg, - code=ErrorCode.SERVER_NONE_AVAILABLE, - ) - merged = [ - canonical[name] - for name in sorted(canonical, key=lambda value: (value.casefold(), value)) - ] - target_config.upsert_servers(merged) - if save: - target_config.save() - return merged - - def activate( idp: str, selector: str | None = None, @@ -240,8 +98,8 @@ def activate( if selector is None: active_server = _active_server_for_idp(target_config, idp) if active_server is not None: - target_config.set_active_selection(idp, active_server) - target_config.save() + _store_active_selection(target_config, idp, active_server) + target_config.editor.save() return ServerActivation(server=active_server, reason="active") servers = _servers_for_idp(target_config, idp) @@ -255,11 +113,11 @@ def activate( ) remembered = _remembered_server_for_idp(target_config, idp, servers) - if remembered is not None and remembered.uri is not None: - selector = str(remembered.uri) + if remembered is not None and remembered.name is not None: + selector = remembered.name reason = "remembered" - elif len(servers) == 1 and servers[0].uri is not None: - selector = str(servers[0].uri) + elif len(servers) == 1 and servers[0].name is not None: + selector = servers[0].name reason = "single" else: raise ServerSelectionRequiredError(idp, servers) @@ -290,11 +148,38 @@ def activate( dev=dev, timeout=timeout, ) - target_config.set_active_selection(idp, validated) - target_config.save() + _store_active_selection(target_config, idp, validated) + target_config.editor.save() return ServerActivation(server=validated, reason=reason) +def activate_authentication( + idp: str, + *, + config: Configuration | None = None, +) -> None: + """Activate an Authentication Record and its remembered Server Selection.""" + target_config = config or Configuration() # ty: ignore[missing-argument] + if idp not in target_config.authentication: + msg = f"Authentication record for IDP '{idp}' not found." + raise KeyError(msg) + servers = _servers_for_idp(target_config, idp) + remembered = _remembered_server_for_idp(target_config, idp, servers) + if remembered is not None: + _store_active_selection(target_config, idp, remembered) + else: + active_server = _active_server_for_idp(target_config, idp) + active = target_config.active.model_copy( + update={ + "authentication": idp, + "server": active_server.name if active_server is not None else None, + "servers": _server_selection_history(target_config), + }, + ) + target_config.editor.set("active", active) + target_config.editor.save() + + def list_servers( *, discover_if_empty: bool = True, @@ -324,7 +209,7 @@ def list_servers( return servers discover(active_idp, config=config, dev=dev, timeout=timeout, save=False) - config.save() + config.editor.save() return [server for server in config.servers.values() if server.idp == active_idp] @@ -364,9 +249,8 @@ def _servers_for_idp(config: Configuration, idp: str) -> list[Server]: def _active_server_for_idp(config: Configuration, idp: str) -> Server | None: if config.active.server is None: return None - try: - active_server = config.get_active_server() - except KeyError: + active_server = config.servers.get(config.active.server) + if active_server is None: return None if active_server.idp != idp: return None @@ -378,16 +262,61 @@ def _remembered_server_for_idp( idp: str, servers: list[Server], ) -> Server | None: - remembered = config.get_remembered_server_for_idp(idp) - if remembered is None or remembered.uri is None: + name = _server_selection_history(config).get(idp) + if name is None: return None - if remembered.name is None: + remembered = config.servers.get(name) + if remembered is None or remembered.idp != idp: return None - if not any(server.name == remembered.name for server in servers): + if not any(server.name == name for server in servers): return None return remembered +def _server_selection_history(config: Configuration) -> dict[str, str]: + """Return remembered Server Selections seeded by the active pair.""" + selections = dict(config.active.servers) + active_name = config.active.server + if active_name is None: + return selections + + active_server = config.servers.get(active_name) + if ( + active_server is not None + and active_server.idp == config.active.authentication + and active_server.name is not None + ): + selections[config.active.authentication] = active_server.name + return selections + + +def _store_active_selection( + config: Configuration, + idp: str, + server: Server, +) -> None: + """Store a Server Selection and its history through the editor boundary.""" + if server.name is None: + msg = "Server name is required for active selection." + raise ValueError(msg) + + selected = server.model_copy(update={"idp": idp}, deep=True) + servers = {**config.servers, server.name: selected} + selections = _server_selection_history(config) + selections[idp] = server.name + active = config.active.model_copy( + update={ + "authentication": idp, + "server": server.name, + "servers": selections, + }, + ) + config.editor._set_top_level( # noqa: SLF001 + servers=servers, + active=active, + ) + + def _resolve_selector( config: Configuration, selector: str, @@ -415,548 +344,3 @@ def _resolve_selector( if server.uri is not None and str(server.uri) == selector: return server return None - - -async def _discover_for_idp( - idp: str, - *, - config: Configuration | None = None, - dev: bool = False, - timeout: int = 2, -) -> list[Server]: - """Discover active servers for a single Identity Provider. - - Args: - idp: Canonical IDP key. - config: Configuration whose Authentication Record authorizes enrichment. - dev: Include development registries and endpoints. - timeout: HTTP timeout in seconds for discovery requests. - - Returns: - list[Server]: Validated HTTP server models for reachable endpoints. - - Raises: - ServerDiscoveryError: If registry retrieval fails. - """ - evidence = await registry.evidence( - idp, - dev=dev, - timeout=timeout, - check_platforms=True, - ) - if not evidence.available: - errors = "; ".join(evidence.errors) - msg = f"Failed to discover servers for IDP '{idp}': {errors}" - raise ServerDiscoveryError(msg) - - endpoints = [ - resource - for resource in evidence.resources - if resource.uri.endswith("/skaha") and resource.status == 200 - ] - if not endpoints: - return [] - - storage_resources = [ - resource - for resource in evidence.resources - if resource.uri.endswith(f"/{evidence.leaf}") - ] - workers = await registry.workers( - config, - idp, - endpoint=endpoints[0], - count=len(endpoints), - ) - if workers is None: - return [_registry_resource_to_server(endpoint, idp) for endpoint in endpoints] - - return list( - await asyncio.gather( - *( - asyncio.to_thread( - _discovered_to_server, - endpoint, - idp, - config=worker_config, - token=workers.token, - certificate=workers.certificate, - timeout=timeout, - storage_resource=_select_storage( - endpoint, - storage_resources, - strict=False, - ), - ) - for endpoint, worker_config in zip( - endpoints, - workers.configs, - strict=True, - ) - ) - ) - ) - - -def _host_slug(uri: AnyUrl) -> str | None: - """Return a Server Name slug derived from a URI host (dots -> hyphens).""" - if uri.host is None: - return None - return uri.host.replace(".", "-") - - -def _select_storage( - endpoint: RegistryResource, - resources: list[RegistryResource], - *, - strict: bool, -) -> RegistryResource | None: - """Map private registry ambiguity to the public server fetch error.""" - try: - return registry.select_storage(endpoint, resources, strict=strict) - except RegistryEvidenceError as exc: - raise ServerFetchError(str(exc)) from exc - - -async def _discover_storage( - server: Server, - idp: str, - *, - dev: bool, - timeout: int, -) -> RegistryResource | None: - """Return fresh registry evidence for a server's primary VOSpace service.""" - try: - return await registry.discover_storage( - str(server.uri) if server.uri is not None else None, - str(server.url) if server.url is not None else None, - server.name, - idp, - dev=dev, - timeout=timeout, - ) - except RegistryEvidenceError as exc: - raise ServerFetchError(str(exc)) from exc - - -def _configured_storage_resource(server: Server) -> RegistryResource | None: - """Convert the persisted primary VOSpace service to inspection evidence.""" - if server.name is None: - return None - service = server.storage.get(server.name) - if service is None: - return None - return RegistryResource( - registry="configuration", - uri=str(service.uri), - url=str(service.url), - ) - - -def _registry_resource_to_server(endpoint: RegistryResource, idp: str) -> Server: - """Convert registry endpoint identity without performing capability I/O.""" - uri = AnyUrl(endpoint.uri) - return Server( - idp=idp, - name=endpoint.name or _host_slug(uri), - uri=uri, - url=AnyHttpUrl(endpoint.url), - ) - - -def _discovered_to_server( - endpoint: RegistryResource, - idp: str, - *, - config: Configuration | None = None, - token: str | None = None, - certificate: Path | None = None, - timeout: int = 2, - storage_resource: RegistryResource | None = None, -) -> Server: - """Convert a registry discovery record to a persisted HTTP server model. - - The registry-provided name wins as the Server Name; endpoints without a - registry name are named by a slug of the URI host. - - Args: - endpoint: Discovered registry endpoint. - idp: Canonical IDP key. - config: Configuration whose Authentication Record authorizes enrichment. - token: Pre-materialized runtime bearer token for worker isolation. - certificate: Pre-materialized runtime certificate for worker isolation. - timeout: HTTP timeout in seconds for VOSI capabilities requests. - storage_resource: Same-namespace preferred VOSpace registry record, if any. - - Returns: - Server: Persisted server model with capabilities metadata when available. - """ - server = _registry_resource_to_server(endpoint, idp) - return enrich( - server, - config=config, - token=token, - certificate=certificate, - strict=False, - timeout=timeout, - storage_resource=storage_resource, - ) - - -def enrich( - server: Server, - *, - config: Configuration | None = None, - authentication_idp: str | None = None, - token: str | None = None, - certificate: Path | None = None, - strict: bool = True, - timeout: int = 2, - storage_resource: RegistryResource | None | object = _STORAGE_RESOURCE_UNSET, -) -> Server: - """Return a validated Server enriched from its VOSI capabilities. - - Args: - server: Server record to enrich. - config: Configuration whose Authentication Record should authorize the - capability request. The transient selector does not change or persist - Authentication or Server Selection. - authentication_idp: Optional Authentication Record selector. Defaults to - the Server IDP, then the active Authentication. - token: Optional runtime bearer token for capability requests. - certificate: Optional runtime certificate for capability requests. - strict: When ``False``, keep usable registry and existing storage data - when session or storage capabilities cannot be retrieved or parsed. - Other successful enrichment may still be returned, so the result can - be partial. Discovery uses non-strict mode so one malformed endpoint - does not abort listing for an IDP. - timeout: HTTP timeout in seconds for VOSI capabilities requests. - storage_resource: Retained same-namespace VOSpace registry record. Passing - ``None`` records that the preferred resource was absent; omitting the - argument leaves storage outside this inspection. - - Returns: - Server: Copy with version and auth modes populated when discoverable. - - Raises: - ServerFetchError: If ``strict`` is ``True`` and capabilities cannot - be retrieved, parsed, or contain no session capabilities. - """ - base_config = config or Configuration() # ty: ignore[missing-argument] - active_idp = authentication_idp or server.idp or base_config.active.authentication - if storage_resource is not _STORAGE_RESOURCE_UNSET: - server = _enrich_storage( - server, - storage_resource=( - storage_resource - if isinstance(storage_resource, RegistryResource) - else None - ), - config=base_config, - authentication_idp=active_idp, - token=token, - certificate=certificate, - strict=strict, - timeout=timeout, - ) - if server.url is None: - msg = "Server URL is required to inspect capabilities." - raise ServerFetchError(msg) - try: - capabilities = vosi.capabilities( - xml=_fetch_capabilities( - server.url, - config=base_config, - authentication_idp=active_idp, - token=token, - certificate=certificate, - timeout=timeout, - ) - ) - except ( - httpx.HTTPError, - OSError, - AuthContextError, - AuthExpiredError, - CertificateError, - HTTPAuthenticationError, - ParseError, - DefusedXmlException, - ) as exc: - return _keep_or_raise( - server, - strict=strict, - error=f"Failed to fetch capabilities for {server.url}: {exc}", - cause=exc, - debug="Skipping capability enrichment for %s during discovery: %s", - args=(server.url, exc), - ) - - primary = next( - ( - capability - for capability in capabilities - if capability.get("version") and capability.get("auth_modes") - ), - None, - ) - if primary is None: - return _keep_or_raise( - server, - strict=strict, - error=f"No complete session capabilities found for {server.url}.", - debug=( - "No complete session capabilities found for %s during discovery; " - "keeping registry metadata only." - ), - args=(server.url,), - ) - - try: - return Server.model_validate( - { - **server.model_dump(mode="python"), - "url": primary["baseurl"], - "version": primary["version"], - "auths": primary["auth_modes"], - } - ) - except ValidationError as exc: - return _keep_or_raise( - server, - strict=strict, - error=f"Invalid capabilities for {server.url}: {exc}", - cause=exc, - debug=( - "Ignoring invalid capability enrichment for %s during discovery: %s" - ), - args=(server.url, exc), - ) - - -def _enrich_storage( - server: Server, - *, - storage_resource: RegistryResource | None, - config: Configuration, - authentication_idp: str, - token: str | None, - certificate: Path | None, - strict: bool, - timeout: int, -) -> Server: - """Validate and attach one retained primary VOSpace registry resource.""" - error: BaseException | None = None - if storage_resource is None: - leaf = get_idp(authentication_idp).leaf - subject = f"same-namespace '{leaf}' registry record" - error = ValueError( - f"No {subject} found for Science Platform Server '{server.name}'." - ) - else: - subject = storage_resource.uri - try: - xml = _fetch_capabilities( - AnyHttpUrl(storage_resource.url), - config=config, - authentication_idp=authentication_idp, - token=token, - certificate=certificate, - timeout=timeout, - ) - valid = vosi.is_vospace_service(xml) - except ( - httpx.HTTPError, - OSError, - AuthContextError, - AuthExpiredError, - CertificateError, - HTTPAuthenticationError, - ParseError, - DefusedXmlException, - ValueError, - ) as exc: - error = exc - else: - if not valid: - error = ValueError( - "required VOSpace node capability is missing or malformed" - ) - elif server.name is None: - error = ValueError("Science Platform Server has no Server Name") - - if error is not None: - message = ( - f"Failed to inspect VOSpace Service '{subject}' for Science " - f"Platform Server '{server.name}': {error}" - ) - return _keep_or_raise( - server, - strict=strict, - error=message, - cause=error, - debug="Skipping VOSpace Service %s during discovery: %s", - args=(subject, error), - ) - assert storage_resource is not None - assert server.name is not None - service = VOSpaceService.model_validate( - {"uri": storage_resource.uri, "url": storage_resource.url} - ) - - return server.model_copy( - update={"storage": {**server.storage, server.name: service}}, - deep=True, - ) - - -def _fetch_capabilities( - url: AnyHttpUrl, - *, - config: Configuration, - authentication_idp: str, - token: str | None = None, - certificate: Path | None = None, - timeout: int, -) -> str: - """Fetch one VOSI capabilities document through the existing HTTP seam.""" - from canfar.client import HTTPClient # noqa: PLC0415 - - with HTTPClient.build( - config=config, - authentication_idp=authentication_idp, - url=url, - token=token, - certificate=certificate, - timeout=timeout, - raise_http_errors=False, - ) as client: - request_client = client.client - request_client.headers["Accept"] = "application/xml" - request_client.headers.pop("Content-Type", None) - request_client.headers.pop("X-Skaha-Registry-Auth", None) - response = request_client.get("capabilities") - response.raise_for_status() - return response.text - - -def _keep_or_raise( - server: Server, - *, - strict: bool, - error: str, - debug: str, - args: tuple[object, ...] = (), - cause: BaseException | None = None, -) -> Server: - """Raise on strict enrich failures; otherwise keep the original server.""" - if strict: - raise ServerFetchError(error) from cause - log.debug(debug, *args) - return server.model_copy(deep=True) - - -def _validate_server( - server: Server, - *, - config: Configuration | None = None, - idp: str | None = None, - dev: bool = False, - timeout: int = 2, -) -> Server: - """Fetch and validate a server before persisting it as active. - - Args: - server: Candidate server record. - config: Configuration to use while validating the candidate selection. - idp: Authentication IDP to pair with the candidate server. - dev: Include development registry evidence during validation. - timeout: HTTP timeout in seconds for validation requests. - - Returns: - Server: Enriched, validated server model. - - Raises: - ServerFetchError: If capability enrichment fails or URL/version are missing. - """ - base_config = config or Configuration() # ty: ignore[missing-argument] - active_idp = idp or server.idp or base_config.active.authentication - storage_resource = _configured_storage_resource(server) - if storage_resource is None: - storage_resource = asyncio.run( - _discover_storage( - server, - active_idp, - dev=dev, - timeout=timeout, - ) - ) - enriched = enrich( - server, - config=base_config, - authentication_idp=active_idp, - strict=True, - timeout=timeout, - storage_resource=storage_resource, - ) - if enriched.url is None or enriched.version is None: - msg = "Server URL and version are required before activation." - raise ServerFetchError(msg) - - return _fetch_resources( - enriched, - timeout=timeout, - config=base_config, - authentication_idp=active_idp, - ) - - -def _fetch_resources( - server: Server, - *, - config: Configuration, - authentication_idp: str, - timeout: int, -) -> Server: - """Return a Server populated from its authenticated context endpoint.""" - from canfar.client import HTTPClient # noqa: PLC0415 - - if server.url is None or server.version is None: - msg = "Server URL and version are required for resource enrichment." - raise ValueError(msg) - client = HTTPClient( - config=config, - authentication_idp=authentication_idp, - url=AnyHttpUrl(f"{server.url}/{server.version}"), - timeout=timeout, - raise_http_errors=False, - ) - try: - with client: - response = client.client.get("context") - response.raise_for_status() - data = dict(response.json()) - - cores_data = data.get("cores") or {} - ram_data = data.get("memoryGB") or {} - gpus_data = data.get("gpus") or {} - cores = cores_data.get("defaultLimit") - ram = ram_data.get("defaultLimit") - gpu_options = gpus_data.get("options") or [] - return Server.model_validate( - { - **server.model_dump(mode="python"), - "cores": cores if cores is not None else DEFAULT_SERVER_CORES, - "ram": ram if ram is not None else DEFAULT_SERVER_RAM_GB, - "gpus": max(gpu_options) if gpu_options else DEFAULT_SERVER_GPUS, - } - ) - except (httpx.HTTPError, OSError, ValueError, TypeError): - return server.model_copy( - update={ - "cores": DEFAULT_SERVER_CORES, - "ram": DEFAULT_SERVER_RAM_GB, - "gpus": DEFAULT_SERVER_GPUS, - }, - deep=True, - ) diff --git a/canfar/sessions.py b/canfar/sessions.py index 22e953d3..efae8239 100644 --- a/canfar/sessions.py +++ b/canfar/sessions.py @@ -3,13 +3,13 @@ from __future__ import annotations import asyncio +import logging import re -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeVar from webbrowser import open_new_tab from httpx import HTTPError, Response -from canfar import get_logger from canfar.client import HTTPClient from canfar.models.session import CreateRequest from canfar.utils import build @@ -18,7 +18,8 @@ from collections.abc import Mapping from canfar.models.types import Kind, Status, View -log = get_logger(__name__) +log = logging.getLogger(__name__) +_Result = TypeVar("_Result") def _log_http_task_failure(operation: str, context: object, exc: BaseException) -> None: @@ -30,11 +31,53 @@ def _log_http_task_failure(operation: str, context: object, exc: BaseException) log.error("%s: %s (%s)", operation, context, type(exc).__name__) +def _task_result( + operation: str, + context: object, + result: _Result | Exception, +) -> _Result | None: + """Keep one failure and logging policy for collected transport results.""" + if isinstance(result, Exception): + _log_http_task_failure(operation, context, result) + return None + return result + + +def _destroy_failure(session_id: str, exc: BaseException | None = None) -> bool: + """Log a failed Session deletion and preserve the false result policy.""" + msg = f"Failed to destroy session {session_id}" + if exc is not None: + msg += f": {exc}" + # Both callers invoke this from their HTTPError handler; keep traceback logging. + log.exception(msg) # noqa: LOG004 + return False + + def _ids(value: str | list[str]) -> list[str]: """Normalize one or many Session identifiers without changing their order.""" return [value] if isinstance(value, str) else value +def _session_url(session_id: str) -> str: + """Build the endpoint path for one Session identifier.""" + return f"session/{session_id}" + + +def _view_parameters(view: str) -> dict[str, str]: + """Build the query parameters for a Session detail view.""" + return {"view": view} + + +def _response_session_id(response: Response) -> str: + """Interpret a create response as a clean Session identifier.""" + return response.text.rstrip("\r\n") + + +def _response_event(session_id: str, response: Response) -> dict[str, str]: + """Interpret an events response with its requested Session identifier.""" + return {session_id: response.text} + + def _session_name_pattern(selector: str) -> re.Pattern[str]: """Compile a regular expression or an anchored literal Session selector.""" meta = frozenset(".^$*+?{}[]()|") @@ -135,7 +178,7 @@ def stats(self) -> dict[str, Any]: 'maxCores': {'cores': 32, 'withRam': '147Gi'}}, 'ram': {'maxRAM': {'ram': '226Gi', 'withCores': 32}}} """ - parameters = {"view": "stats"} + parameters = _view_parameters("stats") response: Response = self.client.get("session", params=parameters) data: dict[str, Any] = response.json() return data @@ -157,10 +200,10 @@ def info(self, ids: list[str] | str) -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] for value in ids: try: - response: Response = self.client.get(url=f"session/{value}") + response: Response = self.client.get(url=_session_url(value)) results.append(response.json()) except HTTPError as err: - _log_http_task_failure("failed to fetch session info for", value, err) + _task_result("failed to fetch session info for", value, err) return results def logs( @@ -182,18 +225,18 @@ def logs( >>> session.logs(ids=["hjko98yghj", "ikvp1jtp"]) """ ids = _ids(ids) - parameters: dict[str, str] = {"view": "logs"} + parameters: dict[str, str] = _view_parameters("logs") results: dict[str, str] = {} for value in ids: try: response: Response = self.client.get( - url=f"session/{value}", + url=_session_url(value), params=parameters, ) results[value] = response.text except HTTPError as err: - _log_http_task_failure("failed to fetch logs for session", value, err) + _task_result("failed to fetch logs for session", value, err) if verbose: for key, value in results.items(): @@ -203,7 +246,7 @@ def logs( return results - def create( + def create( # noqa: PLR0917 self, name: str | CreateRequest, image: str | None = None, @@ -281,9 +324,9 @@ def create( for replica, payload in enumerate(payloads, start=1): try: response: Response = self.client.post(url="session", params=payload) - results.append(response.text.rstrip("\r\n")) + results.append(_response_session_id(response)) except HTTPError as err: - _log_http_task_failure( + _task_result( "Failed to create session", f"replica {replica}/{len(payloads)}", err, @@ -315,16 +358,16 @@ def events( """ ids = _ids(ids) results: list[dict[str, str]] = [] - parameters: dict[str, str] = {"view": "events"} + parameters: dict[str, str] = _view_parameters("events") for value in ids: try: response: Response = self.client.get( - url=f"session/{value}", + url=_session_url(value), params=parameters, ) - results.append({value: response.text}) + results.append(_response_event(value, response)) except HTTPError as err: - _log_http_task_failure("Failed to fetch events for session", value, err) + _task_result("Failed to fetch events for session", value, err) if verbose and results: for result in results: for key, value in result.items(): @@ -352,12 +395,10 @@ def destroy(self, ids: str | list[str]) -> dict[str, bool]: results: dict[str, bool] = {} for value in ids: try: - self.client.delete(url=f"session/{value}") + self.client.delete(url=_session_url(value)) results[value] = True except HTTPError: - msg = f"Failed to destroy session {value}" - log.exception(msg) - results[value] = False + results[value] = _destroy_failure(value) return results def destroy_with( @@ -511,7 +552,7 @@ async def stats(self) -> dict[str, Any]: 'maxCores': {'cores': 32, 'withRam': '147Gi'}}, 'ram': {'maxRAM': {'ram': '226Gi', 'withCores': 32}}} """ - parameters = {"view": "stats"} + parameters = _view_parameters("stats") response: Response = await self.asynclient.get("session", params=parameters) data: dict[str, Any] = response.json() return data @@ -533,21 +574,18 @@ async def info(self, ids: list[str] | str) -> list[dict[str, Any]]: """ ids = _ids(ids) results: list[dict[str, Any]] = [] - semaphore: asyncio.Semaphore = asyncio.Semaphore(self.concurrency) - async def bounded(value: str) -> dict[str, Any]: - async with semaphore: - response = await self.asynclient.get(url=f"session/{value}") - data: dict[str, Any] = response.json() - return data + async def request(value: str) -> dict[str, Any]: + response = await self.asynclient.get(url=_session_url(value)) + data: dict[str, Any] = response.json() + return data - tasks = [bounded(value) for value in ids] + tasks = [request(value) for value in ids] responses = await asyncio.gather(*tasks, return_exceptions=True) for value, reply in zip(ids, responses, strict=True): - if isinstance(reply, Exception): - _log_http_task_failure("failed to fetch session info for", value, reply) - elif isinstance(reply, dict): - results.append(reply) + result = _task_result("failed to fetch session info for", value, reply) + if isinstance(result, dict): + results.append(result) log.debug("Session info records collected: %s", results) return results @@ -572,26 +610,22 @@ async def logs( >>> await session.logs(ids=["hjko98yghj", "ikvp1jtp"]) """ ids = _ids(ids) - parameters: dict[str, str] = {"view": "logs"} + parameters: dict[str, str] = _view_parameters("logs") results: dict[str, str] = {} - semaphore: asyncio.Semaphore = asyncio.Semaphore(self.concurrency) - - async def bounded(value: str) -> tuple[str, str]: - async with semaphore: - response = await self.asynclient.get( - url=f"session/{value}", - params=parameters, - ) - return value, response.text + async def request(value: str) -> tuple[str, str]: + response = await self.asynclient.get( + url=_session_url(value), + params=parameters, + ) + return value, response.text - tasks = [bounded(value) for value in ids] + tasks = [request(value) for value in ids] responses = await asyncio.gather(*tasks, return_exceptions=True) for value, reply in zip(ids, responses, strict=True): - if isinstance(reply, Exception): - _log_http_task_failure("failed to fetch logs for session", value, reply) - elif isinstance(reply, tuple): - results[reply[0]] = reply[1] + result = _task_result("failed to fetch logs for session", value, reply) + if isinstance(result, tuple): + results[result[0]] = result[1] # Print logs to stdout if verbose is set to True if verbose: @@ -601,7 +635,7 @@ async def bounded(value: str) -> tuple[str, str]: return None return results - async def create( + async def create( # noqa: PLR0917 self, name: str | CreateRequest, image: str | None = None, @@ -674,27 +708,24 @@ async def create( replicas, ) results: list[str] = [] - semaphore: asyncio.Semaphore = asyncio.Semaphore(self.concurrency) - async def bounded(parameters: list[tuple[str, Any]]) -> Any: - async with semaphore: - response = await self.asynclient.post(url="session", params=parameters) - return response.text.rstrip("\r\n") + async def request_session(parameters: list[tuple[str, Any]]) -> str: + response = await self.asynclient.post(url="session", params=parameters) + return _response_session_id(response) - tasks = [bounded(payload) for payload in payloads] + tasks = [request_session(payload) for payload in payloads] session_kind = name.kind if isinstance(name, CreateRequest) else kind msg = f"Creating {len(payloads)} {session_kind} session[s]." log.debug(msg) responses = await asyncio.gather(*tasks, return_exceptions=True) for replica, reply in enumerate(responses, start=1): - if isinstance(reply, Exception): - _log_http_task_failure( - "Failed to create session", - f"replica {replica}/{len(payloads)}", - reply, - ) - elif isinstance(reply, str): - results.append(reply) + result = _task_result( + "Failed to create session", + f"replica {replica}/{len(payloads)}", + reply, + ) + if isinstance(result, str): + results.append(result) log.debug("Session IDs collected from create: %s", results) return results @@ -723,28 +754,25 @@ async def events( """ ids = _ids(ids) results: list[dict[str, str]] = [] - parameters: dict[str, str] = {"view": "events"} - semaphore: asyncio.Semaphore = asyncio.Semaphore(self.concurrency) + parameters: dict[str, str] = _view_parameters("events") - async def bounded(value: str) -> dict[str, str]: - async with semaphore: - response = await self.asynclient.get( - url=f"session/{value}", - params=parameters, - ) - return {value: response.text} + async def request(value: str) -> dict[str, str]: + response = await self.asynclient.get( + url=_session_url(value), + params=parameters, + ) + return _response_event(value, response) - tasks = [bounded(value) for value in ids] + tasks = [request(value) for value in ids] responses = await asyncio.gather(*tasks, return_exceptions=True) for value, reply in zip(ids, responses, strict=True): - if isinstance(reply, Exception): - _log_http_task_failure( - "Failed to fetch events for session", - value, - reply, - ) - elif isinstance(reply, dict): - results.append(reply) + result = _task_result( + "Failed to fetch events for session", + value, + reply, + ) + if isinstance(result, dict): + results.append(result) if verbose and results: for result in results: @@ -772,20 +800,16 @@ async def destroy(self, ids: str | list[str]) -> dict[str, bool]: """ ids = _ids(ids) results: dict[str, bool] = {} - semaphore: asyncio.Semaphore = asyncio.Semaphore(self.concurrency) - - async def bounded(value: str) -> tuple[str, bool]: - async with semaphore: - try: - await self.asynclient.delete(url=f"session/{value}") - except HTTPError as err: - msg = f"Failed to destroy session {value}: {err}" - log.exception(msg) - return value, False - else: - return value, True - - tasks = [bounded(value) for value in ids] + + async def request(value: str) -> tuple[str, bool]: + try: + await self.asynclient.delete(url=_session_url(value)) + except HTTPError as err: + return value, _destroy_failure(value, err) + else: + return value, True + + tasks = [request(value) for value in ids] responses = await asyncio.gather(*tasks, return_exceptions=True) for reply in responses: if isinstance(reply, tuple): diff --git a/canfar/storage.py b/canfar/storage.py index c41095a4..c0219372 100644 --- a/canfar/storage.py +++ b/canfar/storage.py @@ -1,4 +1,4 @@ -"""Adapters for the configured VOSpace Services and the local filesystem.""" +"""Explicit access to configured VOSpace Services and the local filesystem.""" from __future__ import annotations @@ -24,9 +24,9 @@ from vosfs import VOSpaceFileSystem from canfar.models.auth import RuntimeCredential + from canfar.models.http import Server, VOSpaceService -__all__ = ["LOCAL", "filesystem", "identifiers", "sources"] -"""Public surface; Storage Identifiers resolve through ``__getattr__``.""" +__all__ = ["filesystem", "identifiers"] _LISTINGS_EXPIRY_SECONDS = 30 """Seconds a cached directory listing stays valid on one filesystem.""" @@ -35,6 +35,33 @@ """Maximum directory listings retained by one filesystem.""" +def _configured( + config: Configuration, +) -> dict[str, tuple[Server, VOSpaceService]]: + """Return the private Storage Identifier to service mapping.""" + return { + identifier: (server, service) + for server in config.servers.values() + for identifier, service in server.storage.items() + } + + +def _service(config: Configuration, identifier: str) -> tuple[str, str]: + """Return a service endpoint and its parent server's IDP.""" + try: + server, service = _configured(config)[identifier] + except KeyError: + msg = f"Storage Identifier '{identifier}' is not configured." + raise KeyError(msg) from None + if server.idp is None: + msg = ( + f"Storage Identifier '{identifier}' belongs to a Science Platform " + "Server without an IDP." + ) + raise ValueError(msg) + return str(service.url), server.idp + + async def _resolve( identifier: str, token: str | SecretStr | None = None, @@ -54,7 +81,7 @@ async def _resolve( AuthContextError: If the credential cannot be materialized. """ config = Configuration() # ty: ignore[missing-argument] - endpoint, idp = config._resolve_storage(identifier) # noqa: SLF001 + endpoint, idp = _service(config, identifier) try: client = HTTPClient.build( config=config, @@ -152,7 +179,7 @@ async def _local() -> AsyncIterator[AbstractFileSystem]: ) -def sources() -> dict[str, AsyncFilesystemSource]: +def _sources() -> dict[str, AsyncFilesystemSource]: """Build the mapped storage sources for one data command invocation. Every configured VOSpace Service is mapped by its Storage Identifier, plus @@ -163,9 +190,7 @@ def sources() -> dict[str, AsyncFilesystemSource]: """ config = Configuration() # ty: ignore[missing-argument] mapped: dict[str, AsyncFilesystemSource] = { - identifier: _vospace(identifier) - for identifier in config.storage_identifiers() - if identifier != LOCAL + identifier: _vospace(identifier) for identifier in _configured(config) } mapped[LOCAL] = _local return mapped @@ -178,7 +203,7 @@ def identifiers() -> list[str]: list[str]: Configured Storage Identifiers plus the reserved ``local``. """ config = Configuration() # ty: ignore[missing-argument] - return config.storage_identifiers() + return [*sorted(_configured(config)), LOCAL] def filesystem( @@ -206,46 +231,3 @@ def filesystem( # fsspec's background loop, so this works inside a running loop too. endpoint, credential = sync(get_loop(), _resolve, identifier, token, certificate) return _build(endpoint, credential, asynchronous=False) - - -def __getattr__(identifier: str) -> AbstractFileSystem: - """Return a filesystem for a Storage Identifier accessed as an attribute. - - Makes ``from canfar.storage import vault`` resolve to a ready filesystem - for the ``vault`` Storage Identifier. - - Args: - identifier: Attribute name, treated as a Storage Identifier. - - Returns: - AbstractFileSystem: A ready, authenticated filesystem. - - Raises: - AttributeError: If ``identifier`` is not a configured Storage - Identifier. - """ - if identifier.startswith("_"): - message = f"module {__name__!r} has no attribute {identifier!r}" - raise AttributeError(message) - known = identifiers() - if identifier not in known: - message = ( - f"module {__name__!r} has no attribute {identifier!r}; " - f"configured Storage Identifiers are: {', '.join(known)}" - ) - raise AttributeError(message) - # Built outside the membership check so a failure to authenticate surfaces - # as itself rather than as a missing attribute. - return filesystem(identifier) - - -def __dir__() -> list[str]: - """List the module's own names plus every Storage Identifier. - - Returns: - list[str]: Names available on this module, for tab completion. - """ - try: - return sorted({*__all__, *identifiers()}) - except (OSError, ValueError): # pragma: no cover - unreadable configuration - return sorted(__all__) diff --git a/canfar/utils/build.py b/canfar/utils/build.py index f21577c1..83efd9e3 100644 --- a/canfar/utils/build.py +++ b/canfar/utils/build.py @@ -33,7 +33,7 @@ def fetch_parameters( ) -def create_parameters( +def create_parameters( # noqa: PLR0917 name: str | CreateRequest, image: str | None = None, cores: int | None = None, diff --git a/canfar/utils/console.py b/canfar/utils/console.py index 37786e34..9c1d2889 100644 --- a/canfar/utils/console.py +++ b/canfar/utils/console.py @@ -2,12 +2,19 @@ from __future__ import annotations +from contextvars import ContextVar from functools import lru_cache +from typing import TYPE_CHECKING from rich.console import Console from canfar.models.config import Configuration +if TYPE_CHECKING: + from typer.models import Context + +_CLI_ROOT_ACTIVE: ContextVar[bool] = ContextVar("canfar_cli_root_active", default=False) + @lru_cache(maxsize=2) def get_console(*, stderr: bool = False) -> Console: @@ -29,7 +36,25 @@ def emit_active_server_banner() -> None: if not cfg.console.banner: return try: - name = cfg.get_active_server().name + name = ( + cfg.servers[cfg.active.server].name + if cfg.active.server is not None + else None + ) except KeyError: + name = None + if name is None: name = "unknown" get_console().print(f"@{name}", style="dim underline") + + +def activate_cli_root(ctx: Context) -> None: + """Mark callbacks dispatched by the root CLI until their context closes.""" + token = _CLI_ROOT_ACTIVE.set(True) + ctx.call_on_close(lambda: _CLI_ROOT_ACTIVE.reset(token)) + + +def emit_cli_active_server_banner() -> None: + """Emit the banner only when a command runs through the root CLI.""" + if _CLI_ROOT_ACTIVE.get(): + emit_active_server_banner() diff --git a/canfar/utils/logging.py b/canfar/utils/logging.py index af75242b..8da3671b 100644 --- a/canfar/utils/logging.py +++ b/canfar/utils/logging.py @@ -200,11 +200,9 @@ def _resolve_log_level( class CanfarLogger: """Configure stdlib logging for the CANFAR logger name.""" - _configured = False - _rich_handler: RichHandler | None = None - def __init__(self) -> None: """Initialize per-instance file-handler state.""" + self._rich_handler: RichHandler | None = None self._file_handler: logging.handlers.RotatingFileHandler | None = None @property @@ -223,8 +221,7 @@ def configure( target = _resolve_log_file_path(log_file) if log_file is not None else None with _LOCK: install_rich_traceback(show_locals=False, suppress=[]) - if self._configured: - self._cleanup_handlers() + self._cleanup_handlers() if isinstance(loglevel, str): loglevel = getattr(logging, loglevel.upper()) logger = self.logger @@ -251,7 +248,6 @@ def configure( ) logger.propagate = False - self._configured = True def _setup_file_logging( self, diff --git a/canfar/utils/registry.py b/canfar/utils/registry.py index 79f78a3a..db777065 100644 --- a/canfar/utils/registry.py +++ b/canfar/utils/registry.py @@ -3,11 +3,11 @@ from __future__ import annotations import asyncio +import logging from pathlib import Path from pydantic import AnyHttpUrl, BaseModel, ConfigDict -from canfar import get_logger from canfar.auth.x509 import CertificateError from canfar.exceptions.context import AuthContextError, AuthExpiredError from canfar.idp import get_idp, registry_sources @@ -16,7 +16,7 @@ from canfar.models.registry import Server as RegistryResource from canfar.utils.discover import Discover -log = get_logger(__name__) +log = logging.getLogger(__name__) class RegistryEvidenceError(RuntimeError): diff --git a/docs/about/home.md b/docs/about/home.md index 1a807504..bc15c34c 100644 --- a/docs/about/home.md +++ b/docs/about/home.md @@ -1,14 +1,13 @@ # Organization - - -The day-to-day operation of the CANFAR platform is coordinated by the CADC teams. The team management consists of: - -| Name | Role | -|------|---------------------| -| Sharon Goliath | Operations Lead | -| Brian Major | Software Development Lead | -| JJ Kavelaars | CADC Lead | - +CANFAR is operated by teams at the Canadian Astronomy Data Centre (CADC) in +collaboration with research and infrastructure partners. Operations, +platform engineering, user support, and scientific engagement are coordinated +through CADC and the Science Platform project. + +For help with an installed Science Platform, contact +[CANFAR support](mailto:support@canfar.net). For project history and +collaboration, see the [partners](partners.md) and [terms of reference](terms.md) +pages. diff --git a/docs/about/nrc.png b/docs/about/nrc.png deleted file mode 100644 index f513ada6..00000000 Binary files a/docs/about/nrc.png and /dev/null differ diff --git a/docs/agents/architecture.md b/docs/agents/architecture.md index fe86f916..a459aed0 100644 --- a/docs/agents/architecture.md +++ b/docs/agents/architecture.md @@ -13,7 +13,7 @@ Use these notes as navigation guardrails. They are not a refactor backlog. ## Current Seams -- `Configuration` is the persistent config seam. Tests that construct it must isolate `CONFIG_PATH` from the developer's real `~/.canfar/config.yaml`. +- `Configuration` is the validated persisted data seam; `config.editor` owns dotted edits and atomic saves. Tests that construct it must isolate `CONFIG_PATH` from the developer's real `~/.canfar/config.yaml`. - `HTTPClient` is the transport seam. It decides runtime credential precedence before creating `httpx` clients. - `Session` and `AsyncSession` duplicate many operations in sync/async form. Keep behavior aligned when changing either adapter. - CLI modules are adapters over library modules. Prefer testing command parsing/output separately from library behavior. @@ -23,12 +23,13 @@ Use these notes as navigation guardrails. They are not a refactor backlog. ## Authentication Configuration `OIDCCredential` and `X509Credential` Authentication Records live in -`Configuration.authentication`, accessed through `Configuration.get_credential`, -`upsert_credential`, and `update_credential`. `ActiveConfig` owns the active -Authentication and Server Selection references; `HTTPClient` composes +`Configuration.authentication`. The bound `config.editor` owns validated edits +and atomic persistence; Authentication and Platform operations own decisions +about credentials, servers, and Server Selection. `ActiveConfig` stores the +active Authentication and Server Selection references; `HTTPClient` composes `Configuration` and resolves those records for transport. Server Selection -history lives on `Configuration` / `ActiveConfig` directly (there is no -separate `selection.py` shim). +history lives on `ActiveConfig` and the Platform operation (there is no separate +`selection.py` shim). ## Test Caveats diff --git a/docs/agents/research/2026-07-25-storage-python-api.md b/docs/agents/research/2026-07-25-storage-python-api.md index 38c4c4af..b81beff2 100644 --- a/docs/agents/research/2026-07-25-storage-python-api.md +++ b/docs/agents/research/2026-07-25-storage-python-api.md @@ -31,10 +31,11 @@ What changed, VERIFIED against the live service on 0.8.0: `AttributeError: 'StagedReadFile' object has no attribute 'blocksize'`. `Range` is honoured for byte reads, not through the file-object path. -The recommended module has since shipped as `canfar/storage.py`, adding -attribute access (`from canfar.storage import vault, arc`) on top of the -`filesystem` / `fetch` / `sources` surface proposed below, so any text quoting -the old "no public CANFAR storage Python API" wording is historical. +The shipped module's public surface is now `identifiers()` and `filesystem()`. +The earlier proposals for dynamic attributes, `fetch()`, and public `sources()` +were superseded: callers pass Storage Identifiers explicitly, materialize with +standard fsspec methods, and the fsspec-cli source mapping remains private +(`_sources()`). No dynamic protocols or module attributes are registered. Sections 2 and 4 describe the 0.7.0 behaviour and are kept as the historical record. The single-cache-layer recommendation in the Verdict stands, but its diff --git a/docs/changelog.md b/docs/changelog.md index 90cb31c6..afefe06e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1 +1,10 @@ ---8<-- "CHANGELOG.md" \ No newline at end of file +--8<-- "CHANGELOG.md" +# Changelog + +Release notes are published on the [CANFAR releases](releases/releases.md) +page. The complete version history, including changes that predate this site, +is maintained in the repository's +[`CHANGELOG.md`](https://github.com/opencadc/canfar/blob/main/CHANGELOG.md). + +For the current package version and upgrade notes, see the release page for +your version before updating a client or deployment. diff --git a/docs/cli/authentication-contexts.md b/docs/cli/authentication-contexts.md index e22cc637..b9db4c70 100644 --- a/docs/cli/authentication-contexts.md +++ b/docs/cli/authentication-contexts.md @@ -1,44 +1,76 @@ # Authentication and Servers -CANFAR separates identity from routing: +CANFAR keeps identity and routing separate: -- **Authentication** answers "who am I?" -- **Identity Provider (IDP)** names the organization that authenticates you, - such as `cadc` or `srcnet`. -- **Science Platform Server** is the endpoint that runs Sessions. -- **Server selection** chooses which compatible Server receives new requests. +- **Authentication** owns the user's identity and credentials. +- An **Identity Provider (IDP)** issues that identity. The built-in keys are + `cadc` (X.509) and `srcnet` (OIDC Device Authorization). +- A **Science Platform Server** runs Sessions. +- **Server Selection** chooses the Science Platform Server for new requests. -`canfar login` handles the full interactive path: choose an IDP, authenticate, -discover compatible Servers, select one, and save the active pair. +The active Authentication Record and Server Selection are saved together in +the local Configuration. Existing Sessions remain on the Science Platform +Server where they were launched. -## Log in +## Login + +```bash +canfar login [IDP] +``` + +When `IDP` is omitted, the CLI prompts for one. Use these options when needed: + +| Option | Effect | +| --- | --- | +| `--force`, `-f` | Obtain new credentials and rediscover instead of refusing an existing record. | +| `--dev` | Include development registries and endpoints during Server Discovery. | +| `--timeout`, `-t` | HTTP timeout in seconds for login requests; default `10`. | + +Login authenticates the selected IDP, discovers compatible Science Platform +Servers, selects one when necessary, and saves the Authentication Record and +Server Selection. A saved record causes a repeat login to fail unless +`--force` is supplied. + +### CADC X.509 ```bash -canfar login canfar login cadc +``` + +The CLI reuses a usable X.509 certificate when possible. Use `--force` to +obtain a replacement. If the certificate is expired, login reports that and +continues with certificate acquisition. + +### SRCNet OIDC Device Authorization + +```bash canfar login srcnet ``` -Useful options: +The CLI performs OIDC discovery and dynamic client registration, then presents +the Device Authorization challenge in the terminal: -| Option | Use | -| --- | --- | -| `--force`, `-f` | Re-authenticate an IDP that already has saved credentials. | -| `--dev` | Include development Servers during discovery. | -| `--timeout`, `-t` | Increase HTTP timeout for login, discovery, and validation. | +1. It prints a verification URL, user code, and a terminal QR code. +2. It opens the verification URL in the default browser when possible. +3. You sign in and approve the request in the browser. +4. The CLI polls for approval and shows progress until the IDP returns tokens. + +If the browser cannot be opened, visit the printed URL manually. The device +challenge has an IDP-provided expiry; `--timeout` controls HTTP requests and +does not extend that challenge. Denial, expiry, malformed responses, and +network failures stop login with an error; run the command again after fixing +the cause. -Logging controls are root options. Put them before `login` when troubleshooting: +For troubleshooting, put root logging options before `login`: ```bash +canfar --log-level debug login srcnet canfar --log-level debug login cadc --force ``` -See [Logging](logging.md) for level precedence and stream separation. +See [Logging](logging.md) for precedence and stream routing. -`canfar auth login` still exists as a deprecated compatibility alias. New docs -and scripts should use `canfar login`. - -## Inspect authentication +## Inspect Authentication ```bash canfar auth @@ -46,73 +78,77 @@ canfar auth show canfar auth ls ``` -`canfar auth` defaults to `canfar auth show`. - -Use machine output when a script needs stable stdout: +Bare `canfar auth` defaults to `auth show`. Human output includes the active +Server Selection when one is available. For scripts, use machine output on the +data-producing form: ```bash -canfar auth show --json -canfar auth ls --yaml +canfar auth show -o json +canfar auth ls --output yaml ``` -## Switch IDP +The machine payload is an Authentication object or list of Authentication +objects. It contains the canonical IDP key, display name, Authentication Mode, +expiry, active state, and associated Server Name; it does not contain raw +credential material. + +## Change or remove state + +Switch to a saved Authentication Record by canonical IDP key: ```bash canfar auth use srcnet canfar auth use cadc ``` -When you switch IDP, CANFAR tries to keep routing usable: +When switching, CANFAR reuses a compatible remembered Server Selection when +one exists. If several compatible Servers need a choice, the CLI prompts for a +Server URI or list number. -1. Reuse the current Server when it belongs to the target IDP. -2. Reuse the remembered Server for that IDP when it is still valid. -3. Auto-select the only compatible Server. -4. Prompt when multiple compatible Servers exist. - -## Manage Servers +Remove one Authentication Record and its associated Servers with: ```bash -canfar server ls -canfar server use canfar -canfar server use ivo://cadc.nrc.ca/skaha +canfar auth rm srcnet +canfar auth rm srcnet --force ``` -Server names are convenient for humans. Server URIs are stable for scripts. - -`canfar server ls` shows Servers for the active IDP. If no saved Servers exist -for that IDP, the command runs discovery and stores the results. - -## Remove saved auth state +Removing the active Authentication asks for confirmation unless `--force` is +used. To reset all Authentication and Server state, use the required force +flag: ```bash -canfar auth rm srcnet canfar auth purge --force ``` -`auth rm ` removes the Authentication record and Servers associated with -that IDP. Removing the active IDP asks for confirmation unless `--force` is -passed. +The purge restores built-in defaults and preserves unrelated registry and +console settings. + +## Server Selection -`auth purge --force` resets Authentication and Server state while preserving -unrelated configuration such as console and Container Registry settings. +```bash +canfar server ls +canfar server ls -o json +canfar server use SELECTOR +``` -## Python equivalents +`server ls` lists the saved Servers for the active IDP. If none are saved, it +runs discovery and persists the result. Its machine payload is a list of +Server records and is data-only on stdout. -```python -import canfar +`server use` accepts either a Server Name or an IVOA URI: -canfar.login("cadc") -canfar.server.use("ivo://cadc.nrc.ca/skaha") -canfar.authentication.use("srcnet") +```bash +canfar server use canfar +canfar server use ivo://cadc.nrc.ca/skaha ``` -Python helpers are noninteractive. CLI commands own prompts and human rendering. +The persisted `active.server` value is the Server Name. The IVOA URI is +discovery metadata and can still be used as a selector. -## Configuration +## Configuration shape -The current config shape is versioned with `version: 1`. Authentication -records are keyed by IDP and Servers are keyed by Server Name; `active.server` -references a Server by name. +The persisted shape separates Authentication Records and Science Platform +Servers. `active.server` refers to a Server Name, not an IVOA URI: ```yaml version: 1 @@ -129,34 +165,19 @@ servers: url: https://ws-uv.canfar.net/skaha ``` -Dict keys make every value reachable with dotted paths: +Use `canfar config get` and `canfar config set` for dotted configuration +paths. Values passed to `config set` are parsed as YAML: ```bash +canfar config get active.server canfar config get servers.canfar.url -canfar config get authentication.cadc.path +canfar config set console.banner false ``` -Environment overrides use nested active fields: +Environment overrides use the same nested names, for example: ```bash -CANFAR_ACTIVE__AUTHENTICATION=srcnet -CANFAR_ACTIVE__SERVER=canfar +CANFAR_CONSOLE__BANNER=false canfar auth show ``` -Legacy or unsupported config files are backed up to -`..back` before a default config is written. - -## Machine output rules - -| Rule | Behavior | -| --- | --- | -| Supported flags | `--json` and `--yaml` | -| Placement | Put the flag after the command that emits data, for example `canfar auth ls --json`. | -| Unsupported placement | `canfar auth --json ls` exits 2. | -| Conflicts | `--json --yaml` exits 2. | -| stdout | Data only in machine mode. | -| stderr | Diagnostics and errors. | -| Unsupported commands | Exit 1 with `machine output not supported for this command yet` and `use default human output for now`. | - -Lists have no ordering guarantee. Scripts should select by IDP key, Server -Name, URI, Session ID, or another stable field. +The complete command and machine-output contract is in the [CLI reference](cli-help.md). diff --git a/docs/cli/cli-help.md b/docs/cli/cli-help.md index 0f3f93db..fee6e153 100644 --- a/docs/cli/cli-help.md +++ b/docs/cli/cli-help.md @@ -1,15 +1,29 @@ -# CLI Reference +# CLI reference -Use `canfar` for Authentication, Science Platform Server selection, Session -management, Container Images, and client configuration. +The `canfar` command manages Authentication, Science Platform Server +selection, Sessions, Container Images, data sources, and client configuration. +Run `canfar --help` or append `--help` to any command for the installed +options. -```bash -canfar --help -``` +## Canonical command surface + +The root keeps Session and information operations as leaves. Authentication, +Server, data, image, and configuration operations remain grouped: + +| Area | Commands | +| --- | --- | +| Authentication and Server Selection | `login`, `auth`, `server` | +| Sessions | `create`, `ps`, `events`, `info`, `open`, `logs`, `delete`, `prune` | +| Platform information | `stats`, `image ls` | +| Client configuration | `config show`, `config get`, `config set`, `config path`, `version` | +| Data | `data` and its embedded file commands | -## Root logging controls +There are no extra command names between the root and these leaves. In +particular, Session creation is `canfar create`, and login is `canfar login`. -Put logging options before the command: +## Root logging options + +Root logging options must come before the command: ```bash canfar --log-level debug ps @@ -17,40 +31,41 @@ canfar -vvv ps canfar --log-file ./logs/canfar.jsonl ps ``` -| Option | Use | +| Option | Effect | | --- | --- | -| `--log-level LEVEL` | Select `critical`, `error`, `warning`, `info`, or `debug`. | -| `-v` | Increase verbosity; repeat through `-vvvv` for `debug`. | +| `--log-level LEVEL` | `critical`, `error`, `warning`, `info`, or `debug`. | +| `-v` | Increase verbosity; four or more repetitions select `debug`. | | `--log-file PATH` | Add the rotating JSON Lines file sink. | -See [Logging](logging.md) for the exact mapping, precedence, streams, file -schema, and stable error codes. +See [Logging](logging.md) for precedence, stream routing, file records, and +setup errors. ## Authentication and Servers -### `canfar login` +### Login ```bash canfar login [IDP] [OPTIONS] ``` -| Option | Use | -| --- | --- | -| `--force`, `-f` | Force re-authentication. | -| `--dev` | Include development Servers during discovery. | -| `--timeout`, `-t` | HTTP timeout in seconds during login. Default: `10`. | +`IDP` is an optional canonical Identity Provider key. Without it, the CLI +prompts for one. The options are: -Examples: +| Option | Effect | +| --- | --- | +| `--force`, `-f` | Re-authenticate an existing Authentication Record. | +| `--dev` | Include development registries and endpoints during discovery. | +| `--timeout`, `-t` | HTTP timeout in seconds; default `10`. | ```bash -canfar login canfar login cadc canfar login srcnet --force ``` -`canfar auth login` is a deprecated compatibility alias. +The interactive credential flow and Server Selection are described in +[Authentication and Servers](authentication-contexts.md). -### `canfar auth` +### Authentication ```bash canfar auth @@ -61,169 +76,155 @@ canfar auth rm IDP [--force] canfar auth purge --force ``` -| Command | Use | -| --- | --- | -| `auth` / `auth show` | Show active Authentication state. | -| `auth ls` | List saved Authentication records. | -| `auth use IDP` | Switch active Authentication by canonical IDP key. | -| `auth rm IDP` | Remove one Authentication record and its Servers. | -| `auth purge --force` | Reset Authentication and Server state. | - -Machine output is supported for `auth` / `auth show` and `auth ls`: - -```bash -canfar auth show --json -canfar auth ls --yaml -``` +`canfar auth` is the same active-Authentication view as `canfar auth show`. +`auth use` selects a saved Authentication Record; `auth rm` also removes the +associated Servers; `auth purge --force` resets Authentication and Server state. -### `canfar server` +### Server Selection ```bash canfar server ls canfar server use SELECTOR ``` -| Command | Use | -| --- | --- | -| `server ls` | List Servers for the active IDP, discovering them when needed. | -| `server use SELECTOR` | Select a Server by name or URI. | - -Use URIs in scripts because names can be ambiguous: - -```bash -canfar server use ivo://cadc.nrc.ca/skaha -``` +`SELECTOR` may be a Server Name or an IVOA URI. `server ls` lists Servers for +the active IDP and discovers them when no saved Servers are available. ## Sessions -### `canfar create` +### Create ```bash canfar create [OPTIONS] KIND IMAGE [-- CMD [ARGS]...] ``` -| Option | Short | Use | -| --- | --- | --- | -| `--name` | `-n` | Session name. Defaults to a generated name. | -| `--cpu` | `-c` | Fixed CPU cores. Omit for flexible allocation. | -| `--memory` | `-m` | Fixed RAM in GB. Omit for flexible allocation. | -| `--gpu` | `-g` | GPU count. | -| `--env` | `-e` | Environment variable as `KEY=VALUE`. | -| `--replicas` | `-r` | Number of replicas. Default: `1`. | -| `--debug` | | Print parsed Session request details. | -| `--dry-run` | | Parse parameters and exit. | -| `--json` | | Emit created Session IDs as a JSON list. | -| `--yaml` | | Emit created Session IDs as a YAML list. | +`KIND` is one of `desktop`, `notebook`, `carta`, `headless`, `firefly`, or +`contributed`. `IMAGE` is a CANFAR Container Image such as +`skaha/astroml:latest`; the client adds the CANFAR registry and `:latest` when +they are omitted. -`--json` and `--yaml` are mutually exclusive. `--dry-run` is human-output only. +| Option | Effect | +| --- | --- | +| `--name`, `-n` | Session name; a generated name is used by default. | +| `--cpu`, `-c` | Requested CPU cores. | +| `--memory`, `-m` | Requested RAM in GB. | +| `--gpu`, `-g` | Requested GPU count. | +| `--env`, `-e KEY=VALUE` | Set an environment variable; repeat as needed. | +| `--replicas`, `-r` | Number of Sessions; default `1`. | +| `--debug` | Print the parsed Session request. | +| `--dry-run` | Validate and print the request without creating a Session. | +| `--output`, `-o` | Emit created Session IDs as `json` or `yaml`. | -Examples: +Everything after `--` belongs to the container command. The first token is +the command and the remaining tokens are its arguments, even when they look +like CANFAR options: ```bash -canfar create notebook skaha/astroml:latest -canfar create --cpu 4 --memory 16 notebook skaha/astroml:latest canfar create headless skaha/terminal:1.1.2 -- python /arc/projects/demo/run.py +canfar create headless skaha/terminal:1.1.2 -- worker --output json -o yaml ``` -### `canfar ps` +The `--output` option must appear before `--`. `--dry-run` cannot be combined +with machine output. + +### List Sessions ```bash canfar ps [OPTIONS] ``` -| Option | Short | Use | -| --- | --- | --- | -| `--all` | `-a` | Show all Sessions. Default shows running Sessions. | -| `--quiet` | `-q` | Print only Session IDs. | -| `--kind` | `-k` | Filter by Session Kind. | -| `--status` | `-s` | Filter by status. | -| `--debug` | | Show Session response warnings. | +| Option | Effect | +| --- | --- | +| `--all`, `-a` | Include all statuses; otherwise show `Pending` and `Running`. | +| `--quiet`, `-q` | Print matching Session IDs in human mode. | +| `--kind`, `-k` | Filter by Session Kind. | +| `--status`, `-s` | Pass a status filter to the Science Platform. | +| `--debug` | Print Session response warnings. | +| `--output`, `-o` | Emit the filtered Session response array as `json` or `yaml`. | -Machine output: +`ps` stays a thin operation over `Session.fetch()`: `--kind` and `--status` +are sent to the fetch call, then the CLI applies the running/default or +`--all` view to the returned responses. `--quiet` follows that same filter, +including when combined with `--all`, and is not available with `--output`. ```bash -canfar ps --json -canfar ps --yaml +canfar ps +canfar ps --all --kind headless +canfar ps -o json ``` -`--quiet` is a human-output shortcut and is incompatible with machine output. +### Inspect, open, and remove -### Inspect and clean up +These leaves accept one or more Session IDs and produce human-readable output: -```bash -canfar events SESSION_ID... -canfar info SESSION_ID... -canfar logs SESSION_ID... -canfar open SESSION_ID... -canfar delete SESSION_ID... [--force] -canfar prune PREFIX [KIND] [STATUS] -``` - -Quote `PREFIX` when it contains shell metacharacters -(for example `canfar prune 'rabi.*' headless Completed`). - -`canfar info SESSION_ID --debug` shows Session response warnings. This is -a command diagnostic, not a logging-level control. +| Command | Purpose | +| --- | --- | +| `canfar events SESSION_ID...` | List Science Platform events. | +| `canfar info SESSION_ID...` | Show Session details; `--debug` adds response warnings. | +| `canfar logs SESSION_ID...` | Show Session logs. | +| `canfar open SESSION_ID...` | Open ready Sessions in new browser tabs. | +| `canfar delete SESSION_ID... [--force]` | Delete Sessions, confirming unless `--force` is used. | +| `canfar prune PREFIX [KIND] [STATUS]` | Delete matching names; defaults to `headless` and `Succeeded`. | -Common flow: +`prune` treats a plain `PREFIX` as a literal prefix. A value containing regex +metacharacters is treated as a regular expression. Quote such values so the +shell does not expand them: ```bash -canfar ps -canfar info $(canfar ps -q) -canfar open $(canfar ps -q) -canfar delete $(canfar ps -q) +canfar prune 'notebook.*' notebook Completed ``` -## Images and platform state +### Platform information ```bash -canfar image ls canfar stats +canfar image ls +canfar image ls --kind notebook +canfar version +canfar version --debug ``` -Use `canfar image --help` and `canfar stats --help` for command-specific -options. +`stats` prints platform usage tables. `image ls` lists Container Images and +accepts the image-kind filter shown above. `version --debug` prints client, +Python, operating-system, and dependency details for a bug report; it is not a +logging control. -## Data +## Data and configuration ```bash -canfar data ls -lh arc:/home/[username] -canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits +canfar data --help +canfar config show +canfar config get console.width +canfar config set console.width 132 +canfar config path ``` -Data operands use explicit `storage-identifier:/absolute/path` or -`local:/absolute/path` syntax. See [Data commands](data.md) for recursive copy, -cross-source copy and verification, recursive-removal policy, and the exact -supported boundary. +Data operands use an explicit `Storage Identifier`, for example +`arc:/home/user/file.fits` or `local:/tmp/file.fits`. See [Data commands](data.md) +for the embedded command surface. Configuration keys use dotted paths; values +passed to `config set` are parsed as YAML. See [Authentication and Servers](authentication-contexts.md) +for the persisted Authentication, Server, and active-selection shape. + +## Machine output -## Client configuration +The leaf option `-o/--output` accepts only `json` or `yaml`. It is available on +the data-producing forms `auth` (the default active view), `auth show`, +`auth ls`, `server ls`, `create`, `ps`, `config show`, and `config get`: ```bash -canfar config show -canfar config path -canfar config get console.width -canfar config get servers.canfar.url -canfar config set console.width 132 -canfar config set console.banner false -canfar version -canfar version --debug +canfar auth show -o json +canfar server ls --output yaml +canfar create headless skaha/terminal:1.1.2 -o json +canfar config get active.server -o json ``` -`config set` parses values as YAML. -`console.banner` defaults to `true`. Set it to `false` to hide the -`@` prefix from human-readable CLI output. The banner is always -disabled for `--json` and `--yaml` output. -`version --debug` shows environment and dependency details for bug reports; it -does not change the logging level. +The payload is the same Python result used by that command after its normal +filtering. For example, `create -o json` is a raw list of Session IDs and +`ps -o json` is a filtered array of Session response objects; neither is +wrapped in a command-specific envelope. Human active-Server banners and logs +never precede the machine payload on stdout. Diagnostics and errors use stderr. -## Machine output contract - -| Rule | Behavior | -| --- | --- | -| Flags | `--json` or `--yaml`. | -| Placement | Put the flag after the command, for example `canfar ps --json`. | -| Conflict | `--json --yaml` exits 2. | -| stdout | Data payload only. | -| stderr | Diagnostics and errors. | -| Unsupported command | Exits 1 with a clear unsupported-machine-output message. | -| Ordering | List ordering is not guaranteed. | +Put `-o/--output` after the command that owns it. Root `-o`, `--json`, and +`--yaml` are not options, and commands without machine output reject `-o`. +For `create`, `-o` is parsed only before `--`; after the delimiter it is a +container-command token. diff --git a/docs/cli/data.md b/docs/cli/data.md index 322af2bb..476ff8a3 100644 --- a/docs/cli/data.md +++ b/docs/cli/data.md @@ -1,219 +1,88 @@ # Data commands -Use `canfar data` to work with configured VOSpace Services and the local -filesystem through the embedded `fsspec-cli` command application. +`canfar data` delegates to the embedded `fsspec-cli` command application. It +maps every configured VOSpace Service by its Storage Identifier and always +adds the reserved `local` source for the machine running the command. -## Install and authenticate +## Sources and operands -Data commands are included in the standard installation: - -```bash -pip install canfar -canfar login cadc -``` - -## Address mapped sources - -Every operand starts with a Storage Identifier: the handle of a configured -VOSpace Service, or the reserved `local` identifier for the machine where the -command runs. Operands always pair an identifier with an absolute path: +Run `canfar login` for the IDP that owns a remote VOSpace Service, then use an +explicit source-qualified operand: ```text storage-identifier:/absolute/path local:/absolute/path ``` -Every mapped source is available concurrently, regardless of the active Server -Selection. A default CADC login maps two VOSpace Services, `arc` and `vault`, -plus `local`: +The default CADC configuration provides `arc`, `vault`, and `local` when its +default Server record is present: ```bash -canfar data ls -lh arc:/ -canfar data ls -lh arc:/home/[username] +canfar data ls -lh arc:/home/user canfar data ls -lh vault:/ +canfar data ls -lh local:/tmp ``` -Use `ls -lh`, or the `ll -h` long-form command, for a human-readable listing. -The `-h` flag reports human-readable sizes and requires a long listing, so -`ls -h` on its own exits with `ls: -h: requires long listing`. +Storage Identifiers are configuration keys, not protocols. There is no +`active:/` source, bare local-path shorthand, empty `:/path` source, or +`canfar storage` command. A data command sees all configured sources, not only +the active Server Selection. -Operands must name a mapped source. Empty `:/path`, bare local paths such as -`/tmp/file`, and `active:/path` are all rejected; there is no `active` alias -and no `canfar storage` command. +## Command surface -## Copy files and directories +Use `canfar data --help` for the installed upstream options. The available +commands are: -Copy one file between local and remote sources: +| Command | Purpose | +| --- | --- | +| `basename`, `dirname` | Transform a path string. | +| `info`, `size`, `stat`, `test` | Inspect a file or evaluate a predicate. | +| `ls`, `ll` | List directory contents; `ls` accepts `-A`, `-l`, and `-h`, while `ll` accepts `-A` and `-h`. | +| `du` | Estimate file space usage; supports `-s` and `-h`. | +| `find`, `tree` | Traverse recursively; `find` supports `--maxdepth` and `--type f|d`, while `tree` supports `--maxdepth`. | +| `head`, `tail`, `cat` | Read leading bytes, trailing bytes, or file contents. | +| `cp` | Copy files or one directory; `-R`/`-r` enables recursive copy. | +| `mv` | Move or rename files on one mapped filesystem. | +| `mkdir`, `rmdir`, `unlink`, `rm` | Create directories, remove empty directories, remove one file, or remove files. | -```bash -canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits -canfar data cp arc:/home/[username]/file.fits local:/absolute/path/file.fits -``` +Recursive removal is disabled by CANFAR policy, so `data rm` has no `-R` or +`-r` option. `data mv` does not implement a cross-source move; copy between +sources and verify the destination before removing the source separately. -Copy between two VOSpace Services, for example a public test cutout from -`vault` into your `arc` home directory: +## Examples + +List and inspect a remote object: ```bash -canfar data cp vault:/ALMA/test-data/cutouts/test-4d-cube-cutout.fits arc:/home/[username]/test-4d-cube-cutout.fits -canfar data ls -lh arc:/home/[username]/test-4d-cube-cutout.fits +canfar data ls -lh arc:/home/user +canfar data info arc:/home/user/file.fits +canfar data cat arc:/home/user/file.fits ``` -Recursive copy is enabled for admitted local and remote source pairs: +Copy between local and remote sources: ```bash -canfar data cp -R local:/absolute/path/dataset arc:/home/[username]/dataset +canfar data cp local:/tmp/file.fits arc:/home/user/file.fits +canfar data cp arc:/home/user/file.fits local:/tmp/file.fits ``` -The tagged upstream implementation builds a bounded manifest, copies files -through host-local staging when sources differ, and verifies destination -metadata. Recursive copy is not atomic and does not create a snapshot; inspect -the destination before removing any source data. - -## Move data between sources - -Cross-source `mv` is unsupported and exits with status 2: +Recursive copy is available for one directory and its descendants: -```text -mv: cross-source move unsupported +```bash +canfar data cp -R local:/tmp/dataset arc:/home/user/dataset ``` -Move a file between sources explicitly by copying it, verifying the -destination, and only then issuing a separate source removal: +For a cross-source move, make the verification and removal explicit: ```bash -canfar data cp vault:/folder/file.fits arc:/home/[username]/file.fits -canfar data ls -lh arc:/home/[username]/file.fits +canfar data cp vault:/folder/file.fits arc:/home/user/file.fits +canfar data info arc:/home/user/file.fits canfar data rm vault:/folder/file.fits ``` -Do not use this sequence as a one-command or atomic move. - -Recursive removal is disabled by application policy, so `rm` accepts no `-R` or -`-r` flag at all and exits with status 2: - -```text -No such option: -R -``` - -Because recursive directory removal is disabled, directory movement is not a -supported workflow in this release. Any future one-command relocation would be a -separately named, opt-in orchestration feature with stronger destination -verification and residual-state semantics—not portable `mv`. - -## Cache data locally - -### Directory listings - -Directory listings are cached automatically. Each command builds and closes its -own filesystem, so a cached listing only ever serves the command that produced -it and can never return a listing that outlives it. Repeated lookups while one -command walks a tree are served without another round trip. - -### Files and byte ranges - -There is no CLI flag for caching file contents, but `vosfs` is a normal -[fsspec](https://filesystem-spec.readthedocs.io/) filesystem, so any fsspec -cache can wrap it from Python. On a CANFAR session, `/scratch` is fast local -disk and is the right place to point a cache; it is not backed up and is -cleared when the session ends, which is exactly what a cache wants. - -Cache whole files under a named directory. The first read fetches over the -network, and later reads come from `/scratch`: - -```python -from pathlib import Path - -from fsspec.implementations.cached import WholeFileCacheFileSystem -from vosfs import VOSpaceFileSystem - -vault = VOSpaceFileSystem( - "https://cadc-west-01.canfar.net/vault", - certfile=str(Path.home() / ".ssl" / "cadcproxy.pem"), -) -cached = WholeFileCacheFileSystem(fs=vault, cache_storage="/scratch/vault-cache") - -data = cached.cat_file("/ALMA/test-data/cutouts/test-4d-cube-cutout.fits") -``` - -Use `SimpleCacheFileSystem` instead when you do not need the expiry and -staleness metadata that `WholeFileCacheFileSystem` keeps. Passing -`cache_storage` a list of directories tries each in order and treats only the -last as writable, so a shared read-only cache can back your own. - -### Cache byte ranges - -`vosfs` sends an HTTP `Range` header and uses the response when the byte -endpoint answers `206`, so a partial read such as `cat_file(path, start, end)` -transfers only the bytes you asked for. Range support is per-backend: - -| Storage Identifier | Backend | Ranged reads | -| --- | --- | --- | -| `vault` | `minoc` | Yes — a partial read returns `206` and transfers only that slice | -| `arc` | Cavern | No — the whole object is fetched and sliced, which is correct but not cheaper | - -Because a range is now a real partial transfer against `vault`, a block cache -is worth using there. `MMapCache` keeps fetched blocks in a sparse file, so -only the blocks you touch occupy disk: - -```python -from fsspec.caching import MMapCache - -path = "/ALMA/test-data/cutouts/test-4d-cube.fits" -size = vault.info(path)["size"] -blocks = MMapCache( - blocksize=1 << 20, - fetcher=lambda start, end: vault.cat_file(path, start, end), - size=size, - location="/scratch/vault-cache/test-4d-cube.blocks", -) - -header = blocks._fetch(0, 2880) # one FITS header block, one 1 MiB range request -``` - -Reading a FITS header from a 3.4 MB cube this way issues a single ranged -request and materialises one block of four; a second read of the same range is -served from `/scratch`. The saving is in bytes transferred rather than seconds -on small files, because VOSpace transfer negotiation dominates a short request. -It grows with file size, and matters most when many reads hit different parts -of one large cube. - -Against `arc` a block cache still costs a whole download per block, so cache -whole files there instead. - -The `blockcache` filesystem remains unavailable over a VOSpace Service. `Range` -is honoured for byte reads, not through the file-object path, so wrapping -`CachingFileSystem` still fails: - -```text -AttributeError: 'StagedReadFile' object has no attribute 'blocksize' -``` - -Stacked caches do not help either: chaining them (`filecache::simplecache::`) -builds the layers, but the inner layer is never filled and never serves, so use -exactly one cache layer on your fastest local disk. - -These caches use the synchronous filesystem interface. Build the filesystem -without `asynchronous=True`, as above. - -## Output and accepted omissions - -Data command stdout belongs to the embedded command; CANFAR does not prepend -the active-Server banner or add JSON/YAML envelopes. Diagnostics are written to -stderr. - -The Python equivalent of these commands is documented in -[Data Access](../client/data.md). - -This release intentionally provides no FUSE mount, signed-URL extension, -progress display, confirmation prompt, `:/path` or bare-path shorthand, -`active` alias, `canfar storage` alias, recursive removal, or cross-source `mv` -workflow. - -## Upstream releases +Data command stdout belongs to the embedded command. CANFAR does not prepend +the active-Server banner or add a JSON/YAML envelope, and data commands do not +provide the CANFAR `-o/--output` option. -CANFAR installs pinned, tagged releases of -[`vosfs`](https://github.com/shinybrar/vosfs/releases/tag/v0.8.0) and -[`fsspec-cli`](https://github.com/shinybrar/vosfs/releases/tag/fsspec-cli-v0.7.0). -CANFAR tests its composition, configuration, authentication, and output seams; -exhaustive filesystem-command and backend matrices remain in the upstream -project. +For Python access, explicit `Storage Identifier` resolution and cache guidance +are documented in [Data Access](../client/data.md). diff --git a/docs/cli/logging.md b/docs/cli/logging.md index 54d52035..8cdddff0 100644 --- a/docs/cli/logging.md +++ b/docs/cli/logging.md @@ -1,16 +1,12 @@ # Logging -CANFAR configures logging only at an explicit application entry point. The CLI -does this once before dispatching a command. Python applications can call -`canfar.configure_logging(...)` themselves; importing `canfar` does not configure -handlers or consoles. +The CLI configures Python standard-library logging once at the root entry +point. Human logs use Rich on stderr. File logging is opt-in and writes a +rotating JSON Lines file. -Logging uses Python's standard `logging` library with Rich for human stderr -output, plus an optional rotating JSON Lines file sink. +## Controls and precedence -## CLI controls and precedence - -Logging controls are root options, so put them before the command: +Root controls must precede the command: ```bash canfar --log-level debug ps @@ -18,185 +14,94 @@ canfar -vvv ps canfar --log-file ./logs/canfar.jsonl ps ``` -The supported controls are: - -| Control | Effect | +| Control | Result | | --- | --- | | No CLI control | Use `CANFAR_LOGLEVEL`, or `critical` when it is unset. | -| `-v` | `error` | -| `-vv` | `warning` | -| `-vvv` | `info` | -| `-vvvv` or more | `debug` | +| `-v` | `error`; `-vv` is `warning`; `-vvv` is `info`; `-vvvv` and above are `debug`. | | `--log-level LEVEL` | Select `critical`, `error`, `warning`, `info`, or `debug`. | -| `--log-file PATH` | Add the rotating JSON Lines file sink described below. | - -Precedence is: - -1. `--log-level` -2. repeated `-v` -3. `CANFAR_LOGLEVEL` -4. the packaged default, `critical` - -`--log-level` therefore wins when it is combined with `-v`. Level names are -case-insensitive. +| `--log-file PATH` | Add the rotating JSON Lines file sink. | -The machine-output flags remain leaf options after the command. For example: +Precedence is `--log-level`, repeated `-v`, `CANFAR_LOGLEVEL`, then the +packaged `critical` default. Level names are case-insensitive. A value in +`CANFAR_LOGLEVEL` is validated only when it is the effective source; unknown +`CANFAR_*` variables do not affect logging. -```bash -canfar --log-level debug ps --json -``` - -The corresponding environment setting is: +The machine-output option remains owned by the leaf command: ```bash -CANFAR_LOGLEVEL=info canfar ps -``` - -Only the documented CANFAR logging variables affect this policy. Unknown -CANFAR logging variables are ignored. - -## Python applications - -Call the same runtime seam explicitly in a Python application: - -```python -from canfar import configure_logging - -configure_logging(loglevel="debug") -``` - -Calling `configure_logging()` without a level uses `CANFAR_LOGLEVEL`, then the -packaged `critical` default. A Python process that never calls this function -retains the logging configuration chosen by its application. - -To add a file sink, pass a `pathlib.Path`: - -```python -from pathlib import Path - -from canfar import configure_logging - -configure_logging( - loglevel="info", - log_file=Path("logs/canfar.jsonl"), -) +canfar --log-level debug ps -o json ``` -There is no per-`HTTPClient`, `Session`, or `AsyncSession` log-level setting. +## Streams and machine output -## HTTP request and response debug - -At `debug` (for example `--log-level debug` or `-vvvv`), every Science Platform -HTTP call logs the request method and full URL, then the response status and -body: - -```text -DEBUG GET https://ws-uv.canfar.net/skaha/v0/session?status=Running -DEBUG HTTP STATUS CODE -> 200 -[{"id":"...","status":"Running",...}] -``` - -These lines follow the normal logging policy: stderr (and the optional file -sink), never mixed into `--json`/`--yaml` stdout payloads. - -## stdout and stderr - -CANFAR keeps command data separate from diagnostics: - -| Stream | Content | -| --- | --- | -| stdout | Human command results or the selected JSON/YAML data payload. | -| stderr | Rich-oriented logs, warnings, and errors; structured diagnostics in machine mode. | - -JSON and YAML stdout remain data-only at every log level. Redirect the streams -independently when a script needs both: +Human command results go to stdout. Logs, warnings, and errors go to stderr. +With `-o json` or `--output yaml`, stdout contains only the selected command +payload; logs and structured diagnostics remain on stderr: ```bash -canfar --log-level debug ps --json \ +canfar --log-level debug ps -o json \ > sessions.json \ 2> diagnostics.log ``` -In `--json` or `--yaml` mode, logging setup failures and file-sink warnings are -serialized in the selected format on stderr. They never add a banner or log -line to the command payload on stdout. - -## Rotating JSON Lines file sink +Logging setup warnings use the selected machine format on stderr when a leaf +output mode is present. They never add a banner or log record to the machine +payload on stdout. -File logging is opt-in. Use the root option or the Python `Path` argument shown -above. Relative paths resolve from the current working directory, and missing -parent directories are created. +At `debug`, Science Platform HTTP hooks log the request method and URL and the +response status and body. These records follow the same stderr/file routing; +do not enable debug logging if response bodies must remain private. -When enabled, events go to both stderr and the file. The file policy is fixed: +## JSON Lines file sink -- UTF-8 JSON Lines, one object per physical line; -- size-based rotation at 10 MiB; -- 10 backup files; and -- escaped exception and stack text so a record never spans physical lines. +File logging is enabled only with `--log-file PATH`. Relative paths resolve +from the current working directory and missing parent directories are created. +There is no default log file and no temporary-file fallback. `-` and an +existing directory are invalid targets. -There is no default log file, configuration-file log path, or temporary-file -fallback. In particular, CANFAR does not write `~/.canfar/client.log` unless -that exact path is explicitly requested. +The sink is UTF-8 JSON Lines with size-based rotation at 10 MiB and ten backup +files. Each event contains: -An existing directory and the pseudo-target `-` are invalid file paths. Other -initialization, write, or rollover failures disable only the file sink, keep -stderr logging and command execution active, and emit one -`logging.file_sink_unavailable` warning. - -### JSON Lines schema - -Every file event contains: - -| Field | Required | Meaning | -| --- | --- | --- | -| `timestamp` | Yes | UTC RFC3339 timestamp with `Z` suffix and millisecond precision. | -| `level` | Yes | Logging level name, such as `INFO` or `ERROR`. | -| `logger` | Yes | Logger name, such as `canfar.sessions`. | -| `message` | Yes | Rendered message. | -| `exception` | No | Escaped exception or stack text. | - -Example shape: - -```json -{"timestamp":"2026-07-11T12:34:56.789Z","level":"INFO","logger":"canfar.sessions","message":"Session request accepted"} -``` - -Authentication Record secrets use Pydantic `SecretStr` and render as masked -values in Configuration dumps. Do not log raw token or certificate material. +| Field | Meaning | +| --- | --- | +| `timestamp` | UTC RFC3339 timestamp with millisecond precision and a `Z` suffix. | +| `level` | Logging level such as `INFO` or `ERROR`. | +| `logger` | Logger name such as `canfar.sessions`. | +| `message` | Rendered message. | +| `exception` | Escaped exception or stack text when present. | -## Stable logging diagnostics +One JSON object occupies one physical line. Authentication Record secrets are +masked by their secret types; do not treat log output as a place to expose +credential material. -Logging diagnostics use stable dotted-domain codes: +## Setup diagnostics -| Code | Behavior and details | +| Code | Meaning | | --- | --- | -| `logging.invalid_env_value` | Fatal setup error. Includes `env_var`, `provided_value`, and `expected`. | -| `logging.invalid_file_path` | Fatal setup error for `-` or an existing directory. Includes the standard `code`, `message`, and `hint` fields. | -| `logging.file_sink_unavailable` | Non-fatal warning for initialization, write, or rollover failure. The command continues with stderr logging; machine mode emits a structured warning on stderr. | +| `logging.invalid_env_value` | `CANFAR_LOGLEVEL` is invalid. Setup stops before the command. | +| `logging.invalid_file_path` | `--log-file` is `-` or an existing directory. Setup stops before the command. | +| `logging.file_sink_unavailable` | The file sink cannot initialize, write, or rotate. The command continues with stderr logging. | -Fatal logging setup errors exit with status 2 before the command executes. -The file-sink warning does not replace the command's normal exit status. +The first two are fatal setup errors and exit with status `2`. A file-sink +failure is non-fatal and disables only that sink. In machine mode its warning +is a structured payload on stderr; the command's normal exit status is kept. -## Domain `--debug` flags +## Domain `--debug` options -Root logging controls and command diagnostics are separate. These retained -leaf flags have domain-specific meanings and do not select the logging level: +Root logging controls and command diagnostics are separate. These are the +retained leaf `--debug` meanings: -| Command | `--debug` meaning | +| Command | Meaning | | --- | --- | | `canfar version --debug` | Show environment and dependency details for a bug report. | | `canfar info SESSION_ID --debug` | Show Session response warnings. | | `canfar ps --debug` | Show Session response warnings. | | `canfar create KIND IMAGE --debug` | Print parsed Session request details. | -Logging-only `--debug` flags were removed from `login`, the deprecated -`auth login` alias, `delete`, `events`, `logs`, `open`, `prune`, and `stats`. -Use a root control instead: +Other leaves do not use `--debug` as a logging switch. Use a root control, +for example: ```bash -canfar --log-level debug login cadc --force -canfar --log-level debug info abc123 --debug +canfar --log-level debug login srcnet +canfar --log-level debug info SESSION_ID --debug ``` - -The second example requests both debug-level logs and the independent Session -response-anomaly details. diff --git a/docs/cli/quick-start.md b/docs/cli/quick-start.md index d5658afb..42642033 100644 --- a/docs/cli/quick-start.md +++ b/docs/cli/quick-start.md @@ -1,131 +1,108 @@ -# CLI Quickstart +# CLI quickstart -Create a notebook Session from a terminal, open it, inspect it, and clean it up. +This walkthrough logs in, launches a Session, checks it, opens it, and removes +it. Replace `SESSION_ID` with the ID printed by `canfar create` or `canfar ps`. -## 1. Install - -```bash -pip install canfar --upgrade -``` - -## 2. Log in +## Install and log in ```bash +pip install --upgrade canfar canfar login cadc ``` -Use SRCNet when that is your IDP: +Use SRCNet OIDC instead when that is your Identity Provider: ```bash canfar login srcnet ``` -Check the active Authentication and available Servers: +Inspect the active Authentication and available Servers: ```bash canfar auth show canfar server ls ``` -## 3. Work with data - -The standard installation includes data commands. Use a configured Storage Identifier -or the reserved `local` name with an absolute path. A default CADC login maps -`arc` and `vault`: +For the OIDC Device Authorization steps, see [Authentication and Servers](authentication-contexts.md). -```bash -canfar data ls -lh arc:/home/[username] -canfar data cp local:/absolute/path/file.fits arc:/home/[username]/file.fits -``` +## Optional data check -Cross-source `mv` is unsupported. Copy the file, verify the destination, and -then remove the source with a separate command: +Use a configured Storage Identifier and an absolute path: ```bash -canfar data cp vault:/folder/file.fits arc:/home/[username]/file.fits -canfar data ls -lh arc:/home/[username]/file.fits -canfar data rm vault:/folder/file.fits +canfar data ls -lh arc:/home/user ``` -## 4. Create a notebook +See [Data commands](data.md) for copy, recursive-copy, and cross-source +workflows. + +## Create a notebook Session ```bash canfar create notebook skaha/astroml:latest ``` -The image shorthand `skaha/astroml:latest` resolves to the CANFAR image -registry. Use the full image name when you want to be explicit: +The image shorthand is normalized to the CANFAR Container Registry. Fixed +resources are optional: ```bash -canfar create notebook images.canfar.net/skaha/astroml:latest +canfar create notebook skaha/astroml:latest --cpu 4 --memory 16 ``` -## 5. Check status +For a headless command, put the command delimiter before the container +command. Every token after `--` belongs to that command: ```bash -canfar ps -canfar info $(canfar ps -q) +canfar create headless skaha/terminal:1.1.2 -- python /arc/projects/demo/run.py ``` -Use machine output in scripts: +## Check and open the Session ```bash -canfar ps --json +canfar ps +canfar info SESSION_ID +canfar open SESSION_ID ``` -## 6. Open the notebook +`ps` shows `Pending` and `Running` Sessions by default. Use `--all` for every +status, or `-o json` when a script needs a data-only payload: ```bash -canfar open $(canfar ps -q) +canfar ps --all +canfar ps -o json ``` -Notebook Sessions usually take 60-120 seconds to become reachable. +`ps -q` is a human-only ID shortcut and can include the active-Server banner; +use machine output when passing results between tools. -## 7. Inspect startup events +Inspect startup events or logs when a Session is not ready: ```bash -canfar events $(canfar ps -q) -canfar logs $(canfar ps -q) +canfar events SESSION_ID +canfar logs SESSION_ID ``` -## 8. Clean up +## Remove the Session ```bash -canfar delete $(canfar ps -q) +canfar delete SESSION_ID ``` -Skip the confirmation prompt when you are scripting: +The command asks for confirmation. Use `--force` in a controlled script: ```bash -canfar delete $(canfar ps -q) --force +canfar delete SESSION_ID --force ``` -## Fixed resources - -Omit resources for flexible allocation. Set `--cpu`, `--memory`, or `--gpu` -when you need fixed resources. - -```bash -canfar create notebook skaha/astroml:latest --cpu 4 --memory 16 -``` - -## Headless job +## Troubleshooting -Use `--` before the command that should run inside the container. +Put logging controls before the command: ```bash -canfar create headless skaha/terminal:1.1.2 -- python /arc/projects/demo/run.py +canfar --log-level debug login cadc --force +canfar --log-level debug ps ``` -## Troubleshooting - -| Symptom | Command | -| --- | --- | -| Need fresh credentials | `canfar --log-level debug login cadc --force` | -| Need another Server | `canfar server ls` then `canfar server use ` | -| Session is pending | `canfar events $(canfar ps -q)` | -| Need cluster capacity | `canfar stats` | -| Need structured output | `canfar ps --json` | - -Logging controls belong before the command. See -[Logging and observability](logging.md) for the complete policy. +See [Logging](logging.md) for level precedence, stderr routing, and the +optional JSON Lines file sink. See the [CLI reference](cli-help.md) for every +leaf and option. diff --git a/docs/client/advanced-examples.md b/docs/client/advanced-examples.md index ef9fc0dc..05fa0ad1 100644 --- a/docs/client/advanced-examples.md +++ b/docs/client/advanced-examples.md @@ -1,274 +1,109 @@ # Advanced Examples -Complex use cases and power-user examples for CANFAR Science Platform. +These patterns combine the public Session API with +`canfar.helpers.distributed`. They assume that the Session's Container Image +contains the application code and that the caller has already authenticated. -!!! info - `canfar` automatically sets these environment variables in each container: +## Replicated headless processing - - `REPLICA_ID`: Current container ID (1, 2, 3, ...) - - `REPLICA_COUNT`: Total number of containers +`replicas` creates multiple headless Sessions. Each container receives +`REPLICA_ID` (1-based) and `REPLICA_COUNT`: -## Massively Parallel Processing - -Let's assume you have a large dataset of 1000 FITS files that you want to process in parallel. You have a Python script that can process a single FITS file, and you want to run this script in parallel on 100 different CANFAR sessions, with each container processing a subset of the files. This is a common pattern for distributed computing on CANFAR, and can be achieved with a few lines of code. - -```python title="Batch Processing Script" -from canfar.helpers import distributed -from glob import glob -from your.code import analysis - -# Find all FITS files to process -datafiles = glob("/path/to/data/files/*.fits") - -# Each replica processes its assigned chunk of files -# The chunk function automatically handles 1-based REPLICA_ID values -for datafile in distributed.chunk(datafiles): - analysis(datafile) -``` - -### Large Scale Parallel Processing - -=== ":material-language-python: Flexible Mode (Recommended)" - - ```python - from canfar.sessions import AsyncSession - - async with AsyncSession() as session: - # Flexible resource allocation - adapts to cluster availability - sessions = await session.create( - name="fits-processing", - image="images.canfar.net/your/analysis-container:latest", - kind="headless", - cmd="python", - args="/path/to/batch_processing.py", - replicas=100, - ) - return sessions - ``` - -=== ":material-language-python: Fixed Mode" +```python +from canfar.sessions import AsyncSession - ```python - from canfar.sessions import AsyncSession +async def launch_batch() -> list[str]: async with AsyncSession() as session: - # Fixed resource allocation - guaranteed resources - sessions = await session.create( + return await session.create( name="fits-processing", - image="images.canfar.net/your/analysis-container:latest", + image="images.canfar.net/project/analysis:latest", kind="headless", - cores=8, - ram=32, cmd="python", - args="/path/to/batch_processing.py", + args="/app/process_observations.py", replicas=100, ) - return sessions - ``` - -=== ":simple-gnubash: CLI Flexible Mode" - - ```bash - # Flexible resource allocation (default) - canfar create -r 100 -n fits-processing headless images.canfar.net/your/analysis-container:latest -- python /path/to/batch_processing.py - ``` - -=== ":simple-gnubash: CLI Fixed Mode" - - ```bash - # Fixed resource allocation - canfar create -c 8 -m 32 -r 100 -n fits-processing headless images.canfar.net/your/analysis-container:latest -- python /path/to/batch_processing.py - ``` - -## Advanced Resource Allocation Strategies - -For complex workflows, choosing the right resource allocation mode can significantly impact performance: - -### Mixed Resource Allocation - -You can combine flexible and fixed modes within the same workflow: - -```python -from canfar.sessions import AsyncSession - -async def mixed_workflow(): - async with AsyncSession() as session: - # Use flexible mode for data preprocessing (variable workload) - preprocessing_sessions = await session.create( - name="preprocess", - image="images.canfar.net/your/preprocessing:latest", - kind="headless", - cmd="python", - args="preprocess.py", - replicas=50, - ) - - # Use fixed mode for intensive analysis (predictable workload) - analysis_sessions = await session.create( - name="analysis", - image="images.canfar.net/your/analysis:latest", - kind="headless", - cores=16, - ram=64, - cmd="python", - args="analyze.py", - replicas=10, - ) - - return preprocessing_sessions + analysis_sessions ``` -### Resource Allocation Guidelines for Advanced Workflows +Omitting `cores` and `ram` keeps each replica on the platform's flexible +allocation policy. Add both arguments to the same `create()` call (for example, +`cores=8, ram=32`) when every replica needs a fixed resource allocation. -| Workflow Type | Recommended Mode | Reasoning | -|---------------|------------------|-----------| -| **Data Preprocessing** | Flexible | Variable I/O patterns, benefits from burst capacity | -| **Machine Learning Training** | Fixed | Consistent performance needed for convergence | -| **Monte Carlo Simulations** | Flexible | Independent tasks, can handle variable performance | -| **Image Processing Pipelines** | Fixed | Memory-intensive, predictable resource needs | -| **Interactive Development** | Flexible | Exploratory work, cost-effective | -| **Production Batch Jobs** | Fixed | Reliable performance for scheduled workflows | +The return value contains only successfully launched Session IDs. Check for an +empty list before treating the batch as submitted. -## Distributed Processing Strategies +## Partition work inside a container -The `canfar.helpers.distributed` module provides two main strategies for distributing data across replicas: +Use `chunk()` for contiguous ranges and `stripe()` for round-robin assignment: -### Chunking (`distributed.chunk`) - -The `chunk` function divides your data into contiguous blocks, with each replica processing a consecutive chunk. The function uses 1-based replica IDs (matching `canfar` `REPLICA_ID` environment variable): +```python +from pathlib import Path -```python title="Chunking Example" from canfar.helpers import distributed -from glob import glob -# With 1000 files and 100 replicas: -# - Replica 1 processes files 0-9 -# - Replica 2 processes files 10-19 -# - Replica 3 processes files 20-29 -# - And so on... +files = list(Path("/data/observations").glob("*.fits")) -datafiles = glob("/path/to/data/*.fits") -for datafile in distributed.chunk(datafiles): - process_datafile(datafile) -``` +for path in distributed.chunk(files): + print(path) -### Striping (`distributed.stripe`) - -The `stripe` function distributes data in a round-robin fashion, which is useful when file sizes vary significantly: - -```python title="Striping Example" -from canfar.helpers import distributed -from glob import glob - -# With 1000 files and 100 replicas: -# - Replica 1 processes files 0, 100, 200, 300, ... -# - Replica 2 processes files 1, 101, 201, 301, ... -# - Replica 3 processes files 2, 102, 202, 302, ... -# - And so on... - -datafiles = glob("/path/to/data/*.fits") -for datafile in distributed.stripe(datafiles): - process_datafile(datafile) +for path in distributed.stripe(files): + print(path) ``` -### When to Use Each Strategy - -- **Use `chunk`** when files are similar in size and you want each replica to process a contiguous block of data -- **Use `stripe`** when file sizes vary significantly, as it distributes the workload more evenly across replicas - -## Real-World Example: Processing Astronomical Data +When `replica` and `total` are omitted, both helpers read `REPLICA_ID` and +`REPLICA_COUNT` at call time. They default to one replica when those variables +are absent. Explicit arguments override the environment: ```python -import os -import json -from pathlib import Path -from canfar.helpers.distributed import chunk - -def process_observations(): - """Process FITS files across multiple containers.""" - - # Get all observation files - fits_files = list(Path("/data/observations").glob("*.fits")) - my_files = list(chunk(fits_files)) - - if not my_files: - print("No files assigned to this container") - return - - replica_id = os.environ.get('REPLICA_ID') - print(f"Container {replica_id} processing {len(my_files)} files") - - # Process each file - results = [] - for fits_file in my_files: - # Your analysis code here - result = {"file": fits_file.name, "stars_detected": analyze_fits(fits_file)} - results.append(result) - - # Save results with container ID - output_file = f"/results/container_{replica_id}_results.json" - with open(output_file, 'w') as f: - json.dump(results, f, indent=2) - - print(f"Saved {len(results)} results to {output_file}") +from canfar.helpers import distributed -def analyze_fits(fits_path): - """Your FITS analysis logic here.""" - return 42 # Placeholder +items = list(range(10)) +assert list(distributed.chunk(items, replica=1, total=4)) == [0, 1] +assert list(distributed.chunk(items, replica=4, total=4)) == [6, 7, 8, 9] +assert list(distributed.stripe(items, replica=2, total=4)) == [1, 5, 9] ``` -## Best Practices +`chunk()` raises `ValueError` for invalid replica settings. `stripe()` keeps its +legacy empty result for an out-of-range replica, but raises `ValueError` when +`total` is non-positive. See the [Helpers API](helpers.md) for the exact +contract. -**Choose the right function:** -- Use `chunk()` when you need contiguous data blocks -- Use `stripe()` for round-robin distribution +## Process a VOSpace Service in each replica -**Handle empty containers:** -```python -my_data = list(chunk(data)) -if not my_data: - print("No data for this container") - return -``` +Use the explicit Storage Identifier API inside a Session. The same identifier +and path remain separate Python arguments; no dynamic fsspec scheme is needed: -**Save results with container ID:** ```python -import os -replica_id = os.environ.get('REPLICA_ID') -output_file = f"/results/container_{replica_id}_results.json" -``` +from canfar.storage import filesystem -**Combine results from all containers:** -```python -from pathlib import Path -import json +from canfar.helpers import distributed -def combine_results(): - """Merge results from all containers.""" - all_results = [] - for result_file in Path("/results").glob("container_*_results.json"): - with open(result_file) as f: - all_results.extend(json.load(f)) +paths = ["/project/observations/a.fits", "/project/observations/b.fits"] - with open("/results/final_results.json", 'w') as f: - json.dump(all_results, f, indent=2) +vault = filesystem("vault") +try: + for path in distributed.chunk(paths): + raw = vault.cat_file(path) + print(path, len(raw)) +finally: + vault.close() ``` -## Common Issues +For staged or memory-mapped access, use an explicit fsspec cache or +`get_file()` destination; see [Data Access](data.md). -**Some containers get no data** -This happens when you have more containers than data items. Handle it gracefully: -```python -my_data = list(chunk(data)) -if not my_data: - print("No data assigned to this container") - return -``` +## Cleanup a batch + +Use the keyword-only filters on `destroy_with()` after the literal prefix: -**Debugging distribution** ```python -import os -replica_id = os.environ.get('REPLICA_ID') -replica_count = os.environ.get('REPLICA_COUNT') -print(f"Container {replica_id} of {replica_count} processing {len(my_data)} items") +from canfar.sessions import Session + +with Session() as session: + result = session.destroy_with( + "fits-processing-", + kind="headless", + status="Completed", + ) + print(result) ``` diff --git a/docs/client/async_session.md b/docs/client/async_session.md index f0ae1d19..4a177a62 100644 --- a/docs/client/async_session.md +++ b/docs/client/async_session.md @@ -1,28 +1,85 @@ # Asynchronous Sessions -!!! info "Overview" - `canfar` supports asynchronous sessions using the `AsyncSession` class while maintaining 1-to-1 compatibility with the `Session` class. +`AsyncSession` is the native asynchronous counterpart to `Session`. It uses a +native `httpx.AsyncClient`; it is not a synchronous client wrapped in an event +loop. Use it inside an existing async application and close it with `async with`. -## Creating sessions +## Return shapes and parity -`AsyncSession.create` matches `Session.create`: it returns a `list` of session -IDs, omits failed launches without raising, and returns an empty list if every -attempt fails. Check the list after awaiting the call. HTTP and timeout details -are logged when the application configures logging. Call -`canfar.configure_logging()` at the Python application entry point; it honors -`CANFAR_LOGLEVEL`. In the CLI, use a root control such as -`canfar --log-level debug create ...`. See -[Logging](../cli/logging.md). +The async methods preserve the synchronous result contracts: + +| Method | Result | +| --- | --- | +| `await fetch(kind=None, status=None, view=None)` | `list[dict[str, str]]` | +| `await create(...)` | `list[str]` containing the IDs that launched successfully | +| `await info(ids)` | `list[dict[str, Any]]` | +| `await logs(ids, verbose=False)` | `dict[str, str]`, or `None` when `verbose=True` | +| `await events(ids, verbose=False)` | `list[dict[str, str]]`, or `None` when `verbose=True` | +| `await destroy(ids)` / `await destroy_with(...)` | `dict[str, bool]` | +| `await connect(ids)` | `None`; opens ready Session URLs | + +`create()` omits failed launches and returns `[]` when every launch fails. +Invalid request values raise before the HTTP request. Verbose logs and events +are sent to the `canfar.sessions` logger and return `None`. + +`fetch(view="all")` requests the server's all-Sessions view when authorized, +and `stats()` returns aggregate Science Platform Server resource statistics. + +## Async workflow + +```python +from canfar.sessions import AsyncSession + + +async def main() -> None: + async with AsyncSession() as session: + ids = await session.create( + name="async-analysis", + image="images.canfar.net/skaha/astroml:latest", + kind="notebook", + ) + if ids: + await session.connect(ids) + print(await session.fetch(kind="notebook", status="Running")) + print(await session.info(ids)) + print(await session.logs(ids)) + print(await session.events(ids)) + print(await session.destroy(ids)) +``` + +For a headless Session, add `cmd`, `args`, and `env`. `cores` and `ram` request +fixed resources; omitting them uses the Science Platform Server's flexible +allocation policy. + +## Select Sessions for cleanup + +`destroy_with` is keyword-only after `prefix`: + +```python +async with AsyncSession() as session: + result = await session.destroy_with( + "batch-", + kind="headless", + status="Completed", + ) +``` + +Its signature is `destroy_with(prefix, *, kind="headless", status="Completed")`. +Literal prefixes are anchored at the beginning; prefixes containing regular +expression metacharacters are treated as regular expressions. ::: canfar.sessions.AsyncSession handler: python selection: members: - fetch + - stats - create - info - logs + - events - destroy + - destroy_with - connect rendering: members_order: source diff --git a/docs/client/client.md b/docs/client/client.md index f38ba127..57cd1438 100644 --- a/docs/client/client.md +++ b/docs/client/client.md @@ -1,75 +1,109 @@ # HTTPClient -The `canfar.client` module provides a comprehensive HTTP client for interacting with CANFAR Science Platform services. Built on the powerful [`httpx`](https://www.python-httpx.org/) library, it offers both synchronous and asynchronous interfaces with advanced authentication capabilities. +`canfar.client.HTTPClient` is the lower-level transport used by the Session, +Image, Context, and Overview clients. It composes native synchronous and +asynchronous `httpx` clients and is useful when an application needs a CANFAR +request outside those higher-level modules. +## Construct a client +```python +from canfar.client import HTTPClient +from canfar.models.config import Configuration + +client = HTTPClient( + config=Configuration(), + timeout=60, + concurrency=64, +) +``` + +The main settings are: -## Features +- `url`: an explicit Science Platform Server URL. +- `config`: the persisted `Configuration` to resolve by default. +- `authentication_idp`: a transient Identity Provider selector for this client. +- `token`: a runtime bearer token. +- `certificate`: a runtime X.509 certificate path. +- `timeout`: request timeout in seconds (1–300). +- `concurrency`: maximum async connection count (1–128). -!!! tip "Key Capabilities" - - **Multiple Authentication Methods**: X.509 certificates, OIDC tokens, and bearer tokens - - **Automatic SSL Configuration**: Seamless certificate-based authentication - - **Async/Sync Support**: Both synchronous and asynchronous HTTP clients - - **Connection Pooling**: Optimized for concurrent requests - - **Application Logging**: Explicit, secret-safe runtime configuration - - **Context Managers**: Proper resource management +When no explicit `url` is supplied, the active Server Selection in +`Configuration` supplies the Science Platform Server. Without a runtime +credential, `authentication_idp` selects a saved Authentication Record for this +client; otherwise the active Authentication Record is used. -*This is a low-level client that is used by all other API clients in CANFAR. It is not intended to be used directly by users, but rather as a building block for other clients and contributors.* +## Credential precedence and lifecycle -## Authentication Modes +Runtime credentials take precedence over saved Authentication Records. A +non-empty runtime `token` wins over a saved X.509 or OIDC record; a runtime +`certificate` likewise wins. An empty runtime token is treated as absent and +falls back to saved state. Runtime credentials are not persisted. -The client supports multiple authentication modes that can be configured through the authentication system: +Without runtime credentials, the client resolves the selected saved record when +its sync or async HTTPX client is first created. An expired OIDC record is +refreshed through the saved Authentication Record and the refreshed token is +persisted. If the record cannot refresh or an X.509 certificate is invalid, +client construction/request setup raises the established authentication error +instead of silently sending an unauthenticated request. -## Logging +Use the native context manager that matches the transport: ```python -from canfar import configure_logging from canfar.client import HTTPClient -# Configure the application once, then construct clients normally. -configure_logging(loglevel="debug") -client = HTTPClient() +with HTTPClient(token="runtime-token", url="https://example.test/skaha/v1") as client: + response = client.client.get("context") + + +async def request() -> None: + async with HTTPClient( + token="runtime-token", + url="https://example.test/skaha/v1", + ) as client: + response = await client.asynclient.get("context") ``` -Logging is an application concern rather than an `HTTPClient` constructor -setting. See [Logging](../cli/logging.md) for environment precedence, stream -separation, and the optional `--log-file` JSON Lines sink. +`client` is the native `httpx.Client`; `asynclient` is the native +`httpx.AsyncClient`. They are created lazily and closed by their matching +context manager. Do not use the async client from synchronous code or expect a +sync client to run through an event loop. -## Configuration +## Errors and logging -The client composes a `Configuration` object through its `config` field: +HTTP response hooks raise `httpx.HTTPStatusError` by default for unsuccessful +responses. Catch it at the application boundary when a request can fail: ```python +from httpx import HTTPStatusError + from canfar.client import HTTPClient -from canfar.models.config import Configuration -client = HTTPClient( - config=Configuration(), - timeout=60, # Request timeout in seconds - concurrency=64, # Max concurrent connections -) +with HTTPClient(token="runtime-token", url="https://example.test/skaha/v1") as client: + try: + response = client.client.get("invalid-endpoint") + except HTTPStatusError as exc: + print(exc.response.status_code) ``` -Runtime `token` or `certificate` arguments take precedence over saved -Authentication Records, including authentication hook selection. Without -runtime credentials, `authentication_idp` selects an Authentication Record for -that client; otherwise the active Authentication Record and Server Selection -from `Configuration` are used. - -## Error Handling - -The client includes built-in error handling for HTTP responses: +Authentication and configuration failures use CANFAR's authentication/error +types. Do not log tokens, certificates, or raw credential records. Configure +the application logger explicitly when diagnostics are needed: ```python -from httpx import HTTPStatusError +from canfar import configure_logging -try: - response = client.client.get("/invalid-endpoint") - response.raise_for_status() -except HTTPStatusError as e: - print(f"HTTP error: {e.response.status_code}") +configure_logging("debug") ``` +## Configuration editing + +`HTTPClient` consumes the persisted `Configuration`; it does not own edits or +alternate configuration services. Use `config.editor.get()`, `set()`, and +`save()` to make validated, atomic changes while preserving the stable +`version`/`active`/`authentication`/`servers`/`registry`/`console` shape. See +[Install and set up](get-started.md#edit-and-save-configuration). + ## API Reference ::: canfar.client.HTTPClient @@ -78,6 +112,9 @@ except HTTPStatusError as e: members: - client - asynclient + - uses_runtime_credentials + - authentication_record + - build show_root_heading: true show_source: false heading_level: 3 diff --git a/docs/client/context.md b/docs/client/context.md index 149e9357..ed117d0a 100644 --- a/docs/client/context.md +++ b/docs/client/context.md @@ -1,37 +1,21 @@ # Context API -!!! info "Overview" +`Context` reads the resource information advertised by a Science Platform +Server. The returned mapping can guide `Session.create()` resource requests. - The Context API allows the user to get information about the resources available to be requested for a session on the CANFAR Science Platform. This information can be used to configure the session to request the appropriate resources for your session. - -```python title="Get context information" +```python from canfar.context import Context -context = Context() -context.resources() +with Context() as context: + resources = context.resources() + print(resources["cores"]) + print(resources.get("memoryGB")) + print(resources.get("gpus")) ``` -```python -{ - "cores": { - "default": 1, - "defaultRequest": 1, - "defaultLimit": 16, - "defaultHeadless": 1, - "options": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], - }, - "memoryGB": { - "default": 2, - "defaultRequest": 4, - "defaultLimit": 192, - "defaultHeadless": 4, - "options": [1, 2, ..., 192], - }, - "gpus": { - "options": [1, ..., 8], - }, -} -``` +The response is a `dict[str, Any]` whose keys and values are supplied by the +server; common keys include `cores`, `memoryGB`, and `gpus`. Resource limits are +server metadata, not a second Configuration model. ::: canfar.context.Context handler: python @@ -42,4 +26,4 @@ context.resources() members_order: source show_root_heading: true show_source: true - heading_level: 3 + heading_level: 2 diff --git a/docs/client/data.md b/docs/client/data.md index 78c6377d..934a2894 100644 --- a/docs/client/data.md +++ b/docs/client/data.md @@ -1,288 +1,179 @@ # Data Access -Read and write CANFAR VOSpace Services from Python. `canfar` resolves the -endpoints and credentials; [`vosfs`](https://github.com/shinybrar/vosfs) is the -[fsspec](https://filesystem-spec.readthedocs.io/) filesystem that talks to them, -so every tool that already speaks fsspec — astropy, pandas, dask, zarr — works -without an adapter. +The Python data API gives explicit access to configured VOSpace Services through +standard [fsspec](https://filesystem-spec.readthedocs.io/) filesystems. A +**Storage Identifier** names one VOSpace Service; a path is the path inside that +service. `canfar` does not register Storage Identifiers as fsspec protocols, +module attributes, or dynamic schemes. -The same Storage Identifiers the CLI uses are importable by name. +## Find and open a Storage Identifier -## Open a VOSpace Service - -Import a Storage Identifier and you get a ready, authenticated filesystem. Run -`canfar login` first; the credential resolution is the same one the CLI uses. - -```python -from canfar.storage import arc, vault, local -``` - -Any configured Storage Identifier works this way. To see which are available: +Authenticate first, for example with `canfar login cadc`. Then pass the +Storage Identifier to `identifiers()` or `filesystem()`: ```python -from canfar.storage import identifiers +from canfar.storage import filesystem, identifiers -identifiers() # ['arc', 'vault', 'local'] -``` - -Import binds the filesystem once, which is what you usually want. To build one -explicitly — to override a credential, or to name an identifier held in a -variable — use `filesystem`: - -```python -from canfar.storage import filesystem +available = identifiers() # e.g. ["arc", "vault", "local"] vault = filesystem("vault") -staging = filesystem("vault", token="...") # runtime bearer token -archive = filesystem("arc", certificate="/path/to/proxy.pem") +try: + path = "/ALMA/test-data/cutouts/test-4d-cube-cutout.fits" + print(vault.info(path)["size"]) + raw = vault.cat_file(path) +finally: + vault.close() ``` -`local` is reserved for the machine your code runs on and needs no credential. +The `local` identifier is always available and does not require a credential. +Every other identifier must be present in the saved Configuration. The returned +object is a normal synchronous fsspec filesystem; use its standard methods +rather than a CANFAR-specific wrapper. -## Filesystem operations - -The object is a standard fsspec filesystem, so the usual verbs apply: - -```python -cutouts = "/ALMA/test-data/cutouts" -target = f"{cutouts}/test-4d-cube-cutout.fits" - -vault.ls(cutouts, detail=False) # ['/ALMA/.../test-4d-cube-cutout.fits', ...] -vault.info(target)["size"] # 169920 -vault.exists(target) # True -vault.isdir(cutouts) # True -vault.glob(f"{cutouts}/*cutout.fits") -vault.find(cutouts) # recursive listing -vault.du(cutouts) # 3712320 -``` - -Reads come in whole-object, ranged, and file-like forms: +`filesystem()` accepts runtime credentials when a caller must override saved +state. A non-empty `token` takes precedence over a saved Authentication Record; +`certificate` supplies a runtime X.509 certificate. Runtime credentials are +used only for that filesystem and do not rewrite the saved Configuration. ```python -whole = vault.cat_file(target) # 169920 bytes -header = vault.cat_file(target, 0, 2880) # first 2880 bytes only -first, second = vault.cat_ranges([target, target], [0, 100], [80, 180]) +from canfar.storage import filesystem -with vault.open(target, "rb") as handle: - handle.read(80) +vault = filesystem("vault", token="runtime-bearer-token") +try: + # use vault here + ... +finally: + vault.close() -vault.head(target, 100) -vault.tail(target, 100) +archive = filesystem("arc", certificate="/path/to/cadcproxy.pem") +try: + # use archive here + ... +finally: + archive.close() ``` -Writes use `put_file`, `pipe_file`, `mkdir`, and `rm`. Directory listings are -cached in memory for the lifetime of the filesystem object, so a long-lived -object can serve a stale listing; build a fresh one, or pass -`use_listings_cache=False`, when you need to observe another writer's changes. +An unknown Storage Identifier raises `KeyError`. If a saved credential is +missing, expired, invalid, or cannot be materialized, `filesystem()` raises +`AuthContextError` with a login hint; credential contents are not included in +the error. -## Get a local path +## Standard fsspec operations -Some libraries want a real path rather than a file object — anything that -memory-maps, or a C extension that opens by name. Materialise the file: +Storage operations use the ordinary fsspec vocabulary: ```python -vault.get_file(target, "/scratch/cutout.fits") -``` - -`get_file` is the standard fsspec verb; `put_file` is its counterpart for -uploads. - -## Cache +from canfar.storage import filesystem -Nothing is cached to disk unless you ask. On a CANFAR session `/scratch` is -fast local NVMe, is not backed up, and is cleared when the session ends, which -is exactly what a cache wants. Locally, any directory works. +vault = filesystem("vault") +try: + directory = "/ALMA/test-data/cutouts" + target = f"{directory}/test-4d-cube-cutout.fits" -### Whole files + names = vault.ls(directory, detail=False) + metadata = vault.info(target) + assert vault.exists(target) + matches = vault.glob(f"{directory}/*cutout.fits") + all_names = vault.find(directory) -```python -from fsspec.implementations.cached import WholeFileCacheFileSystem + whole = vault.cat_file(target) + header = vault.cat_file(target, 0, 2880) + ranges = vault.cat_ranges([target, target], [0, 100], [80, 180]) -cached = WholeFileCacheFileSystem(fs=vault, cache_storage="/scratch/vault-cache") - -cached.cat_file(target) # cold: fetched over the network -cached.cat_file(target) # warm: served from /scratch + with vault.open(target, "rb") as handle: + first_bytes = handle.read(80) +finally: + vault.close() ``` -A cold read of the 166 KiB cutout took 2.96 s; the warm read took 0.4 ms. - -Use `SimpleCacheFileSystem` when you do not need the expiry and staleness -metadata `WholeFileCacheFileSystem` keeps. Passing `cache_storage` a list of -directories tries each in order and treats only the last as writable, so a -shared read-only cache can back your own. +`cat_file(path, start, end)` returns the requested slice. Whether the VOSpace +Service transfers only that slice or downloads the object and slices it locally +depends on the service and its deployed data endpoint. `open()` provides a +seekable file-like object and may stage the complete object locally. Do not +assume that a partial read reduces network traffic. -### Byte ranges - -`vosfs` sends an HTTP `Range` header and uses the response when the byte -endpoint answers `206`, so a partial read transfers only the bytes you asked -for. Support is per-backend: - -| Storage Identifier | Backend | Ranged reads | -| --- | --- | --- | -| `vault` | `minoc` | Yes — a partial read returns `206` and transfers only that slice | -| `arc` | Cavern | No — the whole object is fetched and sliced, correct but not cheaper | - -Against `vault`, cache blocks instead of whole files when you touch small parts -of a large cube. `MMapCache` keeps fetched blocks in a sparse file: +VOSpace writes use the corresponding fsspec methods (`put_file`, `pipe_file`, +`mkdir`, and `rm`) when the authenticated account has permission: ```python -from fsspec.caching import MMapCache - -cube = "/ALMA/test-data/cutouts/test-4d-cube.fits" -size = vault.info(cube)["size"] - -blocks = MMapCache( - blocksize=1 << 20, - fetcher=lambda start, end: vault.cat_file(cube, start, end), - size=size, - location="/scratch/vault-cache/cube.blocks", -) - -header = blocks._fetch(0, 2880) # one ranged request, one block of four +vault = filesystem("vault") +try: + vault.pipe_file("/tmp/example.txt", b"hello CANFAR\n") + vault.rm("/tmp/example.txt") +finally: + vault.close() ``` -Reading a FITS header from the 3.4 MB cube materialises one block of four. The -saving is in bytes transferred rather than seconds on small files, because -VOSpace transfer negotiation dominates a short request; it grows with file -size. Against `arc` a block cache still costs a whole download per block, so -cache whole files there. +Directory listings use an in-memory fsspec listing cache for the lifetime of +the filesystem. Create a new filesystem when another writer's changes must be +observed. -### RAM +## Materialize a local file -Omit `location` and `MMapCache` uses an anonymous memory map — a RAM cache that -never touches disk, for when `/scratch` is absent or you want the data gone -when the process exits: +Libraries that require a pathname can use the standard fsspec `get_file()` +operation. The destination is explicit and its cleanup is the caller's +responsibility: ```python -blocks = MMapCache( - blocksize=1 << 20, - fetcher=lambda start, end: vault.cat_file(cube, start, end), - size=size, -) -``` - -A warm re-read of a cached range returned in 18 µs. - -### What does not work - -`blockcache` (`CachingFileSystem`) cannot wrap a VOSpace Service. `Range` is -honoured for byte reads, not through the file-object path: +from canfar.storage import filesystem -```text -AttributeError: 'StagedReadFile' object has no attribute 'blocksize' +vault = filesystem("vault") +try: + vault.get_file( + "/ALMA/test-data/cutouts/test-4d-cube-cutout.fits", + "/scratch/cutout.fits", + ) +finally: + vault.close() ``` -Stacked caches do not help either: chaining them (`filecache::simplecache::`) -builds the layers, but the inner layer is never filled and never serves. Use -exactly one cache layer, on your fastest local disk. - -## Scientific tools - -### astropy - -Read a header without downloading the file, using one ranged request: +For example, a local path can then be opened by a memory-mapping library: ```python from astropy.io import fits -raw = vault.cat_file(target, 0, 2880) -header = fits.Header.fromstring(raw.decode("latin-1")) -header["NAXIS"], header["OBJECT"] # 4, 'hers1' -``` - -Or hand the file object straight to astropy: - -```python -with vault.open(target, "rb") as handle, fits.open(handle) as hdul: - hdul[0].data.shape # (1, 96, 26, 16) -``` - -To memory-map, materialise the file first — `memmap=True` needs a real path: - -```python -vault.get_file(target, "/scratch/cutout.fits") - with fits.open("/scratch/cutout.fits", memmap=True) as hdul: - data = hdul[0].data # paged in on demand, not loaded up front + data = hdul[0].data ``` -### numpy +## Content caching -```python -import numpy as np - -raw = vault.cat_file(target) -values = np.frombuffer(raw[2880:2880 + 64], dtype=">f4") -``` - -### pandas - -Astronomy tables usually arrive as FITS rather than CSV. Read one remotely and -convert: +`canfar.storage.filesystem()` does not configure a persistent content cache. +The `/scratch` volume on a Science Platform Server Session is useful for +ephemeral staging, but it is not selected implicitly and is cleared with the +Session. If a workflow needs whole-file caching, compose one standard fsspec +cache wrapper and choose its directory explicitly: ```python -from astropy.table import Table +from fsspec.implementations.cached import SimpleCacheFileSystem -with vault.open("/APASS/north/091106/n091106.0101.cat", "rb") as handle: - table = Table.read(handle, format="fits") - -frame = table.to_pandas() # 2373 rows -frame.columns[:3] # ['NUMBER', 'MAG_AUTO', 'MAGERR_AUTO'] -``` - -For delimited text, pass the file object to pandas directly: - -```python -import pandas as pd +from canfar.storage import filesystem -with vault.open("/path/to/table.csv", "rb") as handle: - frame = pd.read_csv(handle) +remote = filesystem("vault") +try: + cached = SimpleCacheFileSystem( + fs=remote, + cache_storage="/scratch/canfar-vault", + ) + cached.cat_file("/ALMA/test-data/cutouts/test-4d-cube-cutout.fits") +finally: + remote.close() ``` -### dask - -Memory-map a materialised cube and chunk it, so only the blocks a computation -touches are paged in: - -```python -import dask.array as da -from astropy.io import fits - -with fits.open("/scratch/cutout.fits", memmap=True) as hdul: - array = da.from_array(hdul[0].data, chunks=(1, 24, 26, 16)) - array.mean().compute() -``` +One whole-file cache layer is the supported simple choice. Do not stack cache +wrappers, pass a URL as `cache_storage`, or advertise `blockcache`: the staged +VOSpace file object does not provide the fsspec block-cache interface. The +cache wrapper forwards `close()` only when its wrapped filesystem provides it, +so close the known VOSpace client (`remote`) explicitly. -## Async +## Python API boundary -Every read has an async twin. Build the filesystem with `asynchronous=True`, -use the underscore-prefixed coroutines, and close it when done: +The public Python storage surface is deliberately small: ```python -import asyncio - -from canfar.models.config import Configuration -from vosfs import VOSpaceFileSystem - - -async def main() -> None: - config = Configuration() - service = config.servers["canfar"].storage["vault"] - vault = VOSpaceFileSystem( - str(service.url), - certfile=str(config.get_credential("cadc").path), - asynchronous=True, - ) - try: - entries = await vault._ls(cutouts, detail=False) - header = await vault._cat_file(cube, 0, 2880) - finally: - await vault.aclose() - - -asyncio.run(main()) +from canfar.storage import filesystem, identifiers ``` -Async filesystems cannot open file objects — `open()` is rejected, and the -fsspec caches are synchronous — so use the synchronous form for caching and for -libraries that want a file handle. +The fsspec-cli source mapping is private. There is no configuration helper, +automatic `vos://` scheme, or module member for each configured identifier. +Keep the Storage Identifier and the path as separate arguments. diff --git a/docs/client/examples.md b/docs/client/examples.md index d647095a..aa9e3686 100644 --- a/docs/client/examples.md +++ b/docs/client/examples.md @@ -1,37 +1,25 @@ # Python Client Examples -These examples show the sync and async Session interfaces. +The examples assume that an Authentication Record exists: -!!! note "Assumption" - ```bash title="Authenticated via CLI" - canfar login cadc - ``` +```bash +canfar login cadc +``` + +They use the public `Session` and `AsyncSession` classes directly. Both clients +preserve the same result shapes. ## Create Sessions ### Notebook -=== "Flexible Mode (Default)" - - ```python - from canfar.sessions import Session - - session = Session() - ids = session.create( - name="my-notebook", - image="images.canfar.net/skaha/astroml:latest", - kind="notebook", - ) - print(ids) # ["d1tsqexh"] - session.connect(ids) - ``` - -=== "Fixed Mode" +Omit `cores` and `ram` for the server's flexible allocation policy, or pass them +for a fixed resource request: - ```python - from canfar.sessions import Session +```python +from canfar.sessions import Session - session = Session() +with Session() as session: ids = session.create( name="my-notebook", image="images.canfar.net/skaha/astroml:latest", @@ -39,317 +27,149 @@ These examples show the sync and async Session interfaces. cores=2, ram=4, ) - print(ids) # ["d1tsqexh"] - session.connect(ids) - ``` - -=== "`async`" - - ```python - from canfar.sessions import AsyncSession - - session = AsyncSession() - ids = await session.create( - name="my-notebook", - image="images.canfar.net/skaha/astroml:latest", - kind="notebook", - ) - print(ids) # ["d1tsqexh"] - await session.connect(ids) - ``` + print(ids) # list[str] + if ids: + session.connect(ids) +``` ### Headless -- Headless Sessions are containers that execute a command and exit when complete without user interaction. -- They are useful for batch processing and distributed computing. - - -=== "Replicated Headless Sessions" +Headless Sessions run a command and exit. `cmd`, `args`, and `env` are valid +for headless Sessions: - ```python - from canfar.sessions import Session +```python +from canfar.sessions import Session - session = Session() +with Session() as session: ids = session.create( name="my-headless", - image="images.canfar.net/skaha/astroml:latest", + image="images.canfar.net/skaha/terminal:latest", kind="headless", - cmd="echo", - args="Hello, World!", - ) - print(ids) # ["d1tsqexh"] - ``` - -=== "`async`" - - ```python - from canfar.sessions import AsyncSession - - session = AsyncSession() - ids = await session.create( - name="my-headless", - image="images.canfar.net/skaha/astroml:latest", - kind="headless", - cmd="echo", - args="Hello, World!", - ) - print(ids) # ["d1tsqexh"] - ``` - - -!!! example "Replica Environment Variables" - All containers receive the following environment variables: - - `REPLICA_COUNT` — common total number of replicas - - `REPLICA_ID` — 1-based index of the replica (1..N) - - Use these to partition work deterministically. See [Helpers API Reference](helpers.md) for `chunk` and `stripe`. - -!!! warning "Private Container Registry Access" - Use a private Harbor image by providing registry credentials via configuration. - ```python - import asyncio - from canfar.sessions import AsyncSession - from canfar.models.registry import ContainerRegistry - from canfar.models.config import Configuration - - async def main(): - cfg = Configuration(registry=ContainerRegistry(username="username", secret="CLI_SECRET")) - session = AsyncSession(config=cfg) - ids = await session.create( - name="private-job", - image="images.canfar.net/your/private-image:latest", - kind="headless", - cmd="python", - args="/app/run.py", - ) - print(ids) - - asyncio.run(main()) - ``` - -## Resource Allocation Modes - -CANFAR supports two resource allocation modes for your sessions. See the [resource allocation guide](../platform/concepts.md#resource-allocation-modes) for more information. - -### Examples - -=== "Flexible Mode (Default)" - ```python - from canfar.sessions import Session - - session = Session() - # No cores/ram specification - uses flexible allocation - ids = session.create( - name="flexible-notebook", - image="images.canfar.net/skaha/astroml:latest", - kind="notebook" + cmd="python", + args="/arc/projects/demo/run.py", + env={"DATASET": "example"}, + replicas=3, ) - ``` + print(ids) # successful Session IDs only +``` -=== "Fixed Mode" - ```python - from canfar.sessions import Session +If one replica fails, `create()` logs the failure and omits that ID. If all +replicas fail, it returns `[]`. - session = Session() - # Specify exact resources for guaranteed allocation - ids = session.create( - name="fixed-notebook", - image="images.canfar.net/skaha/astroml:latest", - kind="notebook", - cores=4, - ram=8 - ) - ``` +## Discover and filter Sessions -## Discover and Filter Sessions +`fetch()` returns the server's list of dictionaries as +`list[dict[str, str]]`: -=== "Fetch All Sessions" +```python +from canfar.sessions import Session - ```python - from canfar.sessions import Session - - session = Session() +with Session() as session: all_sessions = session.fetch() - print(len(all_sessions)) - ``` - -=== "`async`" - - ```python - from canfar.sessions import AsyncSession - - async with AsyncSession() as session: - all_sessions = await session.fetch() - print(len(all_sessions)) - ``` -
- -=== "Fetch Running Notebooks" - - ```python - from canfar.sessions import Session - - session = Session() running = session.fetch(kind="notebook", status="Running") - print(running) - session.connect([item["id"] for item in running]) - ``` - -=== "`async`" - - ```python - from canfar.sessions import AsyncSession - - async with AsyncSession() as session: - running = await session.fetch(kind="notebook", status="Running") - print(running) - await session.connect([item["id"] for item in running]) - ``` -
- -=== "Fetch Completed Headless Sessions" - - ```python - from canfar.sessions import Session - - session = Session() - completed = session.fetch(kind="headless", status="Succeeded") - print(completed) - ``` - -=== "`async`" - - ```python - from canfar.sessions import AsyncSession - - async with AsyncSession() as session: - completed = await session.fetch(kind="headless", status="Succeeded") - print(completed) - ``` - -!!! success "Kinds & Status" + print(len(all_sessions), running) +``` - You can use any combination of the following kinds and status to filter sessions: +Supported kind values are `desktop`, `notebook`, `carta`, `headless`, `firefly`, +`desktop-app`, and `contributed`. Status filters include `Pending`, `Running`, +`Terminating`, `Succeeded`, `Completed`, `Error`, and `Failed`. - - Kinds: `desktop`, `notebook`, `carta`, `headless`, `firefly`, `desktop-app`, `contributed` - - Statuses: `Pending`, `Running`, `Terminating`, `Succeeded`, `Error`, `Failed` +## Inspect, events, and logs +```python +from canfar.sessions import Session -## Inspect Sessions +with Session() as session: + ids = ["session-id"] + details = session.info(ids) # list[dict[str, Any]] + events = session.events(ids) # list[dict[str, str]] + logs = session.logs(ids) # dict[str, str] + print(details, events, logs) +``` -Detailed information about the session, including resource usage, user IDs, and more. +Passing `verbose=True` to `events()` or `logs()` sends the result through the +`canfar.sessions` logger and returns `None`. Configure logging with +`canfar.configure_logging()` when an application needs to display it. -=== "Detailed Session Information" +## Async workflow - ```python - from canfar.sessions import Session +`AsyncSession` uses the native asynchronous transport. Keep the work inside an +async function and close the client with `async with`: - session = Session() - info = session.info(ids) - print(info) - ``` +```python +from canfar.sessions import AsyncSession -=== "`async`" - - ```python - from canfar.sessions import AsyncSession +async def main() -> None: async with AsyncSession() as session: - info = await session.info(ids) - print(info) - ``` - -## Events - -Events describe the steps taken by the Science Platform to launch your session - -=== "Session Events" - - ```python - from canfar.sessions import Session - - session = Session() - events = session.events(ids, verbose=True) - print(events) - ``` - -=== "`async`" - - ```python - from canfar.sessions import AsyncSession - - async with AsyncSession() as session: - events = await session.events(ids, verbose=True) - print(events) - ``` - -## Logs - -Logs contain the output from your session's containers. - -!!! tip "Log Retention" - Logs are retained until your session is deleted. A completed session, i.e., `Succeeded`, `Failed`, or `Error` is kept for 24 hours before being deleted. - -=== "Session Logs" - - ```python - from canfar.sessions import Session - - session = Session() - logs = session.logs(ids, verbose=True) - ``` - -=== "`async`" - - ```python - from canfar.sessions import AsyncSession - - async with AsyncSession() as session: - logs = await session.logs(ids, verbose=True) - ``` - -## Cleanup Sessions - -!!! warning "Permanent Action" - - Deleted sessions cannot be recovered. - -=== "Destroy Session(s)" - - ```python - from canfar.sessions import Session + ids = await session.create( + name="async-headless", + image="images.canfar.net/skaha/terminal:latest", + kind="headless", + cmd="python", + args="/arc/projects/demo/run.py", + ) + if ids: + running = await session.fetch(kind="headless", status="Running") + details = await session.info(ids) + events = await session.events(ids) + logs = await session.logs(ids) + print(running, details, events, logs) +``` + +## Cleanup + +`destroy()` returns a `dict[str, bool]` keyed by requested Session ID. The +filters after `prefix` in `destroy_with()` are keyword-only: + +```python +from canfar.sessions import Session + +with Session() as session: + print(session.destroy("session-id")) + print( + session.destroy_with( + "my-headless-", + kind="headless", + status="Completed", + ) + ) +``` - session = Session() - result = session.destroy(ids) - print(result) # {"id": True, ...} - ``` +The equivalent async call is: -=== "`async`" +```python +from canfar.sessions import AsyncSession - ```python - from canfar.sessions import AsyncSession +async def cleanup() -> None: async with AsyncSession() as session: - result = await session.destroy(ids) - print(result) # {"id": True, ...} - ``` -
-=== "Bulk Destroy" - - ```python - from canfar.sessions import Session + result = await session.destroy_with( + "my-headless-", + kind="headless", + status="Completed", + ) + print(result) +``` - session = Session() - result = session.destroy_with(prefix="test-", kind="headless", status="Succeeded") - print(result) # {"id": True, ...} - ``` +## Private Container Registry -=== "`async`" +Use the Configuration data model when an image is private: - ```python - from canfar.sessions import AsyncSession +```python +from canfar.models.config import Configuration +from canfar.models.registry import ContainerRegistry +from canfar.sessions import Session - async with AsyncSession() as session: - result = await session.destroy_with(prefix="test-", kind="headless", status="Succeeded") - print(result) # {"id": True, ...} - ``` +config = Configuration( + registry=ContainerRegistry(username="username", secret="CLI_SECRET") +) +with Session(config=config) as session: + ids = session.create( + name="private-session", + image="images.canfar.net/project/private-image:latest", + kind="headless", + cmd="python", + args="/app/run.py", + ) +``` diff --git a/docs/client/get-started.md b/docs/client/get-started.md index 9b65d92c..2bbd9920 100644 --- a/docs/client/get-started.md +++ b/docs/client/get-started.md @@ -1,13 +1,13 @@ # Install and Set Up -Use the CANFAR Python package when you want to automate Science Platform work: +Use the CANFAR Python package to automate work on a Science Platform Server: launch Sessions, list Container Images, fetch logs, inspect state, and clean up -resources from Python. +resources. ## Install ```bash -pip install canfar --upgrade +pip install --upgrade canfar ``` With `uv`: @@ -16,88 +16,100 @@ With `uv`: uv add canfar ``` -## Log in +## Authenticate -Authenticate once from the CLI. Python then uses the active Authentication and -Server selection. +The simplest path is to authenticate with the CLI. Python then uses the saved +Authentication Record and Server Selection: ```bash canfar login cadc ``` -For SRCNet: +Use `canfar login srcnet` for an OIDC Identity Provider. -```bash -canfar login srcnet -``` +Python also exposes native synchronous and asynchronous login functions: -Force a fresh login when credentials expire or you want to replace saved state: +```python +import canfar -```bash -canfar login cadc --force +canfar.login("srcnet", force=True) ``` -Check what Python will use: +For OIDC, Python login prints only the device-flow presentation data to the +terminal, then waits for approval. The output has this shape: -```bash -canfar auth show -canfar server ls +```text +Verification URL: https://example.com/device +Verification URL (complete): https://example.com/device?user_code=ABC123 +Device code: ABC123 ``` -## Create a Session +The device code above is the user-facing code; the private OAuth device token is +never printed. The Python API does not open a browser, render a QR code, or show +CLI progress. The CLI login command owns those interactive presentation +features. -```python -from canfar.sessions import Session +Inside an existing event loop, use `alogin()`; it performs native asynchronous +OIDC I/O and does not call `asyncio.run()`: -session = Session() -ids = session.create( - kind="notebook", - image="images.canfar.net/skaha/astroml:latest", - name="my-analysis", -) -print(ids) -``` +```python +import canfar -Open it in your browser: -```python -session.connect(ids) +async def authenticate() -> None: + await canfar.alogin("srcnet", force=True) ``` -## Use fixed resources +Both functions save the Authentication Record and discovered Science Platform +Servers but do not change the active Authentication or Server Selection. They +return `None`; an unknown Identity Provider raises `KeyError`, and credential or +discovery failures raise `canfar.authentication.AuthenticationError`. -Omit resources for flexible allocation. Pass `cores`, `ram`, and `gpus` when -you need fixed resources. +## Create a Session ```python -ids = session.create( - kind="headless", - image="images.canfar.net/skaha/astroml:latest", - name="batch-job", - cmd="python", - args="/arc/projects/demo/run.py", - cores=4, - ram=16, -) +from canfar.sessions import Session + +with Session() as session: + ids = session.create( + kind="notebook", + image="images.canfar.net/skaha/astroml:latest", + name="my-analysis", + ) + print(ids) ``` -## Use async workflows +`create()` returns `list[str]`. A failed replica is omitted and a total HTTP or +network failure returns `[]`; request validation errors still raise. + +## Edit and save Configuration + +`Configuration` is the persisted data shape. Its top-level fields are +`version`, `active`, `authentication`, `servers`, `registry`, and `console`. +Authentication Records are keyed by Identity Provider, Science Platform Servers +by Server Name, and `active` stores the selected references. The bound +`config.editor` is the supported editing surface: ```python -from canfar.sessions import AsyncSession +from canfar.models.config import Configuration -async with AsyncSession() as session: - ids = await session.create( - kind="notebook", - image="images.canfar.net/skaha/astroml:latest", - name="async-analysis", - ) - await session.connect(ids) +config = Configuration() +width = config.editor.get("console.width") +config.editor.set("console.width", 132) +config.editor.set("servers.canfar.auths", ["x509", "oidc"]) +config.editor.save() ``` +`get()` can return a scalar, mapping, or whole list through a dotted path. List +indices are not supported. `set()` validates before mutating the bound model; +invalid updates leave it unchanged. `save()` persists the validated +Configuration atomically. The editor itself is not part of the serialized +Configuration shape. + ## Private Container Images -Configure Container Registry credentials when you need private images. +Pass a `ContainerRegistry` in the Configuration when creating a Session from a +private image: ```python from canfar.models.config import Configuration @@ -107,18 +119,18 @@ from canfar.sessions import Session config = Configuration( registry=ContainerRegistry(username="username", secret="CLI_SECRET") ) -session = Session(config=config) - -ids = session.create( - kind="notebook", - image="images.canfar.net/my-project/private-image:latest", - name="private-image-test", -) +with Session(config=config) as session: + ids = session.create( + kind="notebook", + image="images.canfar.net/my-project/private-image:latest", + name="private-image-test", + ) ``` ## Read next - [Python quickstart](quick-start.md) - [Examples](examples.md) +- [Data access](data.md) - [Authentication and Servers](../cli/authentication-contexts.md) - [Session API](session.md) diff --git a/docs/client/helpers.md b/docs/client/helpers.md index 5c894960..8ab234cb 100644 --- a/docs/client/helpers.md +++ b/docs/client/helpers.md @@ -40,8 +40,13 @@ for item in distributed.chunk(work): ``` ### Validation and errors -- `replica` must be >= 1 and <= `total` -- `total` must be > 0 + +`chunk()` requires `total > 0` and `1 <= replica <= total`, raising +`ValueError` otherwise. `stripe()` intentionally keeps a looser compatibility +contract: an out-of-range replica yields no items, while a non-positive `total` +raises `ValueError`. Both functions use `REPLICA_ID` and `REPLICA_COUNT` when +their corresponding arguments are omitted, defaulting to one replica when the +environment is absent. ## API Reference @@ -55,4 +60,4 @@ for item in distributed.chunk(work): show_source: false heading_level: 3 docstring_style: google - show_signature_annotations: true \ No newline at end of file + show_signature_annotations: true diff --git a/docs/client/home.md b/docs/client/home.md index 131fb8c4..7ae958a6 100644 --- a/docs/client/home.md +++ b/docs/client/home.md @@ -1,15 +1,19 @@ # Python Client -The CANFAR Python client wraps the Science Platform APIs for Sessions, -Container Images, Authentication-aware HTTP clients, and automation scripts. +The CANFAR Python client provides authenticated access to Sessions, Container +Images, VOSpace Services, and Science Platform metadata. ## Install and authenticate ```bash -pip install canfar --upgrade +pip install --upgrade canfar canfar login cadc ``` +The CLI stores the Authentication Record and Server Selection used by default +by Python clients. Python OIDC login is also available as `canfar.login()` and +`canfar.alogin()`; see [Install and set up](get-started.md). + ## Core workflows
@@ -17,60 +21,58 @@ canfar login cadc - **Create Sessions** Launch notebooks, desktops, CARTA, Firefly, contributed apps, or headless - jobs. + workloads with `Session` or `AsyncSession`. [:octicons-arrow-right-16: Quickstart](quick-start.md) -- **Automate jobs** - - Use `Session` or `AsyncSession` to create, inspect, log, and destroy - Sessions from Python. - - [:octicons-arrow-right-16: Examples](examples.md) +- **Read data** -- **Choose Servers** + Address VOSpace Services through explicit Storage Identifiers and standard + fsspec methods. - Authenticate with an IDP, select a Science Platform Server, and keep scripts - noninteractive. + [:octicons-arrow-right-16: Data access](data.md) - [:octicons-arrow-right-16: Auth and Servers](../cli/authentication-contexts.md) +- **Inspect platform state** -- **Use references** + Query available resources, Container Images, and Science Platform Server + capacity. - Jump to generated API pages for method signatures and model details. - - [:octicons-arrow-right-16: API reference](session.md) + [:octicons-arrow-right-16: Python API reference](session.md)
-## Minimal example +## Minimal synchronous example ```python from canfar.sessions import Session -session = Session() -ids = session.create( - kind="notebook", - image="images.canfar.net/skaha/astroml:latest", - name="my-analysis", -) -session.connect(ids) +with Session() as session: + ids = session.create( + kind="notebook", + image="images.canfar.net/skaha/astroml:latest", + name="my-analysis", + ) + if ids: + session.connect(ids) ``` -## Async example +## Minimal asynchronous example ```python from canfar.sessions import AsyncSession -async with AsyncSession() as session: - ids = await session.create( - kind="headless", - image="images.canfar.net/skaha/astroml:latest", - name="batch-job", - cmd="python", - args="/arc/projects/demo/run.py", - ) - await session.events(ids, verbose=True) + +async def main() -> None: + async with AsyncSession() as session: + ids = await session.create( + kind="headless", + image="images.canfar.net/skaha/astroml:latest", + name="batch-session", + cmd="python", + args="/arc/projects/demo/run.py", + ) + if ids: + await session.events(ids) ``` ## Main modules @@ -78,14 +80,17 @@ async with AsyncSession() as session: | Module | Use | | --- | --- | | `canfar.sessions` | Create, fetch, inspect, connect, log, and destroy Sessions. | -| `canfar.images` | List and inspect available Container Images. | -| `canfar.authentication` | Noninteractive Authentication helpers. | -| `canfar.server` | Server discovery, validation, and selection helpers. | -| `canfar.client` | Lower-level HTTP client composition. | +| `canfar.images` | List Container Images or fetch parsed image details. | +| `canfar.storage` | List Storage Identifiers and build explicit fsspec filesystems. | +| `canfar.context` | Read resource limits advertised by a Science Platform Server. | +| `canfar.overview` | Check Science Platform Server capacity. | +| `canfar.authentication` | Login and manage saved Authentication Records. | +| `canfar.client` | Compose lower-level synchronous or asynchronous HTTP clients. | ## Read next - [Install and set up](get-started.md) - [Python quickstart](quick-start.md) - [Examples](examples.md) +- [Data access](data.md) - [Migration guide](migration.md) diff --git a/docs/client/images.md b/docs/client/images.md index b39839b5..456f7085 100644 --- a/docs/client/images.md +++ b/docs/client/images.md @@ -1,46 +1,39 @@ -# Images API +# Container Images API -!!! info "Overview" - The Image API allows you to get information about the **publicly available** images on the CANFAR Science Platform through the CANFAR Harbor Registry. It can be used to get information about all images, or filter by a specific image kind. +`Images` lists the Container Images advertised by a CANFAR Science Platform +Server. It inherits `HTTPClient`, so the same saved or runtime credential +selection applies. -## Getting Image Information +## Image identifiers -```python title="Get image information" +```python from canfar.images import Images -images = Images() -images.fetch() -[ - "images.canfar.net/canfar/base-3.12:v0.4.1", - "images.canfar.net/canucs/test:1.2.5", - "images.canfar.net/canucs/canucs:1.2.9", - ..., -] +with Images() as images: + all_images = images.fetch() # list[str] + headless = images.fetch(kind="headless") # list[str] + print(headless) ``` -But most of the time, you are only interested in images of a particular type. For example, if you want to get all the images that are available for `headless` sessions, you can do the following: +Each string is an image identifier such as +`images.canfar.net/skaha/terminal:latest`. The optional `kind` is sent as the +server's image-type filter. -```python title="Get headless image information" -images.fetch(kind="headless") -``` +## Parsed image details + +Use `details()` when the digest and supported kinds are needed: ```python -[ - "images.canfar.net/chimefrb/testing:keep", - "images.canfar.net/lsst/lsst_v19_0_0:0.1", - "images.canfar.net/skaha/lensfit:22.11", - "images.canfar.net/skaha/lensfit:22.10", - "images.canfar.net/skaha/lensingsim:22.07", - "images.canfar.net/skaha/phosim:5.6.11", - "images.canfar.net/skaha/terminal:1.1.2", - "images.canfar.net/skaha/terminal:1.1.1", - "images.canfar.net/uvickbos/pycharm:0.1", - "images.canfar.net/uvickbos/swarp:0.1", - "images.canfar.net/uvickbos/isis:2.2", - "images.canfar.net/uvickbos/find_moving:0.1", -] +from canfar.images import Images + +with Images() as images: + for image in images.details(): + print(image.id, image.types, image.digest) ``` +`details()` returns `list[canfar.models.containers.Image]`; each model has `id`, +`types`, and `digest` fields. + ## API Reference ::: canfar.images.Images @@ -48,6 +41,7 @@ images.fetch(kind="headless") selection: members: - fetch + - details rendering: members_order: source show_root_heading: true diff --git a/docs/client/migration.md b/docs/client/migration.md index e3595b4e..f8630455 100644 --- a/docs/client/migration.md +++ b/docs/client/migration.md @@ -1,83 +1,88 @@ -# skaha → canfar +# `skaha` to `canfar` -In summer 2025, the CANFAR Python client was moved from [shinybrar/skaha](https://github.com/shinybrar/skaha) to [opencadc/canfar](https://github.com/opencadc/canfar) to be officially supported by the Canadian Astronomy Data Centre (CADC). As part of this move, the Python package was renamed from `skaha` to `canfar` to better reflect a unified naming scheme across the CANFAR Science Platform. +The supported Python package is now `canfar`, published from +[opencadc/canfar](https://github.com/opencadc/canfar). The service endpoints +and the historical `X-Skaha-*` request headers remain server-side contracts. -This guide helps you migrate from the `skaha` Python package to `canfar`. +## Imports -## Summary of changes +| Old import | Current import | +| --- | --- | +| `from skaha.session import Session` | `from canfar.sessions import Session` | +| `from skaha.session import AsyncSession` | `from canfar.sessions import AsyncSession` | +| `from skaha.client import SkahaClient` | `from canfar.client import HTTPClient` | -- Package name: `skaha` → `canfar`. -- **Breaking Changes** - - `skaha.session` → `canfar.sessions`. - - `headless` session `kind` parameter is no longer required. - - `session.info()` query now returns `Completed` instead of `Succeeded`. -- Configuration path: `~/.skaha/config.yaml` → `~/.canfar/config.yaml`. -- Logger name: `canfar`. Current releases write no default log file; file output - is explicit through root `--log-file` or a `pathlib.Path` passed to - `canfar.configure_logging()`. See - [Logging](../cli/logging.md). -- Environment variables: prefix change `SKAHA_…` → `CANFAR_…`. -- CLI entry point: `canfar` (single entry point). -- User-Agent header: `python-canfar/{version}`. -- Protocol contracts: server URLs and custom headers remain unchanged (see notes below). +`Session` and `AsyncSession` are separate native clients with equivalent public +operations. `create()` returns `list[str]`; `fetch()` returns +`list[dict[str, str]]`. The filters after `destroy_with(prefix)` are +keyword-only in both clients. -## Code Examples +```python +from canfar.sessions import Session -- Python client session - - ```python title="Before" - from skaha.session import AsyncSession, Session - ``` - - ```python title="After" - from canfar.sessions import AsyncSession, Session - ``` +with Session() as session: + ids = session.create( + name="migrated-session", + image="images.canfar.net/skaha/terminal:latest", + ) +``` -- Client composition +`kind` defaults to `"headless"` when it is omitted. For headless Sessions, +`cmd`, `args`, and `env` remain available; interactive kinds do not accept +headless command fields. - ```python title="Before" - from skaha.client import SkahaClient +## Configuration - client = SkahaClient(...) - ``` +The default file moves from `~/.skaha/config.yaml` to +`~/.canfar/config.yaml`. The current persisted shape separates: - ```python title="After" - from canfar.client import HTTPClient +- `authentication`: Authentication Records keyed by Identity Provider; +- `servers`: Science Platform Servers keyed by Server Name; and +- `active.authentication` / `active.server`: the active references. - client = HTTPClient(...) - ``` +Use the bound `Configuration.editor` for validated dotted updates and atomic +persistence. Do not call the removed Configuration service methods: -## Environment variables +```python +from canfar.models.config import Configuration -```bash title="Before" -SKAHA_TIMEOUT, SKAHA_CONCURRENCY, SKAHA_TOKEN, SKAHA_URL, SKAHA_LOGLEVEL -``` -```bash title="After" -CANFAR_TIMEOUT, CANFAR_CONCURRENCY, CANFAR_TOKEN, CANFAR_URL, CANFAR_LOGLEVEL +config = Configuration() +config.editor.set("console.width", 132) +config.editor.save() ``` -## Configuration +See [Install and set up](get-started.md#edit-and-save-configuration) for the +full editor contract. + +## Authentication and data + +Use `canfar login` or the Python `canfar.login()` / `canfar.alogin()` helpers to +create Authentication Records. Python OIDC login prints the verification URL +and user-facing device code to the terminal; the CLI owns browser, QR, and +progress presentation. See [Install and set up](get-started.md#authenticate). -- The default config file moves from `~/.skaha/config.yaml` to `~/.canfar/config.yaml`. -- Current config files are versioned with `version: 1`. -- Authentication records and Science Platform Servers are stored separately. -- Active routing is stored under `active.authentication` and `active.server`. -- Legacy or unsupported config files are backed up to - `..back` before a default config is written. +For VOSpace access, replace implicit or package-specific storage helpers with +explicit Storage Identifiers: -After a legacy reset, run: +```python +from canfar.storage import filesystem, identifiers -```bash -canfar login cadc +print(identifiers()) +vault = filesystem("vault") +try: + data = vault.cat_file("/project/observations/example.fits") +finally: + vault.close() ``` -## Documentation and links +Storage Identifiers are not dynamic module members or fsspec schemes. See +[Data Access](data.md) for standard fsspec operations. -- Repo: `https://github.com/opencadc/canfar` -- Docs: `https://opencadc.github.io/canfar/` -- Changelog: `https://opencadc.github.io/canfar/changelog/` +## Runtime configuration and logging -## Notes on protocol stability +The package-level `canfar.configure_logging()` function configures application +logging. `HTTPClient` accepts runtime `token` and `certificate` values, which +take precedence over saved Authentication Records for that client only. -- Server base path segments under `/skaha` are server-side contracts and remain unchanged (for example, `https://ws-uv.canfar.net/skaha`). -- Historical header names remain unchanged (for example, `X-Skaha-Authentication-Type`, `X-Skaha-Registry-Auth`). +For CLI command names and output, use the [CLI documentation](../cli/cli-help.md) +rather than the Python migration guide. diff --git a/docs/client/overview.md b/docs/client/overview.md index 189d1a80..162d6024 100644 --- a/docs/client/overview.md +++ b/docs/client/overview.md @@ -1,5 +1,20 @@ -!!! info "Overview API" - The Overview API provides information about the availability of the CANFAR Science Platform. +# Overview API + +`Overview` checks whether a Science Platform Server reports itself as +available. It inherits `HTTPClient` and uses the selected or explicitly supplied +credentials. + +```python +from canfar.overview import Overview + +with Overview() as overview: + if overview.availability(): + print("Science Platform Server is available") +``` + +`availability()` returns `True` only when the VOSI availability response says +the server is available. Empty or malformed availability data returns `False` +and is logged. ::: canfar.overview.Overview handler: python @@ -10,4 +25,4 @@ members_order: source show_root_heading: true show_source: true - heading_level: 1 + heading_level: 2 diff --git a/docs/client/quick-start.md b/docs/client/quick-start.md index a7aaa58d..5b9a6e38 100644 --- a/docs/client/quick-start.md +++ b/docs/client/quick-start.md @@ -1,58 +1,80 @@ # Python Quickstart -This guide creates a notebook Session, inspects it, and deletes it from Python. +This guide creates a notebook Session, checks its state, and deletes it from +Python. ## 1. Authenticate ```bash -pip install canfar --upgrade +pip install --upgrade canfar canfar login cadc ``` -Use `canfar login srcnet` for SRCNet. +Use `canfar login srcnet` for an SRCNet Identity Provider. For direct Python +OIDC login, see [login and device flow](get-started.md#authenticate). ## 2. Create a notebook ```python from canfar.sessions import Session -session = Session() -ids = session.create( - kind="notebook", - image="images.canfar.net/skaha/astroml:latest", - name="quickstart-notebook", - cores=2, - ram=4, -) -print(ids) +with Session() as session: + ids = session.create( + kind="notebook", + image="images.canfar.net/skaha/astroml:latest", + name="quickstart-notebook", + cores=2, + ram=4, + ) + print(ids) ``` -`create()` returns a list of Session IDs. +`create()` returns `list[str]`; it contains only successfully launched Session +IDs. -## 3. Open the notebook +## 3. Inspect and connect ```python -session.connect(ids) +with Session() as session: + running = session.fetch(kind="notebook", status="Running") + print(running) # list[dict[str, str]] + session.connect([item["id"] for item in running]) ``` -The notebook can take a minute or two to become reachable. Check status: +`connect()` opens the `connectURL` for Sessions that are ready. It returns +`None`; a Session that is not running is logged and skipped. + +## 4. Read events and logs ```python -running = session.fetch(kind="notebook", status="Running") -print(running) +with Session() as session: + events = session.events("session-id") + logs = session.logs("session-id") + print(events) + print(logs) ``` -## 4. Inspect events and logs +The default return values are `list[dict[str, str]]` for events and +`dict[str, str]` for logs. With `verbose=True`, both methods log their output +through `canfar.sessions` and return `None`. + +## 5. Clean up ```python -session.events(ids, verbose=True) -session.logs(ids, verbose=True) +with Session() as session: + result = session.destroy("session-id") + print(result) # dict[str, bool] ``` -## 5. Clean up +For bulk cleanup, the filters after `prefix` are keyword-only: ```python -session.destroy(ids) +with Session() as session: + result = session.destroy_with( + "quickstart-", + kind="notebook", + status="Completed", + ) ``` ## Async version @@ -60,25 +82,23 @@ session.destroy(ids) ```python from canfar.sessions import AsyncSession -async with AsyncSession() as session: - ids = await session.create( - kind="notebook", - image="images.canfar.net/skaha/astroml:latest", - name="quickstart-notebook", - ) - await session.connect(ids) - await session.events(ids, verbose=True) - await session.destroy(ids) -``` -## Troubleshooting +async def main() -> None: + async with AsyncSession() as session: + ids = await session.create( + kind="notebook", + image="images.canfar.net/skaha/astroml:latest", + name="async-notebook", + ) + if ids: + await session.connect(ids) + print(await session.fetch(kind="notebook", status="Running")) + print(await session.events(ids)) + print(await session.logs(ids)) + print(await session.destroy(ids)) -| Symptom | Check | -| --- | --- | -| Authentication fails | Run `canfar --log-level debug login cadc --force`. | -| Session does not start | Run `canfar stats`, then try smaller `cores` or `ram`. | -| Browser URL fails | Wait 60-120 seconds and confirm the Session is `Running`. | -| Script needs stable output | Use CLI machine output such as `canfar ps --json`. | +``` -See [Logging](../cli/logging.md) for the CLI and Python -logging controls. +For CLI diagnostics, see [Logging](../cli/logging.md). Authentication-dependent +full integration tests are separate from the deterministic Python examples; +see [Testing](testing.md). diff --git a/docs/client/session.md b/docs/client/session.md index 4a2cc1b8..87ef7f84 100644 --- a/docs/client/session.md +++ b/docs/client/session.md @@ -1,27 +1,103 @@ # Session API -!!! info "Overview" - The `Session` API is the core of canfar, enabling you to create, manage, and destroy sessions on the CANFAR Science Platform. +`Session` is the native synchronous Python client for user-owned Sessions on a +Science Platform Server. It inherits the HTTP and credential behavior of +`HTTPClient`. -## Creating sessions +## Return shapes and failure behavior -`Session.create` returns a `list` of session IDs (strings) for each successful launch. If the platform rejects a request or -the call hits a network or HTTP error, that launch is skipped and logging records the failure. If every launch fails, you get an -empty list. The method does not raise for those errors, so callers can check `if not ids:` (or compare the length to `replicas`) -after the call. For details in Python, call `canfar.configure_logging()` at the -application entry point; it honors `CANFAR_LOGLEVEL`. In the CLI, use a root -control such as `canfar --log-level debug create ...`. See -[Logging](../cli/logging.md). +The released Python contracts are: + +| Method | Result | +| --- | --- | +| `fetch(kind=None, status=None, view=None)` | `list[dict[str, str]]` | +| `create(...)` | `list[str]` containing the IDs that launched successfully | +| `info(ids)` | `list[dict[str, Any]]` | +| `logs(ids, verbose=False)` | `dict[str, str]`, or `None` when `verbose=True` | +| `events(ids, verbose=False)` | `list[dict[str, str]]`, or `None` when `verbose=True` | +| `destroy(ids)` / `destroy_with(...)` | `dict[str, bool]` | +| `connect(ids)` | `None`; opens ready Session URLs | + +`create()` skips an individual launch after an HTTP or network failure and logs +the failure without raising. If all requested launches fail, it returns `[]`. +Validation errors in the request are raised before the HTTP call. + +`fetch(view="all")` requests the server's all-Sessions view when the caller is +authorized; the response remains a list of dictionaries. `stats()` returns the +Science Platform Server's aggregate resource statistics as a dictionary. + +`logs(..., verbose=True)` and `events(..., verbose=True)` route their results to +the `canfar.sessions` logger and return `None`; configure application logging +with `canfar.configure_logging()` if the output should be visible. + +## Create and manage a Session + +```python +from canfar.sessions import Session + +with Session() as session: + ids = session.create( + name="my-analysis", + image="images.canfar.net/skaha/astroml:latest", + kind="notebook", + ) + if ids: + session.connect(ids) + print(session.fetch(kind="notebook", status="Running")) + print(session.info(ids)) + print(session.logs(ids)) + print(session.events(ids)) + print(session.destroy(ids)) +``` + +When `kind="headless"`, pass `cmd`, `args`, and `env` for the command. `cores` +and `ram` request fixed resources; omitting them uses the server's flexible +allocation policy. `replicas` requests multiple Sessions and the return value +contains the successful IDs only. + +```python +from canfar.sessions import Session + +with Session() as session: + ids = session.create( + name="batch", + image="images.canfar.net/skaha/terminal:latest", + kind="headless", + cmd="python", + args="/arc/projects/demo/run.py", + replicas=3, + ) +``` + +## Select Sessions for cleanup + +`destroy_with` has a keyword-only filter contract: + +```python +session.destroy_with( + "batch-", + kind="headless", + status="Completed", +) +``` + +Its signature is `destroy_with(prefix, *, kind="headless", status="Completed")`. +A prefix is matched literally unless it contains regular-expression +metacharacters; such a value is treated as a regular expression. ::: canfar.sessions.Session handler: python selection: members: - fetch + - stats - create - info - logs + - events - destroy + - destroy_with + - connect rendering: members_order: source show_root_heading: true diff --git a/docs/client/testing.md b/docs/client/testing.md index 42b1625a..44c9e614 100644 --- a/docs/client/testing.md +++ b/docs/client/testing.md @@ -1,168 +1,70 @@ # Testing -This document provides comprehensive information about testing [opencadc/canfar](https://github.com/opencadc/canfar). - -## Overview - -Canfar uses [pytest](https://pytest.org/) as its testing framework. The test suite includes unit tests, integration tests, and end-to-end tests that verify the functionality of the client library. +CANFAR uses [pytest](https://pytest.org/). Tests mirror the `canfar/` module +layout and include deterministic unit/contract tests plus +Authentication-dependent integration tests. ## Prerequisites -To run tests for Canfar, you need: - -1. **Valid CANFAR Account**: Access to the CANFAR Science Platform -2. **X.509 Certificate**: For authentication with CANFAR services -3. **Python Environment**: Set up with uv +Install the project environment with `uv`. Full integration coverage requires a +valid CANFAR Authentication Record and X.509 certificate/configuration. Do not +run that suite against a real account unless the test workflow is explicitly +intended to create or clean up platform resources. -For certificate generation, refer to the [get started](get-started.md) section. +## Local validation -## Running Tests +The default local test gate excludes slow tests and avoids external +Authentication: -### Basic Test Execution - -Run all tests: ```bash -uv run pytest +uv run --no-sync pytest tests -m "not slow" --no-cov -q \ + -o cache_dir=/tmp/canfar-pytest-cache ``` -Run tests with verbose output: -```bash -uv run pytest -v -``` +Useful focused checks include: -Run tests with coverage report: ```bash -uv run pytest --cov +uv run --no-sync pytest tests/test_sessions_fetch.py tests/test_sessions_lifecycle.py \ + tests/test_storage.py tests/test_config_editor.py -q +uv run --no-sync ruff check . --no-cache +uv run ty check canfar +uv run --group docs mkdocs build ``` -### Test Categories - -Canfar tests are organised with markers to help you run specific subsets: - -#### Slow Tests +The documentation build is the check for broken navigation, Markdown, and +generated Python API pages. -Some tests are marked as "slow" because they involve: -- Network operations with CANFAR services -- Waiting for session state changes -- Authentication timeouts -- Long-running operations +## Test markers -**Skip slow tests for faster development:** -```bash -uv run pytest -m "not slow" -``` +Use markers to select known test categories: -**Run only slow tests:** ```bash -uv run pytest -m "slow" +uv run --no-sync pytest -m unit +uv run --no-sync pytest -m integration +uv run --no-sync pytest -m slow ``` -#### Integration Tests +Integration and slow tests may contact CANFAR services and require valid +credentials. `-m "not slow"` is the deterministic default; it is not a claim +that every test marked `integration` is safe without Authentication. -Tests that interact with external services: -```bash -uv run pytest -m "integration" -``` +## Full suite -#### Unit Tests +Run the full suite only in an environment prepared for the credentialed gate: -Fast, isolated tests: ```bash -uv run pytest -m "unit" +uv run --no-sync pytest ``` -### Test Methodology - -Tests are organised in the `tests/` directory and follow a specific naming convention that mirrors the source code structure. This approach ensures that tests are easy to locate and maintain. - -The naming convention is as follows: - -- If the source file is `canfar/path/to/file.py`, the corresponding test file will be `tests/test_path_to_file.py`. -- If the source file is `canfar/module.py`, the corresponding test file will be `tests/test_module.py`. - -For example: - -- The tests for `canfar/client.py` are located in `tests/test_client.py`. -- The tests for `canfar/auth/oidc.py` are located in `tests/test_auth_oidc.py`. - -This structure makes it straightforward to find the tests associated with a particular module or file. - -## Development Workflow - -For efficient development, follow this testing workflow: - -1. **During Development**: Run fast tests only - ```bash - uv run pytest -m "not slow" - ``` - -2. **Before Committing**: Run the full test suite - ```bash - uv run pytest - ``` - -3. **Debugging Specific Issues**: Run individual test files - ```bash - uv run pytest tests/test_session.py - ``` - -## Test Configuration - -Test configuration is defined in `pyproject.toml`: - -```toml -[tool.pytest.ini_options] -markers = [ - "integration: marks tests as integration tests", - "unit: marks tests as unit tests", - "slow: marks tests as slow (deselect with '-m \"not slow\"')", - "order: marks tests that need to run in a specific order", -] -``` - -## Continuous Integration - -In CI environments, all tests (including slow ones) are executed to ensure complete validation. The CI pipeline: - -1. Sets up authentication with CANFAR -2. Runs the complete test suite -3. Generates coverage reports -4. Cleans up authentication artifacts - -## Writing Tests - -When contributing new tests: - -1. **Follow the naming convention**: Create a test file that mirrors the source file's path and name. -2. **Mark slow tests**: Add `@pytest.mark.slow` to any test that involves network operations, interacts with external services, or has long execution times. This allows developers to skip these tests for a faster development cycle. -3. **Use appropriate markers**: Mark tests as `unit`, `integration`, etc. -4. **Add docstrings**: Document what each test verifies. - -Example of a slow test: -```python -import pytest - -@pytest.mark.slow -def test_long_running_operation(): - """Test that involves waiting or network operations.""" - # Test implementation - pass -``` - - -## Troubleshooting - -### Authentication Issues -- Ensure your X.509 certificate is valid and not expired -- Check that you have access to the CANFAR Science Platform -- Verify your certificate is in the correct location (`~/.ssl/`) +The full run includes Authentication-dependent tests, Session lifecycle work, +and network operations. A local failure can therefore indicate missing +credentials or unavailable Science Platform infrastructure rather than a +library regression. -### Slow Test Timeouts -- Slow tests have built-in timeouts (typically 60 seconds) -- If tests consistently timeout, check your network connection -- Platform availability may affect test execution times +## Adding tests -### Test Failures -- Check if the CANFAR Science Platform is accessible -- Verify your authentication credentials -- Review test logs for specific error messages +- Mirror the source module path under `tests/`. +- Mark network or long-running tests with `integration` and/or `slow`. +- Prefer observable public seams such as `httpx.MockTransport`, `CliRunner`, + and the public Configuration editor. +- Keep sync tests synchronous and async tests on the native async path. diff --git a/docs/client/updates.md b/docs/client/updates.md index 6c81e057..b5408924 100644 --- a/docs/client/updates.md +++ b/docs/client/updates.md @@ -1,179 +1,28 @@ # What's New in CANFAR -Stay up to date with the latest features, improvements, and changes in CANFAR. - -## Recent Updates - -!!! tip "New in v1.1+" - - ### **🛡️ Improved Session Data Validation** - - The CANFAR CLI now features enhanced resilience when handling session data from the Science Platform API. This update improves the user experience when the API returns incomplete or malformed session information. - - **What Changed:** - - - **Graceful Degradation**: The CLI commands (`canfar info`, `canfar ps`) now continue to work even when the API returns incomplete session data, displaying partial information instead of crashing. - - **Better Error Reporting**: Missing or invalid fields are tracked internally and can be viewed with the `--debug` flag for troubleshooting. - - **Enhanced Display**: Resource usage metrics for flexible sessions is now reported with better readability. - - **Type Safety**: Session type validation has been strengthened using Pydantic's built-in validators. - - **Example:** - - ```bash title="Flexible Session Resource Usage" - $ canfar info n2tr1rpf - - CANFAR Session Info for n2tr1rpf - - Session ID n2tr1rpf - Name spy-panda - Status Running - Type notebook - CPU Usage 0.001 core(s) - RAM Usage 0.22 GB - GPU Usage Unknown # (GPU not requested) - ``` - - ```bash title="Debug Mode for Troubleshooting" - $ canfar info --debug n2tr1rpf - - # Shows additional warnings about missing/invalid fields - ⚠️ Session Response Warnings: - • missing or invalid startTime in response - • missing or invalid expiryTime in response - ``` - -!!! success "v1.0" - - :fontawesome-solid-exclamation-triangle: **Breaking Changes** - - - Deprecation of support for Python 3.8 and 3.9. - - The Python package has been renamed from `skaha` to `canfar`. - - The `skaha.session` API has been deprecated in favor of `canfar.sessions`. - - See [Migration guide to migrate from skaha → canfar](migration.md). - - :simple-gnubash: **CLI Support** - - - Comprehensive CLI support has been added to the client under the `canfar` entry point. See [CLI Reference](../cli/cli-help.md) for more information. - - The `canfar` CLI is the recommended way to manage Authentication and Server selection. See [Authentication and Servers](../cli/authentication-contexts.md) for more information. - - **🌎 SRCnet Support** - - - CANFAR now supports launching sessions on all the SRCnet CANFAR Science Platform instances worldwide. - - **:fontawesome-brands-connectdevelop:** OIDC Authentication - - - OpenID Connect (OIDC) authentication is now supported for all SRCnet Science Platform servers where applicable. - - **:material-book-outline: Documentation** - - - Complete overhaul to bring all documentation sources under a single roof. - - Significant improvements to the Python client and brand new CLI documentation. - -!!! info "New in v0.7+" - - ### **🔐 Enhanced Authentication System** - Canfar now features a comprehensive authentication system with support for multiple authentication modes and automatic credential management. - - ```python title="Authentication Examples" - from canfar.client import HTTPClient - from pathlib import Path - - # X.509 certificate authentication - client = HTTPClient(certificate=Path("/path/to/cert.pem")) - - # OIDC token authentication (configured) - client = HTTPClient() # Uses auth.mode = "oidc" - - # Bearer token authentication - from pydantic import SecretStr - client = HTTPClient(token=SecretStr("your-token")) - ``` - - ### **🚀 Asynchronous Sessions** - Canfar now supports asynchronous sessions using the `AsyncSession` class while maintaining 1-to-1 compatibility with the `Session` class. - - ```python title="Asynchronous Session Creation" - from canfar.sessions import AsyncSession - - asession = AsyncSession() - response = await asession.create( - name="test", - image="images.canfar.net/skaha/astroml:latest", - cores=2, - ram=8, - gpu=1, - kind="headless", - cmd="env", - env={"KEY": "VALUE"}, - replicas=3, - ) - ``` - - ### **🗄️ Backend Upgrades** - - - 📡 Canfar now uses the `httpx` library for making HTTP requests instead of `requests`. This adds asynchronous support and also to circumvent the `requests` dependence on `urllib3` which was causing SSL issues on MacOS. See [this issue](https://github.com/urllib3/urllib3/issues/3020) for more details. - - 🔑 Canfar now supports multiple authentication methods including X.509 certificates, OIDC tokens, and bearer tokens with automatic SSL context management. - - 🏎️💨 Added `loglevel` and `concurrency` support to manage the new explosion in functionality! - - 🔍 Comprehensive debug logging for authentication flow and client creation troubleshooting. - - ### **🧾 Logs to `stdout`** - - The `[Session|AsyncSession].logs` method now prints colored output to `stdout` instead of returning them as a string with `verbose=True` flag. - - ```python title="Session Logs" - from canfar.sessions import AsyncSession - - asession = AsyncSession() - await asession.logs(ids=["some-uuid"], verbose=True) - ``` - - ### **🪰 Firefly Support** - Canfar now supports launching `firefly` session on the CANFAR Science Platform. - - ```python title="Firefly Session Creation" - session.create( - name="firefly", - image="images.canfar.net/skaha/firefly:latest", - kind="firefly", - ) - ``` - -!!! info "New in v0.4+" - - ### **🔐 Private Images** - - Starting October 2024, to create a session with a private container image from the [CANFAR Harbor Registry](https://images.canfar.net/), you will need to provide your harbor `username` and the `CLI Secret` through a `ContainerRegistry` object. - - ```python title="Private Image Registry Configuration" - from canfar.models import ContainerRegistry - from canfar.sessions import Session - - registry = ContainerRegistry(username="username", secret="sUp3rS3cr3t") - session = Session(registry=registry) - ``` - - Alternatively, if you have environment variables, `CANFAR_REGISTRY_USERNAME` and `CANFAR_REGISTRY_SECRET`, you can create a `ContainerRegistry` object without providing the `username` and `secret`. - - ```python title="Private Image Registry with Environment Variables" - from canfar.models import ContainerRegistry - - registry = ContainerRegistry() - ``` - - ### **💣 Destroy Sessions** - ```python title="Destroying Sessions" - from canfar.sessions import Session - - session = Session() - session.destroy_with(prefix="test", kind="headless", status="Running") - session.destroy_with(prefix=".*-analysis", kind="headless", status="Pending") - ``` - -## Previous Versions - -For a complete history of changes, see the [Changelog](../changelog.md). - -## Stay Updated - -- 📢 [GitHub Releases](https://github.com/opencadc/canfar/releases) -- 💬 [Discussions](https://github.com/opencadc/canfar/discussions) +This page highlights the current Python client contracts. For release-by-release +changes, see the [changelog](../changelog.md) and +[GitHub Releases](https://github.com/opencadc/canfar/releases). + +## Current Python API + +- `Session` and `AsyncSession` are native synchronous and asynchronous clients + with equivalent public operations. +- `Session.create()` and `AsyncSession.create()` return `list[str]`, omitting + failed replicas instead of raising for an individual HTTP/network failure. +- `fetch()` returns the server response as `list[dict[str, str]]`. +- `destroy_with(prefix, *, kind=..., status=...)` keeps its keyword-only filter + contract in both clients. +- `canfar.login()` and `canfar.alogin()` provide synchronous and asynchronous + Python Authentication flows. OIDC device login prints a verification URL and + user-facing code to the terminal; browser, QR, and progress presentation stay + in the CLI. +- `config.editor.get()`, `set()`, and `save()` are the supported Configuration + editing operations. The persisted Configuration shape remains stable. +- `canfar.storage.identifiers()` and `canfar.storage.filesystem(identifier)` are + the explicit VOSpace Python surface. Storage Identifiers are not dynamic + module members or fsspec schemes. +- `Overview` and `canfar.helpers.distributed` remain supported public APIs. + +Start with the [Python quickstart](quick-start.md), then see the [Session API](session.md), +[Data Access](data.md), and [Migration Guide](migration.md). diff --git a/docs/conduct.md b/docs/conduct.md index c2cf0224..4f2817cb 100644 --- a/docs/conduct.md +++ b/docs/conduct.md @@ -1 +1,12 @@ ---8<-- "CODE_OF_CONDUCT.md" \ No newline at end of file +--8<-- "CODE_OF_CONDUCT.md" +# Code of Conduct + +CANFAR follows the [Contributor Covenant Code of Conduct](https://github.com/opencadc/canfar/blob/main/CODE_OF_CONDUCT.md). +It applies to project discussions, issues, pull requests, documentation, and +community events. + +Report harassment or other unacceptable behaviour privately to +`shiny.brar@nrc-cnrc.gc.ca`. Reports are reviewed fairly and confidentially. + +See the [full code of conduct](https://github.com/opencadc/canfar/blob/main/CODE_OF_CONDUCT.md) +for the community standards and enforcement process. diff --git a/docs/contributing.md b/docs/contributing.md index e079654f..5032892c 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1 +1,42 @@ ---8<-- "CONTRIBUTING.md" \ No newline at end of file +--8<-- "CONTRIBUTING.md" +# Contributing + +Contributions to CANFAR include code, tests, documentation, examples, issue +reports, and improvements to the user experience. Read the +[Code of Conduct](conduct.md) before participating. + +## Set up + +CANFAR uses [uv](https://docs.astral.sh/uv/) for development: + +```bash +uv sync --all-extras --dev +uv run pre-commit install --hook-type commit-msg +``` + +Use a valid CANFAR account and certificate only for integration tests. Most +pull-request checks are deterministic: + +```bash +uv run --no-sync pytest tests -m "not slow" --no-cov -q -o cache_dir=/tmp/canfar-pytest-cache +uv run --group docs mkdocs build +``` + +The full test suite contacts the Science Platform and requires valid +credentials. Run it only when those credentials are available. + +## Documentation + +Keep examples aligned with the installed CLI and Python API. Build the site +locally with `uv run --group docs mkdocs serve` and check navigation, links, +images, and code blocks before opening a pull request. + +## Submit a change + +Open an issue or pull request on [GitHub](https://github.com/opencadc/canfar). +Use a focused branch and a [Conventional Commit](https://www.conventionalcommits.org/) +message. Include the tests or documentation build you ran in the pull request +description. + +The repository's [contributor guide](https://github.com/opencadc/canfar/blob/main/CONTRIBUTING.md) +contains the complete project policy. diff --git a/docs/demos/srcnet-workshop.md b/docs/demos/srcnet-workshop.md index 002da714..4376b91e 100644 --- a/docs/demos/srcnet-workshop.md +++ b/docs/demos/srcnet-workshop.md @@ -5,7 +5,7 @@ - **Run code on the cloud** without complex setup. - Use familiar tools like **Jupyter Notebooks**. - - Scale their analysis from a single interactive session to **hundreds of parallel jobs**. + - Scale their analysis from a single interactive Session to **hundreds of parallel Sessions**. - **Process large datasets** efficiently. **Whether you're new to coding or a seasoned power-user, these tools are designed to be intuitive and powerful.** @@ -61,7 +61,8 @@ pipx install canfar ### Step 2: First Contact (Authentication) -Tell `canfar` who you are. This command discovers all available servers worldwide and guides you through a one-time login. +Tell `canfar` who you are. This command discovers the servers available to the +selected identity and guides you through a one-time login. ```bash canfar login srcnet -f @@ -70,13 +71,14 @@ canfar login srcnet -f ??? info "Auth Walkthrough" #pragma: allowlist secret -You'll be prompted for your credentials, and the CLI handles the rest, saving a secure token for future commands. +You'll be prompted for your credentials, and the CLI handles the rest, saving +an Authentication Record for future commands. !!! success "What just happened?" - We installed the `canfar` python package, which provides the `canfar` command-line interface (CLI). - We authenticated with the CANFAR Science Platform. - - All future commands will use this active Authentication and Server selection automatically. + - Future commands use the selected Authentication Record and Server. --- @@ -87,8 +89,9 @@ Let's launch a Jupyter notebook that comes pre-loaded with common astronomy libr ### Step 1: Create the Notebook ```bash -# Launch a notebook using a pre-built astronomy image -canfar create notebook skaha/astroml:latest +# See the images available on this server, then launch one +canfar image ls --kind notebook +canfar create notebook IMAGE_NAME ``` ??? "Create Notebook Walkthrough" @@ -138,13 +141,13 @@ What if you have a Python script that runs your analysis, and you don't need the Let's say you have a script named `echo.py`. ```bash -canfar create headless skaha/astroml:latest -- python echo.py +canfar create headless IMAGE_NAME -- python echo.py ``` !!! tip "Interactive to Batch, Seamlessly" You can develop your analysis interactively in a **notebook** session, save your code to a python script, and then run it at scale using a **headless** session. **No changes to your environment are needed.** -To check the output of your headless job, you can use the `logs` command. +To check the output of your headless Session, you can use the `logs` command. ```bash canfar logs @@ -154,10 +157,10 @@ canfar logs ## Scaling Up: From One to Many -Need to process hundreds of files? You can launch multiple copies (replicas) of your headless job with a single command. +Need to process hundreds of files? You can launch multiple copies (replicas) of your headless Session with a single command. ```bash -canfar create --replicas 10 headless skaha/astroml:latest -- python echo.py +canfar create --replicas 10 headless IMAGE_NAME -- python echo.py ``` You now have 10 containers in parallel. But how do you divide the work? @@ -166,7 +169,7 @@ You now have 10 containers in parallel. But how do you divide the work? ## The Python Client: Distributing Your Workload -For complex logic like distributing data across many jobs, we switch to the `canfar` Python Client. +For complex logic like distributing data across many Sessions, we switch to the `canfar` Python Client. ### The Problem @@ -187,10 +190,10 @@ all_files = glob("/arc/projects/your_project/*.fits") # 2. 'chunk' automatically gives each replica its unique subset of files # It reads environment variables ($REPLICA_ID, $REPLICA_COUNT) set by CANFAR. -my_files = distributed.chunk(all_files) +my_files = list(distributed.chunk(all_files)) # 3. Process only your assigned files -print(f"This replica will process {len(list(my_files))} files.") +print(f"This replica will process {len(my_files)} files.") for datafile in my_files: run_analysis(datafile) @@ -205,23 +208,22 @@ print("Done!") ## Putting It All Together: A Complete Workflow -Here is the complete workflow, from launching jobs programmatically to processing data in parallel. +Here is the complete workflow, from launching Sessions programmatically to processing data in parallel. -```python title="Launching Jobs Programmatically" +```python title="Launching Sessions Programmatically" from canfar.sessions import Session # This uses the same Authentication from `canfar login` -session = Session() - -# Launch 100 replicas, each running our processing script -ids = session.create( - name="galaxy-processing-batch", - kind="headless", - image="skaha/astroml:latest", - cmd="python", - args="my_script.py", - replicas=100, -) - -print(f"Successfully launched {len(ids)} processing jobs!") +with Session() as session: + # Launch 100 replicas, each running our processing script + ids = session.create( + name="galaxy-processing-batch", + kind="headless", + image="IMAGE_NAME", + cmd="python", + args="my_script.py", + replicas=100, + ) + +print(f"Successfully launched {len(ids)} processing Sessions!") ``` diff --git a/docs/index.md b/docs/index.md index 3521b990..55d95122 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,7 +14,7 @@ hide:

**Canadian Advanced Network for Astronomical Research is a scalable, cloud-native workspace for astronomy research.** - *Spin up JupyterLab, submit batch jobs, and collaborate in shared project spaces.
The CANFAR Science Platform gives researchers the tools they need with minimal setup.* + *Spin up JupyterLab, submit batch Sessions, and collaborate in shared project spaces.
The CANFAR Science Platform gives researchers the tools they need with minimal setup.*

@@ -35,7 +35,7 @@ hide: - [:simple-python: __Python API__ for access and automation](client/home.md) - [:simple-gnubash: __CLI__ for terminal users](cli/quick-start.md) - [:simple-doi: __Publications__ of DataCite DOIs](platform/doi.md) - - [:material-kubernetes: __Platform Operations__ Deployments and infrastructure](https://www.opencadc.org/deployments/) + - [:material-cog: __Platform Operations__ Deployments and infrastructure](https://www.opencadc.org/deployments/) - [:octicons-sparkles-fill-16: __Release Notes__ for the latest updates](releases/releases.md) - [:simple-rocket: __Try out__ CANFAR Science Platform](https://www.canfar.net) - [:octicons-telescope-fill-16: and much more...](platform/concepts.md) @@ -53,5 +53,3 @@ hide: The authors acknowledge the use of the Canadian Advanced Network for Astronomy Research (CANFAR) Science Platform operated by the Canadian Astronomy Data Centre (CADC) and the Digital Research Alliance of Canada, with support from the National Research Council of Canada (NRC), the Canadian Space Agency (CSA), CANARIE, and the Canadian Foundation for Innovation (CFI). - - diff --git a/docs/license.md b/docs/license.md index 64ad9b45..b7b44b7e 100644 --- a/docs/license.md +++ b/docs/license.md @@ -1 +1,10 @@ ---8<-- "LICENSE" \ No newline at end of file +--8<-- "LICENSE" +# License + +CANFAR is distributed under the [GNU Affero General Public License, version +3](https://www.gnu.org/licenses/agpl-3.0.html). The complete license text is +available in the repository's [`LICENSE`](https://github.com/opencadc/canfar/blob/main/LICENSE) +file. + +Unless a file says otherwise, contributions to this repository are licensed +under the same terms. diff --git a/docs/overrides/partials/comments.html b/docs/overrides/partials/comments.html deleted file mode 100644 index 9680248d..00000000 --- a/docs/overrides/partials/comments.html +++ /dev/null @@ -1,16 +0,0 @@ - \ No newline at end of file diff --git a/docs/platform/best-practices.md b/docs/platform/best-practices.md index 576bc813..ae21bf38 100644 --- a/docs/platform/best-practices.md +++ b/docs/platform/best-practices.md @@ -1,141 +1,103 @@ -# Best Practices for Astronomy Pipelines on CANFAR Science Platform +# Best practices for research workflows -Developing astronomy data-processing pipelines for modern, cloud-native platforms (*like CANFAR Science Platform*) requires combining solid software practices with an understanding of scalable, containerized environments. Below are some of the key best practices, aimed at students and researchers building astronomy pipelines. +Keep the science code independent from the way it is launched. Develop in an +interactive Session, validate on a small input, and run the same command in a +`headless` Session when the workflow is ready for automation. -## Writing Scalable and Batch-Friendly Code +## Make commands reproducible -- **Design for Batch Execution**: Your pipeline code should run to completion without any manual intervention. This means no GUI pop-ups, no `input()` prompts, and no reliance on interactive environments. The science platform can execute your code in a batch session that has no interactive interface. Write your scripts to take parameters (like `filepaths` or `data_ids`) and then run autonomously, logging progress as needed. +- Accept input and output paths as arguments or environment variables. +- Avoid prompts, GUI-only steps, and assumptions about the current directory in + a headless workflow. +- Record the Container Image, resource request, input identifiers, and code + revision with each run. +- Write useful progress and error messages to the Session log. +- Give replicas disjoint inputs and use `REPLICA_ID`/`REPLICA_COUNT` or the + [distributed helpers](../client/helpers.md) to partition work. -- **Use Command-Line Arguments or Environment Variables**: Never hard-code dataset paths, filenames, or other configuration inside your code. Instead, pass them in so the pipeline is flexible. For example, use CLI tools like `argparse, click, or typer` to parse an `--input` file path and `--output` directory. Alternatively, read environment variables that the platform or user sets (e.g., your code might read an `$INPUT_DATA` env var). +```python +import argparse +from pathlib import Path - ```python - import os, argparse +parser = argparse.ArgumentParser() +parser.add_argument("input", type=Path) +parser.add_argument("output", type=Path) +args = parser.parse_args() - if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--input", default=os.getenv("INPUT_FILE")) - parser.add_argument("--output_dir", default=os.getenv("OUTPUT_DIR", ".")) - args = parser.parse_args() - - infile = args.input - outfile = args.output_dir - print(f"Processing {infile} and saving to {outfile}") - ``` - - ``` - python3 example.py --input something.fits --output_dir save/something/here - ``` - -- **One Script for Interactive and Batch:** It’s helpful if the same code can run in JupyterLab (for debugging or exploration) and in batch mode (for large-scale runs). For example, if you develop your pipeline in a Jupyter notebook, you can export a Jupyter notebook to a Python script using, - - ```bash - jupyter nbconvert --to script notebook.ipynb - ``` - -## Scaling Out vs. Scaling Up - -- **Prefer Many Small Containers over One Big One**: Embrace horizontal scaling. The CANFAR Science Platform can easily run hundreds of container jobs in parallel, each on separate resources, but requesting a single container with 100× resources is impractical. For example, if you need to process 100 independent data files, it’s more efficient to run 100 containers with 1 CPU and 4 GB RAM each, than to run one container with 100 CPUs and 400 GB RAM doing it serially. The platform is optimized to handle large-scale parallel workloads for big datasets - - - *Why not one huge job?* Large containers (e.g. 32 cores, 128 GB RAM) are harder to schedule and could sit waiting in a queue. They also encourage monolithic processing that can’t be easily checkpointed. By contrast, many 1-core jobs can be scheduled as resources free up and can drastically reduce overall processing time by working concurrently. - - - *Memory is at a premium:* Splitting data into smaller chunks means each container uses less memory. This is crucial when dealing with terabyte-scale data – no single machine might have enough RAM to hold everything, but distributed processing can handle it piecewise. In practice: If each container processes, say, 50 GB of data at a time, a 100 TB dataset can be processed by 2000 such tasks in parallel. - -- **Avoid Unnecessary Resource Requests:** Don’t request more CPU or RAM than your job actually needs. Start with a modest amount (the platform’s “flexible mode” will give resources as available) to prototype and test your pipeline. Only scale up to batch jobs after understanding the your resource requirements. - -- **Utilize Parallel Libraries Cautiously:** Some Python libraries (NumPy, TensorFlow, etc.) will use multi-threading or multi-processing under the hood. Be mindful of this when running many containers to set these to the actual number of cores requested by your job. e.g. If you request 4 cores, set `OMP_NUM_THREADS=4` in your environment. - - ```python - from canfar.sessions import Session - - session = Session() - ids = session.create( - name="threaded-pipeline", - kind="headless", - image="images.canfar.net/library/pipeline:latest", - cores=4, - ram=16, - env={"OMP_NUM_THREADS": "4"}, - ) - ``` - -## Memory and I/O Efficiency - -When dealing with large astronomy datasets, how you handle memory and I/O can make or break your pipeline and can be the difference between a job that finishes in minutes and one that takes hours. - -- **Stream or Chunk Your Data:** Avoid loading extremely large datasets entirely into memory if possible. Libraries like `asttropy` can read FITS files in a memory-mapped mode (so data is loaded on-the-fly), and HDF5 (via `h5py`) allow chunked reads. If you have a 100 GB table, use `pandas` or `polars` to read it in chunks instead of `pd.read_csv` on the whole file at once. By processing data in streaming fashion, your job can handle inputs larger than RAM. - -- **Free Memory Early:** In long-running containers that processes multiple files sequentially, make sure to free resources between files. Delete large arrays or data structures after use (e.g., `del big_array`) or wrap them in functions so they go out of scope. Python’s garbage collector will reclaim memory, but you can encourage it by not holding references longer than necessary. This is important when one container processes many tasks in sequence. If each file is 5 GB in memory and you process 10 of them in one container, you need to release each one before moving to the next to stay within a 5 GB budget. - -## Saving Results - -- **Write to Persistent Storage:** Remember that container file systems are ephemeral – once a session ends, anything written to the container’s own disk (other than mounted volumes) is lost. Always direct your outputs to the mounted storage provided by the platform e.g., your home directory, project space. - -- **Unique and Descriptive Output Names:** When running many tasks in parallel, never have them all write to the same filename (like `output.fits` in the same folder). This would cause conflicts and overwrites. Instead, generate output filenames that incorporate a unique element, such as the input name or an index. - -- **Organize Outputs Predictably:** Consider writing outputs of different pipeline stages to separate directories. For instance, raw data stays in `raw/`, calibration results in `calib/`, intermediate analysis in `analysis/`, and final catalogs or plots in `results/`. This structure helps both humans and programs (like a next-stage script) to locate what they need. It’s much easier to point a plotting script at a `results/` directory of processed files than to pick through one huge directory of mixed files. - -- **Implement Checkpoints and Resume Logic:** If your pipeline can take hours or days to run, add checkpointing to save progress periodically. Break the pipeline into logical stages e.g., data reduction -> feature extraction -> modeling -> results. After each stage, output data to disk (or a database). If a later stage needs to be re-run, you don’t have to redo earlier stages. - - -## Headless Processing vs. GUI Tools - -- **Avoid GUI Tools in Batch Pipelines:** Many traditional astronomy tools (like DS9, TOPCAT, IRAF GUI, etc.) require X11 or a graphical interface. These are not suited for automated pipelines running on a cluster. In a containerized platform, there may not be an easy display available for GUI apps, and attempts to use virtual displays or X forwarding can be fragile. It’s best to use command-line or library equivalents for any analysis. +# Load args.input, perform the analysis, and write args.output. +``` -- **Separate Interactive Analysis from Batch Jobs:** If you do need to use a GUI-based tool for some part of your work (for instance, visually inspecting a subset of data or interactive data exploration), do that in a dedicated interactive session separate from the batch pipeline. The platform provides specialized sessions for this purpose. +## Request measured resources -## Managing Dependencies and Python Environments +Start with the flexible request while testing. Once a workload is understood, +request the CPU, memory, and GPU it actually needs. Oversized fixed requests +can wait for matching capacity. Set threaded libraries to the requested CPU +count: -- **Using Modern Dependency Managers:** The platform’s base images come with tools like uv, pipx, and conda pre-installed *(Soon, Work in Progress)*. Prefer these for managing Python packages: - - **uv:** a fast Python package manager that can replace pip/venv workflows. It allows you to declare dependencies inside your scripts and automatically handles virtual environments for each run - - **pipx:** for installing and running standalone CLI tools in isolated environments, ensuring they don’t pollute your main env. - - **conda/mamba:** for packages that are easier to get from conda (especially if C/C++ libs are needed). Base containers *(Soon, Work in Progress)* include conda (via mamba for speed). +```bash +canfar create \ + --cpu 4 \ + --memory 16 \ + --env OMP_NUM_THREADS=4 \ + headless IMAGE_NAME \ + -- python run.py --input /arc/projects//input.fits +``` +The platform does not promise that many small requests always outrun one large +request. Measure the workload and account for queue time, startup, data +movement, and downstream coordination. -- **Inline Dependency Declaration with uv:** One powerful pattern is to declare your script’s requirements using `PEP 723 style` metadata. For example, using `uv`, you can add a header to your Python script listing its packages. When you run this script with uv run, uv will create an isolated environment with these packages installed, so your script runs with exactly the needed libraries. This approach ensures anyone running the script (or any container executing it) gets the correct dependencies without manual setup. +## Separate interactive and batch work - ```python - # /// script.py - # dependencies = [ - # "astropy", - # "photutils>=1.9", - # "polars" - # ] - # /// - import astropy - import photutils - import polars - ``` +Use a Notebook, Desktop, CARTA, or Firefly Session for exploration and visual +inspection. Use `headless` for unattended reductions and parameter sweeps. Keep +the command-line pipeline free of display dependencies; save a small sample +for visual checks in an interactive Session. - ```bash - uv run python script.py - ``` +## Plan data movement -- **Keep Environments Reproducible:** Avoid “it works on my machine | session” issues by documenting dependencies. If you installed something interactively in Jupyter, add it to your with your manager of choice. +- Read data already under `/arc` through its normal path. +- Transfer remote objects once to `/scratch` when a path-oriented tool or + repeated reads need local files. +- Use an explicit fsspec cache under `/scratch` only when reuse justifies it. +- Write final products, checkpoints, and logs that must survive the Session to + `/arc` or a persistent VOSpace Service. +- Use unique output names when replicas run concurrently. +See [Storage](storage/index.md), [Data transfers](storage/transfers.md), and +[Filesystem and Python tools](storage/filesystem.md) for the transfer and cache +boundary. -## Container Packaging +## Keep environments reproducible -- **Group tools by logical pipeline step:** - Don't create a monolithic container with all tools for every stage or split every tool into its own micro-container. Create one container per logical pipeline step, bundling all tools needed for that step—whether they're interactive or batch-oriented. +Declare Python dependencies in the Container Image or a versioned environment +file. Test imports and command entrypoints before submitting many replicas. +If a runtime installation is necessary, record exactly what was installed and +move stable dependencies into a rebuilt image for the next run. Do not assume +that an example image tag or package is available on every deployment. -- **Reuse** the same container for both interactive testing (e.g. JupyterLab) and headless batch execution of the same code. This ensures consistency, debuggability, and minimizes surprises in batch jobs. +When building a custom image: -- **Test Locally:** Before scaling up, test your container build and functionality on a small dataset locally or in an interactive session. This ensures the environment has everything needed. +- start from a documented image available in your registry; +- pin important dependencies and base-image versions where practical; +- keep credentials and large datasets out of the image; +- remove package-manager caches and temporary build files; and +- run the command as a non-root user where the base image supports it. -#### Keep Containers Lean +See [Building Containers](containers/build.md) for the image workflow. -- **Use Official Base Images (Soon, Work in Progress):** If building your own container image for a pipeline, start with a provided base image rather than starting from scratch. For example, the `base:22.04` image is a good general starting point – and it comes with `uv, pipx, conda` pre-installed and configured. - ```dockerfile - FROM images.canfar.net/library/base:22.04 - ``` +## Checkpoint long workflows -- **Keep Images Lightweight:** Minimize what you add to the container. Uninstall unnecessary packages and avoid including large test data or docs inside the image. A smaller image pulls faster and uses less storage, benefiting batch runs. Use a .dockerignore file to exclude files like docs, tests, and git directories from the build context +For reductions that can run for hours, split the work into stages and write a +checkpoint after each stage. Make reruns safe: do not overwrite a valid output +unless the command explicitly requests it, and record which inputs completed. +Inspect `canfar events SESSION_ID` and `canfar logs SESSION_ID` when a Session +terminates unexpectedly. -- **Optimize Dockerfile Layers:** Combine related commands into single `RUN` statements and clean up after installations to reduce image size. For example, update and install Linux packages in one layer, then remove package lists and caches: +## Related guides -```dockerfile -# Combining update, install, and cleanup in one layer -RUN apt-get update && apt-get install -y \ - astrometry.net sextractor \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -``` +- [Batch processing](sessions/batch.md) +- [Sessions](sessions/index.md) +- [Containers](containers/index.md) +- [Permissions](permissions.md) +- [Support](support/index.md) diff --git a/docs/platform/cloud.md b/docs/platform/cloud.md index b9430e21..a1e64693 100644 --- a/docs/platform/cloud.md +++ b/docs/platform/cloud.md @@ -1,422 +1,30 @@ -# Legacy Cloud Platform (OpenStack VMs) +# Legacy cloud workflows -!!! warning "Legacy Platform" - This documentation covers the legacy CANFAR cloud platform based on OpenStack virtual machines. For new users, we recommend the modern [CANFAR Science Platform](get-started.md) which provides container-based sessions, improved workflows, and better resource management. +This page is retained as a signpost for deployments that still operate the +older OpenStack-based CANFAR Cloud workflow. It is not the recommended path for +new research workloads. The current user-facing model is the [CANFAR Science +Platform](index.md): launch a Session from a Container Image and use mounted +storage or configured VOSpace Services. -!!! abstract "🎯 What You'll Learn" - - How to access legacy CANFAR cloud services via Digital Research Alliance Canada OpenStack - - Differences between Digital Research Alliance Canada OpenStack cloud and modern CANFAR platform access - - VM management and batch processing workflows - - Migration strategies to the modern platform +## If you already have a legacy VM -The legacy CANFAR cloud services are hosted on the Digital Research Alliance Canada [OpenStack](https://www.openstack.org/) infrastructure. This platform provides traditional virtual machines for users who require persistent compute environments or have specific legacy workflow requirements. +Follow the instructions supplied by the operator of that deployment for VM +creation, network access, firewall rules, quotas, and image selection. Those +values are deployment configuration and are not defined by the Python client +or CLI documentation in this repository. -## 🔄 Migration to Modern Platform +Use the legacy VM only for work that depends on its existing environment. Keep +new scripts and data in a supported persistent location, and do not assume that +a VM path is mounted in a Science Platform Session. -Before proceeding with OpenStack VMs, consider whether the [modern CANFAR Science Platform](index.md) meets your needs: +## Moving a workflow to Sessions -### Modern Platform Advantages +1. Put the command and its dependencies in a [Container Image](containers/index.md). +2. Store persistent inputs and outputs under `/arc` or a VOSpace Service. +3. Use a Notebook or Desktop Session for exploration and a [headless Session](sessions/batch.md) + for unattended commands. +4. Stage remote files to `/scratch` when a path-oriented program needs a local + file, then copy results to persistent storage. -- **Container-based sessions**: Faster startup, better resource utilisation -- **Browser-native access**: No SSH or complex networking required -- **Automated resource management**: Dynamic scaling and optimized allocation -- **Integrated storage**: Seamless access to shared `/arc/` -- **Pre-configured environments**: Ready-to-use astronomy software stacks - -### When to Use Legacy Platform - -- **Persistent services**: Long-running applications that need to stay active -- **Custom system configurations**: Root access requirements -- **Legacy workflows**: Existing scripts and pipelines that require VMs -- **Old VM Batch processing**: Large-scale automated job processing which was installed on a VM - -## 🔑 Access and Authentication - -### Key Differences from Digital Research Alliance Canada Defaults - -- **Credentials**: Sign in with **CADC Username/Password** (not a Digital Research Alliance Canada account) -- **Portal**: Use the [arbutus-canfar portal](https://arbutus-canfar.cloud.computecanada.ca/) (instead of [arbutus](https://arbutus.cloud.computecanada.ca/)) -- **Resource policy**: Interactive analysis gets reasonable quotas; **batch processing** can scale to large footprints - -### Registration & Allocation - -A CADC account is required to access cloud services. - -1. **Register a CADC account** (if you don't have one) -2. **Email CANFAR support** with: - - Project Name - - CADC Account Username - - Estimated resources (storage, compute; whether you need batch) - - A short description of your use case (2–3 sentences) - -CANFAR will review and coordinate project/quotas on the Digital Research Alliance Canada side. - -## 🖥️ Virtual Machine Management - -### Creating and Configuring VMs - -#### 1. Create a VM - -Use the Digital Research Alliance Canada web dashboard: - -1. Sign in to [Dashboard](https://arbutus-canfar.cloud.computecanada.ca/) with CADC username/password -2. Each CANFAR allocation maps to an OpenStack **Project**. Use the top-left project picker to switch if you belong to multiple -3. Follow Digital Research Alliance Canada's [Creating a Linux VM](https://docs.alliancecan.ca/wiki/Creating_a_Linux_VM) documentation - -#### 2. Import an SSH Public Key - -OpenStack prefers SSH key pairs over passwords: - -- If you do not have a key pair, run `ssh-keygen` locally or follow Digital Research Alliance Canada's SSH Keys documentation -- In **Compute → Key Pairs**, click **Import Key Pair** -- Name the key and paste your public key (default path `~/.ssh/id_rsa.pub`) - -#### 3. Allocate a Public IP - -- Go to **Network → Floating IPs** -- If none is listed, click **Allocate IP to Project** -- Typically, each project has one public IP; if exhausted you'll see _Quota Exceeded_ - -#### 4. Launch an Instance - -- In **Compute → Instances → Launch Instance**, choose: - - **Source**: _canfar-ubuntu-20.04_ (important for batch compatibility) - - **Flavour**: e.g., `c2-7.5gb-30` (2 vCPU / 7.5 GiB RAM / ~31 GiB ephemeral disk) - - **Key Pair**: select your SSH key -- Click **Launch** - -#### 5. Connect to the Instance - -After status becomes **Running**: - -1. **Associate floating IP** (menu → **Associate Floating IP**) -2. **SSH to the instance**: - - ```bash - ssh ubuntu@[floating_ip] - ``` - -3. **Create a local user** matching your CADC account: - - ```bash - sudo canfar_create_user [user] - ``` - -## 🔧 VM Configuration and Tools - -### Pre-built VM Helpers - -The `canfar-ubuntu-20.04` and `canfar-rocky-8` images include helpful tools: - -```bash -# Obtain a CADC proxy (legacy helper) -cadc_cert -u [user] - -# Create/update ~/.netrc for CADC services -cadc_dotnetrc - -# Set up /mnt/scratch for temporary storage -canfar_setup_scratch - -# Create a local user and grant sudo access -canfar_create_user [user] - -# Update CANFAR scripts and CADC clients -canfar_update -``` - -### System Maintenance - -Keep your VM updated and secure: - -```bash -# Ubuntu/Debian systems -sudo apt update && sudo apt dist-upgrade - -# Rocky/CentOS systems -sudo dnf update -``` - -## 🚀 Batch Processing Workflow - -### Setting Up Batch Processing - -This tutorial demonstrates building a basic **source detection** pipeline for CFHT MegaCam images on a CANFAR VM with fast access to the CADC archive and VOSpace. - -!!! note "You will learn to" - - Create/manage VMs on Digital Research Alliance Canada OpenStack / CANFAR - - Access CADC VOSpace from VMs - - Submit batch jobs that run your pipeline - -### 1. Create a VM - -Use the Digital Research Alliance Canada web dashboard. - -1. Sign in to [Dashboard](https://arbutus-canfar.cloud.computecanada.ca/) with CADC `[user]`/password. -2. Each CANFAR allocation maps to an OpenStack **`[project]`**. Use the top-left project picker to switch if you belong to multiple. - -Follow Digital Research Alliance Canada's [Creating a Linux VM](https://docs.alliancecan.ca/wiki/Creating_a_Linux_VM). Summary below. - -### 2. Import an SSH Public Key - -OpenStack prefers SSH key pairs over passwords. - -- If you do not have a key pair, run `ssh-keygen` locally or follow Digital Research Alliance Canada's SSH Keys documentation. -- In **Compute → Key Pairs**, click **Import Key Pair**. -- Name the key and paste your public key (default path `~/.ssh/id_rsa.pub`). - -### 3. Allocate a Public IP - -- Go to **Network → Floating IPs**. -- If none is listed, click **Allocate IP to Project**. -- Typically, each project has one public IP; if exhausted you'll see _Quota Exceeded_. - -### 4. Launch an Instance - -- In **Compute → Instances → Launch Instance**, choose: - - **Source**: _canfar-ubuntu-20.04_ (important for batch) - - **Flavour**: e.g., `c2-7.5gb-30` (2 vCPU / 7.5 GiB RAM / ~31 GiB ephemeral disk) - - **Key Pair**: select your SSH key -- Click **Launch**. - -### 5. Connect to the Instance - -After status becomes **Running**, first **associate** the floating IP (menu → **Associate Floating IP**), then SSH: - -```bash -ssh ubuntu@[floating_ip] -``` - -Create a local user matching your CADC account (for audit/minimal access): - -```bash -sudo canfar_create_user [user] -logout -ssh [user]@[floating_ip] -``` - -!!! note "Default image users" - - Ubuntu images: `ubuntu` - - Rocky Linux images: `rocky` - -### Install software - -The base VM image comes with only a minimal set of packages. -For this example, we need to install two additional tools: - -- [Source Extractor](https://sextractor.readthedocs.io/) (source detection): software used to detect astronomical sources in FITS images, producing catalogues of stars and galaxies. -- [funpack](https://heasarc.gsfc.nasa.gov/fitsio/fpack/) (FITS decompressor; Ubuntu package `libcfitsio-bin`): a decompression utility for FITS images. Most FITS images provided by CADC are Rice-compressed and stored with an `.fz` extension. Since Source Extractor only accepts uncompressed images, we will use `funpack` to uncompress them. The `funpack` executable is distributed as part of the `libcfitsio-bin` package in Debian/Ubuntu. - -Because both tools are available from the Ubuntu software repository, we can install them system-wide after updating the package index: - -```bash title="Install packages" -sudo apt update -y -sudo apt install -y source-extractor libcfitsio-bin -``` - -### Test on the VM - -Use the ephemeral disk (mounted at `/mnt`) for scratch. - -```bash -sudo canfar_setup_scratch # create /mnt/scratch with proper permissions -cd /mnt/scratch -cp /usr/share/source-extractor/default* . -cat > default.param <<'EOF' -NUMBER -MAG_AUTO -X_IMAGE -Y_IMAGE -EOF -cadcget cadc:CFHT/1056213p.fits.fz -funpack -D 1056213p.fits.fz -source-extractor 1056213p.fits -CATALOG_NAME 1056213p.cat -``` - -!!! warning "Scratch space" - - Run `canfar_setup_scratch` each time you boot a **new** instance. - - In **batch mode**, each job gets its own scratch directory (not `/mnt/scratch`). - -### Persist results to VOSpace - -Ephemeral storage is wiped when the VM terminates. Upload the output `1056213p.cat` to **VOSpace** (the VM includes the `vos` client). - -Obtain a proxy certificate for automated access: - -```bash -cadc_dotnetrc # one-time helper to create ~/.netrc -cadc-get-cert -n # generate an X509 proxy (default 10 days) -``` - -```bash -vcp 1056213p.cat vos:[project]/ -``` - -!!! danger "Credential hygiene" - `.netrc` stores credentials in plaintext. Use only on controlled hosts and restrict permissions: `chmod 600 ~/.netrc`. - -### Snapshot the instance - -In the **Instances** view, click **Create Snapshot** (e.g., name it `image-reduction-2023-08-21`). - -!!! warning - Avoid writes on the VM while a snapshot is being created. - -Without a snapshot, **ephemeral** data is lost when the instance is deleted. **Volume-backed** VMs persist data but are **not suitable for batch**. - -### Automate as a batch script - -CANFAR batch is powered by **[HTCondor](https://htcondor.org/)**; Cloud Scheduler launches worker VMs on demand. - -Create `~/do_catalogue.bash`: - -```bash -#!/usr/bin/env bash -set -euo pipefail -id="$1" -cadcget "cadc:CFHT/${id}.fits.fz" -funpack -D "${id}.fits.fz" -cp /usr/share/source-extractor/default* . -cat > default.param <<'EOF' -NUMBER -MAG_AUTO -X_IMAGE -Y_IMAGE -EOF -source-extractor "${id}.fits" -CATALOG_NAME "${id}.cat" -vcp "${id}.cat" "vos:[project]/" -``` - -### Write a submission file - -Submit four image IDs: `1056215p 1056216p 1056217p 1056218p`. - -```text title="do_catalogue.sub" -executable = do_catalogue.bash - -output = do_catalogue-$(arguments).out -error = do_catalogue-$(arguments).err -log = do_catalogue-$(arguments).log - -queue arguments from ( - 1056215p - 1056216p - 1056217p - 1056218p -) -``` - -### Submit jobs - -Two authorizations are needed: - -- Access to snapshots in `[project]` -- Write access to `vos:[project]` - -On the batch login node `batch.canfar.net`: - -```bash -ssh [user]@batch.canfar.net -. [project]-openrc.sh # set OpenStack env (once per session) -``` - -Submit: - -```bash -canfar_submit do_catalogue.sub image-reduction-2023-08-21 c2-7.5gb-30 -``` - -Where: - -- `do_catalogue.sub`: submission file -- `image-reduction-2023-08-21`: snapshot image name -- `c2-7.5gb-30`: VM flavour (list via `openstack flavor list`) - -Monitor: - -```bash -condor_q -condor_q -all # all users summary -``` - -When the interactive VM is no longer needed, delete it from the dashboard (**Delete Instances**). - -## Extras: Helpful Commands & VM Maintenance - -**Keep the OS updated:** - -```bash -# Ubuntu/Debian systems -sudo apt update && sudo apt dist-upgrade - -# Rocky/CentOS systems -sudo dnf update -``` - -**Prebuilt VM helpers** (`canfar-ubuntu-20.04` / `canfar-rocky-8`): - -- `cadc_cert -u [user]`: obtain a CADC proxy (legacy helper) -- `cadc_dotnetrc`: create/update `~/.netrc` -- `canfar_setup_scratch`: set up `/mnt/scratch` -- `canfar_create_user [user]`: create a local user and grant sudo -- `canfar_update`: update CANFAR scripts and CADC clients - -### Migration Strategies - -#### Option 1: Containerise Your Workflow - -Convert your VM-based pipeline to containers: - -1. **Create a Dockerfile** based on your VM configuration -2. **Test the container** on the modern platform -3. **Submit container-based jobs** instead of VM jobs - -#### Option 2: Hybrid Approach - -Use both platforms as appropriate: - -- **Development**: Modern platform for interactive analysis -- **Production**: VM batch jobs for large-scale processing -- **Data sharing**: Common storage accessible from both - -#### Option 3: Gradual Migration - -Migrate components incrementally: - -1. **Start with interactive work** on the modern platform -2. **Keep batch processing** on VMs initially -3. **Gradually containerise** pipeline components -4. **Complete migration** when ready - -## 🔗 Migration Resources - -### Modern Platform Documentation - -- **[CANFAR Science Platform →](index.md)**: Overview of modern container-based platform -- **[Interactive Sessions →](sessions/index.md)**: Browser-based computing environments -- **[Container Usage →](containers/index.md)**: Working with pre-built and custom containers -- **[Batch Jobs →](sessions/batch.md)**: Modern batch processing workflows - -### Support and Migration Assistance - -- **Email**: [support@canfar.net](mailto:support@canfar.net) for migration planning -- **Documentation**: Platform comparison and migration guides -- **Consultation**: Schedule time to discuss your specific use case - -## 📋 Platform Comparison - -| Feature | Legacy OpenStack VMs | Modern CANFAR Platform | -|---------|---------------------|----------------------| -| **Access Method** | SSH, web dashboard | Browser-based interface | -| **Startup Time** | 5-10 minutes | 30-60 seconds | -| **Resource Management** | Manual VM sizing | Dynamic allocation | -| **Software Installation** | Manual setup required | Pre-configured containers | -| **Collaboration** | Shared VM access | Session sharing, unified storage | -| **Maintenance** | User responsibility | Platform managed | -| **Best For** | Persistent services, custom configs | Interactive analysis, quick workflows | - -!!! tip "Choosing the Right Platform" - - **New users**: Start with the [modern CANFAR platform](index.md) - - **Existing VM users**: Consider migration for better efficiency - - **Persistent services**: Continue using VMs where appropriate - - **Hybrid workflows**: Use both platforms as needed +For deployment and operator documentation, see [OpenCADC Deployments](https://www.opencadc.org/deployments/). +For help choosing a supported workflow, contact [CANFAR support](support/index.md). diff --git a/docs/platform/community/CASA_and_more.md b/docs/platform/community/CASA_and_more.md index 55a066a3..4efb0bf1 100644 --- a/docs/platform/community/CASA_and_more.md +++ b/docs/platform/community/CASA_and_more.md @@ -1,60 +1,33 @@ -# CASA containers and adjacent software +# CASA and adjacent astronomy software -This page contains a summary of additional packages included in CASA containers, known issues and work-arounds for specific containers, as well as a brief summary on other radio astronomy tools available. +CASA and related radio-astronomy tools are supplied through selected Container +Images. Availability and version-specific behavior belong to the image you +choose; check `canfar image ls` and the image documentation before writing a +workflow that depends on a package. -## CASA add-on tools and packages +## Choose and inspect an image -### Astroquery / Astropy -The [astroquery tool](https://astroquery.readthedocs.io/en/latest/) is presently only installed on newer CASA containers (6.4.4 and above). To use astroquery from an appropriate CASA container, type the following to initiate an astroquery-compatible version of python: ```bash -/opt/casa/bin/python3 +canfar image ls --kind desktop +canfar create desktop IMAGE_NAME --name casa-work ``` -As per the [astroquery documentation](https://astroquery.readthedocs.io/en/latest/), the tool can then be used on the command line within the python environment. For example, the following sequence of commands yield a one-line table listing some basic information about M1. -```python -from astroquery.simbad import Simbad -result_table = Simbad.query_object("m1") -result_table.pprint() -``` - -### Analysis Utilities -The [analysisUtils package](https://casaguides.nrao.edu/index.php/Analysis_Utilities) package is pre-installed on every CASA container, and is ready to use. You may need to type the following to load the package: - -```python -import analysisUtils as au -``` - -### Firefox -The Firefox web-browser, needed for CASA commands where you are interacting with the weblogs, should available for CASA versions 6.1.0 to 6.4.3. Error messages will pop up in your terminal window, but minimal testing suggests that it is sufficiently functional. - -### UVMultiFit -The [UVMultiFit](https://github.com/onsala-space-observatory/UVMultiFit/blob/master/INSTALL.md) package is presently installed and working for all CASA 5.X versions except 5.8. To load the UVMultiFit package, initiate casa and then type - -```python -from NordicARC import uvmultifit as uvm -``` -## Known Issues - -1. CASA versions `6.5.0` to `6.5.2` initially launch with some display errors in the logger window. Exiting casa (but not the container) and re-starting casa fixes the issue, i.e., - - ```bash - casa - exit - casa - ``` - -2. Running multi-thread pipeline scripts (MPI CASA) may generate error messages, as described [here](https://casadocs.readthedocs.io/en/latest/notebooks/frequently-asked-questions.html) under the 'Running pipeline in non-interactive mode' section. A CANFAR ALMA user reports success initiating MPI CASA in a Desktop container as follows: - - ```bash - xvfb-run -a mpicasa casa —nologger —nogui -agg -c casa_script.py - ``` - -## CASA Adjacent Containers +Inside the running Session, check the installed CASA version and invoke the +command documented for that release. Keep the reduction script and measurement +sets under `/arc` or copy them from `/scratch` before the Session ends. -### Galario -The UV data analysis package [galario](https://mtazzari.github.io/galario) is available under the radio-submm menu. Note that this container has had minimal testing, and the uvplot package commands in the quickstart.py script are not presently working, although all preceeding commands in the quickstart.py script do work. +For package-specific usage, use the upstream documentation: -### Starlink -The JCMT's [Starlink](https://starlink.eao.hawaii.edu/starlink) package is available under the radio-submm menu, including image analysis tools and the gaia image viewer. Note that the [starlink-pywrapper](https://starlink-pywrapper.readthedocs.io/en/latest/) add-on package is presently not working. Minimal testing has been done on the Starlink container. +- [CASA documentation](https://casadocs.readthedocs.io/) +- [Astropy](https://docs.astropy.org/) +- [Astroquery](https://astroquery.readthedocs.io/) +- [Analysis Utilities](https://casaguides.nrao.edu/index.php/Analysis_Utilities) +- [UVMultiFit](https://github.com/onsala-space-observatory/UVMultiFit) +- [Galario](https://mtazzari.github.io/galario/) +- [Starlink](https://starlink.eao.hawaii.edu/starlink/) +If an application is missing or behaves differently from its upstream +documentation, record the Container Image name and version, the Session Kind, +the command, and the error before contacting [CANFAR support](../support/index.md). +For an end-to-end ALMA example, see the [ALMA analysis workflow](alma/index.md). diff --git a/docs/platform/community/alma/ALMA_Desktop/archive_download.md b/docs/platform/community/alma/ALMA_Desktop/archive_download.md deleted file mode 100644 index 021b1ce9..00000000 --- a/docs/platform/community/alma/ALMA_Desktop/archive_download.md +++ /dev/null @@ -1,4 +0,0 @@ -# Downloading from the ALMA archive (web) - -Instructions for downloading ALMA data via the web interface. For large downloads prefer scripted methods or transfer tools. - diff --git a/docs/platform/community/alma/ALMA_Desktop/archive_script_download.md b/docs/platform/community/alma/ALMA_Desktop/archive_script_download.md deleted file mode 100644 index b32dc550..00000000 --- a/docs/platform/community/alma/ALMA_Desktop/archive_script_download.md +++ /dev/null @@ -1,14 +0,0 @@ -# Using scripts to download ALMA archive data - -How to download ALMA archive data in bulk using URL lists and command-line tools securely. - -Use URL lists and command-line tools (wget, curl) or transfer tools for bulk downloads; ensure credentials are handled securely. - -For large datasets obtain an URL list from the archive and use `wget` with the certificate you fetched with `cadc-get-cert`: - -```sh -cadc-get-cert -u [username] -wget --content-disposition -i url_list.txt --certificate ~/.ssl/cadcproxy.pem --ca-certificate ~/.ssl/cadcproxy.pem -``` - -Alternatively use `vcp` to transfer directly into a VOSpace location. diff --git a/docs/platform/community/alma/ALMA_Desktop/casa_containers.md b/docs/platform/community/alma/ALMA_Desktop/casa_containers.md deleted file mode 100644 index 39d06db0..00000000 --- a/docs/platform/community/alma/ALMA_Desktop/casa_containers.md +++ /dev/null @@ -1,44 +0,0 @@ -# CASA container images - -Notes about CASA container images available in Desktop sessions and compatibility considerations. - -- Container images may differ by CASA major/minor version. -- Some CASA tasks rely on external system libraries; these are usually bundled in the container but check the release notes if you see missing symbols. - -If you need GPU acceleration or special libraries, consult the container documentation for appropriate tags and runtime flags. - -## Astroquery / astropy - -The `astroquery` tool is installed on newer CASA containers (6.4.4-6.6.3). To use astroquery from the CASA Python: - -```py -from astroquery.simbad import Simbad -result_table = Simbad.query_object("m1") -result_table.pprint() -``` - -## Analysis Utilities - -The `analysisUtils` package is pre-installed on many CASA containers. You may need to run `import analysisUtils as au` to load it. - -## ADMIT - -ADMIT (ALMA Data Mining Tool) is available on some CASA containers (typically CASA >= 4.5); newer containers may exclude it. - -## Known Container Notes - -- Firefox is available on some CASA versions for minimal web-browser interaction. - -## Example: restarting CASA (known issue workaround) - -```sh -casa -exit -casa -``` - -## Example: run CASA with MPI and Xvfb (non-interactive) - -```sh -xvfb-run -a mpicasa casa --nologger --nogui -agg -c casa_script.py -``` diff --git a/docs/platform/community/alma/ALMA_Desktop/start_casa.md b/docs/platform/community/alma/ALMA_Desktop/start_casa.md deleted file mode 100644 index e5fc85f1..00000000 --- a/docs/platform/community/alma/ALMA_Desktop/start_casa.md +++ /dev/null @@ -1,34 +0,0 @@ -# Starting CASA in a Desktop session - -How to launch CASA inside a Desktop session and run reduction or imaging scripts. - -CASA (Common Astronomy Software Applications) is typically provided as a container in the Desktop session. To start CASA: - -1. Launch a Desktop session and open a terminal. -2. Start the CASA container. Depending on the configuration the command may be as simple as: - -```sh -casa -``` - -3. If you have a reduction script (e.g., `scriptForPI.py`) you can run it within CASA: - -```py -execfile('scriptForPI.py') -``` - -If your dataset requires a specific CASA version, choose the container image for that version. -the scripts distributed with ALMA Cycle 0 data on the archive). -to use a non-CASA terminal for all regular linux uses. - -Once you have launched a Desktop session it is straightforward to run CASA in a terminal. - -![image](../../../sessions/images/desktop/1_launch_desktop.png) - -To start a CASA-enabled terminal, click the `Applications` menu at the top-left of the screen and choose the desired CASA version from the `AstroSoftware` menu. - -Select the CASA version you want. All versions back to CASA 3.4.0 are available; choose the one appropriate for your scripts. - -Clicking a CASA version opens a terminal where you can start CASA with `casa` or `casa --pipeline` (two dashes before `pipeline`). - -You can open a regular (non-CASA) terminal by double-clicking the `terminal` icon. CASA terminals accept a limited set of commands; use the non-CASA terminal for general Linux work. diff --git a/docs/platform/community/alma/ALMA_Desktop/typical_reduction.md b/docs/platform/community/alma/ALMA_Desktop/typical_reduction.md deleted file mode 100644 index 15ec1547..00000000 --- a/docs/platform/community/alma/ALMA_Desktop/typical_reduction.md +++ /dev/null @@ -1,28 +0,0 @@ -# Typical ALMA data reduction workflow - -A concise overview of a typical reduction and imaging workflow using CASA in Desktop sessions. - -First, download your ALMA data onto your Desktop Session (see the archive download tutorials). If you already have the data locally, use one of the file transfer methods to move it into your session. - -Next, open a CASA container (see the Start CASA tutorial for the correct version). Start CASA in interactive or pipeline mode depending on your script: - -```sh -casa -casa --pipeline -``` - -Inside CASA run the reduction script (commonly named `scriptForPI.py`): - -```py -execfile('scriptForPI.py') -``` - -After the reduction finishes you will find calibrated measurement sets in a `calibrated/` directory. The `scriptForImaging.py` script (if provided) can be used to create images and is often run from the `calibrated/` directory. - -When analysis is complete, transfer final files off the system using one of the transfer options (VOSpace, web download, vcp, etc.). Example using `vcp`: - -```sh -vcp calibrated_final_cont_image_162622-24225.* vos:helenkirk/ -``` - -Note: use VOSpace for long-term storage; the Science Portal sessions are not intended as persistent archival storage. diff --git a/docs/platform/community/alma/General_tools/File_transfers.md b/docs/platform/community/alma/General_tools/File_transfers.md deleted file mode 100644 index eaf999de..00000000 --- a/docs/platform/community/alma/General_tools/File_transfers.md +++ /dev/null @@ -1,19 +0,0 @@ -# File transfer overview - -Summary of available file transfer methods: Notebook upload, web storage, VOS Tools, SSHFS, and direct URLs. - -There are several ways to move files into and out of the Science Portal. Common options include: - -- Notebook upload: small files can be uploaded via the Notebook file browser (see the Notebook transfer tutorial). -- Web storage: use the web interface to upload/download and manage files. -- VOS Tools: command-line tools (`vcp`, `vls`, `vrm`) for copying files to/from VOSpace and Science Portal locations. -- SSHFS: mount the remote file system locally and use rsync or other tools to sync files. -- Direct URL: obtain a direct ARC URL list and use `wget` with the appropriate certificate to download files. - -See the individual tutorials for step-by-step instructions: - -- Notebook uploads: /science-containers/general/Notebook/transfer_file -- Web storage: /science-containers/general/General_tools/Using_webstorage -- VOS Tools: /science-containers/general/General_tools/Using_vostools -- SSHFS: /science-containers/general/General_tools/Using_sshfs -- Direct URL downloads: /science-containers/general/TipsTricks/Direct_url diff --git a/docs/platform/community/alma/General_tools/Group_management.md b/docs/platform/community/alma/General_tools/Group_management.md deleted file mode 100644 index 77387f8d..00000000 --- a/docs/platform/community/alma/General_tools/Group_management.md +++ /dev/null @@ -1,7 +0,0 @@ -# Group management - -How to create and manage project groups and permissions via the Science Portal web UI. - -Create and manage groups via the CANFAR web UI. Use the edit membership controls to add users and administrators. For complex setups, group permissions can also be managed via command-line tools; see the CANFAR docs for details. - -For automated or complex permission changes, prefer the command-line tools described in the CANFAR documentation. diff --git a/docs/platform/community/alma/General_tools/Using_sshfs.md b/docs/platform/community/alma/General_tools/Using_sshfs.md deleted file mode 100644 index 180a36c6..00000000 --- a/docs/platform/community/alma/General_tools/Using_sshfs.md +++ /dev/null @@ -1,63 +0,0 @@ -# Using SSHFS - -Mount the remote Science Platform file system locally with `sshfs` to access files directly from your machine. - -## Installation - -Software installation is required to use this tool. - -*Linux*: SSHFS is Linux-based software that needs to be installed on your local computer. On Ubuntu and Debian based systems, it can be installed through apt-get: - - sudo apt-get install sshfs - -*Mac OSX*: Often SSHFS is already installed; if not, you will need to download FUSE and SSHFS from the [osxfuse site](https://osxfuse.github.io) - -## Prepare your Arc account - -A public SSH key will need to be installed into your Arc home directory. Ensure you have a `.ssh` folder in your `/home/[your_cadc_username]` folder. You can do this through the UI: https://www.canfar.net/storage/arc/list/home - -1. Ensure you are logged in using the pulldown in the top right menu. - -2. Visit your `Home` folder. - -3. If there is no `.ssh` folder listed, create one. - -4. Ensure you have a file called `authorized_keys` with your SSH public key in it. This public key should match whichever private key you are using to authenticate with. For example, if your private key on your local machine is `${HOME}/.ssh/id_rsa`, then your public key is likely `${HOME}/.ssh/id_rsa.pub`. - -## Mount the Remote File System - -For Ubuntu/Debian Linux or Mac OSX, the instructions are below. - -To start, we will need to create a local directory in which to mount the file system, "arc": - - mkdir $HOME/arc - -Now we can mount the file system locally using the following command, based on which OS you are running. You will be asked for your CADC password during this step. - -*On Ubuntu/Debian*: - - sshfs -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=10 -p 64022 [your_cadc_username]@ws-uv.canfar.net:/ $HOME/arc - -*On Mac OSX*: - - sshfs -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=10,defer_permissions -p 64022 [your_cadc_username]@ws-uv.canfar.net:/ $HOME/arc - -The extra `defer_permissions` switch works around issues with OSX permission handling. - -## Synch Local and Remote Directories with rsync - -With the steps above in place, the rsync ("remote synch") command can be used. rsync uses an algorithm that minimizes the amount of data copied by only moving the portions of files that have changed. - -The synch is performed using the following: - - rsync -vrltP source_dir $HOME/arc/destination_dir/ - -Pro tip: including a `/` after source_dir in the command above will transfer the directory contents without the main directory itself. - -## Unmounting the File System - -If you have finished working with your files and want to disconnect from the remote file system, you can do this by: - - umount $HOME/arc - -NB: If you run into problems with the original sshfs command and need to run it again, you will likely need to unmount first. diff --git a/docs/platform/community/alma/General_tools/Using_vostools.md b/docs/platform/community/alma/General_tools/Using_vostools.md deleted file mode 100644 index 2f66769b..00000000 --- a/docs/platform/community/alma/General_tools/Using_vostools.md +++ /dev/null @@ -1,67 +0,0 @@ -# Using VOS Tools - -Examples for using VOSpace command-line tools (vcp, vls, vrm) and notes on authentication and certificates. - -VOS Tools provide command-line access to CANFAR VOSpace and the Science Portal storage. - -Install instructions and details are available on CANFAR's storage documentation (see CANFAR storage docs). - -Typical commands: - -```sh -vcp localfile arc:home/[username] -vcp vos:[username]/remotefile ./ -vls vos:[username] -vrm vos:[username]/file -``` - -If you see an expired certificate error, update it with: - -```sh -cadc-get-cert -u [username] -``` - -The most efficient way to transfer files in and out of CANFAR's Science -Portal is to use the VOS Tools, which are also used for interacting with CANFAR's VOSpace. - -Instructions for installing VOS Tools on your personal computer are -located in CANFAR's storage documentation under the section "The vos Python module and command line client". - -Instructions on how to use this tool, including some basic examples, are -found on the same webpage. In brief, this tool runs on the command line -with syntax similar to the linux `scp` command. File locations within -CANFAR systems are specified with *vos* for VOSpace and *arc* for the -Science Portal. For example, to copy a file from your personal computer -to your home directory in the Science Portal, you would type the -following on your local computer: - -```sh -vcp myfile.txt arc:home/[username] -``` - -To copy a file from VOSpace to your personal computer, you would use: - -```sh -vcp vos:[username]/myfile.txt ./ -``` - -To copy files from the Science Portal to VOSpace, you would similarly -use the command: - -```sh -vcp myfile.txt vos:[username] -``` - -Note that VOS Tools use a security certificate which needs to be updated periodically. If you get an error message stating: - -```sh -ERROR:: Expired cert. -``` - -Update by running: - -```sh -cadc-get-cert -u [username] -``` - -and enter your password for CADC/CANFAR services at the prompt. diff --git a/docs/platform/community/alma/General_tools/Using_webstorage.md b/docs/platform/community/alma/General_tools/Using_webstorage.md deleted file mode 100644 index d55c462f..00000000 --- a/docs/platform/community/alma/General_tools/Using_webstorage.md +++ /dev/null @@ -1,31 +0,0 @@ -# Using web storage - -Using the web UI to upload, download, and manage files in VOSpace or project storage; use URL lists for scripting. - -## Upload File(s) - -To upload one or more files (or folders), navigate to the desired directory, then click the `Add` button along the top, selecting the appropriate option. Follow the instructions on the pop-up box that appears to choose and upload your files. - -## Download Files - -Downloading files is also straightforward, and three options are outlined here: `URL List`, `HTML List`, and `Zip`. The `Zip` option will usually be the most practical, but the `HTML List` option may be preferred when downloading only a few files, and `URL List` may be best for scripting. - -### Download - URL List Option - -First, choose the `URL List` option, then select the desired directory and file name and click `save`. - -If the file(s) is/are not publicly available, update your security certificates by running: - - cadc-get-cert -u [username] - -Then download the files using `wget` with the provided URL list and certificates: - - wget --content-disposition -i cadcUrlList.txt --certificate ~/.ssl/cadcproxy.pem --ca-certificate ~/.ssl/cadcproxy.pem - -### Download - HTML List Option - -Clicking the `HTML List` option will bring up a pop up window with a series of long URL strings - each entry is a clickable direct link to your individual files. - -### Download - Zip Option - -The `Zip` option allows you to download a single zip file containing all of your requested files. Choose the `zip` option, and click `save` in the pop-up window after adjusting your preferred directory and zip file name. diff --git a/docs/platform/community/alma/NewUser/LaunchCARTA.md b/docs/platform/community/alma/NewUser/LaunchCARTA.md deleted file mode 100644 index 0ccd0ef2..00000000 --- a/docs/platform/community/alma/NewUser/LaunchCARTA.md +++ /dev/null @@ -1,5 +0,0 @@ -# Launching CARTA - -How to open CARTA in a Desktop session to visualize spectral cubes and images. - -In the Desktop session choose CARTA from the application menu or launch from a terminal if available. Use File -> Open to select files from your project space or VOSpace. diff --git a/docs/platform/community/alma/NewUser/LaunchDesktop.md b/docs/platform/community/alma/NewUser/LaunchDesktop.md deleted file mode 100644 index ac36df57..00000000 --- a/docs/platform/community/alma/NewUser/LaunchDesktop.md +++ /dev/null @@ -1,33 +0,0 @@ -# Launching a Desktop session - -Steps to start a Desktop session from the Science Portal web UI and connect to it. - -To start a Desktop session use the Science Portal web UI. Choose the desired container image and allocate resources (CPU, memory). When the desktop is ready connect using the provided browser window. - -After logging in to the Science Portal and clicking the plus sign to -launch a new session, choose a session type of `desktop`. - -> ![image](../../../sessions/images/desktop/1_launch_desktop.png) - -Note that the remaining menu bars and options update automatically after -your session type selection. There is currently only one option for `container image`, so no selection is needed. - -Give your session a descriptive name; this will later appear on your Science Portal page if you need to log in again later - -> ![image](../../../sessions/images/desktop/3_choose_name.png) - -Now, hit the launch button and wait for your session to launch - -> ![image](../../../sessions/images/desktop/4_launch.png) - -Your Desktop session now appears on the main Science Portal page as an icon with your chosen descriptive name. - -> ![image](../../../sessions/images/desktop/5_active_desktop.png) - -This takes you to the landing page for your Desktop session. Click the connect button to connect to the session. - -> ![image](../../../sessions/images/desktop/6_connect_desktop.png) - -When your session becomes inactive for some time, you are automatically returned to this page, but you can return to the session exactly where you left off by once again clicking the connect button. - -> ![image](../../../sessions/images/desktop/7_desktop_connected.png) diff --git a/docs/platform/community/alma/NewUser/LaunchNotebook.md b/docs/platform/community/alma/NewUser/LaunchNotebook.md deleted file mode 100644 index e79592b5..00000000 --- a/docs/platform/community/alma/NewUser/LaunchNotebook.md +++ /dev/null @@ -1,3 +0,0 @@ -# Launching a Notebook - -Create and open a Notebook session (JupyterLab) via the Science Portal and use the file browser to manage files. diff --git a/docs/platform/community/alma/NewUser/Login.md b/docs/platform/community/alma/NewUser/Login.md deleted file mode 100644 index 08402df2..00000000 --- a/docs/platform/community/alma/NewUser/Login.md +++ /dev/null @@ -1,17 +0,0 @@ -# Logging in to the Science Portal - -Authenticate using CANFAR credentials or supported OIDC providers via the web UI. - -You will need a CADC account to access the system. If you do not have one, you can request one at: - - -To request authorization to use the Science Portal, send an email to . You may also wish to consider the following: - -- *Project space*: If you intend to work on a dataset with collaborators, it is recommended that you set up a [project space](ProjectSpace.md), where a designated group of users all has common access to the files contained within it. -- *Communications on Discord*: a Discord workspace is used for some aspects of communication around the Science Portal, including notice of service outages and some trouble-shooting support. You can request that you be added to this space also by contacting the email address listed above. - -Once your access has been confirmed, go to the CANFAR page: and log in to access the Science Portal. - -Start a new session by clicking the plus sign. - -There are four different types of sessions that you can choose to launch: Desktop, CARTA, Notebook, and Contributed. All are described below; in brief, Desktop provides a linux desktop-like working environment, CARTA corresponds to ALMA's CARTA visualization tool, Notebook provides a Jupyter Notebook environment, and Contributed contains community-contributed tools such as a time estimator for the CASTOR mission. diff --git a/docs/platform/community/alma/NewUser/Overview.md b/docs/platform/community/alma/NewUser/Overview.md deleted file mode 100644 index 1842023d..00000000 --- a/docs/platform/community/alma/NewUser/Overview.md +++ /dev/null @@ -1,3 +0,0 @@ -# New user overview - -Quick-start notes for logging in, launching sessions, and using project spaces. diff --git a/docs/platform/community/alma/NewUser/ProjectSpace.md b/docs/platform/community/alma/NewUser/ProjectSpace.md deleted file mode 100644 index 6a0b6228..00000000 --- a/docs/platform/community/alma/NewUser/ProjectSpace.md +++ /dev/null @@ -1,15 +0,0 @@ -# Project spaces - -Short guide to requesting and sharing a project space for collaborative work. - -## What is a Project Space? - -Users can work with files in two different main directories. The first is their personal home directory, found in `/home/[username]`. The second is within a project directory, found in `/project/[project_name]`. These project directories provide a space where files can easily be shared and analyzed with collaborators. - -## How to Request a Project Space - -In order to request a project space, you will need to have a name for the space (the name of the project directory). As with home directories, the default diskspace allocation is 200GB. If you anticipate needing more than this, you will need an estimate of the amount of diskspace that you will need. Once you have this information, if you are on the Discord workspace, the easiest way to request a project space is to post the request there, with a note to Kevin Casteels. Alternatively, you can email the request to `support@canfar.net`. - -## How to Give Collaborators Access - -Collaborators will require a free CADC account in order to be added as people with designated access to the project space files. A CADC account can be requested at . If your collaborators only wish to download the files and analyze them on their own computers, this step is sufficient; they can access the files on the web at or use the VOS Tools to download them. diff --git a/docs/platform/community/alma/Notebook/transfer_file.md b/docs/platform/community/alma/Notebook/transfer_file.md deleted file mode 100644 index 127251ad..00000000 --- a/docs/platform/community/alma/Notebook/transfer_file.md +++ /dev/null @@ -1,39 +0,0 @@ -# Uploading files in a Notebook session - -Use the Jupyter file browser to upload small files; for large datasets prefer VOSpace or command-line transfer tools. - -## Transfer file into Notebook session - -Smaller files can be uploaded into a Notebook session easily in two different ways. These are outlined in turn below. - -### Directly Upload the File - -Once you have navigated into your directory of interest using the browser in the left-hand side, click the upward-pointing arrow on the top menu bar. - -> ![image](../../../sessions/images/transfer_file/1_landing_click_upload.png) - -This will bring up a window that will let you select the file you wish to upload. Click the `Open` button as usual to confirm your choice of files. - -> ![image](../../../sessions/images/transfer_file/2_upload_window.png) - -Success! Your file is now visible in the browser, and would also be accessible in the same location in a Desktop or a CARTA session. - -> ![image](../../../sessions/images/transfer_file/3_file_is_uploaded.png) - -### Copy-Paste Text - -Alternatively, you can copy and paste text directly into a file within your Notebook session. This might be preferred if, for example, you want to copy a snippet of code into an already existing file in your session. Start by opening up a terminal by double-clicking on the icon. - -> ![image](../../../sessions/images/transfer_file/4_open_terminal.png) - -This opens a terminal on the right hand side of the screen which you can interact with as usual. In the example shown, the text editor vi is being initiated on the command line. - -> ![image](../../../sessions/images/transfer_file/5_new_terminal.png) - -On your local computer, you would select and copy the text of interest. - -> ![image](../../../sessions/images/transfer_file/6_copy_local_text.png) - -You can then paste this text into a text editor in the terminal. Once the file is saved, it is accessible from the file browser in the current directory, and would be visible in a Desktop or a CARTA session as well. - -> ![image](../../../sessions/images/transfer_file/8_file_saved.png) diff --git a/docs/platform/community/alma/TipsTricks/Direct_url.md b/docs/platform/community/alma/TipsTricks/Direct_url.md deleted file mode 100644 index f8d2de51..00000000 --- a/docs/platform/community/alma/TipsTricks/Direct_url.md +++ /dev/null @@ -1,3 +0,0 @@ -# Direct URL downloads - -Notes on using `wget` with URL lists and certificates for direct downloads of private data. diff --git a/docs/platform/community/alma/TipsTricks/Increase_font.md b/docs/platform/community/alma/TipsTricks/Increase_font.md deleted file mode 100644 index f2250c7b..00000000 --- a/docs/platform/community/alma/TipsTricks/Increase_font.md +++ /dev/null @@ -1,7 +0,0 @@ -# Increase font size - -Tips to increase font sizes in Desktop containers and Jupyter notebooks for readability. - -In Desktop sessions increase font sizes via application preferences or browser zoom. In JupyterLab use the View menu or browser zoom settings. - -Images show how to change the terminal font size via the terminal's preferences or context menu. diff --git a/docs/platform/community/alma/TipsTricks/Using_clipboard.md b/docs/platform/community/alma/TipsTricks/Using_clipboard.md deleted file mode 100644 index f13c1906..00000000 --- a/docs/platform/community/alma/TipsTricks/Using_clipboard.md +++ /dev/null @@ -1,5 +0,0 @@ -# Using the clipboard - -How to copy and paste small text between Desktop and Notebook sessions using built-in clipboard features. - -Copy selected text with Ctrl-Shift-C and paste with Ctrl-Shift-V inside Desktop containers. For larger file transfers use VOSpace or file upload/download. diff --git a/docs/platform/community/alma/index.md b/docs/platform/community/alma/index.md index dd74a4d9..1aeb6e95 100644 --- a/docs/platform/community/alma/index.md +++ b/docs/platform/community/alma/index.md @@ -1,37 +1,170 @@ -# ALMA resources +# ALMA analysis workflow -This section collects ALMA-specific tutorials and tips for using the CANFAR Science Platform. +Use this workflow to move ALMA data into a CANFAR Session, reduce it with +CASA, inspect the products, and keep the results in persistent storage. It is +text-first so commands can be copied and the workflow remains useful when the +Science Portal layout changes. -## Desktop +## Prerequisites -- [Archive download (web)](ALMA_Desktop/archive_download.md) -- [Archive download (script)](ALMA_Desktop/archive_script_download.md) -- [CASA containers](ALMA_Desktop/casa_containers.md) -- [Starting CASA](ALMA_Desktop/start_casa.md) -- [Typical reduction & imaging](ALMA_Desktop/typical_reduction.md) +- A [CADC account](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/auth/request.html) + and access to the [CANFAR Science Platform](https://www.canfar.net/). Contact + [support@canfar.net](mailto:support@canfar.net) if your account or project + access is not ready. +- A persistent destination: use `/arc/home/[username]` for personal files or + `/arc/projects/[project]` for shared work. Treat `/scratch` as temporary; + it is suitable for fast intermediate files, not final results. +- A Session Kind that matches the task: use **Desktop** for CASA and other GUI + applications, **Notebook** for Python-based analysis, and **CARTA** for + inspecting images and spectral cubes. +- Optional command-line access. Install the shipped client and log in: -## General tools + ```bash + pip install canfar --upgrade + canfar login cadc + canfar auth show + canfar server ls + ``` -- [File transfers overview](General_tools/File_transfers.md) -- [Group management](General_tools/Group_management.md) -- [Using SSHFS](General_tools/Using_sshfs.md) -- [Using VOS Tools](General_tools/Using_vostools.md) -- [Using web storage](General_tools/Using_webstorage.md) +See [Getting Started](../../get-started.md) for account and portal setup. -## New users +## Workflow -- [Overview](NewUser/Overview.md) -- [Login](NewUser/Login.md) -- [Launching Desktop](NewUser/LaunchDesktop.md) -- [Launching Notebook](NewUser/LaunchNotebook.md) -- [Project spaces](NewUser/ProjectSpace.md) +### 1. Choose a Session and storage location -## Notebook +From the Science Portal, create a Desktop, Notebook, or CARTA Session and +choose an image that supports the application you need. From a terminal, list +the available images before creating a Session: -- [Transfer files in Notebook](Notebook/transfer_file.md) +```bash +canfar image ls --kind desktop +canfar create desktop IMAGE_NAME --name alma-reduction +canfar ps --kind desktop +canfar open [session-id] +``` -## Tips & tricks +Replace bracketed values with the Session ID, username, and project name used +by your account. Replace `IMAGE_NAME` with an image returned by +`canfar image ls --kind desktop` that includes the CASA version required by the +reduction. The image listing is the source of truth; do not assume that a CASA +tag remains unchanged. -- [Direct URL downloads](TipsTricks/Direct_url.md) -- [Increase font size](TipsTricks/Increase_font.md) -- [Using clipboard](TipsTricks/Using_clipboard.md) +### 2. Stage the archive data + +Download the requested data from the [ALMA Science Archive](https://almascience.nrao.edu/) +to your local machine, or use a configured CANFAR data source. Create a +destination and verify the copy with the CANFAR data commands: + +```bash +canfar data mkdir -p arc:/home/[username]/alma/raw +canfar data cp local:/path/to/alma-data.tar.gz \ + arc:/home/[username]/alma/raw/alma-data.tar.gz +canfar data ls -lh arc:/home/[username]/alma/raw +``` + +The `local:` source is the machine where the command runs. To copy data that +is already in Vault, use a `vault:` source instead: + +```bash +canfar data cp vault:/ALMA/test-data/cutouts/test-4d-cube.fits \ + arc:/home/[username]/alma/raw/test-4d-cube.fits +``` + +Inside a Session, the same `/arc` paths are available from the file browser +and terminal. For a large reduction, copy working data to `/scratch` and keep +the source and final products under `/arc`. + +### 3. Start CASA in a Desktop Session + +In a Desktop Session, open **Applications → Astro Software**, choose the CASA +version required by the project, and open its terminal. Start CASA in the +mode required by the supplied reduction script: + +```bash +casa +casa --pipeline +``` + +Run the archive's supplied calibration and imaging scripts using the command +form documented for that CASA release. Keep the scripts, measurement sets, +and intermediate products under a persistent project directory or copy them +back to `/arc` when the reduction finishes. The [CASA containers guide](../CASA_and_more.md) +lists version-specific package notes and known issues. + +### 4. Inspect and analyse the products + +Open calibrated measurement sets or image products from a CARTA Session when +you need cube navigation, spectra, regions, or moment-map inspection. Use a +Notebook Session for Python analysis. Both Session Kinds can read files saved +under `/arc/home/[username]` and `/arc/projects/[project]`. + +For a local or scripted check, list the result before opening it: + +```bash +canfar data ls -lh arc:/home/[username]/alma/results +canfar data info arc:/home/[username]/alma/results/result.fits +``` + +### 5. Save results and clean up + +Copy final products out of `/scratch` before stopping the Session. From inside +the Session, use ordinary filesystem commands: + +```bash +mkdir -p /arc/home/[username]/alma/results +cp -a /scratch/alma-reduction/results/. \ + /arc/home/[username]/alma/results/ +``` + +From the machine where the files are local, use the data command instead: + +```bash +canfar data cp local:/path/to/result.fits \ + arc:/home/[username]/alma/results/result.fits +canfar data ls -lh arc:/home/[username]/alma/results +``` + +When the results are safely stored, inspect and remove the Session if it is no +longer needed: + +```bash +canfar info [session-id] +canfar delete [session-id] --force +``` + +## Expected results + +At the end of the workflow: + +1. `canfar auth show` reports the active Authentication and + `canfar server ls` lists an available Science Platform Server. +2. The selected Session reaches `Running` and opens in the Science Portal. +3. Raw data and final products are visible under the chosen `/arc` path from + the Desktop, Notebook, or CARTA Session. +4. CASA scripts complete using a compatible image, and the reduction products + are stored outside `/scratch`. +5. Deleting the Session does not remove the files saved under `/arc`. + +## Troubleshooting + +| Symptom | Check or fix | +| --- | --- | +| Login fails or credentials are stale | Run `canfar --log-level debug login cadc --force`, then `canfar auth show`. | +| No suitable image appears | Run `canfar image ls --kind desktop` (or `--kind notebook`/`--kind carta`) and choose an image that includes the required software. | +| Session remains pending | Run `canfar ps --all`, `canfar events [session-id]`, and `canfar stats`; reduce requested resources if the Science Platform Server has no capacity. | +| A data copy fails | Confirm the `local:`, `arc:`, or `vault:` source and destination with `canfar data ls -lh IDENTIFIER:/path`; check that the active Authentication Record can access the target. | +| Files disappear after the Session ends | Move them from `/scratch` to `/arc/home/[username]` or `/arc/projects/[project]` before deleting the Session. | +| A shared project path is denied | Ask the project administrator to add your CADC account to the project group; see [Permissions](../../permissions.md). | +| CASA is missing or incompatible | Select a Desktop image that lists the required CASA version and match the version to the archive scripts. If CASA 6.5.0–6.5.2 opens with display errors, exit CASA and start it again. | +| A transfer reports expired credentials | Renew or select the Authentication Record used by the active server, then retry. See [support](../../support/index.md) if the record or certificate cannot be refreshed. | +| The browser cannot connect to a new Session | Wait for the Session to reach `Running`, then retry `canfar open [session-id]`; inspect `canfar info [session-id]` and `canfar logs [session-id]` if it still fails. | + +## Related documentation + +- [Interactive Sessions](../../sessions/index.md), [Desktop](../../sessions/desktop.md), + [Notebooks](../../sessions/notebook.md), and [CARTA](../../sessions/carta.md) +- [Storage overview](../../storage/index.md), [data transfers](../../storage/transfers.md), + and [VOSpace](../../storage/vospace.md) +- [CLI quickstart](../../../cli/quick-start.md) and [CLI reference](../../../cli/cli-help.md) +- [Python quickstart](../../../client/quick-start.md) +- [Support](../../support/index.md) diff --git a/docs/platform/community/index.md b/docs/platform/community/index.md index 89ddcf05..c6a75a9f 100644 --- a/docs/platform/community/index.md +++ b/docs/platform/community/index.md @@ -1,5 +1,13 @@ -# CANFAR Community Resources +# Community workflows -As teams and projects grow, it's important to share knowledge and best practices. This section provides resources and tutorials for specific communities and their use cases within the CANFAR ecosystem. +These pages collect domain-specific workflows that build on the general +[Sessions](../sessions/index.md), [Storage](../storage/index.md), and +[Container Images](../containers/index.md) guides. -If you have a specific use case or community you'd like to see represented here, please contribute to the [documentation](https://github.com/opencadc/canfar) or [contact us](mailto:support@canfar.net). +- [ALMA analysis workflow](alma/index.md) — a text-first reduction and + inspection path. +- [CASA and adjacent software](CASA_and_more.md) — image selection and links + to upstream package documentation. + +Suggest a workflow through the [CANFAR repository](https://github.com/opencadc/canfar) +or contact [support@canfar.net](mailto:support@canfar.net). diff --git a/docs/platform/concepts.md b/docs/platform/concepts.md index ddf1488a..1957ef74 100644 --- a/docs/platform/concepts.md +++ b/docs/platform/concepts.md @@ -1,694 +1,131 @@ -# CANFAR Platform Concepts +# Platform concepts -**Understanding the architecture and core concepts behind the CANFAR Science Platform for astronomical research.** - -!!! abstract "🎯 Core Concepts" - **Essential platform knowledge for all users:** - - - **Cloud Architecture**: Container-based platform design - - **Container Environments**: Pre-built software stacks for astronomy - - **Session Management**: Interactive and batch computing resources - - **Storage Systems**: Data persistence and collaboration - - **Browser Access**: Minimal-installation web-based workflows - - -## 🚀 CANFAR Science Platform Overview - -The **Canadian Advanced Network for Astronomy Research (CANFAR)** Science Platform is a cloud-native computing environment designed specifically for astronomical research workflows. - -### Platform Design Philosophy - -CANFAR eliminates traditional barriers to astronomical computing: - -- **Minimal Software Installation**: Pre-built environments with astronomy packages ready to use -- **Browser-Based Access**: Complete workflows accessible through web interfaces -- **Scalable Resources**: Computing power that grows with your project needs -- **Collaborative Infrastructure**: Shared storage and standardized environments -- **Reproducible Science**: Container-based workflows ensure consistent results - -### Core Benefits - -=== "Individual Researchers" - - **Minimal Setup**: Pre-configured containers ready to use immediately - - **Hardware Liberation**: Access powerful computing without owning servers - - **Location Independence**: Work from anywhere with just a web browser - - **Data Protection**: Automatic backups and managed storage systems - -=== "Research Teams" - - **Environment Standardisation**: Identical software stacks across the team - - **Seamless Collaboration**: Shared workspaces and data access - - **Session Sharing**: Live collaboration on analysis workflows - - **Project Management**: Centralised resource and permission management - -=== "Large Projects" - - **Dynamic Scaling**: Resources adjust to computational demands - - **Batch Processing**: Automated workflows for large dataset processing - - **Custom Environments**: Specialised containers for unique requirements - - **Archive Integration**: Fast access to astronomical data repositories - -## 🏗️ Platform Architecture - -CANFAR is built on modern cloud-native technologies designed for scalability, reliability, and ease of use. Understanding the architecture helps you leverage the platform effectively. - -### System Components +CANFAR separates authenticated compute from the storage that holds research +data. A Science Platform Server launches user-owned Sessions from Container +Images, while VOSpace Services and mounted filesystems provide data access. ```mermaid -graph LR - %% User Entry Point - User["👤 Scientist"]:::user - - %% Portal Layer - Portal["🌐 Science Portal
canfar.net"]:::portal - Auth["🔐 CADC Authentication"]:::auth - Sessions["🖥️ Session Manager
Skaha"]:::sessions - - %% Infrastructure Layer - K8s["☸️ Kubernetes Cluster"]:::k8s - Containers["🐳 Container Images
Harbor Registry"]:::containers - Storage["💾 Storage Systems"]:::storage - - %% Storage Systems - arc["📁 arc POSIX Storage
Shared Filesystem"]:::arc - vault["☁️ VOSpace Object Store
Long-term Storage"]:::vospace - scratch["⚡ Scratch
Temporary SSDs"]:::scratch - - %% Session Types - Types["Session Types"]:::types - Notebook["📓 Jupyter Notebooks"]:::notebooks - Desktop["🖥️ Desktop Environment"]:::desktop - CARTA["📊 CARTA Viewer"]:::carta - Firefly["🔥 Firefly Viewer"]:::firefly - Contrib["⚙️ Contributed Apps"]:::contrib - Batch["🏭 Batch Jobs"]:::batch - - %% Connections - User --> Portal - Portal --> Auth - Portal --> Sessions - - Auth --> K8s - Sessions --> K8s - - K8s --> Containers - K8s --> Storage - - Storage --> arc - Storage --> vault - Storage --> scratch - - Sessions --> Types - Types --> Notebook - Types --> Desktop - Types --> CARTA - Types --> Firefly - Types --> Contrib - Types --> Batch - - %% Styling - classDef user fill:#e3f2fd,stroke:#1976d2,stroke-width:3px,color:#000 - classDef portal fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px,color:#000 - classDef auth fill:#ffebee,stroke:#c62828,stroke-width:2px,color:#000 - classDef sessions fill:#e8f5e8,stroke:#2e7d32,stroke-width:2px,color:#000 - classDef k8s fill:#fff3e0,stroke:#ef6c00,stroke-width:3px,color:#000 - classDef containers fill:#f1f8e9,stroke:#558b2f,stroke-width:2px,color:#000 - classDef storage fill:#e0f2f1,stroke:#00695c,stroke-width:2px,color:#000 - classDef arc fill:#fce4ec,stroke:#ad1457,stroke-width:2px,color:#000 - classDef vospace fill:#f3e5f5,stroke:#6a1b9a,stroke-width:2px,color:#000 - classDef scratch fill:#fff8e1,stroke:#f57f17,stroke-width:2px,color:#000 - classDef types fill:#e1f5fe,stroke:#0277bd,stroke-width:2px,color:#000 - classDef notebooks fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px,color:#000 - classDef desktop fill:#e8f5e8,stroke:#388e3c,stroke-width:2px,color:#000 - classDef carta fill:#fff3e0,stroke:#f57c00,stroke-width:2px,color:#000 - classDef firefly fill:#fce4ec,stroke:#c2185b,stroke-width:2px,color:#000 - classDef contrib fill:#e0f2f1,stroke:#00695c,stroke-width:2px,color:#000 - classDef batch fill:#ffebee,stroke:#d32f2f,stroke-width:2px,color:#000 -``` - -### Architecture Components - -**Browser-Based Portal** (`canfar.net`) -: Single entry point to the platform - usually no software installation required. Provides access to all CANFAR services through web interfaces. - -**Authentication System** (CADC / OIDC) -: Secure identity management through the Canadian Astronomy Data Centre, providing single sign-on and access control across astronomical data archives and user-created group management - -**Container Orchestration** (kubernetes) -: Manages computing resources automatically, handling container deployment, scaling, and resource allocation behind the scenes. - -**Software Environments** (Harbor Registry) -: Pre-built and customized container images with astronomy software packages, from basic Python environments to specialised tools like CASA and CARTA. - -**Session Management** (`skaha`) -: Orchestrates your computing sessions, connecting containers with storage systems and managing resource allocation. - -**Storage Infrastructure** -: Multiple storage systems optimized for different use cases - from high-performance computing to long-term archival. - -### Key Architectural Principles - -**Container-First Design** -: All software runs in containers, ensuring consistency, reproducibility, and easy distribution of complex software environments. - -**Kubernetes-Native** -: Built on Kubernetes for automatic scaling, resource management, and high availability without manual intervention. - -**Storage Separation** -: Data persistence is handled separately from computing, allowing containers to be ephemeral while keeping your data safe. - -**Web-Based Access** -: Everything accessible through standard web browsers for portability and ease of installation. - -**API-Driven** -: All platform functions available through REST APIs, enabling automation and integration with external tools. - -!!! info "For Developers" - The platform provides REST APIs for programmatic access. See the **[CANFAR Python Client](../client/home.md)** for automation and scripting examples. - - -## 🐳 Container Environments - -Containers are the foundation of CANFAR's flexibility and reproducibility. They provide complete, portable software environments with all astronomy tools pre-configured and ready to use. - -### Container Fundamentals - -**What are Containers?** -: Lightweight, portable packages that include an application and all its dependencies (libraries, tools, system settings) in a single, consistent environment. - -**Why Containers for Astronomy?** -: Solve the traditional "dependency hell" of astronomical software by packaging complex tool chains into reproducible, shareable environments. - -### Traditional vs Container Workflows - -=== "Traditional Software Installation" - **Common Problems:** - - - Conflicting library versions and dependencies - - Missing system requirements and packages - - Different behaviour across different machines - - Time-consuming setup and configuration - - Version compatibility issues between tools - - "It works on my machine" syndrome - -=== "Container Approach" - **Solutions Provided:** - - - Consistent environment that works identically everywhere - - All dependencies pre-installed and tested together - - No installation or configuration required - - Easy sharing and collaboration - - Reproducible analysis results - - Instant access to complex software stacks - -### Popular CANFAR Containers - -| Container | Purpose | Key Software | Best For | -|-----------|---------|--------------|----------| -| **astroml** | General astronomy analysis | scipy, astropy, matplotlib, pandas, scikit-learn, pytorch, STILTS | Data analysis, visualization, ML, research | -| **improc** | Image processing | CASUTools, SExtractor, SWarp, IRAF, STILTS | Photometry, astrometry, source detection, PSF fitting | -| **casa** | Radio/MM astronomy | CASA software suite, Python | Radio astronomy, interferometry | -| **lsst** | LSST Analysis | LSST Software Stack | Image processing, LSST software stack and data access | -| **carta** | Data visualization | CARTA viewer, analysis tools | Interactive datacubes visualization | - -!!! tip "Container Selection" - **Start with `astroml`** for general astronomy work - it includes most common packages and is regularly updated with the latest astronomy and machine learning software. - -### Container Lifecycle & Performance - -**First Launch** (2-3 minutes) -: kubernetes downloads the container image to node-local storage. This only happens once per container type. - -**Subsequent Launches** (30-60 seconds) -: Fast startup using cached images. Container starts with your storage systems already connected. - -**During Session** -: Full access to pre-configured software environment with your data mounted and ready to use. - -**Session End** -: Container is destroyed, scratch is wiped out, but all data in persistent storage remains safely preserved. - -### Container Registry & Management - -**Harbor Registry** (`images.canfar.net`) -: Browse all available container images. - -**Image Updates** -: Containers should be regularly updated with latest software versions and security patches. - -**Custom Containers** -: Advanced users can build and maintain specialized containers for unique workflows or software requirements. - -!!! success "Reproducible Science" - Containers ensure your analysis runs identically for you, your collaborators, and future researchers. This is crucial for reproducible scientific workflows. - -!!! tip "Advanced Usage" - Use the **[CANFAR CLI](../cli/cli-help.md)** to list available containers, check versions, and manage sessions programmatically. - - -## ☸️ Sessions & Computing Resources - -CANFAR uses Kubernetes to manage your computing sessions automatically. Sessions connect container environments with storage systems and provide different interfaces optimized for various workflows. - -### Session Fundamentals - -**Session Lifecycle** -: Each session creates a fresh container instance that runs until you stop it or it times out. Your data persists independently in storage systems. - -**Resource Management** -: Kubernetes automatically handles resource allocation, scaling, and availability without requiring infrastructure knowledge. - -**Data Persistence** -: Container instances are temporary and destroyed at session end, but your files persist through the storage systems. - -### Session Types & Interfaces - -CANFAR provides different session types, each optimized for specific workflows: - -=== "📓 Notebook Sessions" - **JupyterLab Interface** for interactive data science workflows - - **Best For**: Data exploration, visualization, prototyping, interactive analysis - - **Features**: - - - Rich text, code, and visualization in unified interface - - Python default, and other languagecustom kernels possible - - Cell-based execution for iterative development - - Built-in file browser and terminal access - - Collaborative sharing and version control - -=== "🖥️ Desktop Sessions" - **Linux Desktop Environment** for traditional GUI applications - - **Best For**: CASA, DS9, TOPCAT, Aladin, traditional desktop workflows - - **Features**: - - - Minimal Ubuntu desktop with window manager - - Multiple applications, each running on a different cluster node - - GUI-based tools and traditional software - - File managers and system utilities - - Firefox browser-on-browser - -=== "📊 CARTA Sessions" - **Visualization and analysis** for FITS and HDF5 astronomy data - - **Best For**: Radio astronomy data analysis, data cubes visualization - - **Features**: - - - Interactive data exploration and analysis - - Optimized for large radio astronomy datasets - - Advanced visualization and measurement tools - - Browser-native interface with desktop-class performance - -=== "🔥 Firefly Sessions" - **Table and Image Visualization** tools - - **Best For**: Catalogue analysis, image display, multi-wavelength data, LSST - - **Features**: - - - Astronomical table viewing and analysis - - FITS image display and manipulation - - Cross-matching and catalog operations - - Browser-native interface for data exploration - - Integration with astronomical databases - -=== "⚙️ Contributed Sessions" - **Community-Maintained Applications** and specialised tools - - **Best For**: Communitity-supported web apps, Specialised workflows, experimental features, niche applications - - **Features**: - - - Custom applications contributed by the community - - Specialised tools for specific research areas - - Experimental features and beta software - - Domain-specific analysis environments - - Research group customisations - -=== "🏭 Batch Sessions" - **Automated Processing** without interactive interfaces - - **Best For**: Large-scale processing, automated workflows, production pipelines - - **Features**: - - - Headless execution for automated processing - - Script-based workflows and command execution - - Integration with workflow management systems - - Scalable processing for large datasets - - Programmatic job submission and monitoring - -### Resource Allocation Modes - -CANFAR supports two resource definition approaches: - -=== "🔄 Flexible Mode (Default)" - **Dynamic resource allocation** that adapts to cluster availability - - **Characteristics**: - - - **Adaptive Usage**: Can use more CPU/memory when cluster resources available - - **Fast Scheduling**: Sessions start quickly as they're easier to place - - **Variable Performance**: Performance adapts to cluster load - - **Efficient Sharing**: Resources shared optimally across users - - **Best For**: Interactive work, development, data exploration, most research workflows - - **CLI Usage**: - ```bash - canfar create notebook skaha/astroml:latest - ``` - -=== "🎯 Fixed Mode" - **Guaranteed resource allocation** with dedicated resources - - **Characteristics:** - - - **Predictable Performance**: Consistent CPU/memory regardless of cluster load - - **Resource Reservation**: Resources reserved exclusively for your session - - **Potential Delays**: May wait for exact resources to become available - - **Dedicated Resources**: No sharing with other users - - **Best For:** Production workflows, time-sensitive analysis, performance-critical tasks - - **CLI Usage:** - ```bash - canfar create --cpu 4 --memory 8 notebook skaha/astroml:latest - ``` - -### Resource Selection Guidelines - -| Workflow Type | Recommended Mode | Reasoning | -|---------------|------------------|-----------| -| **Interactive Analysis** | Flexible | Variable resource needs, benefits from burst capacity | -| **Data Exploration** | Flexible | Unpredictable resource patterns, fast startup preferred | -| **Production Processing** | Fixed | Predictable performance requirements | -| **Time-Critical Analysis** | Fixed | Deadline-driven work requiring consistent performance | -| **Large Batch Jobs** | Fixed | Known resource requirements, consistent runtime needed | -| **Development & Testing** | Flexible | Variable needs, frequent session creation/destruction | - -!!! tip "Getting Started" - **Start with flexible mode** (the default) for most research work. Only use fixed mode when you have specific performance requirements or time constraints. - -!!! tip "Advanced Session Management" - Use the **[CANFAR CLI](../cli/cli-help.md)** to monitor sessions with `canfar stats`, get detailed information with `canfar info`, and manage multiple sessions programmatically. - - -## 💾 Storage Systems & Data Management - -CANFAR provides multiple storage systems optimized for different use cases in astronomical research. Understanding data persistence is crucial for effective platform use. - -### Data Persistence Fundamentals - -!!! warning "Critical: Understanding Data Persistence" - **Where your files are saved determines whether they survive session restarts:** - - | Storage Location | Persistence | Purpose | Performance | - |------------------|-------------|---------|-------------| - | `/arc/projects/[project]/` | ✅ **Permanent, backed up** | Shared project data, results | Network-based shared POSIX | - | `/arc/home/[user]/` | ✅ **Permanent, backed up** | Personal configs, scripts | Network-based shared POSIX | - | `vos:[user\|project]` | ✅ **Permanent, archived** | Long-term storage, sharing | Network-based shared object store | - | [`/cvmfs/`](cvmfs.md) | ✅ **Permanent, read-only** | Global software repositories | Distributed read-only filesystem | - | `/scratch/` | ❌ **Wiped at session end** | Large temporary computations | Local SSD POSIX| - -### ARC Storage (`/arc/`) - Active Research Storage - -**High-Performance POSIX Filesystem** for active research workflows: - -**Key Features:** - -- **Speed**: Direct filesystem access optimized for large computations -- **Collaboration**: Group-based access control for team projects -- **Backup**: Daily snapshots for data protection -- **Quotas**: Managed per-project and per-user allocations -- **POSIX Compliance**: Standard Unix/Linux filesystem operations - -**Directory Structure:** -``` -/arc/ -├── home/[user]/ # Personal user space -├── projects/[project]/ # Shared project directories -/scratch/ # Fast temporary storage (session-local) +flowchart TD + Identity[Identity Provider] --> Auth[Authentication Record] + Auth --> Server[Science Platform Server] + Server --> Session[Session] + Session --> Image[Container Image] + Session --> Mounted[Mounted /arc and /scratch] + Server --> VOS[VOSpace Service] + Auth --> VOS + VOS --> Identifier[Storage Identifier] ``` -**Best For:** - -- Active data analysis and processing -- Shared datasets within research teams -- Large computational workflows requiring fast I/O -- Collaborative software development - -### Vault VOSpace (`vos:[user|project]`) - Long-Term Object Storage +## Authentication and Server Selection -**Vault is an [IVOA-Compliant VOSpace](https://www.ivoa.net/documents/VOSpace/)** for archival and sharing. +An **Identity Provider** issues the identity used to access CANFAR. An +**Authentication Record** stores the local credential state and its +Authentication Mode, such as X.509 or OIDC. A **Science Platform Server** is a +named endpoint discovered for that identity. **Server Selection** chooses the +Science Platform Server for new requests. -**Key Features**: +Changing Authentication or Server Selection affects new requests. It does not +move or change an existing Session. -- **Standards-Based**: International Virtual Observatory Alliance (IVOA) VOSpace standard -- **Web Access**: RESTful APIs and web interfaces -- **Metadata Support**: Rich astronomical metadata and annotation capabilities -- **Versioning**: Track changes to datasets over time -- **Geo-Redundant**: Multiple copies across different locations -- **Access Control**: Fine-grained permissions and sharing +From the CLI: -**Access Methods:** ```bash -# Command-line tools -vcp myfile.fits vos:[user|project]/ # Copy to VOSpace -vls vos:[user|project]/ # List VOSpace contents -vmkdir vos:newproject/ # Create directories - -# Web interface -https://www.canfar.net/storage/list - -# Python APIs -import vos +canfar login cadc +canfar auth show +canfar server ls +canfar server use NAME ``` -NB: `arc` is also available through the VOSpace API (`arc:`). - -**Best For**: - -- Long-term data archival and preservation -- Sharing datasets with external collaborators -- Metadata-rich astronomical data -- Cross-institutional data exchange -- Backup copies of important results - -### Scratch Storage (`/scratch/`) - High-Performance Temporary +The Python authentication and platform modules provide noninteractive +operations for scripts. See the [CLI reference](../cli/cli-help.md) and the +[Python client guide](../client/get-started.md). -**Fast SSD Storage** for intensive computations: +## Sessions -**Key Features**: +A **Session** is a compute environment for one user on one Science Platform +Server. It starts from a **Container Image**, has a **Session Kind**, and +requests a Resource Allocation Mode. Interactive kinds expose a browser +application; `headless` runs a command and exits. -- **Performance**: Fastest available storage for I/O-intensive operations -- **Temporary**: Automatically cleared when sessions end -- **Capacity**: Up to few hundreds GBs per session for big computational jobs -- **No Backup**: Data is not preserved or backed up +Creation returns IDs before a Session is necessarily ready. `Pending` means the +platform is still admitting, scheduling, pulling, or initializing the Session; +it is not an application log or a guarantee of a queue position. Use +`canfar events`, `canfar info`, and `canfar ps --all` to inspect it. See +[Sessions](sessions/index.md) and [Batch processing](sessions/batch.md). -**Best For**: +## Container Images -- Large intermediate files during processing -- I/O-intensive computations requiring maximum speed -- Temporary datasets that don't need preservation -- Cache storage for repeated computations +A **Container Image** is a reusable software environment. The image determines +the tools available after startup; the Session request determines how it is +used. Choose an image from `canfar image ls` or the Science Portal rather than +assuming that an example tag exists at every deployment. -!!! danger "Scratch Storage Warning" - **All data in `/scratch/` is permanently deleted when your session ends.** Always copy important results to `/arc/` or `vos:` storage before ending sessions. +Keep stable software and reproducible configuration in the image or a versioned +project repository. Treat changes made inside a running container as temporary +unless they are written to persistent storage or rebuilt into an image. -### Storage Strategy & Best Practices +See [Containers](containers/index.md) and [Building Containers](containers/build.md). -=== "📊 Active Research Computing" - **Use `/arc/` for active work** - Example of structure: - - 1. **Input Data**: Store working datasets in `/arc/projects/[project]/data/` - 2. **Analysis Scripts**: Keep analysis code in `/arc/projects/[project]/scripts/` - 3. **Results**: Save outputs to `/arc/projects/[project]/results/` - 4. **Collaboration**: Share via project directory access permissions +## Storage -=== "🗄️ Long-Term Archival Workflow" - **Use `vos:` for preservation** - - 1. **Final Results**: Archive completed analysis results - 2. **Publication Data**: Store data associated with published papers - 3. **Metadata**: Add rich descriptions and provenance information - 4. **Sharing**: Grant access to external collaborators +A Science Platform Session may expose persistent `/arc` paths and ephemeral +`/scratch`. Use `/arc/home/` for personal files and +`/arc/projects/` for shared project data when those mounts are +provided. `/scratch` is fast Session-local working space and is deleted with +the Session. -=== "⚡ High-Performance Computing" - **Use `/scratch/` for intensive processing:** - - 1. **Large Intermediates**: Store temporary large files during processing - 2. **Cache**: Keep frequently accessed data for fast retrieval - 3. **I/O Intensive**: Use for operations requiring maximum disk speed - 4. **Copy Results**: Always copy important outputs to persistent storage +A Science Platform Server can expose one or more VOSpace Services. Each service +has a **Storage Identifier**. Use `canfar data` in a shell or +`canfar.storage.filesystem(identifier)` in Python. The identifier is a +configuration argument; CANFAR does not register dynamic `vault://` or `arc://` +protocols. -### Storage Integration & Automation - -**Command-Line Tools:** ```bash -# ARC storage (standard Unix commands) -cp analysis.py /arc/projects/[project]/scripts/ -ls -la /arc/home/[user]/ - -# VOSpace operations -vcp /arc/projects/[project]/results/ vos:[project]/analysis-v1/ -vmv vos:[user]/oldname vos:[user]/newname +canfar data cp vault:/project/input.fits local:/scratch/input.fits ``` -**Programmatic Access:** -```python -# Python integration examples -import vos - -filename = "myimage.fits" -vclient = vos.Client() -vclient.copy(filename, 'vos:[project]/public/{filename}') -``` +Use the mounted `/arc` path for data already there, stage remote data once when +a tool needs a local path, and write durable results outside `/scratch`. See +[Storage](storage/index.md). -### Storage Quotas & Management +## Resource allocation modes -**Quota Information**: +CPU, memory, and GPU requests are inputs to platform admission. Omit CPU and +memory for the flexible request or supply measured fixed values. A fixed request +can wait for matching capacity. Queue order and relative priority between +interactive and headless work are deployment policy; the Python client and CLI +do not promise an ordering. -- **ARC Storage**: Project-based quotas managed by CANFAR administrators, request increase when necessary anytime -- **Vault**: User and project allocations with expansion available, increase when necessary anytime -- **Scratch**: Per-session allocation, automatically managed +Use `canfar stats` as a Science Platform Server-level capacity signal, not as a per-Session +explanation. For one workload, inspect its events and status. +## Scientific workflow -!!! tip "Storage Efficiency" - **Optimize your storage strategy:** - - - Use `/arc/` for active work requiring file system access - - Archive to `vos:` for long-term preservation and sharing - - Leverage `/scratch/` for temporary high-performance needs - - Regularly clean up unnecessary files to stay within quotas +A typical workflow is: +1. Authenticate and select an available Science Platform Server. +2. Launch a suitable Session from a Container Image. +3. Read mounted data directly, or transfer remote data to `/scratch`. +4. Run the analysis and write durable outputs under `/arc` or a VOSpace Service. +5. Inspect outputs, then delete the Session when it is no longer needed. -## 🌐 Browser-Based Access & Automation - -CANFAR provides comprehensive browser-based access to all platform features, eliminating the need for local software installation while supporting advanced automation workflows. - -### Web-Based Computing - -**Science Portal** (`canfar.net`) -: Complete platform access through standard web browsers - no plugins or software installation required. - -**Session Interfaces** -: All session types accessible through web interfaces, from Jupyter notebooks to full desktop environments delivered via browser. - -**Data Management** -: Web-based file browsers, transfer tools, and storage management interfaces integrated into the portal. - -### Programmatic Platform Access - -CANFAR provides REST APIs for programmatic access, enabling automation and integration with external tools: - -**CANFAR Python Client** -: Comprehensive Python library for session management, data operations, and workflow automation. See the **[CANFAR Python Client](../client/home.md)** documentation. - -**VOSpace API** -: IVOA-standard APIs for programmatic data storage operations and metadata management. - -**Authentication APIs** -: CADC integration providing secure programmatic access to platform resources and astronomical data archives. - -### Key API Services - -| Service | Purpose | Documentation | -|---------|---------|---------------| -| **Session Management** | Launch, monitor, and control computing sessions | [Python Client](../client/home.md) | -| **VOSpace Operations** | File transfer, storage, and metadata operations | [VOSpace API](storage/vospace.md) | -| **Access Control** | Authentication and authorization management | [CADC Services](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/ac) | - -### Automation Examples - -**Session Automation:** - -```python -# Launch and manage sessions programmatically -from canfar.sessions import Session - -session = Session() -ids = session.create( - name="analysis-notebook", - image="skaha/astroml:latest", - kind="notebook", -) -session.info(ids) -``` - -**Data Workflow Automation:** - -```bash -# VOSpace tools handle storage movement; canfar handles Sessions. -vcp /local/data.fits vos:project/input/data.fits -canfar create headless skaha/astroml:latest -- python /arc/projects/myproject/run.py -vcp vos:project/results/output.fits /local/output.fits +```mermaid +flowchart LR + Login[Authenticate] --> Launch[Launch Session] + Launch --> Stage[Stage or open input] + Stage --> Analyse[Run analysis] + Analyse --> Save[Save durable result] + Save --> Delete[Delete Session] ``` -!!! tip "Integration Options" - **External Workflow Integration:** - - - **GitHub Actions**: Automate CANFAR workflows from code repositories - - **Jupyter Notebooks**: Embed CANFAR operations in interactive analysis - - **CI/CD Pipelines**: Include CANFAR processing in continuous integration - - **Custom Applications**: Build specialised tools using CANFAR APIs - - -## Platform Integration - -### Understanding Platform Connections - -CANFAR integrates with the broader astronomical ecosystem through standards-based interfaces and established protocols: - -**Data Archive Integration** -: Direct access to CADC and international observatory data archives through authenticated connections. - -**VO Standards Compliance** -: IVOA-compliant services enabling interoperability with other Virtual Observatory tools and services. - -**Collaborative Networks** -: Integration with academic institutions, research networks, and international astronomical organizations. - -### Recommended Learning Path - -Now that you understand CANFAR's core concepts, explore specific platform areas: - -1. **[Get Started Guide](get-started.md)** - Hands-on tutorials and first steps -2. **[Permissions & Access](permissions.md)** - User management and collaboration -3. **[Storage Systems](storage/index.md)** - Master data management workflows -4. **[Container Environments](containers/index.md)** - Work with software environments -5. **[Interactive Sessions](sessions/index.md)** - Start analyzing data -6. **[Legacy Cloud Platform](cloud.md)** - Understanding legacy VM infrastructure - -### Advanced Platform Usage - -**For Power Users:** - -- **[CANFAR CLI](../cli/cli-help.md)** - Command-line tools for platform automation -- **[Python Client](../client/home.md)** - Programmatic access and workflow development -- **[Container Building](containers/build.md)** - Create custom software environments -- **[Batch Processing](sessions/batch.md)** - Large-scale automated workflows - -**For Administrators:** - -- **[Project Management](permissions.md#group-management-collaboration)** - Managing research teams and resources -- **[Resource Allocation](permissions.md#group-resource-access)** - Understanding quotas and limits -- **[Access Control](permissions.md#access-control-lists-acls)** - Fine-grained permission management +For reproducible batch workflows, use `headless` plus the [distributed +helpers](../client/helpers.md). For visual analysis, choose Notebook, +Desktop, CARTA, or Firefly as appropriate. ---- +## Related concepts -!!! success "Key Platform Concepts" - **CANFAR provides the computing power of a research institution without the infrastructure overhead.** - - **Core Principles:** - - - **Container-first**: All software runs in reproducible, portable environments - - **Browser-based**: Complete workflows accessible through web interfaces - - **Storage-centric**: Data persistence separate from computing resources - - **Kubernetes-native**: Automatic resource management and scaling - - **API-driven**: Full platform functionality available programmatically - - **Focus on your science** - let CANFAR handle the infrastructure, software, and data management. +- [Getting started](get-started.md) +- [Sessions](sessions/index.md) +- [Storage](storage/index.md) +- [Permissions](permissions.md) +- [Container Images](containers/index.md) diff --git a/docs/platform/containers/build.md b/docs/platform/containers/build.md index 0053af51..3b7ea193 100644 --- a/docs/platform/containers/build.md +++ b/docs/platform/containers/build.md @@ -1,516 +1,115 @@ -# Building Custom Containers +# Build a Container Image -**Creating your own astronomy software environments for CANFAR - from development through deployment and maintenance.** +Build a custom image when the required software is not available in an image +listed by the Science Portal. Keep the Dockerfile, dependency declarations, and +entrypoint in a version-controlled repository so another researcher can rebuild +the same environment. -!!! abstract "🎯 Container Building Overview" - **Master custom container development:** - - - **Development Setup**: Local environments for container building and testing - - **CANFAR Requirements**: Platform-specific configurations and best practices - - **Testing & Debugging**: Ensuring containers work correctly in CANFAR sessions - - **Harbor Registry**: Publishing and maintaining custom containers +## Before you build -Building custom containers becomes necessary when existing CANFAR containers don't meet your specific software requirements or when creating standardized environments for research teams. This guide covers the complete development workflow from initial setup through deployment and maintenance. +1. Check `canfar image ls` for an existing image that already contains the + required tools. +2. Decide which Session Kind will run the image: Notebook, Desktop, a + contributed application, or `headless`. +3. Test the smallest useful workflow and identify persistent input/output paths. +4. Keep credentials, certificates, and research data outside the build context. -## 📋 When to Build Custom Containers - -### Scenarios Requiring Custom Containers - -**Missing Software Packages:** -- Proprietary or licensed software not available in public containers -- Cutting-edge research tools not yet in CANFAR containers -- Specific versions of software required for reproducibility -- Legacy software with complex dependency requirements - -**Team Standardization:** -- Consistent environments across research groups -- Custom analysis pipelines and workflows -- Institutional software licensing requirements -- Project-specific data processing tools - -**Performance Optimization:** -- GPU-optimized builds for specific hardware -- Memory-efficient configurations for large datasets -- Custom compilation flags for scientific software -- Minimized container size for batch processing - -### Alternatives to Consider First - -Before building custom containers, consider these alternatives: - -```bash -# Runtime package installation (temporary) -pip install --user new-package # Installs to /arc/home/[user]/.local/ -mamba install -c conda-forge package # If mamba or conda is available - -# Development in existing containers -# Use astroml as base and install packages per session -# Keep development scripts in /arc/home/ or /arc/projects/ -``` - -!!! tip "Start Simple" - Try adding software to existing containers at runtime first. Only build custom containers when you need permanent, reproducible environments or when runtime installation isn't feasible. - -## 🛠️ Development Environment Setup - -### Local Development Prerequisites - -**Required software:** -- [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine -- Git for version control -- Text editor or IDE (VS Code recommended for Dockerfile support) -- Terminal access for command-line operations - -**CANFAR-specific requirements:** -- Harbor registry access ([images.canfar.net](https://images.canfar.net/)) -- Understanding of CANFAR storage mounting (`/arc/`, `/scratch/`) -- Knowledge of target session types (notebook, desktop-app, headless) - -### Development Workflow Setup - -Create a structured development environment: - -```bash -# Set up development directory -mkdir ~/canfar-containers -cd ~/canfar-containers - -# Create container project -mkdir my-analysis-container -cd my-analysis-container - -# Initialize version control -git init -git remote add origin https://github.com/myteam/my-analysis-container.git - -# Create basic structure -touch Dockerfile -touch README.md -mkdir scripts/ -mkdir tests/ -mkdir docs/ - -# Create test data directories (for local testing) -mkdir test-data/ -mkdir test-home/ -``` - -**Recommended project structure:** -``` -my-analysis-container/ -├── Dockerfile # Container definition -├── README.md # Documentation and usage -├── requirements.txt # Python dependencies -├── environment.yml # Conda environment (if using) -├── scripts/ # Custom scripts to include -├── tests/ # Container functionality tests -├── docs/ # Additional documentation -├── test-data/ # Sample data for testing -├── test-home/ # Mock user home for testing -└── .github/workflows/ # CI/CD automation (optional) -``` - -## 🏗️ Container Development Process - -### Starting from CANFAR Base Images - -Always extend existing CANFAR base images rather than starting from scratch: - -```dockerfile -# For general astronomy work -FROM images.canfar.net/skaha/astroml:latest - -# For radio astronomy -FROM images.canfar.net/skaha/casa:[version] - -# For minimal environments -FROM images.canfar.net/skaha/base:latest - -``` - -### Basic Dockerfile Patterns - -#### Notebook Container Extension +## Minimal Dockerfile ```dockerfile -FROM images.canfar.net/skaha/astroml:latest - -# Container metadata -LABEL maintainer="research-team@university.edu" -LABEL description="Custom astronomy analysis environment with X-ray tools" -LABEL version="1.0.0" - -# Install system dependencies as root -USER root - - -# Install specialized X-ray analysis tools -RUN pip install --no-cache-dir \ - xspec-models-cxc \ - pyxspec \ - sherpa +FROM python:3.13-slim -# Install custom analysis tools from source -RUN git clone https://github.com/myteam/xray-analysis-tools.git /tmp/tools && \ - cd /tmp/tools && \ - pip install --no-cache-dir -e . && \ - rm -rf /tmp/tools +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 -# Set up environment variables -ENV XRAY_TOOLS_PATH=/opt/custom-tools -ENV PYTHONPATH=${PYTHONPATH}:/opt/custom-tools +RUN python -m pip install --no-cache-dir astropy numpy pandas +COPY requirements.txt /tmp/requirements.txt +RUN python -m pip install --no-cache-dir -r /tmp/requirements.txt \ + && rm /tmp/requirements.txt +COPY workflow.py /opt/workflow/workflow.py +ENTRYPOINT ["python", "/opt/workflow/workflow.py"] ``` -#### Desktop-App Container +Use a base image appropriate to the software and architecture you need. The +example is a generic Python image, not a promise that it is the best base for a +particular astronomy application. Follow upstream licensing and installation +instructions for CASA, GPU libraries, and other system packages. -```dockerfile -FROM ubuntu:24.04 - -# Avoid prompts during installation -ENV DEBIAN_FRONTEND=noninteractive - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - # X11 and GUI libraries - libx11-6 \ - libxext6 \ - libxrender1 \ - libxtst6 \ - libxrandr2 \ - libxss1 \ - libgtk-3-0 \ - saods9 \ - xterm \ - wget \ - curl \ - vim \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Create startup script for CANFAR desktop integration -RUN mkdir -p /skaha +## Keep the image reproducible -# Create the startup script -COPY startup.sh /skaha/ +- Pin important Python and system dependencies where practical. +- Keep one logical installation in each build layer and remove package caches. +- Use `.dockerignore` to exclude `.git`, datasets, local environments, and test + output. +- Avoid `latest` in a production pipeline; record the immutable digest or a + version tag used for the run. +- Run the image as a non-root user when the base image and application support + it. -# Make startup script executable -RUN chmod +x /skaha/startup.sh +Do not bake a CADC certificate, bearer token, private key, or password into an +image. Supply runtime credentials through the supported authentication flow or +the deployment's secret mechanism. -# Set the startup script as entrypoint -ENTRYPOINT ["/skaha/startup.sh"] -``` +## Test locally -with the `startup.sh` being: +Build and run a small smoke test before pushing: ```bash -#!/bin/bash - -# Set up X11 environment -export DISPLAY=${DISPLAY:-:1} - -# Navigate to user home directory -cd /arc/home/$USER || cd /tmp - -# Launch the application -exec ds9 -title "Custom DS9 - $USER" & - -# Keep container running -wait -``` - -### Advanced Container Features - -#### Multi-stage Builds for Complex Software - -```dockerfile -# Build stage for compiling software -FROM ubuntu:24.04 AS builder - -RUN apt-get update && apt-get install -y \ - build-essential \ - cmake \ - git \ - libfftw3-dev \ - libcfitsio-dev - -# Clone and build complex software -RUN git clone https://github.com/radio-astro/complex-software.git /src -WORKDIR /src -RUN cmake . && make -j$(nproc) && make install - -# Production stage -FROM images.canfar.net/skaha/astroml:latest - -# Copy only the built binaries -COPY --from=builder /usr/local/bin/complex-software /usr/local/bin/ -COPY --from=builder /usr/local/lib/libcomplex* /usr/local/lib/ - -# Update library cache -USER root -RUN ldconfig +docker build -t canfar-workflow:test . +docker run --rm canfar-workflow:test --help ``` -#### GPU-Enabled Containers - -```dockerfile -FROM images.canfar.net/skaha/astroml-cuda:latest - -# Install additional GPU-accelerated packages -RUN pip install --no-cache-dir \ - # GPU-accelerated arrays - cupy-cuda11x \ - # GPU machine learning - rapids-singlecell \ - # GPU image processing - cucim \ - # GPU signal processing - cusignal - -RUN cd /opt/cuda_kernels && \ - nvcc -o gpu_analysis analysis.cu -lcufft -lcublas -``` +For a `headless` image, test the exact command passed after the CLI `--` +delimiter. For an interactive image, confirm that the expected application +starts in the chosen Session Kind. The image cannot be validated solely by a +successful build. -## 🧪 Testing and Debugging +## Publish and launch -### Local Testing Strategy - -Test containers thoroughly before deploying to CANFAR: +Log in to the Container Registry using its documented credentials, then push a +versioned tag: ```bash -# Build container locally -docker build -t myteam/analysis-env:test . - -# Test basic functionality -docker run --rm myteam/analysis-env:test python -c "import astropy; print('Astropy works!')" - -# Test with mounted directories (simulate CANFAR environment) -docker run -it --rm \ - -v $(pwd)/test-data:/arc/projects/test \ - -v $(pwd)/test-home:/arc/home/testuser \ - -e USER=testuser \ - -e HOME=/arc/home/testuser \ - myteam/analysis-env:test \ - /bin/bash +docker tag canfar-workflow:test images.canfar.net//workflow:0.1.0 +docker push images.canfar.net//workflow:0.1.0 ``` -### Testing Notebook Containers +Confirm that the image is visible to the account that will launch it, then use +the exact published name: ```bash -# Test Jupyter startup -docker run -it --rm \ - -p 8888:8888 \ - -v $(pwd)/test-notebooks:/arc/home/testuser \ - -e USER=testuser \ - myteam/analysis-env:test \ - jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root +canfar image ls +canfar create headless images.canfar.net//workflow:0.1.0 \ + -- python /arc/projects//run.py ``` -### Testing Desktop-App Containers +The image project and tag are deployment data. Replace the placeholders and do +not copy an example registry path unchanged. -```bash -# Test X11 application (requires X11 forwarding setup) -docker run -it --rm \ - -e DISPLAY=${DISPLAY} \ - -v /tmp/.X11-unix:/tmp/.X11-unix \ - myteam/desktop-app:test \ - xterm -``` +## Storage and runtime behavior -### Automated Testing Framework +The image is not a data archive. In a Session, use mounted `/arc` paths for +persistent files and `/scratch` for temporary staging. A custom image should +not assume that a particular user's data, project, or VOSpace path exists. -Create test scripts to validate container functionality: +For remote VOSpace data, use `canfar data` or the explicit +`canfar.storage.filesystem(identifier)` helper and stage path-oriented inputs +to `/scratch`. See [Storage](../storage/index.md). -```python -# tests/test_container.py -import subprocess -import pytest +## Troubleshooting -def test_python_packages(): - """Test that required Python packages are installed.""" - packages = ['astropy', 'numpy', 'scipy', 'matplotlib'] - - for package in packages: - result = subprocess.run([ - 'docker', 'run', '--rm', 'myteam/analysis-env:test', - 'python', '-c', f'import {package}; print(f"{package} version: {{package.__version__}}")' - ], capture_output=True, text=True) - - assert result.returncode == 0, f"Package {package} not available" - print(result.stdout) - -def test_custom_scripts(): - """Test that custom analysis scripts work.""" - result = subprocess.run([ - 'docker', 'run', '--rm', 'myteam/analysis-env:test', - 'python', '/opt/custom-tools/test_analysis.py' - ], capture_output=True, text=True) - - assert result.returncode == 0, "Custom analysis script failed" - assert "Analysis completed" in result.stdout - -def test_file_permissions(): - """Test that file permissions work correctly.""" - result = subprocess.run([ - 'docker', 'run', '--rm', - '-v', '$(pwd)/test-data:/arc/projects/test', - 'myteam/analysis-env:test', - 'ls', '-la', '/arc/projects/test' - ], capture_output=True, text=True) - - assert result.returncode == 0, "Cannot access mounted directories" - -if __name__ == "__main__": - pytest.main([__file__]) -``` +| Symptom | Check | +| --- | --- | +| Build cannot install a dependency | Confirm the base image architecture, package name, and upstream installation instructions. | +| Image starts but command is missing | Test the entrypoint and command locally; check the Session Kind. | +| Pull is denied | Confirm the image name and project membership/registry credentials. | +| Session remains Pending | Check `canfar events SESSION_ID`; image pull and resource admission happen after creation. | +| Data is missing | Use a mounted `/arc` path or an explicit transfer; image layers do not contain Session storage. | -### Debugging Common Issues +## Related guides - -#### Package Installation Failures - -```dockerfile -# Clean package caches to reduce image size and avoid corruption -RUN apt-get update && apt-get install -y package1 package2 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/* /var/tmp/* - -# if necessary, you can pin exact package versions to avoid conflicts -RUN pip install --no-cache-dir \ - astropy==5.3.4 \ - numpy==1.24.3 \ - scipy==1.10.1 -``` - -#### Container Size Issues - -```dockerfile -# Use multi-stage builds -FROM ubuntu:24.04 AS builder -# ... build software ... - -FROM images.canfar.net/skaha/astroml:latest -COPY --from=builder /output /final-location - -# Minimize layers -RUN apt-get update && apt-get install -y pkg1 pkg2 pkg3 && apt-get clean && rm -rf /var/lib/apt/lists/* -# Instead of: -# RUN apt-get update -# RUN apt-get install -y pkg1 -# RUN apt-get install -y pkg2 -``` - -## 📦 Building and Optimization - -### Efficient Docker Practices - -#### Layer Optimization - -```dockerfile -# Good: Combine related operations -RUN apt-get update && apt-get install -y \ - package1 \ - package2 \ - package3 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Good: Order layers by change frequency -FROM base-image -# System packages (change rarely) -RUN apt-get update && apt-get install -y system-packages -# Python packages (change occasionally) -RUN pip install stable-packages -# Custom code (changes frequently) -COPY . /app/ -``` - -#### Size Minimization - -```dockerfile -# Use .dockerignore to exclude unnecessary files -# .dockerignore contents: -# .git -# *.md -# tests/ -# docs/ -# .DS_Store -# __pycache__ - -# Clean up in same layer -RUN apt-get update && apt-get install -y packages \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf /tmp/* /var/tmp/* - -# Use --no-cache-dir for pip -RUN pip install --no-cache-dir package-name -``` - -### Performance Optimization - -#### Parallel Builds - -```bash -# Build with multiple cores -docker build --build-arg MAKEFLAGS=-j$(nproc) . - -# Use BuildKit for faster builds -export DOCKER_BUILDKIT=1 -docker build . -``` - -#### Build Arguments for Flexibility - -```dockerfile -# Flexible package versions -ARG PYTHON_VERSION=3.11 -ARG ASTROPY_VERSION=5.3.4 - -FROM python:${PYTHON_VERSION}-slim - -RUN pip install --no-cache-dir astropy==${ASTROPY_VERSION} -``` - -```bash -# Build with custom arguments -docker build --build-arg PYTHON_VERSION=3.10 --build-arg ASTROPY_VERSION=5.2.0 . -``` - -### Version Management and Tagging - -```bash -# Build with specific tags -docker build -t myteam/analysis-env:latest . -docker build -t myteam/analysis-env:v1.2.3 . -docker build -t myteam/analysis-env:2024.03 . - -# Tag for Harbor registry -docker tag myteam/analysis-env:latest images.canfar.net/myteam/analysis-env:latest -docker tag myteam/analysis-env:v1.2.3 images.canfar.net/myteam/analysis-env:v1.2.3 -``` - -## 🚀 Publishing to Harbor Registry - -### Registry Authentication - -```bash -# Login to CANFAR Harbor registry -docker login images.canfar.net - -# Or use credentials directly -echo "your-harbor-password" | docker login images.canfar.net -u your-harbor-username --password-stdin -``` - -### Pushing Images - -```bash -# Push specific version -docker push images.canfar.net/myteam/analysis-env:v1.2.3 - -# Push latest -docker push images.canfar.net/myteam/analysis-env:latest - -# Push all tags -docker push --all-tags images.canfar.net/myteam/analysis-env -``` +- [Container Images](index.md) +- [Registry](registry.md) +- [Batch processing](../sessions/batch.md) +- [Storage](../storage/index.md) diff --git a/docs/platform/containers/index.md b/docs/platform/containers/index.md index e29bf723..d5eb301e 100644 --- a/docs/platform/containers/index.md +++ b/docs/platform/containers/index.md @@ -1,456 +1,84 @@ -# Containers +# Container Images -**Working and building software containers on CANFAR.** +A Container Image bundles the operating-system libraries, astronomy software, +and application entrypoint used by a Session. The image is the software +environment; the Session request supplies the Session Kind, resources, command, +and mounted storage. -!!! abstract "🎯 Container Guide Overview" - **Master CANFAR's containerized environments:** - - - **[Container Concepts](#what-are-containers)**: Understanding reproducible software environments - - **[Available Containers](#canfar-supported-containers)**: Pre-built astronomy software stacks - - **[Container Building](build.md)**: Creating custom environments for specialised workflows - - **[Registry Management](registry.md)**: Harbor registry access and image distribution +## Choose an image -Containers provide pre-packaged software environments that include everything needed to run astronomy applications. On CANFAR, containers eliminate the "works on my machine" problem by ensuring consistent, reproducible computational environments across different sessions and workflows. - -## 📋 What Are Containers? - -Think of containers as complete software packages that bundle an operating system (typically Ubuntu Linux), astronomy software like CASA or Python packages, programming tools, system libraries, and environment configuration into a single portable unit. When you launch a session on CANFAR, you're essentially starting up one of these pre-configured environments with your data and home directory automatically mounted and accessible. - -!!! success "Key Concept: Reproducible Environments" - Containers provide consistent, reproducible software environments for astronomy work across sessions and teams. - -### Why Containers Matter for Astronomy - -#### Traditional Software Installation - -- Struggle with dependencies and conflicting versions -- Missing libraries and system requirements -- Different behaviour across different machines -- Time-consuming setup and configuration - -#### CANFAR Containers - -- **Consistent environment**: Works the same everywhere -- **Pre-configured**: Astronomy packages included -- **No installation hassles**: Ready to use immediately -- **Easy sharing**: Reproducible results across teams - -!!! tip "Research Reproducibility" - Containers ensure your analysis runs the same way for you, your collaborators, and future researchers. This is crucial for reproducible science. - -### Container Architecture on CANFAR - -The container ecosystem on CANFAR follows a layered approach: - -```mermaid -graph TB - BaseOS[Ubuntu Linux Base] --> SystemLibs["OS Packages"] - SystemLibs --> CondaPython["conda-forge packages"] - - CondaPython --> Astroml["astroml Container"] - Astroml --> Casa["casa Container"] - CondaPython --> Custom[Custom Containers] - - Runtime[CANFAR Runtime] --> Storage[Storage Mounting] - Runtime --> UserContext[User Context] - Runtime --> Resources[Resource Allocation] -``` - -**Base containers** provide fundamental tools and the conda package manager, while **specialised containers** build upon these foundations to offer domain-specific software stacks. This architecture ensures consistency while allowing flexibility for different research needs. - -## 🏗️ Build Time vs Runtime - -Understanding the distinction between build time and runtime is crucial for effective container usage: - -### Build Time - -**What happens when containers are created:** - -- **Base image selection**: Choose Ubuntu, Python, or specialised astronomy base -- **Software installation**: Install system packages, Python libraries, astronomy tools -- **Environment configuration**: Set up paths, environment variables, user permissions -- **Code packaging**: Include stable scripts and analysis tools -- **Image optimization**: Layer caching, size reduction, security patches - -```dockerfile -# Build time example -FROM ubuntu:24.04 - -# Install system dependencies (build time) -RUN apt-get update && apt-get install -y \ - python3-dev \ - libcfitsio-dev \ - && apt-get clean - -# Install Python packages (build time) -RUN pip install astropy numpy matplotlib - -# Package stable code (build time) -COPY analysis_tools/ /opt/tools/ -``` - -### Runtime - -**What happens when you launch a session:** - -- **User context**: Container runs as your CADC username (not root) -- **Storage mounting**: `/arc/home`, `/arc/projects`, `/scratch` mounted automatically -- **Resource allocation**: CPU, memory, GPU assigned based on session request -- **Network access**: Internet connectivity for downloading data or documentation -- **Session integration**: Jupyter, desktop, or headless execution mode activated +List images that support a Session Kind before creating a Session: ```bash -# Runtime environment (inside your running container) -echo $USER # [user] -echo $HOME # /arc/home/[user] -ls /arc/projects/ # Your accessible project directories -df -h /scratch/ # Temporary high-speed storage +canfar image ls --kind notebook +canfar image ls --kind headless +canfar create notebook IMAGE_NAME --name analysis ``` -!!! warning "Persistence Boundary" - **Build time changes** are permanent and part of the container image. **Runtime changes** (like `pip install --user package`) are on `/arc` and not on the container. Keep stable software in the image; keep development scripts in `/arc/home/[user]` or `/arc/projects/[project]`. +The image list and Science Portal are authoritative for available names, +versions, and supported kinds. Examples in these docs are illustrative and may +not exist in every deployment. A private image also requires access to its +Container Registry project. -## 🔗 How Containers Relate to Sessions +## Build time and run time -CANFAR containers are designed to work seamlessly with different session types, each optimized for specific workflows: - -### Session Type Integration - -```mermaid -graph TB - Container[Container Image] --> SessionType{Session Type} - - SessionType --> Notebook[notebook] - SessionType --> Desktop[desktop] - SessionType --> Carta[carta] - SessionType --> Firefly[firefly] - SessionType --> Contributed[contributed] - SessionType --> Batch[headless] - - Notebook --> JupyterLab[JupyterLab Interface] - Desktop --> DesktopEnv[noVNC Ubuntu GUI] - Carta --> CartaWeb[CARTA Interface] - Firefly --> FireflyWeb[Firefly Interface] - Contributed --> WebApp[Custom Web Interface] - Batch --> ScriptExec[Script Execution] - - JupyterLab --> Storage[ \/arc and \/scratch Storage] - DesktopEnv --> Storage - CartaWeb --> Storage - FireflyWeb --> Storage - WebApp --> Storage - ScriptExec --> Storage -``` +Build-time changes are part of the image and can be reproduced by anyone with +the Dockerfile and build inputs. Run-time changes inside a Session are not an +image release. Keep stable dependencies in the image and save scripts or +configuration under persistent `/arc` storage. -Same container, different interfaces: The `astroml` container can run as a **notebook** (`JupyterLab`) session, as a **desktop app**lication (as an `xterm`) in a **desktop** session, or batch job (**headless**) session as an executable script. - -### Notebook Sessions - -**Requirements for notebook containers:** - -1. **JupyterLab installed**: The `jupyterlab` package must be installed and in path in the container -2. **Container labelling**: Tagged as **notebook** in the registry - -When you launch a notebook session, CANFAR automatically: - -- Starts JupyterLab on dedicated port -- Mounts your storage directories -- Provides web-based access to the Python environment - -**The `astroml` container** exemplifies this perfectly - it's a comprehensive Python astronomy stack with `astropy`, `scipy`, `pandas`, `matplotlib`, `numpy`, `scikit-learn`, `pytorch` and many more packages pre-installed. - -```bash -# Check installed packages in a running astroml container -mamba list # conda/mamba/pip installed system packages -apt list --installed # OS system packages (Ubuntu) -ls /build_info/ # Container build information -``` - -**Runtime package installation:** - -```bash -# Install Python packages at runtime (will install to /arc/home/[user]/.local) -# in non-astroml containers, add the --user flag. -pip install fireducks - -# Show where it was installed -pip show -f fireducks # Should show ~/.local/lib/python*/ -``` - -!!! tip "GPU Support" - For GPU acceleration, use the `astroml-cuda` container which extends `astroml` with CUDA libraries and GPU-enabled `pytorch`, `pyarrow`, and many other CUDA-capable libraries. - -### Desktop Container - -There is a single Desktop container maintained by CANFAR that provides a full Ubuntu desktop environment with GUI applications like Firefox, file managers, and terminal access. This container is ideal for workflows requiring graphical interfaces, such as legacy astronomy software or interactive data visualization tools. - -The Desktop Session provides access to the full desktop environment through a VNC connection in your browser via a web-based VNC client. - -#### Technical Details -Given that Desktop Application run in their own containers in the cluster (see [below](#desktop-app-containers)), the Desktop Session is hard-coded with low resource requirements: -```yaml -resources: - requests: - memory: "1Gi" - cpu: "250m" - ephemeral-storage: "2Gi" - limits: - memory: "4Gi" - cpu: "1" - ephemeral-storage: "10Gi" -``` - -The Desktop container uses SupervisorD to manage two main processes: -- [TigerVNC](https://tigervnc.org/) server for remote desktop access -- [noVNC](https://novnc.com/info.html) web client for browser-based access - - The noVNC client connects to the TigerVNC server over WebSockets, and has a slightly customized interface to better fit within the CANFAR web portal. - -Other technical features: -- X11 with remote sharing enabled to allow multiple simultaneous connections to the same desktop session (through `xhost`). Wayland support is disabled. -- Ubuntu 24.04 LTS base with standard desktop packages and CANFAR-specific configurations. -- Auto (remote) resize of the desktop session to fit the browser window. - -#### Desktop-App Containers - -Specialised containers that run specific GUI applications within desktop sessions. - -**Requirements for desktop-app containers:** - -1. **X11/Xorg application**: Must have at least one GUI application installed and available -2. **Startup script**: Application launcher at `/skaha/startup.sh` (if not specified, assumed to be `xterm` which must be installed in the container) -3. **Container labelling**: Tagged as `desktop-app` in the registry - -**How it works:** - -- Each desktop-app container runs on its own worker node -- Applications connect to your desktop session via X11 forwarding -- Shared storage provides data access across all containers -- Applications appear in the **Astro Software** menu - -```bash -# Example desktop-app startup script (/skaha/startup.sh) -#!/bin/bash -export DISPLAY=${DISPLAY} -cd /arc/home/$USER -exec your-gui-application -``` - -The same **astroml** container can run as both notebook (has JupyterLab) and desktop-app (has xterm), demonstrating the flexibility of container usage. - -### Batch/Headless Sessions - -Headless containers execute without graphical interfaces, perfect for automated processing: - -- **No GUI requirements**: Command-line tools only -- **Script execution**: Runs your specified command and exits -- **Background processing**: Perfect for large datasets and automation -- **Resource optimization**: Can use different resource priorities - -```bash -# Example headless execution -python /arc/projects/[project]/scripts/reduce_data.py --input=/arc/projects/[project]/data/ --output=/arc/projects/[project]/results/ -``` - -### Contributed Application Sessions - -Contributed applications are custom web-based tools that integrate with CANFAR: - -**Requirements:** - -1. **Web service**: Application serves HTTP on port 5000 -2. **Startup script**: Service launcher at `/skaha/startup.sh` -3. **Container labelling**: Tagged appropriately for discovery as **contributed**. - -Examples include Marimo (reactive notebooks) and VSCode (browser IDE). - -## 💾 Storage Mounting and Integration - -### CANFAR Storage Integration - -CANFAR automatically mounts storage systems into your container at runtime, providing seamless access to persistent data: - -```mermaid -graph LR - Container[Running Container] --> ArcMount["/arc"] - - ArcMount --> Home["/arc/home/[user]"] - ArcMount --> Projects["/arc/projects/[project]"] - - Container --> Scratch["/scratch"] - - Home --> HomeData["Personal Data 10GB Quota"] - Projects --> ProjectData["Shared Project Data Variable Quota"] - Scratch --> FastStorage["Fast Temporary Storage Node-local"] -``` - -#### Storage Hierarchy - -| Mount Point | Purpose | Persistence | Quota | Sharing | -|-------------|---------|-------------|-------|---------| -| `/arc/home/[user]` | Personal files, notebooks, configs | Permanent | 10GB | Private | -| `/arc/projects/[project]` | Research data, collaboration | Permanent | Variable | Team-based | -| `/scratch` | High-speed processing | Session only | Node-dependent | Private | - -#### Storage Best Practices - -**Personal Development (`/arc/home`):** - -```bash -/arc/home/[user]/ -├── notebooks/ # Jupyter notebooks -├── scripts/ # Analysis scripts -├── .local/ # pip install --user packages -├── .config/ # Application configurations -└── small_datasets/ # Personal research data -``` - -**Project Collaboration (`/arc/projects`):** +```dockerfile +FROM python:3.13-slim -```bash -/arc/projects/[project]/ -├── raw_data/ # Input datasets -├── processed/ # Reduced data products -├── scripts/ # Shared analysis code -├── docs/ # Project documentation -└── results/ # Final outputs +RUN python -m pip install --no-cache-dir astropy numpy +COPY reduce.py /opt/workflow/reduce.py +ENTRYPOINT ["python", "/opt/workflow/reduce.py"] ``` -**Temporary Processing (`/scratch`):** - -```bash -# Copy large datasets to fast storage for processing -cp /arc/projects/[project]/large_data.fits /scratch/ -process_data /scratch/large_data.fits /scratch/output.fits -cp /scratch/output.fits /arc/projects/[project]/results/ -``` +The base image, packages, and entrypoint must match the Session Kind. A +`headless` image must provide the command that the batch request invokes; a +Notebook or application image must provide the application expected by its +Session launcher. -### User Context and Permissions +## Runtime storage -Containers run with your CADC user identity, not as root or container-defined users: +When the deployment provides them, Sessions mount persistent `/arc` paths and +ephemeral `/scratch` storage. The image does not own those data lifetimes: -```bash -# Inside any CANFAR container -whoami # [user] -id # uid=1234([user]) gid=1234([user]) groups=[user] (and-all-your-CADC-groups) -echo $HOME # /arc/home/[user] -groups # Shows your CANFAR project group memberships +```text +/arc/home/ persistent personal files +/arc/projects/ persistent shared project data +/scratch Session-local staging and intermediates ``` -**Security model:** - -- **No root access**: Containers cannot perform system administration at runtime. -- **File permissions**: Respect standard Unix permissions on `/arc` -- **Group membership**: Access to `/arc/projects` based on CANFAR group membership -- **Network isolation**: Containers have internet access but cannot access other users' sessions - -## 🔧 CANFAR-Supported Containers - -The CANFAR team maintains several core containers that cover most astronomy research needs in the **`skaha`** namespace: - -| Container | Description | -|-----------|-------------| -| **base** | Basic UNIX tools, conda, CADC packages | -| **astroml** | Many astro (STILTS, astropy ecosystem), data sciences (pandas, pyarrow,...), machine learning (sklearn, pytorch) packages. JupyterLab, xterm. | -| **marimo** | Same as astroml stack, with marimo notebook as web interface | -| **vscode** | Same as astroml, with VSCode on browser as interface | -| **\*-cuda** | Same as all above containers, with CUDA-enabled | -| **improc** | Image processing tools (SWarp, SExtractor, SourceExtractor++, IRAF, CASUTools...) | -| **casa** | CASA installations | - -### Visualisation Containers - -#### `carta` - Radio Astronomy Visualisation - -**Purpose**: Interactive visualisation of radio astronomy data - -**Features:** - -- **CARTA application**: Cube Analysis and Rendering Tool for Astronomy -- **Multi-dimensional data**: Spectral cubes, moment maps, polarisation -- **Interactive analysis**: Region statistics, profile extraction -- **Collaboration support**: Session sharing capabilities - -#### `firefly` - Catalogue Data Analysis - -**Purpose**: Advanced catalogue queries and visualisation - -**Features:** - -- **Multi-mission support**: LSST, Spitzer, WISE, 2MASS, ... -- **Interactive catalogues**: Source overlays and cross-matching -- **Multi-wavelength workflows**: RGB composites and band comparisons -- **Large dataset handling**: Efficient rendering of survey-scale data - -### Development and Desktop Containers +Use `/scratch` for fast temporary work and copy results to `/arc` or a +persistent VOSpace Service before stopping the Session. See [Storage](../storage/index.md). -#### `desktop` - Ubuntu Environment +## Resource and security considerations -**Purpose**: Complete Linux desktop for GUI applications and legacy software. Astronomy software applications will each run on dedicated nodes. +- Keep credentials out of Dockerfiles, image layers, and command arguments. +- Pin important dependencies and rebuild when security fixes are needed. +- Keep images small so pulls and startup do not dominate short workloads. +- Request only the CPU, memory, and GPU that measurements support. +- Test the exact image with a small input before launching replicas. -**Features:** +## Build and publish -- **Ubuntu**: Linux environment -- **Desktop environment**: Full GNOME-based interface -- **Applications**: Firefox, file managers, terminals, editors -- **X11 forwarding**: Support for launching astronomy GUI applications - -#### `notebook` - Jupyter Environment - -**Purpose**: Minimal Jupyter environment for basic Python work - -**Features:** - -- **Jupyter Lab**: Web-based notebook interface -- **Extensible**: Foundation for custom development -- **Fast startup**: Minimal software for quick sessions - -### Container Selection Guide - -| Workflow Type | Recommended Container | Session Type | Typical Resources | -|---------------|----------------------|--------------|-------------------| -| **Python core** | `base` | Headless | 1 core, 1GB | -| **Python data analysis** | `astroml` | Notebook | 2-4 cores, 8-16GB | -| **GPU machine learning** | `astroml-cuda` | Notebook | 4-8 cores, 16-32GB, 1 GPU | -| **Radio interferometry** | `casa` | Notebook/Desktop | 4-8 cores, 16-32GB | -| **Data visualisation** | `carta` | CARTA session | 2-4 cores, 8-16GB | -| **Catalogue analysis** | `firefly` | Firefly session | 2-4 cores, 8-16GB | -| **GUI applications** | `desktop` | Desktop | 2-4 cores, 8-16GB | -| **Legacy software** | `desktop` | Desktop | Variable | -| **Batch processing** | `astroml` or `casa` | Headless | Variable | - -!!! tip "Container Selection Strategy" - Start with `astroml` for most astronomy work. It includes comprehensive libraries and is actively maintained. Use specialised containers (`casa`, `carta`, `firefly`) only when you need their specific tools. - -### Version Management - -CANFAR containers follow semantic versioning: - -- **`:latest`** - Current stable release (recommended for most work) -- **`:YY.MM`** - Monthly snapshots for reproducibility -- **`:commit-hash`** - Specific builds for exact reproducibility +Use the build toolchain documented by your registry or deployment. A generic +local workflow is: ```bash -# Use latest stable version (recommended) -images.canfar.net/skaha/astroml:latest - -# Use specific monthly snapshot for reproducible research -images.canfar.net/skaha/astroml:25.09 - -# Use exact commit for critical reproducibility -images.canfar.net/skaha/astroml:a1b2c3d4 +docker build -t images.canfar.net//: . +docker push images.canfar.net//: ``` -### Container Updates and Maintenance - -CANFAR containers receive regular updates: - -**Monthly releases**: Security patches, library updates, new features -**Quarterly reviews**: Major version updates, new software additions -**Community feedback**: Feature requests and bug reports incorporated - -**Update notifications:** +The registry project must grant the account permission to push. Use a stable +version tag for reproducible workflows and reserve moving tags such as `latest` +for development. See [Building Containers](build.md) and [Registry](registry.md). -- Science Portal notifications for major changes +## Related guides -!!! warning "Version Pinning for Reproducibility" - For published research, specify exact container versions (monthly tags or commit hashes) to ensure long-term reproducibility of your analysis. +- [Building Containers](build.md) +- [Container Registry](registry.md) +- [Sessions](../sessions/index.md) +- [Batch processing](../sessions/batch.md) diff --git a/docs/platform/containers/registry.md b/docs/platform/containers/registry.md index 2f32a542..355fdae5 100644 --- a/docs/platform/containers/registry.md +++ b/docs/platform/containers/registry.md @@ -1,838 +1,56 @@ -# Harbor Container Registry +# Container Registry -**Managing container images on CANFAR's Harbor-based registry for secure distribution and deployment.** +A Container Image is pulled from the registry named by its image reference. +The Science Platform may provide public images and project-scoped private +images; availability and permissions are deployment-specific. Use the image +listing and the project’s published instructions as the source of truth. -!!! abstract "🎯 Registry Management Overview" - **Master Harbor registry operations:** - - - **Harbor Platform**: Understanding CANFAR's enterprise container registry features - - **Access Control**: Projects, repositories, and role-based permissions - - **Image Management**: Tagging, versioning, and metadata organization - - **Security Features**: Vulnerability scanning and compliance monitoring +## Find an image - -Harbor serves as CANFAR's container registry, providing a secure, feature-rich platform for storing, managing, and distributing container images. Built on Docker Registry v2, Harbor adds enterprise features like role-based access control, vulnerability scanning, and image replication. - -## 📋 Harbor Registry Overview - -### What is Harbor? - -[Harbor](https://goharbor.io/) is an open-source container registry that provides: - -- **Secure storage**: Role-based access control and authentication integration -- **Image management**: Tagging, versioning, and metadata handling -- **Vulnerability scanning**: Automated security analysis of container images -- **Replication**: Multi-site synchronisation and backup capabilities -- **Web interface**: User-friendly management portal -- **API access**: Programmatic integration with development workflows - -### CANFAR Harbor Instance - -**Registry URL**: [https://images.canfar.net](https://images.canfar.net) - -**Key features on CANFAR:** - -- Integration with CADC authentication system -- Project-based organization for research teams -- Automated vulnerability scanning for public containers -- Role-based permissions aligned with CANFAR groups -- API access for automated workflows - -```mermaid -graph TB - Users[CANFAR Users] --> Harbor[Harbor Registry] - Harbor --> Projects[Projects/Organizations] - - Projects --> Public[Public Projects] - Projects --> Private[Private Projects] - - Public --> CADCContainers[CADC/Official Containers] - Public --> CommunityContainers[Community Containers] - - Private --> TeamProjects[Research Team Projects] - Private --> PersonalProjects[Individual Projects] - - Harbor --> Features[Harbor Features] - Features --> RBAC[Role-Based Access] - Features --> Scanning[Vulnerability Scanning] - Features --> Replication[Image Replication] - Features --> API[REST API] -``` - -## 🏗️ Projects and Organization - -### Project Structure - -Harbor organizes containers into **projects**, which serve as top-level namespaces: - -```text -images.canfar.net/ -├── skaha/ # Official CANFAR containers -│ ├── astroml:latest -│ ├── casa:latest -│ └── desktop:latest -├── [project]/ # Research team project -│ ├── custom-pipeline:latest -│ └── analysis-env:v2.1 -└── [user]/ # Personal project - ├── development:latest - └── testing:experimental -``` - -### Project Types - -#### Public Projects - -**Characteristics:** - -- Visible to all CANFAR users -- Images can be pulled without authentication -- Suitable for community-shared tools -- Used for official CANFAR containers - -**Examples:** - -- `skaha/` - Core CANFAR containers -- `lsst/` - LSST Community-contributed containers - -#### Private Projects - -**Characteristics:** - -- Access restricted to project members -- Require authentication for all operations -- Support proprietary or sensitive software -- Can be shared with specific research teams - -**Examples:** - -- `myuniversity-xray/` - Institutional X-ray analysis tools -- `survey-collaboration/` - Multi-institutional survey project -- `proprietary-software/` - Licensed commercial software - -### Project Creation and Management - -#### Requesting New Projects - -Contact [support@canfar.net](mailto:support@canfar.net) to create new projects: - -**Required information:** - -- Project name (must be unique, lowercase, alphanumeric) -- Description and purpose -- Visibility (public/private) -- Initial project members and their roles -- Resource requirements (storage quota) - -**Naming conventions:** - -```bash -# Good project names -myteam-radio-analysis -survey-processing-tools -xray-spectroscopy - -# Avoid -MyTeam_Radio_Analysis # Mixed case, underscores -my team radio # Spaces -special-chars-@#$ # Special characters -``` - -#### Project Membership Management - -Project owners can manage membership through the Harbor web interface: - -**Role levels:** - -- **Guest**: Pull images only -- **Developer**: Pull and push images, manage tags -- **Master**: Full project management, member administration -- **ProjectAdmin**: Complete project control including deletion - -## 🗂️ Repository Management - -### Understanding Repositories - -Within each project, **repositories** contain the actual container images: - -```text -[project]/ # Project -├── analysis-pipeline/ # Repository -│ ├── latest # Tag -│ ├── v1.0.0 # Tag -│ └── 2024.03 # Tag -└── visualization-tools/ # Repository - ├── latest # Tag - └── beta # Tag -``` - -### Repository Naming - -Follow consistent naming conventions for repositories: - -```bash -# Good repository names -analysis-pipeline -data-processing-tools -visualization-suite -radio-astronomy-env - -# Specific use cases -survey-reduction-v2 -xray-spectral-analysis -machine-learning-gpu -``` - -### Tagging Strategy - -Implement systematic tagging for version control: - -#### Semantic Versioning - -```bash -# Major.Minor.Patch format -myteam/analysis-env:1.0.0 # Initial release -myteam/analysis-env:1.1.0 # New features -myteam/analysis-env:1.1.1 # Bug fixes -myteam/analysis-env:2.0.0 # Breaking changes -``` - -#### Date-Based Versioning +List images before creating a Session: ```bash -# Monthly releases -myteam/analysis-env:2024.03 # March 2024 release -myteam/analysis-env:2024.04 # April 2024 release - -# Daily builds (development) -myteam/analysis-env:2024.03.15 -myteam/analysis-env:2024.03.16 +canfar image ls +canfar image ls --kind headless ``` -#### Feature and Environment Tags - -```bash -# Environment-specific -myteam/analysis-env:production -myteam/analysis-env:development -myteam/analysis-env:testing - -# Feature branches -myteam/analysis-env:feature-gpu-support -myteam/analysis-env:experimental-ml - -# Special purpose -myteam/analysis-env:conference-demo -myteam/analysis-env:paper-reproduction -``` - -### Managing Image Metadata - -Harbor stores rich metadata for each image: - -```json -{ - "name": "analysis-env", - "version": "v1.2.0", - "description": "Custom astronomy analysis environment", - "created": "2024-03-15T10:30:00Z", - "size": "2.1GB", - "labels": { - "maintainer": "research-team@university.edu", - "version": "1.2.0", - "description": "X-ray astronomy analysis with XSPEC", - "ca.nrc.cadc.skaha.type": "notebook" - }, - "vulnerabilities": "scanned", - "signature": "verified" -} -``` - -## 🔐 Access Control and Security - -### Authentication Methods - -#### Web Interface Access +Use the complete image reference returned by that command. Keep a stable tag +or digest in reproducible workflows rather than relying on `latest`. -1. **Visit**: [https://images.canfar.net](https://images.canfar.net) -2. **Login**: Use your CADC credentials -3. **Navigate**: Browse projects and repositories -4. **Manage**: Create, tag, and delete images (with appropriate permissions) +## Publish an image -#### Docker CLI Authentication +Container builds and registry credentials are controlled by the deployment. +When a project gives you a registry endpoint, use the standard container tools +with that endpoint and follow its access instructions: ```bash -# Login to Harbor registry -docker login images.canfar.net -# Enter CADC username and password when prompted -# Paste the Habor CLI Secret copied from the images.canfar.net User Profile-> - -# Verify authentication -docker info | grep -A 5 "Registry Mirrors" +docker build -t REGISTRY/PROJECT/IMAGE:TAG . +docker push REGISTRY/PROJECT/IMAGE:TAG ``` -#### API Access - -```bash -# Get authentication token -curl -X POST "https://images.canfar.net/api/v2.0/users/current" \ - -u "username:password" \ - -H "accept: application/json" - -# Use token for API calls -curl -X GET "https://images.canfar.net/api/v2.0/projects" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -H "accept: application/json" -``` - -### Permission Matrix - -| Action | Guest | Developer | Master | ProjectAdmin | -|--------|-------|-----------|---------|--------------| -| **View public projects** | ✅ | ✅ | ✅ | ✅ | -| **View private projects** | ❌ | ✅* | ✅* | ✅* | -| **Pull images** | ✅** | ✅ | ✅ | ✅ | -| **Push images** | ❌ | ✅ | ✅ | ✅ | -| **Delete images** | ❌ | ❌ | ✅ | ✅ | -| **Manage tags** | ❌ | ✅ | ✅ | ✅ | -| **Scan images** | ❌ | ✅ | ✅ | ✅ | -| **Add/remove members** | ❌ | ❌ | ✅ | ✅ | -| **Delete project** | ❌ | ❌ | ❌ | ✅ | - -*Only if member of project -**Public images only - - - -## 🔍 Harbor Web Interface - -### Navigation and Features - -#### Project Dashboard - -The project dashboard provides an overview of: +Do not put passwords or access tokens in a Dockerfile, an image layer, a +notebook, or a Session command. Prefer a short-lived credential mechanism +provided by the registry operator. -- **Repositories**: List of container repositories -- **Members**: Project access control -- **Logs**: Activity and audit trail -- **Configuration**: Project settings and policies -- **Summary**: Storage usage and statistics +## Image contents -#### Repository View +Keep images small and reproducible. Pin operating-system and language-package +inputs where practical, remove build-only material from runtime layers, and +run the image as a non-root user when the workload permits. Store data in +Science Platform storage rather than baking project data into an image. -For each repository, you can: +The registry does not make a container a Session: [create a Session](../sessions/index.md) +with an image that the target Science Platform Server can pull. -- **Browse tags**: View all available versions -- **Inspect images**: Examine layers, metadata, and vulnerabilities -- **Manage artifacts**: Add/remove tags, delete images -- **View history**: Track changes and updates -- **Configure policies**: Set retention and scanning rules +## Troubleshooting -#### Image Details +- A `not found` or pull error usually means the image reference is wrong, the + tag was removed, or the target server cannot reach that registry. +- A private image may require project membership or a server-specific login. +- A successful push does not guarantee that every Science Platform deployment + can pull the image. +- For a `Pending` Session, inspect `canfar info SESSION_ID` and + `canfar events SESSION_ID` before changing the resource request. -Each image provides detailed information: - -```yaml -Image Information: - Digest: sha256:a1b2c3d4e5f6... - Size: 2.1 GB - Created: 2024-03-15 10:30:00 UTC - OS/Arch: linux/amd64 - -Labels: - maintainer: research-team@university.edu - version: 1.2.0 - ca.nrc.cadc.skaha.type: notebook - -Vulnerabilities: - Critical: 0 - High: 1 - Medium: 3 - Low: 12 - -Build Information: - Dockerfile: Available - Build Args: Recorded - Layers: 15 layers, optimized -``` - -### Vulnerability Scanning - -Harbor automatically scans container images for security vulnerabilities: - -#### Scanning Process - -1. **Automatic scanning**: Public containers scanned on push -2. **Manual scanning**: Trigger scans for private repositories -3. **Scheduled scanning**: Regular updates with latest vulnerability database -4. **Policy enforcement**: Block pulls based on vulnerability thresholds - -#### Vulnerability Reports - -```yaml -Vulnerability Report: - Scanner: Trivy - Scan Time: 2024-03-15 11:00:00 UTC - Database Version: 2024-03-14 - -Critical Vulnerabilities: - - None found - -High Vulnerabilities: - - CVE-2024-1234: OpenSSL vulnerability - Severity: High - Package: openssl 3.0.1 - Fix: Upgrade to openssl 3.0.2 - -Medium Vulnerabilities: - - CVE-2024-5678: Python vulnerability - Severity: Medium - Package: python 3.11.1 - Fix: Upgrade to python 3.11.2 -``` - -#### Addressing Vulnerabilities - -```dockerfile -# Update base image to address vulnerabilities -FROM images.canfar.net/skaha/astroml:latest - -# Update system packages -USER root -RUN apt-get update && apt-get upgrade -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Update Python packages -RUN pip install --upgrade package-with-vulnerability -``` - -## 🛠️ CLI and API Usage - -### Harbor CLI Operations - -#### Basic Image Operations - -```bash -# List repositories in a project -curl -X GET "https://images.canfar.net/api/v2.0/projects/myteam/repositories" \ - -H "Authorization: Basic $(echo -n username:password | base64)" - -# Get repository information -curl -X GET "https://images.canfar.net/api/v2.0/projects/myteam/repositories/analysis-env" \ - -H "Authorization: Basic $(echo -n username:password | base64)" - -# List tags for a repository -curl -X GET "https://images.canfar.net/api/v2.0/projects/myteam/repositories/analysis-env/artifacts" \ - -H "Authorization: Basic $(echo -n username:password | base64)" -``` - -#### Docker Registry v2 API - -```bash -# Get manifest for specific tag -curl -X GET "https://images.canfar.net/v2/myteam/analysis-env/manifests/latest" \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - -H "Authorization: Basic $(echo -n username:password | base64)" - -# Get blob (layer) information -curl -X GET "https://images.canfar.net/v2/myteam/analysis-env/blobs/sha256:digest" \ - -H "Authorization: Basic $(echo -n username:password | base64)" -``` - -### Automated Workflows - -#### CI/CD Integration - -```yaml -# GitHub Actions example -name: Build and Push to Harbor - -on: - push: - branches: [main] - tags: ['v*'] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Login to Harbor - uses: docker/login-action@v3 - with: - registry: images.canfar.net - username: ${{ secrets.HARBOR_USERNAME }} - password: ${{ secrets.HARBOR_PASSWORD }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: images.canfar.net/myteam/analysis-env - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push - uses: docker/build-push-action@v5 - with: - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} -``` - -#### Automated Scanning and Deployment - -```python -#!/usr/bin/env python3 -# automated_scan_deploy.py - -import requests -import time -import sys -from base64 import b64encode - -class HarborManager: - def __init__(self, registry_url, username, password): - self.registry_url = registry_url - self.auth_header = self._create_auth_header(username, password) - - def _create_auth_header(self, username, password): - credentials = f"{username}:{password}" - encoded_credentials = b64encode(credentials.encode()).decode() - return {"Authorization": f"Basic {encoded_credentials}"} - - def scan_repository(self, project, repository): - """Trigger vulnerability scan for repository.""" - url = f"{self.registry_url}/api/v2.0/projects/{project}/repositories/{repository}/artifacts" - - response = requests.get(url, headers=self.auth_header) - if response.status_code != 200: - print(f"Failed to get artifacts: {response.status_code}") - return False - - artifacts = response.json() - for artifact in artifacts: - scan_url = f"{url}/{artifact['digest']}/scan" - scan_response = requests.post(scan_url, headers=self.auth_header) - - if scan_response.status_code == 202: - print(f"Scan triggered for {artifact['digest'][:12]}") - else: - print(f"Scan failed for {artifact['digest'][:12]}: {scan_response.status_code}") - - return True - - def get_vulnerability_report(self, project, repository, tag="latest"): - """Get vulnerability scan results.""" - url = f"{self.registry_url}/api/v2.0/projects/{project}/repositories/{repository}/artifacts/{tag}/scan/overview" - - response = requests.get(url, headers=self.auth_header) - if response.status_code == 200: - return response.json() - else: - print(f"Failed to get scan results: {response.status_code}") - return None - - def check_vulnerabilities(self, scan_results, max_critical=0, max_high=5): - """Check if vulnerability levels are within acceptable limits.""" - if not scan_results: - return False - - # Extract vulnerability counts - summary = scan_results.get('vulnerabilities', {}) - critical = summary.get('critical', 0) - high = summary.get('high', 0) - - print(f"Vulnerabilities found - Critical: {critical}, High: {high}") - - return critical <= max_critical and high <= max_high - -# Usage example -def main(): - harbor = HarborManager( - "https://images.canfar.net", - "your-username", - "your-password" - ) - - project = "myteam" - repository = "analysis-env" - - # Trigger scan - if harbor.scan_repository(project, repository): - print("Scan triggered, waiting for completion...") - time.sleep(60) # Wait for scan to complete - - # Check results - results = harbor.get_vulnerability_report(project, repository) - if harbor.check_vulnerabilities(results): - print("✅ Vulnerability check passed") - sys.exit(0) - else: - print("❌ Vulnerability check failed") - sys.exit(1) - -if __name__ == "__main__": - main() -``` - -## 📊 Registry Maintenance - -### Storage Management - -#### Repository Cleanup - -```bash -# Remove old tags (manual approach) -# List all tags first -docker images images.canfar.net/myteam/analysis-env - -# Remove specific old tags -docker rmi images.canfar.net/myteam/analysis-env:old-tag - -# Remove all untagged images -docker image prune -f - -# Remove all unused images -docker system prune -a -f -``` - -#### Automated Retention Policies - -Configure Harbor retention policies through web interface: - -**Policy examples:** - -- Keep latest 10 versions -- Retain images pushed within last 30 days -- Preserve all tagged releases (v*.*.*) -- Delete untagged artifacts after 7 days - -### Monitoring and Analytics - -#### Usage Statistics - -Harbor provides metrics on: - -- Storage usage per project -- Pull/push activity -- Popular repositories -- User access patterns -- Vulnerability trends - -#### Audit Logging - -Track all registry activities: - -```yaml -Audit Log Entry: - Timestamp: 2024-03-15T10:30:00Z - User: research-user - Action: PUSH - Resource: myteam/analysis-env:v1.2.0 - IP Address: 192.168.1.100 - User Agent: docker/24.0.0 - Status: SUCCESS -``` - -### Backup and Disaster Recovery - -#### Export/Import Procedures - -```bash -# Export project (including images) -# Contact CANFAR support for full project exports - -# Export image metadata only -curl -X GET "https://images.canfar.net/api/v2.0/projects/myteam/repositories/analysis-env/artifacts" \ - -H "Authorization: Basic $(echo -n username:password | base64)" \ - > repository-metadata.json - -# Backup individual images -docker pull images.canfar.net/myteam/analysis-env:v1.2.0 -docker save images.canfar.net/myteam/analysis-env:v1.2.0 > analysis-env-v1.2.0.tar - -# Restore from backup -docker load < analysis-env-v1.2.0.tar -docker push images.canfar.net/myteam/analysis-env:v1.2.0 -``` - -## 🚀 Best Practices - -### Registry Best Practices - -#### Registry Organization - -```bash -# Recommended project organization -university-astronomy/ # Institution-wide project -├── public-tools/ # Publicly available tools -├── course-materials/ # Educational containers -└── research-environments/ # General research tools - -survey-collaboration/ # Multi-institutional project -├── data-processing/ # Survey data pipeline -├── analysis-tools/ # Shared analysis software -└── visualization/ # Survey-specific viz tools - -personal-research/ # Individual development -├── experimental/ # Development and testing -├── paper-environments/ # Publication reproducibility -└── conference-demos/ # Presentation materials -``` - -#### Naming Conventions - -```bash -# Clear, descriptive names -radio-interferometry-pipeline -optical-photometry-tools -xray-spectral-analysis - -# Version-specific names for major releases -survey-pipeline-v2 -analysis-environment-2024 - -# Environment-specific variants -ml-environment-gpu -processing-tools-cpu -analysis-suite-minimal -``` - -### Performance Optimization - -#### Registry Performance - -```bash -# Use layer caching effectively -# Order Dockerfile instructions by change frequency - -# Use multi-stage builds to reduce final image size -FROM ubuntu:24.04 AS builder -# ... build dependencies ... -FROM images.canfar.net/skaha/astroml:latest -COPY --from=builder /app/binary /usr/local/bin/ - -# Optimize layer sizes -RUN apt-get update && apt-get install -y pkg1 pkg2 pkg3 \ - && apt-get clean && rm -rf /var/lib/apt/lists/* -# Instead of multiple RUN commands -``` - -#### Network Optimization - -```bash -# Use Harbor proximity -# CANFAR Harbor is optimized for Canadian academic networks - -# Leverage local caching -# Images are cached at compute nodes for faster session startup - -# Consider image size for session startup time -# Smaller images start faster, especially for batch jobs -``` - -## 🔗 Integration with CANFAR Services - - -### Storage Integration - -Containers automatically receive CANFAR storage mounts: - -```bash -# Inside any Harbor-deployed container on CANFAR -ls /arc/home/[user]/ # Personal storage -ls /arc/projects/[project]/ # Project storage -ls /scratch/ # Temporary storage -``` - -### Authentication Integration - -Harbor integrates with CADC authentication: - -- Single sign-on with CADC credentials -- Group membership determines project access -- API tokens respect CADC account policies -- Audit logs integrate with CADC security monitoring - -## 🆘 Troubleshooting - -### Common Issues - -#### Authentication Problems - -```bash -# Check Docker login status -docker system info | grep -A 5 "Registry" - -# Clear cached credentials -docker logout images.canfar.net - -# Login with explicit credentials -docker login images.canfar.net -u username - -# Test authentication -docker pull images.canfar.net/skaha/astroml:latest -``` - -#### Push/Pull Failures - -```bash -# Check repository permissions -curl -X GET "https://images.canfar.net/api/v2.0/projects/myteam" \ - -H "Authorization: Basic $(echo -n username:password | base64)" - -# Verify image format -docker inspect local-image:tag - -# Check Harbor service status -curl -X GET "https://images.canfar.net/api/v2.0/systeminfo" -``` - -#### Vulnerability Scan Issues - -```bash -# Manually trigger scan -curl -X POST "https://images.canfar.net/api/v2.0/projects/myteam/repositories/image/artifacts/tag/scan" \ - -H "Authorization: Basic $(echo -n username:password | base64)" - -# Check scan status -curl -X GET "https://images.canfar.net/api/v2.0/projects/myteam/repositories/image/artifacts/tag/scan/overview" \ - -H "Authorization: Basic $(echo -n username:password | base64)" -``` - -### Performance Issues - -#### Slow Push/Pull Operations - -```bash -# Check network connectivity -ping images.canfar.net - -# Use Docker BuildKit for faster builds -export DOCKER_BUILDKIT=1 -docker build . - -# Enable Docker layer caching -docker build --cache-from=images.canfar.net/myteam/image:latest . -``` - -#### Large Image Size - -```bash -# Analyze image layers -docker history images.canfar.net/myteam/image:latest - -# Use dive tool for detailed analysis -dive images.canfar.net/myteam/image:latest - -# Optimize Dockerfile layers -# Combine RUN commands -# Remove package caches -# Use multi-stage builds -``` -``` +See [container builds](build.md), [Session troubleshooting](../sessions/batch.md#monitor-and-troubleshoot), +and [support](../support/index.md). diff --git a/docs/platform/cvmfs.md b/docs/platform/cvmfs.md index 0cbaa90f..0a7a7653 100644 --- a/docs/platform/cvmfs.md +++ b/docs/platform/cvmfs.md @@ -1,117 +1,60 @@ -# Software Repositories (CVMFS) - -**Accessing thousands of scientific software packages through global read-only repositories.** - -!!! abstract "🎯 What is CVMFS?" - The **CernVM File System (CVMFS)** is a distributed, read-only filesystem designed to deliver software to large-scale computing environments. On CANFAR, all sessions (Notebooks, Desktops, and Batch) have access to CVMFS, providing instant access to software stacks maintained by the **Digital Research Alliance of Canada (Alliance)**. - - CVMFS is **not** a general-purpose writable project storage system: it is optimized for publishing and distributing versioned, mostly immutable software/reference trees (read-many, write-by-maintainers), while your active notebooks, code, intermediate results, and team-shared working files should live in writable storage (for example, `/arc` on CANFAR). - -## 🚀 Why Use CVMFS? - -Traditional software management often involves complex installations, dependency conflicts, and large container images. CVMFS changes this by providing: - -* **Instant Access**: Thousands of pre-built packages are available without any installation. -* **Consistency**: The same software environment used on Alliance clusters (like Fir or Nibi) is available directly in your CANFAR session. -* **Resource Efficiency**: Software is downloaded on-demand and cached, keeping container images small and fast to launch. - -## ✅ Pros and ⚠️ Trade-offs (User Perspective) - -From a user perspective, CVMFS is often the fastest way to access large shared scientific software stacks because repositories are mounted read-only, fetched on demand, and cached locally. - -**Pros** - -* **Fast access to large software stacks**: You can use pre-installed tools without rebuilding a large container or reinstalling packages in every session. -* **On-demand, cached delivery**: CVMFS fetches only the files you actually touch, then reuses them from cache, which is efficient for interactive work. -* **Consistent shared environments**: The same published software stack can be exposed across many systems, which helps reproducibility and collaboration. -* **Curated by maintainers**: Shared stacks are typically built and maintained by dedicated teams, which reduces user setup burden for common tools. -* **Works well with containers**: A small container can provide the runtime base while CVMFS provides the heavy software stack. - -**Trade-offs** - -* **Read-only applies to `/cvmfs`, not your workspace**: You cannot install into `/cvmfs`, but you can still install in your home/project space (on CANFAR, under mounted `/arc`) or build software into your own container. -* **First use can be slower**: Initial access may take longer while metadata/files are fetched and cached; repeated use is usually faster. -* **What is available depends on maintainers**: If a package/version is not published in the shared repository, you may need a local environment or custom container. -* **Site-specific tooling may vary**: On CANFAR/Alliance, using CVMFS software often involves environment modules (`module load`); other CVMFS deployments may use different setup methods. -* **Custom environments are still important**: For writable installs, rapid iteration, or tightly pinned dependencies, use a virtualenv/conda env in your workspace and/or a custom container on top of the shared stack. - -!!! warning "Common Gotcha: Browsing `/cvmfs`" - A common surprise is that running `ls /cvmfs` may appear empty. This is because repositories are mounted *lazily* only when you access a known path. You cannot simply browse `/cvmfs` like a normal directory to discover software. - - **Practical remedies:** - - * **Use documented repository paths**: On CANFAR, always start with `/cvmfs/soft.computecanada.ca/`. - * **Provide shortcuts in docs/examples**: Include copy/paste-ready `source` and `module` commands in your own team's documentation to avoid guesswork. - -## 🧠 Advanced: CVMFS + Containers (Hybrid Model) - -In modern platforms, CVMFS usually complements containers rather than replacing them: a small container image provides the base OS/runtime, CVMFS provides large shared software trees on demand, and your home/project storage remains the writable layer for notebooks, code, and custom packages. This hybrid approach improves startup time and reduces image size while preserving flexibility for user-specific environments. - -On CANFAR, CVMFS caching happens on the Kubernetes worker node and is shared by sessions running on that node. This means one user's first access to a tool may warm the cache for later sessions scheduled on the same node, while a newly scaled or different node may behave like a cold cache. - -### Platform Evolution -Where this can go next: - -* **Richer shared environments**: Publish more curated stacks (for example, domain-specific Python/Conda environments) for common workflows. -* **Team-maintained software repositories**: Small groups can publish and version their own CVMFS software trees (with appropriate operational support), reducing reliance on one central software stack. -* **Container + CVMFS hybrid workflows**: Keep containers small and stable, and move large, frequently reused software into CVMFS for faster startup and less image churn. -* **Improved cache/proxy topology**: Node-local caches plus local HTTP proxy caches can significantly reduce repeated downloads and external bandwidth usage at scale. -* **Collaborative reproducibility**: Treat CVMFS-published stacks as versioned, documented shared environments, while keeping writable collaboration artifacts (code, notebooks, data products) in `/arc` or other project storage. - -### Pointers for Advanced Readers - -* **[CVMFS Official Documentation](https://cvmfs.readthedocs.io/en/stable/)**: architecture, client/cache behaviour, and operations guidance. -* **[Kubernetes CVMFS CSI driver](https://github.com/cvmfs-contrib/cvmfs-csi)**: mounting CVMFS repositories into pods in cloud-native environments. -* **[EESSI (European Environment for Scientific Software Installations)](https://www.eessi.io/)**: an example of a large cross-site scientific software stack distributed via CVMFS. -## 🛠️ Accessing the Software - -The Alliance software stack is mounted at `/cvmfs/soft.computecanada.ca/`. Accessing it is a two-step process: - -1. **Initialize the environment**: Source the profile to enable `module` commands. -2. **Load your software**: Use `module load` to add specific packages to your path. - -### Example: Before vs. After CVMFS - -Suppose you need a specific Python version or a package not included in the standard `astroml` container. - -=== "Step 0: Before CVMFS" - Notice that the environment is limited to what is pre-installed in your container. - ```bash - # Current python version might be 3.12 - python --version - # Output: Python 3.12.x - ``` - -=== "Step 1: Enable CVMFS" - Source the Alliance bash profile to enable the environment module system. - ```bash - source /cvmfs/soft.computecanada.ca/config/profile/bash.sh - ``` - -=== "Step 2: Find & Load Software" - Search for the software you need and load it. - ```bash - # See available python versions - module avail python - - # Load Python 3.10 - module load python/3.10 - ``` - -=== "Step 3: Verification" - Verify that your environment has changed. - ```bash - python --version - # Output: Python 3.10.x - ``` - -## 🔗 Learning More - -The Alliance provides extensive documentation on their software environment. Since CANFAR mounts the same CVMFS repositories, these guides apply directly to your sessions: - -* **[Accessing CVMFS](https://docs.alliancecan.ca/wiki/Accessing_CVMFS)**: Technical overview of the filesystem. -* **[Using Modules](https://docs.alliancecan.ca/wiki/Using_modules)**: Detailed guide on the `module` command (avail, load, list, purge). -* **[Available Software](https://docs.alliancecan.ca/wiki/Available_software)**: Searchable list of the thousands of packages available via CVMFS. - -!!! tip "Persistence" - Environment changes made via `module load` are session-specific. If you want specific modules to be loaded every time you open a terminal, you can add the `source` and `module load` commands to your `~/.bashrc` file (located in `/arc/home/[user]/.bashrc`). +# Software repositories (CVMFS) + +[CVMFS](https://cvmfs.readthedocs.io/en/stable/) is a read-only, on-demand +filesystem for distributing large, versioned software trees. A Science +Platform deployment may mount one or more CVMFS repositories in Sessions. The +available repositories and software are deployment data; a missing path is not +an instruction to install a CVMFS client yourself. + +CVMFS is for shared software and reference data. Keep notebooks, source code, +intermediate files, and research products in writable [Science Platform +storage](storage/index.md), not under `/cvmfs`. + +## Check the repository + +If your Session includes the Alliance software tree, the common path is: + +```bash +ls /cvmfs/soft.computecanada.ca +source /cvmfs/soft.computecanada.ca/config/profile/bash.sh +module avail +``` + +Only run the `source` command when that path exists. The module list and +version names change as maintainers publish new software. Check the +[Alliance CVMFS](https://docs.alliancecan.ca/wiki/Accessing_CVMFS) and +[available software](https://docs.alliancecan.ca/wiki/Available_software) +guides for the current repository contents. + +For a documented module, load it and record the module name and version with +the workflow: + +```bash +module load python/3.10 +python --version +module list +``` + +Module changes apply to the current process and Session. Put stable, required +dependencies in a versioned Container Image or environment rather than +depending on an unrecorded interactive shell. + +## Trade-offs + +- The first access can be slower while metadata and files are fetched. +- The filesystem is read-only; use `/arc` or `/scratch` for writable paths. +- A repository that is mounted on one server or Session Kind may not be + mounted on another. +- A software version published in CVMFS is not automatically installed in a + Container Image. + +If a required repository is absent, ask the platform operator which supported +image or software distribution to use. Do not modify `/cvmfs` or rely on a +private node-local cache for reproducibility. + +## Related guides + +- [Container Images](containers/index.md) +- [Best practices](best-practices.md) +- [Storage](storage/index.md) +- [Alliance Using modules](https://docs.alliancecan.ca/wiki/Using_modules) diff --git a/docs/platform/doi.md b/docs/platform/doi.md index 22f4e4af..68c024ca 100644 --- a/docs/platform/doi.md +++ b/docs/platform/doi.md @@ -1,273 +1,71 @@ -# Data Publication Service (DOIs) - -CANFAR's Data Publication Service (DPS) provides permanent Digital Object Identifiers (DOIs) for research data packages, ensuring long-term accessibility and proper citation of datasets supporting astronomical publications. - -!!! abstract "🎯 DOI Service Overview" - **Essential data publication workflows:** - - - **DOI Registration**: Reserve permanent identifiers through DataCite - - **Data Packaging**: Organize and upload research datasets - - **Referee Access**: Provide controlled access during peer review - - **Publication**: Mint final DOIs with locked data directories - - **Long-term Preservation**: Ensure data accessibility and citation - -## 🚀 Service Purpose & Access - -### Data Publication Service Overview - -The **CANFAR Data Publication Service (DPS)** creates permanent links between research papers and their supporting data packages. DPS provides: - -- **Permanent Storage**: Reliable hosting for research data packages -- **DOI Registration**: Official Digital Object Identifiers through DataCite -- **Landing Pages**: Professional presentation of datasets and metadata -- **Citation Support**: Proper attribution for data reuse and collaboration - -### Access Points - -**Web Interface** -: [CANFAR Science Portal](https://www.canfar.net/) → **Data Publication** - -**Direct Service** -: [Data Publication Service](https://www.canfar.net/citation/) - -**Account Requirements** -: First author requires a CADC account for DPS access and VOSpace data management. - -## 📋 DOI Workflow Guide - -### Step 1: Request a DOI - -**Reserve Your DOI:** - -- A permanent DOI is assigned to your data package (e.g., [10.11570/20.0006](http://doi.org/10.11570/20.0006)) -- A dedicated [Data Directory (VOSpace)](https://www.canfar.net/storage/vault/list/AstroDataCitationDOI/CISTI.CANFAR/20.0006/data) is created -- A professional [landing page](https://www.canfar.net/citation/landing?doi=20.0006) is generated - -### Step 2: Upload Data Package - -**Choose Upload Method Based on Data Size:** - -**Small/Few Files:** -: Use the Web Storage UI for direct browser upload - -**Large/Many Files:** -: Use [`vos` CLI tools](storage/vospace.md) for efficient bulk transfer - -See [Data Package Guidelines](#data-package-requirements) for content organization recommendations. - -### Step 3: Referee Access (Optional) - -**Peer Review Support:** - -CADC can create read-only accounts for editors/referees to access your data directory during the review process. The temporary account is disabled after review completion. - -### Step 4: Publish with DataCite - -**Final Publication:** - -Click **Publish** in the DPS interface to: - -- Complete DOI registration with DataCite -- Lock the data directory (preventing further changes) -- Make the landing page publicly accessible - -!!! warning "Important: Publication Locks Data" - After publishing, the data directory becomes read-only. Metadata changes require contacting [CANFAR support](mailto:support@canfar.net). - -## 🔧 Using the Data Publication Service - -### Managing Your DOIs - -**DOI Dashboard:** -: The [DPS interface](https://www.canfar.net/citation/) displays all your DOIs with status, title, landing page links, and data directory access. - -**Creating New DOIs:** -: Use **New** from the dashboard or go directly to the [request page](https://www.canfar.net/citation/request). - -### DOI Request Requirements - -**Required Information:** - -- **First Author**: Primary researcher responsible for the data package -- **Title**: Descriptive title for the dataset - -**Optional Information (editable later):** - -- **Additional Authors**: Contributing researchers -- **Journal Reference**: Journal name, volume, page numbers -- **Publication Details**: Can be added after manuscript acceptance - -### DOI Management Interface - -**DOI Details Page** (e.g., [DOI.20.0016](https://www.canfar.net/citation/request?doi=20.0016)): - -- DOI reference number and dataset title -- Author list and journal reference information -- Current publication status -- Direct links to landing page and data directory -- Lock status indicator for published DOIs - -**Editing Capabilities:** - -- **Unpublished DOIs**: Full editing access via **Update** button -- **Published DOIs**: Changes require [CANFAR support](mailto:support@canfar.net) request - -**Landing Page Access:** - -- **DOI Link**: [10.11570/20.0016](http://doi.org/10.11570/20.0016) (permanent identifier) -- **Landing Page**: [Direct access](https://www.canfar.net/citation/landing?doi=20.0016) (publicly accessible after publication) - -**DOI Lifecycle Management:** - -- **Unpublished**: Can be edited or deleted by the author -- **Published**: Permanent and locked, requires support for modifications - -## 📦 Data Package Requirements - -### Storage Implementation - -**VOSpace Data Directory:** -: Each DOI receives a dedicated folder in the CANFAR Vault (VOSpace) with a `data/` subdirectory under your control. - -**Example Structure:** -: [Data Directory Example](https://www.canfar.net/storage/vault/list/AstroDataCitationDOI/CISTI.CANFAR/21.0002/data) - -### Content Organization - -**Recommended Package Contents:** - -- **Primary Data**: Core datasets supporting the research -- **Analysis Code**: Scripts and software used in data processing -- **Documentation**: README files describing structure and usage -- **Supplementary Materials**: Figures, tables, additional analysis outputs - -**Best Practices:** - -- Include a top-level README describing package layout and usage instructions -- Organize files in logical subdirectories (e.g., `raw_data/`, `processed/`, `scripts/`, `figures/`) -- Use descriptive filenames and provide metadata where appropriate - -### Upload Methods - -**Web Interface Upload:** -: Web Storage UI for small datasets and simple uploads - -**Command-Line Upload:** -: `vcp` and `vos` CLI tools for large datasets and automated transfers - -### Publication & Access Control - -**Pre-Publication (Referee Access):** - -- Contact [CANFAR support](mailto:support@canfar.net) for read-only reviewer accounts -- Temporary access provided during peer review process -- Reviewers may request changes before publication approval - -**Post-Publication (Public Access):** - -- **Publish** button mints the final DOI and locks data directory -- Landing page becomes publicly discoverable through DataCite search -- Minimal discovery metadata appears in DataCite registry - -### Final Publication Integration - -**Linking DOIs:** - -After manuscript acceptance, coordinate the connection between your data package DOI and journal publication DOI: - -1. **Notify Journal**: Provide your data package DOI for inclusion in the published paper -2. **Update Metadata**: Email [CANFAR support](mailto:support@canfar.net) with: - - Publication DOI from the journal - - Updated reference details (journal, volume, pages) - - Any additional metadata corrections - -!!! tip "Data Package Success" - **Plan your data package early in the research process** to ensure all necessary files, documentation, and metadata are preserved and organized for publication. - -## Using the DPS - -### Listing current DOIs - -[DPS](https://www.canfar.net/citation/) shows your DOIs (status, title, landing page, data directory). From here, you can request, view, edit, or publish depending on status. - -### Requesting a new DOI - -Use **New** from the list or go to the [request page](https://www.canfar.net/citation/request). - -!!! question "Required" - - First Author - - Title - -!!! note "Optional (can be edited later)" - - Journal reference (journal, volume, page) - - Additional Authors - -After submission, a **DOI Reference** number is assigned and displayed. - -### DOI Details - -On the details page (e.g., [DOI.20.0016](https://www.canfar.net/citation/request?doi=20.0016)) you'll find: - -- DOI number / Title -- Authors / Journal reference -- DOI status -- Landing page link -- Data Directory link (shows 🔒 when frozen) - -### Editing details - -- **Unpublished** DOIs can be edited by authenticated users; click **Update**. -- **Published** DOIs require a request to [CANFAR support](mailto:support@canfar.net). - -### Viewing the landing page - -- DOI: [10.11570/20.0016](http://doi.org/10.11570/20.0016) -- Landing page: [landing page](https://www.canfar.net/citation/landing?doi=20.0016) - -Published landing pages are publicly accessible. - -### Publishing a DOI - -If not yet published, a **Publish** button appears at the top right. Publishing: - -- Completes registration with DataCite -- Locks the Data Directory - -Related publication info can be added later via support. - -### Deleting unpublished DOIs - -Unpublished records can be deleted via **Delete** on the request page. **Published** DOIs cannot be deleted. - -## DOI Data Package - -DPS hosts a Data Directory in the **Vault (VOSpace)** implementation for each DOI. A folder named `data/` is created under the DOI root; you control the structure beneath it. - -Example: [Data Directory](https://www.canfar.net/storage/vault/list/AstroDataCitationDOI/CISTI.CANFAR/21.0002/data) - -!!! warning "Locked after publish" - After publishing, the directory is **locked**. To modify contents or metadata, contact [CANFAR support](mailto:support@canfar.net). - -### Contents - -You decide what to include: data, figures, software, etc. We recommend a top‑level `README` describing layout and usage. - -### Uploading - -- Few/small files: [Web Storage UI](storage/transfers.md#upload-methods). -- Large/many files: [Use `vcp`, `vos` CLI Tools](storage/transfers.md#large-files-100gb-advanced-methods). - -### Refereeing access - -Contact support to obtain a read‑only account and share with the editor/referee. They may request changes prior to publication. - -### Publish & discoverability - -After acceptance, click **Publish** to mint the DOI. The directory and metadata freeze; minimal discovery metadata will appear in DataCite search. - -### Final linking - -Finally, link the **data package DOI** to the **journal DOI** (currently manual): - -- Email support with the publication DOI and updated reference details. -- Provide the data package DOI to the journal so it appears in the paper. \ No newline at end of file +# Publish data with a DOI + +The CANFAR Data Publication Service (DPS) packages research data with a +landing page and a persistent Digital Object Identifier (DOI). Use the +[Science Portal](https://www.canfar.net/) or open the +[DPS](https://www.canfar.net/citation/) directly. + +The service workflow is web-led. A CADC account and access to the data service +are required; if the portal does not show the publication tools, ask CANFAR +support about account or project access. + +## Workflow + +1. Open **Data Publication** and choose **New**. +2. Enter the required title and first-author details. Add authors and journal + information when available. +3. Upload or copy the package into the data directory shown by DPS. For a + command-line transfer, use the configured Storage Identifier and the + [CANFAR data commands](storage/transfers.md): + + ```bash + canfar data ls vault:/path/to/doi/data + canfar data cp local:/scratch/catalog.csv vault:/path/to/doi/data/catalog.csv + ``` + + `vault:` is an example; use the identifier and path supplied by the + publication service on your deployment. +4. Add a top-level `README` that describes the layout, provenance, software, + units, and any restrictions. Include the data, code, figures, or tables + needed to interpret the research, while excluding credentials and + unnecessary temporary files. +5. Share the unpublished record or its data directory with referees using the + access workflow provided by DPS. +6. Select **Publish** when the package and metadata are final. Publishing + registers the DOI and locks the published data directory; contact + [CANFAR support](mailto:support@canfar.net) if a published record needs a + correction. + +## Build a useful package + +Keep the package self-describing and reproducible: + +- record the source observations, processing steps, and software versions; +- use stable, descriptive filenames and a short directory structure; +- include machine-readable metadata alongside tables and images; +- document missing values, coordinate systems, units, and selection criteria; +- include a license and citation guidance for reuse; and +- verify that the package can be read from the published data directory before + selecting **Publish**. + +Use `/scratch` only for staging during a Session. Store the package and any +checkpoint that must survive outside `/arc` or the DPS data directory. The +[storage guide](storage/index.md) explains the distinction between mounted +storage, staged files, and persistent remote data. + +## After publication + +Use the DOI landing page in the paper and cite the dataset as a research +output. If the journal DOI or bibliographic details become available later, +update the unpublished metadata or contact support for the published record. + +The DPS landing page and data-directory URLs are the source of truth for the +individual package. Do not hard-code a sample DOI or assume that a particular +Vault path is available to another user or deployment. + +## Related guides + +- [Data transfers](storage/transfers.md) +- [Storage and Python tools](storage/filesystem.md) +- [Permissions](permissions.md) +- [CANFAR support](support/index.md) diff --git a/docs/platform/get-started.md b/docs/platform/get-started.md index ecf249a0..9aee9d4b 100644 --- a/docs/platform/get-started.md +++ b/docs/platform/get-started.md @@ -1,116 +1,72 @@ -# 🚀 Getting Started with CANFAR +# Get started -**Guide to setting up and using the CANFAR Science Platform for astronomical research.** +The CANFAR Science Platform combines authenticated compute, Container Images, +and research storage. Start with the portal for interactive work, or install +the Python client and CLI for repeatable workflows. - **Essential Resources:** - - - [Permissions Guide](permissions.md): Account set-up and group management - - [Sessions Overview](sessions/index.md): Interactive computing environments - - [Storage Guide](storage/index.md): Data management and file systems - - [Container Guide](containers/index.md): Software environments and registries - - [Support Centre](support/index.md): Help resources and FAQ +## 1. Get an account and access +Request a [CADC account](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/auth/request.html) +and join the project or group that owns the data you need. Group membership +controls access to shared project paths and private Container Images. If you +need help, contact [CANFAR support](support/index.md). -## 1️⃣ Get Your CADC Account +## 2. Launch a first Session -If you are a first-time user, request a Canadian Astronomy Data Centre (CADC) account: +From the [Science Portal](https://www.canfar.net/), select a Session Kind and +Container Image, then launch the Session. Start with a Notebook for Python +exploration; choose Desktop, CARTA, or Firefly when the workflow needs a +specialized interface. Use `headless` for a command that should run without an +interactive interface. -[🔗 Request CADC Account](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/auth/request.html){ .md-button .md-button--primary } +The portal's image list is authoritative for available images. Do not assume +that an example tag in this guide is published by every deployment. -See [Accounts & Permissions](permissions.md) for more details. +## 3. Install the client (optional) +Install the released package on the machine from which you want to manage +Sessions or transfer data: -!!! info "Account Processing Time" - CADC accounts are typically approved within 1–2 business days. - For troubleshooting account issues, see [FAQ](support/faq.md) or [Contact Support](support/index.md). +```bash +python -m pip install --upgrade canfar +canfar login cadc +canfar auth show +canfar server ls +``` +The CLI stores Authentication Records and discovered Science Platform Servers +in its local configuration. See the [CLI reference](../cli/cli-help.md) and +[Python client guide](../client/get-started.md). +## 4. Put data in the right place -## 2️⃣ Join or Create Your Research Group +Inside a Session, use mounted paths when they are available: -Once you have a CADC account: +| Location | Use | +| --- | --- | +| `/arc/home/` | Personal scripts and results | +| `/arc/projects/` | Shared project data and outputs | +| `/scratch` | Temporary staging and intermediates; deleted with the Session | -=== "Joining an Existing Group" - Ask your collaboration administrator to add you via the [CADC Group Management Interface](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/groups/). +For remote data, use [canfar data](storage/transfers.md) or +`canfar.storage.filesystem(identifier)` in Python. Storage Identifiers are +explicit configuration names; do not invent a `vault://` URL or import a +configured identifier as a Python attribute. -=== "New Collaboration" - Email [support@canfar.net](mailto:support@canfar.net) with: - - - Your project description - - Expected team size - - Storage requirements - - Timeline +## 5. Run and preserve a workflow +1. Test the command on a small input in an interactive Session. +2. Write a durable input/output path into the command or environment. +3. Submit a [headless Session](sessions/batch.md) for unattended work. +4. Inspect `Pending` Sessions with `canfar ps --all`, `canfar info`, and + `canfar events`. +5. Copy final results out of `/scratch` before deleting the Session. -See [Permissions Guide](permissions.md) for group management details. For advanced collaboration, see [Storage Guide](storage/index.md). - - -## 3️⃣ First Login and Set-up - -1. Login to [canfar.net](https://www.canfar.net) with your CADC credentials. -2. Accept Terms of Service to complete set-up. -3. Optional (for private containers): Access the [Image Registry](https://images.canfar.net) - -See [Container Guide](containers/index.md) for more about images and custom software. For building your own containers, see [Building Containers](containers/build.md). - - -## 4️⃣ Launch Your First Session - -To start analyzing data, launch a Jupyter notebook: - -1. Click **Science Portal** from the main menu. -2. Use the default settings. -3. Click **Launch**. -4. Wait about 30 seconds, then open your session. - -🎉 You're ready to go! Your session includes Python, common astronomy packages, and access to shared storage. - - -See [Sessions Overview](sessions/index.md) for more session types and workflows. For automation, see [CANFAR Python Client](../client/home.md). - - -!!! tip "Recommended Starting Point" - Start with the default `astroml` container – it includes most common astronomy packages and is regularly updated. If you need thousands of pre-built software packages, see [Accessing CVMFS](cvmfs.md). - -!!! tip "Advanced: Custom Containers" - - Build your own containers for specialised workflows. See [Building Containers](containers/build.md). - - Use [Harbor Registry](containers/registry.md) at [images.canfar.net](https://images.canfar.net/) to browse and manage images. - - - -## 📁 Understanding Your Workspace - - -See the [Storage Guide](storage/index.md) for full details. For VOSpace scripting, see [VOSpace API](storage/vospace.md). - -| Location | Purpose | Persistence | Best For | -|----------|---------|-------------|----------| -| `/arc/projects/[project]/` | Shared research data | ✅ Permanent, backed up | Datasets, results, shared code | -| `/arc/home/[user]/` | Personal files | ✅ Permanent, backed up | Personal configs, small files | -| `/scratch/` | Fast temporary space | ❌ Wiped at session end | Large computations, temporary files | - - - -## 🤝 Collaboration Features - - -See [Permissions Guide](permissions.md) and [Storage Guide](storage/index.md) for collaboration details. For team onboarding, see [Getting Started](get-started.md). - -### Storage Sharing - -All group members have access to `/arc/projects/[project]/` – perfect for: - -- Sharing datasets and results -- Collaborative analysis scripts -- Common software environments -- Project documentation - - -## 💬 Need Help? - -- **[💬 Discord Community](https://discord.gg/vcCQ8QBvBa)** – Chat with other users -- **[🆘 Support Centre](support/index.md)** – Help resources and contact information - ---- - +## Next steps +- [Platform concepts](concepts.md) +- [Sessions](sessions/index.md) +- [Storage](storage/index.md) +- [Containers](containers/index.md) +- [Permissions](permissions.md) +- [Support](support/index.md) diff --git a/docs/platform/index.md b/docs/platform/index.md index b2b0c789..c2ff5649 100644 --- a/docs/platform/index.md +++ b/docs/platform/index.md @@ -1,112 +1,42 @@ # CANFAR Science Platform -The Canadian Advanced Network for Astronomy Research lets you access software environments and large datasets on the cloud designed specifically for astronomical research. - -!!! abstract "🚀 Platform Overview" - - **Scalable** cloud compute and storage to analyze very large datasets from anywhere. - - **Browser-based** interactive environments for instant, device-independent data exploration. - - **Collaborative** tools that make teamwork and data sharing simple. - - **Well‑tested** default scientific containers plus customizable images and registries for large-team workflows. - -## 🚀 Quick Access - -=== "🆕 New to CANFAR?" - - **Start Your Research Journey** - - **[📖 Getting Started Guide](get-started.md)** - : Complete onboarding with tutorials, examples, and first session setup - - **[🧩 Platform Concepts](concepts.md)** - : Understanding CANFAR architecture, containers, sessions, and storage systems - - **[🔑 Account Setup](permissions.md)** - : User management, groups, and collaboration access - -=== "👤 Experienced Users" - - **Direct Platform Access** - - **[🌐 Science Portal](https://www.canfar.net/science-portal/)** - : Launch interactive sessions and manage computing resources - - **[📁 File Manager](https://www.canfar.net/storage/arc/list)** - : Access and organize your research data and project files - - **[👥 Group Management](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/groups/)** - : Manage research teams and collaborative permissions - -=== "⚙️ Developers & Automation" - - **Programmatic Access** - - **[⚙️ CANFAR CLI](../cli/cli-help.md)** - : Command-line tools for session management and automation - - **[🐍 Python Client](../client/home.md)** - : Programmatic API access for workflows and integration - - **[🐳 Container Registry](https://images.canfar.net)** - : Browse and manage software environments - -## 📚 Platform Documentation - -### Core Platform Guides - -**[🧩 Platform Concepts](concepts.md)** -: **Comprehensive platform understanding** - Architecture, containers, sessions, storage systems, and browser-based workflows - -**[🔑 User Management & Permissions](permissions.md)** -: **Collaboration and access control** - Accounts, groups, ACLs, container registry, and API authentication - -**[📄 Data Publication Service](doi.md)** -: **DOI management and data preservation** - Request DOIs, referee access, and freeze your data for publication - -**[☁️ Legacy Cloud Platform](cloud.md)** -: **Traditional VM infrastructure** - OpenStack cloud, VM batch processing - -### Getting Started Resources - -**[📖 Getting Started Guide](get-started.md)** -: **Hands-on tutorials** - Account setup, first sessions, data management, and workflow examples - -**[🗄️ Storage Systems](storage/index.md)** -: **Data management mastery** - ARC storage, VOSpace, quotas, and collaboration workflows - -**[🐳 Container Environments](containers/index.md)** -: **Software environment usage** - Available containers, custom builds, and workflow integration - -**[🖥️ Sessions](sessions/index.md)** -: **Computing workflows** - Notebooks, desktops, CARTA, Firefly, specialized browser-based apps - -### Advanced Topics - -**[🏭 Batch Processing](sessions/batch.md)** -: **Automated workflows** - Large-scale processing, job management, and production pipelines - -**[⚙️ Command Line Interface](../cli/cli-help.md)** -: **Platform automation** - CLI tools, scripting, and workflow management - -**[🐍 Python Client](../client/home.md)** -: **API integration** - Programmatic access, custom applications, and automation - -## 🆘 Support & Community - -### Getting Help - -**[🆘 Help & Support](support/index.md)** -: **Technical assistance** - Support channels, issue reporting, and community resources - -**[❓ Frequently Asked Questions](support/faq.md)** -: **Common questions and solutions** - Platform usage, troubleshooting, and best practices - - -## 🔗 External Resources - -### CANFAR Ecosystem - -**[Canadian Astronomy Data Centre (CADC)](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/)** -: **Data archives and services** - Search observatory data, VO services - -**[CANFAR Main Website](https://www.canfar.net/)** -: **Project information** - Access various CANFAR services, news, partnerships, and organizational details +CANFAR provides authenticated compute and storage for astronomical research. +Launch a Session from a Container Image, work with mounted project storage or +configured VOSpace Services, and preserve results outside temporary compute. + +## Choose a starting point + +- [Get started](get-started.md) — account, first Session, and first workflow. +- [Platform concepts](concepts.md) — Authentication, Server Selection, Sessions, + Container Images, and Storage Identifiers. +- [Sessions](sessions/index.md) — Notebook, Desktop, CARTA, Firefly, contributed, + and headless workflows. +- [Storage](storage/index.md) — `/arc`, `/scratch`, VOSpace, fsspec, and data + transfers. +- [Containers](containers/index.md) — choose or build a software environment. +- [Permissions](permissions.md) — accounts, groups, and access control. +- [Support](support/index.md) — troubleshooting and contact options. + +## Typical workflow + +```mermaid +flowchart LR + Account[Account and group access] --> Login[Authenticate] + Login --> Session[Launch a Session] + Session --> Data[Read mounted or remote data] + Data --> Work[Run analysis] + Work --> Result[Save result to persistent storage] +``` + +Use the [Python client](../client/get-started.md) or [CLI](../cli/cli-help.md) +when the workflow should be repeatable from a script. Use the [Demos](../demos/srcnet-workshop.md) +for a guided command-line exercise. + +## Community and publication + +The [Community](community/index.md) pages collect domain workflows such as +ALMA. The [DOI guide](doi.md) describes the Data Publication Service. If you +use CANFAR in a publication, follow the [acknowledgement](../about/acknowledgement.md) +wording. + +Deployment and operator material lives in [OpenCADC Deployments](https://www.opencadc.org/deployments/). diff --git a/docs/platform/javascripts/mobile-fix.js b/docs/platform/javascripts/mobile-fix.js deleted file mode 100644 index 4852595b..00000000 --- a/docs/platform/javascripts/mobile-fix.js +++ /dev/null @@ -1,41 +0,0 @@ -// Mobile navigation visibility with better breakpoints -document.addEventListener('DOMContentLoaded', function() { - function ensureMobileMenu() { - const hamburgerButton = document.querySelector('.md-header__button.md-icon[for="__drawer"]'); - const tabs = document.querySelector('.md-tabs'); - const header = document.querySelector('.md-header'); - - // Show hamburger menu on screens smaller than 1440px (instead of 1219px) - if (hamburgerButton && window.innerWidth <= 1440) { - hamburgerButton.style.display = 'block'; - hamburgerButton.style.visibility = 'visible'; - console.log('Mobile menu visible at width:', window.innerWidth); - } else if (hamburgerButton && window.innerWidth > 1440) { - // Hide hamburger on large screens where tabs should be visible - hamburgerButton.style.display = 'none'; - } - - // Ensure tabs are visible on desktop - if (tabs && window.innerWidth > 1440) { - tabs.style.display = 'block'; - } - - // Force refresh of navigation state - if (header) { - header.style.position = 'relative'; - setTimeout(() => { - header.style.position = ''; - }, 10); - } - } - - // Run on load - ensureMobileMenu(); - - // Run on resize - window.addEventListener('resize', ensureMobileMenu); - - // Run after Material theme initializes - setTimeout(ensureMobileMenu, 100); - setTimeout(ensureMobileMenu, 500); -}); diff --git a/docs/platform/permissions.md b/docs/platform/permissions.md index f950c5e8..f1f3dab5 100644 --- a/docs/platform/permissions.md +++ b/docs/platform/permissions.md @@ -1,495 +1,101 @@ -# User Management & Permissions +# Accounts, groups, and permissions -**Accounts, groups, access control, and API authentication on the CANFAR platform for collaborative astronomical research.** +CANFAR access is evaluated at several boundaries: your CADC identity, the +active Science Platform Server, the Storage Identifier you use, the project +groups that grant access, and the Container Image you request. A successful +login does not grant access to every server, project, image, or data object. -!!! abstract "🎯 Permission System Overview" - **Essential access control concepts for all users:** - - - **Account Management**: CADC identity and authentication systems - - **Group Collaboration**: Team-based resource sharing and project management - - **Access Control Lists**: Fine-grained file and directory permissions - - **Container Registry**: Software environment access and distribution - - **API Authentication**: Programmatic access and automation +## Identity and server access -## 🔐 CANFAR Permission Architecture - -CANFAR's security model consists of multiple integrated layers providing flexible, secure access control for astronomical research collaboration. - -### Authentication & Authorisation Layers - -**CADC Identity System** -: Your foundational identity for all Canadian astronomy services, providing single sign-on across CANFAR, data archives, and VO services. - -**Group-Based Collaboration** -: Teams and projects organized through hierarchical group membership with shared resource access and management capabilities. - -**Harbor Container Registry** -: Software environment access control determining who can access, modify, and distribute container images. - -**Access Control Lists (ACLs)** -: Fine-grained POSIX-extended permissions for precise file and directory access control on shared storage systems. - -**API Authentication Framework** -: Secure programmatic access enabling automation, integration, and custom application development. - -### Permission Model Benefits - -=== "Individual Researchers" - - **Single Identity**: One CADC account for all astronomical services - - **Self-Service**: Manage personal permissions and group memberships - - **Secure Access**: Multi-factor authentication and token-based API access - - **Data Protection**: Granular control over personal and shared data - -=== "Research Teams" - - **Collaborative Workspaces**: Shared storage, containers, and computing resources - - **Role-Based Access**: Flexible administrator and member roles - - **Project Isolation**: Security boundaries between different research projects - - **External Collaboration**: Controlled access for external partners and institutions - -=== "System Administrators" - - **Centralized Management**: Unified interface for user and resource administration - - **Audit Capabilities**: Comprehensive logging and permission tracking - - **Scalable Security**: Supports large multi-institutional collaborations - - **Automated Workflows**: API-driven permission management and integration - -## 👥 Group Management & Collaboration - -Groups form the foundation of collaborative research on CANFAR, providing shared access to computing resources, storage systems, and container environments while maintaining security boundaries between projects. - -### Group-Based Resource Sharing - -```mermaid -graph TD - Admin["👑 Group Administrator"] - Members["👤 Group Members"] - Resources["💾 Shared Resources"] - - Admin --> |"Manages"| Members - Admin --> |"Controls access to"| Resources - Members --> |"Access"| Resources - - Resources --> Projects["📁 /arc/projects/[project]/"] - Resources --> Storage["💾 Storage Quotas"] - Resources --> Containers["🐳 Container Access"] -``` - -### Group Administration Interface - -**Access Group Management:** -: [CADC Group Management Portal](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/groups/) - -### Creating Research Groups - -**Step 1: Create New Group** - -1. Navigate to the CADC Group Management portal -2. Click **"New Group"** -3. Provide descriptive group name (e.g., `cfhtls-survey`, `exoplanet-collab`) -4. Add comprehensive project description -5. Click **Create** to establish the group - -**Step 2: Add Team Members** - -1. Locate your group in the management interface -2. Click **"Edit"** in the Membership column -3. Search by real names (not CADC usernames) -4. Select appropriate users from search results -5. Click **"Add member"** to grant access - -**Step 3: Assign Administrative Roles** - -1. Click **"Edit"** in the Administrators column -2. Add users requiring group management capabilities -3. Administrators gain full group control and resource allocation rights - -!!! tip "User Discovery" - **Search by full names** (e.g., "John Smith") rather than CADC usernames. The system will find users and display their associated usernames. - -### Group Role Hierarchy - -| Role | Access Level | Responsibilities | Best For | -|------|--------------|------------------|----------| -| **Administrator** | Full group management, resource allocation, member control | Group creation, permission management, resource requests | Project PIs, team leads, institutional coordinators | -| **Member** | Shared resource access, collaboration capabilities | Data analysis, research participation, resource usage | Researchers, students, collaborators, external partners | - -### Group Resource Access - -**Shared Storage Access:** -: Groups automatically receive shared directories in `/arc/projects/[project]/` with managed quotas and backup policies. - -**Container Image Sharing:** -: Group-specific namespaces in Harbor container registry for sharing custom software environments. - -**Computing Resource Allocation:** -: Shared computational quotas and session management across group members. - -**Collaborative Session Management:** -: Ability to share and handoff interactive sessions between group members. - -### Multi-Institutional Collaboration - -**External User Integration:** -: Add researchers from other institutions to your CANFAR groups while maintaining institutional security boundaries. - -**Cross-Project Permissions:** -: Users can belong to multiple groups, enabling interdisciplinary collaboration and resource sharing. - -**Temporary Access:** -: Grant time-limited access for visiting researchers, students, or short-term collaborations. - -!!! success "Collaboration Benefits" - **Groups enable seamless research collaboration** by providing standardized environments, shared data access, and unified resource management across institutional boundaries. - -## 🐳 Container Registry Access (Harbor) - -Harbor serves as CANFAR's container registry for storing, managing, and distributing software environments. Understanding Harbor permissions is essential for teams building custom containers or managing specialized software stacks. - -### Registry Overview - -**Harbor Registry Access:** -: [https://images.canfar.net](https://images.canfar.net) - -**Purpose:** -: Centralized repository for container images with role-based access control, vulnerability scanning, and automated build integration. - -### Harbor Permission Levels - -| Role | Repository Access | Image Management | Project Control | -|------|------------------|------------------|-----------------| -| **Guest** | Pull public images only | View public metadata | Browse public projects | -| **Developer** | Pull all group images, push to assigned repositories | Upload, tag, and delete own images | View project configurations | -| **Master** | Full repository access within project | Complete image lifecycle management | Project settings, user management | - -### Harbor Access Management - -**Permission Requests:** -: Harbor permissions are managed by CANFAR administrators. Contact [support@canfar.net](mailto:support@canfar.net) for: - -- **Repository Access**: Request developer or master access to existing projects -- **New Projects**: Set up dedicated projects for your research group -- **Team Management**: Add or modify permissions for team members -- **Repository Configuration**: Set up automated builds and integration workflows - -### Working with Harbor - -**Authentication:** +Authenticate with the Identity Provider (IDP) that owns the target Science Platform Server: ```bash -# Login to Harbor registry -docker login images.canfar.net +canfar login cadc +canfar server ls +canfar server use SERVER_NAME +canfar config get active.server ``` -**Pulling Images:** - -```bash -# Pull public container images -docker pull images.canfar.net/skaha/astroml:latest - -# Pull private group images (requires permissions) -docker pull images.canfar.net/[project]/[container]:[tag] -``` - -**Pushing Images (Developer/Master roles):** - -```bash -# Build and tag your container -docker build -t images.canfar.net/[project]/[container]:[tag] . - -# Push to project repository -docker push images.canfar.net/[project]/[container]:[tag] -``` - -### Project Organization - -**Public Projects:** -: CANFAR-maintained base images available to all users (e.g., `skaha/astroml`) +The names and capabilities in `canfar server ls` are deployment data. If the +server is missing or login succeeds but a request is forbidden, contact the +operator for that Science Platform deployment. See the [client +overview](../client/overview.md) for credential and server selection details. -**Group Projects:** -: Private repositories for research teams with controlled access and custom software environments +## Groups and project data -**Personal Projects:** -: Individual user spaces for development and testing before team integration +Project storage is intended for collaboration. A project administrator grants +membership through the [CADC group management portal](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/groups/); +the resulting permissions are enforced by the storage service. Use the paths +and Storage Identifiers supplied by your project rather than assuming that a +project name, quota, or sharing policy is the same on every deployment. -!!! tip "Container Strategy" - **Start with public base images** and extend them for your specific needs. Request team projects for sharing custom environments across your research group. - -## 🛡️ Access Control Lists (ACLs) - -Access Control Lists provide fine-grained file and directory permissions beyond traditional POSIX capabilities, enabling flexible collaboration across research teams while maintaining security boundaries. - -### ACL Fundamentals - -**What ACLs Provide:** -: Extended POSIX permissions allowing multiple users and groups to have different access levels to the same files and directories. - -**Why ACLs Matter for Research:** -: Enable complex collaborative scenarios where traditional owner/group/other permissions are insufficient for multi-institutional projects. - -### Traditional POSIX vs ACL Permissions - -=== "POSIX Limitations" - **Traditional Issues:** - - - Only one group can own a file or directory - - No granular control for multiple collaborating groups - - Sharing across research teams requires complex workarounds - - Binary all-or-nothing access for group members - -=== "ACL Advantages" - **Extended Capabilities:** - - - Multiple users and groups with different permissions per file - - Granular read/write access for specific researchers - - Selective collaboration without compromising security - - Fine-tuned access for external partners and institutions - -### ACL vs POSIX Comparison - -| Collaboration Scenario | POSIX Solution | ACL Solution | -|------------------------|----------------|--------------| -| **Single Team Project** | `rwxrwx---` (group access) | Same as POSIX, no advantage | -| **Multi-Group Collaboration** | Must choose one primary group | Grant specific access to multiple groups | -| **External Researcher Access** | Add to group or make world-readable | Grant individual read access only | -| **Selective Write Permissions** | All group members get write access | Grant write access to specific users only | -| **Cross-Institutional Sharing** | Complex group management | Flexible user and group combinations | - -### Viewing ACL Permissions - -**Check Current ACLs:** +For data operations, start by listing the identifiers and paths you can use: ```bash -# View detailed ACL information for files or directories -getfacl /arc/projects/[project]/[directory]/ +canfar data ls -lh arc:/projects/ +canfar data info arc:/projects//catalog.csv ``` -**Example ACL Output:** +`local:` refers to the machine running the command. `arc:` and `vault:` are +examples of configured remote identifiers; use the identifiers returned by +your platform configuration and the [storage guide](storage/index.md) for your +deployment. A forbidden +operation usually means that your account is not a member of the owning group, +the path is outside the project's allocation, or the active credentials do not +match the service. -``` -# file: sensitive_data/ -# owner: alice -# group: myproject-team -user::rwx # Owner permissions -user:bob:r-- # Bob has read-only access -user:carol:rw- # Carol can read and write -group::r-- # Primary group has read-only -group:external-team:r-- # External group has read access -mask::rwx # Maximum effective permissions -other::--- # No access for others -``` +Do not put credentials in a path, a notebook, an image, or a Session command. +Use the configured Authentication Record or an explicitly supplied runtime +credential as described in the [client overview](../client/overview.md) and +[storage guide](storage/index.md). -!!! warning "ACL Mask Behaviour" - The ACL "mask" entry limits maximum effective permissions for named users and groups. If permissions seem restricted, check the mask value. +## Container images -### Setting and Managing ACLs - -**Grant User Access:** +Image visibility and push rights are controlled by the registry project. Find +images visible to the active server with: ```bash -# Give user 'bob' read access to a directory -setfacl -m u:bob:r-- /arc/projects/[project]/shared_data/ - -# Grant user 'alice' read and write access to specific files -setfacl -m u:alice:rw- /arc/projects/[project]/scripts/analysis.py +canfar image ls +canfar image ls --kind headless ``` -**Grant Group Access:** +Use the complete reference returned by the listing. A pull failure can mean a +missing tag, a private project, or a registry that the target server cannot +reach; ask the registry or platform operator which case applies. See +[Container Registry](containers/registry.md). -```bash -# Allow external group read access to results -setfacl -m g:external-collab:r-- /arc/projects/[project]/public_results/ - -# Grant write access to multiple collaborating groups -setfacl -m g:partner-institution:rw- /arc/projects/[project]/shared_analysis/ -``` - -**Remove ACL Entries:** - -```bash -# Remove specific user access -setfacl -x u:bob /arc/projects/[project]/sensitive_data/ - -# Remove all ACL entries (revert to POSIX only) -setfacl -b /arc/projects/[project]/temp_data/ -``` +## Session resources -**Recursive Operations:** +Resource requests are checked when a Session is admitted. They do not change +data permissions or project membership. For a rejected or long-running +request, inspect the Session and its events: ```bash -# Apply ACLs to entire directory trees -setfacl -R -m g:collaborators:r-- /arc/projects/[project]/results/ +canfar ps --all +canfar info SESSION_ID +canfar events SESSION_ID ``` -**Recommended Directory Structure:** - -``` -/arc/projects/[project]/ -├── public/ # World-readable results -│ └── (ACL: group:world:r--) -├── team/ # Full team access -│ └── (ACL: group:myproject-team:rw-) -├── admin/ # Administrator-only access -│ └── (ACL: user:pi:rw-, group:admins:rw-) -├── external/ # Controlled external collaboration -│ └── (ACL: user:collaborator:r--, group:external-team:r--) -└── sensitive/ # Restricted access with specific permissions - └── (ACL: user:analyst1:rw-, user:analyst2:r--) -``` - -**Security Best Practices:** - -- **Principle of Least Privilege**: Grant minimum access required for each user or group -- **Regular Audits**: Review ACLs periodically using `getfacl` to ensure appropriate access -- **Documentation**: Maintain records of why specific ACLs were set and who requested them -- **Group Preference**: Use group-based permissions when possible for easier management -- **Inheritance Planning**: Set default ACLs on directories to automatically apply to new files - -**ACL Troubleshooting:** - -If ACL changes don't take effect as expected: - -1. **Check the ACL mask**: `getfacl filename` and verify mask entry -2. **Update mask if needed**: `setfacl -m m::rwx filename` -3. **Set default ACLs for directories**: `setfacl -d -m g:groupname:rw directory/` -4. **Verify group membership**: Ensure users belong to specified groups - -!!! success "Collaboration Success" - **ACLs enable sophisticated research collaboration** across institutional boundaries while maintaining data security and access control granularity. - -## 🔌 API Authentication & Programmatic Access - -CANFAR provides comprehensive REST APIs enabling automation, integration, and custom application development. Understanding authentication methods is essential for programmatic platform usage. - -### Authentication Framework - -**API Access Purpose:** -: Enable automation, workflow integration, and custom tool development using CANFAR platform capabilities. - -**Authentication Requirements:** -: All API calls require proper authentication tokens or certificates for secure access to platform resources. - -### Authentication Methods - -=== "🔧 CANFAR CLI (Recommended)" - - **Best for:** Interactive use, development, short-term automation - - **Setup:** - - ```bash - # Login and store authentication token - canfar login cadc - - # Subsequent commands use stored credentials - canfar ps - canfar create notebook skaha/astroml:latest - canfar info [session-id] - ``` - - **Benefits:** - - Easy setup and token management - - Automatic token refresh handling - - Integrated with all CANFAR platform services - - Ideal for development and testing workflows - -=== "🔒 Proxy Certificates" - - **Best for:** Long-term automation, production scripts, file operations - - **Setup:** - ```bash - # Install CADC utilities - pip install cadcutils - - # Generate 10-day proxy certificate - cadc-get-cert -u [username] - - # Certificate stored in ~/.ssl/cadcproxy.pem - # Automatically used by CADC tools and APIs - ``` - - **Benefits:** - - Extended validity (10 days) - - Compatible with all CADC services - - Suitable for production automation - - Works with VOSpace and data archive APIs - -### API Integration Examples - -**Session Management:** - -```python -from canfar.sessions import Session - -session = Session() -ids = session.create( - name="permission-check", - image="skaha/astroml:latest", - kind="notebook", -) - -# Monitor session status -status = session.info(ids) - -# List all active sessions -active_sessions = session.fetch() -``` - -**Batch Processing Integration:** - -```python -from canfar.sessions import Session - -session = Session() -job_ids = session.create( - name="automated-analysis", - image="skaha/astroml:latest", - kind="headless", - cmd="python", - args="analysis.py", - cores=4, - ram=8, -) -``` - -## 🚨 Common Issues & Troubleshooting - -For solutions to common permission and access issues, including: -- Permission Denied Accessing `/arc/projects/[project]` -- Harbor Container Registry Access Issues -- API Authentication Failures -- ACL Changes Not Taking Effect -- Group Changes Not Visible - -Please refer to the **[Troubleshooting section of the FAQ](support/faq.md#troubleshooting)**. - -!!! warning "Security Best Practices" - **Protect Your Credentials:** - - - Never share CADC passwords or authentication tokens - - Use group-based permissions instead of individual token sharing - - Regularly review access permissions for sensitive data - - Report suspected security issues immediately to support - -## 🔗 Advanced Permission Management - -### Multi-Institutional Collaboration - -**Cross-Institution Access:** -: CANFAR supports researchers from multiple institutions through flexible group membership and external user integration. +See [batch troubleshooting](sessions/batch.md#monitor-and-troubleshoot) for +`Pending` and image-pull checks. -**Guest Researcher Workflows:** -: Temporary access patterns for visiting researchers, students, and short-term collaborations. +## When access is denied -**Resource Delegation:** -: Project administrators can delegate specific permissions without granting full administrative access. +Collect the smallest useful diagnostic set without exposing credentials: -### Enterprise Integration +1. the active Server Name and Storage Identifier; +2. the command shape and redacted path or image reference; +3. the Session ID and status, if a Session is involved; and +4. the exact error and relevant `info`/`events` output. -**LDAP/Active Directory:** -: Contact CANFAR administrators for integration with institutional identity management systems. +Ask the project administrator to confirm group membership and path ownership. +If membership is correct, contact the service operator through +[support](support/index.md). Do not retry destructive operations while the +ownership or path is uncertain. -**Single Sign-On:** -: CADC authentication integrates with Canadian academic identity federations and international collaborations. +## Related guides -**Compliance Requirements:** -: Support for institutional data governance and compliance requirements through audit logging and access controls. +- [Getting started](get-started.md) +- [Storage](storage/index.md) +- [Container Registry](containers/registry.md) +- [Support](support/index.md) diff --git a/docs/platform/sessions/batch.md b/docs/platform/sessions/batch.md index e7bdb227..55a6c6df 100644 --- a/docs/platform/sessions/batch.md +++ b/docs/platform/sessions/batch.md @@ -1,178 +1,190 @@ -# Batch Processing +# Batch processing -Use a `headless` Session when a container should run a command and exit without -an interactive notebook, desktop, CARTA, or Firefly interface. +Use a `headless` Session when a container should run a command without a +Notebook, Desktop, CARTA, or Firefly interface. The command runs in the same +container and mounted-storage environment as an interactive Session; only the +entrypoint and lifecycle differ. -Batch work uses the same Container Images and storage mounts as interactive -Sessions. The main difference is that you pass a command after `--` in the CLI, -or `cmd` and `args` in Python. +This page keeps the guidance requested in [#209](https://github.com/opencadc/canfar/issues/209): +creation output, `Pending` status, queue expectations, and checks for a Session that +does not start. -## Submit From the CLI +## Submit a headless Session -Authenticate once, then create a headless Session: +Authenticate and put the command after `--`: ```bash canfar login cadc -canfar create --name data-reduction headless skaha/astroml:latest -- python /arc/projects/myproject/scripts/reduce_data.py +canfar create \ + --name data-reduction \ + headless skaha/astroml:latest \ + -- python /arc/projects/myproject/scripts/reduce_data.py ``` -Omit resource options for flexible allocation. Use fixed resources only when the -workload has measured requirements: +The image listing is the source of truth for names and available Session Kinds: ```bash -canfar create \ - --name large-simulation \ - --cpu 16 \ - --memory 64 \ - headless skaha/astroml:latest \ - -- python /arc/projects/myproject/scripts/simulation.py +canfar image ls --kind headless ``` -Pass environment variables with repeated `--env` flags: +Resource options are optional. Omit them for the platform's flexible request, +or set measured requirements explicitly: ```bash canfar create \ - --name omp-test \ - --env OMP_NUM_THREADS=4 \ - --cpu 4 \ + --name simulation \ + --cpu 16 \ + --memory 64 \ headless skaha/astroml:latest \ - -- python /arc/projects/myproject/scripts/run.py + -- python /arc/projects/myproject/scripts/simulate.py ``` -Create replicas when each container can process an independent slice: +Repeat `--env KEY=VALUE` for environment variables and use `--replicas` when +each container can process an independent slice: ```bash canfar create \ --name parameter-study \ --replicas 10 \ + --env OMP_NUM_THREADS=4 \ headless skaha/astroml:latest \ - -- python /arc/projects/myproject/scripts/analyze.py + -- python /arc/projects/myproject/scripts/analyse.py ``` -Each replica receives `REPLICA_ID` and `REPLICA_COUNT`. Use those values, or the -helpers in [Distributed Computing](../../client/helpers.md), to split work +Replicas receive `REPLICA_ID` and `REPLICA_COUNT`. Use those values or the +[distributed helpers](../../client/helpers.md) to partition work deterministically. -`canfar run` and `canfar launch` are compatibility aliases for `canfar create`. -New examples should use `canfar create`. +## What `create` prints -## Submit From Python +The library returns the Session IDs accepted by the platform. Human CLI output +reports one successful Session as: -```python -from datetime import datetime +```text +Successfully created session 'data-reduction' (ID: SESSION_ID) +``` -from canfar.sessions import Session +For multiple replicas it reports the count and lists each ID. `--output json` +or `--output yaml` emits the raw list of returned IDs as data-only output; the +same option must appear before `--`, because everything after `--` belongs to +the container command: -session = Session() -project = "/arc/projects/myproject" -data_path = f"{project}/data/{datetime.now().strftime('%Y%m%d')}" +```bash +canfar create headless skaha/astroml:latest --output json -- python run.py +``` -job_ids = session.create( - name=f"nightly-reduction-{datetime.now().strftime('%Y%m%d')}", - image="images.canfar.net/skaha/casa:6.5", - kind="headless", - cmd="python", - args=f"{project}/pipelines/reduce_night.py {data_path}", -) +An empty result is a creation/transport failure, not evidence that a Session is +queued. A partial replica result returns the IDs that were accepted. Use +`--debug` for request diagnostics on stderr or increase `CANFAR_TIMEOUT` when +image pulls or platform requests need more time. -print(job_ids) -``` +## Pending means waiting, not running -Fixed resources use `cores` and `ram`: +An accepted ID can remain `Pending` while the Science Platform admits the +request, waits for requested resources, pulls the image, or completes Session +initialization. `create` does not wait for `Running`, and `canfar open` only +opens a Session once it is ready. -```python -job_ids = session.create( - name="heavy-computation", - image="images.canfar.net/myproject/processor:latest", - kind="headless", - cores=8, - ram=32, - cmd="/opt/scripts/heavy_process.sh", - args="/arc/projects/myproject/data/input.h5 /arc/projects/myproject/results/", - env={"PROCESSING_THREADS": "8"}, -) -``` +The client does not promise a relative priority between `headless` and +interactive Sessions. Queue order and admission policy are deployment-owned; +do not assume that a headless Session will outrank an interactive Session, or that +increasing a request will make it run sooner. If queue policy matters for a +project, ask the platform operator for the current policy. -For async workflows, use `AsyncSession`: +## Monitor and troubleshoot -```python -from canfar.sessions import AsyncSession +The default `ps` view shows `Pending` and `Running` Sessions. Include all +statuses when checking a new batch request: -async with AsyncSession() as session: - job_ids = await session.create( - name="async-batch", - image="images.canfar.net/skaha/astroml:latest", - kind="headless", - cmd="python", - args="/arc/projects/myproject/scripts/analyze.py", - replicas=10, - ) +```bash +canfar ps --all +canfar info SESSION_ID +canfar events SESSION_ID +canfar stats ``` -## Monitor and Clean Up +Use `canfar logs SESSION_ID` after the container has started. A Pending Session +may have no application logs yet. Interpret the checks together: + +| Observation | Next step | +| --- | --- | +| ID is absent from `ps` | Use `canfar ps --all`; confirm the create command returned an ID. | +| Status is `Pending` and events show admission/resource waiting | Reduce fixed CPU, memory, or GPU requests, or wait for capacity. | +| Status is `Pending` and events show image or initialization work | Check the image name and registry access; wait for readiness before opening it. | +| Status is `Running` but the command failed | Read `canfar logs SESSION_ID` and inspect the command, paths, and environment. | +| Status is terminal and output is missing | Check the command's exit/log output and the persistent destination; container-local paths are temporary. | +| Pending persists beyond a reasonable queue interval | Save the Session ID, `info`, `events`, `stats`, image, and resource request, then contact [support](../support/index.md). | + +`canfar stats` is a Science Platform Server-level capacity signal; it does not +explain every admission decision. Events are the most useful next check for one +Session. -Use Session IDs returned by `create`: +When a request should be cancelled, delete by ID: ```bash -canfar ps -canfar info SESSION_ID -canfar events SESSION_ID -canfar logs SESSION_ID canfar delete SESSION_ID ``` -`canfar stats` reports cluster-wide load, not per-Session usage. +## Python API -Python equivalents: +`Session.create` and `AsyncSession.create` return `list[str]` Session IDs. They +do not wait for `Running`; inspect status and events separately: ```python -info = session.info(job_ids) -events = session.events(job_ids) -logs = session.logs(job_ids) -deleted = session.destroy(job_ids) -``` +from canfar.sessions import Session + +with Session() as session: + ids = session.create( + name="nightly-reduction", + image="images.canfar.net/skaha/astroml:latest", + kind="headless", + cmd="python", + args="/arc/projects/myproject/scripts/reduce.py", + env={"OMP_NUM_THREADS": "4"}, + ) -## Resource Guidance + print(ids) + print(session.fetch(status="Pending")) + print(session.events(ids)) +``` -Start flexible. Fixed requests can be harder to schedule and should be based on -measured CPU and memory use. +For native asynchronous code: -| Workload | First request | -| --- | --- | -| Script smoke test | Flexible | -| Single-file reduction | Flexible or `cores=1`, `ram=4` | -| Known memory-heavy job | Fixed `cores` and `ram` | -| Independent parameter sweep | Replicas plus modest fixed resources | +```python +import asyncio -Tune threaded libraries to match requested cores: +from canfar.sessions import AsyncSession -```bash -canfar create \ - --name threaded-job \ - --cpu 4 \ - --env OMP_NUM_THREADS=4 \ - headless skaha/astroml:latest \ - -- python /arc/projects/myproject/scripts/threaded.py +async def main() -> None: + async with AsyncSession() as session: + ids = await session.create( + name="async-batch", + image="images.canfar.net/skaha/astroml:latest", + kind="headless", + cmd="python", + args="/arc/projects/myproject/scripts/analyse.py", + replicas=10, + ) + print(ids) + + +if __name__ == "__main__": + asyncio.run(main()) ``` -## Storage Rules - -Write durable outputs to mounted storage such as `/arc/home//` or -`/arc/projects//`. Treat container-local paths as temporary, and treat -`/scratch/` as ephemeral high-speed working space. +The `args` value is a command-line string. Keep command arguments after the +CLI's `--` delimiter so the client does not parse them as Session options. -## Private Images +## Storage and cleanup -Private images require Container Registry credentials. Configure those through -the Python `Configuration` model or the CLI config before submitting the Session; -see the [registry guide](../containers/registry.md). +Write inputs and durable outputs to `/arc/home//`, +`/arc/projects//`, or a configured VOSpace Service. Use `/scratch` +for high-speed staging and intermediates only; it is deleted when the Session +ends. See [Storage](../storage/index.md) and [Data transfers](../storage/transfers.md). -## Troubleshooting +## Related guides -| Symptom | Check | -| --- | --- | -| Job does not start | `canfar events SESSION_ID` | -| Command exits unexpectedly | `canfar logs SESSION_ID` | -| Resource request waits too long | Try flexible mode or smaller fixed resources | -| Image pull fails | Verify the image name and registry credentials | -| Need structured Session data | `canfar ps --json` | +- [Interactive Sessions](index.md) +- [Container Images](../containers/index.md) +- [Distributed helpers](../../client/helpers.md) +- [CLI reference](../../cli/cli-help.md) diff --git a/docs/platform/sessions/carta.md b/docs/platform/sessions/carta.md index 55c84bcd..0cfae7dc 100644 --- a/docs/platform/sessions/carta.md +++ b/docs/platform/sessions/carta.md @@ -1,298 +1,86 @@ # CARTA Sessions -**CARTA (Cube Analysis and Rendering Tool for Astronomy) for astronomy data visualisation** +[CARTA](https://cartavis.org/) is a browser-based tool for inspecting +astronomical images and data cubes. Use it for interactive visualisation and +region or spectral exploration; use a [headless Session](batch.md) for a +repeatable reduction. -!!! abstract "🎯 What You'll Learn" - - How to launch CARTA sessions and choose the right version - - Loading data from CANFAR storage and working with radio data cubes - - Key features for spectral analysis, region analysis, and animations - - Performance tips and troubleshooting guidance +## Launch CARTA -CARTA is a specialised image visualisation and analysis tool designed specifically for radio astronomy data. It excels at handling multi-dimensional data cubes, providing powerful tools for spectral analysis, and enabling real-time collaborative workflows. - -## 📋 Overview - -CARTA provides specialised capabilities for: - -### Key Features - -| Feature | Capability | -|---------|------------| -| **Image Visualisation** | Multi-dimensional data cube exploration with WCS support | -| **Spectral Analysis** | Line profiles, moment maps, and velocity analysis | -| **Region Analysis** | Statistical analysis of user-defined image regions | -| **Animation** | Time-series and frequency animations through data cubes | -| **Collaboration** | Real-time session sharing with multiple users | -| **Performance** | Optimized rendering for large astronomical datasets | - -### Data Format Support - -- **FITS files:** Standard astronomical format with full WCS support -- **HDF5 files:** High-performance format for large datasets -- **CASA images:** Native support for CASA image formats -- **Compressed formats:** Automatic handling of gzipped files - -## 🚀 Creating a CARTA Session - -### Step 1: Select Session Type - -From the Science Portal dashboard, click the **plus sign (+)** to create a new session, then select **carta** as your session type. - -### Step 2: Choose Container Version - -Note that the menu options update automatically after your session type selection. Choose the CARTA version that meets your needs: - -#### Available Versions - -- **CARTA 5.1.0** (recommended): Latest features and bug fixes -- **CARTA 4.x:** Previous stable releases for compatibility - -!!! tip "Version Selection" - Use the latest version (5.1.0+) unless you specifically need compatibility with older workflows. New versions include performance improvements and additional features. - -### Step 3: Configure Session - -#### Session Name - -Choose a descriptive session name to help you identify it later: - -**Good session names:** -- `m87-analysis` -- `ngc-1300-cube` -- `alma-co-line-study` -- `vla-continuum-imaging` - -#### Resource Allocation - -Start with a "flexible" session for most analyzes. Switch to a fixed resource allocation if you need guaranteed performance for demanding visualisations. - -**Resource Guidelines:** -- **Flexible:** Good for most CARTA workflows -- **Fixed:** Use for large datasets (>1GB) or guaranteed performance - -### Step 4: Launch Session - -Click the **Launch** button and wait for your session to initialize. CARTA sessions typically start within 30-60 seconds. - -## 🧭 Using CARTA - -### First Steps - -Once connected to your CARTA session: - -1. **File Menu:** Use "Open Image" to load your data -2. **File Browser:** Navigate to `/arc/projects/[project]/` or `/arc/home/[user]/` -3. **Load Data:** Select FITS or HDF5 files to visualize - -### Data Loading - -#### From CANFAR Storage +Choose an image published for the active server: ```bash -# CARTA can access files from: -/arc/home/[user]/ # Your personal data -/arc/projects/[project]/ # Shared project data -/scratch/ # Temporary high-speed storage +canfar login cadc +canfar image ls --kind carta +canfar create carta IMAGE_NAME --name cube-inspection ``` -#### Supported File Paths - -- **Local files:** Any file accessible in the session filesystem -- **Remote files:** HTTP/HTTPS URLs (limited support) -- **Archive data:** Files downloaded to CANFAR storage - -### Interface Overview - -#### Main Components - -- **Image Viewer:** Central panel showing the astronomical image -- **File Browser:** Left panel for navigating and opening files -- **Region List:** Panel for managing analysis regions -- **Statistics:** Real-time statistics for selected regions -- **Spectral Profiler:** Panel for line profile analysis -- **Animation:** Controls for cycling through cube slices - -#### Essential Controls - -| Control | Function | -|---------|----------| -| **Mouse wheel** | Zoom in/out | -| **Click + drag** | Pan around image | -| **Right-click** | Context menu with additional options | -| **Keyboard shortcuts** | See Help menu for complete list | - -## 🔬 Analysis Features - -### Spectral Analysis - -#### Line Profiles - -1. **Draw regions** on the image -2. **Open Spectral Profiler** panel -3. **Select region** to view spectrum -4. **Analyze lines** with built-in fitting tools - -#### Moment Maps +The Science Portal offers the same Session Kind. Available versions, resource +controls, and supported file formats are deployment- and image-specific. Use +the image description rather than assuming a version or startup time. -CARTA can generate: - -- **Moment 0:** Integrated intensity -- **Moment 1:** Velocity field -- **Moment 2:** Velocity dispersion - -### Region Analysis - -#### Creating Regions - -1. **Select region tool** from toolbar -2. **Draw on image:** Rectangle, ellipse, polygon, or point -3. **View statistics** in the Statistics panel -4. **Export regions** in DS9 or CRTF format - -#### Statistical Analysis - -CARTA automatically computes: -- **Sum, mean, RMS** within regions -- **Min/max values** and positions -- **Flux measurements** with proper units -- **Histogram analysis** of pixel values - -### Animation and Navigation - -#### Data Cube Navigation - -- **Slider controls:** Navigate through spectral channels or Stokes parameters -- **Animation playback:** Automatic cycling through cube slices -- **Frame rate control:** Adjust animation speed -- **Custom ranges:** Focus on specific velocity ranges - -#### Multi-Panel Views - -- **Compare datasets:** Load multiple images simultaneously -- **Linked panels:** Synchronize zoom, pan, and navigation -- **Layout control:** Arrange panels as needed - -## ⚡ Performance Optimisation - -### Large Dataset Handling - -#### Memory Management +Monitor the Session and open it when ready: ```bash -# Monitor session resources -htop # Check memory usage -df -h # Check disk space +canfar ps --all +canfar info SESSION_ID +canfar open SESSION_ID ``` -#### Optimisation Tips - -- **Use /scratch for large files:** Copy data to high-speed storage -- **Close unused files:** Reduce memory consumption -- **Reduce image resolution:** For initial exploration -- **Use data subsets:** Work with spatial/spectral sub-cubes - -### Network Performance - -#### For Remote Access - -- **Stable connection:** CARTA requires consistent network connectivity -- **Bandwidth:** Higher bandwidth improves responsiveness -- **Close other applications:** Reduce network competition - -## 🤝 Collaboration Features - -### Real-Time Sharing +For a Session that remains `Pending`, inspect [events and resource +troubleshooting](batch.md#monitor-and-troubleshoot). -CARTA supports collaborative analysis: +## Open data -1. **Share session URL** with team members -2. **Simultaneous access:** Multiple users can connect -3. **Synchronized views:** All users see the same state -4. **Coordinate activities:** Communicate to avoid conflicts +CARTA can open files that the Session can read. Common workflows use FITS +images or cubes under mounted Science Platform storage: -### Best Practices for Collaboration - -- **Designate a lead:** Have one person control navigation -- **Use voice/chat:** Coordinate complex operations -- **Save work frequently:** Export regions and analysis results -- **Plan sessions:** Organize collaborative time in advance - -## 🔧 Advanced Features - -### Scripting and Automation - -#### Export Capabilities - -- **Image exports:** PNG, JPEG, PDF formats -- **Region files:** DS9 or CRTF format for other tools -- **Spectral data:** CSV format for further analysis -- **Session state:** Save and restore CARTA configurations - -#### Integration with Other Tools - -```python -# Load CARTA regions in Python -from astropy.io import fits -from regions import Regions - -# Read CARTA-exported region file -regions = Regions.read('carta_regions.crtf', format='crtf') - -# Use with other astronomy software +```text +/arc/home// +/arc/projects// +/scratch/ ``` -### Custom Colour Maps - -- **Built-in maps:** Scientific colour schemes -- **Custom maps:** Import your own colour tables -- **Accessibility:** Colour-blind friendly options -- **Publication quality:** High-contrast options for papers +For an object in a configured VOSpace Service, transfer it explicitly before +opening it: -## 🔧 Troubleshooting - -### Common Issues - -#### Session Won't Load Data - -**Problem:** CARTA cannot open FITS files - -**Solutions:** - -1. Check file permissions and location -2. Verify file format is supported -3. Try copying file to `/scratch/` first -4. Check file isn't corrupted - -#### Slow Performance - -**Problem:** CARTA responds slowly to interactions - -**Solutions:** +```bash +canfar data cp IDENTIFIER:/path/to/cube.fits local:/scratch/cube.fits +``` -1. Check available memory with `htop` -2. Close other browser tabs/applications -3. Reduce image size or use sub-cubes -4. Restart session if memory is exhausted +Use the Storage Identifier and path supplied by your project. `/scratch` is +temporary; copy any regions, tables, or derived products that must survive to +`/arc` or a persistent VOSpace Service. See [Data transfers](../storage/transfers.md). -#### Connection Issues +## Interactive analysis -**Problem:** Lost connection to CARTA session +The exact controls depend on the CARTA version, but typical workflows are: -**Solutions:** +1. open an image or cube from the Session filesystem; +2. inspect WCS, channels, Stokes axes, and image statistics; +3. draw regions and examine spectra or moment summaries; and +4. export regions, tables, or figures to a persistent path. -1. Refresh browser page -2. Check internet connection stability -3. Clear browser cache if persistent -4. Try different browser +Treat exported files as products of the analysis and record the input path, +image version, and CARTA version with them. -#### Display Problems +## Performance -**Problem:** Images don't render correctly +Large cubes can require substantial memory, storage, and network traffic. Start +with a representative sub-cube, close unused views, and stage a remote object +to `/scratch` when repeated reads are required. Request only measured CPU or +memory needs; fixed oversized requests may wait for matching capacity. -**Solutions:** +## Troubleshooting -1. Try different browser (Chrome/Firefox recommended) -2. Update browser to latest version -3. Disable browser extensions temporarily -4. Check graphics drivers on local machine +- If the image is not listed, confirm the image with `canfar image ls --kind + carta` and ask the deployment operator about availability. +- If a file does not open, verify the path with `canfar data info` or stage a + copy and check its format. +- If the interface is slow, reduce the data subset and inspect Session + resources before increasing the request. +- If the browser disconnects, check `canfar ps --all` and `canfar info` before + restarting the Session. +See [Support](../support/index.md) for the diagnostic information to include +when reporting a persistent problem. diff --git a/docs/platform/sessions/contributed.md b/docs/platform/sessions/contributed.md index d422e43f..5dc3fc13 100644 --- a/docs/platform/sessions/contributed.md +++ b/docs/platform/sessions/contributed.md @@ -1,45 +1,51 @@ -# Contributed Applications +# Contributed applications -**Contributed Web-based Applications on CANFAR** +Contributed Sessions expose community-provided web applications through the +Science Platform. The catalogue, image names, ports, and access requirements +are deployment-specific. -!!! abstract "🎯 What You'll Learn" - - How to launch contributed applications on CANFAR - - Where your data is stored and how to save results - - How to contribute your own web application - - Troubleshooting common issues +## Launch an application -Contributed applications are specialised, community-developed web tools that expand CANFAR's capabilities. They integrate with CANFAR storage and authentication, are web-based for easy collaboration, and require no local installation. The catalogue of available applications evolves as the community contributes new tools. +List the images currently available for this Session Kind: -!!! tip "Suggest New Applications" - Have an idea for a new application? Jump on [discord](https://discord.gg/vcCQ8QBvBa), or contact [support@canfar.net](mailto:support@canfar.net) to discuss it. +```bash +canfar login cadc +canfar image ls --kind contributed +canfar create contributed IMAGE_NAME --name my-application +``` -## 🚀 Getting Started +Open the Session when it is ready. If it remains Pending, use the checks in +[batch troubleshooting](batch.md#monitor-and-troubleshoot). If it is Running +but the application page is unavailable, report the Session ID and image +reference to the platform operator. -1. Log into the [CANFAR Science Portal](https://www.canfar.net), click **+** to create a new session, and select `contributed`. -2. Choose an application from the dropdown menu. -3. Give your session a descriptive name and click "Launch." +## Data and persistence -Your application will start in 30-90 seconds. +Use the storage paths supplied by your project. Save inputs, notebooks, and +results under `/arc/home//`, `/arc/projects//`, or a configured +VOSpace Service. Use `/scratch` only for temporary files; it is not a durable +application workspace. -!!! warning "Data Persistence" - Contributed applications can access your files at `/arc/projects/[project]/` and `/arc/home/[user]/`. Always save important results to these paths, as other locations may not persist. +The application may have its own file picker or path conventions. Confirm the +path with the application documentation rather than assuming that every +contributed image exposes the same directories. -**Currently Available Applications:** +## Contribute an application -- **[marimo](https://marimo.io)** (`skaha/marimo:latest`): Reactive Python notebooks for reproducible analysis. -- **[VSCode on Browser](https://github.com/coder/code-server)** (`skaha/vscode:latest`): A browser-based development environment for collaborative projects (based on Visual Studio Code). +Start with a container that can run the application without interactive setup. +The platform operator must confirm the web endpoint, health check, command, +resource needs, image registry, and security requirements for the target +deployment. Do not assume a fixed port or startup path from another +application. -## 🧑‍💻 Contributing Your App +For the image workflow, see [Container Images](../containers/index.md) and +[Container Registry](../containers/registry.md). Contact +[support@canfar.net](mailto:support@canfar.net) before requesting that an image +be added to the catalogue. -If you have a containerized web application, you can contribute it to the platform. The main requirements are that your application must expose a web interface on port 5000 and include a startup script at `/skaha/startup.sh`. +## Related guides -For detailed instructions, see the [Container Development](../containers/build.md) guide. We recommend contacting [support@canfar.net](mailto:support@canfar.net) to discuss your idea before you start. - -## 🆘 Troubleshooting - -- **Application doesn't load**: If your session doesn't start after 90 seconds, try a hard refresh of your browser page. -- **Data access issues**: These usually stem from incorrect file paths or permissions. Verify you are using the correct paths and have the necessary permissions. - -## 🔗 What's Next? - -To make the most of contributed applications, match the right tool to your workflow. Explore each application's capabilities and consider combining them with other CANFAR services like [Batch Processing](batch.md) for more powerful analysis. The [Storage Guide](../storage/index.md) will help you effectively manage your data. +- [Sessions overview](index.md) +- [Storage](../storage/index.md) +- [Permissions](../permissions.md) +- [Support](../support/index.md) diff --git a/docs/platform/sessions/desktop.md b/docs/platform/sessions/desktop.md index 06aef3b0..1a62bce2 100644 --- a/docs/platform/sessions/desktop.md +++ b/docs/platform/sessions/desktop.md @@ -1,326 +1,74 @@ # Desktop Sessions -**Linux graphical environment in your browser with astronomy software** +A Desktop Session provides a browser-accessible graphical Linux environment. +Use it for tools that need a window manager, multiple graphical applications, +or an interactive terminal. Use a Notebook or headless Session when a browser +desktop is unnecessary. -!!! abstract "🎯 What You'll Learn" - - How to launch, connect, and configure desktop sessions - - Available software and how to launch astronomy applications - - Managing files and storage within desktop sessions - - Tips for collaboration, performance, and troubleshooting +## Launch a Desktop -Desktop sessions on CANFAR provide a full Linux graphical environment directly in your browser, with access to CANFAR storage. Most astronomy software runs in dedicated containers on separate worker nodes and connects to your browser session using X11 protocols. This provides a familiar desktop experience for GUI applications and traditional workflows. +List the images available on the active server and choose one that contains +the software you need: -## 📋 Overview - -Desktop sessions offer: - -- **Full Linux desktop**: Accessed in your browser, with CANFAR storage integration -- **Multi-application workflow**: Run multiple programs and containers simultaneously -- **Traditional interfaces**: Use graphical astronomy software and desktop tools -- **File management**: Visual file browser and management tools -- **Session persistence**: Resume work exactly where you left off - -### Common Use Cases - -- **Running astronomy software in containers**: DS9, Aladin, TOPCAT, CASA, etc. -- **Multi-step workflows**: Combine several applications in sequence -- **Teaching and demonstrations**: Share desktop for educational purposes -- **Legacy software**: Applications requiring a desktop environment -- **Visual file management**: Organize data with graphical tools - -### How Desktop Sessions Work - -```mermaid -graph TB - Browser[Your Browser] --> Desktop[Desktop Session] - Desktop --> FileManager[File Manager] - Desktop --> Terminal[Terminal] - Desktop --> Shortcuts[Application Shortcuts] - - Shortcuts --> DS9[DS9 Container] - Shortcuts --> TOPCAT[TOPCAT Container] - Shortcuts --> Aladin[Aladin Container] - - Desktop --> AstroMenu[Astro Software Menu] - AstroMenu --> CASA[CASA Container] - AstroMenu --> UserApps[User-Contributed Apps] - - Desktop --> Storage[CANFAR Storage] - Storage --> ArcHome[/arc/home/user/] - Storage --> ArcProjects[/arc/projects/project/] - Storage --> Scratch[/scratch/] +```bash +canfar login cadc +canfar image ls --kind desktop +canfar create desktop IMAGE_NAME --name visual-analysis ``` -**Desktop Runtime**: Provides the graphical environment in your browser -**Application Containers**: Most astronomy software runs in separate containers -**X11 Forwarding**: Applications display through the desktop session -**Storage Integration**: Direct access to CANFAR filesystems - -## 🚀 Creating a Desktop Session - -### Step 1: Select Session Type and Name - -From the Science Portal dashboard, click the **plus sign (+)** to create a new session, then select **desktop** as your session type. - -Choose a descriptive session name to help you identify it later: - -**Good session names:** -- `data-reduction` -- `teaching-session` -- `multi-instrument-analysis` -- `collaborative-work` - -### Step 2: Configure Resources - -Desktop sessions use the default container and resource allocation. For most desktop work, the default settings are appropriate. - -**Resource Guidelines:** -- **Memory**: 16GB is typically sufficient for most desktop workflows -- **CPU**: 2-4 cores handle most desktop applications well -- **Storage**: Use persistent storage in `/arc/` for important work - -### Step 3: Launch Session - -Click the **Launch** button and wait for your session to initialize: - -Desktop sessions may take slightly longer to start than other session types as they need to set up the full graphical environment. - -Your session will appear on the Science Portal dashboard. - -!!! note "Connection Timing" - Sometimes it takes a few seconds for the session link to work properly. If you see a "Bad gateway" error, wait a moment and try again. - -## 🖥️ Connecting to Your Desktop - -### Initial Connection - -Click the desktop icon to access the connection page, then click **Connect** to access your desktop environment. - -### Desktop Environment - -When you connect, you'll see a full Linux desktop in your browser with the following features: - -#### Key Desktop Features - -- **Taskbar**: Application launcher and system controls at bottom -- **File Manager**: Browse CANFAR storage and manage files -- **Terminal**: Command-line access for advanced operations -- **Application Shortcuts**: Quick launch icons for common tools -- **System Menu**: Access to additional applications and settings - -### Session Persistence - -When your session becomes inactive, you'll be returned to the connection page. Click **Connect** again to resume exactly where you left off - all your applications and work remain open. - -**Session Persistence Features:** -- Open applications remain running -- File locations and window positions preserved -- Terminal sessions maintain history -- Application states saved automatically - -## 🛠️ Available Software - -### Desktop Architecture - -The desktop session provides access to astronomy software in two main ways: - -#### 1. Desktop Shortcuts - -Quick access icons available directly on the desktop: -- **DS9**: FITS image viewer and analysis -- **Aladin**: Interactive sky atlas and visualisation -- **TOPCAT**: Tool for Operations on Catalogues And Tables -- **Firefox**: Web browser for documentation and online tools - -#### 2. Astro Software Menu - -Access CANFAR-supported and user-contributed astronomy containers: - -1. Click **Applications** menu in the taskbar -2. Select **Astro Software** to browse available containers -3. Choose your desired application to launch in a dedicated container - -#### Available Applications - -| Application | Type | Best For | -|-------------|------|----------| -| **DS9** | Image Viewer | FITS file display, region analysis | -| **Aladin** | Sky Atlas | Multi-survey visualisation, catalog overlay | -| **TOPCAT** | Table Tool | Catalogue analysis, cross-matching | -| **CASA** | Radio Astronomy | Interferometry data reduction | -| **astroml** | Analysis | python software stack for astronomy and ML | - -### Native vs Container Applications - -**Native Applications** (few): -- Basic file manager and terminal -- Simple text editors -- System utilities - -**Container Applications** (most astronomy software): -- Run in dedicated containers on worker nodes -- Connect via X11 forwarding to your desktop -- Provide full functionality with isolated environments -- Include DS9, CASA, TOPCAT, Aladin, and contributed applications +You can also launch from the Science Portal. Image contents, desktop +environment, GPU availability, and resource controls are deployment-specific. +Do not assume that an application shortcut or astronomy package is present in +every image. -#### CVMFS Software Repositories - -In addition to containerized applications, you can access a wide range of scientific software via the **CVMFS** repositories mounted at `/cvmfs/`. - -See the **[Software Repositories (CVMFS)](../cvmfs.md)** guide for more information and examples. - -!!! important "Application Launch Method" - You cannot start astronomy applications by simply running commands like `ds9 &` in a terminal. These applications must be launched through desktop shortcuts or the Astro Software menu, as they run in separate containers. - -## 🧭 Working with Applications - -### Launching Applications - -#### Method 1: Desktop Shortcuts - -Click the shortcut icon on the desktop for immediate access to: -- DS9 (FITS viewer) -- Aladin (sky atlas) -- TOPCAT (table analysis) -- Firefox (web browser) - -#### Method 2: Astro Software Menu - -1. Click **Applications** in the taskbar -2. Navigate to **Astro Software** -3. Select the application or container you need -4. Application launches in a new window - -#### Method 3: File Association - -- Double-click FITS files to open in DS9 (if available) -- Right-click files for "Open with" options -- File manager remembers your preferred applications - -### Example Multi-Application Workflow - -**Optical Astronomy Analysis:** - -1. **File Management**: Organize data using the graphical file manager -2. **Image Display**: Open FITS files in DS9 for visual inspection -3. **Catalogue Analysis**: Load source lists in TOPCAT for analysis -4. **Cross-matching**: Use TOPCAT to cross-match with online catalogues -5. **Documentation**: Use Firefox to access documentation and references -6. **Scripting**: Open terminal for command-line operations as needed - -### CASA Desktop Usage - -To use CASA with its graphical interface: - -1. Launch CASA from the **Astro Software** menu -2. Start CASA in the terminal by typing either `casa` or `casa --pipeline` as appropriate. -3. Run CASA tasks as usual via command line or scripts (e.g., calibration or imaging). -4. Access CASA's plotting and visualization tools (e.g., plotms or interactive clean). - -## 🔧 Desktop Session Features - -### Copy & Paste Between Containers - -Since different containers may run on separate remote computers, text transfer between applications requires the **Clipboard** application. - -#### Accessing the Clipboard - -1. **Click the arrow** at the far left of the desktop taskbar -2. **Find "Clipboard"** in the application menu (middle of list) -3. **Click to open** the Clipboard application - -#### Using the Clipboard for Text Transfer - -The Clipboard functions as an intermediary for text transfer: - -**Transfer Process:** - -1. **Copy text**: Highlight text in source application, use `Ctrl+Shift+C` -2. **Transfer to Clipboard**: Text appears in the Clipboard application -3. **Select in Clipboard**: Highlight the text and copy with `Ctrl+Shift+C` -4. **Paste to target**: Click in destination application, use `Ctrl+Shift+V` - -!!! tip "Keyboard Shortcuts" - - **Copy**: `Ctrl+Shift+C` - - **Paste**: `Ctrl+Shift+V` - - These shortcuts work consistently across desktop containers - -### Font Size Adjustment - -Desktop containers support adjustable font sizes for better readability: - -#### Changing Terminal Font Size - -1. **Access font menu**: Hold `Ctrl` and right-click in a terminal window -2. **Select size**: Choose from Small, Medium, or Large options -3. **Apply immediately**: Font changes take effect instantly - -**Compatible Applications:** -- Terminal windows -- CASA command-line interface -- Text-based applications - -!!! note "Font Persistence" - Font size changes apply only to the current session. You'll need to readjust when starting new sessions. - -## 💾 File Management - -### Storage Access - -Your desktop session provides access to all CANFAR storage systems: +Monitor and open the Session when it is ready: ```bash -/arc/home/[user]/ # Personal persistent storage (10GB) -/arc/projects/[project]/ # Shared project storage -/scratch/ # Temporary high-speed storage +canfar ps --all +canfar info SESSION_ID +canfar open SESSION_ID ``` -### File Operations - -Use the graphical file manager for: +If it remains `Pending`, inspect `canfar events SESSION_ID`; see [batch +troubleshooting](batch.md#monitor-and-troubleshoot). -- **Visual browsing**: Navigate directories with point-and-click -- **Drag-and-drop**: Move files between directories easily -- **Preview**: View image thumbnails and file properties -- **Batch operations**: Select multiple files for operations -- **Permissions**: Set file and directory permissions graphically +## Files and applications -### File Transfer +The desktop terminal and file manager see the Session's mounted paths: -**Small Files**: Drag and drop from your local computer to the file manager -**Large Files**: Use [data transfer methods](../storage/transfers.md) -**Between Sessions**: Files in `/arc/` are accessible from all session types - -!!! warning "Persistence Reminder" - Save important work to `/arc/projects/` or `/arc/home/`. Files in `/scratch/` will not persist after the session ends. - -## 🤝 Collaboration and Sharing - -### Session Sharing - -Desktop sessions can be shared for collaborative work: +```text +/arc/home// personal persistent files +/arc/projects// project files, when available +/scratch/ temporary staging +``` -1. **Copy session URL** from browser address bar -2. **Share with team members** who have CANFAR accounts -3. **Coordinate activities** to avoid conflicts -4. **Use shared storage** in `/arc/projects/[project]/` for collaboration +Save data, scripts, and exported results under `/arc` or a persistent VOSpace +Service. `/scratch` is deleted when the Session ends. For remote objects that +are not mounted, use [CANFAR data transfers](../storage/transfers.md) before +opening them in a path-oriented application. -### Collaborative Workflows +Some deployments expose scientific software through [CVMFS](../cvmfs.md). +It is read-only and its repositories and modules vary by server. Record the +module and version if a workflow depends on it. -**Teaching and Training:** -- Share desktop session URL with students -- Demonstrate software usage in real-time -- Students can follow along with same tools +## Good desktop practice -**Team Analysis:** -- Multiple researchers access same desktop -- Share applications and data simultaneously -- Coordinate complex multi-step analyzes +- Keep the graphical Session for exploration, visual inspection, and tools + that genuinely need a display. +- Save work before deleting or restarting a Session; a browser Session is not + a durable data store. +- Use a named Container Image or versioned environment for repeatable software + rather than installing packages only in the current Session. +- Copy final products to persistent storage before closing the Session. -### Best Practices for Collaboration +## Troubleshooting -- **Communicate clearly** about who is controlling what -- **Use shared storage** for data that everyone needs to access -- **Plan ahead** for resource-intensive operations -- **Save work frequently** to avoid conflicts +- If the image is missing, confirm it with `canfar image ls --kind desktop`. +- If the Session is Running but the desktop does not open, retry the link in a + current browser and report the Session ID and endpoint error. +- If a file is missing or forbidden, verify the identifier/path and project + membership with `canfar data info` and the project administrator. +- If the desktop is slow, close unused applications and inspect the Session's + resource usage before requesting more CPU or memory. +See [Support](../support/index.md) for a minimal diagnostic report. diff --git a/docs/platform/sessions/firefly.md b/docs/platform/sessions/firefly.md index 9df97169..aee739a2 100644 --- a/docs/platform/sessions/firefly.md +++ b/docs/platform/sessions/firefly.md @@ -1,426 +1,76 @@ # Firefly Sessions -**The LSST table and image visualizer for astronomical data exploration** +[Firefly](https://github.com/Caltech-IPAC/firefly) is a browser-based +astronomy application for viewing images, tables, and supported archive data. +The available Firefly image and features are deployment-specific. -!!! abstract "🎯 What You'll Learn" - - How to launch a Firefly session and choose the right version - - How to load images, tables, and access CANFAR storage - - How to perform catalogue overlays, plotting, and cutouts - - Performance tips for large surveys and troubleshooting guidance - -Firefly is a powerful web-based visualisation tool originally developed for the Rubin Observatory LSST. It provides advanced capabilities for viewing images, overlaying catalogues, and analysing tabular data - making it perfect for survey data analysis and multi-wavelength astronomy. - -## 🎯 What is Firefly? - -Firefly offers specialised tools for: - -- **Image visualisation** with advanced stretch and colour controls -- **Catalogue overlay** and source analysis tools -- **Table viewer** with filtering, plotting, and statistical tools -- **Multi-wavelength data** comparison and analysis -- **Large survey datasets** like LSST, HSC, and WISE - -### Key Features - -| Feature | Capability | -|---------|------------| -| **Image Display** | FITS images with WCS support, multiple panels | -| **Catalogue Overlay** | Plot sources on images, interactive selection | -| **Table Analysis** | Sort, filter, plot columns, statistical analysis | -| **Multi-band** | RGB colour composites, band switching | -| **Cutout Services** | Extract subimages from large surveys | -| **Coordinate Systems** | Support for all standard astronomical coordinates | - -## 🚀 Launching Firefly - -### Step 1: Create New Session - -1. **Login** to the [CANFAR Science Portal](https://www.canfar.net/science-portal) -2. **Click** the plus sign (**+**) to create a new session -3. **Select** `firefly` as your session type - -### Step 2: Choose Container - -The container selection updates automatically after choosing the session type. Select the Firefly container version you need: - -#### Available Versions - -- **firefly:latest** - Most recent stable version (recommended) -- **firefly:X.X** - Specific version for reproducible analysis - -!!! tip "Version Selection" - Use the latest version unless you need a specific version for reproducibility. The latest version includes performance improvements and new features. - -### Step 3: Configure Session - -#### Session Name - -Choose a descriptive name that helps identify your work: - -**Good session names:** -- `lsst-photometry` -- `hsc-catalogue-analysis` -- `multiband-survey` -- `gaia-cross-match` - -#### Memory Requirements - -If using a fixed resource session, select RAM based on your data size: - -- **8GB**: Small catalogues, single images -- **16GB**: Default, suitable for most work -- **32GB**: Large catalogues, multiple images -- **64GB**: Very large survey datasets - -!!! tip "Memory Planning" - Large tables and multi-image layouts benefit from 32GB+ RAM. Start with 8GB and scale up if needed. - -#### CPU Cores - -Most Firefly work is I/O bound rather than CPU intensive: - -- **2 cores**: Default, sufficient for most visualisation tasks -- **4 cores**: Large table operations, complex filtering - -### Step 4: Launch Session - -1. **Click** "Launch" button -2. **Wait** for container initialisation (~30-60 seconds) -3. **Session appears** on your portal dashboard -4. **Click** the session icon to access Firefly - -## 🔥 Using Firefly - -### Interface Overview - -Firefly's interface consists of several main areas: - -```mermaid -graph TD - Interface[Firefly Interface] - Interface --> Upload[File Upload Area] - Interface --> Images[Image Display] - Interface --> Tables[Table Viewer] - Interface --> Tools[Analysis Tools] - - Upload --> Local[Local Files] - Upload --> URLs[Remote URLs] - Upload --> Storage[CANFAR Storage] - - Images --> Display[Image Canvas] - Images --> Controls[Display Controls] - Images --> Overlays[Catalogue Overlays] - - Tables --> Browse[Data Browser] - Tables --> Filter[Filtering Tools] - Tables --> Plot[Plotting Tools] -``` - -#### Main Components - -- **File Upload Area**: Load local files, URLs, or access CANFAR storage -- **Image Display**: Multi-panel image viewer with WCS support -- **Table Viewer**: Advanced table browser with analysis tools -- **Control Panels**: Image display controls, colour maps, overlays - -### Loading Data - -#### Upload Local Files - -**FITS Images:** - -1. Click "Images" tab -2. Select "Upload" -3. Choose FITS file from your computer -4. Image loads automatically with WCS if available - -**Catalogue Tables:** - -1. Click "Tables" tab -2. Select "Upload" -3. Choose CSV, FITS table, or VOTable -4. Table opens in browser interface - -#### Access CANFAR Storage - -**From ARC Projects:** +## Launch Firefly ```bash -# Files in your project directory are accessible via: -/arc/projects/[project]/data/image.fits -/arc/projects/[project]/data/image_sources.csv -``` - -**From VOSpace:** - -1. In Firefly, use "File" → "Open" -2. Navigate to VOSpace URLs -3. Access: `vos://cadc.nrc.ca~vault/[user]/` - -#### Remote Data Access - -**Survey Archives:** - -```text -# Example URLs for Firefly -https://archive.stsci.edu/hlsp/data.fits -https://irsa.ipac.caltech.edu/data/WISE/cutouts/ -``` - -**Supported Formats:** - -- **Images**: FITS, JPEG, PNG -- **Tables**: CSV, FITS tables, VOTable, IPAC tables -- **Archives**: Gzipped files automatically handled - -### Image Analysis - -#### Basic Image Display - -**Display Controls:** - -1. Load FITS image -2. Adjust stretch (log, linear, sqrt) -3. Set scale limits (min/max values) -4. Choose colour table (heat, cool, rainbow) - -**Navigation:** - -- **Zoom**: Mouse wheel or zoom controls -- **Pan**: Click and drag -- **Centre**: Double-click to centre -- **Reset**: Reset zoom and pan to default - -#### Multi-band RGB - -**Creating RGB Composites:** - -1. Load three images (e.g., g, r, i bands) -2. Select "RGB" mode -3. Assign each image to R, G, or B channel -4. Adjust relative scaling -5. Fine-tune colour balance - -#### Coordinate Systems - -Firefly supports standard coordinate systems: - -- **Equatorial**: RA/Dec (J2000, B1950) -- **Galactic**: Galactic longitude/latitude -- **Ecliptic**: Ecliptic coordinates -- **Pixel**: Image pixel coordinates - -### Catalogue Analysis - -#### Table Operations - -**Basic Navigation:** - -- **Sort columns**: Click headers to sort -- **Filter rows**: Use search box for text filtering -- **Select rows**: Click rows, Ctrl+click for multiple -- **Pagination**: Navigate large tables with page controls - -**Advanced Filtering:** - -```javascript -// Example filters (use in filter box): -magnitude < 20.5 // Bright sources -colour_g_r > 0.5 && colour_g_r < 1.5 // Colour selection -distance < 100 // Distance constraint -ra > 180 && ra < 200 // RA range +canfar login cadc +canfar image ls --kind firefly +canfar create firefly IMAGE_NAME --name archive-exploration ``` -#### Statistical Analysis - -**Built-in Statistics:** - -- **Column statistics**: Mean, median, std deviation -- **Histogram analysis**: Distribution plots -- **Cross-correlation**: Compare columns -- **Selection statistics**: Stats on filtered data - -#### Plotting Tools - -**Column Plots:** - -1. Select table columns for X and Y axes -2. Choose plot type (scatter, histogram, line) -3. Apply colour coding by third column -4. Add error bars if available -5. Customize symbols and colours - -**Image-Catalogue Overlay:** - -1. Load image and catalogue table -2. Match coordinate columns (RA, Dec) -3. Select overlay symbol (circle, cross, diamond) -4. Adjust symbol size and colour -5. Sources appear overlaid on image - -### Advanced Features - -#### Cutout Services - -Extract subimages from large surveys: - -**Manual Cutouts:** - -1. Right-click on image location -2. Select "Create Cutout" -3. Specify size (arcmin) -4. Choose format (FITS, JPEG, PNG) -5. Download or save to CANFAR storage - -**Programmatic Cutouts:** - -```python -# Example using Python and Firefly -import requests +The Science Portal provides the same Session Kind. Monitor the Session and +open it when ready: -url = "https://irsa.ipac.caltech.edu/cgi-bin/Cutouts/nph-cutouts" -params = { - 'mission': 'wise', - 'locstr': '10.68 +41.27', - 'sizeX': '300', - 'sizeY': '300' -} -response = requests.get(url, params=params) +```bash +canfar ps --all +canfar info SESSION_ID +canfar open SESSION_ID ``` -#### Multi-wavelength Analysis - -**Cross-band Analysis:** - -1. Load images in different bands -2. Use "Blink" mode to compare -3. Create RGB composite -4. Overlay catalogue with colour-magnitude selection -5. Identify sources across wavelengths - -**Spectral Energy Distributions:** - -1. Load multi-band photometry table -2. Select source of interest -3. Plot flux vs wavelength -4. Fit SED models if available - -#### Data Export - -**Save Results:** +If a Session remains `Pending`, inspect `canfar events SESSION_ID`; see [batch +troubleshooting](batch.md#monitor-and-troubleshoot). -- **Modified tables**: CSV, FITS, VOTable formats -- **Image displays**: PNG, PDF for publications -- **Analysis plots**: Vector formats for papers -- **Session state**: Save/restore workspace +## Load data -**Export Options:** +Use the paths and URLs supported by the selected Firefly image. A common +workflow is to place data under mounted storage: ```text -File → Export → [Format] -- Tables: CSV, FITS table, VOTable -- Images: FITS, PNG, JPEG, PDF -- Plots: PNG, PDF, SVG -- Session: Save current state -``` - -## 🛠️ Common Workflows - -### Survey Photometry - -**HSC/LSST Photometry Workflow:** - -1. Load survey image (HSC, LSST, etc.) -2. Upload photometric catalogue -3. Overlay sources on image -4. Filter by magnitude and colour -5. Create colour-magnitude diagram -6. Export selected sources - -```javascript -// Example: Filter for main sequence stars -// In Firefly filter box: -(g_mag - r_mag) > 0.2 && (g_mag - r_mag) < 1.0 && r_mag < 22 -``` - -### Multi-object Analysis - -**Target List Processing:** - -1. Load target list (CSV with coordinates) -2. Create cutouts around each target -3. Measure properties in each cutout -4. Compile results in table -5. Plot trends and correlations -6. Save analysis products - -### Cross-matching Catalogues - -**Gaia Cross-match Example:** - -1. Load your source catalogue -2. Load Gaia reference catalogue -3. Perform spatial cross-match -4. Analyze proper motions and parallaxes -5. Create clean stellar sample -6. Export matched catalogue - -### Time Series Visualisation - -**Light Curve Analysis:** - -1. Load time-series table (time, magnitude, error) -2. Create light curve plot -3. Apply period folding if needed -4. Identify outliers and trends -5. Export cleaned data - -## 🔧 Integration with CANFAR - -### Storage Access - -**ARC Projects:** - -```bash -# Your project data appears in Firefly file browser -/arc/projects/[project]/ -├── images/ # FITS images -├── catalogues/ # Source tables -├── results/ # Analysis products -└── plots/ # Exported figures +/arc/home// personal persistent files +/arc/projects// shared project files, when available +/scratch/ temporary staging ``` -**VOSpace Integration:** +For a remote object in a configured VOSpace Service, transfer it explicitly +before opening it in a path-oriented application: ```bash -# Access archived data -vos://cadc.nrc.ca~vault/[user]/ -├── published_data/ # Public datasets -├── working_data/ # Analysis in progress -└── final_products/ # Paper-ready results +canfar data cp IDENTIFIER:/path/to/image.fits local:/scratch/image.fits ``` -### Collaborative Features +Use Firefly's own archive or URL features only as documented by the selected +image. Do not assume that a VOSpace URL, archive endpoint, table format, or +remote service is enabled on every deployment. -**Session Sharing:** +Save figures, tables, regions, and other products under `/arc` or a persistent +VOSpace Service. `/scratch` is deleted when the Session ends. See [Storage](../storage/index.md) +and [Data transfers](../storage/transfers.md). -1. Copy Firefly session URL -2. Share with team members (same CANFAR group) -3. Collaborate on analysis in real-time -4. Each user sees same data and visualisations +## Typical workflow -**Data Sharing:** +1. Open a FITS image, table, or archive result supported by the image. +2. Inspect image metadata and WCS before interpreting the display. +3. Use the image, table, and catalogue tools provided by the selected Firefly + version. +4. Export products and record the source path, image reference, and relevant + display settings. -1. Save analysis results to shared project space -2. Export publication-quality figures -3. Share VOSpace links for external collaborators -4. Version control important datasets +For analysis that must be rerun, use Python or a headless Session and keep +Firefly for visual inspection. -### Working with Other CANFAR Tools +## Troubleshooting -**Integration Patterns:** +- If the image is unavailable, check `canfar image ls --kind firefly` and ask + the platform operator about the deployment's catalogue. +- If a local file does not open, verify its path and permissions or stage it + to `/scratch` with `canfar data cp`. +- If a remote URL fails, confirm that the image supports that endpoint and + that the Session can reach it; do not embed credentials in the URL. +- If the browser disconnects, check `canfar ps --all` and `canfar info` before + restarting the Session. -- **Notebooks → Firefly**: Prepare data in Python, visualize in Firefly -- **Firefly → Desktop**: Export results for further analysis in CASA/DS9 -- **Batch → Firefly**: Process large datasets, visualize results -- **CARTA → Firefly**: Radio analysis in CARTA, optical follow-up in Firefly +See [Support](../support/index.md) when the cause remains unclear. diff --git a/docs/platform/sessions/images/carta/10_navigate_files.png b/docs/platform/sessions/images/carta/10_navigate_files.png deleted file mode 100644 index de5cf6fa..00000000 Binary files a/docs/platform/sessions/images/carta/10_navigate_files.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/11_select_file.png b/docs/platform/sessions/images/carta/11_select_file.png deleted file mode 100644 index 22d0392d..00000000 Binary files a/docs/platform/sessions/images/carta/11_select_file.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/12_success_CARTA.png b/docs/platform/sessions/images/carta/12_success_CARTA.png deleted file mode 100644 index 6d3e0f5b..00000000 Binary files a/docs/platform/sessions/images/carta/12_success_CARTA.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/1_select_carta_session.png b/docs/platform/sessions/images/carta/1_select_carta_session.png deleted file mode 100644 index e8aa7af8..00000000 Binary files a/docs/platform/sessions/images/carta/1_select_carta_session.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/2_select_carta_container.png b/docs/platform/sessions/images/carta/2_select_carta_container.png deleted file mode 100644 index 8615182c..00000000 Binary files a/docs/platform/sessions/images/carta/2_select_carta_container.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/3_choose_carta_name.png b/docs/platform/sessions/images/carta/3_choose_carta_name.png deleted file mode 100644 index 892d15ce..00000000 Binary files a/docs/platform/sessions/images/carta/3_choose_carta_name.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/4_choose_carta_ram.png b/docs/platform/sessions/images/carta/4_choose_carta_ram.png deleted file mode 100644 index 2c585b2a..00000000 Binary files a/docs/platform/sessions/images/carta/4_choose_carta_ram.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/5_choose_carta_cores.png b/docs/platform/sessions/images/carta/5_choose_carta_cores.png deleted file mode 100644 index a107671a..00000000 Binary files a/docs/platform/sessions/images/carta/5_choose_carta_cores.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/6_launch_carta.png b/docs/platform/sessions/images/carta/6_launch_carta.png deleted file mode 100644 index 3dbfac74..00000000 Binary files a/docs/platform/sessions/images/carta/6_launch_carta.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/7_click_carta.png b/docs/platform/sessions/images/carta/7_click_carta.png deleted file mode 100644 index 07afd682..00000000 Binary files a/docs/platform/sessions/images/carta/7_click_carta.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/8_carta_loading.png b/docs/platform/sessions/images/carta/8_carta_loading.png deleted file mode 100644 index be78ce1d..00000000 Binary files a/docs/platform/sessions/images/carta/8_carta_loading.png and /dev/null differ diff --git a/docs/platform/sessions/images/carta/9_main_carta_landing.png b/docs/platform/sessions/images/carta/9_main_carta_landing.png deleted file mode 100644 index ec2c06c8..00000000 Binary files a/docs/platform/sessions/images/carta/9_main_carta_landing.png and /dev/null differ diff --git a/docs/platform/sessions/images/contributed/1_placeholder.png b/docs/platform/sessions/images/contributed/1_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/1_select_contributed_session.png b/docs/platform/sessions/images/contributed/1_select_contributed_session.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/2_placeholder.png b/docs/platform/sessions/images/contributed/2_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/2_select_contributed_container.png b/docs/platform/sessions/images/contributed/2_select_contributed_container.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/3_choose_contributed_name.png b/docs/platform/sessions/images/contributed/3_choose_contributed_name.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/3_placeholder.png b/docs/platform/sessions/images/contributed/3_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/4_choose_contributed_resources.png b/docs/platform/sessions/images/contributed/4_choose_contributed_resources.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/4_placeholder.png b/docs/platform/sessions/images/contributed/4_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/5_launch_contributed.png b/docs/platform/sessions/images/contributed/5_launch_contributed.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/5_placeholder.png b/docs/platform/sessions/images/contributed/5_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/contributed/6_placeholder.png b/docs/platform/sessions/images/contributed/6_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/desktop/1_launch_desktop.png b/docs/platform/sessions/images/desktop/1_launch_desktop.png deleted file mode 100644 index bc191747..00000000 Binary files a/docs/platform/sessions/images/desktop/1_launch_desktop.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/3_choose_name.png b/docs/platform/sessions/images/desktop/3_choose_name.png deleted file mode 100644 index b40bb927..00000000 Binary files a/docs/platform/sessions/images/desktop/3_choose_name.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/4_launch.png b/docs/platform/sessions/images/desktop/4_launch.png deleted file mode 100644 index 92dfbe2b..00000000 Binary files a/docs/platform/sessions/images/desktop/4_launch.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/5_active_desktop.png b/docs/platform/sessions/images/desktop/5_active_desktop.png deleted file mode 100644 index d9c51c72..00000000 Binary files a/docs/platform/sessions/images/desktop/5_active_desktop.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/6_connect_desktop.png b/docs/platform/sessions/images/desktop/6_connect_desktop.png deleted file mode 100644 index e37805d1..00000000 Binary files a/docs/platform/sessions/images/desktop/6_connect_desktop.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/7_desktop_connected.png b/docs/platform/sessions/images/desktop/7_desktop_connected.png deleted file mode 100644 index 680dba2c..00000000 Binary files a/docs/platform/sessions/images/desktop/7_desktop_connected.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/clipboard/1_desktop_landing.png b/docs/platform/sessions/images/desktop/clipboard/1_desktop_landing.png deleted file mode 100644 index 4de8c535..00000000 Binary files a/docs/platform/sessions/images/desktop/clipboard/1_desktop_landing.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/clipboard/2_desktop_with_clipboard_menu.png b/docs/platform/sessions/images/desktop/clipboard/2_desktop_with_clipboard_menu.png deleted file mode 100644 index 23a6b14f..00000000 Binary files a/docs/platform/sessions/images/desktop/clipboard/2_desktop_with_clipboard_menu.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/clipboard/3_clipboard_open.png b/docs/platform/sessions/images/desktop/clipboard/3_clipboard_open.png deleted file mode 100644 index ae1b400b..00000000 Binary files a/docs/platform/sessions/images/desktop/clipboard/3_clipboard_open.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/clipboard/4_text_into_clipboard.png b/docs/platform/sessions/images/desktop/clipboard/4_text_into_clipboard.png deleted file mode 100644 index 3c840285..00000000 Binary files a/docs/platform/sessions/images/desktop/clipboard/4_text_into_clipboard.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/clipboard/5_copy_text_to_casa.png b/docs/platform/sessions/images/desktop/clipboard/5_copy_text_to_casa.png deleted file mode 100644 index d596f2a1..00000000 Binary files a/docs/platform/sessions/images/desktop/clipboard/5_copy_text_to_casa.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/font/1_terminal_original_font.png b/docs/platform/sessions/images/desktop/font/1_terminal_original_font.png deleted file mode 100644 index 1ad83a54..00000000 Binary files a/docs/platform/sessions/images/desktop/font/1_terminal_original_font.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/font/2_fontsize_popup.png b/docs/platform/sessions/images/desktop/font/2_fontsize_popup.png deleted file mode 100644 index db88f860..00000000 Binary files a/docs/platform/sessions/images/desktop/font/2_fontsize_popup.png and /dev/null differ diff --git a/docs/platform/sessions/images/desktop/font/3_terminal_new_fontsize.png b/docs/platform/sessions/images/desktop/font/3_terminal_new_fontsize.png deleted file mode 100644 index 9aa75ee4..00000000 Binary files a/docs/platform/sessions/images/desktop/font/3_terminal_new_fontsize.png and /dev/null differ diff --git a/docs/platform/sessions/images/firefly/1_placeholder.png b/docs/platform/sessions/images/firefly/1_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/1_select_firefly_session.png b/docs/platform/sessions/images/firefly/1_select_firefly_session.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/2_placeholder.png b/docs/platform/sessions/images/firefly/2_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/2_select_firefly_container.png b/docs/platform/sessions/images/firefly/2_select_firefly_container.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/3_choose_firefly_name.png b/docs/platform/sessions/images/firefly/3_choose_firefly_name.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/3_placeholder.png b/docs/platform/sessions/images/firefly/3_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/4_choose_firefly_ram.png b/docs/platform/sessions/images/firefly/4_choose_firefly_ram.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/4_placeholder.png b/docs/platform/sessions/images/firefly/4_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/5_choose_firefly_cores.png b/docs/platform/sessions/images/firefly/5_choose_firefly_cores.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/5_placeholder.png b/docs/platform/sessions/images/firefly/5_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/6_launch_firefly.png b/docs/platform/sessions/images/firefly/6_launch_firefly.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/firefly/6_placeholder.png b/docs/platform/sessions/images/firefly/6_placeholder.png deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/platform/sessions/images/notebook/10_example_casa_in_ipy_notebook.png b/docs/platform/sessions/images/notebook/10_example_casa_in_ipy_notebook.png deleted file mode 100644 index 580bc4a0..00000000 Binary files a/docs/platform/sessions/images/notebook/10_example_casa_in_ipy_notebook.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/1_select_notebook_session.png b/docs/platform/sessions/images/notebook/1_select_notebook_session.png deleted file mode 100644 index cef4b914..00000000 Binary files a/docs/platform/sessions/images/notebook/1_select_notebook_session.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/2_choose_notebook.png b/docs/platform/sessions/images/notebook/2_choose_notebook.png deleted file mode 100644 index a154e782..00000000 Binary files a/docs/platform/sessions/images/notebook/2_choose_notebook.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/3_choose_casa_container.png b/docs/platform/sessions/images/notebook/3_choose_casa_container.png deleted file mode 100644 index 809940f5..00000000 Binary files a/docs/platform/sessions/images/notebook/3_choose_casa_container.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/4_choose_name.png b/docs/platform/sessions/images/notebook/4_choose_name.png deleted file mode 100644 index 54f2a8fa..00000000 Binary files a/docs/platform/sessions/images/notebook/4_choose_name.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/5_select_RAM.png b/docs/platform/sessions/images/notebook/5_select_RAM.png deleted file mode 100644 index 2c585b2a..00000000 Binary files a/docs/platform/sessions/images/notebook/5_select_RAM.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/6_choose_cores.png b/docs/platform/sessions/images/notebook/6_choose_cores.png deleted file mode 100644 index a107671a..00000000 Binary files a/docs/platform/sessions/images/notebook/6_choose_cores.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/7_launch_notebook.png b/docs/platform/sessions/images/notebook/7_launch_notebook.png deleted file mode 100644 index 838e12a3..00000000 Binary files a/docs/platform/sessions/images/notebook/7_launch_notebook.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/8_notebook_created.png b/docs/platform/sessions/images/notebook/8_notebook_created.png deleted file mode 100644 index 4336f2f0..00000000 Binary files a/docs/platform/sessions/images/notebook/8_notebook_created.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/9_notebook_landing.png b/docs/platform/sessions/images/notebook/9_notebook_landing.png deleted file mode 100644 index 081f8d3f..00000000 Binary files a/docs/platform/sessions/images/notebook/9_notebook_landing.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/1_landing_click_upload.png b/docs/platform/sessions/images/notebook/transfer_file/1_landing_click_upload.png deleted file mode 100644 index 8cbb7d73..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/1_landing_click_upload.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/2_upload_window.png b/docs/platform/sessions/images/notebook/transfer_file/2_upload_window.png deleted file mode 100644 index 3eebc16d..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/2_upload_window.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/3_file_is_uploaded.png b/docs/platform/sessions/images/notebook/transfer_file/3_file_is_uploaded.png deleted file mode 100644 index d66b6d10..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/3_file_is_uploaded.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/4_open_terminal.png b/docs/platform/sessions/images/notebook/transfer_file/4_open_terminal.png deleted file mode 100644 index c4ab6542..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/4_open_terminal.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/5_new_terminal.png b/docs/platform/sessions/images/notebook/transfer_file/5_new_terminal.png deleted file mode 100644 index 92bc197a..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/5_new_terminal.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/6_copy_local_text.png b/docs/platform/sessions/images/notebook/transfer_file/6_copy_local_text.png deleted file mode 100644 index f4b1521c..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/6_copy_local_text.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/7_paste_text.png b/docs/platform/sessions/images/notebook/transfer_file/7_paste_text.png deleted file mode 100644 index 3a6af3bd..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/7_paste_text.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/8_file_saved.png b/docs/platform/sessions/images/notebook/transfer_file/8_file_saved.png deleted file mode 100644 index f9112880..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/8_file_saved.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/orig/1_landing_click_upload.png b/docs/platform/sessions/images/notebook/transfer_file/orig/1_landing_click_upload.png deleted file mode 100644 index 10f93eaf..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/orig/1_landing_click_upload.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/orig/2_upload_window.png b/docs/platform/sessions/images/notebook/transfer_file/orig/2_upload_window.png deleted file mode 100644 index 3eebc16d..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/orig/2_upload_window.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/orig/3_file_is_uploaded.png b/docs/platform/sessions/images/notebook/transfer_file/orig/3_file_is_uploaded.png deleted file mode 100644 index cea4dbbb..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/orig/3_file_is_uploaded.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/orig/4_open_terminal.png b/docs/platform/sessions/images/notebook/transfer_file/orig/4_open_terminal.png deleted file mode 100644 index cea4dbbb..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/orig/4_open_terminal.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/orig/5_new_terminal.png b/docs/platform/sessions/images/notebook/transfer_file/orig/5_new_terminal.png deleted file mode 100644 index 8d3d0c62..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/orig/5_new_terminal.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/orig/6_copy_local_text.png b/docs/platform/sessions/images/notebook/transfer_file/orig/6_copy_local_text.png deleted file mode 100644 index f4b1521c..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/orig/6_copy_local_text.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/orig/7_paste_text.png b/docs/platform/sessions/images/notebook/transfer_file/orig/7_paste_text.png deleted file mode 100644 index 3a6af3bd..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/orig/7_paste_text.png and /dev/null differ diff --git a/docs/platform/sessions/images/notebook/transfer_file/orig/8_file_saved.png b/docs/platform/sessions/images/notebook/transfer_file/orig/8_file_saved.png deleted file mode 100644 index fcf07787..00000000 Binary files a/docs/platform/sessions/images/notebook/transfer_file/orig/8_file_saved.png and /dev/null differ diff --git a/docs/platform/sessions/images/transfer_file/1_landing_click_upload.png b/docs/platform/sessions/images/transfer_file/1_landing_click_upload.png deleted file mode 100644 index 8cbb7d73..00000000 Binary files a/docs/platform/sessions/images/transfer_file/1_landing_click_upload.png and /dev/null differ diff --git a/docs/platform/sessions/images/transfer_file/2_upload_window.png b/docs/platform/sessions/images/transfer_file/2_upload_window.png deleted file mode 100644 index 3eebc16d..00000000 Binary files a/docs/platform/sessions/images/transfer_file/2_upload_window.png and /dev/null differ diff --git a/docs/platform/sessions/images/transfer_file/3_file_is_uploaded.png b/docs/platform/sessions/images/transfer_file/3_file_is_uploaded.png deleted file mode 100644 index d66b6d10..00000000 Binary files a/docs/platform/sessions/images/transfer_file/3_file_is_uploaded.png and /dev/null differ diff --git a/docs/platform/sessions/images/transfer_file/4_open_terminal.png b/docs/platform/sessions/images/transfer_file/4_open_terminal.png deleted file mode 100644 index c4ab6542..00000000 Binary files a/docs/platform/sessions/images/transfer_file/4_open_terminal.png and /dev/null differ diff --git a/docs/platform/sessions/images/transfer_file/5_new_terminal.png b/docs/platform/sessions/images/transfer_file/5_new_terminal.png deleted file mode 100644 index 92bc197a..00000000 Binary files a/docs/platform/sessions/images/transfer_file/5_new_terminal.png and /dev/null differ diff --git a/docs/platform/sessions/images/transfer_file/6_copy_local_text.png b/docs/platform/sessions/images/transfer_file/6_copy_local_text.png deleted file mode 100644 index f4b1521c..00000000 Binary files a/docs/platform/sessions/images/transfer_file/6_copy_local_text.png and /dev/null differ diff --git a/docs/platform/sessions/images/transfer_file/7_paste_text.png b/docs/platform/sessions/images/transfer_file/7_paste_text.png deleted file mode 100644 index 3a6af3bd..00000000 Binary files a/docs/platform/sessions/images/transfer_file/7_paste_text.png and /dev/null differ diff --git a/docs/platform/sessions/images/transfer_file/8_file_saved.png b/docs/platform/sessions/images/transfer_file/8_file_saved.png deleted file mode 100644 index f9112880..00000000 Binary files a/docs/platform/sessions/images/transfer_file/8_file_saved.png and /dev/null differ diff --git a/docs/platform/sessions/index.md b/docs/platform/sessions/index.md index e73d9c28..e04b494e 100644 --- a/docs/platform/sessions/index.md +++ b/docs/platform/sessions/index.md @@ -1,183 +1,96 @@ -# Interactive Sessions - -**CANFAR computing environments for astronomical research - Jupyter notebooks, and non-interactive applications.** - -!!! abstract "🎯 Session Types Overview" - **Choose the right interface for your research:** - - - **[Jupyter Notebooks](notebook.md)**: Interactive data analysis and visualisation - - **[Desktop Environment](desktop.md)**: Full Linux desktop with GUI applications - - **[CARTA Viewer](carta.md)**: Radio astronomy visualisation and analysis - - **[Firefly Viewer](firefly.md)**: Table and image viewing for surveys - - **[Contributed Apps](contributed.md)**: Specialised community applications - - **[Batch Processing](batch.md)**: Automated and large-scale workflows - -## 🚀 Session Fundamentals - -### What are Interactive Sessions? - -Interactive sessions provide on-demand access to pre-configured computing environments running in containers. Each session type offers different interfaces optimized for specific astronomical workflows. - -### Key Benefits - -**No Installation Required** -: Access complex astronomy software through your web browser without local installation or configuration. - -**Pre-Configured Environments** -: Containers include popular astronomy packages like AstroPy, CASA, and scientific Python libraries ready to use. - -**Persistent Data Access** -: All sessions automatically connect to your ARC storage and can access VOSpace for long-term data management. - -**Scalable Resources** -: Choose flexible or fixed resource allocation based on your computational requirements. - -## 📊 Session Type Comparison - -| Session Type | Interface | Best For | GUI Support | -|--------------|-----------|----------|-------------| -| **[Notebook](notebook.md)** | JupyterLab | Data analysis, prototyping, documentation | ✅ Web-based | -| **[Desktop](desktop.md)** | Full Linux desktop | CASA, image viewers, traditional software | ✅ Desktop GUI | -| **[CARTA](carta.md)** | CARTA interface | Radio astronomy visualisation | ✅ Specialised | -| **[Firefly](firefly.md)** | Firefly viewer | Catalogue analysis, image display | ✅ Web-based | -| **[Contributed](contributed.md)** | Various | Specialised applications | ⚠️ Varies | -| **[Batch](batch.md)** | None (headless) | Large-scale processing | ❌ Headless | - -## 🔧 Session Management - -### Creating Sessions - -**Via Science Portal:** -: Launch sessions through the [CANFAR Science Portal](https://www.canfar.net/science-portal/) web interface with point-and-click simplicity. - -**Via Command Line:** -: Use the [CANFAR CLI](../../cli/cli-help.md) for scripted session creation and automation. - -**Via Python API:** -: Integrate session management into custom workflows using the [CANFAR Python Client](../../client/home.md). - -### Session Lifecycle - -**Creation** (30 seconds - 3 minutes) -: Container download (first time) and startup with storage mounting - -**Active Use** -: Full access to computing resources and storage systems - -**Idle Management** -: Sessions automatically suspend after periods of inactivity to conserve resources - -**Termination** -: Container deletion with data preserved in persistent storage - -!!! warning "Data Persistence" - **Important**: Session containers are temporary. Always save important work to `/arc/` storage or VOSpace before ending sessions. - -## 📈 Resource Allocation - -### Flexible Allocation (Default) - -**Advantages:** -- Faster session startup -- Can burst to higher resource usage when available -- Optimal for interactive work and development - -**Best For:** -- Data exploration and analysis -- Development and testing -- Educational workshops - -### Fixed Allocation - -**Advantages:** -- Guaranteed consistent performance -- Predictable resource availability -- Better for production workloads - -**Best For:** -- Large-scale processing -- Performance-critical analysis -- Time-sensitive computations - -### Resource Selection Guide - -| Workflow Type | Recommended Mode | CPU/Memory | Duration | -|---------------|------------------|------------|----------| -| **Interactive Analysis** | Flexible | 2-4 CPU, 4-8GB | Hours | -| **Large Dataset Processing** | Fixed | 4-8 CPU, 16-32GB | Hours-Days | -| **Development & Testing** | Flexible | 1-2 CPU, 2-4GB | Hours | -| **Production Pipelines** | Fixed | Varies by workload | Days | - -## 🔗 Integration with Platform Services - -### Storage Integration - -All interactive sessions automatically mount: - -- **ARC Home** (`/arc/home/[user]/`): Personal configurations and scripts -- **ARC Projects** (`/arc/projects/[project]/`): Shared research data and results -- **Scratch** (`/scratch/`): High-speed temporary storage for processing - -Additional storage accessible via API: -- **VOSpace** (`vos:`): Long-term archives and data sharing - -### Container Environments - -Sessions run in [container environments](../containers/index.md) that include: - -- Operating system (typically Ubuntu Linux) -- Astronomy software packages (AstroPy, CASA, etc.) -- Scientific computing libraries (NumPy, SciPy, Matplotlib) -- Development tools and utilities - -### CVMFS Software Repositories - -All CANFAR sessions provide access to read-only **CVMFS** (CernVM File System) software repositories. This feature provides instant access to the vast collections of pre-built scientific software maintained by the **Digital Research Alliance of Canada (Alliance)**. - -See the **[Software Repositories (CVMFS)](../cvmfs.md)** guide for more information and examples. - -### Authentication & Permissions - -Sessions inherit your [CANFAR permissions](../permissions.md): - -- Automatic access to your group projects -- Secure integration with CADC services -- API access for automated workflows - -## 🎯 Choosing Your Session Type - -### For Data Analysis - -**New to CANFAR?** → Start with **[Jupyter Notebooks](notebook.md)** -: Familiar interface combining code, documentation, and visualisation - -**Need GUI Applications?** → Use **[Desktop Sessions](desktop.md)** -: Full Linux desktop for CASA, image viewers, and traditional software - -### For Astronomy Specialisations - -**Radio Astronomy** → **[CARTA Viewer](carta.md)** -: Optimized for radio interferometry data visualisation and analysis - -**Survey Data** → **[Firefly Viewer](firefly.md)** -: Efficient table and image viewing for large astronomical catalogues - -**Specialised Tools** → **[Contributed Applications](contributed.md)** -: Community-maintained applications for specific research domains - -### For Production Work - -**Large-Scale Processing** → **[Batch Sessions](batch.md)** -: Automated workflows for processing large datasets without interactive interfaces - -### API Access to Sessions - -If you have a session running and it exposes an API, you can programatically -interact with it remotely, either from another session or from your laptop. - -The base of the session API will be available at the session URL. For more -details, consult the API docs for each of the session types: -- [CARTA API Docs](https://carta-python.readthedocs.io/en/latest/quickstart.html#opening-and-appending-images) -- [Firefly API Docs](https://caltech-ipac.github.io/firefly_client/) -- [JupyterLab API Docs](https://jupyterlab-server.readthedocs.io/en/latest/api/rest.html) - +# Sessions + +A Session is a user-owned compute environment launched from a Container Image +on a Science Platform Server. Choose the Session Kind that matches the way you +want to work: + +| Kind | Interface | Typical use | +| --- | --- | --- | +| `notebook` | JupyterLab | Python analysis and interactive notebooks | +| `desktop` | Browser desktop | CASA and other graphical applications | +| `carta` | CARTA | Image and spectral-cube exploration | +| `firefly` | Firefly | Tables and image visualization | +| `contributed` | Application-specific | Community-maintained tools | +| `headless` | No interactive interface | Batch commands and pipelines | + +See the individual guides for [Notebook](notebook.md), [Desktop](desktop.md), +[CARTA](carta.md), [Firefly](firefly.md), [Contributed](contributed.md), and +[Batch](batch.md) workflows. + +## Start a Session + +You can launch a Session from the [Science Portal](https://www.canfar.net/), +from the CLI, or with the Python client. The CLI uses a Container Image and +Session Kind as positional arguments: + +```bash +canfar login cadc +canfar image ls --kind notebook +canfar create notebook IMAGE_NAME --name analysis +canfar ps --all +``` + +Use `canfar server ls` and `canfar server use NAME` when more than one Science +Platform Server is available for the active Identity Provider. The image list +is the source of truth for image names and supported kinds; examples are +illustrative tags, not a guarantee that every deployment publishes them. + +## Lifecycle and status + +Creation returns Session IDs before the Session is necessarily ready. A Session +can be `Pending` while it waits for admission, resources, image pulls, or +initialization, then become `Running` or a terminal state. Queue order and +resource policy are deployment-owned. + +```bash +canfar ps --all +canfar info SESSION_ID +canfar events SESSION_ID +canfar logs SESSION_ID +``` + +The default `canfar ps` view shows `Pending` and `Running` Sessions; use +`--all` to include terminal states as well. `canfar open` only opens a ready +Session. Delete a Session when its work is complete: + +```bash +canfar delete SESSION_ID +``` + +## Storage boundary + +Sessions are temporary compute environments. A Session can use mounted +`/arc/home/` and `/arc/projects/` paths when the deployment +provides them, plus `/scratch` for fast Session-local work. `/scratch` is +deleted with the Session. Save scripts, inputs, and results under `/arc` or a +persistent VOSpace Service before stopping or deleting the Session. + +For remote VOSpace data, use [canfar data](../storage/transfers.md) or the +explicit Python [storage helper](../storage/filesystem.md). Do not assume that +opening a remote object provides server-side random access; stage once when a +path-oriented tool needs a local filename. + +## Resource requests + +Omit `--cpu` and `--memory` for the platform's flexible request, or set values +based on measured workload needs. Fixed requests can wait longer when matching +capacity is unavailable. For headless workloads, see the [batch queue and +troubleshooting guide](batch.md). + +## Session APIs + +The Python client exposes synchronous and asynchronous Session operations. The +public library returns ordinary Python values; it does not replace the mounted +filesystem or add a second storage adapter. Start with the [Python client +guide](../../client/get-started.md) and [Session reference](../../client/session.md). + +Some interactive applications expose their own API after the Session reaches +`Running`. Consult the application documentation linked from its individual +guide rather than assuming that every Session has an HTTP API. + +## Related guides + +- [Storage](../storage/index.md) +- [Containers](../containers/index.md) +- [Permissions](../permissions.md) +- [Support](../support/index.md) diff --git a/docs/platform/sessions/notebook.md b/docs/platform/sessions/notebook.md index 91edb306..9ca9dd56 100644 --- a/docs/platform/sessions/notebook.md +++ b/docs/platform/sessions/notebook.md @@ -1,300 +1,94 @@ # Notebook Sessions -**Interactive Jupyter Lab sessions for data analysis and computational astronomy** +A Notebook Session provides a browser-based Jupyter environment for interactive +analysis. It is a good place to explore data, test a reduction, and prepare a +command for a [headless Session](batch.md). -!!! abstract "🎯 What You'll Learn" - - How to launch and configure Jupyter notebook sessions - - Available containers and when to use each - - File management, uploads, and storage integration - - Performance tips, collaboration, and troubleshooting +## Launch a Notebook -Jupyter notebooks combine code execution, rich text documentation, and inline visualisations in a single interface. CANFAR's notebook sessions include pre-configured astronomy software stacks, persistent storage access, and collaborative sharing capabilities. - -## 📋 Overview - -Notebook sessions provide: - -- **Jupyter Lab:** Full-featured development environment with file browser, terminal, and extensions -- **Pre-configured containers:** Astronomy-specific software stacks with popular libraries -- **Persistent storage:** Direct access to your `/arc/home/` and `/arc/projects/` data -- **Terminal access:** Built-in terminal for command-line operations -- **File transfers:** Upload/download capabilities for data management - -## 🚀 Creating a Notebook Session - -### Step 1: Access Session Creation - -From the Science Portal dashboard, click the **plus sign (+)** to create a new session, then select **notebook** as your session type. - -### Step 2: Choose Your Container - -Select a container image that includes the software you need. Each container comes pre-configured with specific tools and libraries: - -#### Available Containers -There are quite a few containers available, some from the CANFAR team, and the community. Some examples: - -| Container | Contents | Best For | -|-----------|----------|----------| -| **astroml** ⭐ | Modern Python astronomy stack ([`astropy`](https://docs.astropy.org/), [`numpy`](https://numpy.org), [`scipy`](https://scipy.org), [`matplotlib`](https://matplotlib.org), [`pandas`](https://pandas.pydata.org)) | General astronomy analysis, data science | -| **casa-notebook** | [CASA](https://casa.nrao.edu/) + Python stack | Radio astronomy data reduction | - -!!! tip "Container Selection" - Start with **astroml** for most astronomy workflows. It includes the latest astronomy libraries and is actively maintained. Use CASA containers only when you specifically need CASA functionality. - -### Step 3: Configure Session Resources - -#### Session Name - -Choose a descriptive name that helps you identify this session later: - -**Good session names:** -- `galaxy-photometry` -- `pulsar-analysis` -- `alma-data-reduction` -- `exoplanet-search` - -#### Memory Allocation - -Select the maximum amount of RAM you anticipate requiring: - -**Memory Guidelines:** -- **4GB:** Basic analysis, small datasets -- **16GB:** Standard workflows, moderate datasets (recommended default) -- **32GB:** Large datasets, memory-intensive operations -- **64GB+:** Very large datasets, specialized workflows - -!!! warning "Resource Sharing" - Choose the smallest value reasonable for your needs. Computing resources are shared amongst all users. Excessive requests may slow or prevent session launch. - -#### CPU Cores - -Select the maximum number of computing cores you anticipate requiring: - -**CPU Guidelines:** -- **1-2 cores:** Most single-threaded analysis (recommended default) -- **4-8 cores:** Parallel processing, multi-threaded libraries -- **16+ cores:** Highly parallel workflows - -### Step 4: Launch Session - -Click the **Launch** button to start your notebook session. - -Wait until a notebook icon appears on your dashboard, then click it to access your session: - -## 🧭 Using Jupyter Lab - -### Interface Overview - -Once connected, you'll see the Jupyter Lab interface with several key areas: - -- **File Browser (left):** Navigate your filesystem and open files -- **Main Work Area (centre):** Notebooks, terminals, and file editors -- **Launcher:** Create new notebooks, terminals, and files -- **Menu Bar:** File operations, edit functions, and view options - -### Starting Your First Notebook - -1. **Click** the Python 3 (ipykernel) notebook icon in the launcher -2. **Start coding** in the first cell -3. **Run cells** with `Shift+Enter` - -### File Management - -#### Persistent Storage Locations - -Your notebook session has access to: +List the images available on the active server before choosing one: ```bash -/arc/home/[user]/ # Your personal 10GB space -/arc/projects/[project]/ # Shared project spaces -/scratch/ # Temporary high-speed storage +canfar login cadc +canfar image ls --kind notebook +canfar create notebook IMAGE_NAME --name analysis ``` -#### Uploading Files - -**Method 1: Direct Upload (< 100MB)** - -1. **Navigate** to your target directory in the file browser -2. **Click** the upload arrow in the top menu bar -3. **Select files** and click "Open" -4. **Files appear** in the browser - -**Method 2: Copy-Paste Text** +You can also create a Notebook from the Science Portal. Image names, package +versions, GPU availability, and resource limits are deployment-specific. Use +the image description and the current `canfar create --help` output as the +source of truth. -For code snippets or small text files: - -1. **Open a terminal** by double-clicking the terminal icon -2. **Create/edit files** using command-line editors -3. **Copy text** from your local computer -4. **Paste into the editor** - -### Working with CASA - -If using a CASA container, you can run CASA commands directly in notebook cells: - -```python -# Import CASA tasks -import casatasks as casa - -# Example: Import UV data -casa.importuvfits(fitsfile='data.uvfits', vis='data.ms') - -# List measurement set contents -casa.listobs(vis='data.ms') -``` - -## 🔧 Advanced Features - -### Terminal Access - -Access the built-in terminal for command-line operations: - -1. **Click** the terminal icon in the launcher -2. **Run commands** as you would in any Linux terminal -3. **Install packages** with pip or conda (where permissions allow) +Monitor the Session and open it when it is ready: ```bash -# Example terminal commands -ls /arc/projects/[project]/ -python script.py -git clone https://github.com/username/repo.git +canfar ps --all +canfar info SESSION_ID +canfar open SESSION_ID ``` -#### Accessing CVMFS Software - -All sessions have access to read-only scientific software repositories maintained by the **Digital Research Alliance of Canada (Alliance)** via CVMFS. These repositories provide thousands of pre-configured software packages and environment modules. - -See the **[Software Repositories (CVMFS)](../cvmfs.md)** guide for detailed instructions and examples of how to use this feature. - -### Jupyter Extensions - -Many useful extensions are pre-installed: +If it remains `Pending`, inspect `canfar events SESSION_ID` and the +[batch troubleshooting guide](batch.md#monitor-and-troubleshoot). A Session +that is not ready has no application logs yet; use `canfar logs` after it has +started. -- **Variable Inspector:** View variable contents -- **Table of Contents:** Navigate large notebooks -- **Git Integration:** Version control directly in Jupyter +## Work with files -### Python Package Management +Use the mounted paths supplied by your server: -#### Installing Additional Packages - -```bash -# Prefer python -m pip for clarity; installs to user site if needed -python -m pip install --user package-name - -# Check installed packages -python -m pip list | less +```text +/arc/home// personal persistent files +/arc/projects// project files, when available +/scratch/ temporary Session-local staging ``` -## 🤝 Collaboration - -Focus collaboration on shared project storage and version control, not session URL sharing. - -### Best Practices for Collaboration +Save notebooks, code, and results under `/arc` or a persistent VOSpace +Service. `/scratch` is useful for high-I/O intermediates and is deleted when +the Session ends. For remote data that is not mounted, use [CANFAR data +transfers](../storage/transfers.md) or the [Python filesystem +helpers](../storage/filesystem.md). -- **Use descriptive cell comments** for clarity -- **Save frequently** to persistent storage -- **Use version control** (git) for important work -- **Coordinate changes** via pull requests or issue tracking +The Jupyter file browser and terminal operate inside the Session. Large local +uploads are often more reliable when copied explicitly with `canfar data cp`. -### Sharing Notebooks +## Use a specialised image -```bash -# Save notebook to shared location -cp my-analysis.ipynb /arc/projects/[project]/notebooks/ +If a published image includes CASA or another astronomy package, follow that +image's documentation. Do not assume that every Notebook has the same Python +packages or that installing a package in one Session changes another. For +repeatable work, pin dependencies in a Container Image or versioned +environment. -# Share via git repository -git add my-analysis.ipynb -git commit -m "Add analysis notebook" -git push origin main -``` +Some deployments also expose shared software through [CVMFS](../cvmfs.md). +Treat it as read-only and record the module and version used by a workflow. -## ⚡ Performance Optimisation +## Move from exploration to automation -### Memory Management +Keep the notebook for inspection and use a script with explicit input/output +paths for repeatable reductions: ```python -import psutil -print(f"Memory usage: {psutil.virtual_memory().percent}%") +from pathlib import Path -# Free up memory by deleting large variables -if 'large_array' in globals(): - del large_array -import gc -gc.collect() +input_path = Path("/arc/projects//input.fits") +output_path = Path("/arc/projects//results/output.fits") +# Load input_path, run the reduction, and write output_path. ``` -### Storage Performance - -```python -# Use /scratch for intensive I/O operations -import shutil, pathlib - -source = '/arc/projects/[project]/large_file.fits' -target = '/scratch/large_file.fits' -shutil.copy(source, target) - -# ... your analysis code ... - -# Copy results back -pathlib.Path('/arc/projects/[project]/outputs/').mkdir(parents=True, exist_ok=True) -shutil.copy('/scratch/results.fits', '/arc/projects/[project]/outputs/results.fits') -``` - -### Efficient Data Loading - -```python -from astropy.io import fits - -# Efficient FITS access with context manager and memmap -with fits.open('huge_file.fits', memmap=True) as hdul: - header = hdul[0].header # Primary header - data_section = hdul[1].data # Access required extension lazily - -# If only header needed -from astropy.io.fits import getheader -primary_header = getheader('large_file.fits') -``` - -## 🔧 Troubleshooting - -### Common Issues - -#### Kernel Not Starting - -**Problem:** Python kernel fails to start - -**Solutions:** -1. Restart the kernel: `Kernel` → `Restart Kernel` -2. Clear output: `Cell` → `All Output` → `Clear` -3. Check memory usage and restart session if needed - -#### Out of Memory Errors - -**Problem:** `MemoryError` or kernel crashes - -**Solutions:** -1. Restart kernel and clear variables -2. Process data in smaller chunks -3. Use more memory-efficient data types -4. Launch session with more RAM - -#### Slow Performance - -**Problem:** Notebooks running slowly - -**Solutions:** -1. Check system resources with `htop` in terminal -2. Close unused notebooks and terminals -3. Clear notebook output: `Cell` → `All Output` → `Clear` -4. Use `/scratch` for temporary files +Then submit the same command as a headless Session, following the +[batch guide](batch.md). Record the image, resource request, Storage +Identifiers, and code revision with the run. -#### File Upload Issues +## Troubleshooting -**Problem:** Cannot upload files or uploads fail +- A kernel that will not start may indicate an image or resource problem; + inspect `info`, `events`, and the Session resource request. +- A missing file is usually a path, identifier, or group-permission issue; + confirm it with `canfar data info IDENTIFIER:/path`. +- A slow notebook may be reading a remote object repeatedly. Stage it once to + `/scratch` or use the opt-in cache guidance in [Filesystem and Python + tools](../storage/filesystem.md). +- A lost browser connection does not necessarily mean the Session stopped; + check `canfar ps --all` first. -**Solutions:** -1. Check file size (< 100MB for web upload) -2. Use command-line tools for larger files -3. Check available disk space -4. Try uploading to `/scratch` first, then moving +See [Support](../support/index.md) when the checks do not identify the cause. diff --git a/docs/platform/static/logo.png b/docs/platform/static/logo.png deleted file mode 100644 index 110e21d9..00000000 Binary files a/docs/platform/static/logo.png and /dev/null differ diff --git a/docs/platform/storage/filesystem.md b/docs/platform/storage/filesystem.md index d9ba5845..877876cb 100644 --- a/docs/platform/storage/filesystem.md +++ b/docs/platform/storage/filesystem.md @@ -1,749 +1,219 @@ -# Filesystem Access +# Filesystem and Python tools -**CANFAR's ARC (Cavern) storage systems as filesystems, SSHFS mounting from external computers, and permission management.** +Use a normal `/arc` path inside a Science Platform Session whenever the data is +already mounted. Use `canfar.storage` when Python is running outside the +Session, or when the object lives in a configured VOSpace Service. The module +keeps the CANFAR-owned work at one explicit boundary and returns the upstream +fsspec filesystem for everything else. The [Python client data +guide](../../client/data.md) is the canonical contract for Storage Identifier +lookup, credential selection, fsspec operations, local staging, and +`SimpleCacheFileSystem`. -!!! abstract "🎯 Filesystem Guide Overview" - **Master direct storage access:** - - - **Session Access**: How ARC storage appears within CANFAR computing sessions - - **SSHFS Mounting**: Accessing CANFAR storage from your local computer - - **Access Control Lists**: Fine-grained permissions for collaborative research - - **Performance Tips**: Optimising filesystem operations and troubleshooting +## Construct a filesystem explicitly -ARC storage (Home and Projects) can be accessed as standard Unix filesystems both within CANFAR sessions and from external computers via SSHFS. This provides familiar file operations and integrates seamlessly with existing tools and workflows. +See [Find and open a Storage Identifier](../../client/data.md#find-and-open-a-storage-identifier) +for the construction examples and the complete contract, including the +reserved `local` identifier, runtime credentials, saved Authentication Records, +error behavior, and the fact that identifiers are explicit arguments rather +than dynamic fsspec schemes. -## 🗂️ ARC Storage as Filesystems +## Read shape and backend capability -### Within CANFAR Sessions +The [client data guide](../../client/data.md#standard-fsspec-operations) defines +the supported read methods and staging behavior. This page adds the +deployment-specific capability guidance: -When you start any CANFAR session (Notebook, Desktop, or batch job), ARC storage is automatically mounted as standard directories: +- `vosfs` validates a ranged `206` response for explicit byte reads and falls + back to a complete response when the service does not provide ranges. A + successful call is therefore correct on both kinds of backend, but a fallback + still transfers the whole object. +- The read-only CADC measurements used for this guide saw validated ranges from + Vault/minoc and whole-object fallback from ARC/Cavern. -!!! abstract "🎯 Storage Naming" - **ARC or Cavern:** - - Both use the same VOSpace image. +The response capability belongs to the deployment, not the name `vault` or +`arc`. Treat another deployment or Storage Identifier as unknown until its +responses are observed. - - **ARC**: The CANFAR user storage system - - **Cavern**: The generic deployment for user storage +For data already mounted at `/arc`, use the path directly. For remote data used +once, a single explicit transfer is often clearer: ```bash -# Automatic mounts in every session -/arc/home/[user]/ # Your personal 10GB space -/arc/projects/[project]/ # Shared project spaces (if member) -/scratch/ # Temporary session storage +canfar data cp vault:/project/cube.fits local:/scratch/cube.fits ``` -### Directory Structure and Conventions +## Ephemeral and persistent caches -#### ARC Home Directory (`/arc/home/[user]/`) +The [client data guide's content-caching +contract](../../client/data.md#content-caching) explains the distinction between +the fsspec directory-listing cache and object-byte caching. Select one content +cache explicitly when repeated reads justify it. -Typically the home directory tree structure will be as follows: +### Session-local cache -```text -/arc/home/[user]/ -├── .ssh/ # SSH keys and config -│ ├── authorized_keys # Public keys for SSHFS access -│ └── config # SSH client configuration -├── .jupyter/ # Jupyter configuration -├── .bashrc # Shell configuration -├── .profile # Environment setup -├── bin/ # Personal scripts and tools -├── config/ # Application configurations -└── work/ # Personal analysis work -``` - -**Recommended Use:** -- Configuration files and dotfiles -- Personal code, scripts and utilities -- SSH keys for external access -- Small reference files and notes - -#### ARC Projects Directory (`/arc/projects/[project]/`) - -Used for team project use. For example, for a propcessing pipeline analysis: -```text -/arc/projects/[project]/ -├── data/ -│ ├── raw/ # Original datasets -│ ├── processed/ # Reduced/calibrated data -│ ├── catalogs/ # Reference catalogs -│ └── archives/ # Archived datasets -├── code/ -│ ├── pipelines/ # Data processing workflows -│ ├── analysis/ # Analysis scripts -│ ├── notebooks/ # Jupyter notebooks -│ └── tools/ # Project-specific utilities -├── results/ -│ ├── plots/ # Figures and visualisations -│ ├── tables/ # Output catalogues and measurements -│ ├── papers/ # Manuscripts and drafts -│ └── presentations/ # Conference materials -├── docs/ -│ ├── README.md # Project documentation -│ ├── data_notes.md # Dataset descriptions -│ └── procedures.md # Analysis procedures -└── scratch_archive/ # Backed up scratch work -``` - -## 🏠 Direct Filesystem Access (Within Sessions) - -### Basic Operations - -All standard Unix filesystem commands work directly: - -```bash -# Navigation -cd /arc/projects/[project]/ -pwd -ls -la - -# File operations -cp source.fits destination.fits -mv old_name.fits new_name.fits -rm unwanted_file.fits - -# Directory operations -mkdir -p data/2024/observations/ -rmdir empty_directory/ -find . -name "*.fits" -type f - -# Permissions -chmod 644 data_file.fits # Read/write owner, read others -chmod 755 analysis_script.py # Executable script -chgrp projectgroup shared_data/ # Change group ownership -``` - -### Creating a Project Allocation - -A project allocation under `/arc/projects/[project]` is **not** another folder created with `mkdir`. It is a VOSpace container node that carries a **quota** (in bytes) and an associated **team Group** for membership and access. Ordinary directory operations only work *inside* an allocation that already exists. - -!!! danger "🔐 Admin only — Allocations owner required" - Creating an allocation must be invoked **as the Allocations owner** (the admin identity used for allocation nodes; currently `storops`). **This procedure is not for general users.** End users cannot create project allocations this way — request one via [support@canfar.net](mailto:support@canfar.net) (see the [FAQ](../support/faq.md#how-much-storage-do-i-get-and-where-should-i-put-data)). Group membership for access is managed separately through [Group Management](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/groups/). - -The steps below adapt the low-level node create process for ARC project allocations. The same VOSpace node model applies to related services (for example vault); for ARC, use the arc nodes endpoint and authority. - -#### Create the allocation - -1. Create an XML file from this minimal template: - -```xml - - - - USERNAME - true - NUMBYTES - - - -``` - -`AUTHORITY` is the VOSpace authority for that site’s ARC (Cavern) service (it differs by deployment). For example, on CANFAR ARC it is typically `cadc.nrc.ca~arc`, so a project named `myproject` would use: - -```text -vos://cadc.nrc.ca~arc/projects/myproject -``` - -For SRCNet deployments, it would look like that site's identification. For `canSRC`, for example: - -```text -vos://canfar.net~staging-src~cavern -``` - -Always confirm the value from the site’s root node (step 5 below) rather than hard-coding an authority from another environment. - -2. Edit the file: - - 1. Put the project name in the `uri` path (replace `NAME`). The path must match the mounted tree: `/arc/projects/[project]` ↔ `projects/NAME`. - 2. Put the owner’s username in the `#creator` property (replace `USERNAME`). - 3. Put the desired quota in bytes in the `#quota` property (replace `NUMBYTES`). - 4. Leave `#inheritPermissions` as `true` (sane default). - 5. Set the authority part of the `uri` from the ARC root node for that site: - -```bash -curl https://example.org/arc/nodes?limit=0 -``` - -3. Create the container node authenticated as the Allocations owner. Certificate and Bearer token authentication are both accepted: - -```bash -curl --cert certificate.pem --header "content-type: text/xml" \ - --upload-file \ - https://example.org/arc/nodes/projects/NAME - -curl --header "Authorization: Bearer TOKEN" \ - --header "content-type: text/xml" \ - --upload-file \ - https://example.org/arc/nodes/projects/NAME -``` - -`NAME` in the URL must match the name in the XML `uri`, or the create request is rejected. On success, the service returns an XML representation of the created node (it may include additional default properties). - -#### Check allocation status - -```bash -curl https://example.org/arc/nodes/projects/NAME?limit=0 -``` - -`limit=0` means “do not list children.” Auth is optional for a publicly readable parent, but you can use Allocations-owner credentials (certificate or Bearer token) as above. - -#### Delete an empty allocation - -If you make a mistake and the node has no children: - -```bash -curl --cert certificate.pem -X DELETE \ - https://example.org/arc/nodes/projects/NAME - -curl --header "Authorization: Bearer TOKEN" -X DELETE \ - https://example.org/arc/nodes/projects/NAME -``` - -This fails if the container has any child nodes. - -#### Update quota on an existing allocation - -```bash -curl --cert certificate.pem --header "content-type: text/xml" \ - --data-binary @ \ - https://example.org/arc/nodes/projects/NAME - -curl --header "Authorization: Bearer TOKEN" \ - --header "content-type: text/xml" \ - --data-binary @ \ - https://example.org/arc/nodes/projects/NAME -``` - -The XML only needs the properties you intend to change (typically `#quota`). Include only that property so other node properties are not changed by accident. Changing the owner of a node via this path is not implemented. - -### Working with Large Datasets - -```bash -# Check available space -df -h /arc/projects/[project]/ -df -h /arc/home/[user]/ - -# Monitor space usage -du -sh /arc/projects/[project]/* -du -h --max-depth=2 /arc/projects/[project]/ - -# Efficient data movement -rsync -avP /scratch/processed_data/ /arc/projects/[project]/results/ - -# Archive old data -tar -czf old_observations_2023.tar.gz data/2023/ -mv old_observations_2023.tar.gz archives/ -``` - -### Linking and Shortcuts - -```bash -# Create symbolic links for easy access -ln -s /arc/projects/survey/data/master_catalogue.fits ~/current_catalogue.fits -ln -s /arc/projects/[project]/ ~/project - -# Hard links (same filesystem only) -ln /arc/projects/shared/reference.fits /arc/home/[user]/my_reference.fits - -# Quick navigation with variables -export PROJECT_DIR="/arc/projects/[project]" -cd $PROJECT_DIR/data -``` - -## 🌐 SSHFS: Remote Filesystem Access - -SSHFS allows you to mount CANFAR's ARC storage on your local computer as if it were a local directory, enabling seamless integration with local tools and workflows. - -### Prerequisites - -#### Local Computer Setup - -=== "macOS" - ```bash - # Install macFUSE and SSHFS - brew install --cask macfuse - brew install sshfs - - # Restart or logout/login after installation - ``` - -=== "Linux (Ubuntu/Debian)" - ```bash - # Install SSHFS - sudo apt update - sudo apt install sshfs - - # Add user to fuse group - sudo usermod -a -G fuse $USER - # Logout and login again - ``` - -=== "Linux (Fedora/RedHat)" - ```bash - # Install SSHFS - sudo dnf install sshfs - - # Add user to fuse group - sudo usermod -a -G fuse $USER - ``` - -#### CANFAR Side Setup - -You need to set up SSH key authentication on your CANFAR account: - -1. **Create SSH key pair** (on your local computer): - ```bash - ssh-keygen -t rsa -b 4096 -f ~/.ssh/canfar_key - # Enter passphrase when prompted (recommended) - ``` - -2. **Upload public key to CANFAR**: - - **Method 1: Via Web Interface** - - Navigate to [ARC File Manager](https://www.canfar.net/storage/arc/list/home) - - Go to your home directory - - Create `.ssh` folder if it doesn't exist - - Upload your `~/.ssh/canfar_key.pub` as `authorized_keys` (if it already exists, you will have to append to the end of the file) - - **Method 2: Via existing session** - ```bash - # In a CANFAR session, copy your public key content to: - mkdir -p /arc/home/[user]/.ssh - # Paste your public key content into authorized_keys file - nano /arc/home/[user]/.ssh/authorized_keys - chmod 700 /arc/home/[user]/.ssh - chmod 600 /arc/home/[user]/.ssh/authorized_keys - ``` - -### Mounting ARC Storage - -#### Basic Mount - -```bash -# Create local mount point -mkdir ~/canfar_arc - -# Mount ARC storage -sshfs -p 64022 -i ~/.ssh/canfar_key \ - -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=10 \ - [user]@ws-uv.canfar.net:/ ~/canfar_arc/ - -# On macOS, you may have to add defer_permissions option: -sshfs -p 64022 -i ~/.ssh/canfar_key \ - -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=10,defer_permissions \ - [user]@ws-uv.canfar.net:/ ~/canfar_arc/ -``` - -#### Advanced Mount Options - -```bash -# Mount with optimizations for large files -sshfs -p 64022 -i ~/.ssh/canfar_key \ - -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=10 \ - -o cache=yes,kernel_cache,compression=yes \ - -o Ciphers=aes128-ctr \ - [user]@ws-uv.canfar.net:/ ~/canfar_arc/ - -# Mount specific project only -sshfs -p 64022 -i ~/.ssh/canfar_key \ - -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=10 \ - [user]@ws-uv.canfar.net:/arc/projects/[project] ~/project_mount/ -``` - -#### Connection Configuration - -Create `~/.ssh/config` for easier connections: - -```text -Host canfar - HostName ws-uv.canfar.net - Port 64022 - User [user] - IdentityFile ~/.ssh/canfar_key - ServerAliveInterval 15 - ServerAliveCountMax 10 - Compression yes -``` - -Then mount with simpler command: -```bash -sshfs canfar:/ ~/canfar_arc/ -``` - -### Using Mounted Storage - -Once mounted, use CANFAR storage like any local directory: - -```bash -# Navigate to your project -cd ~/canfar_arc/arc/projects/[project]/ - -# Copy files from local to CANFAR -cp ~/local_analysis.py ~/canfar_arc/arc/projects/[project]/code/ - -# Edit files with local editor -code ~/canfar_arc/arc/home/[user]/.bashrc - -# Run local tools on CANFAR data -python analyze_data.py ~/canfar_arc/arc/projects/[project]/data/observations.fits - -# Sync directories -rsync -avz ~/local_scripts/ ~/canfar_arc/arc/projects/[project]/code/ -``` +See the [client data guide's SimpleCache example](../../client/data.md#content-caching) +for the supported whole-file composition. `/scratch` is a useful default inside +a Science Platform Session because it is local and is deleted with the Session. +Key the directory by Storage Identifier so objects from different endpoints +cannot collide. This cache is opt-in, does not make a staged file into a ranged +reader, and does not survive Session deletion; remove it when its data is no +longer needed or when `/scratch` is under pressure. -### Unmounting +### Persistent cache -```bash -# Unmount when finished -umount ~/canfar_arc -# or on macOS: -diskutil unmount ~/canfar_arc - -# Force unmount if needed -umount -f ~/canfar_arc -# or -fusermount -u ~/canfar_arc -``` - -## 🔐 Access Control and Permissions - -### Understanding ARC Permissions - -ARC storage uses traditional Unix permissions combined with group-based access control: - -#### Permission Types - -```bash -# View detailed permissions -ls -l /arc/projects/[project]/ - -# Example output: -# drwxrwxr-- projectgroup data/ -# -rw-rw-r-- projectgroup analysis.py -# -rwx------ username private_script.py - -# Permission breakdown: -# d = directory, - = file -# rwx = owner permissions (read/write/execute) -# rwx = group permissions -# r-- = other permissions -``` - -#### User and Group Information - -```bash -# Check your user ID and groups -id -whoami -groups - -# Check file ownership -stat /arc/projects/[project]/somefile.fits - -# View group membership -getent group [project] -``` - -### Managing Permissions - -#### Setting File Permissions - -```bash -# Make file readable by group -chmod g+r data_file.fits - -# Make script executable -chmod +x analysis_script.py - -# Set specific permission modes -chmod 664 shared_data.fits # rw-rw-r-- -chmod 755 public_script.py # rwxr-xr-x -chmod 600 private_config.txt # rw------- - -# Recursive permission changes -chmod -R g+rw shared_directory/ -``` - -#### Group Management - -Group membership is managed through CANFAR's Group Management system: - -1. **Navigate to**: [Group Management](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/groups/) -2. **Create or modify groups**: Add/remove users from project groups -3. **Apply permissions**: Use `chgrp` to assign files to groups - -```bash -# Change group ownership -chgrp projectgroup /arc/projects/[project]/shared_data.fits - -# Change recursively -chgrp -R projectgroup /arc/projects/[project]/shared_results/ - -# Set default group for new files in directory -chmod g+s /arc/projects/[project]/shared_directory/ -``` - -### Access Control Lists (ACLs) - -For fine-grained permissions beyond standard Unix permissions: - -```bash -# View current ACLs -getfacl /arc/projects/[project]/sensitive_data.fits - -# Set ACL for specific user -setfacl -m u:collaborator:r /arc/projects/[project]/data.fits - -# Set ACL for group -setfacl -m g:external_collaborators:r /arc/projects/[project]/ - -# Remove ACL -setfacl -x u:former_collaborator /arc/projects/[project]/data.fits - -# Set default ACLs for directory -setfacl -d -m g:projectgroup:rw /arc/projects/[project]/shared/ -``` - -## 🔧 Optimization and Best Practices - -### Performance Optimization - -#### Local Filesystem Operations - -```bash -# Use rsync for efficient synchronization -rsync -avz --progress ~/local_data/ /arc/projects/[project]/backup/ - -# Monitor I/O performance -iostat -x 1 # Live I/O statistics -iotop # Process I/O usage - -# Optimize for large files -# Use /scratch/ for intensive processing -cp /arc/projects/[project]/large_dataset.fits /scratch/ -process_data /scratch/large_dataset.fits -cp /scratch/results.fits /arc/projects/[project]/outputs/ -``` - -#### SSHFS Performance Tips - -```bash -# Optimize SSHFS for different use cases - -# For frequent small file access: -sshfs -o cache=yes,kernel_cache,attr_timeout=3600,entry_timeout=3600 \ - canfar:/ ~/canfar_arc/ - -# For large file transfers: -sshfs -o cache=no,compression=yes,Ciphers=aes128-ctr \ - canfar:/ ~/canfar_arc/ - -# For read-only access (faster): -sshfs -o ro,cache=yes,kernel_cache \ - canfar:/ ~/canfar_arc/ -``` - -### Workflow Integration - -#### Local Development with CANFAR Data +Choose a persistent cache directory only when retaining a local copy across +processes or Sessions is intentional. `WholeFileCacheFileSystem` can check +remote metadata and expire entries; the application owns capacity, freshness, +permissions, and cleanup: -```bash -# Create development environment -mkdir ~/canfar_project/ -cd ~/canfar_project/ - -# Mount CANFAR storage as subdirectory -mkdir canfar_data -sshfs canfar:/arc/projects/[project] canfar_data/ - -# Create local working directory -mkdir local_work -cd local_work +```python +from fsspec.implementations.cached import WholeFileCacheFileSystem -# Symlink to CANFAR data for easy access -ln -s ../canfar_data/data ./data -ln -s ../canfar_data/code ./shared_code +from canfar.storage import filesystem -# Work locally with CANFAR data -python shared_code/analysis.py data/observations.fits +remote = filesystem("vault") +try: + cached = WholeFileCacheFileSystem( + fs=remote, + cache_storage="/arc/home//.cache/canfar/vault", + expiry_time=24 * 60 * 60, + check_files=True, + ) + with cached.open("/project/catalog.csv", "rb") as handle: + process(handle) +finally: + remote.close() ``` -#### Automated Backup Scripts +Do not silently choose `/scratch`, stack cache layers, use a `memory://` cache +location, or recommend `blockcache` for `vosfs` staged file objects. These +choices hide lifetime or capability decisions and can turn every block into a +whole-object transfer. -```bash -#!/bin/bash -# backup_to_canfar.sh - Automated backup script +## Scientific Python recipes -LOCAL_DIR="$HOME/important_work" -CANFAR_MOUNT="$HOME/canfar_arc" -BACKUP_DIR="$CANFAR_MOUNT/arc/home/[user]/backups" +The recipes below use standard library APIs and the filesystem returned by +`canfar.storage`. None of them adds a CANFAR-specific adapter. -# Check if CANFAR is mounted -if ! mountpoint -q "$CANFAR_MOUNT"; then - echo "Mounting CANFAR storage..." - sshfs canfar:/ "$CANFAR_MOUNT" -fi +### pandas -# Create backup with timestamp -DATE=$(date +%Y%m%d_%H%M%S) -BACKUP_PATH="$BACKUP_DIR/backup_$DATE" +Pandas accepts a file-like object. A remote `open` stages the complete object; +wrap the filesystem in an explicit `SimpleCacheFileSystem` for repeated reads, +or stage the file once to `/scratch`: -echo "Creating backup: $BACKUP_PATH" -rsync -avz --progress "$LOCAL_DIR/" "$BACKUP_PATH/" +```python +import pandas as pd -# Keep only last 5 backups -cd "$BACKUP_DIR" -ls -t | tail -n +6 | xargs rm -rf +from canfar.storage import filesystem -echo "Backup completed successfully" +remote = filesystem("vault") +try: + with remote.open("/project/catalog.csv", "rb") as handle: + frame = pd.read_csv(handle) +finally: + remote.close() ``` +### NumPy +`numpy.load` accepts a seekable binary handle. Memory mapping requires a real +local filename, so transfer the object to `/scratch` first when using +`mmap_mode`: -## 🛠️ Troubleshooting +```python +import numpy as np -### Common Issues and Solutions +from canfar.storage import filesystem -#### SSHFS Connection Problems +remote = filesystem("vault") +try: + with remote.open("/project/array.npy", "rb") as handle: + array = np.load(handle, allow_pickle=False) + remote.get_file("/project/array.npy", "/scratch/array.npy") +finally: + remote.close() -```bash -# Debug SSHFS connection -sshfs -d -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=10,defer_permissions -p 64022 [user]@ws-uv.canfar.net:/ $HOME/canfar_arc -# Check mount status -mount | grep sshfs -df -h | grep sshfs +mapped = np.load("/scratch/array.npy", mmap_mode="r", allow_pickle=False) ``` -#### Permission Denied Errors - -```bash -# Check your group membership -groups -id - -# Verify file permissions -ls -la /arc/projects/[project]/problematic_file - -# Check directory execute permissions -ls -ld /arc/projects/[project]/ - -``` +### Astropy FITS -#### Performance Issues +Astropy can consume the staged file-like object directly. For repeated access, +or for a workflow that needs a local path, stage once and use `memmap=True`: -```bash -# Check filesystem I/O -iostat -x 1 +```python +from astropy.io import fits -# Monitor network usage (for SSHFS) -netstat -i -iftop +from canfar.storage import filesystem -# Test SSHFS performance -time ls -la ~/canfar_arc/projects/[project]/ +remote = filesystem("vault") +try: + remote.get_file("/project/cube.fits", "/scratch/cube.fits") +finally: + remote.close() -# Remount with performance options -umount ~/canfar_arc -sshfs -o cache=yes,compression=yes canfar:/ ~/canfar_arc/ +with fits.open("/scratch/cube.fits", memmap=True) as hdul: + image = hdul[0].data ``` -#### Storage Space Issues +If a service is known to return a valid range response, an explicit +`cat_file(path, start, end)` can retrieve a small byte slice. Do not infer that +FITS header access through `open()` is ranged; the staged file path downloads +the complete object. -```bash -# Check quota usage -df -h /arc/home/[user]/ -df -h /arc/projects/[project]/ +### Dask -# Find large files -find /arc/projects/[project]]/ -type f -size +100M -exec ls -lh {} \; +For a batch workload in a Science Platform Session, stage input files to +`/scratch` and let Dask read local paths. This avoids asking workers to split a +single staged remote object by byte offset: -# Clean up space -du -sh /arc/projects/[project]/* | sort -hr -# Remove or archive large unnecessary files -``` - -### Diagnostic Commands +```python +import dask.dataframe as dd -```bash -# System information -uname -a -mount | grep arc -df -h - -# Network connectivity -ping ws-uv.canfar.net -telnet ws-uv.canfar.net 64022 - -# SSH key verification -ssh-keygen -lf ~/.ssh/canfar_key.pub -ssh-add -l - -# SSHFS troubleshooting -fusermount -V -sshfs --version - -# Permission debugging -getfacl /arc/projects/[project]/ -namei -l /arc/projects/[project]/path/to/file +frame = dd.read_csv("/scratch/catalog/part-*.csv") +result = frame.groupby("source_id").flux.mean().compute() ``` -## 🔗 Integration Examples - -### IDE and Editor Integration - -#### VS Code with Remote Filesystem +When workers read remote data directly, construct the filesystem in the worker +environment and ensure every worker has CANFAR/vosfs and usable credentials. +Do not rely on a notebook-only protocol registration or inherit an open +filesystem across a process fork. Dask URL protocols and `storage_options` are +worker-side concerns; explicit construction with `filesystem(identifier)` keeps +the CANFAR endpoint and authentication lookup visible. -```json -// .vscode/settings.json -{ - "python.defaultInterpreterPath": "/usr/bin/python", - "files.watcherExclude": { - "**/canfar_arc/**": true - }, - "search.exclude": { - "**/canfar_arc/**": true - } -} -``` +### Zarr -#### Jupyter Lab with SSHFS +Zarr 3 stores chunks as separate objects. Build its documented fsspec mapper +from the explicit filesystem and keep the chunks reasonably large for the +service's metadata and transfer costs: ```python -# In Jupyter Lab, access CANFAR data via mounted filesystem on your laptop -import pandas as pd -from astropy.io import fits +import dask.array as da +from zarr.storage import FsspecStore -# Read data from mounted CANFAR storage -data_path = "$HOME/canfar_arc/arc/projects/[project]/data/" -catalog = pd.read_csv(f"{data_path}/catalog.csv") +from canfar.storage import filesystem -# Process and save results back to CANFAR -results = process_data(catalog) -results.to_csv(f"{data_path}/processed_catalog.csv") +remote = filesystem("vault") +try: + store = FsspecStore.from_mapper(remote.get_mapper("/project/cube.zarr")) + cube = da.from_zarr(store, component="science") + mean = cube.mean().compute() +finally: + remote.close() ``` -### Automated Workflows +The Zarr, Dask, and fsspec packages must be available in the running +environment. A Zarr chunk is already an object-level read; do not wrap the +whole store in a block cache and assume it creates server-side ranges. -#### Git Repository Sync +### Other path-oriented tools -```bash -#!/bin/bash -# sync_code_to_canfar.sh - -LOCAL_REPO="$HOME/my_analysis_code" -CANFAR_CODE="$HOME/canfar_arc/arc/projects/[project]/code" - -cd "$LOCAL_REPO" +For h5py, CASA, or a C/C++ application that opens filenames itself, stage the +object to `/scratch` and pass the local path. This is clearer and safer than +assuming that a path-only consumer can use a VOSpace file-like object. -# Push local changes to git -git add . -git commit -m "Update analysis code" -git push origin main +## Related guides -# Sync to CANFAR -rsync -avz --exclude='.git' . "$CANFAR_CODE/" - -echo "Code synchronized to CANFAR" -``` +- [Storage overview](index.md) +- [Data transfers](transfers.md) +- [VOSpace](vospace.md) +- [Python client data access](../../client/data.md) diff --git a/docs/platform/storage/images/groupmanagement/10_edit_permissions2.png b/docs/platform/storage/images/groupmanagement/10_edit_permissions2.png deleted file mode 100644 index 7f2ecfa1..00000000 Binary files a/docs/platform/storage/images/groupmanagement/10_edit_permissions2.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/11_edit_permissions3.png b/docs/platform/storage/images/groupmanagement/11_edit_permissions3.png deleted file mode 100644 index 15941833..00000000 Binary files a/docs/platform/storage/images/groupmanagement/11_edit_permissions3.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/12_edit_permissions4.png b/docs/platform/storage/images/groupmanagement/12_edit_permissions4.png deleted file mode 100644 index e83e6f34..00000000 Binary files a/docs/platform/storage/images/groupmanagement/12_edit_permissions4.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/13_permissions_updated.png b/docs/platform/storage/images/groupmanagement/13_permissions_updated.png deleted file mode 100644 index 22c8c3e0..00000000 Binary files a/docs/platform/storage/images/groupmanagement/13_permissions_updated.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/1_canfar_landing.png b/docs/platform/storage/images/groupmanagement/1_canfar_landing.png deleted file mode 100644 index c5e6dd1d..00000000 Binary files a/docs/platform/storage/images/groupmanagement/1_canfar_landing.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/2_group_management_landing.png b/docs/platform/storage/images/groupmanagement/2_group_management_landing.png deleted file mode 100644 index 1124e671..00000000 Binary files a/docs/platform/storage/images/groupmanagement/2_group_management_landing.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/3_create_group.png b/docs/platform/storage/images/groupmanagement/3_create_group.png deleted file mode 100644 index 5d608a6b..00000000 Binary files a/docs/platform/storage/images/groupmanagement/3_create_group.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/4_group_landing_add.png b/docs/platform/storage/images/groupmanagement/4_group_landing_add.png deleted file mode 100644 index 10444fa5..00000000 Binary files a/docs/platform/storage/images/groupmanagement/4_group_landing_add.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/5_add_members.png b/docs/platform/storage/images/groupmanagement/5_add_members.png deleted file mode 100644 index 02b2dc6a..00000000 Binary files a/docs/platform/storage/images/groupmanagement/5_add_members.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/6_updated_members.png b/docs/platform/storage/images/groupmanagement/6_updated_members.png deleted file mode 100644 index d3fb4dca..00000000 Binary files a/docs/platform/storage/images/groupmanagement/6_updated_members.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/7_add_admin.png b/docs/platform/storage/images/groupmanagement/7_add_admin.png deleted file mode 100644 index 9afc3c9d..00000000 Binary files a/docs/platform/storage/images/groupmanagement/7_add_admin.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/8_browse_projects.png b/docs/platform/storage/images/groupmanagement/8_browse_projects.png deleted file mode 100644 index 0a37fd6f..00000000 Binary files a/docs/platform/storage/images/groupmanagement/8_browse_projects.png and /dev/null differ diff --git a/docs/platform/storage/images/groupmanagement/9_edit_permissions1.png b/docs/platform/storage/images/groupmanagement/9_edit_permissions1.png deleted file mode 100644 index 704a106b..00000000 Binary files a/docs/platform/storage/images/groupmanagement/9_edit_permissions1.png and /dev/null differ diff --git a/docs/platform/storage/images/sshfs/add-file.png b/docs/platform/storage/images/sshfs/add-file.png deleted file mode 100644 index f13dbca4..00000000 Binary files a/docs/platform/storage/images/sshfs/add-file.png and /dev/null differ diff --git a/docs/platform/storage/images/sshfs/add-folder.png b/docs/platform/storage/images/sshfs/add-folder.png deleted file mode 100644 index a79d9ad7..00000000 Binary files a/docs/platform/storage/images/sshfs/add-folder.png and /dev/null differ diff --git a/docs/platform/storage/images/sshfs/auth-keys.png b/docs/platform/storage/images/sshfs/auth-keys.png deleted file mode 100644 index 4d2edfb2..00000000 Binary files a/docs/platform/storage/images/sshfs/auth-keys.png and /dev/null differ diff --git a/docs/platform/storage/images/sshfs/file-added.png b/docs/platform/storage/images/sshfs/file-added.png deleted file mode 100644 index 35ccb0b9..00000000 Binary files a/docs/platform/storage/images/sshfs/file-added.png and /dev/null differ diff --git a/docs/platform/storage/images/sshfs/folder-created.png b/docs/platform/storage/images/sshfs/folder-created.png deleted file mode 100644 index f10e7bbf..00000000 Binary files a/docs/platform/storage/images/sshfs/folder-created.png and /dev/null differ diff --git a/docs/platform/storage/images/sshfs/home.png b/docs/platform/storage/images/sshfs/home.png deleted file mode 100644 index 8792f2e2..00000000 Binary files a/docs/platform/storage/images/sshfs/home.png and /dev/null differ diff --git a/docs/platform/storage/images/sshfs/login.png b/docs/platform/storage/images/sshfs/login.png deleted file mode 100644 index edd9b221..00000000 Binary files a/docs/platform/storage/images/sshfs/login.png and /dev/null differ diff --git a/docs/platform/storage/images/sshfs/new-folder-name.png b/docs/platform/storage/images/sshfs/new-folder-name.png deleted file mode 100644 index a6e78681..00000000 Binary files a/docs/platform/storage/images/sshfs/new-folder-name.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/10_html_list.png b/docs/platform/storage/images/webstorage/10_html_list.png deleted file mode 100644 index 04767c5a..00000000 Binary files a/docs/platform/storage/images/webstorage/10_html_list.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/11_zip_download_popup.png b/docs/platform/storage/images/webstorage/11_zip_download_popup.png deleted file mode 100644 index 39dda6bc..00000000 Binary files a/docs/platform/storage/images/webstorage/11_zip_download_popup.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/12_open_zip.png b/docs/platform/storage/images/webstorage/12_open_zip.png deleted file mode 100644 index bfe2eede..00000000 Binary files a/docs/platform/storage/images/webstorage/12_open_zip.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/1_click_add.png b/docs/platform/storage/images/webstorage/1_click_add.png deleted file mode 100644 index 632f31d2..00000000 Binary files a/docs/platform/storage/images/webstorage/1_click_add.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/2_upload_popup.png b/docs/platform/storage/images/webstorage/2_upload_popup.png deleted file mode 100644 index 2ce1f663..00000000 Binary files a/docs/platform/storage/images/webstorage/2_upload_popup.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/3_choose_file.png b/docs/platform/storage/images/webstorage/3_choose_file.png deleted file mode 100644 index 81aaf49f..00000000 Binary files a/docs/platform/storage/images/webstorage/3_choose_file.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/4_click_upload.png b/docs/platform/storage/images/webstorage/4_click_upload.png deleted file mode 100644 index f20923c3..00000000 Binary files a/docs/platform/storage/images/webstorage/4_click_upload.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/5_click_ok.png b/docs/platform/storage/images/webstorage/5_click_ok.png deleted file mode 100644 index 4ed79df9..00000000 Binary files a/docs/platform/storage/images/webstorage/5_click_ok.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/6_file_uploaded.png b/docs/platform/storage/images/webstorage/6_file_uploaded.png deleted file mode 100644 index cd2e1a74..00000000 Binary files a/docs/platform/storage/images/webstorage/6_file_uploaded.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/7_start_download.png b/docs/platform/storage/images/webstorage/7_start_download.png deleted file mode 100644 index 4678ff5c..00000000 Binary files a/docs/platform/storage/images/webstorage/7_start_download.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/8_url_download_popup.png b/docs/platform/storage/images/webstorage/8_url_download_popup.png deleted file mode 100644 index 4a66c7c7..00000000 Binary files a/docs/platform/storage/images/webstorage/8_url_download_popup.png and /dev/null differ diff --git a/docs/platform/storage/images/webstorage/9_show_url_download.png b/docs/platform/storage/images/webstorage/9_show_url_download.png deleted file mode 100644 index ed67f84b..00000000 Binary files a/docs/platform/storage/images/webstorage/9_show_url_download.png and /dev/null differ diff --git a/docs/platform/storage/index.md b/docs/platform/storage/index.md index 1ab778bf..2a4ab28c 100644 --- a/docs/platform/storage/index.md +++ b/docs/platform/storage/index.md @@ -1,112 +1,96 @@ -# CANFAR Storage Systems +# Storage -**A guide to choosing the right storage, understanding how sessions interact with storage, and optimizing your data workflows on the CANFAR platform.** +Choose storage by lifetime and by where your code runs. A Science Platform +Session has mounted POSIX storage for working with files and a separate local +scratch volume for temporary work. VOSpace Services provide authenticated +remote access when data is not already mounted. -CANFAR provides four distinct storage systems, each optimized for different stages of the research lifecycle. Understanding how these systems work together with CANFAR sessions is essential for efficient data management and analysis. +## Storage at a glance -!!! abstract "Storage Guides" - - **[Filesystem Access](filesystem.md)**: ARC storage, SSHFS mounting, and permissions. - - **[Data Transfers](transfers.md)**: Moving data between systems and external sources. - - **[VOSpace Guide](vospace.md)**: Long-term storage, sharing, and archival. +| Location | Lifetime | Use it for | +| --- | --- | --- | +| `/arc/home/` | Persistent | Personal scripts, configuration, and results | +| `/arc/projects/` | Persistent | Project data and shared results | +| `/scratch` | Session-local; deleted when the Session ends | Staging, intermediate files, and explicitly selected caches | +| A configured VOSpace Service | Service-defined | Remote data and transfers through `canfar data` or fsspec | +| `local` | The machine running the command | Local input and output in `canfar data` and the Python helper | -## Storage Options Overview +The exact mounts, quotas, and retention policy belong to the deployment and +your project. Do not treat `/scratch` as a backup. Copy anything you need after +the Session to `/arc` or another persistent destination. -| Storage | Path/URI | Access (Session/External) | Speed | Persistence & Backup | Default Quota | Best For | -|-----------------|---------------------------|--------------------------------|---------------------|---------------------------|-------------------------------|------------------------------------------------| -| **Scratch** | `/scratch` | Direct FS / N/A | Fastest (local SSD) | Ephemeral (no backup) | ~200GB (per session) | High-speed temporary processing, staging I/O. | -| **ARC Home** | `/arc/home/[user]` | Direct FS / SSHFS | Fast (CephFS) | Permanent (daily snapshots) | 10GB | Personal configs, scripts, small files. | -| **ARC Projects**| `/arc/projects/[project]` | Direct FS / SSHFS | Fast (CephFS) | Permanent (daily snapshots) | 200GB | Active collaborative research data and results.| -| **Vault** | `vos:[project|user]` | API / Web UI | Medium | Permanent (geo-redundant) | Project-dependent | Long-term archives, sharing, publication. | +## Use the mounted filesystem first -### Checking Quotas and Requesting More Space +If data is already under `/arc` in a Science Platform Session, use its normal +POSIX path. This avoids an unnecessary VOSpace request and is the simplest +path for CASA, FITS tools, NumPy, pandas, and other software that expects a +filename. -You can monitor your storage usage with the following commands: +For data outside the Session, stage one copy into `/scratch` when the workflow +will read it more than once, then write final products to `/arc`: -```bash -# Check ARC storage usage -df -h /arc/home/[user]/ -df -h /arc/projects/[project]/ - -# For a detailed breakdown of a project directory -du -sh /arc/projects/[project]/* +```text +remote VOSpace Service -> /scratch/input.fits -> analysis -> /arc/projects//results/ ``` -Vault usage can be monitored via the [web interface](https://www.canfar.net/storage/vault/list/). - -To request a quota increase, email `support@canfar.net` with the project name, current usage, requested space, and a brief justification. +## Address a VOSpace Service explicitly -## Storage in a Session +CANFAR calls the configured handle for a VOSpace Service a **Storage +Identifier**. The identifier is configuration data, not a Python module member +or a new fsspec protocol. List identifiers and construct a filesystem explicitly: -When you start a CANFAR session (like a Notebook or Desktop), the storage systems are integrated seamlessly. +```python +from canfar.storage import filesystem, identifiers -- **ARC Home and Projects** are automatically mounted as standard directories. You can interact with them just like any other folder on a Linux system. -- **Scratch space** is provided as a temporary, high-speed directory at `/scratch`. - -This setup allows for a simple and powerful workflow: - -```mermaid -graph TD - Start([Session Starts]) --> Mounts["/arc/home & /arc/projects mounted"] - Mounts --> Scratch["Empty /scratch created"] - Scratch --> DataWork["Analyze data, using /scratch for temporary files"] - DataWork --> Save["Save results to /arc/projects"] - Save --> End([Session Ends]) - End --> Cleanup["/scratch is wiped clean"] +print(identifiers()) # configured identifiers, plus the reserved "local" +remote = filesystem("vault") +try: + entries = remote.ls("/project", detail=False) +finally: + remote.close() ``` -!!! warning "Scratch is Temporary" - Any data left in `/scratch` is **permanently deleted** when your session ends. Always copy important files to `/arc` or `vos:` before stopping a session. - -## Storage Strategy and Performance +`filesystem("local")` returns a filesystem for the machine where Python is +running. It does not require a CANFAR Authentication Record. A configured +identifier resolves its endpoint and parent Identity Provider through the saved +CANFAR configuration; runtime credentials can be supplied to `filesystem()` +when needed. See [Filesystem and Python tools](filesystem.md). -Choosing the right storage for each task is key to an efficient workflow. The general principle is to **move data to the fastest storage for processing**. +For shell workflows, use the embedded `canfar data` command application: -**Storage Speed Hierarchy:** -1. Fastest: `/scratch` (local SSD) -2. Medium: `/arc/projects` & `/arc/home` (Shared network filesystem) -3. Slower: `vos:` (Vault) (optimized for archival) - -### Common Workflows - -#### Interactive Analysis -- **Your data source:** `/arc/projects/[project]` -- **For large files:** Copy them to `/scratch` before processing. -- **Save results to:** `/arc/projects/[project]/results` - -*Example:* ```bash -# 1. Copy data to fast, temporary storage -cp /arc/projects/my_project/large_dataset.fits /scratch/ - -# 2. Process the data in /scratch -run_analysis.py /scratch/large_dataset.fits - -# 3. Save the results back to permanent project storage -mv /scratch/results.csv /arc/projects/my_project/ +canfar data ls -lh vault:/project +canfar data cp vault:/project/input.fits local:/scratch/input.fits +canfar data cp local:/scratch/result.fits arc:/projects//result.fits ``` -#### Batch Processing -- **Input:** Stage data from Vault (`vos:`), ARC, or the internet into `/scratch`. -- **Processing:** Run your code on the data in `/scratch`. -- **Output:** Save results to ARC for collaboration or to Vault for long-term archival. +The `local:` operand is always the machine running `canfar`. `vault:` and +`arc:` are examples of Storage Identifiers; use the names returned by your +configuration rather than assuming that every deployment has those identifiers. +See [Data transfers](transfers.md) for command details. -#### Data Sharing and Collaboration -- **Active collaboration:** Use `/arc/projects` for shared data and code among team members. -- **External sharing:** Use Vault (`vos:`) to share data with collaborators outside of CANFAR, or for public data releases. +## Remote-read performance -## Troubleshooting Common Issues +`vosfs` and fsspec provide two different read shapes: -**"No space left on device"** -- This usually means your `/arc/home` or `/arc/projects` quota is full. -- Use `du -sh /path/to/storage/*` to find large files and clean up anything you don't need. +- `cat_file(path, start, end)` and `cat_ranges(...)` can request explicit byte + ranges. The response is validated; a backend that returns a complete `200` + response is read and sliced correctly, but it still transferred the whole + object. +- `open(path, "rb")` provides a convenient seekable file object by staging the + complete object. Passing that handle to Astropy, NumPy, pandas, or h5py does + not make random access network-efficient. -**"Can't access project directory"** -- You may not be a member of the project's group. Contact the project PI to be added. +The capability is deployment-specific. In the measured CADC deployment, the +Vault/minoc service accepts validated `206` responses while ARC/Cavern falls +back to a whole-object response. Do not infer capability from the spelling of a +Storage Identifier. See [Filesystem and Python tools](filesystem.md) for the +backend boundary and scientific-library recipes. -**"Session is slow or unresponsive"** -- If you are performing I/O-intensive operations directly in `/arc`, it can slow down your session. -- For better performance, move large files to `/scratch` for processing. +## Related guides -**"My files are gone!"** -- You likely saved them to `/scratch` and the session ended. This data is not recoverable. -- Always save important results to `/arc` or `vos:` before your session ends. +- [Filesystem and Python tools](filesystem.md) +- [Data transfers](transfers.md) +- [VOSpace](vospace.md) +- [Session storage](../sessions/index.md) +- [Permissions](../permissions.md) diff --git a/docs/platform/storage/transfers.md b/docs/platform/storage/transfers.md index df13a219..a31fc866 100644 --- a/docs/platform/storage/transfers.md +++ b/docs/platform/storage/transfers.md @@ -1,470 +1,118 @@ -# Data Transfers +# Data transfers -**Moving data between CANFAR storage systems, external sources, and your local computer.** +Use `canfar data` for authenticated transfers between configured VOSpace +Services and the local filesystem. The command is the shell front door for the +same Storage Identifier mapping used by `canfar.storage`. -!!! abstract "🎯 Transfer Methods Overview" - **Efficient data movement strategies:** - - - **Web Interfaces**: Simple uploads and downloads for small files - - **Command-Line Tools**: Efficient transfers for large datasets - - **Automated Workflows**: Scripted transfers and synchronisation - - **Performance Optimisation**: Choosing the right method for your data size +## Operand syntax -Efficient data transfer is essential for astronomy workflows. CANFAR provides multiple transfer methods optimized for different scenarios, from small file uploads to large dataset synchronisation. +Every operand is an explicit Storage Identifier followed by an absolute path: -## 🔄 Transfer Overview - -### Transfer Types by Method - -| Method | Best For | Speed | Complexity | Interactive | Automated | -|--------|----------|-------|------------|-------------|-----------| -| **Web Upload/Download** | Small files (<1GB) | Slow | Simple | ✅ | ❌ | -| **Direct URLs** | Medium files, scripting | Medium | Simple | ⚠️ | ✅ | -| **VOSpace CLI** | All sizes, Vault access | Medium | Medium | ✅ | ✅ | -| **SSHFS Mount** | Local file operations | Medium | Medium | ✅ | ⚠️ | -| **rsync via SSHFS** | Large datasets, sync | Fast | Advanced | ⚠️ | ✅ | - -### Storage System Access - -| Source → Destination | Method | Command Example | -|---------------------|--------|-----------------| -| **Local → ARC Projects** | SSHFS, Direct URL, VOSpace | `vcp file.fits vos:/arc:projects/[project]/` | -| **Local → Vault** | VOSpace CLI, Web | `vcp file.fits vos:[user]/` | -| **Local → Scratch** | Only during sessions | `cp file.fits /scratch/` (within session) | -| **ARC → Vault** | VOSpace CLI | `vcp /arc/projects/[project]/file.fits vos:[user]/` | -| **Vault → ARC** | VOSpace CLI | `vcp vos:[user]/file.fits /arc/projects/[project]/` | -| **Scratch ↔ ARC** | Direct copy | `cp /scratch/file.fits /arc/projects/[project]/` | - -## 📤 Upload Methods - -### Small Files (<1GB): Web Interface - -#### ARC Projects and Home - -1. **Navigate to storage**: [ARC File Manager](https://www.canfar.net/storage/arc/list/) -2. **Select destination**: Choose your home or project directory -3. **Upload files**: Click "Add" → "Upload Files" -4. **Select files**: Choose files from your computer -5. **Confirm upload**: Click "Upload" then "OK" - -Note on a notebook session you can also use the JupyterLab **Upload** button. - -#### Vault (VOSpace) - -1. **Navigate to Vault**: [VOSpace File Manager](https://www.canfar.net/storage/vault/list/) -2. **Select destination**: Browse to your space -3. **Upload files**: Same process as ARC storage -4. **Set permissions**: Right-click → Properties to set sharing permissions - -### Medium Files (1-100GB): Command Line - -#### Using Direct URLs (ARC only) - -```bash -# Authenticate first -cadc-get-cert --user [user] - -# Upload to ARC Home -curl --cert ~/.ssl/cadcproxy.pem \ - --upload-file myfile.fits \ - https://ws-uv.canfar.net/arc/files/home/[user]/myfile.fits - -# Upload to ARC Projects -curl --cert ~/.ssl/cadcproxy.pem \ - --upload-file myfile.fits \ - https://ws-uv.canfar.net/arc/files/projects/[project]/myfile.fits +```text +identifier:/absolute/path ``` -#### Using VOSpace CLI +`local` is always available and means the machine where `canfar` is running. +Configured names such as `arc` or `vault` are deployment data; list the names +with the configuration tools or use the names shown in your setup. The embedded +application does not register those names as Python or fsspec protocols. ```bash -# Install VOS tools (if not already available) -pip install vos - -# Authenticate -cadc-get-cert --user [user] - -# Upload to Vault -vcp myfile.fits vos:[user]/data/ - -# Upload to ARC via VOSpace API -vcp myfile.fits arc:projects/[project]/data/ - -# Upload with progress monitoring -vcp --verbose myfile.fits vos:[user]/large_files/ +canfar data ls -lh local:/tmp +canfar data ls -lh vault:/project +canfar data ls -lh arc:/projects/ ``` -### Large Files (>100GB): Advanced Methods - -#### SSHFS Mount + rsync - -```bash -# 1. Mount CANFAR storage locally -mkdir ~/canfar_mount -sshfs -p 64022 [user]@ws-uv.canfar.net:/ ~/canfar_mount - -# 2. Sync large datasets with rsync -rsync --archive --verbose --compress --progress --partial \ - ./large_dataset/ \ - ~/canfar_mount/arc/projects/[project]/data/ - -# 3. Unmount when complete -umount ~/canfar_mount -``` - -#### VOSpace Bulk Transfer - -```bash -# Sync entire directories -vsync ./local_data/ vos:[user]/backup/ - -# Parallel transfers (faster for many files) -vsync --nstreams=4 large_file.tar vos:[user]/archives/ -``` - -## 📥 Download Methods - -### From ARC Storage - -#### Web Interface - -1. **Navigate**: [ARC File Manager](https://www.canfar.net/storage/arc/list/) -2. **Select files**: Check boxes next to desired files -3. **Download options**: - - **ZIP**: Single archive (recommended for multiple files) - - **URL List**: Generate download links for scripting - - **HTML List**: Individual download links - -#### Command Line - -```bash -# Direct URL download -curl --cert ~/.ssl/cadcproxy.pem \ - https://ws-uv.canfar.net/arc/files/home/[user]/myfile.fits \ - --output myfile.fits - -# Via VOSpace API -vcp arc:home/[user]/myfile.fits ./ - -# Multiple files with wildcards -vcp "arc:projects/[project]/data/*.fits" ./local_data/ -``` - -### From Vault (VOSpace) - -#### Command Line - -```bash -# Single file -vcp vos:[user]/data.fits ./ - -# Directory with all contents -vcp vos:[user]/survey_data/ ./local_survey/ -``` - -#### Python API - -```python -import vos - -client = vos.Client() - -# Download single file -client.copy("vos:[user]/data.fits", "./local_data.fits") - -# Download with progress callback -def progress_callback(bytes_transferred, total_bytes): - percent = (bytes_transferred / total_bytes) * 100 - print(f"Progress: {percent:.1f}%") - -client.copy("vos:[user]/large_file.fits", - "./large_file.fits", - callback=progress_callback) -``` - -## 🔄 Inter-Storage Transfers - -### Moving Data Between Storage Systems - -#### Scratch to ARC (Within Sessions) - -```bash -# Process data in scratch for speed -cp /arc/projects/[project]/raw_data.fits /scratch/ -python reduce_data.py /scratch/raw_data.fits - -# Save results to permanent storage -cp /scratch/processed_data.fits /arc/projects/[project]/results/ -cp /scratch/analysis_plots/ /arc/projects/[project]/figures/ -``` +Authentication and endpoint resolution happen when the command opens a +configured source. If a credential is missing or expired, log in to the +corresponding Identity Provider and retry. -#### ARC to Vault (Archival) +## Inspect data ```bash -# Archive completed project results -vcp /arc/projects/[project]/final_results/ vos:[user]/archives/project2024/ +canfar data ls -lh vault:/project +canfar data info vault:/project/catalog.csv +canfar data size vault:/project/catalog.csv +canfar data stat vault:/project/catalog.csv +canfar data find vault:/project --type f ``` -#### Vault to ARC (Project Setup) - -```bash -# Import archived data for new analysis -vcp vos:shared_project/calibrated_data/ /arc/projects/[project]/data/ +Use `canfar data --help` and the individual command help for the complete +upstream fsspec-cli surface. The command output is intentionally owned by that +application; CANFAR does not add an active-server banner to it. -# Import specific datasets -vcp "vos:public_surveys/gaia_dr3/*.fits" /arc/projects/[project]/catalogues/ -``` +## Copy files and directories -### Automated Workflow Example +Copy one file in either direction: ```bash -#!/bin/bash -# Complete data processing workflow - -set -e # Exit on error - -PROJECT_DIR="/arc/projects/[project]" -SCRATCH_DIR="/scratch" - -echo "Starting data processing pipeline..." - -# 1. Download raw data from Vault to scratch -echo "Downloading raw data..." -vcp vos:[user]/raw_observations/obs_*.fits ${SCRATCH_DIR}/ - -# 2. Process data in scratch (fastest storage) -echo "Processing data..." -cd ${SCRATCH_DIR} -for file in obs_*.fits; do - python calibrate.py "$file" "cal_${file}" -done - -# 3. Save intermediate results to ARC -echo "Saving calibrated data..." -mkdir --parents ${PROJECT_DIR}/calibrated/ -cp cal_*.fits ${PROJECT_DIR}/calibrated/ - -# 4. Further analysis -echo "Running analysis..." -python analyze_all.py ${PROJECT_DIR}/calibrated/ > analysis_results.txt - -# 5. Save final results to ARC and archive to Vault -echo "Saving final results..." -cp analysis_results.txt ${PROJECT_DIR}/results/ -cp final_plots/*.png ${PROJECT_DIR}/figures/ - -# Archive to Vault -vcp ${PROJECT_DIR}/results/ vos:[user]/completed_projects/$(date +%Y%m%d)/ - -echo "Pipeline completed successfully!" +canfar data cp local:/data/result.fits vault:/project/results/result.fits +canfar data cp vault:/project/input.fits local:/scratch/input.fits ``` -## 📊 Performance Optimization - -### Transfer Speed Optimization - -#### For Many Small Files +Use `-R` (or `-r`) for a directory copy: ```bash -# Bundle small files into archives -tar --create --gzip --file analysis_scripts.tar.gz scripts/ -vcp analysis_scripts.tar.gz vos:[user]/code/ - -# Use directory sync instead of individual copies -vsync --nstreams=4 ./many_small_files/ vos:[user]/collection/ +canfar data cp -R local:/data/run-42 vault:/project/runs/run-42 ``` -### Network Performance Tips - -#### Optimal Transfer Times - -- **Best performance**: Off-peak hours (evenings, weekends) -- **Avoid**: Peak research hours (9 AM - 5 PM Pacific) - -#### Connection Optimization +Create a destination first when that makes the workflow clearer: ```bash -# Check network speed to CANFAR -ping ws-uv.canfar.net - -# Test transfer speed with small file -time vcp test_file.fits vos:[user]/speed_test/ +canfar data mkdir -p vault:/project/results +canfar data cp local:/scratch/result.fits vault:/project/results/result.fits ``` -## 🚨 Error Handling and Recovery - -### Common Transfer Issues - -#### Authentication Errors +For a transfer between two remote VOSpace Services, use an explicit copy and +verify the destination. Do not assume that a cross-source `mv` is supported: ```bash -# Certificate expired -ERROR:: Expired cert. Update by running cadc-get-cert - -# Solution: Refresh certificate -cadc-get-cert --user [user] - -# Check certificate validity -cadc-get-cert --days-valid +canfar data cp vault:/project/input.fits arc:/projects//input.fits +canfar data info arc:/projects//input.fits ``` -#### Network Timeouts +The CLI keeps recursive removal disabled. Delete individual files with `rm` or +empty directories with `rmdir` only after checking the path: ```bash -# Retry with exponential backoff -for i in {1..3}; do - vcp file.fits vos:[user]/ && break - sleep $((2**i)) -done +canfar data rm vault:/project/results/old.fits +canfar data rmdir vault:/project/results/empty-directory ``` -### Robust Transfer Script +## Session workflow -```python -#!/usr/bin/env python -""" -Robust file transfer with retry logic -""" -import vos -import time -import sys -from pathlib import Path - -def robust_transfer(source, destination, max_retries=3): - """Transfer file with retry logic""" - client = vos.Client() - - for attempt in range(max_retries): - try: - print(f"Transfer attempt {attempt + 1}: {source} → {destination}") - client.copy(source, destination) - print(f"✓ Transfer successful") - return True - - except Exception as e: - print(f"✗ Attempt {attempt + 1} failed: {e}") - if attempt < max_retries - 1: - wait_time = 2 ** attempt # Exponential backoff - print(f"Waiting {wait_time} seconds before retry...") - time.sleep(wait_time) - else: - print(f"Transfer failed after {max_retries} attempts") - return False - -# Usage -if __name__ == "__main__": - if len(sys.argv) != 3: - print("Usage: python robust_transfer.py ") - sys.exit(1) - - source, destination = sys.argv[1], sys.argv[2] - success = robust_transfer(source, destination) - sys.exit(0 if success else 1) -``` - -## 📋 Transfer Checklists - -### Pre-Transfer Checklist - -- [ ] **Authentication**: Valid CADC certificate (`cadc-get-cert`) -- [ ] **Permissions**: Write access to destination directory -- [ ] **Space**: Sufficient quota in destination storage -- [ ] **Network**: Stable connection for large transfers -- [ ] **Backup**: Important data backed up before moving - -### Post-Transfer Verification +Inside a Science Platform Session, prefer a mounted `/arc` path for data that is +already present there. For one remote input, copy directly to `/scratch`, run +the analysis locally, and copy final products to `/arc` or a persistent VOSpace +destination: ```bash -# Verify file integrity -vls --long vos:[user]/transferred_file.fits # Check size and timestamp - -# Compare checksums (if available) -vcp --head vos:[user]/data.fits | grep MD5 - -# Test file readability -python -c "from astropy.io import fits; fits.open('test_file.fits')" +canfar data cp vault:/project/cube.fits local:/scratch/cube.fits +python reduce.py /scratch/cube.fits /scratch/result.fits +canfar data cp local:/scratch/result.fits arc:/projects//result.fits ``` -### Transfer Planning Template +`/scratch` is Session-local and is deleted when the Session ends. It is a good +staging location, not a backup. For repeated Python reads, select an explicit +fsspec whole-file cache under `/scratch`; see [Filesystem and Python tools](filesystem.md). -```markdown -## Transfer Plan: [Project Name] +## Transfer failures -**Data Description**: -- Size: ___GB -- File count: ___ -- Type: Raw/Processed/Results +| Symptom | What to check | +| --- | --- | +| Unknown Storage Identifier | Use the configured identifier exactly; `local` is the only reserved name. | +| Authentication failure | Run the appropriate `canfar login ` and confirm the saved Authentication Record. | +| Permission denied | Confirm the VOSpace path and project/group membership. | +| Destination is missing | Create parent directories with `canfar data mkdir -p`. | +| Copy is slow | Avoid many small remote reads; stage once to `/scratch` or use one explicit cache. | +| Files disappear after a Session | Move results from `/scratch` to `/arc` or a persistent VOSpace Service before deletion. | -**Source**: _______________ -**Destination**: ___________ -**Method**: _______________ +For a service outage or persistent authorization issue, contact [CANFAR +support](../support/index.md). -**Timeline**: -- Start: ____________ -- Estimated completion: ___________ +## Related guides -**Verification**: -- [ ] File count matches -- [ ] Total size matches -- [ ] Sample files readable -- [ ] Permissions set correctly - -**Backup**: _______________ -``` - -## 🔗 Integration Examples - -### Jupyter Notebook Upload - -Within a CANFAR Jupyter session: - -```python -# Upload files using the Jupyter interface -# 1. Click the "Upload" button in file browser -# 2. Select files from your computer -# 3. Files appear in current directory - -# Move uploaded files to appropriate storage -import shutil -shutil.move('uploaded_data.fits', '/arc/projects/[project]/data/') - -# Or copy to scratch for processing -shutil.copy('/arc/projects/[project]/data.fits', '/scratch/') -``` - -### Batch Job Data Staging - -```bash -#!/bin/bash -# Batch job with data staging - -# Download input data -vcp vos:project/input_data.tar.gz /scratch/ -cd /scratch -tar --extract --gzip --file input_data.tar.gz - -# Process data -python analysis.py input_data/ - -# Upload results -tar --create --gzip --file results_$(date +%Y%m%d).tar.gz results/ -vcp results_*.tar.gz vos:[user]/job_outputs/ - -# Cleanup -rm --recursive --force /scratch/* -``` - -### External Data Import - -```bash -# Download from astronomical archives -wget --output-document=survey_data.fits "https://archive.eso.org/..." - -# Upload to CANFAR -vcp survey_data.fits vos:[user]/external_data/ - -# Or direct to project space -curl --cert ~/.ssl/cadcproxy.pem \ - --upload-file survey_data.fits \ - https://ws-uv.canfar.net/arc/files/projects/[project]/survey_data.fits -``` +- [Storage overview](index.md) +- [Filesystem and Python tools](filesystem.md) +- [VOSpace](vospace.md) +- [Permissions](../permissions.md) diff --git a/docs/platform/storage/vospace.md b/docs/platform/storage/vospace.md index 8f71e11b..e2a4f2fe 100644 --- a/docs/platform/storage/vospace.md +++ b/docs/platform/storage/vospace.md @@ -1,1154 +1,101 @@ -# VOSpace +# VOSpace Services -!!! abstract "🎯 VOSpace Guide Overview" - **Master CANFAR's long-term storage system:** - - - **VOSpace Concepts**: Understanding IVOA standards and when to use Vault - - **Web Interface**: Browser-based file management and sharing - - **Command-Line Tools**: Efficient bulk operations and automation - - **Python API**: Programmatic access for workflows and integration - - **Metadata & Sharing**: Rich data descriptions and collaborative access +[VOSpace](https://www.ivoa.net/documents/VOSpace/) is the IVOA protocol for +remote astronomical storage. A CANFAR Science Platform Server can expose one or +more VOSpace Services. Each service has a user-facing **Storage Identifier** in +the CANFAR configuration and an endpoint discovered from the platform. -VOSpace is CANFAR's implementation of the International Virtual Observatory Alliance (IVOA) [VOSpace standard](https://www.ivoa.net/documents/VOSpace/), providing long-term, secure, and collaborative storage for astronomy data. It serves as both an archive and a data sharing platform. +VOSpace is remote object storage, not a promise that every deployment behaves +like a mounted POSIX filesystem. Use a mounted `/arc` path inside a Session when +the data is already there; use a VOSpace Service for authenticated remote reads, +writes, sharing, and transfer between machines. -## 🌐 VOSpace Overview +## Choose an access path -### What is VOSpace? +| Need | Recommended path | +| --- | --- | +| Work with data already mounted in a Session | `/arc/home/` or `/arc/projects/` | +| Copy one object or directory from a remote service | `canfar data cp` | +| List or inspect a remote service from a shell | `canfar data ls`, `info`, `stat`, or `find` | +| Use a remote object from Python | `filesystem(identifier)` from `canfar.storage` | +| Feed a path-only program such as CASA | Stage with `get_file()` or `canfar data cp` to `/scratch` | +| Reuse an input during one Session | An explicit `SimpleCacheFileSystem` under `/scratch` | -VOSpace is a distributed storage service that allows astronomers to: +## CLI access -- **Store data persistently** with geographic redundancy -- **Share data** with collaborators and the public -- **Organize data** with hierarchical directories and metadata -- **Access data** programmatically via standardized APIs -- **Integrate** with Virtual Observatory tools and services - -### Vault VOSpace vs ARC VOSpace vs Scratch - -| Feature | Vault | ARC Projects | ARC Home | Scratch | -|---------|-----------------|--------------|----------|---------| -| **Persistence** | ✅ Permanent | ✅ Permanent | ✅ Permanent | ❌ Session only | -| **Backup** | ✅ Geo-redundant | ⚠️ Basic | ⚠️ Basic | ❌ None | -| **Sharing** | ✅ Flexible permissions | ⚠️ Group-based | ⚠️ User-based | ❌ Session only | -| **Public access** | ✅ Public URLs | ❌ Private | ❌ Private | ❌ Session only | -| **Metadata** | ✅ Rich metadata | ⚠️ Basic | ⚠️ Basic | ❌ None | -| **API access** | ✅ VOSpace API | ✅ VOSpace API | ✅ VOSpace API | ❌ None | -| **Speed** | Slow (network) | Medium (network) | Medium (network) | Fast (SSD) | - -## 🌍 Web Interface - -### Accessing VOSpace - -1. **Navigate to**: - - [Vault VOSpace File Manager](https://www.canfar.net/storage/vault/list/) - - [ARC VOSpace File Manager](https://www.canfar.net/storage/arc/list/) -2. **Login**: Use your CADC credentials -3. **Browse**: Navigate through your space and shared spaces - -### Web Interface Features - -#### File Operations - -- **Upload**: Drag and drop or click "Add" → "Upload Files" -- **Download**: Select files → "Download" (ZIP, URL list, or HTML list) -- **Create folders**: "Add" → "Create Folder" -- **Delete**: Select items → "Delete" -- **Move/Copy**: Drag and drop or cut/paste - -#### Sharing and Permissions - -```text -Right-click file/folder → Properties → Permissions - -Permission Types: -- Read (r): View and download -- Write (w): Modify and delete -- Execute (x): Navigate directories - -Target Groups: -- Owner: You (full control) -- Group: Project members -- Other: Public access -``` - -## 💻 Command Line Interface - -### Installation - -VOSpace tools are pre-installed in CANFAR sessions in CANFAR-maintained containers such as `astroml`. -For local or custom installation, use `pip`: +The embedded data command maps every configured VOSpace Service and the reserved +`local` filesystem for each invocation: ```bash -# Install VOS python module with vcp/vsync/vls/vchmod/vmkdir commands -pip install vos - -# Verify installation -vls --help -vcp --help +canfar data ls -lh vault:/project +canfar data cp vault:/project/input.fits local:/scratch/input.fits +canfar data cp local:/scratch/result.fits arc:/projects//result.fits ``` -### Authentication - -```bash -# Get security certificate (valid 24 hours) -cadc-get-cert -u [user] - -# Verify authentication -vls vos:[user] -``` - -### Basic Operations - -#### Directory Operations +The `vault:` and `arc:` names are examples, not universal protocol names. Use +the Storage Identifiers configured for the active installation. See [Data +transfers](transfers.md) for copy, directory, removal, and troubleshooting +guidance. -```bash -# List directories and files -vls vos:[user]/ # Your root directory -vls vos:[user]/projects/ # Subdirectory -vls -l vos:[user]/data/ # Detailed listing - -# Create directories -vmkdir vos:[user]/new_project/ -vmkdir vos:[user]/data/2024/ - -# Navigate hierarchically -vls vos:[user]/projects/survey/data/ -``` - -#### File Operations - -```bash -# Upload files -vcp mydata.fits vos:[user]/data/ -vcp *.fits vos:[user]/observations/ -# vcp is recursive -vcp ./analysis_scripts/ vos:[user]/code/ - -# Download files -vcp vos:[user]/data/results.fits ./ -vcp "vos:[user]/observations/*.fits" ./data/ -vcp vos:[user]/code/ ./local_scripts/ - -# Copy between VOSpace locations -vcp vos:[user]/data/obs1.fits vos:[user]/backup/ -``` - -#### File Management - -```bash -# Move/rename files -vmv vos:[user]/old_name.fits vos:[user]/new_name.fits -vmv vos:[user]/temp/ vos:[user]/archive/ - -# Delete files and directories -vrm vos:[user]/old_file.fits -vrm vos:[user]/old_directory/ - -# View file contents (for text files) -vcat vos:[user]/catalog.csv -``` - -### Advanced Operations - -#### Bulk Operations - -```bash -# Synchronize directories -vsync ./local_data/ vos:[user]/backup/ -vsync vos:[user]/analysis/ ./local_analysis/ - -# Parallel transfers for speed -vsync --nstreams=4 huge_dataset.tar vos:[user]/archives/ -``` +## Python access -#### Permission Management - -```bash -# Make file publicly readable -vchmod o+r vos:[user]/public_catalog.fits - -# Grant group access -vchmod g+rw vos:[user]/shared_data.fits - -# Set permissions for specific groups -vchmod g+r:external-collaborators vos:[user]/collaboration_data/ - -# View current permissions -vls -l vos:[user]/myfile.fits -``` - -### Data Cutouts and Processing - -```bash -# FITS cutouts (pixel coordinates) -vcp "vos:[user]/image.fits[100:200,100:200]" ./cutout.fits - -# Header-only download -vcp --head vos:[user]/large_image.fits ./headers.txt - -# Inspect headers without downloading -vcat --head vos:[user]/observation.fits - -``` - -## 🐍 Python API - -### Basic Setup - -```python -import vos -from vos import Client - -# Initialize client (uses existing authentication) -client = Client() - -# Alternative: specify authentication -client = Client(username='[user]', password='[password]') -``` - -### File Operations - -```python -# List directory contents -files = client.listdir('vos:[user]/') -print(f"Found {len(files)} files") - -# Check if file exists -exists = client.isfile('vos:[user]/data.fits') -if not exists: - print("File not found") - -# Get file information -info = client.get_info('vos:[user]/data.fits') -print(f"Size: {info['size']} bytes") -print(f"Modified: {info['date']}") - -# Copy files -client.copy('mydata.fits', 'vos:[user]/uploads/mydata.fits') -client.copy('vos:[user]/results.txt', './local_results.txt') - -# Create directories -client.mkdir('vos:[user]/new_project/') - -# Delete files -client.delete('vos:[user]/old_file.fits') -``` - -### Advanced Python Usage - -#### Batch Processing - -```python -import os -from pathlib import Path - -def process_vospace_directory(vospace_path, local_temp_dir): - """Download, process, and re-upload files from VOSpace""" - - # Create local working directory - Path(local_temp_dir).mkdir(exist_ok=True) - - # List files in VOSpace - files = client.listdir(vospace_path) - fits_files = [f for f in files if f.endswith('.fits')] - - for fits_file in fits_files: - vospace_file = f"{vospace_path}/{fits_file}" - local_file = f"{local_temp_dir}/{fits_file}" - processed_file = f"{local_temp_dir}/processed_{fits_file}" - - # Download - print(f"Downloading {fits_file}") - client.copy(vospace_file, local_file) - - # Process (example: your analysis here) - process_fits_file(local_file, processed_file) - - # Upload processed version - processed_vospace = f"{vospace_path}/processed_{fits_file}" - client.copy(processed_file, processed_vospace) - - # Cleanup local files - os.remove(local_file) - os.remove(processed_file) - -# Usage -process_vospace_directory('vos:[user]/raw_data', './temp_processing') -``` - -#### Metadata Management +The public Python surface is deliberately small: ```python -# Get file node (for metadata operations) -node = client.get_node('vos:[user]/observation.fits') - -# Set metadata -node.props['TELESCOPE'] = 'ALMA' -node.props['OBJECT'] = 'NGC1365' -node.props['DATE-OBS'] = '2024-03-15T10:30:00' - -# Update node with new metadata -client.update(node) +from canfar.storage import filesystem, identifiers -# Read metadata -props = node.props -telescope = props.get('TELESCOPE', 'Unknown') -object_name = props.get('OBJECT', 'Unknown') - -print(f"Observation of {object_name} with {telescope}") -``` - -#### Progress Monitoring - -```python -def upload_with_progress(local_file, vospace_path): - """Upload file with progress monitoring""" - - file_size = os.path.getsize(local_file) - - def progress_callback(bytes_transferred): - percent = (bytes_transferred / file_size) * 100 - print(f"\rProgress: {percent:.1f}% ({bytes_transferred}/{file_size} bytes)", end='') - - try: - client.copy(local_file, vospace_path, callback=progress_callback) - print("\nUpload completed successfully!") - except Exception as e: - print(f"\nUpload failed: {e}") - -# Usage -upload_with_progress('large_dataset.fits', 'vos:[user]/archives/dataset.fits') -``` - -## 🔒 Sharing and Collaboration - -### Permission Levels - -#### Owner Permissions -- **Full control**: Read, write, delete, change permissions -- **Default**: Only owner has access to new files - -#### Group Permissions -- **Read**: Group members can view and download -- **Write**: Group members can modify and upload -- **Execute**: Group members can navigate directories - -#### Public Permissions -- **Read**: Anyone with the URL can download -- **Useful for**: Publishing datasets, sharing with external collaborators - -### Setting Up Sharing - -#### Command Line Sharing - -```bash -# Make dataset publicly available -vchmod o+r vos:[user]/public_datasets/gaia_subset.fits - -# Share with research group -vchmod g+rw:my_research_group vos:[user]/shared_analysis/ - -# Create public directory -vmkdir vos:[user]/public/ -vchmod o+r vos:[user]/public/ - -# Share specific project data -vchmod g+r:external_collaborators vos:[user]/collaboration/survey_data/ -``` - -#### Public URLs - -```bash -# Files with public read permissions get accessible URLs: -# https://ws-cadc.canfar.net/vault/nodes/[user]/public_file.fits - -# Direct download links for shared data: -curl -O https://ws-cadc.canfar.net/vault/nodes/[user]/public/catalog.csv -``` - -### Collaboration Workflows - -#### Multi-Institutional Project - -```bash -# Project coordinator sets up shared space -vmkdir vos:[project]/data -vmkdir vos:[project]/public -vmkdir vos:[project]/results -vchmod g+rw:all_collaborators vos:[project]/data/ - -# Collaborators contribute data -vcp local_observations.fits vos:[project]/data/institution_a/ -vcp analysis_results.csv vos:[project]/results/ - -# Public data release -vcp vos:[project]/data/final_catalogue.fits vos:[project]/public/ -vchmod o+r vos:[project]/public/final_catalogue.fits -``` - -#### Data Publication - -```python -import vos - -def publish_dataset(local_files, publication_space): - """Publish dataset with proper metadata""" - - client = vos.Client() - - # Create publication directory - client.mkdir(publication_space) - - for local_file in local_files: - filename = os.path.basename(local_file) - vospace_path = f"{publication_space}/{filename}" - - # Upload file - client.copy(local_file, vospace_path) - - # Set metadata - node = client.get_node(vospace_path) - node.props['AUTHOR'] = 'Dr. Astronomer' - node.props['PUBLICATION'] = 'ApJ 2024, 123, 456' - node.props['DOI'] = '10.1088/example' - client.update(node) - - # Make publicly accessible - client.set_permissions(vospace_path, public_read=True) - - print(f"Published: {vospace_path}") - -# Usage -files_to_publish = ['final_catalog.fits', 'processed_images.tar.gz'] -publish_dataset(files_to_publish, 'vos:[user]/publications/survey2024') -``` - -## 🔧 Integration with Astronomical Tools - -### FITS File Handling - -```python -from astropy.io import fits -import tempfile -import os - -def analyze_vospace_fits(vospace_path): - """Analyze FITS file stored in VOSpace""" - - # Download to temporary file - with tempfile.NamedTemporaryFile(suffix='.fits', delete=False) as tmp: - client.copy(vospace_path, tmp.name) - - # Open with astropy - with fits.open(tmp.name) as hdul: - header = hdul[0].header - data = hdul[0].data - - # Perform analysis - mean_value = data.mean() - max_value = data.max() - - print(f"Image stats: mean={mean_value:.2f}, max={max_value:.2f}") - - # Extract key information - telescope = header.get('TELESCOP', 'Unknown') - object_name = header.get('OBJECT', 'Unknown') - - # Cleanup - os.unlink(tmp.name) - - return {'mean': mean_value, 'max': max_value, 'telescope': telescope} - -# Usage -stats = analyze_vospace_fits('vos:[user]/observations/ngc1365.fits') -``` - -### Integration with Archives - -```python -def mirror_archive_data(archive_url, vospace_destination): - """Download from astronomical archive and store in VOSpace""" - - import requests - import tempfile - - # Download from archive - response = requests.get(archive_url) - - # Save to temporary file - with tempfile.NamedTemporaryFile(delete=False) as tmp: - tmp.write(response.content) - tmp_path = tmp.name - - try: - # Upload to VOSpace - client.copy(tmp_path, vospace_destination) - - # Set metadata about source - node = client.get_node(vospace_destination) - node.props['ARCHIVE_URL'] = archive_url - node.props['DOWNLOAD_DATE'] = datetime.now().isoformat() - client.update(node) - - print(f"Mirrored {archive_url} to {vospace_destination}") - - finally: - os.unlink(tmp_path) - -# Example: Mirror HST data -mirror_archive_data( - 'https://archive.stsci.edu/missions/hubble/...', - 'vos:[user]/hst_data/observation_123.fits' -) -``` - -## 📊 Performance and Optimization - -### Transfer Performance - - -#### Caching and Local Mirrors - -```python -import hashlib -from pathlib import Path - -class VOSpaceCache: - def __init__(self, cache_dir='./vospace_cache'): - self.cache_dir = Path(cache_dir) - self.cache_dir.mkdir(exist_ok=True) - self.client = vos.Client() - - def get_cached_file(self, vospace_path, force_refresh=False): - """Get file from cache or download if needed""" - - # Generate cache filename - cache_name = hashlib.md5(vospace_path.encode()).hexdigest() - cache_file = self.cache_dir / cache_name - - # Check if cache is valid - if not force_refresh and cache_file.exists(): - # Compare modification times - local_mtime = cache_file.stat().st_mtime - try: - remote_info = self.client.get_info(vospace_path) - remote_mtime = remote_info['date'] - - if local_mtime >= remote_mtime: - print(f"Using cached version: {cache_file}") - return str(cache_file) - except: - pass - - # Download fresh copy - print(f"Downloading {vospace_path} to cache") - self.client.copy(vospace_path, str(cache_file)) - return str(cache_file) - -# Usage -cache = VOSpaceCache() -local_file = cache.get_cached_file('vos:[user]/large_catalog.fits') -``` - -### Monitoring and Logging - -```python -import logging -import time - -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def monitored_transfer(source, destination): - """Transfer with monitoring and timing""" - - start_time = time.time() - logger.info(f"Starting transfer: {source} → {destination}") - - try: - client.copy(source, destination) - - end_time = time.time() - duration = end_time - start_time - - # Get file size for speed calculation - if source.startswith('vos:'): - info = client.get_info(source) - size_mb = info['size'] / (1024 * 1024) - else: - size_mb = os.path.getsize(source) / (1024 * 1024) - - speed = size_mb / duration if duration > 0 else 0 - - logger.info(f"Transfer completed: {size_mb:.1f} MB in {duration:.1f}s ({speed:.1f} MB/s)") - return True - - except Exception as e: - logger.error(f"Transfer failed: {e}") - return False - -# Usage -success = monitored_transfer('large_file.fits', 'vos:[user]/archives/') -``` - -## 🛠️ Troubleshooting - -### Common Issues - -#### Authentication Problems - -```bash -# Certificate expired -cadc-get-cert -u [user] - -# Check certificate validity -cadc-get-cert --days-valid - -# Clear certificate cache -rm ~/.ssl/cadcproxy.pem -cadc-get-cert -u [user] -``` - -#### Permission Errors - -```bash -# Check file permissions -vls -l vos:[user]/file.fits - -# Verify directory permissions -vls -l vos:[user]/ - -# Check group membership -# (Contact CANFAR support if needed) -``` - -#### Network and Transfer Issues - -```bash -# Test connectivity -ping ws-cadc.canfar.net - -# Check VOSpace service status -vls vos: - -# Retry with different parameters -vcp --timeout=3600 large_file.fits vos:[user]/ # Increase timeout -vcp --nstreams=1 problematic_file.fits vos:[user]/ # Reduce streams -``` - -### Debugging and Diagnostics - -```python -import vos -import logging - -# Enable debug logging -logging.basicConfig(level=logging.DEBUG) - -# Get detailed client information -client = vos.Client() -print(f"VOSpace endpoint: {client.vospace_url}") -print(f"Authentication: {client.get_auth()}") - -# Test basic operations +print(identifiers()) +remote = filesystem("vault") try: - files = client.listdir('vos:') - print(f"Root access successful, found {len(files)} items") -except Exception as e: - print(f"Root access failed: {e}") - -# Check specific paths -test_paths = ['vos:[user]/', 'vos:[user]/data/'] -for path in test_paths: - try: - contents = client.listdir(path) - print(f"✓ {path}: {len(contents)} items") - except Exception as e: - print(f"✗ {path}: {e}") -``` - -## Related Storage Docs - -- **[Data Transfers →](transfers.md)** - Moving data between storage systems -- **[Filesystem Access →](filesystem.md)** - ARC storage and SSHFS mounting -- **[Storage Overview →](index.md)** - Understanding all CANFAR storage types -- **[Interactive Sessions →](../sessions/index.md)** - Using VOSpace within CANFAR sessions -``` - - -#### ARC (Inside CANFAR session) - -```bash -# List files and directories -ls /arc/projects/[project]/ - -# Copy files -cp mydata.fits /arc/projects/[project]/data/ - -# Create directories -mkdir /arc/projects/[project]/survey_analysis/ - -# Move/rename files -mv /arc/projects/[project]/old.fits /arc/projects/[project]/new.fits - -# Remove files -rm /arc/projects/[project]/temp/old_data.fits -``` - - - -### Bulk Operations - -> Note: `vsync` and `vcp` are always recursive; no `--recursive` flag is needed. - - -#### Vault (VOSpace API) - -```bash -# Sync entire directories to Vault -vsync ./local_data/ vos:[user]/backup/ - -# Download project data from Vault -vsync vos:[project]/survey_data/ ./project_data/ - -# Upload analysis results to Vault -vsync ./results/ vos:[user]/analysis_outputs/ -``` - - -#### ARC (VOSpace API, outside CANFAR) - -```bash -# Sync entire directories to ARC -vsync ./local_data/ arc:projects/[project]/backup/ - -# Download project data from ARC -vsync arc:projects/[project]/survey_data/ ./project_data/ - -# Upload analysis results to ARC -vsync ./results/ arc:projects/[project]/analysis_outputs/ -``` - - - -## Python API - - -### Basic Usage - -```python -import vos - -# Initialize client -client = vos.Client() - - -# List directory contents in Vault -files_vault = client.listdir("vos:[user]/") -print(files_vault) - -# List directory contents in ARC -files_arc = client.listdir("arc:projects/[project]/") -print(files_arc) - -# Check if file exists in Vault -exists_vault = client.isfile("vos:[user]/data.fits") - -# Check if file exists in ARC -exists_arc = client.isfile("arc:projects/[project]/data.fits") - -# Get file info from Vault -info_vault = client.get_info("vos:[user]/data.fits") -print(f"Size: {info_vault['size']} bytes") -print(f"Modified: {info_vault['date']}") - -# Get file info from ARC -info_arc = client.get_info("arc:projects/[project]/data.fits") -print(f"Size: {info_arc['size']} bytes") -print(f"Modified: {info_arc['date']}") -``` - - -### File Operations - -```python - -# Copy file to Vault -client.copy("mydata.fits", "vos:[user]/data/mydata.fits") - -# Copy file to ARC -client.copy("mydata.fits", "arc:projects/[project]/data/mydata.fits") - -# Copy file from Vault -client.copy("vos:[user]/data/results.txt", "./results.txt") - -# Copy file from ARC -client.copy("arc:projects/[project]/data/results.txt", "./results.txt") - -# Create directory in Vault -client.mkdir("vos:[user]/new_project/") - -# Create directory in ARC -client.mkdir("arc:projects/[project]/new_project/") - -# Delete file in Vault -client.delete("vos:[user]/temp/old_file.txt") - -# Delete file in ARC -client.delete("arc:projects/[project]/temp/old_file.txt") -``` - - -### Advanced Operations - -```python -import os -from astropy.io import fits - -def process_fits_files(vospace_dir, output_dir): - """Process all FITS files in a Vault or ARC directory""" - - # List all FITS files - files = client.listdir(vospace_dir) - fits_files = [f for f in files if f.endswith(".fits")] - - for fits_file in fits_files: - vospace_path = f"{vospace_dir}/{fits_file}" - local_path = f"./temp_{fits_file}" - - # Download file - client.copy(vospace_path, local_path) - - # Process with astropy - with fits.open(local_path) as hdul: - # Your processing here - processed_data = hdul[0].data * 2 # Example processing - - # Save processed file - output_path = f"{output_dir}/processed_{fits_file}" - fits.writeto(output_path, processed_data, overwrite=True) - - - # Upload to Vault or ARC - if vospace_dir.startswith("vos:"): - client.copy(output_path, f"vos:[user]/processed/{fits_file}") - else: - client.copy(output_path, f"arc:projects/[project]/processed/{fits_file}") - - # Clean up temporary file - os.remove(local_path) - -# Usage - -process_fits_files("vos:[user]/raw_data", "./processed/") -process_fits_files("arc:projects/[project]/raw_data", "./processed/") -``` - - -## Automation Workflows - - -### Batch Processing Script - -```python -#!/usr/bin/env python3 -""" -Automated data processing pipeline using Vault (VOSpace API) and ARC -""" -import vos -import sys -import logging -from pathlib import Path - -# Setup logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def setup_vospace(): - """Initialize VOSpace client with authentication""" - try: - client = vos.Client() - # Test connection - client.listdir("vos:[project]/") - return client - except Exception as e: - logger.error(f"VOSpace authentication failed: {e}") - sys.exit(1) - -def sync_input_data(client, remote_dir, local_dir): - """Download input data from Vault or ARC""" - logger.info(f"Syncing {remote_dir} to {local_dir}") - - Path(local_dir).mkdir(parents=True, exist_ok=True) - - # Get list of files - files = client.listdir(remote_dir) - - for file in files: - if file.endswith((".fits", ".txt", ".csv")): - remote_path = f"{remote_dir}/{file}" - local_path = f"{local_dir}/{file}" - - if not Path(local_path).exists(): - logger.info(f"Downloading {file}") - client.copy(remote_path, local_path) - -def upload_results(client, local_dir, remote_dir): - """Upload processing results to Vault or ARC""" - logger.info(f"Uploading results from {local_dir} to {remote_dir}") - - # Ensure remote directory exists - try: - client.mkdir(remote_dir) - except: - pass # Directory might already exist - - for file_path in Path(local_dir).glob("*"): - if file_path.is_file(): - remote_path = f"{remote_dir}/{file_path.name}" - logger.info(f"Uploading {file_path.name}") - client.copy(str(file_path), remote_path) - -def main(): - """Main processing pipeline""" - client = setup_vospace() - - # Configuration - input_remote_vault = "vos:[project]/raw_data" - input_remote_arc = "arc:projects/[project]/raw_data" - output_remote_vault = "vos:[user]/processed_results" - output_remote_arc = "arc:projects/[project]/processed_results" - local_input = "./input_data" - local_output = "./output_data" - - # Download input data from Vault - sync_input_data(client, input_remote_vault, local_input) - # Download input data from ARC - sync_input_data(client, input_remote_arc, local_input) - - # Your processing code here - logger.info("Processing data...") - # ... processing logic ... - - # Upload results to Vault - upload_results(client, local_output, output_remote_vault) - # Upload results to ARC - upload_results(client, local_output, output_remote_arc) - - logger.info("Pipeline completed successfully") - -if __name__ == "__main__": - main() -``` - -## Monitoring and Logging - -### Transfer Progress - -```python -def copy_with_progress(client, source, destination): - """Copy file with progress monitoring""" - import time - - # Start transfer - start_time = time.time() - client.copy(source, destination) - end_time = time.time() - - # Get file size for speed calculation - if source.startswith("vos:"): - info = client.get_info(source) - size_mb = info["size"] / (1024 * 1024) - else: - size_mb = os.path.getsize(source) / (1024 * 1024) - - duration = end_time - start_time - speed = size_mb / duration if duration > 0 else 0 - - print(f"Transfer completed: {size_mb:.1f} MB in {duration:.1f}s ({speed:.1f} MB/s)") -``` - -### Error Handling - -```python -def robust_copy(client, source, destination, max_retries=3): - """Copy with retry logic""" - import time - - for attempt in range(max_retries): - try: - client.copy(source, destination) - return True - except Exception as e: - logger.warning(f"Copy attempt {attempt + 1} failed: {e}") - if attempt < max_retries - 1: - time.sleep(2**attempt) # Exponential backoff - else: - logger.error(f"Copy failed after {max_retries} attempts") - return False -``` - -## Performance Optimization - -### Parallel Transfers - -```python -import concurrent.futures -import threading - - -def parallel_upload(client, file_list, remote_dir, max_workers=4): - """Upload multiple files in parallel""" - - def upload_file(file_path): - remote_path = f"{remote_dir}/{file_path.name}" - try: - client.copy(str(file_path), remote_path) - return f"✓ {file_path.name}" - except Exception as e: - return f"✗ {file_path.name}: {e}" - - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [executor.submit(upload_file, f) for f in file_list] - - for future in concurrent.futures.as_completed(futures): - result = future.result() - print(result) -``` - -### Caching Strategy - -```python -import hashlib -from pathlib import Path - - -def cached_download(client, vospace_path, local_path, force_refresh=False): - """Download file only if it has changed""" - - local_file = Path(local_path) - cache_file = Path(f"{local_path}.cache_info") - - # Get remote file info - remote_info = client.get_info(vospace_path) - remote_hash = remote_info.get("MD5", "") - - # Check if we have cached info - if not force_refresh and local_file.exists() and cache_file.exists(): - cached_hash = cache_file.read_text().strip() - if cached_hash == remote_hash: - print(f"Using cached version of {local_file.name}") - return local_path - - # Download file - print(f"Downloading {local_file.name}") - client.copy(vospace_path, local_path) - - # Save cache info - cache_file.write_text(remote_hash) - - return local_path -``` - -## Integration Examples - -### With Astropy - -```python -from astropy.io import fits -from astropy.table import Table - - -def analyze_vospace_catalog(client, catalog_path): - """Analyze a catalog stored in VOSpace""" - - # Download catalog - local_path = "./temp_catalog.fits" - client.copy(catalog_path, local_path) - - # Load and analyze - table = Table.read(local_path) - - # Example analysis - bright_sources = table[table["magnitude"] < 15] - print(f"Found {len(bright_sources)} bright sources") - - # Save filtered results - result_path = "./bright_sources.fits" - bright_sources.write(result_path, overwrite=True) - - # Upload results - result_vospace = catalog_path.replace(".fits", "_bright.fits") - client.copy(result_path, result_vospace) - - # Cleanup - os.remove(local_path) - os.remove(result_path) -``` - - -### With Batch Jobs - -```bash -#!/bin/bash -# Batch job script using Vault and ARC via VOSpace API - -# Authenticate -cadc-get-cert --cert ~/.ssl/cadcproxy.pem - - -# Download input data from Vault -vcp vos:[project]/input/data.fits ./input.fits -# Download input data from ARC -vcp arc:projects/[project]/input/data.fits ./input_arc.fits - -# Process data -python analysis_script.py input.fits output.fits - -# Upload results to Vault -vcp output.fits vos:[project]/results/processed_$(date +%Y%m%d).fits -# Upload results to ARC -vcp output.fits arc:projects/[project]/results/processed_$(date +%Y%m%d).fits - -# Cleanup -rm input.fits input_arc.fits output.fits -``` - - -## Troubleshooting - -### Common Issues - -**Authentication Problems:** -```bash -# Refresh certificate -cadc-get-cert --cert ~/.ssl/cadcproxy.pem - -# Check certificate validity -cadc-get-cert --cert ~/.ssl/cadcproxy.pem --days-valid -``` - -**Network Timeouts:** -```python -# Increase timeout for large files -import vos - -client = vos.Client() -client.timeout = 300 # 5 minutes -``` - - -**Permission Errors:** -```bash - -# Check file permissions in Vault -vls -l vos:[user]/file.fits -# Check file permissions in ARC -vls -l arc:home/[user]/script.py - -# Check directory access in Vault -vls vos:[project]/ -# Check directory access in ARC -vls arc:projects/[project]/ -``` + remote.get_file("/project/catalog.fits", "/scratch/catalog.fits") +finally: + remote.close() +``` + +CANFAR resolves the Storage Identifier to its service endpoint and parent +Identity Provider, materializes the saved Authentication Record, and returns a +normal fsspec filesystem. There is no public `storage.configure()`, dynamic +`from canfar.storage import vault` attribute, or runtime `vault://`/`arc://` +registration. The private source factories used by `canfar data` are not part of +the Python API. + +## Files, ranges, and staging + +`vosfs` exposes standard fsspec methods. Explicit `cat_file(path, start, end)` +and `cat_ranges(...)` may use server-side byte ranges when the negotiated data +endpoint returns a valid `206` response. If the backend returns a complete +response, `vosfs` falls back to a whole-object read and slices it. This preserves +correctness but not range efficiency. + +An `open(path, "rb")` handle is a seekable staged file. It transfers the complete +object before a scientific library reads it. Do not infer range efficiency from +the fact that Astropy, NumPy, pandas, h5py, or Xarray accepts a file-like object. +For random access or memory mapping, stage once to `/scratch` and pass the local +path to the library. For chunked Zarr data, use the fsspec mapper and choose +chunks that suit the service; a block cache does not add ranges to a staged +file object. + +Measured CADC behaviour is deployment-specific: Vault/minoc accepted validated +ranges, while ARC/Cavern returned whole objects. A federated or custom Storage +Identifier can differ. The stable contract is response-driven fallback, not a +guarantee based on the identifier name. + +## Lifetime and permissions + +VOSpace retention, quotas, sharing, and permissions are service and project +policies. Confirm the destination and access rights before a large transfer. +Store reproducible results in `/arc` or a persistent VOSpace location, not in +`/scratch`. A cache under `/scratch` is ephemeral; a cache under `/arc` is a +separate local copy whose freshness and cleanup are your responsibility. + +Close the Python filesystem when the workflow is finished. Do not keep a +filesystem or open handle alive across a Session shutdown or pass an open +filesystem into a worker process; reconstruct it in the worker with the same +Storage Identifier and usable credentials. + +## Related guides + +- [Storage overview](index.md) +- [Filesystem and Python tools](filesystem.md) +- [Data transfers](transfers.md) +- [Permissions](../permissions.md) diff --git a/docs/platform/stylesheets/extra.css b/docs/platform/stylesheets/extra.css deleted file mode 100644 index 61f3c58d..00000000 --- a/docs/platform/stylesheets/extra.css +++ /dev/null @@ -1,46 +0,0 @@ -/* Mobile navigation improvements */ - -/* Show hamburger menu on tablets and smaller screens */ -@media screen and (max-width: 1440px) { - .md-header__button.md-icon[for="__drawer"] { - display: block !important; - color: var(--md-primary-bg-color); - } -} - -/* Ensure mobile navigation works properly */ -@media screen and (max-width: 76.1875em) { - .md-header__button.md-icon { - display: block !important; - } - - .md-nav__toggle { - display: block !important; - } - - .md-nav--primary .md-nav__list { - display: block; - } -} - -/* Force hamburger menu on medium screens (tablets) */ -@media screen and (max-width: 1200px) { - .md-header__button { - display: block !important; - } - - .md-nav__toggle ~ .md-nav { - display: block; - } -} - -/* Make sure tabs still work on larger screens */ -@media screen and (min-width: 1441px) { - .md-tabs { - display: block; - } - - .md-header__button.md-icon[for="__drawer"] { - display: none; - } -} diff --git a/docs/platform/support/faq.md b/docs/platform/support/faq.md index e5244170..d5053354 100644 --- a/docs/platform/support/faq.md +++ b/docs/platform/support/faq.md @@ -1,390 +1,120 @@ -# Frequently Asked Questions +# Frequently asked questions -!!! abstract "🎯 Quick Navigation" - **Find answers by topic:** - - - **[Platform Questions](#platform)**: Getting started, sessions, storage, performance - - **[Session Resources](#session-resources)**: Resource management and optimisation - - **[Client Questions](#client)**: Python client automation and REST API usage - - **[CLI Questions](#cli)**: Command-line interface and authentication - - **[Troubleshooting](#troubleshooting)**: Common problems and solutions +## What is the CANFAR Science Platform? -This unified FAQ covers the CANFAR Science Platform across all areas: Platform, Client, and CLI usage. +CANFAR provides interactive and unattended computing Sessions for astronomy, +with access to project storage, configured VOSpace Services, and published +Container Images. The available Servers, images, resources, and Storage +Identifiers depend on the deployment and your account. -## Platform +## How do I get access? -### What is the CANFAR Science Platform? -The CANFAR Science Platform is a national cloud computing environment tailored for astronomy. It provides interactive notebooks and desktops, browser-native visualization (e.g., CARTA, Firefly), user-contributed web applications, batch jobs, and direct access to CADC data holdings. +You need a CADC account and access to a Science Platform Server. Request a +[CADC account](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/auth/request.html) +if needed, then ask the platform or project administrator for CANFAR and group +access. See [Getting started](../get-started.md). -### Who can use it and what does it cost? -CANFAR is free for astronomical research. Canadian astronomers and their collaborators can use it subject to fair‑use and allocation limits. For larger needs, request additional resources via the Digital Research Alliance of Canada Resource Allocation Competition. +## How do I log in and select a server? -### How do I get access? -1. To start, you must have a CADC account. If you don't have a CADC account, you can request one at: [https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/auth/request.html](https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/auth/request.html) -2. To get CANFAR access, send a short e-mail to support@canfar.net with a short note about who you are, your research and what you plan to do with CANFAR. Include your CADC username. Turnaround is typically 1-2 business days. This can be done in parallel with requesting a CADC account. -3. Alternatively, if you already have a CADC account and you are part of a research group that is already using CANFAR, you can ask your PI to add you to the appropriate project groups. - -### What session types are available and when should I use them? -- Notebook: Jupyter Lab for interactive analysis and prototyping. -- Desktop: Full Linux desktop for GUI workflows and multi‑app sessions. -- Firefly: Interactive database access and visualization. -- CARTA: Specialised image/cube visualisation. -- Headless: Non‑GUI batch processing and automation. - -### How long can sessions run? -- Interactive sessions: up to 7 days of continuous runtime, with auto‑shutdown after prolonged inactivity; resumable if not deleted. -- Batch jobs: no strict time limit; queue priority depends on resource usage. - -### Can I run GPU‑accelerated workloads? -Yes. Request NVIDIA GPUs in the session configuration from the command line or the API. Ensure your chosen container supports CUDA libraries (i.e. `astroml-cuda`) - - -### How much storage do I get and where should I put data? - -- Personal: `/arc/home/[user]/` (typically 10 GB). -- Project/group: `/arc/projects/[project]/` (hundreds of GB to TBs, varies by project). If you do not have a project space, request one by emailing support@canfar.net. -- Temporary: `/scratch/` inside sessions (cleared when the session ends). - -Suggested layout: - -- Raw data: `/arc/projects/[project]/raw/` -- Working data: `/arc/projects/[project]/working/` -- Data: `/arc/projects/[project]/data/` -- Results: `/arc/projects/[project]/results/` -- Scripts: `/arc/projects/[project]/scripts/` - - -### How do I transfer large datasets? - -- For files <1 GB, the Science Portal file manager is convenient. - -or the VOSpace client `vcp` from the `vos` python package: - -- For larger transfers, you can use `sshfs`: - - ```bash - sshfs -o reconnect,ServerAliveInterval=15,ServerAliveCountMax=10,defer_permissions -p 64022 [user]@ws-uv.canfar.net:/ $HOME/arc - cp largedata.fits $HOME/arc/projects/[project]/data/ - ``` - -- Or use the VOSpace client `vcp` from the `vos` python package: - - ```bash - # to vault VOSpace for long-term access - vcp largefile.fits vos:[project]/data/ - # to arc file system (and VOSpace) for short-term access - vcp largefile.fits arc:[project]/data/ - ``` - - -### What software/containers are available? - -Containers include general astronomy stacks (AstroPy ecosystem), Jupyter, data science tools, machine learning libraries, full Linux desktops, and specialised astronomy tools (CASA, CARTA, DS9, TOPCAT). You can also build and use custom containers. See the Container Guide at `platform/containers/index.md`. - - -### Can I install additional software? - -- `pip install --user ...` (the `--user` option may not be necessary depending on container) or within environments. Software will be persisted on `/arc` -- Permanent: build a custom container with your required stack (see `platform/containers/index.md`). Software will be persisted on the container. - - -### Collaboration and sharing - -- Share data via project groups and `/arc/projects/[project]/` or in vault VOSPace with appropriate permissions. -- Share container images via the Harbor registry on images.canfar.net -- Share code via Git and group storage; document workflows. - -### Troubleshooting slow or failing sessions -- **Resource constraints**: Try flexible mode (default) for faster scheduling, or use fixed mode with fewer cores/less RAM if needed. Consider different times of day when cluster load varies. -- **Variable performance in flexible mode**: This is normal - performance adapts to cluster load. For consistent performance, use fixed mode with specific resource values. -- **Container issues**: Verify name/version; try a maintained baseline image. -- **Account/group issues**: Confirm group membership and active account status. -- **Performance optimization**: Process data in fast scratch (e.g., `/scratch/`), parallelize where appropriate, monitor with `htop`, `df -h`, `iotop`. - -### Getting help and community -- Documentation: start at `platform/index.md`. -- Help & Support: `platform/support/index.md` (how to contact support and what to include). -- Community: Discord for Q&A and announcements; workshops and office hours are announced there. - -## Session Resources - -### Why is my session performance variable? -If you're using flexible mode (the default), performance variation is normal and expected. Your session can use more resources when the cluster has capacity available, but may use fewer resources during peak usage times. This adaptive behaviour allows for better overall cluster utilization. - -For consistent performance, use fixed mode by specifying exact `--cpu` and `--memory` values (CLI) or `cores` and `ram` parameters (Python API). - -## Client - -### Can I automate session management with the Python client? -Yes. The Python client supports creating, monitoring, and cleaning up sessions programmatically. - -Example: -```python -import time -from canfar.sessions import Session - -session = Session() -ids = session.create( - name="automated", - image="skaha/astroml:latest", - kind="headless", - cmd="python", - args="script.py", -) - -while session.info(ids)[0]["status"] not in {"Succeeded", "Failed", "Error"}: - time.sleep(60) - -session.logs(ids, verbose=True) -session.destroy(ids) -``` - -### How do I call the REST API directly? -You can use REST endpoints for jobs and sessions if you prefer low‑level control. - -Example: -```python -from canfar.sessions import Session - -session = Session() -job_ids = session.create( - name="automated-analysis", - image="images.canfar.net/skaha/astroml:latest", - cores=4, - ram=16, - kind="headless", - cmd="python", - args="/arc/projects/[project]/scripts/analyze.py", -) +```bash +canfar login cadc +canfar server ls +canfar server use SERVER_NAME +canfar config get active.server ``` -### Authentication options for programs -- X.509 certificates (typical for many users). -- OIDC tokens via SRCNet for advanced and cross‑site workflows. See `cli/authentication-contexts.md` for options and flows. - - -## CLI - - -### How do I authenticate? +`canfar auth` manages saved Authentication Records and `canfar server` manages +the active Server. The removed `canfar context` command is not part of the +current CLI. -- Certificates: +## Which Session Kind should I use? - ```bash - cadc-get-cert -u [user] - # enter CADC password when prompted - ``` +- `notebook` for interactive Python and exploratory work; +- `desktop` for a graphical Linux environment; +- `carta` for image and cube visualisation; +- `firefly` for supported browser-based astronomy applications; and +- `headless` for scripts, reductions, and parameter sweeps. -- OIDC (SRCNet‑aware): +Use `canfar image ls --kind KIND` to see images available for a kind. See +[Sessions](../sessions/index.md). - ```bash - canfar login srcnet - ``` +## What does `Pending` mean? -Certificates typically last ~10 days; renew as needed. - - -### How do I check platform status and quotas from the CLI? +`Pending` means the platform accepted the request but has not made the Session +ready. Admission, requested resources, image pulling, or initialization can +all occur while a Session is Pending. `canfar create` returns accepted IDs; it +does not wait for `Running`. ```bash +canfar ps --all +canfar info SESSION_ID +canfar events SESSION_ID canfar stats ``` +Do not assume that headless work has priority over interactive work. Queue +policy is deployment-owned. See [batch processing](../sessions/batch.md). -### Why is my session stuck in "Pending"? +## What does `canfar create` print? -Possible reasons: insufficient resources, image issues, quota limits, or maintenance windows. Inspect events: +Human mode reports the accepted Session ID. JSON and YAML modes emit the raw +list of returned IDs, suitable for a script: ```bash -canfar events +canfar create headless IMAGE_NAME --output json -- python run.py ``` +Put CLI options before the `--` delimiter; everything after it is the command +given to the container. A partial result contains the IDs accepted by the +platform. An empty result indicates a creation or transport failure, not a +queued Session. -### I can’t connect to my session URL - -1. Ensure the session is Running (`canfar ps`). -2. Check for VPN/firewall interference. -3. Try another browser or clear cache/private mode. - - -### Can I run multiple sessions at once? - -Yes. You can run multiple sessions concurrently subject to fair‑use and any configured limits per session type. Prefer batch/headless for automation. +## Where should I put data? +- `/arc/home//` for personal persistent files; +- `/arc/projects//` for project files, when that path is available; +- a configured VOSpace Service for persistent remote data; and +- `/scratch` for temporary, high-speed staging only. -### Where can I find more CLI help? +Use explicit `IDENTIFIER:/path` operands with `canfar data`. See [Storage](../storage/index.md) +and [Data transfers](../storage/transfers.md). -- Quick start: `cli/quick-start.md` -- Auth contexts: `cli/authentication-contexts.md` -- Command reference: `cli/cli-help.md` +## How do I use storage from Python? -## 🔧 Troubleshooting +The public API uses explicit identifiers: -### Common Platform Issues - -#### Sessions won't start or take too long to launch - -**Possible causes:** - -- High cluster usage during peak hours -- Resource requirements too high -- Container image issues -- Insufficient group permissions - -**Solutions:** - -- Try flexible mode for faster scheduling -- Reduce CPU/memory requirements -- Use off-peak hours (evenings, weekends) -- Verify container image name and availability -- Check group membership for required projects - -#### Can't access files or storage - -**Possible causes:** - -- Incorrect file paths -- Missing group permissions -- Storage quota exceeded -- Network connectivity issues - -**Solutions:** - -- Verify file paths: `/arc/home/[user]/` vs `/arc/projects/[project]/` -- Check group membership with project administrators -- Clean up old files to free space -- Use `ls -la` and `getfacl` to check permissions - -#### Performance is slow or variable - -**In flexible mode (default):** - -- Performance varies with cluster load (this is normal) -- More resources available during off-peak hours -- Better overall cluster utilisation - -**For consistent performance:** - -- Use fixed mode with specific CPU/memory values -- Consider batch jobs for large processing tasks -- Use `/scratch/` storage for temporary files - -#### Browser or interface problems - -**Symptoms:** - -- Interface won't load -- Features don't work properly -- Connection timeouts - -**Solutions:** - -- Use Chrome or Firefox (recommended browsers) -- Clear browser cache and cookies -- Try incognito/private mode -- Disable ad blockers for canfar.net -- Check network connection stability - -### Authentication and Access Issues - -#### Certificate problems - -**Symptoms:** - -- Can't login or authenticate -- "Certificate expired" errors -- CLI commands fail with auth errors - -**Solutions:** - -```bash -# Renew certificate -cadc-get-cert -u [username] - -# Check certificate status -cadc-get-cert --days-valid - -# For OIDC authentication -canfar login srcnet +```python +from canfar.storage import filesystem, identifiers + +print(identifiers()) +remote = filesystem("IDENTIFIER") +try: + print(remote.ls("/path")) +finally: + remote.close() ``` -#### Permission denied errors - -**Symptoms:** - -- Can't access project directories -- File operation failures -- Session creation blocked - -**Solutions:** - -- Verify group membership with project PI -- Check project access through Science Portal -- Ensure account is active and in good standing -- Contact support if permissions seem incorrect - -## 🆘 Getting Help - -### When to Use Each Support Channel - -#### Use Discord for - -- Quick questions with fast community response -- Sharing tips and tricks with other users -- General platform discussions -- Finding collaborators - -#### Use GitHub Issues for - -- Bug reports with reproducible problems -- Feature requests and suggestions -- Documentation improvements -- Technical discussions - -#### Email Support for - -- Account access problems -- Resource allocation requests -- Data recovery needs -- Complex technical issues -- Security concerns - -### Before Contacting Support - -1. **Check this FAQ** for common solutions -2. **Search existing issues** on GitHub -3. **Try basic troubleshooting steps** (restart browser, clear cache) -4. **Gather diagnostic information** using commands in [index.md](index.md#gather-information-before-asking-for-help) - -### What to Include in Support Requests - -**Essential information:** - -- CANFAR username -- Date/time of issue -- Session type and container used -- Browser and version -- Complete error messages -- Steps to reproduce the problem - -**Helpful additional details:** +`local` is the machine running the Python process. CANFAR does not register a +dynamic `vault://` or `arc://` protocol and there is no `storage.configure()` +call. See [Filesystem and Python tools](../storage/filesystem.md). -- Screenshots of error screens -- Session IDs for failed jobs -- File paths for access issues -- What you've already tried +## Can I use a GPU or request more resources? -### Response Time Expectations +Use the resource options supported by the current `canfar create --help` and +the image's requirements. A larger fixed request can wait longer for matching +capacity. Record the image and resource request with a reproducible workflow. +Ask the platform operator about deployment-specific GPU availability. -| Issue Type | Response Time | Examples | -|------------|---------------|----------| -| **Critical** | Same day | System outages, data loss, security | -| **High** | 1-2 business days | Session failures, access problems | -| **Normal** | 2-3 business days | General questions, how-to requests | -| **Low** | 3-5 business days | Feature requests, documentation | +## How do I build or publish a Container Image? -### Community Resources +Start with [Container Images](../containers/index.md) and the registry +instructions for your project. Use a stable tag or digest and never store +credentials in an image layer. A pushed image is usable only where the target +Science Platform Server can pull it. -- **[Discord Community](https://discord.gg/vcCQ8QBvBa)**: Active community chat +## How do I get help? -Remember: The CANFAR community is here to help! Don't hesitate to ask questions, share your experiences, or contribute solutions that might help other users. +Run the checks in [Support](index.md), then email +[support@canfar.net](mailto:support@canfar.net) with a Server Name, timestamp, +exact error, and relevant Session diagnostics. Remove tokens, certificates, +passwords, and private data before sharing logs. diff --git a/docs/platform/support/index.md b/docs/platform/support/index.md index 02cfba79..dfa57150 100644 --- a/docs/platform/support/index.md +++ b/docs/platform/support/index.md @@ -1,240 +1,91 @@ -# Getting Help and Support +# Support -!!! abstract "🎯 Support Resources Overview" - **Find the help you need:** - - - **Self-service**: Documentation, troubleshooting guides, and FAQs - - **Community**: User discussions, office hours, and peer assistance - - **Direct support**: Help from CANFAR platform specialists - - **Emergency**: Rapid response for critical incidents +Start with the guide for the component that failed, then contact the service +operator with a small, reproducible diagnostic set. -The CANFAR Science Platform offers several ways to get assistance. Start with self-service resources, then move to community channels or direct support as needed. +## Check the documentation first -## 🚀 Quick Start for Support +- [Getting started](../get-started.md) covers login, server selection, and a + first Session. +- [Storage](../storage/index.md) covers Storage Identifiers, `/arc`, + `/scratch`, and Python filesystems. +- [Batch processing](../sessions/batch.md) explains `Pending`, queueing, + creation output, and headless troubleshooting. +- [Permissions](../permissions.md) explains identity, project groups, images, + and access-denied errors. +- [FAQ](faq.md) collects short answers and command examples. -### New to CANFAR? +## Fast checks -- **[Get Started Guide](../get-started.md)**: 10-minute overview -- **[First Login](../permissions.md)**: Account activation and access -- **[Choose Your Interface](../sessions/index.md)**: Pick the right session type - -### Having Problems? - -- **[FAQ](faq.md)**: Quick answers to common questions -- **[Troubleshooting](#troubleshooting)**: Diagnostic steps for common issues -- **[Contact Support](#contact-support)**: Reach the CANFAR team for help - -## 📚 Self-Help Resources - -- **Documentation search**: Use the search box or browse by topic -- **[Concepts](../concepts.md)**: Platform architecture and terminology -- **[Storage](../storage/index.md)**: Managing data effectively -- **[Containers](../containers/index.md)**: Using and building software environments -- **[Interactive Sessions](../sessions/index.md)**: Jupyter, Desktop, CARTA, Firefly -- **[Batch Jobs](../sessions/batch.md)**: Automated and large-scale processing - -## 🔧 Troubleshooting - -### Quick Checks - -1. Confirm there are no current maintenance announcements -2. Try Chrome or Firefox and clear the browser cache -3. Use a private/incognito window to rule out extensions -4. Verify your network connection is stable - -### Frequent Issues - -#### Session won't start - -- Lower memory or CPU requests and retry -- Try a different container image or launch time -- Ensure your account has the required group memberships - -#### Cannot access files +### Login or server selection ```bash -# Check locations and permissions -ls /arc/home/[user]/ -ls /arc/projects/[project]/ -ls -la /arc/projects/[project]/ -getfacl /arc/projects/[project]/ +canfar login cadc +canfar server ls +canfar server use SERVER_NAME +canfar config get active.server -o json ``` -- Confirm the path and project name -- Verify you belong to the correct project group -- Contact the project administrator if permissions are missing - -#### Performance feels slow - -- Monitor resource usage with `htop` -- Close unused applications and tabs -- Use `/scratch/` for temporary, high-I/O workloads -- Submit a support request if performance remains degraded +Use the Identity Provider (IDP) and Server Name supplied by your platform operator. +Do not paste certificates, tokens, or passwords into a support request. -#### Browser quirks +### A Session is not ready -- Stick to Chrome or Firefox and keep them updated -- Enable JavaScript and cookies for `canfar.net` -- Disable ad blockers or privacy extensions for the site - -### Gather Information Before Asking for Help - -Run these commands to capture context for a support request: +`Pending` means the platform accepted the request but has not made the Session +ready. It can include admission, resource, image-pull, or initialization work. +Inspect the Session and its events before changing the request: ```bash -# Platform status -canfar info [session-id] +canfar ps --all +canfar info SESSION_ID +canfar events SESSION_ID canfar stats - -# Session details -echo $USER -groups -env | grep -E "(CANFAR|SKAHA)" ``` -## 📧 Contact Support - -### When to Reach Out - -Email [support@canfar.net](mailto:support@canfar.net) when you encounter: - -- **Account issues**: Login failures, certificate problems, group membership -- **Technical problems**: Persistent errors, failed sessions, system outages -- **Data concerns**: Missing files, data corruption, recovery requests -- **Resource changes**: Requests for additional storage, CPU, or RAM -- **Software help**: Complex installations or container customization - -### What to Include - -Provide clear, specific details to speed up triage: - -- **Subject**: Short summary of the problem -- **Contact**: CANFAR username and email -- **Timeline**: Date and time (with timezone) when the issue occurred -- **Environment**: Session type, container, operating system, browser -- **Steps to reproduce**: Numbered list of actions leading to the issue -- **Observed vs expected**: What happened and what you expected -- **Error output**: Copy exact error text and attach screenshots when available -- **What you tried**: Mention any workarounds attempted - -### Expected Response Times - -| Priority | Response Time | Examples | -|----------|---------------|----------| -| **Critical** | Same day | System outages, data loss, security issues | -| **High** | 1–2 business days | Session failures, access problems | -| **Normal** | 2–3 business days | General questions, documentation requests | -| **Low** | 3–5 business days | Feature requests, enhancement suggestions | - -### Escalation - -If a ticket is not progressing within the expected timeframe: +`canfar logs SESSION_ID` is useful after the container has started. A Pending +Session may not have application logs yet. See [batch troubleshooting](../sessions/batch.md#monitor-and-troubleshoot). -1. Reply to the original email and add "URGENT" to the subject -2. Share any new details or screenshots gathered since the initial report -3. For emergencies, follow the contacts listed in [🚨 Emergency Contacts](#emergency-contacts) +### A data operation fails -## 👥 Community Support +Confirm the identifier and path, then check group membership with the project +administrator: -### Discord - -Join the [CANFAR Discord](https://discord.gg/vcCQ8QBvBa) for real-time conversations with other users and staff. - -- Search existing threads before posting -- Use the channel that matches your topic -- Share concise questions and relevant context -- Never publish sensitive data or credentials - -### GitHub - -Use [GitHub Issues](https://github.com/opencadc/canfar/issues) to track bugs, suggest enhancements, or contribute documentation updates. - -- Reference related documentation pages or example workflows -- Tag issues appropriately (e.g., `bug`, `documentation`, `feature-request`) -- Follow up on discussions to confirm fixes or add clarifications - -## 🐛 Helpful Bug Reports - -### Before Filing - -1. Search the documentation and [FAQ](faq.md) for related answers -2. Look for existing issues on GitHub to avoid duplicates -3. Ask quick questions on Discord if you are unsure whether something is a bug - -### What Maintainers Need - -- Clear, descriptive title -- Environment details (OS, browser, session type, container) -- Steps to reproduce, numbered and complete -- Expected result versus what actually happened -- Complete error output and supporting screenshots or logs -- Notes on any temporary workarounds you discovered - -This template can help structure a report: - -```markdown -## Bug Description -[Short summary] - -## Environment -- OS: [...] -- Browser: [...] -- Session Type: [...] -- Container: [...] - -## Steps to Reproduce -1. [...] -2. [...] - -## Expected Behaviour -[...] - -## Actual Behaviour -[...] - -## Error Messages -```text -[Paste exact text] -``` - -## Screenshots -If applicable, add screenshots to help explain the problem. - -## Additional Context -[Anything else that helps] +```bash +canfar data ls -lh IDENTIFIER:/path +canfar data info IDENTIFIER:/path/to/file +canfar data stat IDENTIFIER:/path/to/file ``` -After submitting, monitor the issue for follow-up questions, provide additional details promptly, and test proposed fixes when available. - -## 🚨 Emergency Contacts - -### System Outages - -- Planned maintenance notices go out at least 48 hours in advance via email and Discord -- For unexpected outages, email [support@canfar.net](mailto:support@canfar.net) and request a status update - -### Critical Data Issues +Use `/scratch` for temporary staged data and `/arc` or a persistent VOSpace +Service for outputs that must survive a Session. See [data transfers](../storage/transfers.md). -1. Stop affected jobs or sessions immediately -2. Document what happened and when -3. Email [support@canfar.net](mailto:support@canfar.net) with **URGENT** in the subject -4. Preserve files and logs so recovery is possible +### A browser Session does not open -Daily snapshots of `/arc/` storage are retained for 30 days; support can coordinate point-in-time recovery when necessary. +Confirm that the Session is ready with `canfar ps --all` and `canfar info +SESSION_ID`. Then retry the link in a current browser or private window. If +the Session is Running but the endpoint remains unreachable, report the +Session ID, Server Name, timestamp, and browser error to support. -### Security Incidents +## Contact CANFAR support -1. Revoke and reissue credentials right away -2. Report the incident to [support@canfar.net](mailto:support@canfar.net) -3. Describe what you observed, including timestamps and IP addresses if known -4. Follow instructions from the security team before resuming activity +Email [support@canfar.net](mailto:support@canfar.net) for account access, +project membership, persistent data errors, service outages, or a Session that +remains Pending after the platform's normal queue interval. Include: -## 📝 Contributing +- the Server Name and Identity Provider (IDP); +- the command or UI action, with paths and image references redacted as needed; +- the Session ID and status, if applicable; +- the time and timezone; +- exact error text; and +- relevant `info`, `events`, or `stats` output. -Documentation is community-driven. If you spot something to improve: +Send security vulnerabilities privately; see the [security policy](../../security.md). +Do not include passwords, tokens, certificates, or private data in email or +public issues. -1. Browse the source on [GitHub](https://github.com/opencadc/canfar) -2. Follow the contribution guidelines in `CONTRIBUTING.md` -3. Submit a pull request or open an issue describing the change +## Community and bug reports -For help getting started, ask in Discord or email [support@canfar.net](mailto:support@canfar.net). +Use the [CANFAR Discord](https://discord.gg/vcCQ8QBvBa) for community +questions and the [GitHub issue tracker](https://github.com/opencadc/canfar/issues) +for reproducible bugs or documentation changes. Search existing issues first +and include a minimal reproduction. diff --git a/docs/presentations/srcnet-june-2026-demo.typ b/docs/presentations/srcnet-june-2026-demo.typ index b7c6b534..b083e305 100644 --- a/docs/presentations/srcnet-june-2026-demo.typ +++ b/docs/presentations/srcnet-june-2026-demo.typ @@ -32,8 +32,8 @@ Ubiquitous *front door* to the CANFAR Science Platform. [ *Before — server-first* (≤ 1.3.5) ```bash - canfar auth login # deprecated - canfar context # removed + canfar login + # Then select the target Server before starting work ``` - Auth bound to a Server / context up front - Switching meant re-wiring config @@ -56,11 +56,11 @@ Ubiquitous *front door* to the CANFAR Science Platform. ```bash canfar server use sweSRC canfar create headless skaha/base-notebook:latest -- env -canfar logs $(canfar ps -a --json | jq -r ".[0].id") +canfar logs $(canfar ps -a -o json | jq -r ".[0].id") canfar server use canSRC canfar create headless skaha/base-notebook:latest -- env -canfar logs $(canfar ps -a --json | jq -r ".[0].id") +canfar logs $(canfar ps -a -o json | jq -r ".[0].id") ``` - Pick a node, browse images, launch a headless session, read its logs @@ -72,11 +72,11 @@ canfar logs $(canfar ps -a --json | jq -r ".[0].id") == Machine output you can pipe ```bash -canfar config get active.server --json -canfar open $(canfar ps --json | jq -r ".[0].id") +canfar config get active.server -o json +canfar open $(canfar ps -o json | jq -r ".[0].id") ``` -- `--json` / `--yaml` on `auth`, `server`, `ps`, `config` +- `-o/--output json|yaml` on `auth`, `server`, `ps`, `config` - Data on *stdout*, diagnostics on *stderr* → safe to pipe The same flow at package level, for notebooks and pipelines: @@ -84,10 +84,10 @@ The same flow at package level, for notebooks and pipelines: ```python from canfar import login, server, sessions login("srcnet") -for node in ["canSRC", "sweSRC"]: - server.use(node) - session = sessions.AsyncSession() - await session.create(image="skaha/base-notebook:latest", cmd="env") +async with sessions.AsyncSession() as session: + for node in ["canSRC", "sweSRC"]: + server.use(node) + await session.create(image="skaha/base-notebook:latest", cmd="env") ``` == Data, same front door diff --git a/docs/releases/2025-1.md b/docs/releases/2025-1.md index b62794e8..54bbc186 100644 --- a/docs/releases/2025-1.md +++ b/docs/releases/2025-1.md @@ -1,74 +1,31 @@ -# CANFAR Science Platform Release 2025.1 - Sept 9, 2025 +# CANFAR Science Platform 2025.1 -!!! success "CanfarSP 2025.1 - Sept 9, 2025" +**September 9, 2025** - **Dear CANFAR Community,** +This was the first production release in the 2025 platform release cycle. - We are pleased to announce a major milestone for the CANFAR Science Platform: On September 9, 2025, we completed a transition from a beta system, initially released in 2021, to our first production release, CANFAR Science Platoform 2025.1, marking the beginning of an official production release cycle. +## User-visible changes - This latest version is ready for use on www.canfar.net, and is also available for deployments to pick up across SRCNet. +- The [CANFAR Python client and CLI](../client/home.md) became the supported + interface for scripts and command-line workflows. +- Session creation exposed flexible and fixed resource requests through the + Science Portal and client interfaces. +- The portal added clearer home-directory and storage-quota information. +- CARTA 5 and Firefly workflows were available on deployments that published + the corresponding images. +- API users moving from the old `skaha/v0` endpoint needed to move to the + supported v1 endpoint or, preferably, the CANFAR client. - !!! danger "" - - If you use scripts to launch sessions on the science platform via the now deprecated [skaha python package](https://github.com/shinybrar/skaha) or with curl, please switch to the new [CANFAR Python Client or CLI](../client/home.md). If you access the API directly, please switch the reference to `skaha/v0` to `skaha/v1` as soon as possible. +## Compatibility notes - ### ✨ Highlights - - [**New & Improved** User Documentation Hub](../index.md) - - **Official Release of the CANFAR Python Client & CLI** — see [clients docs](../client/home.md) - - **Smart Session Launching** — choose between **flexible** (auto-scaling) and **fixed** modes - - **Science Portal UI Improvements** — added display for home directory & storage quota usage - - **CARTA 5.0**: latest radio astronomy visualization tool ([August 2025 Release](https://docs.google.com/document/d/1kBtYjclOn5bxlvkV5a588DtUKy3UEqPXL78IiTVAMUk/edit?tab=t.0#heading=h.9m3bw7vn40ea)) - - **Firefly**: IVOA-compliant catalog browsing and visualization platform +Use the current [Session](../platform/sessions/index.md), [batch](../platform/sessions/batch.md), +and [CLI](../cli/cli-help.md) documentation rather than copying historical +resource or status examples from this release. Image names, status values, and +deployment endpoints are not global constants. - ### 📝 Changes & Deprecations - - **Breaking Changes**: - - For API users, `headless` sessions no longer require the `type` parameter - - For Python Client & CLI users, `headless` sessions no longer require the `kind` parameter and the `headless` session `kind` will be deprecated in a future release. - - `Succeeded` status is now `Completed` for all session types, e.g. when performing a `session.info()` query. - - **Skaha API `v1` Released** — [`v0`](https://ws-uv.canfar.net/skaha/v0) API will be sunset with the next major release. Portal users are unaffected; API users should plan to migrate to `v1` as soon as possible. - - **Container Image Labels** are no longer required in the [Harbor Image Registry](https://images.canfar.net/). They are only used to populate dropdown menu options in the Science Portal UI. - - **Session Types** — launching via API, omit the `type` parameter for headless mode; interactive sessions require the `type` parameter. - - **Status Changes** — Job status `Succeeded` is now `Completed` for all session types. +## Operators - - ### 🐛 Fixes - - **Resource Monitoring** — RAM and CPU usage for sessions now display correctly in the Science Portal UI. - - ### ⚙️ Technical Changes - - CANFAR deployment requires Kubernetes v1.29 or later - - **Kueue Scheduling** — optional advanced job scheduling system that can be enabled per namespace to reduce cluster pressure and provide queue management. - - **Monitoring Fixes** — Skaha API now uses the the Job API instead of the Pod API internally to provide more accurate resource usage information. - - **Flexible** sessions use the `Burstable` Kubernetes Quality of Service (QoS) class instead of `Guaranteed`, which provides better resource efficiency on the cluster. Currently, *flexible* sessions can grow up to 8 cores and 32GB of RAM. - - Internal API's have been updated to use the `Job` API instead of the `Pod` API. This provides better resource monitoring and usage information. - - - ### 📦 Deployment Notes - - - Use the offically supported helm charts in the [opencadc/deployments](https://github.com/opencadc/deployments/tree/main/helm/applications/skaha) for CANFAR 2025.1 deployments. - - To test, profile and setup the Kueue scheduling system, see the [deployment guide](https://github.com/opencadc/deployments/tree/main/configs/kueue) for detailed instructions. - - #### Python Client & CLI - - | Component | Version | - |---------|--------------| - | canfar | [v1.0.2](https://pypi.org/project/canfar/) | - - #### Helm Charts & Container Images - - | Component | Helm Chart Version | Container Image | - |-----------|-------------------|-----------------| - | base | 0.4.0 | N/A | - | cavern | 0.7.0 | images.opencadc.org/platform/cavern:0.9.0 | - | skaha | 1.0.3 | images.opencadc.org/platform/skaha:1.0.2 | - | posix-mapper | 0.4.4 | images.opencadc.org/platform/posix-mapper:0.3.2 | - | science-portal | 1.0.0 | images.opencadc.org/platform/science-portal:1.0.0 | - | storage-ui | 0.6.0 | images.opencadc.org/client/storage-ui:1.3.0 | - - ### 💬 Contact & Support - - For any questions about this release, or for information relating to CANFAR issues or deployment support, head over to the [CANFAR Discord Server](https://discord.gg/vcCQ8QBvBa) or please contact us at [support@canfar.net](mailto:support@canfar.net). - -
-
- Built with :heart:{.heart} at CADC -
+Deployment-specific chart and service changes belong in the +[OpenCADC deployments repository](https://github.com/opencadc/deployments). +Users with questions about an installed deployment can contact +[CANFAR support](mailto:support@canfar.net). diff --git a/docs/releases/2025-2.md b/docs/releases/2025-2.md index 0f7ef1ea..7fdad848 100644 --- a/docs/releases/2025-2.md +++ b/docs/releases/2025-2.md @@ -1,54 +1,25 @@ -# CANFAR Science Platform Release 2025.2 - Nov 25, 2025 +# CANFAR Science Platform 2025.2 -!!! success "CanfarSP 2025.2 - Nov 25, 2025" +**November 25, 2025** - ### ✨ Features - - **Modified resource selection controls** - On the CANFAR Science Portal, when launching a session in Fixed-mode, there is more fine-tuned control of the resource selections. - - **Cluster-aware resource selection on CANFAR Science Portal** — Memory and Core options reflect characteristics of the underlying kubernetes cluster - - **GPU selection** — In CANFAR clusters where GPUs are available, a GPU selection option presented in the science portal +## User-visible changes - ### 🐛 Fixes - - Addresses problem with the incorrect enforcement of the maximum number of sessions allowed - - Improved accuracy of Global session statistics on CANFAR portal - - canfar CLI fixes - - Graceful Degradation: The CLI commands (canfar info, canfar ps) now continue to work even when the API returns incomplete session data, displaying partial information instead of crashing - - Better Error Reporting: Missing or invalid fields are tracked internally and can be viewed with the --debug flag for troubleshooting - - Enhanced Display: Resource usage metrics for flexible sessions is now reported with better readability - - Type Safety: Session type validation has been strengthened using Pydantic's built-in validators +- Science Portal resource controls became more aware of deployment capacity. +- Deployments with compatible hardware could expose GPU choices. +- Session listings and information commands handled incomplete platform data + more gracefully and reported clearer diagnostics with `--debug`. +- Resource usage display and validation were refined for flexible Sessions. - ### ⚙️ Technical Changes - - Cavern controlls user allocations - called by authorized clients such as skaha and prepareData - - All registry lookups benefit from registry mirrroring and failover - - ### 📦 Deployment Notes - - Deployers now specify limits of their cluster as LimitRange objects from helm charts [DONE] - - Deployers must specify properties about their cavern installation - - Ability to specify a default project for each of the harbor instances configured - - Ability to define multiple registries to support mirroring - - Use the offically supported helm charts in the [opencadc/deployments](https://github.com/opencadc/deployments/tree/main/helm/applications/skaha) for CANFAR 2025.2 deployments. +## Compatibility notes - #### Python Client & CLI +Resource choices, GPUs, image names, and project allocations remain +deployment-specific. Start with the current [Session](../platform/sessions/index.md) +and [batch](../platform/sessions/batch.md) guides, and inspect available +images with `canfar image ls`. - | Component | Version | - |---------|--------------| - | canfar | [v1.1+](https://pypi.org/project/canfar/) | - - #### Helm Charts & Container Images - - | Component | Helm Chart Version | Container Image | - |-----------|-------------------|-----------------| - | base | 0.4.0 | N/A | - | cavern | 0.9.0 | images.opencadc.org/platform/cavern:0.9.2 | - | skaha | 1.3.2 | images.opencadc.org/platform/skaha:1.1.7 | - | posix-mapper | 0.5.0 | images.opencadc.org/platform/posix-mapper:0.3.2 | - | science-portal | 1.1.2 | images.opencadc.org/platform/science-portal:1.2.5 | - | storage-ui | 0.8.0 | images.opencadc.org/client/storage-ui:1.4.1 | +## Operators - ### 💬 Contact & Support - - For any questions about this release, or for information relating to CANFAR issues or deployment support, head over to the [CANFAR Discord Server](https://discord.gg/vcCQ8QBvBa) or please contact us at [support@canfar.net](mailto:support@canfar.net). - -
-
- Built with :heart:{.heart} at CADC -
+Chart values and service deployment changes are maintained in the +[OpenCADC deployments repository](https://github.com/opencadc/deployments), not +in the user documentation. Contact [CANFAR support](mailto:support@canfar.net) +for a problem on an installed deployment. diff --git a/docs/releases/2026-1.md b/docs/releases/2026-1.md index c151d8af..a63d299f 100644 --- a/docs/releases/2026-1.md +++ b/docs/releases/2026-1.md @@ -1,74 +1,28 @@ -# CANFAR Science Platform Release 2026.1 - February 26, 2026 +# CANFAR Science Platform 2026.1 -!!! success "CanfarSP 2026.1 - February 26, 2026" +**February 26, 2026** - ### ✨ Features - - Updated **Desktop** and **Desktop Terminal** — The CANFAR Desktop session and the associated Desktop Terminal (1.2.0) have been updated and refreshed. The previous version of the Terminal (1.1.2) is still available from the drop-down menu under: Applications → AstroSoftware → skaha +## User-visible changes - - In the **CANFAR CLI** and **Python Library** - - **Active Server Context** — Every `canfar` cli command now shows the active server context (`@`) in the CLI output. - - **New CLI Commands** — `canfar config get/set` to retrieve and update configuration values with dotted paths with editing YAML by hand. For example: - - `canfar config get console.width` -- get the current console width - - `canfar config get contexts.default.server` -- get the default server configuration - - `canfar config set console.width 160` -- set the console width to 160 characters - - **Configurable CLI Width** — control table and log/event output width via configuration parameter `console.width` which can be set via `canfar config set console.width 160`. - - **Regex-aware prune** — `canfar prune` now accepts literal prefixes or regex patterns for bulk cleanup. +- Desktop and Desktop Terminal images were refreshed on deployments that + adopted the release. +- CVMFS support became available to deployments that mounted the Alliance + software repositories; see [CVMFS](../platform/cvmfs.md). +- New contributed applications could be published by a deployment. +- `canfar open` no longer tries to open a `Pending` Session; inspect its + status and events until it is ready. +- Client timeout and HTTP-error messages improved for synchronous, asynchronous, + and replica creation workflows. - - **CVMFS** enabled — All sessions and jobs now have access to the CernVM File System, providing instant access to software stacks maintained by the **Digital Research Alliance of Canada (Alliance)**. [More on using CVMFS on CANFAR.](https://www.opencadc.org/canfar/latest/platform/cvmfs/) +## Compatibility notes - - Two new contributed sessions: - - **WebTerminal** — A simple but effective way to interact with your data in CANFAR, based on `ttyd` - - **Globus** — Manage CANFAR data transfers on Globus Connect Personal endpoints +The current client uses `canfar auth` and `canfar server` for saved identities +and Server selection. Use [CLI help](../cli/cli-help.md) and [client +guides](../client/home.md) for the current configuration shape rather than +historical context examples. - - **Scripting functions** now available on **CARTA sessions**. Like with Firefly sessions, you can now programatically interact with your CARTA sessions through the CARTA API. See the documentation on [Session API Access](https://www.opencadc.org/canfar/latest/platform/sessions/#api-access-to-sessions) for more information. +## Operators - ### 🐛 Fixes - - On the Science Portal, Desktop sessions won't be available until fully intialized, thus avoiding the "503 Service Unavailable" message that would occur for about 30 seconds. - - An issue was fixed where, in the SRCNet deployments, the incorrect userid was sometimes shown on the Science Portal. - - - CANFAR CLI and Python Library bug fixes: - - **Pending Session Open** — `canfar open` now skip `Pending` sessions and warn until they are `Running`. - - **Improved Timeout + HTTP Error Messages** — clearer diagnostics for stalled or failing requests (sync and async clients), including replica creation workflows. - - **Logging fallback** — CLI logging falls back to a temp directory when the default log path is not writable. - - `canfar ps` reporting correct values and match with those reported by top - - ### ⚙️ Technical Changes - - **Backwards Compatability Note: Removal of the skaha API 'v0' endpoint** — The v0 endpoint of skaha was replaced by v1 in the CanfarSP 2025.1 release of September 2025. v0 continued in parallel with v1 for the last few months, but has now been removed. This change only affects people with clients or scripts that communicate with the API directly (e.g. with curl), not those using the Science Portal or the canfar CLI and python package which all use v1. If there are any remaining users of the old skaha python package (that uses v0), they should update their workflows with the new canfar package: `pip install --upgrade canfar` - - - CANFAR CLI and Python Library: - - **Destroy-with matching** — `destroy_with` uses regex when metacharacters are present, otherwise a literal prefix. - - **Shared console output** — CLI commands now use a single Rich console configured from `canfar.utils.console`, and always shows the active server context. - - **Registry discovery updates** — expanded SRCNet registry names/endpoints (Australia, SKAO, Italy-CINECA) and excluded preprod endpoints. - - ### 📦 Deployment Notes - - Initial values for flexible jobs now configurable in the skaha helm charts - - Logos on the CANFAR Portal can be customized. SRCNet nodes can brand their portals with the national logos. - - Removal of the skaha/v0 API endpoint - - Official CVMFS support (must also be supported in your kubernetes infrastructure). - - The CANFAR Portal reflects image registry options when multiple are configured - - #### Python Client & CLI - - | Component | Version | - |---------|--------------| - | canfar | [v1.2+](https://pypi.org/project/canfar/) | - - #### Helm Charts & Container Images - - | Component | Helm Chart Version | Container Image | - |-----------|-------------------|-----------------| - | base | 0.4.0 | - | - | cavern | 0.9.0 | 0.9.2 | - | skaha | 1.5.0 | 1.1.7 | - | posix-mapper | 0.5.0 | 0.3.2 | - | science-portal | 1.2.0 | 1.2.6 | - | storage-ui | 0.9.0 | 1.4.3 | - - ### 💬 Contact & Support - - For any questions about this release, or for information relating to CANFAR issues or deployment support, head over to the [CANFAR Discord Server](https://discord.gg/vcCQ8QBvBa) or please contact us at [support@canfar.net](mailto:support@canfar.net). - -
-
- Built with :heart:{.heart} at CADC -
+Use the [OpenCADC deployments repository](https://github.com/opencadc/deployments) +for deployment instructions. Report an installed-deployment problem to +[CANFAR support](mailto:support@canfar.net). diff --git a/docs/releases/2026-2.md b/docs/releases/2026-2.md index e6b53213..1d406af5 100644 --- a/docs/releases/2026-2.md +++ b/docs/releases/2026-2.md @@ -1,167 +1,47 @@ -# CANFAR Science Platform Release 2026.2 -## June 11, 2026 - -!!! success "CSP 2026.2" - - ### **📢 Upcoming Infrastructure Upgrade** - - Over the next few months, the CANFAR Science Platform is moving to new hardware. We will transition all users from our existing compute and storage infrastructure to a new, faster, and more reliable cluster. - - **What is changing?** - - - **Compute** — sessions and batch jobs will run on new hardware with faster CPUs, more memory per node, and improved GPU availability. - - **Storage** — your home and project spaces (`/arc`) are moving to a new storage system with better performance and more capacity. - - **What stays the same?** - - - Your account, your container images, and the way you log in. - - **Your data.** Everything in home and project directories will be migrated. - - !!! info "Nothing to do yet" - - There is no action required right now. Watch your inbox over the coming weeks for the detailed migration schedule and step-by-step guidance. - - ### **✨ New Features** - - New Science Portal UI with improved mobile device experience. - - [CARTA 5.1.0](https://github.com/CARTAvis/carta/releases/tag/v5.1.0) available. - - ### **🐍 Python Client & CLI Improvements** - - !!! warning "Configuration Reset Required" - - This release moves the CANFAR client to a new revision controlled configuration format. After upgrading, you will be required to re-login, - - ```bash - rm ~/.canfar/config.yaml - canfar login cadc - ``` - - That single command authenticates you, discovers available Science Platform servers, and selects one. - - - **New login flow** — `canfar login` is now the one front door for authentication. Pass your identity provider directly, or run it bare to pick from a list: - - ```bash - # Log in to CADC directly - canfar login cadc - # Or choose your identity provider interactively - canfar login - ``` - - The same flow is available at package level for scripts and notebooks: - - ```python - import canfar - canfar.login("cadc") - canfar.server.use("canfar") - ``` - - - **Manage identities and servers** — new `canfar auth` and `canfar server` command groups let you inspect and switch authentication and servers without editing config files: - - ```bash - canfar login srcnet # Login with SRCNet - canfar auth # show active authentication - canfar auth ls # list all saved identities - canfar auth use srcnet # switch identity provider - canfar server ls # list servers for the active identity - canfar server use canSRC # pick a server - ``` - - The CLI remembers the last valid server for each identity provider, so switching back and forth is seamless. - - - **Script-friendly machine output** — `--json` and `--yaml` are now supported on `auth`, `server`, `ps` and `config` commands. Output is clean, stable-keyed data on stdout — errors and diagnostics go to stderr — so piping into tools like `jq` is safe, - - ```bash - canfar config get active.server --json - ``` - - - **Scientist session names** — sessions launched without an explicit `--name` now get memorable defaults like `einstein`, `curie`, or `ramanujan` instead of random word pairs. - - ### 🐛 Platform Fixes - - Enabled the deletion of non-empty user storage folders through the Storage UI - - CANFAR CLI and Python Library: - - The active-server banner (`@`) now appears only in human-readable output, never in JSON/YAML, keeping machine output safe to pipe. - - Documentation and docstring examples corrected to match the implemented code signatures. - - `canfar.helpers.distributed` (`chunk`, `stripe`) batch helpers documented with corrected, runnable examples. - - The **skaha** service is enjoying some optimizations that should help speed up session creation - - Now relies on the Cavern self-allocation for new users. - - Small internal optimizations where effort was previously wasted. - - **Cavern** - Now enables the ability for new users to "self-allocate". Permissions are controlled either by Group setting, or using the SRCNet Permissions API. - - See the new [Helm Chart configuration](https://github.com/opencadc/deployments/blob/main/helm/applications/cavern/values.yaml#L89) for allocation authorization. - - ### ⚙️ Technical Changes - - Science Portal UI/UX components are now based on a modern technology stack: Next.js, TypeScript, and React. This will simplify and expedite our capability to iterate and improve the portal components in the future. - - Readiness checks added for all session types. Sessions aren't put in the 'Running' state until session initialization procedures are complete. - - CARTA 5.1.0 with [psrecord](https://github.com/astrofrog/psrecord) available (Requires CanfarSP 2026.2) - - CANFAR CLI and Python Library - - !!! warning "Compatibility Notes" - - `canfar context` has been removed. Use `canfar auth`and `canfar server` commands instead. - - `canfar auth login` is deprecated and will be removed in a future release. Switch to `canfar login`. - - Persisted client configuration now uses an explicit `version: 1` schema. - - `canfar.authentication` and `canfar.server` Python modules provide noninteractive helpers for authentication and server discovery, validation, and selection in scripts. - - Runtime `token`/`certificate` skip saved-auth expiry hooks — no false `AuthExpiredError` with stale x509 config ([#115](https://github.com/opencadc/canfar/issues/115)). - - Bug Fix: Removal of `fsGroupChangePolicy` and `fsGroup` - Ensures that kubernetes will avoid performing a recursive permission modification on job startup. - - ### 📦 Deployment Notes - - Improved and simplified deployment documentation - - Complexity of matching keys between skaha and cavern no longer required - - Improved Kueue deployment documentation - - The CANFAR Portal Branding Logo is now configurable - - Deployers have the option to configure skaha and cavern to use the SRCNet Permissions API for authorization decisions. - - For updated deployment instructions, refer to the [OpenCADC Deployments repository](https://github.com/opencadc/deployments). - - Looking for a reference deployment to get past a deployment issue? You can now see what [canSRC](https://src.canfar.net/science-portal) deploys as our [ArgoCD Application](https://github.com/cadc-ccda-infra/keel-deploy/tree/main/helm/values/src.canfar.net). - - !!! warning "Potential Breaking Changes" - - - The **skaha** service is the only one that requires a Service Account as it will interact with the Kubernetes Cluster. The skaha helm chart`serviceAccountName` value will be required to be converted to the new `serviceAccount` object: - ```yaml - # Old Helm Configuration - deployment: - skaha: - serviceAccountName: skaha - # New Helm Configuration - serviceAccount: - name: skaha - create: false - ``` - - - Beware of maintaining RBAC rules through Helm Charts. Changes have been made to make the charts a little more ArgoCD friendly. - ```yaml - rbac: - create: true - clusterRole: - create: true - ``` - The `deployment.skaha.sessions.limitRange` and `deployment.skaha.sessions.kueue` also maintain their own RBAC settings. See [science platform helm chart](https://github.com/opencadc/science-platform/blob/main/helm) for full options. - - - If **Cavern** was previously installed with a local PostgreSQL in the cluster, it may now require authentication provided. This can be done with a secret, or just setup username and password. The UWS database is volatile. See [cavern helm chart](https://github.com/opencadc/deployments/blob/main/helm/applications/cavern/values.yaml#L144) for more information. - - ### 🔖 Releases - - #### Python Client & CLI - - | Component | Version | - |---------|--------------| - | canfar | [1.4+](https://pypi.org/project/canfar/) | - - #### Helm Charts & Container Images - - | Component | Helm Chart Version | Container Image | - |-----------|-------------------|-----------------| - | base | 0.4.0 | N/A | - | cavern | 0.10.0 | 0.10.0 | - | skaha | 1.6.0 | 1.3.0 | - | posix-mapper | 0.5.0 | 0.3.2 | - | science-portal | 2.0.0 | 2.0.0 | - | storage-ui | 0.9.0 | 1.4.3 | - - See [**detailed upgrade and deployment guidance**](https://www.opencadc.org/deployments/helm/#canfar-science-platform) for administrators and operators. *These docs are going through an overhaul and are expected to improve significantly over the coming months.* - - ### 💬 Contact & Support - - For any questions about this release, or for information relating to CANFAR issues or deployment support, head over to the [CANFAR Discord Server](https://discord.gg/vcCQ8QBvBa) or please contact us at [support@canfar.net](mailto:support@canfar.net). - -
-
- Built with :heart:{.heart} at CADC -
+# CANFAR Science Platform 2026.2 + +**June 11, 2026** + +## User-visible changes + +- The Science Portal UI was refreshed, including improved small-screen + behaviour. +- Readiness checks keep a Session in `Pending` until initialization is + complete; use `info` and `events` while it is waiting. +- The `canfar` login flow became the front door for identity and Server + discovery. `canfar auth` manages saved Authentication Records and + `canfar server` selects the active Server. +- Machine-readable JSON and YAML output is available on supported inspection + commands. Human-only server banners stay off data output. +- `canfar.helpers.distributed.chunk` and `.stripe` document replica-aware + partitioning through `REPLICA_ID` and `REPLICA_COUNT`. +- Runtime token or certificate credentials bypass saved-credential expiry hooks + in the client. +- Deployments that publish the relevant images can offer CARTA 5.1 and updated + contributed or desktop workflows. + +## Configuration and migration + +After upgrading a client from an older configuration format, log in again if +the client asks for it: + +```bash +canfar login cadc +canfar auth +canfar server ls +``` + +The obsolete `canfar context` command is not part of the current CLI. Use +`canfar auth` and `canfar server` instead. See [CLI help](../cli/cli-help.md) +and [client guides](../client/home.md) for the current commands. + +## Deployment notes + +Hardware, storage migration, Session images, CVMFS, GPU availability, and +resource policy are deployment-owned. A release note does not promise that a +feature is enabled on every Science Platform Server. Operators should follow +the [OpenCADC deployments repository](https://github.com/opencadc/deployments) +for upgrade instructions. + +Report problems with an installed deployment to [CANFAR +support](mailto:support@canfar.net). diff --git a/docs/releases/releases.md b/docs/releases/releases.md index 845c2e20..53f70155 100644 --- a/docs/releases/releases.md +++ b/docs/releases/releases.md @@ -1,49 +1,18 @@ -# CANFAR Releases - -!!! abstract "CanfarSP Releases" - - Jun 11, 2026 **[2026.2](2026-2.md)** - - Feb 26, 2026 **[2026.1](2026-1.md)** - - Nov 25, 2025 **[2025.2](2025-2.md)** - - Sept 9, 2025 **[2025.1](2025-1.md)** - -## 🎯 CANFAR Releases - -The CADC team releases the CANFAR Science Platform (CanfarSP) on a fixed, predictable schedule so users and deployers can plan upgrades. - -### Who is this page for? - -If you use the CANFAR Python Client or CLI: - -- Use the release notes in this section to see user-visible changes in the client, CLI, and documentation. -- Each release page focuses on what changed and how it might affect your workflows. - -If you deploy the Canfar Science Platform: - -- Use these release notes to understand user-facing changes that may drive support or rollout timing. -- Track backend deployment changes in the [opencadc/deployments](https://github.com/opencadc/deployments) repository. - -### Release Naming - -CanfarSP releases are named using the year and number, in the form `CanfarSP YYYY.MM`. For example, the first release of 2025 was named **CanfarSP 2025.1**. - -We currently aim for a CanfarSP release every 3 months. - -In the subpages of this section of documentation, detailed notes about each release can be found. - -CanfarSP releases include dedicated integration and regression testing, plus internal user testing. Generally: - -- New features can only be added to CanfarSP in a fixed cycle release -- Bug fixes are added to CanfarSP immediately and result in a patch release -- Collaborators can get access to new features prior to their target release, but only as experimental software because the features have not been thoroughly tested. - -### Roadmap - -Currently, the CanfarSP Roadmap is managed internally by CADC, though we plan on making it public in the near future. For questions or suggestions about the CanfarSP Roadmap, please reach out to the CADC team. - -### Development Contributions - -We welcome and encourage contributions to all areas of the CANFAR and OpenCADC code bases! For information on how to start, please see the contributing guidelines in these OpenCADC repositories: - -- [CANFAR Python Client, CLI, and Documentation](https://github.com/opencadc/canfar) -- [CANFAR Backend Services](https://github.com/opencadc/science-platform) -- [CANFAR Helm Charts](https://github.com/opencadc/deployments) +# CANFAR releases + +These pages summarise user-visible changes to the CANFAR Science Platform and +the `canfar` Python client and CLI. Confirm the Server, Container Image, and +deployment version with the operator of the platform you use; release notes do +not imply that every deployment exposes every feature. + +- [2026.2 — June 11, 2026](2026-2.md) +- [2026.1 — February 26, 2026](2026-1.md) +- [2025.2 — November 25, 2025](2025-2.md) +- [2025.1 — September 9, 2025](2025-1.md) + +For package-level changes after the latest platform release, see the +[`canfar` package history](https://pypi.org/project/canfar/) and the +repository [CHANGELOG](https://github.com/opencadc/canfar/blob/main/CHANGELOG.md). +Deployment operators should use the [OpenCADC deployments +repository](https://github.com/opencadc/deployments) for chart and service +upgrade instructions. diff --git a/docs/security.md b/docs/security.md index 390dbe3b..d2fe3ea9 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1 +1,13 @@ ---8<-- "SECURITY.md" \ No newline at end of file +--8<-- "SECURITY.md" +# Security + +Please report security vulnerabilities privately rather than opening a public +issue. Email `shiny.brar@nrc-cnrc.gc.ca` with a description, affected version, +reproduction steps, and any relevant logs or proof of concept. Do not include +secrets or personal data in the report. + +We will acknowledge reports and coordinate a fix and disclosure timeline. Keep +CANFAR and its dependencies up to date while a report is being assessed. + +See the repository's [security policy](https://github.com/opencadc/canfar/blob/main/SECURITY.md) +for the canonical contact and policy text. diff --git a/docs/static/logo.ico b/docs/static/logo.ico deleted file mode 100644 index 24f87e4c..00000000 Binary files a/docs/static/logo.ico and /dev/null differ diff --git a/docs/stylesheets/overrides.css b/docs/stylesheets/overrides.css deleted file mode 100644 index 502cde40..00000000 --- a/docs/stylesheets/overrides.css +++ /dev/null @@ -1,12 +0,0 @@ -/* Admonition title weight */ -.md-typeset .admonition-title { font-weight: 600; } - -/* Compact code blocks */ -.md-typeset pre > code { line-height: 1.3; } - -/* Simple badges */ -.badge { display: inline-block; padding: .1rem .4rem; border-radius: .25rem; font-size: .75rem; } -.badge.success { background: #2e7d32; color: #fff; } -.badge.warning { background: #f9a825; color: #000; } -.badge.info { background: #0288d1; color: #fff; } - diff --git a/mkdocs.yml b/mkdocs.yml index a7df070d..3cd2fad6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -93,6 +93,14 @@ plugins: branch: main - termynal +# Dated engineering records stay in the repository for maintainers but are not +# user-facing site pages. Excluding them also keeps MkDocs' unlisted-page +# warning focused on accidental navigation omissions. +exclude_docs: | + agents/adrs/** + agents/research/** + agents/reviews/** + extra_css: - stylesheets/extra.css @@ -200,33 +208,7 @@ nav: - Community: - Overview: platform/community/index.md - CASA: platform/community/CASA_and_more.md - # - ALMA: - # - Overview: platform/community/alma/index.md - # - Desktop: - # - Archive Download: platform/community/alma/ALMA_Desktop/archive_download.md - # - Archive Script Download: platform/community/alma/ALMA_Desktop/archive_script_download.md - # - CASA Containers: platform/community/alma/ALMA_Desktop/casa_containers.md - # - Start CASA: platform/community/alma/ALMA_Desktop/start_casa.md - # - Typical Reduction: platform/community/alma/ALMA_Desktop/typical_reduction.md - # - General Tools: - # - File Transfers: platform/community/alma/General_tools/File_transfers.md - # - Group Management: platform/community/alma/General_tools/Group_management.md - # - SSHFS: platform/community/alma/General_tools/Using_sshfs.md - # - VOS Tools: platform/community/alma/General_tools/Using_vostools.md - # - Web Storage: platform/community/alma/General_tools/Using_webstorage.md - # - New Users: - # - Overview: platform/community/alma/NewUser/Overview.md - # - Login: platform/community/alma/NewUser/Login.md - # - Launch CARTA: platform/community/alma/NewUser/LaunchCARTA.md - # - Launch Desktop: platform/community/alma/NewUser/LaunchDesktop.md - # - Launch Notebook: platform/community/alma/NewUser/LaunchNotebook.md - # - Project Space: platform/community/alma/NewUser/ProjectSpace.md - # - Notebook: - # - Transfer Files: platform/community/alma/Notebook/transfer_file.md - # - Tips: - # - Direct URL: platform/community/alma/TipsTricks/Direct_url.md - # - Increase Font: platform/community/alma/TipsTricks/Increase_font.md - # - Clipboard: platform/community/alma/TipsTricks/Using_clipboard.md + - ALMA: platform/community/alma/index.md - Support: - Help: platform/support/index.md - FAQ: platform/support/faq.md diff --git a/pyproject.toml b/pyproject.toml index 3dbf489a..347b1fd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,20 +61,17 @@ dependencies = [ "typer>=0.16.0", "vosfs @ git+https://github.com/shinybrar/vosfs@v0.8.0", "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.7.0#subdirectory=src/fsspec-cli", + "cryptography>=42.0.0", ] -[tool.uv] [dependency-groups] dev = [ - "ipython>=8.37.0", "pre-commit>=4.2.0", "pytest>=8.3.5", "pytest-asyncio>=1.0.0", "pytest-cov>=6.1.1", - "pytest-order>=1.3.0", "pytest-xdist>=3.7.0", "ruff>=0.11.11", - "tomli>=2.0.1; python_version < '3.11'", "ty>=0.0.51", ] docs = [ @@ -99,7 +96,6 @@ canfar = "canfar.cli.main:main" # Ruff configuration - handles formatting, linting, and import sorting [tool.ruff] target-version = "py39" -line-length = 88 exclude = [ ".bzr", ".direnv", @@ -223,16 +219,9 @@ convention = "google" max-args = 8 [tool.ruff.format] -quote-style = "double" -indent-style = "space" -skip-magic-trailing-comma = false -line-ending = "auto" docstring-code-format = true -docstring-code-line-length = "dynamic" # ty configuration (replaces mypy) -[tool.ty] - [tool.ty.rules] unused-ignore-comment = "ignore" @@ -256,7 +245,6 @@ addopts = [ "--cov-report=xml", "--cov-fail-under=75", "-n=auto", # Enable parallel execution with pytest-xdist - "--dist=loadfile", # Use loadfile distribution to ensure tests in same file run in same worker (required for pytest-order) ] testpaths = ["tests"] asyncio_mode = "auto" @@ -264,7 +252,6 @@ markers = [ "integration: marks tests as integration tests (deselect with '-m \"not integration\"')", "unit: marks tests as unit tests", "slow: marks tests as slow (deselect with '-m \"not slow\"')", - "order: marks tests that need to run in a specific order (will run sequentially)", ] filterwarnings = [ "error", @@ -296,37 +283,16 @@ exclude_lines = [ "pass", "@(abc\\.)?abstractmethod", "class .*\\bProtocol\\):", - "@(abc\\.)?abstractmethod", - "raise NotImplementedError", "if 0:", "if False:", - "if TYPE_CHECKING:", ] show_missing = true -skip_covered = false precision = 2 -[tool.coverage.html] -directory = "htmlcov" - -[tool.coverage.xml] -output = "coverage.xml" - [tool.interrogate] ignore-init-method = true -ignore-init-module = false -ignore-magic = false -ignore-semiprivate = false -ignore-private = false -ignore-property-decorators = false -ignore-module = false -ignore-nested-functions = false ignore-nested-classes = true -ignore-setters = false fail-under = 80 exclude = ["setup.py", "docs", "build"] ignore-regex = ["^get$", "^mock_.*", ".*BaseClass.*"] verbose = 3 -quiet = false -whitelist-regex = [] -color = true diff --git a/tests/conftest.py b/tests/conftest.py index b6046f78..53240c08 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,19 +8,14 @@ import pytest -ISOLATED_HOME = Path( - os.environ.get( - "CANFAR_TEST_HOME", - str(Path(tempfile.gettempdir()) / "canfar-empty-home"), - ) -) - - -def pytest_configure(config: pytest.Config) -> None: # noqa: ARG001 - """Isolate ``HOME`` so tests never consume developer configuration.""" - ISOLATED_HOME.mkdir(parents=True, exist_ok=True) - os.environ["CANFAR_TEST_HOME"] = str(ISOLATED_HOME) - os.environ["HOME"] = str(ISOLATED_HOME) +_REQUESTED_TEST_HOME = os.environ.get("CANFAR_TEST_HOME") +if _REQUESTED_TEST_HOME is None: + _REQUESTED_TEST_HOME = tempfile.mkdtemp(prefix="canfar-test-home-") + +ISOLATED_HOME = Path(_REQUESTED_TEST_HOME) +os.environ["CANFAR_TEST_HOME"] = _REQUESTED_TEST_HOME +os.environ["HOME"] = _REQUESTED_TEST_HOME +ISOLATED_HOME.mkdir(parents=True, exist_ok=True) @pytest.fixture(autouse=True) diff --git a/tests/test_alma_docs.py b/tests/test_alma_docs.py new file mode 100644 index 00000000..c9e9474c --- /dev/null +++ b/tests/test_alma_docs.py @@ -0,0 +1,23 @@ +from pathlib import Path + +ALMA_GUIDE = ( + Path(__file__).parents[1] / "docs" / "platform" / "community" / "alma" / "index.md" +) +ALMA_DOCS = ALMA_GUIDE.parent + + +def test_alma_guide_is_a_text_first_workflow() -> None: + text = ALMA_GUIDE.read_text() + + for heading in ( + "## Prerequisites", + "## Workflow", + "## Expected results", + "## Troubleshooting", + ): + assert heading in text + + assert "canfar login cadc" in text + assert "canfar create" in text + assert "canfar data cp" in text + assert all("![" not in page.read_text() for page in ALMA_DOCS.rglob("*.md")) diff --git a/tests/test_async_session.py b/tests/test_async_session.py deleted file mode 100644 index bfbe9cf2..00000000 --- a/tests/test_async_session.py +++ /dev/null @@ -1,429 +0,0 @@ -"""Test the async session.""" - -from asyncio import sleep -from time import time -from unittest.mock import AsyncMock, MagicMock, patch -from uuid import uuid4 - -import httpx -import pytest -from pydantic import SecretStr, ValidationError - -from canfar.sessions import AsyncSession - - -@pytest.fixture(scope="module") -def name(): - """Return a random name.""" - return str(uuid4().hex[:7]) - - -@pytest.fixture -def asession(): - """Test images.""" - return AsyncSession() - - -@pytest.mark.asyncio -@pytest.mark.integration -@pytest.mark.slow -async def test_fetch_with_kind(asession: AsyncSession) -> None: - """Test fetching images with kind.""" - await asession.fetch(kind="headless") - - -def _sync_response(json_data=None, text: str = "") -> MagicMock: - response = MagicMock() - response.json.return_value = json_data - response.text = text - return response - - -def _http_error(method: str, url: str) -> httpx.HTTPStatusError: - request = httpx.Request(method, url) - response = httpx.Response(500, request=request) - return httpx.HTTPStatusError("boom", request=request, response=response) - - -@pytest.mark.asyncio -async def test_async_session_methods_handle_success_and_failures() -> None: - """AsyncSession methods return parsed data and tolerate per-ID failures.""" - session = AsyncSession( - token=SecretStr("token"), url="https://example.test/skaha/v1", concurrency=2 - ) - client = MagicMock() - client.get = AsyncMock() - client.post = AsyncMock() - client.delete = AsyncMock() - session._asynclient = client # noqa: SLF001 - - client.get.return_value = _sync_response([{"id": "s1"}]) - assert await session.fetch(kind="headless") == [{"id": "s1"}] - - client.get.return_value = _sync_response({"cores": {}}) - assert await session.stats() == {"cores": {}} - - client.get.side_effect = [ - _sync_response({"id": "s1"}), - RuntimeError("info failed"), - ] - with patch("canfar.sessions._log_http_task_failure") as log_failure: - assert await session.info(["s1", "s2"]) == [{"id": "s1"}] - log_failure.assert_called_once() - - client.get.side_effect = [ - _sync_response(text="log text"), - RuntimeError("logs failed"), - ] - with patch("canfar.sessions._log_http_task_failure") as log_failure: - assert await session.logs(["s1", "s2"]) == {"s1": "log text"} - log_failure.assert_called_once() - - client.post.side_effect = [ - _sync_response(text="created-1\n"), - RuntimeError("create failed"), - ] - with patch("canfar.sessions._log_http_task_failure") as log_failure: - assert await session.create( - name="batch", - image="skaha/terminal:latest", - kind="headless", - replicas=2, - ) == ["created-1"] - log_failure.assert_called_once() - - client.get.side_effect = [ - _sync_response(text="event text"), - RuntimeError("events failed"), - ] - with patch("canfar.sessions._log_http_task_failure") as log_failure: - assert await session.events(["s1", "s2"]) == [{"s1": "event text"}] - log_failure.assert_called_once() - - client.get.side_effect = [_sync_response(text="event text")] - assert await session.events("s1", verbose=True) is None - - client.delete.side_effect = [ - None, - _http_error("DELETE", "https://example.test/skaha/v1/session/s2"), - ] - assert await session.destroy(["s1", "s2"]) == {"s1": True, "s2": False} - - -@pytest.mark.asyncio -async def test_fetch_malformed_kind(asession: AsyncSession) -> None: - """Test fetching images with malformed kind.""" - with pytest.raises(ValidationError): - await asession.fetch(kind="invalid") - - -@pytest.mark.asyncio -async def test_fetch_with_malformed_view(asession: AsyncSession) -> None: - """Test fetching images with malformed view.""" - with pytest.raises(ValidationError): - await asession.fetch(view="invalid") - - -@pytest.mark.asyncio -@pytest.mark.slow -async def test_get_session_stats(asession: AsyncSession) -> None: - """Test fetching images with kind.""" - response = await asession.stats() - assert "cores" in response - assert "ram" in response - - -@pytest.mark.asyncio -async def test_create_session_invalid(asession: AsyncSession, name: str) -> None: - """Test creating a session with malformed kind.""" - with pytest.raises(ValidationError): - await asession.create( - name=name, - kind="invalid", - image="jupyter/base-notebook", - ) - - -@pytest.mark.asyncio -@pytest.mark.order(1) -@pytest.mark.slow -async def test_create_session(asession: AsyncSession, name: str) -> None: - """Test creating a session.""" - identity: list[str] = await asession.create( - name=name, - kind="headless", - cores=1, - ram=1, - image="images.canfar.net/skaha/terminal:1.1.2", - cmd="env", - replicas=1, - env={"TEST": "test"}, - ) - assert len(identity) == 1 - assert identity[0] != "" - pytest.IDENTITY = identity - - -@pytest.mark.asyncio -@pytest.mark.order(2) -@pytest.mark.slow -async def test_get_succeeded(asession: AsyncSession) -> None: - """Test getting succeeded sessions.""" - limit: float = time() + 60 # 1 minute - while time() < limit: - response = await asession.fetch() - for result in response: - await sleep(1) - if result["id"] == pytest.IDENTITY[0]: - break - - -@pytest.mark.asyncio -@pytest.mark.order(3) -@pytest.mark.slow -async def test_get_logs(asession: AsyncSession) -> None: - """Test getting logs for a session.""" - logs = await asession.logs(ids=pytest.IDENTITY) - assert logs != "" - assert "TEST" in logs[pytest.IDENTITY[0]] - no_logs = await asession.logs(ids=pytest.IDENTITY, verbose=True) - assert no_logs is None - - -@pytest.mark.asyncio -@pytest.mark.order(4) -@pytest.mark.slow -async def test_session_events(asession: AsyncSession) -> None: - """Test getting session events.""" - done = False - limit = time() + 60 - while not done and time() < limit: - await sleep(1) - events = await asession.events(pytest.IDENTITY) - if events: - done = True - assert pytest.IDENTITY[0] in events[0] - assert done, "No events found for the session." - - -@pytest.mark.asyncio -@pytest.mark.order(5) -@pytest.mark.slow -async def test_delete_session(asession: AsyncSession, name: str) -> None: - """Test deleting a session.""" - # Delete the session - done = False - while not done: - info = await asession.info(ids=pytest.IDENTITY) - for status in info: - if status["status"] == "Completed": - done = True - deletion = await asession.destroy_with(prefix=name) - assert deletion == {pytest.IDENTITY[0]: True} - - -@pytest.mark.asyncio -async def test_destroy_with_regex_match(asession: AsyncSession) -> None: - """Regex pattern should match anywhere in the session name.""" - mock_sessions = [ - { - "id": "abc123", - "name": "directwarp-13", - "status": "Running", - "kind": "headless", - } - ] - pattern = "directwarp-.*" - - with ( - patch.object( - AsyncSession, "fetch", AsyncMock(return_value=mock_sessions) - ) as mock_fetch, - patch.object( - AsyncSession, "destroy", AsyncMock(return_value={"abc123": True}) - ) as mock_destroy, - ): - result = await asession.destroy_with( - prefix=pattern, - kind="headless", - status="Running", - ) - - mock_fetch.assert_called_once_with(kind="headless", status="Running") - mock_destroy.assert_called_once_with(["abc123"]) - assert result == {"abc123": True} - - -@pytest.mark.asyncio -async def test_destroy_with_name_deprecation(asession: AsyncSession) -> None: - """Deprecated name parameter removed; retained for backward-compat check.""" - with pytest.raises(TypeError): - await asession.destroy_with(name="directwarp-.*") # type: ignore[arg-type] - - -# Unit tests for connect method (covers lines 798-804) -class TestAsyncSessionConnect: - """Test the AsyncSession.connect method.""" - - @patch("canfar.sessions.open_new_tab") - @patch.object(AsyncSession, "info") - @pytest.mark.asyncio - async def test_connect_single_session_string( - self, mock_info, mock_open_tab - ) -> None: - """Test connect with single session ID as string.""" - asession = AsyncSession() - - # Mock the info method to return session data with connectURL - mock_info.return_value = [ - { - "id": "session-123", - "status": "Running", - "connectURL": "https://example.com/connect", - } - ] - - await asession.connect("session-123") - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-123"]) - - # Verify open_new_tab was called with the connectURL - mock_open_tab.assert_called_once_with("https://example.com/connect") - - @patch("canfar.sessions.open_new_tab") - @patch.object(AsyncSession, "info") - @pytest.mark.asyncio - async def test_connect_multiple_sessions_list( - self, mock_info, mock_open_tab - ) -> None: - """Test connect with multiple session IDs as list.""" - asession = AsyncSession() - - # Mock the info method to return session data for all IDs - mock_info.return_value = [ - { - "id": "session-1", - "status": "Running", - "connectURL": "https://example.com/connect1", - }, - { - "id": "session-2", - "status": "Running", - "connectURL": "https://example.com/connect2", - }, - ] - - await asession.connect(["session-1", "session-2"]) - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-1", "session-2"]) - - # Verify open_new_tab was called for each connectURL - assert mock_open_tab.call_count == 2 - mock_open_tab.assert_any_call("https://example.com/connect1") - mock_open_tab.assert_any_call("https://example.com/connect2") - - @patch("canfar.sessions.open_new_tab") - @patch.object(AsyncSession, "info") - @pytest.mark.asyncio - async def test_connect_session_without_connect_url( - self, mock_info, mock_open_tab - ) -> None: - """Test connect when some sessions don't have connectURL.""" - asession = AsyncSession() - - # Mock the info method to return mixed session data - mock_info.return_value = [ - { - "id": "session-1", - "status": "Running", - "connectURL": "https://example.com/connect1", - }, - {"id": "session-2", "status": "Running"}, # No connectURL - { - "id": "session-3", - "status": "Running", - "connectURL": "https://example.com/connect3", - }, - ] - - await asession.connect(["session-1", "session-2", "session-3"]) - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-1", "session-2", "session-3"]) - - # Verify open_new_tab was called only for sessions with connectURL - assert mock_open_tab.call_count == 2 - mock_open_tab.assert_any_call("https://example.com/connect1") - mock_open_tab.assert_any_call("https://example.com/connect3") - # session-2 should be skipped because it has no connectURL - - @patch("canfar.sessions.open_new_tab") - @patch.object(AsyncSession, "info") - @pytest.mark.asyncio - async def test_connect_string_to_list_conversion( - self, mock_info, mock_open_tab - ) -> None: - """Test that single string ID is converted to list internally.""" - asession = AsyncSession() - - # Mock the info method - mock_info.return_value = [ - { - "id": "session-123", - "status": "Running", - "connectURL": "https://example.com/connect", - } - ] - - # Call with string (should be converted to list internally) - await asession.connect("session-123") - - # The method should have processed it as a single-item list - mock_info.assert_called_once_with(["session-123"]) - mock_open_tab.assert_called_once_with("https://example.com/connect") - - @patch("canfar.sessions.open_new_tab") - @patch.object(AsyncSession, "info") - @pytest.mark.asyncio - async def test_connect_empty_info_response(self, mock_info, mock_open_tab) -> None: - """Test connect when info returns empty list.""" - asession = AsyncSession() - - # Mock the info method to return empty list - mock_info.return_value = [] - - # Should not raise any exception, just do nothing - await asession.connect("session-123") - - # Verify info was called - mock_info.assert_called_once_with(["session-123"]) - - # Verify open_new_tab was not called - mock_open_tab.assert_not_called() - - @patch("canfar.sessions.open_new_tab") - @patch.object(AsyncSession, "info") - @pytest.mark.asyncio - async def test_connect_non_running_session(self, mock_info, mock_open_tab) -> None: - """Test connect when session is not in Running status.""" - asession = AsyncSession() - - # Mock the info method to return session with non-Running status - mock_info.return_value = [ - { - "id": "session-123", - "status": "Pending", - "connectURL": "https://example.com/connect", - } - ] - - # Should not raise any exception, just skip the session - await asession.connect("session-123") - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-123"]) - - # Verify open_new_tab was not called because status is not Running - mock_open_tab.assert_not_called() diff --git a/tests/test_auth_oidc_authenticate.py b/tests/test_auth_oidc_authenticate.py index f506e189..8bb9c28c 100644 --- a/tests/test_auth_oidc_authenticate.py +++ b/tests/test_auth_oidc_authenticate.py @@ -21,6 +21,7 @@ async def _authenticate_with_tokens( *, credential: OIDCCredential | None = None, userinfo_error: Exception | None = None, + authenticated: list[str | None] | None = None, ) -> OIDCCredential: credential = credential or OIDCCredential( idp="test", @@ -66,6 +67,9 @@ async def _authenticate_with_tokens( credential, expected_issuer="https://example.com", device_flow=AsyncMock(return_value=tokens), + on_authenticated=( + authenticated.append if authenticated is not None else None + ), ) @pytest.mark.asyncio @@ -132,6 +136,18 @@ async def test_authenticate_persists_issued_tokens(self) -> None: assert result.expiry.access == 1234567890 assert result.expiry.refresh is None + @pytest.mark.asyncio + async def test_authenticate_notifies_authenticated_username(self) -> None: + """Authentication reports the same UserInfo username as the sync flow.""" + authenticated: list[str | None] = [] + + await self._authenticate_with_tokens( + {"access_token": "test_access_token"}, + authenticated=authenticated, + ) + + assert authenticated == ["testuser"] + @pytest.mark.asyncio async def test_authenticate_accepts_token_without_refresh_token(self) -> None: """Authentication succeeds when the token response omits a refresh token.""" diff --git a/tests/test_auth_oidc_sync.py b/tests/test_auth_oidc_sync.py new file mode 100644 index 00000000..6141b964 --- /dev/null +++ b/tests/test_auth_oidc_sync.py @@ -0,0 +1,233 @@ +"""Contract tests for the native synchronous OIDC device flow.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, call, patch + +import httpx +import pytest +from authlib.integrations.httpx_client import OAuth2Client +from pydantic import SecretStr + +from canfar.auth.oidc import sync_authenticate_credential, sync_poll_device_token +from canfar.models.auth import Client, DeviceAuthorization, Endpoint, OIDCCredential + + +def _challenge( + *, + expires_in: int = 60, + interval: int = 5, + device_code: str = "device_code_123", +) -> DeviceAuthorization: + """Return a deterministic RFC 8628 challenge.""" + return DeviceAuthorization( + verification_uri="https://example.com/device", + user_code="ABC123", + expires_in=expires_in, + interval=interval, + device_code=device_code, + ) + + +def _oauth_client( + *responses: httpx.Response | Exception, +) -> tuple[OAuth2Client, list[httpx.Request]]: + """Return an Authlib client backed by deterministic token responses.""" + remaining = iter(responses) + requests: list[httpx.Request] = [] + + def token_endpoint(request: httpx.Request) -> httpx.Response: + """Return the next deterministic token response.""" + requests.append(request) + response = next(remaining) + if isinstance(response, Exception): + raise response + return response + + return ( + OAuth2Client( + "client_id", + "client_secret", + token_endpoint_auth_method="client_secret_basic", + transport=httpx.MockTransport(token_endpoint), + ), + requests, + ) + + +def test_sync_poll_device_token_waits_at_protocol_interval() -> None: + """A pending authorization waits before the next token request.""" + client, requests = _oauth_client( + httpx.Response(400, json={"error": "authorization_pending"}), + httpx.Response(200, json={"access_token": "access-token"}), + ) + + with client, patch("canfar.auth.oidc.time.sleep") as sleep: + tokens = sync_poll_device_token( + "https://example.com/token", + _challenge(), + client, + ) + + assert tokens["access_token"] == "access-token" + assert len(requests) == 2 + assert sleep.call_args_list == [call(5)] + + +def test_sync_poll_device_token_expires_at_challenge_deadline() -> None: + """Pending authorization stops at the challenge expiry deadline.""" + client, requests = _oauth_client( + httpx.Response(400, json={"error": "authorization_pending"}), + httpx.Response(400, json={"error": "authorization_pending"}), + httpx.Response(200, json={"access_token": "too-late"}), + ) + clock = 0.0 + + def monotonic() -> float: + return clock + + def advance(seconds: float) -> None: + nonlocal clock + clock += seconds + + with ( + client, + patch("canfar.auth.oidc.time.monotonic", side_effect=monotonic), + patch("canfar.auth.oidc.time.sleep", side_effect=advance) as sleep, + pytest.raises(TimeoutError, match="Device flow timed out"), + ): + sync_poll_device_token( + "https://example.com/token", + _challenge(expires_in=6), + client, + ) + + assert len(requests) == 2 + assert sleep.call_args_list == [call(5), call(1)] + + +def test_sync_poll_device_token_reports_denial_without_secrets() -> None: + """Terminal denial omits OIDC Identity Provider response data.""" + client, requests = _oauth_client( + httpx.Response( + 400, + json={ + "error": "access_denied", + "error_description": "secret-error-description", + }, + ) + ) + + with client, pytest.raises(PermissionError, match="authorization was denied"): + sync_poll_device_token( + "https://example.com/token", + _challenge(device_code="secret-device-code"), + client, + ) + + assert len(requests) == 1 + + +def test_sync_poll_device_token_retries_transport_failure() -> None: + """A transport failure backs off and retries before succeeding.""" + client, _ = _oauth_client( + httpx.ConnectTimeout("network timeout"), + httpx.Response(200, json={"access_token": "access-token"}), + ) + + with client, patch("canfar.auth.oidc.time.sleep") as sleep: + tokens = sync_poll_device_token( + "https://example.com/token", + _challenge(), + client, + ) + + assert tokens["access_token"] == "access-token" + sleep.assert_called_once_with(10) + + +def test_sync_authenticate_credential_runs_complete_native_flow() -> None: + """Sync authentication performs discovery, registration, device, and userinfo.""" + credential = OIDCCredential( + idp="srcnet", + endpoints=Endpoint( + discovery="https://example.com/.well-known/openid-configuration" + ), + client=Client(), + ) + discovery = httpx.Response( + 200, + request=httpx.Request( + "GET", "https://example.com/.well-known/openid-configuration" + ), + json={ + "issuer": "https://example.com", + "device_authorization_endpoint": "https://example.com/device", + "registration_endpoint": "https://example.com/register", + "token_endpoint": "https://example.com/token", + "userinfo_endpoint": "https://example.com/userinfo", + }, + ) + registration = httpx.Response( + 200, + request=httpx.Request("POST", "https://example.com/register"), + json={"client_id": "client-id", "client_secret": "client-secret"}, + ) + challenge = httpx.Response( + 200, + request=httpx.Request("POST", "https://example.com/device"), + json={ + "verification_uri": "https://example.com/device", + "user_code": "ABC123", + "expires_in": 600, + "interval": 5, + "device_code": "device-code", + }, + ) + userinfo = httpx.Response( + 200, + request=httpx.Request("GET", "https://example.com/userinfo"), + json={"preferred_username": "test-user"}, + ) + sync_client = MagicMock() + sync_client.get.side_effect = [discovery, userinfo] + sync_client.post.return_value = registration + oauth_client = MagicMock() + oauth_client.post.return_value = challenge + oauth_client.fetch_token.return_value = { + "access_token": "access-token", + "refresh_token": "refresh-token", + "token_type": "Bearer", + "scope": "openid profile", + "expires_at": 1893456000, + } + presented: list[DeviceAuthorization] = [] + authenticated: list[str | None] = [] + + with ( + patch("canfar.auth.oidc.httpx.Client") as client_class, + patch("authlib.integrations.httpx_client.OAuth2Client") as oauth_client_class, + patch("canfar.auth.oidc.time.sleep"), + ): + client_class.return_value.__enter__.return_value = sync_client + oauth_client_class.return_value.__enter__.return_value = oauth_client + result = sync_authenticate_credential( + credential, + expected_issuer="https://example.com", + on_challenge=presented.append, + on_authenticated=authenticated.append, + ) + + assert result.client.identity == "client-id" + assert result.client.secret == SecretStr("client-secret") + assert result.token.access == SecretStr("access-token") + assert result.token.refresh == SecretStr("refresh-token") + assert presented[0].user_code == SecretStr("ABC123") + assert authenticated == ["test-user"] + sync_client.get.assert_any_call( + "https://example.com/.well-known/openid-configuration" + ) + sync_client.get.assert_any_call( + "https://example.com/userinfo", + headers={"Authorization": "Bearer access-token"}, + ) diff --git a/tests/test_auth_x509.py b/tests/test_auth_x509.py index d59c058f..6acf0fb1 100644 --- a/tests/test_auth_x509.py +++ b/tests/test_auth_x509.py @@ -165,28 +165,34 @@ def test_expiry_error_message_contains_times(tmp_path) -> None: assert "current time" in message -def test_expiry_handles_missing_not_valid_before_utc(monkeypatch, tmp_path) -> None: - """Expiry should fall back to naive datetime attributes when needed.""" +def test_expiry_uses_current_cryptography_certificate_api( + monkeypatch, tmp_path +) -> None: + """Expiry reads certificates through the current Cryptography API.""" cert_path = tmp_path / "cert.pem" generate_cert(cert_path, valid_for_days=3) original_loader = x509_auth.x509.load_pem_x509_certificate + original_cert = original_loader(cert_path.read_bytes()) - class MinimalCert: - """Certificate exposing only naive validity attributes.""" + class ModernCertificate: + """Certificate exposing only the supported UTC-aware validity fields.""" - def __init__(self, cert: x509.Certificate) -> None: - self.not_valid_before = cert.not_valid_before - self.not_valid_after = cert.not_valid_after + not_valid_before_utc = original_cert.not_valid_before_utc + not_valid_after_utc = original_cert.not_valid_after_utc - def fake_loader(data: bytes, backend) -> MinimalCert: # type: ignore[override] - cert = original_loader(data, backend) - return MinimalCert(cert) + def load_certificate(data: bytes) -> ModernCertificate: + assert data == cert_path.read_bytes() + return ModernCertificate() - monkeypatch.setattr(x509_auth.x509, "load_pem_x509_certificate", fake_loader) + monkeypatch.setattr( + x509_auth.x509, + "load_pem_x509_certificate", + load_certificate, + ) expiry_ts = x509_auth.expiry(cert_path) - assert isinstance(expiry_ts, float) + assert expiry_ts == pytest.approx(original_cert.not_valid_after_utc.timestamp()) # --- Tests for canfar.auth.x509.inspect --- # diff --git a/tests/test_authentication.py b/tests/test_authentication.py index 704c8e97..1edb01e1 100644 --- a/tests/test_authentication.py +++ b/tests/test_authentication.py @@ -29,9 +29,13 @@ def _merge_servers( config: canfar.models.config.Configuration, discovered: list, ) -> None: - for server in discovered: - if server.name is not None: - config.servers[server.name] = server + config.editor.set( + "servers", + { + **config.servers, + **{server.name: server for server in discovered if server.name is not None}, + }, + ) class TestAuthenticationList: @@ -436,8 +440,8 @@ def test_login_saves_auth_and_servers_without_changing_active( assert config.active.authentication == "cadc" assert config.active.server == "canfar" - assert config.get_credential("cadc").path == Path("/new/cert.pem") - assert str(config.get_server_by_uri("ivo://cadc.nrc.ca/skaha").url) == ( + assert config.authentication["cadc"].path == Path("/new/cert.pem") + assert str(config.servers["CADC-CANFAR"].url) == ( "https://ws-uv.canfar.net/skaha" ) @@ -492,8 +496,8 @@ def test_login_with_force_replaces_existing_credential( with _patch_config(config_path): config = canfar.models.config.Configuration() - assert config.get_credential("cadc").path == Path("/new/cert.pem") - assert config.get_credential("cadc").expiry == 888.0 + assert config.authentication["cadc"].path == Path("/new/cert.pem") + assert config.authentication["cadc"].expiry == 888.0 class TestAuthenticationModuleExports: diff --git a/tests/test_authentication_oidc_login.py b/tests/test_authentication_oidc_login.py new file mode 100644 index 00000000..d5068fb0 --- /dev/null +++ b/tests/test_authentication_oidc_login.py @@ -0,0 +1,135 @@ +"""Contract tests for public synchronous and asynchronous OIDC login.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import canfar +from canfar.models.auth import ( + Client, + DeviceAuthorization, + Endpoint, + Expiry, + OIDCCredential, + Token, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def _patch_config(path: Path): + return patch("canfar.models.config.CONFIG_PATH", path) + + +def _credential() -> OIDCCredential: + """Return a saved-ready SRCNet Authentication Record.""" + return OIDCCredential( + idp="srcnet", + endpoints=Endpoint( + discovery="https://example.com/.well-known/openid-configuration" + ), + client=Client(identity="client-id", secret="client-secret"), + token=Token( + access="access-token", + refresh="refresh-token", + token_type="Bearer", + scope="openid profile", + ), + expiry=Expiry(access=1893456000, refresh=None), + ) + + +def _challenge() -> DeviceAuthorization: + """Return the presentation data exposed by the device protocol.""" + return DeviceAuthorization( + verification_uri="https://example.com/device", + verification_uri_complete="https://example.com/device?user_code=ABC123", + user_code="ABC123", + expires_in=600, + interval=5, + device_code="secret-device-code", + ) + + +def test_login_runs_plain_sync_oidc_flow_and_persists_record( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """The sync Python API presents a challenge without CLI presentation tools.""" + config_path = tmp_path / "config.yaml" + coordinator = MagicMock() + + def authenticate(credential: OIDCCredential, **kwargs) -> OIDCCredential: + """Expose the challenge through the API's plain terminal callback.""" + assert credential.idp == "srcnet" + kwargs["on_challenge"](_challenge()) + coordinator(**kwargs) + return _credential() + + with ( + _patch_config(config_path), + patch( + "canfar.authentication.oidc.sync_authenticate_credential", + side_effect=authenticate, + ) as authenticate_credential, + patch("canfar.authentication.server_service.discover", return_value=[]), + ): + canfar.login("srcnet") + + output = capsys.readouterr().out + assert "https://example.com/device" in output + assert "ABC123" in output + assert "secret-device-code" not in output + authenticate_credential.assert_called_once() + with _patch_config(config_path): + saved = canfar.models.config.Configuration() + credential = saved.authentication["srcnet"] + assert isinstance(credential, OIDCCredential) + assert credential.token.access is not None + assert credential.token.access.get_secret_value() == "access-token" + + +@pytest.mark.asyncio +async def test_alogin_uses_native_async_flow_without_asyncio_run( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """The async API awaits native protocol work inside an existing event loop.""" + config_path = tmp_path / "config.yaml" + + async def authenticate(credential: OIDCCredential, **kwargs) -> OIDCCredential: + """Expose the challenge through the async terminal callback.""" + assert credential.idp == "srcnet" + kwargs["on_challenge"](_challenge()) + await asyncio.sleep(0) + return _credential() + + with ( + _patch_config(config_path), + patch( + "canfar.authentication.oidc.authenticate_credential", + new=AsyncMock(side_effect=authenticate), + ) as authenticate_credential, + patch("canfar.authentication.server_service.discover", return_value=[]), + patch( + "asyncio.run", side_effect=AssertionError("library API used asyncio.run") + ), + ): + await canfar.alogin("srcnet") + + output = capsys.readouterr().out + assert "https://example.com/device" in output + assert "ABC123" in output + assert "secret-device-code" not in output + authenticate_credential.assert_awaited_once() + with _patch_config(config_path): + saved = canfar.models.config.Configuration() + credential = saved.authentication["srcnet"] + assert isinstance(credential, OIDCCredential) + assert credential.token.access is not None + assert credential.token.access.get_secret_value() == "access-token" diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py new file mode 100644 index 00000000..927cb935 --- /dev/null +++ b/tests/test_ci_workflows.py @@ -0,0 +1,193 @@ +from pathlib import Path + +import yaml + +ROOT = Path(__file__).parents[1] + + +def load_workflow(name: str) -> dict: + return yaml.safe_load((ROOT / ".github" / "workflows" / name).read_text()) + + +def workflow_events(workflow: dict) -> dict: + return workflow["on"] if "on" in workflow else workflow[True] + + +def run_commands(workflow: dict) -> list[str]: + return [ + step["run"] + for job in workflow["jobs"].values() + for step in job.get("steps", []) + if "run" in step + ] + + +def test_reusable_tests_preserve_fast_and_credentialed_commands(): + workflow = load_workflow("reusable-tests.yml") + events = workflow_events(workflow) + commands = "\n".join(run_commands(workflow)) + + assert events["workflow_call"]["inputs"]["full-suite"] == { + "default": False, + "required": False, + "type": "boolean", + } + assert set(events["workflow_call"]["secrets"]) == { + "CANFAR_BASEURL", + "CANFAR_USERNAME", + "CANFAR_PASSWORD", + "CODECOV_TOKEN", + } + assert 'uv run pytest -m "not slow" tests' in commands + assert 'CANFAR_TEST_HOME="$HOME" uv run pytest' in commands + assert "uv run cadc-get-cert" in commands + assert "CANFAR credentials are required for the full test suite" in commands + assert "rm -rf ~/.ssl/" in commands + + fast_step = next( + step + for step in workflow["jobs"]["tests"]["steps"] + if step.get("name") == "Run fast test suite" + ) + full_step = next( + step + for step in workflow["jobs"]["tests"]["steps"] + if step.get("name") == "Run full test suite" + ) + verify_step = next( + step + for step in workflow["jobs"]["tests"]["steps"] + if step.get("name") == "Verify CANFAR credentials" + ) + login_step = next( + step + for step in workflow["jobs"]["tests"]["steps"] + if step.get("name") == "Login to CANFAR" + ) + cleanup_step = next( + step + for step in workflow["jobs"]["tests"]["steps"] + if step.get("name") == "Remove CANFAR Certificate" + ) + assert fast_step["if"] == "${{ !inputs.full-suite }}" + assert full_step["if"] == "${{ inputs.full-suite }}" + assert verify_step["if"] == "${{ inputs.full-suite }}" + assert login_step["if"] == "${{ inputs.full-suite }}" + assert cleanup_step["if"] == "${{ always() && inputs.full-suite }}" + assert '-m "not slow"' not in full_step["run"] + + +def test_release_and_edge_delegate_to_one_container_build_contract(): + release = load_workflow("release.yml") + edge = load_workflow("edge.yml") + + assert workflow_events(release) == { + "repository_dispatch": {"types": ["release-build"]}, + } + assert workflow_events(edge) == { + "repository_dispatch": {"types": ["edge-build"]}, + } + + release_job = release["jobs"]["release-build"] + edge_job = edge["jobs"]["edge-build"] + assert release_job["uses"] == "./.github/workflows/reusable-container.yml" + assert edge_job["uses"] == "./.github/workflows/reusable-container.yml" + assert ( + release_job["permissions"] + == edge_job["permissions"] + == { + "packages": "write", + "attestations": "write", + "id-token": "write", + } + ) + assert release_job["with"]["checkout-ref"] == ( + "${{ github.event.client_payload.tag_name }}" + ) + assert "checkout-ref" not in edge_job["with"] + assert release_job["with"]["image-version"] == ( + "${{ github.event.client_payload.tag_name }}" + ) + assert edge_job["with"]["image-version"] == "edge" + assert release_job["with"]["image-description"] == ( + "Python Client for CANFAR Science Portal" + ) + assert edge_job["with"]["image-description"] == ( + "Python Client for CANFAR Science Platform" + ) + assert release_job["with"]["image-tags"].strip().endswith(":latest") + assert edge_job["with"]["image-tags"].strip().endswith(":edge") + + +def test_reusable_container_preserves_build_and_attestation_guards(): + workflow = load_workflow("reusable-container.yml") + events = workflow_events(workflow) + job = workflow["jobs"]["container-build"] + steps = {step["name"]: step for step in job["steps"]} + + assert set(events["workflow_call"]["inputs"]) == { + "checkout-ref", + "client-payload", + "image-description", + "image-tags", + "image-version", + } + assert workflow["permissions"] == {"contents": "read"} + assert job["permissions"] == { + "packages": "write", + "attestations": "write", + "id-token": "write", + } + assert steps["Build & Push Docker Image"]["uses"].startswith( + "docker/build-push-action@" + ) + build_with = steps["Build & Push Docker Image"]["with"] + assert build_with["platforms"] == "linux/amd64,linux/arm64" + assert build_with["provenance"] == "mode=max" + assert build_with["sbom"] is True + assert build_with["push"] is True + + attest = steps["Attest GHCR Container Image"] + assert attest["uses"].startswith("actions/attest-build-provenance@") + assert attest["with"]["push-to-registry"] is True + assert attest["with"]["subject-digest"] == "${{ steps.build.outputs.digest }}" + + +def test_pull_requests_use_the_fast_suite_for_maintained_branches(): + workflow = load_workflow("ci.yml") + events = workflow_events(workflow) + + assert set(events["pull_request"]["branches"]) == {"main", "feat/interfaces"} + assert "paths-ignore" not in events["pull_request"] + job = workflow["jobs"]["tests"] + assert job["uses"] == "./.github/workflows/reusable-tests.yml" + assert job["with"] == {"full-suite": False} + assert job["needs"] == "pre-commit-checks" + assert job["secrets"] == { + "CODECOV_TOKEN": "${{ secrets.CODECOV_TOKEN }}", + } + + +def test_full_suite_is_limited_to_merged_code_prs_into_main(): + workflow = load_workflow("full-tests.yml") + events = workflow_events(workflow) + + assert set(events) == {"pull_request_target"} + assert events["pull_request_target"] == { + "types": ["closed"], + "branches": ["main"], + "paths": ["canfar/**", "tests/**"], + } + assert "github.event.pull_request.merged == true" in workflow["jobs"]["tests"]["if"] + assert ( + "github.event.pull_request.base.ref == 'main'" + in workflow["jobs"]["tests"]["if"] + ) + assert workflow["jobs"]["tests"]["uses"] == "./.github/workflows/reusable-tests.yml" + assert workflow["jobs"]["tests"]["with"] == {"full-suite": True} + assert workflow["jobs"]["tests"]["secrets"] == { + "CANFAR_BASEURL": "${{ secrets.CANFAR_BASEURL }}", + "CANFAR_USERNAME": "${{ secrets.CANFAR_USERNAME }}", + "CANFAR_PASSWORD": "${{ secrets.CANFAR_PASSWORD }}", + "CODECOV_TOKEN": "${{ secrets.CODECOV_TOKEN }}", + } diff --git a/tests/test_cli_auth.py b/tests/test_cli_auth.py index edccc1ef..af1fd7f2 100644 --- a/tests/test_cli_auth.py +++ b/tests/test_cli_auth.py @@ -13,7 +13,6 @@ from canfar.authentication import Authentication from canfar.cli.auth import auth -from canfar.cli.main import cli from canfar.models.config import Configuration if TYPE_CHECKING: @@ -157,18 +156,6 @@ def test_auth_ls_lists_saved_records(tmp_path: Path) -> None: assert "srcnet" in result.stdout -def test_auth_list_alias_is_not_supported() -> None: - """``auth list`` is not a supported command alias.""" - result = runner.invoke(auth, ["list"]) - assert result.exit_code != 0 - - -def test_auth_remove_alias_is_not_supported() -> None: - """``auth remove`` is not a supported command alias.""" - result = runner.invoke(auth, ["remove", "cadc"]) - assert result.exit_code != 0 - - def test_auth_use_switches_by_idp(tmp_path: Path) -> None: """``auth use`` selects Authentication by canonical IDP key.""" config_path = tmp_path / "config.yaml" @@ -280,22 +267,10 @@ def test_auth_purge_force_preserves_registry_and_console(tmp_path: Path) -> None with _patch_config(config_path): before = Configuration() before.console = before.console.model_copy(update={"width": 99}) - before.save() + before.editor.save() result = runner.invoke(auth, ["purge", "--force"]) assert result.exit_code == 0 with _patch_config(config_path): after = Configuration() assert after.console.width == 99 - - -def test_authentication_alias_is_wired(tmp_path: Path) -> None: - """``canfar authentication`` aliases ``canfar auth``.""" - config_path = tmp_path / "config.yaml" - _write_config(config_path) - - with _patch_config(config_path): - result = runner.invoke(cli, ["authentication"]) - - assert result.exit_code == 0 - assert "cadc" in result.stdout diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py index 56a71fb7..f48ae5a4 100644 --- a/tests/test_cli_config.py +++ b/tests/test_cli_config.py @@ -81,7 +81,10 @@ def test_config_get_full_server_record_includes_runtime_name(tmp_path: Path) -> patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - json_result = runner.invoke(config, ["get", "servers.canfar", "--json"]) + json_result = runner.invoke( + config, + ["get", "servers.canfar", "--output", "json"], + ) human_result = runner.invoke(config, ["get", "servers.canfar"]) assert json_result.exit_code == 0 @@ -115,7 +118,10 @@ def test_config_get_full_credential_record_includes_runtime_idp(tmp_path: Path) patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - json_result = runner.invoke(config, ["get", "authentication.cadc", "--json"]) + json_result = runner.invoke( + config, + ["get", "authentication.cadc", "--output", "json"], + ) human_result = runner.invoke(config, ["get", "authentication.cadc"]) assert json_result.exit_code == 0 @@ -190,14 +196,14 @@ def test_config_show_path_format_and_errors(tmp_path: Path) -> None: assert isinstance(result.exception, RuntimeError) with patch("canfar.cli.config.Configuration") as cfg: - cfg.return_value.get_value.side_effect = KeyError("missing") + cfg.return_value.editor.get.side_effect = KeyError("missing") result = runner.invoke(config, ["get", "missing"]) assert result.exit_code == 1 assert "missing" in result.stderr with patch("canfar.cli.config.Configuration") as cfg: - cfg.return_value.set_value.side_effect = TypeError("wrong") + cfg.return_value.editor.set.side_effect = TypeError("wrong") result = runner.invoke(config, ["set", "console.width", "120"]) assert result.exit_code == 1 @@ -205,14 +211,14 @@ def test_config_show_path_format_and_errors(tmp_path: Path) -> None: def test_config_show_json_emits_configuration_model(tmp_path: Path) -> None: - """``config show --json`` emits the Configuration model with stable keys.""" + """``config show -o json`` emits the Configuration model with stable keys.""" config_path = tmp_path / "config.yaml" with ( _patch_config_path(config_path), patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - result = runner.invoke(config, ["show", "--json"]) + result = runner.invoke(config, ["show", "-o", "json"]) assert result.exit_code == 0 assert not result.stdout.startswith("@") @@ -222,14 +228,14 @@ def test_config_show_json_emits_configuration_model(tmp_path: Path) -> None: def test_config_show_yaml_emits_configuration_model(tmp_path: Path) -> None: - """``config show --yaml`` emits the Configuration model on stdout.""" + """``config show -o yaml`` emits the Configuration model on stdout.""" config_path = tmp_path / "config.yaml" with ( _patch_config_path(config_path), patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - result = runner.invoke(config, ["show", "--yaml"]) + result = runner.invoke(config, ["show", "-o", "yaml"]) assert result.exit_code == 0 payload = yaml.safe_load(result.stdout) @@ -237,7 +243,7 @@ def test_config_show_yaml_emits_configuration_model(tmp_path: Path) -> None: def test_config_show_json_redacts_oidc_secrets(tmp_path: Path) -> None: - """``config show --json`` must not emit raw OIDC secrets from saved auth.""" + """``config show -o json`` must not emit raw OIDC secrets from saved auth.""" config_path = tmp_path / "config.yaml" config_path.write_text( yaml.dump( @@ -278,7 +284,7 @@ def test_config_show_json_redacts_oidc_secrets(tmp_path: Path) -> None: patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - result = runner.invoke(config, ["show", "--json"]) + result = runner.invoke(config, ["show", "-o", "json"]) assert result.exit_code == 0 rendered = result.stdout @@ -331,7 +337,7 @@ def test_config_show_json_keeps_null_secrets_null(tmp_path: Path) -> None: patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - result = runner.invoke(config, ["show", "--json"]) + result = runner.invoke(config, ["show", "--output", "json"]) assert result.exit_code == 0 oidc = json.loads(result.stdout)["authentication"]["srcnet"] @@ -341,14 +347,14 @@ def test_config_show_json_keeps_null_secrets_null(tmp_path: Path) -> None: def test_config_get_json_emits_scalar_value(tmp_path: Path) -> None: - """``config get --json`` emits the resolved value without human formatting.""" + """``config get -o json`` emits the resolved value without human formatting.""" config_path = tmp_path / "config.yaml" with ( _patch_config_path(config_path), patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - result = runner.invoke(config, ["get", "console.width", "--json"]) + result = runner.invoke(config, ["get", "console.width", "-o", "json"]) assert result.exit_code == 0 assert not result.stdout.startswith("@") @@ -356,21 +362,24 @@ def test_config_get_json_emits_scalar_value(tmp_path: Path) -> None: def test_config_get_yaml_emits_scalar_value(tmp_path: Path) -> None: - """``config get --yaml`` emits the resolved scalar value on stdout.""" + """``config get -o yaml`` emits the resolved scalar value on stdout.""" config_path = tmp_path / "config.yaml" with ( _patch_config_path(config_path), patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - result = runner.invoke(config, ["get", "console.width", "--yaml"]) + result = runner.invoke( + config, + ["get", "console.width", "--output", "yaml"], + ) assert result.exit_code == 0 assert yaml.safe_load(result.stdout) == 120 def test_config_get_json_redacts_sensitive_paths(tmp_path: Path) -> None: - """``config get --json`` masks sensitive OIDC credential values.""" + """``config get -o json`` masks sensitive OIDC credential values.""" config_path = tmp_path / "config.yaml" config_path.write_text( yaml.dump( @@ -410,7 +419,12 @@ def test_config_get_json_redacts_sensitive_paths(tmp_path: Path) -> None: ): result = runner.invoke( config, - ["get", "authentication.srcnet.client.secret", "--json"], + [ + "get", + "authentication.srcnet.client.secret", + "--output", + "json", + ], ) assert result.exit_code == 0 @@ -421,7 +435,7 @@ def test_config_get_json_redacts_sensitive_paths(tmp_path: Path) -> None: def test_config_get_json_redacts_secrets_inside_credential_record( tmp_path: Path, ) -> None: - """``config get authentication. --json`` masks nested OIDC secrets.""" + """``config get authentication. -o json`` masks nested OIDC secrets.""" config_path = tmp_path / "config.yaml" config_path.write_text( yaml.dump( @@ -459,10 +473,13 @@ def test_config_get_json_redacts_secrets_inside_credential_record( patch("canfar.cli.config.CONFIG_PATH", config_path), patch("canfar.models.config.CONFIG_PATH", config_path), ): - record = runner.invoke(config, ["get", "authentication.srcnet", "--json"]) + record = runner.invoke( + config, + ["get", "authentication.srcnet", "-o", "json"], + ) token = runner.invoke( config, - ["get", "authentication.srcnet.token", "--json"], + ["get", "authentication.srcnet.token", "-o", "json"], ) assert record.exit_code == 0 diff --git a/tests/test_cli_create.py b/tests/test_cli_create.py index 38b29c34..358513bb 100644 --- a/tests/test_cli_create.py +++ b/tests/test_cli_create.py @@ -8,7 +8,7 @@ import yaml from typer.testing import CliRunner -from canfar.cli.create import create +from canfar.cli.main import cli from canfar.errors import ErrorCode, StructuredError from canfar.models.session import CreateRequest @@ -26,8 +26,9 @@ def test_create_command_success(self, mock_session_cls): mock_session.create.return_value = ["id-1", "id-2"] result = runner.invoke( - create, + cli, [ + "create", "headless", "skaha/worker:v1", "--name", @@ -80,8 +81,8 @@ def test_create_command_single_keeps_human_success_message( mock_session.create.return_value = ["session-id"] result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--name", "single"], + cli, + ["create", "headless", "skaha/worker:v1", "--name", "single"], ) assert result.exit_code == 0 @@ -95,8 +96,8 @@ def test_create_command_multiple(self, mock_session_cls): mock_session.create.return_value = ["id-1", "id-2"] result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--replicas", "2"], + cli, + ["create", "headless", "skaha/worker:v1", "--replicas", "2"], ) assert result.exit_code == 0 @@ -111,12 +112,48 @@ def test_create_command_json_success_is_a_raw_id_list(self, mock_session_cls): mock_session_cls.return_value.__aenter__.return_value = mock_session mock_session.create.return_value = ["session-id"] - result = runner.invoke(create, ["headless", "skaha/worker:v1", "--json"]) + result = runner.invoke( + cli, + ["create", "headless", "skaha/worker:v1", "--output", "json"], + ) assert result.exit_code == 0 assert json.loads(result.stdout) == ["session-id"] assert result.stderr == "" + @patch("canfar.cli.create.AsyncSession") + def test_create_output_option_stops_at_command_delimiter( + self, + mock_session_cls, + ): + """Only the output option before ``--`` is parsed by ``create``.""" + mock_session = AsyncMock() + mock_session_cls.return_value.__aenter__.return_value = mock_session + mock_session.create.return_value = ["session-id"] + + result = runner.invoke( + cli, + [ + "create", + "headless", + "skaha/worker:v1", + "--output", + "json", + "--", + "echo", + "-o", + "yaml", + "--output", + "json", + ], + ) + + assert result.exit_code == 0 + assert json.loads(result.stdout) == ["session-id"] + request = mock_session.create.await_args.args[0] + assert request.cmd == "echo" + assert request.args == "-o yaml --output json" + @patch("canfar.cli.create.AsyncSession") def test_create_command_debug_keeps_machine_stdout_data_only( self, @@ -128,8 +165,8 @@ def test_create_command_debug_keeps_machine_stdout_data_only( mock_session.create.return_value = ["session-id"] result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--debug", "--json"], + cli, + ["create", "headless", "skaha/worker:v1", "--debug", "--output", "json"], ) assert result.exit_code == 0 @@ -148,8 +185,16 @@ def test_create_command_yaml_partial_success_keeps_a_list( mock_session.create.return_value = ["id-1"] result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--replicas", "2", "--yaml"], + cli, + [ + "create", + "headless", + "skaha/worker:v1", + "--replicas", + "2", + "--output", + "yaml", + ], ) assert result.exit_code == 0 @@ -163,17 +208,20 @@ def test_create_command_failure(self, mock_session_cls): mock_session_cls.return_value.__aenter__.return_value = mock_session mock_session.create.return_value = [] - result = runner.invoke(create, ["headless", "skaha/worker:v1"]) + result = runner.invoke(cli, ["create", "headless", "skaha/worker:v1"]) assert result.exit_code == 1 - assert result.stdout == "" + assert result.stdout.startswith("@") assert "Failed to create session(s)" in result.stderr assert "CANFAR_TIMEOUT" in result.stderr assert "canfar --log-level debug create" in result.stderr @pytest.mark.parametrize( ("flag", "load"), - [("--json", json.loads), ("--yaml", yaml.safe_load)], + [ + (["--output", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), + ], ) @patch("canfar.cli.create.AsyncSession") def test_create_command_machine_empty_is_transport_failure( @@ -187,7 +235,7 @@ def test_create_command_machine_empty_is_transport_failure( mock_session_cls.return_value.__aenter__.return_value = mock_session mock_session.create.return_value = [] - result = runner.invoke(create, ["headless", "skaha/worker:v1", flag]) + result = runner.invoke(cli, ["create", "headless", "skaha/worker:v1", *flag]) assert result.exit_code == 1 assert result.stdout == "" @@ -196,7 +244,10 @@ def test_create_command_machine_empty_is_transport_failure( @pytest.mark.parametrize( ("flag", "load"), - [("--json", json.loads), ("--yaml", yaml.safe_load)], + [ + (["--output", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), + ], ) @pytest.mark.parametrize( "invalid_args", @@ -212,8 +263,8 @@ def test_create_command_machine_validation_failure_is_structured( ): """Invalid command input fails before the Session boundary opens.""" result = runner.invoke( - create, - ["headless", "skaha/worker:v1", *invalid_args, flag], + cli, + ["create", "headless", "skaha/worker:v1", *invalid_args, *flag], ) assert result.exit_code == 1 @@ -229,8 +280,8 @@ def test_create_command_validation_failure_keeps_human_diagnostics( ): """Human validation failure keeps detailed diagnostics and exit one.""" result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--cpu", "-1"], + cli, + ["create", "headless", "skaha/worker:v1", "--cpu", "-1"], ) assert result.exit_code == 1 @@ -246,8 +297,8 @@ def test_create_command_malformed_environment_keeps_human_message( ): """Human malformed-environment input keeps its existing message.""" result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--env", "BROKEN"], + cli, + ["create", "headless", "skaha/worker:v1", "--env", "BROKEN"], ) assert result.exit_code == 1 @@ -258,8 +309,8 @@ def test_create_command_malformed_environment_keeps_human_message( def test_create_command_dry_run(self): """Test create command dry run.""" result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--dry-run"], + cli, + ["create", "headless", "skaha/worker:v1", "--dry-run"], ) assert result.exit_code == 0 @@ -270,25 +321,21 @@ def test_create_command_dry_run(self): def test_create_command_rejects_dry_run_with_machine_output(self): """Dry-run diagnostics cannot contaminate machine stdout.""" result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--dry-run", "--json"], + cli, + [ + "create", + "headless", + "skaha/worker:v1", + "--dry-run", + "--output", + "json", + ], ) assert result.exit_code == 2 assert result.stdout == "" assert "--dry-run" in result.stderr - def test_create_command_rejects_conflicting_machine_formats(self): - """The shared resolver rejects simultaneous JSON and YAML output.""" - result = runner.invoke( - create, - ["headless", "skaha/worker:v1", "--json", "--yaml"], - ) - - assert result.exit_code == 2 - assert result.stdout == "" - assert "Conflicting machine output flags" in result.stderr - @patch("canfar.cli.create.AsyncSession") def test_create_command_exception(self, mock_session_cls): """Test create command exception handling.""" @@ -296,14 +343,17 @@ def test_create_command_exception(self, mock_session_cls): mock_session_cls.return_value.__aenter__.return_value = mock_session mock_session.create.side_effect = httpx.HTTPError("API Error") - result = runner.invoke(create, ["headless", "skaha/worker:v1"]) + result = runner.invoke(cli, ["create", "headless", "skaha/worker:v1"]) assert result.exit_code == 1 assert "Error: API Error" in result.stderr @pytest.mark.parametrize( ("flag", "load"), - [("--json", json.loads), ("--yaml", yaml.safe_load)], + [ + (["--output", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), + ], ) @pytest.mark.parametrize("phase", ["enter", "body", "exit"]) @patch("canfar.cli.create.AsyncSession") @@ -326,7 +376,7 @@ def test_create_command_machine_exception_is_secret_safe_transport_failure( }[phase] failing_call.side_effect = httpx.HTTPError(secret) - result = runner.invoke(create, ["headless", "skaha/worker:v1", flag]) + result = runner.invoke(cli, ["create", "headless", "skaha/worker:v1", *flag]) assert result.exit_code == 1 assert result.stdout == "" @@ -345,15 +395,18 @@ def test_create_command_keyboard_interrupt_keeps_human_exit( mock_session_cls.return_value.__aenter__.return_value = mock_session mock_session.create.side_effect = KeyboardInterrupt - result = runner.invoke(create, ["headless", "skaha/worker:v1"]) + result = runner.invoke(cli, ["create", "headless", "skaha/worker:v1"]) assert result.exit_code == 130 - assert result.stdout == "" + assert result.stdout.startswith("@") assert "Operation cancelled by user" in result.stderr @pytest.mark.parametrize( ("flag", "load"), - [("--json", json.loads), ("--yaml", yaml.safe_load)], + [ + (["--output", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), + ], ) @pytest.mark.parametrize("phase", ["enter", "body", "exit"]) @patch("canfar.cli.create.AsyncSession") @@ -376,7 +429,7 @@ def test_create_command_machine_keyboard_interrupt_is_structured( }[phase] failing_call.side_effect = KeyboardInterrupt(secret) - result = runner.invoke(create, ["headless", "skaha/worker:v1", flag]) + result = runner.invoke(cli, ["create", "headless", "skaha/worker:v1", *flag]) assert result.exit_code == 130 assert result.stdout == "" diff --git a/tests/test_cli_data.py b/tests/test_cli_data.py index 0ee1c871..e6b8e359 100644 --- a/tests/test_cli_data.py +++ b/tests/test_cli_data.py @@ -33,7 +33,6 @@ def _configuration(*storage_names: str) -> SimpleNamespace: storage = dict.fromkeys(storage_names, object()) return SimpleNamespace( servers={"server": SimpleNamespace(storage=storage)}, - storage_identifiers=lambda: [*storage_names, "local"], ) @@ -61,7 +60,6 @@ def source_factory(_name: str) -> object: "first": SimpleNamespace(storage={"arc": object()}), "second": SimpleNamespace(storage={"cavern": object()}), }, - storage_identifiers=lambda: ["arc", "cavern", "local"], ), SimpleNamespace( active=SimpleNamespace(server="second"), @@ -71,7 +69,6 @@ def source_factory(_name: str) -> object: storage={"cavern": object(), "vault": object()} ), }, - storage_identifiers=lambda: ["arc", "cavern", "vault", "local"], ), ] ) @@ -246,41 +243,6 @@ async def unused_source() -> AsyncIterator[AbstractFileSystem]: assert result.stderr == "mv: cross-source move unsupported\n" -@pytest.mark.parametrize("operand", [":/path", "/bare/local/path"]) -def test_data_deprecated_operand_grammar_is_unsupported( - monkeypatch, - operand: str, -) -> None: - """Only the upstream explicit ``name:/absolute/path`` grammar is accepted.""" - monkeypatch.setattr(storage, "Configuration", _configuration) - - result = runner.invoke(cli, ["data", "ls", operand]) - - assert result.exit_code != 0 - - -@pytest.mark.parametrize( - ("arguments", "diagnostic"), - [ - (["storage", "ls"], "No such command 'storage'"), - (["data", "ls", "active:/"], "unknown filesystem"), - (["data", "ls", "-h", "local:/"], "-h: requires long listing"), - ], -) -def test_data_retired_aliases_and_standalone_h_are_unsupported( - monkeypatch, - arguments: list[str], - diagnostic: str, -) -> None: - """Retired host aliases do not widen the upstream mapped grammar.""" - monkeypatch.setattr(storage, "Configuration", _configuration) - - result = runner.invoke(cli, arguments) - - assert result.exit_code == 2 - assert diagnostic in result.stderr - - def test_importing_data_module_does_not_load_configuration() -> None: """Registering the command performs no configuration filesystem I/O.""" package = sys.modules["canfar.cli"] diff --git a/tests/test_cli_dead_modules.py b/tests/test_cli_dead_modules.py deleted file mode 100644 index 5b22958c..00000000 --- a/tests/test_cli_dead_modules.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Characterization tests guarding the dead-module cleanup (issue #137). - -Two empty placeholder modules (``canfar/cli/run.py`` and ``canfar/cli/alias.py``) -are removed by this cleanup. Neither is imported anywhere; the ``run``/``launch`` -and ``del`` CLI aliases are wired in :mod:`canfar.cli.main` via the -:class:`~canfar.hooks.typer.aliases.AliasGroup`, not by those files. These tests -pin both invariants so the deletion cannot silently regress the alias feature. -""" - -from __future__ import annotations - -import importlib.util - -import pytest -from typer.testing import CliRunner - -from canfar.cli.main import cli - -runner = CliRunner() - - -@pytest.mark.parametrize("module", ["canfar.cli.run", "canfar.cli.alias"]) -def test_dead_cli_module_is_absent(module: str) -> None: - """The empty placeholder modules must not be importable after cleanup.""" - assert importlib.util.find_spec(module) is None - - -@pytest.mark.parametrize("alias", ["run", "launch", "del"]) -def test_cli_aliases_still_resolve(alias: str) -> None: - """The ``run``/``launch``/``del`` aliases keep resolving through the app.""" - result = runner.invoke(cli, [alias, "--help"]) - assert result.exit_code == 0 - assert "Usage" in result.stdout diff --git a/tests/test_cli_delete.py b/tests/test_cli_delete.py index 3156729b..dccba400 100644 --- a/tests/test_cli_delete.py +++ b/tests/test_cli_delete.py @@ -6,7 +6,7 @@ from typer.testing import CliRunner -from canfar.cli.delete import delete +from canfar.cli.main import cli runner = CliRunner() @@ -22,7 +22,7 @@ def test_delete_force_success_error_and_cancel() -> None: with patch("canfar.cli.delete.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.destroy.return_value = {"abc": True} - result = runner.invoke(delete, ["--force", "abc"]) + result = runner.invoke(cli, ["delete", "--force", "abc"]) assert result.exit_code == 0 assert "Successfully deleted" in result.stdout @@ -30,12 +30,12 @@ def test_delete_force_success_error_and_cancel() -> None: with patch("canfar.cli.delete.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.destroy.side_effect = RuntimeError("delete failed") - result = runner.invoke(delete, ["--force", "abc"]) + result = runner.invoke(cli, ["delete", "--force", "abc"]) assert result.exit_code == 0 assert "Error during deletion: delete failed" in result.stderr with patch("canfar.cli.delete.Confirm.ask", return_value=False): - result = runner.invoke(delete, ["abc"]) + result = runner.invoke(cli, ["delete", "abc"]) assert result.exit_code == 0 diff --git a/tests/test_cli_events.py b/tests/test_cli_events.py index b7258701..7fb9ca97 100644 --- a/tests/test_cli_events.py +++ b/tests/test_cli_events.py @@ -4,7 +4,7 @@ from typer.testing import CliRunner -from canfar.cli.events import events +from canfar.cli.main import cli runner = CliRunner() @@ -28,7 +28,7 @@ def test_events_command_success(self, mock_session_cls): ] mock_session.events.return_value = mock_events - result = runner.invoke(events, ["session-id"]) + result = runner.invoke(cli, ["events", "session-id"]) assert result.exit_code == 0 assert "Server events for session-id" in result.stdout @@ -42,7 +42,7 @@ def test_events_command_no_events(self, mock_session_cls): mock_session_cls.return_value.__aenter__.return_value = mock_session mock_session.events.return_value = [] - result = runner.invoke(events, ["session-id"]) + result = runner.invoke(cli, ["events", "session-id"]) assert result.exit_code == 0 assert "No events found" in result.stderr diff --git a/tests/test_cli_help.py b/tests/test_cli_help.py index d837f9d7..a615090e 100644 --- a/tests/test_cli_help.py +++ b/tests/test_cli_help.py @@ -14,7 +14,6 @@ ["login"], ["auth"], ["auth", "show"], - ["auth", "login"], ["auth", "ls"], ["auth", "use"], ["auth", "rm"], diff --git a/tests/test_cli_info.py b/tests/test_cli_info.py index 4d6ea8f3..251b7b40 100644 --- a/tests/test_cli_info.py +++ b/tests/test_cli_info.py @@ -5,7 +5,8 @@ from typer.testing import CliRunner -from canfar.cli.info import _format, _utilization, info +from canfar.cli.info import _format, _utilization +from canfar.cli.main import cli from canfar.models.session import FetchResponse runner = CliRunner() @@ -75,7 +76,7 @@ def test_info_command_success(self, mock_session_cls): } mock_session.info.return_value = [mock_response] - result = runner.invoke(info, ["test-id"]) + result = runner.invoke(cli, ["info", "test-id"]) assert result.exit_code == 0 assert "test-id" in result.stdout @@ -88,7 +89,7 @@ def test_info_command_not_found(self, mock_session_cls): mock_session_cls.return_value.__aenter__.return_value = mock_session mock_session.info.return_value = [] - result = runner.invoke(info, ["non-existent"]) + result = runner.invoke(cli, ["info", "non-existent"]) assert result.exit_code == 0 assert "No information found" in result.stderr @@ -100,7 +101,7 @@ def test_info_debug_retains_response_anomaly_details(self, mock_session_cls): mock_session_cls.return_value.__aenter__.return_value = mock_session mock_session.info.return_value = [{"id": "test-id", "name": "test-name"}] - result = runner.invoke(info, ["test-id", "--debug"]) + result = runner.invoke(cli, ["info", "test-id", "--debug"]) assert result.exit_code == 0 assert "Session Response Warnings" in result.stderr diff --git a/tests/test_cli_login.py b/tests/test_cli_login.py index 9e2094a7..712b7c7b 100644 --- a/tests/test_cli_login.py +++ b/tests/test_cli_login.py @@ -45,11 +45,11 @@ def test_login_help_is_available() -> None: @pytest.mark.parametrize( "arguments", - [["login", "cadc"], ["auth", "login", "cadc"]], - ids=["canonical", "compatibility-alias"], + [["login", "cadc"]], + ids=["canonical"], ) def test_login_defaults_to_ten_second_timeout(arguments: list[str]) -> None: - """Canonical and compatibility login commands default to ten seconds.""" + """The canonical login command defaults to ten seconds.""" with patch("canfar.cli.login._login_flow") as login_flow: result = runner.invoke(cli, arguments) @@ -101,57 +101,6 @@ def test_login_without_config_file_does_not_require_force(tmp_path: Path) -> Non assert "already exists" not in result.stdout -def test_auth_login_alias_delegates_to_login_flow(tmp_path: Path) -> None: - """``canfar auth login`` remains a compatibility alias.""" - config_path = tmp_path / "config.yaml" - credential = X509Credential( - idp="cadc", - path=Path("/new/cert.pem"), - expiry=123.0, - ) - discovered = [ - Server( - idp="cadc", - name="CADC-CANFAR", - uri=AnyUrl(_CADC_URI), - url=AnyHttpUrl("https://ws-uv.canfar.net/skaha"), - version="v1", - auths=["x509"], - ) - ] - validated = discovered[0].model_copy(deep=True) - - with ( - _patch_config(config_path), - patch("canfar.cli.login.CONFIG_PATH", config_path), - patch("canfar.models.config.CONFIG_PATH", config_path), - patch("canfar.cli.login.authenticate_for_cli", return_value=credential), - patch("canfar.server._validate_server", return_value=validated), - patch( - "canfar.cli.login.discover", - side_effect=lambda idp, *, config, **_kwargs: ( - _merge_servers( - config, - discovered, - idp, - ) - or discovered - ), - ), - ): - result = runner.invoke(cli, ["auth", "login", "cadc", "--force"]) - - assert result.exit_code == 0 - assert "canfar auth login will be removed soon" in result.stderr - assert "canfar login" in result.stderr - with ( - patch("canfar.models.config.CONFIG_PATH", config_path), - ): - saved = Configuration() - assert saved.active.authentication == "cadc" - assert saved.active.server == "CADC-CANFAR" - - def test_login_saves_auth_and_server_atomically(tmp_path: Path) -> None: """Login persists active Authentication and Server in one save.""" config_path = tmp_path / "config.yaml" @@ -197,7 +146,7 @@ def test_login_saves_auth_and_server_atomically(tmp_path: Path) -> None: saved = Configuration() assert saved.active.authentication == "cadc" assert saved.active.server == "CADC-CANFAR" - assert saved.get_credential("cadc").path == Path("/new/cert.pem") + assert saved.authentication["cadc"].path == Path("/new/cert.pem") def test_login_passes_dev_and_timeout_to_http_steps(tmp_path: Path) -> None: @@ -262,18 +211,6 @@ def discover( assert isinstance(validate.call_args.kwargs["config"], Configuration) -def test_auth_login_alias_passes_dev_and_timeout_to_login_flow() -> None: - """Compatibility alias keeps the same discovery options as canfar login.""" - with patch("canfar.cli.login._login_flow") as login_flow: - result = runner.invoke( - cli, - ["auth", "login", "cadc", "--dev", "--timeout", "7"], - ) - - assert result.exit_code == 0 - login_flow.assert_called_once_with("cadc", force=False, dev=True, timeout=7) - - def test_login_existing_without_force_exits_nonzero(tmp_path: Path) -> None: """Repeated login without --force is rejected.""" config_path = tmp_path / "config.yaml" @@ -310,3 +247,22 @@ def test_login_existing_without_force_exits_nonzero(tmp_path: Path) -> None: assert result.exit_code == 1 assert "already exists" in result.stderr + + +def test_login_presents_device_flow_failure_on_terminal(tmp_path: Path) -> None: + """CLI login renders terminal device-flow failures on stderr.""" + config_path = tmp_path / "config.yaml" + + with ( + _patch_config(config_path), + patch("canfar.cli.login.CONFIG_PATH", config_path), + patch("canfar.models.config.CONFIG_PATH", config_path), + patch( + "canfar.cli.login.authenticate_for_cli", + side_effect=PermissionError("OIDC device authorization was denied"), + ), + ): + result = runner.invoke(cli, ["login", "srcnet"]) + + assert result.exit_code == 1 + assert "OIDC device authorization was denied" in result.stderr diff --git a/tests/test_cli_logs.py b/tests/test_cli_logs.py index 732bf2d3..18d514c8 100644 --- a/tests/test_cli_logs.py +++ b/tests/test_cli_logs.py @@ -6,7 +6,7 @@ from typer.testing import CliRunner -from canfar.cli.logs import logs +from canfar.cli.main import cli runner = CliRunner() @@ -22,7 +22,7 @@ def test_logs_outputs_logs_and_empty_message() -> None: with patch("canfar.cli.logs.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.logs.return_value = {"abc": "hello\nworld"} - result = runner.invoke(logs, ["abc"]) + result = runner.invoke(cli, ["logs", "abc"]) assert result.exit_code == 0 assert "Logs for session abc" in result.stdout @@ -31,7 +31,7 @@ def test_logs_outputs_logs_and_empty_message() -> None: with patch("canfar.cli.logs.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.logs.return_value = {} - result = runner.invoke(logs, ["abc"]) + result = runner.invoke(cli, ["logs", "abc"]) assert result.exit_code == 0 assert "No logs found" in result.stderr @@ -42,7 +42,7 @@ def test_logs_reports_fetch_error() -> None: with patch("canfar.cli.logs.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.logs.side_effect = RuntimeError("boom") - result = runner.invoke(logs, ["abc"]) + result = runner.invoke(cli, ["logs", "abc"]) assert result.exit_code == 1 assert "Could not fetch logs" not in result.stdout diff --git a/tests/test_cli_machine_output.py b/tests/test_cli_machine_output.py index 7a74da7e..63d944a1 100644 --- a/tests/test_cli_machine_output.py +++ b/tests/test_cli_machine_output.py @@ -130,12 +130,17 @@ def test_invalid_console_banner_value_fails_validation(tmp_path: Path) -> None: @pytest.mark.parametrize( ("flag", "parser"), - [("--json", json.loads), ("--yaml", yaml.safe_load)], + [ + (["-o", "json"], json.loads), + (["--output", "json"], json.loads), + (["-o", "yaml"], yaml.safe_load), + (["--output", "yaml"], yaml.safe_load), + ], ) @pytest.mark.parametrize("banner", [True, False]) def test_auth_ls_machine_stdout_is_data_only( tmp_path: Path, - flag: str, + flag: list[str], parser: Callable[[str], object], banner: bool, ) -> None: @@ -144,15 +149,25 @@ def test_auth_ls_machine_stdout_is_data_only( _write_config(config_path, banner=banner) with _patch_config(config_path): - result = runner.invoke(cli, ["auth", "ls", flag]) + result = runner.invoke(cli, ["auth", "ls", *flag]) assert result.exit_code == 0 assert not result.stdout.startswith("@") + assert result.stderr == "" parser(result.stdout) -def test_passthrough_json_argument_keeps_human_banner(tmp_path: Path) -> None: - """A container argument named ``--json`` does not select machine output.""" +def test_auth_ls_invalid_output_format_is_rejected() -> None: + """Output formats are validated at the CLI boundary.""" + result = runner.invoke(cli, ["auth", "ls", "--output", "toml"]) + + assert result.exit_code == 2 + assert result.stdout == "" + assert "Invalid value" in click.unstyle(result.stderr) + + +def test_passthrough_output_options_keep_human_banner(tmp_path: Path) -> None: + """Output-looking container arguments remain verbatim after ``--``.""" config_path = tmp_path / "config.yaml" _write_config(config_path) @@ -166,16 +181,19 @@ def test_passthrough_json_argument_keeps_human_banner(tmp_path: Path) -> None: "example.invalid/image", "--", "echo", - "--json", + "-o", + "yaml", + "--output", + "json", ], ) assert result.exit_code == 0 assert result.stdout.startswith("@CADC-CANFAR") - assert "Arguments: --json" in result.stdout + assert "Arguments: -o yaml --output json" in result.stdout -@pytest.mark.parametrize("name", ["--json", "--yaml"]) +@pytest.mark.parametrize("name", ["--output", "--json"]) def test_machine_flag_spelling_as_option_value_keeps_human_banner( tmp_path: Path, name: str, @@ -203,18 +221,22 @@ def test_machine_flag_spelling_as_option_value_keeps_human_banner( def test_auth_group_flag_before_subcommand_is_rejected(tmp_path: Path) -> None: - """Group-level ``--json``/``--yaml`` placement exits 2 with guidance.""" + """Group-level output placement exits 2 with guidance.""" config_path = tmp_path / "config.yaml" _write_config(config_path) with _patch_config(config_path): - json_result = runner.invoke(cli, ["auth", "--json", "ls"]) - yaml_result = runner.invoke(cli, ["auth", "--yaml", "show"]) + json_result = runner.invoke(cli, ["auth", "-o", "json", "ls"]) + yaml_result = runner.invoke(cli, ["auth", "--output", "yaml", "show"]) assert json_result.exit_code == 2 - assert "Place --json or --yaml after the subcommand." in json_result.stderr + assert "Place --output json or --output yaml after the subcommand." in ( + json_result.stderr + ) assert yaml_result.exit_code == 2 - assert "Place --json or --yaml after the subcommand." in yaml_result.stderr + assert "Place --output json or --output yaml after the subcommand." in ( + yaml_result.stderr + ) def test_ps_human_mode_emits_banner(tmp_path: Path) -> None: @@ -236,13 +258,13 @@ def test_ps_human_mode_emits_banner(tmp_path: Path) -> None: def test_auth_default_json_matches_show(tmp_path: Path) -> None: - """Default ``auth`` emits the same payload as ``auth show --json``.""" + """Default ``auth`` emits the same payload as ``auth show -o json``.""" config_path = tmp_path / "config.yaml" _write_config(config_path) with _patch_config(config_path): - default_result = runner.invoke(cli, ["auth", "--json"]) - show_result = runner.invoke(cli, ["auth", "show", "--json"]) + default_result = runner.invoke(cli, ["auth", "-o", "json"]) + show_result = runner.invoke(cli, ["auth", "show", "-o", "json"]) assert default_result.exit_code == 0 assert show_result.exit_code == 0 @@ -250,12 +272,12 @@ def test_auth_default_json_matches_show(tmp_path: Path) -> None: def test_auth_show_json_payload_shape(tmp_path: Path) -> None: - """``auth show --json`` emits a domain Authentication object without envelopes.""" + """``auth show -o json`` emits a domain Authentication object without envelopes.""" config_path = tmp_path / "config.yaml" _write_config(config_path) with _patch_config(config_path): - result = runner.invoke(cli, ["auth", "show", "--json"]) + result = runner.invoke(cli, ["auth", "show", "-o", "json"]) assert result.exit_code == 0 payload = json.loads(result.stdout) @@ -265,12 +287,12 @@ def test_auth_show_json_payload_shape(tmp_path: Path) -> None: def test_auth_ls_json_payload_shape(tmp_path: Path) -> None: - """``auth ls --json`` emits a JSON array of Authentication objects.""" + """``auth ls -o json`` emits a JSON array of Authentication objects.""" config_path = tmp_path / "config.yaml" _write_config(config_path) with _patch_config(config_path): - result = runner.invoke(cli, ["auth", "ls", "--json"]) + result = runner.invoke(cli, ["auth", "ls", "-o", "json"]) assert result.exit_code == 0 payload = json.loads(result.stdout) @@ -282,25 +304,18 @@ def test_auth_ls_json_payload_shape(tmp_path: Path) -> None: assert srcnet["server"] is None -def test_root_json_flag_before_command_path_is_not_supported( +def test_root_output_option_before_command_path_is_not_supported( tmp_path: Path, ) -> None: - """Root ``--json`` is rejected; supported commands own machine output.""" + """Root output options are rejected; supported commands own machine output.""" config_path = tmp_path / "config.yaml" _write_config(config_path) with _patch_config(config_path): - result = runner.invoke(cli, ["--json", "auth", "ls"]) - - assert result.exit_code == 2 - assert "--json" in click.unstyle(result.stderr) - + result = runner.invoke(cli, ["-o", "json", "auth", "ls"]) -def test_conflicting_output_flags_exit_two() -> None: - """Conflicting machine output flags exit with code 2.""" - result = runner.invoke(cli, ["auth", "ls", "--json", "--yaml"]) assert result.exit_code == 2 - assert "Conflicting machine output flags" in result.stderr + assert "-o" in click.unstyle(result.stderr) def test_unsupported_command_rejects_leaf_json_flag() -> None: @@ -308,3 +323,12 @@ def test_unsupported_command_rejects_leaf_json_flag() -> None: result = runner.invoke(cli, ["auth", "purge", "--json", "--force"]) assert result.exit_code == 2 assert "--json" in click.unstyle(result.stderr) + + +def test_unsupported_command_rejects_leaf_output_option() -> None: + """Commands without machine flags let Click reject the ``-o`` option.""" + result = runner.invoke(cli, ["auth", "purge", "-o", "json", "--force"]) + + assert result.exit_code == 2 + assert result.stdout == "" + assert "No such option: -o" in click.unstyle(result.stderr) diff --git a/tests/test_cli_main.py b/tests/test_cli_main.py index b32701ab..de902250 100644 --- a/tests/test_cli_main.py +++ b/tests/test_cli_main.py @@ -106,13 +106,6 @@ def test_human_cli_runs_when_active_server_is_null(tmp_path: Path) -> None: assert result.stdout.startswith("@unknown") -def test_context_command_group_is_removed() -> None: - """``canfar context`` is no longer a supported command group.""" - result = runner.invoke(cli, ["context", "show"]) - assert result.exit_code != 0 - assert "No such command 'context'" in result.output - - def test_version_debug_retains_bug_report_diagnostics() -> None: """The domain-specific version flag remains separate from root logging.""" result = runner.invoke(cli, ["version", "--debug"]) diff --git a/tests/test_cli_open.py b/tests/test_cli_open.py index 459b37a5..3f8a61c1 100644 --- a/tests/test_cli_open.py +++ b/tests/test_cli_open.py @@ -6,7 +6,7 @@ from typer.testing import CliRunner -from canfar.cli.open import open_command +from canfar.cli.main import cli runner = CliRunner() @@ -37,7 +37,7 @@ def test_open_command_opens_url_and_reports_missing_data() -> None: }, {"id": "missing"}, ] - result = runner.invoke(open_command, ["abc", "stopped", "missing"]) + result = runner.invoke(cli, ["open", "abc", "stopped", "missing"]) assert result.exit_code == 0 assert "Opening session abc" in result.stdout @@ -52,7 +52,7 @@ def test_open_command_no_session_info() -> None: with patch("canfar.cli.open.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.info.return_value = [] - result = runner.invoke(open_command, ["abc"]) + result = runner.invoke(cli, ["open", "abc"]) assert result.exit_code == 0 assert "No information found" in result.stderr diff --git a/tests/test_cli_output.py b/tests/test_cli_output.py index eace57fd..9d3a6026 100644 --- a/tests/test_cli_output.py +++ b/tests/test_cli_output.py @@ -6,7 +6,6 @@ from io import StringIO import pytest -import typer import yaml from pydantic import BaseModel @@ -31,27 +30,20 @@ def test_output_mode_values() -> None: def test_resolve_mode_defaults_to_human() -> None: - """No machine flags resolve to human output mode.""" - assert machine.resolve_mode(json_output=False, yaml_output=False) == ( - output.OutputMode.HUMAN - ) + """No output option resolves to human output mode.""" + assert machine.resolve_mode(None) == output.OutputMode.HUMAN def test_resolve_mode_json_and_yaml() -> None: - """Single machine flags select the matching output mode.""" - assert machine.resolve_mode(json_output=True, yaml_output=False) == ( - output.OutputMode.JSON - ) - assert machine.resolve_mode(json_output=False, yaml_output=True) == ( - output.OutputMode.YAML - ) + """The output option selects the matching machine output mode.""" + assert machine.resolve_mode("json") == output.OutputMode.JSON + assert machine.resolve_mode("yaml") == output.OutputMode.YAML -def test_resolve_mode_conflict_exits_two() -> None: - """Conflicting machine output flags exit with code 2.""" - with pytest.raises(typer.Exit) as exc_info: - machine.resolve_mode(json_output=True, yaml_output=True) - assert exc_info.value.exit_code == output.OUTPUT_CONFLICT_EXIT_CODE +def test_resolve_mode_rejects_unknown_format() -> None: + """The resolver rejects formats outside the extensible output contract.""" + with pytest.raises(ValueError, match="not a valid OutputMode"): + machine.resolve_mode("toml") def test_model_dump_includes_null_fields() -> None: @@ -94,13 +86,13 @@ def test_render_stderr_error_json_on_stderr_channel() -> None: error = StructuredError( code="output.conflict", message="Conflicting output flags.", - hint="Use only one of --json or --yaml.", + hint="Use --output json or --output yaml.", ) rendered = output.render_stderr_error(error, output.OutputMode.JSON) payload = json.loads(rendered) assert payload["code"] == "output.conflict" assert payload["message"] == "Conflicting output flags." - assert payload["hint"] == "Use only one of --json or --yaml." + assert payload["hint"] == "Use --output json or --output yaml." def test_render_stderr_error_yaml_on_stderr_channel() -> None: diff --git a/tests/test_cli_prune.py b/tests/test_cli_prune.py index 4876fd17..ae3c215a 100644 --- a/tests/test_cli_prune.py +++ b/tests/test_cli_prune.py @@ -4,9 +4,10 @@ from unittest.mock import AsyncMock, MagicMock, patch +import click from typer.testing import CliRunner -from canfar.cli.prune import PruneUsageMessage, prune +from canfar.cli.main import cli runner = CliRunner() @@ -22,7 +23,7 @@ def test_prune_success_and_usage_message() -> None: with patch("canfar.cli.prune.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.destroy_with.return_value = {"abc": True, "def": False} - result = runner.invoke(prune, ["batch", "headless", "Succeeded"]) + result = runner.invoke(cli, ["prune", "batch", "headless", "Succeeded"]) assert result.exit_code == 0 assert "Deleted 2 sessions" in result.stdout @@ -30,9 +31,10 @@ def test_prune_success_and_usage_message() -> None: prefix="batch", kind="headless", status="Succeeded" ) - usage = PruneUsageMessage(name="prune").get_usage(MagicMock()) - assert "canfar prune" in usage - - help_result = runner.invoke(prune, ["--help"]) + help_result = runner.invoke(cli, ["prune", "--help"]) assert help_result.exit_code == 0 - assert "canfar prune 'session.*' notebook Running" in help_result.output + help_text = " ".join(click.unstyle(help_result.output).split()) + assert ( + "Usage: canfar prune [OPTIONS] PREFIX KIND STATUS COMMAND [ARGS]..." + in help_text + ) diff --git a/tests/test_cli_root.py b/tests/test_cli_root.py new file mode 100644 index 00000000..58ee8bda --- /dev/null +++ b/tests/test_cli_root.py @@ -0,0 +1,140 @@ +"""Root CLI composition contracts for issue #255.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, patch + +import click +from typer.core import TyperGroup +from typer.main import get_command +from typer.testing import CliRunner + +from canfar.cli.main import cli + +runner = CliRunner() + + +def test_root_help_lists_leaf_commands_without_alias_section() -> None: + """Canonical leaf commands remain discoverable at the root.""" + result = runner.invoke(cli, ["--help"]) + + assert result.exit_code == 0 + for command in ( + "create", + "ps", + "events", + "info", + "open", + "logs", + "delete", + "prune", + "stats", + "version", + ): + assert command in result.stdout + + +def test_management_groups_remain_grouped() -> None: + """Authentication and server management retain their subcommands.""" + root = get_command(cli) + auth_result = runner.invoke(cli, ["auth", "--help"]) + server_result = runner.invoke(cli, ["server", "--help"]) + + assert isinstance(root.commands["auth"], TyperGroup) + assert isinstance(root.commands["server"], TyperGroup) + assert auth_result.exit_code == 0 + assert "show" in auth_result.stdout + assert "ls" in auth_result.stdout + assert "use" in auth_result.stdout + assert server_result.exit_code == 0 + assert "ls" in server_result.stdout + assert "use" in server_result.stdout + + +def test_session_and_information_leaves_are_root_commands() -> None: + """Leaf callbacks are commands, not one-callback child groups.""" + root = get_command(cli) + + for name in ( + "create", + "ps", + "events", + "info", + "open", + "logs", + "delete", + "prune", + "stats", + "version", + ): + assert name in root.commands + assert not isinstance(root.commands[name], TyperGroup) + + +def test_create_root_usage_preserves_delimiter_contract() -> None: + """The root create command reserves every token after ``--``.""" + result = runner.invoke( + cli, + [ + "create", + "--dry-run", + "headless", + "example.invalid/image", + "--", + "echo", + "--output", + "json", + "-o", + "yaml", + ], + ) + + assert result.exit_code == 0 + assert "Command: echo" in result.stdout + assert "Arguments: --output json -o yaml" in result.stdout + + +@patch("canfar.cli.create.AsyncSession") +def test_root_create_output_stops_at_delimiter(mock_session_cls: AsyncMock) -> None: + """Root create parses output before ``--`` and reserves the rest.""" + mock_session = AsyncMock() + mock_session_cls.return_value.__aenter__.return_value = mock_session + mock_session.create.return_value = ["session-id"] + + result = runner.invoke( + cli, + [ + "create", + "headless", + "example.invalid/image", + "--output", + "json", + "--", + "echo", + "-o", + "yaml", + "--output", + "json", + ], + ) + + assert result.exit_code == 0 + assert json.loads(result.stdout) == ["session-id"] + request = mock_session.create.await_args.args[0] + assert request.cmd == "echo" + assert request.args == "-o yaml --output json" + + +def test_root_leaf_help_keeps_canonical_usage() -> None: + """Direct leaf commands retain their documented usage lines.""" + create_result = runner.invoke(cli, ["create", "--help"]) + prune_result = runner.invoke(cli, ["prune", "--help"]) + create_help = " ".join(click.unstyle(create_result.stdout).split()) + prune_help = " ".join(click.unstyle(prune_result.stdout).split()) + + assert create_result.exit_code == 0 + assert "Usage: canfar create [OPTIONS] KIND IMAGE [-- CMD [ARGS]...]" in create_help + assert prune_result.exit_code == 0 + prune_usage = "Usage: canfar prune [OPTIONS] PREFIX KIND STATUS COMMAND [ARGS]..." + assert prune_usage in prune_help diff --git a/tests/test_cli_server.py b/tests/test_cli_server.py index 58999efc..867fbc07 100644 --- a/tests/test_cli_server.py +++ b/tests/test_cli_server.py @@ -130,12 +130,12 @@ def test_server_use_selects_by_name(tmp_path: Path) -> None: def test_server_ls_json_output(tmp_path: Path) -> None: - """``server ls --json`` emits a JSON array of Server objects on stdout.""" + """``server ls -o json`` emits a JSON array of Server objects on stdout.""" config_path = tmp_path / "config.yaml" _write_config(config_path) with _patch_config(config_path): - result = runner.invoke(cli, ["server", "ls", "--json"]) + result = runner.invoke(cli, ["server", "ls", "-o", "json"]) assert result.exit_code == 0 payload = json.loads(result.stdout) @@ -165,8 +165,8 @@ def test_server_ls_machine_output_includes_server_name(tmp_path: Path) -> None: _write_config(config_path) with _patch_config(config_path): - json_result = runner.invoke(cli, ["server", "ls", "--json"]) - yaml_result = runner.invoke(cli, ["server", "ls", "--yaml"]) + json_result = runner.invoke(cli, ["server", "ls", "--output", "json"]) + yaml_result = runner.invoke(cli, ["server", "ls", "--output", "yaml"]) assert json_result.exit_code == 0 assert json.loads(json_result.stdout)[0]["name"] == "CADC-CANFAR" diff --git a/tests/test_cli_stats_ps.py b/tests/test_cli_stats_ps.py index a2bada34..84c537e2 100644 --- a/tests/test_cli_stats_ps.py +++ b/tests/test_cli_stats_ps.py @@ -6,13 +6,12 @@ from typing import TYPE_CHECKING from unittest.mock import AsyncMock, MagicMock, patch +import click import pytest import yaml from typer.testing import CliRunner from canfar.cli.main import cli -from canfar.cli.ps import ps -from canfar.cli.stats import stats if TYPE_CHECKING: from collections.abc import Callable @@ -81,10 +80,13 @@ def test_stats_command_help() -> None: assert result.exit_code == 0 -def test_ps_command_help() -> None: - """Test ps command help executes successfully.""" +def test_ps_help_describes_default_status_filter() -> None: + """Test ps help names the statuses shown by the default filter.""" result = runner.invoke(cli, ["ps", "--help"]) + assert result.exit_code == 0 + help_text = " ".join(click.unstyle(result.output).replace("│", " ").split()) + assert "default shows Pending and Running" in help_text def test_ps_outputs_running_table_and_debug_anomalies() -> None: @@ -114,7 +116,7 @@ def test_ps_outputs_running_table_and_debug_anomalies() -> None: with patch("canfar.cli.ps.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.fetch.return_value = payloads - result = runner.invoke(ps, ["--debug"]) + result = runner.invoke(cli, ["ps", "--debug"]) assert result.exit_code == 0 assert "running-1" in result.stdout @@ -152,27 +154,34 @@ def test_ps_quiet_prints_all_matching_session_ids() -> None: with patch("canfar.cli.ps.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.fetch.return_value = payloads - result = runner.invoke(ps, ["--quiet"]) + result = runner.invoke(cli, ["ps", "--quiet"]) assert result.exit_code == 0 - assert result.stdout.splitlines() == ["running-1", "running-2"] + lines = result.stdout.splitlines() + assert lines[0].startswith("@") + assert lines[1:] == ["running-1", "running-2"] with patch("canfar.cli.ps.AsyncSession") as session_cls: session = _mock_async_session(session_cls) session.fetch.return_value = payloads - result = runner.invoke(ps, ["--quiet", "--all"]) + result = runner.invoke(cli, ["ps", "--quiet", "--all"]) assert result.exit_code == 0 - assert result.stdout.splitlines() == ["running-1", "done-1", "running-2"] + lines = result.stdout.splitlines() + assert lines[0].startswith("@") + assert lines[1:] == ["running-1", "done-1", "running-2"] @pytest.mark.parametrize( ("flag", "load"), - [("--json", json.loads), ("--yaml", yaml.safe_load)], + [ + (["-o", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), + ], ) def test_ps_machine_emits_filtered_session_array( tmp_path: Path, - flag: str, + flag: list[str], load: Callable[[str], object], ) -> None: """Machine ``ps`` emits validated session models with running-only filtering.""" @@ -188,7 +197,7 @@ def test_ps_machine_emits_filtered_session_array( ): session = _mock_async_session(session_cls) session.fetch.return_value = payloads - result = runner.invoke(ps, [flag]) + result = runner.invoke(cli, ["ps", *flag]) assert result.exit_code == 0 assert not result.stdout.startswith("@") @@ -212,7 +221,7 @@ def test_ps_json_kind_filter_parity(tmp_path: Path) -> None: ): session = _mock_async_session(session_cls) session.fetch.return_value = [payloads[0]] - result = runner.invoke(ps, ["--kind", "headless", "--json"]) + result = runner.invoke(cli, ["ps", "--kind", "headless", "--output", "json"]) assert result.exit_code == 0 session.fetch.assert_awaited_once_with(kind="headless", status=None) @@ -235,7 +244,7 @@ def test_ps_json_malformed_payload_keeps_stdout_pure(tmp_path: Path) -> None: ): session = _mock_async_session(session_cls) session.fetch.return_value = payloads - result = runner.invoke(ps, ["--json"]) + result = runner.invoke(cli, ["ps", "--output", "json"]) assert result.exit_code == 0 data = json.loads(result.stdout) @@ -244,9 +253,9 @@ def test_ps_json_malformed_payload_keeps_stdout_pure(tmp_path: Path) -> None: assert "validation error" in result.stderr.lower() -def test_ps_quiet_with_json_exits_two() -> None: - """``ps --quiet`` is incompatible with machine output flags.""" - result = runner.invoke(ps, ["--quiet", "--json"]) +def test_ps_quiet_with_machine_output_exits_two() -> None: + """``ps --quiet`` is incompatible with machine output.""" + result = runner.invoke(cli, ["ps", "--quiet", "--output", "json"]) assert result.exit_code == 2 assert "quiet" in result.stderr.lower() @@ -264,7 +273,7 @@ def test_ps_allows_empty_running_view() -> None: "isFixedResources": True, }, ] - result = runner.invoke(ps, []) + result = runner.invoke(cli, ["ps"]) assert result.exit_code == 0 assert "No pending or running sessions found" in result.stderr @@ -279,7 +288,7 @@ def test_stats_outputs_cluster_tables() -> None: "cores": {"requestedCPUCores": 4, "cpuCoresAvailable": 64}, "ram": {"requestedRAM": "8Gi", "ramAvailable": "128Gi"}, } - result = runner.invoke(stats, []) + result = runner.invoke(cli, ["stats"]) assert result.exit_code == 0 assert "CANFAR Platform Load" in result.stdout @@ -301,7 +310,7 @@ def test_stats_renders_only_cpu_and_ram_columns() -> None: "cores": {"requestedCPUCores": 4, "cpuCoresAvailable": 64}, "ram": {"requestedRAM": "8Gi", "ramAvailable": "128Gi"}, } - result = runner.invoke(stats, []) + result = runner.invoke(cli, ["stats"]) assert result.exit_code == 0 # The CPU/RAM table and its values are rendered. diff --git a/tests/test_cli_stream_contracts.py b/tests/test_cli_stream_contracts.py index b3e9d30d..04d2a1a2 100644 --- a/tests/test_cli_stream_contracts.py +++ b/tests/test_cli_stream_contracts.py @@ -92,8 +92,8 @@ def _client_factory( ("flags", "load"), [ ([], None), - (["--json"], json.loads), - (["--yaml"], yaml.safe_load), + (["-o", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), ], ) def test_config_success_warning_and_failure_keep_stream_contracts( @@ -157,13 +157,13 @@ def test_config_success_warning_and_failure_keep_stream_contracts( @pytest.mark.parametrize( ("flag", "load"), [ - ("--json", json.loads), - ("--yaml", yaml.safe_load), + (["-o", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), ], ) def test_real_ps_log_and_payload_stay_on_separate_streams( tmp_path: Path, - flag: str, + flag: list[str], load: Callable[[str], Any], ) -> None: """A real HTTP-backed command emits logs beside one machine payload.""" @@ -181,7 +181,7 @@ def response(request: httpx.Request) -> httpx.Response: side_effect=_async_client_factory(httpx.MockTransport(response)), ), ): - result = runner.invoke(cli, ["-vvvv", "ps", flag]) + result = runner.invoke(cli, ["-vvvv", "ps", *flag]) assert result.exit_code == 0 assert load(result.stdout) == [] @@ -192,13 +192,13 @@ def response(request: httpx.Request) -> httpx.Response: @pytest.mark.parametrize( ("flag", "load"), [ - ("--json", json.loads), - ("--yaml", yaml.safe_load), + (["-o", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), ], ) def test_ps_transport_failure_is_one_structured_machine_error( tmp_path: Path, - flag: str, + flag: list[str], load: Callable[[str], Any], ) -> None: """Expected HTTP transport failure preserves empty stdout and exit one.""" @@ -217,7 +217,7 @@ def unavailable(request: httpx.Request) -> httpx.Response: side_effect=_async_client_factory(httpx.MockTransport(unavailable)), ), ): - result = runner.invoke(cli, ["ps", flag]) + result = runner.invoke(cli, ["ps", *flag]) assert result.exit_code == 1 assert result.stdout == "" @@ -228,13 +228,13 @@ def unavailable(request: httpx.Request) -> httpx.Response: @pytest.mark.parametrize( ("flag", "load"), [ - ("--json", json.loads), - ("--yaml", yaml.safe_load), + (["-o", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), ], ) def test_fresh_server_discovery_keeps_progress_out_of_machine_payload( tmp_path: Path, - flag: str, + flag: list[str], load: Callable[[str], Any], ) -> None: """Fresh registry discovery emits progress on stderr and data on stdout.""" @@ -279,7 +279,7 @@ def capability_response(request: httpx.Request) -> httpx.Response: side_effect=_client_factory(httpx.MockTransport(capability_response)), ), ): - result = runner.invoke(cli, ["server", "ls", flag]) + result = runner.invoke(cli, ["server", "ls", *flag]) assert result.exit_code == 0, result.stderr payload = load(result.stdout) @@ -317,13 +317,13 @@ def unavailable(request: httpx.Request) -> httpx.Response: @pytest.mark.parametrize( ("flag", "load"), [ - ("--json", json.loads), - ("--yaml", yaml.safe_load), + (["-o", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), ], ) def test_fresh_server_discovery_failure_is_one_structured_machine_error( tmp_path: Path, - flag: str, + flag: list[str], load: Callable[[str], Any], ) -> None: """Registry transport failure emits one machine-readable boundary error.""" @@ -341,7 +341,7 @@ def unavailable(request: httpx.Request) -> httpx.Response: side_effect=_async_client_factory(httpx.MockTransport(unavailable)), ), ): - result = runner.invoke(cli, ["server", "ls", flag]) + result = runner.invoke(cli, ["server", "ls", *flag]) assert result.exit_code == 1 assert result.stdout == "" @@ -352,11 +352,11 @@ def unavailable(request: httpx.Request) -> httpx.Response: @pytest.mark.parametrize( "command", [ - ["config", "get", "console.width", "--json"], - ["auth", "show", "--json"], - ["auth", "ls", "--json"], - ["server", "ls", "--json"], - ["ps", "--json"], + ["config", "get", "console.width", "--output", "json"], + ["auth", "show", "--output", "json"], + ["auth", "ls", "--output", "json"], + ["server", "ls", "--output", "json"], + ["ps", "--output", "json"], ], ) def test_malformed_config_is_structured_for_every_machine_command( @@ -379,19 +379,19 @@ def test_malformed_config_is_structured_for_every_machine_command( @pytest.mark.parametrize( ("flag", "load"), [ - ("--json", json.loads), - ("--yaml", yaml.safe_load), + (["-o", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), ], ) def test_invalid_logging_environment_is_one_structured_machine_error( monkeypatch: pytest.MonkeyPatch, - flag: str, + flag: list[str], load: Callable[[str], Any], ) -> None: - """Root dispatch preserves parsed leaf args for callback setup failures.""" + """Machine-capable root groups keep setup diagnostics structured.""" monkeypatch.setenv("CANFAR_LOGLEVEL", "chatty") - result = runner.invoke(cli, ["config", "get", "console.width", flag]) + result = runner.invoke(cli, ["config", "get", "console.width", *flag]) assert result.exit_code == 2 assert result.stdout == "" @@ -402,6 +402,68 @@ def test_invalid_logging_environment_is_one_structured_machine_error( assert error.expected == ["critical", "error", "warning", "info", "debug"] +@pytest.mark.parametrize( + "command", [["config", "get", "console.width"], ["auth", "ls"]] +) +def test_invalid_logging_environment_is_human_without_leaf_output( + monkeypatch: pytest.MonkeyPatch, + command: list[str], +) -> None: + """Setup failures stay human when a machine-capable leaf has no output flag.""" + monkeypatch.setenv("CANFAR_LOGLEVEL", "chatty") + + result = runner.invoke(cli, command) + + assert result.exit_code == 2 + assert result.stdout == "" + assert result.stderr.startswith("logging.invalid_env_value env_var=CANFAR_LOGLEVEL") + assert not result.stderr.lstrip().startswith("{") + + +@pytest.mark.parametrize( + ("flag", "prefix"), + [(["-o", "json"], "{"), (["--output", "yaml"], "code:")], +) +def test_invalid_logging_environment_uses_leaf_output_format( + monkeypatch: pytest.MonkeyPatch, + flag: list[str], + prefix: str, +) -> None: + """Setup failures use the selected leaf format, not the owning group.""" + monkeypatch.setenv("CANFAR_LOGLEVEL", "chatty") + + result = runner.invoke(cli, ["config", "get", "console.width", *flag]) + + assert result.exit_code == 2 + assert result.stdout == "" + assert result.stderr.lstrip().startswith(prefix) + + +def test_setup_failure_ignores_machine_option_after_command_delimiter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Passthrough command arguments cannot change root setup diagnostics.""" + monkeypatch.setenv("CANFAR_LOGLEVEL", "chatty") + + result = runner.invoke( + cli, + [ + "create", + "--dry-run", + "headless", + "example.invalid/image", + "--", + "echo", + "--output", + "json", + ], + ) + + assert result.exit_code == 2 + assert result.stdout == "" + assert result.stderr.startswith("logging.invalid_env_value env_var=CANFAR_LOGLEVEL") + + def test_relative_log_file_creates_parents_without_contaminating_stdout( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -417,7 +479,8 @@ def test_relative_log_file_creates_parents_without_contaminating_stdout( "config", "get", "console.width", - "--json", + "--output", + "json", ], ) @@ -430,14 +493,18 @@ def test_relative_log_file_creates_parents_without_contaminating_stdout( @pytest.mark.parametrize(("target", "directory"), [("-", False), ("logs", True)]) @pytest.mark.parametrize( ("flag", "load"), - [(None, None), ("--json", json.loads), ("--yaml", yaml.safe_load)], + [ + (None, None), + (["--output", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), + ], ) def test_invalid_log_file_target_is_a_structured_setup_error( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, target: str, directory: bool, - flag: str | None, + flag: list[str] | None, load: Callable[[str], Any] | None, ) -> None: """Directory and pseudo-file targets fail at the existing setup boundary.""" @@ -446,7 +513,7 @@ def test_invalid_log_file_target_is_a_structured_setup_error( (tmp_path / target).mkdir() args = ["--log-file", target, "config", "get", "console.width"] if flag: - args.append(flag) + args.extend(flag) result = runner.invoke(cli, args) @@ -461,18 +528,22 @@ def test_invalid_log_file_target_is_a_structured_setup_error( @pytest.mark.parametrize( ("flag", "load"), - [(None, None), ("--json", json.loads), ("--yaml", yaml.safe_load)], + [ + (None, None), + (["--output", "json"], json.loads), + (["--output", "yaml"], yaml.safe_load), + ], ) def test_log_file_initialization_failure_keeps_command_running( tmp_path: Path, - flag: str | None, + flag: list[str] | None, load: Callable[[str], Any] | None, ) -> None: """An unavailable sink emits one mode-aware structured warning.""" log_file = tmp_path / "unavailable.jsonl" args = ["--log-file", str(log_file), "config", "get", "console.width"] if flag: - args.append(flag) + args.extend(flag) with patch( "logging.handlers.RotatingFileHandler.__init__", diff --git a/tests/test_cli_version.py b/tests/test_cli_version.py index 6acffa78..9d09e947 100644 --- a/tests/test_cli_version.py +++ b/tests/test_cli_version.py @@ -3,9 +3,7 @@ import pytest from typer.testing import CliRunner -from canfar.cli.version import ( - version, -) +from canfar.cli.main import cli class TestVersionCLI: @@ -18,14 +16,14 @@ def runner(self) -> CliRunner: def test_version_simple_output(self, runner: CliRunner) -> None: """Test simple version output without debug flag.""" - result = runner.invoke(version, []) + result = runner.invoke(cli, ["version"]) assert result.exit_code == 0 assert "CANFAR Python Client" in result.stdout def test_version_debug_output(self, runner: CliRunner) -> None: """Test detailed debug output with --debug flag.""" - result = runner.invoke(version, ["--debug"]) + result = runner.invoke(cli, ["version", "--debug"]) assert result.exit_code == 0 assert "CANFAR Python Client Debug Information" in result.stdout assert "Client Version" in result.stdout diff --git a/tests/test_client.py b/tests/test_client.py index 2c117418..3fe0f83c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,8 +1,5 @@ """Test CANFAR Python Client API.""" -# ruff: noqa: SLF001 -import re -import ssl import tempfile from datetime import datetime, timedelta, timezone from pathlib import Path @@ -18,6 +15,7 @@ from pydantic import AnyUrl, SecretStr, ValidationError from canfar.client import HTTPClient +from canfar.exceptions.context import AuthExpiredError from canfar.models.auth import X509Credential from canfar.models.config import Configuration from canfar.models.http import Server @@ -37,26 +35,24 @@ def _create_client(**kwargs): return _create_client -@pytest.fixture -def mock_httpx_client(): - """Mock httpx.Client to prevent actual network calls.""" - with patch("httpx.Client") as mock_client: - yield mock_client +def _sync_client_factory(transport: httpx.BaseTransport): + """Build native sync HTTPX clients over a supplied transport.""" + return lambda **kwargs: httpx.Client(transport=transport, **kwargs) -@pytest.fixture -def mock_httpx_async_client(): - """Mock httpx.AsyncClient to prevent actual network calls.""" - with patch("httpx.AsyncClient") as mock_async_client: - yield mock_async_client +def _async_client_factory(transport: httpx.BaseTransport): + """Build native async HTTPX clients over a supplied transport.""" + return lambda **kwargs: httpx.AsyncClient(transport=transport, **kwargs) -@pytest.fixture -def mock_cryptography(): - """Mock cryptography functions for certificate validation.""" - with patch("canfar.auth.x509.inspect") as mock_inspect: - mock_inspect.return_value = {"expiry": 9999999999} # Far future - yield mock_inspect +def _response_transport(requests: list[httpx.Request]) -> httpx.MockTransport: + """Record requests and return one successful native HTTPX response.""" + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={"ok": True}, request=request) + + return httpx.MockTransport(respond) class TestInitializationAndConfiguration: @@ -145,6 +141,55 @@ def test_default_certificate(self, canfar_client_fixture) -> None: class TestRuntimeCredentialHandling: """Test runtime credential handling including mutual exclusivity and validation.""" + def test_empty_token_uses_saved_oidc_authentication( + self, + canfar_client_fixture, + ) -> None: + """A sync request uses saved OIDC state when the runtime token is empty.""" + config = oidc_config(idp="oidc") + requests: list[httpx.Request] = [] + + with ( + patch( + "canfar.client.Client", + side_effect=_sync_client_factory(_response_transport(requests)), + ), + canfar_client_fixture(config=config, token=SecretStr("")) as client, + ): + assert not client.uses_runtime_credentials + response = client.client.get("probe") + + assert response.json() == {"ok": True} + assert len(requests) == 1 + request = requests[0] + assert str(request.url) == "https://oidc.example.com//v1/probe" + assert request.headers["Authorization"] == "Bearer access-token" + assert request.headers["X-Skaha-Authentication-Type"] == "OIDC" + + async def test_empty_token_uses_saved_oidc_authentication_async( + self, + canfar_client_fixture, + ) -> None: + """An async request uses saved OIDC state when the runtime token is empty.""" + requests: list[httpx.Request] = [] + + with patch( + "canfar.client.AsyncClient", + side_effect=_async_client_factory(_response_transport(requests)), + ): + async with canfar_client_fixture( + config=oidc_config(idp="oidc"), + token=SecretStr(""), + ) as client: + response = await client.asynclient.get("probe") + + assert response.json() == {"ok": True} + assert len(requests) == 1 + request = requests[0] + assert str(request.url) == "https://oidc.example.com//v1/probe" + assert request.headers["Authorization"] == "Bearer access-token" + assert request.headers["X-Skaha-Authentication-Type"] == "OIDC" + def test_token_only(self, canfar_client_fixture) -> None: """Test instantiation with token only.""" client = canfar_client_fixture( @@ -211,15 +256,6 @@ def test_invalid_certificate_path_raises_error(self, canfar_client_fixture) -> N certificate=Path("/nonexistent/path.pem"), url="https://example.com" ) - def test_token_setup_headers(self, canfar_client_fixture) -> None: - """Test token setup creates correct headers.""" - token = "abcdef" - client = canfar_client_fixture( - token=SecretStr(token), url="https://example.com" - ) - assert client.token.get_secret_value() == token - assert client.client.headers["Authorization"] == f"Bearer {token}" - def _create_test_certificate( path: Path, expired: bool = False, not_yet_valid: bool = False @@ -293,24 +329,41 @@ class TestBaseURLConstruction: """Test base URL construction based on precedence.""" def test_runtime_url_precedence(self, canfar_client_fixture) -> None: - """Test that runtime URL takes precedence.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://runtime.com/api" - ) - base_url = client._get_base_url() - assert str(base_url) == "https://runtime.com/api" + """A request uses the runtime URL when runtime credentials are supplied.""" + requests: list[httpx.Request] = [] + + with ( + patch( + "canfar.client.Client", + side_effect=_sync_client_factory(_response_transport(requests)), + ), + canfar_client_fixture( + token=SecretStr("test-token"), url="https://runtime.com/api" + ) as client, + ): + response = client.client.get("probe") + + assert response.status_code == 200 + assert str(requests[0].url) == "https://runtime.com/api/probe" + assert requests[0].headers["Authorization"] == "Bearer test-token" def test_configured_url_from_context(self, canfar_client_fixture) -> None: - """Test base URL construction from configuration context.""" - config = x509_config( - server_name="TestServer", - server_url="https://config.example.com", - path=Path("/test/cert.pem"), - ) + """A request uses the configured Science Platform Server URL.""" + requests: list[httpx.Request] = [] + + with ( + patch( + "canfar.client.Client", + side_effect=_sync_client_factory(_response_transport(requests)), + ), + canfar_client_fixture( + config=oidc_config(server_url="https://config.example.com") + ) as client, + ): + response = client.client.get("probe") - client = canfar_client_fixture(config=config) - base_url = client._get_base_url() - assert str(base_url) == "https://config.example.com//v1" + assert response.status_code == 200 + assert str(requests[0].url) == "https://config.example.com//v1/probe" def test_no_server_in_context_raises_error(self, canfar_client_fixture) -> None: """Test that missing server in context raises ValueError.""" @@ -320,11 +373,13 @@ def test_no_server_in_context_raises_error(self, canfar_client_fixture) -> None: ), ) - client = canfar_client_fixture(config=config) - with pytest.raises( - ValueError, match="Server not found for Authentication Record" + with ( + canfar_client_fixture(config=config) as client, + pytest.raises( + ValueError, match="Server not found for Authentication Record" + ), ): - client._get_base_url() + client.client.get("probe") def test_active_server_without_url_raises_error( self, canfar_client_fixture @@ -341,36 +396,60 @@ def test_active_server_without_url_raises_error( ), ) - client = canfar_client_fixture(config=config) - with pytest.raises(ValueError, match="Active server has no URL configured"): - client._get_base_url() + with ( + canfar_client_fixture(config=config) as client, + pytest.raises(ValueError, match="Active server has no URL configured"), + ): + client.client.get("probe") class TestCertificateValidation: """Test certificate validation functionality.""" def test_certificate_validation_with_token_skips_validation(self, tmp_path) -> None: - """Test certificate validation when token provided takes precedence.""" + """A runtime-token request ignores a supplied certificate.""" # Create a valid certificate file for this test cert_path = tmp_path / "valid.pem" _create_test_certificate(cert_path) - # Even with a valid certificate, when token is provided, it should use the token - client = HTTPClient( - token=SecretStr("test-token"), - certificate=cert_path, - url="https://example.com", - ) - assert client.token is not None - assert client.certificate is None # Certificate should be nullified + requests: list[httpx.Request] = [] + with ( + patch( + "canfar.client.Client", + side_effect=_sync_client_factory(_response_transport(requests)), + ), + HTTPClient( + token=SecretStr("test-token"), + certificate=cert_path, + url="https://example.com", + ) as client, + ): + response = client.client.get("probe") - # Verify that the client uses token authentication - headers = client._get_http_headers( - credential=client._resolved_authentication_record() - ) - assert "Authorization" in headers - assert headers["Authorization"] == "Bearer test-token" - assert headers["X-Skaha-Authentication-Type"] == "RUNTIME-TOKEN" + assert response.status_code == 200 + assert len(requests) == 1 + assert str(requests[0].url) == "https://example.com/probe" + assert requests[0].headers["Authorization"] == "Bearer test-token" + assert requests[0].headers["X-Skaha-Authentication-Type"] == "RUNTIME-TOKEN" + + def test_certificate_validates_for_native_request(self, tmp_path) -> None: + """A valid certificate can build a native client and complete a request.""" + cert_path = tmp_path / "valid.pem" + _create_test_certificate(cert_path) + requests: list[httpx.Request] = [] + + with ( + patch( + "canfar.client.Client", + side_effect=_sync_client_factory(_response_transport(requests)), + ), + HTTPClient(certificate=cert_path, url="https://example.com") as client, + ): + response = client.client.get("probe") + + assert response.status_code == 200 + assert len(requests) == 1 + assert requests[0].headers["X-Skaha-Authentication-Type"] == "RUNTIME-X509" def test_certificate_file_not_exists(self, tmp_path) -> None: """Test certificate validation when file doesn't exist.""" @@ -395,359 +474,219 @@ def test_certificate_not_readable(self, tmp_path) -> None: ): HTTPClient(certificate=cert_path, url="https://example.com") - def test_certificate_valid(self, tmp_path) -> None: - """Test certificate validation with valid certificate.""" - cert_path = tmp_path / "valid.pem" - _create_test_certificate(cert_path) - - # Should not raise an error - client = HTTPClient(certificate=cert_path, url="https://example.com") - assert client.certificate == cert_path - class TestHTTPClientCreationAndHeaders: - """Test HTTP client creation and header generation.""" - - def test_lazy_client_initialization(self, canfar_client_fixture) -> None: - """Test that httpx clients are created only on first access.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) - - # Initially, private attributes should be None - assert client._client is None - assert client._asynclient is None - - # Access client property to trigger creation - sync_client = client.client - assert client._client is not None - assert isinstance(sync_client, httpx.Client) - - # Access asynclient property to trigger creation - async_client = client.asynclient - assert client._asynclient is not None - assert isinstance(async_client, httpx.AsyncClient) - - def test_default_headers_present(self, canfar_client_fixture) -> None: - """Test that common headers are present.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) - with patch( - "canfar.client.formatdate", - return_value="Wed, 09 Jun 2026 12:00:00 GMT", - ) as mock_formatdate: - headers = client._get_http_headers( - credential=client._resolved_authentication_record() - ) - - mock_formatdate.assert_called_once_with(usegmt=True) - assert "Content-Type" in headers - assert "Accept" in headers - assert "User-Agent" in headers - assert headers["Date"] == "Wed, 09 Jun 2026 12:00:00 GMT" - assert re.match( - r"^\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT$", - headers["Date"], - ) - assert headers["Content-Type"] == "application/x-www-form-urlencoded" - assert headers["Accept"] == "application/json" - assert "python-canfar" in headers["User-Agent"] - - def test_runtime_token_headers(self, canfar_client_fixture) -> None: - """Test headers for runtime token authentication.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) - headers = client._get_http_headers( - credential=client._resolved_authentication_record() - ) - - assert headers["Authorization"] == "Bearer test-token" - assert headers["X-Skaha-Authentication-Type"] == "RUNTIME-TOKEN" - - def test_runtime_certificate_headers(self, canfar_client_fixture, tmp_path) -> None: - """Test headers for runtime certificate authentication.""" - cert_path = tmp_path / "test.pem" - _create_test_certificate(cert_path) - - client = canfar_client_fixture(certificate=cert_path, url="https://example.com") - headers = client._get_http_headers( - credential=client._resolved_authentication_record() - ) - - assert headers["X-Skaha-Authentication-Type"] == "RUNTIME-X509" - assert "Authorization" not in headers - - def test_oidc_context_headers(self, canfar_client_fixture) -> None: - """Test headers for OIDC context authentication.""" - config = oidc_config( - idp="oidc", - access="oidc-access-token", - access_expiry=9999999999.0, - refresh_expiry=9999999999.0, - ) - - client = canfar_client_fixture(config=config) - headers = client._get_http_headers( - credential=client._resolved_authentication_record() - ) - - assert headers["Authorization"] == "Bearer oidc-access-token" - assert headers["X-Skaha-Authentication-Type"] == "OIDC" + """Test HTTP client requests and header generation.""" - def test_x509_context_headers(self, canfar_client_fixture) -> None: - """Test headers for X509 context authentication.""" - config = x509_config( - idp="x509", - server_name="TestX509", - path=Path("/test/cert.pem"), - ) - - client = canfar_client_fixture(config=config) - headers = client._get_http_headers( - credential=client._resolved_authentication_record() - ) - - assert headers["X-Skaha-Authentication-Type"] == "X509" - assert "Authorization" not in headers - - def test_registry_headers(self, canfar_client_fixture) -> None: - """Test headers with registry authentication.""" - registry = ContainerRegistry(username="test", secret="test") + def test_sync_request_includes_common_and_registry_headers( + self, + ) -> None: + """A native sync request carries common and registry headers.""" config = Configuration() - config.registry = registry - - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com", config=config - ) - headers = client._get_http_headers( - credential=client._resolved_authentication_record() - ) - - assert "X-Skaha-Registry-Auth" in headers - - def test_client_kwargs_timeout_and_concurrency(self, canfar_client_fixture) -> None: - """Test that httpx clients are initialized with correct timeout and limits.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), - url="https://example.com", - timeout=45, - concurrency=16, - ) + config.registry = ContainerRegistry(username="test", secret="test") + requests: list[httpx.Request] = [] - # Test sync client kwargs - sync_kwargs = client._get_client_kwargs( - asynchronous=False, credential=client._resolved_authentication_record() - ) - assert "timeout" in sync_kwargs - # httpx.Timeout doesn't have a .timeout attribute, it's the object itself - assert isinstance(sync_kwargs["timeout"], httpx.Timeout) - assert "limits" not in sync_kwargs # Only for async - - # Test async client kwargs - async_kwargs = client._get_client_kwargs( - asynchronous=True, credential=client._resolved_authentication_record() - ) - assert "timeout" in async_kwargs - assert isinstance(async_kwargs["timeout"], httpx.Timeout) - assert "limits" in async_kwargs - assert async_kwargs["limits"].max_connections == 16 - - def test_oidc_refresh_hook_added(self, canfar_client_fixture) -> None: - """Test that OIDC refresh hook is added for OIDC contexts.""" - config = oidc_config( - idp="oidc", - access="oidc-access-token", - access_expiry=9999999999.0, - refresh_expiry=9999999999.0, - ) + with ( + patch( + "canfar.client.formatdate", + return_value="Wed, 09 Jun 2026 12:00:00 GMT", + ) as mock_formatdate, + patch( + "canfar.client.Client", + side_effect=_sync_client_factory(_response_transport(requests)), + ), + HTTPClient( + token=SecretStr("test-token"), + url="https://example.com", + config=config, + ) as client, + ): + response = client.client.get("probe") - client = canfar_client_fixture(config=config) + assert response.status_code == 200 + mock_formatdate.assert_called_once_with(usegmt=True) + request = requests[0] + assert str(request.url) == "https://example.com/probe" + assert request.headers["Authorization"] == "Bearer test-token" + assert request.headers["X-Skaha-Authentication-Type"] == "RUNTIME-TOKEN" + assert request.headers["Content-Type"] == "application/x-www-form-urlencoded" + assert request.headers["Accept"] == "application/json" + assert request.headers["Date"] == "Wed, 09 Jun 2026 12:00:00 GMT" + assert "python-canfar" in request.headers["User-Agent"] + assert request.headers["X-Skaha-Registry-Auth"] == "dGVzdDp0ZXN0" + + async def test_async_request_includes_common_headers( + self, + canfar_client_fixture, + ) -> None: + """A native async request carries common headers and returns its result.""" + requests: list[httpx.Request] = [] - # Test sync client kwargs - sync_kwargs = client._get_client_kwargs( - asynchronous=False, credential=client._resolved_authentication_record() - ) - assert "event_hooks" in sync_kwargs - assert "request" in sync_kwargs["event_hooks"] + with ( + patch( + "canfar.client.AsyncClient", + side_effect=_async_client_factory(_response_transport(requests)), + ), + canfar_client_fixture( + token=SecretStr("test-token"), url="https://example.com" + ) as client, + ): + response = await client.asynclient.get("probe") - # Test async client kwargs - async_kwargs = client._get_client_kwargs( - asynchronous=True, credential=client._resolved_authentication_record() - ) - assert "event_hooks" in async_kwargs - assert "request" in async_kwargs["event_hooks"] + assert response.status_code == 200 + assert len(requests) == 1 + request = requests[0] + assert str(request.url) == "https://example.com/probe" + assert request.headers["Authorization"] == "Bearer test-token" + assert request.headers["X-Skaha-Authentication-Type"] == "RUNTIME-TOKEN" + assert request.headers["Content-Type"] == "application/x-www-form-urlencoded" + assert request.headers["Accept"] == "application/json" class TestContextManagerBehavior: """Test context manager functionality.""" def test_sync_context_manager_enter_exit(self, canfar_client_fixture) -> None: - """Test synchronous context manager entry and exit.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) + """The sync context manager returns the client and closes HTTPX.""" + requests: list[httpx.Request] = [] + created: list[httpx.Client] = [] - # Test __enter__ - with client as ctx_client: - assert ctx_client is client - # Verify client is created - assert isinstance(ctx_client.client, httpx.Client) + def factory(**kwargs: object) -> httpx.Client: + native = httpx.Client(transport=_response_transport(requests), **kwargs) + created.append(native) + return native - # After exit, client should be closed - assert client._client is None - - def test_close_sync_client(self, canfar_client_fixture) -> None: - """Test closing synchronous client.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) - - # Access client to create it - _ = client.client - assert client._client is not None - - # Close it - client._close() - assert client._client is None - - def test_close_sync_client_when_none(self, canfar_client_fixture) -> None: - """Test closing synchronous client when it's None.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) - - # Don't access client, so it remains None - assert client._client is None + with ( + patch("canfar.client.Client", side_effect=factory), + canfar_client_fixture( + token=SecretStr("test-token"), url="https://example.com" + ) as client, + ): + response = client.client.get("probe") - # Close should not raise an error - client._close() - assert client._client is None + assert response.status_code == 200 + assert len(requests) == 1 + assert created[0].is_closed async def test_async_context_manager_enter_exit( self, canfar_client_fixture ) -> None: - """Test asynchronous context manager entry and exit.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) - - # Test __aenter__ - async with client as ctx_client: - assert ctx_client is client - # Verify asynclient is created - assert isinstance(ctx_client.asynclient, httpx.AsyncClient) - - # After exit, asynclient should be closed - assert client._asynclient is None - - async def test_aclose_async_client(self, canfar_client_fixture) -> None: - """Test closing asynchronous client.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) - - # Access asynclient to create it - _ = client.asynclient - assert client._asynclient is not None - - # Close it - await client._aclose() - assert client._asynclient is None + """The async context manager returns the client and closes HTTPX.""" + requests: list[httpx.Request] = [] + created: list[httpx.AsyncClient] = [] - async def test_aclose_async_client_when_none(self, canfar_client_fixture) -> None: - """Test closing asynchronous client when it's None.""" - client = canfar_client_fixture( - token=SecretStr("test-token"), url="https://example.com" - ) + def factory(**kwargs: object) -> httpx.AsyncClient: + native = httpx.AsyncClient( + transport=_response_transport(requests), **kwargs + ) + created.append(native) + return native - # Don't access asynclient, so it remains None - assert client._asynclient is None + with patch("canfar.client.AsyncClient", side_effect=factory): + async with canfar_client_fixture( + token=SecretStr("test-token"), url="https://example.com" + ) as client: + response = await client.asynclient.get("probe") - # Close should not raise an error - await client._aclose() - assert client._asynclient is None + assert response.status_code == 200 + assert len(requests) == 1 + assert created[0].is_closed class TestSSLContextAndClientKwargs: - """Test SSL context creation and client kwargs generation.""" + """Test observable TLS validation and request-hook behavior.""" def test_expiry_hook_omitted_for_runtime_token( self, canfar_client_fixture, tmp_path ) -> None: - """Runtime token must not install saved-config expiry request hook.""" + """Runtime credentials bypass expiry checks from saved X.509 state.""" cert_path = tmp_path / "expired.pem" generate_cert(cert_path, expired=True) - config = x509_config(idp="x509", path=cert_path) - client = canfar_client_fixture( - config=config, - token=SecretStr("runtime-token"), - url="https://runtime.com", - ) + requests: list[httpx.Request] = [] + config = x509_config(idp="x509", path=cert_path, expiry=0.0) - sync_kwargs = client._get_client_kwargs( - asynchronous=False, credential=client._resolved_authentication_record() - ) - async_kwargs = client._get_client_kwargs( - asynchronous=True, credential=client._resolved_authentication_record() - ) + with ( + patch( + "canfar.client.Client", + side_effect=_sync_client_factory(_response_transport(requests)), + ), + canfar_client_fixture( + config=config, + token=SecretStr("runtime-token"), + url="https://runtime.com", + ) as client, + ): + response = client.client.get("probe") - assert len(sync_kwargs["event_hooks"]["request"]) == 1 - assert len(async_kwargs["event_hooks"]["request"]) == 1 - assert sync_kwargs["event_hooks"]["request"][0].__name__ == "request" - assert async_kwargs["event_hooks"]["request"][0].__name__ == "arequest" + assert response.status_code == 200 + assert len(requests) == 1 + assert requests[0].headers["Authorization"] == "Bearer runtime-token" - def test_expiry_hook_present_for_saved_expired_x509( + async def test_async_expiry_hook_omitted_for_runtime_token( self, canfar_client_fixture, tmp_path ) -> None: - """Saved expired x509 config must still install expiry request hook.""" + """Async runtime credentials bypass saved X.509 expiry checks.""" cert_path = tmp_path / "expired.pem" generate_cert(cert_path, expired=True) - config = x509_config(idp="x509", path=cert_path) - client = canfar_client_fixture(config=config) + requests: list[httpx.Request] = [] - sync_kwargs = client._get_client_kwargs( - asynchronous=False, credential=client._resolved_authentication_record() - ) - async_kwargs = client._get_client_kwargs( - asynchronous=True, credential=client._resolved_authentication_record() - ) + with patch( + "canfar.client.AsyncClient", + side_effect=_async_client_factory(_response_transport(requests)), + ): + async with canfar_client_fixture( + config=x509_config(idp="x509", path=cert_path, expiry=0.0), + token=SecretStr("runtime-token"), + url="https://runtime.com", + ) as client: + response = await client.asynclient.get("probe") - assert len(sync_kwargs["event_hooks"]["request"]) == 2 - assert len(async_kwargs["event_hooks"]["request"]) == 2 - assert sync_kwargs["event_hooks"]["request"][0].__name__ == "hook" - assert async_kwargs["event_hooks"]["request"][0].__name__ == "hook" - assert sync_kwargs["event_hooks"]["request"][1].__name__ == "request" - assert async_kwargs["event_hooks"]["request"][1].__name__ == "arequest" + assert response.status_code == 200 + assert len(requests) == 1 + assert requests[0].headers["Authorization"] == "Bearer runtime-token" - def test_get_client_kwargs_with_certificate( + def test_expiry_hook_present_for_saved_expired_x509( self, canfar_client_fixture, tmp_path ) -> None: - """Test client kwargs with certificate authentication.""" - cert_path = tmp_path / "test.pem" - _create_test_certificate(cert_path) + """A sync request rejects an expired saved X.509 record.""" + cert_path = tmp_path / "expired.pem" + generate_cert(cert_path, expired=True) + requests: list[httpx.Request] = [] + config = x509_config(idp="x509", path=cert_path, expiry=0.0) - client = canfar_client_fixture(certificate=cert_path, url="https://example.com") - kwargs = client._get_client_kwargs( - asynchronous=False, credential=client._resolved_authentication_record() - ) + with ( + patch( + "canfar.client.Client", + side_effect=_sync_client_factory(_response_transport(requests)), + ), + canfar_client_fixture(config=config) as client, + pytest.raises(AuthExpiredError, match="expired"), + ): + client.client.get("probe") - assert "verify" in kwargs - assert isinstance(kwargs["verify"], ssl.SSLContext) + assert requests == [] - def test_get_ssl_context_valid_certificate( + async def test_async_expiry_hook_present_for_saved_expired_x509( self, canfar_client_fixture, tmp_path ) -> None: - """Test SSL context creation with valid certificate.""" - cert_path = tmp_path / "test.pem" - _create_test_certificate(cert_path) + """An async request rejects an expired saved X.509 record.""" + cert_path = tmp_path / "expired.pem" + generate_cert(cert_path, expired=True) + requests: list[httpx.Request] = [] - client = canfar_client_fixture(certificate=cert_path, url="https://example.com") - ssl_context = client._get_ssl_context(cert_path) + with ( + patch( + "canfar.client.AsyncClient", + side_effect=_async_client_factory(_response_transport(requests)), + ), + pytest.raises(AuthExpiredError, match="expired"), + ): + async with canfar_client_fixture( + config=x509_config(idp="x509", path=cert_path, expiry=0.0) + ) as client: + await client.asynclient.get("probe") - assert isinstance(ssl_context, ssl.SSLContext) - assert ssl_context.minimum_version == ssl.TLSVersion.TLSv1_2 + assert requests == [] def test_non_readable_certfile() -> None: diff --git a/tests/test_client_auth_resolution.py b/tests/test_client_auth_resolution.py index 00df58ad..644a86a8 100644 --- a/tests/test_client_auth_resolution.py +++ b/tests/test_client_auth_resolution.py @@ -14,7 +14,7 @@ from canfar.client import HTTPClient from canfar.exceptions.context import AuthContextError, AuthExpiredError -from canfar.hooks.httpx.auth import AuthenticationError +from canfar.hooks.httpx.auth import AuthenticationError, arefresh from canfar.models.active import ActiveConfig from canfar.models.auth import ( Client, @@ -222,14 +222,14 @@ def test_refreshable_record_without_access_refreshes_first_request( HTTPClient(config=config) as client, ): response = client.client.get("probe") - persisted = Configuration().get_credential("test") + persisted = Configuration().authentication["test"] assert response.status_code == 200 assert len(token_requests) == 1 assert [request.headers["Authorization"] for request in platform_requests] == [ f"Bearer {_REFRESHED_TOKEN}" ] - canonical = config.get_credential("test") + canonical = config.authentication["test"] assert isinstance(canonical, OIDCCredential) assert canonical == persisted assert canonical.token == expected_token @@ -417,7 +417,7 @@ async def refresh_token(request: httpx.Request) -> httpx.Response: client.asynclient.get("one"), client.asynclient.get("two"), ) - persisted = Configuration().get_credential("test") + persisted = Configuration().authentication["test"] assert [response.status_code for response in responses] == [200, 200] assert len(token_requests) == 1 @@ -425,7 +425,7 @@ async def refresh_token(request: httpx.Request) -> httpx.Response: f"Bearer {_REFRESHED_TOKEN}", f"Bearer {_REFRESHED_TOKEN}", ] - canonical = config.get_credential("test") + canonical = config.authentication["test"] assert isinstance(canonical, OIDCCredential) assert canonical == persisted assert canonical.token == Token( @@ -436,6 +436,67 @@ async def refresh_token(request: httpx.Request) -> httpx.Response: ) assert canonical.expiry == Expiry(access=1_300.0, refresh=None) + async def test_refresh_hooks_share_one_client_lock( + self, + tmp_path: Path, + ) -> None: + """Independent async hook factories cannot race token persistence.""" + now = 1_000.0 + config = _configuration( + _oidc(access="expired", access_expiry=now - 1, refresh_expiry=now + 1_000) + ) + token_requests: list[httpx.Request] = [] + + async def refresh_token(request: httpx.Request) -> httpx.Response: + token_requests.append(request) + await asyncio.sleep(0.01) + return httpx.Response( + 200, + json={ + "access_token": _REFRESHED_TOKEN, + "refresh_token": "rotated-refresh", + "token_type": "Bearer", + "scope": "openid profile", + "expires_in": 300, + }, + request=request, + ) + + token_transport = httpx.MockTransport(refresh_token) + platform_transport = httpx.MockTransport( + lambda request: httpx.Response(200, request=request) + ) + + with ( + patch("canfar.models.config.CONFIG_PATH", tmp_path / "config.yaml"), + patch("canfar.models.auth.time.time", return_value=now), + patch("authlib.oauth2.rfc6749.wrappers.time.time", return_value=now), + patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: httpx.AsyncClient( + transport=platform_transport, **kwargs + ), + ), + patch( + "authlib.integrations.httpx_client.AsyncOAuth2Client", + side_effect=lambda *args, **kwargs: AsyncOAuth2Client( + *args, transport=token_transport, **kwargs + ), + ), + ): + async with HTTPClient(config=config) as client: + first = arefresh(client) + second = arefresh(client) + await asyncio.gather( + first(httpx.Request("GET", "https://platform.example/one")), + second(httpx.Request("GET", "https://platform.example/two")), + ) + + assert len(token_requests) == 1 + credential = config.authentication["test"] + assert isinstance(credential, OIDCCredential) + assert credential.token.access == SecretStr(_REFRESHED_TOKEN) + async def test_async_refresh_repairs_existing_sync_client( self, tmp_path: Path, @@ -501,14 +562,14 @@ async def test_async_refresh_repairs_existing_sync_client( assert sync.headers["Authorization"] == "Bearer expired" sync_response = sync.get("after-refresh") assert sync.headers["Authorization"] == f"Bearer {_REFRESHED_TOKEN}" - persisted = Configuration().get_credential("test") + persisted = Configuration().authentication["test"] assert [async_response.status_code, sync_response.status_code] == [200, 200] assert [request.headers["Authorization"] for request in platform_requests] == [ f"Bearer {_REFRESHED_TOKEN}", f"Bearer {_REFRESHED_TOKEN}", ] - canonical = config.get_credential("test") + canonical = config.authentication["test"] assert isinstance(canonical, OIDCCredential) assert canonical == persisted @@ -532,7 +593,7 @@ def test_refresh_failure_preserves_last_valid_record( update={"client": Client(identity="client", secret="client-secret")} ) ) - original = config.get_credential("test").model_copy(deep=True) + original = config.authentication["test"].model_copy(deep=True) platform_requests: list[httpx.Request] = [] token_requests: list[httpx.Request] = [] @@ -593,7 +654,7 @@ def token_endpoint(request: httpx.Request) -> httpx.Response: return_value=oauth_client, ), patch( - "canfar.models.config.Configuration.save", + "canfar.config.editor.ConfigurationEditor.save", side_effect=(OSError(sentinel) if failure == "save" else None), ) as save, HTTPClient(config=config) as client, @@ -607,7 +668,7 @@ def token_endpoint(request: httpx.Request) -> httpx.Response: assert exc_info.value.__cause__ is None assert sentinel not in str(exc_info.value) assert sentinel not in caplog.text - assert config.get_credential("test") == original + assert config.authentication["test"] == original assert len(token_requests) == 1 assert platform_requests == [] assert save.call_count == int(failure == "save") @@ -676,16 +737,17 @@ def test_refresh_and_expiry_outcome_matrix( client = stack.enter_context(HTTPClient(config=config)) request_client = client.client if state == "unrefreshable": - credential = config.get_credential("test") + credential = config.authentication["test"] assert isinstance(credential, OIDCCredential) - config.update_credential( + config.editor.set( + "authentication.test", credential.model_copy( update={ "endpoints": credential.endpoints.model_copy( update={"discovery": None} ) } - ) + ), ) if succeeds: response = request_client.get("probe") diff --git a/tests/test_config.py b/tests/test_config.py index b8b92d52..ae762581 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -9,12 +9,11 @@ import yaml from pydantic import ValidationError -from canfar.config.editor import set_value as set_config_value +import canfar.authentication as authentication_service from canfar.config.migration import ( ConfigResetRequiredError, ensure_current_config, ) -from canfar.config.store import save_config from canfar.models.auth import X509Credential from canfar.models.config import Configuration from canfar.models.http import Server @@ -126,36 +125,36 @@ class TestConfigServersPaths: """Test dotted-path access to name-keyed servers.""" def test_get_and_set_servers_canfar_url(self, tmp_path: Path) -> None: - """servers.. paths round-trip through get_value/set_value.""" + """servers.. paths round-trip through the editor.""" config_path = tmp_path / "config.yaml" with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() - assert str(config.get_value("servers.canfar.url")) == ( + assert str(config.editor.get("servers.canfar.url")) == ( "https://ws-uv.canfar.net/skaha" ) - updated = config.set_value( + config.editor.set( "servers.canfar.url", "https://example.test/skaha", ) - assert str(updated.get_value("servers.canfar.url")) == ( + assert str(config.editor.get("servers.canfar.url")) == ( "https://example.test/skaha" ) -class TestConfigServices: - """Test configuration storage and action helpers.""" +class TestConfigEditing: + """Test validated configuration edits and domain-owned persistence.""" def test_invalid_server_selection_preserves_configuration( self, tmp_path: Path, ) -> None: - """An invalid Server Selection changes neither memory nor persisted YAML.""" + """An invalid server mapping changes neither memory nor persisted YAML.""" config_path = tmp_path / "config.yaml" with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() - config.save() + config.editor.save() persisted = config_path.read_bytes() invalid = Server( @@ -167,13 +166,13 @@ def test_invalid_server_selection_preserves_configuration( auths=["x509"], ) with pytest.raises(ValidationError, match="Invalid server name"): - config.set_active_selection("cadc", invalid) + config.editor.set("servers", {"invalid.name": invalid}) assert config.active.server == "canfar" assert set(config.servers) == {"canfar"} assert config_path.read_bytes() == persisted - def test_upsert_credential_preserves_other_authentication_records( + def test_editor_credential_update_preserves_other_authentication_records( self, tmp_path: Path, ) -> None: @@ -182,47 +181,21 @@ def test_upsert_credential_preserves_other_authentication_records( with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() cadc_path = config.authentication["cadc"].path - config.upsert_credential( + config.editor.set( + "authentication.srcnet", X509Credential( idp="srcnet", path=Path("/srcnet/cert.pem"), expiry=42.0, ), ) - config.save() + config.editor.save() loaded = Configuration() assert set(loaded.authentication) == {"cadc", "srcnet"} assert loaded.authentication["cadc"].path == cadc_path assert loaded.authentication["srcnet"].path == Path("/srcnet/cert.pem") - def test_update_unknown_credential_preserves_configuration( - self, - tmp_path: Path, - ) -> None: - """Updating an unknown Authentication Record changes no state.""" - config_path = tmp_path / "config.yaml" - with patch("canfar.models.config.CONFIG_PATH", config_path): - config = Configuration() - config.save() - state = config.model_dump(mode="python") - persisted = config_path.read_bytes() - - with pytest.raises( - KeyError, - match="Authentication record for IDP 'srcnet' not found", - ): - config.update_credential( - X509Credential( - idp="srcnet", - path=Path("/srcnet/cert.pem"), - expiry=42.0, - ), - ) - - assert config.model_dump(mode="python") == state - assert config_path.read_bytes() == persisted - def test_update_known_credential_preserves_unrelated_state( self, tmp_path: Path, @@ -231,7 +204,8 @@ def test_update_known_credential_preserves_unrelated_state( config_path = tmp_path / "config.yaml" with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() - config.upsert_credential( + config.editor.set( + "authentication.srcnet", X509Credential( idp="srcnet", path=Path("/srcnet/cert.pem"), @@ -241,14 +215,15 @@ def test_update_known_credential_preserves_unrelated_state( active = config.active.model_dump(mode="python") servers = config.servers - config.update_credential( + config.editor.set( + "authentication.cadc", X509Credential( idp="cadc", path=Path("/updated/cadc.pem"), expiry=84.0, ), ) - config.save() + config.editor.save() loaded = Configuration() assert loaded.authentication["cadc"].path == Path("/updated/cadc.pem") @@ -264,7 +239,8 @@ def test_upsert_server_preserves_other_science_platform_servers( config_path = tmp_path / "config.yaml" with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() - config.upsert_server( + config.editor.set( + "servers.SRCNet", Server( idp="srcnet", name="SRCNet", @@ -274,7 +250,7 @@ def test_upsert_server_preserves_other_science_platform_servers( auths=["oidc"], ), ) - config.save() + config.editor.save() loaded = Configuration() assert set(loaded.servers) == {"canfar", "SRCNet"} @@ -289,14 +265,16 @@ def test_remove_authentication_preserves_unrelated_records( config_path = tmp_path / "config.yaml" with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() - config.upsert_credential( + config.editor.set( + "authentication.srcnet", X509Credential( idp="srcnet", path=Path("/srcnet/cert.pem"), expiry=42.0, ), ) - config.upsert_server( + config.editor.set( + "servers.SRCNet", Server( idp="srcnet", name="SRCNet", @@ -306,9 +284,18 @@ def test_remove_authentication_preserves_unrelated_records( auths=["oidc"], ), ) - config.set_active_selection("srcnet", config.servers["SRCNet"]) - config.remove_authentication("cadc") - config.save() + config.editor.set( + "active", + config.active.model_copy( + update={ + "authentication": "srcnet", + "server": "SRCNet", + "servers": {"srcnet": "SRCNet"}, + } + ), + ) + config.editor.save() + authentication_service.remove("cadc", force=True) loaded = Configuration() assert set(loaded.authentication) == {"srcnet"} @@ -316,24 +303,22 @@ def test_remove_authentication_preserves_unrelated_records( assert loaded.active.authentication == "srcnet" assert loaded.active.server == "SRCNet" - def test_editor_and_store_update_config_without_model_io( + def test_editor_updates_config_without_model_io( self, tmp_path: Path, ) -> None: """Config editing and persistence live outside the Pydantic model.""" config_path = tmp_path / "config.yaml" - config = Configuration() - - updated = set_config_value(config, "console.width", 132) - save_config(updated, config_path) - with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + config.editor.set("console.width", 132) + config.editor.save() loaded = Configuration() assert loaded.console.width == 132 - def test_selection_service_sets_active_server(self) -> None: - """Active server selection is provided as a config action helper.""" + def test_editor_sets_active_server(self) -> None: + """Active server selection remains ordinary validated data.""" config = Configuration() server = Server( idp="cadc", @@ -344,12 +329,19 @@ def test_selection_service_sets_active_server(self) -> None: auths=["x509"], ) - config.set_active_selection("cadc", server) + config.editor.set("servers.CADC-CANFAR", server) + config.editor.set( + "active", + config.active.model_copy( + update={ + "server": "CADC-CANFAR", + "servers": {"cadc": "CADC-CANFAR"}, + } + ), + ) assert config.active.server == "CADC-CANFAR" - assert config.get_server_by_uri("ivo://cadc.example/skaha").name == ( - "CADC-CANFAR" - ) + assert config.servers["CADC-CANFAR"].name == ("CADC-CANFAR") class TestConfigManualReset: diff --git a/tests/test_config_editor.py b/tests/test_config_editor.py new file mode 100644 index 00000000..0ec18fb0 --- /dev/null +++ b/tests/test_config_editor.py @@ -0,0 +1,212 @@ +"""Tests for the bound persisted Configuration editor.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from unittest.mock import patch + +import pytest +from pydantic import ValidationError + +from canfar.models.config import Configuration + +if TYPE_CHECKING: + from pathlib import Path + + +def test_configuration_exposes_only_data_and_editor_for_editing( + tmp_path: Path, +) -> None: + """Legacy service operations are not part of the persisted data model.""" + config_path = tmp_path / "config.yaml" + removed = ( + "save", + "get_value", + "set_value", + "get_credential", + "storage_identifiers", + "_resolve_storage", + "upsert_credential", + "update_credential", + "set_active_authentication", + "remove_authentication", + "purge_authentication", + "get_server_by_uri", + "get_active_server", + "get_server_for_idp", + "get_remembered_server_for_idp", + "set_active_selection", + "upsert_server", + "upsert_servers", + ) + + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert all(not hasattr(config, name) for name in removed) + assert hasattr(config, "editor") + + +def test_editor_reads_scalars_mappings_and_whole_lists(tmp_path: Path) -> None: + """The editor resolves dotted paths without exposing list indexing.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert config.editor.get("console.width") == 120 + server = config.editor.get("servers.canfar") + assert server["name"] == "canfar" + assert config.editor.get("servers.canfar.auths") == ["x509"] + + assert "editor" not in config.model_dump(mode="python") + + +def test_editor_set_mutates_validated_configuration(tmp_path: Path) -> None: + """Setting a dotted path updates the bound Configuration in memory.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + assert config.editor.set("console.width", 132) is config + assert config.console.width == 132 + config.editor.set("servers.canfar.auths", ["x509", "oidc"]) + + assert config.editor.get("console.width") == 132 + assert config.editor.get("servers.canfar.auths") == ["x509", "oidc"] + + +def test_editor_replaces_top_level_mapping_without_reloading_saved_state( + tmp_path: Path, +) -> None: + """Top-level mapping edits remove records instead of resurrecting YAML keys.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + config.editor.set("authentication.srcnet", config.authentication["cadc"]) + config.editor.save() + config.editor.set("authentication", {"cadc": config.authentication["cadc"]}) + + assert set(config.authentication) == {"cadc"} + + +def test_editor_seeds_idp_for_mapping_form_credentials(tmp_path: Path) -> None: + """Mapping-form Authentication records inherit their Identity Provider key.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + config.editor.set( + "authentication.second", + { + "mode": "x509", + "path": str(tmp_path / "second.pem"), + "expiry": 0, + }, + ) + + assert config.authentication["second"].idp == "second" + + +def test_editor_rejects_auth_replacement_with_active_idp_reference( + tmp_path: Path, +) -> None: + """Replacing records cannot invalidate the persisted active IDP reference.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + config.editor.set( + "authentication.srcnet", + { + "mode": "x509", + "path": str(tmp_path / "srcnet.pem"), + "expiry": 0, + }, + ) + config.editor.set( + "active", + config.active.model_copy(update={"authentication": "srcnet"}), + ) + config.editor.save() + + with pytest.raises(ValidationError): + config.editor.set( + "authentication", + {"cadc": config.authentication["cadc"]}, + ) + + assert config.active.authentication == "srcnet" + assert set(config.authentication) == {"cadc", "srcnet"} + + +def test_editor_rejects_list_indexing(tmp_path: Path) -> None: + """Dotted paths may retrieve a whole list but cannot address its items.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + + with pytest.raises(ValueError, match="List indices are not supported"): + config.editor.get("servers.canfar.auths.0") + with pytest.raises(ValueError, match="List indices are not supported"): + config.editor.set("servers.canfar.auths.0", "oidc") + + +def test_editor_save_persists_bound_configuration(tmp_path: Path) -> None: + """Editor save writes its validated in-memory state to the config file.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + config.editor.set("console.width", 133) + config.editor.save() + loaded = Configuration() + + assert loaded.console.width == 133 + + +def test_editor_failed_set_preserves_configuration(tmp_path: Path) -> None: + """Invalid editor updates do not partially mutate the bound config.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + original = config.model_dump(mode="python") + + with pytest.raises(ValidationError): + config.editor.set("console.width", "not-an-int") + + assert config.model_dump(mode="python") == original + + +def test_editor_failed_top_level_batch_preserves_configuration( + tmp_path: Path, +) -> None: + """Invalid complete-state edits do not partially mutate the bound config.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + original = config.model_dump(mode="python") + + with pytest.raises(ValidationError): + config.editor._set_top_level( # noqa: SLF001 + active=config.active.model_copy(update={"authentication": "missing"}), + authentication={}, + servers={}, + ) + + assert config.model_dump(mode="python") == original + + +def test_editor_save_failure_preserves_existing_file(tmp_path: Path) -> None: + """A failed editor save leaves the existing YAML untouched.""" + config_path = tmp_path / "config.yaml" + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration() + config.editor.save() + original = config_path.read_bytes() + config.editor.set("console.width", 134) + + with ( + patch("canfar.config.editor.os.fsync", side_effect=OSError("disk full")), + pytest.raises(OSError, match="Failed to save configuration"), + ): + config.editor.save() + + assert config_path.read_bytes() == original + assert list(tmp_path.iterdir()) == [config_path] diff --git a/tests/test_context.py b/tests/test_context.py index 108022d0..2dbf1a35 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,8 +1,8 @@ """Test Canfar Context API.""" -from unittest.mock import MagicMock - +import httpx import pytest +from pydantic import SecretStr from canfar.context import Context @@ -11,8 +11,10 @@ def context(): """Test Context.""" context = Context() - yield context - del context + try: + yield context + finally: + context.__exit__(None, None, None) @pytest.mark.integration @@ -24,10 +26,30 @@ def test_context(context) -> None: def test_context_resources_use_http_client() -> None: """Resources returns decoded context payload.""" - context = Context(token="token", url="https://example.test/skaha/v1") - mock_client = MagicMock() - mock_client.get.return_value.json.return_value = {"cores": {"default": 1}} - context._client = mock_client # noqa: SLF001 - - assert context.resources() == {"cores": {"default": 1}} - mock_client.get.assert_called_once_with(url="context") + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json={"cores": {"default": 1}}, + request=request, + ) + + with ( + pytest.MonkeyPatch.context() as monkeypatch, + Context( + token=SecretStr("token"), url="https://example.test/skaha/v1" + ) as context, + ): + monkeypatch.setattr( + "canfar.client.Client", + lambda **kwargs: httpx.Client( + transport=httpx.MockTransport(respond), **kwargs + ), + ) + # The client is lazy, so the transport is installed before the request. + assert context.resources() == {"cores": {"default": 1}} + + assert len(requests) == 1 + assert requests[0].url.path.endswith("/context") diff --git a/tests/test_data_dependencies.py b/tests/test_data_dependencies.py index ffaabbb0..f85f13aa 100644 --- a/tests/test_data_dependencies.py +++ b/tests/test_data_dependencies.py @@ -2,24 +2,17 @@ from __future__ import annotations -from pathlib import Path - -try: - import tomllib -except ModuleNotFoundError: - import tomli as tomllib +from importlib.metadata import distribution def test_tagged_data_dependencies_are_standard_dependencies() -> None: """A standard CANFAR install includes both immutable upstream releases.""" - metadata = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + dist = distribution("canfar") + requirements = dist.requires or [] - assert ( - "vosfs @ git+https://github.com/shinybrar/vosfs@v0.8.0" - in metadata["project"]["dependencies"] - ) + assert "vosfs @ git+https://github.com/shinybrar/vosfs@v0.8.0" in requirements assert ( "fsspec-cli @ git+https://github.com/shinybrar/vosfs@fsspec-cli-v0.7.0" - "#subdirectory=src/fsspec-cli" in metadata["project"]["dependencies"] + "#subdirectory=src/fsspec-cli" in requirements ) - assert "data" not in metadata["project"].get("optional-dependencies", {}) + assert "data" not in (dist.metadata.get_all("Provides-Extra") or []) diff --git a/tests/test_data_smoke.py b/tests/test_data_smoke.py index 69d2fe66..5be1b8ec 100644 --- a/tests/test_data_smoke.py +++ b/tests/test_data_smoke.py @@ -27,8 +27,14 @@ def _require_live_credentials() -> None: _skip("configuration is unavailable") try: - _endpoint, idp = config._resolve_storage("arc") # noqa: SLF001 - credential = config.get_credential(idp) + server = next( + (server for server in config.servers.values() if "arc" in server.storage), + None, + ) + if server is None or server.idp is None: + _skip("the named service has no parent Authentication Record") + idp = server.idp + credential = config.authentication[idp] except (KeyError, ValueError): _skip("the named service or its Authentication Record is unavailable") diff --git a/tests/test_errors.py b/tests/test_errors.py index aaa5e37b..fcb66554 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -73,13 +73,13 @@ def test_structured_error_to_json(self) -> None: error = StructuredError( code=ErrorCode.OUTPUT_CONFLICT, message="Conflicting machine output flags.", - hint="Use only one of --json or --yaml.", + hint="Use --output json or --output yaml.", ) payload = json.loads(structured_error_to_json(error)) assert payload == { "code": "output.conflict", "message": "Conflicting machine output flags.", - "hint": "Use only one of --json or --yaml.", + "hint": "Use --output json or --output yaml.", } def test_structured_error_to_json_includes_null_hint(self) -> None: diff --git a/tests/test_hooks_httpx_auth.py b/tests/test_hooks_httpx_auth.py index dcace7b6..c6337be6 100644 --- a/tests/test_hooks_httpx_auth.py +++ b/tests/test_hooks_httpx_auth.py @@ -39,7 +39,7 @@ def oidc_client() -> HTTPClient: class TestSyncHook: """Tests for the synchronous `hook` function.""" - @patch("canfar.models.config.Configuration.save") + @patch("canfar.config.editor.ConfigurationEditor.save") @patch( "canfar.auth.oidc.sync_refresh", return_value={ @@ -71,9 +71,9 @@ def test_successful_refresh( # Verify the main client's headers were updated assert oidc_client.client.headers["Authorization"] == "Bearer new-access-token" - credential = oidc_client.config.get_credential( + credential = oidc_client.config.authentication[ oidc_client.config.active.authentication - ) + ] assert isinstance(credential, OIDCCredential) assert credential.token.access is not None assert credential.token.access.get_secret_value() == "new-access-token" @@ -110,18 +110,19 @@ def test_skip_if_runtime_credentials_used(self, mock_refresh) -> None: @patch("canfar.auth.oidc.sync_refresh") def test_skip_if_token_not_expired(self, mock_refresh, oidc_client) -> None: """Verify the hook does nothing if the access token is not expired.""" - credential = oidc_client.config.get_credential( + credential = oidc_client.config.authentication[ oidc_client.config.active.authentication - ) + ] assert isinstance(credential, OIDCCredential) - oidc_client.config.update_credential( + oidc_client.config.editor.set( + f"authentication.{credential.idp}", credential.model_copy( update={ "expiry": credential.expiry.model_copy( update={"access": time.time() + 3600} ) } - ) + ), ) hook_func = refresh(oidc_client) request = httpx.Request("GET", "/") @@ -142,7 +143,7 @@ def test_refresh_failure_raises_error(self, mock_refresh, oidc_client) -> None: class TestAsyncHook: """Tests for the asynchronous `ahook` function.""" - @patch("canfar.models.config.Configuration.save") + @patch("canfar.config.editor.ConfigurationEditor.save") @patch( "canfar.auth.oidc.refresh", return_value={ @@ -173,9 +174,9 @@ async def test_successful_async_refresh( oidc_client.asynclient.headers["Authorization"] == "Bearer new-async-token" ) - credential = oidc_client.config.get_credential( + credential = oidc_client.config.authentication[ oidc_client.config.active.authentication - ) + ] assert isinstance(credential, OIDCCredential) assert credential.token.access is not None assert credential.token.access.get_secret_value() == "new-async-token" @@ -216,6 +217,31 @@ async def test_async_refresh_failure_raises_error( class TestSyncAsyncParity: """Characterization tests: sync and async auth hooks share guard behavior.""" + async def test_empty_runtime_token_keeps_saved_oidc_sync_and_async(self) -> None: + """An empty runtime token does not bypass either saved OIDC hook.""" + config = oidc_config( + idp="testoidc", + access="saved-access-token", + access_expiry=time.time() + 3600, + refresh_expiry=time.time() + 7200, + ) + client = HTTPClient( + config=config, + token=SecretStr(""), + url="https://platform.example", + ) + + sync_request = httpx.Request("GET", "https://platform.example/sync") + refresh(client)(sync_request) + assert sync_request.headers["Authorization"] == "Bearer saved-access-token" + + async with client: + async_request = httpx.Request("GET", "https://platform.example/async") + await arefresh(client)(async_request) + + assert async_request.headers["Authorization"] == "Bearer saved-access-token" + client._close() # noqa: SLF001 + @patch("canfar.auth.oidc.sync_refresh") @patch("canfar.auth.oidc.refresh") async def test_refresh_and_arefresh_both_skip_non_oidc( diff --git a/tests/test_hooks_typer_aliases.py b/tests/test_hooks_typer_aliases.py deleted file mode 100644 index d3f020fd..00000000 --- a/tests/test_hooks_typer_aliases.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Tests for the AliasGroup class in canfar.hooks.typer.aliases.""" - -from __future__ import annotations - -from unittest.mock import Mock, patch - -import pytest -from click.core import Command, Context -from typer.core import TyperGroup - -from canfar.hooks.typer.aliases import AliasGroup - - -class TestAliasGroup: - """Test cases for the AliasGroup class.""" - - @pytest.fixture - def alias_group(self) -> AliasGroup: - """Create an AliasGroup instance for testing.""" - return AliasGroup() - - @pytest.fixture - def mock_context(self) -> Context: - """Create a mock Click context.""" - return Mock(spec=Context) - - @pytest.fixture - def mock_command(self) -> Command: - """Create a mock Click command.""" - return Mock(spec=Command) - - def test_alias_group_inherits_from_typer_group( - self, alias_group: AliasGroup - ) -> None: - """Test that AliasGroup inherits from TyperGroup.""" - assert isinstance(alias_group, TyperGroup) - - def test_cmd_split_pattern_regex(self, alias_group: AliasGroup) -> None: - """Test the regex pattern for splitting command names.""" - pattern = alias_group._CMD_SPLIT_P # noqa: SLF001 - - # Test comma separation - assert pattern.split("cmd1,cmd2") == ["cmd1", "cmd2"] - assert pattern.split("cmd1, cmd2") == ["cmd1", "cmd2"] - assert pattern.split("cmd1 ,cmd2") == ["cmd1", "cmd2"] - assert pattern.split("cmd1 , cmd2") == ["cmd1", "cmd2"] - - # Test pipe separation - assert pattern.split("cmd1|cmd2") == ["cmd1", "cmd2"] - assert pattern.split("cmd1| cmd2") == ["cmd1", "cmd2"] - assert pattern.split("cmd1 |cmd2") == ["cmd1", "cmd2"] - assert pattern.split("cmd1 | cmd2") == ["cmd1", "cmd2"] - - # Test mixed separators - assert pattern.split("cmd1,cmd2|cmd3") == ["cmd1", "cmd2", "cmd3"] - - # Test single command (no split) - assert pattern.split("single") == ["single"] - - def test_group_cmd_name_with_exact_match(self, alias_group: AliasGroup) -> None: - """Test _group_cmd_name when the default matches exactly.""" - mock_cmd = Mock() - mock_cmd.name = "show" - alias_group.commands = {"show": mock_cmd} - - result = alias_group._group_cmd_name("show") # noqa: SLF001 - assert result == "show" - - def test_group_cmd_name_with_alias_match(self, alias_group: AliasGroup) -> None: - """Test _group_cmd_name when the default matches an alias.""" - mock_cmd = Mock() - mock_cmd.name = "show | list | ls" - alias_group.commands = {"show": mock_cmd} - - # Test each alias - assert alias_group._group_cmd_name("show") == "show | list | ls" # noqa: SLF001 - assert alias_group._group_cmd_name("list") == "show | list | ls" # noqa: SLF001 - assert alias_group._group_cmd_name("ls") == "show | list | ls" # noqa: SLF001 - - def test_group_cmd_name_with_no_match(self, alias_group: AliasGroup) -> None: - """Test _group_cmd_name when no command matches.""" - mock_cmd = Mock() - mock_cmd.name = "show | list" - alias_group.commands = {"show": mock_cmd} - - result = alias_group._group_cmd_name("nonexistent") # noqa: SLF001 - assert result == "nonexistent" - - def test_group_cmd_name_with_empty_commands(self, alias_group: AliasGroup) -> None: - """Test _group_cmd_name with empty commands dict.""" - alias_group.commands = {} - - result = alias_group._group_cmd_name("anything") # noqa: SLF001 - assert result == "anything" - - def test_group_cmd_name_with_command_without_name( - self, alias_group: AliasGroup - ) -> None: - """Test _group_cmd_name with command that has no name attribute.""" - mock_cmd = Mock() - del mock_cmd.name # Remove the name attribute - alias_group.commands = {"test": mock_cmd} - - result = alias_group._group_cmd_name("test") # noqa: SLF001 - assert result == "test" - - def test_group_cmd_name_with_empty_name(self, alias_group: AliasGroup) -> None: - """Test _group_cmd_name with command that has empty name.""" - mock_cmd = Mock() - mock_cmd.name = "" - alias_group.commands = {"test": mock_cmd} - - result = alias_group._group_cmd_name("test") # noqa: SLF001 - assert result == "test" - - def test_get_command_calls_group_cmd_name( - self, alias_group: AliasGroup, mock_context: Context - ) -> None: - """Test that get_command calls _group_cmd_name and super().get_command.""" - mock_cmd = Mock() - mock_cmd.name = "show | list" - alias_group.commands = {"show": mock_cmd} - - # Mock the parent class method - with patch.object( - alias_group.__class__.__bases__[0], "get_command" - ) as mock_super: - mock_super.return_value = mock_cmd - - result = alias_group.get_command(mock_context, "list") - - # Verify that super().get_command was called with the resolved name - mock_super.assert_called_once_with(mock_context, "show | list") - assert result == mock_cmd - - def test_get_command_with_nonexistent_alias( - self, alias_group: AliasGroup, mock_context: Context - ) -> None: - """Test get_command with a non-existent command/alias.""" - alias_group.commands = {} - - with patch.object( - alias_group.__class__.__bases__[0], "get_command" - ) as mock_super: - mock_super.return_value = None - - result = alias_group.get_command(mock_context, "nonexistent") - - mock_super.assert_called_once_with(mock_context, "nonexistent") - assert result is None - - def test_get_command_integration( - self, alias_group: AliasGroup, mock_context: Context - ) -> None: - """Integration test for get_command with real command setup.""" - # Create a mock command with aliases - mock_cmd = Mock(spec=Command) - mock_cmd.name = "show | list | ls" - alias_group.commands = {"show": mock_cmd} - - with patch.object( - alias_group.__class__.__bases__[0], "get_command" - ) as mock_super: - mock_super.return_value = mock_cmd - - # Test that all aliases resolve to the same command - for alias in ["show", "list", "ls"]: - result = alias_group.get_command(mock_context, alias) - assert result == mock_cmd - - # Verify super was called with the full command name each time - assert mock_super.call_count == 3 - for call in mock_super.call_args_list: - assert ( - call[0][1] == "show | list | ls" - ) # Second argument should be the resolved name - - def test_multiple_commands_with_aliases(self, alias_group: AliasGroup) -> None: - """Test _group_cmd_name with multiple commands having different aliases.""" - cmd1 = Mock() - cmd1.name = "show | list | ls" - cmd2 = Mock() - cmd2.name = "create | new | add" - cmd3 = Mock() - cmd3.name = "delete" - - alias_group.commands = { - "show": cmd1, - "create": cmd2, - "delete": cmd3, - } - - # Test first command aliases - assert alias_group._group_cmd_name("show") == "show | list | ls" # noqa: SLF001 - assert alias_group._group_cmd_name("list") == "show | list | ls" # noqa: SLF001 - assert alias_group._group_cmd_name("ls") == "show | list | ls" # noqa: SLF001 - - # Test second command aliases - assert alias_group._group_cmd_name("create") == "create | new | add" # noqa: SLF001 - assert alias_group._group_cmd_name("new") == "create | new | add" # noqa: SLF001 - assert alias_group._group_cmd_name("add") == "create | new | add" # noqa: SLF001 - - # Test third command (no aliases) - assert alias_group._group_cmd_name("delete") == "delete" # noqa: SLF001 - - # Test non-existent command - assert alias_group._group_cmd_name("nonexistent") == "nonexistent" # noqa: SLF001 diff --git a/tests/test_images.py b/tests/test_images.py index 317301b9..ee05a291 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -1,8 +1,8 @@ """Test Canfar Images API.""" -from unittest.mock import MagicMock, patch - +import httpx import pytest +from pydantic import SecretStr from canfar.images import Images from canfar.models.containers import Image @@ -12,8 +12,10 @@ def images(): """Test images.""" images = Images() - yield images - del images + try: + yield images + finally: + images.__exit__(None, None, None) @pytest.mark.integration @@ -41,11 +43,19 @@ def test_images_details_returns_models() -> None: "digest": "sha256:deadbeef", } ] - images = Images() - with patch("canfar.images.HTTPClient.client") as client: - client.get.return_value.json.return_value = payload - results = images.details() + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=payload, request=request) + + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr( + "canfar.client.Client", + lambda **kwargs: httpx.Client( + transport=httpx.MockTransport(respond), **kwargs + ), + ) + with Images(token=SecretStr("token"), url="https://example.test") as images: + results = images.details() assert isinstance(results[0], Image) assert results[0].id == payload[0]["id"] @@ -55,14 +65,32 @@ def test_images_details_returns_models() -> None: def test_images_fetch_uses_http_client_params() -> None: """Fetch returns image IDs and passes optional kind as request parameter.""" - images = Images(token="token", url="https://example.test/skaha/v1") - mock_client = MagicMock() - mock_client.get.return_value.json.return_value = [ - {"id": "images.canfar.net/skaha/terminal:latest"} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json=[{"id": "images.canfar.net/skaha/terminal:latest"}], + request=request, + ) + + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr( + "canfar.client.Client", + lambda **kwargs: httpx.Client( + transport=httpx.MockTransport(respond), **kwargs + ), + ) + with Images( + token=SecretStr("token"), url="https://example.test/skaha/v1" + ) as images: + assert images.fetch() == ["images.canfar.net/skaha/terminal:latest"] + assert images.fetch(kind="headless") == [ + "images.canfar.net/skaha/terminal:latest" + ] + + assert [request.url.params.multi_items() for request in requests] == [ + [], + [("type", "headless")], ] - images._client = mock_client # noqa: SLF001 - - assert images.fetch() == ["images.canfar.net/skaha/terminal:latest"] - assert images.fetch(kind="headless") == ["images.canfar.net/skaha/terminal:latest"] - mock_client.get.assert_any_call("image", params={}) - mock_client.get.assert_any_call("image", params={"type": "headless"}) diff --git a/tests/test_import_isolation.py b/tests/test_import_isolation.py index 1f201f56..3d4d37f0 100644 --- a/tests/test_import_isolation.py +++ b/tests/test_import_isolation.py @@ -14,6 +14,86 @@ def test_conftest_isolates_home() -> None: assert Path(os.environ["HOME"]) == Path(os.environ["CANFAR_TEST_HOME"]) +def _run_home_probe( + tmp_path: Path, + *, + test_home: str | None = None, +) -> list[tuple[str, str]]: + """Run isolated probe tests and return each worker's HOME values.""" + probe_dir = tmp_path / "probe" + probe_dir.mkdir(parents=True) + output_dir = tmp_path / "output" + output_dir.mkdir() + probe = """ +import os +from pathlib import Path + + +def test_home(): + worker = os.environ.get("PYTEST_XDIST_WORKER", "master") + output_dir = Path(os.environ["PROBE_OUTPUT_DIR"]) + (output_dir / f"{Path(__file__).stem}-{worker}").write_text( + f"{os.environ['HOME']}\\n{os.environ['CANFAR_TEST_HOME']}", + encoding="utf-8", + ) +""" + for name in ("one", "two"): + (probe_dir / f"test_probe_{name}.py").write_text(probe, encoding="utf-8") + + environment = os.environ.copy() + environment.pop("CANFAR_TEST_HOME", None) + environment["PROBE_OUTPUT_DIR"] = str(output_dir) + if test_home is not None: + environment["CANFAR_TEST_HOME"] = test_home + + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + str(probe_dir), + "-p", + "tests.conftest", + "-n2", + "--no-cov", + "-q", + ], + capture_output=True, + check=False, + cwd=Path.cwd(), + env=environment, + text=True, + ) + assert result.returncode == 0, result.stderr + return [ + tuple(path.read_text(encoding="utf-8").splitlines()) + for path in sorted(output_dir.iterdir()) + ] + + +def test_default_home_is_fresh_per_run_and_shared_by_workers(tmp_path: Path) -> None: + """Default test HOME must not retain state between pytest invocations.""" + first = _run_home_probe(tmp_path / "first") + second = _run_home_probe(tmp_path / "second") + + assert len(first) == len(second) == 2 + assert len({home for home, _ in first}) == 1 + assert len({test_home for _, test_home in first}) == 1 + assert first[0][0] == first[0][1] + assert second[0][0] == second[0][1] + assert first[0][0] != second[0][0] + + +def test_explicit_home_is_preserved_for_workers(tmp_path: Path) -> None: + """An explicit test HOME remains unchanged, including its spelling.""" + explicit = f"{tmp_path / 'explicit-home'}/" + + values = _run_home_probe(tmp_path / "explicit", test_home=explicit) + + assert len(values) == 2 + assert values == [(explicit, explicit), (explicit, explicit)] + + def test_stale_list_config_does_not_break_canfar_or_cli_imports( tmp_path: Path, ) -> None: @@ -75,10 +155,7 @@ def test_stale_list_config_does_not_break_canfar_or_cli_imports( ): importlib.import_module(module) -from canfar.utils.logging import _canfar_logger - canfar_logger = logging.getLogger("canfar") -assert not _canfar_logger._configured assert canfar_logger.handlers == [] assert canfar_logger.level == logging.NOTSET assert canfar_logger.propagate diff --git a/tests/test_logging_contraction.py b/tests/test_logging_contraction.py index c74f7d3d..09e4f1ac 100644 --- a/tests/test_logging_contraction.py +++ b/tests/test_logging_contraction.py @@ -15,7 +15,6 @@ "command", [ ["login"], - ["auth", "login"], ["delete"], ["events"], ["logs"], diff --git a/tests/test_models_config.py b/tests/test_models_config.py index 4bd5c046..4f822b2c 100644 --- a/tests/test_models_config.py +++ b/tests/test_models_config.py @@ -80,57 +80,19 @@ def test_default_canfar_server_ships_arc_and_vault_storage( ("arc", "https://ws-uv.canfar.net/arc"), ("vault", "https://cadc-west-01.canfar.net/vault"), ): - resolved, idp = config._resolve_storage(name) # noqa: SLF001 - assert resolved.rstrip("/") == endpoint - assert idp == "cadc" + service = config.servers["canfar"].storage[name] + assert str(service.url).rstrip("/") == endpoint - def test_legacy_server_named_storage_is_healed(self, tmp_path: Path) -> None: - """A Storage Identifier saved as the Server Name is restored to its leaf.""" - config_path = tmp_path / "config.yaml" - config_path.write_text( - "version: 1\n" - "servers:\n" - " canfar:\n" - " idp: cadc\n" - " uri: ivo://cadc.nrc.ca/skaha\n" - " url: https://ws-uv.canfar.net/skaha\n" - " version: v1\n" - " auths: [x509]\n" - " storage:\n" - " canfar:\n" - " uri: ivo://cadc.nrc.ca/arc\n" - " url: https://ws-uv.canfar.net/arc\n", - encoding="utf-8", - ) - with patch("canfar.models.config.CONFIG_PATH", config_path): - config = Configuration() - - storage = config.servers["canfar"].storage - assert set(storage) == {"arc", "vault"} - assert str(storage["arc"].url).rstrip("/") == "https://ws-uv.canfar.net/arc" - - def test_custom_storage_names_are_not_healed(self, tmp_path: Path) -> None: - """Deliberate Storage Identifiers are configuration, not stale defaults.""" - config_path = tmp_path / "config.yaml" - config_path.write_text( - "version: 1\n" - "servers:\n" - " canfar:\n" - " idp: cadc\n" - " uri: ivo://cadc.nrc.ca/skaha\n" - " url: https://ws-uv.canfar.net/skaha\n" - " version: v1\n" - " auths: [x509]\n" - " storage:\n" - " canSRC:\n" - " uri: ivo://cadc.nrc.ca/arc\n" - " url: https://ws-cadc.canfar.net/arc\n", - encoding="utf-8", - ) - with patch("canfar.models.config.CONFIG_PATH", config_path): + def test_storage_resolution_belongs_to_storage_module( + self, + tmp_path: Path, + ) -> None: + """Configuration stores VOSpace metadata without resolving it.""" + with patch("canfar.models.config.CONFIG_PATH", tmp_path / "config.yaml"): config = Configuration() - assert set(config.servers["canfar"].storage) == {"canSRC"} + assert not hasattr(config, "storage_identifiers") + assert not hasattr(config, "_resolve_storage") def test_default_authentication_dict_keyed_by_idp(self, tmp_path: Path) -> None: """Default Authentication Records are keyed by IDP.""" @@ -257,6 +219,21 @@ def test_long_storage_name_allowed(self) -> None: assert list(server.storage) == [storage_name] + @pytest.mark.parametrize("storage_name", ["filesystem", "identifiers", "sources"]) + def test_storage_name_can_match_storage_module_member( + self, + storage_name: str, + ) -> None: + """Module members are not reserved when lookup is explicit.""" + service = { + "uri": "ivo://cadc.nrc.ca/arc", + "url": "https://ws-cadc.canfar.net/arc", + } + + server = Server(storage={storage_name: service}) + + assert list(server.storage) == [storage_name] + def test_storage_name_whitespace_is_trimmed(self) -> None: """Valid surrounding whitespace remains normalized.""" service = { @@ -468,7 +445,7 @@ def test_complex_round_trip_serialization(self, tmp_path: Path) -> None: temp_config_path = tmp_path / "config.yaml" with patch("canfar.models.config.CONFIG_PATH", temp_config_path): - original.save() + original.editor.save() loaded = Configuration() assert loaded.active.authentication == "srcnet" @@ -518,17 +495,17 @@ def test_v1_storage_json_and_yaml_round_trip(self, tmp_path: Path) -> None: config_path = tmp_path / "config.yaml" with patch("canfar.models.config.CONFIG_PATH", config_path): - config.save() + config.editor.save() loaded = Configuration() assert list(loaded.servers["canfar"].storage) == ["canSRC", "canSRCs3"] assert loaded.servers["canfar"].idp == "cadc" assert loaded == config - def test_existing_v1_configuration_without_storage_gains_defaults( + def test_existing_v1_configuration_without_storage_loads_unchanged( self, tmp_path: Path ) -> None: - """A storage-less Server gains defaults without a schema migration.""" + """A released storage-less Server remains unchanged on load.""" config_path = tmp_path / "config.yaml" config_path.write_text(yaml.safe_dump(_sample_config()), encoding="utf-8") @@ -536,14 +513,14 @@ def test_existing_v1_configuration_without_storage_gains_defaults( config = Configuration() assert config.version == 1 - assert set(config.servers["canfar"].storage) == {"arc", "vault"} + assert config.servers["canfar"].storage == {} def test_save_creates_directory(self, tmp_path: Path) -> None: """Save creates parent directories when missing.""" config = Configuration() nested_path = tmp_path / "nested" / "config.yaml" with patch("canfar.models.config.CONFIG_PATH", nested_path): - config.save() + config.editor.save() assert nested_path.exists() def test_failed_serialization_preserves_existing_configuration( @@ -558,12 +535,12 @@ def test_failed_serialization_preserves_existing_configuration( with ( patch("canfar.models.config.CONFIG_PATH", config_path), patch( - "canfar.config.store.yaml.dump", + "canfar.config.editor.yaml.dump", side_effect=TypeError("cannot serialize"), ), pytest.raises(OSError, match="Failed to save configuration"), ): - config.save() + config.editor.save() assert config_path.read_bytes() == original @@ -576,12 +553,12 @@ def test_invalid_assignment_cannot_replace_last_valid_configuration( with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() - config.save() + config.editor.save() original = config_path.read_bytes() config.active.authentication = "missing" with pytest.raises(OSError, match="Failed to save configuration"): - config.save() + config.editor.save() loaded = Configuration() @@ -601,12 +578,12 @@ def test_failed_replacement_preserves_existing_configuration( with ( patch("canfar.models.config.CONFIG_PATH", config_path), patch( - "canfar.config.store.Path.replace", + "canfar.config.editor.Path.replace", side_effect=OSError("cannot replace"), ), pytest.raises(OSError, match="Failed to save configuration"), ): - config.save() + config.editor.save() assert config_path.read_bytes() == original assert set(tmp_path.iterdir()) == {config_path} @@ -618,10 +595,10 @@ def test_failed_first_write_leaves_no_configuration(self, tmp_path: Path) -> Non with ( patch("canfar.models.config.CONFIG_PATH", config_path), - patch("canfar.config.store.os.fsync", side_effect=OSError("disk full")), + patch("canfar.config.editor.os.fsync", side_effect=OSError("disk full")), pytest.raises(OSError, match="Failed to save configuration"), ): - config.save() + config.editor.save() assert not config_path.exists() assert list(tmp_path.iterdir()) == [] @@ -637,7 +614,7 @@ def test_yaml_file_content_structure(self, tmp_path: Path) -> None: ), }, ) - config.save() + config.editor.save() yaml_data = yaml.safe_load(temp_config_path.read_text(encoding="utf-8")) assert yaml_data["version"] == 1 @@ -731,7 +708,7 @@ def test_save_handles_directory_creation_error(self, tmp_path: Path) -> None: patch("pathlib.Path.mkdir", side_effect=OSError("Permission denied")), pytest.raises(OSError, match="Permission denied"), ): - config.save() + config.editor.save() def test_save_handles_file_write_error(self, tmp_path: Path) -> None: """Save surfaces file write errors.""" @@ -743,7 +720,7 @@ def test_save_handles_file_write_error(self, tmp_path: Path) -> None: patch("canfar.models.config.CONFIG_PATH", config_path), pytest.raises(OSError, match=error_msg), ): - config.save() + config.editor.save() def test_save_handles_yaml_serialization_error(self, tmp_path: Path) -> None: """Save surfaces YAML serialization errors.""" @@ -755,7 +732,7 @@ def test_save_handles_yaml_serialization_error(self, tmp_path: Path) -> None: patch("yaml.dump", side_effect=TypeError("Mock YAML error")), pytest.raises(OSError, match=error_msg), ): - config.save() + config.editor.save() def test_settings_customise_sources_order(self) -> None: """Settings sources preserve expected precedence ordering.""" diff --git a/tests/test_overview.py b/tests/test_overview.py index 39c02317..ddf4afc3 100644 --- a/tests/test_overview.py +++ b/tests/test_overview.py @@ -1,6 +1,7 @@ """Test Canfar Overview API.""" -from unittest.mock import MagicMock, patch +import asyncio +from unittest.mock import patch import httpx import pytest @@ -13,8 +14,11 @@ def overview(): """Test overview.""" overview = Overview() - yield overview - del overview + try: + yield overview + finally: + overview.__exit__(None, None, None) + asyncio.run(overview.__aexit__(None, None, None)) @pytest.mark.integration @@ -24,57 +28,75 @@ def test_available(overview: Overview) -> None: assert overview.availability(), "Server should be available" -def _sync_response(text: str) -> MagicMock: - response = MagicMock() - response.text = text - return response - - def test_overview_updates_base_url_and_parses_availability() -> None: """Overview strips version from base URL and parses available true.""" - sync_client = MagicMock() - sync_client.base_url = httpx.URL("https://example.test/skaha/v1/") - async_client = MagicMock() - async_client.base_url = httpx.URL("https://example.test/skaha/v1/") - sync_client.get.return_value = _sync_response( + payload = ( 'true' "ok" ) + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text=payload, request=request) + with ( - patch("canfar.client.HTTPClient._create_sync_client", return_value=sync_client), patch( - "canfar.client.HTTPClient._create_async_client", return_value=async_client + "canfar.client.Client", + side_effect=lambda **kwargs: httpx.Client( + transport=httpx.MockTransport(respond), **kwargs + ), ), - ): - overview = Overview( + patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: httpx.AsyncClient( + transport=httpx.MockTransport(respond), **kwargs + ), + ), + Overview( token=SecretStr("token"), url="https://example.test/skaha/v1" - ) - - assert str(overview.client.base_url) == "https://example.test/skaha" - assert str(overview.asynclient.base_url) == "https://example.test/skaha" - assert overview.availability() is True + ) as overview, + ): + try: + assert str(overview.client.base_url) == "https://example.test/skaha/" + assert str(overview.asynclient.base_url) == "https://example.test/skaha/" + assert overview.availability() is True + finally: + asyncio.run(overview.__aexit__(None, None, None)) def test_overview_availability_false_paths() -> None: """Overview availability returns false for empty or unavailable responses.""" - overview = Overview.model_construct() - client = MagicMock() - overview._client = client # noqa: SLF001 - - client.get.return_value = _sync_response("") - assert overview.availability() is False - - client.get.return_value = _sync_response( - 'missing' + responses = iter( + [ + "", + ( + 'missing' + "" + ), + ( + 'false' + "" + ), + ] ) - assert overview.availability() is False - client.get.return_value = _sync_response( - 'false' - "" - ) - assert overview.availability() is False + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, text=next(responses), request=request) + + with ( + patch( + "canfar.client.Client", + side_effect=lambda **kwargs: httpx.Client( + transport=httpx.MockTransport(respond), **kwargs + ), + ), + Overview(token=SecretStr("token"), url="https://example.test") as overview, + ): + try: + assert overview.availability() is False + assert overview.availability() is False + assert overview.availability() is False + finally: + asyncio.run(overview.__aexit__(None, None, None)) diff --git a/tests/test_platform_enrichment.py b/tests/test_platform_enrichment.py index 8d578b6f..3b5e9f80 100644 --- a/tests/test_platform_enrichment.py +++ b/tests/test_platform_enrichment.py @@ -305,7 +305,7 @@ def test_enrich_handles_invalid_saved_authentication_without_state_change( authentication={"cadc": credential}, servers={"canfar": server}, ) - config.save() + config.editor.save() before_model = config.model_dump(mode="json") before_yaml = config_path.read_bytes() if strict: @@ -536,7 +536,7 @@ def token_response(request: httpx.Request) -> httpx.Response: ), ): config = _active_with_target(_oidc_credential(access_expiry=now - 1)) - config.save() + config.editor.save() before_active = config.active.model_dump(mode="json") before_servers = { name: server.model_dump(mode="json") @@ -547,8 +547,8 @@ def token_response(request: httpx.Request) -> httpx.Response: ) persisted = Configuration() - refreshed = config.get_credential("srcnet") - saved_refreshed = persisted.get_credential("srcnet") + refreshed = config.authentication["srcnet"] + saved_refreshed = persisted.authentication["srcnet"] assert isinstance(refreshed, OIDCCredential) assert isinstance(saved_refreshed, OIDCCredential) assert validated.version == "v2" @@ -615,7 +615,7 @@ def test_refresh_failure_leaves_all_platform_state_unchanged( ), ): config = _active_with_target(_oidc_credential(access_expiry=now - 1)) - config.save() + config.editor.save() before_model = config.model_dump(mode="json") before_yaml = config_path.read_bytes() with pytest.raises( @@ -670,7 +670,7 @@ def __init__(self, **kwargs: object) -> None: ), ): config = _anonymous_config() - config.save() + config.editor.save() before_model = config.model_dump(mode="json") before_yaml = config_path.read_bytes() with pytest.raises( diff --git a/tests/test_pyproject_toolchain.py b/tests/test_pyproject_toolchain.py deleted file mode 100644 index bb13a3d9..00000000 --- a/tests/test_pyproject_toolchain.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Invariant: mypy is gone, ty is configured in pyproject.toml.""" - -from __future__ import annotations - -from pathlib import Path - -try: - import tomllib -except ModuleNotFoundError: - import tomli as tomllib - -_PYPROJECT = Path(__file__).parent.parent / "pyproject.toml" - - -def _load() -> dict[str, object]: - with _PYPROJECT.open("rb") as f: - return tomllib.load(f) - - -def test_mypy_config_absent() -> None: - """The [tool.mypy] section must not exist after migrating to ty.""" - data = _load() - assert "mypy" not in data.get("tool", {}), ( - "[tool.mypy] still present in pyproject.toml; migration to ty is incomplete." - ) - - -def test_ty_config_present() -> None: - """A [tool.ty] section must exist after migrating from mypy.""" - data = _load() - assert "ty" in data.get("tool", {}), ( - "[tool.ty] missing from pyproject.toml; ty is not configured." - ) - - -def test_mypy_dev_dependency_absent() -> None: - """Mypy must not appear in [dependency-groups.dev] after the toolchain swap.""" - data = _load() - dev_deps: list[str] = data.get("dependency-groups", {}).get("dev", []) - assert not any(dep.startswith("mypy") for dep in dev_deps), ( - "mypy still listed as a dev dependency." - ) - - -def test_toolchain_parser_is_not_a_runtime_dependency() -> None: - """The test-only TOML reader must not ship with the CANFAR client.""" - data = _load() - dependencies: list[str] = data.get("project", {}).get("dependencies", []) - assert not any(dep.startswith("toml") for dep in dependencies) diff --git a/tests/test_server.py b/tests/test_server.py index 6d0caae9..db84335a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect from pathlib import Path from threading import Barrier from typing import TYPE_CHECKING @@ -12,10 +13,15 @@ import yaml from pydantic import AnyHttpUrl, AnyUrl +from canfar._server_discovery import ( + _discover_for_idp, + _discovered_to_server, + _select_storage, +) from canfar.errors import ErrorCode from canfar.models.active import ActiveConfig from canfar.models.auth import OIDCCredential, RuntimeCredential, X509Credential -from canfar.models.config import Configuration, default_servers +from canfar.models.config import Configuration from canfar.models.http import Server, VOSpaceService from canfar.models.registry import IVOARegistry, IVOARegistrySearch from canfar.models.registry import Server as DiscoveredServer @@ -23,14 +29,14 @@ ServerDiscoveryError, ServerFetchError, ServerSelectionRequiredError, - _discover_for_idp, - _discovered_to_server, - _select_storage, activate, discover, enrich, use, ) +from canfar.server import ( + __all__ as server_exports, +) from canfar.server import ( list_servers as server_list, ) @@ -54,6 +60,32 @@ """ +def test_public_exports_declare_server_api() -> None: + """The module's public and compatibility exports remain explicit.""" + expected = { + "ServerActivation", + "ServerDiscoveryError", + "ServerFetchError", + "ServerSelectionRequiredError", + "ServerSelectorError", + "activate", + "activate_authentication", + "discover", + "enrich", + "list_servers", + "use", + } + + assert set(server_exports) == expected + + +def test_enrich_preserves_storage_resource_annotation() -> None: + """The public enrichment signature retains its released annotation text.""" + annotation = inspect.signature(enrich).parameters["storage_resource"].annotation + + assert annotation == "RegistryResource | None | object" + + def _http_client_factory( transport: httpx.BaseTransport, ) -> Callable[..., httpx.Client]: @@ -102,7 +134,7 @@ def test_list_returns_servers_for_active_idp(self, tmp_path: Path) -> None: config = Configuration() assign_servers(config, cadc, srcnet) config.active = config.active.model_copy(update={"authentication": "cadc"}) - config.save() + config.editor.save() with patch("canfar.server.Configuration", Configuration): servers = server_list() @@ -137,10 +169,86 @@ def test_list_empty_when_no_servers_for_active_idp(self, tmp_path: Path) -> None assert servers == [] + def test_discover_uses_editor_for_server_state_and_persistence( + self, + tmp_path: Path, + ) -> None: + """Discovery updates and persists Servers through the editor boundary.""" + discovered = _cadc_server(name="Discovered-CADC") + config_path = tmp_path / "config.yaml" + + with ( + patch("canfar.models.config.CONFIG_PATH", config_path), + patch( + "canfar._server_discovery._discover_for_idp", + return_value=[discovered], + ), + ): + config = _anonymous_config() + discover("cadc", config=config) + + assert config.servers["Discovered-CADC"] == discovered + assert Configuration().servers["Discovered-CADC"] == discovered + class TestServerUse: """Tests for canfar.server.use().""" + def test_activate_uses_editor_for_active_selection_and_history( + self, + tmp_path: Path, + ) -> None: + """Activation records a Server Name through the editor boundary.""" + target = _cadc_server(name="Selected-CADC") + fetched = target.model_copy(update={"cores": 8}, deep=True) + config_path = tmp_path / "config.yaml" + + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = _anonymous_config(target) + config.editor.save() + + with patch("canfar.server._validate_server", return_value=fetched): + activation = activate("cadc", "Selected-CADC", config=config) + + assert activation.server == fetched + assert config.active.server == "Selected-CADC" + assert config.active.servers["cadc"] == "Selected-CADC" + saved = Configuration() + + assert saved.active.server == "Selected-CADC" + assert saved.active.servers["cadc"] == "Selected-CADC" + + def test_activate_resolves_remembered_selection_by_server_name( + self, + tmp_path: Path, + ) -> None: + """Remembered Server Selection is resolved by Server Name, not URI.""" + first = _cadc_server(name="First", uri=AnyUrl("ivo://first.example/skaha")) + remembered = _cadc_server( + name="Remembered", + uri=AnyUrl("ivo://remembered.example/skaha"), + ) + config_path = tmp_path / "config.yaml" + + with patch("canfar.models.config.CONFIG_PATH", config_path): + config = Configuration( + active=ActiveConfig( + authentication="cadc", + server=None, + servers={"cadc": "Remembered"}, + ), + authentication={"cadc": X509Credential(idp="cadc")}, + servers={first.name: first, remembered.name: remembered}, + ) + + with patch("canfar.server._validate_server", return_value=remembered): + activation = activate("cadc", config=config) + + assert activation.reason == "remembered" + assert activation.server.name == "Remembered" + assert config.active.server == "Remembered" + assert config.active.servers["cadc"] == "Remembered" + def test_use_by_uri_updates_active_server(self, tmp_path: Path) -> None: """Selecting by URI fetches, validates, and saves the active server.""" target = _cadc_server() @@ -152,7 +260,7 @@ def test_use_by_uri_updates_active_server(self, tmp_path: Path) -> None: with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() assign_servers(config, target) - config.save() + config.editor.save() with ( patch( @@ -175,7 +283,7 @@ def test_use_by_unique_name_updates_active_server(self, tmp_path: Path) -> None: with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() assign_servers(config, target) - config.save() + config.editor.save() with ( patch("canfar.server._validate_server", return_value=fetched), @@ -195,7 +303,7 @@ def test_use_unknown_name_fails_without_changing_active_server( with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() assign_servers(config, target) - config.save() + config.editor.save() previous = Configuration().active.server with ( @@ -224,7 +332,7 @@ def test_use_runs_discovery_on_miss_then_succeeds(self, tmp_path: Path) -> None: with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() assign_servers(config, known) - config.save() + config.editor.save() def merge_discovered( _idp: str, @@ -232,7 +340,7 @@ def merge_discovered( config: Configuration, **_kwargs: object, ) -> list[Server]: - config.upsert_server(discovered) + config.editor.set(f"servers.{discovered.name}", discovered) return [discovered] with ( @@ -258,7 +366,7 @@ def test_use_discovery_failure_leaves_active_unchanged( with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() assign_servers(config, target) - config.save() + config.editor.save() previous = Configuration().active.server with ( @@ -281,7 +389,7 @@ def test_use_fetch_failure_leaves_active_unchanged(self, tmp_path: Path) -> None with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() assign_servers(config, target) - config.save() + config.editor.save() previous = Configuration().active.server with ( @@ -325,7 +433,7 @@ def test_discover_returns_canonical_state_for_duplicate_server_names( with ( patch("canfar.models.config.CONFIG_PATH", config_path), patch( - "canfar.server._discover_for_idp", + "canfar._server_discovery._discover_for_idp", return_value=[winner, alpha, earlier], ), ): @@ -348,7 +456,10 @@ def test_discover_merges_servers_through_public_api(self, tmp_path: Path) -> Non with ( patch("canfar.models.config.CONFIG_PATH", config_path), - patch("canfar.server._discover_for_idp", return_value=[discovered]), + patch( + "canfar._server_discovery._discover_for_idp", + return_value=[discovered], + ), patch("canfar.server.Configuration", Configuration), ): servers = discover("cadc") @@ -356,9 +467,13 @@ def test_discover_merges_servers_through_public_api(self, tmp_path: Path) -> Non assert servers == [discovered] with patch("canfar.models.config.CONFIG_PATH", config_path): saved = Configuration() - assert str(saved.get_server_by_uri("ivo://cadc.example/skaha").url) == ( - "https://cadc.example/skaha" - ) + assert str( + next( + server + for server in saved.servers.values() + if str(server.uri) == "ivo://cadc.example/skaha" + ).url + ) == ("https://cadc.example/skaha") @pytest.mark.asyncio async def test_registry_retains_only_skaha_and_preferred_storage_records( @@ -393,12 +508,7 @@ def test_discover_refreshes_primary_storage_and_preserves_manual_entries( self, tmp_path: Path, ) -> None: - """Rediscovery updates only the generated Storage Identifier entry. - - A Server Name keyed entry saved by an older client is healed to its - registry leaf (``arc``) on load, so rediscovery refreshes that entry in - place instead of generating a second one. - """ + """Rediscovery updates only the generated Storage Identifier entry.""" manual = VOSpaceService( uri="ivo://cadc.nrc.ca/custom", url="https://manual.example/custom", @@ -467,8 +577,8 @@ def capabilities_response(request: httpx.Request) -> httpx.Response: persisted = Configuration().servers["canfar"] assert discovered.storage == persisted.storage - assert set(discovered.storage) == {"arc", "archive", "vault"} - assert discovered.storage["arc"] == VOSpaceService( + assert set(discovered.storage) == {"canfar", "archive"} + assert discovered.storage["canfar"] == VOSpaceService( uri="ivo://cadc.nrc.ca/arc", url="https://storage.example/arc", ) @@ -642,7 +752,7 @@ def enriched( with ( patch("canfar.utils.registry.Discover", return_value=mock_discovery), - patch("canfar.server.enrich", side_effect=enriched), + patch("canfar._server_discovery.enrich", side_effect=enriched), ): servers = await _discover_for_idp("srcnet") @@ -769,7 +879,10 @@ def convert( materialize = AsyncMock(return_value=RuntimeCredential(token="current-token")) with ( patch("canfar.utils.registry.Discover", return_value=mock_discovery), - patch("canfar.server._discovered_to_server", side_effect=convert), + patch( + "canfar._server_discovery._discovered_to_server", + side_effect=convert, + ), patch( "canfar.client.HTTPClient._materialize_credentials", new=materialize, @@ -1048,7 +1161,7 @@ def capabilities_response(request: httpx.Request) -> httpx.Response: with patch("canfar.models.config.CONFIG_PATH", config_path): config = _anonymous_config(known) - config.save() + config.editor.save() with ( patch( @@ -1066,18 +1179,13 @@ def capabilities_response(request: httpx.Request) -> httpx.Response: persisted = Configuration().servers["canfar"] - # A storage-less saved Server gains the default VOSpace Services on load. - healed = known.model_copy( - update={"storage": default_servers["canfar"].storage}, - deep=True, - ) expected = ( - healed.model_copy( + known.model_copy( update={"version": "v2.1", "auths": ["oidc"]}, deep=True, ) if capabilities_case == "success" - else healed + else known ) assert discovered == [expected] assert config.servers["canfar"] == expected @@ -1099,7 +1207,7 @@ def test_activate_without_selector_requires_prompt_for_multiple_servers( config = Configuration() assign_servers(config, first, second) config.active = config.active.model_copy(update={"server": None}) - config.save() + config.editor.save() with ( patch("canfar.server.Configuration", Configuration), @@ -1123,7 +1231,10 @@ def test_discover_keys_named_server_by_registry_name(self, tmp_path: Path) -> No with ( patch("canfar.models.config.CONFIG_PATH", config_path), - patch("canfar.server._discover_for_idp", return_value=[discovered]), + patch( + "canfar._server_discovery._discover_for_idp", + return_value=[discovered], + ), patch("canfar.server.Configuration", Configuration), ): discover("cadc") @@ -1150,9 +1261,15 @@ def test_rediscovery_updates_existing_name_in_place(self, tmp_path: Path) -> Non patch("canfar.models.config.CONFIG_PATH", config_path), patch("canfar.server.Configuration", Configuration), ): - with patch("canfar.server._discover_for_idp", return_value=[first]): + with patch( + "canfar._server_discovery._discover_for_idp", + return_value=[first], + ): discover("cadc") - with patch("canfar.server._discover_for_idp", return_value=[moved]): + with patch( + "canfar._server_discovery._discover_for_idp", + return_value=[moved], + ): discover("cadc") with patch("canfar.models.config.CONFIG_PATH", config_path): @@ -1179,11 +1296,14 @@ def test_registry_rename_inserts_new_key_without_rewriting_old( with patch("canfar.models.config.CONFIG_PATH", config_path): config = Configuration() assign_servers(config, original) - config.save() + config.editor.save() with ( patch("canfar.models.config.CONFIG_PATH", config_path), - patch("canfar.server._discover_for_idp", return_value=[renamed]), + patch( + "canfar._server_discovery._discover_for_idp", + return_value=[renamed], + ), patch("canfar.server.Configuration", Configuration), ): discover("cadc") @@ -1216,7 +1336,7 @@ def test_discover_keys_unnamed_server_by_host_slug(self, tmp_path: Path) -> None patch("canfar.models.config.CONFIG_PATH", config_path), patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( - "canfar.server.enrich", + "canfar._server_discovery.enrich", side_effect=lambda item, **_kwargs: item.model_copy( update={"version": "v1", "auths": ["oidc"]}, deep=True, @@ -1254,7 +1374,7 @@ async def test_discover_for_idp_converts_active_endpoints(self) -> None: with ( patch("canfar.utils.registry.Discover", return_value=mock_discovery), patch( - "canfar.server.enrich", + "canfar._server_discovery.enrich", side_effect=lambda item, **_kwargs: item.model_copy( update={"version": "v1", "auths": ["x509"]}, deep=True, diff --git a/tests/test_session.py b/tests/test_session.py deleted file mode 100644 index 89e90e66..00000000 --- a/tests/test_session.py +++ /dev/null @@ -1,389 +0,0 @@ -"""Test Canfar Session API.""" - -from time import sleep, time -from typing import Any -from unittest.mock import MagicMock, patch -from uuid import uuid4 - -import httpx -import pytest -from pydantic import SecretStr, ValidationError - -from canfar.models.session import CreateRequest -from canfar.sessions import Session - -pytest.IDENTITY: list[str] = [] - - -@pytest.fixture(scope="module") -def name(): - """Return a random name.""" - return str(uuid4().hex[:7]) - - -@pytest.fixture(scope="session") -def session(): - """Test images.""" - session = Session() - yield session - del session - - -@pytest.mark.integration -@pytest.mark.slow -def test_fetch_with_kind(session: Session) -> None: - """Test fetching images with kind.""" - session.fetch(kind="headless") - - -def _sync_response(json_data: Any = None, text: str = "") -> MagicMock: - response = MagicMock() - response.json.return_value = json_data - response.text = text - return response - - -def _http_error(method: str, url: str) -> httpx.HTTPStatusError: - request = httpx.Request(method, url) - response = httpx.Response(500, request=request) - return httpx.HTTPStatusError("boom", request=request, response=response) - - -def test_sync_session_methods_handle_success_and_failures() -> None: - """Sync Session methods return parsed data and tolerate per-ID failures.""" - session = Session(token=SecretStr("token"), url="https://example.test/skaha/v1") - client = MagicMock() - session._client = client # noqa: SLF001 - - client.get.return_value = _sync_response([{"id": "s1"}]) - assert session.fetch(kind="headless") == [{"id": "s1"}] - client.get.assert_called_with(url="session", params={"type": "headless"}) - - client.get.return_value = _sync_response({"cores": {}}) - assert session.stats() == {"cores": {}} - client.get.assert_called_with("session", params={"view": "stats"}) - - client.get.side_effect = [ - _sync_response({"id": "s1"}), - _http_error("GET", "https://example.test/skaha/v1/session/s2"), - ] - with patch("canfar.sessions._log_http_task_failure") as log_failure: - assert session.info(["s1", "s2"]) == [{"id": "s1"}] - log_failure.assert_called_once() - - client.get.side_effect = None - client.get.return_value = _sync_response(text="hello") - assert session.logs("s1") == {"s1": "hello"} - assert session.logs("s1", verbose=True) is None - - client.delete.side_effect = [ - None, - _http_error("DELETE", "https://example.test/skaha/v1/session/s2"), - ] - assert session.destroy(["s1", "s2"]) == {"s1": True, "s2": False} - - -def test_fetch_malformed_kind(session: Session) -> None: - """Test fetching images with malformed kind.""" - with pytest.raises(ValidationError): - session.fetch(kind="invalid") - - -def test_fetch_with_malformed_view(session: Session) -> None: - """Test fetching images with malformed view.""" - with pytest.raises(ValidationError): - session.fetch(view="invalid") - - -def test_fetch_with_malformed_status(session: Session) -> None: - """Test fetching images with malformed status.""" - with pytest.raises(ValidationError): - session.fetch(status="invalid") - - -@pytest.mark.slow -def test_session_stats(session: Session) -> None: - """Test fetching stats with kind.""" - assert "cores" in session.stats() - assert "ram" in session.stats() - - -def test_create_session_with_malformed_kind(session: Session, name: str) -> None: - """Test creating a session with malformed kind.""" - with pytest.raises(ValidationError): - session.create( - name=name, - kind="invalid", - image="ubuntu:latest", - cmd="bash", - replicas=1, - ) - - -def test_create_session_cmd_without_headless(session: Session, name: str) -> None: - """Test creating a session without headless.""" - with pytest.raises(ValidationError): - session.create( - name=name, - kind="notebook", - image="ubuntu:latest", - cmd="bash", - replicas=1, - ) - - -@pytest.mark.slow -def test_create_session(session: Session, name: str) -> None: - """Test creating a session.""" - identity: list[str] = session.create( - name=name, - kind="headless", - cores=1, - ram=1, - image="images.canfar.net/skaha/terminal:1.1.2", - cmd="env", - replicas=1, - env={"TEST": "test"}, - ) - assert len(identity) == 1 - assert identity[0] != "" - pytest.IDENTITY = identity - - -@pytest.mark.slow -def test_get_session_info(session: Session) -> None: - """Test getting session info.""" - info: list[dict[str, Any]] = [{}] - limit = time() + 60 # 1 minute - success: bool = False - while time() < limit: - sleep(1) - info = session.info(pytest.IDENTITY) - if len(info) == 1: - success = True - break - assert success, "Session info not found." - - -@pytest.mark.slow -def test_session_logs(session: Session) -> None: - """Test getting session logs.""" - limit = time() + 60 # 1 minute - logs: dict[str, str] = {} - while time() < limit: - sleep(1) - info = session.info(pytest.IDENTITY) - if info[0]["status"] in ("Succeeded", "Completed"): - logs = session.logs(pytest.IDENTITY) - success = False - for line in logs[pytest.IDENTITY[0]].split("\n"): - if "TEST=test" in line: - success = True - break - session.logs(pytest.IDENTITY, verbose=True) - assert success - - -@pytest.mark.slow -def test_session_events(session: Session) -> None: - """Test getting session events.""" - limit = time() + 60 # 1 minute - events: list[dict[str, str]] = [] - while time() < limit: - sleep(1) - events = session.events(pytest.IDENTITY) - if len(events) > 0: - break - assert pytest.IDENTITY[0] in events[0] - - -@pytest.mark.slow -def test_delete_session(session: Session, name: str) -> None: - """Test deleting a session.""" - # Delete the session - sleep(10) - deletion = session.destroy_with(prefix=name) - assert deletion == {pytest.IDENTITY[0]: True} - - -def test_destroy_with_regex_match(session: Session) -> None: - """Regex pattern should match substring in session name.""" - mock_sessions = [ - { - "id": "xyz789", - "name": "shiny-was-here", - "status": "Running", - "kind": "headless", - } - ] - pattern = ".*-was-" - - with ( - patch.object(Session, "fetch", return_value=mock_sessions) as mock_fetch, - patch.object(Session, "destroy", return_value={"xyz789": True}) as mock_destroy, - ): - result = session.destroy_with(prefix=pattern, kind="headless", status="Running") - - mock_fetch.assert_called_once_with(kind="headless", status="Running") - mock_destroy.assert_called_once_with(["xyz789"]) - assert result == {"xyz789": True} - - -def test_destroy_with_name_deprecation(session: Session) -> None: - """Deprecated name parameter removed; retained for backward-compat check.""" - with pytest.raises(TypeError): - session.destroy_with(name=".*-was-") # type: ignore[arg-type] - - -def test_create_session_with_type_field(name: str) -> None: - """Test creating a session and confirm kind field is changed to type.""" - specification: CreateRequest = CreateRequest( - name=name, - image="images.canfar.net/skaha/terminal:1.1.2", - cores=1, - ram=1, - kind="headless", - cmd="env", - replicas=1, - env={"TEST": "test"}, - ) - data: dict[str, Any] = specification.model_dump(exclude_none=True, by_alias=True) - assert "type" in data - assert data["type"] == "headless" - assert "kind" not in data - - -def test_bad_repica_requests(session: Session) -> None: - """Test error handling.""" - with pytest.raises(ValidationError): - session.create( - name="bad", - kind="firefly", - image="images.canfar.net/skaha/terminal:1.1.2", - replicas=10, - ) - with pytest.raises(ValidationError): - session.create( - name="bad", - kind="desktop", - image="images.canfar.net/skaha/terminal:1.1.2", - replicas=513, - ) - - -# Unit tests for connect method (covers lines 369-374) -class TestSessionConnect: - """Test the Session.connect method.""" - - @patch("canfar.sessions.open_new_tab") - @patch.object(Session, "info") - def test_connect_single_session_string(self, mock_info, mock_open_tab) -> None: - """Test connect with single session ID as string.""" - session = Session() - - # Mock the info method to return session data with connectURL - mock_info.return_value = [ - { - "id": "session-123", - "status": "Running", - "connectURL": "https://example.com/connect", - } - ] - - session.connect("session-123") - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-123"]) - - # Verify open_new_tab was called with the connectURL - mock_open_tab.assert_called_once_with("https://example.com/connect") - - @patch("canfar.sessions.open_new_tab") - @patch.object(Session, "info") - def test_connect_multiple_sessions_list(self, mock_info, mock_open_tab) -> None: - """Test connect with multiple session IDs as list.""" - session = Session() - - # Mock the info method to return session data for all IDs - mock_info.return_value = [ - { - "id": "session-1", - "status": "Running", - "connectURL": "https://example.com/connect1", - }, - { - "id": "session-2", - "status": "Running", - "connectURL": "https://example.com/connect2", - }, - ] - - session.connect(["session-1", "session-2"]) - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-1", "session-2"]) - - # Verify open_new_tab was called for each connectURL - assert mock_open_tab.call_count == 2 - mock_open_tab.assert_any_call("https://example.com/connect1") - mock_open_tab.assert_any_call("https://example.com/connect2") - - @patch("canfar.sessions.open_new_tab") - @patch.object(Session, "info") - def test_connect_empty_info_response(self, mock_info, mock_open_tab) -> None: - """Test connect when info returns empty list.""" - session = Session() - - # Mock the info method to return empty list - mock_info.return_value = [] - - # Should not raise any exception, just do nothing - session.connect("session-123") - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-123"]) - - # Verify open_new_tab was not called - mock_open_tab.assert_not_called() - - @patch("canfar.sessions.open_new_tab") - @patch.object(Session, "info") - def test_connect_missing_connect_url(self, mock_info, mock_open_tab) -> None: - """Test connect when session info lacks connectURL.""" - session = Session() - - # Mock the info method to return session without connectURL - mock_info.return_value = [{"id": "session-123", "status": "Running"}] - - # Should not raise any exception, just skip the session - session.connect("session-123") - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-123"]) - - # Verify open_new_tab was not called - mock_open_tab.assert_not_called() - - @patch("canfar.sessions.open_new_tab") - @patch.object(Session, "info") - def test_connect_non_running_session(self, mock_info, mock_open_tab) -> None: - """Test connect when session is not in Running status.""" - session = Session() - - # Mock the info method to return session with non-Running status - mock_info.return_value = [ - { - "id": "session-123", - "status": "Pending", - "connectURL": "https://example.com/connect", - } - ] - - # Should not raise any exception, just skip the session - session.connect("session-123") - - # Verify info was called with the session ID list - mock_info.assert_called_once_with(["session-123"]) - - # Verify open_new_tab was not called because status is not Running - mock_open_tab.assert_not_called() diff --git a/tests/test_sessions_create_errors.py b/tests/test_sessions_create_errors.py index 06cb1eda..b218b3e8 100644 --- a/tests/test_sessions_create_errors.py +++ b/tests/test_sessions_create_errors.py @@ -1,57 +1,79 @@ -"""Unit tests for Session.create and AsyncSession.create error handling.""" +"""Public Session.create request and failure contracts.""" from __future__ import annotations import logging -from contextlib import contextmanager -from typing import TYPE_CHECKING -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import patch import httpx import pytest -from pydantic import SecretStr +from pydantic import SecretStr, ValidationError from canfar.models.session import CreateRequest from canfar.sessions import AsyncSession, Session -if TYPE_CHECKING: - from collections.abc import Iterator - - -@contextmanager -def _session_log_sink(caplog: pytest.LogCaptureFixture) -> Iterator[None]: - """Attach pytest's real capture handler to the Session logger.""" - logger = logging.getLogger("canfar.sessions") - previous_level = logger.level - logger.addHandler(caplog.handler) - logger.setLevel(logging.DEBUG) - try: - yield - finally: - logger.removeHandler(caplog.handler) - logger.setLevel(previous_level) - - -def _http_status_error() -> httpx.HTTPStatusError: - request = httpx.Request("POST", "https://ws-uv.canfar.net/skaha/v1/session") - response = httpx.Response(500, request=request, text="no capacity") - return httpx.HTTPStatusError("server error", request=request, response=response) - - -@pytest.mark.asyncio -async def test_sync_and_async_create_serialize_the_same_request() -> None: - """Both public clients serialize one CreateRequest identically.""" - sent: dict[str, list[list[tuple[str, str]]]] = {"sync": [], "async": []} - - def handler(lane: str): - def respond(request: httpx.Request) -> httpx.Response: - sent[lane].append(request.url.params.multi_items()) - name = request.url.params["name"] - return httpx.Response(200, text=f"{name}-id\n", request=request) +_BASE_URL = "https://example.test/skaha/v1/" +_SERIALIZED_REQUEST = [ + ("name", "batch-1"), + ("image", "images.canfar.net/custom/image:latest"), + ("cores", "2"), + ("ram", "4"), + ("type", "headless"), + ("gpus", "1"), + ("cmd", "python"), + ("args", "-m worker"), + ("env", "A=1"), + ("env", "REPLICA_ID=1"), + ("env", "REPLICA_COUNT=2"), +] +_SERIALIZED_REQUEST_REPLICA_TWO = [ + ("name", "batch-2"), + ("image", "images.canfar.net/custom/image:latest"), + ("cores", "2"), + ("ram", "4"), + ("type", "headless"), + ("gpus", "1"), + ("cmd", "python"), + ("args", "-m worker"), + ("env", "A=1"), + ("env", "REPLICA_ID=2"), + ("env", "REPLICA_COUNT=2"), +] +_FAILURE_CASES = ( + pytest.param({"batch-2"}, ["batch-1-id"], id="partial"), + pytest.param({"batch-1", "batch-2"}, [], id="total"), +) +_INVALID_REQUEST_CASES = ( + pytest.param( + {"kind": "invalid", "image": "skaha/terminal:latest"}, + id="invalid-kind", + ), + pytest.param( + { + "kind": "notebook", + "image": "skaha/terminal:latest", + "cmd": "python", + }, + id="notebook-command", + ), + pytest.param( + {"kind": "headless", "image": "skaha/terminal:latest", "replicas": 0}, + id="zero-replicas", + ), + pytest.param( + { + "kind": "headless", + "image": "skaha/terminal:latest", + "replicas": 513, + }, + id="too-many-replicas", + ), +) - return respond - request = CreateRequest( +def _create_request() -> CreateRequest: + """Return one deterministic two-replica request.""" + return CreateRequest( name="batch", image="custom/image:latest", cores=2, @@ -63,77 +85,64 @@ def respond(request: httpx.Request) -> httpx.Response: env={"A": "1"}, replicas=2, ) - base_url = "https://example.test/skaha/v1/" - real_client = httpx.Client - real_async_client = httpx.AsyncClient - sync_transport = httpx.MockTransport(handler("sync")) - async_transport = httpx.MockTransport(handler("async")) + +def _create_responder(sent: list[list[tuple[str, str]]]): + """Return a transport handler that records and identifies replicas.""" + + def respond(request: httpx.Request) -> httpx.Response: + params = request.url.params.multi_items() + sent.append(params) + name = request.url.params["name"] + return httpx.Response(200, text=f"{name}-id\n", request=request) + + return respond + + +def test_sync_create_serializes_the_public_request_contract() -> None: + """Sync create serializes every replica through the HTTP boundary.""" + sent: list[list[tuple[str, str]]] = [] + real_client = httpx.Client with ( patch( "canfar.client.Client", side_effect=lambda **kwargs: real_client( - transport=sync_transport, + transport=httpx.MockTransport(_create_responder(sent)), **kwargs, ), ), - patch( - "canfar.client.AsyncClient", - side_effect=lambda **kwargs: real_async_client( - transport=async_transport, - **kwargs, - ), - ), - Session(token=SecretStr("token"), url=base_url) as session, + Session(token=SecretStr("token"), url=_BASE_URL) as session, ): - async with AsyncSession(token=SecretStr("token"), url=base_url) as asession: - assert session.create(request) == ["batch-1-id", "batch-2-id"] - assert await asession.create(request) == ["batch-1-id", "batch-2-id"] - - assert sent["sync"] == sent["async"] - assert sent["sync"] == [ - [ - ("name", "batch-1"), - ("image", "images.canfar.net/custom/image:latest"), - ("cores", "2"), - ("ram", "4"), - ("type", "headless"), - ("gpus", "1"), - ("cmd", "python"), - ("args", "-m worker"), - ("env", "A=1"), - ("env", "REPLICA_ID=1"), - ("env", "REPLICA_COUNT=2"), - ], - [ - ("name", "batch-2"), - ("image", "images.canfar.net/custom/image:latest"), - ("cores", "2"), - ("ram", "4"), - ("type", "headless"), - ("gpus", "1"), - ("cmd", "python"), - ("args", "-m worker"), - ("env", "A=1"), - ("env", "REPLICA_ID=2"), - ("env", "REPLICA_COUNT=2"), - ], - ] + assert session.create(_create_request()) == ["batch-1-id", "batch-2-id"] + + assert sent[0] == _SERIALIZED_REQUEST + assert sent[1] == _SERIALIZED_REQUEST_REPLICA_TWO @pytest.mark.asyncio -@pytest.mark.parametrize( - ("failed_names", "expected"), - [ - ({"batch-2"}, ["batch-1-id"]), - ({"batch-1", "batch-2"}, []), - ], -) -async def test_sync_and_async_create_share_http_failure_policy( - failed_names: set[str], - expected: list[str], -) -> None: - """Per-replica HTTP failures are omitted and total failure returns empty.""" +async def test_async_create_serializes_the_public_request_contract() -> None: + """Async create serializes every replica through the HTTP boundary.""" + sent: list[list[tuple[str, str]]] = [] + real_async_client = httpx.AsyncClient + with patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: real_async_client( + transport=httpx.MockTransport(_create_responder(sent)), + **kwargs, + ), + ): + async with AsyncSession(token=SecretStr("token"), url=_BASE_URL) as session: + assert await session.create(_create_request()) == [ + "batch-1-id", + "batch-2-id", + ] + + assert sent[0] == _SERIALIZED_REQUEST + assert sent[1] == _SERIALIZED_REQUEST_REPLICA_TWO + + +def _failure_responder(failed_names: set[str]): + """Return a transport handler for partial or total replica failures.""" def respond(request: httpx.Request) -> httpx.Response: name = request.url.params["name"] @@ -150,92 +159,163 @@ def respond(request: httpx.Request) -> httpx.Response: ) return httpx.Response(200, text=f"{name}-id\n", request=request) + return respond + + +@pytest.mark.parametrize( + ("failed_names", "expected"), + _FAILURE_CASES, +) +def test_sync_create_omits_failed_replicas( + failed_names: set[str], expected: list[str] +) -> None: + """Sync create omits failed replicas and returns an empty total failure.""" request = CreateRequest( - name="batch", - image="skaha/terminal:latest", - kind="headless", - replicas=2, + name="batch", image="skaha/terminal:latest", kind="headless", replicas=2 ) - base_url = "https://example.test/skaha/v1/" - transport = httpx.MockTransport(respond) real_client = httpx.Client - real_async_client = httpx.AsyncClient - with ( patch( "canfar.client.Client", - side_effect=lambda **kwargs: real_client(transport=transport, **kwargs), - ), - patch( - "canfar.client.AsyncClient", - side_effect=lambda **kwargs: real_async_client( - transport=transport, + side_effect=lambda **kwargs: real_client( + transport=httpx.MockTransport(_failure_responder(failed_names)), **kwargs, ), ), - Session(token=SecretStr("token"), url=base_url) as session, + Session(token=SecretStr("token"), url=_BASE_URL) as session, ): - async with AsyncSession(token=SecretStr("token"), url=base_url) as asession: - assert session.create(request) == expected - assert await asession.create(request) == expected + assert session.create(request) == expected -def test_sync_create_failure_logs_only_safe_replica_context( - caplog: pytest.LogCaptureFixture, +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failed_names", "expected"), + _FAILURE_CASES, +) +async def test_async_create_omits_failed_replicas( + failed_names: set[str], expected: list[str] ) -> None: - """Sync create omits the request payload and raw exception from logs.""" - session = Session() - mock_client = MagicMock() - mock_client.post.side_effect = _http_status_error() - session._client = mock_client # noqa: SLF001 - environment_secret = "sync-environment-secret-sentinel" - - with _session_log_sink(caplog): - result = session.create( - name="test-name", - image="images.example/net/img:latest", - kind="headless", - env={"ACCESS_TOKEN": environment_secret}, - replicas=1, - ) + """Async create omits failed replicas and returns an empty total failure.""" + request = CreateRequest( + name="batch", image="skaha/terminal:latest", kind="headless", replicas=2 + ) + real_async_client = httpx.AsyncClient + with patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: real_async_client( + transport=httpx.MockTransport(_failure_responder(failed_names)), + **kwargs, + ), + ): + async with AsyncSession(token=SecretStr("token"), url=_BASE_URL) as session: + assert await session.create(request) == expected - assert result == [] + +@pytest.mark.parametrize( + "request_kwargs", + _INVALID_REQUEST_CASES, +) +def test_sync_create_rejects_invalid_requests( + request_kwargs: dict[str, object], +) -> None: + """Invalid create requests fail before the synchronous HTTP boundary.""" + with ( + Session(token=SecretStr("token"), url="https://example.test") as session, + pytest.raises(ValidationError), + ): + session.create(name="batch", **request_kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_kwargs", + _INVALID_REQUEST_CASES, +) +async def test_async_create_rejects_invalid_requests( + request_kwargs: dict[str, object], +) -> None: + """Invalid create requests fail before the asynchronous HTTP boundary.""" + with pytest.raises(ValidationError): + async with AsyncSession( + token=SecretStr("token"), url="https://example.test" + ) as session: + await session.create(name="batch", **request_kwargs) + + +def _failure_log_responder(request: httpx.Request) -> httpx.Response: + """Raise one safe-to-log HTTP failure for a create request.""" + response = httpx.Response(500, request=request, text="no capacity") + message = "server error" + raise httpx.HTTPStatusError(message, request=request, response=response) + + +def _assert_safe_create_log(caplog: pytest.LogCaptureFixture, secret: str) -> None: + """Assert create logs only stable replica context and exception type.""" logged = caplog.text - assert environment_secret not in logged + assert secret not in logged assert "no capacity" not in logged assert "server error" not in logged assert "Failed to create session" in logged assert "replica 1/1" in logged assert "HTTPStatusError" in logged - mock_client.post.assert_called_once() + + +def test_sync_create_failure_logs_only_safe_replica_context( + caplog: pytest.LogCaptureFixture, +) -> None: + """Sync create omits request payload and raw exception from logs.""" + environment_secret = "sync-create-environment-secret" + caplog.set_level(logging.ERROR, logger="canfar.sessions") + real_client = httpx.Client + with ( + patch( + "canfar.client.Client", + side_effect=lambda **kwargs: real_client( + transport=httpx.MockTransport(_failure_log_responder), + **kwargs, + ), + ), + Session(token=SecretStr("token"), url=_BASE_URL) as session, + ): + assert ( + session.create( + name="test-name", + image="images.example/net/img:latest", + kind="headless", + env={"ACCESS_TOKEN": environment_secret}, + replicas=1, + ) + == [] + ) + + _assert_safe_create_log(caplog, environment_secret) @pytest.mark.asyncio async def test_async_create_failure_logs_only_safe_replica_context( caplog: pytest.LogCaptureFixture, ) -> None: - """Async create omits the request payload and raw exception from logs.""" - asession = AsyncSession() - mock_client = MagicMock() - mock_client.post = AsyncMock(side_effect=_http_status_error()) - asession._asynclient = mock_client # noqa: SLF001 - environment_secret = "async-environment-secret-sentinel" - - with _session_log_sink(caplog): - result = await asession.create( - name="test-name", - image="images.example/net/img:latest", - kind="headless", - env={"REFRESH_TOKEN": environment_secret}, - replicas=1, - ) + """Async create omits request payload and raw exception from logs.""" + environment_secret = "async-create-environment-secret" + caplog.set_level(logging.ERROR, logger="canfar.sessions") + real_async_client = httpx.AsyncClient + with patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: real_async_client( + transport=httpx.MockTransport(_failure_log_responder), + **kwargs, + ), + ): + async with AsyncSession(token=SecretStr("token"), url=_BASE_URL) as session: + assert ( + await session.create( + name="test-name", + image="images.example/net/img:latest", + kind="headless", + env={"REFRESH_TOKEN": environment_secret}, + replicas=1, + ) + == [] + ) - assert result == [] - logged = caplog.text - assert environment_secret not in logged - assert "no capacity" not in logged - assert "server error" not in logged - assert "Failed to create session" in logged - assert "replica 1/1" in logged - assert "HTTPStatusError" in logged - mock_client.post.assert_awaited_once() + _assert_safe_create_log(caplog, environment_secret) diff --git a/tests/test_sessions_fetch.py b/tests/test_sessions_fetch.py new file mode 100644 index 00000000..8c8c36b1 --- /dev/null +++ b/tests/test_sessions_fetch.py @@ -0,0 +1,132 @@ +"""Paired public contracts for synchronous and asynchronous Session reads.""" + +from __future__ import annotations + +from unittest.mock import patch + +import httpx +import pytest +from pydantic import SecretStr, ValidationError + +from canfar.sessions import AsyncSession, Session + +_FETCH_PAYLOAD = [{"id": "session-1", "name": "notebook", "status": "Running"}] +_FETCH_FILTERS = {"kind": "notebook", "status": "Running", "view": "all"} +_FETCH_PARAMS = [("type", "notebook"), ("status", "Running"), ("view", "all")] +_STATS_PAYLOAD = {"cores": {"available": 4}, "ram": {"available": "8G"}} +_BASE_URL = "https://example.test/skaha/v1/" + + +def test_sync_fetch_preserves_request_and_response_contract() -> None: + """Sync fetch preserves filters and the server response shape.""" + sent: list[tuple[str, str]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.extend(request.url.params.multi_items()) + return httpx.Response(200, json=_FETCH_PAYLOAD, request=request) + + real_client = httpx.Client + with ( + patch( + "canfar.client.Client", + side_effect=lambda **kwargs: real_client( + transport=httpx.MockTransport(respond), + **kwargs, + ), + ), + Session(token=SecretStr("token"), url=_BASE_URL) as session, + ): + assert session.fetch(**_FETCH_FILTERS) == _FETCH_PAYLOAD + + assert sent == _FETCH_PARAMS + + +@pytest.mark.asyncio +async def test_async_fetch_preserves_request_and_response_contract() -> None: + """Async fetch preserves filters and the server response shape.""" + sent: list[tuple[str, str]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.extend(request.url.params.multi_items()) + return httpx.Response(200, json=_FETCH_PAYLOAD, request=request) + + real_async_client = httpx.AsyncClient + with patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: real_async_client( + transport=httpx.MockTransport(respond), + **kwargs, + ), + ): + async with AsyncSession(token=SecretStr("token"), url=_BASE_URL) as session: + assert await session.fetch(**_FETCH_FILTERS) == _FETCH_PAYLOAD + + assert sent == _FETCH_PARAMS + + +def test_sync_stats_preserves_response_shape() -> None: + """Sync stats returns the decoded platform response.""" + sent: list[tuple[str, str]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.extend(request.url.params.multi_items()) + return httpx.Response(200, json=_STATS_PAYLOAD, request=request) + + real_client = httpx.Client + with ( + patch( + "canfar.client.Client", + side_effect=lambda **kwargs: real_client( + transport=httpx.MockTransport(respond), + **kwargs, + ), + ), + Session(token=SecretStr("token"), url=_BASE_URL) as session, + ): + assert session.stats() == _STATS_PAYLOAD + + assert sent == [("view", "stats")] + + +@pytest.mark.asyncio +async def test_async_stats_preserves_response_shape() -> None: + """Async stats returns the decoded platform response.""" + sent: list[tuple[str, str]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.extend(request.url.params.multi_items()) + return httpx.Response(200, json=_STATS_PAYLOAD, request=request) + + real_async_client = httpx.AsyncClient + with patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: real_async_client( + transport=httpx.MockTransport(respond), + **kwargs, + ), + ): + async with AsyncSession(token=SecretStr("token"), url=_BASE_URL) as session: + assert await session.stats() == _STATS_PAYLOAD + + assert sent == [("view", "stats")] + + +@pytest.mark.parametrize("field", ["kind", "status", "view"]) +def test_sync_fetch_rejects_invalid_filter(field: str) -> None: + """Invalid Session filters fail at the synchronous public boundary.""" + with ( + Session(token=SecretStr("token"), url="https://example.test") as session, + pytest.raises(ValidationError), + ): + session.fetch(**{field: "invalid"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["kind", "status", "view"]) +async def test_async_fetch_rejects_invalid_filter(field: str) -> None: + """Invalid Session filters fail at the asynchronous public boundary.""" + async with AsyncSession( + token=SecretStr("token"), url="https://example.test" + ) as session: + with pytest.raises(ValidationError): + await session.fetch(**{field: "invalid"}) diff --git a/tests/test_sessions_info_logs.py b/tests/test_sessions_info_logs.py index 11d40d06..74642fec 100644 --- a/tests/test_sessions_info_logs.py +++ b/tests/test_sessions_info_logs.py @@ -11,84 +11,114 @@ from canfar.sessions import AsyncSession, Session +_BASE_URL = "https://example.test/skaha/v1/" + +_INFO_LOG_CASES = ( + pytest.param([], [], {}, id="empty"), + pytest.param("one", [{"id": "one"}], {"one": "log-one"}, id="one"), + pytest.param( + ["one", "two"], + [{"id": "one"}, {"id": "two"}], + {"one": "log-one", "two": "log-two"}, + id="many", + ), + pytest.param( + ["one", "failed", "three"], + [{"id": "one"}, {"id": "three"}], + {"one": "log-one", "three": "log-three"}, + id="mixed-failure", + ), +) + + +def _respond(request: httpx.Request) -> httpx.Response: + """Return deterministic info/log payloads or one transport failure.""" + session_id = request.url.path.rsplit("/", 1)[-1] + if session_id == "failed": + message = "connection refused" + raise httpx.ConnectError(message, request=request) + if request.url.params.get("view") == "logs": + return httpx.Response(200, text=f"log-{session_id}", request=request) + return httpx.Response(200, json={"id": session_id}, request=request) + + +def _verbose_messages(caplog: pytest.LogCaptureFixture) -> list[str]: + """Return only Session logger messages captured for verbose output.""" + return [ + record.getMessage() + for record in caplog.records + if record.name == "canfar.sessions" and record.levelno == logging.INFO + ] + -@pytest.mark.asyncio @pytest.mark.parametrize( ("ids", "expected_info", "expected_logs"), - [ - ([], [], {}), - ("one", [{"id": "one"}], {"one": "log-one"}), - ( - ["one", "two"], - [{"id": "one"}, {"id": "two"}], - {"one": "log-one", "two": "log-two"}, - ), - ( - ["one", "failed", "three"], - [{"id": "one"}, {"id": "three"}], - {"one": "log-one", "three": "log-three"}, - ), - ], + _INFO_LOG_CASES, ) -async def test_sync_and_async_info_and_logs_share_public_policy( +def test_sync_info_and_logs_share_public_policy( ids: str | list[str], expected_info: list[dict[str, str]], expected_logs: dict[str, str], caplog: pytest.LogCaptureFixture, ) -> None: - """Zero, one, many, and mixed results have matching shape and order.""" - - def respond(request: httpx.Request) -> httpx.Response: - session_id = request.url.path.rsplit("/", 1)[-1] - if session_id == "failed": - message = "connection refused" - raise httpx.ConnectError(message, request=request) - if request.url.params.get("view") == "logs": - return httpx.Response( - 200, - text=f"log-{session_id}", - request=request, - ) - return httpx.Response(200, json={"id": session_id}, request=request) - - base_url = "https://example.test/skaha/v1/" - transport = httpx.MockTransport(respond) + """Sync info/logs preserve shape, order, failures, and verbose routing.""" real_client = httpx.Client - real_async_client = httpx.AsyncClient - with ( patch( "canfar.client.Client", - side_effect=lambda **kwargs: real_client(transport=transport, **kwargs), - ), - patch( - "canfar.client.AsyncClient", - side_effect=lambda **kwargs: real_async_client( - transport=transport, + side_effect=lambda **kwargs: real_client( + transport=httpx.MockTransport(_respond), **kwargs, ), ), - Session(token=SecretStr("token"), url=base_url) as session, + Session(token=SecretStr("token"), url=_BASE_URL) as session, + ): + assert session.info(ids) == expected_info + assert session.logs(ids) == expected_logs + + caplog.set_level(logging.INFO, logger="canfar.sessions") + caplog.clear() + assert session.logs(ids, verbose=True) is None + sync_messages = _verbose_messages(caplog) + + assert sync_messages == [ + message + for session_id, message in expected_logs.items() + for message in (f"Session ID: {session_id}\n", message) + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("ids", "expected_info", "expected_logs"), + _INFO_LOG_CASES, +) +async def test_async_info_and_logs_share_public_policy( + ids: str | list[str], + expected_info: list[dict[str, str]], + expected_logs: dict[str, str], + caplog: pytest.LogCaptureFixture, +) -> None: + """Async info/logs preserve shape, order, failures, and verbose routing.""" + real_async_client = httpx.AsyncClient + with patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: real_async_client( + transport=httpx.MockTransport(_respond), + **kwargs, + ), ): - async with AsyncSession(token=SecretStr("token"), url=base_url) as asession: - assert session.info(ids) == expected_info - assert await asession.info(ids) == expected_info - assert session.logs(ids) == expected_logs - assert await asession.logs(ids) == expected_logs + async with AsyncSession(token=SecretStr("token"), url=_BASE_URL) as session: + assert await session.info(ids) == expected_info + assert await session.logs(ids) == expected_logs caplog.set_level(logging.INFO, logger="canfar.sessions") caplog.clear() - assert session.logs(ids, verbose=True) is None - sync_messages = [ - record.getMessage() - for record in caplog.records - if record.name == "canfar.sessions" - ] - caplog.clear() - assert await asession.logs(ids, verbose=True) is None - async_messages = [ - record.getMessage() - for record in caplog.records - if record.name == "canfar.sessions" - ] - assert sync_messages == async_messages + assert await session.logs(ids, verbose=True) is None + async_messages = _verbose_messages(caplog) + + assert async_messages == [ + message + for session_id, message in expected_logs.items() + for message in (f"Session ID: {session_id}\n", message) + ] diff --git a/tests/test_sessions_integration.py b/tests/test_sessions_integration.py new file mode 100644 index 00000000..167d8356 --- /dev/null +++ b/tests/test_sessions_integration.py @@ -0,0 +1,106 @@ +"""Credentialed Session lifecycle contracts. + +These tests intentionally keep each live workflow self-contained. They are +excluded from the deterministic local gate and require a configured CANFAR +Authentication Record. +""" + +from __future__ import annotations + +import asyncio +from time import monotonic, sleep +from uuid import uuid4 + +import pytest + +from canfar.sessions import AsyncSession, Session + + +def _session_name() -> str: + """Return a unique name for one credentialed lifecycle.""" + return f"contract-{uuid4().hex[:10]}" + + +@pytest.mark.integration +@pytest.mark.slow +def test_sync_session_lifecycle_is_self_contained() -> None: + """Create, observe, and clean up one synchronous Session.""" + name = _session_name() + identity: list[str] = [] + + with Session() as session: + try: + identity = session.create( + name=name, + kind="headless", + cores=1, + ram=1, + image="images.canfar.net/skaha/terminal:1.1.2", + cmd="env", + replicas=1, + env={"TEST": "test"}, + ) + assert len(identity) == 1 + session_id = identity[0] + + deadline = monotonic() + 60 + info: list[dict[str, object]] = [] + while monotonic() < deadline: + info = session.info(session_id) + if info and info[0].get("status") in {"Succeeded", "Completed"}: + break + sleep(1) + assert info + assert info[0].get("status") in {"Succeeded", "Completed"} + + logs = session.logs(session_id) + assert logs is not None + assert "TEST=test" in logs[session_id] + events = session.events(session_id) + assert any(session_id in event for event in events) + finally: + if identity: + session.destroy(identity) + + +@pytest.mark.integration +@pytest.mark.slow +@pytest.mark.asyncio +async def test_async_session_lifecycle_is_self_contained() -> None: + """Create, observe, and clean up one asynchronous Session.""" + name = _session_name() + identity: list[str] = [] + + async with AsyncSession() as session: + try: + identity = await session.create( + name=name, + kind="headless", + cores=1, + ram=1, + image="images.canfar.net/skaha/terminal:1.1.2", + cmd="env", + replicas=1, + env={"TEST": "test"}, + ) + assert len(identity) == 1 + session_id = identity[0] + + deadline = monotonic() + 60 + info: list[dict[str, object]] = [] + while monotonic() < deadline: + info = await session.info(session_id) + if info and info[0].get("status") in {"Succeeded", "Completed"}: + break + await asyncio.sleep(1) + assert info + assert info[0].get("status") in {"Succeeded", "Completed"} + + logs = await session.logs(session_id) + assert logs is not None + assert "TEST=test" in logs[session_id] + events = await session.events(session_id) + assert any(session_id in event for event in events) + finally: + if identity: + await session.destroy(identity) diff --git a/tests/test_sessions_lifecycle.py b/tests/test_sessions_lifecycle.py index e24d9da4..70a5dbe9 100644 --- a/tests/test_sessions_lifecycle.py +++ b/tests/test_sessions_lifecycle.py @@ -2,6 +2,7 @@ from __future__ import annotations +from inspect import signature from typing import Any from unittest.mock import patch @@ -11,6 +12,57 @@ from canfar.sessions import AsyncSession, Session, connection_url +_BASE_URL = "https://example.test/skaha/v1/" +_CONNECT_IDS = ["missing", "stopped", "running", "terminating", "no-url"] +_EXPECTED_OPEN_URL = "https://example.test/running" + +_EXPECTED_SESSION_SIGNATURES: dict[str, str] = { + "fetch": ( + "(self, kind: 'Kind | None' = None, " + "status: 'Status | None' = None, " + "view: 'View | None' = None) -> 'list[dict[str, str]]'" + ), + "stats": "(self) -> 'dict[str, Any]'", + "info": "(self, ids: 'list[str] | str') -> 'list[dict[str, Any]]'", + "logs": ( + "(self, ids: 'list[str] | str', verbose: 'bool' = False) " + "-> 'dict[str, str] | None'" + ), + "create": ( + "(self, name: 'str | CreateRequest', image: 'str | None' = None, " + "cores: 'int | None' = None, ram: 'int | None' = None, " + "kind: 'Kind' = 'headless', gpu: 'int | None' = None, " + "cmd: 'str | None' = None, args: 'str | None' = None, " + "env: 'dict[str, Any] | None' = None, replicas: 'int' = 1) " + "-> 'list[str]'" + ), + "events": ( + "(self, ids: 'str | list[str]', verbose: 'bool' = False) " + "-> 'list[dict[str, str]] | None'" + ), + "destroy": "(self, ids: 'str | list[str]') -> 'dict[str, bool]'", + "destroy_with": ( + "(self, prefix: 'str', *, kind: 'Kind' = 'headless', " + "status: 'Status' = 'Completed') -> 'dict[str, bool]'" + ), + "connect": "(self, ids: 'list[str] | str') -> 'None'", +} + + +@pytest.mark.parametrize( + ("method", "expected"), + list(_EXPECTED_SESSION_SIGNATURES.items()), + ids=list(_EXPECTED_SESSION_SIGNATURES), +) +def test_session_signatures_are_stable_and_parallel( + method: str, + expected: str, +) -> None: + """Sync and async Session methods keep the complete released contract.""" + sync_signature = signature(getattr(Session, method)) + assert str(sync_signature) == expected + assert signature(getattr(AsyncSession, method)) == sync_signature + @pytest.mark.parametrize( ("record", "expected"), @@ -49,100 +101,190 @@ def test_connection_url_is_the_shared_eligibility_policy( assert connection_url(record) == expected -@pytest.mark.asyncio -async def test_sync_and_async_lifecycle_share_public_policy() -> None: - """Events, destruction, selection, and connection have matching outcomes.""" +def _respond(request: httpx.Request) -> httpx.Response: + """Return deterministic lifecycle payloads from one public transport.""" + if request.url.path.endswith("/session"): + return httpx.Response( + 200, + json=[ + {"id": "batch-1", "name": "batch-1"}, + {"id": "other", "name": "other-batch"}, + ], + request=request, + ) + + session_id = request.url.path.rsplit("/", 1)[-1] + if session_id in {"failed", "missing"}: + message = "connection refused" + raise httpx.ConnectError(message, request=request) + if request.method == "DELETE": + return httpx.Response(204, request=request) + if request.url.params.get("view") == "events": + return httpx.Response(200, text=f"event-{session_id}", request=request) + + records: dict[str, dict[str, Any]] = { + "stopped": { + "id": "stopped", + "status": "Stopped", + "connectURL": "https://example.test/stopped", + }, + "running": { + "id": "running", + "status": "Running", + "connectURL": _EXPECTED_OPEN_URL, + }, + "terminating": { + "id": "terminating", + "status": "Terminating", + "connectURL": "https://example.test/terminating", + }, + "no-url": {"id": "no-url", "status": "Running"}, + } + return httpx.Response(200, json=records[session_id], request=request) + + +def _filtered_destroy_responder(requests: list[httpx.Request]): + """Record the filtered list request and accept its matching deletion.""" def respond(request: httpx.Request) -> httpx.Response: - if request.url.path.endswith("/session"): + if request.method == "GET" and request.url.path.endswith("/session"): + requests.append(request) return httpx.Response( 200, json=[ - {"id": "batch-1", "name": "batch-1"}, - {"id": "other", "name": "other-batch"}, + { + "id": "batch-1", + "name": "batch-1", + "kind": "headless", + "status": "Running", + } ], request=request, ) - - session_id = request.url.path.rsplit("/", 1)[-1] - if session_id in {"failed", "missing"}: - message = "connection refused" - raise httpx.ConnectError(message, request=request) if request.method == "DELETE": return httpx.Response(204, request=request) - if request.url.params.get("view") == "events": - return httpx.Response(200, text=f"event-{session_id}", request=request) + message = f"Unexpected request: {request.method} {request.url}" + raise AssertionError(message) - records: dict[str, dict[str, Any]] = { - "stopped": { - "id": "stopped", - "status": "Stopped", - "connectURL": "https://example.test/stopped", - }, - "running": { - "id": "running", - "status": "Running", - "connectURL": "https://example.test/running", - }, - "terminating": { - "id": "terminating", - "status": "Terminating", - "connectURL": "https://example.test/terminating", - }, - } - return httpx.Response(200, json=records[session_id], request=request) + return respond - base_url = "https://example.test/skaha/v1/" - transport = httpx.MockTransport(respond) - real_client = httpx.Client - real_async_client = httpx.AsyncClient +def test_sync_lifecycle_share_public_policy() -> None: + """Sync events, destruction, selection, and connection share one policy.""" + real_client = httpx.Client with ( patch( "canfar.client.Client", - side_effect=lambda **kwargs: real_client(transport=transport, **kwargs), + side_effect=lambda **kwargs: real_client( + transport=httpx.MockTransport(_respond), + **kwargs, + ), ), + patch("canfar.sessions.open_new_tab") as open_tab, + Session(token=SecretStr("token"), url=_BASE_URL) as session, + ): + ids = ["one", "failed", "three"] + assert session.events(ids) == [ + {"one": "event-one"}, + {"three": "event-three"}, + ] + assert session.events("running", verbose=True) is None + assert session.destroy(ids) == { + "one": True, + "failed": False, + "three": True, + } + assert session.destroy_with("batch") == {"batch-1": True} + assert session.destroy_with("other-.*") == {"other": True} + session.connect(_CONNECT_IDS) + session.connect("running") + + assert [call.args[0] for call in open_tab.call_args_list] == [ + _EXPECTED_OPEN_URL, + _EXPECTED_OPEN_URL, + ] + + +@pytest.mark.asyncio +async def test_async_lifecycle_share_public_policy() -> None: + """Async events, destruction, selection, and connection share one policy.""" + real_async_client = httpx.AsyncClient + with ( patch( "canfar.client.AsyncClient", side_effect=lambda **kwargs: real_async_client( - transport=transport, + transport=httpx.MockTransport(_respond), **kwargs, ), ), patch("canfar.sessions.open_new_tab") as open_tab, - Session(token=SecretStr("token"), url=base_url) as session, ): - async with AsyncSession(token=SecretStr("token"), url=base_url) as asession: + async with AsyncSession(token=SecretStr("token"), url=_BASE_URL) as session: ids = ["one", "failed", "three"] - expected_events = [{"one": "event-one"}, {"three": "event-three"}] - expected_destroy = {"one": True, "failed": False, "three": True} - - assert session.events(ids) == expected_events - assert await asession.events(ids) == expected_events - assert session.destroy(ids) == expected_destroy - assert await asession.destroy(ids) == expected_destroy - assert session.destroy_with("batch") == {"batch-1": True} - assert await asession.destroy_with("batch") == {"batch-1": True} + assert await session.events(ids) == [ + {"one": "event-one"}, + {"three": "event-three"}, + ] + assert await session.events("running", verbose=True) is None + assert await session.destroy(ids) == { + "one": True, + "failed": False, + "three": True, + } + assert await session.destroy_with("batch") == {"batch-1": True} + assert await session.destroy_with("other-.*") == {"other": True} + await session.connect(_CONNECT_IDS) - connect_ids = ["missing", "stopped", "running", "terminating"] - session.connect(connect_ids) - await asession.connect(connect_ids) + await session.connect("running") assert [call.args[0] for call in open_tab.call_args_list] == [ - "https://example.test/running", - "https://example.test/running", + _EXPECTED_OPEN_URL, + _EXPECTED_OPEN_URL, ] -@pytest.mark.asyncio -async def test_destroy_with_kind_and_status_are_keyword_only() -> None: - """Kind and status must be keyword-only on Session and AsyncSession.""" - base_url = "https://example.test/skaha/v1/" - token = SecretStr("token") +def test_sync_destroy_with_passes_kind_and_status_filters() -> None: + """Sync destroy_with forwards its filter keywords to Session.fetch.""" + requests: list[httpx.Request] = [] + real_client = httpx.Client + with ( + patch( + "canfar.client.Client", + side_effect=lambda **kwargs: real_client( + transport=httpx.MockTransport(_filtered_destroy_responder(requests)), + **kwargs, + ), + ), + Session(token=SecretStr("token"), url=_BASE_URL) as session, + ): + assert session.destroy_with("batch", kind="headless", status="Running") == { + "batch-1": True + } - with Session(token=token, url=base_url) as session, pytest.raises(TypeError): - session.destroy_with("prefix", "headless", "Completed") + assert requests[0].url.params.multi_items() == [ + ("type", "headless"), + ("status", "Running"), + ] - async with AsyncSession(token=token, url=base_url) as asession: - with pytest.raises(TypeError): - await asession.destroy_with("prefix", "headless", "Completed") + +@pytest.mark.asyncio +async def test_async_destroy_with_passes_kind_and_status_filters() -> None: + """Async destroy_with forwards its filter keywords to Session.fetch.""" + requests: list[httpx.Request] = [] + real_async_client = httpx.AsyncClient + with patch( + "canfar.client.AsyncClient", + side_effect=lambda **kwargs: real_async_client( + transport=httpx.MockTransport(_filtered_destroy_responder(requests)), + **kwargs, + ), + ): + async with AsyncSession(token=SecretStr("token"), url=_BASE_URL) as session: + assert await session.destroy_with( + "batch", kind="headless", status="Running" + ) == {"batch-1": True} + + assert requests[0].url.params.multi_items() == [ + ("type", "headless"), + ("status", "Running"), + ] diff --git a/tests/test_storage.py b/tests/test_storage.py index e4e7a454..ff76dccf 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock +import fsspec import pytest import vosfs from fsspec.implementations.local import LocalFileSystem @@ -16,7 +17,7 @@ from canfar.models.active import ActiveConfig from canfar.models.config import Configuration from canfar.models.http import Server, VOSpaceService -from canfar.storage import _vospace +from canfar.storage import _sources, _vospace from tests.helpers.config import oidc_credential, x509_credential if TYPE_CHECKING: @@ -84,7 +85,7 @@ async def test_source_reloads_config_and_runtime_token_wins( ) -> None: """Entry reloads endpoint state and keeps token-over-certificate precedence.""" config = _config(credential=oidc_credential("inactive")) - config.save() + config.editor.save() source = _vospace( "archive", token="runtime-token", @@ -94,7 +95,7 @@ async def test_source_reloads_config_and_runtime_token_wins( config.servers["inactive"].storage["archive"].url = AnyHttpUrl( "https://changed.example/vospace" ) - config.save() + config.editor.save() monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) async with source() as filesystem: @@ -116,7 +117,9 @@ async def test_source_factory_constructs_fresh_filesystem_per_acquisition( monkeypatch: pytest.MonkeyPatch, ) -> None: """Repeated acquisition constructs and closes distinct VOSpace filesystems.""" - _config(credential=oidc_credential("inactive", access="current-token")).save() + _config( + credential=oidc_credential("inactive", access="current-token") + ).editor.save() filesystems: list[_Filesystem] = [] def build(endpoint: str, **kwargs: Any) -> _Filesystem: @@ -144,7 +147,7 @@ async def test_environment_token_preserves_runtime_precedence( monkeypatch: pytest.MonkeyPatch, ) -> None: """The source leaves an omitted token open to HTTPClient environment settings.""" - _config(credential=x509_credential("inactive")).save() + _config(credential=x509_credential("inactive")).editor.save() monkeypatch.delenv("CANFAR_CERTIFICATE", raising=False) monkeypatch.setenv("CANFAR_TOKEN", "environment-token") monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) @@ -165,7 +168,7 @@ async def test_environment_certificate_preserves_runtime_precedence( ) -> None: """The source leaves an omitted certificate open to settings sources.""" certificate = tmp_path / "environment.pem" - _config(credential=oidc_credential("inactive")).save() + _config(credential=oidc_credential("inactive")).editor.save() monkeypatch.delenv("CANFAR_TOKEN", raising=False) monkeypatch.setenv("CANFAR_CERTIFICATE", certificate.as_posix()) monkeypatch.setattr( @@ -200,7 +203,7 @@ async def test_expired_inactive_oidc_refreshes_once_and_persists( access_expiry=1.0, ) ) - config.save() + config.editor.save() refresh = AsyncMock( return_value={ "access_token": "new-access-secret", @@ -220,7 +223,7 @@ async def test_expired_inactive_oidc_refreshes_once_and_persists( "refresh-secret", ) persisted = Configuration() # ty: ignore[missing-argument] - saved = persisted.get_credential("inactive") + saved = persisted.authentication["inactive"] assert saved.mode == "oidc" assert saved.token.access is not None assert saved.token.access.get_secret_value() == "new-access-secret" @@ -231,7 +234,9 @@ async def test_valid_saved_oidc_access_token_is_reused( monkeypatch: pytest.MonkeyPatch, ) -> None: """A current saved OIDC Authentication Record needs no refresh.""" - _config(credential=oidc_credential("inactive", access="current-token")).save() + _config( + credential=oidc_credential("inactive", access="current-token") + ).editor.save() refresh = AsyncMock() monkeypatch.setattr("canfar.client.oidc.refresh", refresh) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) @@ -249,7 +254,7 @@ async def test_saved_x509_is_validated_before_construction( ) -> None: """Saved X.509 material becomes only an inspected literal certfile path.""" certificate = tmp_path / "saved.pem" - _config(credential=x509_credential("inactive", path=certificate)).save() + _config(credential=x509_credential("inactive", path=certificate)).editor.save() inspect = Mock() def inspect_certificate(path: Path) -> dict[str, object]: @@ -272,7 +277,7 @@ async def test_runtime_x509_overrides_saved_authentication_record( ) -> None: """A validated runtime certificate wins over the saved Authentication Record.""" certificate = tmp_path / "runtime.pem" - _config(credential=oidc_credential("inactive")).save() + _config(credential=oidc_credential("inactive")).editor.save() monkeypatch.setattr( "canfar.client.x509.inspect", lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, @@ -294,7 +299,7 @@ async def test_invalid_saved_x509_fails_before_vospace( ) -> None: """An invalid X.509 Authentication Record fails with a clean login hint.""" certificate = tmp_path / "invalid.pem" - _config(credential=x509_credential("inactive", path=certificate)).save() + _config(credential=x509_credential("inactive", path=certificate)).editor.save() monkeypatch.setattr( "canfar.client.x509.inspect", Mock(side_effect=ValueError("certificate parse detail")), @@ -317,7 +322,7 @@ async def test_source_closes_on_failure_and_cancellation( monkeypatch: pytest.MonkeyPatch, ) -> None: """Context exit always closes a yielded filesystem.""" - _config(credential=oidc_credential("inactive")).save() + _config(credential=oidc_credential("inactive")).editor.save() filesystems: list[_Filesystem] = [] def build(endpoint: str, **kwargs: Any) -> _Filesystem: @@ -359,7 +364,7 @@ async def test_unrefreshable_oidc_fails_secret_safe_before_vospace( access_expiry=1.0, refresh_expiry=1.0, ) - ).save() + ).editor.save() constructor = AsyncMock() monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) @@ -378,7 +383,7 @@ async def test_empty_saved_oidc_token_fails_cleanly( monkeypatch: pytest.MonkeyPatch, ) -> None: """An empty saved token cannot fall through to certificate construction.""" - _config(credential=oidc_credential("inactive", access="")).save() + _config(credential=oidc_credential("inactive", access="")).editor.save() constructor = Mock() monkeypatch.setattr(vosfs, "VOSpaceFileSystem", constructor) @@ -394,51 +399,69 @@ class TestPublicSurface: def test_identifiers_lists_configured_services_and_local(self) -> None: """Every configured Storage Identifier is listed, with local last.""" - _config(credential=x509_credential("inactive")).save() + _config(credential=x509_credential("inactive")).editor.save() assert storage.identifiers() == ["archive", "local"] + def test_identifiers_discovers_services_on_every_server(self) -> None: + """Discovery does not narrow the list to the active Server Selection.""" + config = _config(credential=x509_credential("inactive")) + config.servers["other"] = Server( + idp="inactive", + uri=AnyUrl("ivo://other.example/skaha"), + url=AnyHttpUrl("https://other.example/skaha"), + storage={ + "second": VOSpaceService( + uri=AnyUrl("ivo://other.example/second"), + url=AnyHttpUrl("https://other.example/second"), + ) + }, + ) + config.editor.save() + + assert storage.identifiers() == ["archive", "second", "local"] + def test_local_identifier_returns_a_local_filesystem(self) -> None: """The reserved local identifier needs no credential.""" assert isinstance(storage.filesystem("local"), LocalFileSystem) - assert isinstance(storage.local, LocalFileSystem) + assert fsspec.get_filesystem_class("file") is LocalFileSystem - def test_attribute_access_builds_a_filesystem( + def test_filesystem_builds_a_configured_identifier( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - """A Storage Identifier resolves as a module attribute.""" + """The explicit filesystem API resolves a configured identifier.""" certificate = tmp_path / "saved.pem" - _config(credential=x509_credential("inactive", path=certificate)).save() + _config(credential=x509_credential("inactive", path=certificate)).editor.save() monkeypatch.setattr( "canfar.client.x509.inspect", lambda path: {"path": path.as_posix(), "expiry": 9_999_999_999.0}, ) monkeypatch.setattr(vosfs, "VOSpaceFileSystem", _Filesystem) - filesystem = storage.archive + filesystem = storage.filesystem("archive") assert filesystem.endpoint == "https://inactive.example/vospace" assert filesystem.kwargs["certfile"] == certificate.as_posix() assert filesystem.kwargs["use_listings_cache"] is True - def test_unknown_identifier_raises_attribute_error(self) -> None: - """An unconfigured name is an AttributeError naming what is available.""" - _config(credential=x509_credential("inactive")).save() + def test_unknown_identifier_raises_key_error(self) -> None: + """An unconfigured Storage Identifier fails explicitly.""" + _config(credential=x509_credential("inactive")).editor.save() - with pytest.raises(AttributeError, match="archive"): - _ = storage.missing + with pytest.raises(KeyError, match="missing"): + storage.filesystem("missing") - def test_private_names_are_not_treated_as_identifiers(self) -> None: - """Dunder lookups must not attempt a filesystem build.""" - with pytest.raises(AttributeError): - _ = storage.__wrapped__ + def test_data_source_mapping_is_private(self) -> None: + """The fsspec-cli source mapping is not a public storage API.""" + _config(credential=x509_credential("inactive")).editor.save() - def test_dir_offers_identifiers_for_completion(self) -> None: - """Tab completion exposes identifiers alongside the module functions.""" - _config(credential=x509_credential("inactive")).save() + assert not hasattr(storage, "sources") + assert set(_sources()) == {"archive", "local"} - listed = dir(storage) + def test_storage_identifiers_are_not_dynamic_module_imports(self) -> None: + """Storage Identifiers must be passed to the explicit filesystem API.""" + _config(credential=x509_credential("inactive")).editor.save() - assert {"archive", "local", "filesystem", "identifiers"} <= set(listed) + assert "archive" not in storage.__dict__ diff --git a/tests/test_utils_logging.py b/tests/test_utils_logging.py index 87464b92..be53992d 100644 --- a/tests/test_utils_logging.py +++ b/tests/test_utils_logging.py @@ -5,6 +5,7 @@ import json import logging from contextlib import ExitStack +from logging.handlers import RotatingFileHandler from typing import TYPE_CHECKING from unittest.mock import Mock, patch @@ -29,16 +30,29 @@ from pathlib import Path +def _emit_exception_log(logger: logging.Logger) -> None: + """Emit one exception record for the JSONL formatter contract.""" + try: + json.loads("not-json") + except json.JSONDecodeError: + logger.exception("operation failed") + + +def _clear_handlers() -> None: + """Release handlers through the stdlib logger boundary.""" + logger = logging.getLogger(LOGGER_NAME) + for handler in logger.handlers[:]: + handler.close() + logger.removeHandler(handler) + + @pytest.fixture def canfar_logger() -> Generator[CanfarLogger]: """Fresh CanfarLogger cleaned after each test.""" logger = CanfarLogger() - # Shared stdlib logger may already have handlers from earlier suite tests. - logger._cleanup_handlers() # noqa: SLF001 - logger._configured = False # noqa: SLF001 + _clear_handlers() yield logger - logger._cleanup_handlers() # noqa: SLF001 - logger._configured = False # noqa: SLF001 + _clear_handlers() def test_configure_rich_stderr_defaults(canfar_logger: CanfarLogger) -> None: @@ -48,23 +62,47 @@ def test_configure_rich_stderr_defaults(canfar_logger: CanfarLogger) -> None: logger = canfar_logger.logger rich_handlers = [h for h in logger.handlers if isinstance(h, RichHandler)] assert logger.level == logging.INFO - assert rich_handlers - assert canfar_logger._rich_handler in rich_handlers # noqa: SLF001 + assert len(rich_handlers) == 1 assert not logger.propagate - assert canfar_logger._configured # noqa: SLF001 def test_reconfigure_replaces_handlers(canfar_logger: CanfarLogger) -> None: """Reconfiguration replaces previous handlers.""" canfar_logger.configure(loglevel=logging.INFO) - first = canfar_logger._rich_handler # noqa: SLF001 + first = next( + handler + for handler in canfar_logger.logger.handlers + if isinstance(handler, RichHandler) + ) canfar_logger.configure(loglevel=logging.DEBUG) - assert canfar_logger._rich_handler is not None # noqa: SLF001 - assert canfar_logger._rich_handler is not first # noqa: SLF001 - assert canfar_logger._rich_handler in canfar_logger.logger.handlers # noqa: SLF001 + second = next( + handler + for handler in canfar_logger.logger.handlers + if isinstance(handler, RichHandler) + ) + assert second is not first assert first not in canfar_logger.logger.handlers +def test_separate_logger_lifecycles_do_not_accumulate_handlers( + tmp_path: Path, +) -> None: + """Repeated application lifecycles leave one stderr/file sink pair.""" + first = CanfarLogger() + second = CanfarLogger() + try: + first.configure(loglevel=logging.INFO, log_file=tmp_path / "first.jsonl") + second.configure(loglevel=logging.INFO, log_file=tmp_path / "second.jsonl") + + logger = logging.getLogger(LOGGER_NAME) + assert len([h for h in logger.handlers if isinstance(h, RichHandler)]) == 1 + files = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)] + assert len(files) == 1 + assert files[0].baseFilename.endswith("second.jsonl") + finally: + _clear_handlers() + + @pytest.mark.parametrize( ("kwargs", "expected"), [ @@ -131,9 +169,23 @@ def test_jsonl_file_sink_writes_flat_events( assert event["message"] == "hello" assert set(event.keys()) == {"timestamp", "level", "logger", "message"} finally: - for handler in get_logger().handlers[:]: - handler.close() - get_logger().removeHandler(handler) + _clear_handlers() + + +def test_jsonl_file_sink_includes_exception_text(tmp_path: Path) -> None: + """Exception diagnostics remain available as one escaped JSONL field.""" + log_file = tmp_path / "exception.jsonl" + try: + configure_logging(loglevel="INFO", log_file=log_file) + _emit_exception_log(get_logger("jsonl")) + for handler in get_logger().handlers: + handler.flush() + + event = json.loads(log_file.read_text(encoding="utf-8")) + assert "exception" in event + assert "JSONDecodeError" in event["exception"] + finally: + _clear_handlers() def test_jsonl_rotates_with_small_max_size(tmp_path: Path) -> None: @@ -151,7 +203,7 @@ def test_jsonl_rotates_with_small_max_size(tmp_path: Path) -> None: assert log_file in files assert tmp_path / "rotating.jsonl.1" in files finally: - logger._cleanup_handlers() # noqa: SLF001 + _clear_handlers() @pytest.mark.parametrize("failure", ["write", "rollover"]) @@ -168,8 +220,13 @@ def test_file_sink_failure_keeps_stderr_and_warns_once( log_file=tmp_path / f"{failure}.jsonl", warning_writer=warning_writer, ) - handler = logger._file_handler # noqa: SLF001 - assert handler is not None + handlers = [ + candidate + for candidate in logger.logger.handlers + if isinstance(candidate, RotatingFileHandler) + ] + assert len(handlers) == 1 + handler = handlers[0] try: with ExitStack() as stack: if failure == "write": @@ -203,4 +260,4 @@ def test_file_sink_failure_keeps_stderr_and_warns_once( == ErrorCode.LOGGING_FILE_SINK_UNAVAILABLE.value ) finally: - logger._cleanup_handlers() # noqa: SLF001 + _clear_handlers() diff --git a/uv.lock b/uv.lock index ab873201..c6863a22 100644 --- a/uv.lock +++ b/uv.lock @@ -39,15 +39,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] -[[package]] -name = "asttokens" -version = "3.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/1e/faf0f247f6f881b98fc4d6d07e14085cb89d13665084e6d6ac1dc2c03d0b/asttokens-3.0.2.tar.gz", hash = "sha256:3ecdbd8f2cc195f53ccada3a613538bb5f9ef6f6869129f13e03c30a677b8fe2", size = 63136, upload-time = "2026-07-12T03:31:49.084Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/2b/04b8a15f3a1c77bc79ddf5c73875327f34b4fa75982df2b76e45e402d364/asttokens-3.0.2-py3-none-any.whl", hash = "sha256:9da13157f5b28becde0bd374fc677dcd3c290614264eff096f167c469cd9f933", size = 28702, upload-time = "2026-07-12T03:31:47.542Z" }, -] - [[package]] name = "authlib" version = "1.7.2" @@ -117,6 +108,7 @@ dependencies = [ { name = "authlib" }, { name = "cadcutils" }, { name = "click" }, + { name = "cryptography" }, { name = "defusedxml" }, { name = "fsspec-cli" }, { name = "httpx", extra = ["http2"] }, @@ -133,16 +125,12 @@ dependencies = [ [package.dev-dependencies] dev = [ - { name = "ipython", version = "8.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "ipython", version = "9.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, - { name = "pytest-order" }, { name = "pytest-xdist" }, { name = "ruff" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "ty" }, ] docs = [ @@ -159,6 +147,7 @@ requires-dist = [ { name = "authlib", specifier = ">=1.7.2" }, { name = "cadcutils", specifier = ">=1.5.4" }, { name = "click", specifier = ">=8.4.1" }, + { name = "cryptography", specifier = ">=42.0.0" }, { name = "defusedxml", specifier = ">=0.7.1" }, { name = "fsspec-cli", git = "https://github.com/shinybrar/vosfs?subdirectory=src%2Ffsspec-cli&rev=fsspec-cli-v0.7.0" }, { name = "httpx", extras = ["http2"], specifier = ">=0.28.1" }, @@ -175,15 +164,12 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ - { name = "ipython", specifier = ">=8.37.0" }, { name = "pre-commit", specifier = ">=4.2.0" }, { name = "pytest", specifier = ">=8.3.5" }, { name = "pytest-asyncio", specifier = ">=1.0.0" }, { name = "pytest-cov", specifier = ">=6.1.1" }, - { name = "pytest-order", specifier = ">=1.3.0" }, { name = "pytest-xdist", specifier = ">=3.7.0" }, { name = "ruff", specifier = ">=0.11.11" }, - { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.1" }, { name = "ty", specifier = ">=0.0.51" }, ] docs = [ @@ -591,15 +577,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] -[[package]] -name = "decorator" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, -] - [[package]] name = "defusedxml" version = "0.7.1" @@ -648,15 +625,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] -[[package]] -name = "executing" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, -] - [[package]] name = "filelock" version = "3.29.7" @@ -838,82 +806,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "ipython" -version = "8.39.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "exceptiongroup" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/56/4cc7fc9e9e3f38fd324f24f8afe0ad8bb5fa41283f37f1aaf9de0612c968/ipython-8.39.0-py3-none-any.whl", hash = "sha256:bb3c51c4fa8148ab1dea07a79584d1c854e234ea44aa1283bcb37bc75054651f", size = 831849, upload-time = "2026-03-27T10:02:07.846Z" }, -] - -[[package]] -name = "ipython" -version = "9.15.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version >= '3.11' and python_full_version < '3.14'", -] -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/3a/948263ca3b9d65bb2b1b0c521b3a49fad5d59ada58724bd87d2bd5ff3f36/ipython-9.15.0-py3-none-any.whl", hash = "sha256:515ad9c3cdf0c932a5a9f6245419e8aba706b7bd03c3e1d3a1c83d9351d6aa6e", size = 630895, upload-time = "2026-06-26T11:03:33.809Z" }, -] - -[[package]] -name = "ipython-pygments-lexers" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, -] - -[[package]] -name = "jedi" -version = "0.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "parso" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" }, -] - [[package]] name = "jinja2" version = "3.1.6" @@ -1162,18 +1054,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] -[[package]] -name = "matplotlib-inline" -version = "0.2.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "traitlets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, -] - [[package]] name = "mdurl" version = "0.1.2" @@ -1385,15 +1265,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746, upload-time = "2024-08-25T14:17:22.55Z" }, ] -[[package]] -name = "parso" -version = "0.8.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, -] - [[package]] name = "pathspec" version = "1.1.1" @@ -1403,18 +1274,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] -[[package]] -name = "pexpect" -version = "4.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "ptyprocess" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, -] - [[package]] name = "platformdirs" version = "4.10.0" @@ -1461,52 +1320,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, ] -[[package]] -name = "psutil" -version = "7.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, - { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, - { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, - { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, - { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, - { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, - { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, - { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, - { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, - { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, - { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, - { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, - { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, - { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, - { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, - { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, - { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, - { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, - { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, - { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, -] - -[[package]] -name = "ptyprocess" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, -] - -[[package]] -name = "pure-eval" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, -] - [[package]] name = "pycparser" version = "3.0" @@ -1751,18 +1564,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] -[[package]] -name = "pytest-order" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/97/d8/82a5dc2aad3392f66c9741960fcbeeb45dd4d62eedb218aed6c52eea36eb/pytest_order-1.5.0.tar.gz", hash = "sha256:96acd7587b5a2855dcaa4a898288103d202894a61afd813adbc9b77aab04d90d", size = 54136, upload-time = "2026-06-13T05:39:41.636Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/5c/bd6a85b44beb8bd74c0e86ab699ef0e096e260dc90458b84fc54c28e4335/pytest_order-1.5.0-py3-none-any.whl", hash = "sha256:667ba2b7303fe2c529848663e0a41dfb0cc4d64f09f87272e4a9a1751efb52b2", size = 16634, upload-time = "2026-06-13T05:39:40.431Z" }, -] - [[package]] name = "pytest-xdist" version = "3.8.0" @@ -1996,20 +1797,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, ] -[[package]] -name = "stack-data" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asttokens" }, - { name = "executing" }, - { name = "pure-eval" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, -] - [[package]] name = "termynal" version = "0.14.0" @@ -2076,15 +1863,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] -[[package]] -name = "traitlets" -version = "5.15.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/57/a9/a2584b8313b89f94869ddb3c4074617a691de1812a614d2d50e32ca5a7a6/traitlets-5.15.1.tar.gz", hash = "sha256:7b1c07854fe25acb39e009bae49f11b79ff6cbb2f27999104e9110e7a6b53722", size = 163344, upload-time = "2026-06-03T12:26:06.181Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/8d/1080ee4c231f361b6ce4470d556c8c435b67c7e0753aaa641497ee92f88b/traitlets-5.15.1-py3-none-any.whl", hash = "sha256:770a53705f84b81ac107e83a1b3328ff2dae16094d8fc3cfc004e4b22dfd8e92", size = 85858, upload-time = "2026-06-03T12:26:04.395Z" }, -] - [[package]] name = "ty" version = "0.0.64"