diff --git a/.github/README.md b/.github/README.md index b8356d2..da06762 100644 --- a/.github/README.md +++ b/.github/README.md @@ -10,10 +10,11 @@ This repository holds the PKI Maturity Model maintained by the PKI Consortium PK ## For contributors -The authoritative content lives in two places: +The authoritative machine-readable content uses three versioned contracts: - `data/pkimm-model-.yaml` — the machine-readable model (modules, categories, requirements, levels). This is the source of truth for downstream tooling. - `data/pkimm-references.yaml` — the shared catalog of standards, regulations, and publications cited from the model. Independently versioned so reference-metadata updates do not require a model release. +- `data/pkimm-self-assessment-profile-.yaml` — versioned, declarative runtime configuration for the web assessment tool. It references a released model and defines the weighted scoring strategy, subject fields, assurance boundary, and report/signing policy without adding PKIMM-specific application code. The per-category markdown under `categories/` and the references summary at `model/references/_index.md` are **generated** from the YAML files. Edit the YAML, run the generator, commit both. @@ -42,12 +43,13 @@ pip install -r scripts/requirements-dev.txt 4. Run the consistency validator until it passes: ```sh python scripts/check_model_docs_consistency.py --repo-root . + python scripts/validate_assessment_profile.py ``` 5. Commit. The same checks run on every PR via [`.github/workflows/check-consistency.yml`](workflows/check-consistency.yml). ### Repository layout -- `data/` — versioned model YAML files and their JSON schemas, plus the references catalog and its schema. +- `data/` — versioned model and assessment-profile YAML files and their JSON schemas, plus the references catalog and its schema. - `categories/` — generated per-category markdown (one folder per category, named by stable kebab-case id). - `model/` — narrative pages: vocabulary, modules, categories overview, references summary. - `release-notes/` — per-version release notes. Each version subdirectory uses the template at `release-notes/templates/_index.md`. @@ -57,6 +59,16 @@ pip install -r scripts/requirements-dev.txt - Excel-based assessment tools have been retired in favor of the web self-assessment; the last Excel tools remain available under the `1.0.0` tag / website section. - `scripts/` — authoring scripts (generator, validator) and their tests. +### Assessment-tool compatibility + +The model and assessment profile are intentionally separate. The released model remains the normative source for modules, categories, levels, requirements, and weights. The profile selects the generic `weighted-maturity` experience and `weighted-average` strategy and provides tool and report policy. The web tool can therefore load PKIMM through the same model/profile interfaces used by other assessments, with no PKIMM-specific scoring branch. + +The browser profile exposes only self-assessment. Qualified third-party assessment and PKI Consortium certification remain external workflow states and cannot be self-selected in the browser. The profile declares optional executive and security-executive PAdES fields, permits additional signatures, and leaves actual signer identity and authority to the signing workflow. + +### Releases + +Semantic model tags use the existing `` convention, for example `2.0.0`. The release workflow validates the model, generated pages, and assessment profile before publishing individual model/profile/schema assets, checksums, and complete `.tar.gz` and `.zip` source packages. Downstream builds should use a pinned GitHub release asset rather than an unversioned file from `main`. + ### Conventions - **Stable identifiers**: category and requirement `id` fields are kebab-case strings (`strategy-and-vision`, `sponsor-support`), not positional numbers. Ordering is expressed via the YAML array position and Hugo's `weight:` front-matter field. diff --git a/.github/workflows/check-consistency.yml b/.github/workflows/check-consistency.yml index 6429964..b5b193d 100644 --- a/.github/workflows/check-consistency.yml +++ b/.github/workflows/check-consistency.yml @@ -1,18 +1,47 @@ name: Check model/docs consistency on: + workflow_call: pull_request: push: branches: [main] +permissions: + contents: read + +concurrency: + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" + cancel-in-progress: true + jobs: consistency: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 #v6.2.0 + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - - run: pip install -r scripts/requirements-dev.txt - - run: pytest scripts/ -q - - run: python scripts/check_model_docs_consistency.py --repo-root . + + - name: Install validation dependencies + run: pip install -r scripts/requirements-dev.txt + + - name: Run positive and negative tests + run: pytest scripts/ -q + + - name: Validate data schemas + run: | + check-jsonschema --schemafile data/pkimm-model.schema-1.0.0.json data/pkimm-model-1.0.0.yaml + check-jsonschema --schemafile data/pkimm-model.schema-2.0.0.json data/pkimm-model-2.0.0.yaml + check-jsonschema --schemafile data/pkimm-references.schema-1.0.0.json data/pkimm-references.yaml + + - name: Validate model and generated documentation + run: python scripts/check_model_docs_consistency.py --repo-root . + + - name: Validate assessment profile compatibility + run: python scripts/validate_assessment_profile.py + + - name: Check whitespace errors + run: git diff --check diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..bcf87ac --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,81 @@ +name: Publish model release + +on: + push: + tags: ["[0-9]*.[0-9]*.[0-9]*"] + +permissions: + contents: write + +jobs: + validate: + uses: ./.github/workflows/check-consistency.yml + + release: + needs: validate + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Build release package + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + test -f "data/pkimm-model-${RELEASE_TAG}.yaml" + test -f "data/pkimm-model.schema-${RELEASE_TAG}.json" + PACKAGE_DIR="dist/pkimm-${RELEASE_TAG}" + mkdir -p "${PACKAGE_DIR}" + cp -R assessment categories data extensions model release-notes "${PACKAGE_DIR}/" + cp _index.md LICENSE "${PACKAGE_DIR}/" + cp .github/README.md "${PACKAGE_DIR}/README.md" + cp "data/pkimm-model-${RELEASE_TAG}.yaml" dist/ + cp "data/pkimm-model.schema-${RELEASE_TAG}.json" dist/ + cp data/pkimm-references.yaml dist/ + cp data/pkimm-references.schema-1.0.0.json dist/ + cp data/pkimm-self-assessment-profile-1.0.0.yaml dist/ + cp data/assessment-profile.schema-1.0.0.json dist/ + SOURCE_DATE_EPOCH="$(git show -s --format=%ct)" + find "${PACKAGE_DIR}" -exec touch -d "@${SOURCE_DATE_EPOCH}" {} + + tar \ + --sort=name \ + --mtime="@${SOURCE_DATE_EPOCH}" \ + --owner=0 \ + --group=0 \ + --numeric-owner \ + -C dist \ + -cf - \ + "pkimm-${RELEASE_TAG}" \ + | gzip -n > "dist/pkimm-${RELEASE_TAG}.tar.gz" + cd dist + find "pkimm-${RELEASE_TAG}" -type f -print \ + | LC_ALL=C sort \ + | zip -Xq "pkimm-${RELEASE_TAG}.zip" -@ + sha256sum \ + assessment-profile.schema-1.0.0.json \ + "pkimm-model-${RELEASE_TAG}.yaml" \ + "pkimm-model.schema-${RELEASE_TAG}.json" \ + pkimm-references.schema-1.0.0.json \ + pkimm-references.yaml \ + pkimm-self-assessment-profile-1.0.0.yaml \ + "pkimm-${RELEASE_TAG}.tar.gz" \ + "pkimm-${RELEASE_TAG}.zip" > SHA256SUMS + + - name: Create GitHub release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + run: >- + gh release create "${RELEASE_TAG}" + dist/assessment-profile.schema-1.0.0.json + "dist/pkimm-model-${RELEASE_TAG}.yaml" + "dist/pkimm-model.schema-${RELEASE_TAG}.json" + dist/pkimm-references.schema-1.0.0.json + dist/pkimm-references.yaml + dist/pkimm-self-assessment-profile-1.0.0.yaml + "dist/pkimm-${RELEASE_TAG}.tar.gz" + "dist/pkimm-${RELEASE_TAG}.zip" + dist/SHA256SUMS + --verify-tag + --generate-notes + --title "PKI Maturity Model ${RELEASE_TAG}" diff --git a/CLAUDE.md b/CLAUDE.md index 668f99e..b54be1a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this repository is -Content repository for the **PKI Maturity Model (PKIMM)**, maintained by the PKI Consortium PKIMM working group. The content is rendered at https://pkic.org/pkimm by an external Hugo-based site — this repo holds the source markdown, the canonical model data, and the assessment methodology pages. There is **no build system, no test suite, and no application code** here. Treat changes as documentation/data edits, not software changes. +Content repository for the **PKI Maturity Model (PKIMM)**, maintained by the PKI Consortium PKIMM working group. The content is rendered at https://pkic.org/pkimm by an external Hugo-based site — this repo holds the source markdown, canonical model data, assessment runtime profile, and assessment methodology pages. There is no runtime application code here; Python scripts and tests validate authored data and generated documentation. ## Repository layout @@ -19,6 +19,8 @@ Content repository for the **PKI Maturity Model (PKIMM)**, maintained by the PKI - `data/pkimm-model.schema-1.0.0.json` — JSON Schema for the 1.0.0 YAML shape (retroactively renamed from `pkimm-model.schema.json`). - `data/pkimm-references.yaml` — **independently-versioned references catalog**. Per-requirement `references` fields in the 2.0.0 model contain arrays of IDs from this catalog. Edit here to update reference metadata without touching the model YAML. - `data/pkimm-references.schema-1.0.0.json` — JSON Schema for the references catalog. +- `data/pkimm-self-assessment-profile-1.0.0.yaml` — versioned generic assessment runtime configuration for PKIMM 2.0.0. It selects weighted-maturity behavior, subject fields, assurance states, and report/signing policy without introducing PKIMM-specific application logic. +- `data/assessment-profile.schema-1.0.0.json` — shared JSON Schema for assessment runtime profiles. - `extensions/` — extension framework: schema (`extension.schema-1.0.0.json`), structure and scoring documentation. The extension framework defines the non-destructive, composable overlay/relevance model (schema/structure/scoring); the catalog of published extension YAML definitions now lives in the separate `pkimm-extensions` repository, rendered at https://pkic.org/wg/pkimm/extensions/. - `scripts/` — authoring and validation scripts (see "Authoring workflow" below). - Integration converters (e.g., the Eramba CSV package generators) now live in the separate `pkimm-integrations` repository, rendered at https://pkic.org/wg/pkimm/integrations/. @@ -43,9 +45,10 @@ The markdown category pages and `data/pkimm-model-2.0.0.yaml` describe the same 2. **Regenerate markdown**: run `python scripts/generate_category_docs.py` to regenerate all category pages under `categories/` from the updated YAML. 3. **Update narrative docs**: manually update `model/` pages, `_index.md` mindmap, and `release-notes/` notes if the change is consumer-facing. 4. **Validate**: run `python scripts/check_model_docs_consistency.py --repo-root .` — must exit `0 error(s), 0 warning(s)` before committing. -5. **Tag**: when releasing, bump `version` in `data/pkimm-model-2.0.0.yaml`, copy the file to `data/pkimm-model-.yaml`, add a schema file for the new shape, and author release notes under `release-notes//`. +5. **Validate the assessment profile**: run `python scripts/validate_assessment_profile.py` and confirm that the profile still references the model version and a supported generic methodology. +6. **Tag**: when releasing, bump `version` in `data/pkimm-model-2.0.0.yaml`, copy the file to `data/pkimm-model-.yaml`, add a schema file for the new shape, and author release notes under `release-notes//`. -CI runs the validator on every PR and push to main (`.github/workflows/check-consistency.yml`). +CI runs the tests and both validators on every PR and push to main (`.github/workflows/check-consistency.yml`). Model-version tags publish pinned release assets through `.github/workflows/release.yml`. **Key scripts in `scripts/`:** - `generate_category_docs.py` — regenerates `categories/` markdown from YAML. diff --git a/data/assessment-profile.schema-1.0.0.json b/data/assessment-profile.schema-1.0.0.json new file mode 100644 index 0000000..7703bc5 --- /dev/null +++ b/data/assessment-profile.schema-1.0.0.json @@ -0,0 +1,339 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://pkic.org/assessment-profile.schema-1.0.0.json", + "title": "Assessment runtime profile", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "profile", + "runtime", + "assurance", + "report", + "futureServices" + ], + "properties": { + "schemaVersion": { "const": "1.0.0" }, + "profile": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version", "family", "model", "title", "description"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "version": { "$ref": "#/$defs/version" }, + "family": { + "enum": ["maturity", "knowledge-diagnostic", "certification-exam"] + }, + "model": { + "type": "object", + "additionalProperties": false, + "required": ["id", "version"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "version": { "$ref": "#/$defs/version" } + } + }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 } + } + }, + "runtime": { + "type": "object", + "additionalProperties": false, + "required": ["experience", "methodology", "subjectFields"], + "properties": { + "experience": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "methodology": { + "type": "object", + "additionalProperties": false, + "required": ["strategy", "version", "parameters"], + "properties": { + "strategy": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "version": { "$ref": "#/$defs/version" }, + "parameters": { "type": "object" } + } + }, + "subjectFields": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "items": { "$ref": "#/$defs/subjectField" } + }, + "subjectRules": { + "type": "array", + "maxItems": 64, + "items": { "$ref": "#/$defs/subjectRule" } + }, + "criterion": { + "type": "object", + "additionalProperties": false, + "required": ["statuses", "evidence"], + "properties": { + "statuses": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "items": { "$ref": "#/$defs/status" } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "requiredForStatuses", + "requiredFromLevel", + "statementLabel", + "statementHint", + "maxFileBytes", + "maxPackageBytes", + "acceptedMediaTypes", + "validators" + ], + "properties": { + "requiredForStatuses": { + "type": "array", + "maxItems": 64, + "uniqueItems": true, + "items": { "type": "string" } + }, + "requiredFromLevel": { "type": "integer", "minimum": 0 }, + "statementLabel": { "type": "string", "minLength": 1 }, + "statementHint": { "type": "string", "minLength": 1 }, + "maxFileBytes": { + "type": "integer", + "minimum": 1, + "maximum": 26214400 + }, + "maxPackageBytes": { + "type": "integer", + "minimum": 1, + "maximum": 104857600 + }, + "acceptedMediaTypes": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "uniqueItems": true, + "items": { "type": "string" } + }, + "validators": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[a-z0-9-]+$" } + } + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "experience": { "const": "evidence-gated-maturity" } + } + }, + "then": { "required": ["criterion"] } + } + ] + }, + "assurance": { + "type": "object", + "additionalProperties": false, + "required": ["defaultProfile", "profiles"], + "properties": { + "defaultProfile": { "type": "string" }, + "profiles": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "items": { "$ref": "#/$defs/assuranceProfile" } + } + } + }, + "report": { + "type": "object", + "additionalProperties": false, + "required": [ + "title", + "claimLabel", + "machineAttachmentName", + "includeEvidenceAttachments", + "includeEvidenceManifest", + "includeAttestation" + ], + "properties": { + "title": { "type": "string", "minLength": 1 }, + "claimLabel": { "type": "string", "minLength": 1 }, + "machineAttachmentName": { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]+$" + }, + "includeEvidenceAttachments": { "type": "boolean" }, + "includeEvidenceManifest": { "type": "boolean" }, + "includeAttestation": { "type": "boolean" }, + "signing": { "$ref": "#/$defs/reportSigningPolicy" } + } + }, + "futureServices": { + "type": "object", + "additionalProperties": false, + "required": ["pkicSubmission"], + "properties": { + "pkicSubmission": { + "type": "object", + "additionalProperties": false, + "required": ["implemented", "processingMayRequirePayment", "covers"], + "properties": { + "implemented": { "const": false }, + "processingMayRequirePayment": { "type": "boolean" }, + "covers": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + } + } + } + } + } + }, + "$defs": { + "version": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" }, + "subjectField": { + "type": "object", + "additionalProperties": false, + "required": ["key", "label", "component", "required"], + "properties": { + "key": { "type": "string", "pattern": "^[a-z][a-zA-Z0-9]*$" }, + "label": { "type": "string", "minLength": 1 }, + "component": { "enum": ["text", "textarea", "date"] }, + "required": { "type": "boolean" }, + "rows": { "type": "integer", "minimum": 1 }, + "hint": { "type": "string" }, + "format": { "enum": ["cpe-2.3", "package-url", "uri"] } + } + }, + "subjectRule": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "fields", "message"], + "properties": { + "kind": { "const": "at-least-one" }, + "fields": { + "type": "array", + "minItems": 2, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[a-z][a-zA-Z0-9]*$" } + }, + "message": { "type": "string", "minLength": 1 } + } + }, + "status": { + "type": "object", + "additionalProperties": false, + "required": ["value", "label"], + "properties": { + "value": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "label": { "type": "string", "minLength": 1 } + } + }, + "assuranceProfile": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "label", + "reportLabel", + "availability", + "independentVerification", + "certification", + "requiredFacets", + "notice" + ], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "label": { "type": "string", "minLength": 1 }, + "reportLabel": { "type": "string", "minLength": 1 }, + "availability": { "enum": ["browser", "external-workflow"] }, + "independentVerification": { "type": "boolean" }, + "certification": { "type": "boolean" }, + "requiredFacets": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "identity", + "organization-binding", + "role-authority", + "document-signature", + "trusted-time", + "evidence-review", + "assessor-qualification", + "certification-issuance" + ] + } + }, + "notice": { "type": "string", "minLength": 1 }, + "attestation": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "signerRole", "method", "declaration"], + "properties": { + "enabled": { "type": "boolean" }, + "signerRole": { "type": "string", "minLength": 1 }, + "method": { + "enum": [ + "electronic-acknowledgement", + "external-digital-signature" + ] + }, + "declaration": { "type": "string", "minLength": 1 } + } + } + } + }, + "reportSigningPolicy": { + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "fields", + "allowAdditionalSignatures", + "targetLevel", + "trustedTime", + "acceptedAuthorityEvidence" + ], + "properties": { + "format": { "const": "PAdES" }, + "fields": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/reportSignatureFieldPolicy" } + }, + "allowAdditionalSignatures": { "type": "boolean" }, + "targetLevel": { "enum": ["B-T", "B-LT", "B-LTA"] }, + "trustedTime": { "const": "signature-or-document-timestamp" }, + "acceptedAuthorityEvidence": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^[a-z0-9-]+$" } + } + } + }, + "reportSignatureFieldPolicy": { + "type": "object", + "additionalProperties": false, + "required": ["name", "label", "role", "required"], + "properties": { + "name": { + "type": "string", + "pattern": "^[a-zA-Z][a-zA-Z0-9._-]+$" + }, + "label": { "type": "string", "minLength": 1 }, + "role": { "type": "string", "minLength": 1 }, + "required": { "type": "boolean" } + } + } + } +} diff --git a/data/pkimm-self-assessment-profile-1.0.0.yaml b/data/pkimm-self-assessment-profile-1.0.0.yaml new file mode 100644 index 0000000..20c49e1 --- /dev/null +++ b/data/pkimm-self-assessment-profile-1.0.0.yaml @@ -0,0 +1,121 @@ +schemaVersion: 1.0.0 +profile: + id: pkimm-self-assessment + version: 1.0.0 + family: maturity + model: + id: pkimm + version: 2.0.0 + title: PKIMM Assessment + description: Organization PKI maturity assessment. Data remains in this browser unless the user exports it. +runtime: + experience: weighted-maturity + methodology: + strategy: weighted-average + version: 1.0.0 + parameters: + minimumLevel: 0 + maximumLevel: 5 + rounding: floor + categoryWeightField: weight + requirementWeightField: weight + excludeNotApplicable: true + subjectFields: + - { + key: assessmentName, + label: Assessment name, + component: text, + required: false, + } + - { + key: organizationName, + label: Organization, + component: text, + required: false, + } + - { key: assessorName, label: Assessor, component: text, required: false } + - key: useCaseDescription + label: Description of the use case + component: textarea + rows: 4 + required: false +assurance: + defaultProfile: self + profiles: + - id: self + label: Self-assessment + reportLabel: Self-assessment — not independently verified — not PKI Consortium certified + availability: browser + independentVerification: false + certification: false + requiredFacets: [] + notice: The maturity result is calculated from ratings supplied by the organization and has not been independently verified. + - id: qualified-third-party + label: Qualified third-party assessment + reportLabel: Qualified third-party assessment + availability: external-workflow + independentVerification: true + certification: false + requiredFacets: + [ + identity, + organization-binding, + role-authority, + document-signature, + trusted-time, + evidence-review, + assessor-qualification, + ] + notice: Qualification and assessor signature must be verified by the PKI Consortium service before this assurance label can be issued. + - id: pkic-certified + label: PKI Consortium certified + reportLabel: PKI Consortium certified assessment + availability: external-workflow + independentVerification: true + certification: true + requiredFacets: + [ + identity, + organization-binding, + role-authority, + document-signature, + trusted-time, + evidence-review, + assessor-qualification, + certification-issuance, + ] + notice: This status can only be issued by the future PKI Consortium review, signing, publication, and status service. +report: + title: PKIMM Self-Assessment Report + claimLabel: Claimed maturity level + machineAttachmentName: pkimm-assessment-credential.json + includeEvidenceAttachments: true + includeEvidenceManifest: true + includeAttestation: true + signing: + format: PAdES + fields: + - name: ExecutiveApproval + label: Executive approval signature + role: CEO or accountable executive + required: false + - name: SecurityExecutiveApproval + label: Security executive approval signature + role: CISO or security executive + required: false + allowAdditionalSignatures: true + targetLevel: B-LTA + trustedTime: signature-or-document-timestamp + acceptedAuthorityEvidence: + [vlei-oor, vlei-ecr, eudi-wallet-credential, manually-verified-authority] +futureServices: + pkicSubmission: + implemented: false + processingMayRequirePayment: true + covers: + - intake and validation + - qualified assessor verification + - human and AI-assisted evidence review + - formal digital signatures + - publication and status registry + - long-term report and evidence storage diff --git a/scripts/test_validate_assessment_profile.py b/scripts/test_validate_assessment_profile.py new file mode 100644 index 0000000..efd4aea --- /dev/null +++ b/scripts/test_validate_assessment_profile.py @@ -0,0 +1,77 @@ +"""Positive and negative tests for the PKIMM assessment profile contract.""" +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest +import yaml + +from scripts.validate_assessment_profile import ( + MODEL_PATH, + PROFILE_PATH, + PROFILE_SCHEMA_PATH, + validate_profile_data, +) + + +ROOT = Path(__file__).resolve().parent.parent + + +def _fixtures() -> tuple[dict, dict, dict]: + profile = yaml.safe_load((ROOT / PROFILE_PATH).read_text()) + model = yaml.safe_load((ROOT / MODEL_PATH).read_text()) + schema = json.loads((ROOT / PROFILE_SCHEMA_PATH).read_text()) + return profile, model, schema + + +def test_current_profile_matches_schema_model_and_runtime() -> None: + profile, model, schema = _fixtures() + validate_profile_data(profile, model, schema) + + +def test_rejects_profile_for_another_model_version() -> None: + profile, model, schema = _fixtures() + profile = copy.deepcopy(profile) + profile["profile"]["model"]["version"] = "1.0.0" + + with pytest.raises(ValueError, match="packaged PKIMM model version"): + validate_profile_data(profile, model, schema) + + +def test_rejects_model_specific_or_unsupported_scoring() -> None: + profile, model, schema = _fixtures() + profile = copy.deepcopy(profile) + profile["runtime"]["methodology"]["strategy"] = "pkimm-special-case" + + with pytest.raises(ValueError, match="generic weighted-maturity"): + validate_profile_data(profile, model, schema) + + +def test_rejects_negative_model_weights() -> None: + profile, model, schema = _fixtures() + model = copy.deepcopy(model) + model["modules"][0]["categories"][0]["requirements"][0]["weight"] = -1 + + with pytest.raises(ValueError, match="requirement weights cannot be negative"): + validate_profile_data(profile, model, schema) + + +def test_rejects_browser_claim_of_independent_verification() -> None: + profile, model, schema = _fixtures() + profile = copy.deepcopy(profile) + profile["assurance"]["profiles"][0]["independentVerification"] = True + + with pytest.raises(ValueError, match="cannot claim verification"): + validate_profile_data(profile, model, schema) + + +def test_rejects_duplicate_pdf_signature_fields() -> None: + profile, model, schema = _fixtures() + profile = copy.deepcopy(profile) + fields = profile["report"]["signing"]["fields"] + fields[1]["name"] = fields[0]["name"] + + with pytest.raises(ValueError, match="PDF signature field names must be unique"): + validate_profile_data(profile, model, schema) diff --git a/scripts/validate_assessment_profile.py b/scripts/validate_assessment_profile.py new file mode 100644 index 0000000..50db859 --- /dev/null +++ b/scripts/validate_assessment_profile.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Validate the PKIMM assessment profile and its runtime compatibility.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import yaml +from jsonschema import Draft202012Validator, FormatChecker + + +PROFILE_PATH = "data/pkimm-self-assessment-profile-1.0.0.yaml" +PROFILE_SCHEMA_PATH = "data/assessment-profile.schema-1.0.0.json" +MODEL_PATH = "data/pkimm-model-2.0.0.yaml" + + +def _assert_unique(values: list[str], label: str) -> None: + if len(values) != len(set(values)): + raise ValueError(f"{label} must be unique") + + +def _validate_model_weights(model: dict[str, Any]) -> None: + for module in model["modules"]: + for category in module["categories"]: + if category["weight"] < 0: + raise ValueError("PKIMM category weights cannot be negative") + for requirement in category.get("requirements", []): + if requirement["weight"] < 0: + raise ValueError("PKIMM requirement weights cannot be negative") + + +def validate_profile_data( + profile: dict[str, Any], + model: dict[str, Any], + schema: dict[str, Any], +) -> None: + """Validate schema conformance and model/runtime compatibility.""" + validator = Draft202012Validator(schema, format_checker=FormatChecker()) + errors = sorted(validator.iter_errors(profile), key=lambda error: list(error.path)) + if errors: + details = "; ".join(error.message for error in errors) + raise ValueError(f"Assessment profile schema validation failed: {details}") + + model_reference = profile["profile"]["model"] + if model_reference != {"id": "pkimm", "version": model["version"]}: + raise ValueError( + "Assessment profile must reference the packaged PKIMM model version" + ) + _validate_model_weights(model) + + runtime = profile["runtime"] + methodology = runtime["methodology"] + parameters = methodology["parameters"] + supported_parameters = ( + parameters.get("minimumLevel") == 0 + and parameters.get("maximumLevel") == 5 + and parameters.get("rounding") in {"floor", "round", "ceil"} + and parameters.get("categoryWeightField") == "weight" + and parameters.get("requirementWeightField") == "weight" + and parameters.get("excludeNotApplicable") is True + ) + if ( + runtime["experience"] != "weighted-maturity" + or methodology["strategy"] != "weighted-average" + or not supported_parameters + ): + raise ValueError( + "PKIMM requires the supported generic weighted-maturity methodology" + ) + + field_keys = [field["key"] for field in runtime["subjectFields"]] + _assert_unique(field_keys, "Assessment subject field keys") + known_fields = set(field_keys) + for rule in runtime.get("subjectRules", []): + unknown_fields = set(rule["fields"]) - known_fields + if unknown_fields: + raise ValueError( + "Assessment subject rule references unknown fields: " + + ", ".join(sorted(unknown_fields)) + ) + + assurance = profile["assurance"] + assurance_ids = [item["id"] for item in assurance["profiles"]] + _assert_unique(assurance_ids, "Assurance profile ids") + default_assurance = next( + ( + item + for item in assurance["profiles"] + if item["id"] == assurance["defaultProfile"] + ), + None, + ) + if not default_assurance or default_assurance["availability"] != "browser": + raise ValueError("Default assurance profile must be available in the browser") + for item in assurance["profiles"]: + if item["availability"] == "browser" and ( + item["independentVerification"] or item["certification"] + ): + raise ValueError( + "Browser assurance profiles cannot claim verification or certification" + ) + + report = profile["report"] + signing = report.get("signing") + if report["includeAttestation"] and not signing: + raise ValueError("Attestation reports require a signing policy") + if signing: + _assert_unique( + [field["name"] for field in signing["fields"]], + "PDF signature field names", + ) + + +def validate_repository(repo_root: Path) -> tuple[dict[str, Any], dict[str, Any]]: + """Load and validate the current profile, schema, and model files.""" + profile = yaml.safe_load((repo_root / PROFILE_PATH).read_text()) + model = yaml.safe_load((repo_root / MODEL_PATH).read_text()) + schema = json.loads((repo_root / PROFILE_SCHEMA_PATH).read_text()) + validate_profile_data(profile, model, schema) + return profile, model + + +def main() -> int: + repo_root = Path(__file__).resolve().parent.parent + profile, model = validate_repository(repo_root) + print( + "Validated PKIMM assessment profile " + f"{profile['profile']['version']} for model {model['version']}." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())