Messagebox feature + letter transfer

.github/workflows/ghcr-tag-cleanup.yml

@@ -0,0 +1,69 @@
+name: GHCR tag cleanup
+
+on:
+  schedule:
+    - cron: '23 2 * * *'   # 02:23 UTC daily
+  workflow_dispatch: {}
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  PACKAGE_OWNER: worth-nl
+  PACKAGE_NAME: notifynl-api
+  KEEP_PR_TAGS: 10
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Round up all image tags (daily report)
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+
+          if ! gh api --paginate \
+               "/orgs/${PACKAGE_OWNER}/packages/container/${PACKAGE_NAME}/versions?per_page=100" \
+               > /tmp/pages.jsonl 2> /tmp/gh-api.err; then
+            echo "::warning::Could not list package versions for ${PACKAGE_OWNER}/${PACKAGE_NAME}"
+            cat /tmp/gh-api.err
+            echo "## GHCR tag report — ${PACKAGE_NAME}" >> "$GITHUB_STEP_SUMMARY"
+            echo "Could not list package versions (see job log)." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+          jq -s 'add // []' /tmp/pages.jsonl > /tmp/versions.json
+
+          {
+            echo "## GHCR tag report — ${PACKAGE_NAME}"
+            echo
+            echo "_Run: $(date -u +'%Y-%m-%d %H:%M UTC') — $(jq 'length' /tmp/versions.json) total versions in \`ghcr.io/${PACKAGE_OWNER}/${PACKAGE_NAME}\`_"
+            echo
+            echo "| Tag(s) | Created (UTC) |"
+            echo "|---|---|"
+            jq -r '
+              sort_by(.created_at) | reverse | .[]
+              | [ ((.metadata.container.tags // []) | if length == 0 then "_(untagged)_" else join(", ") end), .created_at ]
+              | "| " + .[0] + " | " + .[1] + " |"
+            ' /tmp/versions.json
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Delete stale -pr tags (keep newest 10)
+        uses: snok/container-retention-policy@v3.1.0
+        with:
+          account: ${{ env.PACKAGE_OWNER }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          image-names: ${{ env.PACKAGE_NAME }}
+          image-tags: "*-pr"
+          tag-selection: tagged
+          cut-off: 1s
+          keep-n-most-recent: ${{ env.KEEP_PR_TAGS }}
+          dry-run: false

.github/workflows/pr.yml

@@ -209,70 +209,157 @@ jobs:
       contents: read
       pull-requests: write
 
+    env:
+      HELM_SET_ARGS: >-
+        --set apps.api.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerRetryTasks.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerLetters.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerJobs.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerJobsSave.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerJobsSaveDocuments.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerResearch.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerSender.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerSenderLetters.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerPeriodic.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerReporting.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerInternal.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerReceipts.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerServiceCallbacks.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerServiceCallbacksRetry.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerMessagebox.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.apiWorkerReportRequestsNotificationsTasks.image.tag=${{ needs.build.outputs.tag }}
+        --set apps.celeryBeat.image.tag=${{ needs.build.outputs.tag }}
+        --set migrations.apiUpgrade.image.tag=${{ needs.build.outputs.tag }}
+
     steps:
+      # Order matters: actions/checkout defaults to `clean: true`, which runs
+      # `git clean -ffdx` in its checkout path. A checkout with no `path:`
+      # (this one) targets the workspace root, and that clean can wipe a
+      # sibling directory checked out earlier by a separate actions/checkout
+      # step (https://github.com/orgs/community/discussions/27030) — so the
+      # root-targeting checkout must run first, before notifynl-api exists.
       - uses: actions/checkout@v4
         with:
           repository: Worth-NL/notifynl-charts-private
           ref: main
           token: ${{ secrets.RELEASE_TOKEN }}
         name: Checkout Worth-NL/notifynl-charts-private
 
-      - uses: bwvolleyball/k8s-toolkit@v1.0.0
+      - uses: actions/checkout@v4
+        with:
+          path: notifynl-api
+        name: Checkout notifynl-api (this PR)
+
+      # K8S_CONFIG is stored base64-encoded (shared with the other submodules'
+      # workflows, which still use bwvolleyball/k8s-toolkit and decode it
+      # internally). azure/k8s-set-context's `kubeconfig` method expects raw
+      # kubeconfig YAML, not base64, so decode it here first.
+      - name: Decode kubeconfig
+        id: decode-kubeconfig
+        run: |
+          {
+            echo "content<<KUBECONFIG_EOF"
+            echo "${{ secrets.K8S_CONFIG }}" | base64 -d
+            echo "KUBECONFIG_EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - uses: azure/k8s-set-context@v4
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ steps.decode-kubeconfig.outputs.content }}
+
+      - uses: azure/setup-helm@v4
         with:
-          config: ${{ secrets.K8S_CONFIG }}
+          version: v4.2.2
 
       - name: Helm chart dependency build
         run: helm dependency build
         working-directory: ${{ github.workspace }}/notifynl-full
 
+      # notifynl-full pins migrations.apiUpgrade.{alphagovRevision,nlRevision}
+      # explicitly per release rather than implicitly running whatever HEAD
+      # the image bundles. A PR build's image can bundle newer migrations
+      # than the chart's currently pinned defaults, so derive the two heads
+      # from this PR's own migration trees (mirrors tests/test_migrations.py's
+      # "sorted glob, last file wins" head detection) and override them below.
+      - name: Determine migration revisions
+        id: migration-revisions
+        run: |
+          set -euo pipefail
+          alphagov_revision=$(basename "$(ls notifynl-api/migrations/versions/*.py | sort | tail -1)" .py)
+          nl_revision=$(basename "$(ls notifynl-api/migrations_nl/versions/*.py | sort | tail -1)" .py | cut -d_ -f1)
+          echo "alphagov-revision=$alphagov_revision" >> "$GITHUB_OUTPUT"
+          echo "nl-revision=$nl_revision" >> "$GITHUB_OUTPUT"
+
+      # Renders the chart and validates against the live cluster API/CRDs
+      # (required/fail template checks included) without persisting anything
+      # or scheduling any pods. Uses the exact same flags as the real deploy
+      # below so it can't drift out of sync and give a false pass.
+      - name: Pre-flight validation (dry-run, no cluster mutation)
+        id: helm-dry-run
+        run: |
+          set -x
+          helm upgrade --install notifynl notifynl-full/ \
+          -n ${{ vars.NAMESPACE }} \
+          --reset-then-reuse-values \
+          $HELM_SET_ARGS \
+          --set migrations.apiUpgrade.alphagovRevision=${{ steps.migration-revisions.outputs.alphagov-revision }} \
+          --set migrations.apiUpgrade.nlRevision=${{ steps.migration-revisions.outputs.nl-revision }} \
+          --rollback-on-failure \
+          --dry-run=server
+
       - name: Deploy new tag
         id: helm-deploy
         run: |
           set -x
           helm upgrade --install notifynl notifynl-full/ \
           -n ${{ vars.NAMESPACE }} \
-          --reuse-values \
-          --set apps.api.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerRetryTasks.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerLetters.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerJobs.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerResearch.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerSender.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerSenderLetters.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerPeriodic.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerReporting.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerInternal.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerReceipts.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerServiceCallbacks.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.apiWorkerReportRequestsNotificationsTasks.image.tag=${{ needs.build.outputs.tag }} \
-          --set apps.celeryBeat.image.tag=${{ needs.build.outputs.tag }} \
-          --wait \
+          --reset-then-reuse-values \
+          $HELM_SET_ARGS \
+          --set migrations.apiUpgrade.alphagovRevision=${{ steps.migration-revisions.outputs.alphagov-revision }} \
+          --set migrations.apiUpgrade.nlRevision=${{ steps.migration-revisions.outputs.nl-revision }} \
+          --rollback-on-failure \
           --timeout 300s
 
       - name: Comment on PR if deployment fails
-        if: failure() && steps.helm-deploy.outcome == 'failure' && github.event_name == 'pull_request'
+        if: failure() && (steps.helm-dry-run.outcome == 'failure' || steps.helm-deploy.outcome == 'failure') && github.event_name == 'pull_request'
         uses: actions/github-script@v7
         with:
           script: |
+            const dryRunFailed = "${{ steps.helm-dry-run.outcome }}" === "failure";
+            const body = dryRunFailed
+              ? [
+                  "⚠️ **Test environment deployment blocked (pre-flight check failed)**",
+                  "",
+                  "- Environment: `Test`",
+                  "- Image tag: `${{ needs.build.outputs.tag }}`",
+                  "",
+                  "The pre-flight `helm upgrade --dry-run=server` check failed, so the real deploy never ran — **the cluster was not touched**, nothing to clean up.",
+                  "",
+                  "Check the workflow logs for the Helm error output. Most likely cause: `notifynl-full` added a new value with no chart default that this release doesn't have set yet — seed it once with a manual `helm upgrade --set <key>=<value>`, and future deploys will carry it forward automatically."
+                ].join("\n")
+              : [
+                  "❌ **Test environment deployment failed**",
+                  "",
+                  "- Environment: `Test`",
+                  "- Image tag: `${{ needs.build.outputs.tag }}`",
+                  "",
+                  "The pre-flight check passed but the real deploy failed — the cluster may be in a partial state.",
+                  "",
+                  "Check the workflow logs for the Helm error output.",
+                  "",
+                  "Check the test env k8s state/logs.",
+                  "",
+                  "Check if the helm release needs a rollback.",
+                  "",
+                  "Most likely issues:",
+                  "- DB migration desync",
+                  "- Package visibility",
+                  "- ghcr.io credentials expiration or change"
+                ].join("\n");
             github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: context.issue.number,
-              body: [
-                "❌ **Test environment deployment failed**",
-                "",
-                "- Environment: `Test`",
-                "- Image tag: `${{ needs.build.outputs.tag }}`",
-                "",
-                "Check the workflow logs for the Helm error output.",
-                "",
-                "Check the test env k8s state/logs.",
-                "",
-                "Check if the helm release needs a rollback.",
-                "",
-                "Most likely issues:",
-                "- DB migration desync",
-                "- Package visibility",
-                "- ghcr.io credentials expiration or change"
-              ].join("\n")
+              body
             })

.gitignore

@@ -896,7 +896,7 @@ target/
 environment.sh
 .envrc
 
-celerybeat-schedule
+celerybeat-schedule*
 
 # CloudFoundry
 .cf

.pre-commit-config.yaml

@@ -14,3 +14,22 @@ repos:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]
     - id: ruff-format
+- repo: https://github.com/astral-sh/uv-pre-commit
+  # uv version.
+  rev: 0.12.3
+  hooks:
+    - id: pip-compile
+      name: pip-compile requirements.in
+      args: [requirements.in, -o, requirements.txt]
+    - id: pip-compile
+      name: pip-compile requirements_for_test.in
+      args: [requirements_for_test.in, -o, requirements_for_test.txt]
+      files: ^requirements_for_test\.(in|txt)$
+    - id: pip-compile
+      name: pip-compile requirements_nl.in
+      args: [requirements_nl.in, -o, requirements_nl.txt]
+      files: ^requirements_nl\.(in|txt)$
+    - id: pip-compile
+      name: pip-compile requirements_nl_test.in
+      args: [requirements_nl_test.in, -o, requirements_nl_test.txt]
+      files: ^requirements_nl_test\.(in|txt)$

.vscode/launch.json

@@ -28,10 +28,13 @@
         "run_celery.notify_celery",
         "worker",
         "-l",
-        "INFO",
+        "DEBUG",
         "-E",
         "-P",
-        "solo"
+        "solo",
+        "--without-gossip",
+        "--without-mingle",
+        "--without-heartbeat"
       ],
       "consoleName": "API worker",
       "preLaunchTask": "API pre-launch"
@@ -47,7 +50,7 @@
         "run_celery.notify_celery",
         "beat",
         "-l",
-        "INFO"
+        "DEBUG"
       ],
       "consoleName": "Celery beat",
       "preLaunchTask": "API pre-launch"

.vscode/settings.json

@@ -1,11 +1,26 @@
 {
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
     "python.testing.pytestEnabled": true,
-    "python.defaultInterpreterPath": ".venv/bin/python",
+    "python.terminal.useEnvFile": true,
     "python-envs.pythonProjects": [
         {
             "path": ".",
             "envManager": "ms-python.python:venv",
             "packageManager": "ms-python.python:pip"
         }
-    ]
+    ],
+    "emeraldwalk.runonsave": {
+        "commands": [
+            {
+                "match": "requirements_nl_test\\.in$",
+                "cmd": "cd ${workspaceFolder} && uv pip compile requirements_nl_test.in -o requirements_nl_test.txt",
+                "isAsync": false
+            },
+            {
+                "match": "requirements_nl\\.in$",
+                "cmd": "cd ${workspaceFolder} && uv pip compile requirements_nl.in -o requirements_nl.txt && uv pip compile requirements_nl_test.in -o requirements_nl_test.txt",
+                "isAsync": false
+            }
+        ]
+    }
 }