Save verified 10M-row CSV results and recover interrupted promotions

.github/workflows/ci.yml

@@ -26,6 +26,7 @@ jobs:
           "Pillow>=10,<13"
           "requests>=2.31,<3"
           "requests-toolbelt>=1,<2"
+          "duckdb>=1.2,<2"
           pytest pytest-django build setuptools wheel
       - run: >-
           python -m pip install

.github/workflows/query-compatibility.yml

@@ -0,0 +1,42 @@
+name: Data query compatibility
+on:
+  pull_request:
+    paths:
+      - 'src/omero_analysis/data_query*'
+      - 'src/omero_analysis/views.py'
+      - 'tests/test*query*'
+      - 'tests/test_worker_contract.py'
+      - '.github/workflows/query-compatibility.yml'
+  workflow_dispatch:
+    inputs:
+      worker_ref:
+        description: Exact worker commit or branch to test with this Analysis revision
+        required: true
+        default: 80dae969f8ff0d4a9cb1699caed31a36fef9328f
+permissions:
+  contents: read
+jobs:
+  http-matrix:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: OMERO.Analysis
+      - uses: actions/checkout@v4
+        with:
+          repository: NL-BioImaging/OMERO.DataQueryWorker
+          ref: ${{ inputs.worker_ref || '80dae969f8ff0d4a9cb1699caed31a36fef9328f' }}
+          fetch-depth: 0
+          path: OMERO.DataQueryWorker
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -m venv OMERO.DataQueryWorker/.venv
+      - run: OMERO.DataQueryWorker/.venv/bin/pip install -e 'OMERO.DataQueryWorker[test]'
+      - run: python -m pip install --no-deps -e OMERO.Analysis
+      - run: python -m pip install 'Django>=3.2,<6' cryptography requests requests-toolbelt numpy Pillow pytest pytest-django duckdb
+      - run: python -m pytest tests/test_worker_contract.py -q
+        working-directory: OMERO.Analysis
+        env:
+          DQW_RUN_INTEGRATION: '1'
+

.gitignore

@@ -12,4 +12,6 @@ frontend/node_modules/
 src/omero_analysis/static/omero_analysis/pyodide/
 src/omero_analysis/static/omero_analysis/assets/
 .DS_Store
-benchmark-data/
+benchmark-data/
+.local-query-deployment/
+.local-query-gates/

README.md

@@ -1,5 +1,9 @@
 # OMERO.Analysis
 
+For paired DataQueryWorker upgrades, verified CSV saving, authorization, audit,
+mTLS, test commands and rollback, see the
+[production foundations guide](docs/data-query-production-foundations.md).
+
 Administrators enabling BIOMERO.importer-backed storage should read the [`.analysis` storage compatibility contract](docs/importer-analysis-storage-compatibility.md). It documents the optional runtime APIs, capability checks, versions, deployment, fallback rules, and integration-test matrix.
 
 The maintained user manual is available in [docs/MANUAL.md](docs/MANUAL.md)

deploy/compose.large-export.yaml

@@ -0,0 +1,13 @@
+# Apply alongside the worker's deploy/compose.large-export.yaml.
+services:
+  omeroweb:
+    environment:
+      CONFIG_omero_web_wsgi__timeout: "1800"
+      OMERO_ANALYSIS_DATA_QUERY_REQUEST_TIMEOUT_SECONDS: "930"
+      OMERO_ANALYSIS_DATA_QUERY_SOURCE_UPLOAD_TIMEOUT_SECONDS: "1800"
+      OMERO_ANALYSIS_NOTEBOOK_CELL_TIMEOUT_SECONDS: "3600"
+      OMERO_ANALYSIS_DATA_QUERY_RESULT_TTL_SECONDS: "3600"
+      OMERO_ANALYSIS_MAX_DOWNLOAD_BYTES: "2147483648"
+      OMERO_ANALYSIS_DATA_QUERY_PROMOTION_MAX_BYTES: "2147483648"
+# For OMERO Docker images using CONFIG_ environment translation. Also configure
+# proxy_read_timeout and proxy_send_timeout to 1800s in the reverse proxy.

deploy/compose.query-gates.yaml

@@ -0,0 +1,73 @@
+# Entirely disposable OMERO stack. Never attach production volumes/networks.
+services:
+  database:
+    image: postgres:16
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: omero
+      POSTGRES_DB: omero
+      POSTGRES_PASSWORD: ${GATE_OMERO_PASSWORD:?Generate a private gate environment file}
+    volumes:
+      - database:/var/lib/postgresql/data
+    networks: [gate]
+  omeroserver:
+    image: cellularimagingcf/omeroserver:1.7.0-beta.11
+    restart: unless-stopped
+    depends_on: [database]
+    environment:
+      CONFIG_omero_db_host: database
+      CONFIG_omero_db_user: omero
+      CONFIG_omero_db_name: omero
+      CONFIG_omero_db_pass: ${GATE_OMERO_PASSWORD}
+      ROOTPASS: ${GATE_OMERO_PASSWORD}
+    volumes:
+      - omero:/OMERO
+    networks: [gate]
+  web:
+    image: omero-analysis-web:recovery
+    restart: unless-stopped
+    depends_on: [omeroserver, data-query-worker]
+    environment:
+      OMEROHOST: omeroserver
+      CONFIG_omero_web_wsgi__timeout: "1800"
+      ROOTPASS: ${GATE_OMERO_PASSWORD}
+      OMERO_ANALYSIS_DATA_QUERY_WORKER_URL: http://data-query-worker:8080
+      OMERO_ANALYSIS_DATA_QUERY_WORKER_TOKEN: ${GATE_WORKER_TOKEN:?Generate a private gate environment file}
+      OMERO_ANALYSIS_DATA_QUERY_REQUEST_TIMEOUT_SECONDS: "930"
+      OMERO_ANALYSIS_DATA_QUERY_SOURCE_UPLOAD_TIMEOUT_SECONDS: "1800"
+      OMERO_ANALYSIS_DATA_QUERY_RESULT_TTL_SECONDS: "3600"
+      OMERO_ANALYSIS_DATA_QUERY_STATE_DIR: /opt/omero/web/OMERO.web/var/query-state
+      OMERO_ANALYSIS_DATA_QUERY_AUDIT_FILE: /opt/omero/web/OMERO.web/var/log/data-query-audit.jsonl
+      OMERO_ANALYSIS_DATA_QUERY_PROMOTION_MAX_BYTES: "2147483648"
+    volumes:
+      - web:/opt/omero/web/OMERO.web/var
+    networks: [gate]
+  data-query-worker:
+    image: omero-data-query-worker:recovery
+    init: true
+    restart: unless-stopped
+    read_only: true
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]
+    mem_limit: 2g
+    memswap_limit: 2g
+    cpus: 2.0
+    pids_limit: 128
+    environment:
+      DQW_API_TOKEN: ${GATE_WORKER_TOKEN}
+      DQW_MAX_RESULT_ROWS: "10000000"
+      DQW_MAX_RESULT_BYTES: "2147483648"
+      DQW_DUCKDB_MEMORY_LIMIT: 256MB
+      DQW_QUERY_TIMEOUT_SECONDS: "900"
+      DQW_INGESTION_TIMEOUT_SECONDS: "1800"
+    volumes:
+      - cache:/var/lib/omero-data-query-worker
+    networks: [gate]
+volumes:
+  database:
+  omero:
+  web:
+  cache:
+networks:
+  gate:
+    internal: true