diff --git a/.env b/.env index 0af2f734..7a7da33d 100644 --- a/.env +++ b/.env @@ -11,7 +11,7 @@ EXTERNAL_PORT=8018 # SUBPATH: The subpath of the istSOS server. # Default: /istsos4 -SUBPATH=/v4 +SUBPATH=/istsos4 # VERSION: The version of the istSOS offered STA. # Default: /v1.1 diff --git a/.env.example b/.env.example index b63e1e67..7a7da33d 100644 --- a/.env.example +++ b/.env.example @@ -1,138 +1,180 @@ -# Copy this file to .env and fill in your values before starting the stack. -# cp .env.example .env -# -# Variables marked REQUIRED have no default and the application will not work -# correctly without them. +# This file contains environment variables for the istsos4 project. -# --------------------------------------------------------------------------- -# Server -# --------------------------------------------------------------------------- - -# Public hostname of the server (should match EXTERNAL_PORT). +# HOSTNAME: The hostname of the server where the istsos4 project is running. +# Default: http://localhost:8018 +# Note: The port should match the EXTERNAL_PORT variable. HOSTNAME=http://localhost:8018 -# Port Docker exposes the API on. +# EXTERNAL_PORT: The external port on which the istSOS server is accessible. +# Default: 8018 EXTERNAL_PORT=8018 -# URL subpath for the API. +# SUBPATH: The subpath of the istSOS server. +# Default: /istsos4 SUBPATH=/istsos4 -# SensorThings API version segment. +# VERSION: The version of the istSOS offered STA. +# Default: /v1.1 VERSION=/v1.1 -# Set to 1 to enable verbose debug output. +# DEBUG: Enable or disable debug mode. +# 0 - disabled +# 1 - enabled +# Default: 0 DEBUG=0 -# --------------------------------------------------------------------------- -# PostgreSQL -# --------------------------------------------------------------------------- +# VERSIONING: Enable or disable versioning of data. +# 0 - disabled +# 1 - enabled +# Default: 0 +VERSIONING=0 +# POSTGRES_DB: The name of the PostgreSQL database used by istSOS. +# Default: istsos POSTGRES_DB=istsos + +# POSTGRES_USER: The username for connecting to the PostgreSQL database. +# Default: postgres POSTGRES_USER=postgres -# REQUIRED — choose a strong password. -POSTGRES_PASSWORD=change_me +# POSTGRES_PASSWORD: The password for connecting to the PostgreSQL database. +# Default: postgres +POSTGRES_PASSWORD=postgres +# POSTGRES_HOST: Specifies the hostname of the PostgreSQL database server. +# Default: database POSTGRES_HOST=database -POSTGRES_PORT=5432 -# External port Docker exposes Postgres on (useful for local psql access). +# POSTGRES_EXTERNAL_PORT: The external port for the PostgreSQL database. +# Default: 45432 POSTGRES_EXTERNAL_PORT=45432 -# Set to a write-replica host:port to split reads/writes. Leave empty for -# single-node setups. -POSTGRES_PORT_WRITE= - -# --------------------------------------------------------------------------- -# istSOS administrator account -# --------------------------------------------------------------------------- - +# ISTSOS_ADMIN: The username for the istSOS administrator. +# Default: admin ISTSOS_ADMIN=admin -# REQUIRED — choose a strong password. -ISTSOS_ADMIN_PASSWORD=change_me - -# --------------------------------------------------------------------------- -# Connection pool -# --------------------------------------------------------------------------- +# ISTSOS_ADMIN_PASSWORD: The password for the istSOS administrator. +# Default: admin +ISTSOS_ADMIN_PASSWORD=admin +# PG_MAX_OVERFLOW: The maximum number of connections that can be opened above the maximum number of connections specified by PG_MAX_CONNECTIONS. +# Default: 0 PG_MAX_OVERFLOW=0 + +# PG_POOL_SIZE: The maximum number of connections to the PostgreSQL database pool. +# Default: 10 PG_POOL_SIZE=10 -PG_POOL_TIMEOUT=30 -# --------------------------------------------------------------------------- -# Query behaviour -# --------------------------------------------------------------------------- +# PG_POOL_TIMEOUT: The maximum time in seconds that a client can wait for a connection from the connection pool. +# Default: 30 +PG_POOL_TIMEOUT=30 -# FULL | LIMIT_ESTIMATE | ESTIMATE_LIMIT -# FULL always accurate; ESTIMATE_LIMIT fastest for large datasets. -COUNT_MODE=FULL +# DUMMY_DATA: Set to 1 if you want to use dummy data for testing purposes. +# 0 - disabled +# 1 - enabled +# Default: 1 +DUMMY_DATA=0 -COUNT_ESTIMATE_THRESHOLD=10000 -TOP_VALUE=100 -PARTITION_CHUNK=10000 - -# --------------------------------------------------------------------------- -# Features -# --------------------------------------------------------------------------- +# CLEAR_DATA: Set to 1 if you want to clear existing data before running the program. +# 0 - disabled +# 1 - enabled +# Default: 0 +CLEAR_DATA=0 -# Enable Redis-based token blacklisting (0 = disabled, 1 = enabled). -REDIS=0 +# N_THINGS: Number of things to generate data for. +# Default: 10 +N_THINGS=5 -# Allow duplicate observations (0 = disabled, 1 = enabled). -DUPLICATES=0 +# N_OBSERVED_PROPERTIES: Number of observed properties to generate data for. +# Default: 2 +N_OBSERVED_PROPERTIES=4 -# Default coordinate reference system (EPSG code). -EPSG=4326 +# INTERVAL: Interval for generating data. +# Default: P1Y (1 year) +INTERVAL=P7D -# Enable row-level authorization (0 = disabled, 1 = enabled). -AUTHORIZATION=0 +# FREQUENCY: Frequency of data generation. +# Default: PT5M (every 5 minutes) +FREQUENCY=PT5M -# Allow unauthenticated read access (0 = disabled, 1 = enabled). -ANONYMOUS_VIEWER=0 +# START_DATETIME: The start date for phenomenonTime of observation. +# Format: YYYY-MM-DDTHH:MM:SS.SSS+HH:MM +# Default: 2020-01-01T12:00:00.000+01:00 +START_DATETIME=2020-01-01T12:00:00.000+01:00 -# Enable network entity support (0 = disabled, 1 = enabled). -NETWORK=0 +# CHUNK_INTERVAL: The interval for chunking data. +# Default: P1Y (1 year) +CHUNK_INTERVAL=P7D -# Enable data versioning (0 = disabled, 1 = enabled). -VERSIONING=0 +# COUNT_MODE: Specifies the count mode for estimation. +# FULL - Fully count all entities. Can be very slow on large result sets, but always gives accurate results. +# LIMIT_ESTIMATE - First do a count, with a limit of countEstimateThreshold. If the limit is reached, do an +# estimate using EXPLAIN. For large result can be wildly inaccurate for fields that are not backed by an index. +# For low counts this method is still guaranteed to give accurate results. +# ESTIMATE_LIMIT - First do an estimate using EXPLAIN and if the estimate is below the threshold, do a count with +# a limit of countEstimateThreshold. This is the fastest method, but if the estimate is inaccurate, +# it can give an incorrect estimate for low counts. +# Default: FULL +COUNT_MODE=FULL -# Observed-area aggregation function: CONVEX_HULL | EXTENT -ST_AGGREGATE=CONVEX_HULL +# COUNT_ESTIMATE_THRESHOLD: Specifies the threshold for count estimation. +# Default: 10000 +COUNT_ESTIMATE_THRESHOLD=10000 -# --------------------------------------------------------------------------- -# Authentication (REQUIRED when AUTHORIZATION=1) -# --------------------------------------------------------------------------- +# TOP_VALUE: Specifies the default top value. +# Default: 100 +TOP_VALUE=100 -# REQUIRED — generate with: openssl rand -hex 32 -SECRET_KEY=replace_with_output_of__openssl_rand_-hex_32 +# PARTITION_CHUNK: Determines the number of records to be processed in each partition. +# Default: 10000 +PARTITION_CHUNK=10000 -ALGORITHM=HS256 +# REDIS: Indicates whether Redis is enabled. +# 0 - disabled +# 1 - enabled +# Default: 0 +REDIS=0 -# Access token lifetime in minutes. -ACCESS_TOKEN_EXPIRE_MINUTES=60 +# DUPLICATES: Indicates whether duplicate entries are allowed. +# 0 - disabled +# 1 - enabled +# Default: 0 +DUPLICATES=1 -# --------------------------------------------------------------------------- -# Dummy data generator (dev_docker-compose.yml only) -# --------------------------------------------------------------------------- +# EPSG: Specifies the coordinate reference system to be used. +# Default: 4326 +EPSG=4326 -# Load synthetic data on first startup (0 = disabled, 1 = enabled). -DUMMY_DATA=1 +# AUTHORIZATION: Specifies the authorization level. +# 0 - disabled +# 1 - enabled +# Default: 0 +AUTHORIZATION=0 -# Wipe existing data before loading (0 = disabled, 1 = enabled). -CLEAR_DATA=0 +# SECRET_KEY: A secret key used for encoding tokens generated with 'openssl rand -hex 32'. +# Default: 09d25e094faa6ca2556c818166b7a9563b93f7099f6f0f4caa6cf63b88e8d3e7 +SECRET_KEY=09d25e094faa6ca2556c818166b7a9563b93f7099f6f0f4caa6cf63b88e8d3e7 -N_THINGS=5 -N_OBSERVED_PROPERTIES=4 +# ALGORITHM: The algorithm used for encoding tokens. +# Default: HS256 +ALGORITHM=HS256 -# ISO 8601 duration: total time span of generated observations. -INTERVAL=P7D +# ACCESS_TOKEN_EXPIRE_MINUTES: The expiration time for access tokens in minutes. +# Default: 5 +ACCESS_TOKEN_EXPIRE_MINUTES=60 -# ISO 8601 duration: gap between observations. -FREQUENCY=PT5M +# ANONYMOUS_VIEWER: Indicates whether anonymous viewer is enabled. +# 0 - disabled +# 1 - enabled +ANONYMOUS_VIEWER=0 -# ISO 8601 datetime: start of the generated observation series. -START_DATETIME=2020-01-01T12:00:00.000+01:00 +# NETWORK: Indicates whether network access is enabled. +# 0 - disabled +# 1 - enabled +NETWORK=0 -# ISO 8601 duration: time-series partition size. -CHUNK_INTERVAL=P7D +# ST_AGGREGATE: Specifies the function for updating the observedArea of datastreams +# CONVEX_HULL - A convex hull is used to determine an affected area based on a aggregate set of geometries based on their sets of point observations. +# EXTENT - a two-dimensional bounding box enclosing a geometry or collection of geometries is used instead. +# Default: CONVEX_HULL +ST_AGGREGATE=CONVEX_HULL diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..79056b93 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,75 @@ +name: CI + +on: + push: + branches: [main, dev] + pull_request: + +env: + STA_BASE_URL: http://localhost:8018/istsos4/v1.1 + +jobs: + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Start stack + run: | + cp .env.example .env + docker compose -f dev_docker-compose.yml up -d --build --wait database api + + - name: Wait for API + run: timeout 120 bash -c 'until curl -sf "$STA_BASE_URL" >/dev/null; do sleep 2; done' + + - name: Install test deps + run: pip install -r tests/conformance/requirements.txt + + - name: OGC 18-088 conformance (405 tests) + run: pytest tests/conformance + + - name: API logs + if: failure() + run: docker compose -f dev_docker-compose.yml logs api + + publish: + if: github.ref == 'refs/heads/main' + needs: conformance + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + component: [api, database] + steps: + - uses: actions/checkout@v4 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve image tag + id: image + run: | + ref=$(grep -oE "ghcr\.io/istsos/istsos4/${{ matrix.component }}:[^[:space:]]+" docker-compose.yml) + echo "ref=$ref" >> "$GITHUB_OUTPUT" + if docker manifest inspect "$ref" >/dev/null 2>&1; then + echo "$ref already published, skipping" + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - uses: docker/build-push-action@v6 + if: steps.image.outputs.exists == 'false' + with: + context: ./${{ matrix.component }} + push: true + tags: ${{ steps.image.outputs.ref }} diff --git a/.gitignore b/.gitignore index fb778b2f..11476589 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,13 @@ -# Python stuff -__pycache__/ +# Python *.py[cod] -.cache/ +.cache + +# Local config & secrets .env -*.pkl + +# Generated output docs/mkdocs/site database/schemaspy -.venv/ -venv/ -env/ -.codex -.vscode -.python-version \ No newline at end of file + +# Editors +.vscode \ No newline at end of file diff --git a/tests/conformance/README.md b/tests/conformance/README.md index 290ec5d5..5c76d3ec 100644 --- a/tests/conformance/README.md +++ b/tests/conformance/README.md @@ -36,7 +36,7 @@ lead; test files are owned per the table above. ## Install -The tests need `pytest`, `httpx`, `pytest-xdist`. Use an isolated venv: +The tests need `pytest` and `httpx`. Use an isolated venv: ```bash uv venv tests/conformance/.venv --python 3.12 @@ -48,24 +48,27 @@ uv pip install --python tests/conformance/.venv/bin/python -r tests/conformance/ ## Run The istSOS4 API must be running. Default target: -`http://localhost:8018/v4/v1.1` (override with `STA_BASE_URL`). +`http://localhost:8018/istsos4/v1.1` (the `SUBPATH` shipped in `.env.example`; +override with `STA_BASE_URL`). ```bash PYBIN=tests/conformance/.venv/bin/python # one class at a time (three 18-088 classes) -$PYBIN -m pytest tests/conformance -m c01 # 203 passed -$PYBIN -m pytest tests/conformance -m c02 # 73 passed +$PYBIN -m pytest tests/conformance -m c01 # 204 passed +$PYBIN -m pytest tests/conformance -m c02 # 81 passed $PYBIN -m pytest tests/conformance -m c03 # 120 passed # or a single subfolder -$PYBIN -m pytest tests/conformance/c01 +$PYBIN -m pytest tests/conformance/c01 # 204 passed +$PYBIN -m pytest tests/conformance/c02 # 81 passed +$PYBIN -m pytest tests/conformance/c03 # 120 passed -# whole suite, in parallel (isolation must hold) -> 396 passed (18-088 only) -$PYBIN -m pytest tests/conformance -n auto +# whole suite -> 405 passed (18-088 only) +$PYBIN -m pytest tests/conformance # point at another deployment -STA_BASE_URL=http://host:port/v4/v1.1 $PYBIN -m pytest tests/conformance +STA_BASE_URL=http://host:port/istsos4/v1.1 $PYBIN -m pytest tests/conformance ``` `pytest tests/conformance` auto-selects this directory's `pytest.ini` (not the diff --git a/tests/conformance/client.py b/tests/conformance/client.py index 224eb96c..5ef60f44 100644 --- a/tests/conformance/client.py +++ b/tests/conformance/client.py @@ -24,7 +24,7 @@ import httpx -DEFAULT_BASE_URL = "http://localhost:8018/v4/v1.1" +DEFAULT_BASE_URL = "http://localhost:8018/istsos4/v1.1" # Characters left un-escaped in query VALUES. Space is deliberately excluded so # it becomes %20. Single quotes (string literals) and STA punctuation are kept diff --git a/tests/conformance/requirements.txt b/tests/conformance/requirements.txt index 0157533e..782e2e66 100644 --- a/tests/conformance/requirements.txt +++ b/tests/conformance/requirements.txt @@ -4,4 +4,3 @@ # uv pip install --python tests/conformance/.venv/bin/python -r tests/conformance/requirements.txt pytest>=8.0 httpx>=0.27 -pytest-xdist>=3.5 diff --git a/tests/extensions/client.py b/tests/extensions/client.py index 1b59bb80..be9c7e08 100644 --- a/tests/extensions/client.py +++ b/tests/extensions/client.py @@ -13,7 +13,7 @@ import httpx -DEFAULT_BASE_URL = "http://localhost:8018/v4/v1.1" +DEFAULT_BASE_URL = "http://localhost:8018/istsos4/v1.1" _SAFE_VALUE = "$()/:,'=;.-"