diff --git a/.devcontainer/dev_container.dockerfile b/.devcontainer/dev_container.dockerfile new file mode 100644 index 0000000..f602887 --- /dev/null +++ b/.devcontainer/dev_container.dockerfile @@ -0,0 +1,29 @@ +# Base image for the development container +ARG BASE_URL=python:3.12-slim +FROM ${BASE_URL} + +USER root + +# We'll be running as a non-root user in a container and may want root permissions +RUN apt update && apt -y install nano ssh sudo && apt clean + +# Install setup tools and dependencies +WORKDIR /app +COPY . /app +RUN pip install --upgrade pip setuptools && pip install .[dev] + +# Set up user to match the host OS (https://stackoverflow.com/a/78621662/415551) +ARG HOST_USER +ARG HOST_UID +ARG HOST_GID + +RUN addgroup --gid ${HOST_GID} ${HOST_USER} \ + && adduser --gecos "" --disabled-password --uid ${HOST_UID} --gid ${HOST_GID} ${HOST_USER} \ + && usermod -aG sudo ${HOST_USER} \ + && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +ENV HOME /home/${HOST_USER} +ENV TMPDIR=/tmp +WORKDIR /home/${HOST_USER} + +USER ${HOST_USER} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..9c1d714 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,51 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "Unit Test Image", + "remoteUser": "${localEnv:USER}", + "build": { + // Sets the build context to one level up instead of the .devcontainer folder. + "context": "..", + "dockerfile": "dev_container.dockerfile", + "args": { + "HOST_USER": "${localEnv:USER}", + "HOST_UID": "${localEnv:HOST_UID:1000}", + "HOST_GID": "${localEnv:HOST_GID:1000}" + } + }, + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers/features/git:1": {} + }, + "customizations": { + "vscode": { + "extensions": [ + "donjayamanne.git-extension-pack", + "eamodio.gitlens", + "mhutchie.git-graph", + "ms-azuretools.vscode-docker", + "ms-python.python", + "ms-python.debugpy", + "ms-python.black-formatter", + "ms-python.flake8", + "ms-python.vscode-pylance", + "njpwerner.autodocstring", + "redhat.vscode-yaml", + "SonarSource.sonarlint-vscode" + ], + "settings": { + "flake8.args": [ + "--max-line-length=120" + ] + } + } + } + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + // Uncomment the next line to run commands after the container is created. + // "postCreateCommand": "cat /etc/os-release", + // Configure tool-specific properties. + // "customizations": {}, + // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "devcontainer" +} \ No newline at end of file diff --git a/.github/workflows/github-python-workflow.yml b/.github/workflows/github-python-workflow.yml new file mode 100644 index 0000000..3e3e5a9 --- /dev/null +++ b/.github/workflows/github-python-workflow.yml @@ -0,0 +1,105 @@ +# This workflow handles linting, testing, and SonarQube scanning for the OGC repository. +name: Python Workflow + +on: + push: + branches: [ main, develop ] + pull_request: + types: [ opened, synchronize, reopened ] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install --upgrade pip setuptools && pip install .[dev] + + - name: Run flake8 lint checks + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --ignore=E,W,D,I,N806,N815,N818,Q000,Q001,Q002,S001,B008,B028 --show-source --statistics --max-line-length=120 --output-file=flake8-report.txt + + - name: Run black format checks + run: black --check --diff -l 120 ogc example > black-report.txt + + - name: Artifact flake8 report + uses: actions/upload-artifact@v4 + with: + name: flake8-report + path: flake8-report.txt + retention-days: 1 + + - name: Artifact black report + uses: actions/upload-artifact@v4 + with: + name: black-report + path: black-report.txt + retention-days: 1 + + unit_testing: + name: Unit Testing + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install --upgrade pip setuptools && pip install .[dev] + + - name: Run pytest with coverage + run: | + coverage run --data-file=coverage.bin --branch \ + -m pytest --continue-on-collection-errors || \ + { echo "WARNING: Some tests have failed. (Temporarily ignored)"; true; } + coverage xml --data-file=coverage.bin -o coverage.xml + sed -i 's,.*/ogc,ogc,' coverage.xml + - name: Artifact coverage document + uses: actions/upload-artifact@v4 + with: + name: coverage + path: coverage.xml + retention-days: 1 + + sonarqube_scan: + name: SonarQube Scan + needs: [ unit_testing ] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Download coverage artifact + uses: actions/download-artifact@v4 + with: + name: coverage + path: . + + - name: SonarQube Scan + uses: SonarSource/sonarqube-scan-action@fd88b7d7ccbaefd23d8f36f73b59db7a3d246602 + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..08cb765 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: +- repo: https://github.com/ambv/black + rev: stable + hooks: + - id: black + language_version: python3 + files: (^ogc|^example) \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1873b22 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog +## 0.5.0 + +### Introduction +Support for Environmental Data Retrieval (EDR) was added in this version to extend OGC capabilities. +EDR is optional and can be enabled via the OGC_SUPPORTED_FORMATS environmental variable. +The currently supported version of EDR is v1.1.0. Additionally, the repository was modernized by +adding a development container, pyproject.toml, and Github Actions for linting, formatting, +unit testing and scanning. + +### Features +* Added support for EDR v1.1.0. + +### Maintenance +* Added unit testing for code coverage requirements. +* Added development container and pyproject.toml file. +* Added Github Actions for linting, formatting, unit testing, and scanning. + +### Bugfixes +* Fixed formatting, linting, and SonarQube errors to pass CI/CD scans. \ No newline at end of file diff --git a/example/app.py b/example/app.py index 8aff167..b8e11e4 100755 --- a/example/app.py +++ b/example/app.py @@ -2,33 +2,43 @@ Demo Flask app using PODPAC Node backed layers. """ -from flask import Flask from datetime import datetime import re from ogc import servers from ogc import core from ogc import podpac as pogc +from ogc.settings import EDR_TIME_INSTANCE_DIMENSION import podpac import numpy as np +# Setup new dimension +podpac.core.coordinates.utils.add_valid_dimension(EDR_TIME_INSTANCE_DIMENSION) + # create some podpac nodes -data = np.random.rand(11, 21) +data = np.random.default_rng(1).random((11, 21)) lat = np.linspace(90, -90, 11) lon = np.linspace(-180, 180, 21) coords = podpac.Coordinates([lat, lon], dims=["lat", "lon"]) node1 = podpac.data.Array(source=data, coordinates=coords) -data2 = np.random.rand(11, 21) +data2 = np.random.default_rng(1).random((11, 21)) node2 = podpac.data.Array(source=data2, coordinates=coords) +time = np.array(["2025-10-24T12:00:00"], dtype="datetime64") +instance = np.array(["2025-10-24T00:00:00"], dtype="datetime64") +coords = podpac.Coordinates([lat, lon, time, instance], dims=["lat", "lon", "time", EDR_TIME_INSTANCE_DIMENSION]) +data3 = np.random.default_rng(1).random((11, 21, 1, 1)) +node3 = podpac.data.Array(source=data3, coordinates=coords) + # use podpac nodes to create some OGC layers layer1 = pogc.Layer( node=node1, identifier="layer1", title="OGC/POPAC layer containing random data", abstract="This layer contains some random data", + group="Layers", ) layer2 = pogc.Layer( @@ -37,9 +47,19 @@ title="FOUO: Another OGC/POPAC layer containing random data", abstract="Marked as FOUO. This layer contains some random data. Same coordinates as layer1, but different values.", is_fouo=True, + group="Layers", +) + +layer3 = pogc.Layer( + node=node3, + identifier="layer3", + title="OGC/POPAC layer containing random data with time instances available.", + abstract="This layer contains some random data with time instances available.", + group="Layers", + valid_times=[dt.astype(datetime) for dt in time], ) -all_layers = [layer1, layer2] +all_layers = [layer1, layer2, layer3] non_fouo_layers = [layer for layer in all_layers if not layer.is_fouo] # create a couple of different ogc endpoints @@ -69,16 +89,25 @@ def api_home(endpoint):
  • WMS GetLegend Example (PNG) (v1.3.0)
    • +
  • EDR: Open Geospatial Consortium (OGC) Environmental Data Retrieval (EDR) (v1.0.1) + +
    • """ app = servers.FlaskServer(__name__, ogcs=[NonFouoOGC, FouoOGC], home_func=api_home) + # add in some other endpoints. @app.route("/") def home(): - return f'This is an example OGC flask app. See FULL and PARTIAL endpoints.' + return """This is an example OGC flask app. + See FULL and PARTIAL endpoints.""" @app.route("/layers/") @@ -87,7 +116,7 @@ def check_layers(layer): if match_object: clean_layer = match_object.group(0) - if clean_layer in [l.identifier for l in all_layers]: + if clean_layer in [available_layer.identifier for available_layer in all_layers]: return "{} is an available layer id".format(clean_layer) else: return "No layer available with that id" diff --git a/ogc/__init__.py b/ogc/__init__.py index 7e130a5..b8aa1a1 100755 --- a/ogc/__init__.py +++ b/ogc/__init__.py @@ -1,5 +1,5 @@ """ -OGC WMS/WCS (v1.3.0/v1.0.0) server +OGC WMS/WCS (v1.3.0/v1.0.0) server """ import traitlets as tl @@ -70,10 +70,9 @@ class Layer(tl.HasTraits): identifier = tl.Unicode() title = tl.Unicode(default_value="An OGC Layer") abstract = tl.Unicode(default_value="This is an example OGC Layer") + group = tl.Unicode(default_value="Default") is_fouo = tl.Bool(default_value=False) - grid_coordinates = tl.Instance( - klass=GridCoordinates, default_value=GridCoordinates() - ) + grid_coordinates = tl.Instance(klass=GridCoordinates, default_value=GridCoordinates()) valid_times = tl.List( trait=tl.Instance(datetime.datetime), default_value=tl.Undefined, @@ -101,9 +100,7 @@ def __init__(self, *args, **kwargs): elif "title" in kwargs: string_repr = kwargs["title"] if "is_enumerated" in kwargs: - self._style = Style( - string_repr=string_repr, is_enumerated=kwargs["is_enumerated"] - ) + self._style = Style(string_repr=string_repr, is_enumerated=kwargs["is_enumerated"]) else: self._style = Style(string_repr=string_repr) if self.valid_times is not tl.Undefined: diff --git a/ogc/core.py b/ogc/core.py index ec9e392..776161a 100755 --- a/ogc/core.py +++ b/ogc/core.py @@ -3,6 +3,7 @@ Currently holds some definitions for interface classes. """ + import gc import logging import traitlets as tl @@ -12,12 +13,16 @@ from . import wms_request_1_3_0 from . import wcs_response_1_0_0 from . import wms_response_1_3_0 -from . import ogc_common +from .edr import EdrRoutes from ogc.ogc_common import WCSException logger = logging.getLogger(__name__) +LOAD_FAILURE = "Failed to load and validate: " +INVALID_ARGUMENTS = "Invalid arguments" + + class OGC(tl.HasTraits): wms_capabilities = tl.Instance(klass=wms_response_1_3_0.Capabilities) @@ -25,9 +30,7 @@ class OGC(tl.HasTraits): endpoint = tl.Unicode(default_value="/ogc", allow_none=True) service_title = tl.Unicode(default_value="OGC Server", allow_none=True) - service_abstract = tl.Unicode( - default_value="An example OGC Server", allow_none=True - ) + service_abstract = tl.Unicode(default_value="An example OGC Server", allow_none=True) server_address = tl.Unicode(default_value="http://127.0.0.1:5000", allow_none=True) service_group_title = tl.Unicode(default_value="Data Products", allow_none=True) @@ -35,8 +38,9 @@ class OGC(tl.HasTraits): def base_url(self): return "{}{}?".format(self.server_address, self.endpoint) - def __init__(self, layers=[], **kwargs): + def __init__(self, layers=None, **kwargs): super().__init__(**kwargs) + layers = layers if layers is not None else [] coverages = [ wcs_response_1_0_0.Coverage( layer=layer, @@ -59,7 +63,11 @@ def __init__(self, layers=[], **kwargs): service_abstract=self.service_abstract, service_group_title=self.service_group_title, ) - return + self.edr_routes = ( + EdrRoutes(base_url=f"{self.server_address}{self.endpoint}/edr", layers=layers) + if settings.EDR_ENABLED + else None + ) def get_coverage_from_id(self, identifier): for coverage in self.wcs_capabilities.coverages: @@ -71,22 +79,81 @@ def get_coverage_from_id(self, identifier): exception_text="Invalid coverage {}".format(identifier), ) - def handle_wcs_kv(self, args): - if args["request"] == "GetCapabilities": - get_capabilities = wcs_request_1_0_0.GetCapabilities() - try: - get_capabilities.load_from_kv(args) - get_capabilities.validate() - except: - logger.error("Failed to load and validate: ", exc_info=True) - raise WCSException(exception_text="Invalid arguments") + def get_capabilities_wcs(self, args): + get_capabilities = wcs_request_1_0_0.GetCapabilities() + try: + get_capabilities.load_from_kv(args) + get_capabilities.validate() + except Exception: + logger.error(LOAD_FAILURE, exc_info=True) + raise WCSException(exception_text=INVALID_ARGUMENTS) + + capabilities = self.wcs_capabilities + + if args["base_url"]: + capabilities.base_url = args["base_url"] + + return capabilities.to_xml() + + def describe_coverage_wcs(self, args, wcs_request, wcs_response): + describe_coverage = wcs_request.DescribeCoverage() + try: + describe_coverage.load_from_kv(args) + describe_coverage.validate() + except Exception: + logger.error(LOAD_FAILURE, exc_info=True) + raise WCSException(exception_text=INVALID_ARGUMENTS) + + coverages = [self.get_coverage_from_id(identifier.value) for identifier in describe_coverage.identifiers] + coverage_description = wcs_response.CoverageDescription(coverages=coverages) + + return coverage_description.to_xml() + + def get_coverage_wcs(self, args, wcs_request): + get_coverage = wcs_request.GetCoverage() + try: + get_coverage.load_from_kv(args) + get_coverage.validate() + except Exception: + logger.error(LOAD_FAILURE, exc_info=True) + raise WCSException(exception_text=INVALID_ARGUMENTS) + + coverage = self.get_coverage_from_id(get_coverage.identifier.value) + + if get_coverage.width == 0: + raise WCSException( + exception_code="InvalidParameterValue", + locator="VERSION", + exception_text="Grid coordinates x_size must be greater than 0", + ) + if get_coverage.height == 0: + raise WCSException( + exception_code="InvalidParameterValue", + locator="VERSION", + exception_text="Grid coordinates y_size must be greater than 0", + ) + if get_coverage.height * get_coverage.width > settings.MAX_GRID_COORDS_REQUEST_SIZE: + raise WCSException( + exception_code="InvalidParameterValue", + locator="VERSION", + exception_text="Grid coordinates x_size * y_size must be less than %d" + % settings.MAX_GRID_COORDS_REQUEST_SIZE, + ) + + fp = coverage.layer.get_coverage(args) + + fn = coverage.identifier.split(".")[-1] + ".tif" + + # Collect garbage + gc.collect() - capabilities = self.wcs_capabilities + response = {"fp": fp, "fn": fn} - if args["base_url"]: - capabilities.base_url = args["base_url"] + return response - return capabilities.to_xml() + def handle_wcs_kv(self, args): + if args["request"] == "GetCapabilities": + return self.get_capabilities_wcs(args) if "version" in args and args["version"] == "1.0.0": wcs_response = wcs_response_1_0_0 @@ -99,87 +166,96 @@ def handle_wcs_kv(self, args): ) if args["request"] == "DescribeCoverage": + return self.describe_coverage_wcs(args, wcs_request, wcs_response) - describe_coverage = wcs_request.DescribeCoverage() - try: - describe_coverage.load_from_kv(args) - describe_coverage.validate() - except: - logger.error("Failed to load and validate: ", exc_info=True) - raise WCSException(exception_text="Invalid arguments") + elif args["request"] == "GetCoverage": + return self.get_coverage_wcs(args, wcs_request) - coverages = [ - self.get_coverage_from_id(identifier.value) - for identifier in describe_coverage.identifiers - ] - coverage_description = wcs_response.CoverageDescription(coverages=coverages) + raise WCSException(exception_text="KV Request not handled properly: " + str(args)) - return coverage_description.to_xml() + def get_capabilities_wms(self, args): + get_capabilities = wms_request_1_3_0.GetCapabilities() + try: + get_capabilities.load_from_kv(args) + get_capabilities.validate() + except Exception: + logger.error(LOAD_FAILURE, exc_info=True) + raise WCSException(exception_text=INVALID_ARGUMENTS) - elif args["request"] == "GetCoverage": - get_coverage = wcs_request.GetCoverage() - try: - get_coverage.load_from_kv(args) - get_coverage.validate() - except: - logger.error("Failed to load and validate: ", exc_info=True) - raise WCSException(exception_text="Invalid arguments") + wms_capabilities = self.wms_capabilities - coverage = self.get_coverage_from_id(get_coverage.identifier.value) + if args["base_url"]: + wms_capabilities.base_url = args["base_url"] + return wms_capabilities.to_xml() - from dateutil.parser import parse + def get_legend_graphic(self, args, wms_request): + get_legend_graphic = wms_request.GetLegendGraphic() + try: + get_legend_graphic.load_from_kv(args) + get_legend_graphic.validate() + except Exception: + logger.error(LOAD_FAILURE, exc_info=True) + raise WCSException(exception_text=INVALID_ARGUMENTS) + coverage = self.get_coverage_from_id(get_legend_graphic.layer.value) - if get_coverage.width == 0: - raise WCSException( - exception_code="InvalidParameterValue", - locator="VERSION", - exception_text="Grid coordinates x_size must be greater than 0", - ) - if get_coverage.height == 0: - raise WCSException( - exception_code="InvalidParameterValue", - locator="VERSION", - exception_text="Grid coordinates y_size must be greater than 0", - ) - if get_coverage.height * get_coverage.width > settings.MAX_GRID_COORDS_REQUEST_SIZE: - raise WCSException( - exception_code="InvalidParameterValue", - locator="VERSION", - exception_text="Grid coordinates x_size * y_size must be less than %d" % settings.MAX_GRID_COORDS_REQUEST_SIZE, - ) + fp = coverage.layer.get_legend_graphic(args) + fn = coverage.identifier.split(".")[-1] + ".png" - fp = coverage.layer.get_coverage(args) + response = {"fp": fp, "fn": fn} + return response + + def get_map(self, args, wms_request): + get_map = wms_request.GetMap() + try: + get_map.load_from_kv(args) + get_map.validate() + except Exception: + logger.error(LOAD_FAILURE, exc_info=True) + raise WCSException(exception_text=INVALID_ARGUMENTS) + + coverage = self.get_coverage_from_id(get_map.layer.value) + + # Make sure the request size is correct + if get_map.width == 0: + raise WCSException( + exception_code="InvalidParameterValue", + locator="VERSION", + exception_text="Grid coordinates x_size must be greater than 0", + ) + if get_map.height == 0: + raise WCSException( + exception_code="InvalidParameterValue", + locator="VERSION", + exception_text="Grid coordinates y_size must be greater than 0", + ) + if get_map.height * get_map.width > settings.MAX_GRID_COORDS_REQUEST_SIZE: + raise WCSException( + exception_code="InvalidParameterValue", + locator="VERSION", + exception_text="Grid coordinates x_size * y_size must be less than %d" + % settings.MAX_GRID_COORDS_REQUEST_SIZE, + ) - fn = coverage.identifier.split(".")[-1] + ".tif" + try: + fp = coverage.layer.get_map(args) + except Exception: + logger.error("Failed to get_map from layer: ", exc_info=True) + raise WCSException(exception_text=INVALID_ARGUMENTS) - # Collect garbage - gc.collect() + fn = coverage.identifier.split(".")[-1] + ".png" - response = {"fp": fp, "fn": fn} + # Collect garbage + gc.collect() - return response + response = {"fp": fp, "fn": fn} - raise WCSException( - exception_text="KV Request not handled properly: " + str(args) - ) + return response def handle_wms_kv(self, args): if args["request"] == "GetCapabilities": - get_capabilities = wms_request_1_3_0.GetCapabilities() - try: - get_capabilities.load_from_kv(args) - get_capabilities.validate() - except: - logger.error("Failed to load and validate: ", exc_info=True) - raise WCSException(exception_text="Invalid arguments") - - wms_capabilities = self.wms_capabilities - - if args["base_url"]: - wms_capabilities.base_url = args["base_url"] - return wms_capabilities.to_xml() + return self.get_capabilities_wms(args) if args["request"] == "GetFeatureInfo": raise WCSException( @@ -198,70 +274,9 @@ def handle_wms_kv(self, args): ) if args["request"].lower() == "getlegendgraphic": - get_legend_graphic = wms_request.GetLegendGraphic() - try: - get_legend_graphic.load_from_kv(args) - get_legend_graphic.validate() - except: - logger.error("Failed to load and validate: ", exc_info=True) - raise WCSException(exception_text="Invalid arguments") - - coverage = self.get_coverage_from_id(get_legend_graphic.layer.value) - - fp = coverage.layer.get_legend_graphic(args) - - fn = coverage.identifier.split(".")[-1] + ".png" - - response = {"fp": fp, "fn": fn} - return response + return self.get_legend_graphic(args, wms_request) if args["request"].lower() == "getmap": + return self.get_map(args, wms_request) - get_map = wms_request.GetMap() - try: - get_map.load_from_kv(args) - get_map.validate() - except: - logger.error("Failed to load and validate: ", exc_info=True) - raise WCSException(exception_text="Invalid arguments") - - coverage = self.get_coverage_from_id(get_map.layer.value) - - # Make sure the request size is correct - if get_map.width == 0: - raise WCSException( - exception_code="InvalidParameterValue", - locator="VERSION", - exception_text="Grid coordinates x_size must be greater than 0", - ) - if get_map.height == 0: - raise WCSException( - exception_code="InvalidParameterValue", - locator="VERSION", - exception_text="Grid coordinates y_size must be greater than 0", - ) - if get_map.height * get_map.width > settings.MAX_GRID_COORDS_REQUEST_SIZE: - raise WCSException( - exception_code="InvalidParameterValue", - locator="VERSION", - exception_text="Grid coordinates x_size * y_size must be less than %d" % settings.MAX_GRID_COORDS_REQUEST_SIZE, - ) - - try: - fp = coverage.layer.get_map(args) - except: - logger.error("Failed to get_map from layer: ", exc_info=True) - raise WCSException(exception_text="Invalid arguments") - - fn = coverage.identifier.split(".")[-1] + ".png" - - # Collect garbage - gc.collect() - - response = {"fp": fp, "fn": fn} - - return response - - raise WCSException( - exception_text="KV Request not handled properly: " + str(args) - ) + raise WCSException(exception_text="KV Request not handled properly: " + str(args)) diff --git a/ogc/edr/__init__.py b/ogc/edr/__init__.py new file mode 100644 index 0000000..68fde54 --- /dev/null +++ b/ogc/edr/__init__.py @@ -0,0 +1,3 @@ +from .edr_routes import EdrRoutes + +__all__ = ["EdrRoutes"] diff --git a/ogc/edr/config/default.json b/ogc/edr/config/default.json new file mode 100644 index 0000000..6aea353 --- /dev/null +++ b/ogc/edr/config/default.json @@ -0,0 +1,47 @@ +{ + "server": { + "mimetype": "application/json; charset=UTF-8", + "encoding": "utf-8", + "language": "en-US", + "cors": true, + "pretty_print": true, + "limits": { + "default_items": 50, + "max_items": 1000, + "max_distance_x": 999999999999, + "max_distance_y": 999999999999, + "max_distance_units": "km", + "on_exceed": "error" + }, + "admin": false, + "map": { + "url": "https://tile.openstreetmap.org/{z}/{x}/{y}.png", + "attribution": "© OpenStreetMap contributors" + } + }, + "logging": { + "level": "ERROR" + }, + "metadata": { + "identification": { + "title": "EDR Server", + "description": "Environmental Data Retrieval Server", + "keywords": ["geospatial", "podpac"], + "keywords_type": "theme", + "terms_of_service": "http://www.apache.org/licenses/LICENSE-2.0", + "url": "https://github.com/creare-com/ogc" + }, + "license": { + "name": "Apache 2.0 license", + "url": "http://www.apache.org/licenses/LICENSE-2.0" + }, + "provider": { + "name": "Creare LLC", + "url": "https://github.com/creare-com" + }, + "contact": { + "name": "Creare LLC", + "url": "https://github.com/creare-com" + } + } +} diff --git a/ogc/edr/edr_api.py b/ogc/edr/edr_api.py new file mode 100644 index 0000000..7758e1e --- /dev/null +++ b/ogc/edr/edr_api.py @@ -0,0 +1,813 @@ +import json +import pyproj +import numpy as np +import pygeoapi.api +import pygeoapi.api.environmental_data_retrieval as pygeoedr +from http import HTTPStatus +from datetime import datetime, timezone +from typing import Tuple, List, Dict, Any, Union +from ogc import podpac as pogc +from pygeoapi.plugin import load_plugin +from pygeoapi.util import filter_dict_by_key_value, to_json, get_provider_by_type +from pygeoapi.api import API, APIRequest +from pygeoapi.linked_data import jsonldify +from .edr_provider import EdrProvider +from .. import settings + + +class EdrAPI: + """Used to modify the default responses before returning data to the user.""" + + CONFORMANCE_CLASSES = sorted( + { + "https://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core", + "https://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections", + "https://www.opengis.net/spec/ogcapi-edr-1/1.1/conf/core", + } + ) + SCHEMA_CLASS = "https://schemas.opengis.net/ogcapi/edr/1.1/openapi" + + @jsonldify + @staticmethod + def landing_page(api: API, request: APIRequest) -> Tuple[dict, int, str]: + """Provide the API landing page. + + Parameters + ---------- + api : API + The API which handles the request. + request : APIRequest + The request object. + + Returns + ------- + Tuple[dict, int, str] + Headers, HTTP Status, and Content returned as a tuple. + """ + return pygeoapi.api.landing_page(api, request) + + @staticmethod + def openapi_(api: API, request: APIRequest) -> Tuple[dict, int, str]: + """Provide the OpenAPI documentation. + + Parameters + ---------- + api : API + The API which handles the request. + request : APIRequest + The request object. + + Returns + ------- + Tuple[dict, int, str] + Headers, HTTP Status, and Content returned as a tuple. + """ + html_path = "openapi/redoc.html" if request._args.get("ui") == "redoc" else "openapi/swagger.html" + headers = request.get_response_headers(**api.api_headers) + + if request.format == pygeoapi.api.F_HTML: + data = {"openapi-document-path": f"{api.base_url}/openapi"} + content = pygeoapi.api.render_j2_template( + api.tpl_config, api.config["server"]["templates"], html_path, data, request.locale + ) + + return headers, HTTPStatus.OK, content + + headers["Content-Type"] = "application/vnd.oai.openapi+json;version=3.0" + + if isinstance(api.openapi, dict): + openapi = EdrAPI._openapi_update(api.openapi) + return headers, HTTPStatus.OK, to_json(openapi, api.pretty_print) + else: + return headers, HTTPStatus.OK, api.openapi + + @staticmethod + def conformance(api: API, request: APIRequest) -> Tuple[dict, int, str]: + """Provide the conformance definition. + + Parameters + ---------- + api : API + The API which handles the request. + request : APIRequest + The request object. + + Returns + ------- + Tuple[dict, int, str] + Headers, HTTP Status, and Content returned as a tuple. + """ + html_path = "conformance.html" + conformance = {"conformsTo": list(EdrAPI.CONFORMANCE_CLASSES)} + + headers = request.get_response_headers(**api.api_headers) + if request.format == pygeoapi.api.F_HTML: + content = pygeoapi.api.render_j2_template( + api.tpl_config, api.config["server"]["templates"], html_path, conformance, request.locale + ) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(conformance, api.pretty_print) + + @jsonldify + @staticmethod + def describe_collections(api: API, request: APIRequest, dataset: str | None = None) -> Tuple[dict, int, str]: + """Provide the collection/collections metadata. + + Overrides default functionality to append additional metadata to collections. + + Parameters + ---------- + api : API + The API which handles the request. + request : APIRequest + The request object. + dataset : str | None, optional + The dataset (collection) to be described or None for all collections, by default None. + + Returns + ------- + Tuple[dict, int, str] + Headers, HTTP Status, and Content returned as a tuple. + """ + headers, status, content = pygeoapi.api.describe_collections(api, request, dataset) + if request.format != pygeoapi.api.F_JSON or status != HTTPStatus.OK: + return headers, status, content + + collection_description = json.loads(content) + collection_configuration = filter_dict_by_key_value(api.config["resources"], "type", "collection") + collections = [collection_description] if dataset is not None else collection_description.get("collections", []) + + for collection in collections: + collection_id = collection["id"] + provider = get_provider_by_type(collection_configuration[collection_id]["providers"], "edr") + provider_plugin = load_plugin("provider", provider) + provider_parameters = provider_plugin.get_fields() + collection_layers = EdrProvider.get_layers(provider["base_url"], collection_id) + collection["extent"] = EdrAPI._generate_extents(collection_layers, None) + collection["output_formats"] = collection_configuration[collection_id].get("output_formats", []) + + collection_queryable = EdrProvider.is_collection_queryable(provider["base_url"], collection_id) + collection_without_queryables = EdrAPI._remove_query_metadata(collection) + collection["links"] = collection_without_queryables["links"] # Always remove unnecessary query links + if not collection_queryable: + collection["data_queries"] = collection_without_queryables["data_queries"] + + height_units = EdrAPI._vertical_units(collection_layers) + query_formats = collection_configuration[collection_id].get("query_formats", {}) + for query_type in collection["data_queries"]: + data_query_additions = { + "query_type": query_type, + **(query_formats.get(query_type) if query_formats.get(query_type) is not None else {}), + **({"height_units": height_units} if query_type == "cube" else {}), + } + variables = collection["data_queries"][query_type]["link"].get("variables", {}) + collection["data_queries"][query_type]["link"]["variables"] = variables | data_query_additions + + for parameter in collection["parameter_names"]: + collection["parameter_names"][parameter] = collection["parameter_names"][parameter] | { + "description": provider_parameters[parameter].get("description", "") + } + + return headers, status, to_json(collection_description, api.pretty_print) + + @staticmethod + def get_collection_edr_instances( + api: API, request: APIRequest, dataset: str, instance_id: str | None = None + ) -> Tuple[dict, int, str]: + """Provide the instance/instances metadata. + + Overrides default functionality to append additional metadata to instances. + + Parameters + ---------- + api : API + The API which handles the request. + request : APIRequest + The request object. + dataset : str + The dataset (collection) to be described. + instance_id : str | None, optional + The instance to be described or None for all instances, by default None. + + Returns + ------- + Tuple[dict, int, str] + Headers, HTTP Status, and Content returned as a tuple. + """ + headers, status, content = pygeoedr.get_collection_edr_instances(api, request, dataset, instance_id) + if request.format != pygeoapi.api.F_JSON or status != HTTPStatus.OK: + return headers, status, content + + instance_description = json.loads(content) + collection_configuration = filter_dict_by_key_value(api.config["resources"], "type", "collection") + provider = get_provider_by_type(collection_configuration[dataset]["providers"], "edr") + provider_plugin = load_plugin("provider", provider) + provider_parameters = provider_plugin.get_fields() + instances = [instance_description] if instance_id is not None else instance_description.get("instances", []) + collection_layers = EdrProvider.get_layers(provider["base_url"], dataset) + + for instance in instances: + instance_id = instance["id"] + instance["extent"] = EdrAPI._generate_extents(collection_layers, instance_id) + instance["output_formats"] = collection_configuration[dataset].get("output_formats", []) + + height_units = EdrAPI._vertical_units(collection_layers) + query_formats = collection_configuration[dataset].get("query_formats", {}) + for query_type in instance["data_queries"]: + data_query_additions = { + "query_type": query_type, + **(query_formats.get(query_type) if query_formats.get(query_type) is not None else {}), + **({"height_units": height_units} if query_type == "cube" else {}), + } + variables = instance["data_queries"][query_type]["link"].get("variables", {}) + instance["data_queries"][query_type]["link"]["variables"] = variables | data_query_additions + + instance["parameter_names"] = EdrAPI._instance_parameters( + collection_layers, provider_parameters, instance["id"] + ) + + return headers, status, to_json(instance_description, api.pretty_print) + + @staticmethod + def get_collection_edr_query( + api: API, + request: APIRequest, + dataset: str, + instance: str | None, + query_type: str, + location_id: str | None = None, + ) -> Tuple[dict, int, str]: + """Query the collection or instance. + + Parameters + ---------- + api : API + The API which handles the request. + request : APIRequest + The request object. + dataset : str + The dataset (collection) to be queried. + instance_id : str | None + The instance to be queried or None if querying a collection. + query_type : str + The query type. + location_id : str | None, optional + Location identifier for location queries or None, by default None. + + Returns + ------- + Tuple[dict, int, str] + Headers, HTTP Status, and Content returned as a tuple. + """ + return pygeoedr.get_collection_edr_query(api, request, dataset, instance, query_type, location_id) + + @staticmethod + def _temporal_extents(times: List[Union[np.datetime64, datetime]], trs: str | None) -> Dict[str, Any]: + """Get the temporal extents for the provided times and reference system. + + Parameters + ---------- + times : List[Union[np.datetime64, datetime]] + Times used to create the temporal extent. + trs : str | None + The reference system for the times. + + Returns + ------- + Dict[str, Any] + The temporal extent object or an empty dictionary.. + """ + iso_times = [] + for time in sorted(times): + dt = time.astype("datetime64[ms]").astype(datetime) if isinstance(time, np.datetime64) else time + if dt.tzinfo is None: + time_utc = dt.replace(tzinfo=timezone.utc) + else: + time_utc = dt.astimezone(timezone.utc) + iso_times.append(time_utc.isoformat().replace("+00:00", "Z")) + + return ( + { + "temporal": { + "interval": [iso_times[0], iso_times[-1]], + "values": iso_times, + "trs": trs, + } + } + if len(iso_times) > 0 + else {} + ) + + @staticmethod + def _vertical_extents(vertical_levels: List[float], vrs: str | None) -> Dict[str, Any]: + """Get the vertical extents for the provided levels and reference system. + + Parameters + ---------- + vertical_levels : List[float] + Vertical levels used to create the vertical extent. + vrs : str | None + The reference system for the vertical levels. + + Returns + ------- + Dict[str, Any] + The vertical extent object or an empty dictionary. + """ + return ( + { + "vertical": { + "interval": [vertical_levels[0], vertical_levels[-1]], + "values": vertical_levels, + "vrs": vrs, + } + } + if len(vertical_levels) > 0 + else {} + ) + + @staticmethod + def _spatial_extents(bbox: List[float], crs: str | None) -> Dict[str, Any]: + """Get the spatial extents for the provided bbox and reference system. + + Parameters + ---------- + bbox : List[float] + Bounding box used to create the spatial extent. + crs : str | None + The reference system for the bounding box. + + Returns + ------- + Dict[str, Any] + The spatial extent object. + """ + return { + "spatial": { + "bbox": bbox, + **({"crs": crs} if crs is not None else {}), + } + } + + @staticmethod + def _vertical_units(layers: List[pogc.Layer]) -> List[str]: + """Retrieve the vertical units for the layers. + + Parameters + ---------- + layer : List[pogc.Layer] + The layers from which to get the vertical units. + + Returns + ------- + List[str] + The vertical units available for the layers. + """ + vertical_units = set() + for layer in layers: + coordinates = layer.get_coordinates() + if coordinates is not None and coordinates.alt_units: + vertical_units.add(coordinates.alt_units) + + return list(vertical_units) + + @staticmethod + def _crs84_bounding_box(layer: pogc.Layer) -> Tuple[float, float, float, float]: + """Retrieve the bounding box for the layer with a default fallback. + + Parameters + ---------- + layer : pogc.Layer + The layer from which to get the bounding box coordinates. + + Returns + ------- + Tuple[float, float, float, float] + Lower-left longitude, lower-left latitude, upper-right longitude, upper-right latitude. + """ + try: + return ( + layer.grid_coordinates.LLC.lon, + layer.grid_coordinates.LLC.lat, + layer.grid_coordinates.URC.lon, + layer.grid_coordinates.URC.lat, + ) + except Exception: + crs_extents = settings.EDR_CRS[settings.crs_84_uri_format] + return (crs_extents["minx"], crs_extents["miny"], crs_extents["maxx"], crs_extents["maxy"]) + + @staticmethod + def _generate_extents(layers: List[pogc.Layer], instance: str | None) -> Dict[str, Any]: + """Generate the extents for the provided layers. + + Parameters + ---------- + layers : List[pogc.Layer] + The layers for temporal and spatial extent generation. + instance : str | None + The instance for extent generation or None for collection extents. + + Returns + ------- + Dict[str, Any] + The extents dictionary for the provided layers. + """ + bbox = [] + crs = pyproj.CRS(settings.crs_84_uri_format).to_wkt() + + time_range = set() + vertical_range = set() + + for layer in layers: + coordinates = layer.get_coordinates() + if coordinates is not None: + llc_lon_tmp, llc_lat_tmp, urc_lon_tmp, urc_lat_tmp = EdrAPI._crs84_bounding_box(layer) + if len(bbox) != 4: + bbox = [llc_lon_tmp, llc_lat_tmp, urc_lon_tmp, urc_lat_tmp] + else: + llc_lon = min(bbox[0], llc_lon_tmp) + llc_lat = min(bbox[1], llc_lat_tmp) + urc_lon = max(bbox[2], urc_lon_tmp) + urc_lat = max(bbox[3], urc_lat_tmp) + bbox = [llc_lon, llc_lat, urc_lon, urc_lat] + + if "alt" in coordinates.udims: + vertical_range.update(coordinates["alt"].coordinates) + + if "time" in coordinates.udims: + if instance in layer.time_instances() and settings.EDR_TIME_INSTANCE_DIMENSION in coordinates.udims: + instance_datetime = np.datetime64(instance) + instance_coordinates = coordinates.select( + {settings.EDR_TIME_INSTANCE_DIMENSION: [instance_datetime, instance_datetime]} + ) + selected_time_coordinates = instance_coordinates["time"].coordinates + time_range.update(selected_time_coordinates) + elif not instance and settings.EDR_TIME_INSTANCE_DIMENSION not in coordinates.udims: + time_range.update(coordinates["time"].coordinates) + + sorted_time_range = sorted(time_range) + sorted_vertical_range = sorted(vertical_range) + + return { + **(EdrAPI._spatial_extents(bbox, crs)), + **(EdrAPI._temporal_extents(sorted_time_range, "https://www.opengis.net/def/uom/ISO-8601/0/Gregorian")), + **(EdrAPI._vertical_extents(sorted_vertical_range, "https://www.opengis.net/def/uom/EPSG/0/9001")), + } + + @staticmethod + def _instance_parameters( + collection_layers: List[pogc.Layer], provider_parameters: Dict[str, Any], instance: str + ) -> Dict[str, Any]: + """Get the parameter metadata for the specific instance provided. + + Parameters + ---------- + collection_layers : List[pogc.Layer] + The layers available in the collection. + provider_parameters: Dict[str, Any] + The metadata for all available parameters in the collection from provider fields. + instance: str + The instance to determine parameters for. + + Returns + ------- + Dict[str, Any] + The metadata for available parameters in the instance. + """ + instance_parameters = {} + for key, value in provider_parameters.items(): + layer = next((layer for layer in collection_layers if layer.identifier == key), None) + if layer is not None and instance in layer.time_instances(): + instance_parameters[key] = { + "id": key, + "type": "Parameter", + "name": value["title"], + "observedProperty": { + "label": {"id": key, "en": value["title"]}, + }, + "description": value["description"], + "unit": { + "label": {"en": value["title"]}, + "symbol": { + "value": value["x-ogc-unit"], + "type": "http://www.opengis.net/def/uom/UCUM/", + }, + }, + } + return instance_parameters + + @staticmethod + def _remove_query_metadata(data: Dict[str, Any]) -> Dict[str, Any]: + """Remove metadata from the dictionary which relates to query types such as position and cube. + + Parameters + ---------- + data : Dict[str, Any] + The dictionary to remove query metadata from. + + Returns + ------- + Dict[str, Any] + The updated dictionary with metadata removed. + """ + filtered_data = data.copy() + data_queries = data.get("data_queries", {}) + filtered_data["data_queries"] = {"instances": data_queries.get("instances")} + filtered_data["links"] = [] + for link in data["links"]: + if link.get("rel") != "data" or "/instances" in link.get("href", ""): + filtered_data["links"].append(link) + + return filtered_data + + @staticmethod + def _openapi_update(api: Dict[str, Any]) -> Dict[str, Any]: + """Update the default OpenAPI definition to a custom format. + + Parameters + ---------- + api : Dict[str, Any] + The OpenAPI definition to be updated. + + Returns + ------- + Dict[str, Any] + The customized OpenAPI definition. + """ + server_tag = "Server" + collection_tag = "Collection Information" + instance_tag = "Instance Information" + query_tag = "Query" + + resource_not_found_error = {"description": "Resource not found."} + internal_application_error = { + "description": "Internal application error", + "content": { + "application/xml": { + "schema": { + "type": "object", + "format": "xml", + "xml": {"name": "ExceptionReport"}, + }, + "example": ( + '' + "" + '' + "Internal application error" + "" + "" + ), + } + }, + } + + query_base_parameters = [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/crs.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/datetime.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/f.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/parameter-name.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/z.yaml"}, + ] + query_responses = { + "200": { + "$ref": f"{EdrAPI.SCHEMA_CLASS}/responses/queries/200.yaml", + }, + "400": { + "$ref": f"{EdrAPI.SCHEMA_CLASS}/responses/queries/400.yaml", + }, + "404": resource_not_found_error, + "default": internal_application_error, + } + + openapi = {} + openapi["openapi"] = api.get("openapi") + openapi["info"] = api.get("info") + openapi["servers"] = api.get("servers") + openapi["tags"] = [server_tag, collection_tag, instance_tag, query_tag] + openapi["paths"] = { + "/": { + "get": { + "summary": "Landing Page", + "description": "Landing page of the API.", + "tags": [server_tag], + "operationId": "getLandingPage", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/f.yaml"}, + ], + "responses": { + "200": { + "$ref": f"{EdrAPI.SCHEMA_CLASS}/responses/core/landingPage.yaml", + }, + "default": internal_application_error, + }, + }, + }, + "/api": { + "get": { + "summary": "Capabilities of the API.", + "description": "API", + "tags": [server_tag], + "operationId": "getApi", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/f.yaml"}, + ], + "responses": { + "200": { + "description": "API capabilities", + }, + "default": internal_application_error, + }, + }, + }, + "/conformance": { + "get": { + "summary": "Conformance classes defining standard compliance of the API.", + "description": "Conformance Classes", + "tags": [server_tag], + "operationId": "getConformance", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/f.yaml"}, + ], + "responses": { + "200": { + "$ref": f"{EdrAPI.SCHEMA_CLASS}/responses/core/conformance.yaml", + }, + "default": internal_application_error, + }, + }, + }, + "/collections": { + "get": { + "summary": "Collection information for all available collections.", + "description": "Collections", + "tags": [collection_tag], + "operationId": "getCollections", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/f.yaml"}, + ], + "responses": { + "200": { + "$ref": f"{EdrAPI.SCHEMA_CLASS}/responses/collections/collections.yaml", + }, + "default": internal_application_error, + }, + }, + }, + "/collections/{collectionId}": { + "get": { + "summary": "Collection information for a single collection.", + "description": "Collection", + "tags": [collection_tag], + "operationId": "getCollection", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/f.yaml"}, + ], + "responses": { + "200": { + "$ref": f"{EdrAPI.SCHEMA_CLASS}/responses/collections/collection.yaml", + }, + "404": resource_not_found_error, + "default": internal_application_error, + }, + }, + }, + "/collections/{collectionId}/area": { + "get": { + "summary": "Query a collection for an area.", + "description": "Collection Area Query", + "tags": [query_tag], + "operationId": "getCollectionArea", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/areaCoords.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/resolution-x.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/resolution-y.yaml"}, + *query_base_parameters, + ], + "responses": query_responses, + }, + }, + "/collections/{collectionId}/cube": { + "get": { + "summary": "Query a collection for a cube.", + "description": "Collection Cube Query", + "tags": [query_tag], + "operationId": "getCollectionCube", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/bbox.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/resolution-x.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/resolution-y.yaml"}, + *query_base_parameters, + ], + "responses": query_responses, + }, + }, + "/collections/{collectionId}/position": { + "get": { + "summary": "Query a collection for a position.", + "description": "Collection Position Query", + "tags": [query_tag], + "operationId": "getCollectionPosition", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/positionCoords.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + *query_base_parameters, + ], + "responses": query_responses, + }, + }, + "/collections/{collectionId}/instances/": { + "get": { + "summary": "Instance information for all available instances in a collection.", + "description": "Collection Instances", + "tags": [instance_tag], + "operationId": "getCollectionInstances", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/f.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + ], + "responses": { + "200": { + "$ref": f"{EdrAPI.SCHEMA_CLASS}/responses/queries/instances.yaml", + }, + "404": resource_not_found_error, + "default": internal_application_error, + }, + }, + }, + "/collections/{collectionId}/instances/{instanceId}": { + "get": { + "summary": "Instance information for a single instance in a collection.", + "description": "Collection Instance", + "tags": [instance_tag], + "operationId": "getCollectionInstance", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/f.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/queries/instanceId.yaml"}, + ], + "responses": { + "200": { + "$ref": f"{EdrAPI.SCHEMA_CLASS}/responses/queries/instances.yaml", + }, + "404": resource_not_found_error, + "default": internal_application_error, + }, + }, + }, + "/collections/{collectionId}/instances/{instanceId}/area": { + "get": { + "summary": "Query a collection instance for an area.", + "description": "Instance Area Query", + "tags": [query_tag], + "operationId": "getCollectionInstanceArea", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/areaCoords.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/queries/instanceId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/resolution-x.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/resolution-y.yaml"}, + *query_base_parameters, + ], + "responses": query_responses, + }, + }, + "/collections/{collectionId}/instances/{instanceId}/cube": { + "get": { + "summary": "Query a collection instance for a cube.", + "description": "Instance Cube Query", + "tags": [query_tag], + "operationId": "getCollectionInstanceCube", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/bbox.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/queries/instanceId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/resolution-x.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/resolution-y.yaml"}, + *query_base_parameters, + ], + "responses": query_responses, + }, + }, + "/collections/{collectionId}/instances/{instanceId}/position": { + "get": { + "summary": "Query a collection instance for a position.", + "description": "Instance Position Query", + "tags": [query_tag], + "operationId": "getCollectionInstancePosition", + "parameters": [ + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/core/positionCoords.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/collections/collectionId.yaml"}, + {"$ref": f"{EdrAPI.SCHEMA_CLASS}/parameters/queries/instanceId.yaml"}, + *query_base_parameters, + ], + "responses": query_responses, + }, + }, + } + + return openapi diff --git a/ogc/edr/edr_config.py b/ogc/edr/edr_config.py new file mode 100644 index 0000000..2d9716f --- /dev/null +++ b/ogc/edr/edr_config.py @@ -0,0 +1,147 @@ +import os +import json +import logging +from typing import List, Dict, Any +from ogc import podpac as pogc +from .. import settings + + +class EdrConfig: + """Defines the configuration for the pygeoapi based server. + + This configuration is used to replace the typical YAML based configurations in order to provide dynamic properties. + """ + + @staticmethod + def get_configuration(base_url: str, layers: List[pogc.Layer]) -> Dict[str, Any]: + """Generate the configuration for the API. + + Parameters + ---------- + base_url : str + The base URL for the EDR endpoints. + layers : List[pogc.Layer] + The layers which define the data sources for the EDR server. + + Returns + ------- + Dict[str, Any] + The configuration for the API as a dictionary. + """ + configuration_path = settings.EDR_CONFIGURATION_PATH + if configuration_path is None: + configuration_path = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/config/default.json")) + + configuration = {} + with open(configuration_path) as f: + configuration = json.load(f) + + # Add default static files with an absolute path + server = configuration.get("server", {}) + configuration["server"] = server | { + "templates": { + "path": os.path.abspath(os.path.join(os.path.dirname(__file__) + "/templates/")), + "static": os.path.abspath(os.path.join(os.path.dirname(__file__) + "/static/")), + } + } + configuration["server"]["url"] = base_url + + # Add the data resources and provider information + resources = configuration.get("resources", {}) + configuration["resources"] = resources | EdrConfig._resources_definition(base_url, layers) + + # Force the log level based on the configuration as it is loaded, otherwise it is ignored + if configuration.get("logging", {}).get("level"): + api_logger = logging.getLogger("pygeoapi") + api_logger.setLevel(configuration["logging"]["level"]) + + return configuration + + @staticmethod + def _resources_definition(base_url: str, layers: List[pogc.Layer]) -> Dict[str, Any]: + """Define resource related data for the configuration. + + The resources dictionary holds the information needed to generate the collections. + Each group is mapped to a collection with the layers in the group forming the collection parameters. + The custom provider is specified with a data value of the group name. + This allows for the provider to generate the collection data for each group. + + Parameters + ---------- + base_url : str + The base URL used as an identifier for the given layers. + layers : List[pogc.Layer] + The layers which define the data sources for the EDR server. + + Returns + ------- + Dict[str, Any] + The resources configuration for the API as a dictionary. + """ + + resources = {} + groups = {layer.group for layer in layers} + + # Generate collection resources based on groups + for group_name in groups: + resource = { + group_name: { + "type": "collection", + "visibility": "default", + "title": group_name, + "description": f"Collection of data related to {group_name}", + "keywords": ["podpac"], + "extents": { + "spatial": { + "bbox": [-180, -90, 180, 90], # Placeholder extents + "crs": settings.crs_84_uri_format, + } + }, + "output_formats": list({item for values in settings.EDR_QUERY_FORMATS.values() for item in values}), + "query_formats": EdrConfig.data_query_formats(), + "providers": [ + { + "type": "edr", + "default": True, + "name": "ogc.edr.edr_provider.EdrProvider", + "data": group_name, + "base_url": base_url, + "crs": list(settings.EDR_CRS.keys()), + "format": { + "name": settings.GEOTIFF, + "mimetype": "image/tiff", + }, + } + ], + "formatters": [ + { + "name": "ogc.edr.edr_formatter.GeoTiffFormatter", + "mimetype": "image/tiff", + }, + { + "name": "ogc.edr.edr_formatter.CoverageJsonFormatter", + "mimetype": "application/prs.coverage+json", + }, + ], + } + } + resources.update(resource) + + return resources + + @staticmethod + def data_query_formats() -> Dict[str, Any]: + """Get data related to the available query output formats and the default format. + + Returns + ------- + Dict[str, Any] + Query format data for each query type. + """ + query_formats = {} + for query_type, formats in settings.EDR_QUERY_FORMATS.items(): + query_formats[query_type] = { + "output_formats": formats, + "default_output_format": settings.EDR_QUERY_DEFAULTS.get(query_type), + } + return query_formats diff --git a/ogc/edr/edr_formatter.py b/ogc/edr/edr_formatter.py new file mode 100644 index 0000000..104d570 --- /dev/null +++ b/ogc/edr/edr_formatter.py @@ -0,0 +1,69 @@ +from pygeoapi.formatter.base import BaseFormatter +from pygeoapi.util import to_json + + +class BaseEdrFormatter(BaseFormatter): + """Base formatter for EDR data.""" + + def __init__(self, formatter_def: dict): + """Initialize the formatter. + + Parameters + ---------- + formatter_def : dict + The formatter definition. + """ + + super().__init__(formatter_def) + self.mimetype = formatter_def["mimetype"] + + def write(self, options: dict | None = None, data: dict | None = None) -> str: + """Generate data in the specified format. + + Parameters + ---------- + options : dict, optional + Formatting options, by default None. + data : dict | None, optional + Dictionary representation of the data, by default None. + + Returns + ------- + str + String representation of the data. + """ + return to_json(data, True) if data is not None else "" + + +class GeoTiffFormatter(BaseEdrFormatter): + """Formatter for GeoTIFF data. Defined for format link information to be populated.""" + + def __init__(self, formatter_def: dict): + """Initialize the formatter. + + Parameters + ---------- + formatter_def : dict + The formatter definition. + """ + + super().__init__(formatter_def) + self.f = "geotiff" + self.extension = "tiff" + + +class CoverageJsonFormatter(BaseEdrFormatter): + """Formatter for CoverageJSON data. Defined for format link information to be populated.""" + + def __init__(self, formatter_def: dict): + """Initialize the formatter. + + Parameters + ---------- + formatter_def : dict + The formatter definition. + """ + + super().__init__(formatter_def) + self.f = "coveragejson" + self.extension = "json" diff --git a/ogc/edr/edr_provider.py b/ogc/edr/edr_provider.py new file mode 100644 index 0000000..4167200 --- /dev/null +++ b/ogc/edr/edr_provider.py @@ -0,0 +1,1122 @@ +import json +import tempfile +import numpy as np +import zipfile +import pyproj +from datetime import datetime +from collections import defaultdict +from typing import List, Dict, Tuple, Any +from pyproj.exceptions import CRSError +from shapely.geometry.base import BaseGeometry +from pygeoapi.provider.base import ProviderConnectionError, ProviderInvalidQueryError +from pygeoapi.provider.base_edr import BaseEDRProvider +from ogc import podpac as pogc +import podpac + +from .. import settings + + +class EdrProvider(BaseEDRProvider): + """Custom provider to be used with layer data sources.""" + + _layers_dict = defaultdict(list) + _extra_args = defaultdict() + + @classmethod + def set_extra_query_args(cls, args: Dict[str, Any]): + """Set the extra arguments which will be available to the provider on a request. + + Parameters + ---------- + args : Dict[str, Any] + The extra arguments which are not included in the default requests. + """ + cls._extra_args = args + + @classmethod + def set_layers(cls, base_url: str, layers: List[pogc.Layer]): + """Set the layer resources which will be available to the provider. + + Parameters + ---------- + base_url : str + The base URL that the layers are available on. + layers : List[pogc.Layer] + The layers which the provider will have access to. + """ + cls._layers_dict[base_url] = layers + + @classmethod + def get_layers(cls, base_url: str, group: str | None = None) -> List[pogc.Layer]: + """Get the layer resources for a specific base URL and group. + + Parameters + ---------- + base_url : str + The base URL for the layers. + group : str | None, optional + Optional group to filter layers, by default None. + + + Returns + ------- + List[pogc.Layer] + The layers associated with the base URL. + """ + layers = cls._layers_dict.get(base_url, []) + if group is not None: + return [layer for layer in layers if layer.group.lower() == group.lower()] + else: + return layers + + @classmethod + def is_collection_queryable(cls, base_url: str, group: str) -> bool: + """Determine whether a collection contains directly queryable data or is only queryable through instances. + + Parameters + ---------- + base_url : str + The base URL for the layers. + group : str + Collection to check if direct querying is possible. + + Returns + ------- + bool + True if the collection can be queried directly, false otherwise. + """ + layers = cls.get_layers(base_url, group) + for layer in layers: + coordinates = layer.get_coordinates() + if coordinates is not None and settings.EDR_TIME_INSTANCE_DIMENSION not in coordinates.udims: + return True + return False + + def __init__(self, provider_def: Dict[str, Any]): + """Construct the provider using the provider definition. + + Parameters + ---------- + provider_def : Dict[str, Any] + The provider configuration definition. + + Raises + ------ + ProviderConnectionError + Raised if the specified collection is not found within any layers. + ProviderConnectionError + Raised if the provider does not specify any base URL. + """ + super().__init__(provider_def) + collection_id = provider_def.get("data") + if collection_id is None: + raise ProviderConnectionError("Data not found.") + + self.collection_id = str(collection_id) + + self.base_url = provider_def.get("base_url", "") + if not self.base_url: + raise ProviderConnectionError("Valid URL identifier not found for the data.") + + @property + def parameters(self) -> Dict[str, pogc.Layer]: + """The parameters which are defined in a given collection. + + The parameters map to the layers which are a part of the group, with keys of the layer identifiers. + + Returns + ------- + Dict[str, pogc.Layer] + The parameters as a dictionary of layer identifiers and layer objects. + """ + return {layer.identifier: layer for layer in self.get_layers(self.base_url, self.collection_id)} + + def handle_query(self, requested_coordinates: podpac.Coordinates, **kwargs): + """Handle the requests to the EDR server at the specified requested coordinates. + The coordinates are expected to be latitude and longitude values determined by the specific query function. + + Parameters + ---------- + requested_coordinates : podpac.Coordinates + The coordinates for evaluation, it is expected that the coordinates passed in only hold lat and lon. + instance : str + The time instance for the request. + select_properties : List[str] + The selected properties (parameters) for the request. + format_ : str + The requested output format of the data. + datetime_ : str + The requested datetime/datetimes for data retrieval. + z : str + The requested vertical level/levels for data retrieval. + resolution-x : str + The number of requested data points, as a string, in the x-direction. + resolution-y : str + The number of requested data points, as a string, in the y-direction. + + Returns + ------- + Any + Coverage data as a dictionary of CoverageJSON or native format. + + Raises + ------ + ProviderInvalidQueryError + Raised if an invalid instance is provided. + ProviderInvalidQueryError + Raised if an invalid parameter is provided. + ProviderInvalidQueryError + Raised if a datetime string is provided but cannot be interpreted. + ProviderInvalidQueryError + Raised if an altitude string is provided but cannot be interpreted. + ProviderInvalidQueryError + Raised if a GeoTIFF request includes multiple time bands. + ProviderInvalidQueryError + Raised if a GeoTIFF request includes multiple vertical bands. + ProviderInvalidQueryError + Raised if native coordinates could not be found. + ProviderInvalidQueryError + Raised if the request queries for native coordinates exceeding the max allowable size. + ProviderInvalidQueryError + Raised if no parameters could not be evaluated. + """ + instance = kwargs.get("instance") + requested_parameters = kwargs.get("select_properties") + output_format = kwargs.get("format_") + datetime_arg = kwargs.get("datetime_") + z_arg = kwargs.get("z") + resolution_x = kwargs.get("resolution-x") + resolution_y = kwargs.get("resolution-y") + + instance = self.validate_instance(instance) + requested_parameters = self.validate_parameters(requested_parameters) + resolution_x, resolution_y = self.validate_resolution(resolution_x, resolution_y) + + crs = self.interpret_crs(requested_coordinates.crs) + available_times = self.get_datetimes(list(self.parameters.values()), instance) + available_altitudes = self.get_altitudes(list(self.parameters.values())) + time_coords = self.interpret_time_coordinates( + available_times, datetime_arg, instance, requested_coordinates.crs + ) + altitude_coords = self.interpret_altitude_coordinates(available_altitudes, z_arg, requested_coordinates.crs) + + if time_coords is not None: + self.check_query_condition( + any(dimension > 1 for dimension in time_coords.shape) and output_format == settings.GEOTIFF.lower(), + "GeoTIFF output currently only supports single time requests.", + ) + requested_coordinates = podpac.coordinates.merge_dims([time_coords, requested_coordinates]) + if altitude_coords is not None: + self.check_query_condition( + len(altitude_coords["alt"].coordinates) > 1 and output_format == settings.GEOTIFF.lower(), + "GeoTIFF output currently only supports single altitude requests.", + ) + requested_coordinates = podpac.coordinates.merge_dims([altitude_coords, requested_coordinates]) + + # Handle defining native coordinates for the query, these should match between each layer + coordinates = next(iter(requested_parameters.values())).get_coordinates() + self.check_query_condition(coordinates is None, "Native coordinates not found.") + resolution_lon, resolution_lat = self.crs_converter(resolution_x, resolution_y, crs) + requested_native_coordinates = self.get_native_coordinates( + requested_coordinates, coordinates, resolution_lat, resolution_lon + ) + + self.check_query_condition( + bool(requested_native_coordinates.size > settings.MAX_GRID_COORDS_REQUEST_SIZE), + "Coordinates size must be less than %d" % settings.MAX_GRID_COORDS_REQUEST_SIZE, + ) + + dataset = {} + for requested_parameter, layer in requested_parameters.items(): + units_data_array = EdrProvider.evaluate_layer(requested_native_coordinates, layer) + if units_data_array is not None: + dataset[requested_parameter] = units_data_array + + self.check_query_condition(len(dataset) == 0, "No matching parameters found.") + + if output_format == settings.COVERAGE_JSON.lower(): + layers = self.get_layers(self.base_url, self.collection_id) + return self.to_coverage_json(layers, dataset, self.collection_id, crs) + + return self.to_geotiff_response(dataset, self.collection_id) + + def position(self, **kwargs): + """Handles requests for the position query type. + + Parameters + ---------- + wkt : shapely.geometry + WKT geometry + format_ : str + The requested output format of the data. + + Returns + ------- + Any + Coverage data as a dictionary of CoverageJSON or native format. + + Raises + ------ + ProviderInvalidQueryError + Raised if an invalid output format is provided. + ProviderInvalidQueryError + Raised if the wkt string is not provided. + ProviderInvalidQueryError + Raised if the wkt string is an unknown type. + """ + lat, lon = [], [] + wkt = kwargs.get("wkt") + crs = self._extra_args.get("crs") + crs = EdrProvider.interpret_crs(crs) + kwargs["format_"] = self.validate_output_format(kwargs["format_"], "position") + + if not isinstance(wkt, BaseGeometry): + msg = "Invalid WKT string provided for the position query." + raise ProviderInvalidQueryError(msg, user_msg=msg) + elif wkt.geom_type == "Point": + lon, lat = EdrProvider.crs_converter([wkt.x], [wkt.y], crs) + else: + msg = "Unknown WKT string type for the position query (use Point)." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + requested_coordinates = podpac.Coordinates([lat, lon], dims=["lat", "lon"], crs=crs) + + return self.handle_query(requested_coordinates, **kwargs) + + def cube(self, **kwargs): + """Handles requests for the cube query type. + + Parameters + ---------- + bbox : List[float] + Bbox geometry (for cube queries) + format_ : str + The requested output format of the data. + + Returns + ------- + Any + Coverage data as a dictionary of CoverageJSON or native format. + + Raises + ------ + ProviderInvalidQueryError + Raised if an invalid output format is provided. + ProviderInvalidQueryError + Raised if the bounding box is invalid. + """ + bbox = kwargs.get("bbox") + crs = self._extra_args.get("crs") + crs = EdrProvider.interpret_crs(crs) + kwargs["format_"] = self.validate_output_format(kwargs["format_"], "cube") + kwargs["resolution-x"] = self._extra_args.get("resolution-x") + kwargs["resolution-y"] = self._extra_args.get("resolution-y") + + if not isinstance(bbox, List) or (len(bbox) != 4 and len(bbox) != 6): + msg = ( + "Invalid bounding box provided, " + "expected bounding box of (minx, miny, maxx, maxy) or (minx, miny, minz, maxx, maxy, maxz)." + ) + raise ProviderInvalidQueryError(msg, user_msg=msg) + + if len(bbox) == 6: + xmin, ymin, zmin, xmax, ymax, zmax = bbox + # Set the z argument if not specified using a closed interval from the bounding box data + if kwargs.get("z") is None: + kwargs["z"] = f"{zmin}/{zmax}" + else: + xmin, ymin, xmax, ymax = bbox + + lon, lat = EdrProvider.crs_converter([xmin, xmax], [ymin, ymax], crs) + requested_coordinates = podpac.Coordinates([lat, lon], dims=["lat", "lon"], crs=crs) + + return self.handle_query(requested_coordinates, **kwargs) + + def area(self, **kwargs): + """Handles requests for the area query type. + + Parameters + ---------- + wkt : shapely.geometry + WKT geometry + format_ : str + The requested output format of the data. + + Returns + ------- + Any + Coverage data as a dictionary of CoverageJSON or native format. + + Raises + ------ + ProviderInvalidQueryError + Raised if an invalid output format is provided. + ProviderInvalidQueryError + Raised if the wkt string is not provided. + ProviderInvalidQueryError + Raised if the wkt string is an unknown type. + """ + lat, lon = [], [] + wkt = kwargs.get("wkt") + crs = self._extra_args.get("crs") + crs = EdrProvider.interpret_crs(crs) + kwargs["format_"] = self.validate_output_format(kwargs["format_"], "area") + kwargs["resolution-x"] = self._extra_args.get("resolution-x") + kwargs["resolution-y"] = self._extra_args.get("resolution-y") + + if not isinstance(wkt, BaseGeometry): + msg = "Invalid WKT string provided for the area query." + raise ProviderInvalidQueryError(msg, user_msg=msg) + elif wkt.geom_type == "Polygon": + lon, lat = EdrProvider.crs_converter(wkt.exterior.xy[0], wkt.exterior.xy[1], crs) + else: + msg = "Unknown WKT string type for the area query (use Polygon)." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + requested_coordinates = podpac.Coordinates([lat, lon], dims=["lat", "lon"], crs=crs) + + return self.handle_query(requested_coordinates, **kwargs) + + def get_instance(self, instance: str) -> str | None: + """Validate instance identifier. + + Parameters + ---------- + instance : str + The instance identifier to validate. + + Returns + ------- + str + The instance identifier if valid, otherwise returns None. + """ + return instance if instance in self.instances() else None + + def instances(self, **kwargs) -> List[str]: + """The instances in the collection. + + Returns + ------- + List[str] + The instances available in the collection. + """ + instances = set() + collection_layers = self.get_layers(self.base_url, self.collection_id) + for layer in collection_layers: + instances.update(layer.time_instances()) + return list(instances) + + def get_fields(self) -> Dict[str, Any]: + """The observed property fields (parameters) in the collection. + + Returns + ------- + Dict[str, Any] + The fields based on the available parameters. + """ + fields = {} + for parameter_key, layer in self.parameters.items(): + fields[parameter_key] = { + "type": "number", + "title": parameter_key, + "description": layer.abstract, + "x-ogc-unit": layer.get_units(), + } + return fields + + def validate_output_format(self, output_format: str | None, query_type: str) -> str: + """Validate the output format for a query. + + If None provided, return the default. + If the provided output format is invalid, raise an error. + + Parameters + ---------- + output_format : str | None + The specified output format which needs to be validated. + query_type: str + The query type to validate output formats against. + + Returns + ------- + str + Output format string. + + Raises + ------ + ProviderInvalidQueryError + Raised if the provided output format is invalid. + """ + if output_format is None: + return settings.EDR_QUERY_DEFAULTS.get(query_type, "") + + if output_format.lower() not in [key.lower() for key in settings.EDR_QUERY_FORMATS.get(query_type, [])]: + msg = ( + f"Invalid format provided, expected one of {', '.join(settings.EDR_QUERY_FORMATS.get(query_type, []))}" + ) + raise ProviderInvalidQueryError(msg, user_msg=msg) + + return output_format.lower() + + def validate_resolution(self, resolution_x: str | None, resolution_y: str | None) -> Tuple[int, int]: + """Validate the resolutions and return the values as integers. + + If no resolution is provided in a specific direction a zero value should be used to indicate native resolution. + + Parameters + ---------- + resolution_x : str | None + The resolution in the x-direction or None. + resolution_y : str | None + The resolution in the y-direction or None. + + Returns + ------- + Tuple[int, int] + Resolution x and y as integers. + + Raises + ------ + ProviderInvalidQueryError + Raised if either of the provided resolutions is invalid. + """ + valid_resolutions = True + validated_resolution_x = 0 + validated_resolution_y = 0 + try: + validated_resolution_x = int(0 if resolution_x is None else resolution_x) + validated_resolution_y = int(0 if resolution_y is None else resolution_y) + valid_resolutions = validated_resolution_x >= 0 and validated_resolution_y >= 0 + except ValueError: + valid_resolutions = False + + if not valid_resolutions: + msg = "Invalid resolution provided, expected positive integer." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + return validated_resolution_x, validated_resolution_y + + def validate_instance(self, instance: str | None) -> str | None: + """Validate the instance for a query. + + If None provided, the collection is being queried. + If the instance is invalid, raise an error. + + Parameters + ---------- + instance : str | None + The instance which needs to be validated. + + Returns + ------- + str | None + The validated instance or None if the collection is being queried. + + Raises + ------ + ProviderInvalidQueryError + Raised if the provided instance is invalid. + """ + if instance is None: + return None + + if instance not in self.instances(): + msg = "Invalid instance provided." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + return instance + + def validate_parameters(self, parameters: List[str] | None) -> Dict[str, pogc.Layer]: + """Validate the parameters for a query. + + If None provided or an list is empty, return all parameters. + If the provided parameter list is invalid, raise an error. + + Parameters + ---------- + parameters : List[str] | None + The specified parameters for a query. + + Returns + ------- + Dict[str, pogc.Layer] + The validated parameters dictionary containing associated layers. + + Raises + ------ + ProviderInvalidQueryError + Raised if the provided parameters are invalid. + """ + if parameters is None or len(parameters) == 0: + return self.parameters + + parameters_lower = [param.lower() for param in parameters] + parameters_filtered = { + key: value + for key, value in self.parameters.items() + if key.lower() in parameters_lower and value is not None + } + if len(parameters_filtered) != len(parameters): + msg = "Invalid parameters provided." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + return parameters_filtered + + @staticmethod + def evaluate_layer(requested_coordinates: podpac.Coordinates, layer: pogc.Layer) -> podpac.UnitsDataArray | None: + """Evaluate a layer using the requested coordinates. + + Parameters + ---------- + requested_coordinates : podpac.Coordinates + The requested coordinates for the evaluation. + layer : pogc.Layer + The layer to evaluate. + + Returns + ------- + podpac.UnitsDataArray + The units data array returned from evaluation or None if the node was not found. + """ + coordinates = layer.get_coordinates() + layer_requested_coordinates = requested_coordinates + units_data_array = None + if coordinates is None: + return units_data_array + + layer_has_instances = settings.EDR_TIME_INSTANCE_DIMENSION in coordinates.udims + request_has_instances = settings.EDR_TIME_INSTANCE_DIMENSION in requested_coordinates.udims + if layer_has_instances ^ request_has_instances: + return units_data_array + + if "time" not in coordinates.udims: + layer_requested_coordinates = layer_requested_coordinates.udrop(["time"], ignore_missing=True) + if layer.node is not None: + units_data_array = layer.node.eval(layer_requested_coordinates) + units_data_array = units_data_array.unstack() + if settings.EDR_TIME_INSTANCE_DIMENSION in units_data_array.dims: + # Use scalar selection to drop the extra dimension + units_data_array = units_data_array.sel( + { + settings.EDR_TIME_INSTANCE_DIMENSION: layer_requested_coordinates[ + settings.EDR_TIME_INSTANCE_DIMENSION + ].coordinates[0] + } + ) + units_data_array = units_data_array.drop_vars({settings.EDR_TIME_INSTANCE_DIMENSION}) + if units_data_array.attrs.get("bounds", None): + filtered_bounds = { + coord: bnd + for coord, bnd in units_data_array.attrs["bounds"].items() + if coord in units_data_array.coords.dims + } + units_data_array.attrs["bounds"] = filtered_bounds + + return units_data_array + + @staticmethod + def get_altitudes(layers: List[pogc.Layer]) -> List[float]: + """The list of available altitudes for the provided layers. + + Parameters + ---------- + layers : List[pogc.Layer] + The list of layers to determine altitudes for. + + Returns + ------- + List[float] + Available altitudes for the providers layers. + """ + + available_altitudes = set() + for layer in layers: + coordinates = layer.get_coordinates() + if coordinates is not None and "alt" in coordinates.udims: + available_altitudes.update(coordinates["alt"].coordinates) + + return list(available_altitudes) + + @staticmethod + def get_datetimes(layers: List[pogc.Layer], instance_time: str | None) -> List[np.datetime64]: + """The list of available times for the provided layers. + + Parameters + ---------- + layers : List[pogc.Layer] + The list of layers to determine datetimes for. + instance_time: str | None + The optional instance time to get datetimes for, if not provided collection datetimes are returned. + Returns + ------- + List[np.datetime64] + Available time values for the provider layers. + """ + + available_times = set() + for layer in layers: + coordinates = layer.get_coordinates() + if coordinates is not None and "time" in coordinates.udims: + if ( + instance_time in layer.time_instances() + and settings.EDR_TIME_INSTANCE_DIMENSION in coordinates.udims + ): + instance_datetime = np.datetime64(instance_time) + instance_coordinates = coordinates.select( + {settings.EDR_TIME_INSTANCE_DIMENSION: [instance_datetime, instance_datetime]} + ) + selected_time_coordinates = instance_coordinates["time"].coordinates + available_times.update(selected_time_coordinates) + elif not instance_time and settings.EDR_TIME_INSTANCE_DIMENSION not in coordinates.udims: + available_times.update(coordinates["time"].coordinates) + + return list(available_times) + + @staticmethod + def interpret_crs(crs: str | None) -> str: + """Interpret the CRS id string into a valid WKT CRS format. + + If None provided, return the default. + If the provided CRS is invalid, raise an error. + + Parameters + ---------- + crs : str | None + The input CRS id string which needs to be validated/converted. + + Returns + ------- + str + WKT CRS string. + + Raises + ------ + ProviderInvalidQueryError + Raised if the provided CRS string is unknown. + """ + if crs is None: + return pyproj.CRS(settings.crs_84_uri_format).to_wkt() # Pyproj acceptable format + + try: + wkt_crs = pyproj.CRS(crs).to_wkt() + wkt_options = [pyproj.CRS(key).to_wkt() for key in settings.EDR_CRS.keys()] + except CRSError: + wkt_crs = None + wkt_options = [] + + if wkt_crs is None or wkt_crs not in wkt_options: + error_msg = msg = f"Invalid CRS provided, expected one of {', '.join(settings.EDR_CRS.keys())}" + raise ProviderInvalidQueryError(msg, user_msg=error_msg) + + return wkt_crs + + @staticmethod + def crs_converter(x: Any, y: Any, crs: str) -> Tuple[Any, Any]: + """Convert the X, Y data to Longitude, Latitude data with the provided crs. + + Parameters + ---------- + x : Any + X data in any form. + y: Any + Y data in any form. + crs : str + The input CRS id string to apply to convert the X,Y data. + + Returns + ------- + Tuple[Any, Any] + The X,Y as Longitude/Latitude data. + """ + wkt_crs = pyproj.CRS(crs).to_wkt() + wkt_epsg_4326 = pyproj.CRS(settings.epsg_4326_uri_format).to_wkt() + if wkt_crs == wkt_epsg_4326: + return (y, x) + + return (x, y) + + @staticmethod + def interpret_altitude_coordinates( + available_altitudes: List[float], altitude_string: str | None, crs: str | None + ) -> podpac.Coordinates | None: + """Interpret the altitude string into altitude coordinates using known formats. + + Specification: + single-level = level + interval-closed = min-level "/" max-level + repeating-interval = "R"number of intervals "/" min-level "/" height to increment by + level-list = level1 "," level2 "," level3 + + Parameters + ---------- + available_altitudes: List[float] + The available altitudes for interpretation. + altitude_string : str | None + The string representation of the requested altitudes. + crs : str + The CRS that the coordinates need to match. + + Returns + ------- + podpac.Coordinates | None + Altitude coordinates for the request or None if no matching altitudes were found. + + Raises + ------ + ProviderInvalidQueryError + Raised if the provided altitude string is invalid. + """ + + if not altitude_string and len(available_altitudes) == 0: + return None + + try: + altitudes = None + if not altitude_string: + altitudes = available_altitudes + elif "/" in altitude_string: + altitudes_split = altitude_string.split("/") + if len(altitudes_split) == 2: + minimum = float(altitudes_split[0]) + maximum = float(altitudes_split[1]) + altitudes = [alt for alt in available_altitudes if minimum <= alt <= maximum] + if len(altitudes_split) == 3: + if altitudes_split[0].startswith("R"): + altitudes = float(altitudes_split[1]) + np.arange(float(altitudes_split[0][1:])) * float( + altitudes_split[2] + ) + else: + altitudes = [float(alt) for alt in altitude_string.split(",")] + except ValueError: + msg = "Invalid vertical level requested." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + if altitudes is None: + msg = "Invalid vertical level requested." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + return podpac.Coordinates([altitudes], dims=["alt"], crs=crs) if altitudes is not None else None + + @staticmethod + def interpret_time_coordinates( + available_times: List[np.datetime64], time_string: str | None, instance_time: str | None, crs: str | None + ) -> podpac.Coordinates | None: + """Interpret the time string and instance into time coordinates using known formats. + + Specification: + interval-closed = date-time "/" date-time + interval-open-start = "../" date-time + interval-open-end = date-time "/.." + interval = interval-closed / interval-open-start / interval-open-end + datetime = date-time / interval + + Parameters + ---------- + available_times: List[np.datetime64] + The available times for interpretation. + time_string : str | None + The string representation of the requested times. + instance_time: str | None + The string representation of the requested instance time. + crs : str + The CRS that the coordinates need to match. + + Returns + ------- + podpac.Coordinates | None + Time coordinates for the request or None if no matching times were found. + + Raises + ------ + ProviderInvalidQueryError + Raised if the provided time string is invalid. + """ + + if not time_string and len(available_times) == 0: + return None + + try: + times = None + if not time_string: + times = available_times + elif "/" in time_string: + times_split = time_string.split("/") + if len(times_split) == 2: + minimum = times_split[0] + maximum = times_split[1] + if minimum == "..": + times = [time for time in available_times if time <= np.datetime64(maximum)] + elif maximum == "..": + times = [time for time in available_times if time >= np.datetime64(minimum)] + else: + times = [ + time for time in available_times if np.datetime64(minimum) <= time <= np.datetime64(maximum) + ] + else: + times = [np.datetime64(time_string)] + except ValueError: + msg = "Invalid datetime requested." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + if times is None: + msg = "Invalid datetime requested." + raise ProviderInvalidQueryError(msg, user_msg=msg) + + if instance_time: + return podpac.Coordinates( + [times, [instance_time]], dims=["time", settings.EDR_TIME_INSTANCE_DIMENSION], crs=crs + ) + + return podpac.Coordinates([times], dims=["time"], crs=crs) + + @staticmethod + def to_coverage_json( + layers: List[pogc.Layer], dataset: Dict[str, podpac.UnitsDataArray], collection_id: str, crs: str + ) -> Dict[str, Any]: + """Generate a CoverageJSON of the data for the provided parameters. + + The returned object must be serializable, so a temporary file is returned to reference the data. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers which were used in the dataset creation for metadata information. + dataset : Dict[str, podpac.UnitsDataArray] + Data in an units data array format with matching parameter key. + collection_id : str + The collection id of the data used in naming the output file. + crs : str + The CRS associated with the requested coordinates and data response. + + Returns + ------- + Dict[str, Any] + A dictionary of the desired output file name and data path. + """ + + # Determine the bounding coordinates, assume they all are the same + coordinates = next(iter(dataset.values())).coords + x_arr, y_arr = EdrProvider.crs_converter(coordinates["lon"].values, coordinates["lat"].values, crs) + + # Convert numpy array coordinates to a flattened list. + x_arr = list(x_arr.flatten()) + y_arr = list(y_arr.flatten()) + + lon_map_value, lat_map_value = EdrProvider.crs_converter("x", "y", crs) + dimension_map = {"time": "t", "alt": "z", "lon": lon_map_value, "lat": lat_map_value} + + coverage_json = { + "type": "Coverage", + "domain": { + "type": "Domain", + "domainType": "Grid", + "axes": { + "x": { + "start": x_arr[0] if len(x_arr) > 0 else None, + "stop": x_arr[-1] if len(x_arr) > 0 else None, + "num": len(x_arr), + }, + "y": { + "start": y_arr[0] if len(y_arr) > 0 else None, + "stop": y_arr[-1] if len(y_arr) > 0 else None, + "num": len(y_arr), + }, + }, + "referencing": [ + { + "coordinates": ["x", "y"], + "system": {"type": "GeographicCRS", "id": crs}, + }, + *( + [ + { + "coordinates": ["t"], + "system": { + "type": "TemporalRS", + "calendar": "Gregorian", + }, + } + ] + if "time" in coordinates.dims + else [] + ), + *( + [ + { + "coordinates": ["z"], + "system": {"type": "VerticalCRS"}, + } + ] + if "alt" in coordinates.dims + else [] + ), + ], + "parameters": {}, + "ranges": {}, + }, + } + if "time" in coordinates.dims: + coverage_json["domain"]["axes"]["t"] = { + "values": [ + time.astype("datetime64[ms]").astype(datetime).isoformat() + "Z" + for time in coordinates["time"].values + ] + } + + for param, data_array in dataset.items(): + layer = next((layer for layer in layers if layer.identifier == param), None) + if layer is not None: + units = layer.get_units() + parameter_definition = { + param: { + "type": "Parameter", + "observedProperty": { + "label": { + "id": param, + "en": param, + } + }, + "description": layer.abstract, + "unit": { + "label": {"en": param}, + "symbol": { + "value": units, + "type": None, + }, + }, + } + } + coverage_json["domain"]["parameters"].update(parameter_definition) + + data = [x if np.isfinite(x) else None for x in data_array.values.flatten()] + coverage_json["domain"]["ranges"].update( + { + param: { + "type": "NdArray", + "dataType": "float", + "axisNames": [dimension_map.get(str(key), str(key)) for key in data_array.coords.keys()], + "shape": data_array.shape, + "values": data, # Row Major Order + } + } + ) + + encoder = json.JSONEncoder() + with tempfile.NamedTemporaryFile(mode="w+", suffix=".json", delete=False) as named_file: + for chunk in encoder.iterencode(coverage_json): + named_file.write(chunk) + + return {"fp": named_file.name, "fn": f"{collection_id}.json"} + + @staticmethod + def check_query_condition(conditional: bool, message: str): + """Check the provided conditional and raise a ProviderInvalidQueryError if true. + + Parameters + ---------- + conditional : bool + The conditional value to check for raising a query error. + message : str + The message to include if the query error is raised. + + Raises + ------ + ProviderInvalidQueryError + Raised if the conditional provided is true. + """ + if conditional: + raise ProviderInvalidQueryError(message, user_msg=message) + + @staticmethod + def to_geotiff_response(dataset: Dict[str, podpac.UnitsDataArray], collection_id: str) -> Dict[str, Any]: + """Generate a geotiff of the data for the provided parameters. + + The returned object must be serializable, so a temporary file is returned to reference the data. + + Parameters + ---------- + dataset : Dict[str, podpac.UnitsDataArray] + Data in an units data array format with matching parameter key. + collection_id : str + The collection id of the data used in naming the zip file if needed. + + Returns + ------- + Dict[str, Any] + A dictionary of the desired output file name and data path. + """ + if len(dataset) == 1: + units_data_array = next(iter(dataset.values())) + geotiff_bytes = units_data_array.to_format("geotiff").read() + with tempfile.NamedTemporaryFile(mode="wb+", suffix=".tif", delete=False) as named_file: + named_file.write(geotiff_bytes) + return { + "fp": named_file.name, + "fn": f"{next(iter(dataset.keys()))}.tif", + } + else: + with tempfile.NamedTemporaryFile(mode="wb+", suffix=".zip", delete=False) as named_file: + with zipfile.ZipFile(named_file, "w", zipfile.ZIP_DEFLATED) as zip_file: + for parameter, data_array in dataset.items(): + geotiff_memory_file = data_array.to_format("geotiff") + tiff_filename = f"{parameter}.tif" + zip_file.writestr(tiff_filename, geotiff_memory_file.read()) + + return {"fp": named_file.name, "fn": f"{collection_id}.zip"} + + @staticmethod + def get_native_coordinates( + source_coordinates: podpac.Coordinates, + target_coordinates: podpac.Coordinates, + resolution_lat: int, + resolution_lon: int, + ) -> podpac.Coordinates: + """Find the intersecting latitude and longitude coordinates between the source and target. + + Parameters + ---------- + source_coordinates : podpac.Coordinates + The source coordinates to be converted. + target_coordinates : podpac.Coordinates + The target coordinates to find intersections on. + resolution_lat: int + The desired resolution in the latitudinal direction, with a zero value using native resolution. + resolution_lon: int + The desired resolution in the longitudinal direction, with a zero value using native resolution. + + Returns + ------- + podpac.Coordinates + The converted coordinates source coordinates intersecting with the target coordinates. + """ + if len(source_coordinates["lat"].coordinates) == 1 and len(source_coordinates["lon"].coordinates) == 1: + return source_coordinates + + latitudes = ( + target_coordinates["lat"].coordinates + if resolution_lat == 0 + else np.linspace( + np.min(source_coordinates["lat"].coordinates), + np.max(source_coordinates["lat"].coordinates), + resolution_lat, + ) + ) + longitudes = ( + target_coordinates["lon"].coordinates + if resolution_lon == 0 + else np.linspace( + np.min(source_coordinates["lon"].coordinates), + np.max(source_coordinates["lon"].coordinates), + resolution_lon, + ) + ) + + # Find intersections with target keeping source crs + target_spatial_coordinates = podpac.Coordinates( + [latitudes, longitudes], dims=["lat", "lon"], crs=target_coordinates.crs + ) + source_intersection_coordinates = target_spatial_coordinates.intersect(source_coordinates, dims=["lat", "lon"]) + source_intersection_coordinates = source_intersection_coordinates.transform(source_coordinates.crs) + return podpac.coordinates.merge_dims( + [source_intersection_coordinates, source_coordinates.udrop(["lat", "lon"], ignore_missing=True)] + ) diff --git a/ogc/edr/edr_routes.py b/ogc/edr/edr_routes.py new file mode 100644 index 0000000..b03435b --- /dev/null +++ b/ogc/edr/edr_routes.py @@ -0,0 +1,326 @@ +import os +import mimetypes +import json +import traitlets as tl +import weakref +import pygeoapi.l10n +import pygeoapi.plugin +import pygeoapi.api +from typing import Tuple, Any, Dict, Generator +from http import HTTPStatus +from copy import deepcopy +from pygeoapi.openapi import get_oas +from ogc import podpac as pogc + +from .edr_api import EdrAPI +from .edr_config import EdrConfig +from .edr_provider import EdrProvider + + +class EdrRoutes(tl.HasTraits): + """Class responsible for routing EDR requests to the appropriate pygeoapi API method.""" + + base_url = tl.Unicode(default_value="http://127.0.0.1:5000/edr") + layers = tl.List(trait=tl.Instance(pogc.Layer)) + + def __init__(self, **kwargs): + """Initialize the API based on the available layers.""" + super().__init__(**kwargs) + self.api = self.create_api() + + @tl.observe("layers") + def layers_change(self, change: Dict[str, Any]): + """Monitor the layers and update the API when a change occurs. + + Parameters + ---------- + change : Dict[str, Any] + Dictionary holding type of modification and name of the attribute that triggered it. + """ + self.api = self.create_api() + + @tl.observe("base_url") + def base_url_change(self, change: Dict[str, Any]): + """Monitor the base url and update the API when a change occurs. + + Parameters + ---------- + change : Dict[str, Any] + Dictionary holding type of modification and name of the attribute that triggered it. + """ + self.api = self.create_api() + + def create_api(self) -> pygeoapi.api.API: + """Create the pygeoapi API using a custom configuration. + + Returns + ------- + pygeoapi.api.API + The API which handles all EDR requests. + """ + # Allow specifying GeoTiff or CoverageJSON in the format argument. + # This is a bypass which is needed to get by a conditional check in pygeoapi. + pygeoapi.plugin.PLUGINS["formatter"]["geotiff"] = "" + pygeoapi.plugin.PLUGINS["formatter"]["coveragejson"] = "" + EdrProvider.set_layers(self.base_url, self.layers) + config = EdrConfig.get_configuration(self.base_url, self.layers) + open_api = get_oas(config, fail_on_invalid_collection=False) + return pygeoapi.api.API(config=deepcopy(config), openapi=open_api) + + def clean_configuration_cache(self): + """Clean a pygeoapi internal translation cache so that multiple configurations can be used simultaneously.""" + pygeoapi.l10n._cfg_cache = {} + + def update_configuration_base_url(self, request: pygeoapi.api.APIRequest): + """Update the EDR configuration base URL based on the provided request. + The EDR configuration base URL does not necessarily match the full path of the request base URL. + + Parameters + ---------- + request : pygeoapi.api.APIRequest + The API request containing a base URL. + The base URL should include scheme, host, and path for the request. + """ + request_base_url = request.params.get("base_url", "") + # Limit the base URL from the request to the EDR subdirectory + base_url_partitioned = request_base_url.partition("/edr") + if len(base_url_partitioned[0]) > 0: + configuration_base_url = base_url_partitioned[0] + base_url_partitioned[1] + if configuration_base_url != self.base_url: + self.base_url = configuration_base_url + + def static_files(self, request: pygeoapi.api.APIRequest, file_path: str) -> Tuple[dict, int, str | bytes]: + """Handle static file requests using the custom static file folder or the pygeoapi default folder. + + Parameters + ---------- + file_path : str + The file path of the requested static resource. + + Returns + ------- + Tuple[dict, int, str | bytes] + Headers, HTTP Status, and Content returned as a tuple to make the server response. + """ + self.clean_configuration_cache() + self.update_configuration_base_url(request) + static_path = os.path.join(os.path.dirname(pygeoapi.__file__), "static") + if "templates" in self.api.config["server"]: + static_path = self.api.config["server"]["templates"].get("static", static_path) + file_path = os.path.join(static_path, file_path) + if os.path.isfile(file_path): + mime_type, _ = mimetypes.guess_type(file_path) + mime_type = mime_type or "application/octet-stream" + with open(file_path, "rb") as f: + content = f.read() + return {"Content-Type": mime_type}, HTTPStatus.OK, content + else: + return {}, HTTPStatus.NOT_FOUND, b"File not found" + + def landing_page(self, request: pygeoapi.api.APIRequest) -> Tuple[dict, int, str | bytes]: + """Handle landing page requests for the server. + + Parameters + ---------- + request : pygeoapi.api.APIRequest + The pygeoapi request for the server. + + Returns + ------- + Tuple[dict, int, str | bytes] + Headers, HTTP Status, and Content returned as a tuple to make the server response. + """ + self.clean_configuration_cache() + self.update_configuration_base_url(request) + return EdrAPI.landing_page(self.api, request) + + def openapi(self, request: pygeoapi.api.APIRequest) -> Tuple[dict, int, str | bytes]: + """Handle API documentation requests for the server. + + Parameters + ---------- + request : pygeoapi.api.APIRequest + The pygeoapi request for the server. + + Returns + ------- + Tuple[dict, int, str | bytes] + Headers, HTTP Status, and Content returned as a tuple to make the server response. + """ + self.clean_configuration_cache() + self.update_configuration_base_url(request) + return EdrAPI.openapi_(self.api, request) + + def conformance(self, request: pygeoapi.api.APIRequest) -> Tuple[dict, int, str | bytes]: + """Handle conformance requests for the server. + + Parameters + ---------- + request : pygeoapi.api.APIRequest + The pygeoapi request for the server. + + Returns + ------- + Tuple[dict, int, str | bytes] + Headers, HTTP Status, and Content returned as a tuple to make the server response. + """ + self.clean_configuration_cache() + self.update_configuration_base_url(request) + return EdrAPI.conformance(self.api, request) + + def describe_collections( + self, + request: pygeoapi.api.APIRequest, + collection_id: str | None, + ) -> Tuple[dict, int, str | bytes]: + """Handle describe collection requests for the server. + + Parameters + ---------- + request : pygeoapi.api.APIRequest + The pygeoapi request for the server. + collection_id : str | None + The collection ID to describe. + + Returns + ------- + Tuple[dict, int, str | bytes] + Headers, HTTP Status, and Content returned as a tuple to make the server response. + """ + self.clean_configuration_cache() + self.update_configuration_base_url(request) + return EdrAPI.describe_collections(self.api, request, collection_id) + + def describe_instances( + self, + request: pygeoapi.api.APIRequest, + collection_id: str, + instance_id: str | None, + ) -> Tuple[dict, int, str | bytes]: + """Handle collection instances requests for the server. + + Parameters + ---------- + request : pygeoapi.api.APIRequest + The pygeoapi request for the server. + collection_id : str + The collection ID for the instances. + instance_id: str + The instance ID to describe. + + Returns + ------- + Tuple[dict, int, str | bytes] + Headers, HTTP Status, and Content returned as a tuple to make the server response. + """ + self.clean_configuration_cache() + self.update_configuration_base_url(request) + return EdrAPI.get_collection_edr_instances(self.api, request, collection_id, instance_id=instance_id) + + def collection_query( + self, + request: pygeoapi.api.APIRequest, + collection_id: str, + instance_id: str | None, + query_type: str, + ) -> Tuple[dict, int, Any]: + """Handle collection and instance query requests for the server. + + Parameters + ---------- + request : pygeoapi.api.APIRequest + The pygeoapi request for the server. + query_type: str + The query type for the request. + collection_id : str + The collection ID for the query. + instance_id: str + The instance ID for the query. + + Returns + ------- + Tuple[dict, int, Any] + Headers, HTTP Status, and Content returned as a tuple to make the server response. + """ + self.clean_configuration_cache() + self.update_configuration_base_url(request) + EdrProvider.set_extra_query_args(request._args) + headers, http_status, content = EdrAPI.get_collection_edr_query( + self.api, request, collection_id, instance_id, query_type=query_type, location_id=None + ) + + if "text/html" in headers.get("Content-Type", ""): + return headers, http_status, content + + content = json.loads(content) + if "fn" in content and "fp" in content: + data_path = content["fp"] + binary_mode = "rb" + text_mode = "r" + mode = text_mode + if data_path.endswith(".tif"): + mode = binary_mode + headers["Content-Type"] = "image/tiff" + elif data_path.endswith(".zip"): + mode = binary_mode + headers["Content-Type"] = "application/zip" + elif data_path.endswith(".json"): + mode = text_mode + headers["Content-Type"] = "application/prs.coverage+json" + + headers["Content-Disposition"] = f"attachment; filename={content["fn"]}" + content = self.file_generator_with_cleanup(data_path, mode) + + return headers, http_status, content + + @staticmethod + def file_generator_with_cleanup(path: str, mode: str, chunk_size=1024 * 1024) -> Generator[str | bytes, None, None]: + """Create a generator for file data with cleanup. + + Parameters + ---------- + path : str + The path to the file. + mode : str + The mode to open the file with. + chunk_size : int, optional + The character or byte size of each chunk read from the file, by default 1024*1024 + + Yields + ------ + Generator[str | bytes, None, None] + Chunks of data from the file. + """ + + def _cleanup(path: str): + """Cleanup a file if it exists. + + Parameters + ---------- + path : str + The path to cleanup. + """ + if os.path.exists(path): + os.remove(path) + + def _generator() -> Generator[str | bytes, None, None]: + """Create the generator for reading file data. + + Yields + ------ + Generator[str | bytes, None, None] + Chunks of file data. + """ + try: + with open(path, mode) as f: + while True: + chunk = f.read(chunk_size) + if not chunk: + break + yield chunk + finally: + _cleanup(path) + + generator = _generator() + weakref.finalize(generator, _cleanup, path) # Ensure file removal even if generator is never used + return generator diff --git a/ogc/edr/static/css/default.css b/ogc/edr/static/css/default.css new file mode 100644 index 0000000..36f9261 --- /dev/null +++ b/ogc/edr/static/css/default.css @@ -0,0 +1,86 @@ +.flat { + border: 0px; +} + +header { + display: inline-block; +} + +main { + background-color: white; +} + +.crumbs { + background-color: rgb(230, 230, 230); + padding: 6px; +} + +.crumbs a { + padding: 0px 6px; + color: black; + text-decoration: none; + /* text-transform: capitalize;*/ +} + +#items-map, +#collection-map { + width: 100%; + height: 400px; +} + +#coverages-map { + width: 100%; + height: 80vh; +} + +.c3-tooltip-container { + z-index: 300; +} + +/* cancel mini-css header>button uppercase */ +header button, +header [type="button"], +header .button, +header [role="button"] { + text-transform: none; +} + +html { + background-color: #fff; +} + +body { + display: flex; + flex-direction: column; + min-height: 100vh; + background-color: #fff; +} + +footer.sticky-bottom { + margin-top: auto; +} + +main { + padding-bottom: 65px; + /* prevent from falling under the footer */ +} + +table:not(.horizontal) { + max-height: none; +} + +mark.successful { + background-color: green; +} + +mark.accepted { + background-color: default; +} + +mark.failed { + background-color: red; +} + +mark.running { + background-color: orange; +} \ No newline at end of file diff --git a/ogc/edr/static/img/favicon.ico b/ogc/edr/static/img/favicon.ico new file mode 100644 index 0000000..1e20f9c Binary files /dev/null and b/ogc/edr/static/img/favicon.ico differ diff --git a/ogc/edr/static/img/logo.png b/ogc/edr/static/img/logo.png new file mode 100644 index 0000000..4b8ebad Binary files /dev/null and b/ogc/edr/static/img/logo.png differ diff --git a/ogc/edr/static/img/pygeoapi.png b/ogc/edr/static/img/pygeoapi.png new file mode 100644 index 0000000..0477111 Binary files /dev/null and b/ogc/edr/static/img/pygeoapi.png differ diff --git a/ogc/edr/test/conftest.py b/ogc/edr/test/conftest.py new file mode 100644 index 0000000..d566273 --- /dev/null +++ b/ogc/edr/test/conftest.py @@ -0,0 +1,134 @@ +import os +import pytest +import numpy as np +import datetime +import podpac +from ogc import podpac as pogc +from typing import Dict, List, Any +from ogc.settings import EDR_TIME_INSTANCE_DIMENSION + +# Setup new dimension +podpac.core.coordinates.utils.add_valid_dimension(EDR_TIME_INSTANCE_DIMENSION) + +lat = np.linspace(90, -90, 11) +lon = np.linspace(-180, 180, 21) +time = np.array(["2025-10-24T12:00:00"], dtype="datetime64") +instance = np.array(["2025-10-24T00:00:00"], dtype="datetime64") +data = np.random.default_rng(1).random((11, 21, 1, 1)) +coords = podpac.Coordinates([lat, lon, time, instance], dims=["lat", "lon", "time", EDR_TIME_INSTANCE_DIMENSION]) +data_without_instance = np.random.default_rng(1).random((11, 21, 1)) +coords_without_instance = podpac.Coordinates([lat, lon, time], dims=["lat", "lon", "time"]) + +# Define test layers using sample data and coordinates +node1 = podpac.data.Array(source=data, coordinates=coords) +layer1 = pogc.Layer( + node=node1, + identifier="layer1", + title="Layer 1", + abstract="Layer1 Data", + group="Layers", + valid_times=[dt.astype(datetime.datetime) for dt in time], +) +node2 = podpac.data.Array(source=data, coordinates=coords) +layer2 = pogc.Layer( + node=node2, + identifier="layer2", + title="Layer 2", + abstract="Layer2 Data", + group="Layers", + valid_times=[dt.astype(datetime.datetime) for dt in time], +) +node3 = podpac.data.Array(source=data_without_instance, coordinates=coords_without_instance) +layer3 = pogc.Layer( + node=node3, + identifier="layer3", + title="Layer 3", + abstract="Layer3 Data (No instance)", + group="Layers", + valid_times=[dt.astype(datetime.datetime) for dt in time], +) + + +@pytest.fixture(scope="session", autouse=True) +def set_env_vars(): + """Setup the environmental variables for the session to support EDR.""" + os.environ["OGC_SUPPORTED_FORMATS"] = "edr" + + +@pytest.fixture() +def layers() -> List[pogc.Layer]: + """List of test layers. + + Returns + ------- + List[pogc.Layer] + The test layers. + """ + return [layer1, layer2] + + +@pytest.fixture() +def layers_no_instance() -> List[pogc.Layer]: + """List of test layers without instances. + + Returns + ------- + List[pogc.Layer] + The test layers. + """ + return [layer3] + + +@pytest.fixture() +def single_layer_cube_args() -> Dict[str, Any]: + """Dictionary of valid request arguments that align to a single test layer cube request. + + Returns + ------- + Dict[str, Any] + Valid cube request arguments for a single test layer. + """ + + return { + "f": "coveragejson", + "bbox": "-180, -90, 180, 90", + "datetime": str(time[0]), + "parameter-name": [layer1.identifier], + } + + +@pytest.fixture() +def single_layer_cube_args_internal() -> Dict[str, Any]: + """Dictionary of valid arguments that align to a single test layer request with internal pygeoapi keys. + + Returns + ------- + Dict[str, Any] + Valid internal cube arguments for a single test layer. + """ + + return { + "format_": "coveragejson", + "instance": str(instance[0]), + "bbox": [-180, -90, 180, 90], + "datetime_": str(time[0]), + "select_properties": [layer1.identifier], + } + + +@pytest.fixture() +def single_layer_cube_args_no_instance_internal() -> Dict[str, Any]: + """Dictionary of valid arguments that align to a single non-instance test layer request with internal pygeoapi keys. + + Returns + ------- + Dict[str, Any] + Valid internal cube arguments for a single non-instance test layer. + """ + + return { + "format_": "coveragejson", + "bbox": [-180, -90, 180, 90], + "datetime_": str(time[0]), + "select_properties": [layer3.identifier], + } diff --git a/ogc/edr/test/test_edr_config.py b/ogc/edr/test/test_edr_config.py new file mode 100644 index 0000000..e651c49 --- /dev/null +++ b/ogc/edr/test/test_edr_config.py @@ -0,0 +1,64 @@ +from typing import Dict, List, Any +from ogc import podpac as pogc +from ogc.edr.edr_config import EdrConfig + + +def test_edr_default_configuration_has_required_keys(): + """Test the EDR default configuration loads the required keys.""" + configuration = EdrConfig.get_configuration("/ogc", []) + + assert configuration.keys() == {"server", "logging", "metadata", "resources"} + + +def test_edr_configuration_contains_layer_groups(layers: List[pogc.Layer]): + """Test the EDR configuration contains the layer groups. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + group_keys = {layer.group for layer in layers} + configuration = EdrConfig.get_configuration("/ogc", layers) + + assert len(group_keys) > 0 + for key in group_keys: + assert configuration["resources"].get(key) is not None + + +def test_edr_configuration_contains_spatial_extent(layers: List[pogc.Layer], single_layer_cube_args: Dict[str, Any]): + """Test the EDR configuration contains the spatial extent. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args : Dict[str, Any] + Single layer arguments for validation checking provided by a test fixture. + """ + group_keys = {layer.group for layer in layers} + configuration = EdrConfig.get_configuration("/ogc", layers) + + assert len(group_keys) > 0 + for key in group_keys: + assert configuration["resources"][key]["extents"]["spatial"]["bbox"] == list( + map(float, single_layer_cube_args["bbox"].split(",")) + ) + + +def test_edr_configuration_contains_custom_provider(layers: List[pogc.Layer]): + """Test the EDR configuration contains the custom provider. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + group_keys = {layer.group for layer in layers} + configuration = EdrConfig.get_configuration("/ogc", layers) + + assert len(group_keys) > 0 + for key in group_keys: + assert configuration["resources"][key]["providers"][0]["type"] == "edr" + assert configuration["resources"][key]["providers"][0]["name"] == "ogc.edr.edr_provider.EdrProvider" diff --git a/ogc/edr/test/test_edr_provider.py b/ogc/edr/test/test_edr_provider.py new file mode 100644 index 0000000..c081781 --- /dev/null +++ b/ogc/edr/test/test_edr_provider.py @@ -0,0 +1,781 @@ +import pytest +import numpy as np +import zipfile +import io +import json +import os +import podpac +import pyproj +from shapely import Point, Polygon +from typing import Dict, List, Any +from ogc import settings +from ogc import podpac as pogc +from ogc.edr.edr_provider import EdrProvider +from pygeoapi.provider.base import ProviderInvalidQueryError + + +def get_json_with_cleanup(path: str) -> Dict[str, Any]: + """Get JSON data from a path and remove the file after retrieval. + + Parameters + ---------- + path : str + The JSON file path. + + Returns + ------- + Dict[str, Any] + JSON data from the path. + """ + with open(path, "r") as f: + response = json.load(f) + + os.remove(path) + return response + + +def get_bytes_with_cleanup(path: str) -> bytes: + """Get byte data from a path and remove the file after retrieval. + + Parameters + ---------- + path : str + The file path. + + Returns + ------- + bytes + Binary data from the path. + """ + with open(path, "rb") as f: + response = f.read() + + os.remove(path) + return response + + +def get_provider_definition(base_url: str) -> Dict[str, Any]: + """Define the provider definition which is typically handled by pygeoapi. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers for the provider to use in defining available data sources. + + Returns + ------- + Dict[str, Any] + The provider definition which defines data sources. + """ + return { + "type": "edr", + "default": True, + "name": "ogc.edr.edr_provider.EdrProvider", + "data": "Layers", + "base_url": base_url, + "crs": list(settings.EDR_CRS.keys()), + "format": {"name": "GeoJSON", "mimetype": "application/json"}, + } + + +def test_edr_provider_resources(layers: List[pogc.Layer]): + """Test the available resources of the EDR Provider class. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + base_url = "/" + identifiers = [layer.identifier for layer in layers] + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + assert len(provider.get_layers(base_url)) == len(layers) + assert all(layer.identifier in identifiers for layer in provider.get_layers(base_url)) + + +def test_edr_provider_resources_limited_by_url(layers: List[pogc.Layer]): + """Test the available resources of the EDR Provider class are limited by URL. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + base_url = "/" + invalid_url = "/invalid" + + provider = EdrProvider(provider_def=get_provider_definition(invalid_url)) + provider.set_layers(base_url, layers) + + assert len(provider.get_layers(invalid_url)) == 0 + assert len(provider.get_layers(base_url)) == len(layers) + + +def test_edr_provider_get_instance_valid_id(layers: List[pogc.Layer]): + """Test the get_instance method of the EDR Provider class with a valid id. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + base_url = "/" + time_instance = next(iter(layers[0].time_instances())) + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + assert provider.get_instance(time_instance) == time_instance + + +def test_edr_provider_get_instance_invalid_id(layers: List[pogc.Layer]): + """Test the get_instance method of the EDR Provider class with an invalid id. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + base_url = "/" + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + assert provider.get_instance("invalid") is None + + +def test_edr_provider_parameter_keys(layers: List[pogc.Layer]): + """Test the parameters property of the EDR Provider class. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + base_url = "/" + identifiers = [layer.identifier for layer in layers] + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + parameters = provider.parameters + + assert len(list(parameters.keys())) == len(layers) + assert all(identifier in identifiers for identifier in parameters.keys()) + + +def test_edr_provider_instances(layers: List[pogc.Layer]): + """Test the instances method of the EDR Provider class. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + base_url = "/" + instance_sets = [layer.time_instances() for layer in layers] + time_instances = set().union(*instance_sets) + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + instances = provider.instances() + + assert len(instances) == len(time_instances) + assert instances == [str(t) for t in time_instances] + + +def test_edr_provider_get_fields(layers: List[pogc.Layer]): + """Test the get fields method of the EDR Provider class. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + base_url = "/" + identifiers = [layer.identifier for layer in layers] + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + fields = provider.get_fields() + + assert len(fields.keys()) == len(layers) + assert all(identifier in identifiers for identifier in fields.keys()) + + +def test_edr_provider_position_request_valid_wkt( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the position method of the EDR Provider class with a valid WKT. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + del args["bbox"] + args["wkt"] = Point(5.2, 52.1) + parameter_name = single_layer_cube_args_internal["select_properties"][0] + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + response = provider.position(**args) + response = get_json_with_cleanup(response["fp"]) + + assert set(response["domain"]["ranges"][parameter_name]["axisNames"]) == {"x", "y", "t"} + assert np.prod(np.array(response["domain"]["ranges"][parameter_name]["shape"])) == len( + response["domain"]["ranges"][parameter_name]["values"] + ) + + +def test_edr_provider_position_request_invalid_wkt( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the position method of the EDR Provider class with an invalid WKT. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + del args["bbox"] + args["wkt"] = "invalid" + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + with pytest.raises(ProviderInvalidQueryError): + provider.position(**args) + + +def test_edr_provider_position_request_invalid_format( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the position method of the EDR Provider class with an invalid format. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + del args["bbox"] + args["wkt"] = Point(5.2, 52.1) + args["format_"] = settings.GEOTIFF + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + with pytest.raises(ProviderInvalidQueryError): + provider.position(**args) + + +def test_edr_provider_position_request_invalid_property( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the position method of the EDR Provider class with an invalid property. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + del args["bbox"] + args["wkt"] = Point(5.2, 52.1) + args["select_properties"] = "invalid" + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + with pytest.raises(ProviderInvalidQueryError): + provider.position(**args) + + +def test_edr_provider_cube_request_valid_bbox( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the cube method of the EDR Provider class with a valid bounding box. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + parameter_name = single_layer_cube_args_internal["select_properties"][0] + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + response = provider.cube(**args) + response = get_json_with_cleanup(response["fp"]) + + assert set(response["domain"]["ranges"][parameter_name]["axisNames"]) == {"x", "y", "t"} + assert np.prod(np.array(response["domain"]["ranges"][parameter_name]["shape"])) == len( + response["domain"]["ranges"][parameter_name]["values"] + ) + + +def test_edr_provider_cube_request_valid_bbox_with_resolution( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the cube method of the EDR Provider class with a valid bounding box and a specific resolution. + + The tested node is adjusted to ensure interpolation is used. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + parameter_name = single_layer_cube_args_internal["select_properties"][0] + resolution_x = 15 + resolution_y = 20 + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + layers[0].node = layers[0].node.interpolate() + provider.set_layers(base_url, layers) + provider.set_extra_query_args({"resolution-x": resolution_x, "resolution-y": resolution_y}) + + response = provider.cube(**args) + response = get_json_with_cleanup(response["fp"]) + + assert set(response["domain"]["ranges"][parameter_name]["axisNames"]) == {"x", "y", "t"} + assert np.prod(np.array(response["domain"]["ranges"][parameter_name]["shape"])) == resolution_x * resolution_y + assert np.prod(np.array(response["domain"]["ranges"][parameter_name]["shape"])) == len( + response["domain"]["ranges"][parameter_name]["values"] + ) + + +def test_edr_provider_cube_request_invalid_bbox( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the cube method of the EDR Provider class with an invalid bounding box. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + args["bbox"] = "invalid" + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + with pytest.raises(ProviderInvalidQueryError): + provider.cube(**args) + + +def test_edr_provider_cube_request_without_instance( + layers_no_instance: List[pogc.Layer], single_layer_cube_args_no_instance_internal: Dict[str, Any] +): + """Test the cube method of the EDR Provider class with a valid layer without an instance. + + Parameters + ---------- + layers_no_instance : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_no_instance_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_no_instance_internal + parameter_name = single_layer_cube_args_no_instance_internal["select_properties"][0] + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers_no_instance) + + response = provider.cube(**args) + response = get_json_with_cleanup(response["fp"]) + + assert set(response["domain"]["ranges"][parameter_name]["axisNames"]) == {"x", "y", "t"} + assert np.prod(np.array(response["domain"]["ranges"][parameter_name]["shape"])) == len( + response["domain"]["ranges"][parameter_name]["values"] + ) + + +def test_edr_provider_cube_request_invalid_instance( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the cube method of the EDR Provider class with an invalid instance. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + args["instance"] = "invalid" + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + with pytest.raises(ProviderInvalidQueryError): + provider.cube(**args) + + +def test_edr_provider_cube_request_invalid_altitude( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the cube method of the EDR Provider class with an invalid altitude. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + args["z"] = "invalid" + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + with pytest.raises(ProviderInvalidQueryError): + provider.position(**args) + + +def test_edr_provider_area_request_valid_wkt(layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any]): + """Test the area method of the EDR Provider class with a valid wkt. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + del args["bbox"] + args["wkt"] = Polygon(((-180.0, -90.0), (-180.0, 90.0), (180.0, -90.0), (180.0, 90.0))) + parameter_name = single_layer_cube_args_internal["select_properties"][0] + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + response = provider.area(**args) + response = get_json_with_cleanup(response["fp"]) + + assert set(response["domain"]["ranges"][parameter_name]["axisNames"]) == {"x", "y", "t"} + assert np.prod(np.array(response["domain"]["ranges"][parameter_name]["shape"])) == len( + response["domain"]["ranges"][parameter_name]["values"] + ) + + +def test_edr_provider_area_request_invalid_wkt( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the area method of the EDR Provider class with an invalid wkt. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + del args["bbox"] + args["wkt"] = "invalid" + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + with pytest.raises(ProviderInvalidQueryError): + provider.area(**args) + + +def test_edr_provider_cube_request_invalid_datetime( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the area method of the EDR Provider class with an invalid datetime. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + args["datetime_"] = "10_24/2025" + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + with pytest.raises(ProviderInvalidQueryError): + provider.cube(**args) + + +def test_edr_provider_cube_request_valid_geotiff_format( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the query method of the EDR Provider class with a valid geotiff request. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + args["format_"] = "geotiff" + parameter_name = single_layer_cube_args_internal["select_properties"][0] + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + response = provider.cube(**args) + data = get_bytes_with_cleanup(response["fp"]) + + assert response["fn"] == f"{parameter_name}.tif" + assert len(data) > 0 + + +def test_edr_provider_cube_request_valid_geotiff_format_multiple_parameters( + layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] +): + """Test the query method of the EDR Provider class with a valid geotiff request. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_internal + args["format_"] = "geotiff" + + # Set the properties argument as multiple layers from the same group/collection + group = layers[0].group + selected_layers = [layer.identifier for layer in layers if layer.group == group] + args["select_properties"] = selected_layers + + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers) + + response = provider.cube(**args) + buffer = io.BytesIO(get_bytes_with_cleanup(response["fp"])) + assert response["fn"] == f"{group}.zip" + assert zipfile.is_zipfile(buffer) + with zipfile.ZipFile(buffer, "r") as zf: + namelist = zf.namelist() + assert len(namelist) > 0 + assert all(f"{layer}.tif" in namelist for layer in selected_layers) + + +def test_edr_provider_datetime_single_value(): + """Test the datetime interpreter method of the EDR Provider class with a single datetime value.""" + time_string = "2025-10-24" + available_times = ["2025-10-24", "2025-10-25", "2025-10-26", "2025-10-27", "2025-10-28"] + available_times = [np.datetime64(time) for time in available_times] + expected_times = available_times[0] + + time_coords = EdrProvider.interpret_time_coordinates(available_times, time_string, None, None) + + assert time_coords is not None + np.testing.assert_array_equal(time_coords["time"].coordinates, expected_times) + + +def test_edr_provider_datetime_range_closed(): + """Test the datetime interpreter method of the EDR Provider class with a closed datetime range.""" + time_string = "2025-10-24/2025-10-26" + available_times = ["2025-10-24", "2025-10-25", "2025-10-26", "2025-10-27", "2025-10-28"] + available_times = [np.datetime64(time) for time in available_times] + expected_times = available_times[0:3] + + time_coords = EdrProvider.interpret_time_coordinates(available_times, time_string, None, None) + + assert time_coords is not None + np.testing.assert_array_equal(time_coords["time"].coordinates, expected_times) + + +def test_edr_provider_datetime_open_start(): + """Test the datetime interpreter method of the EDR Provider class with a open datetime start.""" + time_string = "../2025-10-27" + available_times = ["2025-10-24", "2025-10-25", "2025-10-26", "2025-10-27", "2025-10-28"] + available_times = [np.datetime64(time) for time in available_times] + expected_times = available_times[0:4] + + time_coords = EdrProvider.interpret_time_coordinates(available_times, time_string, None, None) + + assert time_coords is not None + np.testing.assert_array_equal(time_coords["time"].coordinates, expected_times) + + +def test_edr_provider_datetime_open_end(): + """Test the datetime interpreter method of the EDR Provider class with a open datetime end.""" + time_string = "2025-10-25/.." + available_times = ["2025-10-24", "2025-10-25", "2025-10-26", "2025-10-27", "2025-10-28"] + available_times = [np.datetime64(time) for time in available_times] + expected_times = available_times[1:] + + time_coords = EdrProvider.interpret_time_coordinates(available_times, time_string, None, None) + + assert time_coords is not None + np.testing.assert_array_equal(time_coords["time"].coordinates, expected_times) + + +def test_edr_provider_datetime_invalid_string(): + """Test the datetime interpreter method of the EDR Provider class with an invalid string.""" + time_string = "2025-10-25/../../.." + available_times = ["2025-10-24", "2025-10-25", "2025-10-26", "2025-10-27", "2025-10-28"] + available_times = [np.datetime64(time) for time in available_times] + + with pytest.raises(ProviderInvalidQueryError): + EdrProvider.interpret_time_coordinates(available_times, time_string, None, None) + + +def test_edr_provider_get_altitudes(): + """Test the get altitudes method of the EDR Provider class with a layer containing altitude data.""" + latitude = np.arange(1, 5) + longitude = np.arange(1, 5) + altitude = np.arange(1, 10) + data = np.random.default_rng(1).random((len(latitude), len(longitude), len(altitude))) + coords = podpac.Coordinates([latitude, longitude, altitude], dims=["lat", "lon", "alt"]) + node = podpac.data.Array(source=data, coordinates=coords) + layer = pogc.Layer(node=node, identifier="Test") + + np.testing.assert_array_equal(EdrProvider.get_altitudes([layer]), altitude) + + +def test_edr_provider_altitude_single_value(): + """Test the altitude interpreter method of the EDR Provider class with a single datetime value.""" + altitude_string = "10" + available_altitudes = [0.0, 5.0, 10.0, 15.0, 20.0] + expected_altitudes = [10.0] + + altitude_coords = EdrProvider.interpret_altitude_coordinates(available_altitudes, altitude_string, None) + + assert altitude_coords is not None + np.testing.assert_array_equal(altitude_coords["alt"].coordinates, expected_altitudes) + + +def test_edr_provider_altitude_range_closed(): + """Test the altitude interpreter method of the EDR Provider class with a closed datetime range.""" + altitude_string = "10/20" + available_altitudes = [0.0, 5.0, 10.0, 15.0, 20.0] + expected_altitudes = [10.0, 15.0, 20.0] + + altitude_coords = EdrProvider.interpret_altitude_coordinates(available_altitudes, altitude_string, None) + + assert altitude_coords is not None + np.testing.assert_array_equal(altitude_coords["alt"].coordinates, expected_altitudes) + + +def test_edr_provider_altitude_repeating_interval(): + """Test the altitude interpreter method of the EDR Provider class with a repeating interval.""" + altitude_string = "R2/5/5" + available_altitudes = [0.0, 5.0, 10.0, 15.0, 20.0] + expected_altitudes = [5.0, 10.0] + + altitude_coords = EdrProvider.interpret_altitude_coordinates(available_altitudes, altitude_string, None) + + assert altitude_coords is not None + np.testing.assert_array_equal(altitude_coords["alt"].coordinates, expected_altitudes) + + +def test_edr_provider_altitude_list(): + """Test the altitude interpreter method of the EDR Provider class with a list.""" + altitude_string = "5,10,15" + available_altitudes = [0.0, 5.0, 10.0, 15.0, 20.0] + expected_altitudes = [5.0, 10.0, 15.0] + + altitude_coords = EdrProvider.interpret_altitude_coordinates(available_altitudes, altitude_string, None) + + assert altitude_coords is not None + np.testing.assert_array_equal(altitude_coords["alt"].coordinates, expected_altitudes) + + +def test_edr_provider_altitude_invalid_string(): + """Test the altitude interpreter method of the EDR Provider class with an invalid string.""" + altitude_string = "../20" + available_altitudes = [0.0, 5.0, 10.0, 15.0, 20.0] + + with pytest.raises(ProviderInvalidQueryError): + EdrProvider.interpret_altitude_coordinates(available_altitudes, altitude_string, None) + + +def test_edr_provider_crs_interpreter_default_value(): + """Test the CRS interpretation returns a default value when the argument is None.""" + + assert EdrProvider.interpret_crs(None) == pyproj.CRS(settings.crs_84_uri_format).to_wkt() + + +def test_edr_provider_crs_interpreter_valid_value(): + """Test the CRS interpretation returns a valid value when the argument is acceptable.""" + assert ( + EdrProvider.interpret_crs(settings.epsg_4326_uri_format) == pyproj.CRS(settings.epsg_4326_uri_format).to_wkt() + ) + + +def test_edr_provider_crs_interpreter_invalid_value(): + """Test the CRS interpretation raises an exception when an invalid argument is provided.""" + with pytest.raises(ProviderInvalidQueryError): + EdrProvider.interpret_crs("epsp:4444") + + +def test_edr_provider_crs_converter(): + """Test the CRS converter returns latitude and longitude data properly.""" + x = [1, 2, 3] + y = [3, 4, 5] + + # EPSG:4326 specifies x (latitude) and y (longitude) + lon = y + lat = x + + assert EdrProvider.crs_converter(x, y, crs=settings.epsg_4326_uri_format) == (lon, lat) diff --git a/ogc/edr/test/test_edr_routes.py b/ogc/edr/test/test_edr_routes.py new file mode 100644 index 0000000..4931f99 --- /dev/null +++ b/ogc/edr/test/test_edr_routes.py @@ -0,0 +1,426 @@ +import os +import json +import numpy as np +import tempfile +from pygeoapi.api import APIRequest +from http import HTTPStatus +from typing import Dict, List, Any +from werkzeug.test import create_environ +from werkzeug.wrappers import Request +from werkzeug.datastructures import ImmutableMultiDict +from ogc import podpac as pogc +from ogc.edr.edr_routes import EdrRoutes + + +def mock_request(request_args: Dict[str, Any] | None = None) -> APIRequest: + """Creates a mock request for EDR routes to use. + + + Parameters + ---------- + request_args: Dict[str, Any] | None, optional + The dictionary for query string arguments, by default None. + + Returns + ------- + APIRequest + Mock API request for route testing. + """ + request_args = request_args if request_args is not None else {} + environ = create_environ(base_url="http://127.0.0.1:5000/ogc/edr") + request = Request(environ) + request.args = ImmutableMultiDict(request_args.items()) + return APIRequest(request, ["en"]) + + +def test_edr_routes_static_files_valid_path(): + """Test the EDR static routes with a valid static file path.""" + request = mock_request() + edr_routes = EdrRoutes(layers=[]) + + headers, status, _ = edr_routes.static_files(request, "img/logo.png") + + assert status == HTTPStatus.OK + assert headers["Content-Type"] == "image/png" + + +def test_edr_routes_static_files_invalid_path(): + """Test the EDR static routes with an invalid static file path.""" + request = mock_request() + edr_routes = EdrRoutes(layers=[]) + + _, status, _ = edr_routes.static_files(request, "invalid") + + assert status == HTTPStatus.NOT_FOUND + + +def test_edr_routes_landing_page(): + """Test the EDR landing page for a response.""" + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=[]) + + headers, status, _ = edr_routes.landing_page(request) + + assert status == HTTPStatus.OK + assert headers["Content-Type"] == "application/json" + + +def test_edr_routes_landing_page_html(): + """Test the EDR landing page for a response.""" + request = mock_request({"f": "html"}) + edr_routes = EdrRoutes(layers=[]) + + headers, status, _ = edr_routes.landing_page(request) + + assert status == HTTPStatus.OK + assert headers["Content-Type"] == "text/html" + + +def test_edr_routes_conformance(layers: List[pogc.Layer]): + """Test the EDR conformance for a response. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers) + + _, status, content = edr_routes.conformance(request) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert len(response["conformsTo"]) > 0 + assert "https://www.opengis.net/spec/ogcapi-edr-1/1.1/conf/core" in response["conformsTo"] + + +def test_edr_routes_api(): + """Test the EDR api documentation for a response.""" + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=[]) + + _, status, content = edr_routes.openapi(request) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert response["paths"]["/"] + assert response["paths"]["/api"] + assert response["paths"]["/conformance"] + assert response["paths"]["/collections"] + + +def test_edr_routes_describe_collections(layers: List[pogc.Layer]): + """Test the EDR collections description for a response. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers) + collections = {layer.group for layer in layers} + + _, status, content = edr_routes.describe_collections(request, collection_id=None) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert len(response["collections"]) == len(collections) + + response_collection_ids = [collection["id"] for collection in response["collections"]] + + assert response_collection_ids == list(collections) + + +def test_edr_routes_describe_collection(layers: List[pogc.Layer]): + """Test the EDR collection description for a response. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers) + collection_id = layers[0].group + collection_layers = [layer for layer in layers if layer.group == collection_id] + + _, status, content = edr_routes.describe_collections(request, collection_id=collection_id) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert response["id"] == collection_id + assert list(response["parameter_names"].keys()) == [layer.identifier for layer in collection_layers] + assert list(response["data_queries"].keys()) == ["instances"] + + +def test_edr_routes_describe_collection_without_instance(layers_no_instance: List[pogc.Layer]): + """Test the EDR collection description for a response without instances. + The instance data query should still be available so it can return empty. + + Parameters + ---------- + layers_no_instance : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers_no_instance) + collection_id = layers_no_instance[0].group + collection_layers = [layer for layer in layers_no_instance if layer.group == collection_id] + + _, status, content = edr_routes.describe_collections(request, collection_id=collection_id) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert response["id"] == collection_id + assert list(response["parameter_names"].keys()) == [layer.identifier for layer in collection_layers] + assert set(response["data_queries"].keys()) == {"position", "cube", "area", "instances"} + + +def test_edr_routes_describe_instances(layers: List[pogc.Layer]): + """Test the EDR instances description for a response. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers) + collection_id = layers[0].group + time_instances = set() + for layer in layers: + if layer.group == collection_id: + time_instances.update(layer.time_instances()) + + _, status, content = edr_routes.describe_instances(request, collection_id=collection_id, instance_id=None) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert len(response["instances"]) == len(time_instances) + + response_time_instances_ids = [instance["id"] for instance in response["instances"]] + assert response_time_instances_ids == list(time_instances) + + +def test_edr_routes_describe_instances_without_instances(layers_no_instance: List[pogc.Layer]): + """Test the EDR instances description for a response when no instances are available. + + Parameters + ---------- + layers_no_instance : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers_no_instance) + collection_id = layers_no_instance[0].group + + _, status, content = edr_routes.describe_instances(request, collection_id=collection_id, instance_id=None) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert len(response["instances"]) == 0 + + +def test_edr_routes_describe_instance(layers: List[pogc.Layer]): + """Test the EDR instance description for a response. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers) + collection_id = layers[0].group + instance_id = next(iter(layers[0].time_instances())) + + _, status, content = edr_routes.describe_instances(request, collection_id=collection_id, instance_id=instance_id) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert response["id"] == instance_id + assert list(response["data_queries"].keys()) == ["position", "cube", "area"] + + +def test_edr_routes_collection_query(layers: List[pogc.Layer], single_layer_cube_args: Dict[str, Any]): + """Test the EDR collection query for a reponse. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args : Dict[str, Any] + Single layer arguments provided by a test fixture. + """ + collection_id = layers[0].group + instance_id = next(iter(layers[0].time_instances())) + parameter_name = single_layer_cube_args["parameter-name"][0] + request = mock_request(single_layer_cube_args) + edr_routes = EdrRoutes(layers=layers) + + _, status, content = edr_routes.collection_query( + request, + collection_id=collection_id, + instance_id=instance_id, + query_type="cube", + ) + content = json.loads("".join(content)) + + assert status == HTTPStatus.OK + + assert set(content["domain"]["ranges"][parameter_name]["axisNames"]) == {"x", "y", "t"} + assert np.prod(np.array(content["domain"]["ranges"][parameter_name]["shape"])) == len( + content["domain"]["ranges"][parameter_name]["values"] + ) + + +def test_edr_routes_collection_query_geotiff_format(layers: List[pogc.Layer], single_layer_cube_args: Dict[str, Any]): + """Test the EDR collection query for a GeoTiff formatted reponse. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args : Dict[str, Any] + Single layer arguments provided by a test fixture. + """ + collection_id = layers[0].group + instance_id = next(iter(layers[0].time_instances())) + single_layer_cube_args["f"] = "geotiff" + request = mock_request(single_layer_cube_args) + edr_routes = EdrRoutes(layers=layers) + + headers, status, _ = edr_routes.collection_query( + request, + collection_id=collection_id, + instance_id=instance_id, + query_type="cube", + ) + + assert status == HTTPStatus.OK + assert headers["Content-Disposition"] == f"attachment; filename={layers[0].identifier}.tif" + + +def test_edr_routes_collection_query_invalid_type(layers: List[pogc.Layer], single_layer_cube_args: Dict[str, Any]): + """Test the EDR collection query with an invalid query type. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args : Dict[str, Any] + Single layer arguments provided by a test fixture. + """ + collection_id = layers[0].group + instance_id = next(iter(layers[0].time_instances())) + request = mock_request(single_layer_cube_args) + edr_routes = EdrRoutes(layers=layers) + + _, status, _ = edr_routes.collection_query( + request, + collection_id=collection_id, + instance_id=instance_id, + query_type="corridor", + ) + + assert status == HTTPStatus.BAD_REQUEST + + +def test_edr_routes_collection_query_invalid_bbox(layers: List[pogc.Layer], single_layer_cube_args: Dict[str, Any]): + """Test the EDR collection query with an invalid bounding box. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args : Dict[str, Any] + Single layer arguments provided by a test fixture. + """ + single_layer_cube_args["bbox"] = "invalid" + request = mock_request(single_layer_cube_args) + edr_routes = EdrRoutes(layers=layers) + + _, status, _ = edr_routes.collection_query( + request, + collection_id=layers[0].group, + instance_id=next(iter(layers[0].time_instances())), + query_type="cube", + ) + + assert status == HTTPStatus.BAD_REQUEST + + +def test_edr_routes_collection_query_missing_parameter( + layers: List[pogc.Layer], single_layer_cube_args: Dict[str, Any] +): + """Test the EDR colletion query with a missing parameter. All parameters are expected to be returned. + + Parameters + ---------- + layers : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args : Dict[str, Any] + Single layer arguments provided by a test fixture. + """ + del single_layer_cube_args["parameter-name"] + request = mock_request(single_layer_cube_args) + edr_routes = EdrRoutes(layers=layers) + + _, status, content = edr_routes.collection_query( + request, + collection_id=layers[0].group, + instance_id=next(iter(layers[0].time_instances())), + query_type="cube", + ) + content = json.loads("".join(content)) + + assert status == HTTPStatus.OK + assert content["domain"]["ranges"].keys() == {layer.identifier for layer in layers} + + +def test_edr_routes_request_url_updates_configuration_url(): + """Test the EDR routes request base URL updates the configuration URL.""" + request_url = "http://test:5000/ogc/edr/static/img/logo.png" + expected_config_url = "http://test:5000/ogc/edr" + request = mock_request({"base_url": request_url}) + edr_routes = EdrRoutes(layers=[]) + + _, status, _ = edr_routes.static_files(request, "img/logo.png") + + assert status == HTTPStatus.OK + assert edr_routes.api.config["server"]["url"] == expected_config_url + + +def test_edr_routes_file_cleanup_unused_generator(): + """Test the EDR routes properly cleans up files if the generator goes unused.""" + named_file = tempfile.NamedTemporaryFile(mode="w+", delete=False) + generator = EdrRoutes.file_generator_with_cleanup(named_file.name, "r") + + assert os.path.exists(named_file.name) + + del generator + + assert not os.path.exists(named_file.name) + + +def test_edr_routes_file_cleanup_generator_error(): + """Test the EDR routes properly cleans up files if the generator has an error.""" + named_file = tempfile.NamedTemporaryFile(mode="w+", delete=False) + generator = EdrRoutes.file_generator_with_cleanup(named_file.name, "r") + + assert os.path.exists(named_file.name) + + try: + next(generator) + generator.close() + except StopIteration: + pass + + assert not os.path.exists(named_file.name) diff --git a/ogc/ogc_common.py b/ogc/ogc_common.py index 0341201..4bf3dd4 100755 --- a/ogc/ogc_common.py +++ b/ogc/ogc_common.py @@ -1,6 +1,7 @@ import logging -import lxml, lxml.etree +import lxml +import lxml.etree import numpy as np import traitlets as tl @@ -27,9 +28,8 @@ def to_xml(self): raise NotImplementedError("XML Serialization not implemented.") def _load_xml_doc(self, xml_doc): - """ Override this method with code that unpacks contents of XML into the traits object.""" + """Override this method with code that unpacks contents of XML into the traits object.""" raise NotImplementedError("XML Parsing not implemented.") - return self def load_from_kv(self, args): self._load_from_kv(args) @@ -72,7 +72,6 @@ def validate(self): def to_xml(self): raise NotImplementedError() - return "%s" % self.value class TemporalSubset(XMLNode): @@ -89,17 +88,9 @@ class TemporalSubset(XMLNode): def validate(self): raise NotImplementedError() - for val in ( - self.lower_corner[0], - self.lower_corner[1], - self.upper_corner[0], - self.upper_corner[1], - ): - assert np.isfinite(val), "error: time values must be finite" def to_xml(self): raise NotImplementedError() - return "%s" % self.value class WCSException(Exception): @@ -112,7 +103,7 @@ def __init__( """ exception_code: 'NoApplicableCode', 'InvalidFormat', 'CoverageNotDefined', 'MissingParameterValue', 'InvalidParameterValue' """ - super(WCSException, self).__init__(exception_text) + super().__init__(exception_text, exception_code, locator) self.exception_text = exception_text self.exception_code = exception_code @@ -133,7 +124,5 @@ def to_xml(self): {exception_text} -""".format( - self=self, exception_text=exception_text - ) +""".format(self=self, exception_text=exception_text) return xml diff --git a/ogc/podpac.py b/ogc/podpac.py index 2490435..e2103cd 100755 --- a/ogc/podpac.py +++ b/ogc/podpac.py @@ -6,17 +6,16 @@ import podpac from podpac.core.coordinates import Coordinates import traitlets as tl - +from typing import List from matplotlib import pyplot as plt import matplotlib as mpl import io -from PIL import Image import numpy as np -import xarray as xr import json import textwrap import re - +from datetime import datetime +from ogc.settings import EDR_TIME_INSTANCE_DIMENSION def _uppercase_for_dict_keys(lower_dict): @@ -49,12 +48,76 @@ class Layer(ogc.Layer): node = tl.Instance(klass=podpac.Node, allow_none=True) convert_requests_to_default_crs = tl.Bool(default_value=True) + crs84_colon_format = "CRS:84" def __init__(self, **kwargs): super().__init__(**kwargs) if self.node is not None and self.node.style.enumeration_legend: self._style.is_enumerated = True + def time_instances(self) -> List[str]: + """Retrieve the time instances available for the layer. + + Returns + ------- + List[str] + List of available time instances as a strings. + """ + time_instances = set() + coordinates = self.get_coordinates() + + # Time instances are created if a node has a time instance dimension. + if coordinates is not None and EDR_TIME_INSTANCE_DIMENSION in coordinates.udims: + time_instances.update( + [ + time.astype("datetime64[ms]").astype(datetime).isoformat() + for time in coordinates[EDR_TIME_INSTANCE_DIMENSION].coordinates + ] + ) + + return list(time_instances) + + def get_coordinates(self) -> Coordinates | None: + """Retrieve the coordinates from the node. + + Returns + ------- + Coordinates | None + Coordinates from the node or None if not found. + """ + if self.node is None: + return None + + coordinates_list = self.node.find_coordinates() + dimension_set = set() + coordinates = None + + for coords in coordinates_list: + dimension_set.update(coords.udims) + if coordinates is None or len(coords.udims) > len(coordinates.udims): + coordinates = coords + + if coordinates is not None and not all(dim in coordinates.udims for dim in dimension_set): + raise ValueError("Not all node coordinate dimensions contained in the layer coordinates.") + + return coordinates + + def get_units(self) -> str | None: + """Retrieve the units from the node. + + Returns + ------- + str | None + The units from the node. + """ + units = None + if self.node is not None and self.node.units is not None: + units = self.node.units + elif self.node is not None and self.node.style is not None: + units = self.node.style.units + + return units + def get_node(self, args): return self.node @@ -70,24 +133,18 @@ def get_map(self, args): orig_w = args["WIDTH"] # don't reduce # pixels to less than MIN_N (unless orig request was smaller) MIN_N = 8 - args["HEIGHT"] = max( - min(int(orig_h), MIN_N), int(round(int(orig_h) / rescale)) - ) - args["WIDTH"] = max( - min(int(orig_w), MIN_N), int(round(int(orig_w) / rescale)) - ) + args["HEIGHT"] = max(min(int(orig_h), MIN_N), int(round(int(orig_h) / rescale))) + args["WIDTH"] = max(min(int(orig_w), MIN_N), int(round(int(orig_w) / rescale))) else: rescale = 0 - if "CRS" in args and args["CRS"].upper() == "CRS:84".upper(): + if "CRS" in args and args["CRS"].upper() == self.crs84_colon_format.upper(): args["CRS"] = "CRS84" # for pyproj if self.convert_requests_to_default_crs and "DEFAULT_CRS" in podpac.settings: # PODPAC transforms input coords to crs of datasource recursively # every time eval is used in a Node included in its dependency tree. # This optimization can be used if most datasources are stored in the same crs. - coords = Coordinates.from_url(args).transform( - podpac.settings["DEFAULT_CRS"] - ) + coords = Coordinates.from_url(args).transform(podpac.settings["DEFAULT_CRS"]) else: coords = Coordinates.from_url(args) @@ -98,17 +155,10 @@ def get_map(self, args): if rescale > 1.0: args["HEIGHT"] = orig_h args["WIDTH"] = orig_w - if ( - self.convert_requests_to_default_crs - and "DEFAULT_CRS" in podpac.settings - ): - rescaledcoords = Coordinates.from_url(args).transform( - podpac.settings["DEFAULT_CRS"] - ) + if self.convert_requests_to_default_crs and "DEFAULT_CRS" in podpac.settings: + rescaledcoords = Coordinates.from_url(args).transform(podpac.settings["DEFAULT_CRS"]) else: rescaledcoords = Coordinates.from_url(args) - # rescaled_node = podpac.data.Array(source=output, coordinates=coords, style = node.style) - # output = rescaled_node.eval(rescaledcoords) output = output.interp( lat=rescaledcoords["lat"].coordinates + 1e-6, lon=rescaledcoords["lon"].coordinates + 1e-6, @@ -122,15 +172,13 @@ def get_map(self, args): def get_coverage(self, args): args = _uppercase_for_dict_keys(args) - if "CRS" in args and args["CRS"].upper() == "CRS:84".upper(): + if "CRS" in args and args["CRS"].upper() == self.crs84_colon_format.upper(): args["CRS"] = "CRS84" # for pyproj if self.convert_requests_to_default_crs and "DEFAULT_CRS" in podpac.settings: # PODPAC transforms input coords to crs of datasource recursively # every time eval is used in a Node included in its dependency tree. # This optimization can be used if most datasources are stored in the same crs. - coords = Coordinates.from_url(args).transform( - podpac.settings["DEFAULT_CRS"] - ) + coords = Coordinates.from_url(args).transform(podpac.settings["DEFAULT_CRS"]) else: coords = Coordinates.from_url(args) @@ -184,17 +232,18 @@ class LegendGraphic(tl.HasTraits): """ A class for generating legend graphics with adjustable properties such as width, height, fonts, and color mappings. """ + width = tl.Float(default_value=0.7) # inches max_width = tl.Float(default_value=1.5) # inches - min_width = tl.Float(default_value=0.8) # inches + min_width = tl.Float(default_value=0.8) # inches height = tl.Float(default_value=2.5) # inches max_heigth = tl.Float(default_value=5.5) # inches dpi = tl.Float(default_value=100) # pixels per inch - units_fontsize = tl.Float(default_value=13) # used for units text at top of legend - colorbar_fontsize = tl.Float(default_value=10) # used for tick marks of colorbar - enumeration_fontsize = tl.Float(default_value=16) # used for labels of categorical legends - enumeration_min_fontsize = tl.Float(default_value=5)# minimum fontsize for categorical legends with lots of labels - max_unit_chars = tl.Float(default_value=16) # maximum characters allowed in a line for units + units_fontsize = tl.Float(default_value=13) # used for units text at top of legend + colorbar_fontsize = tl.Float(default_value=10) # used for tick marks of colorbar + enumeration_fontsize = tl.Float(default_value=16) # used for labels of categorical legends + enumeration_min_fontsize = tl.Float(default_value=5) # minimum fontsize for categorical legends with lots of labels + max_unit_chars = tl.Float(default_value=16) # maximum characters allowed in a line for units units = tl.Unicode(default_value=tl.Undefined, allow_none=True) img_format = tl.Enum(values=["png", "pdf", "ps", "eps", "svg"], default_value="png") cmap = tl.Instance(klass=mpl.colors.Colormap, default_value=mpl.cm.viridis) @@ -205,9 +254,7 @@ class LegendGraphic(tl.HasTraits): default_value=None, allow_none=True, ) - enumeration_colors = tl.Dict( - key_trait=tl.Int(), default_value=None, allow_none=True - ) + enumeration_colors = tl.Dict(key_trait=tl.Int(), default_value=None, allow_none=True) clim = tl.List(default_value=[None, None]) def __init__(self, *args, **kwargs): @@ -219,7 +266,7 @@ def __init__(self, *args, **kwargs): def legend_image(self): """ Generates the legend image based on provided parameters. - + Returns: io.BytesIO: A byte stream containing the legend image in the specified format. """ @@ -257,19 +304,21 @@ def check_if_units_need_to_be_wrapped(self, fig): """ Checks if the unit text needs wrapping. If so, it divieds the string into bins the size of max_unit_chars. - + Args: fig (matplotlib.figure.Figure): The figure object. - + Returns: tuple: Updated figure, boolean flag indicating wrapping, and the wrapped unit string. """ units = "[%s]" % self.units - needs_wrap = len(units)>self.max_unit_chars #if characters are greater than 16 then wrap text, shrink colorbar + needs_wrap = ( + len(units) > self.max_unit_chars + ) # if characters are greater than 16 then wrap text, shrink colorbar # currently only allows for 2 lines wrapped_units = self.wrap_text(units, self.max_unit_chars) # format exponents - units = re.sub(r"\^(\d+)", r"$^{\1}\!$", units) + units = re.sub(r"\^(\d+)", r"$^{\1}\!$", units) units = re.sub(r"\^-(\d+)", r"$^{-\1}\!$", units) # add units to figure fig.text( @@ -279,28 +328,30 @@ def check_if_units_need_to_be_wrapped(self, fig): fontsize=self.units_fontsize, horizontalalignment="center", verticalalignment="top", - wrap=True + wrap=True, ) return fig, needs_wrap, wrapped_units - + def adjust_fig_height_for_wrapped_units(self, fig, wrapped_units): """ Adjusts figure height to accommodate wrapped units. Will increase figure size for each line of wrapped text until maximum figure height is reached - + Args: fig (matplotlib.figure.Figure): The figure object. wrapped_units (str): The wrapped unit string. - + Returns: tuple: Updated figure and adjusted axis. """ - + # wrap text and increase height of figure added_lines = wrapped_units.count("\n") added_height = self.additional_height_for_wrapped_text(added_lines, self.units_fontsize) - fig_height = min(self.max_heigth, added_height + self.height) # add extra height to figure ensure it is less than 6.5 in - # adjust fig size to fit units + fig_height = min( + self.max_heigth, added_height + self.height + ) # add extra height to figure ensure it is less than 6.5 in + # adjust fig size to fit units fig.set_size_inches(self.max_width, fig_height, forward=True) # Standard height ratio (before adjustments) @@ -314,118 +365,122 @@ def adjust_fig_height_for_wrapped_units(self, fig, wrapped_units): def adjust_fig_width_for_unwrapped_units(self, fig, units_str): """ - Adjusts figure width to accommodate units. + Adjusts figure width to accommodate units. Expects units to be under max_unit_chars - + Args: fig (matplotlib.figure.Figure): The figure object. wrapped_units (str): The wrapped unit string. - + Returns: tuple: Updated figure and adjusted axis. """ # add space for units ax = fig.add_axes([0.25, 0.05, 0.15, 0.80]) # Estimates the max label width assuming fontsize 10 - max_label_width_units = self.get_max_text_width([units_str], self.units_fontsize) - + max_label_width_units = self.get_max_text_width([units_str], self.units_fontsize) + # add color bar and see if fig width needs to be bigger for tick marks norm = mpl.colors.Normalize(vmin=self.clim[0], vmax=self.clim[1]) cb = mpl.colorbar.ColorbarBase(ax, cmap=self.cmap, norm=norm) - + # Convert ticks to float32 to avoid errors converting float64 to string - tick_labels = [str(t) for t in cb.ax.get_yticks().astype('f4')] + tick_labels = [str(t) for t in cb.ax.get_yticks().astype("f4")] max_label_width_ticks = self.get_max_text_width(tick_labels, self.colorbar_fontsize) - #define minimum width need or max_label width + some extra margin - fig_width = max(self.min_width, max_label_width_units+0.2, max_label_width_ticks+0.4) + # define minimum width need or max_label width + some extra margin + fig_width = max(self.min_width, max_label_width_units + 0.2, max_label_width_ticks + 0.4) fig.set_size_inches(fig_width, self.height, forward=True) return fig, ax - + def create_enumeration_legend(self, fig, ax): """ - Creates a legend for categorical data. + Creates a legend for categorical data. Dynamically adjusts figure size based on number of labels and label text length Adds colorbar to figure - + Args: fig (matplotlib.figure.Figure): The figure object. ax (matplotlib.axes.Axes): The axis object. - + Returns: matplotlib.figure.Figure: The updated figure. """ enum_values = list(self.enumeration_colors.keys()) enum_colors = list(self.enumeration_colors.values()) enum_labels = list(self.enumeration_legend.values()) - - # Dynamically adjust font size based on the number of ticks - font_size = max(self.enumeration_min_fontsize, self.enumeration_fontsize - (len(enum_values) * 0.35)) # Scale font size - # Change legend dynamically - max_label_width = self.get_max_text_width(enum_labels, font_size) # Estimates the max label width assuming fontsize 10 + # Dynamically adjust font size based on the number of ticks + font_size = max( + self.enumeration_min_fontsize, self.enumeration_fontsize - (len(enum_values) * 0.35) + ) # Scale font size + + # Change legend dynamically + max_label_width = self.get_max_text_width( + enum_labels, font_size + ) # Estimates the max label width assuming fontsize 10 fig_width = 0.5 + max_label_width # Base width + label-dependent width fig_height = min(self.max_heigth, len(enum_colors) * 0.25) # Adjust height based on number of labels fig.set_size_inches(fig_width, fig_height, forward=True) - - self.cmap = mpl.colors.ListedColormap(enum_colors) #create categorical colomap to replace previous cmap - bounds = np.array([val-0.5 for val in np.arange(1,len(enum_values)+2)]) + + self.cmap = mpl.colors.ListedColormap(enum_colors) # create categorical colomap to replace previous cmap + bounds = np.array([val - 0.5 for val in np.arange(1, len(enum_values) + 2)]) norm = mpl.colors.BoundaryNorm(bounds, self.cmap.N) cb = mpl.colorbar.ColorbarBase( ax, cmap=self.cmap, norm=norm, - ticks=np.arange(1,len(self.enumeration_legend)+1), + ticks=np.arange(1, len(self.enumeration_legend) + 1), ) if self.enumeration_legend: cb.ax.set_yticklabels(enum_labels, fontsize=font_size) return fig - + def adjust_fig_width_for_long_tick_marks(self, fig, ax): """ - Adjusts figure width to accommodate tick marks that are long. - + Adjusts figure width to accommodate tick marks that are long. + Args: fig (matplotlib.figure.Figure): The figure object. ax (matplotlib.axes.Axes): The axis object. - + Returns: matplotlib.figure.Figure: The updated figure. """ norm = mpl.colors.Normalize(vmin=self.clim[0], vmax=self.clim[1]) cb = mpl.colorbar.ColorbarBase(ax, cmap=self.cmap, norm=norm) - tick_labels = [str(t) for t in cb.ax.get_yticks().astype('f4')] # Convert ticks to strings + tick_labels = [str(t) for t in cb.ax.get_yticks().astype("f4")] # Convert ticks to strings max_label_width = self.get_max_text_width(tick_labels, self.colorbar_fontsize) fig_width = max_label_width + 0.4 - fig.set_size_inches(fig_width, self.height,forward=True) + fig.set_size_inches(fig_width, self.height, forward=True) return fig def get_max_text_width(self, labels, font_size=10): """ Estimates the maximum width of given text labels in inches. - + Args: labels (list of str): List of text labels. font_size (int): Font size used for labels. - + Returns: float: The maximum width of the labels in inches. """ fig, ax = plt.subplots() # Create a temporary figure renderer = fig.canvas.get_renderer() # Get renderer to measure text - + text_widths = [] for label in labels: text = ax.text(0, 0, label, fontsize=font_size) # Attach text to the figure text_widths.append(text.get_window_extent(renderer).width) - + plt.close(fig) # Close temporary figure return max(text_widths) / self.dpi # Convert pixels to inches - + def wrap_text(self, text, max_width_chars=16): """ Split string into sections of length max_width_chars @@ -435,16 +490,16 @@ def wrap_text(self, text, max_width_chars=16): def additional_height_for_wrapped_text(self, added_lines_num, font_size): """ Calculates additional height needed for wrapped text based on number lines needed - + Args: added_lines_num (int): Number of additional wrapped lines. font_size (int): Font size of the text. - + Returns: float: Additional height required in inches. """ font_height_px = font_size * (self.dpi / 72) # Convert to pixels font_height_in = font_height_px / self.dpi # Convert pixels to inches - additional_height = font_height_in*added_lines_num + additional_height = font_height_in * added_lines_num return additional_height diff --git a/ogc/servers.py b/ogc/servers.py index 835dd3f..2c1093f 100755 --- a/ogc/servers.py +++ b/ogc/servers.py @@ -7,21 +7,36 @@ """ import re +import xml.sax.saxutils from flask import Flask, request, Response, make_response, send_file import six -import traceback import logging +from typing import Callable +from werkzeug.datastructures import ImmutableMultiDict from ogc.ogc_common import WCSException +from pygeoapi.api import APIRequest +from pygeoapi.util import get_api_rules +from . import settings logger = logging.getLogger(__name__) +def _check_query_string(raw_qs: bytes) -> None: + """Raise WCSException if the raw query string exceeds the maximum allowed length or contains invalid UTF-8.""" + if len(raw_qs) > settings.MAX_QUERY_STRING_BYTES: + raise WCSException("Request query string exceeds maximum allowed length.") + try: + raw_qs.decode("utf-8") + except UnicodeDecodeError: + raise WCSException("Request contains invalid UTF-8 encoding.") + + def respond_xml(doc, status=200): # First, validate that XML can be parsed. from lxml import etree - root = etree.fromstring(doc.encode("ascii")) + etree.fromstring(doc.encode("ascii")) # Then, return w/ proper content type return Response(doc, mimetype="text/xml", status=status) @@ -30,10 +45,9 @@ def home(endpoint): """Example API home page. Developers should make their own page similar to this one.""" test_layer = "testLayerName" test_layer_time = "12:59:59" # HH:MM:SS - return f"""

    OGC Server API

    -

    This is the API endpoint served at {endpoint}. Add example usage here for your users.

    -