From 7585320640642eef39b576d864af734e6cf00984 Mon Sep 17 00:00:00 2001 From: Sam Cranford Date: Tue, 14 Apr 2026 14:54:12 +0000 Subject: [PATCH 1/3] ENH: Move to a custom referenceTime dimension --- example/app.py | 7 +++--- ogc/edr/edr_api.py | 12 +++++---- ogc/edr/edr_provider.py | 54 +++++++++++++++++++++------------------- ogc/edr/test/conftest.py | 9 ++++--- ogc/podpac.py | 7 +++--- ogc/settings.py | 3 +++ 6 files changed, 52 insertions(+), 40 deletions(-) diff --git a/example/app.py b/example/app.py index f0ddf19..0b20ad3 100755 --- a/example/app.py +++ b/example/app.py @@ -8,12 +8,13 @@ from ogc import servers from ogc import core from ogc import podpac as pogc +from ogc.settings import EDR_TIME_INSTANCE_DIMENSION import podpac import numpy as np # Setup new dimension -podpac.core.coordinates.utils.add_valid_dimension("forecastOffsetHr") +podpac.core.coordinates.utils.add_valid_dimension(EDR_TIME_INSTANCE_DIMENSION) # create some podpac nodes data = np.random.default_rng(1).random((11, 21)) @@ -26,8 +27,8 @@ node2 = podpac.data.Array(source=data2, coordinates=coords) time = np.array(["2025-10-24T12:00:00"], dtype="datetime64") -offsets = [np.timedelta64(0, "h")] -coords = podpac.Coordinates([lat, lon, time, offsets], dims=["lat", "lon", "time", "forecastOffsetHr"]) +instance = np.array(["2025-10-24T00:00:00"], dtype="datetime64") +coords = podpac.Coordinates([lat, lon, time, instance], dims=["lat", "lon", "time", EDR_TIME_INSTANCE_DIMENSION]) data3 = np.random.default_rng(1).random((11, 21, 1, 1)) node3 = podpac.data.Array(source=data3, coordinates=coords) diff --git a/ogc/edr/edr_api.py b/ogc/edr/edr_api.py index ed18663..7758e1e 100644 --- a/ogc/edr/edr_api.py +++ b/ogc/edr/edr_api.py @@ -437,12 +437,14 @@ def _generate_extents(layers: List[pogc.Layer], instance: str | None) -> Dict[st vertical_range.update(coordinates["alt"].coordinates) if "time" in coordinates.udims: - if instance in layer.time_instances() and "forecastOffsetHr" in coordinates.udims: + if instance in layer.time_instances() and settings.EDR_TIME_INSTANCE_DIMENSION in coordinates.udims: instance_datetime = np.datetime64(instance) - instance_coordinates = coordinates.select({"time": [instance_datetime, instance_datetime]}) - selected_offset_coordinates = instance_coordinates["forecastOffsetHr"].coordinates - time_range.update([instance_datetime + offset for offset in selected_offset_coordinates]) - elif not instance and "forecastOffsetHr" not in coordinates.udims: + instance_coordinates = coordinates.select( + {settings.EDR_TIME_INSTANCE_DIMENSION: [instance_datetime, instance_datetime]} + ) + selected_time_coordinates = instance_coordinates["time"].coordinates + time_range.update(selected_time_coordinates) + elif not instance and settings.EDR_TIME_INSTANCE_DIMENSION not in coordinates.udims: time_range.update(coordinates["time"].coordinates) sorted_time_range = sorted(time_range) diff --git a/ogc/edr/edr_provider.py b/ogc/edr/edr_provider.py index 0b94842..765e659 100644 --- a/ogc/edr/edr_provider.py +++ b/ogc/edr/edr_provider.py @@ -88,7 +88,7 @@ def is_collection_queryable(cls, base_url: str, group: str) -> bool: layers = cls.get_layers(base_url, group) for layer in layers: coordinates = layer.get_coordinates() - if coordinates is not None and "forecastOffsetHr" not in coordinates.udims: + if coordinates is not None and settings.EDR_TIME_INSTANCE_DIMENSION not in coordinates.udims: return True return False @@ -584,25 +584,25 @@ def evaluate_layer(requested_coordinates: podpac.Coordinates, layer: pogc.Layer) if coordinates is None: return units_data_array - layer_has_instances = "forecastOffsetHr" in coordinates.udims - request_has_instances = "forecastOffsetHr" in requested_coordinates.udims + layer_has_instances = settings.EDR_TIME_INSTANCE_DIMENSION in coordinates.udims + request_has_instances = settings.EDR_TIME_INSTANCE_DIMENSION in requested_coordinates.udims if layer_has_instances ^ request_has_instances: return units_data_array - if "time" not in coordinates[0].udims: - layer_requested_coordinates = layer_requested_coordinates.udrop( - ["time", "forecastOffsetHr"], ignore_missing=True - ) + if "time" not in coordinates.udims: + layer_requested_coordinates = layer_requested_coordinates.udrop(["time"], ignore_missing=True) if layer.node is not None: units_data_array = layer.node.eval(layer_requested_coordinates) - if "forecastOffsetHr" in units_data_array.dims or "time_forecastOffsetHr" in units_data_array.dims: - if "time_forecastOffsetHr" not in units_data_array.dims: - units_data_array = units_data_array.stack(time_forecastOffsetHr=("time", "forecastOffsetHr")) - forecast_offsets = units_data_array.forecastOffsetHr.data.copy() - time_data = units_data_array.time.data.copy() - units_data_array = units_data_array.drop_vars({"time", "time_forecastOffsetHr", "forecastOffsetHr"}) - units_data_array = units_data_array.rename(time_forecastOffsetHr="time") - units_data_array = units_data_array.assign_coords(time=time_data + forecast_offsets) + units_data_array = units_data_array.unstack() + if settings.EDR_TIME_INSTANCE_DIMENSION in units_data_array.dims: + units_data_array = units_data_array.sel( + { + settings.EDR_TIME_INSTANCE_DIMENSION: layer_requested_coordinates[ + settings.EDR_TIME_INSTANCE_DIMENSION + ].coordinates[0] + } + ) + units_data_array = units_data_array.drop_vars({settings.EDR_TIME_INSTANCE_DIMENSION}) if units_data_array.attrs.get("bounds", None): filtered_bounds = { coord: bnd @@ -645,7 +645,7 @@ def get_datetimes(layers: List[pogc.Layer], instance_time: str | None) -> List[n layers : List[pogc.Layer] The list of layers to determine datetimes for. instance_time: str | None - The optional instance time which forecast datetimes are being requested for. + The optional instance time to get datetimes for, if not provided collection datetimes are returned. Returns ------- List[np.datetime64] @@ -656,14 +656,17 @@ def get_datetimes(layers: List[pogc.Layer], instance_time: str | None) -> List[n for layer in layers: coordinates = layer.get_coordinates() if coordinates is not None and "time" in coordinates.udims: - if instance_time in layer.time_instances() and "forecastOffsetHr" in coordinates.udims: - # Retrieve available forecastOffSetHr and instance time combinations + if ( + instance_time in layer.time_instances() + and settings.EDR_TIME_INSTANCE_DIMENSION in coordinates.udims + ): instance_datetime = np.datetime64(instance_time) - instance_coordinates = coordinates.select({"time": [instance_datetime, instance_datetime]}) - selected_offset_coordinates = instance_coordinates["forecastOffsetHr"].coordinates - available_times.update([instance_datetime + offset for offset in selected_offset_coordinates]) - elif not instance_time and "forecastOffsetHr" not in coordinates.udims: - # Retrieve layer times for non-instance requests + instance_coordinates = coordinates.select( + {settings.EDR_TIME_INSTANCE_DIMENSION: [instance_datetime, instance_datetime]} + ) + selected_time_coordinates = instance_coordinates["time"].coordinates + available_times.update(selected_time_coordinates) + elif not instance_time and settings.EDR_TIME_INSTANCE_DIMENSION not in coordinates.udims: available_times.update(coordinates["time"].coordinates) return list(available_times) @@ -859,8 +862,9 @@ def interpret_time_coordinates( raise ProviderInvalidQueryError(msg, user_msg=msg) if instance_time: - offsets = [np.timedelta64(time - np.datetime64(instance_time), "h") for time in times] - return podpac.Coordinates([[instance_time], offsets], dims=["time", "forecastOffsetHr"], crs=crs) + return podpac.Coordinates( + [times, [instance_time]], dims=["time", settings.EDR_TIME_INSTANCE_DIMENSION], crs=crs + ) return podpac.Coordinates([times], dims=["time"], crs=crs) diff --git a/ogc/edr/test/conftest.py b/ogc/edr/test/conftest.py index 2c36dd2..90b92bd 100644 --- a/ogc/edr/test/conftest.py +++ b/ogc/edr/test/conftest.py @@ -4,16 +4,17 @@ import podpac from ogc import podpac as pogc from typing import Dict, List, Any +from ogc.settings import EDR_TIME_INSTANCE_DIMENSION # Setup new dimension -podpac.core.coordinates.utils.add_valid_dimension("forecastOffsetHr") +podpac.core.coordinates.utils.add_valid_dimension(EDR_TIME_INSTANCE_DIMENSION) lat = np.linspace(90, -90, 11) lon = np.linspace(-180, 180, 21) time = np.array(["2025-10-24T12:00:00"], dtype="datetime64") -offsets = [np.timedelta64(0, "h")] +instance = np.array(["2025-10-24T00:00:00"], dtype="datetime64") data = np.random.default_rng(1).random((11, 21, 1, 1)) -coords = podpac.Coordinates([lat, lon, time, offsets], dims=["lat", "lon", "time", "forecastOffsetHr"]) +coords = podpac.Coordinates([lat, lon, time, instance], dims=["lat", "lon", "time", EDR_TIME_INSTANCE_DIMENSION]) # Define test layers using sample data and coordinates node1 = podpac.data.Array(source=data, coordinates=coords) @@ -78,7 +79,7 @@ def single_layer_cube_args_internal() -> Dict[str, Any]: return { "format_": "coveragejson", - "instance": str(time[0]), + "instance": str(instance[0]), "bbox": [-180, -90, 180, 90], "datetime_": str(time[0]), "select_properties": [layer1.identifier], diff --git a/ogc/podpac.py b/ogc/podpac.py index b136a30..3e6b183 100755 --- a/ogc/podpac.py +++ b/ogc/podpac.py @@ -15,6 +15,7 @@ import textwrap import re from datetime import datetime +from ogc.settings import EDR_TIME_INSTANCE_DIMENSION def _uppercase_for_dict_keys(lower_dict): @@ -65,10 +66,10 @@ def time_instances(self) -> List[str]: time_instances = set() coordinates = self.get_coordinates() - # Time instances are created if a node has both time and offsets. - if coordinates is not None and "time" in coordinates.udims and "forecastOffsetHr" in coordinates.udims: + # Time instances are created if a node has a time instance dimension. + if coordinates is not None and EDR_TIME_INSTANCE_DIMENSION in coordinates.udims: time_instances.update( - [time.astype("datetime64[ms]").astype(datetime).isoformat() for time in coordinates["time"].coordinates] + [time.astype(datetime).isoformat() for time in coordinates[EDR_TIME_INSTANCE_DIMENSION].coordinates] ) return list(time_instances) diff --git a/ogc/settings.py b/ogc/settings.py index 6859455..7796b66 100755 --- a/ogc/settings.py +++ b/ogc/settings.py @@ -55,6 +55,9 @@ "position": COVERAGE_JSON, } +# The dimension associated with time instances of a collection +EDR_TIME_INSTANCE_DIMENSION = "referenceTime" + # WMS Capabilities timestamp format USE_TIMES_LIST = False PAST_DAYS_INCLUDED = 7 From 9be58e4e11a97e7fd594af49a58b996ec8c3f618 Mon Sep 17 00:00:00 2001 From: Sam Cranford Date: Wed, 15 Apr 2026 20:27:05 +0000 Subject: [PATCH 2/3] FIX: Time conversion to int --- ogc/podpac.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ogc/podpac.py b/ogc/podpac.py index 3e6b183..e2103cd 100755 --- a/ogc/podpac.py +++ b/ogc/podpac.py @@ -69,7 +69,10 @@ def time_instances(self) -> List[str]: # Time instances are created if a node has a time instance dimension. if coordinates is not None and EDR_TIME_INSTANCE_DIMENSION in coordinates.udims: time_instances.update( - [time.astype(datetime).isoformat() for time in coordinates[EDR_TIME_INSTANCE_DIMENSION].coordinates] + [ + time.astype("datetime64[ms]").astype(datetime).isoformat() + for time in coordinates[EDR_TIME_INSTANCE_DIMENSION].coordinates + ] ) return list(time_instances) From 7aebc97485e4a88640387c42d7738c366e34baf4 Mon Sep 17 00:00:00 2001 From: Sam Cranford Date: Fri, 17 Apr 2026 10:39:50 -0400 Subject: [PATCH 3/3] Add test coverage for non-instance layers --- ogc/edr/edr_provider.py | 1 + ogc/edr/test/conftest.py | 41 ++++++++++++++++++++++++++++++ ogc/edr/test/test_edr_provider.py | 28 +++++++++++++++++++++ ogc/edr/test/test_edr_routes.py | 42 +++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) diff --git a/ogc/edr/edr_provider.py b/ogc/edr/edr_provider.py index 765e659..4167200 100644 --- a/ogc/edr/edr_provider.py +++ b/ogc/edr/edr_provider.py @@ -595,6 +595,7 @@ def evaluate_layer(requested_coordinates: podpac.Coordinates, layer: pogc.Layer) units_data_array = layer.node.eval(layer_requested_coordinates) units_data_array = units_data_array.unstack() if settings.EDR_TIME_INSTANCE_DIMENSION in units_data_array.dims: + # Use scalar selection to drop the extra dimension units_data_array = units_data_array.sel( { settings.EDR_TIME_INSTANCE_DIMENSION: layer_requested_coordinates[ diff --git a/ogc/edr/test/conftest.py b/ogc/edr/test/conftest.py index 90b92bd..6ccd37d 100644 --- a/ogc/edr/test/conftest.py +++ b/ogc/edr/test/conftest.py @@ -15,6 +15,8 @@ instance = np.array(["2025-10-24T00:00:00"], dtype="datetime64") data = np.random.default_rng(1).random((11, 21, 1, 1)) coords = podpac.Coordinates([lat, lon, time, instance], dims=["lat", "lon", "time", EDR_TIME_INSTANCE_DIMENSION]) +data_without_instance = np.random.default_rng(1).random((11, 21, 1)) +coords_without_instance = podpac.Coordinates([lat, lon, time], dims=["lat", "lon", "time"]) # Define test layers using sample data and coordinates node1 = podpac.data.Array(source=data, coordinates=coords) @@ -35,6 +37,15 @@ group="Layers", valid_times=[dt.astype(datetime.datetime) for dt in time], ) +node3 = podpac.data.Array(source=data_without_instance, coordinates=coords_without_instance) +layer3 = pogc.Layer( + node=node3, + identifier="layer3", + title="Layer 3", + abstract="Layer3 Data (No instance)", + group="Layers", + valid_times=[dt.astype(datetime.datetime) for dt in time], +) @pytest.fixture() @@ -49,6 +60,18 @@ def layers() -> List[pogc.Layer]: return [layer1, layer2] +@pytest.fixture() +def layers_no_instance() -> List[pogc.Layer]: + """List of test layers without instances. + + Returns + ------- + List[pogc.Layer] + The test layers. + """ + return [layer3] + + @pytest.fixture() def single_layer_cube_args() -> Dict[str, Any]: """Dictionary of valid request arguments that align to a single test layer cube request. @@ -84,3 +107,21 @@ def single_layer_cube_args_internal() -> Dict[str, Any]: "datetime_": str(time[0]), "select_properties": [layer1.identifier], } + + +@pytest.fixture() +def single_layer_cube_args_no_instance_internal() -> Dict[str, Any]: + """Dictionary of valid arguments that align to a single non-instance test layer request with internal pygeoapi keys. + + Returns + ------- + Dict[str, Any] + Valid internal cube arguments for a single non-instance test layer. + """ + + return { + "format_": "coveragejson", + "bbox": [-180, -90, 180, 90], + "datetime_": str(time[0]), + "select_properties": [layer3.identifier], + } diff --git a/ogc/edr/test/test_edr_provider.py b/ogc/edr/test/test_edr_provider.py index fddd6bc..c081781 100644 --- a/ogc/edr/test/test_edr_provider.py +++ b/ogc/edr/test/test_edr_provider.py @@ -401,6 +401,34 @@ def test_edr_provider_cube_request_invalid_bbox( provider.cube(**args) +def test_edr_provider_cube_request_without_instance( + layers_no_instance: List[pogc.Layer], single_layer_cube_args_no_instance_internal: Dict[str, Any] +): + """Test the cube method of the EDR Provider class with a valid layer without an instance. + + Parameters + ---------- + layers_no_instance : List[pogc.Layer] + Layers provided by a test fixture. + + single_layer_cube_args_no_instance_internal : Dict[str, Any] + Single layer arguments with internal pygeoapi keys provided by a test fixture. + """ + base_url = "/" + args = single_layer_cube_args_no_instance_internal + parameter_name = single_layer_cube_args_no_instance_internal["select_properties"][0] + provider = EdrProvider(provider_def=get_provider_definition(base_url)) + provider.set_layers(base_url, layers_no_instance) + + response = provider.cube(**args) + response = get_json_with_cleanup(response["fp"]) + + assert set(response["domain"]["ranges"][parameter_name]["axisNames"]) == {"x", "y", "t"} + assert np.prod(np.array(response["domain"]["ranges"][parameter_name]["shape"])) == len( + response["domain"]["ranges"][parameter_name]["values"] + ) + + def test_edr_provider_cube_request_invalid_instance( layers: List[pogc.Layer], single_layer_cube_args_internal: Dict[str, Any] ): diff --git a/ogc/edr/test/test_edr_routes.py b/ogc/edr/test/test_edr_routes.py index e9eb11f..cc8442e 100644 --- a/ogc/edr/test/test_edr_routes.py +++ b/ogc/edr/test/test_edr_routes.py @@ -154,6 +154,29 @@ def test_edr_routes_describe_collection(layers: List[pogc.Layer]): assert list(response["data_queries"].keys()) == ["instances"] +def test_edr_routes_describe_collection_without_instance(layers_no_instance: List[pogc.Layer]): + """Test the EDR collection description for a response without instances. + The instance data query should still be available so it can return empty. + + Parameters + ---------- + layers_no_instance : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers_no_instance) + collection_id = layers_no_instance[0].group + collection_layers = [layer for layer in layers_no_instance if layer.group == collection_id] + + _, status, content = edr_routes.describe_collections(request, collection_id=collection_id) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert response["id"] == collection_id + assert list(response["parameter_names"].keys()) == [layer.identifier for layer in collection_layers] + assert set(response["data_queries"].keys()) == {"position", "cube", "area", "instances"} + + def test_edr_routes_describe_instances(layers: List[pogc.Layer]): """Test the EDR instances description for a response. @@ -180,6 +203,25 @@ def test_edr_routes_describe_instances(layers: List[pogc.Layer]): assert response_time_instances_ids == list(time_instances) +def test_edr_routes_describe_instances_without_instances(layers_no_instance: List[pogc.Layer]): + """Test the EDR instances description for a response when no instances are available. + + Parameters + ---------- + layers_no_instance : List[pogc.Layer] + Layers provided by a test fixture. + """ + request = mock_request({"f": "json"}) + edr_routes = EdrRoutes(layers=layers_no_instance) + collection_id = layers_no_instance[0].group + + _, status, content = edr_routes.describe_instances(request, collection_id=collection_id, instance_id=None) + response = json.loads(content) + + assert status == HTTPStatus.OK + assert len(response["instances"]) == 0 + + def test_edr_routes_describe_instance(layers: List[pogc.Layer]): """Test the EDR instance description for a response.