fix support for custom formatters (#1997) (#2144)

* fix support for custom formatters (#1997)

* fix typo

* add formatters for EDR

Author: tomkralidis

docs/source/configuration.rst

@@ -257,6 +257,10 @@ default.
                 storage_crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84  # optional CRS in which data is stored, default: as 'crs' field
                 storage_crs_coordinate_epoch: 2017.23  # optional, if storage_crs is a dynamic coordinate reference system
                 always_xy: false  # optional should CRS respect axis ordering
+          formatters:  # list of 1..n formatter definitions
+              - name: path.to.formatter  # Python path of formatter definition
+                attachment: true  # whether or not to provide as an attachment or normal response
+                geom: false  # whether or not to include geometry
 
       hello-world:  # name of process
           type: process  # REQUIRED (collection, process, or stac-collection)

docs/source/plugins.rst

@@ -435,7 +435,10 @@ The below template provides a minimal example (let's call the file ``mycooljsonf
            """Inherit from parent class"""
 
            super().__init__({'name': 'cooljson', 'geom': None})
-           self.mimetype = 'application/json; subtype:mycooljson'
+           self.f = 'cooljson'  # f= value
+           self.mimetype = 'application/json; subtype:mycooljson'  # response media type
+           self.attachment = False  # whether to provide as an attachment (default False)
+           self.extension = 'cooljson'  # filename extension if providing as an attachment
 
        def write(self, options={}, data=None):
            """custom writer"""

pygeoapi/__init__.py

@@ -68,7 +68,7 @@ def decorator(click_group):
             try:
                 click_group.add_command(entry_point.load())
             except Exception as err:
-                print(err)
+                click.echo(err)
         return click_group
 
     return decorator

pygeoapi/api/__init__.py

@@ -68,7 +68,8 @@
     TEMPLATESDIR, UrlPrefetcher, dategetter,
     filter_dict_by_key_value, filter_providers_by_type, get_api_rules,
     get_base_url, get_provider_by_type, get_provider_default, get_typed_value,
-    render_j2_template, to_json, get_choice_from_headers, get_from_headers
+    render_j2_template, to_json, get_choice_from_headers, get_from_headers,
+    get_dataset_formatters
 )
 
 LOGGER = logging.getLogger(__name__)
@@ -319,11 +320,14 @@ def _get_locale(self, headers: dict,
 
         return raw, default_locale
 
-    def _get_format(self, headers: dict) -> Union[str, None]:
+    def _get_format(self, headers: dict,
+                    extra_formats: dict = {}) -> Union[str, None]:
         """
         Get `Request` format type from query parameters or headers.
 
         :param headers: Dict of Request headers
+        :param extra_formats: Dict of extra dataset specific formats
+
         :returns: format value or None if not found/specified
         """
 
@@ -339,10 +343,14 @@ def _get_format(self, headers: dict) -> Union[str, None]:
         if types_ is None:
             return
 
-        (fmts, mimes) = zip(*FORMAT_TYPES.items())
+        merged_format_types = FORMAT_TYPES | extra_formats
+
+        (fmts, mimes) = zip(*merged_format_types.items())
+        mimes2 = [m.split(';')[0] for m in mimes]
+
         for type_ in types_:
-            if type_ in mimes:
-                idx_ = mimes.index(type_)
+            if type_ in mimes2:
+                idx_ = mimes2.index(type_)
                 return fmts[idx_]
 
     @property
@@ -1042,6 +1050,14 @@ def describe_collections(api: API, request: APIRequest,
                 'href': f'{api.get_collections_url()}/{k}/items?f={F_HTML}'  # noqa
             })
 
+            for key, value in get_dataset_formatters(v).items():
+                collection['links'].append({
+                    'type': value.mimetype,
+                    'rel': 'items',
+                    'title': l10n.translate(f'Items as {key}', request.locale),  # noqa
+                    'href': f'{api.get_collections_url()}/{k}/items?f={value.f}'  # noqa
+                })
+
         # OAPIF Part 2 - list supported CRSs and StorageCRS
         if collection_data_type in ['edr', 'feature']:
             collection['crs'] = get_supported_crs_list(collection_data)
@@ -1221,6 +1237,15 @@ def describe_collections(api: API, request: APIRequest,
                     'href': f'{api.get_collections_url()}/{k}/{qt}?f={F_HTML}'
                 })
 
+                for key, value in get_dataset_formatters(v).items():
+                    title3 = f'{qt} query for this collection as {key}'
+                    collection['links'].append({
+                        'type': value.mimetype,
+                        'rel': 'data',
+                        'title': title3,
+                        'href': f'{api.get_collections_url()}/{k}/{qt}?f={value.f}'  # noqa
+                    })
+
         if dataset is not None and k == dataset:
             fcm = collection
             break

pygeoapi/api/environmental_data_retrieval.py

@@ -8,7 +8,7 @@
 #          Ricardo Garcia Silva <ricardo.garcia.silva@geobeyond.it>
 #          Bernhard Mallinger <bernhard.mallinger@eox.at>
 #
-# Copyright (c) 2025 Tom Kralidis
+# Copyright (c) 2026 Tom Kralidis
 # Copyright (c) 2025 Francesco Bartoli
 # Copyright (c) 2022 John A Stevenson and Colin Blackburn
 # Copyright (c) 2023 Ricardo Garcia Silva
@@ -37,7 +37,7 @@
 #
 # =================================================================
 
-
+from copy import deepcopy
 from http import HTTPStatus
 import logging
 from typing import Tuple
@@ -49,13 +49,15 @@
 
 from pygeoapi import l10n
 from pygeoapi.api import evaluate_limit
+from pygeoapi.formatter.base import FormatterSerializationError
 from pygeoapi.crs import (create_crs_transform_spec, set_content_crs_header)
+from pygeoapi.openapi import get_oas_30_parameters
 from pygeoapi.plugin import load_plugin, PLUGINS
 from pygeoapi.provider.base import (
     ProviderGenericError, ProviderItemNotFoundError)
 from pygeoapi.util import (
-    filter_providers_by_type, get_provider_by_type, get_typed_value,
-    render_j2_template, to_json, filter_dict_by_key_value
+    filter_providers_by_type, get_dataset_formatters, get_provider_by_type,
+    get_typed_value, render_j2_template, to_json, filter_dict_by_key_value
 )
 
 from . import (APIRequest, API, F_COVERAGEJSON, F_HTML, F_JSON, F_JSONLD,
@@ -253,8 +255,6 @@ def get_collection_edr_query(api: API, request: APIRequest,
     :returns: tuple of headers, status code, content
     """
 
-    if not request.is_valid(PLUGINS['formatter'].keys()):
-        return api.get_format_exception(request)
     headers = request.get_response_headers(api.default_locale,
                                            **api.api_headers)
     collections = filter_dict_by_key_value(api.config['resources'],
@@ -287,6 +287,20 @@ def get_collection_edr_query(api: API, request: APIRequest,
             HTTPStatus.BAD_REQUEST, headers, request.format,
             'InvalidParameterValue', msg)
 
+    LOGGER.debug('Validating requested format')
+    dataset_formatters = get_dataset_formatters(collections[dataset])
+
+    if dataset_formatters:
+        LOGGER.debug(f'Dataset formatters: {dataset_formatters}')
+        request._format = request._get_format(
+            request.get_request_headers(request.headers),
+            {v.f: v.mimetype for v in dataset_formatters.values()})
+
+        LOGGER.debug(f'Request format: {request.format}')
+
+    if not request.is_valid(dataset_formatters.keys()):
+        return api.get_format_exception(request)
+
     crs_transform_spec = None
     query_crs_uri = request.params.get('crs')
     if query_crs_uri is not None:
@@ -442,6 +456,30 @@ def get_collection_edr_query(api: API, request: APIRequest,
         content = render_j2_template(api.tpl_config, tpl_config,
                                      'collections/edr/query.html', data,
                                      api.default_locale)
+    elif request.format in [df.f for df in dataset_formatters.values()]:
+        formatter = [v for v in dataset_formatters.values() if
+                     v.f == request.format][0]
+
+        try:
+            content = formatter.write(
+                data=data,
+                options={
+                    'provider_def': get_provider_by_type(
+                        collections[dataset]['providers'],
+                        'edr')
+                }
+            )
+        except FormatterSerializationError:
+            msg = 'Error serializing output'
+            return api.get_exception(
+                HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format,
+                'NoApplicableCode', msg)
+
+        if formatter.attachment:
+            filename = f'{dataset}.{formatter.extension}'
+            cd = f'attachment; filename="{filename}"'
+            headers['Content-Disposition'] = cd
+
     else:
         content = to_json(data, api.pretty_print)
 
@@ -515,6 +553,12 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str,
                     spatial_parameter = {
                         '$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{eqe['qt']}Coords.yaml"  # noqa
                     }
+
+                dataset_formatters = get_dataset_formatters(v)
+                coll_f_parameter = deepcopy(get_oas_30_parameters(cfg, locale))['f']  # noqa
+                for key, value in dataset_formatters.items():
+                    coll_f_parameter['schema']['enum'].append(value.f)
+
                 paths[eqe['path']] = {
                     'get': {
                         'summary': f"query {description} by {eqe['qt']}",
@@ -527,7 +571,7 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str,
                             {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"},  # noqa
                             {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"},  # noqa
                             {'$ref': '#/components/parameters/crs'},
-                            {'$ref': '#/components/parameters/f'}
+                            coll_f_parameter,
                         ],
                         'responses': {
                             '200': {

pygeoapi/api/itemtypes.py

@@ -7,7 +7,7 @@
 #          Colin Blackburn <colb@bgs.ac.uk>
 #          Ricardo Garcia Silva <ricardo.garcia.silva@geobeyond.it>
 #
-# Copyright (c) 2025 Tom Kralidis
+# Copyright (c) 2026 Tom Kralidis
 # Copyright (c) 2025 Francesco Bartoli
 # Copyright (c) 2022 John A Stevenson and Colin Blackburn
 # Copyright (c) 2023 Ricardo Garcia Silva
@@ -55,13 +55,15 @@
                           set_content_crs_header)
 from pygeoapi.formatter.base import FormatterSerializationError
 from pygeoapi.linked_data import geojson2jsonld
+from pygeoapi.openapi import get_oas_30_parameters
 from pygeoapi.plugin import load_plugin, PLUGINS
 from pygeoapi.provider.base import (
     ProviderGenericError, ProviderTypeError, SchemaType)
 
 from pygeoapi.util import (filter_providers_by_type, to_json,
                            filter_dict_by_key_value, str2bool,
-                           get_provider_by_type, render_j2_template)
+                           get_provider_by_type, render_j2_template,
+                           get_dataset_formatters)
 
 from . import (
     APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD,
@@ -241,9 +243,6 @@ def get_collection_items(
     :returns: tuple of headers, status code, content
     """
 
-    if not request.is_valid(PLUGINS['formatter'].keys()):
-        return api.get_format_exception(request)
-
     # Set Content-Language to system locale until provider locale
     # has been determined
     headers = request.get_response_headers(SYSTEM_LOCALE,
@@ -352,6 +351,20 @@ def get_collection_items(
             err.http_status_code, headers, request.format,
             err.ogc_exception_code, err.message)
 
+    LOGGER.debug('Validating requested format')
+    dataset_formatters = get_dataset_formatters(collections[dataset])
+
+    if dataset_formatters:
+        LOGGER.debug(f'Dataset formatters: {dataset_formatters}')
+        request._format = request._get_format(
+            request.get_request_headers(request.headers),
+            {v.f: v.mimetype for v in dataset_formatters.values()})
+
+        LOGGER.debug(f'Request format: {request.format}')
+
+    if not request.is_valid(dataset_formatters.keys()):
+        return api.get_format_exception(request)
+
     crs_transform_spec = None
     if provider_type == 'feature':
         # crs query parameter is only available for OGC API - Features
@@ -581,6 +594,14 @@ def get_collection_items(
         'href': f'{uri}?f={F_HTML}{serialized_query_params}'
     }])
 
+    for key, value in dataset_formatters.items():
+        content['links'].append({
+            'type': value.mimetype,
+            'rel': 'alternate',
+            'title': f'This document as {key}',
+            'href': f'{uri}?f={value.name}{serialized_query_params}'
+        })
+
     next_link = False
     prev_link = False
 
@@ -656,9 +677,9 @@ def get_collection_items(
                                      'collections/items/index.html',
                                      content, request.locale)
         return headers, HTTPStatus.OK, content
-    elif request.format == 'csv':  # render
-        formatter = load_plugin('formatter',
-                                {'name': 'CSV', 'geom': True})
+    elif request.format in [df.f for df in dataset_formatters.values()]:
+        formatter = [v for v in dataset_formatters.values() if
+                     v.f == request.format][0]
 
         try:
             content = formatter.write(
@@ -677,13 +698,14 @@ def get_collection_items(
 
         headers['Content-Type'] = formatter.mimetype
 
-        if p.filename is None:
-            filename = f'{dataset}.csv'
-        else:
-            filename = f'{p.filename}'
+        if formatter.attachment:
+            if p.filename is None:
+                filename = f'{dataset}.{formatter.extension}'
+            else:
+                filename = f'{p.filename}'
 
-        cd = f'attachment; filename="{filename}"'
-        headers['Content-Disposition'] = cd
+            cd = f'attachment; filename="{filename}"'
+            headers['Content-Disposition'] = cd
 
         return headers, HTTPStatus.OK, content
 
@@ -1073,14 +1095,19 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str,
                 v.get('limits', {})
             )
 
+            dataset_formatters = get_dataset_formatters(v)
+            coll_f_parameter = deepcopy(get_oas_30_parameters(cfg, locale))['f']  # noqa
+            for key, value in dataset_formatters.items():
+                coll_f_parameter['schema']['enum'].append(value.f)
+
             paths[items_path] = {
                 'get': {
                     'summary': f'Get {title} items',
                     'description': description,
                     'tags': [k],
                     'operationId': f'get{k.capitalize()}Features',
                     'parameters': [
-                        {'$ref': '#/components/parameters/f'},
+                        coll_f_parameter,
                         {'$ref': '#/components/parameters/lang'},
                         {'$ref': '#/components/parameters/bbox'},
                         coll_limit,

pygeoapi/formatter/base.py

@@ -2,7 +2,7 @@
 #
 # Authors: Tom Kralidis <tomkralidis@gmail.com>
 #
-# Copyright (c) 2022 Tom Kralidis
+# Copyright (c) 2026 Tom Kralidis
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -39,23 +39,28 @@ def __init__(self, formatter_def: dict):
         """
         Initialize object
 
-        :param formatter_def: formatter definition
+        param formatter_def: formatter definition
 
         :returns: pygeoapi.formatter.base.BaseFormatter
         """
 
+        self.extension = None
+        self.f = None
         self.mimetype = None
-        self.geom = False
 
-        self.name = formatter_def['name']
-        if 'geom' in formatter_def:
-            self.geom = formatter_def['geom']
+        try:
+            self.name = formatter_def['name']
+        except KeyError:
+            raise RuntimeError('name is required')
+
+        self.geom = formatter_def.get('geom', False)
+        self.attachment = formatter_def.get('attachment', False)
 
     def write(self, options: dict = {}, data: dict | None = None) -> str:
         """
         Generate data in specified format
 
-        :param options: CSV formatting options
+        :param options: formatting options
         :param data: dict representation of GeoJSON object
 
         :returns: string representation of format