diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index f24e45f1..96a250e8 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -124,6 +124,101 @@ ], "description": "Logical data type for fields and metrics, independent of role (e.g. dimension vs fact) and physical representation. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. `DateTime` has no timezone or offset, while `DateTimeTz` identifies an instant using offset or timezone context but does not guarantee preservation of a named timezone. Omit `datatype` when unknown; use `Opaque` plus `custom_extensions` for a known type outside the portable vocabulary." }, + "SemanticType": { + "description": "High-level semantic classification of a field or metric. Either a well-known token or a URI pointing to an external type system.", + "oneOf": [ + { + "type": "string", + "enum": ["categorical", "quantitative", "monetary", "temporal", "geographic", "ordinal", "identifier"], + "description": "Well-known semantic type token" + }, + { + "type": "string", + "format": "uri", + "description": "URI pointing to an external type system (e.g., ISO 20022, FIBO, SKOS vocabulary)" + } + ] + }, + "DesiredDirection": { + "type": "string", + "enum": ["higher_is_better", "lower_is_better", "neutral"], + "description": "KPI polarity indicating whether higher or lower values are preferred" + }, + "DefaultAggregation": { + "type": "string", + "enum": ["sum", "avg", "min", "max", "count", "count_distinct"], + "description": "Default aggregation function to apply when this field is used as a measure" + }, + "DefaultTimeGranularity": { + "type": "string", + "enum": ["day", "week", "month", "quarter", "year"], + "description": "Default time bucket for temporal fields" + }, + "UnitSystem": { + "type": "string", + "enum": ["si", "imperial", "custom"], + "description": "Unit system classification" + }, + "Measurement": { + "type": "object", + "description": "Describes what a numeric value represents (units and measurement)", + "properties": { + "quantity_kind": { + "type": "string", + "description": "The kind of quantity (e.g., currency, length, weight, temperature, percentage, duration)" + }, + "unit": { + "type": "string", + "description": "Specific unit. For currency use ISO 4217 (e.g., usd, eur). For others use UCUM or descriptive strings." + }, + "unit_system": { + "$ref": "#/$defs/UnitSystem" + } + }, + "additionalProperties": false + }, + "DefaultSort": { + "type": "object", + "description": "Default sorting behavior", + "properties": { + "direction": { + "type": "string", + "enum": ["asc", "desc"], + "description": "Sort direction" + }, + "nulls": { + "type": "string", + "enum": ["first", "last"], + "description": "Null value positioning" + }, + "by_field": { + "type": "string", + "description": "Sort by a different field (e.g., sort month names by month number)" + } + }, + "additionalProperties": false + }, + "SemanticMapping": { + "type": "object", + "description": "Link to an external ontology or standard using a SKOS-based predicate. The predicate vocabulary is intentionally open to allow non-positive assertions (e.g. DISTINCT_FROM) via extensions.", + "properties": { + "target": { + "type": "string", + "format": "uri", + "description": "URI of the external concept (e.g., https://schema.org/MonetaryAmount, https://spec.edmcouncil.org/fibo/...)" + }, + "predicate": { + "type": "string", + "description": "Relationship type. SKOS baseline: exactMatch, closeMatch, broadMatch, narrowMatch, relatedMatch. Defaults to exactMatch. Vocabulary is open — non-standard predicates (e.g., DISTINCT_FROM) are permitted." + }, + "provenance": { + "type": "string", + "description": "Optional: origin of this mapping (e.g., 'manual', 'FIBO 4.1', a tool name)" + } + }, + "required": ["target"], + "additionalProperties": false + }, "Dimension": { "type": "object", "description": "Dimension metadata", @@ -153,6 +248,10 @@ "type": "string", "description": "Label for categorization" }, + "display_label": { + "type": "string", + "description": "Human-readable display name for UI and AI interaction" + }, "description": { "type": "string", "description": "Human-readable description" @@ -163,6 +262,40 @@ "ai_context": { "$ref": "#/$defs/AIContext" }, + "semantic_type": { + "$ref": "#/$defs/SemanticType" + }, + "measurement": { + "$ref": "#/$defs/Measurement" + }, + "display_format": { + "type": "string", + "description": "Excel-compatible display format string (e.g., $#,##0.00, 0.0%, #,##0)" + }, + "default_aggregation": { + "$ref": "#/$defs/DefaultAggregation" + }, + "default_sort": { + "$ref": "#/$defs/DefaultSort" + }, + "default_time_granularity": { + "$ref": "#/$defs/DefaultTimeGranularity" + }, + "semantic_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/SemanticMapping" + }, + "description": "Links to external ontologies or standards" + }, + "hidden": { + "type": "boolean", + "description": "Whether this field should be hidden from consumer UIs" + }, + "group_label": { + "type": "string", + "description": "Organizational grouping label for UI presentation" + }, "custom_extensions": { "type": "array", "items": { @@ -291,6 +424,41 @@ "ai_context": { "$ref": "#/$defs/AIContext" }, + "display_label": { + "type": "string", + "description": "Human-readable display name for UI and AI interaction" + }, + "semantic_type": { + "$ref": "#/$defs/SemanticType" + }, + "measurement": { + "$ref": "#/$defs/Measurement" + }, + "display_format": { + "type": "string", + "description": "Excel-compatible display format string (e.g., $#,##0.00, 0.0%, #,##0)" + }, + "desired_direction": { + "$ref": "#/$defs/DesiredDirection" + }, + "default_sort": { + "$ref": "#/$defs/DefaultSort" + }, + "semantic_mappings": { + "type": "array", + "items": { + "$ref": "#/$defs/SemanticMapping" + }, + "description": "Links to external ontologies or standards" + }, + "hidden": { + "type": "boolean", + "description": "Whether this metric should be hidden from consumer UIs" + }, + "group_label": { + "type": "string", + "description": "Organizational grouping label for UI presentation" + }, "custom_extensions": { "type": "array", "items": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 156cb1db..d31ef335 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -234,9 +234,19 @@ Fields represent row-level attributes that can be used for grouping, filtering, | `expression` | object | Yes | Expression definition with dialect support | | `dimension` | object | No | Dimension metadata (e.g., `is_time` flag) | | `label` | string | No | Label for categorization | +| `display_label` | string | No | Human-readable display name for UI and AI interaction | | `description` | string | No | Human-readable description | | `datatype` | string (enum) | No | Logical data type for this field. See [Data types](#data-types). | | `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms) | +| `semantic_type` | string | No | High-level semantic classification (see [Semantic Type](#semantic-type)) | +| `measurement` | object | No | Unit and quantity metadata (see [Measurement](#measurement)) | +| `display_format` | string | No | Excel-compatible format string (e.g., `$#,##0.00`, `0.0%`) | +| `default_aggregation` | string | No | Default aggregation when used as a measure: `sum`, `avg`, `min`, `max`, `count`, `count_distinct` | +| `default_sort` | object | No | Default sorting behavior (see [Default Sort](#default-sort)) | +| `default_time_granularity` | string | No | Default time bucket for temporal fields: `day`, `week`, `month`, `quarter`, `year` | +| `semantic_mappings` | array | No | Links to external ontologies (see [Semantic Mappings](#semantic-mappings)) | +| `hidden` | boolean | No | Whether this field should be hidden from consumer UIs | +| `group_label` | string | No | Organizational grouping label for UI presentation | | `custom_extensions` | array | No | Vendor-specific attributes | ### Expression Object @@ -369,6 +379,15 @@ Quantitative measures defined on business data, representing key calculations li | `description` | string | No | Human-readable description of what the metric measures | | `datatype` | string (enum) | No | Logical data type for this metric. See [Data types](#data-types). | | `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms) | +| `display_label` | string | No | Human-readable display name for UI and AI interaction | +| `semantic_type` | string | No | High-level semantic classification (see [Semantic Type](#semantic-type)) | +| `measurement` | object | No | Unit and quantity metadata (see [Measurement](#measurement)) | +| `display_format` | string | No | Excel-compatible format string (e.g., `$#,##0.00`, `0.0%`) | +| `desired_direction` | string | No | KPI polarity: `higher_is_better`, `lower_is_better`, `neutral` | +| `default_sort` | object | No | Default sorting behavior (see [Default Sort](#default-sort)) | +| `semantic_mappings` | array | No | Links to external ontologies (see [Semantic Mappings](#semantic-mappings)) | +| `hidden` | boolean | No | Whether this metric should be hidden from consumer UIs | +| `group_label` | string | No | Organizational grouping label for UI presentation | | `custom_extensions` | array | No | Vendor-specific attributes | ### Expression Object @@ -417,6 +436,210 @@ expression: --- +## Extended Metadata Types + +The following types are used by the extended metadata fields on both fields and metrics. All extended metadata is **optional** and **non-executional** — it does not affect query execution but enables consumers (BI tools, AI agents, developers) to correctly interpret, render, and present data. + +### Semantic Type + +High-level classification of a field or metric value. Accepts either a **well-known token** or a **URI** pointing to an external type system. Using URIs keeps the field registry-agnostic and allows governed external type systems to carry the long tail of domain-specific types. + +**Well-known tokens:** + +| Value | Description | +|-------|-------------| +| `categorical` | Unordered categorical values (e.g., status, color) | +| `quantitative` | Numeric values representing quantities | +| `monetary` | Currency/financial values | +| `temporal` | Time or date values | +| `geographic` | Location-related values (country, lat/lng, region) | +| `ordinal` | Ordered categorical values (e.g., Low/Medium/High, ratings) | +| `identifier` | Unique identifiers (e.g., IDs, codes) | + +**URI examples** (for regulated or domain-specific types): + +```yaml +semantic_type: https://www.iso20022.org/glossary/LEI +semantic_type: https://fpml.org/types/ISIN +semantic_type: https://spec.edmcouncil.org/fibo/ontology/FBC/ProductsAndServices/FinancialProductsAndServices/UPI +``` + +### Measurement + +Describes what a numeric value represents. Enables unit-aware reasoning and formatting. + +| Field | Type | Description | +|-------|------|-------------| +| `quantity_kind` | string | The kind of quantity (e.g., `currency`, `length`, `weight`, `temperature`, `percentage`, `duration`) | +| `unit` | string | Specific unit. For currency, use ISO 4217 codes (e.g., `usd`, `eur`, `gbp`). For others, use UCUM or descriptive strings (e.g., `meters`, `kg`, `celsius`) | +| `unit_system` | string | Unit system: `si`, `imperial`, `custom` | + +**Example:** + +```yaml +measurement: + quantity_kind: currency + unit: usd + unit_system: custom +``` + +### Default Sort + +Defines default sorting behavior for a field or metric. + +| Field | Type | Description | +|-------|------|-------------| +| `direction` | string | Sort direction: `asc`, `desc` | +| `nulls` | string | Null positioning: `first`, `last` | +| `by_field` | string | Sort by a different field (e.g., sort month names by month number) | + +**Example:** + +```yaml +default_sort: + direction: desc + nulls: last +``` + +### Semantic Mappings + +Links a field or metric to external ontologies or standards using a SKOS-based predicate. The predicate vocabulary is **intentionally open** — the SKOS predicates provide a well-understood baseline, but non-standard predicates (e.g., `DISTINCT_FROM` for regulatory disambiguation) are permitted and can be expressed via extensions without being schema-invalid. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `target` | string (URI) | Yes | URI of the external concept | +| `predicate` | string | No | Relationship type. Defaults to `exactMatch`. SKOS baseline predicates listed below. Vocabulary is open. | +| `provenance` | string | No | Origin of this mapping (e.g., `"manual"`, `"FIBO 4.1"`, a tool name) | + +**SKOS baseline predicates:** + +| Predicate | Meaning | +|-----------|--------| +| `exactMatch` | Concepts are sufficiently similar to be used interchangeably (default) | +| `closeMatch` | Concepts are similar enough to be useful in some contexts | +| `broadMatch` | Target concept is broader (more general) | +| `narrowMatch` | Target concept is narrower (more specific) | +| `relatedMatch` | Concepts are associatively related | + +**Example — standard alignment:** + +```yaml +semantic_mappings: + - target: https://schema.org/MonetaryAmount + predicate: exactMatch + provenance: manual +``` + +**Example — regulatory disambiguation (open predicate):** + +```yaml +semantic_mappings: + - target: https://spec.edmcouncil.org/fibo/ontology/DER/RateDerivatives/IRSwaps/Counterparty + predicate: exactMatch + provenance: FIBO 4.1 + - target: https://www.esma.europa.eu/emir/Counterparty + predicate: DISTINCT_FROM + provenance: EMIR-vs-MiFIR mapping review 2024 +``` + +### Display Format + +The `display_format` string follows Excel-compatible custom number format conventions. Consumers MAY support a subset, but interoperability is improved when adhering to common patterns. + +**Common Patterns:** + +| Pattern | Description | Example Output | +|---------|-------------|----------------| +| `$#,##0.00` | Currency with 2 decimals | $1,234.56 | +| `#,##0` | Integer with grouping | 12,345 | +| `0.0%` | Percentage with 1 decimal | 12.3% | +| `#,##0.00;(#,##0.00)` | Positive/negative | 1,234.56 or (1,234.56) | +| `0.00E+00` | Scientific notation | 1.23E+04 | +| `yyyy-mm-dd` | Date format | 2024-01-15 | + +--- + +## Extended Metadata Examples + +**Field with full extended metadata:** + +```yaml +- name: sales_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_amount + display_label: "Sales Amount" + semantic_type: monetary + measurement: + quantity_kind: currency + unit: usd + display_format: "$#,##0.00" + default_aggregation: sum + default_sort: + direction: desc + nulls: last + semantic_mappings: + - target: https://schema.org/MonetaryAmount + predicate: exactMatch + group_label: "Revenue" +``` + +**Metric with extended metadata:** + +```yaml +- name: total_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.sales_amount) + display_label: "Total Sales" + description: Total revenue from all completed orders + semantic_type: monetary + measurement: + quantity_kind: currency + unit: usd + display_format: "$#,##0.00" + desired_direction: higher_is_better + default_sort: + direction: desc + group_label: "Revenue" + semantic_mappings: + - target: https://schema.org/MonetaryAmount + predicate: exactMatch +``` + +**Temporal field with time granularity:** + +```yaml +- name: order_date + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date + dimension: + is_time: true + display_label: "Order Date" + semantic_type: temporal + default_time_granularity: month + default_sort: + direction: desc +``` + +**Hidden field used only in expressions:** + +```yaml +- name: internal_cost_basis + expression: + dialects: + - dialect: ANSI_SQL + expression: raw_cost * adjustment_factor + hidden: true + description: Internal cost calculation used by margin metrics +``` + +--- + ## Custom Extensions Custom extensions allow vendors to add platform-specific metadata without breaking core compatibility. Each extension includes a vendor name and arbitrary JSON data. diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 32fbb3e1..8f19c574 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -211,6 +211,10 @@ fields: # Optional: Label for categorization (e.g., "filter") label: string + # Optional: Human-readable display name for UI and AI interaction + # Provides a user-friendly name distinct from the technical field name + display_label: string + # Optional: Human-readable description of the field description: string @@ -226,6 +230,71 @@ fields: # Helps LLMs understand the field meaning and generate better queries ai_context: string + # Optional: High-level semantic classification of the field + # Helps consumers determine appropriate visualization and handling + # Either a well-known token OR a URI pointing to an external type system + # Well-known tokens: categorical, quantitative, monetary, temporal, geographic, ordinal, identifier + # URI examples: https://www.iso20022.org/glossary/LEI, https://fpml.org/types/ISIN + semantic_type: string + + # Optional: Describes what a numeric value represents (units and measurement) + # Enables unit-aware reasoning and formatting + measurement: + # Optional: The kind of quantity being measured + # Examples: currency, length, weight, temperature, percentage, duration + quantity_kind: string + # Optional: The specific unit of measurement + # For currency, use ISO 4217 codes (e.g., usd, eur, gbp) + # For other units, use UCUM or descriptive strings (e.g., meters, kg, celsius) + unit: string + # Optional: The unit system + # Values: si, imperial, custom + unit_system: string + + # Optional: Display format string for presentation + # Uses Excel-compatible custom number format conventions + # Examples: "$#,##0.00", "0.0%", "#,##0", "yyyy-mm-dd" + # Consumers MAY fall back to default formatting if unsupported + display_format: string + + # Optional: Default aggregation behavior when this field is used as a measure + # Removes ambiguity in query generation for consumers + # Values: sum, avg, min, max, count, count_distinct + default_aggregation: string + + # Optional: Default sorting behavior for this field + default_sort: + # Optional: Sort direction + # Values: asc, desc + direction: string + # Optional: Null handling + # Values: first, last + nulls: string + # Optional: Sort by a different field (e.g., sort month names by month number) + by_field: string + + # Optional: Default time granularity for temporal fields + # Only applicable when dimension.is_time is true + # Values: day, week, month, quarter, year + default_time_granularity: string + + # Optional: Links to external ontologies or standards + # Uses SKOS-based predicates as a baseline but the predicate vocabulary is open, + # allowing extensions to express non-positive assertions (e.g., DISTINCT_FROM) + semantic_mappings: + - target: string # URI of the external concept (e.g., https://schema.org/MonetaryAmount) + predicate: string # Relationship type. SKOS baseline: exactMatch, closeMatch, broadMatch, + # narrowMatch, relatedMatch. Defaults to exactMatch. Vocabulary is open. + provenance: string # Optional: source of this mapping (e.g., "manual", "FIBO 4.1", a tool name) + + # Optional: Whether this field should be hidden from consumer UIs + # Hidden fields remain available for expressions but are not surfaced to end users + hidden: boolean + + # Optional: Organizational grouping label for UI presentation + # Used to organize fields into logical folders or categories + group_label: string + # Optional: Vendor-specific attributes for extensibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor @@ -263,6 +332,46 @@ metrics: # Helps LLMs understand the metric meaning and suggest it appropriately ai_context: string + # Optional: Human-readable display name for UI and AI interaction + display_label: string + + # Optional: High-level semantic classification of the metric + # Either a well-known token OR a URI pointing to an external type system + # Well-known tokens: categorical, quantitative, monetary, temporal, geographic, ordinal, identifier + semantic_type: string + + # Optional: Describes what the metric value represents (units and measurement) + measurement: + quantity_kind: string + unit: string + unit_system: string + + # Optional: Display format string for presentation + # Uses Excel-compatible custom number format conventions + display_format: string + + # Optional: Indicates KPI polarity for scorecards and AI summarization + # Values: higher_is_better, lower_is_better, neutral + desired_direction: string + + # Optional: Default sorting behavior for this metric + default_sort: + direction: string # asc, desc + nulls: string # first, last + + # Optional: Links to external ontologies or standards + # Uses SKOS-based predicates as a baseline but the predicate vocabulary is open + semantic_mappings: + - target: string # URI of the external concept + predicate: string # Relationship type (SKOS baseline, open vocabulary). Default: exactMatch + provenance: string # Optional: source of this mapping + + # Optional: Whether this metric should be hidden from consumer UIs + hidden: boolean + + # Optional: Organizational grouping label for UI presentation + group_label: string + # Optional: Vendor-specific attributes for extensibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor