Fix JSON-null datapoint queries

container/src/main/java/org/openremote/container/timer/TimerService.java

@@ -22,7 +22,6 @@
 
 import java.time.*;
 import java.time.temporal.Temporal;
-import java.util.TimeZone;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Logger;
@@ -83,13 +82,11 @@ public void reset() {
     PSEUDO {
       protected AtomicLong offset = new AtomicLong();
       protected volatile Long stopTime;
-      // Explicitly define a UTC Clock
+      // Use UTC internally without changing the JVM's default time zone.
       protected java.time.Clock utcClock = java.time.Clock.systemUTC();
 
       @Override
       public void init() {
-        // Force JVM timezone to UTC for legacy date conversions to avoid DST issues in tests
-        TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
         long current = getCurrentTimeMillis();
         LOG.info("Initialized pseudo clock to: " + current + " / " + Instant.ofEpochMilli(current));
       }

container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovy

@@ -23,17 +23,62 @@ import spock.lang.Specification
 import java.time.LocalDate
 import java.time.LocalTime
 import java.time.ZoneId
+import java.util.concurrent.TimeUnit
 
 class PseudoClockTest extends Specification {
 
   TimerService.Clock clock
+  TimeZone originalTimeZone
 
   def setup() {
+    originalTimeZone = TimeZone.getDefault()
     clock = TimerService.Clock.PSEUDO
+    clock.reset()
     clock.init()
     clock.stop()
   }
 
+  def cleanup() {
+    try {
+      clock.reset()
+    } finally {
+      TimeZone.setDefault(originalTimeZone)
+    }
+  }
+
+  def "initializing the pseudo clock preserves the JVM time zone #zone"() {
+    given:
+    def timeZone = TimeZone.getTimeZone(zone)
+    TimeZone.setDefault(timeZone)
+
+    when:
+    clock.init()
+
+    then:
+    TimeZone.getDefault() == timeZone
+
+    where:
+    zone << ["Europe/Brussels", "America/New_York", "Asia/Kathmandu"]
+  }
+
+  def "setTime respects the #hours hour day on #date in Europe/Brussels"() {
+    given:
+    def zone = ZoneId.of("Europe/Brussels")
+    def startDate = LocalDate.parse(date)
+    def start = clock.setTime(startDate, LocalTime.MIDNIGHT, zone)
+
+    when:
+    def end = clock.setTime(startDate.plusDays(1), LocalTime.MIDNIGHT, zone)
+
+    then:
+    end - start == TimeUnit.HOURS.toMillis(hours)
+
+    where:
+    date | hours
+    "2026-03-29" | 23
+    "2026-10-25" | 25
+  }
+
   def "setTime sets #date at midnight in #zone to #expected milliseconds"() {
     expect:
     clock.setTime(LocalDate.parse(date), LocalTime.MIDNIGHT, ZoneId.of(zone)) == expected

model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java

@@ -29,7 +29,9 @@
     description =
         "Returns every stored datapoint in the inclusive range, ordered from newest to oldest. "
             + "Use this when the complete series is needed without aggregation or downsampling. Supports numeric, "
-            + "boolean, text, and JSON-compatible complex attribute values.")
+            + "boolean, text, and JSON-compatible complex attribute values. "
+            + "Unavailable values retain their stored timestamps and are returned as null. "
+            + "Boolean values are returned as 1 for true, 0 for false, and null for unavailable values.")
 public class AssetDatapointAllQuery extends AssetDatapointQuery {
 
   public AssetDatapointAllQuery() {}
@@ -50,11 +52,15 @@ public String getSQLQuery(String tableName, Class<?> attributeType) throws Illeg
     String conditionSuffix =
         " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? order by timestamp desc";
     if (isNumber) {
-      return "select timestamp as X, cast(value as numeric) as Y from "
+      return "select timestamp as X, cast("
+          + NULLABLE_VALUE_SQL
+          + " as numeric) as Y from "
           + tableName
           + conditionSuffix;
     } else if (isBoolean) {
-      return "select timestamp as X, (case when cast(cast(value as text) as boolean) is true then 1 else 0 end) as Y from "
+      return "select timestamp as X, ("
+          + BOOLEAN_AS_NUMBER_SQL
+          + ") as Y from "
           + tableName
           + conditionSuffix;
     } else {

model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java

@@ -47,17 +47,26 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {
       description =
           "When true, returns empty buckets across the requested range using time-bucket gap "
               + "filling; their aggregate is generally null. When false, only buckets containing datapoints are "
-              + "returned. Defaults to false.",
+              + "returned. MODE currently returns only buckets containing datapoints regardless of this flag. "
+              + "Defaults to false.",
       example = "true")
   public boolean gapFill;
 
   @Schema(
       description =
           "Aggregation applied per bucket: `MIN`, `AVG`, and `MAX` calculate the corresponding "
               + "value; `DIFFERENCE` calculates the change between the last values of consecutive buckets; "
-              + "`COUNT` counts datapoints; `SUM` totals numeric values (or true values for booleans); `MODE` "
+              + "`COUNT` counts stored datapoints, including null-valued rows; `SUM` totals numeric values "
+              + "(or true values for booleans); `MODE` "
               + "selects the most frequent value; and `MEDIAN` selects the middle value. `DIFFERENCE` is numeric "
-              + "only.",
+              + "only. For numeric and boolean attributes, MIN, AVG, MAX, SUM, MODE and MEDIAN ignore null values and return null "
+              + "for buckets with no available values. Empty gap-filled buckets return null for COUNT. "
+              + "MODE ties select the smaller numeric value or true for booleans. Boolean MEDIAN returns 1 "
+              + "when the median of available 1/0 values is at least 0.5, otherwise 0; it returns null when unavailable. "
+              + "DIFFERENCE uses the last stored value per bucket. With gap filling enabled, empty buckets carry "
+              + "the previous bucket value, while explicit null values interrupt carry-forward. A difference is "
+              + "null if either endpoint is unavailable. The first returned bucket has a difference of zero "
+              + "if its value is available; observations outside the query range are not used.",
       requiredMode = Schema.RequiredMode.REQUIRED)
   public Formula formula;
 
@@ -117,12 +126,14 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
         return "WITH interval_data AS ("
             + "SELECT "
             + function
-            + "(cast(? as interval), timestamp) AS x, public.locf(public.last(value::DOUBLE PRECISION, timestamp)) AS numeric_value "
+            + "(cast(? as interval), timestamp) AS x, public.locf(public.last("
+            + NULLABLE_VALUE_SQL
+            + "::DOUBLE PRECISION, timestamp)) AS numeric_value "
             + "FROM "
             + tableName
             + " "
             + "WHERE ENTITY_ID = ? AND ATTRIBUTE_NAME = ? AND TIMESTAMP >= ? AND TIMESTAMP <= ? GROUP BY x )"
-            + "SELECT x, COALESCE(numeric_value - LAG(numeric_value, 1, numeric_value) OVER (ORDER BY x), numeric_value) AS delta FROM interval_data ORDER BY x ASC";
+            + "SELECT x, numeric_value - LAG(numeric_value, 1, numeric_value) OVER (ORDER BY x) AS delta FROM interval_data ORDER BY x ASC";
       case COUNT:
         return "SELECT "
             + function
@@ -137,7 +148,9 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
         return "SELECT "
             + function
             + "(cast(? as interval), timestamp) AS x, "
-            + "SUM(cast(value as numeric)) AS total_sum "
+            + "SUM(cast("
+            + NULLABLE_VALUE_SQL
+            + " as numeric)) AS total_sum "
             + "FROM "
             + tableName
             + " "
@@ -148,7 +161,9 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
             + "  SELECT "
             + function
             + "(cast(? as interval), timestamp) AS x, "
-            + "         cast(value as numeric) AS num_value "
+            + "         cast("
+            + NULLABLE_VALUE_SQL
+            + " as numeric) AS num_value "
             + "  FROM "
             + tableName
             + " "
@@ -158,7 +173,7 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
             + "    SELECT num_value FROM ( "
             + "         SELECT num_value, COUNT(*) AS freq "
             + "         FROM bucketed b2 "
-            + "         WHERE b2.x = d.x "
+            + "         WHERE b2.x = d.x AND num_value IS NOT NULL "
             + "         GROUP BY num_value "
             + "         ORDER BY freq DESC, num_value "
             + "         LIMIT 1 "
@@ -170,7 +185,9 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
         return "SELECT "
             + function
             + "(cast(? as interval), timestamp) AS x, "
-            + "PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY cast(value as numeric)) as median_value "
+            + "PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY cast("
+            + NULLABLE_VALUE_SQL
+            + " as numeric)) as median_value "
             + "FROM "
             + tableName
             + " "
@@ -181,7 +198,9 @@ protected String getSQLQueryForNumbers(String tableName, String function) {
             + function
             + "(cast(? as interval), timestamp) AS x, "
             + this.formula.toString().toLowerCase()
-            + "(cast(value as numeric)) FROM "
+            + "(cast("
+            + NULLABLE_VALUE_SQL
+            + " as numeric)) FROM "
             + tableName
             + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? GROUP BY x ORDER by x ASC";
     }
@@ -206,7 +225,9 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
         return "SELECT "
             + function
             + "(cast(? as interval), timestamp) AS x, "
-            + "SUM(CASE WHEN cast(value as text)::boolean THEN 1 ELSE 0 END) AS true_count_sum "
+            + "SUM("
+            + BOOLEAN_AS_NUMBER_SQL
+            + ") AS true_count_sum "
             + "FROM "
             + tableName
             + " "
@@ -217,17 +238,19 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
             + "   SELECT "
             + function
             + "(cast(? as interval), timestamp) AS x, "
-            + "          cast(value as text)::boolean AS bool_value "
+            + "          cast("
+            + NULLABLE_VALUE_SQL
+            + " as text)::boolean AS bool_value "
             + "   FROM "
             + tableName
             + " "
             + "   WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "
             + ") "
             + "SELECT d.x, ( "
-            + "    SELECT bool_value FROM ( "
+            + "    SELECT bool_value::integer FROM ( "
             + "         SELECT bool_value, COUNT(*) AS freq "
             + "         FROM bucketed b2 "
-            + "         WHERE b2.x = d.x "
+            + "         WHERE b2.x = d.x AND bool_value IS NOT NULL "
             + "         GROUP BY bool_value "
             + "         ORDER BY freq DESC, bool_value DESC "
             + "         LIMIT 1 "
@@ -239,8 +262,9 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
         return "SELECT "
             + function
             + "(cast(? as interval), timestamp) AS x, "
-            + "CASE WHEN PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY (CASE WHEN cast(value as text)::boolean THEN 1 ELSE 0 END)) >= 0.5 "
-            + "THEN true ELSE false END AS median_value "
+            + "(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ("
+            + BOOLEAN_AS_NUMBER_SQL
+            + ")) >= 0.5)::integer AS median_value "
             + "FROM "
             + tableName
             + " "
@@ -251,7 +275,9 @@ protected String getSQLQueryForBooleans(String tableName, String function) {
             + function
             + "(cast(? as interval), timestamp) AS x, "
             + this.formula.toString().toLowerCase()
-            + "(case when cast(cast(value as text) as boolean) is true then 1 else 0 end) FROM "
+            + "("
+            + BOOLEAN_AS_NUMBER_SQL
+            + ") FROM "
             + tableName
             + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? GROUP BY x ORDER by x ASC";
     }

model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java

@@ -30,13 +30,15 @@
         "Downsamples numeric or boolean datapoints from the inclusive range with the "
             + "Largest-Triangle-Three-Buckets (LTTB) algorithm while preserving the series' visual shape. Use this "
             + "to reduce a large series for charts. Results are chronological; boolean values are processed as "
-            + "1 for true and 0 for false. Other attribute value types are rejected.")
+            + "1 for true and 0 for false. Null-valued observations are omitted before downsampling, so their "
+            + "timestamps are not returned as gap markers. A range with no available values returns no points. "
+            + "Other attribute value types are rejected.")
 public final class AssetDatapointLTTBQuery extends AssetDatapointQuery {
 
   @Schema(
       description =
           "Requested maximum number of representative datapoints after downsampling. Fewer "
-              + "points are returned when the range contains fewer datapoints.",
+              + "points are returned when the range contains fewer available datapoints.",
       example = "500",
       requiredMode = Schema.RequiredMode.REQUIRED)
   public int amountOfPoints;
@@ -60,11 +62,15 @@ public String getSQLQuery(String tableName, Class<?> attributeType) throws Illeg
     boolean isNumber = Number.class.isAssignableFrom(attributeType);
     boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);
     if (isNumber) {
-      return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), cast(value as double precision), ?) from "
+      return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), cast("
+          + NULLABLE_VALUE_SQL
+          + " as double precision), ?) from "
           + tableName
           + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ?))";
     } else if (isBoolean) {
-      return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), (case when cast(cast(value as text) as boolean) is true then 1 else 0 end), ?) from "
+      return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), ("
+          + BOOLEAN_AS_NUMBER_SQL
+          + "), ?) from "
           + tableName
           + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ?))";
     } else {

model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.java

@@ -31,7 +31,8 @@
     description =
         "Returns at most one numeric datapoint: the closest value at or before the requested "
             + "time. Use `fromTimestamp` as a Unix-seconds timestamp at the HTTP API boundary. `toTimestamp`, "
-            + "`fromTime`, and `toTime` are ignored. The equality boundary is inclusive.")
+            + "`fromTime`, and `toTime` are ignored. The equality boundary is inclusive. "
+            + "If the nearest datapoint has a null value, its timestamp and null value are returned.")
 public class AssetDatapointNearestQuery extends AssetDatapointQuery {
 
   public AssetDatapointNearestQuery() {}
@@ -43,7 +44,9 @@ public AssetDatapointNearestQuery(long timestamp) {
 
   @Override
   public String getSQLQuery(String tableName, Class<?> attributeType) throws IllegalStateException {
-    return "SELECT timestamp as X, cast(value as numeric) as Y "
+    return "SELECT timestamp as X, cast("
+        + NULLABLE_VALUE_SQL
+        + " as numeric) as Y "
         + "    FROM "
         + tableName
         + " "

model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java

@@ -40,9 +40,11 @@
     defaultImpl = AssetDatapointAllQuery.class)
 @Schema(
     description =
-        "Polymorphic historical-datapoint query. Select `all` to return every value, "
+        "Polymorphic historical or predicted datapoint query. Select `all` to return every value, "
             + "`interval` to aggregate values into time buckets, `lttb` to downsample a numeric "
-            + "or boolean series, or `nearest` to retrieve the last value at or before a time.",
+            + "or boolean series, or `nearest` to retrieve the last numeric value at or before a time. "
+            + "Returned datapoints use x for the timestamp and y for the value. When a query retains an "
+            + "unavailable datapoint, its JSON representation contains x but omits y.",
     discriminatorProperty = "type",
     discriminatorMapping = {
       @DiscriminatorMapping(value = "all", schema = AssetDatapointAllQuery.class),
@@ -52,6 +54,15 @@
     })
 public abstract class AssetDatapointQuery implements Serializable {
 
+  /** Converts stored JSON null to SQL NULL before casting or aggregating a value. */
+  protected static final String NULLABLE_VALUE_SQL = "nullif(value, 'null'::jsonb)";
+
+  /** Maps true/false to 1/0 while preserving unavailable boolean values as SQL NULL. */
+  protected static final String BOOLEAN_AS_NUMBER_SQL =
+      "CASE cast("
+          + NULLABLE_VALUE_SQL
+          + " as text)::boolean WHEN true THEN 1 WHEN false THEN 0 ELSE NULL END";
+
   @Schema(
       description =
           "Inclusive lower range bound as Unix time in milliseconds for `all`, `interval`, "