[feat] initial support of bjdata draft 4 extension types

Author: fangq

jdataencode.m

@@ -45,6 +45,7 @@
 %                  toeplitz, and use _ArrayShape_ to encode the matrix
 %         MapAsStruct: [0|1] if set to 1, convert containers.Map into
 %                  struct; otherwise, keep it as map
+%         DateTime: [1|0] if set to 1, convert datetime to string
 %         Compression: ['zlib'|'gzip','lzma','lz4','lz4hc'] - use zlib method
 %                  to compress data array
 %         CompressArraySize: [300|int]: only to compress an array if the
@@ -93,6 +94,7 @@
 opt.messagepack = jsonopt('MessagePack', 0, opt);
 opt.usearrayshape = jsonopt('UseArrayShape', 0, opt) && exist('bandwidth');
 opt.annotatearray = jsonopt('AnnotateArray', 0, opt);
+opt.datetime = jsonopt('DateTime', 1, opt);
 
 % Performance optimization: pre-compute prefixed field names to avoid
 % repeated string concatenation in hot loops
@@ -467,6 +469,10 @@
 
 %% -------------------------------------------------------------------------
 function newitem = matlabobject2jd(item, opt)
+if (~opt.datetime && (isa(item, 'datetime') || isa(item, 'duration')))
+    newitem = item;
+    return
+end
 try
     if numel(item) == 0 % empty object
         newitem = struct();

loadbj.m

@@ -107,10 +107,8 @@
     else
         inputstr = urlread(fname);
     end
-elseif (~isempty(fname) && any(fname(1) == '[{SCBHiUIulmLMhdDTFZN'))
-    inputstr = fname;
 else
-    error('input file does not exist or buffer is invalid');
+    inputstr = fname;
 end
 
 inputlen = length(inputstr);
@@ -192,7 +190,7 @@
             else
                 [data{jsoncount}, pos] = parse_array(pos, opt);
             end
-        case {'S', 'C', 'B', 'H', 'i', 'U', 'I', 'u', 'l', 'm', 'L', 'M', 'h', 'd', 'D', 'T', 'F', 'Z', 'N'}
+        case {'S', 'C', 'B', 'H', 'i', 'U', 'I', 'u', 'l', 'm', 'L', 'M', 'h', 'd', 'D', 'T', 'F', 'Z', 'N', 'E'}
             [data{jsoncount}, pos] = parse_value(pos, [], opt);
         otherwise
             error_pos('Root level structure must start with a valid marker "{[SCBHiUIulmLMhdDTFZN"', opt, pos);
@@ -639,6 +637,8 @@
         end
         val = [];
         pos = pos + 1;
+    case 'E'
+        [val, pos] = parse_extension(pos, opt);
     otherwise
         error_pos('Value expected at position %d', opt, pos);
 end
@@ -1162,3 +1162,73 @@
         values{i} = '';
     end
 end
+
+%% -------------------------------------------------------------------------
+
+function [val, pos] = parse_extension(pos, opt)
+% Parse BJData Extension: [E][type-id][byte-length][payload]
+pos = pos + 1;
+[typeid, pos] = parse_number(pos, opt);
+[bytelen, pos] = parse_number(pos, opt);
+typeid = double(typeid);
+bytelen = double(bytelen);
+
+if bytelen > 0
+    payload = uint8(opt.inputstr_(pos:pos + bytelen - 1));
+    pos = pos + bytelen;
+else
+    payload = uint8([]);
+end
+
+% Swap bytes for Little-Endian (UUID excluded - Big-Endian per RFC 4122)
+doswap = opt.flipendian_ && typeid ~= 10;
+sw = @(d) d(end:-1:1);  % byte reversal helper
+
+switch typeid
+    case 1  % epoch_s: uint32
+        if doswap
+            payload = sw(payload);
+        end
+        val = datetime(double(typecast(payload, 'uint32')), 'ConvertFrom', 'posixtime', 'TimeZone', 'UTC');
+    case {2, 6}  % epoch_us, datetime_us: int64
+        if doswap
+            payload = sw(payload);
+        end
+        val = datetime(double(typecast(payload, 'int64')) / 1e6, 'ConvertFrom', 'posixtime', 'TimeZone', 'UTC');
+    case 3  % epoch_ns: int64 + uint32
+        if doswap
+            payload = [sw(payload(1:8)), sw(payload(9:12))];
+        end
+        val = datetime(double(typecast(payload(1:8), 'int64')) + double(typecast(payload(9:12), 'uint32')) / 1e9, ...
+                       'ConvertFrom', 'posixtime', 'TimeZone', 'UTC');
+    case 4  % date: int16 + 2*uint8
+        if doswap
+            payload(1:2) = sw(payload(1:2));
+        end
+        val = datetime(double(typecast(payload(1:2), 'int16')), double(payload(3)), double(payload(4)));
+    case 5  % time_s: 3*uint8 + reserved
+        val = duration(double(payload(1)), double(payload(2)), double(payload(3)));
+    case 7  % timedelta_us: int64
+        if doswap
+            payload = sw(payload);
+        end
+        val = duration(0, 0, double(typecast(payload, 'int64')) / 1e6);
+    case 8  % complex64: 2*float32
+        if doswap
+            payload = [sw(payload(1:4)), sw(payload(5:8))];
+        end
+        p = typecast(payload, 'single');
+        val = complex(double(p(1)), double(p(2)));
+    case 9  % complex128: 2*float64
+        if doswap
+            payload = [sw(payload(1:8)), sw(payload(9:16))];
+        end
+        p = typecast(payload, 'double');
+        val = complex(p(1), p(2));
+    case 10 % uuid: 16 bytes Big-Endian
+        h = lower(reshape(dec2hex(payload, 2)', 1, []));
+        val = jdict([h(1:8) '-' h(9:12) '-' h(13:16) '-' h(17:20) '-' h(21:32)], ...
+                    'schema', struct('type', 'string', 'format', 'uuid'));
+    otherwise % unknown extension
+        val = jdict(payload, 'schema', struct('type', 'bytes', 'exttype', int32(typeid)));
+end

savebj.m

@@ -243,13 +243,13 @@
 skippreencode = false;
 if (opt.formatversion >= 4 && ~opt.messagepack && ~opt.ubjson)
     if (isa(obj, 'table') && size(obj, 1) > 1)
-        [cansoa, ~] = table2soa(obj, opt);
+        cansoa = table2soa(obj, opt);
         skippreencode = cansoa;
     end
 end
 
 if (~skippreencode && jsonopt('PreEncode', 1, opt))
-    obj = jdataencode(obj, 'Base64', 0, 'UseArrayZipSize', opt.messagepack, opt);
+    obj = jdataencode(obj, 'Base64', 0, 'UseArrayZipSize', opt.messagepack, 'DateTime', 0, opt);
 end
 
 dozip = opt.compression;
@@ -321,6 +321,12 @@
     txt = struct2ubjson(name, item, level, opt);
 elseif (isnumeric(item) || islogical(item))
     txt = mat2ubjson(name, item, level, opt);
+elseif (isa(item, 'datetime'))
+    txt = ext2ubjson(name, item, 'datetime', opt);
+elseif (isa(item, 'duration'))
+    txt = ext2ubjson(name, item, 'duration', opt);
+elseif (isa(item, 'jdict'))
+    txt = jdict2ubjson(name, item, level, opt);
 elseif (ischar(item))
     if (numel(item) >= opt.compressstringsize)
         txt = mat2ubjson(name, item, level, opt);
@@ -339,6 +345,8 @@
     txt = struct2ubjson(name, jdataencode(item), level, opt);
 elseif (isobject(item))
     txt = matlabobject2ubjson(name, item, level, opt);
+elseif (~isreal(item) && isnumeric(item))
+    txt = ext2ubjson(name, item, 'complex', opt);
 else
     txt = any2ubjson(name, item, level, opt);
 end
@@ -1744,3 +1752,130 @@
 else
     newdata = data;
 end
+
+%% -------------------------------------------------------------------------
+function txt = ext2ubjson(name, item, dtype, opt)
+% Unified extension encoder for datetime, duration, complex
+if opt.messagepack || opt.ubjson
+    switch dtype
+        case 'datetime'
+            txt = str2ubjson(name, char(item), 0, opt);
+        case 'duration'
+            txt = mat2ubjson(name, seconds(item), 0, opt);
+        case 'complex'
+            txt = struct2ubjson(name, struct('re', real(item), 'im', imag(item)), 0, opt);
+    end
+    return
+end
+
+if numel(item) > 1
+    parts = cell(1, numel(item) + 2);
+    parts{1} = opt.AM_{1};
+    for i = 1:numel(item)
+        parts{i + 1} = encode_ext_scalar(item(i), dtype, opt);
+    end
+    parts{end} = opt.AM_{2};
+    txt = [parts{:}];
+else
+    txt = encode_ext_scalar(item, dtype, opt);
+end
+if ~isempty(name)
+    txt = [N_(decodevarname(name, opt.unpackhex), opt) txt];
+end
+
+%% -------------------------------------------------------------------------
+function txt = encode_ext_scalar(val, dtype, opt)
+% Unified encoder for datetime, duration, complex scalars
+% Returns 'Z' for NaT/NaN, otherwise [E][typeid][len][payload]
+switch dtype
+    case 'datetime'
+        if isnat(val)
+            txt = opt.ZM_;
+            return
+        end
+        tz = val.TimeZone;
+        if isempty(tz)
+            val.TimeZone = 'UTC';
+        end
+        pt = posixtime(val);
+        hasTime = hour(val) || minute(val) || second(val);
+        if ~hasTime && isempty(tz)
+            typeid = 4;
+            payload = [typecast(int16(year(val)), 'uint8'), uint8([month(val), day(val)])];
+        elseif mod(second(val), 1) ~= 0 || pt < 0 || pt > 4294967295
+            typeid = 6;
+            payload = typecast(int64(round(pt * 1e6)), 'uint8');
+        else
+            typeid = 1;
+            payload = typecast(uint32(pt), 'uint8');
+        end
+    case 'duration'
+        if isnan(val)
+            txt = opt.ZM_;
+            return
+        end
+        typeid = 7;
+        payload = typecast(int64(round(seconds(val) * 1e6)), 'uint8');
+    case 'complex'
+        if isa(val, 'single')
+            typeid = 8;
+            payload = [typecast(single(real(val)), 'uint8'), typecast(single(imag(val)), 'uint8')];
+        else
+            typeid = 9;
+            payload = [typecast(double(real(val)), 'uint8'), typecast(double(imag(val)), 'uint8')];
+        end
+end
+% Apply endian swap based on typeid
+if opt.flipendian_
+    n = length(payload);
+    if typeid == 4  % date: only swap first 2 bytes (int16)
+        payload(1:2) = payload(2:-1:1);
+    elseif typeid == 8  % complex64: swap two 4-byte floats
+        payload = [payload(4:-1:1), payload(8:-1:5)];
+    elseif typeid == 9  % complex128: swap two 8-byte floats
+        payload = [payload(8:-1:1), payload(16:-1:9)];
+    elseif n > 1  % all others: simple reversal
+        payload = payload(n:-1:1);
+    end
+end
+txt = ['E' I_(uint8(typeid), opt) I_(uint8(length(payload)), opt) char(payload)];
+
+%% -------------------------------------------------------------------------
+function txt = jdict2ubjson(name, item, level, opt)
+% Handle jdict objects - check schema for special types
+if ~isa(item, 'jdict')
+    txt = struct2ubjson(name, item, level, opt);
+    return
+end
+s = item.schema;
+if item.getattr('$', '') && strcmp(s.format, 'uuid')
+    % UUID string
+    if opt.messagepack || opt.ubjson
+        txt = str2ubjson(name, char(item), level, opt);
+        return
+    end
+    uuidstr = char(item);
+    hexstr = strrep(uuidstr, '-', '');
+    payload = uint8(zeros(1, 16));
+    for i = 1:16
+        payload(i) = hex2dec(hexstr(2 * i - 1:2 * i));
+    end
+    txt = ['E' I_(uint8(10), opt) I_(uint8(16), opt) char(payload)];
+    if ~isempty(name)
+        txt = [N_(decodevarname(name, opt.unpackhex), opt) txt];
+    end
+elseif isfield(s, 'type') && strcmp(s.type, 'bytes')
+    % Raw extension bytes
+    payload = uint8(item.data);
+    typeid = uint32(0);
+    if isfield(s, 'exttype')
+        typeid = uint32(s.exttype);
+    end
+    txt = ['E' I_(typeid, opt) I_(uint32(length(payload)), opt) char(payload)];
+    if ~isempty(name)
+        txt = [N_(decodevarname(name, opt.unpackhex), opt) txt];
+    end
+else
+    % Generic jdict - encode as struct
+    txt = struct2ubjson(name, struct(item), level, opt);
+end