diff --git a/stone/backends/python_rsrc/stone_serializers.py b/stone/backends/python_rsrc/stone_serializers.py index 4317ee34..f6e21f2f 100644 --- a/stone/backends/python_rsrc/stone_serializers.py +++ b/stone/backends/python_rsrc/stone_serializers.py @@ -878,7 +878,7 @@ def make_stone_friendly(self, data_type, val, validate): elif isinstance(data_type, bv.Bytes): if self.for_msgpack: if isinstance(val, str): - ret = val.encode('utf-8') + ret = val.encode('utf-8', errors='surrogateescape') else: ret = val else: @@ -987,18 +987,17 @@ def _strftime(dt, fmt): def msgpack_encode(data_type, obj): return msgpack.dumps( - msgpack_compat_obj_encode(data_type, obj), encoding='utf-8') + msgpack_compat_obj_encode(data_type, obj), use_bin_type=True) msgpack_compat_obj_decode = functools.partial(json_compat_obj_decode, for_msgpack=True) def msgpack_decode( data_type, serialized_obj, alias_validators=None, strict=True): - # We decode everything as utf-8 because we want all object keys to be - # unicode. Otherwise, we need to do a lot more refactoring to make - # json/msgpack share the same code. We expect byte arrays to fail - # decoding, but when they don't, we have to convert them to bytes. + # Decode string keys as unicode while keeping bin values as bytes. + # Older encoders used raw strings for bytes; surrogateescape preserves + # those bytes until the Bytes validator converts them back. deserialized_obj = msgpack.loads( - serialized_obj, encoding='utf-8', unicode_errors='ignore') + serialized_obj, raw=False, unicode_errors='surrogateescape') return msgpack_compat_obj_decode( - data_type, deserialized_obj, alias_validators, strict) + data_type, deserialized_obj, alias_validators=alias_validators, strict=strict) diff --git a/test/requirements.txt b/test/requirements.txt index 52df1c78..79fc9cc1 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,3 @@ coverage>=7.16.0 -pytest>=9.1.1 \ No newline at end of file +pytest>=9.1.1 +msgpack>=1.0.0 diff --git a/test/test_msgpack.py b/test/test_msgpack.py new file mode 100644 index 00000000..83b6e933 --- /dev/null +++ b/test/test_msgpack.py @@ -0,0 +1,43 @@ +import msgpack +import pytest + +from stone.backends.python_rsrc import stone_serializers as ss +from stone.backends.python_rsrc import stone_validators as bv + + +@pytest.mark.parametrize('validator,value', [ + (bv.String(), 'hello \u2650'), + (bv.Bytes(), b'\x00\xff\x80hello'), + (bv.List(bv.Bytes()), [b'\xff', b'hello', b'']), + (bv.Map(bv.String(), bv.Bytes()), {'data': b'\xff\x80'}), +]) +def test_msgpack_round_trip(validator, value): + encoded = ss.msgpack_encode(validator, value) + assert ss.msgpack_decode(validator, encoded) == value + + +@pytest.mark.parametrize('value', [b'', b'hello', b'\xff\x80\x00hello']) +def test_msgpack_legacy_raw_bytes(value): + encoded = msgpack.packb(value, use_bin_type=False) + assert ss.msgpack_decode(bv.Bytes(), encoded) == value + + +def test_msgpack_alias_validators(): + validator = bv.String() + calls = [] + + def reject(value): + calls.append(value) + raise bv.ValidationError('alias rejected') + + with pytest.raises(bv.ValidationError, match='alias rejected'): + ss.msgpack_decode( + validator, msgpack.packb('hello'), alias_validators={validator: reject}) + assert calls == ['hello'] + + +def test_msgpack_strict_option(): + encoded = msgpack.packb('unexpected') + with pytest.raises(bv.ValidationError, match='expected null'): + ss.msgpack_decode(bv.Void(), encoded) + assert ss.msgpack_decode(bv.Void(), encoded, strict=False) is None