Fix experimental MessagePack round trips with current msgpack

stone/backends/python_rsrc/stone_serializers.py

@@ -878,7 +878,7 @@ def make_stone_friendly(self, data_type, val, validate):
         elif isinstance(data_type, bv.Bytes):
             if self.for_msgpack:
                 if isinstance(val, str):
-                    ret = val.encode('utf-8')
+                    ret = val.encode('utf-8', errors='surrogateescape')
                 else:
                     ret = val
             else:
@@ -987,18 +987,17 @@ def _strftime(dt, fmt):
 
     def msgpack_encode(data_type, obj):
         return msgpack.dumps(
-            msgpack_compat_obj_encode(data_type, obj), encoding='utf-8')
+            msgpack_compat_obj_encode(data_type, obj), use_bin_type=True)
 
     msgpack_compat_obj_decode = functools.partial(json_compat_obj_decode,
                                                   for_msgpack=True)
 
     def msgpack_decode(
             data_type, serialized_obj, alias_validators=None, strict=True):
-        # We decode everything as utf-8 because we want all object keys to be
-        # unicode. Otherwise, we need to do a lot more refactoring to make
-        # json/msgpack share the same code. We expect byte arrays to fail
-        # decoding, but when they don't, we have to convert them to bytes.
+        # Decode string keys as unicode while keeping bin values as bytes.
+        # Older encoders used raw strings for bytes; surrogateescape preserves
+        # those bytes until the Bytes validator converts them back.
         deserialized_obj = msgpack.loads(
-            serialized_obj, encoding='utf-8', unicode_errors='ignore')
+            serialized_obj, raw=False, unicode_errors='surrogateescape')
         return msgpack_compat_obj_decode(
-            data_type, deserialized_obj, alias_validators, strict)
+            data_type, deserialized_obj, alias_validators=alias_validators, strict=strict)

test/requirements.txt

@@ -1,2 +1,3 @@
 coverage>=7.16.0
-pytest>=9.1.1
+pytest>=9.1.1
+msgpack>=1.0.0

test/test_msgpack.py

@@ -0,0 +1,43 @@
+import msgpack
+import pytest
+
+from stone.backends.python_rsrc import stone_serializers as ss
+from stone.backends.python_rsrc import stone_validators as bv
+
+
+@pytest.mark.parametrize('validator,value', [
+    (bv.String(), 'hello \u2650'),
+    (bv.Bytes(), b'\x00\xff\x80hello'),
+    (bv.List(bv.Bytes()), [b'\xff', b'hello', b'']),
+    (bv.Map(bv.String(), bv.Bytes()), {'data': b'\xff\x80'}),
+])
+def test_msgpack_round_trip(validator, value):
+    encoded = ss.msgpack_encode(validator, value)
+    assert ss.msgpack_decode(validator, encoded) == value
+
+
+@pytest.mark.parametrize('value', [b'', b'hello', b'\xff\x80\x00hello'])
+def test_msgpack_legacy_raw_bytes(value):
+    encoded = msgpack.packb(value, use_bin_type=False)
+    assert ss.msgpack_decode(bv.Bytes(), encoded) == value
+
+
+def test_msgpack_alias_validators():
+    validator = bv.String()
+    calls = []
+
+    def reject(value):
+        calls.append(value)
+        raise bv.ValidationError('alias rejected')
+
+    with pytest.raises(bv.ValidationError, match='alias rejected'):
+        ss.msgpack_decode(
+            validator, msgpack.packb('hello'), alias_validators={validator: reject})
+    assert calls == ['hello']
+
+
+def test_msgpack_strict_option():
+    encoded = msgpack.packb('unexpected')
+    with pytest.raises(bv.ValidationError, match='expected null'):
+        ss.msgpack_decode(bv.Void(), encoded)
+    assert ss.msgpack_decode(bv.Void(), encoded, strict=False) is None