fix(findrefs): reach non-BMP strings and class names in the locators

droidasc/asc_client/apk_handler.py

@@ -366,9 +366,8 @@ def get_class_dex(self, dalvik_class : str):
         # search in _find_type_idx relies on.
         #
         # The two encodings differ only above the BMP (and for an embedded NUL,
-        # which a descriptor cannot hold), so only such a name pays for the
-        # encoder - a compiled extension costing ~19ms to import, which every
-        # ordinary lookup would otherwise drag in on this path.
+        # which a descriptor cannot hold), so only such a name pays for the encoder
+        # and an ordinary lookup stays on the plain UTF-8 path.
         target_bytes = dalvik_class.encode("utf-8")
         if any(ord(ch) > 0xFFFF for ch in dalvik_class):
             from mutf8 import encode_modified_utf8

droidasc/asc_core/findrefs/locator/field_locator.py

@@ -48,14 +48,17 @@ def _find_type_idx_precisely(self, clz : str):
         right = type_ids_size - 1
         buf = self.buf
         strings = self.dex.strings
+        # see MethodLocator._find_type_idx_precisely: type_ids is sorted by UTF-16
+        # code unit, so the search has to compare in that order, not Python's
+        key = clz.encode('utf-16-be', 'surrogatepass')
 
         while left <= right:
             mid = (left + right) >> 1
             desc_idx = _STRUCT_I.unpack_from(buf, type_ids_off + (mid << 2))[0]
             desc = strings[desc_idx]
             if desc == clz:
                 return mid
-            if desc < clz:
+            if desc.encode('utf-16-be', 'surrogatepass') < key:
                 left = mid + 1
             else:
                 right = mid - 1

droidasc/asc_core/findrefs/locator/method_locator.py

@@ -47,14 +47,21 @@ def _find_type_idx_precisely(self, clz : str):
         right = type_ids_size - 1
         buf = self.buf
         strings = self.dex.strings
+        # type_ids is sorted by UTF-16 code unit, which is not Python's order above
+        # the BMP: a surrogate pair is 0xD800-0xDFFF, so U+10000 sorts before
+        # U+E000, while Python compares those two the other way round. Comparing
+        # string values prunes the wrong half and misses entries; comparing the
+        # UTF-16BE encoding compares in the table's own order. surrogatepass because
+        # a malformed dex can decode to lone surrogates, which utf-16 refuses.
+        key = clz.encode('utf-16-be', 'surrogatepass')
 
         while left <= right:
             mid = (left + right) >> 1
             desc_idx = _STRUCT_I.unpack_from(buf, type_ids_off + (mid << 2))[0]
             desc = strings[desc_idx]
             if desc == clz:
                 return mid
-            if desc < clz:
+            if desc.encode('utf-16-be', 'surrogatepass') < key:
                 left = mid + 1
             else:
                 right = mid - 1

droidasc/asc_core/findrefs/locator/string_locator.py

@@ -46,13 +46,20 @@ def _build_map(self):
 
     def _match_string_offset(self, string : str):
         buf = self.buf
-        string = string.encode('utf-8')
+        # the region below holds MUTF-8, where a code point above the BMP is stored
+        # as a surrogate pair (CESU-8) that a UTF-8 pattern can never match. The two
+        # encodings agree below the BMP, so only such a pattern pays for the encoder
+        # and an ordinary search stays on the plain UTF-8 path.
+        pattern_bytes = string.encode('utf-8')
+        if any(ord(ch) > 0xFFFF for ch in string):
+            from mutf8 import encode_modified_utf8
+            pattern_bytes = encode_modified_utf8(string)
         strdata_start = self.strdata_start
         strdata_end = self.strdata_end
         submem = buf[strdata_start: strdata_end]
 
         mm = submem.obj
-        pattern = re.compile(string)
+        pattern = re.compile(pattern_bytes)
 
         # offsets = []
         for match in pattern.finditer(submem):

tests/test_references.py

@@ -155,6 +155,15 @@ def test_empty_string_table(self):
         struct.pack_into('<II', data, 56, 0, 0)
         self.assertEqual(self.locate(data, 'token'), set())
 
+    def test_non_bmp_string_is_searchable(self):
+        # the region holds MUTF-8, where a code point above the BMP is stored as a
+        # surrogate pair (CESU-8), so a UTF-8 pattern never matches it
+        from test_mutf8_strings import _make_class_dex
+        emoji = '\U0001f600'
+        data = _make_class_dex(['Ll/' + emoji + ';'])
+        self.assertEqual(self.locate(data, emoji), {0})
+        self.assertEqual(self.locate(data, 'Ll/'), {0})
+
 
 class DependencyTests(unittest.TestCase):
     def test_reference_search_without_site_packages(self):

tests/test_source_edit.py

@@ -441,6 +441,21 @@ def test_exact_method_locator_does_not_use_substring_matching(self):
             "method": ["first", True],
         }), {0})
 
+    def test_precise_class_lookup_survives_utf16_code_unit_order(self):
+        # type_ids is sorted by UTF-16 code unit, where U+10000 (a surrogate pair)
+        # sorts before U+E000 - the opposite of Python's code point order. A value
+        # comparison prunes the wrong half and walks past the entry it wants.
+        from test_mutf8_strings import _make_class_dex
+
+        names = ['Ll/\U00010000;', 'Ll/;']
+        dex = DEX.parse(memoryview(bytes(_make_class_dex(names))), 'utf16-order.dex')
+        method = FindRefManager(dex)._get_method_locator(True)
+        field = FindRefManager(dex)._get_field_locator(True)
+        for index, name in enumerate(names):
+            with self.subTest(name=name):
+                self.assertEqual(method._find_type_idx_precisely(name), index)
+                self.assertEqual(field._find_type_idx_precisely(name), index)
+
     def test_exact_field_locator_does_not_use_substring_matching(self):
         from dex_fixture import make_static_field_dex