diff --git a/droidasc/asc_client/apk_handler.py b/droidasc/asc_client/apk_handler.py index b648eff..f943455 100644 --- a/droidasc/asc_client/apk_handler.py +++ b/droidasc/asc_client/apk_handler.py @@ -366,9 +366,8 @@ def get_class_dex(self, dalvik_class : str): # search in _find_type_idx relies on. # # The two encodings differ only above the BMP (and for an embedded NUL, - # which a descriptor cannot hold), so only such a name pays for the - # encoder - a compiled extension costing ~19ms to import, which every - # ordinary lookup would otherwise drag in on this path. + # which a descriptor cannot hold), so only such a name pays for the encoder + # and an ordinary lookup stays on the plain UTF-8 path. target_bytes = dalvik_class.encode("utf-8") if any(ord(ch) > 0xFFFF for ch in dalvik_class): from mutf8 import encode_modified_utf8 diff --git a/droidasc/asc_core/findrefs/locator/field_locator.py b/droidasc/asc_core/findrefs/locator/field_locator.py index 4192746..4a6bcf5 100644 --- a/droidasc/asc_core/findrefs/locator/field_locator.py +++ b/droidasc/asc_core/findrefs/locator/field_locator.py @@ -48,6 +48,9 @@ def _find_type_idx_precisely(self, clz : str): right = type_ids_size - 1 buf = self.buf strings = self.dex.strings + # see MethodLocator._find_type_idx_precisely: type_ids is sorted by UTF-16 + # code unit, so the search has to compare in that order, not Python's + key = clz.encode('utf-16-be', 'surrogatepass') while left <= right: mid = (left + right) >> 1 @@ -55,7 +58,7 @@ def _find_type_idx_precisely(self, clz : str): desc = strings[desc_idx] if desc == clz: return mid - if desc < clz: + if desc.encode('utf-16-be', 'surrogatepass') < key: left = mid + 1 else: right = mid - 1 diff --git a/droidasc/asc_core/findrefs/locator/method_locator.py b/droidasc/asc_core/findrefs/locator/method_locator.py index bc9b7c8..f46bcfe 100644 --- a/droidasc/asc_core/findrefs/locator/method_locator.py +++ b/droidasc/asc_core/findrefs/locator/method_locator.py @@ -47,6 +47,13 @@ def _find_type_idx_precisely(self, clz : str): right = type_ids_size - 1 buf = self.buf strings = self.dex.strings + # type_ids is sorted by UTF-16 code unit, which is not Python's order above + # the BMP: a surrogate pair is 0xD800-0xDFFF, so U+10000 sorts before + # U+E000, while Python compares those two the other way round. Comparing + # string values prunes the wrong half and misses entries; comparing the + # UTF-16BE encoding compares in the table's own order. surrogatepass because + # a malformed dex can decode to lone surrogates, which utf-16 refuses. + key = clz.encode('utf-16-be', 'surrogatepass') while left <= right: mid = (left + right) >> 1 @@ -54,7 +61,7 @@ def _find_type_idx_precisely(self, clz : str): desc = strings[desc_idx] if desc == clz: return mid - if desc < clz: + if desc.encode('utf-16-be', 'surrogatepass') < key: left = mid + 1 else: right = mid - 1 diff --git a/droidasc/asc_core/findrefs/locator/string_locator.py b/droidasc/asc_core/findrefs/locator/string_locator.py index f410f92..c42f6de 100644 --- a/droidasc/asc_core/findrefs/locator/string_locator.py +++ b/droidasc/asc_core/findrefs/locator/string_locator.py @@ -46,13 +46,20 @@ def _build_map(self): def _match_string_offset(self, string : str): buf = self.buf - string = string.encode('utf-8') + # the region below holds MUTF-8, where a code point above the BMP is stored + # as a surrogate pair (CESU-8) that a UTF-8 pattern can never match. The two + # encodings agree below the BMP, so only such a pattern pays for the encoder + # and an ordinary search stays on the plain UTF-8 path. + pattern_bytes = string.encode('utf-8') + if any(ord(ch) > 0xFFFF for ch in string): + from mutf8 import encode_modified_utf8 + pattern_bytes = encode_modified_utf8(string) strdata_start = self.strdata_start strdata_end = self.strdata_end submem = buf[strdata_start: strdata_end] mm = submem.obj - pattern = re.compile(string) + pattern = re.compile(pattern_bytes) # offsets = [] for match in pattern.finditer(submem): diff --git a/tests/test_references.py b/tests/test_references.py index f607114..cef4548 100644 --- a/tests/test_references.py +++ b/tests/test_references.py @@ -155,6 +155,15 @@ def test_empty_string_table(self): struct.pack_into('