wip: implementing character enconding in boxes_to_text and ToString

Author: mmatera

mathics/builtin/strings.py

@@ -16,6 +16,8 @@
 from mathics.core.expression import (Expression, Symbol, SymbolFailed, SymbolFalse, SymbolTrue, String, Integer,
                                      from_python, string_list)
 from mathics.builtin.lists import python_seq, convert_seq
+from mathics_scanner.characters import replace_wl_with_plain_text
+from mathics.settings import SYSTEM_CHARACTER_ENCODING
 
 
 _regex_longest = {
@@ -254,6 +256,20 @@ def unpack_bytes(codes):
         return unpack('B' * len(codes), codes)
 
 
+class SystemCharacterEncoding(Predefined):
+    """
+    <dl>
+    <dt>$SystemCharacterEncoding
+
+    </dl>
+    """
+    name = "$SystemCharacterEncoding"
+
+    rules = {
+        '$CharacterEncoding': SYSTEM_CHARACTER_ENCODING,
+    }
+
+
 class CharacterEncoding(Predefined):
     """
     <dl>
@@ -1576,19 +1592,19 @@ class ToString(Builtin):
      = U2
     """
 
-    options = {'CharacterEncoding' : '"Unicode"',
+    options = { 'CharacterEncoding' : '"Unicode"',
                 'FormatType' : 'OutputForm',
                 'NumberMarks': '$NumberMarks',
                 'PageHeight' : 'Infinity',
                 'PageWidth' : 'Infinity',
                 'TotalHeight' : 'Infinity',
                 'TotalWidth' : 'Infinity'}
 
-    def apply(self, value, evaluation):
-        'ToString[value_]'
-
+    def apply(self, value, evaluation, **options):
+        'ToString[value_, OptionsPattern[ToString]]'
+        encoding = options['options']["System`CharacterEncoding"]
         text = value.format(evaluation, 'System`OutputForm').boxes_to_text(
-            evaluation=evaluation)
+            evaluation=evaluation, encoding=encoding)
         return String(text)
 
 

mathics/core/expression.py

@@ -17,6 +17,7 @@
 from mathics.core.convert import sympy_symbol_prefix, SympyExpression
 import base64
 
+from mathics_scanner.characters import replace_wl_with_plain_text
 
 def fully_qualified_symbol_name(name) -> bool:
     return (
@@ -1904,7 +1905,9 @@ def do_copy(self) -> "Symbol":
         return Symbol(self.name)
 
     def boxes_to_text(self, **options) -> str:
-        return str(self.name)
+        if options.get("encoding", "UTF-8") in ("UTF-8", "Unicode"):
+            return str(self.name)
+        return replace_wl_with_plain_text(str(self.name), False)
 
     def atom_to_boxes(self, f, evaluation) -> "String":
         return String(evaluation.definitions.shorten_name(self.name))
@@ -2723,7 +2726,9 @@ def boxes_to_text(self, show_string_characters=False, **options) -> str:
         ):
             value = value[1:-1]
 
-        return value
+        if options.get("encoding", "UTF-8") in ("UTF-8", "Unicode"):
+            return value
+        return replace_wl_with_plain_text(value, False)
 
     def boxes_to_xml(self, show_string_characters=False, **options) -> str:
         from mathics.core.parser import is_symbol_name

mathics/settings.py

@@ -72,3 +72,4 @@
 # whatever it is that setting this thing did.
 default_pymathics_modules = []
 
+SYSTEM_CHARACTER_ENCODING = "UTF-8" if sys.getdefaultencoding()=='utf-8' else 'ASCII'