Merge pull request #195 from frescobaldi/fix-ascii-digit-regexes

Match only ASCII digits in the lexer number patterns

Author: bmjcode

ly/lex/html.py

@@ -95,8 +95,9 @@ class StringSQEnd(String, _token.StringEnd, _token.Leaver):
     rx = r"'"
 
 
+# [0-9], not \d: a numeric character reference is ASCII digits only
 class EntityRef(_token.Character):
-    rx = r"\&(#\d+|#[xX][0-9A-Fa-f]+|[A-Za-z_:][\w.:_-]*);"
+    rx = r"\&(#[0-9]+|#[xX][0-9A-Fa-f]+|[A-Za-z_:][\w.:_-]*);"
 
 
 class LilyPondTag(Tag):

ly/lex/lilypond.py

@@ -28,6 +28,9 @@
 from . import _token
 from . import Parser, FallthroughParser
 
+# digit patterns use [0-9]: \d also matches Unicode digits that are not legal
+# LilyPond. \d in a negated class excludes those too, so it stays
+
 # an identifier allowing letters and single hyphens in between
 re_identifier = r"[^\W\d_]+([_-][^\W\d_]+)*"
 
@@ -45,7 +48,7 @@
 re_duration = rf"(\\(maxima|longa|breve){re_identifier_end}|(1|2|4|8|16|32|64|128|256|512|1024|2048)(?!\d))"
 
 re_dot = r"\."
-re_scaling = r"\*[\t ]*\d+(/\d+)?"
+re_scaling = r"\*[\t ]*[0-9]+(/[0-9]+)?"
 
 
 
@@ -72,15 +75,15 @@ class Value(_token.Item, _token.Numeric):
 
 
 class DecimalValue(Value):
-    rx = r"-?\d+(\.\d+)?"
+    rx = r"-?[0-9]+(\.[0-9]+)?"
 
 
 class IntegerValue(DecimalValue):
-    rx = r"\d+"
+    rx = r"[0-9]+"
 
 
 class Fraction(Value):
-    rx = r"\d+/\d+"
+    rx = r"[0-9]+/[0-9]+"
 
 
 class Delimiter(_token.Token):
@@ -289,11 +292,11 @@ class ScriptAbbreviation(Articulation, _token.Leaver):
 
 
 class Fingering(Articulation, _token.Leaver):
-    rx = r"\d+"
+    rx = r"[0-9]+"
 
 
 class StringNumber(Articulation):
-    rx = r"\\\d+"
+    rx = r"\\[0-9]+"
 
 
 class Slur(_token.Token):
@@ -379,7 +382,7 @@ class ChordSeparator(ChordItem):
 
 
 class ChordStepNumber(ChordItem):
-    rx = r"\d+[-+]?"
+    rx = r"[0-9]+[-+]?"
 
 
 class DotChord(ChordItem):
@@ -571,7 +574,7 @@ def update_state(self, state):
 
 
 class TempoSeparator(Delimiter):
-    rx = r"[-~](?=\s*\d)"
+    rx = r"[-~](?=\s*[0-9])"
 
 
 class Partial(Command):
@@ -740,7 +743,7 @@ class FigureBracket(Figure):
 
 class FigureStep(Figure):
     """A step figure number or the underscore."""
-    rx = r"_|\d+"
+    rx = r"_|[0-9]+"
 
 
 class FigureAccidental(Figure):

ly/lex/scheme.py

@@ -138,20 +138,21 @@ def test_match(cls, match):
         return match.group() in data.scheme_constants()
 
 
+# [0-9], not \d, which also matches Unicode digits that are not legal LilyPond
 class Number(_token.Item, _token.Numeric):
     rx = (r"("
-          r"-?\d+|"
+          r"-?[0-9]+|"
           r"#(b[0-1]+|o[0-7]+|x[0-9a-fA-F]+)|"
           r"[-+]inf.0|[-+]?nan.0"
           r")(?=$|[)\s])")
 
 
 class Fraction(Number):
-    rx = r"-?\d+/\d+(?=$|[)\s])"
+    rx = r"-?[0-9]+/[0-9]+(?=$|[)\s])"
 
 
 class Float(Number):
-    rx = r"-?((\d+(\.\d*)|\.\d+)(E\d+)?)(?=$|[)\s])"
+    rx = r"-?(([0-9]+(\.[0-9]*)|\.[0-9]+)(E[0-9]+)?)(?=$|[)\s])"
 
 
 class VectorStart(OpenParen):

tests/test_lex.py

@@ -0,0 +1,61 @@
+"""Tests for the lexers."""
+import ly.document
+import ly.lex._token
+import ly.lex.html
+import ly.lex.lilypond
+
+
+DIGIT_TOKENS = (ly.lex._token.Numeric,
+                ly.lex.lilypond.Fingering,
+                ly.lex.lilypond.StringNumber,
+                ly.lex.lilypond.ChordStepNumber,
+                ly.lex.lilypond.FigureStep,
+                ly.lex.lilypond.TempoSeparator,
+                ly.lex.lilypond.Scaling)
+
+TO_FULLWIDTH = str.maketrans('0123456789', '０１２３４５６７８９')
+
+SOURCES = [
+    '#(display 42)',
+    '#(x 1/2)',
+    '#(x 1.5)',
+    '\\tempo 4 = 60 - 80',
+    '\\time 3/4',
+    'c4',
+    'c-4',
+    '\\override NoteHead.font-size = 2',
+    '\\chords { c1:5 }',
+    '\\figures { <1> }',
+    '\\score { { c1*2 } }',
+    '\\score { \\new TabStaff { c4\\4 } }',
+]
+
+
+def digit_tokens(text, mode=None):
+    """Return the text of every token the lexer only reads as such because of
+    the digits in it."""
+    doc = ly.document.Document(text, mode)
+    return [str(t) for block in doc
+            for t in doc.tokens(block) if isinstance(t, DIGIT_TOKENS)]
+
+
+def test_ascii_digits_are_read():
+    for source in SOURCES:
+        assert digit_tokens(source), source
+
+
+def test_fullwidth_digits_are_not_read():
+    for source in SOURCES:
+        fullwidth = source.translate(TO_FULLWIDTH)
+        assert digit_tokens(fullwidth) == [], fullwidth
+
+
+def entity_refs(text):
+    doc = ly.document.Document(text, 'html')
+    return [str(t) for block in doc
+            for t in doc.tokens(block) if isinstance(t, ly.lex.html.EntityRef)]
+
+
+def test_only_ascii_digits_form_a_numeric_entity():
+    assert entity_refs('&#65;') == ['&#65;']
+    assert entity_refs('&#65;'.translate(TO_FULLWIDTH)) == []