diff --git a/src/commonMain/kotlin/com/notkamui/keval/AbstractSyntaxTree.kt b/src/commonMain/kotlin/com/notkamui/keval/AbstractSyntaxTree.kt index 48c0cf0..efefa72 100644 --- a/src/commonMain/kotlin/com/notkamui/keval/AbstractSyntaxTree.kt +++ b/src/commonMain/kotlin/com/notkamui/keval/AbstractSyntaxTree.kt @@ -64,7 +64,7 @@ internal interface Node { } /** - * An binary operator node + * A binary operator node * * @property left is its left child * @property op is the actual operator diff --git a/src/commonMain/kotlin/com/notkamui/keval/Tokenizer.kt b/src/commonMain/kotlin/com/notkamui/keval/Tokenizer.kt index b76a11f..ffdfa4e 100644 --- a/src/commonMain/kotlin/com/notkamui/keval/Tokenizer.kt +++ b/src/commonMain/kotlin/com/notkamui/keval/Tokenizer.kt @@ -47,7 +47,10 @@ private fun Sequence.normalizeTokens(symbols: Map ret.add(token) } else { throw KevalInvalidSymbolException( - token, joinToString(""), currentPos, "comma can only be used in the context of a function" + token, + joinToString(""), + currentPos, + "comma can only be used in the context of a function" ) } } @@ -85,13 +88,20 @@ internal fun String.isKevalOperator(symbolsSet: Set): Boolean = this in * @return the list of tokens * @throws KevalInvalidSymbolException if the expression contains an invalid symbol */ -internal fun String.tokenize(symbolsSet: Map): List = - TOKENIZER_REGEX.findAll(this) +internal fun String.tokenize(symbolsSet: Map): List { + val constantsSet = symbolsSet.filter { it.value is KevalConstant } + + val tokenizerRegex = if (constantsSet.isNotEmpty()) { + val constantsRegexString = constantsSet.keys.joinToString(separator = "|") + """(\d+\.\d+|\d+|$constantsRegexString|[a-zA-Z_]\w*|[^\w\s])""" + } else { + """(\d+\.\d+|\d+|[a-zA-Z_]\w*|[^\w\s])""" + }.toRegex() + + return tokenizerRegex.findAll(this) .map(MatchResult::value) .filter(String::isNotBlank) .map { SANITIZE_REGEX.replace(it, "") } .normalizeTokens(symbolsSet) - -private val SANITIZE_REGEX = """\s+""".toRegex() - -private val TOKENIZER_REGEX = """(\d+\.\d+|\d+|[a-zA-Z_]\w*|[^\w\s])""".toRegex() +} +private val SANITIZE_REGEX = """\s+""".toRegex() \ No newline at end of file diff --git a/src/commonTest/kotlin/com/notkamui/keval/TokenizerTest.kt b/src/commonTest/kotlin/com/notkamui/keval/TokenizerTest.kt index 0901b5d..3d83c37 100644 --- a/src/commonTest/kotlin/com/notkamui/keval/TokenizerTest.kt +++ b/src/commonTest/kotlin/com/notkamui/keval/TokenizerTest.kt @@ -103,4 +103,24 @@ class TokenizerTest { val nodes = "f(((1)))".tokenize(k.resourcesView()) assertEquals("f(((1)))", nodes.joinToString(separator = "")) } + + @Test + fun symbolsAndNumbers() { + listOf( + listOf("PI","9"), + listOf("9", "e"), + listOf("PI", "e"), + listOf("PI", "PI"), + listOf("PI", "e", "e", "PI") + ).forEach { tokenList -> + // Intersperses the assumed "*" + val expected = tokenList.dropLast(1).flatMap { listOf(it, "*") } + tokenList.last() + assertEquals( + expected, + tokenList + .joinToString(separator = "") + .tokenize(KevalBuilder.DEFAULT_RESOURCES) + ) + } + } }