From a7d333c9a3f8be717eb31b98615750e4f229d115 Mon Sep 17 00:00:00 2001 From: Arsym <123092893+arsym-dev@users.noreply.github.com> Date: Sat, 30 May 2026 10:01:41 -0400 Subject: [PATCH 1/3] Generate tokens on build --- scripts/generate.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/generate.py b/scripts/generate.py index e2c19a4..2c48595 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -67,6 +67,7 @@ def convert_file(filename: pathlib.Path): pass destination.write_text(output) + return True def main(): @@ -75,6 +76,11 @@ def main(): generated = False + # Force pattern generation on fresh clones where build artifacts are absent. + generated_dir = ROOT / "src" / "tokenizer" / "generated" + if not generated_dir.exists() or not any(generated_dir.iterdir()): + generated = True + for filename in ROOT.glob("syntaxes/*.tmLanguage.yaml"): generated = generated or convert_file(filename) From c2cf9aa12a0ea2f0d2a383848104f07ec61e293b Mon Sep 17 00:00:00 2001 From: Arsym <123092893+arsym-dev@users.noreply.github.com> Date: Sat, 30 May 2026 11:12:26 -0400 Subject: [PATCH 2/3] Fix build errors --- scripts/generate.py | 8 ++------ scripts/syntax_to_token_pattern.py | 7 ++++++- src/diagnostics.ts | 2 +- src/tokenizer/renpy-tokens.ts | 1 + src/tokenizer/token-definitions.ts | 1 + 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/generate.py b/scripts/generate.py index 2c48595..5883dff 100644 --- a/scripts/generate.py +++ b/scripts/generate.py @@ -76,13 +76,9 @@ def main(): generated = False - # Force pattern generation on fresh clones where build artifacts are absent. - generated_dir = ROOT / "src" / "tokenizer" / "generated" - if not generated_dir.exists() or not any(generated_dir.iterdir()): - generated = True - for filename in ROOT.glob("syntaxes/*.tmLanguage.yaml"): - generated = generated or convert_file(filename) + file_changed = convert_file(filename) + generated = generated or file_changed print("Generated .tmLanguage.json files from .tmLanguage.yaml files.") diff --git a/scripts/syntax_to_token_pattern.py b/scripts/syntax_to_token_pattern.py index f797ccd..c6e4835 100644 --- a/scripts/syntax_to_token_pattern.py +++ b/scripts/syntax_to_token_pattern.py @@ -489,7 +489,9 @@ def transform_pattern(state: GeneratorState, indent: int, value: dict[str, Any], language = source.split(".")[-1] language_accessor = titleCase(language) + "Patterns." - if reference == None: + if language == "python" and reference == "python-block-tester": + include = "RenpyPatterns.pythonBlockTester" + elif reference == None: include = language_accessor + language else: include = language_accessor + camelCase(reference) @@ -604,6 +606,9 @@ def generate_token_patterns(): style_state = GeneratorState() python_state = GeneratorState() + generated_dir = ROOT / "src" / "tokenizer" / "generated" + generated_dir.mkdir(parents=True, exist_ok=True) + generate_file(renpy_state, "./syntaxes/renpy.tmLanguage.json", "src/tokenizer/generated/renpy-token-patterns.g.ts") generate_file(atl_state, "./syntaxes/renpy.atl.tmLanguage.json", "src/tokenizer/generated/atl-token-patterns.g.ts") generate_file(screen_state, "./syntaxes/renpy.screen.tmLanguage.json", "src/tokenizer/generated/screen-token-patterns.g.ts") diff --git a/src/diagnostics.ts b/src/diagnostics.ts index c1b89a4..3b61f02 100644 --- a/src/diagnostics.ts +++ b/src/diagnostics.ts @@ -119,7 +119,7 @@ function refreshDiagnostics(doc: TextDocument, diagnosticCollection: DiagnosticC //Filenames must begin with a letter or number, //and may not begin with "00", as Ren'Py uses such files for its own purposes. - const ignoreThisFilename: string = !!doc.lineAt(0).text.match(rxIgnoreThisFilename); + const ignoreThisFilename: boolean = !!doc.lineAt(0).text.match(rxIgnoreThisFilename); const checkFilenames: string = config.warnOnInvalidFilenameIssues; if (checkFilenames.toLowerCase() !== "disabled" && !ignoreThisFilename) { let severity = DiagnosticSeverity.Error; diff --git a/src/tokenizer/renpy-tokens.ts b/src/tokenizer/renpy-tokens.ts index ebec302..c144452 100644 --- a/src/tokenizer/renpy-tokens.ts +++ b/src/tokenizer/renpy-tokens.ts @@ -119,6 +119,7 @@ export const enum KeywordTokenType { Circles, Clockwise, Counterclockwise, + Knot, Event, On, Function, diff --git a/src/tokenizer/token-definitions.ts b/src/tokenizer/token-definitions.ts index f657ae3..1fa6f46 100644 --- a/src/tokenizer/token-definitions.ts +++ b/src/tokenizer/token-definitions.ts @@ -545,6 +545,7 @@ const tokenTypeDefinitions: EnumToString = { Circles: { name: "Circles", value: KeywordTokenType.Circles }, Clockwise: { name: "Clockwise", value: KeywordTokenType.Clockwise }, Counterclockwise: { name: "Counterclockwise", value: KeywordTokenType.Counterclockwise }, + Knot: { name: "Knot", value: KeywordTokenType.Knot }, Event: { name: "Event", value: KeywordTokenType.Event }, On: { name: "On", value: KeywordTokenType.On }, Function: { name: "Function", value: KeywordTokenType.Function }, From b05e79815c1355010fe526888f5062e8039ae16f Mon Sep 17 00:00:00 2001 From: Arsym <123092893+arsym-dev@users.noreply.github.com> Date: Sat, 30 May 2026 11:19:47 -0400 Subject: [PATCH 3/3] Test token patterns --- scripts/syntax_to_token_pattern.py | 20 +++++++++++++++++++- src/tokenizer/renpy-tokens.ts | 16 ++++++++++++++++ src/tokenizer/token-definitions.ts | 11 +++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/scripts/syntax_to_token_pattern.py b/scripts/syntax_to_token_pattern.py index c6e4835..377d42c 100644 --- a/scripts/syntax_to_token_pattern.py +++ b/scripts/syntax_to_token_pattern.py @@ -247,6 +247,9 @@ def get_parts(range: slice) -> str: else: return "EntityTokenType.FunctionName" + elif get_part(1) == "constant" and get_part(2) == "property-name": + return "EntityTokenType.PropertyName" + elif get_part(1) == "other": if get_part(2) == "match": if get_part(3) == "any": @@ -304,6 +307,9 @@ def get_parts(range: slice) -> str: else: return "KeywordTokenType." + titleCase(get_part(3)) + elif get_part(2) == "test": + return "KeywordTokenType.Test" + elif get_part(2) == "import": return "KeywordTokenType.Import" @@ -604,6 +610,7 @@ def generate_token_patterns(): atl_state = GeneratorState() screen_state = GeneratorState() style_state = GeneratorState() + test_state = GeneratorState() python_state = GeneratorState() generated_dir = ROOT / "src" / "tokenizer" / "generated" @@ -613,6 +620,7 @@ def generate_token_patterns(): generate_file(atl_state, "./syntaxes/renpy.atl.tmLanguage.json", "src/tokenizer/generated/atl-token-patterns.g.ts") generate_file(screen_state, "./syntaxes/renpy.screen.tmLanguage.json", "src/tokenizer/generated/screen-token-patterns.g.ts") generate_file(style_state, "./syntaxes/renpy.style.tmLanguage.json", "src/tokenizer/generated/style-token-patterns.g.ts") + generate_file(test_state, "./syntaxes/renpy.test.tmLanguage.json", "src/tokenizer/generated/test-token-patterns.g.ts") generate_file(python_state, "./syntaxes/renpy.python.tmLanguage.json", "src/tokenizer/generated/python-token-patterns.g.ts") # Write the typescript entries to a file @@ -625,7 +633,16 @@ def generate_token_patterns(): contents += "\n" # Add all source import from all states, but only the unique ones - source_imports = list(dict.fromkeys(renpy_state.source_imports + atl_state.source_imports + screen_state.source_imports + style_state.source_imports + python_state.source_imports)) + source_imports = list( + dict.fromkeys( + renpy_state.source_imports + + atl_state.source_imports + + screen_state.source_imports + + style_state.source_imports + + test_state.source_imports + + python_state.source_imports + ) + ) for source_import in source_imports: contents += f"import * as {titleCase(source_import)}Patterns from \"./{source_import}-token-patterns.g\";\n" @@ -643,6 +660,7 @@ def add_entries(entries: list[str], prefix: str) -> str: contents += add_entries(atl_state.external_pattern_include_entries, "AtlPatterns") contents += add_entries(screen_state.external_pattern_include_entries, "ScreenPatterns") contents += add_entries(style_state.external_pattern_include_entries, "StylePatterns") + contents += add_entries(test_state.external_pattern_include_entries, "TestPatterns") contents += add_entries(python_state.external_pattern_include_entries, "PythonPatterns") exports: list[str] = [] diff --git a/src/tokenizer/renpy-tokens.ts b/src/tokenizer/renpy-tokens.ts index c144452..6bb4aa3 100644 --- a/src/tokenizer/renpy-tokens.ts +++ b/src/tokenizer/renpy-tokens.ts @@ -124,6 +124,12 @@ export const enum KeywordTokenType { On, Function, + // Test keywords + Test, + OtherTest, + Testcase, + Testsuite, + // Python keywords None, True, @@ -170,6 +176,10 @@ export const enum EntityTokenType { // ATL entities EventName, PropertyName, + + // Test entities + TestcaseName, + TestsuiteName, } export const enum LiteralTokenType { @@ -429,6 +439,12 @@ export const enum MetaTokenType { TransformStatement, TransformParameters, + TestBlock, + TestcaseStatement, + TestcaseParameters, + TestsuiteStatement, + TestsuiteParameters, + MemberAccess, ItemAccess, IndexedName, diff --git a/src/tokenizer/token-definitions.ts b/src/tokenizer/token-definitions.ts index 1fa6f46..dee0677 100644 --- a/src/tokenizer/token-definitions.ts +++ b/src/tokenizer/token-definitions.ts @@ -540,6 +540,10 @@ const tokenTypeDefinitions: EnumToString = { Parallel: { name: "Parallel", value: KeywordTokenType.Parallel }, Block: { name: "Block", value: KeywordTokenType.Block }, Choice: { name: "Choice", value: KeywordTokenType.Choice }, + Test: { name: "Test", value: KeywordTokenType.Test }, + OtherTest: { name: "OtherTest", value: KeywordTokenType.OtherTest }, + Testcase: { name: "Testcase", value: KeywordTokenType.Testcase }, + Testsuite: { name: "Testsuite", value: KeywordTokenType.Testsuite }, Warp: { name: "Warp", value: KeywordTokenType.Warp }, Circles: { name: "Circles", value: KeywordTokenType.Circles }, @@ -578,6 +582,8 @@ const tokenTypeDefinitions: EnumToString = { FunctionName: { name: "FunctionName", value: EntityTokenType.FunctionName }, TagName: { name: "TagName", value: EntityTokenType.TagName }, Identifier: { name: "Identifier", value: EntityTokenType.Identifier }, + TestcaseName: { name: "TestcaseName", value: EntityTokenType.TestcaseName }, + TestsuiteName: { name: "TestsuiteName", value: EntityTokenType.TestsuiteName }, StyleName: { name: "StyleName", value: EntityTokenType.StyleName }, TransformName: { name: "TransformName", value: EntityTokenType.TransformName }, @@ -845,6 +851,11 @@ const tokenTypeDefinitions: EnumToString = { FunctionDecorator: { name: "FunctionDecorator", value: MetaTokenType.FunctionDecorator }, FunctionCall: { name: "FunctionCall", value: MetaTokenType.FunctionCall }, FunctionCallGeneric: { name: "FunctionCallGeneric", value: MetaTokenType.FunctionCallGeneric }, + TestBlock: { name: "TestBlock", value: MetaTokenType.TestBlock }, + TestcaseStatement: { name: "TestcaseStatement", value: MetaTokenType.TestcaseStatement }, + TestcaseParameters: { name: "TestcaseParameters", value: MetaTokenType.TestcaseParameters }, + TestsuiteStatement: { name: "TestsuiteStatement", value: MetaTokenType.TestsuiteStatement }, + TestsuiteParameters: { name: "TestsuiteParameters", value: MetaTokenType.TestsuiteParameters }, Fstring: { name: "Fstring", value: MetaTokenType.Fstring }, InterpolateFlags: { name: "InterpolateFlags", value: MetaTokenType.InterpolateFlags },