diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db3966..bad0ae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Add optional Prism, CodeMirror, and highlight.js adapters, with the website using the shared CodeMirror stream parser. +- Add VS Code and Sublime Text syntax packages for Knap Markdown, with shared generated grammars, template comments, and snippets. + ## 0.4.2 - Add `{# ... #}` template comments, including multiline comments, with syntax errors for unclosed comments. diff --git a/README.md b/README.md index ce7f37c..0438684 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,10 @@ Filters that deliberately preserve their input after invalid runtime data can re ## Syntax +VS Code and Sublime Text packages are available in [editors/](editors/README.md). They highlight Knap inside Markdown and include template snippets. Use `.knap` for Knap templates. The packages also recognize `.knap.md`, or you can select **Knap Markdown** for an existing template. + +JavaScript applications can use the optional [Prism, CodeMirror, and highlight.js adapters](editors/web.md) through `knap/prism`, `knap/codemirror`, and `knap/highlightjs`. + The syntax is inspired by Twig and Liquid. ```liquid diff --git a/editors/README.md b/editors/README.md new file mode 100644 index 0000000..ee63af0 --- /dev/null +++ b/editors/README.md @@ -0,0 +1,60 @@ +# Editor support + +For JavaScript applications, see the [Prism, CodeMirror, and highlight.js integration guide](web.md). + +Knap Markdown highlights Knap expressions alongside the editor's built-in Markdown syntax. Use `.knap` for Knap templates. The packages also recognize the legacy-compatible `.knap.md` suffix, and existing `.md` templates can use the syntax by selecting **Knap Markdown** manually. Installing the package does not change the language of ordinary Markdown files. + +## VS Code and compatible editors + +For local development, launch the editor with this repository's extension folder: + +```sh +code --extensionDevelopmentPath="$PWD/editors/vscode" "$PWD/editors/examples/article.knap" +``` + +Use `cursor` instead of `code` for Cursor. The extension contains only declarative grammars, language configuration, and snippets; it does not run the template engine. + +To associate an existing template folder, add this to workspace settings: + +```json +{ + "files.associations": { + "templates/*.md": "knap" + } +} +``` + +To build an installable VSIX, run `pnpm editors:build` from the repository root, then run `pnpm dlx @vscode/vsce package` from `editors/vscode`. Install the resulting file with **Extensions: Install from VSIX**. Marketplace publication is a separate maintainer action; the manifest's `obsidianmd` publisher must be verified before publishing. + +## Sublime Text 4 + +Choose **Preferences → Browse Packages**, create a `Knap` folder there, and copy the contents of `editors/sublime` into it. Open the example template or select **View → Syntax → Knap Markdown** for an existing template. + +Sublime inherits its built-in Markdown grammar and uses a separate generated Knap expression grammar. Dedicated Markdown, YAML, HTML, and JavaScript adapters highlight templates inside frontmatter, HTML attributes, inline code, and JavaScript fences. Inheritance avoids recursive `with_prototype` expansion across embedded languages. Host scopes are removed temporarily inside Knap tags and restored afterward, so an expression in a quoted YAML value is not colored as YAML string content. A native grammar is necessary because Sublime cannot include TextMate grammars inside native syntax definitions. + +## Included features + +- Output expressions, logic tags, whitespace trimming, and multiline comments. +- Strings and escapes, numbers, constants, variables, property access, filters, operators, and nested map expressions. +- Markdown highlighting, including Knap inside frontmatter, HTML, and code spans. +- Template comment toggling and snippets starting with `knap-` in both editors. +- Template delimiter pairing and selection wrapping in VS Code. + +The syntax highlighter follows the website's convention of styling bare filter arguments as strings, with expression highlighting for `map`. Those arguments can still resolve to data at runtime; color does not determine their meaning. Validation, data-aware completions, rendering, and Markdown preview integration are outside this initial package. The `knap` language mode does not automatically inherit features contributed specifically for VS Code's `markdown` language ID. + +## Maintaining the grammars + +```sh +pnpm editors:build +pnpm check +``` + +`editors/build.mjs` is the source for the TextMate and core Sublime grammars and both sets of snippets. `Knap Expressions.sublime-syntax` contains the generated expression contexts; `Knap.sublime-syntax` selects the Markdown host. The Markdown, YAML, HTML, and JavaScript host adapters are hand-maintained. Generated files are checked in so editor packages can be used without installing Node.js. `pnpm editors:check`, included in `pnpm check`, rejects stale generated files. + +`src/language-syntax.json` shares keyword and constant definitions with the website's existing highlighters. Their colors, Markdown punctuation handling, CodeMirror integration, and HTML rendering remain in the website. The tests port its Knap fixtures and exercise VS Code's actual TextMate/Oniguruma engine with standard Markdown, YAML, HTML, and JavaScript grammars, including HTML's derivative grammar. They also check VS Code's encoded string/comment token types and verify that host highlighting resumes after tags and fences close. TextMate retains parent scopes; `meta.embedded` resets the editor token type inside a template. + +Sublime context references and YAML structure are checked separately; use Sublime's **Tools → Build System → Syntax Tests** on each of `sublime/syntax_test_knap.knap` and `sublime/syntax_test_knap_yaml.knap` after installation for native positive and negative scope assertions. These native checks require Sublime; the Node test suite does not execute its syntax engine. + +The host integration was compared against [Shopify Liquid](https://github.com/Shopify/liquid-tm-grammar), [Twig for VS Code](https://github.com/mblode/vscode-twig-language-2), and [BetterJinja](https://github.com/Sublime-Instincts/BetterJinja). The YAML adapter incorporates BetterJinja's scalar and mapping-key integration techniques; its MIT notice is included in `sublime/ThirdPartyNotices.txt`. + +Use **Developer: Inspect Editor Tokens and Scopes** in VS Code or **Show Scope Name** in Sublime to inspect theme behavior in `examples/article.knap`. diff --git a/editors/build.mjs b/editors/build.mjs new file mode 100644 index 0000000..d3461c6 --- /dev/null +++ b/editors/build.mjs @@ -0,0 +1,185 @@ +// The website's token categories and filter-argument conventions are the basis +// for these grammars. Keep editor-specific state machines out of the runtime. +import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname } from 'node:path'; +import { stringify } from 'yaml'; + +const root = new URL('../', import.meta.url); +const syntax = JSON.parse(readFileSync(new URL('src/language-syntax.json', root), 'utf8')); +const check = process.argv.includes('--check'); +function output(path, contents) { + const file = fileURLToPath(new URL(path, root)); + if (check) { + if (!existsSync(file) || readFileSync(file, 'utf8') !== contents) throw new Error(`${path} is stale. Run pnpm editors:build.`); + } else { + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, contents); + } +} +const json = (path, value) => output(path, JSON.stringify(value, null, 2) + '\n'); +const include = name => ({ include: `#${name}` }); +const match = (match, name) => ({ match, name }); +const capture = name => ({ 0: { name } }); +const identifier = '[A-Za-z_$][\\w$]*'; +const words = values => `\\b(?:${values.join('|')})\\b`; +const boundary = '(?=-?\\}\\}|-?%\\}|\\{[{%])'; +const pipe = '\\|(?!\\|)'; +const filterEnd = `(?=${pipe}|-?\\}\\}|-?%\\}|\\{[{%]|\\))`; +const repository = { + tags: { patterns: [include('comment'), include('output'), include('logic')] }, + comment: { + name: 'comment.block.knap', begin: '\\{#', end: '#\\}', + beginCaptures: capture('punctuation.definition.comment.begin.knap'), + endCaptures: capture('punctuation.definition.comment.end.knap'), + }, + output: { + name: 'meta.embedded.inline.knap', begin: '\\{\\{-?', end: '-?\\}\\}|(?=\\{[{%])', + beginCaptures: capture('punctuation.section.embedded.begin.knap'), + endCaptures: capture('punctuation.section.embedded.end.knap'), + patterns: [include('expression')], + }, + logic: { + name: 'meta.embedded.block.knap', begin: '\\{%-?', end: '-?%\\}|(?=\\{[{%])', + beginCaptures: capture('punctuation.section.embedded.begin.knap'), + endCaptures: capture('punctuation.section.embedded.end.knap'), + patterns: [include('expression')], + }, + expression: { patterns: [ + include('strings'), + // A double pipe is boolean OR, never the start of a filter. + match('\\|\\|', 'keyword.operator.knap'), + include('map-filter'), include('filter'), + include('group'), include('array'), include('object'), + include('atoms'), match(identifier, 'variable.other.knap'), + ] }, + strings: { patterns: ['"', "'"].map(quote => ({ + name: `string.quoted.${quote === '"' ? 'double' : 'single'}.knap`, + begin: quote, end: quote, + beginCaptures: capture('punctuation.definition.string.begin.knap'), + endCaptures: capture('punctuation.definition.string.end.knap'), + patterns: [match('\\\\.', 'constant.character.escape.knap')], + })) }, + 'map-filter': { + begin: `(${pipe})\\s*(map)\\b`, end: filterEnd, + beginCaptures: { 1: { name: 'keyword.operator.pipe.knap' }, 2: { name: 'support.function.filter.knap' } }, + patterns: [include('expression')], + }, + filter: { + begin: `(${pipe})\\s*(${identifier})?`, end: filterEnd, + beginCaptures: { 1: { name: 'keyword.operator.pipe.knap' }, 2: { name: 'support.function.filter.knap' } }, + patterns: [ + // Support a filter name on the next line, including unfinished input. + { begin: ':', beginCaptures: capture('punctuation.separator.knap'), end: filterEnd, patterns: [include('arguments')] }, + match(identifier, 'support.function.filter.knap'), + ], + }, + arguments: { patterns: [ + include('strings'), include('argument-group'), + match(`(\\.)\\s*(${identifier})`, 'string.unquoted.argument.knap'), + include('atoms'), + // Like the website, style bare argument fallbacks as strings. They may + // resolve to data at runtime; highlighting does not validate their value. + match(identifier, 'string.unquoted.argument.knap'), + ] }, + atoms: { patterns: [ + match(`(\\.)\\s*(${identifier})`, 'variable.other.property.knap'), + match(words(syntax.tags), 'keyword.control.knap'), + match(words(syntax.wordOperators), 'keyword.operator.word.knap'), + match(words(syntax.constants), 'constant.language.knap'), + match('(?|==|!=|>=|<=|&&|\\|\\||\\?\\?|[=<>!+*/-]', 'keyword.operator.knap'), + match('\\\\.', 'constant.character.escape.knap'), + match('[.,:]', 'punctuation.separator.knap'), + ] }, +}; +for (const [name, begin, end, patterns] of [ + ['group', '\\(', '\\)', 'expression'], + ['array', '\\[', '\\]', 'expression'], + ['object', '\\{(?![{%#])', '\\}', 'expression'], + ['argument-group', '\\(', '\\)', 'arguments'], +]) repository[name] = { + begin, end: `${end}|${boundary}`, + beginCaptures: capture('punctuation.section.group.begin.knap'), + endCaptures: capture('punctuation.section.group.end.knap'), + patterns: [include(patterns)], +}; + +const core = { name: 'Knap', scopeName: 'source.knap', patterns: [include('tags')], repository }; +json('editors/vscode/syntaxes/knap.tmLanguage.json', core); +json('editors/vscode/syntaxes/knap-markdown.tmLanguage.json', { + name: 'Knap Markdown', scopeName: 'text.html.markdown.knap', + patterns: [{ include: 'source.knap' }, { include: 'text.html.markdown' }], + injections: { + 'L:text.html.markdown.knap - meta.embedded.inline.knap - meta.embedded.block.knap - comment.block.knap': { + patterns: [{ include: 'source.knap' }], + }, + // Markdown normally consumes an entire inline-code span in one match, + // leaving no opportunity to inject template tokens inside it. + 'L:text.html.markdown.knap - meta.embedded - comment - string - markup.inline.raw - markup.fenced_code - meta.tag': { + patterns: [ + // Give the host's fenced-code rules precedence over inline backticks. + { include: 'text.html.markdown#fenced_code_block' }, + { + name: 'markup.inline.raw.string.markdown', begin: '(`+)', end: '(? value && new Map(Object.entries(value).map(([key, value]) => [Number(key), value.name])); +function sublimePatterns(patterns, prefix) { + return patterns.map((rule, index) => { + if (rule.include) return { include: rule.include.slice(1) }; + if (rule.match) return { match: rule.match, scope: rule.name }; + const name = `${prefix}-${index}`; + // Unlike TextMate's meta.embedded token-type reset, Sublime can remove + // host scopes while a tag is active. Restore them automatically on pop. + const embedded = ['comment', 'output', 'logic'].includes(prefix); + contexts[name] = [ + { meta_include_prototype: false }, + ...(embedded ? [{ clear_scopes: true }] : []), + ...(rule.name ? [{ meta_scope: `${embedded ? 'source.knap ' : ''}${rule.name}` }] : []), + { match: rule.end, ...(rule.endCaptures ? { captures: captures(rule.endCaptures) } : {}), pop: true }, + ...sublimePatterns(rule.patterns ?? [], name), + ]; + return { match: rule.begin, ...(rule.beginCaptures ? { captures: captures(rule.beginCaptures) } : {}), push: name }; + }); +} +for (const [name, rule] of Object.entries(repository)) { + contexts[name] = sublimePatterns(rule.begin ? [rule] : rule.patterns, name); +} +output('editors/sublime/Knap Expressions.sublime-syntax', '%YAML 1.2\n---\n# Generated by editors/build.mjs. Do not edit directly.\n' + stringify({ + name: 'Knap expressions', scope: 'source.knap', version: 2, hidden: true, + contexts: { + main: [{ include: 'tags' }], + ...contexts, + }, +}, { lineWidth: 0 })); +output('editors/sublime/Knap.sublime-syntax', '%YAML 1.2\n---\n# Generated by editors/build.mjs. Do not edit directly.\n' + stringify({ + name: 'Knap Markdown', scope: 'text.html.markdown.knap', version: 2, + file_extensions: ['knap', 'knap.md'], + extends: 'Packages/Knap/Knap Markdown.sublime-syntax', +}, { lineWidth: 0 })); + +// Snippet bodies are shared; each editor gets its own package format. +const snippets = { + 'Output a variable': { prefix: 'knap-var', body: '{{ ${1:title} }}', description: 'Output a value' }, + 'Apply a filter': { prefix: 'knap-filter', body: '{{ ${1:title} | ${2:upper} }}', description: 'Output a filtered value' }, + 'Conditional block': { prefix: 'knap-if', body: '{% if ${1:condition} %}\n$0\n{% endif %}', description: 'Conditional block' }, + 'Conditional with fallback': { prefix: 'knap-ifelse', body: '{% if ${1:condition} %}\n\t${2}\n{% else %}\n\t$0\n{% endif %}', description: 'Conditional block with fallback' }, + 'Loop': { prefix: 'knap-for', body: '{% for ${1:item} in ${2:items} %}\n- {{ ${1:item} }}$0\n{% endfor %}', description: 'Loop over a collection' }, + 'Set a variable': { prefix: 'knap-set', body: '{% set ${1:name} = ${2:value} %}$0', description: 'Assign a local variable' }, + 'Template comment': { prefix: 'knap-comment', body: '{# ${1:comment} #}$0', description: 'Comment removed from rendered output' }, +}; +json('editors/vscode/snippets/knap.json', snippets); +for (const snippet of Object.values(snippets)) { + output(`editors/sublime/${snippet.prefix}.sublime-snippet`, `\n \n ${snippet.prefix}\n (text.html.markdown.knap | source.knap) - comment - string\n ${snippet.description}\n\n`); +} diff --git a/editors/examples/article.knap b/editors/examples/article.knap new file mode 100644 index 0000000..30f4967 --- /dev/null +++ b/editors/examples/article.knap @@ -0,0 +1,42 @@ +--- +title: "{{ title | trim }}" +source: "{{ url }}" +{{ tags | yaml_property:"tags" }} +--- + +# {{ title | trim }} + +{# This comment spans multiple lines. + {{ ignored }} and {% if ignored %} are not evaluated here. +#} + +{% if author %} +By **{{ author.name }}** +{% elseif site %} +From [{{ site }}]({{ url }}) +{% else %} +Unknown source +{% endif %} + +{% set heading = title | upper %} +{{ published | date:"YYYY-MM-DD" }} +{{ First name | trim }} +{{ summary ?? "No summary" }} + +{% for tag in tags %} +- #{{ tag | kebab }} +{% endfor %} + +{{ text | highlight:blue }} +{{ people | sort:(details.rank, desc) }} +{{ people | map:item => ({name: item.name}) | first }} + +Inline code: `{{ title }}` + +```text +{{ content }} +``` + +{{ title }} + +{{- "Literal delimiters: {# comment #} and }}" -}} diff --git a/editors/sublime/Comments.tmPreferences b/editors/sublime/Comments.tmPreferences new file mode 100644 index 0000000..4a3ff24 --- /dev/null +++ b/editors/sublime/Comments.tmPreferences @@ -0,0 +1,10 @@ + + + + nameKnap comments + scopetext.html.markdown.knap, source.knap + settingsshellVariables + nameTM_COMMENT_STARTvalue{# + nameTM_COMMENT_ENDvalue #} + + diff --git a/editors/sublime/Knap Expressions.sublime-syntax b/editors/sublime/Knap Expressions.sublime-syntax new file mode 100644 index 0000000..01542b1 --- /dev/null +++ b/editors/sublime/Knap Expressions.sublime-syntax @@ -0,0 +1,199 @@ +%YAML 1.2 +--- +# Generated by editors/build.mjs. Do not edit directly. +name: Knap expressions +scope: source.knap +version: 2 +hidden: true +contexts: + main: + - include: tags + tags: + - include: comment + - include: output + - include: logic + comment-0: + - meta_include_prototype: false + - clear_scopes: true + - meta_scope: source.knap comment.block.knap + - match: "#\\}" + captures: + 0: punctuation.definition.comment.end.knap + pop: true + comment: + - match: \{# + captures: + 0: punctuation.definition.comment.begin.knap + push: comment-0 + output-0: + - meta_include_prototype: false + - clear_scopes: true + - meta_scope: source.knap meta.embedded.inline.knap + - match: -?\}\}|(?=\{[{%]) + captures: + 0: punctuation.section.embedded.end.knap + pop: true + - include: expression + output: + - match: \{\{-? + captures: + 0: punctuation.section.embedded.begin.knap + push: output-0 + logic-0: + - meta_include_prototype: false + - clear_scopes: true + - meta_scope: source.knap meta.embedded.block.knap + - match: -?%\}|(?=\{[{%]) + captures: + 0: punctuation.section.embedded.end.knap + pop: true + - include: expression + logic: + - match: \{%-? + captures: + 0: punctuation.section.embedded.begin.knap + push: logic-0 + expression: + - include: strings + - match: \|\| + scope: keyword.operator.knap + - include: map-filter + - include: filter + - include: group + - include: array + - include: object + - include: atoms + - match: "[A-Za-z_$][\\w$]*" + scope: variable.other.knap + strings-0: + - meta_include_prototype: false + - meta_scope: string.quoted.double.knap + - match: '"' + captures: + 0: punctuation.definition.string.end.knap + pop: true + - match: \\. + scope: constant.character.escape.knap + strings-1: + - meta_include_prototype: false + - meta_scope: string.quoted.single.knap + - match: "'" + captures: + 0: punctuation.definition.string.end.knap + pop: true + - match: \\. + scope: constant.character.escape.knap + strings: + - match: '"' + captures: + 0: punctuation.definition.string.begin.knap + push: strings-0 + - match: "'" + captures: + 0: punctuation.definition.string.begin.knap + push: strings-1 + map-filter-0: + - meta_include_prototype: false + - match: (?=\|(?!\|)|-?\}\}|-?%\}|\{[{%]|\)) + pop: true + - include: expression + map-filter: + - match: (\|(?!\|))\s*(map)\b + captures: + 1: keyword.operator.pipe.knap + 2: support.function.filter.knap + push: map-filter-0 + filter-0-0: + - meta_include_prototype: false + - match: (?=\|(?!\|)|-?\}\}|-?%\}|\{[{%]|\)) + pop: true + - include: arguments + filter-0: + - meta_include_prototype: false + - match: (?=\|(?!\|)|-?\}\}|-?%\}|\{[{%]|\)) + pop: true + - match: ":" + captures: + 0: punctuation.separator.knap + push: filter-0-0 + - match: "[A-Za-z_$][\\w$]*" + scope: support.function.filter.knap + filter: + - match: (\|(?!\|))\s*([A-Za-z_$][\w$]*)? + captures: + 1: keyword.operator.pipe.knap + 2: support.function.filter.knap + push: filter-0 + arguments: + - include: strings + - include: argument-group + - match: (\.)\s*([A-Za-z_$][\w$]*) + scope: string.unquoted.argument.knap + - include: atoms + - match: "[A-Za-z_$][\\w$]*" + scope: string.unquoted.argument.knap + atoms: + - match: (\.)\s*([A-Za-z_$][\w$]*) + scope: variable.other.property.knap + - match: \b(?:if|elseif|else|endif|for|in|endfor|set)\b + scope: keyword.control.knap + - match: \b(?:and|or|not|contains)\b + scope: keyword.operator.word.knap + - match: \b(?:true|false|null)\b + scope: constant.language.knap + - match: (?|==|!=|>=|<=|&&|\|\||\?\?|[=<>!+*/-] + scope: keyword.operator.knap + - match: \\. + scope: constant.character.escape.knap + - match: "[.,:]" + scope: punctuation.separator.knap + group-0: + - meta_include_prototype: false + - match: \)|(?=-?\}\}|-?%\}|\{[{%]) + captures: + 0: punctuation.section.group.end.knap + pop: true + - include: expression + group: + - match: \( + captures: + 0: punctuation.section.group.begin.knap + push: group-0 + array-0: + - meta_include_prototype: false + - match: \]|(?=-?\}\}|-?%\}|\{[{%]) + captures: + 0: punctuation.section.group.end.knap + pop: true + - include: expression + array: + - match: \[ + captures: + 0: punctuation.section.group.begin.knap + push: array-0 + object-0: + - meta_include_prototype: false + - match: \}|(?=-?\}\}|-?%\}|\{[{%]) + captures: + 0: punctuation.section.group.end.knap + pop: true + - include: expression + object: + - match: \{(?![{%#]) + captures: + 0: punctuation.section.group.begin.knap + push: object-0 + argument-group-0: + - meta_include_prototype: false + - match: \)|(?=-?\}\}|-?%\}|\{[{%]) + captures: + 0: punctuation.section.group.end.knap + pop: true + - include: arguments + argument-group: + - match: \( + captures: + 0: punctuation.section.group.begin.knap + push: argument-group-0 diff --git a/editors/sublime/Knap HTML.sublime-syntax b/editors/sublime/Knap HTML.sublime-syntax new file mode 100644 index 0000000..6dae79d --- /dev/null +++ b/editors/sublime/Knap HTML.sublime-syntax @@ -0,0 +1,24 @@ +%YAML 1.2 +--- +name: Knap HTML host +scope: text.html.knap +version: 2 +hidden: true +extends: Packages/HTML/HTML.sublime-syntax + +contexts: + prototype: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + tag-attribute-value-content: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + tag-href-attribute-value-content: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + tag-class-attribute-value-content: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + tag-id-attribute-value-content: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags diff --git a/editors/sublime/Knap JavaScript.sublime-syntax b/editors/sublime/Knap JavaScript.sublime-syntax new file mode 100644 index 0000000..cc89e4d --- /dev/null +++ b/editors/sublime/Knap JavaScript.sublime-syntax @@ -0,0 +1,15 @@ +%YAML 1.2 +--- +name: Knap JavaScript host +scope: source.js.knap +version: 2 +hidden: true +extends: Packages/JavaScript/JavaScript.sublime-syntax + +contexts: + prototype: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + string-content: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags diff --git a/editors/sublime/Knap Markdown.sublime-syntax b/editors/sublime/Knap Markdown.sublime-syntax new file mode 100644 index 0000000..085a861 --- /dev/null +++ b/editors/sublime/Knap Markdown.sublime-syntax @@ -0,0 +1,50 @@ +%YAML 1.2 +--- +# Host-specific integration stays separate from the generated Knap grammar. +name: Knap Markdown host +scope: text.html.markdown.knap.host +version: 2 +hidden: true +extends: Packages/Markdown/Markdown.sublime-syntax + +contexts: + prototype: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + + code-span-body: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + + html-content: + - include: scope:text.html.knap + + fenced-javascript: + - match: '(?x){{fenced_code_block_start}}(?i:\s*(javascript|js|node)){{fenced_code_block_trailing_infostring_characters}}' + captures: + 0: meta.code-fence.definition.begin.markdown-gfm + 2: punctuation.definition.raw.code-fence.begin.markdown + 5: constant.other.language-name.markdown + 6: comment.line.infostring.markdown + 7: meta.fold.code-fence.begin.markdown + embed: scope:source.js.knap + embed_scope: meta.code-fence.body.markdown-gfm markup.raw.code-fence.javascript.markdown-gfm source.js + escape: '{{fenced_code_block_escape}}' + escape_captures: + 0: meta.code-fence.definition.end.markdown-gfm + 1: punctuation.definition.raw.code-fence.end.markdown + 2: meta.fold.code-fence.end.markdown + + frontmatter: + - meta_prepend: true + - match: '^(---)[ \t]*(?i:(yaml|yml)[ \t]*)?$\n?' + captures: + 0: meta.frontmatter.markdown + 1: punctuation.section.frontmatter.begin.markdown + 2: constant.other.language-name.markdown + embed: scope:source.yaml.knap + embed_scope: meta.frontmatter.markdown source.yaml.embedded.markdown + escape: '^(---|\.{3})[ \t]*$\n?' + escape_captures: + 0: meta.frontmatter.markdown + 1: punctuation.section.frontmatter.end.markdown diff --git a/editors/sublime/Knap YAML.sublime-syntax b/editors/sublime/Knap YAML.sublime-syntax new file mode 100644 index 0000000..a393424 --- /dev/null +++ b/editors/sublime/Knap YAML.sublime-syntax @@ -0,0 +1,63 @@ +%YAML 1.2 +--- +# YAML integration adapted from BetterJinja (MIT); see ThirdPartyNotices.txt. +name: Knap YAML host +scope: source.yaml.knap +version: 2 +hidden: true +extends: Packages/YAML/YAML.sublime-syntax + +variables: + # A template at the start of a scalar is not a YAML flow mapping. Quoted + # colons and closing delimiters inside it must not end a mapping-key scan. + knap_interpolation: '\{\{(?:[^{}"'']|"(?:\\.|[^"\\])*"|''(?:\\.|[^''\\])*'')*\}\}' + yaml_plain_in: '(?:[^\s{{c_indicator}}]|[?:-][^\s{{c_flow_indicator}}])' + yaml_plain_out: '(?:[^\s{{c_indicator}}]|[?:-]\S)' + ns_plain_first_plain_in: '(?:{{yaml_plain_in}}|\{\{)' + ns_plain_first_plain_out: '(?:{{yaml_plain_out}}|\{\{)' + _flow_key_in_lookahead: |- + (?x: + (?= + (?: + (?:{{yaml_plain_in}}|{{knap_interpolation}}) + (?:{{knap_interpolation}}|[^\s:{{c_flow_indicator}}]|:[^\s{{c_flow_indicator}}]|\s+(?![#\s]))* + | ".*" + | '.*' + )\s*:(?:\s|$) + ) + ) + _flow_key_out_lookahead: |- + (?x: + (?= + (?: + (?:{{yaml_plain_out}}|{{knap_interpolation}}) + (?:{{knap_interpolation}}|[^\s:]|:\S|\s+(?![#\s]))* + | ".*" + | '.*' + )\s*:(?:\s|$) + ) + ) + +contexts: + block-scalar-body: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + flow-scalar-plain-in-body: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + flow-scalar-plain-out-body: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + flow-scalar-double-quoted-body: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + flow-scalar-single-quoted-body: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + flow-collection: + - meta_prepend: true + - include: Knap Expressions.sublime-syntax#tags + flow-mapping: + - match: '\{(?![{%#])' + scope: punctuation.definition.mapping.begin.yaml + push: flow-mapping-body diff --git a/editors/sublime/Knap.sublime-syntax b/editors/sublime/Knap.sublime-syntax new file mode 100644 index 0000000..2c71939 --- /dev/null +++ b/editors/sublime/Knap.sublime-syntax @@ -0,0 +1,10 @@ +%YAML 1.2 +--- +# Generated by editors/build.mjs. Do not edit directly. +name: Knap Markdown +scope: text.html.markdown.knap +version: 2 +file_extensions: + - knap + - knap.md +extends: Packages/Knap/Knap Markdown.sublime-syntax diff --git a/editors/sublime/LICENSE b/editors/sublime/LICENSE new file mode 100644 index 0000000..0d246bb --- /dev/null +++ b/editors/sublime/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Obsidian + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/editors/sublime/ThirdPartyNotices.txt b/editors/sublime/ThirdPartyNotices.txt new file mode 100644 index 0000000..9f3c2fa --- /dev/null +++ b/editors/sublime/ThirdPartyNotices.txt @@ -0,0 +1,12 @@ +Knap YAML.sublime-syntax adapts YAML integration from BetterJinja. +Source: https://github.com/Sublime-Instincts/BetterJinja/blob/b7b7fc43947ffcad46003479d8e60cde0c74e099/Syntaxes/YAML/YAML%20(Jinja).sublime-syntax + +The MIT License (MIT) + +Copyright 2020-2024 © Ashwin Shenoy + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/editors/sublime/knap-comment.sublime-snippet b/editors/sublime/knap-comment.sublime-snippet new file mode 100644 index 0000000..f75f6d7 --- /dev/null +++ b/editors/sublime/knap-comment.sublime-snippet @@ -0,0 +1,6 @@ + + + knap-comment + (text.html.markdown.knap | source.knap) - comment - string + Comment removed from rendered output + diff --git a/editors/sublime/knap-filter.sublime-snippet b/editors/sublime/knap-filter.sublime-snippet new file mode 100644 index 0000000..ceda7c1 --- /dev/null +++ b/editors/sublime/knap-filter.sublime-snippet @@ -0,0 +1,6 @@ + + + knap-filter + (text.html.markdown.knap | source.knap) - comment - string + Output a filtered value + diff --git a/editors/sublime/knap-for.sublime-snippet b/editors/sublime/knap-for.sublime-snippet new file mode 100644 index 0000000..fde4696 --- /dev/null +++ b/editors/sublime/knap-for.sublime-snippet @@ -0,0 +1,8 @@ + + + knap-for + (text.html.markdown.knap | source.knap) - comment - string + Loop over a collection + diff --git a/editors/sublime/knap-if.sublime-snippet b/editors/sublime/knap-if.sublime-snippet new file mode 100644 index 0000000..c19fd66 --- /dev/null +++ b/editors/sublime/knap-if.sublime-snippet @@ -0,0 +1,8 @@ + + + knap-if + (text.html.markdown.knap | source.knap) - comment - string + Conditional block + diff --git a/editors/sublime/knap-ifelse.sublime-snippet b/editors/sublime/knap-ifelse.sublime-snippet new file mode 100644 index 0000000..5369238 --- /dev/null +++ b/editors/sublime/knap-ifelse.sublime-snippet @@ -0,0 +1,10 @@ + + + knap-ifelse + (text.html.markdown.knap | source.knap) - comment - string + Conditional block with fallback + diff --git a/editors/sublime/knap-set.sublime-snippet b/editors/sublime/knap-set.sublime-snippet new file mode 100644 index 0000000..b5b717a --- /dev/null +++ b/editors/sublime/knap-set.sublime-snippet @@ -0,0 +1,6 @@ + + + knap-set + (text.html.markdown.knap | source.knap) - comment - string + Assign a local variable + diff --git a/editors/sublime/knap-var.sublime-snippet b/editors/sublime/knap-var.sublime-snippet new file mode 100644 index 0000000..6913ca1 --- /dev/null +++ b/editors/sublime/knap-var.sublime-snippet @@ -0,0 +1,6 @@ + + + knap-var + (text.html.markdown.knap | source.knap) - comment - string + Output a value + diff --git a/editors/sublime/syntax_test_knap.knap b/editors/sublime/syntax_test_knap.knap new file mode 100644 index 0000000..ccd0945 --- /dev/null +++ b/editors/sublime/syntax_test_knap.knap @@ -0,0 +1,53 @@ +# SYNTAX TEST "Packages/Knap/Knap.sublime-syntax" + +{{ title | upper }} +# ^^^^^ variable.other.knap +# ^^^^^ support.function.filter.knap + +{% if enabled %} +# ^^ keyword.control.knap + +{% endif %} +# ^^^^^ keyword.control.knap + +{{ text | highlight:blue }} +# ^^^^ string.unquoted.argument.knap + +{{ people | map:item => item.name }} +# ^^^^ variable.other.knap + +{# +{{ ignored }} +# ^^^^^^^ comment.block.knap - variable.other.knap + +#} + +# {{ title }} +# ^^^^^ variable.other.knap - markup.heading + +**before {{ value }} after** +# ^^^^^^ markup.bold.markdown - source.knap +# ^^^^^ variable.other.knap - markup.bold.markdown +# ^^^^^ markup.bold.markdown - source.knap + +`before {{ value }} after` +#^^^^^^ markup.raw.inline.markdown - source.knap +# ^^^^^ variable.other.knap - markup.raw.inline.markdown +# ^^^^^ markup.raw.inline.markdown - source.knap + +link +# ^^^^^^ string.quoted.double.html - source.knap +# ^^^^^ variable.other.knap - string +# ^^^^^ string.quoted.double.html - source.knap + +```javascript +const message = "before {{ value }} after"; +# ^^^^^^ string.quoted.double.js - source.knap +# ^^^^^ variable.other.knap - string +# ^^^^^ string.quoted.double.js - source.knap + +``` + +# Heading +# ^^^^^^^ markup.heading.1.markdown - source.knap - string + diff --git a/editors/sublime/syntax_test_knap_yaml.knap b/editors/sublime/syntax_test_knap_yaml.knap new file mode 100644 index 0000000..69e2b40 --- /dev/null +++ b/editors/sublime/syntax_test_knap_yaml.knap @@ -0,0 +1,46 @@ +# SYNTAX TEST "Packages/Knap/Knap YAML.sublime-syntax" + +key: "before {{ value }} after" +# ^^^^^^ string.quoted.double.yaml - source.knap +# ^^^^^ source.knap variable.other.knap - string - meta.mapping.key.yaml +# ^^^^^ string.quoted.double.yaml - source.knap + +key: 'before {{ value }} after' +# ^^^^^ source.knap variable.other.knap - string +# ^^^^^ string.quoted.single.yaml - source.knap + +key: before {{ value }} after +# ^^^^^ variable.other.knap - string +# ^^^^^ string.unquoted.plain.out.yaml - source.knap + +key: [{{ value }}, "after"] +# ^^^^^ variable.other.knap - string - meta.mapping.yaml +# ^^^^^ string.quoted.double.yaml - source.knap + +key: {{ value }} +next: true +# ^^^^ constant.language.boolean.true.yaml - source.knap - string + +key: | + before {{ value }} after +# ^^^^^^ string.unquoted.block.yaml +# ^^^^^ variable.other.knap - string +# ^^^^^ string.unquoted.block.yaml - source.knap + +next: true +# ^^^^ constant.language.boolean.true.yaml - source.knap - string + +{{ field }}: "after" +# ^^^^^ variable.other.knap - string - meta.mapping.key.yaml +# ^^^^^ string.quoted.double.yaml - source.knap + +key: "before {# {{ ignored }} #} {{ value }} after" +# ^^^^^^^ comment.block.knap - variable.other.knap - string +# ^^^^^ variable.other.knap - comment - string +# ^^^^^ string.quoted.double.yaml - source.knap + +key: "{{ 'literal {{ nested }}' | upper }} after" +# ^^^^^^ string.quoted.single.knap - variable.other.knap - string.quoted.double.yaml +# ^^^^^ support.function.filter.knap - string - comment +# ^^^^^ string.quoted.double.yaml - source.knap + diff --git a/editors/vscode/.vscodeignore b/editors/vscode/.vscodeignore new file mode 100644 index 0000000..41c5092 --- /dev/null +++ b/editors/vscode/.vscodeignore @@ -0,0 +1,2 @@ +**/.DS_Store +**/*.vsix diff --git a/editors/vscode/LICENSE b/editors/vscode/LICENSE new file mode 100644 index 0000000..0d246bb --- /dev/null +++ b/editors/vscode/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Obsidian + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/editors/vscode/README.md b/editors/vscode/README.md new file mode 100644 index 0000000..f4d972a --- /dev/null +++ b/editors/vscode/README.md @@ -0,0 +1,35 @@ +# Knap + +Syntax highlighting and snippets for [Knap](https://github.com/obsidianmd/knap), a template language for creating Markdown. + +Use `.knap` for Knap templates. The extension also recognizes `.knap.md`, or you can select **Knap Markdown** as the language mode for an existing template. Markdown formatting is preserved around expressions: + +```knap +# {{ title | trim }} + +{% if author %} +By {{ author.name }} +{% endif %} + +{% for tag in tags %} +- #{{ tag | kebab }} +{% endfor %} + +{# Comments are removed from the rendered output. #} +``` + +Type `knap-` to find snippets for variables, filters, conditions, loops, assignments, and comments. Use the editor's block-comment command to toggle `{# ... #}`. Template delimiters, parentheses, brackets, and quotes support pairing and selection wrapping. + +To select Knap for an existing template folder, use workspace settings: + +```json +{ + "files.associations": { + "templates/*.md": "knap" + } +} +``` + +This extension supplies highlighting and editing basics. It does not execute templates, validate them, or provide a rendered preview. Bare filter arguments use the same string coloring convention as the Knap website; they can still resolve to variables at runtime. Markdown-specific extensions may require the `markdown` language mode to activate their features. + +Development and installation instructions are in the repository's [editor support guide](https://github.com/obsidianmd/knap/tree/main/editors). diff --git a/editors/vscode/language-configuration.json b/editors/vscode/language-configuration.json new file mode 100644 index 0000000..beb1921 --- /dev/null +++ b/editors/vscode/language-configuration.json @@ -0,0 +1,15 @@ +{ + "comments": { "blockComment": ["{#", "#}"] }, + "brackets": [["{{", "}}"], ["{%", "%}"], ["{#", "#}"], ["(", ")"], ["[", "]"], ["{", "}"]], + "autoClosingPairs": [ + { "open": "{{", "close": "}}", "notIn": ["comment"] }, + { "open": "{%", "close": "%}", "notIn": ["comment"] }, + { "open": "{#", "close": "#}", "notIn": ["comment"] }, + { "open": "(", "close": ")", "notIn": ["string", "comment"] }, + { "open": "[", "close": "]", "notIn": ["string", "comment"] }, + { "open": "\"", "close": "\"", "notIn": ["string", "comment"] }, + { "open": "'", "close": "'", "notIn": ["string", "comment"] } + ], + "surroundingPairs": [["{{", "}}"], ["{%", "%}"], ["{#", "#}"], ["(", ")"], ["[", "]"], ["\"", "\""], ["'", "'"]], + "wordPattern": "[A-Za-z_$][\\w$]*" +} diff --git a/editors/vscode/package.json b/editors/vscode/package.json new file mode 100644 index 0000000..1d780af --- /dev/null +++ b/editors/vscode/package.json @@ -0,0 +1,29 @@ +{ + "name": "knap", + "displayName": "Knap", + "description": "Knap template syntax highlighting and snippets for Markdown.", + "version": "0.1.0", + "publisher": "obsidianmd", + "license": "MIT", + "repository": { "type": "git", "url": "https://github.com/obsidianmd/knap", "directory": "editors/vscode" }, + "engines": { "vscode": "^1.85.0" }, + "categories": ["Programming Languages", "Snippets"], + "keywords": ["knap", "markdown", "template", "obsidian"], + "contributes": { + "languages": [{ + "id": "knap", + "aliases": ["Knap Markdown", "Knap", "knap"], + "extensions": [".knap", ".knap.md"], + "configuration": "./language-configuration.json" + }], + "grammars": [ + { "scopeName": "source.knap", "path": "./syntaxes/knap.tmLanguage.json" }, + { + "language": "knap", + "scopeName": "text.html.markdown.knap", + "path": "./syntaxes/knap-markdown.tmLanguage.json" + } + ], + "snippets": [{ "language": "knap", "path": "./snippets/knap.json" }] + } +} diff --git a/editors/vscode/snippets/knap.json b/editors/vscode/snippets/knap.json new file mode 100644 index 0000000..a3346d4 --- /dev/null +++ b/editors/vscode/snippets/knap.json @@ -0,0 +1,37 @@ +{ + "Output a variable": { + "prefix": "knap-var", + "body": "{{ ${1:title} }}", + "description": "Output a value" + }, + "Apply a filter": { + "prefix": "knap-filter", + "body": "{{ ${1:title} | ${2:upper} }}", + "description": "Output a filtered value" + }, + "Conditional block": { + "prefix": "knap-if", + "body": "{% if ${1:condition} %}\n$0\n{% endif %}", + "description": "Conditional block" + }, + "Conditional with fallback": { + "prefix": "knap-ifelse", + "body": "{% if ${1:condition} %}\n\t${2}\n{% else %}\n\t$0\n{% endif %}", + "description": "Conditional block with fallback" + }, + "Loop": { + "prefix": "knap-for", + "body": "{% for ${1:item} in ${2:items} %}\n- {{ ${1:item} }}$0\n{% endfor %}", + "description": "Loop over a collection" + }, + "Set a variable": { + "prefix": "knap-set", + "body": "{% set ${1:name} = ${2:value} %}$0", + "description": "Assign a local variable" + }, + "Template comment": { + "prefix": "knap-comment", + "body": "{# ${1:comment} #}$0", + "description": "Comment removed from rendered output" + } +} diff --git a/editors/vscode/syntaxes/knap-markdown.tmLanguage.json b/editors/vscode/syntaxes/knap-markdown.tmLanguage.json new file mode 100644 index 0000000..28c59fb --- /dev/null +++ b/editors/vscode/syntaxes/knap-markdown.tmLanguage.json @@ -0,0 +1,48 @@ +{ + "name": "Knap Markdown", + "scopeName": "text.html.markdown.knap", + "patterns": [ + { + "include": "source.knap" + }, + { + "include": "text.html.markdown" + } + ], + "injections": { + "L:text.html.markdown.knap - meta.embedded.inline.knap - meta.embedded.block.knap - comment.block.knap": { + "patterns": [ + { + "include": "source.knap" + } + ] + }, + "L:text.html.markdown.knap - meta.embedded - comment - string - markup.inline.raw - markup.fenced_code - meta.tag": { + "patterns": [ + { + "include": "text.html.markdown#fenced_code_block" + }, + { + "name": "markup.inline.raw.string.markdown", + "begin": "(`+)", + "end": "(?|==|!=|>=|<=|&&|\\|\\||\\?\\?|[=<>!+*/-]", + "name": "keyword.operator.knap" + }, + { + "match": "\\\\.", + "name": "constant.character.escape.knap" + }, + { + "match": "[.,:]", + "name": "punctuation.separator.knap" + } + ] + }, + "group": { + "begin": "\\(", + "end": "\\)|(?=-?\\}\\}|-?%\\}|\\{[{%])", + "beginCaptures": { + "0": { + "name": "punctuation.section.group.begin.knap" + } + }, + "endCaptures": { + "0": { + "name": "punctuation.section.group.end.knap" + } + }, + "patterns": [ + { + "include": "#expression" + } + ] + }, + "array": { + "begin": "\\[", + "end": "\\]|(?=-?\\}\\}|-?%\\}|\\{[{%])", + "beginCaptures": { + "0": { + "name": "punctuation.section.group.begin.knap" + } + }, + "endCaptures": { + "0": { + "name": "punctuation.section.group.end.knap" + } + }, + "patterns": [ + { + "include": "#expression" + } + ] + }, + "object": { + "begin": "\\{(?![{%#])", + "end": "\\}|(?=-?\\}\\}|-?%\\}|\\{[{%])", + "beginCaptures": { + "0": { + "name": "punctuation.section.group.begin.knap" + } + }, + "endCaptures": { + "0": { + "name": "punctuation.section.group.end.knap" + } + }, + "patterns": [ + { + "include": "#expression" + } + ] + }, + "argument-group": { + "begin": "\\(", + "end": "\\)|(?=-?\\}\\}|-?%\\}|\\{[{%])", + "beginCaptures": { + "0": { + "name": "punctuation.section.group.begin.knap" + } + }, + "endCaptures": { + "0": { + "name": "punctuation.section.group.end.knap" + } + }, + "patterns": [ + { + "include": "#arguments" + } + ] + } + } +} diff --git a/editors/web.md b/editors/web.md new file mode 100644 index 0000000..3cb68b4 --- /dev/null +++ b/editors/web.md @@ -0,0 +1,82 @@ +# JavaScript syntax highlighting + +Knap provides optional `knap/prism`, `knap/codemirror`, and `knap/highlightjs` entry points in ESM and CommonJS, with TypeScript declarations. They contain highlighting definitions only and do not import the template renderer or an editor engine. Install the engine you use separately. + +These adapters highlight Knap tags in explicitly selected template text or fenced `knap` code blocks. Text outside tags is left unstyled. They do not change ordinary Markdown highlighting or provide a full Markdown, YAML, or HTML host grammar. The VS Code and Sublime packages provide that broader host integration. + +## Prism 1 + +```ts +import Prism from 'prismjs'; +import { registerKnap } from 'knap/prism'; + +registerKnap(Prism); +const html = Prism.highlight(source, Prism.languages.knap, 'knap'); +``` + +You can also import the default grammar and assign it to `Prism.languages.knap`. Both `Prism.tokenize` and `Prism.highlightElement` work normally. Register the grammar before highlighting elements with `class="language-knap"`. When using a Prism worker, register the grammar in the worker as well. + +In Obsidian, register the grammar in the `after` callback of `prismLoadable` in `src/shared/ext.ts`. This runs before reading view calls `Prism.highlightElement`, including when the loader is used directly rather than through `loadPrism()`. + +## CodeMirror 6 + +```ts +import { StreamLanguage } from '@codemirror/language'; +import { knapStreamParser } from 'knap/codemirror'; + +const knapLanguage = StreamLanguage.define(knapStreamParser); + +// Include knapLanguage in your EditorState extensions. +``` + +The parser uses the CodeMirror 5 token names that CodeMirror 6 already understands: `variable`, `variable-2` for filters, `property`, `keyword`, `operator`, `atom`, `number`, `string`, `comment`, and `punctuation`. In CodeMirror 6, `variable-2` maps to `tags.special(tags.variableName)` and `property` maps to `tags.propertyName`; include those tags in a custom `HighlightStyle` when assigning explicit colors. Comment delimiters and independent copies of multiline parser state are included. Knap's website uses this same parser with its existing theme and Markdown punctuation styling. + +## CodeMirror 5 and Obsidian's HyperMD integration + +```ts +import { registerKnap } from 'knap/codemirror'; + +registerKnap(CodeMirror); +``` + +This registers a `knap` mode, the `text/x-knap` MIME type, and a language metadata entry when `CodeMirror.modeInfo` is available. In Obsidian, call it in the `after` callback of `codemirrorModesLoadable`. Obsidian uses the CodeMirror 5 mode registry inside its CodeMirror 6 HyperMD parser, so this makes fenced `knap` blocks available in both Source mode and Live Preview. + +## highlight.js 11 + +```ts +import hljs from 'highlight.js/lib/core'; +import knap from 'knap/highlightjs'; + +hljs.registerLanguage('knap', knap); +const html = hljs.highlight(source, { language: 'knap' }).value; +``` + +The same registration works with the full `highlight.js` bundle and with `highlightElement`. Knap is excluded from automatic language detection because its delimiters overlap other template languages; use a `knap` fence or `language-knap` class. The adapter uses standard highlight.js classes, including `hljs-title function_` for filters, so existing themes supply the colors. + +In Obsidian Clipper, register it alongside the highlight.js import in `src/utils/reader.ts` to highlight Knap blocks in reader mode. + +## Testing with sibling checkouts + +Build Knap first: + +```sh +pnpm build +``` + +From the sibling Obsidian checkout, link the package: + +```sh +pnpm add -D knap@link:../knap +``` + +Clipper currently uses an older Knap renderer. A separate local dependency name allows testing the new adapter independently: + +```sh +npm install knap-syntax@file:../knap +``` + +Import `knap-syntax/highlightjs` in that setup. Both links resolve the built `dist` files; rebuild Knap after adapter changes and rebuild the consuming app. These sibling links are for local development. Before shipping, replace them with a published Knap version containing the adapters. Clipper can use `knap/highlightjs` after its renderer dependency is upgraded, or retain an npm alias for the newer package. + +## Validation + +`pnpm check` exercises the actual Prism, highlight.js, and CodeMirror 5 and 6 engines against shared fixtures for operators, filter arguments, nested map expressions, escaped and multiline strings, comments, incomplete input, source preservation, and Markdown fence selection. The adapters share the keyword and constant definitions in `src/language-syntax.json`; editor-specific matching and state handling live in their respective modules. diff --git a/package.json b/package.json index 3dcfadf..04af76b 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,15 @@ "*": { "html": [ "dist/html.d.ts" + ], + "prism": [ + "dist/prism.d.ts" + ], + "codemirror": [ + "dist/codemirror.d.ts" + ], + "highlightjs": [ + "dist/highlightjs.d.ts" ] } }, @@ -49,14 +58,31 @@ "types": "./dist/html.d.ts", "import": "./dist/html.js", "require": "./dist/html.cjs" + }, + "./prism": { + "types": "./dist/prism.d.ts", + "import": "./dist/prism.js", + "require": "./dist/prism.cjs" + }, + "./codemirror": { + "types": "./dist/codemirror.d.ts", + "import": "./dist/codemirror.js", + "require": "./dist/codemirror.cjs" + }, + "./highlightjs": { + "types": "./dist/highlightjs.d.ts", + "import": "./dist/highlightjs.js", + "require": "./dist/highlightjs.cjs" } }, "scripts": { + "editors:build": "node editors/build.mjs", + "editors:check": "node editors/build.mjs --check", "build": "tsup", "test": "vitest run", "bench": "vitest bench --run", "typecheck": "tsc --noEmit", - "check": "pnpm typecheck && pnpm test && pnpm build", + "check": "pnpm editors:check && pnpm typecheck && pnpm test && pnpm build", "prepack": "pnpm check" }, "engines": { @@ -72,9 +98,19 @@ "dayjs": "^1.11.23" }, "devDependencies": { + "@codemirror/language": "^6.12.4", + "@lezer/highlight": "^1.2.3", + "@shikijs/langs": "^4.4.3", "@types/node": "^20.19.43", + "@types/prismjs": "^1.26.6", + "codemirror": "^5.65.21", + "highlight.js": "^11.12.0", + "prismjs": "^1.30.0", "tsup": "^8.5.1", "typescript": "^5.9.3", - "vitest": "^4.1.11" + "vitest": "^4.1.11", + "vscode-oniguruma": "^2.0.1", + "vscode-textmate": "^9.3.2", + "yaml": "^2.9.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4d7b860..fa11803 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,21 +15,60 @@ importers: specifier: ^1.11.23 version: 1.11.23 devDependencies: + '@codemirror/language': + specifier: ^6.12.4 + version: 6.12.4 + '@lezer/highlight': + specifier: ^1.2.3 + version: 1.2.3 + '@shikijs/langs': + specifier: ^4.4.3 + version: 4.4.3 '@types/node': specifier: ^20.19.43 version: 20.19.43 + '@types/prismjs': + specifier: ^1.26.6 + version: 1.26.6 + codemirror: + specifier: ^5.65.21 + version: 5.65.21 + highlight.js: + specifier: ^11.12.0 + version: 11.12.0 + prismjs: + specifier: ^1.30.0 + version: 1.30.0 tsup: specifier: ^8.5.1 - version: 8.5.1(postcss@8.5.28)(typescript@5.9.3) + version: 8.5.1(postcss@8.5.28)(typescript@5.9.3)(yaml@2.9.1) typescript: specifier: ^5.9.3 version: 5.9.3 vitest: specifier: ^4.1.11 - version: 4.1.11(@types/node@20.19.43)(vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)) + version: 4.1.11(@types/node@20.19.43)(vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(yaml@2.9.1)) + vscode-oniguruma: + specifier: ^2.0.1 + version: 2.0.1 + vscode-textmate: + specifier: ^9.3.2 + version: 9.3.2 + yaml: + specifier: ^2.9.1 + version: 2.9.1 packages: + '@codemirror/language@6.12.4': + resolution: {integrity: sha512-1q4PaT+o6PbgpkJt4Q8Fv5XJxTy4FUZ4MWETtyiDw3J0Pyr9E2vqcKL+k9wcvjNTIsauxvE7OfmWj3FRPHQ76A==} + + '@codemirror/state@6.7.4': + resolution: {integrity: sha512-QhQIVRY+xHZDxwOSFrJ1eUMapJBUID3IdeAjf7dHO7zBUzSkyooHiodnalz5MG3iHzwixKMlAAyn7244y537EA==} + + '@codemirror/view@6.43.11': + resolution: {integrity: sha512-2+esucbQX6wB2JYi1eDvdCPFTA31BN8oSy6xCmk3G6CloV11yOvEjYk+gH7kLrP0MuHG94E8WDhjs5oMiu3+Wg==} + '@esbuild/aix-ppc64@0.28.2': resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==} engines: {node: '>=18'} @@ -199,6 +238,18 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@lezer/common@1.5.2': + resolution: {integrity: sha512-sxQE460fPZyU3sdc8lafxiPwJHBzZRy/udNFynGQky1SePYBdhkBl1kOagA9uT3pxR8K09bOrmTUqA9wb/PjSQ==} + + '@lezer/highlight@1.2.3': + resolution: {integrity: sha512-qXdH7UqTvGfdVBINrgKhDsVTJTxactNNxLk7+UMwZhU13lMHaOBlJe9Vqp907ya56Y3+ed2tlqzys7jDkTmW0g==} + + '@lezer/lr@1.4.10': + resolution: {integrity: sha512-rnCpTIBafOx4mRp43xOxDJbFipJm/c0cia/V5TiGlhmMa+wsSdoGmUN3w5Bqrks/09Q/D4tNAmWaT8p6NRi77A==} + + '@marijn/find-cluster-break@1.0.4': + resolution: {integrity: sha512-Wy0V7+SGUjnF9/TkiM1hKVDPj7jKXduPNboMVtHTA8dySMURWqfg/JZ9E2Sq8JgSJmkl7k7Qe9FLeMSrSraWmQ==} + '@napi-rs/lzma-linux-x64-gnu@1.5.1': resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==} engines: {node: ^22.20 || ^24.12 || >=25} @@ -426,6 +477,17 @@ packages: cpu: [x64] os: [win32] + '@shikijs/langs@4.4.3': + resolution: {integrity: sha512-ePic0yfAJGOF83D5wBHK/00EjK65oahBYxFk5epgq33WRv7X9UuxLEV8PtR0szC0z8dl7INIpIodB99JRFlR+A==} + engines: {node: '>=20'} + + '@shikijs/types@4.4.3': + resolution: {integrity: sha512-UEJxmRR++MAGR6hugn0vgVS2W/6lWAts84FFSrnlH9sP0LNol7E5+NQ792pH8liWUhyMyjhTgSUH3k7iD7tc5g==} + engines: {node: '>=20'} + + '@shikijs/vscode-textmate@10.0.2': + resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -438,9 +500,18 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/hast@3.0.5': + resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + '@types/node@20.19.43': resolution: {integrity: sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==} + '@types/prismjs@1.26.6': + resolution: {integrity: sha512-vqlvI7qlMvcCBbVe0AKAb4f97//Hy0EBTaiW8AalRnG/xAN5zOiWWyrNqNXeq8+KAuvRewjCVY1+IPxk4RdNYw==} + + '@types/unist@3.0.3': + resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} + '@vitest/expect@4.1.11': resolution: {integrity: sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==} @@ -500,6 +571,9 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + codemirror@5.65.21: + resolution: {integrity: sha512-6teYk0bA0nR3QP0ihGMoxuKzpl5W80FpnHpBJpgy66NK3cZv5b/d/HY8PnRvfSsCG1MTfr92u2WUl+wT0E40mQ==} + commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} @@ -514,6 +588,9 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + crelt@1.0.7: + resolution: {integrity: sha512-aK6BbWfhf4U/wCcLHKPJl/xa6VkVstRaPywWtMKGwuOLc/wZTyQYuoxgvZnNsBvv7Kg3YTBQYYBCggcviQczuA==} + dayjs@1.11.23: resolution: {integrity: sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ==} @@ -562,6 +639,10 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + highlight.js@11.12.0: + resolution: {integrity: sha512-nbfWpyRMcMrPMmDwJB+dhX/eiaPKtc2RB+0QZskqJ3WjRA/FDS0e9hZrx8EC/lbEv8gXy98FcDbNa/dspAaJMg==} + engines: {node: '>=12.0.0'} + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -711,6 +792,10 @@ packages: resolution: {integrity: sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==} engines: {node: ^10 || ^12 || >=14} + prismjs@1.30.0: + resolution: {integrity: sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==} + engines: {node: '>=6'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -746,6 +831,9 @@ packages: std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} + style-mod@4.1.3: + resolution: {integrity: sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==} + sucrase@3.35.1: resolution: {integrity: sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==} engines: {node: '>=16 || 14 >=14.17'} @@ -897,13 +985,47 @@ packages: jsdom: optional: true + vscode-oniguruma@2.0.1: + resolution: {integrity: sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ==} + + vscode-textmate@9.3.2: + resolution: {integrity: sha512-n2uGbUcrjhUEBH16uGA0TvUfhWwliFZ1e3+pTjrkim1Mt7ydB41lV08aUvsi70OlzDWp6X7Bx3w/x3fAXIsN0Q==} + + w3c-keyname@2.2.8: + resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==} + why-is-node-running@2.3.0: resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} engines: {node: '>=8'} hasBin: true + yaml@2.9.1: + resolution: {integrity: sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==} + engines: {node: '>= 14.6'} + hasBin: true + snapshots: + '@codemirror/language@6.12.4': + dependencies: + '@codemirror/state': 6.7.4 + '@codemirror/view': 6.43.11 + '@lezer/common': 1.5.2 + '@lezer/highlight': 1.2.3 + '@lezer/lr': 1.4.10 + style-mod: 4.1.3 + + '@codemirror/state@6.7.4': + dependencies: + '@marijn/find-cluster-break': 1.0.4 + + '@codemirror/view@6.43.11': + dependencies: + '@codemirror/state': 6.7.4 + crelt: 1.0.7 + style-mod: 4.1.3 + w3c-keyname: 2.2.8 + '@esbuild/aix-ppc64@0.28.2': optional: true @@ -996,6 +1118,18 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.6.0 + '@lezer/common@1.5.2': {} + + '@lezer/highlight@1.2.3': + dependencies: + '@lezer/common': 1.5.2 + + '@lezer/lr@1.4.10': + dependencies: + '@lezer/common': 1.5.2 + + '@marijn/find-cluster-break@1.0.4': {} + '@napi-rs/lzma-linux-x64-gnu@1.5.1': optional: true @@ -1123,6 +1257,17 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.63.1': optional: true + '@shikijs/langs@4.4.3': + dependencies: + '@shikijs/types': 4.4.3 + + '@shikijs/types@4.4.3': + dependencies: + '@shikijs/vscode-textmate': 10.0.2 + '@types/hast': 3.0.5 + + '@shikijs/vscode-textmate@10.0.2': {} + '@standard-schema/spec@1.1.0': {} '@types/chai@5.2.3': @@ -1134,10 +1279,18 @@ snapshots: '@types/estree@1.0.9': {} + '@types/hast@3.0.5': + dependencies: + '@types/unist': 3.0.3 + '@types/node@20.19.43': dependencies: undici-types: 6.21.0 + '@types/prismjs@1.26.6': {} + + '@types/unist@3.0.3': {} + '@vitest/expect@4.1.11': dependencies: '@standard-schema/spec': 1.1.0 @@ -1147,13 +1300,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.1 - '@vitest/mocker@4.1.11(vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2))': + '@vitest/mocker@4.1.11(vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(yaml@2.9.1))': dependencies: '@vitest/spy': 4.1.11 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.2.2(@types/node@20.19.43)(esbuild@0.28.2) + vite: 8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(yaml@2.9.1) '@vitest/pretty-format@4.1.11': dependencies: @@ -1198,6 +1351,8 @@ snapshots: dependencies: readdirp: 4.1.2 + codemirror@5.65.21: {} + commander@4.1.1: {} confbox@0.1.8: {} @@ -1206,6 +1361,8 @@ snapshots: convert-source-map@2.0.0: {} + crelt@1.0.7: {} + dayjs@1.11.23: {} debug@4.4.3: @@ -1264,6 +1421,8 @@ snapshots: fsevents@2.3.3: optional: true + highlight.js@11.12.0: {} + joycon@3.1.1: {} lightningcss-android-arm64@1.33.0: @@ -1360,11 +1519,12 @@ snapshots: mlly: 1.8.2 pathe: 2.0.3 - postcss-load-config@6.0.1(postcss@8.5.28): + postcss-load-config@6.0.1(postcss@8.5.28)(yaml@2.9.1): dependencies: lilconfig: 3.1.3 optionalDependencies: postcss: 8.5.28 + yaml: 2.9.1 postcss@8.5.28: dependencies: @@ -1372,6 +1532,8 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + prismjs@1.30.0: {} + readdirp@4.1.2: {} resolve-from@5.0.0: {} @@ -1439,6 +1601,8 @@ snapshots: std-env@4.2.0: {} + style-mod@4.1.3: {} + sucrase@3.35.1: dependencies: '@jridgewell/gen-mapping': 0.3.13 @@ -1474,7 +1638,7 @@ snapshots: ts-interface-checker@0.1.13: {} - tsup@8.5.1(postcss@8.5.28)(typescript@5.9.3): + tsup@8.5.1(postcss@8.5.28)(typescript@5.9.3)(yaml@2.9.1): dependencies: bundle-require: 5.1.0(esbuild@0.28.2) cac: 6.7.14 @@ -1485,7 +1649,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.28) + postcss-load-config: 6.0.1(postcss@8.5.28)(yaml@2.9.1) resolve-from: 5.0.0 rollup: 4.63.1 source-map: 0.7.6 @@ -1508,7 +1672,7 @@ snapshots: undici-types@6.21.0: {} - vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2): + vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(yaml@2.9.1): dependencies: lightningcss: 1.33.0 picomatch: 4.0.7 @@ -1519,11 +1683,12 @@ snapshots: '@types/node': 20.19.43 esbuild: 0.28.2 fsevents: 2.3.3 + yaml: 2.9.1 - vitest@4.1.11(@types/node@20.19.43)(vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)): + vitest@4.1.11(@types/node@20.19.43)(vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(yaml@2.9.1)): dependencies: '@vitest/expect': 4.1.11 - '@vitest/mocker': 4.1.11(vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)) + '@vitest/mocker': 4.1.11(vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(yaml@2.9.1)) '@vitest/pretty-format': 4.1.11 '@vitest/runner': 4.1.11 '@vitest/snapshot': 4.1.11 @@ -1540,14 +1705,22 @@ snapshots: tinyexec: 1.3.1 tinyglobby: 0.2.17 tinyrainbow: 3.1.1 - vite: 8.2.2(@types/node@20.19.43)(esbuild@0.28.2) + vite: 8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(yaml@2.9.1) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 20.19.43 transitivePeerDependencies: - msw + vscode-oniguruma@2.0.1: {} + + vscode-textmate@9.3.2: {} + + w3c-keyname@2.2.8: {} + why-is-node-running@2.3.0: dependencies: siginfo: 2.0.0 stackback: 0.0.2 + + yaml@2.9.1: {} diff --git a/src/codemirror.ts b/src/codemirror.ts new file mode 100644 index 0000000..9614180 --- /dev/null +++ b/src/codemirror.ts @@ -0,0 +1,133 @@ +import { constantNames, tagNames, wordOperatorNames } from './highlighting/rules'; + +type Filter = '' | 'name' | 'args' | 'map'; +export interface KnapState { + close: string; + quote: string; + filter: Filter; + filterName: string; + groups: { close: string; filter: Filter; filterName: string }[]; +} + +/** The small stream surface shared by CodeMirror 5 and CodeMirror 6. */ +export interface KnapStream { + string: string; + pos: number; +} + +const keywords = new Set(tagNames); +const operators = new Set(wordOperatorNames); +const constants = new Set(constantNames); +const word = /[A-Za-z_$][\w$]*/y; +const number = /-?\d+(?:\.\d+)?\b/y; +const symbol = /=>|==|!=|>=|<=|&&|\|\||\?\?|[=<>!+*/-]/y; +const property = /\.\s*[A-Za-z_$][\w$]*/y; + +function consume(stream: KnapStream, pattern: RegExp): string | undefined { + pattern.lastIndex = stream.pos; + const match = pattern.exec(stream.string); + if (!match) return; + stream.pos += match[0].length; + return match[0]; +} + +/** A CM5-compatible stream parser, also accepted by CM6's StreamLanguage.define. */ +export const knapStreamParser = { + name: 'knap', + startState: (): KnapState => ({ close: '', quote: '', filter: '', filterName: '', groups: [] }), + copyState: (state: KnapState): KnapState => ({ ...state, groups: state.groups.map(group => ({ ...group })) }), + languageData: { commentTokens: { block: { open: '{#', close: '#}' } } }, + blockCommentStart: '{#', + blockCommentEnd: '#}', + token(stream: KnapStream, state: KnapState): string | null { + const source = stream.string; + const starts = (value: string) => source.startsWith(value, stream.pos); + if (state.close === '#}') { + const end = source.indexOf('#}', stream.pos); + stream.pos = end < 0 ? source.length : end + 2; + if (end >= 0) state.close = ''; + return 'comment'; + } + if (!state.quote && (starts('{{') || starts('{%') || (!state.close && starts('{#')))) { + state.close = starts('{{') ? '}}' : starts('{%') ? '%}' : '#}'; + state.filter = ''; state.filterName = ''; state.groups = []; + stream.pos += 2; + if (state.close === '#}') { + const end = source.indexOf('#}', stream.pos); + stream.pos = end < 0 ? source.length : end + 2; + if (end >= 0) state.close = ''; + return 'comment'; + } + if (source[stream.pos] === '-') stream.pos++; + return 'punctuation'; + } + if (!state.close) { + // Stop at the next opener so templates still work inside Markdown strings/code. + do { stream.pos++; } while (stream.pos < source.length && !starts('{{') && !starts('{%') && !starts('{#')); + return null; + } + if (!state.quote && state.groups.length === 0 && (starts(state.close) || starts('-' + state.close))) { + stream.pos += state.close.length + (source[stream.pos] === '-' ? 1 : 0); + state.close = ''; state.filter = ''; state.filterName = ''; state.groups = []; + return 'punctuation'; + } + if (state.quote || starts('"') || starts("'")) { + if (!state.quote) state.quote = source[stream.pos++]; + while (stream.pos < source.length) { + const char = source[stream.pos++]; + if (char === '\\') stream.pos = Math.min(stream.pos + 1, source.length); + else if (char === state.quote) { state.quote = ''; break; } + } + return 'string'; + } + if (/\s/.test(source[stream.pos])) { + do { stream.pos++; } while (stream.pos < source.length && /\s/.test(source[stream.pos])); + return null; + } + if (state.filter === 'name' && state.filterName && !starts(':')) { + state.filter = ''; state.filterName = ''; + } + if (starts('||')) { stream.pos += 2; return 'operator'; } + if (starts('|')) { stream.pos++; state.filter = 'name'; state.filterName = ''; return 'operator'; } + if (starts(':') && state.filter === 'name') { + stream.pos++; state.filter = state.filterName === 'map' ? 'map' : 'args'; return 'punctuation'; + } + const char = source[stream.pos]; + const close = ({ '(': ')', '[': ']', '{': '}' } as Record)[char]; + if (close) { + state.groups.push({ close, filter: state.filter, filterName: state.filterName }); + if (state.filter !== 'args') { state.filter = ''; state.filterName = ''; } + stream.pos++; return 'punctuation'; + } + if (state.groups.at(-1)?.close === char) { + const group = state.groups.pop()!; + state.filter = group.filter; state.filterName = group.filterName; + stream.pos++; return 'punctuation'; + } + if (consume(stream, property)) return state.filter === 'args' ? 'string' : 'property'; + if (consume(stream, number)) return 'number'; + if (consume(stream, symbol)) return 'operator'; + const name = consume(stream, word); + if (name) { + if (state.filter === 'name') { state.filterName = name; return 'variable-2'; } + if (keywords.has(name)) return 'keyword'; + if (operators.has(name)) return 'operator'; + if (constants.has(name)) return 'atom'; + return state.filter === 'args' ? 'string' : 'variable'; + } + stream.pos++; return 'punctuation'; + }, +}; + +/** Register the stream parser and fence metadata with a CodeMirror 5 mode registry. */ +export function registerKnap(codeMirror: { + defineMode(name: string, factory: () => typeof knapStreamParser): void; + defineMIME(mime: string, mode: string): void; + modeInfo?: { name: string; mime?: string; mode: string; ext?: string[]; alias?: string[] }[]; +}): void { + codeMirror.defineMode('knap', () => knapStreamParser); + codeMirror.defineMIME('text/x-knap', 'knap'); + if (codeMirror.modeInfo && !codeMirror.modeInfo.some(mode => mode.mode === 'knap')) { + codeMirror.modeInfo.push({ name: 'Knap', mime: 'text/x-knap', mode: 'knap', ext: ['knap'], alias: ['knap'] }); + } +} diff --git a/src/highlighting/rules.ts b/src/highlighting/rules.ts new file mode 100644 index 0000000..4c56266 --- /dev/null +++ b/src/highlighting/rules.ts @@ -0,0 +1,14 @@ +import syntax from '../language-syntax.json'; + +export const tagNames = syntax.tags; +export const wordOperatorNames = syntax.wordOperators; +export const constantNames = syntax.constants; +export const identifier = '[A-Za-z_$][\\w$]*'; +export const words = (values: string[]) => new RegExp(`\\b(?:${values.join('|')})\\b(?!\\$)`); +export const keywords = words(tagNames); +export const wordOperators = words(wordOperatorNames); +export const constants = words(constantNames); +export const operator = /=>|==|!=|>=|<=|&&|\|\||\?\?|[|=<>!+*/-]/; +export const number = /-?\d+(?:\.\d+)?\b/; +// Consume escapes before checking quotes, including strings spanning lines. +export const quoted = String.raw`"(?:\\[\s\S]|[^"\\])*(?:"|$)|'(?:\\[\s\S]|[^'\\])*(?:'|$)`; diff --git a/src/highlightjs.ts b/src/highlightjs.ts new file mode 100644 index 0000000..7fac9d9 --- /dev/null +++ b/src/highlightjs.ts @@ -0,0 +1,80 @@ +import { constants, identifier, keywords, number, operator, wordOperators } from './highlighting/rules'; + +export interface Mode { + scope?: string | Record; + begin?: string | RegExp | RegExp[]; + end?: string | RegExp; + match?: RegExp | RegExp[]; + beginScope?: string | Record; + endScope?: string; + contains?: Mode[]; + relevance?: number; +} +export interface Language { + name: string; + disableAutodetect: boolean; + contains: Mode[]; +} + +/** Native highlight.js 11 language definition. Register it as `knap`. */ +export function knap(): Language { + const strings: Mode[] = ['"', "'"].map(quote => ({ + scope: 'string', begin: quote, end: quote, relevance: 0, + contains: [{ begin: /\\[\s\S]/, relevance: 0 }], + })); + const atoms: Mode[] = [ + { scope: 'property', match: new RegExp(`\\.\\s*${identifier}`), relevance: 0 }, + { scope: 'keyword', match: keywords, relevance: 0 }, + { scope: 'operator', match: wordOperators, relevance: 0 }, + { scope: 'literal', match: constants, relevance: 0 }, + { scope: 'number', match: number, relevance: 0 }, + { scope: 'operator', match: operator, relevance: 0 }, + { scope: 'punctuation', match: /[.,:]/, relevance: 0 }, + ]; + // A pipe ends a filter argument mode. Keeping it out of the child operator + // matcher lets the parent mode see it and start the next filter. + const argumentAtoms = atoms.map(mode => mode.match === operator + ? { ...mode, match: /=>|==|!=|>=|<=|&&|\|\||\?\?|[=<>!+*/-]/ } + : mode); + const expression: Mode[] = []; + const args: Mode[] = []; + const boundary = /(?=\|(?!\|)|\)|-?\}\}|-?%\}|\{[{%])/; + const group = (begin: RegExp, end: RegExp, contains: Mode[]): Mode => ({ + begin, end: new RegExp(`${end.source}|(?=-?\\}\\}|-?%\\}|\\{[{%])`), + beginScope: 'punctuation', endScope: 'punctuation', contains, relevance: 0, + }); + args.push(...strings, group(/\(/, /\)/, args), + { scope: 'string', match: new RegExp(`\\.\\s*${identifier}`), relevance: 0 }, + ...argumentAtoms, { scope: 'string', match: new RegExp(identifier), relevance: 0 }); + expression.push(...strings, + { scope: 'operator', match: /\|\|/, relevance: 0 }, + { + begin: [/\|(?!\|)/, /\s*/, /map\b/], beginScope: { 1: 'operator', 3: 'title.function' }, + end: boundary, contains: expression, relevance: 0, + }, + { + begin: [/\|(?!\|)/, /\s*/, new RegExp(`(?!map\\b)${identifier}`), /\s*/, /:/], + beginScope: { 1: 'operator', 3: 'title.function', 5: 'punctuation' }, + end: boundary, contains: args, relevance: 0, + }, + { + match: [/\|(?!\|)/, /\s*/, new RegExp(identifier)], + scope: { 1: 'operator', 3: 'title.function' }, relevance: 0, + }, + group(/\(/, /\)/, expression), group(/\[/, /\]/, expression), group(/\{(?![{%#])/, /\}/, expression), + ...atoms, { scope: 'variable', match: new RegExp(identifier), relevance: 0 }); + return { + name: 'Knap', + // Template delimiters overlap with many other languages. Require an explicit label. + disableAutodetect: true, + contains: [ + { scope: 'comment', begin: /\{#/, end: /#\}/, relevance: 0 }, + ...([[/\{\{-?/, /-?\}\}/], [/\{%-?/, /-?%\}/]] as const).map(([begin, end]) => ({ + begin, end: new RegExp(`${end.source}|(?=\\{[{%])`), + beginScope: 'punctuation', endScope: 'punctuation', contains: expression, relevance: 0, + })), + ], + }; +} + +export default knap; diff --git a/src/language-syntax.json b/src/language-syntax.json new file mode 100644 index 0000000..0415127 --- /dev/null +++ b/src/language-syntax.json @@ -0,0 +1,5 @@ +{ + "tags": ["if", "elseif", "else", "endif", "for", "in", "endfor", "set"], + "wordOperators": ["and", "or", "not", "contains"], + "constants": ["true", "false", "null"] +} diff --git a/src/prism.ts b/src/prism.ts new file mode 100644 index 0000000..61c3453 --- /dev/null +++ b/src/prism.ts @@ -0,0 +1,80 @@ +import { constants, identifier, keywords, number, operator, quoted, wordOperators } from './highlighting/rules'; + +// Structural types keep the adapter usable without installing Prism's type package. +export interface Grammar { + [token: string]: RegExp | GrammarToken | (RegExp | GrammarToken)[]; +} +export interface GrammarToken { + pattern: RegExp; + lookbehind?: boolean; + greedy?: boolean; + alias?: string; + inside?: Grammar; +} + +const string = { pattern: new RegExp(quoted), greedy: true }; +const balancedBraces = (depth: number): string => depth === 0 + ? String.raw`\{[^{}]*\}` + : String.raw`\{(?:${quoted}|${balancedBraces(depth - 1)}|[^{}"'])*\}`; +const nestedObject = balancedBraces(7); +const loneBrace = String.raw`\{(?![{%])|\}(?!\})`; +const wordToken = (pattern: RegExp): GrammarToken => ({ + pattern: new RegExp(`(^|[^\\w$])(?:${pattern.source})(?![\\w$])`), + lookbehind: true, +}); +const atoms: Grammar = { + string, + property: { pattern: new RegExp(`(\\.)\\s*${identifier}`), lookbehind: true }, + keyword: wordToken(keywords), + boolean: wordToken(constants), + number: wordToken(number), + operator: [wordToken(wordOperators), operator], + punctuation: /[()[\]{},.:]/, +}; +const argumentsGrammar: Grammar = { + ...atoms, + property: { pattern: new RegExp(`(\\.)\\s*${identifier}`), lookbehind: true, alias: 'string' }, + 'bare-argument': { pattern: new RegExp(identifier), alias: 'string' }, +}; +const filterInside: Grammar = { + operator: /^\|/, + function: new RegExp(`${identifier}$`), +}; +const filter: GrammarToken = { + pattern: new RegExp(`\\|(?!\\|)\\s*${identifier}`), + inside: filterInside, +}; +const expressionBody = (end: string) => String.raw`(?:${quoted}|${nestedObject}|${loneBrace}|(?!${end}|\{[{%])[^{}"'])*`; + +/** Prism 1 grammar for Knap tags in plain template text. */ +export const knap: Grammar = { + knap: { + // One ordered alternation keeps comments and quoted delimiters isolated. + pattern: new RegExp(String.raw`\{#[\s\S]*?(?:#\}|$)|\{\{-?${expressionBody('-?\\}\\}')}(?:-?\}\}|(?=\{[{%])|$)|\{%-?${expressionBody('-?%\\}')}(?:-?%\}|(?=\{[{%])|$)`), + greedy: true, + inside: { + comment: /^\{#[\s\S]*/, + 'boolean-or': { pattern: /\|\|/, alias: 'operator' }, + 'filter-with-arguments': { + pattern: new RegExp(String.raw`\|(?!\|)\s*(?!map\b)${identifier}\s*:(?:${quoted}|(?!\|(?!\|)|-?\}\}|-?%\}|\{[{%])[^"'])*`), + greedy: true, + inside: { + filter, + punctuation: /:/, + ...argumentsGrammar, + }, + }, + filter, + 'delimiter': { pattern: /^\{[{%]-?|-?[}%]\}$/, alias: 'punctuation' }, + ...atoms, + variable: new RegExp(identifier), + }, + }, +}; + +/** Register with the caller's Prism instance; importing this module has no side effects. */ +export function registerKnap(prism: { languages: { [name: string]: unknown } }): void { + prism.languages.knap = knap; +} + +export default knap; diff --git a/tests/editor-grammars.test.ts b/tests/editor-grammars.test.ts new file mode 100644 index 0000000..77745f0 --- /dev/null +++ b/tests/editor-grammars.test.ts @@ -0,0 +1,284 @@ +import { beforeAll, describe, expect, test } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { Registry, INITIAL, type IGrammar, type IRawGrammar, type StateStack } from 'vscode-textmate'; +import { loadWASM, OnigScanner, OnigString } from 'vscode-oniguruma'; +import markdown from '@shikijs/langs/markdown'; +import yaml from '@shikijs/langs/yaml'; +import html from '@shikijs/langs/html'; +import htmlDerivative from '@shikijs/langs/html-derivative'; +import javascript from '@shikijs/langs/javascript'; +import { parse as parseYaml, parseDocument, visit, isScalar } from 'yaml'; +import syntax from '../src/language-syntax.json'; +import { tokenize } from '../src/tokenizer'; +import { highlightLine } from '../website/src/lib/highlight'; + +const read = (path: string) => readFileSync(new URL(`../${path}`, import.meta.url), 'utf8'); +let core: IGrammar; +let template: IGrammar; +let plainMarkdown: IGrammar; +beforeAll(async () => { + const require = createRequire(import.meta.url); + const wasm = readFileSync(require.resolve('vscode-oniguruma/release/onig.wasm')); + await loadWASM(wasm.buffer.slice(wasm.byteOffset, wasm.byteOffset + wasm.byteLength)); + const grammars = [ + JSON.parse(read('editors/vscode/syntaxes/knap.tmLanguage.json')), + JSON.parse(read('editors/vscode/syntaxes/knap-markdown.tmLanguage.json')), + ...markdown, ...yaml, ...html, ...htmlDerivative, ...javascript, + ] as IRawGrammar[]; + const registry = new Registry({ + onigLib: Promise.resolve({ createOnigScanner: patterns => new OnigScanner(patterns), createOnigString: value => new OnigString(value) }), + loadGrammar: async scope => grammars.find(grammar => grammar.scopeName === scope) ?? null, + }); + core = (await registry.loadGrammar('source.knap'))!; + template = (await registry.loadGrammar('text.html.markdown.knap'))!; + plainMarkdown = (await registry.loadGrammar('text.html.markdown'))!; +}); + +function tokens(source: string, grammar = core) { + let state: StateStack = INITIAL; + return source.split('\n').flatMap(line => { + const result = grammar.tokenizeLine(line, state); + expect(result.stoppedEarly).toBeFalsy(); + state = result.ruleStack; + return result.tokens.map(token => ({ text: line.slice(token.startIndex, token.endIndex), scopes: token.scopes })); + }); +} +function has(source: string, text: string, scope: string, grammar = core) { + expect(tokens(source, grammar).some(token => token.text === text && token.scopes.includes(scope)), `${JSON.stringify(text)} should have ${scope} in ${source}`).toBe(true); +} + +function tokenAt(source: string, needle: string, grammar = template) { + const offset = source.indexOf(needle); + expect(offset).toBeGreaterThanOrEqual(0); + const lines = source.slice(0, offset).split('\n'); + let state: StateStack = INITIAL; + const sourceLines = source.split('\n'); + for (let line = 0; line < lines.length - 1; line++) state = grammar.tokenizeLine(sourceLines[line], state).ruleStack; + const line = sourceLines[lines.length - 1]; + const column = lines.at(-1)!.length; + const token = grammar.tokenizeLine(line, state).tokens.find(token => token.startIndex <= column && token.endIndex > column)!; + const encoded = grammar.tokenizeLine2(line, state).tokens; + let metadata = 0; + for (let index = 0; index < encoded.length && encoded[index] <= column; index += 2) metadata = encoded[index + 1]; + // TextMate's encoded token attributes store StandardTokenType in bits 8–9: + // 0 = code, 1 = comment, 2 = string, 3 = regular expression. + return { scopes: token.scopes, type: (metadata >>> 8) & 3 }; +} + +describe('Knap editor grammar', () => { + test('uses the runtime keywords and the website token categories', () => { + for (const word of [...syntax.tags, ...syntax.wordOperators, ...syntax.constants]) { + expect(tokenize(`{{ ${word} }}`).tokens[1].type).not.toBe('identifier'); + const scope = syntax.tags.includes(word) ? 'keyword.control.knap' + : syntax.wordOperators.includes(word) ? 'keyword.operator.word.knap' : 'constant.language.knap'; + has(`{{ ${word} }}`, word, scope); + expect(highlightLine(`{{ ${word} }}`, 'knap')).toContain(`syn-${syntax.constants.includes(word) ? 'constant' : 'keyword'}`); + } + }); + + test.each([ + ['{{ First name | trim }}', 'First', 'variable.other.knap'], + ['{{ First name | trim }}', 'name', 'variable.other.knap'], + ['{{ title | my_custom_filter }}', 'my_custom_filter', 'support.function.filter.knap'], + ['{{- title -}}', '{{-', 'punctuation.section.embedded.begin.knap'], + ['{%- if true -%}', '-%}', 'punctuation.section.embedded.end.knap'], + ['{{ value ?? -12.5 }}', '-12.5', 'constant.numeric.knap'], + ['{{ author.if }}', '.if', 'variable.other.property.knap'], + ['{% if a || b %}', 'b', 'variable.other.knap'], + ['{{ items[0].title }}', '.title', 'variable.other.property.knap'], + ['{{ value |\n upper }}', 'upper', 'support.function.filter.knap'], + ['{{ title | upper | trim }}', 'trim', 'support.function.filter.knap'], + ])('highlights %s', (source, text, scope) => has(source, text, scope)); + + // Ported from tests/website-highlight.test.ts: preserve its argument styling. + test.each([ + ['{{ text | highlight:blue }}', 'blue', 'string.unquoted.argument.knap'], + ['{{ footer | hr:before }}', 'before', 'string.unquoted.argument.knap'], + ['{{ blue }}', 'blue', 'variable.other.knap'], + ['{{ people | map:item => item.name }}', 'item', 'variable.other.knap'], + ['{{ text | truncatewords:(20, "…") }}', '…', 'string.quoted.double.knap'], + ['{{ people | sort:(details.rank, desc) }}', 'desc', 'string.unquoted.argument.knap'], + ['{{ people | sort:(details.rank, desc) }}', '.rank', 'string.unquoted.argument.knap'], + ])('retains website conventions for %s', (source, text, scope) => has(source, text, scope)); + + test('comments span lines, do not nest, and resume at the first closing delimiter', () => { + const result = tokens('{#\n{{ ignored }} {% if true %} {#\n#}{{ title }}'); + expect(result.filter(token => token.text.includes('ignored'))[0].scopes).toContain('comment.block.knap'); + expect(result.filter(token => token.text === 'title')[0].scopes).toContain('variable.other.knap'); + expect(tokens('{# unfinished').at(-1)!.scopes).toContain('comment.block.knap'); + }); + + test('strings protect delimiters, escaped quotes, and pipes, including across lines', () => { + const source = '{{ "{# literal #} }} | \\"\nmore" | upper }}'; + const result = tokens(source); + expect(result.some(token => token.scopes.includes('comment.block.knap'))).toBe(false); + has(source, 'more', 'string.quoted.double.knap'); + has(source, 'upper', 'support.function.filter.knap'); + }); + + test('nested map objects do not prematurely close the output tag', () => { + const source = '{{ people | map:item => ({name: item.name, flags: [true, false]}) | first }}'; + has(source, 'first', 'support.function.filter.knap'); + expect(tokens(source).at(-1)!.scopes).toContain('punctuation.section.embedded.end.knap'); + }); + + test('adjacent nested-object braces do not close the output tag', () => { + const source = '{{ x | map:i => ({a: {b: i}}) | first }}'; + has(source, 'first', 'support.function.filter.knap'); + expect(tokens(source).at(-1)!.scopes).toContain('punctuation.section.embedded.end.knap'); + }); + + test('filter chains resume after quoted arguments', () => { + has('{{ date | date:"YYYY-MM-DD" | upper }}', 'upper', 'support.function.filter.knap'); + }); + + test('recovers from unfinished expressions at the next opener', () => { + has('{{ broken | sort:(\n{{ title | upper }}', 'upper', 'support.function.filter.knap'); + has('{% if missing\n{% set title = "Hi" %}', 'set', 'keyword.control.knap'); + }); + + test.each([ + '# Heading {{ title }}', + '- **{{ title }}**', + '[{{ title }}](https://example.com)', + '[link]({{ title }})', + '`{{ title }}`', + '---\ntitle: "{{ title }}"\n---', + 'link', + '```text\n{{ title }}\n```', + '```javascript\nconst title = "{{ title }}";\n```', + ])('embeds Knap in Markdown: %s', source => has(source, 'title', 'variable.other.knap', template)); + + test('keeps Markdown formatting and does not inject into ordinary Markdown', () => { + expect(tokens('# Heading\n\n**bold**', template).some(token => token.scopes.some(scope => scope.startsWith('markup.heading')))).toBe(true); + expect(tokens('# {{ title }}', plainMarkdown).some(token => token.scopes.some(scope => scope.endsWith('.knap')))).toBe(false); + expect(tokens('title if true | upper', template).some(token => token.scopes.includes('variable.other.knap'))).toBe(false); + }); + + test.each([ + ['---\nkey: "before {{ value }} after"\n---\n\n# Heading', 'string.quoted.double.yaml'], + ["---\nkey: 'before {{ value }} after'\n---\n\n# Heading", 'string.quoted.single.yaml'], + ['link\n\n# Heading', 'string.quoted.double.html'], + ['```javascript\nconst x = "before {{ value }} after";\n```\n\n# Heading', 'string.quoted.double.js'], + ])('resets the editor token type inside host strings and restores it after: %s', (source, hostString) => { + expect(tokenAt(source, 'before').scopes).toContain(hostString); + expect(tokenAt(source, 'value')).toMatchObject({ type: 0 }); + expect(tokenAt(source, 'value').scopes).toContain('variable.other.knap'); + const after = tokenAt(source, 'after'); + expect(after.scopes).toContain(hostString); + expect(after.type).toBe(2); + expect(after.scopes).not.toContain('meta.embedded.inline.knap'); + const heading = tokenAt(source, 'Heading'); + expect(heading.scopes.some(scope => scope.startsWith('markup.heading'))).toBe(true); + expect(heading.scopes).not.toContain(hostString); + }); + + test.each([ + '# Before {{ value }} after', + '**Before {{ value }} after**', + '[Before {{ value }} after](https://example.com)', + '`Before {{ value }} after`', + ])('restores the same Markdown scopes after an interpolation: %s', source => { + expect(tokenAt(source, 'after').scopes).toEqual(tokenAt(source, 'Before').scopes); + expect(tokenAt(source, 'after').scopes).not.toContain('meta.embedded.inline.knap'); + }); + + test('Knap comments and strings stay isolated and the next tag resumes normally', () => { + const source = '---\nkey: "before {# {{ ignored }} #} {{ \'literal {{ nested }}\' | upper }} after"\n---'; + expect(tokenAt(source, 'ignored').type).toBe(1); + expect(tokenAt(source, 'ignored').scopes).not.toContain('variable.other.knap'); + expect(tokenAt(source, 'nested').type).toBe(2); + expect(tokenAt(source, 'nested').scopes).toContain('string.quoted.single.knap'); + expect(tokenAt(source, 'nested').scopes).not.toContain('variable.other.knap'); + expect(tokenAt(source, 'upper').scopes).toContain('support.function.filter.knap'); + expect(tokenAt(source, 'after').scopes).toEqual(tokenAt(source, 'before').scopes); + }); + + test.each([ + '---\nkey: {{ value }}\nnext: true\n---\n\n# Heading', + '---\nkey: [{{ value }}, "after"]\n---\n\n# Heading', + '---\nkey: |\n before {{ value }} after\nnext: true\n---\n\n# Heading', + '---\n{{ field }}: "{{ value }}"\n---\n\n# Heading', + ])('keeps YAML collections and block scalars bounded by frontmatter: %s', source => { + expect(tokenAt(source, 'value').scopes).toContain('variable.other.knap'); + const heading = tokenAt(source, 'Heading'); + expect(heading.scopes.some(scope => scope.startsWith('markup.heading'))).toBe(true); + expect(heading.scopes.some(scope => scope.endsWith('.yaml'))).toBe(false); + }); + + test.each(['```', '````', '~~~'])('preserves JavaScript in %s fences, including backticks and the closing fence', fence => { + const source = `${fence}javascript\nconst message = \`before {{ value }} after\`;\n${fence}\n\n# Heading`; + expect(tokenAt(source, 'message').scopes.some(scope => scope.endsWith('.js'))).toBe(true); + expect(tokenAt(source, 'after').scopes).toContain('string.template.js'); + expect(tokenAt(source, 'after').scopes).not.toContain('markup.inline.raw.string.markdown'); + expect(tokenAt(source, 'Heading').scopes.some(scope => scope.endsWith('.js'))).toBe(false); + }); + + test('Sublime inherits Markdown and keeps expression contexts independent of the host', () => { + const entry = parseYaml(read('editors/sublime/Knap.sublime-syntax')); + expect(entry.file_extensions).toEqual(['knap', 'knap.md']); + expect(entry.extends).toBe('Packages/Knap/Knap Markdown.sublime-syntax'); + const grammar = parseYaml(read('editors/sublime/Knap Expressions.sublime-syntax')); + expect(grammar.scope).toBe('source.knap'); + expect(grammar.hidden).toBe(true); + expect(grammar.extends).toBeUndefined(); + for (const context of Object.values(grammar.contexts) as Record[][]) { + for (const rule of context) { + for (const key of ['include', 'push']) { + if (typeof rule[key] === 'string') expect(grammar.contexts[rule[key]]).toBeDefined(); + } + } + } + }); + + test('Sublime capture keys are YAML integers, not quoted strings', () => { + const document = parseDocument(read('editors/sublime/Knap Expressions.sublime-syntax')); + expect(document.errors).toEqual([]); + let captureCount = 0; + visit(document, { + Pair(_, pair) { + if (isScalar(pair.key) && /^\d+$/.test(String(pair.key.value))) { + captureCount++; + expect(typeof pair.key.value).toBe('number'); + } + }, + }); + expect(captureCount).toBeGreaterThan(0); + }); + + test('Sublime host adapters resolve expressions without recursive prototype injection', () => { + const host = parseYaml(read('editors/sublime/Knap Markdown.sublime-syntax')); + expect(host.extends).toBe('Packages/Markdown/Markdown.sublime-syntax'); + expect(host.contexts.frontmatter[1].embed).toBe('scope:source.yaml.knap'); + const yamlHost = parseDocument(read('editors/sublime/Knap YAML.sublime-syntax')); + expect(yamlHost.errors).toEqual([]); + const yamlGrammar = yamlHost.toJS(); + expect(yamlGrammar.scope).toBe('source.yaml.knap'); + expect(yamlGrammar.extends).toBe('Packages/YAML/YAML.sublime-syntax'); + const base = parseYaml(read('editors/sublime/Knap Expressions.sublime-syntax')); + for (const name of ['Knap', 'Knap Expressions', 'Knap Markdown', 'Knap YAML', 'Knap HTML', 'Knap JavaScript']) { + const document = parseDocument(read(`editors/sublime/${name}.sublime-syntax`)); + expect(document.errors).toEqual([]); + // with_prototype across Markdown's embedded grammars exceeds Sublime's + // 25,000-context limit; host inheritance avoids that expansion. + visit(document, { + Pair(_, pair) { + if (isScalar(pair.key)) expect(pair.key.value).not.toBe('with_prototype'); + }, + }); + let references = 0; + for (const context of Object.values(document.toJS().contexts ?? {}) as Record[][]) { + for (const rule of context) { + if (typeof rule.include === 'string' && rule.include.includes('.sublime-syntax#')) { + const [resource, contextName] = rule.include.split('#'); + expect(resource).toBe('Knap Expressions.sublime-syntax'); + expect(base.contexts[contextName]).toBeDefined(); + references++; + } + } + } + if (!['Knap', 'Knap Expressions'].includes(name)) expect(references).toBeGreaterThan(0); + } + }); +}); diff --git a/tests/web-highlighters.test.ts b/tests/web-highlighters.test.ts new file mode 100644 index 0000000..eb59c8f --- /dev/null +++ b/tests/web-highlighters.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, test } from 'vitest'; +import { createRequire } from 'node:module'; +import { readFileSync } from 'node:fs'; +import { runInNewContext } from 'node:vm'; +import Prism from 'prismjs'; +import hljs from 'highlight.js/lib/core'; +import { StreamLanguage, StringStream } from '@codemirror/language'; +import { classHighlighter, highlightTree } from '@lezer/highlight'; +import { knap as prismGrammar, registerKnap as registerPrism } from '../src/prism'; +import highlightjs from '../src/highlightjs'; +import { knapStreamParser, registerKnap as registerCodeMirror } from '../src/codemirror'; +import { templateHighlightStyle, templateLanguage } from '../website/src/scripts/playground-template-editor'; + +registerPrism(Prism); +hljs.registerLanguage('knap', highlightjs); +const cmLanguage = StreamLanguage.define(knapStreamParser); +const require = createRequire(import.meta.url); +const cm5 = require('codemirror/addon/runmode/runmode.node'); +for (const file of ['mode/meta.js', 'mode/markdown/markdown.js']) { + runInNewContext(readFileSync(require.resolve(`codemirror/${file}`), 'utf8'), { CodeMirror: cm5 }); +} +registerCodeMirror(cm5); + +interface Span { text: string; classes: string[] } +function htmlSpans(html: string): Span[] { + const stack: string[][] = []; + const spans: Span[] = []; + for (const part of html.split(/(<\/?span\b[^>]*>)/)) { + if (part.startsWith('') stack.pop(); + else if (part) spans.push({ text: part.replace(/&(amp|lt|gt|quot|#x27|#39);/g, (_, entity: string) => ({ amp: '&', lt: '<', gt: '>', quot: '"', '#x27': "'", '#39': "'" })[entity]!), classes: stack.flat() }); + } + return spans; +} +function cmSpans(source: string): Span[] { + const tree = cmLanguage.parser.parse(source); + const spans: Span[] = []; + let pos = 0; + highlightTree(tree, classHighlighter, (from, to, classes) => { + if (from > pos) spans.push({ text: source.slice(pos, from), classes: [] }); + spans.push({ text: source.slice(from, to), classes: classes.split(' ') }); + pos = to; + }); + if (pos < source.length) spans.push({ text: source.slice(pos), classes: [] }); + return spans; +} +function cm5Spans(source: string, mode: string | object = 'knap'): Span[] { + const spans: Span[] = []; + cm5.runMode(source, mode, (text: string, style: string | null) => { + if (text) spans.push({ text, classes: style?.split(' ') ?? [] }); + }); + return spans; +} +const engines = [ + { name: 'Prism', tokens: (source: string) => htmlSpans(Prism.highlight(source, prismGrammar, 'knap')), classes: { variable: 'variable', filter: 'function', keyword: 'keyword', operator: 'operator', string: 'string', comment: 'comment', number: 'number', constant: 'boolean', property: 'property' } }, + { name: 'highlight.js', tokens: (source: string) => { + const result = hljs.highlight(source, { language: 'knap' }); + expect(result.errorRaised).toBeUndefined(); + return htmlSpans(result.value); + }, classes: { variable: 'hljs-variable', filter: 'hljs-title', keyword: 'hljs-keyword', operator: 'hljs-operator', string: 'hljs-string', comment: 'hljs-comment', number: 'hljs-number', constant: 'hljs-literal', property: 'hljs-property' } }, + { name: 'CodeMirror 6', tokens: cmSpans, classes: { variable: 'tok-variableName', filter: 'tok-variableName2', keyword: 'tok-keyword', operator: 'tok-operator', string: 'tok-string', comment: 'tok-comment', number: 'tok-number', constant: 'tok-atom', property: 'tok-propertyName' } }, + { name: 'CodeMirror 5', tokens: cm5Spans, classes: { variable: 'variable', filter: 'variable-2', keyword: 'keyword', operator: 'operator', string: 'string', comment: 'comment', number: 'number', constant: 'atom', property: 'property' } }, +]; + +const cases = [ + ['{{ title | trim }}', 'title', 'variable'], + ['{{ title12 }}', 'title12', 'variable'], + ['{{ $if }}', '$if', 'variable'], + ['{{ if$ }}', 'if$', 'variable'], + ['{{ title | trim }}', 'trim', 'filter'], + ['{{ title|trim }}', 'title', 'variable'], + ['{{ date | date:"YYYY-MM-DD" | upper }}', 'upper', 'filter'], + ['{{ title |\n upper }}', 'upper', 'filter'], + ['{{ summary ?? "No summary" }}', '??', 'operator'], + ['{{ title | trim ?? true }}', 'true', 'constant'], + ['{% if a || b %}', '||', 'operator'], + ['{% if a || b %}', 'b', 'variable'], + ['{% if a and b %}', 'and', 'operator'], + ['{% elseif enabled %}', 'elseif', 'keyword'], + ['{{ value ?? -12.5 }}', '-12.5', 'number'], + ['{{ true }}', 'true', 'constant'], + ['{{ author.if }}', 'if', 'property'], + ['{{ text | highlight:blue }}', 'blue', 'string'], + ['{{ people | sort:(details.rank, desc) }}', 'rank', 'string'], + ['{{ people | sort:(details.rank, desc) }}', 'desc', 'string'], + ['{{ text | truncatewords:(20, "…") }}', '…', 'string'], + ['{{ people | map:item => item.name }}', 'item', 'variable'], + ['{{ people | map:item => ({name: item.name, flags: [true, false]}) | first }}', 'first', 'filter'], + ['{{ x | map:i => ({a: {b: i}}) | first }}', 'first', 'filter'], + ['{{ value } | upper }}', 'upper', 'filter'], + ['{{ people | map:item => (item.name | upper) | first }}', 'upper', 'filter'], + ['{#\n{{ ignored }} {% if true %} {#\n#}{{ title }}', 'ignored', 'comment'], + ['{#\n{{ ignored }} {% if true %} {#\n#}{{ title }}', 'title', 'variable'], + ['{{ "{# literal #} }} | \\"\nmore" | upper }}', 'more', 'string'], + ['{{ "{# literal #} }} | \\"\nmore" | upper }}', 'upper', 'filter'], + ['{{ broken | sort:(\n{{ title | upper }}', 'upper', 'filter'], + ['{% if missing\n{% set title = "Hi" %}', 'set', 'keyword'], + ['{# unfinished {{ comment', 'comment', 'comment'], + ['{{ "unfinished', 'unfinished', 'string'], + ['{{ title }}', 'url', 'variable'], + ['---\ntitle: "{{ title }}"\n---', 'title }}', 'variable'], + ['```text\n{{ content }}\n```', 'content', 'variable'], +] as const; + +describe.each(engines)('$name Knap adapter', engine => { + test.each(cases)('%s → %s is %s', (source, needle, category) => { + const spans = engine.tokens(source); + expect(spans.map(span => span.text).join('')).toBe(source); + const offset = source.indexOf(needle); + let position = 0; + const span = spans.find(span => { + const start = position; + position += span.text.length; + return start <= offset && position > offset; + }); + expect(span?.classes).toContain(engine.classes[category]); + // Inspect the entire identifier so a correctly styled prefix cannot hide + // a number or keyword incorrectly recognized inside it. + if (category === 'variable' && !needle.includes(' ')) { + let from = 0; + for (const part of spans) { + const to = from + part.text.length; + if (from < offset + needle.length && to > offset) expect(part.classes).toContain(engine.classes.variable); + from = to; + } + } + }); + test('leaves ordinary text unstyled and resumes it after closing a tag', () => { + const spans = engine.tokens('Before {{ value }} after'); + expect(spans[0]).toEqual({ text: 'Before ', classes: [] }); + expect(spans.at(-1)).toEqual({ text: ' after', classes: [] }); + }); + test('preserves hostile HTML as text', () => { + const source = ' {{ " &" }}'; + expect(engine.tokens(source).map(span => span.text).join('')).toBe(source); + }); +}); + +test('Prism grammar supports tokenize directly and does not modify Markdown', () => { + expect(Prism.tokenize('{{ title }}', prismGrammar).some(token => typeof token !== 'string')).toBe(true); + expect(Prism.languages.knap).toBe(prismGrammar); +}); + +test('HTML renderers escape markup both outside tags and inside template strings', () => { + const source = ' {{ " &" }}'; + for (const html of [Prism.highlight(source, prismGrammar, 'knap'), hljs.highlight(source, { language: 'knap' }).value]) { + expect(html).not.toMatch(/<(?:script|img)\b/); + expect(html).toContain('<script'); + expect(html).toContain('<img'); + } +}); + +test('highlight.js does not autodetect Knap in unrelated text', () => { + expect(hljs.autoDetection('knap')).toBe(false); + expect(hljs.highlightAuto('{{ title }}').language).toBeUndefined(); +}); + +test('CodeMirror state copies isolate incremental branches, including nested filters and strings', () => { + const state = knapStreamParser.startState(); + const line = new StringStream('{{ people | map:item => (item | trim:"unfinished', 4, 2); + while (!line.eol()) knapStreamParser.token(line, state); + const copy = knapStreamParser.copyState(state); + const finish = new StringStream('") }}', 4, 2); + while (!finish.eol()) knapStreamParser.token(finish, copy); + expect(copy.close).toBe(''); + expect(state.quote).toBe('"'); + expect(state.groups).toHaveLength(1); +}); + +test('the playground theme distinguishes filters and retains property and constant styling', () => { + const source = '{{ author.name | trim ?? true }}'; + const spans: Span[] = []; + highlightTree(templateLanguage.parser.parse(source), templateHighlightStyle, (from, to, classes) => { + spans.push({ text: source.slice(from, to), classes: classes.split(' ') }); + }); + expect(spans.find(span => span.text === '.name')?.classes).toContain('syn-variable'); + expect(spans.find(span => span.text === 'trim')?.classes).toContain('syn-filter'); + expect(spans.find(span => span.text === 'true')?.classes).toContain('syn-keyword'); +}); + +test('CodeMirror 5 selects Knap inside Markdown fences and resumes Markdown afterward', () => { + registerCodeMirror(cm5); + expect(cm5.modeInfo.filter((mode: { mode: string }) => mode.mode === 'knap')).toHaveLength(1); + expect(cm5.findModeByName('knap').mime).toBe('text/x-knap'); + const source = '```knap\n{{ title | trim }}\n```\n\n# Heading'; + const spans = cm5Spans(source, { name: 'markdown', fencedCodeBlockHighlighting: true }); + expect(spans.find(span => span.text === 'trim')?.classes).toContain('variable-2'); + expect(spans.find(span => span.text.includes('Heading'))?.classes).toContain('header'); +}); diff --git a/tests/website-highlight.test.ts b/tests/website-highlight.test.ts index 78d4a61..73c9c02 100644 --- a/tests/website-highlight.test.ts +++ b/tests/website-highlight.test.ts @@ -192,4 +192,18 @@ describe('website syntax highlighting', () => { expect(highlightLine('{{ people | map:item => item.name }}', 'knap')) .toContain('item'); }); + + test('keeps chained filters after adjacent nested-object braces', () => { + const html = highlightLine('{{ x | map:i => ({a: {b: i}}) | first }}', 'knap'); + expect(html).toContain('first'); + }); + + test('keeps chained filters after quoted arguments', () => { + const html = highlightLine('{{ date | date:"YYYY-MM-DD" | upper }}', 'knap'); + expect(html).toContain('upper'); + }); + + test('returns to unstyled Markdown after an output tag', () => { + expect(highlightLine('{{ title }} after }', 'knap')).toMatch(/}<\/span>}<\/span> after }$/); + }); }); diff --git a/tsup.config.ts b/tsup.config.ts index 689d3c2..e5b9c73 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from 'tsup'; export default defineConfig({ - entry: ['src/index.ts', 'src/html.ts', 'src/cli.ts'], + entry: ['src/index.ts', 'src/html.ts', 'src/cli.ts', 'src/prism.ts', 'src/codemirror.ts', 'src/highlightjs.ts'], format: ['esm', 'cjs'], dts: true, sourcemap: true, diff --git a/website/src/lib/editor-highlighting.ts b/website/src/lib/editor-highlighting.ts index 5e681fe..d960510 100644 --- a/website/src/lib/editor-highlighting.ts +++ b/website/src/lib/editor-highlighting.ts @@ -1,5 +1,6 @@ import { maxHighlightLineLength } from './playground-limits'; import { StreamLanguage, HighlightStyle, syntaxHighlighting } from '@codemirror/language'; +import type { Extension } from '@codemirror/state'; import { Tag } from '@lezer/highlight'; import { highlightLine } from './highlight'; @@ -56,6 +57,6 @@ export function editorLanguage(language: 'json' | 'md') { }); } -export const editorHighlighting = (language: 'json' | 'md') => [ +export const editorHighlighting = (language: 'json' | 'md'): Extension[] => [ editorLanguage(language), syntaxHighlighting(editorHighlightStyle), ]; diff --git a/website/src/lib/highlight.ts b/website/src/lib/highlight.ts index fe648aa..7a2d794 100644 --- a/website/src/lib/highlight.ts +++ b/website/src/lib/highlight.ts @@ -1,5 +1,6 @@ import { maxHighlightLineLength } from './playground-limits'; import { markdownPunctuationAt } from './markdown-punctuation'; +import { knapKeyword, knapConstant } from './knap-syntax'; export type CodeLanguage = 'knap' | 'ts' | 'shell' | 'md' | 'json'; @@ -20,8 +21,8 @@ function tokenClass(token: string, language: CodeLanguage) { if (/^['"`]/.test(token)) return 'syn-string'; if (/^\d/.test(token)) return 'syn-number'; if (language === 'ts' && /^(import|from)$/.test(token)) return 'syn-import'; - if (/^(true|false|null|undefined)$/.test(token)) return 'syn-constant'; - if (/^(if|elseif|else|endif|for|in|endfor|set|and|or|not|contains)$/.test(token)) return 'syn-keyword'; + if (knapConstant.test(token) || token === 'undefined') return 'syn-constant'; + if (knapKeyword.test(token)) return 'syn-keyword'; if (language === 'ts' && /^(const|let|type|async|await|return|new|throw|export)$/.test(token)) return 'syn-keyword'; if (/^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*|\[[^\]]+\])*$/.test(token)) return 'syn-variable'; return undefined; @@ -145,20 +146,41 @@ function highlightShellLine(line: string, state = shellHighlightState()) { } function highlightTokenLine(line: string, language: Exclude, constrainKnapToTags = true) { - const pattern = /(\{\{|\}\}|\{%|%\}|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`|===|!==|==|!=|=>|<=|>=|&&|\|\||\?\?|[{}()[\].,:;=+\-*/<>!?|]|\b(?:if|elseif|else|endif|for|in|endfor|set|and|or|not|contains|true|false|null|undefined|import|from|const|let|type|async|await|return|new|throw|export|pnpm|npm|npx)\b|\b\d(?:_?\d)*(?:\.\d(?:_?\d)*)?\b|[A-Za-z_$][\w$]*)/g; + const pattern = /(\{\{|\{%|%\}|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`|===|!==|==|!=|=>|<=|>=|&&|\|\||\?\?|[{}()[\].,:;=+\-*/<>!?|]|\b(?:if|elseif|else|endif|for|in|endfor|set|and|or|not|contains|true|false|null|undefined|import|from|const|let|type|async|await|return|new|throw|export|pnpm|npm|npx)\b|\b\d(?:_?\d)*(?:\.\d(?:_?\d)*)?\b|[A-Za-z_$][\w$]*)/g; const tokens = line.split(pattern).filter(Boolean); let expectsFilter = false; let inKnapExpression = false; let currentFilter: string | undefined; let inFilterArguments = false; + const groups: string[] = []; + let outputTag = false; + let outputCloseBrace = false; return tokens.map((token, index) => { let className = tokenClass(token, language); if (language === 'knap' && constrainKnapToTags) { - if (/^(\{\{|\{%)$/.test(token)) inKnapExpression = true; - else if (/^(\}\}|%\})$/.test(token)) inKnapExpression = false; - else if (!inKnapExpression) className = undefined; + if (/^(\{\{|\{%)$/.test(token)) { + inKnapExpression = true; + outputTag = token === '{{'; + outputCloseBrace = false; + groups.length = 0; + } + else if (token === '}' && outputTag && inKnapExpression) { + if (groups.at(-1) === '}') groups.pop(); + else if (outputCloseBrace) { + inKnapExpression = false; + outputTag = false; + outputCloseBrace = false; + } else outputCloseBrace = true; + } + else if (token === '%}' && groups.length === 0) inKnapExpression = false; + else { + outputCloseBrace = false; + if (/^[([{]$/.test(token)) groups.push(token === '(' ? ')' : token === '[' ? ']' : '}'); + else if (groups.at(-1) === token) groups.pop(); + else if (!inKnapExpression) className = undefined; + } } if (token === '|' && language === 'knap' && inKnapExpression) { @@ -218,15 +240,40 @@ export function renderInlineCode(value: string): string { } function highlightKnapLine(line: string) { - return line.split(/(\{\{.*?\}\}|\{%.*?%\}|(?:\{\{|\{%).*?$)/g).filter(Boolean).map((segment) => - /^(?:\{\{|\{%)/.test(segment) - ? highlightTokenLine(segment, 'knap') - : highlightMarkdownLine(segment) - ).join(''); + let output = ''; + let start = 0; + while (start < line.length) { + const outputStart = line.indexOf('{{', start); + const logicStart = line.indexOf('{%', start); + const tagStart = outputStart < 0 ? logicStart : logicStart < 0 ? outputStart : Math.min(outputStart, logicStart); + if (tagStart < 0) return output + highlightMarkdownLine(line.slice(start)); + output += highlightMarkdownLine(line.slice(start, tagStart)); + const close = line[tagStart + 1] === '{' ? '}}' : '%}'; + const groups: string[] = []; + let quote = ''; + let index = tagStart + 2; + for (; index < line.length;) { + if (quote) { + if (line[index] === '\\') index += 2; + else { if (line[index] === quote) quote = ''; index++; } + } else if (groups.length === 0 && line.startsWith(close, index)) { + index += close.length; + break; + } else { + const char = line[index++]; + if (char === '"' || char === "'") quote = char; + else if (/^[([{]$/.test(char)) groups.push(char === '(' ? ')' : char === '[' ? ']' : '}'); + else if (groups.at(-1) === char) groups.pop(); + } + } + output += highlightTokenLine(line.slice(tagStart, index), 'knap'); + start = index; + } + return output; } -interface KnapHighlightState { comment: boolean; close: string; quote: string } -const knapHighlightState = (): KnapHighlightState => ({ comment: false, close: '', quote: '' }); +interface KnapHighlightState { comment: boolean; close: string; quote: string; groups: string[] } +const knapHighlightState = (): KnapHighlightState => ({ comment: false, close: '', quote: '', groups: [] }); function highlightKnapComments(line: string, state = knapHighlightState()): string { let output = ''; @@ -246,9 +293,12 @@ function highlightKnapComments(line: string, state = knapHighlightState()): stri index++; } } else if (state.close) { - if (line.startsWith(state.close, index)) { state.close = ''; index += 2; } + if (state.groups.length === 0 && line.startsWith(state.close, index)) { state.close = ''; index += 2; } else { - if (line[index] === '"' || line[index] === "'") state.quote = line[index]; + const char = line[index]; + if (char === '"' || char === "'") state.quote = char; + else if (/^[([{]$/.test(char)) state.groups.push(char === '(' ? ')' : char === '[' ? ']' : '}'); + else if (state.groups.at(-1) === char) state.groups.pop(); index++; } } else if (line.startsWith('{#', index)) { @@ -258,6 +308,7 @@ function highlightKnapComments(line: string, state = knapHighlightState()): stri state.comment = true; } else if (line.startsWith('{{', index) || line.startsWith('{%', index)) { state.close = line[index + 1] === '{' ? '}}' : '%}'; + state.groups = []; index += 2; } else index++; } diff --git a/website/src/lib/knap-syntax.ts b/website/src/lib/knap-syntax.ts new file mode 100644 index 0000000..a12e7d5 --- /dev/null +++ b/website/src/lib/knap-syntax.ts @@ -0,0 +1,6 @@ +import syntax from '../../../src/language-syntax.json'; + +// Shared with the generated desktop editor grammars. Keep each renderer's +// colors and state handling here in the website. +export const knapKeyword = new RegExp(`^(?:${[...syntax.tags, ...syntax.wordOperators].join('|')})$`); +export const knapConstant = new RegExp(`^(?:${syntax.constants.join('|')})$`); diff --git a/website/src/scripts/playground-template-editor.ts b/website/src/scripts/playground-template-editor.ts index a21e6c9..2f3f091 100644 --- a/website/src/scripts/playground-template-editor.ts +++ b/website/src/scripts/playground-template-editor.ts @@ -6,59 +6,40 @@ import { tags } from '@lezer/highlight'; import { templateCompletions, type TemplateSuggestion } from '../lib/playground-completions'; import { emptyTemplatePair, pairTemplateInput } from '../lib/playground-pairs'; import { markdownPunctuationAt } from '../lib/markdown-punctuation'; +import { knapStreamParser } from '../../../src/codemirror'; import { createPlaygroundEditor } from './playground-editor'; export const templateLanguage = StreamLanguage.define({ - startState: () => ({ close: '', quote: '', filter: false }), + ...knapStreamParser, token(stream, state) { if (stream.string.length > maxHighlightLineLength) { - stream.skipToEnd(); state.close = ''; state.quote = ''; state.filter = false; + stream.skipToEnd(); + Object.assign(state, knapStreamParser.startState()); return null; } - if (!state.close) { - if (stream.match('{{')) state.close = '}}'; - else if (stream.match('{%')) state.close = '%}'; - else if (stream.match('{#')) { state.close = '#}'; return 'comment'; } - else { - const length = markdownPunctuationAt(stream.string, stream.pos); - if (length) { stream.pos += length; return 'punctuation'; } - stream.next(); - return null; - } - return 'punctuation'; - } - if (state.close === '#}') { - if (stream.skipTo('#}')) { stream.match('#}'); state.close = ''; } - else stream.skipToEnd(); - return 'comment'; - } - if (!state.quote && stream.match(state.close)) { - state.close = ''; state.filter = false; - return 'punctuation'; - } - if (state.quote || stream.peek() === '"' || stream.peek() === "'") { - if (!state.quote) state.quote = stream.next()!; - while (!stream.eol()) { - const char = stream.next(); - if (char === '\\') stream.next(); - else if (char === state.quote) { state.quote = ''; break; } - } - return 'string'; - } - if (stream.eatSpace()) return null; - if (stream.match('||')) return 'operator'; - if (stream.match('|')) { state.filter = true; return 'operator'; } - if (stream.match(/\d+(?:\.\d+)?/)) return 'number'; - if (stream.match(/[a-zA-Z_$][\w$]*/)) { - if (state.filter) { state.filter = false; return 'filter'; } - return /^(if|else|elseif|endif|for|in|endfor|set|and|or|not|contains|true|false|null)$/.test(stream.current()) ? 'keyword' : 'variableName'; + // Keep the website's Markdown punctuation styling around the shared parser. + if (!state.close && !/^\{[{%#]/.test(stream.string.slice(stream.pos))) { + const length = markdownPunctuationAt(stream.string, stream.pos); + if (length) { stream.pos += length; return 'punctuation'; } + stream.next(); + return null; } - stream.next(); - return 'punctuation'; + return knapStreamParser.token(stream, state); }, - tokenTable: { filter: tags.function(tags.variableName) }, }); +export const templateHighlightStyle = HighlightStyle.define([ + { tag: tags.variableName, class: 'syn-variable' }, + { tag: tags.propertyName, class: 'syn-variable' }, + { tag: tags.special(tags.variableName), class: 'syn-filter' }, + { tag: tags.keyword, class: 'syn-keyword' }, + { tag: tags.atom, class: 'syn-keyword' }, + { tag: tags.string, class: 'syn-string' }, + { tag: tags.number, class: 'syn-number' }, + { tag: tags.comment, class: 'syn-comment' }, + { tag: [tags.punctuation, tags.operator], class: 'syn-punctuation' }, +]); + export function createTemplateEditor(variables: () => Record, wrap = false) { const filters: TemplateSuggestion[] = JSON.parse(document.getElementById('playground-filter-completions')!.textContent!); return createPlaygroundEditor('template', [ @@ -75,15 +56,7 @@ export function createTemplateEditor(variables: () => Record, w }); return true; }), - syntaxHighlighting(HighlightStyle.define([ - { tag: tags.variableName, class: 'syn-variable' }, - { tag: tags.function(tags.variableName), class: 'syn-filter' }, - { tag: tags.keyword, class: 'syn-keyword' }, - { tag: tags.string, class: 'syn-string' }, - { tag: tags.number, class: 'syn-number' }, - { tag: tags.comment, class: 'syn-comment' }, - { tag: [tags.punctuation, tags.operator], class: 'syn-punctuation' }, - ])), + syntaxHighlighting(templateHighlightStyle), autocompletion({ icons: false, optionClass: (completion) => {