Skip to content

perf: rewrite stage_1 lexer with custom(), optimize stages 2-4 #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
NTBBloodbath wants to merge 1 commit into
base: push-tnntnmzouzms
Choose a base branch
from
Open

perf: rewrite stage_1 lexer with custom(), optimize stages 2-4 #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 105 additions & 41 deletions src/stage_1.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use std::fmt::Write as _;

use chumsky::prelude::*;
use chumsky::text::{keyword, Char};
use chumsky::text::Char;
use serde::Serialize;
use unicode_categories::UnicodeCategories;

Expand All @@ -14,6 +14,7 @@ pub enum NorgToken {
SingleNewline,
Newlines(u16),
Regular(char),
Text(String),
Special(char),
Escape(char),
End(char),
Expand All @@ -29,6 +30,7 @@ impl std::fmt::Display for NorgToken {
Self::Newlines(count) => f.write_str(&"\n".repeat(*count as usize)),
Self::Regular(c) | Self::Special(c) => f.write_char(*c),
Self::SingleNewline => f.write_char('\n'),
Self::Text(s) => f.write_str(s),
Self::Whitespace(count) => f.write_str(&" ".repeat(*count as usize)),
}
}
Expand All @@ -43,46 +45,108 @@ impl From<NorgToken> for String {
/// A list of characters which are considered "special", i.e. for parsing of attached modifiers.
const SPECIAL_CHARS: &str = "*-~/_!%^,\"'`$:@|=.#+<>()[]{}\\";

/// List of chars that proceed "end" tags
const TAG_CHARS: &str = "|@=";
/// List of chars that proceed "end" tags are handled inline in the is_tag_char check.

fn coalesce_regular_chars(mut tokens: Vec<NorgToken>) -> Vec<NorgToken> {
let mut result = Vec::with_capacity(tokens.len());
let mut text_buf = String::new();
for token in tokens.drain(..) {
match token {
NorgToken::Regular(c) => {
text_buf.push(c);
}
other => {
if !text_buf.is_empty() {
result.push(NorgToken::Text(std::mem::take(&mut text_buf)));
}
result.push(other);
}
}
}
if !text_buf.is_empty() {
result.push(NorgToken::Text(text_buf));
}
result
}

/// Parses a `.norg` document and breaks it up into tokens.
pub fn stage_1<'src>() -> impl Parser<'src, &'src str, Vec<NorgToken>, extra::Err<Rich<'src, char>>> {
let ws = any::<_, extra::Err<Rich<char>>>()
.filter(|c: &char| c.is_inline_whitespace() || c.is_separator_space())
.repeated()
.at_least(1)
.collect::<String>()
.map(|s| NorgToken::Whitespace(s.len() as u16));

let character = any::<_, extra::Err<Rich<char>>>().map(NorgToken::Regular);

let parse_newline = any::<_, extra::Err<Rich<char>>>()
.filter(|c: &char| *c == '\n' || *c == '\r' || c.is_separator_line() || c.is_separator_paragraph());

let newline = parse_newline
.to(NorgToken::SingleNewline);

let newlines = parse_newline
.repeated()
.at_least(2)
.collect::<String>()
.map(|s| NorgToken::Newlines(s.len() as u16));

let special = one_of(SPECIAL_CHARS).map(NorgToken::Special);

let escape = just('\\').ignore_then(any()).map(NorgToken::Escape);

let tag_end = one_of(TAG_CHARS)
.then_ignore(keyword("end"))
.then_ignore(choice((one_of("\n\r").rewind().map(|_| ()), end())))
.map(NorgToken::End);

let tokens = choice((tag_end, escape, special, newlines, newline, ws, character))
.repeated()
.collect::<Vec<_>>();

tokens
.then(end().to(NorgToken::Eof))
.map(|(mut v, eof)| { v.push(eof); v })
}
custom::<'src, _, &'src str, Vec<NorgToken>, extra::Err<Rich<'src, char>>>(|inp| {
let is_whitespace = |c: char| c.is_inline_whitespace() || c.is_separator_space();
let is_newline =
|c: char| c == '\n' || c == '\r' || c.is_separator_line() || c.is_separator_paragraph();
let is_tag_char = |c: char| matches!(c, '|' | '@' | '=');

let mut tokens = Vec::new();

loop {
let Some(c) = inp.next() else {
tokens.push(NorgToken::Eof);
return Ok(tokens);
};

if is_tag_char(c) {
let saved = inp.save();

let is_end = inp.peek() == Some('e')
&& { inp.next(); inp.peek() == Some('n') }
&& { inp.next(); inp.peek() == Some('d') };

let mut is_tag_end = false;
if is_end {
inp.next();
let after = inp.peek();
is_tag_end = after.map_or(true, |nc| is_newline(nc));
}

if is_tag_end {
tokens.push(NorgToken::End(c));
continue;
}
inp.rewind(saved);
}

if c == '\\' {
if let Some(escaped) = inp.next() {
tokens.push(NorgToken::Escape(escaped));
} else {
tokens.push(NorgToken::Special('\\'));
}
continue;
}

if is_newline(c) {
let mut count = 1u16;
while inp.peek().map_or(false, |nc| is_newline(nc)) {
inp.next();
count += 1;
}
if count == 1 {
tokens.push(NorgToken::SingleNewline);
} else {
tokens.push(NorgToken::Newlines(count));
}
continue;
}

if is_whitespace(c) {
let mut count = 1u16;
while inp.peek().map_or(false, |nc| is_whitespace(nc)) {
inp.next();
count += 1;
}
tokens.push(NorgToken::Whitespace(count));
continue;
}

if SPECIAL_CHARS.contains(c) {
tokens.push(NorgToken::Special(c));
continue;
}

// Regular character, will be coalesced into Text runs at the end
tokens.push(NorgToken::Regular(c));
}
})
.map(coalesce_regular_chars)
}
27 changes: 14 additions & 13 deletions src/stage_2.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ fn tokens_to_paragraph_segment(tokens: Vec<NorgToken>) -> ParagraphTokenList {
}
Some(NorgToken::Special(c)) => Some(ParagraphSegmentToken::Special(c)),
Some(NorgToken::Escape(c)) => Some(ParagraphSegmentToken::Escape(c)),
Some(NorgToken::Text(s)) => Some(ParagraphSegmentToken::Text(s)),
Some(NorgToken::Regular(c)) => {
let mut result: String = it
.peeking_take_while(|token| matches!(token, NorgToken::Regular(_)))
Expand Down Expand Up @@ -150,7 +151,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
.repeated()
.at_least(1)
.collect::<Vec<_>>()
.separated_by(whitespace.repeated().at_least(1).collect::<Vec<_>>())
.separated_by(whitespace.repeated().at_least(1).ignored())
.collect::<Vec<_>>();

let heading = select! {
Expand All @@ -160,7 +161,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
.at_least(1)
.collect::<Vec<_>>()
.map(|chars| chars.len() as u16)
.then_ignore(whitespace.repeated().at_least(1).collect::<Vec<_>>())
.then_ignore(whitespace.repeated().at_least(1).ignored())
.then(extension_section.clone().or_not())
.then(paragraph_segment)
.then_ignore(newlines_or_eof)
Expand Down Expand Up @@ -201,7 +202,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
))
}
})
.then_ignore(whitespace.repeated().at_least(1).collect::<Vec<_>>())
.then_ignore(whitespace.repeated().at_least(1).ignored())
.then(extension_section.clone().or_not())
.map(
|((modifier_type, level), extension_section)| NorgBlock::NestableDetachedModifier {
Expand All @@ -219,7 +220,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
.at_most(2)
.collect::<Vec<_>>()
.map(|chars| (chars[0], chars.len() == 2))
.then_ignore(whitespace.repeated().at_least(1).collect::<Vec<_>>())
.then_ignore(whitespace.repeated().at_least(1).ignored())
.then(extension_section.clone().or_not())
.then(paragraph_segment)
.then_ignore(newlines_or_eof)
Expand Down Expand Up @@ -253,13 +254,13 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
};

let not_tag_end_or_ws = any()
.filter(move |tok: &NorgToken| !matches!(tok, NorgToken::Newlines(_) | NorgToken::SingleNewline | NorgToken::Whitespace(_) | NorgToken::Eof | NorgToken::End(_) if matches!(tok, NorgToken::End(x) if *x == c)));
.filter(move |tok: &NorgToken| !matches!(tok, NorgToken::Newlines(_) | NorgToken::SingleNewline | NorgToken::Whitespace(_) | NorgToken::Eof) && !matches!(tok, NorgToken::End(x) if *x == c));

let tag_parameters = not_tag_end_or_ws
.repeated()
.at_least(1)
.collect::<Vec<_>>()
.separated_by(whitespace.repeated().at_least(1).collect::<Vec<_>>())
.separated_by(whitespace.repeated().at_least(1).ignored())
.collect::<Vec<_>>();

let verbatim_content = any()
Expand All @@ -274,7 +275,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
whitespace
.repeated()
.at_least(1)
.collect::<Vec<_>>()
.ignored()
.ignore_then(tag_parameters)
.or_not(),
)
Expand Down Expand Up @@ -305,7 +306,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
whitespace
.repeated()
.at_least(1)
.collect::<Vec<_>>()
.ignored()
.ignore_then(parameters)
.or_not(),
)
Expand All @@ -332,7 +333,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
whitespace
.repeated()
.at_least(1)
.collect::<Vec<_>>()
.ignored()
.ignore_then(parameters)
.or_not(),
)
Expand All @@ -359,10 +360,10 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec<NorgBlock>, e
.then(not_newlines_ws_or_eof.repeated().at_least(1).collect::<Vec<_>>())
.then(
whitespace
.repeated()
.at_least(1)
.collect::<Vec<_>>()
.ignore_then(parameters)
.repeated()
.at_least(1)
.ignored()
.ignore_then(parameters)
.or_not(),
)
.then_ignore(select! {
Expand Down
66 changes: 32 additions & 34 deletions src/stage_3.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,19 +235,28 @@ fn paragraph_parser_opener_candidates_and_links<'a>() -> impl Parser<
})
}

#[allow(clippy::result_large_err)]
fn dedup_opener_candidates(input: Vec<ParagraphSegment>) -> Vec<ParagraphSegment> {
use ParagraphSegment::*;

input
.into_iter()
.coalesce(|prev, next| match (prev.clone(), next.clone()) {
let mut iter = input.into_iter();
let Some(mut prev) = iter.next() else {
return vec![];
};
let mut result = Vec::with_capacity(1);
for next in iter {
match (&prev, &next) {
(AttachedModifierOpener(_), AttachedModifierOpener(data)) => {
Err((prev, AttachedModifierOpenerFail(data)))
result.push(prev);
prev = AttachedModifierOpenerFail(data.clone());
}
_ => Err((prev, next)),
})
.collect()
_ => {
result.push(prev);
prev = next;
}
}
}
result.push(prev);
result
}

fn paragraph_parser_closer_candidates<'a>(
Expand Down Expand Up @@ -374,31 +383,6 @@ fn paragraph_rollup_candidates<'a>(
choice((attached_modifier, any())).repeated().at_least(1).collect::<Vec<_>>()
}

fn eliminate_invalid_candidates(input: Vec<ParagraphSegment>) -> Vec<ParagraphSegment> {
input
.into_iter()
.fold(Vec::new(), |mut acc: Vec<ParagraphSegment>, segment| {
match segment {
ParagraphSegment::AttachedModifierCandidate {
modifier_type,
content,
closer,
} => {
acc.push(ParagraphSegment::Token(ParagraphSegmentToken::Special(
modifier_type,
)));
acc.extend(content);

if let Some(closer) = closer {
acc.push(*closer);
}
}
_ => acc.push(segment),
};

acc
})
}

#[derive(Clone, Hash, Debug, PartialEq, Eq, Serialize)]
pub enum LinkTarget {
Expand Down Expand Up @@ -488,7 +472,21 @@ fn parse_paragraph(
.into_result()
.unwrap();

eliminate_invalid_candidates(unravel_candidates(stage3_result))
use ParagraphSegment::*;
stage3_result
.into_iter()
.fold(Vec::new(), |mut acc, segment| {
match segment {
t @ Token(_) => acc.push(t),
AttachedModifierCloser(c) => acc.push(Token(ParagraphSegmentToken::Special(c))),
AttachedModifierCandidate { modifier_type, content, .. } => {
acc.push(Token(ParagraphSegmentToken::Special(modifier_type)));
acc.extend(content);
}
other => acc.push(other),
};
acc
})
}

#[derive(Clone, Debug, PartialEq, Hash, Eq, Serialize)]
Expand Down
Loading