diff --git a/src/stage_1.rs b/src/stage_1.rs index b808931..5e4563d 100644 --- a/src/stage_1.rs +++ b/src/stage_1.rs @@ -3,7 +3,7 @@ use std::fmt::Write as _; use chumsky::prelude::*; -use chumsky::text::{keyword, Char}; +use chumsky::text::Char; use serde::Serialize; use unicode_categories::UnicodeCategories; @@ -14,6 +14,7 @@ pub enum NorgToken { SingleNewline, Newlines(u16), Regular(char), + Text(String), Special(char), Escape(char), End(char), @@ -29,6 +30,7 @@ impl std::fmt::Display for NorgToken { Self::Newlines(count) => f.write_str(&"\n".repeat(*count as usize)), Self::Regular(c) | Self::Special(c) => f.write_char(*c), Self::SingleNewline => f.write_char('\n'), + Self::Text(s) => f.write_str(s), Self::Whitespace(count) => f.write_str(&" ".repeat(*count as usize)), } } @@ -43,46 +45,108 @@ impl From for String { /// A list of characters which are considered "special", i.e. for parsing of attached modifiers. const SPECIAL_CHARS: &str = "*-~/_!%^,\"'`$:@|=.#+<>()[]{}\\"; -/// List of chars that proceed "end" tags -const TAG_CHARS: &str = "|@="; +/// List of chars that proceed "end" tags are handled inline in the is_tag_char check. + +fn coalesce_regular_chars(mut tokens: Vec) -> Vec { + let mut result = Vec::with_capacity(tokens.len()); + let mut text_buf = String::new(); + for token in tokens.drain(..) { + match token { + NorgToken::Regular(c) => { + text_buf.push(c); + } + other => { + if !text_buf.is_empty() { + result.push(NorgToken::Text(std::mem::take(&mut text_buf))); + } + result.push(other); + } + } + } + if !text_buf.is_empty() { + result.push(NorgToken::Text(text_buf)); + } + result +} /// Parses a `.norg` document and breaks it up into tokens. pub fn stage_1<'src>() -> impl Parser<'src, &'src str, Vec, extra::Err>> { - let ws = any::<_, extra::Err>>() - .filter(|c: &char| c.is_inline_whitespace() || c.is_separator_space()) - .repeated() - .at_least(1) - .collect::() - .map(|s| NorgToken::Whitespace(s.len() as u16)); - - let character = any::<_, extra::Err>>().map(NorgToken::Regular); - - let parse_newline = any::<_, extra::Err>>() - .filter(|c: &char| *c == '\n' || *c == '\r' || c.is_separator_line() || c.is_separator_paragraph()); - - let newline = parse_newline - .to(NorgToken::SingleNewline); - - let newlines = parse_newline - .repeated() - .at_least(2) - .collect::() - .map(|s| NorgToken::Newlines(s.len() as u16)); - - let special = one_of(SPECIAL_CHARS).map(NorgToken::Special); - - let escape = just('\\').ignore_then(any()).map(NorgToken::Escape); - - let tag_end = one_of(TAG_CHARS) - .then_ignore(keyword("end")) - .then_ignore(choice((one_of("\n\r").rewind().map(|_| ()), end()))) - .map(NorgToken::End); - - let tokens = choice((tag_end, escape, special, newlines, newline, ws, character)) - .repeated() - .collect::>(); - - tokens - .then(end().to(NorgToken::Eof)) - .map(|(mut v, eof)| { v.push(eof); v }) -} \ No newline at end of file + custom::<'src, _, &'src str, Vec, extra::Err>>(|inp| { + let is_whitespace = |c: char| c.is_inline_whitespace() || c.is_separator_space(); + let is_newline = + |c: char| c == '\n' || c == '\r' || c.is_separator_line() || c.is_separator_paragraph(); + let is_tag_char = |c: char| matches!(c, '|' | '@' | '='); + + let mut tokens = Vec::new(); + + loop { + let Some(c) = inp.next() else { + tokens.push(NorgToken::Eof); + return Ok(tokens); + }; + + if is_tag_char(c) { + let saved = inp.save(); + + let is_end = inp.peek() == Some('e') + && { inp.next(); inp.peek() == Some('n') } + && { inp.next(); inp.peek() == Some('d') }; + + let mut is_tag_end = false; + if is_end { + inp.next(); + let after = inp.peek(); + is_tag_end = after.map_or(true, |nc| is_newline(nc)); + } + + if is_tag_end { + tokens.push(NorgToken::End(c)); + continue; + } + inp.rewind(saved); + } + + if c == '\\' { + if let Some(escaped) = inp.next() { + tokens.push(NorgToken::Escape(escaped)); + } else { + tokens.push(NorgToken::Special('\\')); + } + continue; + } + + if is_newline(c) { + let mut count = 1u16; + while inp.peek().map_or(false, |nc| is_newline(nc)) { + inp.next(); + count += 1; + } + if count == 1 { + tokens.push(NorgToken::SingleNewline); + } else { + tokens.push(NorgToken::Newlines(count)); + } + continue; + } + + if is_whitespace(c) { + let mut count = 1u16; + while inp.peek().map_or(false, |nc| is_whitespace(nc)) { + inp.next(); + count += 1; + } + tokens.push(NorgToken::Whitespace(count)); + continue; + } + + if SPECIAL_CHARS.contains(c) { + tokens.push(NorgToken::Special(c)); + continue; + } + + // Regular character, will be coalesced into Text runs at the end + tokens.push(NorgToken::Regular(c)); + } + }) + .map(coalesce_regular_chars) +} diff --git a/src/stage_2.rs b/src/stage_2.rs index a9868fb..b6404b0 100644 --- a/src/stage_2.rs +++ b/src/stage_2.rs @@ -45,6 +45,7 @@ fn tokens_to_paragraph_segment(tokens: Vec) -> ParagraphTokenList { } Some(NorgToken::Special(c)) => Some(ParagraphSegmentToken::Special(c)), Some(NorgToken::Escape(c)) => Some(ParagraphSegmentToken::Escape(c)), + Some(NorgToken::Text(s)) => Some(ParagraphSegmentToken::Text(s)), Some(NorgToken::Regular(c)) => { let mut result: String = it .peeking_take_while(|token| matches!(token, NorgToken::Regular(_))) @@ -150,7 +151,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .repeated() .at_least(1) .collect::>() - .separated_by(whitespace.repeated().at_least(1).collect::>()) + .separated_by(whitespace.repeated().at_least(1).ignored()) .collect::>(); let heading = select! { @@ -160,7 +161,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .at_least(1) .collect::>() .map(|chars| chars.len() as u16) - .then_ignore(whitespace.repeated().at_least(1).collect::>()) + .then_ignore(whitespace.repeated().at_least(1).ignored()) .then(extension_section.clone().or_not()) .then(paragraph_segment) .then_ignore(newlines_or_eof) @@ -201,7 +202,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e )) } }) - .then_ignore(whitespace.repeated().at_least(1).collect::>()) + .then_ignore(whitespace.repeated().at_least(1).ignored()) .then(extension_section.clone().or_not()) .map( |((modifier_type, level), extension_section)| NorgBlock::NestableDetachedModifier { @@ -219,7 +220,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .at_most(2) .collect::>() .map(|chars| (chars[0], chars.len() == 2)) - .then_ignore(whitespace.repeated().at_least(1).collect::>()) + .then_ignore(whitespace.repeated().at_least(1).ignored()) .then(extension_section.clone().or_not()) .then(paragraph_segment) .then_ignore(newlines_or_eof) @@ -253,13 +254,13 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e }; let not_tag_end_or_ws = any() - .filter(move |tok: &NorgToken| !matches!(tok, NorgToken::Newlines(_) | NorgToken::SingleNewline | NorgToken::Whitespace(_) | NorgToken::Eof | NorgToken::End(_) if matches!(tok, NorgToken::End(x) if *x == c))); + .filter(move |tok: &NorgToken| !matches!(tok, NorgToken::Newlines(_) | NorgToken::SingleNewline | NorgToken::Whitespace(_) | NorgToken::Eof) && !matches!(tok, NorgToken::End(x) if *x == c)); let tag_parameters = not_tag_end_or_ws .repeated() .at_least(1) .collect::>() - .separated_by(whitespace.repeated().at_least(1).collect::>()) + .separated_by(whitespace.repeated().at_least(1).ignored()) .collect::>(); let verbatim_content = any() @@ -274,7 +275,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e whitespace .repeated() .at_least(1) - .collect::>() + .ignored() .ignore_then(tag_parameters) .or_not(), ) @@ -305,7 +306,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e whitespace .repeated() .at_least(1) - .collect::>() + .ignored() .ignore_then(parameters) .or_not(), ) @@ -332,7 +333,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e whitespace .repeated() .at_least(1) - .collect::>() + .ignored() .ignore_then(parameters) .or_not(), ) @@ -359,10 +360,10 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .then(not_newlines_ws_or_eof.repeated().at_least(1).collect::>()) .then( whitespace - .repeated() - .at_least(1) - .collect::>() - .ignore_then(parameters) + .repeated() + .at_least(1) + .ignored() + .ignore_then(parameters) .or_not(), ) .then_ignore(select! { diff --git a/src/stage_3.rs b/src/stage_3.rs index 38c748b..764f472 100644 --- a/src/stage_3.rs +++ b/src/stage_3.rs @@ -235,19 +235,28 @@ fn paragraph_parser_opener_candidates_and_links<'a>() -> impl Parser< }) } -#[allow(clippy::result_large_err)] fn dedup_opener_candidates(input: Vec) -> Vec { use ParagraphSegment::*; - input - .into_iter() - .coalesce(|prev, next| match (prev.clone(), next.clone()) { + let mut iter = input.into_iter(); + let Some(mut prev) = iter.next() else { + return vec![]; + }; + let mut result = Vec::with_capacity(1); + for next in iter { + match (&prev, &next) { (AttachedModifierOpener(_), AttachedModifierOpener(data)) => { - Err((prev, AttachedModifierOpenerFail(data))) + result.push(prev); + prev = AttachedModifierOpenerFail(data.clone()); } - _ => Err((prev, next)), - }) - .collect() + _ => { + result.push(prev); + prev = next; + } + } + } + result.push(prev); + result } fn paragraph_parser_closer_candidates<'a>( @@ -374,31 +383,6 @@ fn paragraph_rollup_candidates<'a>( choice((attached_modifier, any())).repeated().at_least(1).collect::>() } -fn eliminate_invalid_candidates(input: Vec) -> Vec { - input - .into_iter() - .fold(Vec::new(), |mut acc: Vec, segment| { - match segment { - ParagraphSegment::AttachedModifierCandidate { - modifier_type, - content, - closer, - } => { - acc.push(ParagraphSegment::Token(ParagraphSegmentToken::Special( - modifier_type, - ))); - acc.extend(content); - - if let Some(closer) = closer { - acc.push(*closer); - } - } - _ => acc.push(segment), - }; - - acc - }) -} #[derive(Clone, Hash, Debug, PartialEq, Eq, Serialize)] pub enum LinkTarget { @@ -488,7 +472,21 @@ fn parse_paragraph( .into_result() .unwrap(); - eliminate_invalid_candidates(unravel_candidates(stage3_result)) + use ParagraphSegment::*; + stage3_result + .into_iter() + .fold(Vec::new(), |mut acc, segment| { + match segment { + t @ Token(_) => acc.push(t), + AttachedModifierCloser(c) => acc.push(Token(ParagraphSegmentToken::Special(c))), + AttachedModifierCandidate { modifier_type, content, .. } => { + acc.push(Token(ParagraphSegmentToken::Special(modifier_type))); + acc.extend(content); + } + other => acc.push(other), + }; + acc + }) } #[derive(Clone, Debug, PartialEq, Hash, Eq, Serialize)] diff --git a/src/stage_4.rs b/src/stage_4.rs index 1a8868a..d3474b3 100644 --- a/src/stage_4.rs +++ b/src/stage_4.rs @@ -126,17 +126,17 @@ fn convert(flat: NorgASTFlat) -> NorgAST { } } -fn consume_heading_content(start_level: &u16, flat: &[NorgASTFlat], i: &mut usize) -> Vec { +fn consume_heading_content(start_level: &u16, flat: &[NorgASTFlat], i: usize) -> (Vec, usize) { let mut heading_level = *start_level as i16; let mut content = vec![]; let mut seen = false; - for j in (*i + 1)..flat.len() { + let mut result_i = flat.len(); + for j in (i + 1)..flat.len() { match &flat[j] { NorgASTFlat::Heading { level, .. } => { if level <= start_level { - // stop. - content = stage_4(flat[(*i + 1)..j].to_vec()); - *i = j - 1; + content = stage_4_from(&flat[(i + 1)..j]); + result_i = j - 1; seen = true; break; } else { @@ -146,15 +146,15 @@ fn consume_heading_content(start_level: &u16, flat: &[NorgASTFlat], i: &mut usiz NorgASTFlat::DelimitingModifier(DelimitingModifier::Weak) => { heading_level -= 1; if heading_level < *start_level as i16 { - content = stage_4(flat[(*i + 1)..j].to_vec()); - *i = j; + content = stage_4_from(&flat[(i + 1)..j]); + result_i = j; seen = true; break; } } NorgASTFlat::DelimitingModifier(DelimitingModifier::Strong) => { - content = stage_4(flat[(*i + 1)..j].to_vec()); - *i = j; + content = stage_4_from(&flat[(i + 1)..j]); + result_i = j; seen = true; break; } @@ -163,9 +163,8 @@ fn consume_heading_content(start_level: &u16, flat: &[NorgASTFlat], i: &mut usiz { if let NorgASTFlat::Heading { level, .. } = **next_object { if level <= *start_level { - // stop. - content = stage_4(flat[(*i + 1)..j].to_vec()); - *i = j - 1; + content = stage_4_from(&flat[(i + 1)..j]); + result_i = j - 1; seen = true; break; } else { @@ -179,10 +178,9 @@ fn consume_heading_content(start_level: &u16, flat: &[NorgASTFlat], i: &mut usiz } } if !seen { - content = stage_4(flat[*i + 1..].to_vec()); - *i = flat.len(); + content = stage_4_from(&flat[i + 1..]); } - content + (content, result_i) } /// Loop over the given flat tree from the given index `i` until a non-NestableDetachedModifier is @@ -195,21 +193,16 @@ fn consume_heading_content(start_level: &u16, flat: &[NorgASTFlat], i: &mut usiz fn consume_nestable_detached_mod_content( start_level: &u16, flat: &[NorgASTFlat], - i: &mut usize, + i: usize, list_type: NestableDetachedModifier, -) -> Vec { - let mut content = vec![]; - for j in (*i + 1)..flat.len() { +) -> (Vec, usize) { + for j in (i + 1)..flat.len() { match &flat[j] { NorgASTFlat::NestableDetachedModifier { level, modifier_type, .. } => { if *modifier_type != list_type || level <= start_level { - content = stage_4(flat[(*i + 1)..j].to_vec()); - *i = j - 1; - break; + return (stage_4_from(&flat[(i + 1)..j]), j - 1); } else if j == flat.len() - 1 { - content = stage_4(flat[(*i + 1)..].to_vec()); - *i = j + 1; - break; + return (stage_4_from(&flat[(i + 1)..]), j + 1); } } NorgASTFlat::CarryoverTag { next_object, .. } @@ -217,32 +210,23 @@ fn consume_nestable_detached_mod_content( { if let NorgASTFlat::NestableDetachedModifier { level, modifier_type, .. } = **next_object { if modifier_type != list_type || level <= *start_level { - content = stage_4(flat[(*i + 1)..j].to_vec()); - *i = j - 1; - break; + return (stage_4_from(&flat[(i + 1)..j]), j - 1); } else if j == flat.len() - 1 { - content = stage_4(flat[(*i + 1)..].to_vec()); - *i = j + 1; - break; + return (stage_4_from(&flat[(i + 1)..]), j + 1); } } else { unreachable!() } } _ => { - content = stage_4(flat[(*i + 1)..j].to_vec()); - *i = j - 1; - // stop immediately if we see something that's not a NestableDetachedModifier - // of lesser level - break; + return (stage_4_from(&flat[(i + 1)..j]), j - 1); } } } - - content + (vec![], flat.len()) } -pub fn stage_4(flat: Vec) -> Vec { +fn stage_4_from(flat: &[NorgASTFlat]) -> Vec { let mut ast = vec![]; let mut i = 0; while i < flat.len() { @@ -253,14 +237,15 @@ pub fn stage_4(flat: Vec) -> Vec { title, extensions, } => { - let content = consume_heading_content(start_level, &flat, &mut i); + let (content, new_i) = consume_heading_content(start_level, flat, i); ast.push(NorgAST::Heading { level: *start_level, title: title.to_vec(), extensions: extensions.to_vec(), content, - }) + }); + i = new_i; } NorgASTFlat::CarryoverTag { tag_type, @@ -274,7 +259,7 @@ pub fn stage_4(flat: Vec) -> Vec { title, extensions, } => { - let content = consume_heading_content(&level, &flat, &mut i); + let (content, new_i) = consume_heading_content(&level, flat, i); ast.push(NorgAST::CarryoverTag { tag_type: tag_type.clone(), name: name.to_vec(), @@ -285,7 +270,10 @@ pub fn stage_4(flat: Vec) -> Vec { extensions, content, }), - }) + }); + i = new_i; + i += 1; + continue; } NorgASTFlat::NestableDetachedModifier { modifier_type, @@ -293,8 +281,8 @@ pub fn stage_4(flat: Vec) -> Vec { extensions, content, } => { - let new_content = - consume_nestable_detached_mod_content(&level, &flat, &mut i, modifier_type); + let (new_content, new_i) = + consume_nestable_detached_mod_content(&level, flat, i, modifier_type); ast.push(NorgAST::CarryoverTag { tag_type: tag_type.clone(), name: name.to_vec(), @@ -306,7 +294,10 @@ pub fn stage_4(flat: Vec) -> Vec { text: content, content: new_content, }), - }) + }); + i = new_i; + i += 1; + continue; } _ => { ast.push(convert(item.clone())) @@ -319,7 +310,7 @@ pub fn stage_4(flat: Vec) -> Vec { extensions, content: text, } => { - let content = consume_nestable_detached_mod_content(start_level, &flat, &mut i, *modifier_type); + let (content, new_i) = consume_nestable_detached_mod_content(start_level, flat, i, *modifier_type); let parsed = NorgAST::NestableDetachedModifier { modifier_type: *modifier_type, level: *start_level, @@ -327,6 +318,7 @@ pub fn stage_4(flat: Vec) -> Vec { text: text.clone(), content, }; + i = new_i; if !ast.is_empty() { let len = ast.len(); if let NorgAST::List { modifier_type: list_modifier, ref mut items } = ast[len - 1] { @@ -349,3 +341,7 @@ pub fn stage_4(flat: Vec) -> Vec { ast } + +pub fn stage_4(flat: Vec) -> Vec { + stage_4_from(&flat) +}