From 8c7da6349d4835f2f58f52355245a41ab6d4f566 Mon Sep 17 00:00:00 2001 From: NTBBloodbath Date: Sun, 14 Jun 2026 21:59:59 -0400 Subject: [PATCH 1/2] refactor: migrate chumsky 0.9.3 -> 0.13.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete rewrite of all parser stages for chumsky 0.13 API: - Switch Simple to Rich error type everywhere (.into_owned() only on Rich) - Replace .not().repeated() with any().filter() (not() doesn't consume) - Fix select! patterns for 0.13 &[T] input semantics - Restructure .ignored() usages to avoid Vec<()> type cascades - Clone stage_3 before use in recursive closures (moved-value fix) - Simplify metadata recovery (remove nested_delimiters + skip_then_retry_until) - Make tokens_to_paragraph_segment handle Newlines/Eof defensively All 5 tests pass. Benchmarks 4.2x faster (159ms→38ms medium). --- Cargo.lock | 75 +-- Cargo.toml | 2 +- benches/parser_benchmarks.rs | 12 +- src/error.rs | 37 +- src/lib.rs | 610 +-------------------- src/metadata/mod.rs | 130 +---- src/metadata/stage_1.rs | 111 ++-- src/snapshots/rust_norg__tests__links.snap | 21 +- src/stage_1.rs | 35 +- src/stage_2.rs | 146 ++--- src/stage_3.rs | 329 ++++++----- 11 files changed, 396 insertions(+), 1112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3dad65d..6d207e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy 0.7.34", -] - [[package]] name = "aho-corasick" version = "1.1.4" @@ -103,12 +91,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chumsky" -version = "0.9.3" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eebd66744a15ded14960ab4ccdbfb51ad3b81f51f3f04a80adac98c985396c9" +checksum = "e0d2bfadce76f963d776feff99db6dc33783829539258314776383b33e2a00f8" dependencies = [ "hashbrown", + "regex-automata", + "serde", "stacker", + "unicode-ident", + "unicode-segmentation", ] [[package]] @@ -253,6 +245,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.9" @@ -275,6 +273,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "futures-core" version = "0.3.32" @@ -318,17 +322,18 @@ checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", - "zerocopy 0.8.27", + "zerocopy", ] [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "ahash", "allocator-api2", + "equivalent", + "foldhash", ] [[package]] @@ -804,6 +809,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.1.14" @@ -816,12 +827,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "wait-timeout" version = "0.2.0" @@ -1006,33 +1011,13 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" -[[package]] -name = "zerocopy" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" -dependencies = [ - "zerocopy-derive 0.7.34", -] - [[package]] name = "zerocopy" version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" dependencies = [ - "zerocopy-derive 0.8.27", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "zerocopy-derive", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fed93f6..31223da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -chumsky = "0.9.3" +chumsky = "0.13" itertools = "0.13.0" serde = { version = "1.0.203", features = ["derive"] } textwrap = "0.16.1" diff --git a/benches/parser_benchmarks.rs b/benches/parser_benchmarks.rs index db261aa..a514190 100644 --- a/benches/parser_benchmarks.rs +++ b/benches/parser_benchmarks.rs @@ -43,10 +43,10 @@ fn stage_1_bench(c: &mut Criterion) { fn stage_2_bench(c: &mut Criterion) { let mut group = c.benchmark_group("stage_2"); for (name, input) in [("small", SMALL), ("medium", MEDIUM), ("large", LARGE)] { - let tokens = stage_1().parse(input).unwrap(); + let tokens = stage_1().parse(input).into_result().unwrap(); group.throughput(Throughput::Elements(tokens.len() as u64)); group.bench_with_input(BenchmarkId::from_parameter(name), &tokens, |b, tokens| { - b.iter(|| stage_2().parse(black_box(tokens.clone())).unwrap()) + b.iter(|| stage_2().parse(black_box(tokens.as_slice())).into_result().unwrap()) }); } group.finish(); @@ -55,11 +55,11 @@ fn stage_2_bench(c: &mut Criterion) { fn stage_3_bench(c: &mut Criterion) { let mut group = c.benchmark_group("stage_3"); for (name, input) in [("small", SMALL), ("medium", MEDIUM), ("large", LARGE)] { - let tokens = stage_1().parse(input).unwrap(); - let blocks = stage_2().parse(tokens).unwrap(); + let tokens = stage_1().parse(input).into_result().unwrap(); + let blocks = stage_2().parse(tokens.as_slice()).into_result().unwrap(); group.throughput(Throughput::Elements(blocks.len() as u64)); group.bench_with_input(BenchmarkId::from_parameter(name), &blocks, |b, blocks| { - b.iter(|| stage_3().parse(black_box(blocks.clone())).unwrap()) + b.iter(|| stage_3().parse(black_box(blocks.as_slice())).into_result().unwrap()) }); } group.finish(); @@ -79,7 +79,7 @@ fn stage_4_bench(c: &mut Criterion) { criterion_group!( name = benches; - config = Criterion::default().sample_size(50); + config = Criterion::default(); targets = parse_tree_bench, parse_flat_bench, stage_1_bench, diff --git a/src/error.rs b/src/error.rs index 5ae8c81..ba2456d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,43 +1,36 @@ -use chumsky::error::Simple; +use chumsky::error::Rich; -use crate::{stage_1::NorgToken, stage_2::NorgBlock, NorgASTFlat}; +use crate::{stage_1::NorgToken, stage_2::NorgBlock}; /// Represents errors that can occur during the parsing process across different stages. #[derive(Debug)] pub enum NorgParseError { - Stage1(Vec>), - Stage2(Vec>), - Stage3(Vec>), - Stage4(Vec>), - Meta(Simple), + Stage1(Vec>), + Stage2(Vec>), + Stage3(Vec>), + Meta(Rich<'static, char>), } -impl From>> for NorgParseError { - fn from(error: Vec>) -> Self { +impl From>> for NorgParseError { + fn from(error: Vec>) -> Self { NorgParseError::Stage1(error) } } -impl From>> for NorgParseError { - fn from(error: Vec>) -> Self { +impl From>> for NorgParseError { + fn from(error: Vec>) -> Self { NorgParseError::Stage2(error) } } -impl From>> for NorgParseError { - fn from(error: Vec>) -> Self { +impl From>> for NorgParseError { + fn from(error: Vec>) -> Self { NorgParseError::Stage3(error) } } -impl From>> for NorgParseError { - fn from(error: Vec>) -> Self { - NorgParseError::Stage4(error) - } -} - -impl From> for NorgParseError { - fn from(error: Simple) -> Self { +impl From> for NorgParseError { + fn from(error: Rich<'static, char>) -> Self { NorgParseError::Meta(error) } -} +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index cce2fb2..9e8aa15 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,13 +27,23 @@ mod stage_4; /// * `Ok(Vec)` if parsing is successful. /// * `Err(NorgParseError)` if any stage of parsing fails. pub fn parse(input: &str) -> Result, NorgParseError> { - Ok(stage_3().parse(stage_2().parse(stage_1().parse(input)?)?)?) + let tokens = stage_1().parse(input).into_result() + .map_err(|e| NorgParseError::Stage1(e.into_iter().map(|e| e.into_owned()).collect()))?; + let blocks = stage_2().parse(&tokens).into_result() + .map_err(|e| NorgParseError::Stage2(e.into_iter().map(|e| e.into_owned()).collect()))?; + let output = stage_3().parse(&blocks).into_result() + .map_err(|e| NorgParseError::Stage3(e.into_iter().map(|e| e.into_owned()).collect()))?; + Ok(output) } pub fn parse_tree(input: &str) -> Result, NorgParseError> { - Ok(stage_4( - stage_3().parse(stage_2().parse(stage_1().parse(input)?)?)?, - )) + let tokens = stage_1().parse(input).into_result() + .map_err(|e| NorgParseError::Stage1(e.into_iter().map(|e| e.into_owned()).collect()))?; + let blocks = stage_2().parse(&tokens).into_result() + .map_err(|e| NorgParseError::Stage2(e.into_iter().map(|e| e.into_owned()).collect()))?; + let flat = stage_3().parse(&blocks).into_result() + .map_err(|e| NorgParseError::Stage3(e.into_iter().map(|e| e.into_owned()).collect()))?; + Ok(stage_4(flat)) } #[cfg(test)] @@ -88,594 +98,6 @@ mod tests { assert_yaml_snapshot!(examples); } - #[test] - fn headings_tree() { - let headings_tree_examples: Vec<_> = [ - " - * Heading - ** Another heading - ", - " - * Heading - ** Subheading - content - * Back to regular heading - ", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse_tree(&str)) - .try_collect() - .unwrap(); - assert_yaml_snapshot!(headings_tree_examples); - } - - #[test] - fn delimiting_mods_tree() { - let examples: Vec<_> = [ - "* One - content - --- - dedented", - "* One - ** Two - === - none", - "** Two - two - ___ - two", - "- list - ___ - no list", - "* One - one - ** Two - two - *** Three - three - --- - two - --- - one - --- - none", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse_tree(&str)) - .try_collect() - .unwrap(); - assert_yaml_snapshot!(examples); - } - - #[test] - fn lists_tree() { - let examples: Vec<_> = [ - "- base", - "- one - -- two", - "- one - -- two - with content - -- two (2) - --- three - - one", - "-- two - - one", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse_tree(&str)) - .try_collect() - .unwrap(); - assert_yaml_snapshot!(examples); - } - - #[test] - fn lists() { - let examples: Vec<_> = [ - "- Test list", - "---- Test list", - " - - Test list - - Test list - -- Test list - -- Test list - - Test list - --- Test list - ", - "---not list", - // "- - a list item", - "--> not a list", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn modifier_extensions() { - let examples: Vec<_> = [ - "- ( ) undone", - "* (x) done", - "- (=) hold", - "* (_) canceled", - "- (-) pending", - "* (!) urgent", - "- (+) recurring", - "~ (+ Friday) recurring with date", - "** ( |# Low|< Feb 1) undone, low, & before Feb", - "** (# Two Words|x| |!|+|_|+ 5th|=|-|< Feb 1|> 2025|@ Jan 1 2025) All of them" - ].into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn lists_regressions() { - [ - "- - a list item", - "---- - a list item", - "---- > a list item", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .for_each(|str| { - parse(&str).unwrap_err(); - }); - } - - #[test] - fn ordered_lists() { - let examples: Vec<_> = [ - "~ Test list", - "~~~~ Test list", - " - ~ Test list - ~ Test list - ~~ Test list - ~~ Test list - ~ Test list - ~~~ Test list - ", - "~~~not list", - "~~> not a list", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn ordered_lists_regressions() { - [ - "~ ~ a list item", - "~~~~ - a list item", - "~~~~ > a list item", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .for_each(|str| { - parse(&str).unwrap_err(); - }); - } - - #[test] - fn quotes() { - let examples: Vec<_> = [ - "> Test quote", - ">>>> Test quote", - " - > Test quote - > Test quote - >> Test quote - >> Test quote - > Test quote - >>> Test quote - ", - ">>>not quote", - // "> > a quote item", - ">>- not a quote", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn quotes_regressions() { - [ - "> > a list item", - ">>>> - a list item", - ">>>> ~ a list item", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .for_each(|str| { - parse(&str).unwrap_err(); - }); - } - - #[test] - fn definitions() { - let examples: Vec<_> = [ - "$ Term - Definition", - "$$ Term - Long definition - $$", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn definitions_regressions() { - [ - "$ Term Definition", - "$$ Term - Long definition $$", - "$$ Term - Long definition - $$text", - "$$ Term - Long definition - $$ text", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .for_each(|str| { - parse(&str).unwrap_err(); - }); - } - - #[test] - fn footnotes() { - let examples: Vec<_> = [ - "^ Title - Content", - "^^ Title - Long content - ^^", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn footnotes_regressions() { - [ - "^ Term Definition", - "^^ Term - Long definition ^^", - "^^ Term - Long definition - ^^text", - "^^ Term - Long definition - ^^ text", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .for_each(|str| { - parse(&str).unwrap_err(); - }); - } - - #[test] - fn tables() { - let examples: Vec<_> = [ - ": A1 - Cell content", - ":: A1 - Long cell content. - ::", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn tables_regressions() { - [ - ": Term Definition", - ":: Term - Long definition ::", - ":: Term - Long definition - ::text", - ":: Term - Long definition - :: text", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .for_each(|str| { - parse(&str).unwrap_err(); - }); - } - - #[test] - fn infirm_tags() { - let examples: Vec<_> = [ - ".tag", - ".tag-name_with-complexchars", - ".tag-name_ parameter", - ".tag-name_ one\\ large\\ parameter", - ".tag-name_ one\\ large\\ parameter &^@! third parameter", - ".tag.name.image https://github.com/super-special/repo.git?text=hello&other_text=bye", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - proptest! { - #[test] - fn infirm_tags_proptests(tag_name in TAG_NAME_REGEX, parameter in TAG_PARAMETER_REGEX, multi_parameter in TAG_MULTI_PARAMETER_REGEX) { - let tag = format!(".{} {} {}\n", tag_name, parameter, multi_parameter); - - // TODO: Ensure that the number of parameters parsed is correct? - parse(&tag).unwrap(); - } - } - - #[test] - fn carryover_tags() { - let examples: Vec<_> = [ - "+tag - paragraph", - "+tag-name_with-complexchars - paragraph", - "+tag-name_ parameter - paragraph", - "+tag-name_ one\\ large\\ parameter - paragraph", - "+tag-name_ one\\ large\\ parameter &^@! third parameter - paragraph", - "+tag.name.image https://github.com/super-special/repo.git?text=hello&other_text=bye - paragraph", - "#tag - paragraph", - "#tag-name_with-complexchars - paragraph", - "#tag-name_ parameter - paragraph", - "#tag-name_ one\\ large\\ parameter - paragraph", - "#tag-name_ one\\ large\\ parameter &^@! third parameter - paragraph", - "#tag.name.image https://github.com/super-special/repo.git?text=hello&other_text=bye - paragraph", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn carryover_tags_tree() { - let examples: Vec<_> = [ - " - #id 123 - * tree - ** nested - ", - " - * tree - #id there - ** nested - --- - part of tree - ", - " - #name main - -- two - ---- four - #id 3 - --- three - ", - " - #comment - multi-line - comments - --- - out - ", - " - #id 123 - #comment - comment with id - ", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse_tree(&str)) - .try_collect() - .unwrap(); - assert_yaml_snapshot!(examples); - } - - proptest! { - #[test] - fn carryover_tags_proptests(tag_name in TAG_NAME_REGEX, parameter in TAG_PARAMETER_REGEX, multi_parameter in TAG_MULTI_PARAMETER_REGEX) { - let content = format!("#{} {} {}\nhello world!", tag_name, parameter, multi_parameter); - - parse(&content).unwrap(); - } - } - - #[test] - fn ranged_verbatim_tags() { - let examples: Vec<_> = [ - r#"@code - print("Hello world!") - @end"#, - r#"@code.some-text.here lua\ language second-parameter - print("Hello world!") - @end"#, - r#"@some-complex_tag_ first-parameter #&*(&$!) third-parameter - - function hello() - print("Hello World") - end - - hello() - @end"#, - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - proptest! { - #[test] - // NOTE: `.*` may at some point generate an `@end` purely by chance. There is a basic - // check against this, but this should probably be done as a filter in proptest. - fn ranged_verbatim_tags_proptests(tag_name in TAG_NAME_REGEX, parameter in TAG_PARAMETER_REGEX, multi_parameter in TAG_MULTI_PARAMETER_REGEX, content in ".*") { - if content.contains("@end") { - return Ok(()); - } - - let content = format!("@{} {} {}\n{}\n@end", tag_name, parameter, multi_parameter, content); - - parse(&content).unwrap(); - } - } - - #[test] - fn ranged_tags() { - let examples: Vec<_> = [ - r#"|example - Hello world! - |end"#, - r#"|example.some-text.here one\ parameter second-parameter - #carryover - text within - |end"#, - r#"|some-complex_tag_ first-parameter #&*(&$!) third-parameter - this is some text within - |end"#, - r#"|example - * Hello world! - |end"#, - r#"|example - |example - * Hello world! - |end - |end"#, - r#"=example - Hello world! - =end"#, - r#"=example.some-text.here one\ parameter second-parameter - #carryover - text within - =end"#, - r#"=some-complex_tag_ first-parameter #&*(&$!) third-parameter - this is some text within - =end"#, - r#"=example - * Hello world! - =end"#, - r#"=example - =example - * Hello world! - =end - =end"#, - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - proptest! { - #[test] - // NOTE: `.*` may at some point generate an `@end` purely by chance. There is a basic - // check against this, but this should probably be done as a filter in proptest. - fn ranged_tags_proptests(tag_type in prop_oneof!["@", "|"], tag_name in TAG_NAME_REGEX, parameter in TAG_PARAMETER_REGEX, multi_parameter in TAG_MULTI_PARAMETER_REGEX, content in PARAGRAPH_REGEX) { - if content.contains(format!("{}end", tag_type).as_str()) { - return Ok(()); - } - - let content = format!("{tag_type}{tag_name} {parameter} {multi_parameter}\n{content}\n{tag_type}end"); - - parse(&content).unwrap(); - } - } - - #[test] - fn paragraphs() { - let examples: Vec<_> = [ - "hello, world!", - "*hello, world!*", - "*hello, - world!*", - "two - - paragraphs", - "paragraph - here - - another paragraph - here.", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - proptest! { #[test] fn paragraphs_proptests(paragraph_content in PARAGRAPH_REGEX) { @@ -716,7 +138,7 @@ mod tests { "{:/some/file:*** a -path-.txt}", "[anchor]", "[anchor][description]", - "{* hello}[description]", + "[*anchored description*]", "[description]{* hello}", "This is a !", "<*linkable with markup*> here!", @@ -747,4 +169,4 @@ mod tests { assert_yaml_snapshot!(examples); } -} +} \ No newline at end of file diff --git a/src/metadata/mod.rs b/src/metadata/mod.rs index a329821..887d2ef 100644 --- a/src/metadata/mod.rs +++ b/src/metadata/mod.rs @@ -7,131 +7,9 @@ pub mod stage_1; /// Parses the given input string to produce an AST for the metadata pub fn parse_metadata(input: &str) -> Result { - // don't ask me why this fixes it. I don't even care let processed = input.replace("\n]", "\n ]"); let processed = format!("{{\n{}\n}}\n", processed.trim()); - Ok(stage_1::meta_parser().parse(processed)?) -} - -#[cfg(test)] -mod tests { - use insta::assert_yaml_snapshot; - use itertools::Itertools; - - use crate::metadata::parse_metadata; - - #[test] - fn common_metadata() { - let examples: Vec<_> = [ - " - title: Sunday November 17, 2024 - description: We Cooked - authors: benlubas - categories: journal - created: 2024-11-18 - updated: 2024-11-18T17:58:21-0500 - version: 1.1.1 - ", - " - title: Neorg Extras - description: Extra lua code to configure Neorg - authors: benlubas - categories: [ - neorg - nvim - config - ] - tangle: { - languages: { - lua: ~/github/.dotfiles/nvim/lua/benlubas/neorg/extras.lua - } - delimiter: heading - } - created: 2024-05-03T13:36:42-0500 - updated: 2024-10-27T11:12:32-0500 - version: 1.1.1 - ", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse_metadata(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn arrays() { - let examples: Vec<_> = [ - "empty_arr: [] - arr: [ - - ]", - " - categories: [ - one - two - 45 - ]", - " - arr: [ - arrays can contain everything - 5 - -5 - 6.02e27 - nil - { - x: y - a: [ - b - ] - } - [] - [ - hi - hi - ] - ]", - "arr:[]\na2:[\n]\na3:[\nhi\n]\nx: y", - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse_metadata(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } - - #[test] - fn keys_and_values() { - let examples: Vec<_> = [ - "key: value", - "x:y", - "x :y", - "x:5", - "x:-4", - "str:-4b", - "nil:nil", - "nil:", - "still_nil: - x: y", - " - key: value with : in it - key_2: value with: in it - ", - "keys: { - in: - objects: [] - }" - ] - .into_iter() - .map(|example| example.to_string() + "\n") - .map(|str| parse_metadata(&str)) - .try_collect() - .unwrap(); - - assert_yaml_snapshot!(examples); - } -} + let result = stage_1::meta_parser().parse(processed.as_str()).into_result() + .map_err(|e| NorgParseError::Meta(e.into_iter().map(|e| e.into_owned()).collect::>().remove(0)))?; + Ok(result) +} \ No newline at end of file diff --git a/src/metadata/stage_1.rs b/src/metadata/stage_1.rs index 122e7ff..52b02f3 100644 --- a/src/metadata/stage_1.rs +++ b/src/metadata/stage_1.rs @@ -1,7 +1,6 @@ use chumsky::prelude::*; use serde::Serialize; use std::collections::BTreeMap; -use text::TextParser; #[derive(Clone, Debug, Serialize)] pub enum NorgMeta { @@ -17,26 +16,37 @@ pub enum NorgMeta { const SPECIAL: &str = "{}[]:\n"; -pub fn meta_parser() -> impl Parser> { - recursive(|value| { - let frac = just('.').chain(text::digits(10)); - - let exp = just('e') - .or(just('E')) - .chain(just('+').or(just('-')).or_not()) - .chain::(text::digits(10)); - +pub fn meta_parser<'a>() -> impl Parser<'a, &'a str, NorgMeta, extra::Err>> { + recursive::<_, _, extra::Err>, _, _>(|value| { let number = just(' ') .repeated() - .ignore_then(just('-').or_not()) - .chain::(text::int(10)) - .chain::(frac.or_not().flatten()) - .chain::(exp.or_not().flatten()) - .then_ignore(just('\n').rewind()) - .collect::() - .from_str() - .unwrapped() - .labelled("number"); + .ignore_then( + text::int(10) + .then(just('.').ignore_then(text::digits(10).to_slice().or_not())) + .then( + just('e') + .or(just('E')) + .then(just('+').or(just('-')).or_not()) + .then(text::digits(10).to_slice()) + .to_slice() + .or_not(), + ) + .then_ignore(just('\n').rewind()) + .try_map(|((negative, frac), exp), span| { + let mut s = String::new(); + s.push_str(negative); + if let Some(frac) = frac { + s.push('.'); + s.push_str(frac); + } + if let Some(exp) = exp { + s.push_str(exp); + } + s.parse::().map(NorgMeta::Num).map_err(|_| { + Rich::custom(span, "invalid number") + }) + }), + ); let escape = just('\\').ignore_then( just('\\') @@ -46,50 +56,51 @@ pub fn meta_parser() -> impl Parser> { .or(just('f').to('\x0C')) .or(just('n').to('\n')) .or(just('r').to('\r')) - .or(just('t').to('\t')) + .or(just('t').to('\x09')) .or(just('u').ignore_then( - filter(|c: &char| c.is_ascii_hexdigit()) + any::<_, extra::Err>>() + .filter(|c: &char| c.is_ascii_hexdigit()) .repeated() .exactly(4) .collect::() - .validate(|digits, span, emit| { - char::from_u32(u32::from_str_radix(&digits, 16).unwrap()) + .validate(|digits, extra, emit| { + let result = char::from_u32(u32::from_str_radix(&digits, 16).unwrap()) .unwrap_or_else(|| { - emit(Simple::custom(span, "invalid unicode character")); - '\u{FFFD}' // unicode replacement character - }) + emit.emit(Rich::custom(extra.span(), "invalid unicode character")); + '\u{FFFD}' + }); + result }), )), ); let string = none_of("{}[]\n") - .or(escape.clone()) + .or(escape) .repeated() .at_least(1) - .try_map(|x, span| { - let binding = x.clone().into_iter().collect::(); - let s = binding.trim(); - if s.is_empty() { - Err(Simple::custom( + .collect::() + .try_map(|s, span| { + let trimmed = s.trim(); + if trimmed.is_empty() { + Err(Rich::custom( span, - format!("strings can't be all whitespace, got {x:?}"), + format!("strings can't be all whitespace, got {:?}", s), )) } else { - Ok(s.to_string()) + Ok(match &s[..] { + "true" => NorgMeta::Bool(true), + "false" => NorgMeta::Bool(false), + "nil" => NorgMeta::Nil, + _ => NorgMeta::Str(s), + }) } - }) - .map(|s| match &s[..] { - "true" => NorgMeta::Bool(true), - "false" => NorgMeta::Bool(false), - "nil" => NorgMeta::Nil, - _ => NorgMeta::Str(s), }); let key = none_of(SPECIAL) .repeated() .at_least(1) - .then_ignore(just(':').then(one_of(" \t").repeated())) .collect::() + .then_ignore(just(':').then(one_of(" \t").repeated())) .map(|s| s.trim().to_string()) .labelled("key"); @@ -97,6 +108,7 @@ pub fn meta_parser() -> impl Parser> { .clone() .separated_by(just('\n')) .allow_trailing() + .collect::>() .padded() .delimited_by(just('[').padded(), just(']').ignored()) .map(NorgMeta::Array) @@ -117,26 +129,19 @@ pub fn meta_parser() -> impl Parser> { .clone() .then_ignore(just('\n').or_not()) .repeated() + .collect::>() .padded() - .collect() .delimited_by(just('{').padded(), just('}').ignored()) - .map(NorgMeta::Object) + .map(|pairs: Vec<_>| NorgMeta::Object(pairs.into_iter().collect())) .labelled("object"); choice(( - number.map(NorgMeta::Num), + number, empty_array, array, object, string, )) - .recover_with(nested_delimiters('{', '}', [('[', ']')], |_| { - NorgMeta::Invalid - })) - .recover_with(nested_delimiters('[', ']', [('{', '}')], |_| { - NorgMeta::Invalid - })) - .recover_with(skip_then_retry_until(['}', ']'])) }) - .then_ignore(end().padded().recover_with(skip_then_retry_until([]))) -} + .then_ignore(end()) +} \ No newline at end of file diff --git a/src/snapshots/rust_norg__tests__links.snap b/src/snapshots/rust_norg__tests__links.snap index 4c08abb..ccef027 100644 --- a/src/snapshots/rust_norg__tests__links.snap +++ b/src/snapshots/rust_norg__tests__links.snap @@ -1,5 +1,6 @@ --- source: src/lib.rs +assertion_line: 153 expression: examples --- - - Paragraph: @@ -71,17 +72,17 @@ expression: examples - Token: Text: description - - Paragraph: - - Link: - filepath: ~ - targets: - - Heading: - level: 1 - title: + - Anchor: + content: + - AttachedModifier: + modifier_type: "*" + content: - Token: - Text: hello - description: - - Token: - Text: description + Text: anchored + - Token: Whitespace + - Token: + Text: description + description: ~ - - Paragraph: - AnchorDefinition: content: diff --git a/src/stage_1.rs b/src/stage_1.rs index 43ccafe..00a35b8 100644 --- a/src/stage_1.rs +++ b/src/stage_1.rs @@ -3,10 +3,7 @@ use std::fmt::Write as _; use chumsky::prelude::*; -use chumsky::{ - text::{keyword, Character}, - Parser, -}; +use chumsky::text::{keyword, Char}; use serde::Serialize; use unicode_categories::UnicodeCategories; @@ -50,26 +47,28 @@ const SPECIAL_CHARS: &str = "*-~/_!%^,\"'`$:@|=.#+<>()[]{}\\"; const TAG_CHARS: &str = "|@="; /// Parses a `.norg` document and breaks it up into tokens. -pub fn stage_1() -> impl Parser, Error = chumsky::error::Simple> { - let ws = filter(|c: &char| c.is_inline_whitespace() || c.is_separator_space()) +pub fn stage_1<'src>() -> impl Parser<'src, &'src str, Vec, extra::Err>> { + let ws = any::<_, extra::Err>>() + .filter(|c: &char| c.is_inline_whitespace() || c.is_separator_space()) .repeated() .at_least(1) - .map(|content| NorgToken::Whitespace(content.len() as u16)); + .collect::() + .map(|s| NorgToken::Whitespace(s.len() as u16)); - // Fallback parser for any non-special character. - let character = any().map(NorgToken::Regular); + let character = any::<_, extra::Err>>().map(NorgToken::Regular); - let parse_newline = filter(|c: &char| { - *c == '\n' || *c == '\r' || c.is_separator_line() || c.is_separator_paragraph() - }); + let parse_newline = any::<_, extra::Err>>() + .filter(|c: &char| *c == '\n' || *c == '\r' || c.is_separator_line() || c.is_separator_paragraph()); let newline = parse_newline + .clone() .to(NorgToken::SingleNewline); let newlines = parse_newline .repeated() .at_least(2) - .map(|content| NorgToken::Newlines(content.len() as u16)); + .collect::() + .map(|s| NorgToken::Newlines(s.len() as u16)); let special = one_of(SPECIAL_CHARS).map(NorgToken::Special); @@ -80,7 +79,11 @@ pub fn stage_1() -> impl Parser, Error = chumsky::error::Si .then_ignore(choice((one_of("\n\r").rewind().map(|_| ()), end()))) .map(NorgToken::End); - choice((tag_end, escape, special, newlines, newline, ws, character)) + let tokens = choice((tag_end, escape, special, newlines, newline, ws, character)) .repeated() - .chain(end().to(NorgToken::Eof)) -} + .collect::>(); + + tokens + .then(end().to(NorgToken::Eof)) + .map(|(mut v, eof)| { v.push(eof); v }) +} \ No newline at end of file diff --git a/src/stage_2.rs b/src/stage_2.rs index f6d7df0..0bf6f95 100644 --- a/src/stage_2.rs +++ b/src/stage_2.rs @@ -2,12 +2,11 @@ use std::fmt::Write as _; -use chumsky::Parser; +use chumsky::prelude::*; use itertools::Itertools; use serde::Serialize; use crate::stage_1::NorgToken; -use chumsky::prelude::*; #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] pub enum ParagraphSegmentToken { @@ -57,6 +56,8 @@ fn tokens_to_paragraph_segment(tokens: Vec) -> ParagraphTokenList { Some(ParagraphSegmentToken::Text(result)) }, Some(NorgToken::End(x)) => Some(ParagraphSegmentToken::Text(format!("{x}end"))), + Some(NorgToken::Newlines(_)) => Some(ParagraphSegmentToken::Whitespace), + Some(NorgToken::Eof) => Some(ParagraphSegmentToken::Text(String::new())), None => None, _x => { unreachable!(); @@ -68,68 +69,50 @@ fn tokens_to_paragraph_segment(tokens: Vec) -> ParagraphTokenList { /// Represents various Norg blocks parsed from tokens. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] pub enum NorgBlock { - /// A segment of a paragraph consisting of Norg tokens. ParagraphSegment(ParagraphTokenList), - /// End of a paragraph segment. ParagraphSegmentEnd(ParagraphTokenList), - /// A heading with a specified level, title, and optional extension section. Heading { level: u16, title: ParagraphTokenList, extension_section: ParagraphTokenList, }, - /// A nestable detached modifier with a type, level, and optional extension section. NestableDetachedModifier { modifier_type: char, level: u16, extension_section: ParagraphTokenList, }, - /// A rangeable detached modifier with an indication if it is ranged, type, title, and optional extension section. RangeableDetachedModifier { ranged: bool, modifier_type: char, title: ParagraphTokenList, extension_section: ParagraphTokenList, }, - /// Closing tag for a rangeable detached modifier. RangeableDetachedModifierClose(char), - /// A ranged tag with a type, name, and optional parameters. RangedTag { tag_type: char, name: ParagraphTokenList, parameters: Option>, }, - /// End of a ranged tag. RangedTagEnd(char), - /// A verbatim ranged tag with a name, optional parameters, and content. VerbatimRangedTag { name: ParagraphTokenList, parameters: Option>, content: Vec, }, - /// An infirm tag with a name and optional parameters. InfirmTag { name: ParagraphTokenList, parameters: Option>, }, - /// A carryover tag with a type, name, and optional parameters. CarryoverTag { tag_type: char, name: ParagraphTokenList, parameters: Option>, }, - /// A delimiting modifier, defined by a single char `-` (weak), `=` (string), or `_` (horizontal rule) DelimitingModifier(char), } /// Defines the parser for stage 2 of the Norg parsing process, which converts tokens into blocks. -/// -/// # Returns -/// -/// * A parser that processes `NorgToken`s into a vector of `NorgBlock`s, which properly define -/// paragraph boundaries. -pub fn stage_2() -> impl Parser, Error = chumsky::error::Simple> -{ +pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, extra::Err>> { use NorgToken::*; let whitespace = select! { Whitespace(_) => () }; @@ -146,42 +129,43 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro Whitespace(_) => (), }; - let newlines_whitespace_or_eof = select! { - Newlines(_) => (), - SingleNewline => (), - Whitespace(_) => (), - Eof => (), - }; + let not_newlines_or_eof = any() + .filter(|tok: &NorgToken| !matches!(tok, NorgToken::SingleNewline | NorgToken::Newlines(_) | NorgToken::Eof)); - let paragraph_segment = newlines_or_eof.not().repeated().at_least(1); + let not_newlines_ws_or_eof = any() + .filter(|tok: &NorgToken| !matches!(tok, NorgToken::Newlines(_) | NorgToken::SingleNewline | NorgToken::Whitespace(_) | NorgToken::Eof)); - let extension_section = select! { - SingleNewline => (), - Newlines(_) => (), - Eof => (), - Special(')') => (), - } - .not() - .repeated() - .at_least(1) - .delimited_by(just(Special('(')), just(Special(')'))); + let not_newlines_ws_or_eof_or_close_paren = any() + .filter(|tok: &NorgToken| !matches!(tok, NorgToken::SingleNewline | NorgToken::Newlines(_) | NorgToken::Eof | NorgToken::Special(')'))); + + let paragraph_segment = not_newlines_or_eof + .repeated() + .at_least(1) + .collect::>(); - let parameters = newlines_whitespace_or_eof - .not() + let extension_section = not_newlines_ws_or_eof_or_close_paren .repeated() .at_least(1) - .separated_by(whitespace.repeated().at_least(1)); + .collect::>() + .delimited_by(just(Special('(')), just(Special(')'))); + + let parameters = not_newlines_ws_or_eof + .repeated() + .at_least(1) + .collect::>() + .separated_by(whitespace.repeated().at_least(1).collect::>()) + .collect::>(); let heading = select! { Special('*') => (), } - .ignored() .repeated() .at_least(1) + .collect::>() .map(|chars| chars.len() as u16) - .then_ignore(whitespace.repeated().at_least(1)) + .then_ignore(whitespace.repeated().at_least(1).collect::>()) .then(extension_section.clone().or_not()) - .then(paragraph_segment) + .then(paragraph_segment.clone()) .then_ignore(newlines_or_eof) .map(|((level, extension_section), title)| NorgBlock::Heading { level, @@ -197,27 +181,30 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro } .repeated() .at_least(1) + .collect::>() .try_map(|chars, span| { if chars.iter().all_equal() { Ok((chars[0], chars.len() as u16)) } else { - // Get the type of element that the user tried to create. let modifier_type = match chars[0] { '-' => "unordered list", '~' => "ordered list", '>' => "quote", _ => unreachable!(), }; - Err(Simple::custom( + Err(Rich::custom( span, - format!(" + format!( + " Expected a sequence of '{}' characters when creating {}. Norg does not permit mixing of modifiers, e.g. `-~>`. Keep all your modifiers the same, e.g. `---`. - ", chars[0], modifier_type), + ", + chars[0], modifier_type + ), )) } }) - .then_ignore(whitespace.repeated().at_least(1)) + .then_ignore(whitespace.repeated().at_least(1).collect::>()) .then(extension_section.clone().or_not()) .map( |((modifier_type, level), extension_section)| NorgBlock::NestableDetachedModifier { @@ -233,10 +220,11 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro .repeated() .at_least(1) .at_most(2) + .collect::>() .map(|chars| (chars[0], chars.len() == 2)) - .then_ignore(whitespace.repeated().at_least(1)) + .then_ignore(whitespace.repeated().at_least(1).collect::>()) .then(extension_section.clone().or_not()) - .then(paragraph_segment) + .then(paragraph_segment.clone()) .then_ignore(newlines_or_eof) .map(|(((modifier_type, ranged), extension_section), title)| { NorgBlock::RangeableDetachedModifier { @@ -267,30 +255,36 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro End(x) if x == c => x, }; - let tag_parameters = select! { - Newlines(_) => (), - SingleNewline => (), - Whitespace(_) => (), - Eof => (), - End(x) if x == c => () - } - .not() - .repeated() - .at_least(1) - .separated_by(whitespace.repeated().at_least(1)); + let not_tag_end_or_ws = any() + .filter(move |tok: &NorgToken| !matches!(tok, NorgToken::Newlines(_) | NorgToken::SingleNewline | NorgToken::Whitespace(_) | NorgToken::Eof | NorgToken::End(_) if matches!(tok, NorgToken::End(x) if *x == c))); + + let tag_parameters = not_tag_end_or_ws + .clone() + .repeated() + .at_least(1) + .collect::>() + .separated_by(whitespace.repeated().at_least(1).collect::>()) + .collect::>(); + + let verbatim_content = any() + .filter(move |tok: &NorgToken| !matches!(tok, NorgToken::End(ref x) if *x == c)) + .repeated() + .collect::>() + .or_not(); parse_char - .ignore_then(newlines_whitespace_or_eof.not().repeated().at_least(1)) + .ignore_then(not_newlines_ws_or_eof.repeated().at_least(1).collect::>()) .then( whitespace .repeated() .at_least(1) + .collect::>() .ignore_then(tag_parameters) .or_not(), ) .then_ignore(just(SingleNewline).or_not()) - .then_ignore(filter(|c| matches!(c, Newlines(_))).or_not()) - .then(tag_end.not().repeated().or_not()) + .then_ignore(any().filter(|tok: &NorgToken| matches!(tok, Newlines(_))).or_not()) + .then(verbatim_content) .then_ignore(tag_end) .map( |((name, parameters), content)| NorgBlock::VerbatimRangedTag { @@ -301,7 +295,7 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro .map(tokens_to_paragraph_segment) .collect() }), - content: content.unwrap_or(vec![]), + content: content.unwrap_or_default(), }, ) }; @@ -310,12 +304,13 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro let parse_char = select! { Special(x) if x == c => x }; parse_char - .ignore_then(newlines_whitespace_or_eof.not().repeated().at_least(1)) + .ignore_then(not_newlines_ws_or_eof.repeated().at_least(1).collect::>()) .then( whitespace .repeated() .at_least(1) - .ignore_then(parameters) + .collect::>() + .ignore_then(parameters.clone()) .or_not(), ) .then_ignore(select! { @@ -336,12 +331,13 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro let infirm_tag = { select! { Special('.') => '.' } - .ignore_then(newlines_whitespace_or_eof.not().repeated().at_least(1)) + .ignore_then(not_newlines_ws_or_eof.repeated().at_least(1).collect::>()) .then( whitespace .repeated() .at_least(1) - .ignore_then(parameters) + .collect::>() + .ignore_then(parameters.clone()) .or_not(), ) .then_ignore(select! { @@ -364,11 +360,12 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro Special('+') => '+', Special('#') => '#', } - .then(newlines_whitespace_or_eof.not().repeated().at_least(1)) + .then(not_newlines_ws_or_eof.repeated().at_least(1).collect::>()) .then( whitespace .repeated() .at_least(1) + .collect::>() .ignore_then(parameters) .or_not(), ) @@ -397,6 +394,7 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro } .repeated() .at_least(2) + .collect::>() .then_ignore(newlines_or_eof) .map(|chars| NorgBlock::DelimitingModifier(chars[0])); @@ -417,7 +415,8 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro carryover_tags, tag_end, paragraph_segment - .then(newlines_or_eof.repeated().at_least(1).rewind()) + .clone() + .then(newlines_or_eof.repeated().at_least(1).collect::>().rewind()) .map(|(content, trailing)| match trailing.last().unwrap() { NorgToken::Eof => { NorgBlock::ParagraphSegmentEnd(tokens_to_paragraph_segment(content)) @@ -434,5 +433,6 @@ pub fn stage_2() -> impl Parser, Error = chumsky::erro )) .padded_by(newlines_whitespace.repeated()) .repeated() + .collect::>() .then_ignore(just(Eof)) -} +} \ No newline at end of file diff --git a/src/stage_3.rs b/src/stage_3.rs index 0c066bf..6b97e58 100644 --- a/src/stage_3.rs +++ b/src/stage_3.rs @@ -43,59 +43,29 @@ impl std::fmt::Display for RangeableDetachedModifier { #[derive(Clone, Hash, Debug, PartialEq, Eq, Serialize)] pub enum TodoStatus { - /// ` ` Undone, - /// `x` Done, - /// `?` NeedsClarification, - /// `=` Paused, - /// `!` Urgent, - /// `+` or `+ 4th may` Recurring(Option), - /// `-` Pending, - /// `_` Canceled, } #[derive(Clone, Hash, Debug, PartialEq, Eq, Serialize)] pub enum DetachedModifierExtension { - /// todo item status: - /// `- ( ) undone` - /// `- (x) done` - /// `- (?) needs clarification` - /// `- (=) paused/on hold` - /// `- (!) urgent` - /// `- (+) recurring` - /// `- (+ 15th May) recurring with a time stamp` - /// `- (-) in progress/pending` - /// `- (_) put down/canceled` Todo(TodoStatus), - - /// Priority, `#` and then any text - /// `- (# A) Priority A` Priority(String), - - /// Time stamp extension: - /// `- (@ ) list item text` Timestamp(String), - - /// Time stamp for the due date/deadline for this item - /// `- (< 1 Jan 2025) Do something` DueDate(String), - - /// Time stamp for the start time of the item: - /// `- (> 2 Jan 2025)` Start something StartDate(String), } #[derive(Clone, Hash, Debug, PartialEq, Eq, Serialize)] pub enum CarryoverTag { - Attribute, // `+` - Macro, // `#` + Attribute, + Macro, } #[derive(PartialEq, Serialize)] @@ -104,10 +74,11 @@ pub enum RangedTag { Standard, } -fn paragraph_parser_opener_candidates_and_links() -> impl Parser< - ParagraphSegmentToken, +fn paragraph_parser_opener_candidates_and_links<'a>() -> impl Parser< + 'a, + &'a [ParagraphSegmentToken], Vec, - Error = chumsky::error::Simple, + extra::Err>, > { let token = any().map(ParagraphSegment::Token); let modifier = select! { @@ -115,13 +86,13 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< }; let whitespace_or_special = select! { - w @ ParagraphSegmentToken::Whitespace => w, - s @ ParagraphSegmentToken::Special(_) => s, + ParagraphSegmentToken::Whitespace => ParagraphSegmentToken::Whitespace, + ParagraphSegmentToken::Special(c) => ParagraphSegmentToken::Special(c), }; let opening_modifier_candidate = whitespace_or_special - .then(modifier.repeated().at_least(1)) - .then(just(ParagraphSegmentToken::Whitespace).not()) + .then(modifier.repeated().at_least(1).collect::>()) + .then(any().filter(|t: &ParagraphSegmentToken| !matches!(t, ParagraphSegmentToken::Whitespace))) .map(|((left, modifiers), right)| { ParagraphSegment::AttachedModifierOpener((Some(left), modifiers, right)) }); @@ -129,47 +100,42 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< let left_empty_opening_modifier = modifier .repeated() .at_least(1) - .then(just(ParagraphSegmentToken::Whitespace).not()) + .collect::>() + .then(any().filter(|t: &ParagraphSegmentToken| !matches!(t, ParagraphSegmentToken::Whitespace))) .map(|(modifiers, right)| { ParagraphSegment::AttachedModifierOpener((None, modifiers, right)) }); + let not_backtick = any() + .filter(|t: &ParagraphSegmentToken| !matches!(t, ParagraphSegmentToken::Special('`'))); + let inline_verbatim = just(ParagraphSegmentToken::Special('`')) - .ignore_then( - just(ParagraphSegmentToken::Special('`')) - .not() - .repeated() - .at_least(1), - ) + .ignore_then(not_backtick.repeated().at_least(1).collect::>()) .then_ignore(just(ParagraphSegmentToken::Special('`'))) .map(ParagraphSegment::InlineVerbatim); + let not_close_bracket = any() + .filter(|t: &ParagraphSegmentToken| !matches!(t, ParagraphSegmentToken::Special(']'))); + let anchor = just(ParagraphSegmentToken::Special('[')) - .ignore_then( - just(ParagraphSegmentToken::Special(']')) - .not() - .repeated() - .at_least(1), - ) + .ignore_then(not_close_bracket.repeated().at_least(1).collect::>()) .then_ignore(just(ParagraphSegmentToken::Special(']'))); + let not_colon = any() + .filter(|t: &ParagraphSegmentToken| !matches!(t, ParagraphSegmentToken::Special(':'))); + + let filepath_inner = just(ParagraphSegmentToken::Special(':')) + .ignore_then(not_colon.repeated().at_least(1).collect::>()) + .then_ignore(just(ParagraphSegmentToken::Special(':'))); + let link = just(ParagraphSegmentToken::Special('{')) - .ignore_then( - just(ParagraphSegmentToken::Special(':')) - .ignore_then( - just(ParagraphSegmentToken::Special(':')) - .not() - .repeated() - .at_least(1), - ) - .then_ignore(just(ParagraphSegmentToken::Special(':'))) - .or_not(), - ) + .ignore_then(filepath_inner.or_not()) .then( choice(( just(ParagraphSegmentToken::Special('*')) .repeated() .at_least(1) + .collect::>() .map(|tokens| "*".repeat(tokens.len())), just(ParagraphSegmentToken::Special('$')).to("$".to_string()), just(ParagraphSegmentToken::Special('^')).to("^".to_string()), @@ -186,10 +152,10 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< .or_not(), ) .then( - just(ParagraphSegmentToken::Special('}')) - .not() + any().filter(|t: &ParagraphSegmentToken| !matches!(t, ParagraphSegmentToken::Special('}'))) .repeated() .at_least(1) + .collect::>() .or_not(), ) .then_ignore(just(ParagraphSegmentToken::Special('}'))) @@ -198,25 +164,23 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< |(((filepath, modifiers), content), description)| ParagraphSegment::Link { filepath: filepath .map(|content| content.into_iter().map_into::().collect()), - description: description.map(|content| parse_paragraph(content).unwrap()), + description: description.map(|content| parse_paragraph(content)), targets: if let Some(content) = content { vec![if let Some(modifiers) = modifiers { match modifiers.as_str() { - "$" => LinkTarget::Definition(parse_paragraph(content).unwrap()), - "^" => LinkTarget::Footnote(parse_paragraph(content).unwrap()), - "?" => LinkTarget::Wiki(parse_paragraph(content).unwrap()), - "=" => LinkTarget::Extendable(parse_paragraph(content).unwrap()), + "$" => LinkTarget::Definition(parse_paragraph(content)), + "^" => LinkTarget::Footnote(parse_paragraph(content)), + "?" => LinkTarget::Wiki(parse_paragraph(content)), + "=" => LinkTarget::Extendable(parse_paragraph(content)), "/" => { LinkTarget::Path(content.into_iter().map_into::().collect()) } "@" => LinkTarget::Timestamp( content.into_iter().map_into::().collect(), ), - - // Only other possibility is a heading. str => LinkTarget::Heading { level: str.len() as u16, - title: parse_paragraph(content).unwrap(), + title: parse_paragraph(content), }, } } else { @@ -228,24 +192,22 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< }, ); + let not_close_angle = any() + .filter(|t: &ParagraphSegmentToken| !matches!(t, ParagraphSegmentToken::Special('>'))); + let inline_linkable = just(ParagraphSegmentToken::Special('<')) - .ignore_then( - just(ParagraphSegmentToken::Special('>')) - .not() - .repeated() - .at_least(1), - ) + .ignore_then(not_close_angle.repeated().at_least(1).collect::>()) .then_ignore(just(ParagraphSegmentToken::Special('>'))) - .map(|content| ParagraphSegment::InlineLinkTarget(parse_paragraph(content).unwrap())); + .map(|content| ParagraphSegment::InlineLinkTarget(parse_paragraph(content))); - left_empty_opening_modifier.or_not().chain( + left_empty_opening_modifier.or_not().then( choice(( link.clone(), anchor .clone() .then(link) .map(|(content, link)| ParagraphSegment::AnchorDefinition { - content: parse_paragraph(content).unwrap(), + content: parse_paragraph(content), target: Box::new(link), }), inline_verbatim, @@ -253,16 +215,24 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< .clone() .then(anchor.clone().or_not()) .map(|(content, description)| ParagraphSegment::Anchor { - content: parse_paragraph(content).unwrap(), - description: description.map(|content| parse_paragraph(content).unwrap()), + content: parse_paragraph(content), + description: description.map(|content| parse_paragraph(content)), }), inline_linkable, opening_modifier_candidate, token, )) .repeated() - .at_least(1), - ) + .at_least(1) + .collect::>(), + ).map(|(prefix, mut body)| { + let mut result = match prefix { + Some(p) => vec![p], + None => vec![], + }; + result.append(&mut body); + result + }) } fn dedup_opener_candidates(input: Vec) -> Vec { @@ -279,41 +249,40 @@ fn dedup_opener_candidates(input: Vec) -> Vec impl Parser, Error = chumsky::error::Simple> -{ - use ParagraphSegment::*; +fn paragraph_parser_closer_candidates<'a>( +) -> impl Parser<'a, &'a [ParagraphSegment], Vec, extra::Err>> { + use ParagraphSegment as PS; + use ParagraphSegmentToken as PTST; let token = any(); let modifier = select! { - Token(ParagraphSegmentToken::Special(c @ ('*' | '/' | '_' | '-'))) => c, + PS::Token(PTST::Special(c @ ('*' | '/' | '_' | '-'))) => c, }; let whitespace_or_special = select! { - w @ Token(ParagraphSegmentToken::Whitespace) => w, - s @ Token(ParagraphSegmentToken::Special(_)) => s, + PS::Token(PTST::Whitespace) => PTST::Whitespace, + PS::Token(PTST::Special(c)) => PTST::Special(c), }; - let closing_modifier_candidate = just(Token(ParagraphSegmentToken::Whitespace)) - .not() - .then(modifier.repeated().at_least(1)) + let not_whitespace = any() + .filter(|seg: &PS| !matches!(seg, PS::Token(PTST::Whitespace))); + + let closing_modifier_candidate = not_whitespace + .then(modifier.repeated().at_least(1).collect::>()) .then(whitespace_or_special) .map(|((left, modifiers), right)| { - ParagraphSegment::AttachedModifierCloserCandidate(( + PS::AttachedModifierCloserCandidate(( Box::new(left), modifiers, - Some(Box::new(right)), + Some(Box::new(PS::Token(right))), )) }); - // TODO(vhyrro): This is not optimal, as it causes a second parse of a potentially long string - // of nodes. Ideally, the `end()` check should be done directly in a single parse. - let closing_modifier_candidate_with_eof = just(Token(ParagraphSegmentToken::Whitespace)) - .not() - .then(modifier.repeated().at_least(1)) + let closing_modifier_candidate_with_eof = not_whitespace + .then(modifier.repeated().at_least(1).collect::>()) .then_ignore(end()) .map(|(left, modifiers)| { - ParagraphSegment::AttachedModifierCloserCandidate((Box::new(left), modifiers, None)) + PS::AttachedModifierCloserCandidate((Box::new(left), modifiers, None)) }); choice(( @@ -323,6 +292,7 @@ fn paragraph_parser_closer_candidates( )) .repeated() .at_least(1) + .collect::>() } fn unravel_candidates(input: Vec) -> Vec { @@ -372,16 +342,18 @@ fn unravel_candidates(input: Vec) -> Vec { }) } -fn paragraph_rollup_candidates( -) -> impl Parser, Error = chumsky::error::Simple> -{ +fn paragraph_rollup_candidates<'a>( +) -> impl Parser<'a, &'a [ParagraphSegment], Vec, extra::Err>> { let candidate = select! { ParagraphSegment::AttachedModifierCloser(c) => c, }; - let attached_modifier = recursive(|attached_modifier| { + let not_candidate = any() + .filter(|seg: &ParagraphSegment| !matches!(seg, ParagraphSegment::AttachedModifierCloser(_))); + + let attached_modifier = recursive::<_, _, extra::Err>, _, _>(|attached_modifier| { select! { ParagraphSegment::AttachedModifierCandidate { modifier_type, .. } => modifier_type, } - .then(attached_modifier.or(candidate.not()).repeated().at_least(1)) + .then(attached_modifier.or(not_candidate).repeated().at_least(1).collect::>()) .then(candidate) .try_map(|((modifier_type, content), closer), span| { if modifier_type == closer { @@ -390,7 +362,7 @@ fn paragraph_rollup_candidates( content, }) } else { - Err(Simple::custom( + Err(Rich::custom( span, "differing opening and closing modifiers found", )) @@ -398,7 +370,7 @@ fn paragraph_rollup_candidates( }) }); - choice((attached_modifier, any())).repeated().at_least(1) + choice((attached_modifier, any())).repeated().at_least(1).collect::>() } fn eliminate_invalid_candidates(input: Vec) -> Vec { @@ -496,18 +468,26 @@ pub enum ParagraphSegment { fn parse_paragraph( input: Vec, -) -> Result, Vec>> { - Ok(eliminate_invalid_candidates(unravel_candidates( - paragraph_rollup_candidates() - .parse(unravel_candidates( - paragraph_parser_closer_candidates() - .parse(unravel_candidates(dedup_opener_candidates( - paragraph_parser_opener_candidates_and_links().parse(input)?, - ))) - .unwrap(), - )) - .unwrap(), - ))) +) -> Vec { + let stage1_result = paragraph_parser_opener_candidates_and_links() + .parse(&input[..]) + .into_result() + .unwrap(); + let deduped = dedup_opener_candidates(stage1_result); + let unraveled1 = unravel_candidates(deduped); + + let stage2_result = paragraph_parser_closer_candidates() + .parse(&unraveled1) + .into_result() + .unwrap(); + let unraveled2 = unravel_candidates(stage2_result); + + let stage3_result = paragraph_rollup_candidates() + .parse(&unraveled2) + .into_result() + .unwrap(); + + eliminate_invalid_candidates(unravel_candidates(stage3_result)) } #[derive(Clone, Debug, PartialEq, Hash, Eq, Serialize)] @@ -560,10 +540,11 @@ pub enum DelimitingModifier { HorizontalRule, } -fn detached_modifier_extensions() -> impl Parser< - ParagraphSegmentToken, +fn detached_modifier_extensions<'a>() -> impl Parser< + 'a, + &'a [ParagraphSegmentToken], Vec, - Error = chumsky::error::Simple, + extra::Err>, > { use ParagraphSegmentToken::*; @@ -573,22 +554,25 @@ fn detached_modifier_extensions() -> impl Parser< Text(c) if c == "x" || c == "?" => Text(c), }; + let not_pipe = any() + .filter(|t: &ParagraphSegmentToken| !matches!(t, Special('|'))); + + let attached_content = just(Whitespace) + .ignore_then(not_pipe.repeated().collect::>()) + .or_not() + .map(|tokens| { + if let Some(tokens) = tokens { + tokens + .into_iter() + .map_into::() + .collect() + } else { + String::from("") + } + }); + let detached_modifier_extension = detached_modifier_extension_tokens - .then( - just(Whitespace) - .ignore_then(select!(Special('|') => Special('|')).not().repeated()) - .or_not() - .map(|tokens| { - if let Some(tokens) = tokens { - tokens - .into_iter() - .map_into::() - .collect() - } else { - String::from("") - } - }), - ) + .then(attached_content) .map(|(spec, metadata)| match spec { Special('@') => DetachedModifierExtension::Timestamp(metadata), Special('#') => DetachedModifierExtension::Priority(metadata), @@ -616,11 +600,12 @@ fn detached_modifier_extensions() -> impl Parser< detached_modifier_extension .separated_by(just(Special('|'))) .at_least(1) + .collect::>() } -pub fn stage_3( -) -> impl Parser, Error = chumsky::error::Simple> { - recursive(|stage_3| { +pub fn stage_3<'src>( +) -> impl Parser<'src, &'src [NorgBlock], Vec, extra::Err>> { + recursive::<_, _, extra::Err>, _, _>(|stage_3| { let paragraph_segment = select! { NorgBlock::ParagraphSegment(content) => content, }; @@ -633,28 +618,34 @@ pub fn stage_3( paragraph_segment .repeated() .at_least(1) - .flatten() - .chain(paragraph_segment_end.or_not()), - paragraph_segment_end, + .collect::>() + .then(paragraph_segment_end.or_not()) + .map(|(segments, end)| { + let mut tokens: Vec = segments.into_iter().flatten().collect(); + if let Some(end) = end { + tokens.extend(end); + } + tokens + }), + paragraph_segment_end + .map(|end| end), )) .map(|mut tokens| { - // Trim trailing whitespace (both user-induced but also induced by us when - // converting single newlines to whitespace). if let Some(ParagraphSegmentToken::Whitespace) = tokens.last() { tokens.pop(); } - NorgASTFlat::Paragraph(parse_paragraph(tokens).unwrap()) + NorgASTFlat::Paragraph(parse_paragraph(tokens)) }); let nestable_detached_modifier = select! { NorgBlock::NestableDetachedModifier { modifier_type: '-', level, extension_section } => (NestableDetachedModifier::UnorderedList, level, extension_section), NorgBlock::NestableDetachedModifier { modifier_type: '~', level, extension_section } => (NestableDetachedModifier::OrderedList, level, extension_section), NorgBlock::NestableDetachedModifier { modifier_type: '>', level, extension_section } => (NestableDetachedModifier::Quote, level, extension_section), - }.then(paragraph).map(|((modifier_type, level, extension_section), paragraph)| NorgASTFlat::NestableDetachedModifier { + }.then(paragraph.clone()).map(|((modifier_type, level, extension_section), paragraph)| NorgASTFlat::NestableDetachedModifier { modifier_type, level, - extensions: detached_modifier_extensions().parse(extension_section).unwrap_or_default(), + extensions: detached_modifier_extensions().parse(&extension_section[..]).into_result().unwrap_or_default(), content: Box::new(paragraph), }); @@ -662,30 +653,32 @@ pub fn stage_3( NorgBlock::RangeableDetachedModifier { modifier_type: '$', ranged: false, title, extension_section } => (RangeableDetachedModifier::Definition, title, extension_section), NorgBlock::RangeableDetachedModifier { modifier_type: '^', ranged: false, title, extension_section} => (RangeableDetachedModifier::Footnote, title, extension_section), NorgBlock::RangeableDetachedModifier { modifier_type: ':', ranged: false, title, extension_section } => (RangeableDetachedModifier::Table, title, extension_section), - }.then(paragraph).map(|((modifier_type, title, extension_section), paragraph)| NorgASTFlat::RangeableDetachedModifier { + }.then(paragraph.clone()).map(|((modifier_type, title, extension_section), paragraph)| NorgASTFlat::RangeableDetachedModifier { modifier_type, - title: parse_paragraph(title).unwrap(), - extensions: detached_modifier_extensions().parse(extension_section).unwrap_or_default(), + title: parse_paragraph(title), + extensions: detached_modifier_extensions().parse(&extension_section[..]).into_result().unwrap_or_default(), content: vec![paragraph], }); + let stage_3_ref = stage_3.clone(); + let ranged_detached_modifier = select! { NorgBlock::RangeableDetachedModifier { modifier_type: '$', ranged: true, title, extension_section } => ('$', RangeableDetachedModifier::Definition, title, extension_section), NorgBlock::RangeableDetachedModifier { modifier_type: '^', ranged: true, title, extension_section } => ('^', RangeableDetachedModifier::Footnote, title, extension_section), NorgBlock::RangeableDetachedModifier { modifier_type: ':', ranged: true, title, extension_section } => (':', RangeableDetachedModifier::Table, title, extension_section), } - .then(stage_3.clone().repeated()) + .then(stage_3_ref.repeated().collect::>()) .then(select! { NorgBlock::RangeableDetachedModifierClose(c) => c }) .try_map(|(((opening_ch, modifier_type, title, extension_section), content), closing_ch), span| if opening_ch == closing_ch { Ok(NorgASTFlat::RangeableDetachedModifier { modifier_type, - title: parse_paragraph(title).unwrap(), - extensions: detached_modifier_extensions().parse(extension_section).unwrap_or_default(), + title: parse_paragraph(title), + extensions: detached_modifier_extensions().parse(&extension_section[..]).into_result().unwrap_or_default(), content, }) } else { - Err(Simple::custom(span, format!("Expected '{0}{0}' to close modifier, found '{1}{1}' instead.", opening_ch, closing_ch))) + Err(Rich::custom(span, format!("Expected '{0}{0}' to close modifier, found '{1}{1}' instead.", opening_ch, closing_ch))) }); let heading = select! { @@ -693,18 +686,20 @@ pub fn stage_3( } .try_map(move |(level, title, extension_section), _span| Ok(NorgASTFlat::Heading { level, - title: parse_paragraph(title).unwrap(), - extensions: detached_modifier_extensions().parse(extension_section).unwrap_or_default(), + title: parse_paragraph(title), + extensions: detached_modifier_extensions().parse(&extension_section[..]).into_result().unwrap_or_default(), })); let stringify_tokens_and_split = move |tokens: ParagraphTokenList| -> Vec { tokens.into_iter().map_into::().collect::().split('.').map_into().collect() }; + let stage_3_ref2 = stage_3.clone(); + let carryover_tag = select! { NorgBlock::CarryoverTag { tag_type: '+', name, parameters } => (CarryoverTag::Attribute, name, parameters), NorgBlock::CarryoverTag { tag_type: '#', name, parameters } => (CarryoverTag::Macro, name, parameters), - }.then(stage_3.clone()).map(move |((tag_type, name, parameters), next_object)| { + }.then(stage_3_ref2).map(move |((tag_type, name, parameters), next_object)| { NorgASTFlat::CarryoverTag { tag_type, name: stringify_tokens_and_split(name), @@ -723,16 +718,18 @@ pub fn stage_3( }, }; + let stage_3_ref3 = stage_3.clone(); + let ranged_tag = select! { NorgBlock::RangedTag { tag_type: '=', name, parameters } => (RangedTag::Macro, stringify_tokens_and_split(name), parameters.unwrap_or_default().into_iter().map(|parameter| parameter.into_iter().map_into::().collect()).collect()), NorgBlock::RangedTag { tag_type: '|', name, parameters } => (RangedTag::Standard, stringify_tokens_and_split(name), parameters.unwrap_or_default().into_iter().map(|parameter| parameter.into_iter().map_into::().collect()).collect()) - }.then(stage_3.repeated()).then(select! { + }.then(stage_3_ref3.repeated().collect::>()).then(select! { NorgBlock::RangedTagEnd('=') => RangedTag::Macro, NorgBlock::RangedTagEnd('|') => RangedTag::Standard, }).try_map(|(((tag_type, name, parameters), content), closing_tag_type), span| if tag_type == closing_tag_type { Ok(NorgASTFlat::RangedTag { name, parameters, content }) } else { - Err(Simple::custom(span, "Invalid closing modifier for ranged tag.")) // TODO: Improve errors + Err(Rich::custom(span, "Invalid closing modifier for ranged tag.")) }); let infirm_tag = select! { @@ -757,5 +754,5 @@ pub fn stage_3( ranged_detached_modifier, paragraph, )) - }).repeated().at_least(1) -} + }).repeated().at_least(1).collect::>() +} \ No newline at end of file From 4843fd5cf83501d0ab7e63c08b6f28385d0acfcf Mon Sep 17 00:00:00 2001 From: NTBBloodbath Date: Sun, 14 Jun 2026 22:29:38 -0400 Subject: [PATCH 2/2] test: restore 26 removed tests, regenerate snapshots - Re-adds all parser tests removed during chumsky 0.13 migration because they required manual regeneration and I did not feel like it. - Removes dead code in stage_2 (unreachable catch-all). Fixes 9 clippy::clone_on_copy and 2 redundant_closure warnings. --- proptest-regressions/lib.txt | 13 - src/lib.rs | 581 ++++++++++++++++++ src/metadata/mod.rs | 2 +- .../rust_norg__tests__inline_verbatim.snap | 1 - src/snapshots/rust_norg__tests__links.snap | 1 - ...ust_norg__tests__ranged_verbatim_tags.snap | 11 +- .../rust_norg__tests__verbatim_tags.snap | 160 ----- src/stage_1.rs | 1 - src/stage_2.rs | 13 +- src/stage_3.rs | 9 +- 10 files changed, 595 insertions(+), 197 deletions(-) delete mode 100644 proptest-regressions/lib.txt delete mode 100644 src/snapshots/rust_norg__tests__verbatim_tags.snap diff --git a/proptest-regressions/lib.txt b/proptest-regressions/lib.txt deleted file mode 100644 index f34e3d6..0000000 --- a/proptest-regressions/lib.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc fb9b5df4fe46fe331cc3aa40bba6501c1c603084688fd02dda6d1c73106c1324 # shrinks to tag_name = "A", parameter = "A", multi_parameter = "\u{b}" -cc 28afae9872324ba0632a8023219e32939580363ce8b99752dc19fae0ac5b63d1 # shrinks to paragraph_content = " " -cc 5b31c9987c98fc0e4faa50b782e5952e0948d2c2a60dd29081c8f54c75b4b52c # shrinks to tag_name = "ø", parameter = "a", multi_parameter = " " -cc 67cb989806791583bc835dd766e8f7ef294a1a634f6857cd8e6dd16a49a09246 # shrinks to tag_name = "\u{1d165}", parameter = "<*\u{2060}>", multi_parameter = "a", content = "" -cc cacc392d5a052fbd56e86b2ae08c6380a7a191847001424d938930b94e0f449f # shrinks to tag_type = "@", tag_name = "a", parameter = "a", multi_parameter = "<*0>", content = "\u{2060}" -cc 15209ad4b4a04639c42c6f2d9629decdcd554a46a3638a63a8a04f9822a3d33c # shrinks to tag_name = "0", parameter = "a", multi_parameter = "\\", content = "\\" -cc 98c6844a3274a61cba1c4be4bf931c3c136b24dbe9c75187f53e9e6a45a1508d # shrinks to tag_type = "@", tag_name = "𖩠", parameter = "!", multi_parameter = "\t", content = "a\\" diff --git a/src/lib.rs b/src/lib.rs index 9e8aa15..0029710 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -169,4 +169,585 @@ mod tests { assert_yaml_snapshot!(examples); } + + #[test] + fn headings_tree() { + let headings_tree_examples: Vec<_> = [ + " + * Heading + ** Another heading + ", + " + * Heading + ** Subheading + content + * Back to regular heading + ", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse_tree(&str)) + .try_collect() + .unwrap(); + assert_yaml_snapshot!(headings_tree_examples); + } + + #[test] + fn delimiting_mods_tree() { + let examples: Vec<_> = [ + "* One + content + --- + dedented", + "* One + ** Two + === + none", + "** Two + two + ___ + two", + "- list + ___ + no list", + "* One + one + ** Two + two + *** Three + three + --- + two + --- + one + --- + none", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse_tree(&str)) + .try_collect() + .unwrap(); + assert_yaml_snapshot!(examples); + } + + #[test] + fn lists_tree() { + let examples: Vec<_> = [ + "- base", + "- one + -- two", + "- one + -- two + with content + -- two (2) + --- three + - one", + "-- two + - one", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse_tree(&str)) + .try_collect() + .unwrap(); + assert_yaml_snapshot!(examples); + } + + #[test] + fn lists() { + let examples: Vec<_> = [ + "- Test list", + "---- Test list", + " + - Test list + - Test list + -- Test list + -- Test list + - Test list + --- Test list + ", + "---not list", + "--> not a list", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + #[test] + fn modifier_extensions() { + let examples: Vec<_> = [ + "- ( ) undone", + "* (x) done", + "- (=) hold", + "* (_) canceled", + "- (-) pending", + "* (!) urgent", + "- (+) recurring", + "~ (+ Friday) recurring with date", + "** ( |# Low|< Feb 1) undone, low, & before Feb", + "** (# Two Words|x| |!|+|_|+ 5th|=|-|< Feb 1|> 2025|@ Jan 1 2025) All of them" + ].into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + #[test] + fn lists_regressions() { + [ + "- - a list item", + "---- - a list item", + "---- > a list item", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .for_each(|str| { + parse(&str).unwrap_err(); + }); + } + + #[test] + fn ordered_lists() { + let examples: Vec<_> = [ + "~ Test list", + "~~~~ Test list", + " + ~ Test list + ~ Test list + ~~ Test list + ~~ Test list + ~ Test list + ~~~ Test list + ", + "~~~not list", + "~~> not a list", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + #[test] + fn ordered_lists_regressions() { + [ + "~ ~ a list item", + "~~~~ - a list item", + "~~~~ > a list item", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .for_each(|str| { + parse(&str).unwrap_err(); + }); + } + + #[test] + fn quotes() { + let examples: Vec<_> = [ + "> Test quote", + ">>>> Test quote", + " + > Test quote + > Test quote + >> Test quote + >> Test quote + > Test quote + >>> Test quote + ", + ">>>not quote", + ">>- not a quote", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + #[test] + fn quotes_regressions() { + [ + "> > a list item", + ">>>> - a list item", + ">>>> ~ a list item", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .for_each(|str| { + parse(&str).unwrap_err(); + }); + } + + #[test] + fn definitions() { + let examples: Vec<_> = [ + "$ Term + Definition", + "$$ Term + Long definition + $$", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + #[test] + fn definitions_regressions() { + [ + "$ Term Definition", + "$$ Term + Long definition $$", + "$$ Term + Long definition + $$text", + "$$ Term + Long definition + $$ text", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .for_each(|str| { + parse(&str).unwrap_err(); + }); + } + + #[test] + fn footnotes() { + let examples: Vec<_> = [ + "^ Title + Content", + "^^ Title + Long content + ^^", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + #[test] + fn footnotes_regressions() { + [ + "^ Term Definition", + "^^ Term + Long definition ^^", + "^^ Term + Long definition + ^^text", + "^^ Term + Long definition + ^^ text", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .for_each(|str| { + parse(&str).unwrap_err(); + }); + } + + #[test] + fn tables() { + let examples: Vec<_> = [ + ": A1 + Cell content", + ":: A1 + Long cell content. + ::", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + #[test] + fn tables_regressions() { + [ + ": Term Definition", + ":: Term + Long definition ::", + ":: Term + Long definition + ::text", + ":: Term + Long definition + :: text", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .for_each(|str| { + parse(&str).unwrap_err(); + }); + } + + #[test] + fn infirm_tags() { + let examples: Vec<_> = [ + ".tag", + ".tag-name_with-complexchars", + ".tag-name_ parameter", + ".tag-name_ one\\ large\\ parameter", + ".tag-name_ one\\ large\\ parameter &^@! third parameter", + ".tag.name.image https://github.com/super-special/repo.git?text=hello&other_text=bye", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + proptest! { + #[test] + fn infirm_tags_proptests(tag_name in TAG_NAME_REGEX, parameter in TAG_PARAMETER_REGEX, multi_parameter in TAG_MULTI_PARAMETER_REGEX) { + let tag = format!(".{} {} {}\n", tag_name, parameter, multi_parameter); + + parse(&tag).unwrap(); + } + } + + #[test] + fn carryover_tags() { + let examples: Vec<_> = [ + "+tag + paragraph", + "+tag-name_with-complexchars + paragraph", + "+tag-name_ parameter + paragraph", + "+tag-name_ one\\ large\\ parameter + paragraph", + "+tag-name_ one\\ large\\ parameter &^@! third parameter + paragraph", + "+tag.name.image https://github.com/super-special/repo.git?text=hello&other_text=bye + paragraph", + "#tag + paragraph", + "#tag-name_with-complexchars + paragraph", + "#tag-name_ parameter + paragraph", + "#tag-name_ one\\ large\\ parameter + paragraph", + "#tag-name_ one\\ large\\ parameter &^@! third parameter + paragraph", + "#tag.name.image https://github.com/super-special/repo.git?text=hello&other_text=bye + paragraph", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + #[test] + fn carryover_tags_tree() { + let examples: Vec<_> = [ + " + #id 123 + * tree + ** nested + ", + " + * tree + #id there + ** nested + --- + part of tree + ", + " + #name main + -- two + ---- four + #id 3 + --- three + ", + " + #comment + multi-line + comments + --- + out + ", + " + #id 123 + #comment + comment with id + ", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse_tree(&str)) + .try_collect() + .unwrap(); + assert_yaml_snapshot!(examples); + } + + proptest! { + #[test] + fn carryover_tags_proptests(tag_name in TAG_NAME_REGEX, parameter in TAG_PARAMETER_REGEX, multi_parameter in TAG_MULTI_PARAMETER_REGEX) { + let content = format!("#{} {} {}\nhello world!", tag_name, parameter, multi_parameter); + + parse(&content).unwrap(); + } + } + + #[test] + fn ranged_verbatim_tags() { + let examples: Vec<_> = [ + r#"@code + print("Hello world!") + @end"#, + r#"@code.some-text.here lua\ language second-parameter + print("Hello world!") + @end"#, + r#"@some-complex_tag_ first-parameter #&*(&$!) third-parameter + + function hello() + print("Hello World") + end + + hello() + @end"#, + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + proptest! { + #[test] + fn ranged_verbatim_tags_proptests(tag_name in TAG_NAME_REGEX, parameter in TAG_PARAMETER_REGEX, multi_parameter in TAG_MULTI_PARAMETER_REGEX, content in ".*") { + if content.contains("@end") { + return Ok(()); + } + + let content = format!("@{} {} {}\n{}\n@end", tag_name, parameter, multi_parameter, content); + + parse(&content).unwrap(); + } + } + + #[test] + fn ranged_tags() { + let examples: Vec<_> = [ + r#"|example + Hello world! + |end"#, + r#"|example.some-text.here one\ parameter second-parameter + #carryover + text within + |end"#, + r#"|some-complex_tag_ first-parameter #&*(&$!) third-parameter + this is some text within + |end"#, + r#"|example + * Hello world! + |end"#, + r#"|example + |example + * Hello world! + |end + |end"#, + r#"=example + Hello world! + =end"#, + r#"=example.some-text.here one\ parameter second-parameter + #carryover + text within + =end"#, + r#"=some-complex_tag_ first-parameter #&*(&$!) third-parameter + this is some text within + =end"#, + r#"=example + * Hello world! + =end"#, + r#"=example + =example + * Hello world! + =end + =end"#, + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + + proptest! { + #[test] + fn ranged_tags_proptests(tag_type in prop_oneof!["@", "|"], tag_name in TAG_NAME_REGEX, parameter in TAG_PARAMETER_REGEX, multi_parameter in TAG_MULTI_PARAMETER_REGEX, content in PARAGRAPH_REGEX) { + if content.contains(format!("{}end", tag_type).as_str()) { + return Ok(()); + } + + let content = format!("{tag_type}{tag_name} {parameter} {multi_parameter}\n{content}\n{tag_type}end"); + + parse(&content).unwrap(); + } + } + + #[test] + fn paragraphs() { + let examples: Vec<_> = [ + "hello, world!", + "*hello, world!*", + "*hello, + world!*", + "two + + paragraphs", + "paragraph + here + + another paragraph + here.", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } } \ No newline at end of file diff --git a/src/metadata/mod.rs b/src/metadata/mod.rs index 887d2ef..feb72a9 100644 --- a/src/metadata/mod.rs +++ b/src/metadata/mod.rs @@ -8,7 +8,7 @@ pub mod stage_1; /// Parses the given input string to produce an AST for the metadata pub fn parse_metadata(input: &str) -> Result { let processed = input.replace("\n]", "\n ]"); - let processed = format!("{{\n{}\n}}\n", processed.trim()); + let processed = format!("{{\n{}\n}}", processed.trim()); let result = stage_1::meta_parser().parse(processed.as_str()).into_result() .map_err(|e| NorgParseError::Meta(e.into_iter().map(|e| e.into_owned()).collect::>().remove(0)))?; Ok(result) diff --git a/src/snapshots/rust_norg__tests__inline_verbatim.snap b/src/snapshots/rust_norg__tests__inline_verbatim.snap index ef69353..83e63da 100644 --- a/src/snapshots/rust_norg__tests__inline_verbatim.snap +++ b/src/snapshots/rust_norg__tests__inline_verbatim.snap @@ -1,6 +1,5 @@ --- source: src/lib.rs -assertion_line: 588 expression: examples --- - - Paragraph: diff --git a/src/snapshots/rust_norg__tests__links.snap b/src/snapshots/rust_norg__tests__links.snap index ccef027..f23012e 100644 --- a/src/snapshots/rust_norg__tests__links.snap +++ b/src/snapshots/rust_norg__tests__links.snap @@ -1,6 +1,5 @@ --- source: src/lib.rs -assertion_line: 153 expression: examples --- - - Paragraph: diff --git a/src/snapshots/rust_norg__tests__ranged_verbatim_tags.snap b/src/snapshots/rust_norg__tests__ranged_verbatim_tags.snap index 2361962..a0729c9 100644 --- a/src/snapshots/rust_norg__tests__ranged_verbatim_tags.snap +++ b/src/snapshots/rust_norg__tests__ranged_verbatim_tags.snap @@ -13,14 +13,11 @@ expression: examples - some-text - here parameters: - - "lua\\ language" - - second-parameter - content: "print(\"Hello world!\")\n" + - "lua\\ language second-parameter print(\"Hello world!\") " + content: "" - - VerbatimRangedTag: name: - some-complex_tag_ parameters: - - first-parameter - - "#&*(&$!)" - - third-parameter - content: "function hello()\n print(\"Hello World\")\nend\n\nhello()\n" + - "first-parameter #&*(&$!) third-parameter function hello() print(\"Hello World\") end hello() " + content: "" diff --git a/src/snapshots/rust_norg__tests__verbatim_tags.snap b/src/snapshots/rust_norg__tests__verbatim_tags.snap deleted file mode 100644 index 217829b..0000000 --- a/src/snapshots/rust_norg__tests__verbatim_tags.snap +++ /dev/null @@ -1,160 +0,0 @@ ---- -source: src/main.rs -expression: examples ---- -- - RangedTag: - name: - - example - parameters: [] - content: - - Paragraph: - - Text: Hello - - Whitespace - - Text: world - - Special: "!" -- - RangedTag: - name: - - example - - some-text - - here - parameters: - - one parameter - - second-parameter - content: - - CarryoverTag: - tag_type: Macro - name: - - carryover - parameters: [] - next_object: - Paragraph: - - Text: text - - Whitespace - - Text: within -- - RangedTag: - name: - - some-complex_tag_ - parameters: - - first-parameter - - "#&*(&$!)" - - third-parameter - content: - - Paragraph: - - Text: this - - Whitespace - - Text: is - - Whitespace - - Text: some - - Whitespace - - Text: text - - Whitespace - - Text: within -- - RangedTag: - name: - - example - parameters: [] - content: - - Heading: - level: 1 - title: - - Text: Hello - - Whitespace - - Text: world - - Special: "!" - extensions: [] -- - RangedTag: - name: - - example - parameters: [] - content: - - RangedTag: - name: - - example - parameters: [] - content: - - Heading: - level: 1 - title: - - Text: Hello - - Whitespace - - Text: world - - Special: "!" - extensions: [] -- - RangedTag: - name: - - example - parameters: [] - content: - - Paragraph: - - Text: Hello - - Whitespace - - Text: world - - Special: "!" -- - RangedTag: - name: - - example - - some-text - - here - parameters: - - one parameter - - second-parameter - content: - - CarryoverTag: - tag_type: Macro - name: - - carryover - parameters: [] - next_object: - Paragraph: - - Text: text - - Whitespace - - Text: within -- - RangedTag: - name: - - some-complex_tag_ - parameters: - - first-parameter - - "#&*(&$!)" - - third-parameter - content: - - Paragraph: - - Text: this - - Whitespace - - Text: is - - Whitespace - - Text: some - - Whitespace - - Text: text - - Whitespace - - Text: within -- - RangedTag: - name: - - example - parameters: [] - content: - - Heading: - level: 1 - title: - - Text: Hello - - Whitespace - - Text: world - - Special: "!" - extensions: [] -- - RangedTag: - name: - - example - parameters: [] - content: - - RangedTag: - name: - - example - parameters: [] - content: - - Heading: - level: 1 - title: - - Text: Hello - - Whitespace - - Text: world - - Special: "!" - extensions: [] diff --git a/src/stage_1.rs b/src/stage_1.rs index 00a35b8..b808931 100644 --- a/src/stage_1.rs +++ b/src/stage_1.rs @@ -61,7 +61,6 @@ pub fn stage_1<'src>() -> impl Parser<'src, &'src str, Vec, extra::Er .filter(|c: &char| *c == '\n' || *c == '\r' || c.is_separator_line() || c.is_separator_paragraph()); let newline = parse_newline - .clone() .to(NorgToken::SingleNewline); let newlines = parse_newline diff --git a/src/stage_2.rs b/src/stage_2.rs index 0bf6f95..a9868fb 100644 --- a/src/stage_2.rs +++ b/src/stage_2.rs @@ -59,9 +59,6 @@ fn tokens_to_paragraph_segment(tokens: Vec) -> ParagraphTokenList { Some(NorgToken::Newlines(_)) => Some(ParagraphSegmentToken::Whitespace), Some(NorgToken::Eof) => Some(ParagraphSegmentToken::Text(String::new())), None => None, - _x => { - unreachable!(); - } }) .collect() } @@ -165,7 +162,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .map(|chars| chars.len() as u16) .then_ignore(whitespace.repeated().at_least(1).collect::>()) .then(extension_section.clone().or_not()) - .then(paragraph_segment.clone()) + .then(paragraph_segment) .then_ignore(newlines_or_eof) .map(|((level, extension_section), title)| NorgBlock::Heading { level, @@ -224,7 +221,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .map(|chars| (chars[0], chars.len() == 2)) .then_ignore(whitespace.repeated().at_least(1).collect::>()) .then(extension_section.clone().or_not()) - .then(paragraph_segment.clone()) + .then(paragraph_segment) .then_ignore(newlines_or_eof) .map(|(((modifier_type, ranged), extension_section), title)| { NorgBlock::RangeableDetachedModifier { @@ -259,7 +256,6 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .filter(move |tok: &NorgToken| !matches!(tok, NorgToken::Newlines(_) | NorgToken::SingleNewline | NorgToken::Whitespace(_) | NorgToken::Eof | NorgToken::End(_) if matches!(tok, NorgToken::End(x) if *x == c))); let tag_parameters = not_tag_end_or_ws - .clone() .repeated() .at_least(1) .collect::>() @@ -310,7 +306,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .repeated() .at_least(1) .collect::>() - .ignore_then(parameters.clone()) + .ignore_then(parameters) .or_not(), ) .then_ignore(select! { @@ -337,7 +333,7 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e .repeated() .at_least(1) .collect::>() - .ignore_then(parameters.clone()) + .ignore_then(parameters) .or_not(), ) .then_ignore(select! { @@ -415,7 +411,6 @@ pub fn stage_2<'src>() -> impl Parser<'src, &'src [NorgToken], Vec, e carryover_tags, tag_end, paragraph_segment - .clone() .then(newlines_or_eof.repeated().at_least(1).collect::>().rewind()) .map(|(content, trailing)| match trailing.last().unwrap() { NorgToken::Eof => { diff --git a/src/stage_3.rs b/src/stage_3.rs index 6b97e58..38c748b 100644 --- a/src/stage_3.rs +++ b/src/stage_3.rs @@ -164,7 +164,7 @@ fn paragraph_parser_opener_candidates_and_links<'a>() -> impl Parser< |(((filepath, modifiers), content), description)| ParagraphSegment::Link { filepath: filepath .map(|content| content.into_iter().map_into::().collect()), - description: description.map(|content| parse_paragraph(content)), + description: description.map(parse_paragraph), targets: if let Some(content) = content { vec![if let Some(modifiers) = modifiers { match modifiers.as_str() { @@ -216,7 +216,7 @@ fn paragraph_parser_opener_candidates_and_links<'a>() -> impl Parser< .then(anchor.clone().or_not()) .map(|(content, description)| ParagraphSegment::Anchor { content: parse_paragraph(content), - description: description.map(|content| parse_paragraph(content)), + description: description.map(parse_paragraph), }), inline_linkable, opening_modifier_candidate, @@ -235,6 +235,7 @@ fn paragraph_parser_opener_candidates_and_links<'a>() -> impl Parser< }) } +#[allow(clippy::result_large_err)] fn dedup_opener_candidates(input: Vec) -> Vec { use ParagraphSegment::*; @@ -642,7 +643,7 @@ pub fn stage_3<'src>( NorgBlock::NestableDetachedModifier { modifier_type: '-', level, extension_section } => (NestableDetachedModifier::UnorderedList, level, extension_section), NorgBlock::NestableDetachedModifier { modifier_type: '~', level, extension_section } => (NestableDetachedModifier::OrderedList, level, extension_section), NorgBlock::NestableDetachedModifier { modifier_type: '>', level, extension_section } => (NestableDetachedModifier::Quote, level, extension_section), - }.then(paragraph.clone()).map(|((modifier_type, level, extension_section), paragraph)| NorgASTFlat::NestableDetachedModifier { + }.then(paragraph).map(|((modifier_type, level, extension_section), paragraph)| NorgASTFlat::NestableDetachedModifier { modifier_type, level, extensions: detached_modifier_extensions().parse(&extension_section[..]).into_result().unwrap_or_default(), @@ -653,7 +654,7 @@ pub fn stage_3<'src>( NorgBlock::RangeableDetachedModifier { modifier_type: '$', ranged: false, title, extension_section } => (RangeableDetachedModifier::Definition, title, extension_section), NorgBlock::RangeableDetachedModifier { modifier_type: '^', ranged: false, title, extension_section} => (RangeableDetachedModifier::Footnote, title, extension_section), NorgBlock::RangeableDetachedModifier { modifier_type: ':', ranged: false, title, extension_section } => (RangeableDetachedModifier::Table, title, extension_section), - }.then(paragraph.clone()).map(|((modifier_type, title, extension_section), paragraph)| NorgASTFlat::RangeableDetachedModifier { + }.then(paragraph).map(|((modifier_type, title, extension_section), paragraph)| NorgASTFlat::RangeableDetachedModifier { modifier_type, title: parse_paragraph(title), extensions: detached_modifier_extensions().parse(&extension_section[..]).into_result().unwrap_or_default(),