base and extension fully compiles, host in good shape

orchid-host/src/parse.rs

@@ -1,10 +1,10 @@
-use std::sync::Arc;
-use std::{iter, thread};
+use std::rc::Rc;
 
+use futures::FutureExt;
+use futures::future::join_all;
 use itertools::Itertools;
 use never::Never;
 use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_err, mk_errv};
-use orchid_base::intern;
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::macros::{MTok, MTree};
@@ -16,8 +16,9 @@ use orchid_base::parse::{
 use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
 
-use crate::extension::{AtomHand, System};
+use crate::atom::AtomHand;
 use crate::macros::MacTree;
+use crate::system::System;
 use crate::tree::{
 	Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, ParsTokTree, Rule, RuleKind,
 };
@@ -29,111 +30,104 @@ pub trait ParseCtx: Send + Sync {
 	fn reporter(&self) -> &impl Reporter;
 }
 
-pub fn parse_items(
+pub async fn parse_items(
 	ctx: &impl ParseCtx,
-	path: Substack<Tok<String>>,
-	items: ParsSnippet,
+	path: Substack<'_, Tok<String>>,
+	items: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	let lines = line_items(items);
-	let mut ok = iter::from_fn(|| None).take(lines.len()).collect_vec();
-	thread::scope(|s| {
-		let mut threads = Vec::new();
-		for (slot, Parsed { output: cmts, tail }) in ok.iter_mut().zip(lines.into_iter()) {
-			let path = &path;
-			threads.push(s.spawn(move || {
-				*slot = Some(parse_item(ctx, path.clone(), cmts, tail)?);
-				Ok::<(), OrcErrv>(())
-			}))
-		}
-		for t in threads {
-			t.join().unwrap().err().into_iter().flatten().for_each(|e| ctx.reporter().report(e))
-		}
-	});
-	Ok(ok.into_iter().flatten().flatten().collect_vec())
+	let lines = line_items(items).await;
+	let line_res =
+		join_all(lines.into_iter().map(|p| parse_item(ctx, path.clone(), p.output, p.tail))).await;
+	Ok(line_res.into_iter().flat_map(|l| l.ok().into_iter().flatten()).collect())
 }
 
-pub fn parse_item(
+pub async fn parse_item(
 	ctx: &impl ParseCtx,
-	path: Substack<Tok<String>>,
+	path: Substack<'_, Tok<String>>,
 	comments: Vec<Comment>,
-	item: ParsSnippet,
+	item: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
 	match item.pop_front() {
 		Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
-			n if *n == intern!(str: "export") => match try_pop_no_fluff(postdisc)? {
+			n if *n == item.i("export").await => match try_pop_no_fluff(postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
-					parse_exportable_item(ctx, path, comments, true, n.clone(), tail),
+					parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
 				Parsed { output: TokTree { tok: Token::NS, .. }, tail } => {
-					let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail)?;
+					let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail).await?;
 					let mut ok = Vec::new();
-					exports.into_iter().for_each(|(e, pos)| match (&e.path.as_slice(), e.name) {
-						([], Some(n)) =>
-							ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }),
-						(_, Some(_)) => ctx.reporter().report(mk_err(
-							intern!(str: "Compound export"),
-							"Cannot export compound names (names containing the :: separator)",
-							[pos.into()],
-						)),
-						(_, None) => ctx.reporter().report(mk_err(
-							intern!(str: "Wildcard export"),
-							"Exports cannot contain the globstar *",
-							[pos.into()],
-						)),
-					});
-					expect_end(tail)?;
+					for (e, pos) in exports {
+						match (&e.path.as_slice(), e.name) {
+							([], Some(n)) =>
+								ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }),
+							(_, Some(_)) => ctx.reporter().report(mk_err(
+								tail.i("Compound export").await,
+								"Cannot export compound names (names containing the :: separator)",
+								[pos.into()],
+							)),
+							(_, None) => ctx.reporter().report(mk_err(
+								tail.i("Wildcard export").await,
+								"Exports cannot contain the globstar *",
+								[pos.into()],
+							)),
+						}
+					}
+					expect_end(tail).await?;
 					Ok(ok)
 				},
-				Parsed { output, .. } => Err(mk_errv(
-					intern!(str: "Malformed export"),
+				Parsed { output, tail } => Err(mk_errv(
+					tail.i("Malformed export").await,
 					"`export` can either prefix other lines or list names inside ::( ) or ::[ ]",
 					[Pos::Range(output.range.clone()).into()],
 				)),
 			},
-			n if *n == intern!(str: "import") => parse_import(ctx, postdisc).map(|v| {
+			n if *n == item.i("import").await => parse_import(ctx, postdisc).await.map(|v| {
 				Vec::from_iter(v.into_iter().map(|(t, pos)| Item {
 					comments: comments.clone(),
 					pos,
 					kind: ItemKind::Import(t),
 				}))
 			}),
-			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc),
+			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
 		},
 		Some(_) =>
-			Err(mk_errv(intern!(str: "Expected a line type"), "All lines must begin with a keyword", [
+			Err(mk_errv(item.i("Expected a line type").await, "All lines must begin with a keyword", [
 				Pos::Range(item.pos()).into(),
 			])),
 		None => unreachable!("These lines are filtered and aggregated in earlier stages"),
 	}
 }
 
-pub fn parse_import(ctx: &impl ParseCtx, tail: ParsSnippet) -> OrcRes<Vec<(Import, Pos)>> {
-	let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail)?;
-	expect_end(tail)?;
+pub async fn parse_import(
+	ctx: &impl ParseCtx,
+	tail: ParsSnippet<'_>,
+) -> OrcRes<Vec<(Import, Pos)>> {
+	let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail).await?;
+	expect_end(tail).await?;
 	Ok(imports)
 }
 
-pub fn parse_exportable_item(
+pub async fn parse_exportable_item(
 	ctx: &impl ParseCtx,
-	path: Substack<Tok<String>>,
+	path: Substack<'_, Tok<String>>,
 	comments: Vec<Comment>,
 	exported: bool,
 	discr: Tok<String>,
-	tail: ParsSnippet,
+	tail: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	let kind = if discr == intern!(str: "mod") {
-		let (name, body) = parse_module(ctx, path, tail)?;
+	let kind = if discr == tail.i("mod").await {
+		let (name, body) = parse_module(ctx, path, tail).await?;
 		ItemKind::Member(Member::new(name, MemberKind::Mod(body)))
-	} else if discr == intern!(str: "const") {
-		let (name, val) = parse_const(tail)?;
-		let locator = CodeLocator::to_const(path.push(name.clone()).unreverse());
+	} else if discr == tail.i("const").await {
+		let (name, val) = parse_const(tail).await?;
+		let locator = CodeLocator::to_const(tail.i(&path.push(name.clone()).unreverse()).await);
 		ItemKind::Member(Member::new(name, MemberKind::Const(Code::from_code(locator, val))))
 	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
-		let line = sys.parse(tail.to_vec(), exported, comments)?;
-		return parse_items(ctx, path, Snippet::new(tail.prev(), &line));
+		let line = sys.parse(tail.to_vec(), exported, comments).await?;
+		return parse_items(ctx, path, Snippet::new(tail.prev(), &line, tail.interner())).await;
 	} else {
 		let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
 		return Err(mk_errv(
-			intern!(str: "Unrecognized line type"),
+			tail.i("Unrecognized line type").await,
 			format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
 			[Pos::Range(tail.prev().range.clone()).into()],
 		));
@@ -141,82 +135,90 @@ pub fn parse_exportable_item(
 	Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }])
 }
 
-pub fn parse_module(
+pub async fn parse_module(
 	ctx: &impl ParseCtx,
-	path: Substack<Tok<String>>,
-	tail: ParsSnippet,
+	path: Substack<'_, Tok<String>>,
+	tail: ParsSnippet<'_>,
 ) -> OrcRes<(Tok<String>, Module)> {
-	let (name, tail) = match try_pop_no_fluff(tail)? {
+	let (name, tail) = match try_pop_no_fluff(tail).await? {
 		Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
 		Parsed { output, .. } => {
 			return Err(mk_errv(
-				intern!(str: "Missing module name"),
-				format!("A name was expected, {output} was found"),
+				tail.i("Missing module name").await,
+				format!("A name was expected, {} was found", output.print().await),
 				[Pos::Range(output.range.clone()).into()],
 			));
 		},
 	};
-	let Parsed { output, tail: surplus } = try_pop_no_fluff(tail)?;
-	expect_end(surplus)?;
-	let body = output.as_s(Paren::Round).ok_or_else(|| {
-		mk_errv(
-			intern!(str: "Expected module body"),
-			format!("A ( block ) was expected, {output} was found"),
+	let Parsed { output, tail: surplus } = try_pop_no_fluff(tail).await?;
+	expect_end(surplus).await?;
+	let Some(body) = output.as_s(Paren::Round, tail.interner()) else {
+		return Err(mk_errv(
+			tail.i("Expected module body").await,
+			format!("A ( block ) was expected, {} was found", output.print().await),
 			[Pos::Range(output.range.clone()).into()],
-		)
-	})?;
+		));
+	};
 	let path = path.push(name.clone());
-	Ok((name, Module::new(parse_items(ctx, path, body)?)))
+	Ok((name, Module::new(parse_items(ctx, path, body).await?)))
 }
 
-pub fn parse_const(tail: ParsSnippet) -> OrcRes<(Tok<String>, Vec<ParsTokTree>)> {
-	let Parsed { output, tail } = try_pop_no_fluff(tail)?;
-	let name = output.as_name().ok_or_else(|| {
-		mk_errv(
-			intern!(str: "Missing module name"),
-			format!("A name was expected, {output} was found"),
-			[Pos::Range(output.range.clone()).into()],
-		)
-	})?;
-	let Parsed { output, tail } = try_pop_no_fluff(tail)?;
-	if !output.is_kw(intern!(str: "=")) {
+pub async fn parse_const(tail: ParsSnippet<'_>) -> OrcRes<(Tok<String>, Vec<ParsTokTree>)> {
+	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
+	let Some(name) = output.as_name() else {
 		return Err(mk_errv(
-			intern!(str: "Missing walrus := separator"),
-			format!("Expected operator := , found {output}"),
+			tail.i("Missing module name").await,
+			format!("A name was expected, {} was found", output.print().await),
+			[Pos::Range(output.range.clone()).into()],
+		));
+	};
+	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
+	if !output.is_kw(tail.i("=").await) {
+		return Err(mk_errv(
+			tail.i("Missing walrus := separator").await,
+			format!("Expected operator := , found {}", output.print().await),
 			[Pos::Range(output.range.clone()).into()],
 		));
 	}
-	try_pop_no_fluff(tail)?;
+	try_pop_no_fluff(tail).await?;
 	Ok((name, tail.iter().flat_map(strip_fluff).collect_vec()))
 }
 
-pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
+pub async fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
 	let mut mtreev = Vec::new();
 	while let Some((ttree, tail)) = snip.pop_front() {
 		let (range, tok, tail) = match &ttree.tok {
-			Token::S(p, b) =>
-				(ttree.range.clone(), MTok::S(*p, parse_mtree(Snippet::new(ttree, b))?), tail),
+			Token::S(p, b) => (
+				ttree.range.clone(),
+				MTok::S(*p, parse_mtree(Snippet::new(ttree, b, snip.interner())).boxed_local().await?),
+				tail,
+			),
 			Token::Name(tok) => {
 				let mut segments = vec![tok.clone()];
 				let mut end = ttree.range.end;
 				while let Some((TokTree { tok: Token::NS, .. }, tail)) = snip.pop_front() {
-					let Parsed { output, tail } = try_pop_no_fluff(tail)?;
-					segments.push(output.as_name().ok_or_else(|| {
-						mk_errv(
-							intern!(str: "Namespaced name interrupted"),
+					let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
+					let Some(seg) = output.as_name() else {
+						return Err(mk_errv(
+							tail.i("Namespaced name interrupted").await,
 							"In expression context, :: must always be followed by a name.\n\
-            ::() is permitted only in import and export items",
+											::() is permitted only in import and export items",
 							[Pos::Range(output.range.clone()).into()],
-						)
-					})?);
+						));
+					};
+					segments.push(seg);
 					snip = tail;
 					end = output.range.end;
 				}
-				(ttree.range.start..end, MTok::Name(Sym::new(segments).unwrap()), snip)
+				(
+					ttree.range.start..end,
+					MTok::Name(Sym::new(segments, snip.interner()).await.unwrap()),
+					snip,
+				)
 			},
 			Token::NS => {
 				return Err(mk_errv(
-					intern!(str: "Unexpected :: in macro pattern"),
+					tail.i("Unexpected :: in macro pattern").await,
 					":: can only follow a name outside export statements",
 					[Pos::Range(ttree.range.clone()).into()],
 				));
@@ -224,8 +226,11 @@ pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
 			Token::Ph(ph) => (ttree.range.clone(), MTok::Ph(ph.clone()), tail),
 			Token::Atom(_) | Token::Macro(_) => {
 				return Err(mk_errv(
-					intern!(str: "Unsupported token in macro patterns"),
-					format!("Macro patterns can only contain names, braces, and lambda, not {ttree}."),
+					tail.i("Unsupported token in macro patterns").await,
+					format!(
+						"Macro patterns can only contain names, braces, and lambda, not {}.",
+						ttree.print().await
+					),
 					[Pos::Range(ttree.range.clone()).into()],
 				));
 			},
@@ -233,50 +238,57 @@ pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
 			Token::Bottom(e) => return Err(e.clone()),
 			Token::LambdaHead(arg) => (
 				ttree.range.start..snip.pos().end,
-				MTok::Lambda(parse_mtree(Snippet::new(ttree, arg))?, parse_mtree(tail)?),
-				Snippet::new(ttree, &[]),
+				MTok::Lambda(
+					parse_mtree(Snippet::new(ttree, arg, snip.interner())).await?,
+					parse_mtree(tail).await?,
+				),
+				Snippet::new(ttree, &[], snip.interner()),
 			),
-			Token::Slot(_) | Token::X(_) => panic!("Did not expect {} in parsed token tree", &ttree.tok),
+			Token::Slot(_) | Token::X(_) =>
+				panic!("Did not expect {} in parsed token tree", &ttree.tok.print().await),
 		};
-		mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Arc::new(tok) });
+		mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Rc::new(tok) });
 		snip = tail;
 	}
 	Ok(mtreev)
 }
 
-pub fn parse_macro(
-	tail: ParsSnippet,
+pub async fn parse_macro(
+	tail: ParsSnippet<'_>,
 	macro_i: u16,
-	path: Substack<Tok<String>>,
+	path: Substack<'_, Tok<String>>,
 ) -> OrcRes<Vec<Rule>> {
-	let (surplus, prev, block) = match try_pop_no_fluff(tail)? {
+	let (surplus, prev, block) = match try_pop_no_fluff(tail).await? {
 		Parsed { tail, output: o @ TokTree { tok: Token::S(Paren::Round, b), .. } } => (tail, o, b),
 		Parsed { output, .. } => {
 			return Err(mk_errv(
-				intern!(str: "m"),
+				tail.i("m").await,
 				"Macro blocks must either start with a block or a ..$:number",
 				[Pos::Range(output.range.clone()).into()],
 			));
 		},
 	};
-	expect_end(surplus)?;
+	expect_end(surplus).await?;
 	let mut errors = Vec::new();
 	let mut rules = Vec::new();
-	for (i, item) in line_items(Snippet::new(prev, block)).into_iter().enumerate() {
-		let Parsed { tail, output } = try_pop_no_fluff(item.tail)?;
-		if !output.is_kw(intern!(str: "rule")) {
+	for (i, item) in
+		line_items(Snippet::new(&prev, block, tail.interner())).await.into_iter().enumerate()
+	{
+		let Parsed { tail, output } = try_pop_no_fluff(item.tail).await?;
+		if !output.is_kw(tail.i("rule").await) {
 			errors.extend(mk_errv(
-				intern!(str: "non-rule in macro"),
-				format!("Expected `rule`, got {output}"),
+				tail.i("non-rule in macro").await,
+				format!("Expected `rule`, got {}", output.print().await),
 				[Pos::Range(output.range.clone()).into()],
 			));
 			continue;
 		};
-		let (pat, body) = match tail.split_once(|t| t.is_kw(intern!(str: "=>"))) {
+		let arrow = tail.i("=>").await;
+		let (pat, body) = match tail.split_once(|t| t.is_kw(arrow.clone())) {
 			Some((a, b)) => (a, b),
 			None => {
 				errors.extend(mk_errv(
-					intern!(str: "no => in macro rule"),
+					tail.i("no => in macro rule").await,
 					"The pattern and body of a rule must be separated by a =>",
 					[Pos::Range(tail.pos()).into()],
 				));
@@ -286,9 +298,9 @@ pub fn parse_macro(
 		rules.push(Rule {
 			comments: item.output,
 			pos: Pos::Range(tail.pos()),
-			pattern: parse_mtree(pat)?,
+			pattern: parse_mtree(pat).await?,
 			kind: RuleKind::Native(Code::from_code(
-				CodeLocator::to_rule(path.unreverse(), macro_i, i as u16),
+				CodeLocator::to_rule(tail.i(&path.unreverse()).await, macro_i, i as u16),
 				body.to_vec(),
 			)),
 		})