base and extension fully compiles, host in good shape

orchid-host/src/lex.rs

@@ -1,18 +1,22 @@
 use std::num::NonZeroU64;
 use std::sync::Arc;
 
+use async_std::sync::Mutex;
+use futures::FutureExt;
 use hashbrown::HashMap;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
-use orchid_base::interner::{Tok, intern};
+use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
+use orchid_base::match_mapping;
 use orchid_base::number::{num_to_err, parse_num};
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::PARENS;
-use orchid_base::tree::Ph;
-use orchid_base::{intern, match_mapping};
+use orchid_base::tree::{AtomRepr, Ph};
 
 use crate::api;
-use crate::extension::{AtomHand, System};
+use crate::atom::AtomHand;
+use crate::ctx::Ctx;
+use crate::system::System;
 use crate::tree::{ParsTok, ParsTokTree};
 
 pub struct LexCtx<'a> {
@@ -20,6 +24,7 @@ pub struct LexCtx<'a> {
 	pub source: &'a Tok<String>,
 	pub tail: &'a str,
 	pub sub_trees: &'a mut HashMap<api::TreeTicket, ParsTokTree>,
+	pub ctx: &'a Ctx,
 }
 impl<'a> LexCtx<'a> {
 	pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
@@ -29,6 +34,7 @@ impl<'a> LexCtx<'a> {
 			tail: &self.source[pos as usize..],
 			systems: self.systems,
 			sub_trees: &mut *self.sub_trees,
+			ctx: &self.ctx,
 		}
 	}
 	pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 }
@@ -70,7 +76,7 @@ impl<'a> LexCtx<'a> {
 	}
 }
 
-pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
+pub async fn lex_once<'a>(ctx: &mut LexCtx<'a>) -> OrcRes<ParsTokTree> {
 	let start = ctx.get_pos();
 	assert!(
 		!ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space),
@@ -82,11 +88,13 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 	} else if ctx.strip_prefix("::") {
 		ParsTok::NS
 	} else if ctx.strip_prefix("--[") {
-		let (cmt, tail) = ctx.tail.split_once("]--").ok_or_else(|| {
-			mk_errv(intern!(str: "Unterminated block comment"), "This block comment has no ending ]--", [
-				Pos::Range(start..start + 3).into(),
-			])
-		})?;
+		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
+			return Err(mk_errv(
+				ctx.ctx.i.i("Unterminated block comment").await,
+				"This block comment has no ending ]--",
+				[Pos::Range(start..start + 3).into()],
+			));
+		};
 		ctx.set_tail(tail);
 		ParsTok::Comment(Arc::new(cmt.to_string()))
 	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
@@ -99,12 +107,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 		while !ctx.strip_char('.') {
 			if ctx.tail.is_empty() {
 				return Err(mk_errv(
-					intern!(str: "Unclosed lambda"),
+					ctx.ctx.i.i("Unclosed lambda").await,
 					"Lambdae started with \\ should separate arguments from body with .",
 					[Pos::Range(start..start + 1).into()],
 				));
 			}
-			arg.push(lex_once(ctx)?);
+			arg.push(lex_once(ctx).boxed_local().await?);
 			ctx.trim_ws();
 		}
 		ParsTok::LambdaHead(arg)
@@ -114,12 +122,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 		while !ctx.strip_char(*rp) {
 			if ctx.tail.is_empty() {
 				return Err(mk_errv(
-					intern!(str: "unclosed paren"),
+					ctx.ctx.i.i("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
 					[Pos::Range(start..start + 1).into()],
 				));
 			}
-			body.push(lex_once(ctx)?);
+			body.push(lex_once(ctx).boxed_local().await?);
 			ctx.trim_ws();
 		}
 		ParsTok::S(*paren, body)
@@ -130,8 +138,10 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 		if ctx.strip_char('(') {
 			let pos = ctx.get_pos();
 			let numstr = ctx.get_start_matches(|x| x != ')').trim();
-			let num = parse_num(numstr).map_err(|e| num_to_err(e, pos))?;
-			ParsTok::Macro(Some(num.to_f64()))
+			match parse_num(numstr) {
+				Ok(num) => ParsTok::Macro(Some(num.to_f64())),
+				Err(e) => return Err(num_to_err(e, pos, &*ctx.ctx.i).await.into()),
+			}
 		} else {
 			ParsTok::Macro(None)
 		}
@@ -139,17 +149,26 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 		for sys in ctx.systems {
 			let mut errors = Vec::new();
 			if ctx.tail.starts_with(|c| sys.can_lex(c)) {
-				let lx =
-					sys.lex(ctx.source.clone(), ctx.get_pos(), |pos| match lex_once(&mut ctx.push(pos)) {
-						Ok(t) => Some(api::SubLexed { pos, ticket: ctx.add_subtree(t) }),
-						Err(e) => {
-							errors.push(e);
-							None
-						},
-					});
+				let (source, pos) = (ctx.source.clone(), ctx.get_pos());
+				let ctx_lck = &Mutex::new(&mut *ctx);
+				let errors_lck = &Mutex::new(&mut errors);
+				let lx = sys
+					.lex(source, pos, |pos| async move {
+						match lex_once(&mut ctx_lck.lock().await.push(pos)).boxed_local().await {
+							Ok(t) => Some(api::SubLexed { pos, ticket: ctx_lck.lock().await.add_subtree(t) }),
+							Err(e) => {
+								errors_lck.lock().await.push(e);
+								None
+							},
+						}
+					})
+					.await;
 				match lx {
-					Err(e) => return Err(errors.into_iter().fold(OrcErrv::from_api(&e), |a, b| a + b)),
-					Ok(Some(lexed)) => return Ok(tt_to_owned(&lexed.expr, &mut ctx.push(lexed.pos))),
+					Err(e) =>
+						return Err(
+							errors.into_iter().fold(OrcErrv::from_api(&e, &*ctx.ctx.i).await, |a, b| a + b),
+						),
+					Ok(Some(lexed)) => return Ok(tt_to_owned(&lexed.expr, &mut ctx.push(lexed.pos)).await),
 					Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
 						Some(errors) => return Err(errors),
 						None => continue,
@@ -158,12 +177,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 			}
 		}
 		if ctx.tail.starts_with(name_start) {
-			ParsTok::Name(intern(ctx.get_start_matches(name_char)))
+			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(name_char)).await)
 		} else if ctx.tail.starts_with(op_char) {
-			ParsTok::Name(intern(ctx.get_start_matches(op_char)))
+			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(op_char)).await)
 		} else {
 			return Err(mk_errv(
-				intern!(str: "Unrecognized character"),
+				ctx.ctx.i.i("Unrecognized character").await,
 				"The following syntax is meaningless.",
 				[Pos::Range(start..start + 1).into()],
 			));
@@ -172,16 +191,18 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 	Ok(ParsTokTree { tok, range: start..ctx.get_pos() })
 }
 
-fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
+async fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
 	let tok = match_mapping!(&api.token, api::Token => ParsTok {
-		Atom(atom => AtomHand::from_api(atom.clone())),
-		Bottom(err => OrcErrv::from_api(err)),
-		LambdaHead(arg => ttv_to_owned(arg, ctx)),
-		Name(name => Tok::from_api(*name)),
-		S(p.clone(), b.iter().map(|t| tt_to_owned(t, ctx)).collect()),
+		Atom(atom =>
+			AtomHand::from_api(atom, Pos::Range(api.range.clone()), &mut ctx.ctx.clone()).await
+		),
+		Bottom(err => OrcErrv::from_api(err, &*ctx.ctx.i).await),
+		LambdaHead(arg => ttv_to_owned(arg, ctx).boxed_local().await),
+		Name(name => Tok::from_api(*name, &*ctx.ctx.i).await),
+		S(p.clone(), b => ttv_to_owned(b, ctx).boxed_local().await),
 		BR, NS,
 		Comment(c.clone()),
-		Ph(ph => Ph::from_api(ph)),
+		Ph(ph => Ph::from_api(ph, &*ctx.ctx.i).await),
 		Macro(*prio),
 	} {
 		api::Token::Slot(id) => return ctx.rm_subtree(*id),
@@ -189,20 +210,24 @@ fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
 	ParsTokTree { range: api.range.clone(), tok }
 }
 
-fn ttv_to_owned<'a>(
+async fn ttv_to_owned<'a>(
 	api: impl IntoIterator<Item = &'a api::TokenTree>,
 	ctx: &mut LexCtx<'_>,
 ) -> Vec<ParsTokTree> {
-	api.into_iter().map(|t| tt_to_owned(t, ctx)).collect()
+	let mut out = Vec::new();
+	for tt in api {
+		out.push(tt_to_owned(&tt, ctx).await)
+	}
+	out
 }
 
-pub fn lex(text: Tok<String>, systems: &[System]) -> OrcRes<Vec<ParsTokTree>> {
+pub async fn lex(text: Tok<String>, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
 	let mut sub_trees = HashMap::new();
-	let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems };
+	let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, ctx };
 	let mut tokv = Vec::new();
 	ctx.trim(unrep_space);
 	while !ctx.tail.is_empty() {
-		tokv.push(lex_once(&mut ctx)?);
+		tokv.push(lex_once(&mut ctx).await?);
 		ctx.trim(unrep_space);
 	}
 	Ok(tokv)