First steps for the macro system

orchid-std/src/macros/instantiate_tpl.rs

@@ -1,9 +1,7 @@
 use std::borrow::Cow;
 use std::pin::Pin;
-use std::rc::Rc;
 
 use futures::AsyncWrite;
-use itertools::Itertools;
 use never::Never;
 use orchid_extension::atom::{Atomic, TypAtom};
 use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant, get_own_instance};

orchid-std/src/macros/let_line.rs

@@ -1,20 +1,17 @@
-use futures::future::LocalBoxFuture;
+use async_std::stream;
+use async_stream::stream;
+use futures::{FutureExt, StreamExt};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::error::{OrcRes, mk_errv};
-use orchid_base::interner::Tok;
-use orchid_base::name::Sym;
-use orchid_base::parse::{Comment, ParseCtx, Parsed, expect_tok, token_errv, try_pop_no_fluff};
-use orchid_extension::gen_expr::GExpr;
-use orchid_extension::parser::{
-	ConstCtx, GenSnippet, ParsCtx, ParsedLine, ParsedLineKind, ParsedMem, ParsedMemKind, Parser,
+use orchid_base::error::OrcRes;
+use orchid_base::parse::{
+	Comment, ParseCtx, Parsed, Snippet, expect_tok, token_errv, try_pop_no_fluff,
+};
+use orchid_extension::parser::{
+	PSnippet, PTok, PTokTree, ParsCtx, ParsedLine, ParsedLineKind, ParsedMem, ParsedMemKind, Parser,
 };
-use substack::Substack;
 
-use crate::macros::mactree::{MacTok, MacTree, map_mactree, map_mactree_v};
-
-type ExprGenerator =
-	Box<dyn for<'a> FnOnce(ConstCtx, Substack<'a, Sym>) -> LocalBoxFuture<'a, GExpr>>;
+use crate::macros::mactree::{MacTok, MacTree, map_mactree_v};
 
 #[derive(Default)]
 pub struct LetLine;
@@ -24,7 +21,7 @@ impl Parser for LetLine {
 		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'a>,
+		line: PSnippet<'a>,
 	) -> OrcRes<Vec<ParsedLine>> {
 		let Parsed { output: name_tok, tail } = try_pop_no_fluff(&ctx, line).await?;
 		let Some(name) = name_tok.as_name() else {
@@ -34,19 +31,11 @@ impl Parser for LetLine {
 			return Err(err.await);
 		};
 		let Parsed { tail, .. } = expect_tok(&ctx, tail, ctx.i().i("=").await).await?;
-		fn parse_tokv(line: GenSnippet<'_>) -> Vec<MacTree> {
-			let first: MacTree = if let Some((idx, arg)) =
-				line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?)))
-			{
-			} else {
-			};
-			todo!()
-		}
 		let mut names = HashMap::new();
-		let aliased = parse_tokv(tail);
+		let aliased = parse_tokv(tail, &ctx).await;
 		map_mactree_v(&aliased, &mut false, &mut |tpl| {
 			if let MacTok::Name(n) = &*tpl.tok {
-				names.insert(n.clone(), n);
+				names.insert(n.clone(), n.clone());
 			}
 			None
 		});
@@ -56,14 +45,76 @@ impl Parser for LetLine {
 			kind: ParsedLineKind::Mem(ParsedMem {
 				exported,
 				name,
-				kind: ParsedMemKind::cnst(async |ctx| {
-					let mut names_str = ctx.names(names.keys());
-					while let Some(()) = names_str.next().await {
-						names[]
+				kind: ParsedMemKind::cnst(async move |ctx| {
+					let keys = names.keys().cloned().collect_vec();
+					let names_mut = &mut names;
+					stream! {
+						for await (canonical, local) in ctx.names(&keys).zip(stream::from_iter(&keys)) {
+							if let Some(name) = canonical {
+								*names_mut.get_mut(local).expect("Queried specifically the keys of this map") = name
+							}
+						}
 					}
-					todo!()
+					.collect::<()>()
+					.await;
+					let macro_input = map_mactree_v(&aliased, &mut false, &mut |tree| match &*tree.tok {
+						MacTok::Name(n) => names.get(n).map(|new_n| MacTok::Name(new_n.clone()).at(tree.pos())),
+						_ => None,
+					});
+					todo!("Run macros then convert this into an expr")
 				}),
 			}),
 		}])
 	}
 }
+
+async fn parse_tokv(line: PSnippet<'_>, ctx: &ParsCtx<'_>) -> Vec<MacTree> {
+	if let Some((idx, arg)) = line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?))) {
+		let (head, lambda) = line.split_at(idx as u32);
+		let (_, body) = lambda.pop_front().unwrap();
+		let body = parse_tokv(body, ctx).boxed_local().await;
+		let mut all = parse_tokv_no_lambdas(&head, ctx).await;
+		match parse_tok(arg, ctx).await {
+			Some(arg) => all.push(MacTok::Lambda(arg, body).at(lambda.sr().pos())),
+			None => ctx.rep().report(
+				token_errv(ctx, arg, "Lambda argument fluff", |arg| {
+					format!("Lambda arguments must be a valid token, found meaningless fragment {arg}")
+				})
+				.await,
+			),
+		};
+		all
+	} else {
+		parse_tokv_no_lambdas(&line, ctx).await
+	}
+}
+
+async fn parse_tokv_no_lambdas(line: &[PTokTree], ctx: &ParsCtx<'_>) -> Vec<MacTree> {
+	stream::from_iter(line).filter_map(|tt| parse_tok(tt, ctx)).collect().await
+}
+
+async fn parse_tok(tree: &PTokTree, ctx: &ParsCtx<'_>) -> Option<MacTree> {
+	let tok = match &tree.tok {
+		PTok::Bottom(errv) => MacTok::Bottom(errv.clone()),
+		PTok::BR | PTok::Comment(_) => return None,
+		PTok::Name(n) => MacTok::Name(ctx.module().suffix([n.clone()], ctx.i()).await),
+		PTok::NS(..) => match tree.as_multiname() {
+			Ok(mn) => MacTok::Name(mn.to_sym(ctx.i()).await),
+			Err(nested) => {
+				ctx.rep().report(
+					token_errv(ctx, tree, ":: can only be followed by a name in an expression", |tok| {
+						format!("Expected name, found {tok}")
+					})
+					.await,
+				);
+				return parse_tok(nested, ctx).boxed_local().await;
+			},
+		},
+		PTok::Handle(expr) => MacTok::Value(expr.clone()),
+		PTok::NewExpr(never) => match *never {},
+		PTok::LambdaHead(_) => panic!("Lambda-head handled in the sequence parser"),
+		PTok::S(p, body) =>
+			MacTok::S(*p, parse_tokv(Snippet::new(tree, body), ctx).boxed_local().await),
+	};
+	Some(tok.at(tree.sr().pos()))
+}

orchid-std/src/macros/mactree.rs

@@ -2,14 +2,17 @@ use std::borrow::Cow;
 use std::fmt::Display;
 use std::rc::Rc;
 
+use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
 use orchid_api::Paren;
+use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::tl_cache;
+use orchid_base::tree::indent;
 use orchid_extension::atom::Atomic;
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
 use orchid_extension::expr::Expr;
@@ -20,7 +23,8 @@ pub struct MacTree {
 	pub tok: Rc<MacTok>,
 }
 impl MacTree {
-	fn tok(&self) -> &MacTok { &*self.tok }
+	pub fn tok(&self) -> &MacTok { &*self.tok }
+	pub fn pos(&self) -> Pos { self.pos.clone() }
 }
 impl Atomic for MacTree {
 	type Data = ();
@@ -46,6 +50,12 @@ pub enum MacTok {
 	/// Only permitted in "pattern" values produced by macro blocks, which are
 	/// never accessed as variables by usercode
 	Ph(Ph),
+	Bottom(OrcErrv),
+}
+impl MacTok {
+	pub fn at(self, pos: impl Into<Pos>) -> MacTree {
+		MacTree { pos: pos.into(), tok: Rc::new(self) }
+	}
 }
 impl Format for MacTok {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -55,7 +65,7 @@ impl Format for MacTok {
 				tl_cache!(Rc<Variants>: Rc::new(Variants::default()
 					.unbounded("\\{0b}.{1l}")
 					.bounded("(\\{0b}.{1b})"))),
-				[arg.print(c).await, mtreev_fmt(b, c).await],
+				[arg.print(c).boxed_local().await, mtreev_fmt(b, c).await],
 			),
 			Self::Name(n) => format!("{n}").into(),
 			Self::Ph(ph) => format!("{ph}").into(),
@@ -68,6 +78,8 @@ impl Format for MacTok {
 				[mtreev_fmt(body, c).await],
 			),
 			Self::Slot => "SLOT".into(),
+			Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()).into(),
+			Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),
 		}
 	}
 }
@@ -117,7 +129,8 @@ pub fn map_mactree<F: FnMut(MacTree) -> Option<MacTree>>(
 				ro(changed, |changed| map_mactree(arg, changed, map)),
 				map_mactree_v(body, changed, map),
 			),
-			MacTok::Name(_) | MacTok::Value(_) | MacTok::Slot | MacTok::Ph(_) => return src.clone(),
+			MacTok::Name(_) | MacTok::Value(_) | MacTok::Slot | MacTok::Ph(_) | MacTok::Bottom(_) =>
+				return src.clone(),
 			MacTok::S(p, body) => MacTok::S(*p, map_mactree_v(body, changed, map)),
 		},
 	};

orchid-std/src/macros/rule/build.rs

@@ -1,5 +1,9 @@
+use futures::FutureExt;
+use futures::future::join_all;
 use itertools::Itertools;
-use orchid_base::interner::Tok;
+use orchid_base::error::{OrcRes, mk_errv};
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::join_ok;
 use orchid_base::side::Side;
 
 use super::shared::{AnyMatcher, ScalMatcher, VecMatcher};
@@ -28,24 +32,31 @@ fn scal_cnt<'a>(iter: impl Iterator<Item = &'a MacTree>) -> usize {
 }
 
 #[must_use]
-pub fn mk_any(pattern: &[MacTree]) -> AnyMatcher {
+pub async fn mk_any(pattern: &[MacTree], i: &Interner) -> OrcRes<AnyMatcher> {
 	let left_split = scal_cnt(pattern.iter());
 	if pattern.len() <= left_split {
-		return AnyMatcher::Scalar(mk_scalv(pattern));
+		return Ok(AnyMatcher::Scalar(mk_scalv(pattern, i).await?));
 	}
 	let (left, not_left) = pattern.split_at(left_split);
 	let right_split = not_left.len() - scal_cnt(pattern.iter().rev());
 	let (mid, right) = not_left.split_at(right_split);
-	AnyMatcher::Vec { left: mk_scalv(left), mid: mk_vec(mid), right: mk_scalv(right) }
+	join_ok! {
+		left = mk_scalv(left, i).await;
+		mid = mk_vec(mid, i).await;
+		right = mk_scalv(right, i).await;
+	}
+	Ok(AnyMatcher::Vec { left, mid, right })
 }
 
 /// Pattern MUST NOT contain vectorial placeholders
 #[must_use]
-fn mk_scalv(pattern: &[MacTree]) -> Vec<ScalMatcher> { pattern.iter().map(mk_scalar).collect() }
+async fn mk_scalv(pattern: &[MacTree], i: &Interner) -> OrcRes<Vec<ScalMatcher>> {
+	join_all(pattern.iter().map(|pat| mk_scalar(pat, i))).await.into_iter().collect()
+}
 
 /// Pattern MUST start and end with a vectorial placeholder
 #[must_use]
-pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
+pub async fn mk_vec(pattern: &[MacTree], i: &Interner) -> OrcRes<VecMatcher> {
 	debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
 	debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
 	debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
@@ -57,39 +68,57 @@ pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
 	let (l_side, l_sep) = left.split_at(left.len() - l_sep_size);
 	let main = VecMatcher::Placeh { key: key.clone(), nonzero };
 	match (left, right) {
-		(&[], &[]) => VecMatcher::Placeh { key, nonzero },
-		(&[], _) => VecMatcher::Scan {
-			direction: Side::Left,
-			left: Box::new(main),
-			sep: mk_scalv(r_sep),
-			right: Box::new(mk_vec(r_side)),
+		(&[], &[]) => Ok(VecMatcher::Placeh { key, nonzero }),
+		(&[], _) => {
+			join_ok! {
+				sep = mk_scalv(r_sep, i).await;
+				right = mk_vec(r_side, i).boxed_local().await;
+			}
+			Ok(VecMatcher::Scan {
+				direction: Side::Left,
+				left: Box::new(main),
+				sep,
+				right: Box::new(right),
+			})
 		},
-		(_, &[]) => VecMatcher::Scan {
-			direction: Side::Right,
-			left: Box::new(mk_vec(l_side)),
-			sep: mk_scalv(l_sep),
-			right: Box::new(main),
+		(_, &[]) => {
+			join_ok! {
+				left = mk_vec(l_side, i).boxed_local().await;
+				sep = mk_scalv(l_sep, i).await;
+			}
+			Ok(VecMatcher::Scan {
+				direction: Side::Right,
+				left: Box::new(left),
+				sep,
+				right: Box::new(main),
+			})
 		},
 		(..) => {
 			let mut key_order =
 				l_side.iter().chain(r_side.iter()).filter_map(vec_attrs).collect::<Vec<_>>();
 			key_order.sort_by_key(|(_, prio, _)| -(*prio as i64));
-			VecMatcher::Middle {
-				left: Box::new(mk_vec(l_side)),
-				left_sep: mk_scalv(l_sep),
-				mid: Box::new(main),
-				right_sep: mk_scalv(r_sep),
-				right: Box::new(mk_vec(r_side)),
-				key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
+			join_ok! {
+				left = mk_vec(l_side, i).boxed_local().await;
+				left_sep = mk_scalv(l_sep, i).await;
+				right_sep = mk_scalv(r_sep, i).await;
+				right = mk_vec(r_side, i).boxed_local().await;
 			}
+			Ok(VecMatcher::Middle {
+				left: Box::new(left),
+				left_sep,
+				mid: Box::new(main),
+				right_sep,
+				right: Box::new(right),
+				key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
+			})
 		},
 	}
 }
 
 /// Pattern MUST NOT be a vectorial placeholder
 #[must_use]
-fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
-	match &*pattern.tok {
+async fn mk_scalar(pattern: &MacTree, i: &Interner) -> OrcRes<ScalMatcher> {
+	Ok(match &*pattern.tok {
 		MacTok::Name(n) => ScalMatcher::Name(n.clone()),
 		MacTok::Ph(Ph { name, kind }) => match kind {
 			PhKind::Vector { .. } => {
@@ -97,10 +126,16 @@ fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
 			},
 			PhKind::Scalar => ScalMatcher::Placeh { key: name.clone() },
 		},
-		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
-		MacTok::Lambda(arg, body) => ScalMatcher::Lambda(Box::new(mk_any(arg)), Box::new(mk_any(body))),
+		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body, i).boxed_local().await?)),
+		MacTok::Lambda(..) =>
+			return Err(mk_errv(
+				i.i("Lambda in matcher").await,
+				"Lambdas can't be matched for, only generated in templates",
+				[pattern.pos()],
+			)),
 		MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
-	}
+		MacTok::Bottom(errv) => return Err(errv.clone()),
+	})
 }
 
 #[cfg(test)]
@@ -150,7 +185,7 @@ mod test {
 				}))
 				.await,
 			];
-			let matcher = mk_any(&pattern);
+			let matcher = mk_any(&pattern, &i).await.expect("This matcher isn't broken");
 			println!("{matcher}");
 		})
 	}

orchid-std/src/macros/rule/matcher.rs

@@ -2,6 +2,7 @@ use std::fmt;
 use std::rc::Rc;
 
 use itertools::Itertools;
+use orchid_base::error::OrcRes;
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
@@ -20,21 +21,21 @@ pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwra
 
 pub struct NamedMatcher(AnyMatcher);
 impl NamedMatcher {
-	pub async fn new(pattern: &[MacTree], i: &Interner) -> Self {
+	pub async fn new(pattern: &[MacTree], i: &Interner) -> OrcRes<Self> {
 		assert!(
 			matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))),
 			"Named matchers must begin with a name"
 		);
 
-		match last_is_vec(pattern) {
-			true => Self(mk_any(pattern)),
+		Ok(Self(match last_is_vec(pattern) {
+			true => mk_any(pattern, i).await,
 			false => {
 				let kind = PhKind::Vector { priority: 0, at_least_one: false };
 				let tok = MacTok::Ph(Ph { name: i.i("::after").await, kind });
 				let suffix = [MacTree { pos: Pos::None, tok: Rc::new(tok) }];
-				Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
+				mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec(), i).await
 			},
-		}
+		}?))
 	}
 	/// Also returns the tail, if any, which should be matched further
 	/// Note that due to how priod works below, the main usable information from
@@ -62,12 +63,12 @@ impl fmt::Debug for NamedMatcher {
 
 pub struct PriodMatcher(VecMatcher);
 impl PriodMatcher {
-	pub fn new(pattern: &[MacTree]) -> Self {
+	pub async fn new(pattern: &[MacTree], i: &Interner) -> OrcRes<Self> {
 		assert!(
 			pattern.first().and_then(vec_attrs).is_some() && pattern.last().and_then(vec_attrs).is_some(),
 			"Prioritized matchers must start and end with a vectorial",
 		);
-		Self(mk_vec(pattern))
+		Ok(Self(mk_vec(pattern, i).await?))
 	}
 	/// tokens before the offset always match the prefix
 	pub fn apply<'a>(

orchid-std/src/macros/rule/scal_match.rs

@@ -20,8 +20,6 @@ pub fn scal_match<'a>(
 			Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
 		(ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>
 			any_match(b_mat, &body[..], save_loc),
-		(ScalMatcher::Lambda(arg_mat, b_mat), MacTok::Lambda(arg, body)) =>
-			Some(any_match(arg_mat, arg, save_loc)?.combine(any_match(b_mat, body, save_loc)?)),
 		_ => None,
 	}
 }

orchid-std/src/macros/rule/shared.rs

@@ -11,7 +11,6 @@ use orchid_base::tokens::{PARENS, Paren};
 pub enum ScalMatcher {
 	Name(Sym),
 	S(Paren, Box<AnyMatcher>),
-	Lambda(Box<AnyMatcher>, Box<AnyMatcher>),
 	Placeh { key: Tok<String> },
 }
 
@@ -62,7 +61,6 @@ impl fmt::Display for ScalMatcher {
 				let (l, r, _) = PARENS.iter().find(|r| r.2 == *t).unwrap();
 				write!(f, "{l}{body}{r}")
 			},
-			Self::Lambda(arg, body) => write!(f, "\\{arg}.{body}"),
 		}
 	}
 }