Fixed macro system by reintroducing MacTok::Resolved

orchid-std/src/macros/let_line.rs

@@ -4,7 +4,7 @@ use futures::{FutureExt, StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_base::{
-	Comment, OrcRes, Paren, Parsed, Snippet, Sym, expect_tok, fmt, is, report, token_errv,
+	Comment, OrcRes, Paren, Parsed, Snippet, Sym, expect_tok, is, report, token_errv,
 	try_pop_no_fluff, with_reporter,
 };
 use orchid_extension::{
@@ -36,17 +36,18 @@ impl Parser for LetLine {
 		};
 		let Parsed { tail, .. } = expect_tok(tail, is("=").await).await?;
 		let aliased = parse_tokv(tail).await;
-		eprintln!("Parsed tokv: {}", fmt(&aliased).await);
-		Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name, async move |mut ctx| {
-			let macro_input =
-				MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&aliased, &ctx)).await?).at(sr.pos());
-			eprintln!("Dealiased tokv: {}", fmt(&macro_input).await);
-			let resolved = resolve(ctx.handle(), macro_input).await;
-			eprintln!("resolves to: {}", fmt(&resolved).await);
-			let x = lower(&resolved, substack::Substack::Bottom).await;
-			eprintln!("created as: {}", fmt(&x).await);
-			Ok(x)
-		})])
+		Ok(vec![ParsedLine::cnst(
+			&line.sr(),
+			&comments,
+			exported,
+			name.clone(),
+			async move |mut ctx| {
+				let macro_input =
+					MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&aliased, &ctx)).await?).at(sr.pos());
+				let resolved = resolve(ctx.handle(), macro_input).await;
+				Ok(lower(&resolved, substack::Substack::Bottom).await)
+			},
+		)])
 	}
 }
 
@@ -70,7 +71,6 @@ pub async fn dealias_mac_v(aliased: &MacTreeSeq, ctx: &ConstCtx<'_>) -> MacTreeS
 
 pub async fn parse_tokv(line: PSnippet<'_>) -> MacTreeSeq {
 	if let Some((idx, arg)) = line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?))) {
-		eprintln!("Found a lambda while parsing tokv");
 		let (head, lambda) = line.split_at(idx as u32);
 		let (_, body) = lambda.split_first().unwrap();
 		let body = parse_tokv(body).boxed_local().await;

orchid-std/src/macros/lower.rs

@@ -18,6 +18,7 @@ fn on_err(e: OrcErrv) -> GExpr {
 
 pub async fn lower(mt: &MacTree, args: Substack<'_, Sym>) -> GExpr {
 	match mt.tok() {
+		MacTok::Resolved(inner) => lower(inner, args).boxed_local().await,
 		MacTok::Bottom(b) => on_err(b.clone()),
 		MacTok::Slot => on_err(mk_errv(
 			is("Lowering intermediary slotted mactree").await,

orchid-std/src/macros/macro_lib.rs

@@ -1,7 +1,9 @@
 use orchid_extension::gen_expr::new_atom;
 use orchid_extension::tree::{GenMember, fun, prefix};
 use orchid_extension::{TAtom, exec};
+use substack::Substack;
 
+use crate::macros::lower::lower;
 use crate::macros::mactree::MacTree;
 use crate::macros::resolve::resolve;
 use crate::macros::utils::{build_macro, mactree, mactreev};
@@ -12,9 +14,10 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 		fun(true, "resolve", async |tpl: TAtom<MacTree>| {
 			exec(async move |mut h| new_atom(resolve(&mut h, tpl.own().await).await)).await
 		}),
+		fun(true, "lower", async |mt: TAtom<MacTree>| lower(&mt.own().await, Substack::Bottom).await),
 		prefix("common", [
 			build_macro(None, ["..", "_", "="]).finish(),
-			build_macro(Some(1), ["+"])
+			build_macro(Some(30), ["+"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::+" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -24,7 +27,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(1), ["-"])
+			build_macro(Some(30), ["-"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::-" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -34,7 +37,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(2), ["*"])
+			build_macro(Some(20), ["*"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::*" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -44,7 +47,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(2), ["/"])
+			build_macro(Some(20), ["/"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::/" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -54,7 +57,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(2), ["%"])
+			build_macro(Some(20), ["%"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::%" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -64,7 +67,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(3), ["."])
+			build_macro(Some(10), ["."])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::." "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {

orchid-std/src/macros/macro_line.rs

@@ -128,7 +128,7 @@ impl Parser for MacroLine {
 				let macro_input =
 					MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&body_mactree, &ctx)).await?)
 						.at(body_sr.pos());
-				Ok(call(sym!(macros::resolve), new_atom(macro_input)))
+				Ok(call(sym!(macros::lower), call(sym!(macros::resolve), new_atom(macro_input))))
 			}))
 		}
 		let mac_cell = Rc::new(OnceCell::new());

orchid-std/src/macros/mactree.rs

@@ -114,6 +114,7 @@ impl MacTree {
 				MacTok::Lambda(arg, body) =>
 					MacTok::Lambda(ro(changed, |changed| arg.map(changed, map)), body.map(changed, map)),
 				MacTok::Name(_) | MacTok::Value(_) => return self.clone(),
+				MacTok::Resolved(inner) => return inner.map(changed, map),
 				MacTok::Slot | MacTok::Ph(_) | MacTok::Bottom(_) => return self.clone(),
 				MacTok::S(p, body) => MacTok::S(*p, body.map(changed, map)),
 			},
@@ -152,11 +153,23 @@ pub enum MacTok {
 	/// never accessed as variables by usercode
 	Ph(Ph),
 	Bottom(OrcErrv),
+	/// This node type cannot be manually constructed in Orchid and should not be
+	/// manually constructed in Rust.
+	///
+	/// - can only match placeholders
+	/// - skipped by Resolve and glossary
+	/// - transparent for all other purposes
+	///
+	/// It is used to wrap macro return values to prevent double-resolve
+	///
+	/// TODO: consider including optional metadata about the match
+	Resolved(MacTree),
 }
 impl MacTok {
 	pub fn build_glossary(&self) -> Rc<HashSet<Sym>> {
 		match self {
-			MacTok::Bottom(_) | MacTok::Ph(_) | MacTok::Slot | MacTok::Value(_) => Rc::default(),
+			MacTok::Bottom(_) | MacTok::Ph(_) | MacTok::Slot | MacTok::Value(_) | MacTok::Resolved(_) =>
+				Rc::default(),
 			MacTok::Name(sym) => Rc::new(HashSet::from([sym.clone()])),
 			MacTok::S(_, body) => union_rc_sets(body.items.iter().map(|mt| mt.glossary.clone())),
 			MacTok::Lambda(arg, body) =>
@@ -185,6 +198,7 @@ impl Format for MacTok {
 			}
 			.units([body.print(c).await]),
 			Self::Slot => "$SLOT".into(),
+			Self::Resolved(res) => res.print(c).boxed_local().await,
 			Self::Bottom(err) => match err.one() {
 				Some(err) => format!("Bottom({err}) ").into(),
 				None => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),

orchid-std/src/macros/resolve.rs

@@ -4,8 +4,8 @@ use futures::FutureExt;
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use orchid_base::{NameLike, Paren, Pos, VPath, fmt, is, log, mk_errv, report};
-use orchid_extension::gen_expr::{GExpr, call_v, new_atom};
-use orchid_extension::{ExecHandle, ReflMemKind, TAtom, ToExpr, refl};
+use orchid_extension::gen_expr::{call_v, new_atom};
+use orchid_extension::{ExecHandle, ReflMemKind, TAtom, refl};
 use subslice_offset::SubsliceOffset;
 
 use crate::macros::macro_value::{Macro, Rule};
@@ -105,7 +105,7 @@ pub async fn resolve(h: &mut ExecHandle<'_>, val: MacTree) -> MacTree {
 			}
 		}
 	}
-	let mut rctx = ResolveCtx { exclusive, priod };
+	let mut rctx = ResolveCtx { exclusive, priod, h: &mut *h };
 	let gex = resolve_one(&mut rctx, &val).await.unwrap_or(val.clone());
 	writeln!(log("debug"), "Macro-resolution over {} yielded {}", fmt(&val).await, fmt(&gex).await)
 		.await;
@@ -119,22 +119,22 @@ pub struct FilteredMacroRecord<'a> {
 	rules: Vec<usize>,
 }
 
-struct ResolveCtx<'a> {
+struct ResolveCtx<'a, 'b> {
 	/// If these overlap, that's a compile-time error
 	pub exclusive: Vec<FilteredMacroRecord<'a>>,
 	/// If these overlap, the priorities decide the order. In case of a tie, the
 	/// order is unspecified
 	pub priod: Vec<FilteredMacroRecord<'a>>,
+	pub h: &'a mut ExecHandle<'b>,
 }
 
-async fn resolve_one(ctx: &mut ResolveCtx<'_>, value: &MacTree) -> Option<MacTree> {
-	eprintln!("Resolving unit {}", fmt(value).await);
+async fn resolve_one(ctx: &mut ResolveCtx<'_, '_>, value: &MacTree) -> Option<MacTree> {
 	match value.tok() {
 		MacTok::Ph(_) | MacTok::Slot => panic!("Forbidden element in value mactree"),
-		MacTok::Bottom(_) | MacTok::Value(_) | MacTok::Name(_) => None,
+		MacTok::Bottom(_) | MacTok::Value(_) | MacTok::Name(_) | MacTok::Resolved(_) => None,
 		MacTok::Lambda(arg, body) => {
 			let new_arg = resolve_one(ctx, arg).boxed_local().await;
-			let new_body = resolve_seq(ctx, body, value.pos()).await;
+			let new_body = resolve_seq(ctx, body, value.pos()).boxed_local().await;
 			if new_arg.is_none() && new_body.is_none() {
 				return None;
 			}
@@ -144,8 +144,8 @@ async fn resolve_one(ctx: &mut ResolveCtx<'_>, value: &MacTree) -> Option<MacTre
 			);
 			Some(tok.at(value.pos()))
 		},
-		MacTok::S(pty, body) =>
-			resolve_seq(ctx, body, value.pos()).await.map(|body| MacTok::S(*pty, body).at(value.pos())),
+		MacTok::S(pty, body) => (resolve_seq(ctx, body, value.pos()).boxed_local().await)
+			.map(|body| MacTok::S(*pty, body).at(value.pos())),
 	}
 }
 
@@ -164,7 +164,7 @@ fn subsection<T>(
 }
 
 async fn resolve_seq(
-	ctx: &mut ResolveCtx<'_>,
+	ctx: &mut ResolveCtx<'_, '_>,
 	val: &MacTreeSeq,
 	fallback_pos: Pos,
 ) -> Option<MacTreeSeq> {
@@ -261,8 +261,8 @@ async fn resolve_seq(
 			// backwards so that the non-overlapping ranges remain valid
 			let pos = (state.names().flat_map(|r| r.1).cloned().reduce(Pos::add))
 				.expect("All macro rules must contain at least one locally defined name");
-			let subex = mk_body_call(mac, rule, &state, pos.clone()).await.to_expr().await;
-			new_val.splice(range, [MacTok::Value(subex).at(pos)]);
+			let subex = call_body(ctx.h, mac, rule, &state, pos.clone()).await;
+			new_val.splice(range, [subex]);
 		}
 	};
 	// TODO: Does this glossary refresh actually pay off?
@@ -280,15 +280,30 @@ async fn resolve_seq(
 			let range = pre.len()..new_val.len() - suf.len();
 			let pos = (state.names().flat_map(|pair| pair.1).cloned().reduce(Pos::add))
 				.expect("All macro rules must contain at least one locally defined name");
-			let subex = mk_body_call(mac, rule, &state, pos.clone()).await.to_expr().await;
+			let subex = call_body(ctx.h, mac, rule, &state, pos.clone()).await;
 			std::mem::drop(state);
-			new_val.splice(range, [MacTok::Value(subex).at(pos)]);
+			new_val.splice(range, [subex]);
 		}
 	}
+	for item in new_val.iter_mut() {
+		let Some(new) = resolve_one(ctx, item).await else { continue };
+		*item = new;
+		any_match = true;
+	}
+	/* TODO:
+	The macro system was previously built on the assumption that whatever is returned by call_body
+	can never be matched. This is no longer true, but now double-matches are possible. Address this.
+	*/
 	any_match.then_some(MacTreeSeq::new(new_val))
 }
 
-async fn mk_body_call(mac: &Macro, rule: &Rule, state: &MatchState<'_>, pos: Pos) -> GExpr {
+async fn call_body(
+	h: &mut ExecHandle<'_>,
+	mac: &Macro,
+	rule: &Rule,
+	state: &MatchState<'_>,
+	pos: Pos,
+) -> MacTree {
 	let mut call_args = vec![];
 	for name in rule.ph_names.iter() {
 		call_args.push(match state.get(name).expect("Missing state entry for placeholder") {
@@ -297,5 +312,9 @@ async fn mk_body_call(mac: &Macro, rule: &Rule, state: &MatchState<'_>, pos: Pos
 				new_atom(MacTok::S(Paren::Round, MacTreeSeq::new(vec.iter().cloned())).at(Pos::None)),
 		});
 	}
-	call_v(mac.0.module.suffix([rule.body.clone()]).await, call_args).await.at(pos.clone())
+	let f_name = mac.0.module.suffix([rule.body.clone()]).await;
+	match h.exec::<TAtom<MacTree>>(call_v(f_name, call_args)).await {
+		Err(e) => MacTok::Bottom(e).at(pos),
+		Ok(mt) => MacTok::Resolved(mt.own().await).at(mt.pos()),
+	}
 }

orchid-std/src/macros/rule/build.rs

@@ -127,6 +127,7 @@ async fn mk_scalar(pattern: &MacTree) -> OrcRes<ScalMatcher> {
 				[pattern.pos()],
 			));
 		},
+		MacTok::Resolved(_) => panic!("Can only appear in macro output, not in matcher"),
 		MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
 		MacTok::Bottom(errv) => return Err(errv.clone()),
 	})

orchid-std/src/macros/utils.rs

@@ -66,10 +66,7 @@ pub struct RuleCtx<'a> {
 }
 impl RuleCtx<'_> {
 	/// Recursively resolve a subexpression
-	pub async fn recur(&mut self, mt: MacTree) -> MacTree {
-		eprintln!("Recursion in macro");
-		resolve(&mut self.handle, mt).await
-	}
+	pub async fn recur(&mut self, mt: MacTree) -> MacTree { resolve(&mut self.handle, mt).await }
 	/// Recursively resolve a value from a delegate macro which is expected to
 	/// match only keywords and return any subexpressions in a datastructure
 	///