Fixed macro system by reintroducing MacTok::Resolved

orchid-api/src/expr.rs

@@ -4,7 +4,9 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{Atom, ExtHostNotif, ExtHostReq, LocalAtom, Location, OrcError, SysId, TStrv};
+use crate::{
+	Atom, ExtHostNotif, ExtHostReq, FormattingUnit, LocalAtom, Location, OrcError, SysId, TStrv,
+};
 
 /// An arbitrary ID associated with an expression on the host side. Incoming
 /// tickets always come with some lifetime guarantee, which can be extended with
@@ -104,11 +106,21 @@ impl Request for Inspect {
 	type Response = Inspected;
 }
 
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(ExprReq, ExtHostReq)]
+pub struct ExprPrint {
+	pub target: ExprTicket,
+}
+impl Request for ExprPrint {
+	type Response = FormattingUnit;
+}
+
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 #[extendable]
 pub enum ExprReq {
 	Inspect(Inspect),
+	ExprPrint(ExprPrint),
 	Create(Create),
 }
 

orchid-extension/src/expr.rs

@@ -166,11 +166,7 @@ impl Expr {
 }
 impl Format for Expr {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		match &self.data().await.kind {
+		FmtUnit::from_api(&request(api::ExprPrint { target: self.handle.0 }).await)
-			ExprKind::Opaque => "OPAQUE".to_string().into(),
-			ExprKind::Bottom(b) => format!("Bottom({b})").into(),
-			ExprKind::Atom(a) => FmtUnit::from_api(&request(api::ExtAtomPrint(a.atom.clone())).await),
-		}
 	}
 }
 impl Eq for Expr {}

orchid-host/src/expr.rs

@@ -173,7 +173,7 @@ async fn print_exprkind<'a>(
 			None => format!("Bottom(\n\t{}\n)", indent(&e.to_string())).into(),
 		},
 		ExprKind::Call(f, x) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
-			.unbounded("{0b} {1l}")
+			// .unbounded("{0b} {1l}")
 			.bounded("({0b} {1})")))
 		.units([print_expr(f, c, visited, id_only).await, print_expr(x, c, visited, id_only).await]),
 		ExprKind::Identity(id) =>

orchid-host/src/extension.rs

@@ -195,6 +195,14 @@ impl Extension {
 												})
 												.await
 										},
+										api::ExprReq::ExprPrint(prt @ api::ExprPrint { target }) => {
+											let msg = match ctx.exprs.get_expr(target) {
+												None => "EXPIRED_TICKET".into(),
+												Some(expr) => expr.print(&FmtCtxImpl::default()).await,
+											}
+											.to_api();
+											handle.reply(&prt, &msg).await
+										},
 										api::ExprReq::Create(cre) => {
 											let req = Witness::of(&cre);
 											let api::Create(sys, expr) = cre;

orchid-std/src/macros/let_line.rs

@@ -4,7 +4,7 @@ use futures::{FutureExt, StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_base::{
-	Comment, OrcRes, Paren, Parsed, Snippet, Sym, expect_tok, fmt, is, report, token_errv,
+	Comment, OrcRes, Paren, Parsed, Snippet, Sym, expect_tok, is, report, token_errv,
 	try_pop_no_fluff, with_reporter,
 };
 use orchid_extension::{
@@ -36,17 +36,18 @@ impl Parser for LetLine {
 		};
 		let Parsed { tail, .. } = expect_tok(tail, is("=").await).await?;
 		let aliased = parse_tokv(tail).await;
-		eprintln!("Parsed tokv: {}", fmt(&aliased).await);
+		Ok(vec![ParsedLine::cnst(
-		Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name, async move |mut ctx| {
+			&line.sr(),
+			&comments,
+			exported,
+			name.clone(),
+			async move |mut ctx| {
 				let macro_input =
 					MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&aliased, &ctx)).await?).at(sr.pos());
-			eprintln!("Dealiased tokv: {}", fmt(&macro_input).await);
 				let resolved = resolve(ctx.handle(), macro_input).await;
-			eprintln!("resolves to: {}", fmt(&resolved).await);
+				Ok(lower(&resolved, substack::Substack::Bottom).await)
-			let x = lower(&resolved, substack::Substack::Bottom).await;
+			},
-			eprintln!("created as: {}", fmt(&x).await);
+		)])
-			Ok(x)
-		})])
 	}
 }
 
@@ -70,7 +71,6 @@ pub async fn dealias_mac_v(aliased: &MacTreeSeq, ctx: &ConstCtx<'_>) -> MacTreeS
 
 pub async fn parse_tokv(line: PSnippet<'_>) -> MacTreeSeq {
 	if let Some((idx, arg)) = line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?))) {
-		eprintln!("Found a lambda while parsing tokv");
 		let (head, lambda) = line.split_at(idx as u32);
 		let (_, body) = lambda.split_first().unwrap();
 		let body = parse_tokv(body).boxed_local().await;

orchid-std/src/macros/lower.rs

@@ -18,6 +18,7 @@ fn on_err(e: OrcErrv) -> GExpr {
 
 pub async fn lower(mt: &MacTree, args: Substack<'_, Sym>) -> GExpr {
 	match mt.tok() {
+		MacTok::Resolved(inner) => lower(inner, args).boxed_local().await,
 		MacTok::Bottom(b) => on_err(b.clone()),
 		MacTok::Slot => on_err(mk_errv(
 			is("Lowering intermediary slotted mactree").await,

orchid-std/src/macros/macro_lib.rs

@@ -1,7 +1,9 @@
 use orchid_extension::gen_expr::new_atom;
 use orchid_extension::tree::{GenMember, fun, prefix};
 use orchid_extension::{TAtom, exec};
+use substack::Substack;
 
+use crate::macros::lower::lower;
 use crate::macros::mactree::MacTree;
 use crate::macros::resolve::resolve;
 use crate::macros::utils::{build_macro, mactree, mactreev};
@@ -12,9 +14,10 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 		fun(true, "resolve", async |tpl: TAtom<MacTree>| {
 			exec(async move |mut h| new_atom(resolve(&mut h, tpl.own().await).await)).await
 		}),
+		fun(true, "lower", async |mt: TAtom<MacTree>| lower(&mt.own().await, Substack::Bottom).await),
 		prefix("common", [
 			build_macro(None, ["..", "_", "="]).finish(),
-			build_macro(Some(1), ["+"])
+			build_macro(Some(30), ["+"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::+" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -24,7 +27,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(1), ["-"])
+			build_macro(Some(30), ["-"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::-" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -34,7 +37,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(2), ["*"])
+			build_macro(Some(20), ["*"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::*" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -44,7 +47,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(2), ["/"])
+			build_macro(Some(20), ["/"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::/" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -54,7 +57,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(2), ["%"])
+			build_macro(Some(20), ["%"])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::%" "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {
@@ -64,7 +67,7 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 					},
 				)
 				.finish(),
-			build_macro(Some(3), ["."])
+			build_macro(Some(10), ["."])
 				.rule(
 					mactreev!("...$" lhs 1 "macros::common::." "...$" rhs 0),
 					async |mut cx, [lhs, rhs]| {

orchid-std/src/macros/macro_line.rs

@@ -128,7 +128,7 @@ impl Parser for MacroLine {
 				let macro_input =
 					MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&body_mactree, &ctx)).await?)
 						.at(body_sr.pos());
-				Ok(call(sym!(macros::resolve), new_atom(macro_input)))
+				Ok(call(sym!(macros::lower), call(sym!(macros::resolve), new_atom(macro_input))))
 			}))
 		}
 		let mac_cell = Rc::new(OnceCell::new());

orchid-std/src/macros/mactree.rs

@@ -114,6 +114,7 @@ impl MacTree {
 				MacTok::Lambda(arg, body) =>
 					MacTok::Lambda(ro(changed, |changed| arg.map(changed, map)), body.map(changed, map)),
 				MacTok::Name(_) | MacTok::Value(_) => return self.clone(),
+				MacTok::Resolved(inner) => return inner.map(changed, map),
 				MacTok::Slot | MacTok::Ph(_) | MacTok::Bottom(_) => return self.clone(),
 				MacTok::S(p, body) => MacTok::S(*p, body.map(changed, map)),
 			},
@@ -152,11 +153,23 @@ pub enum MacTok {
 	/// never accessed as variables by usercode
 	Ph(Ph),
 	Bottom(OrcErrv),
+	/// This node type cannot be manually constructed in Orchid and should not be
+	/// manually constructed in Rust.
+	///
+	/// - can only match placeholders
+	/// - skipped by Resolve and glossary
+	/// - transparent for all other purposes
+	///
+	/// It is used to wrap macro return values to prevent double-resolve
+	///
+	/// TODO: consider including optional metadata about the match
+	Resolved(MacTree),
 }
 impl MacTok {
 	pub fn build_glossary(&self) -> Rc<HashSet<Sym>> {
 		match self {
-			MacTok::Bottom(_) | MacTok::Ph(_) | MacTok::Slot | MacTok::Value(_) => Rc::default(),
+			MacTok::Bottom(_) | MacTok::Ph(_) | MacTok::Slot | MacTok::Value(_) | MacTok::Resolved(_) =>
+				Rc::default(),
 			MacTok::Name(sym) => Rc::new(HashSet::from([sym.clone()])),
 			MacTok::S(_, body) => union_rc_sets(body.items.iter().map(|mt| mt.glossary.clone())),
 			MacTok::Lambda(arg, body) =>
@@ -185,6 +198,7 @@ impl Format for MacTok {
 			}
 			.units([body.print(c).await]),
 			Self::Slot => "$SLOT".into(),
+			Self::Resolved(res) => res.print(c).boxed_local().await,
 			Self::Bottom(err) => match err.one() {
 				Some(err) => format!("Bottom({err}) ").into(),
 				None => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),

orchid-std/src/macros/resolve.rs

@@ -4,8 +4,8 @@ use futures::FutureExt;
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use orchid_base::{NameLike, Paren, Pos, VPath, fmt, is, log, mk_errv, report};
-use orchid_extension::gen_expr::{GExpr, call_v, new_atom};
+use orchid_extension::gen_expr::{call_v, new_atom};
-use orchid_extension::{ExecHandle, ReflMemKind, TAtom, ToExpr, refl};
+use orchid_extension::{ExecHandle, ReflMemKind, TAtom, refl};
 use subslice_offset::SubsliceOffset;
 
 use crate::macros::macro_value::{Macro, Rule};
@@ -105,7 +105,7 @@ pub async fn resolve(h: &mut ExecHandle<'_>, val: MacTree) -> MacTree {
 			}
 		}
 	}
-	let mut rctx = ResolveCtx { exclusive, priod };
+	let mut rctx = ResolveCtx { exclusive, priod, h: &mut *h };
 	let gex = resolve_one(&mut rctx, &val).await.unwrap_or(val.clone());
 	writeln!(log("debug"), "Macro-resolution over {} yielded {}", fmt(&val).await, fmt(&gex).await)
 		.await;
@@ -119,22 +119,22 @@ pub struct FilteredMacroRecord<'a> {
 	rules: Vec<usize>,
 }
 
-struct ResolveCtx<'a> {
+struct ResolveCtx<'a, 'b> {
 	/// If these overlap, that's a compile-time error
 	pub exclusive: Vec<FilteredMacroRecord<'a>>,
 	/// If these overlap, the priorities decide the order. In case of a tie, the
 	/// order is unspecified
 	pub priod: Vec<FilteredMacroRecord<'a>>,
+	pub h: &'a mut ExecHandle<'b>,
 }
 
-async fn resolve_one(ctx: &mut ResolveCtx<'_>, value: &MacTree) -> Option<MacTree> {
+async fn resolve_one(ctx: &mut ResolveCtx<'_, '_>, value: &MacTree) -> Option<MacTree> {
-	eprintln!("Resolving unit {}", fmt(value).await);
 	match value.tok() {
 		MacTok::Ph(_) | MacTok::Slot => panic!("Forbidden element in value mactree"),
-		MacTok::Bottom(_) | MacTok::Value(_) | MacTok::Name(_) => None,
+		MacTok::Bottom(_) | MacTok::Value(_) | MacTok::Name(_) | MacTok::Resolved(_) => None,
 		MacTok::Lambda(arg, body) => {
 			let new_arg = resolve_one(ctx, arg).boxed_local().await;
-			let new_body = resolve_seq(ctx, body, value.pos()).await;
+			let new_body = resolve_seq(ctx, body, value.pos()).boxed_local().await;
 			if new_arg.is_none() && new_body.is_none() {
 				return None;
 			}
@@ -144,8 +144,8 @@ async fn resolve_one(ctx: &mut ResolveCtx<'_>, value: &MacTree) -> Option<MacTre
 			);
 			Some(tok.at(value.pos()))
 		},
-		MacTok::S(pty, body) =>
+		MacTok::S(pty, body) => (resolve_seq(ctx, body, value.pos()).boxed_local().await)
-			resolve_seq(ctx, body, value.pos()).await.map(|body| MacTok::S(*pty, body).at(value.pos())),
+			.map(|body| MacTok::S(*pty, body).at(value.pos())),
 	}
 }
 
@@ -164,7 +164,7 @@ fn subsection<T>(
 }
 
 async fn resolve_seq(
-	ctx: &mut ResolveCtx<'_>,
+	ctx: &mut ResolveCtx<'_, '_>,
 	val: &MacTreeSeq,
 	fallback_pos: Pos,
 ) -> Option<MacTreeSeq> {
@@ -261,8 +261,8 @@ async fn resolve_seq(
 			// backwards so that the non-overlapping ranges remain valid
 			let pos = (state.names().flat_map(|r| r.1).cloned().reduce(Pos::add))
 				.expect("All macro rules must contain at least one locally defined name");
-			let subex = mk_body_call(mac, rule, &state, pos.clone()).await.to_expr().await;
+			let subex = call_body(ctx.h, mac, rule, &state, pos.clone()).await;
-			new_val.splice(range, [MacTok::Value(subex).at(pos)]);
+			new_val.splice(range, [subex]);
 		}
 	};
 	// TODO: Does this glossary refresh actually pay off?
@@ -280,15 +280,30 @@ async fn resolve_seq(
 			let range = pre.len()..new_val.len() - suf.len();
 			let pos = (state.names().flat_map(|pair| pair.1).cloned().reduce(Pos::add))
 				.expect("All macro rules must contain at least one locally defined name");
-			let subex = mk_body_call(mac, rule, &state, pos.clone()).await.to_expr().await;
+			let subex = call_body(ctx.h, mac, rule, &state, pos.clone()).await;
 			std::mem::drop(state);
-			new_val.splice(range, [MacTok::Value(subex).at(pos)]);
+			new_val.splice(range, [subex]);
 		}
 	}
+	for item in new_val.iter_mut() {
+		let Some(new) = resolve_one(ctx, item).await else { continue };
+		*item = new;
+		any_match = true;
+	}
+	/* TODO:
+	The macro system was previously built on the assumption that whatever is returned by call_body
+	can never be matched. This is no longer true, but now double-matches are possible. Address this.
+	*/
 	any_match.then_some(MacTreeSeq::new(new_val))
 }
 
-async fn mk_body_call(mac: &Macro, rule: &Rule, state: &MatchState<'_>, pos: Pos) -> GExpr {
+async fn call_body(
+	h: &mut ExecHandle<'_>,
+	mac: &Macro,
+	rule: &Rule,
+	state: &MatchState<'_>,
+	pos: Pos,
+) -> MacTree {
 	let mut call_args = vec![];
 	for name in rule.ph_names.iter() {
 		call_args.push(match state.get(name).expect("Missing state entry for placeholder") {
@@ -297,5 +312,9 @@ async fn mk_body_call(mac: &Macro, rule: &Rule, state: &MatchState<'_>, pos: Pos
 				new_atom(MacTok::S(Paren::Round, MacTreeSeq::new(vec.iter().cloned())).at(Pos::None)),
 		});
 	}
-	call_v(mac.0.module.suffix([rule.body.clone()]).await, call_args).await.at(pos.clone())
+	let f_name = mac.0.module.suffix([rule.body.clone()]).await;
+	match h.exec::<TAtom<MacTree>>(call_v(f_name, call_args)).await {
+		Err(e) => MacTok::Bottom(e).at(pos),
+		Ok(mt) => MacTok::Resolved(mt.own().await).at(mt.pos()),
+	}
 }

orchid-std/src/macros/rule/build.rs

@@ -127,6 +127,7 @@ async fn mk_scalar(pattern: &MacTree) -> OrcRes<ScalMatcher> {
 				[pattern.pos()],
 			));
 		},
+		MacTok::Resolved(_) => panic!("Can only appear in macro output, not in matcher"),
 		MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
 		MacTok::Bottom(errv) => return Err(errv.clone()),
 	})

orchid-std/src/macros/utils.rs

@@ -66,10 +66,7 @@ pub struct RuleCtx<'a> {
 }
 impl RuleCtx<'_> {
 	/// Recursively resolve a subexpression
-	pub async fn recur(&mut self, mt: MacTree) -> MacTree {
+	pub async fn recur(&mut self, mt: MacTree) -> MacTree { resolve(&mut self.handle, mt).await }
-		eprintln!("Recursion in macro");
-		resolve(&mut self.handle, mt).await
-	}
 	/// Recursively resolve a value from a delegate macro which is expected to
 	/// match only keywords and return any subexpressions in a datastructure
 	///

orcx/src/main.rs

@@ -109,6 +109,12 @@ fn get_all_extensions<'a>(
 	}
 	try_stream(async |mut cx| {
 		for ext_path in args.extension.iter() {
+			let Some(file_name) = ext_path.file_name() else {
+				return Err(io::Error::new(
+					std::io::ErrorKind::IsADirectory,
+					format!("Extensions are always files, but {ext_path} points at a directory"),
+				));
+			};
 			let init = if cfg!(windows) {
 				if ext_path.with_extension("dll").exists() {
 					ext_dylib(ext_path.with_extension("dll").as_std_path(), ctx.clone()).await.unwrap()
@@ -117,12 +123,15 @@ fn get_all_extensions<'a>(
 				} else {
 					return Err(not_found_error(ext_path));
 				}
-			} else if ext_path.with_extension("so").exists() {
+			} else {
-				ext_dylib(ext_path.with_extension("so").as_std_path(), ctx.clone()).await.unwrap()
+				let lib_path = ext_path.with_file_name(format!("lib{file_name}.so"));
+				if lib_path.exists() {
+					ext_dylib(lib_path.as_std_path(), ctx.clone()).await.unwrap()
 				} else if ext_path.exists() {
 					ext_command(Command::new(ext_path.as_os_str()), ctx.clone()).await?
 				} else {
 					return Err(not_found_error(ext_path));
+				}
 			};
 			cx.emit(Extension::new(init, ctx.clone()).await?).await;
 		}

orcx/src/repl.rs

@@ -20,6 +20,7 @@ use crate::Args;
 use crate::print_mod::print_mod;
 
 pub async fn repl(args: &Args, extensions: &[Extension], ctx: Ctx) -> Result<(), String> {
+	eprintln!("Orchid REPL. Commands are prefixed with `:`, use `:help` to learn about the REPL.");
 	let mut counter = 0;
 	let mut imports = Vec::new();
 	let usercode_path = sym!(usercode);
@@ -34,10 +35,18 @@ pub async fn repl(args: &Args, extensions: &[Extension], ctx: Ctx) -> Result<(),
 		if let Some(cmdline) = prompt.trim().strip_prefix(":") {
 			if cmdline == "help" {
 				println!(
-					"Recognized commands are:\n\
+					"All lines not prefixed with `:` are interpreted as Orchid code.\n\
-					- help     print this message\n\
+          Orchid lines are executed in distinct modules to avoid namespace clutter.\n\
-					- modtree  display the current module tree"
+          Use `let <name> = <value>` to define a constant\n\
+          All other lines are \n\
+          \n\
+          Recognized commands prefixed with `:` are:\n\
+					- :help     print this message\n\
+					- :modtree  display the current module tree\n\
+          - :exit     close the REPL"
 				)
+			} else if cmdline == "exit" {
+				break Ok(());
 			} else if cmdline == "modtree" {
 				let root_data = root.0.read().await;
 				print_mod(&root_data.root, VPath::new([]), &root_data).await
@@ -93,7 +102,10 @@ pub async fn repl(args: &Args, extensions: &[Extension], ctx: Ctx) -> Result<(),
 		xctx.set_gas(Some(1000));
 		match xctx.execute().await {
 			ExecResult::Value(val, _) => {
-				println!("{const_name} = {}", take_first(&val.print(&FmtCtxImpl::default()).await, false))
+				println!(
+					"let {const_name} = {}",
+					take_first(&val.print(&FmtCtxImpl::default()).await, true)
+				)
 			},
 			ExecResult::Err(e, _) => println!("error: {e}"),
 			ExecResult::Gas(_) => println!("Ran out of gas!"),