partway through fixes, macro system needs resdesign

orchid-extension/src/gen_expr.rs

@@ -1,3 +1,5 @@
+use std::cell::RefCell;
+use std::marker::PhantomData;
 use std::mem;
 use std::pin::{Pin, pin};
 use std::rc::Rc;
@@ -6,22 +8,38 @@ use futures::{FutureExt, Stream, StreamExt, stream};
 use orchid_base::{
 	FmtCtx, FmtUnit, Format, OrcErr, OrcErrv, Pos, Sym, Variants, match_mapping, tl_cache,
 };
+use substack::Substack;
+use task_local::task_local;
 
 use crate::{AtomFactory, AtomicFeatures, Expr, ToExpr, ToExprFuture, api, request, sys_id};
 
-/// Newly generated AST. Values of this type should not typically be constructed
-/// manually but through the helpers in this module
+#[derive(Clone, Copy, Debug)]
+struct ExprSerializeCx<'a> {
+	closures: Substack<'a, u64>,
+	lambda_counter: &'a RefCell<u64>,
+}
+
+/// Release notifications will not be sent for the slots. Use this with
+/// messages that imply ownership transfer
+pub async fn serialize(expr: GExpr) -> api::Expression {
+	let cx = ExprSerializeCx { closures: Substack::Bottom, lambda_counter: &RefCell::new(0) };
+	expr.serialize(cx).await
+}
+
+/// Smart object representing AST not-yet-sent to the interpreter. This type can
+/// be cloned and persisted, and it must not have unbound arguments. The helper
+/// functions in this module let you build trees of [ToExpr] implementors which
+/// represent lambdas and their arguments separately, and then convert them into
+/// [GExpr] in one pass.
 #[derive(Clone, Debug)]
 pub struct GExpr {
 	/// AST node type
-	pub kind: GExprKind,
+	kind: GExprKind,
 	/// Code location associated with the expression for debugging purposes
-	pub pos: Pos,
+	pos: Pos,
 }
 impl GExpr {
-	/// Release notifications will not be sent for the slots. Use this with
-	/// messages that imply ownership transfer
-	pub async fn serialize(self) -> api::Expression {
+	async fn serialize(self, cx: ExprSerializeCx<'_>) -> api::Expression {
 		if let GExprKind::Slot(ex) = self.kind {
 			let hand = ex.handle();
 			mem::drop(ex);
@@ -32,8 +50,8 @@ impl GExpr {
 			}
 		} else {
 			api::Expression {
-				location: api::Location::Inherit,
-				kind: self.kind.serialize().boxed_local().await,
+				location: self.pos.to_api(),
+				kind: self.kind.serialize(cx).boxed_local().await,
 			}
 		}
 	}
@@ -42,7 +60,7 @@ impl GExpr {
 	/// Send the expression to the interpreter to be compiled and to become
 	/// shareable across extensions
 	pub async fn create(self) -> Expr {
-		Expr::deserialize(request(api::Create(sys_id(), self.serialize().await)).await).await
+		Expr::deserialize(request(api::Create(sys_id(), serialize(self).await)).await).await
 	}
 }
 impl Format for GExpr {
@@ -56,8 +74,8 @@ impl Format for GExpr {
 pub enum GExprKind {
 	/// Function call
 	Call(Box<GExpr>, Box<GExpr>),
-	/// Lambda expression. Argument numbers are matched when equal
-	Lambda(u64, Box<GExpr>),
+	/// Lambda expression. Argument must be the same for slot
+	Lambda(Box<GExpr>),
 	/// Slot for a lambda argument
 	Arg(u64),
 	/// The second expression is only valid after the first one had already been
@@ -80,23 +98,40 @@ pub enum GExprKind {
 	Bottom(OrcErrv),
 }
 impl GExprKind {
-	async fn serialize(self) -> api::ExpressionKind {
+	pub fn at(self, pos: Pos) -> GExpr { GExpr { kind: self, pos } }
+	async fn serialize(self, cx: ExprSerializeCx<'_>) -> api::ExpressionKind {
 		match_mapping!(self, Self => api::ExpressionKind {
 			Call(
-				f => Box::new(f.serialize().await),
-				x => Box::new(x.serialize().await)
+				f => Box::new(f.serialize(cx).await),
+				x => Box::new(x.serialize(cx).await)
 			),
 			Seq(
-				a => Box::new(a.serialize().await),
-				b => Box::new(b.serialize().await)
+				a => Box::new(a.serialize(cx).await),
+				b => Box::new(b.serialize(cx).await)
 			),
-			Lambda(arg, body => Box::new(body.serialize().await)),
-			Arg(arg),
 			Const(name.to_api()),
 			Bottom(err.to_api()),
 			NewAtom(fac.clone().build().await),
 		} {
-			Self::Slot(_) => panic!("processed elsewhere")
+			Self::Slot(_) => panic!("processed elsewhere"),
+			Self::Lambda(body) => {
+				let id: u64;
+				{
+					let mut g = cx.lambda_counter.borrow_mut();
+					id = *g;
+					*g += 1;
+				};
+				let cx = ExprSerializeCx {
+					lambda_counter: cx.lambda_counter,
+					closures: cx.closures.push(id)
+				};
+				api::ExpressionKind::Lambda(id,
+					Box::new(body.serialize(cx).await)
+				)
+			},
+			Self::Arg(arg) => {
+				api::ExpressionKind::Arg(*cx.closures.iter().nth(arg as usize).expect("Unbound arg"))
+			},
 		})
 	}
 }
@@ -106,9 +141,9 @@ impl Format for GExprKind {
 			GExprKind::Call(f, x) =>
 				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0} ({1})")))
 					.units([f.print(c).await, x.print(c).await]),
-			GExprKind::Lambda(arg, body) =>
-				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0}.{1}")))
-					.units([arg.to_string().into(), body.print(c).await]),
+			GExprKind::Lambda(body) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{1}")))
+					.units([body.print(c).await]),
 			GExprKind::Arg(arg) => arg.to_string().into(),
 			GExprKind::Seq(a, b) =>
 				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("[{0}] {1}")))
@@ -123,7 +158,11 @@ impl Format for GExprKind {
 	}
 }
 
-fn inherit(kind: GExprKind) -> GExpr { GExpr { pos: Pos::Inherit, kind } }
+pub fn inherit(kind: GExprKind) -> GExpr { GExpr { pos: Pos::Inherit, kind } }
+
+task_local! {
+	pub static CLOSURE_DEPTH: u64;
+}
 
 impl ToExpr for Sym {
 	async fn to_expr(self) -> Expr
@@ -135,6 +174,8 @@ impl ToExpr for Sym {
 /// Creates an expression from a new atom that we own.
 pub fn new_atom<A: AtomicFeatures>(atom: A) -> GExpr { inherit(GExprKind::NewAtom(atom.factory())) }
 
+pub fn slot(expr: Expr) -> GExpr { GExpr { pos: Pos::SlotTarget, kind: GExprKind::Slot(expr) } }
+
 /// An expression which is only valid if a number of dependencies had already
 /// been normalized
 pub fn seq(
@@ -155,17 +196,49 @@ pub fn seq(
 	})
 }
 
-/// Argument bound by an enclosing [lam] or [dyn_lambda]
-pub fn arg(n: u64) -> GExpr { inherit(GExprKind::Arg(n)) }
-
-/// A lambda expression. The difference from [dyn_lambda] is purely aesthetic
-pub fn lam<const N: u64>(b: impl ToExpr) -> ToExprFuture<impl Future<Output = GExpr>> {
-	dyn_lambda(N, b)
+#[derive(Debug, Clone, Copy)]
+pub enum ArgState {
+	Building,
+	Serializing { depth: u64 },
+	Ready,
 }
 
-/// A lambda expression. The difference from [lam] is purely aesthetic
-pub fn dyn_lambda(n: u64, b: impl ToExpr) -> ToExprFuture<impl Future<Output = GExpr>> {
-	ToExprFuture(async move { inherit(GExprKind::Lambda(n, Box::new(b.to_gen().await))) })
+/// Argument bound by an enclosing [lam] or [dyn_lambda]
+#[derive(Debug, Clone, Copy)]
+pub struct GenArg<'a>(*const RefCell<ArgState>, PhantomData<&'a ()>);
+impl ToExpr for GenArg<'_> {
+	async fn to_gen(self) -> GExpr {
+		// SAFETY: Created from a Rc that lives as long as the lifetime arg, see [lam]
+		let state = unsafe { self.0.as_ref().unwrap() };
+		match (*state.borrow(), CLOSURE_DEPTH.try_with(|r| *r)) {
+			(ArgState::Serializing { .. }, Err(_)) =>
+				panic!("Lambda should have cleared up argstate alongside CLOSURE_DEPTH"),
+			(ArgState::Serializing { depth }, Ok(total)) => inherit(GExprKind::Arg(total - depth)),
+			(ArgState::Building, _) =>
+				panic!("Argument serialized before lambda. Likely an over-eager ToExpr impl"),
+			(ArgState::Ready, _) =>
+				unreachable!("The arg should never be available this long, the GenArg is a convenience"),
+		}
+	}
+}
+
+/// A lambda expression.
+pub fn lam<'a>(
+	cb: impl for<'b> AsyncFnOnce(GenArg<'b>) -> GExpr + 'a,
+) -> ToExprFuture<impl Future<Output = GExpr> + 'a> {
+	let state = Rc::new(RefCell::new(ArgState::Building));
+	ToExprFuture(async move {
+		let rank = CLOSURE_DEPTH.try_with(|r| *r + 1).unwrap_or(0);
+		match *state.borrow_mut() {
+			ref mut state @ ArgState::Building => *state = ArgState::Serializing { depth: rank },
+			ArgState::Serializing { .. } => panic!("Lambda serialized twice, found interrupted"),
+			ArgState::Ready => panic!("Lambda serialized twice"),
+		}
+		let gen_arg = GenArg(Rc::as_ptr(&state), PhantomData);
+		let ret = CLOSURE_DEPTH.scope(rank, async { cb(gen_arg).await.to_gen().await }).await;
+		mem::drop(state);
+		inherit(GExprKind::Lambda(Box::new(ret)))
+	})
 }
 
 /// one or more items that are convertible to expressions. In practice, a