Added support for defining macros in Rust within the macro system

Also fixed a lot of bugs

orchid-extension/Cargo.toml

@@ -7,18 +7,18 @@ edition = "2024"
 
 [dependencies]
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
-async-lock = "3.4.1"
 async-once-cell = "0.5.4"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.20"
 futures = { version = "0.3.31", features = [
-  "std",
-  "async-await",
+	"std",
+	"async-await",
 ], default-features = false }
+futures-locks = "0.7.1"
 hashbrown = "0.16.0"
 include_dir = { version = "0.7.4", optional = true }
 itertools = "0.14.0"
-konst = "0.4.1"
+konst = "0.4.2"
 lazy_static = "1.5.0"
 memo-map = "0.3.3"
 never = "0.1.0"

orchid-extension/src/atom.rs

@@ -109,8 +109,8 @@ impl ForeignAtom {
 		.await?;
 		Some(M::Response::decode(Pin::new(&mut &rep[..])).await)
 	}
-	pub async fn downcast<T: AtomicFeatures>(self) -> Result<TypAtom<T>, NotTypAtom> {
-		TypAtom::downcast(self.ex().handle()).await
+	pub async fn downcast<T: AtomicFeatures>(self) -> Result<TAtom<T>, NotTypAtom> {
+		TAtom::downcast(self.ex().handle()).await
 	}
 }
 impl fmt::Display for ForeignAtom {
@@ -222,11 +222,12 @@ impl<A: AtomCard> Default for MethodSetBuilder<A> {
 }
 
 #[derive(Clone)]
-pub struct TypAtom<A: AtomicFeatures> {
+pub struct TAtom<A: AtomicFeatures> {
 	pub untyped: ForeignAtom,
 	pub value: A::Data,
 }
-impl<A: AtomicFeatures> TypAtom<A> {
+impl<A: AtomicFeatures> TAtom<A> {
+	pub fn ex(&self) -> Expr { self.untyped.clone().ex() }
 	pub fn ctx(&self) -> &SysCtx { self.untyped.ctx() }
 	pub fn i(&self) -> &Interner { self.ctx().i() }
 	pub async fn downcast(expr: Rc<ExprHandle>) -> Result<Self, NotTypAtom> {
@@ -262,11 +263,11 @@ impl<A: AtomicFeatures> TypAtom<A> {
 		.await
 	}
 }
-impl<A: AtomicFeatures> Deref for TypAtom<A> {
+impl<A: AtomicFeatures> Deref for TAtom<A> {
 	type Target = A::Data;
 	fn deref(&self) -> &Self::Target { &self.value }
 }
-impl<A: AtomicFeatures> ToExpr for TypAtom<A> {
+impl<A: AtomicFeatures> ToExpr for TAtom<A> {
 	async fn to_expr(self) -> GExpr { self.untyped.to_expr().await }
 }
 

orchid-extension/src/atom_owned.rs

@@ -1,16 +1,17 @@
 use std::any::{Any, TypeId, type_name};
 use std::borrow::Cow;
 use std::future::Future;
+use std::marker::PhantomData;
 use std::num::NonZero;
 use std::ops::Deref;
 use std::pin::Pin;
 use std::sync::atomic::AtomicU64;
 
-use async_lock::{RwLock, RwLockReadGuard};
 use async_once_cell::OnceCell;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::{LocalBoxFuture, ready};
 use futures::{AsyncRead, AsyncWrite, FutureExt};
+use futures_locks::{RwLock, RwLockReadGuard};
 use itertools::Itertools;
 use memo_map::MemoMap;
 use never::Never;
@@ -22,7 +23,7 @@ use orchid_base::name::Sym;
 use crate::api;
 use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
-	MethodSetBuilder, TypAtom, err_not_callable, err_not_command, get_info,
+	MethodSetBuilder, TAtom, err_not_callable, err_not_command, get_info,
 };
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
@@ -53,16 +54,16 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
 /// While an atom read guard is held, no atom can be removed.
 pub(crate) struct AtomReadGuard<'a> {
 	id: api::AtomId,
-	guard: RwLockReadGuard<'a, MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
+	_lock: PhantomData<&'a ()>,
+	guard: RwLockReadGuard<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
 }
 impl<'a> AtomReadGuard<'a> {
 	async fn new(id: api::AtomId, ctx: &'a SysCtx) -> Self {
 		let guard = ctx.get_or_default::<ObjStore>().objects.read().await;
 		if guard.get(&id).is_none() {
-			let valid = guard.iter().map(|i| i.0).collect_vec();
-			panic!("Received invalid atom ID: {id:?} not in {valid:?}");
+			panic!("Received invalid atom ID: {id:?}");
 		}
-		Self { id, guard }
+		Self { id, guard, _lock: PhantomData }
 	}
 }
 impl Deref for AtomReadGuard<'_> {
@@ -317,7 +318,7 @@ pub(crate) struct ObjStore {
 }
 impl SysCtxEntry for ObjStore {}
 
-pub async fn own<A: OwnedAtom>(typ: TypAtom<A>) -> A {
+pub async fn own<A: OwnedAtom>(typ: TAtom<A>) -> A {
 	let ctx = typ.untyped.ctx();
 	let g = ctx.get_or_default::<ObjStore>().objects.read().await;
 	let atom_id = typ.untyped.atom.drop.expect("Owned atoms always have a drop ID");

orchid-extension/src/conv.rs

@@ -1,11 +1,14 @@
 use std::future::Future;
+use std::pin::Pin;
 
+use dyn_clone::DynClone;
 use never::Never;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
+use trait_set::trait_set;
 
-use crate::atom::{AtomicFeatures, ForeignAtom, ToAtom, TypAtom};
+use crate::atom::{AtomicFeatures, ForeignAtom, TAtom, ToAtom};
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, atom, bot};
 use crate::system::{SysCtx, downcast_atom};
@@ -41,7 +44,7 @@ impl TryFromExpr for ForeignAtom {
 	}
 }
 
-impl<A: AtomicFeatures> TryFromExpr for TypAtom<A> {
+impl<A: AtomicFeatures> TryFromExpr for TAtom<A> {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
 		let f = ForeignAtom::try_from_expr(expr).await?;
 		match downcast_atom::<A>(f).await {
@@ -59,6 +62,29 @@ pub trait ToExpr {
 	fn to_expr(self) -> impl Future<Output = GExpr>;
 }
 
+pub trait ToExprDyn {
+	fn to_expr_dyn<'a>(self: Box<Self>) -> Pin<Box<dyn Future<Output = GExpr> + 'a>>
+	where Self: 'a;
+}
+impl<T: ToExpr> ToExprDyn for T {
+	fn to_expr_dyn<'a>(self: Box<Self>) -> Pin<Box<dyn Future<Output = GExpr> + 'a>>
+	where Self: 'a {
+		Box::pin(self.to_expr())
+	}
+}
+trait_set! {
+	pub trait ClonableToExprDyn = ToExprDyn + DynClone;
+}
+impl ToExpr for Box<dyn ToExprDyn> {
+	async fn to_expr(self) -> GExpr { self.to_expr_dyn().await }
+}
+impl ToExpr for Box<dyn ClonableToExprDyn> {
+	async fn to_expr(self) -> GExpr { self.to_expr_dyn().await }
+}
+impl Clone for Box<dyn ClonableToExprDyn> {
+	fn clone(&self) -> Self { dyn_clone::clone_box(&**self) }
+}
+
 impl ToExpr for GExpr {
 	async fn to_expr(self) -> GExpr { self }
 }

orchid-extension/src/coroutine_exec.rs

@@ -35,15 +35,12 @@ impl BuilderCoroutine {
 		match cmd {
 			None => panic!("Before the stream ends, we should have gotten a Halt"),
 			Some(Command::Halt(expr)) => expr,
-			Some(Command::Execute(expr, reply)) => call([
-				lambda(0, [seq([
-					arg(0),
-					call([Replier { reply, builder: self }.to_expr().await, arg(0)]),
-				])]),
-				expr,
-			]),
+			Some(Command::Execute(expr, reply)) => call(
+				lambda(0, seq([arg(0)], call(Replier { reply, builder: self }.to_expr().await, [arg(0)]))),
+				[expr],
+			),
 			Some(Command::Register(expr, reply)) =>
-				call([Replier { reply, builder: self }.to_expr().await, expr]),
+				call(Replier { reply, builder: self }.to_expr().await, [expr]),
 		}
 	}
 }

orchid-extension/src/entrypoint.rs

@@ -5,11 +5,11 @@ use std::num::NonZero;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use async_lock::RwLock;
 use futures::channel::mpsc::{Receiver, Sender, channel};
 use futures::future::{LocalBoxFuture, join_all};
 use futures::lock::Mutex;
 use futures::{FutureExt, SinkExt, StreamExt, stream, stream_select};
+use futures_locks::RwLock;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_api_traits::{Decode, UnderRoot, enc_vec};
@@ -145,10 +145,7 @@ pub fn extension_init(
 				clone!(exit_send mut);
 				async move {
 					match n {
-						api::HostExtNotif::Exit => {
-							eprintln!("Exit received");
-							exit_send.send(()).await.unwrap()
-						},
+						api::HostExtNotif::Exit => exit_send.send(()).await.unwrap(),
 					}
 				}
 				.boxed_local()
@@ -194,13 +191,14 @@ pub fn extension_init(
 								.then(|mem| {
 									let lazy_mems = &lazy_members;
 									clone!(i, ctx; async move {
+										let name = i.i(&mem.name).await;
 										let mut tia_ctx = TreeIntoApiCtxImpl {
 											lazy_members: &mut *lazy_mems.lock().await,
 											sys: ctx,
 											basepath: &[],
-											path: Substack::Bottom,
+											path: Substack::Bottom.push(name.clone()),
 										};
-										(i.i(&mem.name).await.to_api(), mem.kind.into_api(&mut tia_ctx).await)
+										(name.to_api(), mem.kind.into_api(&mut tia_ctx).await)
 									})
 								})
 								.collect()

orchid-extension/src/expr.rs

@@ -52,10 +52,7 @@ impl ExprHandle {
 	/// nothing, otherwise send an Acquire
 	pub async fn drop_one(self: Rc<Self>) {
 		match Rc::try_unwrap(self) {
-			Err(rc) => {
-				eprintln!("Extending lifetime for {:?}", rc.tk);
-				rc.ctx.reqnot().notify(api::Acquire(rc.ctx.sys_id(), rc.tk)).await
-			},
+			Err(rc) => rc.ctx.reqnot().notify(api::Acquire(rc.ctx.sys_id(), rc.tk)).await,
 			Ok(hand) => {
 				// avoid calling destructor
 				hand.destructure();
@@ -65,10 +62,7 @@ impl ExprHandle {
 	/// Drop the handle and get the ticket without a release notification.
 	/// Use this with messages that imply ownership transfer. This function is
 	/// safe because abusing it is a memory leak.
-	pub fn serialize(self) -> api::ExprTicket {
-		eprintln!("Skipping destructor for {:?}", self.tk);
-		self.destructure().0
-	}
+	pub fn serialize(self) -> api::ExprTicket { self.destructure().0 }
 }
 impl Eq for ExprHandle {}
 impl PartialEq for ExprHandle {

orchid-extension/src/func_atom.rs

@@ -43,7 +43,7 @@ struct FunRecord {
 	fun: Rc<dyn FunCB>,
 }
 
-async fn process_args<I, O, F: ExprFunc<I, O>>(
+fn process_args<I, O, F: ExprFunc<I, O>>(
 	debug: impl AsRef<str> + Clone + 'static,
 	f: F,
 ) -> FunRecord {
@@ -83,7 +83,7 @@ impl Fun {
 		let record = if let Some(record) = fung.get(&path) {
 			record.clone()
 		} else {
-			let record = process_args(path.to_string(), f).await;
+			let record = process_args(path.to_string(), f);
 			fung.insert(path.clone(), record.clone());
 			record
 		};
@@ -134,11 +134,8 @@ pub struct Lambda {
 	record: FunRecord,
 }
 impl Lambda {
-	pub async fn new<I, O, F: ExprFunc<I, O>>(
-		debug: impl AsRef<str> + Clone + 'static,
-		f: F,
-	) -> Self {
-		Self { args: vec![], record: process_args(debug, f).await }
+	pub fn new<I, O, F: ExprFunc<I, O>>(debug: impl AsRef<str> + Clone + 'static, f: F) -> Self {
+		Self { args: vec![], record: process_args(debug, f) }
 	}
 }
 impl Atomic for Lambda {
@@ -176,7 +173,7 @@ mod expr_func_derives {
         impl<
           $($t: TryFromExpr + 'static, )*
 					Out: ToExpr,
-          Func: AsyncFn($($t,)*) -> Out + Clone + Send + Sync + 'static
+          Func: AsyncFn($($t,)*) -> Out + Clone + 'static
         > ExprFunc<($($t,)*), Out> for Func {
           fn argtyps() -> &'static [TypeId] {
             static STORE: OnceLock<Vec<TypeId>> = OnceLock::new();

orchid-extension/src/gen_expr.rs

@@ -105,7 +105,7 @@ fn inherit(kind: GExprKind) -> GExpr { GExpr { pos: Pos::Inherit, kind } }
 pub fn sym_ref(path: Sym) -> GExpr { inherit(GExprKind::Const(path)) }
 pub fn atom<A: ToAtom>(atom: A) -> GExpr { inherit(GExprKind::NewAtom(atom.to_atom_factory())) }
 
-pub fn seq(ops: impl IntoIterator<Item = GExpr>) -> GExpr {
+pub fn seq(deps: impl IntoIterator<Item = GExpr>, val: GExpr) -> GExpr {
 	fn recur(mut ops: impl Iterator<Item = GExpr>) -> Option<GExpr> {
 		let op = ops.next()?;
 		Some(match recur(ops) {
@@ -113,19 +113,15 @@ pub fn seq(ops: impl IntoIterator<Item = GExpr>) -> GExpr {
 			Some(rec) => inherit(GExprKind::Seq(Box::new(op), Box::new(rec))),
 		})
 	}
-	recur(ops.into_iter()).expect("Empty list provided to seq!")
+	recur(deps.into_iter().chain([val])).expect("Empty list provided to seq!")
 }
 
 pub fn arg(n: u64) -> GExpr { inherit(GExprKind::Arg(n)) }
 
-pub fn lambda(n: u64, b: impl IntoIterator<Item = GExpr>) -> GExpr {
-	inherit(GExprKind::Lambda(n, Box::new(call(b))))
-}
+pub fn lambda(n: u64, b: GExpr) -> GExpr { inherit(GExprKind::Lambda(n, Box::new(b))) }
 
-pub fn call(v: impl IntoIterator<Item = GExpr>) -> GExpr {
-	v.into_iter()
-		.reduce(|f, x| inherit(GExprKind::Call(Box::new(f), Box::new(x))))
-		.expect("Empty call expression")
+pub fn call(f: GExpr, argv: impl IntoIterator<Item = GExpr>) -> GExpr {
+	(argv.into_iter()).fold(f, |f, x| inherit(GExprKind::Call(Box::new(f), Box::new(x))))
 }
 
 pub fn bot(ev: impl IntoIterator<Item = OrcErr>) -> GExpr {

orchid-extension/src/reflection.rs

@@ -11,23 +11,25 @@ use orchid_base::reqnot::Requester;
 use crate::api;
 use crate::system::{SysCtx, SysCtxEntry, WeakSysCtx};
 
+#[derive(Debug)]
 pub struct ReflMemData {
 	// None for inferred steps
 	public: OnceCell<bool>,
 	kind: ReflMemKind,
 }
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct ReflMem(Rc<ReflMemData>);
 impl ReflMem {
 	pub fn kind(&self) -> ReflMemKind { self.0.kind.clone() }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub enum ReflMemKind {
 	Const,
 	Mod(ReflMod),
 }
 
+#[derive(Debug)]
 pub struct ReflModData {
 	inferred: Mutex<bool>,
 	path: VPath,
@@ -35,7 +37,7 @@ pub struct ReflModData {
 	members: MemoMap<Tok<String>, ReflMem>,
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct ReflMod(Rc<ReflModData>);
 impl ReflMod {
 	fn ctx(&self) -> SysCtx {
@@ -116,8 +118,8 @@ impl ReflMod {
 				Err(api::LsModuleError::InvalidPath) => Err(InvalidPathError { keep_ancestry: false }),
 				Err(api::LsModuleError::IsConstant) => {
 					let const_mem = default_member(self.is_root(), ReflMemKind::Const);
-					self.0.members.insert(next.clone(), const_mem);
-					Err(InvalidPathError { keep_ancestry: true })
+					self.0.members.insert(next.clone(), const_mem.clone());
+					Ok(const_mem)
 				},
 				Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
 			};
@@ -136,6 +138,7 @@ impl ReflMod {
 struct ReflRoot(ReflMod);
 impl SysCtxEntry for ReflRoot {}
 
+#[derive(Clone, Debug)]
 pub struct InvalidPathError {
 	keep_ancestry: bool,
 }

orchid-extension/src/system.rs

@@ -16,7 +16,7 @@ use orchid_base::name::Sym;
 use orchid_base::reqnot::{Receipt, ReqNot};
 
 use crate::api;
-use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TypAtom, get_info};
+use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TAtom, get_info};
 use crate::coroutine_exec::Replier;
 use crate::entrypoint::ExtReq;
 use crate::func_atom::{Fun, Lambda};
@@ -115,7 +115,7 @@ impl<T: System> DynSystem for T {
 	fn card(&self) -> &dyn DynSystemCard { self }
 }
 
-pub async fn downcast_atom<A>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom>
+pub async fn downcast_atom<A>(foreign: ForeignAtom) -> Result<TAtom<A>, ForeignAtom>
 where A: AtomicFeatures {
 	let mut data = &foreign.atom.data.0[..];
 	let ctx = foreign.ctx().clone();
@@ -128,13 +128,16 @@ where A: AtomicFeatures {
 			.ok_or_else(|| foreign.clone())?
 			.get_card()
 	};
+	if owner.atoms().flatten().all(|dynfo| dynfo.tid() != TypeId::of::<A>()) {
+		return Err(foreign);
+	}
 	let (typ_id, dynfo) = get_info::<A>(owner);
 	if value != typ_id {
 		return Err(foreign);
 	}
 	let val = dynfo.decode(AtomCtx(data, foreign.atom.drop, ctx)).await;
 	let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
-	Ok(TypAtom { value, untyped: foreign })
+	Ok(TAtom { value, untyped: foreign })
 }
 
 #[derive(Clone)]
@@ -142,6 +145,9 @@ pub struct WeakSysCtx(Weak<MemoMap<TypeId, Box<dyn Any>>>);
 impl WeakSysCtx {
 	pub fn upgrade(&self) -> Option<SysCtx> { Some(SysCtx(self.0.upgrade()?)) }
 }
+impl fmt::Debug for WeakSysCtx {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "WeakSysCtx") }
+}
 
 #[derive(Clone)]
 pub struct SysCtx(Rc<MemoMap<TypeId, Box<dyn Any>>>);

orchid-extension/src/tree.rs

@@ -173,7 +173,7 @@ impl GenMember {
 		let name = ctx.sys().i().i::<String>(&self.name).await;
 		let kind = self.kind.into_api(&mut ctx.push_path(name.clone())).await;
 		let comments =
-			join_all(self.comments.iter().map(|cmt| async { ctx.sys().i().i(cmt).await.to_api() })).await;
+			join_all(self.comments.iter().map(async |cmt| ctx.sys().i().i(cmt).await.to_api())).await;
 		api::Member { kind, name: name.to_api(), comments, exported: self.public }
 	}
 }