Correctly halts

Cargo.lock

@@ -969,6 +969,7 @@ name = "orchid-api"
 version = "0.1.0"
 dependencies = [
  "futures",
+ "itertools",
  "orchid-api-derive",
  "orchid-api-traits",
  "ordered-float",
@@ -1113,6 +1114,7 @@ dependencies = [
  "ctrlc",
  "futures",
  "itertools",
+ "orchid-api",
  "orchid-base",
  "orchid-host",
  "substack",

orchid-api/Cargo.toml

@@ -10,6 +10,7 @@ ordered-float = "5.0.0"
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 futures = { version = "0.3.31", features = ["std"], default-features = false }
+itertools = "0.14.0"
 
 [dev-dependencies]
 test_executors = "0.3.2"

orchid-api/src/atom.rs

@@ -1,14 +1,28 @@
+use std::fmt;
 use std::num::NonZeroU64;
 
+use itertools::Itertools;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::{
-	ExprTicket, Expression, ExtHostReq, FormattingUnit, HostExtNotif, HostExtReq, OrcResult, SysId,
+	ExprTicket, Expression, ExtHostReq, FormattingUnit, HostExtReq, OrcResult, SysId, TStrv,
-	TStrv,
 };
 
-pub type AtomData = Vec<u8>;
+#[derive(Clone, Coding)]
+pub struct AtomData(pub Vec<u8>);
+impl fmt::Debug for AtomData {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		let mut byte_strings = self.0.iter().map(|b| format!("{b:02x}"));
+		if self.0.len() < 32 {
+			write!(f, "AtomData({})", byte_strings.join(" "))
+		} else {
+			let data_table =
+				byte_strings.chunks(32).into_iter().map(|mut chunk| chunk.join(" ")).join("\n");
+			write!(f, "AtomData(\n{}\n)", data_table)
+		}
+	}
+}
 
 /// Unique ID associated with atoms that have an identity
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
@@ -16,7 +30,7 @@ pub struct AtomId(pub NonZeroU64);
 
 /// An atom owned by an implied system. Usually used in responses from a system.
 /// This has the same semantics as [Atom] except in that the owner is implied.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct LocalAtom {
 	pub drop: Option<AtomId>,
 	pub data: AtomData,
@@ -27,7 +41,7 @@ impl LocalAtom {
 
 /// An atom representation that can be serialized and sent around. Atoms
 /// represent the smallest increment of work.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct Atom {
 	/// Instance ID of the system that created the atom
 	pub owner: SysId,
@@ -49,7 +63,7 @@ pub struct Atom {
 }
 
 /// Attempt to apply an atom as a function to an expression
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct CallRef(pub Atom, pub ExprTicket);
 impl Request for CallRef {
@@ -59,14 +73,14 @@ impl Request for CallRef {
 /// Attempt to apply an atom as a function, consuming the atom and enabling the
 /// library to reuse its datastructures rather than duplicating them. This is an
 /// optimization over [CallRef] followed by [AtomDrop].
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct FinalCall(pub Atom, pub ExprTicket);
 impl Request for FinalCall {
 	type Response = Expression;
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct SerializeAtom(pub Atom);
 impl Request for SerializeAtom {
@@ -81,14 +95,14 @@ impl Request for DeserAtom {
 }
 
 /// A request blindly routed to the system that provides an atom.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct Fwded(pub Atom, pub TStrv, pub Vec<u8>);
 impl Request for Fwded {
 	type Response = Option<Vec<u8>>;
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct Fwd(pub Atom, pub TStrv, pub Vec<u8>);
 impl Request for Fwd {
@@ -100,7 +114,7 @@ pub enum NextStep {
 	Continue(Expression),
 	Halt,
 }
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct Command(pub Atom);
 impl Request for Command {
@@ -111,17 +125,20 @@ impl Request for Command {
 /// isn't referenced anywhere. This should have no effect if the atom's `drop`
 /// flag is false.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
-#[extends(HostExtNotif)]
+#[extends(HostExtReq)]
 pub struct AtomDrop(pub SysId, pub AtomId);
+impl Request for AtomDrop {
+	type Response = ();
+}
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct AtomPrint(pub Atom);
 impl Request for AtomPrint {
 	type Response = FormattingUnit;
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct ExtAtomPrint(pub Atom);
 impl Request for ExtAtomPrint {
@@ -129,7 +146,7 @@ impl Request for ExtAtomPrint {
 }
 
 /// Requests that apply to an existing atom instance
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 #[extendable]
 pub enum AtomReq {

orchid-api/src/expr.rs

@@ -1,3 +1,4 @@
+use std::fmt;
 use std::num::NonZeroU64;
 
 use orchid_api_derive::{Coding, Hierarchy};
@@ -10,8 +11,13 @@ use crate::{Atom, ExtHostNotif, ExtHostReq, Location, OrcError, SysId, TStrv};
 /// [Acquire].
 ///
 /// The ID is globally unique within its lifetime, but may be reused.
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+#[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ExprTicket(pub NonZeroU64);
+impl fmt::Debug for ExprTicket {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "ExprTicket({:x})", self.0.get())
+	}
+}
 
 /// Acquire a strong reference to an expression. This keeps it alive until a
 /// corresponding [Release] is emitted. The number of times a system has
@@ -62,7 +68,7 @@ pub enum ExpressionKind {
 	Arg(u64),
 	/// Insert the specified host-expression in the template here. When the clause
 	/// is used in the const tree, this variant is forbidden.
-	Slot(ExprTicket),
+	Slot { tk: ExprTicket, by_value: bool },
 	/// The lhs must be fully processed before the rhs can be processed.
 	/// Equivalent to Haskell's function of the same name
 	Seq(Box<Expression>, Box<Expression>),

orchid-api/src/parser.rs

@@ -68,10 +68,7 @@ pub enum ParsedMemberKind {
 /// the macro engine could run here.
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
-pub struct FetchParsedConst {
+pub struct FetchParsedConst(pub SysId, pub ParsedConstId);
-	pub sys: SysId,
-	pub id: ParsedConstId,
-}
 impl Request for FetchParsedConst {
 	type Response = Expression;
 }

orchid-api/src/proto.rs

@@ -120,14 +120,14 @@ pub enum HostExtReq {
 	ParseLine(parser::ParseLine),
 	FetchParsedConst(parser::FetchParsedConst),
 	GetMember(tree::GetMember),
+	SystemDrop(system::SystemDrop),
+	AtomDrop(atom::AtomDrop),
 }
 
 /// Notifications sent from the host to the extension
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extendable]
 pub enum HostExtNotif {
-	SystemDrop(system::SystemDrop),
-	AtomDrop(atom::AtomDrop),
 	/// The host can assume that after this notif is sent, a correctly written
 	/// extension will eventually exit.
 	Exit,

orchid-api/src/system.rs

@@ -5,7 +5,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::{CharFilter, ExtHostReq, HostExtNotif, HostExtReq, MemberKind, TStr, TStrv};
+use crate::{CharFilter, ExtHostReq, HostExtReq, MemberKind, TStr, TStrv};
 
 /// ID of a system type
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
@@ -67,8 +67,11 @@ pub struct NewSystemResponse {
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtNotif)]
+#[extends(HostExtReq)]
 pub struct SystemDrop(pub SysId);
+impl Request for SystemDrop {
+	type Response = ();
+}
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(SysReq, HostExtReq)]

orchid-base/src/reqnot.rs

@@ -1,4 +1,3 @@
-use std::any::Any;
 use std::cell::RefCell;
 use std::future::Future;
 use std::marker::PhantomData;
@@ -41,19 +40,19 @@ fn get_id(message: &[u8]) -> (u64, &[u8]) {
 }
 
 pub trait ReqHandlish {
-	fn defer_drop(&self, val: impl Any + 'static)
+	fn defer(&self, cb: impl Future<Output = ()> + 'static)
 	where Self: Sized {
-		self.defer_drop_objsafe(Box::new(val));
+		self.defer_objsafe(Box::pin(cb));
 	}
-	fn defer_drop_objsafe(&self, val: Box<dyn Any>);
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>);
 }
 impl ReqHandlish for &'_ dyn ReqHandlish {
-	fn defer_drop_objsafe(&self, val: Box<dyn Any>) { (**self).defer_drop_objsafe(val) }
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>) { (**self).defer_objsafe(val) }
 }
 
 #[derive(destructure)]
 pub struct RequestHandle<'a, MS: MsgSet> {
-	defer_drop: RefCell<Vec<Box<dyn Any>>>,
+	defer: RefCell<Vec<Pin<Box<dyn Future<Output = ()>>>>>,
 	fulfilled: AtomicBool,
 	id: u64,
 	_reqlt: PhantomData<&'a mut ()>,
@@ -61,13 +60,7 @@ pub struct RequestHandle<'a, MS: MsgSet> {
 }
 impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
 	fn new(parent: ReqNot<MS>, id: u64) -> Self {
-		Self {
+		Self { defer: RefCell::default(), fulfilled: false.into(), _reqlt: PhantomData, parent, id }
-			defer_drop: RefCell::default(),
-			fulfilled: false.into(),
-			_reqlt: PhantomData,
-			parent,
-			id,
-		}
 	}
 	pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
 	pub async fn handle<U: Request>(&self, _: &U, rep: &U::Response) -> Receipt<'a> {
@@ -83,11 +76,17 @@ impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
 		response.encode(Pin::new(&mut buf)).await;
 		let mut send = clone_box(&*self.reqnot().0.lock().await.send);
 		(send)(&buf, self.parent.clone()).await;
+		let deferred = mem::take(&mut *self.defer.borrow_mut());
+		for item in deferred {
+			item.await
+		}
 		Receipt(PhantomData)
 	}
 }
 impl<MS: MsgSet> ReqHandlish for RequestHandle<'_, MS> {
-	fn defer_drop_objsafe(&self, val: Box<dyn Any>) { self.defer_drop.borrow_mut().push(val); }
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>) {
+		self.defer.borrow_mut().push(val)
+	}
 }
 impl<MS: MsgSet> Drop for RequestHandle<'_, MS> {
 	fn drop(&mut self) {
@@ -236,7 +235,10 @@ impl<This: DynRequester + ?Sized> Requester for This {
 	async fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
 		let req = format!("{data:?}");
 		let rep = R::Response::decode(Pin::new(&mut &self.raw_request(data.into()).await[..])).await;
+		let req_str = req.to_string();
+		if !req_str.starts_with("AtomPrint") && !req_str.starts_with("ExtAtomPrint") {
 			writeln!(self.logger(), "Request {req} got response {rep:?}");
+		}
 		rep
 	}
 }

orchid-extension/src/atom_owned.rs

@@ -8,6 +8,7 @@ use std::sync::atomic::AtomicU64;
 
 use async_lock::{RwLock, RwLockReadGuard};
 use async_once_cell::OnceCell;
+use dyn_clone::{DynClone, clone_box};
 use futures::future::{LocalBoxFuture, ready};
 use futures::{AsyncRead, AsyncWrite, FutureExt};
 use itertools::Itertools;
@@ -15,7 +16,7 @@ use memo_map::MemoMap;
 use never::Never;
 use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
-use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit};
+use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, take_first};
 use orchid_base::name::Sym;
 
 use crate::api;
@@ -39,8 +40,10 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
 			let (typ_id, _) = get_info::<A>(ctx.get::<CtedObj>().inst().card());
 			let mut data = enc_vec(&typ_id).await;
 			self.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
-			ctx.get_or_default::<ObjStore>().objects.read().await.insert(atom_id, Box::new(self));
+			let g = ctx.get_or_default::<ObjStore>().objects.read().await;
-			api::Atom { drop: Some(atom_id), data, owner: ctx.sys_id() }
+			g.insert(atom_id, Box::new(self));
+			std::mem::drop(g);
+			api::Atom { drop: Some(atom_id), data: api::AtomData(data), owner: ctx.sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { OwnedAtomDynfo { msbuild: A::reg_reqs(), ms: OnceCell::new() } }
@@ -55,8 +58,10 @@ pub(crate) struct AtomReadGuard<'a> {
 impl<'a> AtomReadGuard<'a> {
 	async fn new(id: api::AtomId, ctx: &'a SysCtx) -> Self {
 		let guard = ctx.get_or_default::<ObjStore>().objects.read().await;
+		if guard.get(&id).is_none() {
 			let valid = guard.iter().map(|i| i.0).collect_vec();
-		assert!(guard.get(&id).is_some(), "Received invalid atom ID: {id:?} not in {valid:?}");
+			panic!("Received invalid atom ID: {id:?} not in {valid:?}");
+		}
 		Self { id, guard }
 	}
 }
@@ -257,7 +262,7 @@ fn assert_serializable<T: OwnedAtom>() {
 	assert_ne!(TypeId::of::<T::Refs>(), TypeId::of::<Never>(), "{MSG}");
 }
 
-pub trait DynOwnedAtom: 'static {
+pub trait DynOwnedAtom: DynClone + 'static {
 	fn atom_tid(&self) -> TypeId;
 	fn as_any_ref(&self) -> &dyn Any;
 	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()>;
@@ -306,16 +311,38 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 }
 
 #[derive(Default)]
-struct ObjStore {
+pub(crate) struct ObjStore {
-	next_id: AtomicU64,
+	pub(crate) next_id: AtomicU64,
-	objects: RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
+	pub(crate) objects: RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
 }
 impl SysCtxEntry for ObjStore {}
 
 pub async fn own<A: OwnedAtom>(typ: TypAtom<A>) -> A {
 	let ctx = typ.untyped.ctx();
 	let g = ctx.get_or_default::<ObjStore>().objects.read().await;
-	let dyn_atom = (g.get(&typ.untyped.atom.drop.expect("Owned atoms always have a drop ID")))
+	let atom_id = typ.untyped.atom.drop.expect("Owned atoms always have a drop ID");
-		.expect("Atom ID invalid; atom type probably not owned by this crate");
+	let dyn_atom =
+		g.get(&atom_id).expect("Atom ID invalid; atom type probably not owned by this crate");
 	dyn_atom.as_any_ref().downcast_ref().cloned().expect("The ID should imply a type as well")
 }
+
+pub async fn debug_print_obj_store(ctx: &SysCtx, show_atoms: bool) {
+	let store = ctx.get_or_default::<ObjStore>();
+	let keys = store.objects.read().await.keys().cloned().collect_vec();
+	let mut message = "Atoms in store:".to_string();
+	if !show_atoms {
+		message += &keys.iter().map(|k| format!(" {:?}", k)).join("");
+	} else {
+		for k in keys {
+			let g = store.objects.read().await;
+			let Some(atom) = g.get(&k) else {
+				message += &format!("\n{k:?} has since been deleted");
+				continue;
+			};
+			let atom = clone_box(&**atom);
+			std::mem::drop(g);
+			message += &format!("\n{k:?} -> {}", take_first(&atom.dyn_print(ctx.clone()).await, true));
+		}
+	}
+	eprintln!("{message}")
+}

orchid-extension/src/atom_thin.rs

@@ -28,7 +28,7 @@ impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant
 			let (id, _) = get_info::<A>(ctx.get::<CtedObj>().inst().card());
 			let mut buf = enc_vec(&id).await;
 			self.encode(Pin::new(&mut buf)).await;
-			api::Atom { drop: None, data: buf, owner: ctx.sys_id() }
+			api::Atom { drop: None, data: api::AtomData(buf), owner: ctx.sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { ThinAtomDynfo { msbuild: Self::reg_reqs(), ms: OnceCell::new() } }

orchid-extension/src/entrypoint.rs

@@ -5,6 +5,7 @@ use std::num::NonZero;
 use std::pin::Pin;
 use std::rc::Rc;
 
+use async_lock::RwLock;
 use futures::channel::mpsc::{Receiver, Sender, channel};
 use futures::future::{LocalBoxFuture, join_all};
 use futures::lock::Mutex;
@@ -54,7 +55,7 @@ pub enum MemberRecord {
 }
 
 pub struct SystemRecord {
-	lazy_members: HashMap<api::TreeId, MemberRecord>,
+	lazy_members: Mutex<HashMap<api::TreeId, MemberRecord>>,
 	ctx: SysCtx,
 }
 
@@ -72,7 +73,7 @@ pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
 	atom: &'a api::Atom,
 	cb: impl WithAtomRecordCallback<'a, T>,
 ) -> T {
-	let mut data = &atom.data[..];
+	let mut data = &atom.data.0[..];
 	let ctx = get_sys_ctx(atom.owner).await;
 	let inst = ctx.get::<CtedObj>().inst();
 	let id = AtomTypeId::decode(Pin::new(&mut data)).await;
@@ -82,7 +83,7 @@ pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
 
 pub struct ExtensionOwner {
 	_interner_cell: Rc<RefCell<Option<Interner>>>,
-	_systems_lock: Rc<Mutex<HashMap<api::SysId, SystemRecord>>>,
+	_systems_lock: Rc<RwLock<HashMap<api::SysId, SystemRecord>>>,
 	out_recv: Mutex<Receiver<Vec<u8>>>,
 	out_send: Sender<Vec<u8>>,
 }
@@ -106,7 +107,7 @@ pub fn extension_init(
 		.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
 		.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
 		.collect_vec();
-	let systems_lock = Rc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
+	let systems_lock = Rc::new(RwLock::new(HashMap::<api::SysId, SystemRecord>::new()));
 	let ext_header = api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() };
 	let (out_send, in_recv) = channel::<Vec<u8>>(1);
 	let (in_send, out_recv) = channel::<Vec<u8>>(1);
@@ -119,7 +120,7 @@ pub fn extension_init(
 	let get_ctx = clone!(systems_weak; move |id: api::SysId| clone!(systems_weak; async move {
 		let systems =
 			systems_weak.upgrade().expect("System table dropped before request processing done");
-		systems.lock().await.get(&id).expect("System not found").ctx.clone()
+		systems.read().await.get(&id).expect("System not found").ctx.clone()
 	}));
 	let init_ctx = {
 		clone!(interner_weak, spawner, logger);
@@ -139,19 +140,14 @@ pub fn extension_init(
 			Box::pin(async move { in_send.send(a.to_vec()).await.unwrap() })
 		},
 		{
-			clone!(systems_weak, exit_send, get_ctx);
+			clone!(exit_send);
 			move |n, _| {
-				clone!(systems_weak, exit_send mut, get_ctx);
+				clone!(exit_send mut);
 				async move {
 					match n {
-						api::HostExtNotif::Exit => exit_send.send(()).await.unwrap(),
+						api::HostExtNotif::Exit => {
-						api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
+							eprintln!("Exit received");
-							if let Some(rc) = systems_weak.upgrade() {
+							exit_send.send(()).await.unwrap()
-								mem::drop(rc.lock().await.remove(&sys_id))
-							},
-						api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) => {
-							let ctx = get_ctx(sys_id).await;
-							take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await
 						},
 					}
 				}
@@ -165,8 +161,22 @@ pub fn extension_init(
 				async move {
 					let interner_cell = interner_weak.upgrade().expect("Interner dropped before request");
 					let i = interner_cell.borrow().clone().expect("Request arrived before interner set");
+					if !matches!(req, api::HostExtReq::AtomReq(api::AtomReq::AtomPrint(_))) {
 						writeln!(msg_logger, "{} extension received request {req:?}", data.name);
+					}
+
 					match req {
+						api::HostExtReq::SystemDrop(sys_drop) => {
+							if let Some(rc) = systems_weak.upgrade() {
+								mem::drop(rc.write().await.remove(&sys_drop.0))
+							}
+							hand.handle(&sys_drop, &()).await
+						},
+						api::HostExtReq::AtomDrop(atom_drop @ api::AtomDrop(sys_id, atom)) => {
+							let ctx = get_ctx(sys_id).await;
+							take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await;
+							hand.handle(&atom_drop, &()).await
+						},
 						api::HostExtReq::Ping(ping @ api::Ping) => hand.handle(&ping, &()).await,
 						api::HostExtReq::Sweep(sweep @ api::Sweep) =>
 							hand.handle(&sweep, &i.sweep_replica().await).await,
@@ -178,18 +188,17 @@ pub fn extension_init(
 								cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
 									char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
 								});
-							let lazy_mems = Mutex::new(HashMap::new());
+							let lazy_members = Mutex::new(HashMap::new());
 							let ctx = init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await;
 							let const_root = stream::iter(cted.inst().dyn_env())
 								.then(|mem| {
-									let (req, lazy_mems) = (&hand, &lazy_mems);
+									let lazy_mems = &lazy_members;
 									clone!(i, ctx; async move {
 										let mut tia_ctx = TreeIntoApiCtxImpl {
 											lazy_members: &mut *lazy_mems.lock().await,
 											sys: ctx,
 											basepath: &[],
 											path: Substack::Bottom,
-											req
 										};
 										(i.i(&mem.name).await.to_api(), mem.kind.into_api(&mut tia_ctx).await)
 									})
@@ -198,9 +207,9 @@ pub fn extension_init(
 								.await;
 							let prelude =
 								cted.inst().dyn_prelude(&i).await.iter().map(|sym| sym.to_api()).collect();
-							let record = SystemRecord { ctx, lazy_members: lazy_mems.into_inner() };
+							let record = SystemRecord { ctx, lazy_members };
 							let systems = systems_weak.upgrade().expect("System constructed during shutdown");
-							systems.lock().await.insert(new_sys.id, record);
+							systems.write().await.insert(new_sys.id, record);
 							let line_types = join_all(
 								(cted.inst().dyn_parsers().iter())
 									.map(|p| async { i.i(p.line_head()).await.to_api() }),
@@ -212,9 +221,9 @@ pub fn extension_init(
 						api::HostExtReq::GetMember(get_tree @ api::GetMember(sys_id, tree_id)) => {
 							let sys_ctx = get_ctx(sys_id).await;
 							let systems = systems_weak.upgrade().expect("Member queried during shutdown");
-							let mut systems_g = systems.lock().await;
+							let systems_g = systems.read().await;
-							let SystemRecord { lazy_members, .. } =
+							let mut lazy_members =
-								systems_g.get_mut(&sys_id).expect("System not found");
+								systems_g.get(&sys_id).expect("System not found").lazy_members.lock().await;
 							let (path, cb) = match lazy_members.insert(tree_id, MemberRecord::Res) {
 								None => panic!("Tree for ID not found"),
 								Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
@@ -225,8 +234,7 @@ pub fn extension_init(
 								sys: sys_ctx,
 								path: Substack::Bottom,
 								basepath: &path,
-								lazy_members,
+								lazy_members: &mut lazy_members,
-								req: &hand,
 							};
 							hand.handle(&get_tree, &tree.into_api(&mut tia_ctx).await).await
 						},
@@ -237,7 +245,7 @@ pub fn extension_init(
 							sys.dyn_request(hand, payload).await
 						},
 						api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, src, text, pos, id }) => {
-							let sys_ctx = get_ctx(sys).await;
+							let mut sys_ctx = get_ctx(sys).await;
 							let text = Tok::from_api(text, &i).await;
 							let src = Sym::from_api(src, sys_ctx.i()).await;
 							let rep = Reporter::new();
@@ -264,7 +272,7 @@ pub fn extension_init(
 										return hand.handle(&lex, &eopt).await;
 									},
 									Ok((s, expr)) => {
-										let expr = expr.into_api(&mut (), &mut (sys_ctx, &hand)).await;
+										let expr = expr.into_api(&mut (), &mut sys_ctx).await;
 										let pos = (text.len() - s.len()) as u32;
 										expr_store.dispose().await;
 										return hand.handle(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
@@ -294,22 +302,22 @@ pub fn extension_init(
 							let parse_res = parser.parse(pctx, *exported, comments, snip).await;
 							let o_line = match reporter.merge(parse_res) {
 								Err(e) => Err(e.to_api()),
-								Ok(t) => Ok(linev_into_api(t, ctx.clone(), &hand).await),
+								Ok(t) => Ok(linev_into_api(t, ctx.clone()).await),
 							};
+							mem::drop(line);
 							expr_store.dispose().await;
 							hand.handle(&pline, &o_line).await
 						},
-						api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst { id, sys }) => {
+						api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst(sys, id)) => {
 							let ctx = get_ctx(sys).await;
 							let cnst = get_const(id, ctx.clone()).await;
-							hand.handle(fpc, &cnst.api_return(ctx, &hand).await).await
+							hand.handle(fpc, &cnst.api_return(ctx).await).await
 						},
 						api::HostExtReq::AtomReq(atom_req) => {
 							let atom = atom_req.get_atom();
 							let atom_req = atom_req.clone();
 							with_atom_record(&get_ctx, atom, async move |nfo, ctx, id, buf| {
 								let actx = AtomCtx(buf, atom.drop, ctx.clone());
-
 								match &atom_req {
 									api::AtomReq::SerializeAtom(ser) => {
 										let mut buf = enc_vec(&id).await;
@@ -340,21 +348,20 @@ pub fn extension_init(
 										hand.handle(fwded, &some.then_some(reply)).await
 									},
 									api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
-										// SAFETY: function calls borrow their argument implicitly
 										let expr_store = BorrowedExprStore::new();
 										let expr_handle = ExprHandle::borrowed(ctx.clone(), *arg, &expr_store);
 										let ret = nfo.call_ref(actx, Expr::from_handle(expr_handle.clone())).await;
-										expr_handle.drop_one().await;
+										let api_expr = ret.api_return(ctx.clone()).await;
-										let api_expr = ret.api_return(ctx.clone(), &hand).await;
+										mem::drop(expr_handle);
 										expr_store.dispose().await;
 										hand.handle(call, &api_expr).await
 									},
 									api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
-										// SAFETY: function calls borrow their argument implicitly
 										let expr_store = BorrowedExprStore::new();
 										let expr_handle = ExprHandle::borrowed(ctx.clone(), *arg, &expr_store);
 										let ret = nfo.call(actx, Expr::from_handle(expr_handle.clone())).await;
-										let api_expr = ret.api_return(ctx.clone(), &hand).await;
+										let api_expr = ret.api_return(ctx.clone()).await;
+										mem::drop(expr_handle);
 										expr_store.dispose().await;
 										hand.handle(call, &api_expr).await
 									},
@@ -363,7 +370,7 @@ pub fn extension_init(
 										Ok(opt) => match opt {
 											None => hand.handle(cmd, &Ok(api::NextStep::Halt)).await,
 											Some(cont) => {
-												let cont = cont.api_return(ctx.clone(), &hand).await;
+												let cont = cont.api_return(ctx.clone()).await;
 												hand.handle(cmd, &Ok(api::NextStep::Continue(cont))).await
 											},
 										},

orchid-extension/src/expr.rs

@@ -51,14 +51,24 @@ impl ExprHandle {
 	/// Drop one instance of the handle silently; if it's the last one, do
 	/// nothing, otherwise send an Acquire
 	pub async fn drop_one(self: Rc<Self>) {
-		if let Err(rc) = Rc::try_unwrap(self) {
+		match Rc::try_unwrap(self) {
+			Err(rc) => {
+				eprintln!("Extending lifetime for {:?}", rc.tk);
 				rc.ctx.reqnot().notify(api::Acquire(rc.ctx.sys_id(), rc.tk)).await
+			},
+			Ok(hand) => {
+				// avoid calling destructor
+				hand.destructure();
+			},
 		}
 	}
 	/// Drop the handle and get the ticket without a release notification.
 	/// Use this with messages that imply ownership transfer. This function is
 	/// safe because abusing it is a memory leak.
-	pub fn serialize(self) -> api::ExprTicket { self.destructure().0 }
+	pub fn serialize(self) -> api::ExprTicket {
+		eprintln!("Skipping destructor for {:?}", self.tk);
+		self.destructure().0
+	}
 }
 impl Eq for ExprHandle {}
 impl PartialEq for ExprHandle {

orchid-extension/src/gen_expr.rs

@@ -1,3 +1,4 @@
+use std::mem;
 use std::rc::Rc;
 
 use futures::FutureExt;
@@ -5,7 +6,6 @@ use orchid_base::error::{OrcErr, OrcErrv};
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::ReqHandlish;
 use orchid_base::{match_mapping, tl_cache};
 
 use crate::api;
@@ -19,17 +19,21 @@ pub struct GExpr {
 	pub pos: Pos,
 }
 impl GExpr {
-	pub async fn api_return(self, ctx: SysCtx, hand: &impl ReqHandlish) -> api::Expression {
+	pub async fn api_return(self, ctx: SysCtx) -> api::Expression {
 		if let GExprKind::Slot(ex) = self.kind {
-			hand.defer_drop(ex.handle());
+			let hand = ex.handle();
+			mem::drop(ex);
 			api::Expression {
 				location: api::Location::SlotTarget,
-				kind: api::ExpressionKind::Slot(ex.handle().tk),
+				kind: match Rc::try_unwrap(hand) {
+					Ok(h) => api::ExpressionKind::Slot { tk: h.serialize(), by_value: true },
+					Err(rc) => api::ExpressionKind::Slot { tk: rc.tk, by_value: false },
+				},
 			}
 		} else {
 			api::Expression {
 				location: api::Location::Inherit,
-				kind: self.kind.api_return(ctx, hand).boxed_local().await,
+				kind: self.kind.api_return(ctx).boxed_local().await,
 			}
 		}
 	}
@@ -53,17 +57,17 @@ pub enum GExprKind {
 	Bottom(OrcErrv),
 }
 impl GExprKind {
-	pub async fn api_return(self, ctx: SysCtx, hand: &impl ReqHandlish) -> api::ExpressionKind {
+	pub async fn api_return(self, ctx: SysCtx) -> api::ExpressionKind {
 		match_mapping!(self, Self => api::ExpressionKind {
 			Call(
-				f => Box::new(f.api_return(ctx.clone(), hand).await),
+				f => Box::new(f.api_return(ctx.clone()).await),
-				x => Box::new(x.api_return(ctx, hand).await)
+				x => Box::new(x.api_return(ctx).await)
 			),
 			Seq(
-				a => Box::new(a.api_return(ctx.clone(), hand).await),
+				a => Box::new(a.api_return(ctx.clone()).await),
-				b => Box::new(b.api_return(ctx, hand).await)
+				b => Box::new(b.api_return(ctx).await)
 			),
-			Lambda(arg, body => Box::new(body.api_return(ctx, hand).await)),
+			Lambda(arg, body => Box::new(body.api_return(ctx).await)),
 			Arg(arg),
 			Const(name.to_api()),
 			Bottom(err.to_api()),

orchid-extension/src/parser.rs

@@ -12,7 +12,7 @@ use orchid_base::location::SrcRange;
 use orchid_base::match_mapping;
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, ParseCtx, Snippet};
-use orchid_base::reqnot::{ReqHandlish, Requester};
+use orchid_base::reqnot::Requester;
 use orchid_base::tree::{TokTree, Token, ttv_into_api};
 
 use crate::api;
@@ -100,8 +100,8 @@ impl ParseCtx for ParsCtx<'_> {
 type BoxConstCallback = Box<dyn FnOnce(ConstCtx) -> LocalBoxFuture<'static, GExpr>>;
 
 #[derive(Default)]
-struct ParsedConstCtxEntry {
+pub(crate) struct ParsedConstCtxEntry {
-	consts: IdStore<BoxConstCallback>,
+	pub(crate) consts: IdStore<BoxConstCallback>,
 }
 impl SysCtxEntry for ParsedConstCtxEntry {}
 
@@ -136,7 +136,7 @@ impl ParsedLine {
 		let comments = comments.into_iter().cloned().collect();
 		ParsedLine { comments, sr: sr.clone(), kind: line_kind }
 	}
-	pub async fn into_api(self, ctx: SysCtx, hand: &dyn ReqHandlish) -> api::ParsedLine {
+	pub async fn into_api(self, mut ctx: SysCtx) -> api::ParsedLine {
 		api::ParsedLine {
 			comments: self.comments.into_iter().map(|c| c.to_api()).collect(),
 			source_range: self.sr.to_api(),
@@ -149,24 +149,20 @@ impl ParsedLine {
 							ctx.get_or_default::<ParsedConstCtxEntry>().consts.add(cb).id(),
 						)),
 						ParsedMemKind::Mod { lines, use_prelude } => api::ParsedMemberKind::Module {
-							lines: linev_into_api(lines, ctx, hand).boxed_local().await,
+							lines: linev_into_api(lines, ctx).boxed_local().await,
 							use_prelude,
 						},
 					},
 				}),
 				ParsedLineKind::Rec(tv) =>
-					api::ParsedLineKind::Recursive(ttv_into_api(tv, &mut (), &mut (ctx, hand)).await),
+					api::ParsedLineKind::Recursive(ttv_into_api(tv, &mut (), &mut ctx).await),
 			},
 		}
 	}
 }
 
-pub(crate) async fn linev_into_api(
+pub(crate) async fn linev_into_api(v: Vec<ParsedLine>, ctx: SysCtx) -> Vec<api::ParsedLine> {
-	v: Vec<ParsedLine>,
+	join_all(v.into_iter().map(|l| l.into_api(ctx.clone()))).await
-	ctx: SysCtx,
-	hand: &dyn ReqHandlish,
-) -> Vec<api::ParsedLine> {
-	join_all(v.into_iter().map(|l| l.into_api(ctx.clone(), hand))).await
 }
 
 pub enum ParsedLineKind {
@@ -218,7 +214,7 @@ impl ConstCtx {
 }
 
 pub(crate) async fn get_const(id: api::ParsedConstId, ctx: SysCtx) -> GExpr {
-	let ent = ctx.get::<ParsedConstCtxEntry>();
+	let ent = ctx.get_or_default::<ParsedConstCtxEntry>();
 	let rec = ent.consts.get(id.0).expect("Bad ID or double read of parsed const");
 	let ctx = ConstCtx { constid: id, ctx: ctx.clone() };
 	rec.remove()(ctx).await

orchid-extension/src/system.rs

@@ -71,7 +71,7 @@ pub async fn resolv_atom(
 	sys: &(impl DynSystemCard + ?Sized),
 	atom: &api::Atom,
 ) -> Box<dyn AtomDynfo> {
-	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data[..])).await;
+	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data.0[..])).await;
 	atom_by_idx(sys, tid).expect("Value of nonexistent type found")
 }
 
@@ -117,7 +117,7 @@ impl<T: System> DynSystem for T {
 
 pub async fn downcast_atom<A>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom>
 where A: AtomicFeatures {
-	let mut data = &foreign.atom.data[..];
+	let mut data = &foreign.atom.data.0[..];
 	let ctx = foreign.ctx().clone();
 	let value = AtomTypeId::decode(Pin::new(&mut data)).await;
 	let own_inst = ctx.get::<CtedObj>().inst();

orchid-extension/src/tree.rs

@@ -9,7 +9,6 @@ use itertools::Itertools;
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::ReqHandlish;
 use orchid_base::tree::{TokTree, Token, TokenVariant};
 use substack::Substack;
 use trait_set::trait_set;
@@ -27,7 +26,7 @@ pub type GenTok = Token<Expr, GExpr>;
 
 impl TokenVariant<api::Expression> for GExpr {
 	type FromApiCtx<'a> = ();
-	type ToApiCtx<'a> = (SysCtx, &'a dyn ReqHandlish);
+	type ToApiCtx<'a> = SysCtx;
 	async fn from_api(
 		_: &api::Expression,
 		_: &mut Self::FromApiCtx<'_>,
@@ -36,8 +35,8 @@ impl TokenVariant<api::Expression> for GExpr {
 	) -> Self {
 		panic!("Received new expression from host")
 	}
-	async fn into_api(self, (ctx, hand): &mut Self::ToApiCtx<'_>) -> api::Expression {
+	async fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> api::Expression {
-		self.api_return(ctx.clone(), hand).await
+		self.api_return(ctx.clone()).await
 	}
 }
 
@@ -188,7 +187,7 @@ impl MemKind {
 	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
 		match self {
 			Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
-			Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys(), ctx.req()).await),
+			Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys()).await),
 			Self::Mod { members } => api::MemberKind::Module(api::Module {
 				members: stream(async |mut cx| {
 					for m in members {
@@ -207,22 +206,19 @@ pub trait TreeIntoApiCtx {
 	fn sys(&self) -> SysCtx;
 	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
 	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
-	fn req(&self) -> &impl ReqHandlish;
 }
 
-pub struct TreeIntoApiCtxImpl<'a, 'b, RH: ReqHandlish> {
+pub struct TreeIntoApiCtxImpl<'a, 'b> {
 	pub sys: SysCtx,
 	pub basepath: &'a [Tok<String>],
 	pub path: Substack<'a, Tok<String>>,
 	pub lazy_members: &'b mut HashMap<api::TreeId, MemberRecord>,
-	pub req: &'a RH,
 }
 
-impl<RH: ReqHandlish> TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_, RH> {
+impl TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_> {
 	fn sys(&self) -> SysCtx { self.sys.clone() }
 	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx {
 		TreeIntoApiCtxImpl {
-			req: self.req,
 			lazy_members: self.lazy_members,
 			sys: self.sys.clone(),
 			basepath: self.basepath,
@@ -235,5 +231,4 @@ impl<RH: ReqHandlish> TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_, RH> {
 		self.lazy_members.insert(id, MemberRecord::Gen(path, fac));
 		id
 	}
-	fn req(&self) -> &impl ReqHandlish { self.req }
 }

orchid-host/src/atom.rs

@@ -25,11 +25,11 @@ impl AtomData {
 	#[must_use]
 	fn api(self) -> api::Atom {
 		let (owner, drop, data, _display) = self.destructure();
-		api::Atom { data, drop, owner: owner.id() }
+		api::Atom { data: api::AtomData(data), drop, owner: owner.id() }
 	}
 	#[must_use]
 	fn api_ref(&self) -> api::Atom {
-		api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
+		api::Atom { data: api::AtomData(self.data.clone()), drop: self.drop, owner: self.owner.id() }
 	}
 }
 impl Drop for AtomData {
@@ -97,7 +97,11 @@ impl AtomRepr for AtomHand {
 	async fn from_api(atom: &api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
 		let api::Atom { data, drop, owner } = atom.clone();
 		let sys = ctx.system_inst(owner).await.expect("Dropped system created atom");
-		if let Some(id) = drop { sys.new_atom(data, id).await } else { AtomHand::new(data, sys, drop) }
+		if let Some(id) = drop {
+			sys.new_atom(data.0, id).await
+		} else {
+			AtomHand::new(data.0, sys, drop)
+		}
 	}
 	async fn to_api(&self) -> orchid_api::Atom { self.api_ref() }
 }

orchid-host/src/expr.rs

@@ -89,7 +89,10 @@ impl Expr {
 			},
 			api::ExpressionKind::NewAtom(a) =>
 				ExprKind::Atom(AtomHand::from_api(a, pos.clone(), &mut ctx.ctx.clone()).await),
-			api::ExpressionKind::Slot(tk) => return ctx.exprs.get_expr(*tk).expect("Invalid slot"),
+			api::ExpressionKind::Slot { tk, by_value: false } =>
+				return ctx.exprs.get_expr(*tk).expect("Invalid slot"),
+			api::ExpressionKind::Slot { tk, by_value: true } =>
+				return ctx.exprs.take_expr(*tk).expect("Invalid slot"),
 			api::ExpressionKind::Seq(a, b) => {
 				let (apsb, bpsb) = psb.split();
 				ExprKind::Seq(

orchid-host/src/expr_store.rs

@@ -2,6 +2,7 @@ use std::cell::RefCell;
 use std::fmt;
 use std::rc::Rc;
 
+use bound::Bound;
 use hashbrown::HashMap;
 use hashbrown::hash_map::Entry;
 
@@ -12,15 +13,29 @@ use crate::expr::Expr;
 pub struct ExprStoreData {
 	exprs: RefCell<HashMap<api::ExprTicket, (u32, Expr)>>,
 	parent: Option<ExprStore>,
+	tracking_parent: bool,
 }
 #[derive(Clone, Default)]
 pub struct ExprStore(Rc<ExprStoreData>);
 impl ExprStore {
+	/// If tracking_parent is false, get_expr can fall back to the parent if none
+	/// is found here.
+	///
+	/// If tracking_parent is true, get_expr can still fall back to the parent,
+	/// but operations on the parent can access the child exprs too until this
+	/// store is dropped.
 	#[must_use]
-	pub fn derive(&self) -> Self {
+	pub fn derive(&self, tracking_parent: bool) -> Self {
-		Self(Rc::new(ExprStoreData { exprs: RefCell::default(), parent: Some(self.clone()) }))
+		Self(Rc::new(ExprStoreData {
+			exprs: RefCell::default(),
+			parent: Some(self.clone()),
+			tracking_parent,
+		}))
 	}
 	pub fn give_expr(&self, expr: Expr) {
+		if self.0.tracking_parent {
+			self.0.parent.as_ref().unwrap().give_expr(expr.clone());
+		}
 		match self.0.exprs.borrow_mut().entry(expr.id()) {
 			Entry::Occupied(mut oe) => oe.get_mut().0 += 1,
 			Entry::Vacant(v) => {
@@ -29,8 +44,11 @@ impl ExprStore {
 		}
 	}
 	pub fn take_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {
+		if self.0.tracking_parent {
+			self.0.parent.as_ref().unwrap().take_expr(ticket);
+		}
 		match self.0.exprs.borrow_mut().entry(ticket) {
-			Entry::Vacant(_) => None,
+			Entry::Vacant(_) => panic!("Attempted to double-take expression"),
 			Entry::Occupied(oe) if oe.get().0 == 1 => Some(oe.remove().1),
 			Entry::Occupied(mut oe) => {
 				oe.get_mut().0 -= 1;
@@ -43,6 +61,11 @@ impl ExprStore {
 		(self.0.exprs.borrow().get(&ticket).map(|(_, expr)| expr.clone()))
 			.or_else(|| self.0.parent.as_ref()?.get_expr(ticket))
 	}
+	pub fn iter(&self) -> impl Iterator<Item = (u32, Expr)> {
+		let r = Bound::new(self.clone(), |this| this.0.exprs.borrow());
+		let mut iter = Bound::new(r, |r| r.values());
+		std::iter::from_fn(move || iter.wrapped_mut().next().cloned())
+	}
 }
 impl fmt::Display for ExprStore {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -51,3 +74,19 @@ impl fmt::Display for ExprStore {
 		write!(f, "Store holding {rc} refs to {} exprs", r.len())
 	}
 }
+impl Drop for ExprStore {
+	fn drop(&mut self) {
+		if 1 < Rc::strong_count(&self.0) {
+			return;
+		}
+		if !self.0.tracking_parent {
+			return;
+		}
+		let parent = self.0.parent.as_ref().unwrap();
+		for (id, (count, _)) in self.0.exprs.borrow().iter() {
+			for _ in 0..*count {
+				parent.take_expr(*id);
+			}
+		}
+	}
+}

orchid-host/src/extension.rs

@@ -40,6 +40,7 @@ pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 /// upgrading fails.
 #[derive(destructure)]
 pub struct ExtensionData {
+	name: String,
 	ctx: Ctx,
 	reqnot: ReqNot<api::HostMsgSet>,
 	systems: Vec<SystemCtor>,
@@ -67,8 +68,14 @@ impl Extension {
 		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| {
 			let init = Rc::new(init);
 			let (exiting_snd, exiting_rcv) = channel::<()>(0);
-			(ctx.spawn)(clone!(init, weak, ctx; Box::pin(async move {
+			(ctx.spawn)({
-				let rcv_stream = stream(async |mut cx| loop { cx.emit( init.recv().await).await });
+				clone!(init, weak, ctx);
+				Box::pin(async move {
+					let rcv_stream = stream(async |mut cx| {
+						loop {
+							cx.emit(init.recv().await).await
+						}
+					});
 					let mut event_stream = pin!(stream::select(exiting_rcv.map(|()| None), rcv_stream));
 					while let Some(Some(msg)) = event_stream.next().await {
 						if let Some(reqnot) = weak.upgrade().map(|rc| rc.reqnot.clone()) {
@@ -78,10 +85,12 @@ impl Extension {
 							}))
 						}
 					}
-			})));
+				})
+			});
 			ExtensionData {
+				name: init.name.clone(),
 				exiting_snd,
-				exprs: ctx.common_exprs.derive(),
+				exprs: ctx.common_exprs.derive(false),
 				ctx: ctx.clone(),
 				systems: (init.systems.iter().cloned())
 					.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
@@ -95,17 +104,26 @@ impl Extension {
 					clone!(weak; move |notif, _| {
 						clone!(weak; Box::pin(async move {
 						let this = Extension(weak.upgrade().unwrap());
+						if !matches!(notif, api::ExtHostNotif::Log(_)) {
+							writeln!(this.reqnot().logger(), "Host received notif {notif:?}");
+						}
 						match notif {
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
 								let target = this.0.exprs.get_expr(acq.1).expect("Invalid ticket");
 								this.0.exprs.give_expr(target)
 							}
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
-								this.assert_own_sys(rel.0).await;
+								if this.is_own_sys(rel.0).await {
 									this.0.exprs.take_expr(rel.1);
+								} else {
+									writeln!(this.reqnot().logger(), "Not our system {:?}", rel.0)
+								}
 							}
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
-								this.assert_own_sys(mov.dec).await;
+								if !this.is_own_sys(mov.dec).await {
+									writeln!(this.reqnot().logger(), "Not our system {:?}", mov.dec);
+									return;
+								}
 								let recp = this.ctx().system_inst(mov.inc).await.expect("invallid recipient sys id");
 								let expr = this.0.exprs.get_expr(mov.expr).expect("invalid ticket");
 								recp.ext().0.exprs.give_expr(expr);
@@ -120,7 +138,9 @@ impl Extension {
 							clone!(weak, ctx);
 							Box::pin(async move {
 								let this = Self(weak.upgrade().unwrap());
+								if !matches!(req, api::ExtHostReq::ExtAtomPrint(_)) {
 									writeln!(this.reqnot().logger(), "Host received request {req:?}");
+								}
 								let i = this.ctx().i.clone();
 								match req {
 									api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
@@ -235,8 +255,9 @@ impl Extension {
 			}
 		})))
 	}
+	pub fn name(&self) -> &String { &self.0.name }
 	#[must_use]
-	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.reqnot }
+	pub fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.reqnot }
 	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
 	#[must_use]
@@ -246,12 +267,12 @@ impl Extension {
 	pub fn exprs(&self) -> &ExprStore { &self.0.exprs }
 	#[must_use]
 	pub async fn is_own_sys(&self, id: api::SysId) -> bool {
-		let sys = self.ctx().system_inst(id).await.expect("invalid sender sys id");
+		let Some(sys) = self.ctx().system_inst(id).await else {
+			writeln!(self.logger(), "Invalid system ID {id:?}");
+			return false;
+		};
 		Rc::ptr_eq(&self.0, &sys.ext().0)
 	}
-	pub async fn assert_own_sys(&self, id: api::SysId) {
-		assert!(self.is_own_sys(id).await, "Incoming message impersonates separate system");
-	}
 	#[must_use]
 	pub fn next_pars(&self) -> NonZeroU64 {
 		let mut next_pars = self.0.next_pars.borrow_mut();
@@ -293,7 +314,7 @@ impl Extension {
 	pub fn system_drop(&self, id: api::SysId) {
 		let rc = self.clone();
 		(self.ctx().spawn)(Box::pin(async move {
-			rc.reqnot().notify(api::SystemDrop(id)).await;
+			rc.reqnot().request(api::SystemDrop(id)).await;
 			rc.ctx().systems.write().await.remove(&id);
 		}))
 	}

orchid-host/src/lex.rs

@@ -2,16 +2,19 @@ use std::rc::Rc;
 
 use futures::FutureExt;
 use futures::lock::Mutex;
+use orchid_base::clone;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
 use orchid_base::interner::Tok;
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::PARENS;
+use orchid_base::tree::recur;
 
 use crate::api;
 use crate::ctx::Ctx;
 use crate::expr::{Expr, ExprParseCtx};
+use crate::expr_store::ExprStore;
 use crate::parsed::{ParsTok, ParsTokTree, tt_to_api};
 use crate::system::System;
 
@@ -52,14 +55,14 @@ impl<'a> LexCtx<'a> {
 		false
 	}
 	#[must_use]
-	pub async fn ser_subtree(&mut self, subtree: ParsTokTree) -> api::TokenTree {
+	pub async fn ser_subtree(&mut self, subtree: ParsTokTree, exprs: ExprStore) -> api::TokenTree {
-		tt_to_api(&mut self.ctx.common_exprs.clone(), subtree).await
+		tt_to_api(&mut { exprs }, subtree).await
 	}
 	#[must_use]
-	pub async fn des_subtree(&mut self, tree: &api::TokenTree) -> ParsTokTree {
+	pub async fn des_subtree(&mut self, tree: &api::TokenTree, exprs: ExprStore) -> ParsTokTree {
 		ParsTokTree::from_api(
 			tree,
-			&mut self.ctx.common_exprs.clone(),
+			&mut { exprs },
 			&mut ExprParseCtx { ctx: self.ctx, exprs: &self.ctx.common_exprs },
 			self.path,
 			&self.ctx.i,
@@ -145,16 +148,24 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 				let (source, pos, path) = (ctx.source.clone(), ctx.get_pos(), ctx.path.clone());
 				let ctx_lck = &Mutex::new(&mut *ctx);
 				let errors_lck = &Mutex::new(&mut errors);
+				let temp_store = sys.ext().exprs().derive(true);
+				let temp_store_cb = temp_store.clone();
 				let lx = sys
-					.lex(source, path, pos, |pos| async move {
+					.lex(source, path, pos, |pos| {
+						clone!(temp_store_cb);
+						async move {
 							let mut ctx_g = ctx_lck.lock().await;
 							match lex_once(&mut ctx_g.push(pos)).boxed_local().await {
-							Ok(t) => Some(api::SubLexed { pos: t.sr.end(), tree: ctx_g.ser_subtree(t).await }),
+								Ok(t) => Some(api::SubLexed {
+									pos: t.sr.end(),
+									tree: ctx_g.ser_subtree(t, temp_store_cb.clone()).await,
+								}),
 								Err(e) => {
 									errors_lck.lock().await.push(e);
 									None
 								},
 							}
+						}
 					})
 					.await;
 				match lx {
@@ -164,7 +175,14 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 						),
 					Ok(Some(lexed)) => {
 						ctx.set_pos(lexed.pos);
-						return Ok(ctx.des_subtree(&lexed.expr).await);
+						let lexed_tree = ctx.des_subtree(&lexed.expr, temp_store).await;
+						let stable_tree = recur(lexed_tree, &|tt, r| {
+							if let ParsTok::NewExpr(expr) = tt.tok {
+								return ParsTok::Handle(expr).at(tt.sr);
+							}
+							r(tt)
+						});
+						return Ok(stable_tree);
 					},
 					Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
 						Some(errors) => return Err(errors),

orchid-host/src/parsed.rs

@@ -113,7 +113,6 @@ pub enum ParsedMemberKind {
 impl From<ParsedModule> for ParsedMemberKind {
 	fn from(value: ParsedModule) -> Self { Self::Mod(value) }
 }
-
 #[derive(Debug, Default)]
 pub struct ParsedModule {
 	pub exports: Vec<Tok<String>>,

orchid-host/src/sys_parser.rs

@@ -35,11 +35,10 @@ impl Parser {
 		comments: Vec<Comment>,
 		callback: &mut impl AsyncFnMut(ModPath<'_>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
 	) -> OrcRes<Vec<Item>> {
+		let mut temp_store = self.system.ext().exprs().derive(true);
 		let src_path = line.first().expect("cannot be empty").sr.path();
-		let line = join_all(
+		let line =
-			(line.into_iter())
+			join_all((line.into_iter()).map(|t| async { tt_to_api(&mut temp_store.clone(), t).await }))
-				.map(|t| async { tt_to_api(&mut self.system.ext().exprs().clone(), t).await }),
-		)
 				.await;
 		let mod_path = ctx.src_path().suffix(path.unreverse(), self.system.i()).await;
 		let comments = comments.iter().map(Comment::to_api).collect_vec();
@@ -53,18 +52,16 @@ impl Parser {
 			line,
 		};
 		match self.system.reqnot().request(req).await {
-			Ok(parsed_v) => {
+			Ok(parsed_v) =>
-				let mut ext_exprs = self.system.ext().exprs().clone();
 				conv(parsed_v, path, callback, &mut ConvCtx {
 					i: self.system.i(),
 					mod_path: &mod_path,
-					ext_exprs: &mut ext_exprs,
+					ext_exprs: &mut temp_store,
 					pctx: &mut ExprParseCtx { ctx: self.system.ctx(), exprs: self.system.ext().exprs() },
 					src_path: &src_path,
 					sys: &self.system,
 				})
-				.await
+				.await,
-			},
 			Err(e) => Err(OrcErrv::from_api(&e, &self.system.ctx().i).await),
 		}
 	}

orchid-host/src/system.rs

@@ -57,6 +57,10 @@ impl fmt::Debug for SystemInstData {
 #[derive(Clone, Debug)]
 pub struct System(pub(crate) Rc<SystemInstData>);
 impl System {
+	#[must_use]
+	pub async fn atoms(&self) -> impl std::ops::Deref<Target = HashMap<api::AtomId, WeakAtomHand>> {
+		self.0.owned_atoms.read().await
+	}
 	#[must_use]
 	pub fn id(&self) -> api::SysId { self.0.id }
 	#[must_use]
@@ -118,14 +122,17 @@ impl System {
 		owned_g.insert(id, new.downgrade());
 		new
 	}
-	pub(crate) fn drop_atom(&self, drop: api::AtomId) {
+	pub(crate) fn drop_atom(&self, dropped_atom_id: api::AtomId) {
 		let this = self.0.clone();
 		(self.0.ctx.spawn)(Box::pin(async move {
-			this.owned_atoms.write().await.remove(&drop);
+			this.ext.reqnot().request(api::AtomDrop(this.id, dropped_atom_id)).await;
+			this.owned_atoms.write().await.remove(&dropped_atom_id);
 		}))
 	}
 	#[must_use]
-	pub fn downgrade(&self) -> WeakSystem { WeakSystem(Rc::downgrade(&self.0)) }
+	pub fn downgrade(&self) -> WeakSystem {
+		WeakSystem(Rc::downgrade(&self.0), self.0.decl_id, self.ext().downgrade())
+	}
 	/// Implementation of [api::ResolveNames]
 	pub(crate) async fn name_resolver(
 		&self,
@@ -174,10 +181,14 @@ impl Format for System {
 	}
 }
 
-pub struct WeakSystem(Weak<SystemInstData>);
+pub struct WeakSystem(Weak<SystemInstData>, api::SysDeclId, WeakExtension);
 impl WeakSystem {
 	#[must_use]
 	pub fn upgrade(&self) -> Option<System> { self.0.upgrade().map(System) }
+	pub fn ext(&self) -> Option<Extension> { self.2.upgrade() }
+	pub fn ctor(&self) -> Option<SystemCtor> {
+		self.ext()?.system_ctors().find(|ctor| ctor.decl.id == self.1).cloned()
+	}
 }
 
 #[derive(Clone)]

orchid-host/src/tree.rs

@@ -6,6 +6,7 @@ use std::slice;
 
 use async_lock::RwLock;
 use async_once_cell::OnceCell;
+use derive_destructure::destructure;
 use futures::{FutureExt, StreamExt, stream};
 use hashbrown::HashMap;
 use hashbrown::hash_map::Entry;
@@ -88,7 +89,7 @@ impl Root {
 		*this.ctx.root.write().await = new.downgrade();
 		for (path, (sys_id, pc_id)) in deferred_consts {
 			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
-			let api_expr = sys.reqnot().request(api::FetchParsedConst { id: pc_id, sys: sys.id() }).await;
+			let api_expr = sys.reqnot().request(api::FetchParsedConst(sys.id(), pc_id)).await;
 			let mut xp_ctx = ExprParseCtx { ctx: &this.ctx, exprs: sys.ext().exprs() };
 			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), &mut xp_ctx).await;
 			new.0.write().await.consts.insert(path, expr);
@@ -450,6 +451,7 @@ impl MemberKind {
 	}
 }
 
+#[derive(destructure)]
 pub struct LazyMemberHandle {
 	id: api::TreeId,
 	sys: api::SysId,
@@ -457,19 +459,26 @@ pub struct LazyMemberHandle {
 }
 impl LazyMemberHandle {
 	#[must_use]
-	pub async fn run(self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> MemberKind {
+	pub async fn run(mut self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> MemberKind {
 		let sys = ctx.system_inst(self.sys).await.expect("Missing system for lazy member");
 		match sys.get_tree(self.id).await {
 			api::MemberKind::Const(c) => {
 				let mut pctx = ExprParseCtx { ctx: &ctx, exprs: sys.ext().exprs() };
 				let expr = Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await;
-				consts.insert(self.path, expr);
+				let (.., path) = self.destructure();
+				consts.insert(path, expr);
 				MemberKind::Const
 			},
-			api::MemberKind::Module(m) => MemberKind::Module(
+			api::MemberKind::Module(m) => {
-				Module::from_api(m, &mut TreeFromApiCtx { sys: &sys, consts, path: self.path.tok() }).await,
+				let (.., path) = self.destructure();
-			),
+				MemberKind::Module(
-			api::MemberKind::Lazy(id) => Self { id, ..self }.run(ctx, consts).boxed_local().await,
+					Module::from_api(m, &mut TreeFromApiCtx { sys: &sys, consts, path: path.tok() }).await,
+				)
+			},
+			api::MemberKind::Lazy(id) => {
+				self.id = id;
+				self.run(ctx, consts).boxed_local().await
+			},
 		}
 	}
 	#[must_use]

orchid-std/src/macros/recur_state.rs

@@ -3,6 +3,7 @@ use std::fmt;
 use std::rc::Rc;
 
 use never::Never;
+use orchid_base::format::{FmtCtx, FmtUnit};
 use orchid_base::interner::Tok;
 use orchid_base::name::Sym;
 use orchid_extension::atom::Atomic;
@@ -56,4 +57,15 @@ impl OwnedAtom for RecurState {
 			Self::Recursive { .. } => Some(()),
 		})
 	}
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		self.to_string().into()
+	}
+}
+impl fmt::Display for RecurState {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Self::Bottom => write!(f, "RecurState::Bottom"),
+			Self::Recursive { path, prev } => write!(f, "{path}\n{prev}"),
+		}
+	}
 }

orchid.code-workspace

@@ -37,6 +37,12 @@
     // Important; for accessibility reasons, code cannot be wider than 100ch
 		"editor.rulers": [ 100 ],
 		"editor.formatOnSave": true,
+    "files.watcherExclude": {
+      "**/.git/objects/**": true,
+      "**/.git/subtree-cache/**": true,
+      "**/.hg/store/**": true,
+      "target": true,
+    },
 		"git.confirmSync": false,
 		"git.enableSmartCommit": true,
 		"git.autofetch": true,

orcx/Cargo.toml

@@ -12,6 +12,7 @@ clap = { version = "4.5.24", features = ["derive", "env"] }
 ctrlc = "3.4.5"
 futures = "0.3.31"
 itertools = "0.14.0"
+orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 orchid-host = { version = "0.1.0", path = "../orchid-host" }
 substack = "1.1.1"

test-log.txt

@@ -1 +0,0 @@
-Upsending: [ff ff ff ff ff ff ff f7 00 00 00 00 00 00 00 08 22 75 73 65 72 21 22 69]