partway towards commands

I got very confused and started mucking about with "spawn" when in fact all I needed was the "inline" extension type in orcx that allows the interpreter to expose custom constants.

orchid-host/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-event = "0.2.1"
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
 bound = "0.6.0"

orchid-host/src/atom.rs

@@ -5,16 +5,13 @@ use async_once_cell::OnceCell;
 use derive_destructure::destructure;
 #[cfg(feature = "orchid-extension")]
 use orchid_api_traits::{Request, UnderRoot};
-use orchid_base::format::{FmtCtx, FmtUnit, Format, take_first_fmt};
-use orchid_base::location::Pos;
-use orchid_base::reqnot::ClientExt;
-use orchid_base::tree::AtomRepr;
+use orchid_base::{AtomRepr, ClientExt, FmtCtx, FmtUnit, Format, Pos, take_first_fmt};
 #[cfg(feature = "orchid-extension")]
-use orchid_extension::atom::AtomMethod;
+use orchid_extension::AtomMethod;
 
 use crate::api;
 use crate::ctx::Ctx;
-use crate::expr::{Expr, PathSetBuilder};
+use crate::expr::{Expr, ExprFromApiCtx, PathSetBuilder};
 use crate::extension::Extension;
 use crate::system::System;
 
@@ -66,7 +63,7 @@ impl AtomHand {
 		method: M,
 	) -> Option<M::Response> {
 		use orchid_api_traits::{Decode, Encode};
-		use orchid_base::name::Sym;
+		use orchid_base::Sym;
 
 		let name = Sym::parse(<M as UnderRoot>::Root::NAME).await.unwrap();
 		let mut buf = Vec::new();
@@ -76,15 +73,19 @@ impl AtomHand {
 	}
 	#[must_use]
 	pub async fn call(self, arg: Expr) -> Expr {
-		let owner_sys = self.0.owner.clone();
-		let ctx = owner_sys.ctx();
-		let client = owner_sys.client();
+		let owner = self.0.owner.clone();
+		let ctx = owner.ctx();
+		let client = owner.client();
 		ctx.exprs.give_expr(arg.clone());
 		let ret = match Rc::try_unwrap(self.0) {
 			Ok(data) => client.request(api::FinalCall(data.api(), arg.id())).await.unwrap(),
 			Err(hand) => client.request(api::CallRef(hand.api_ref(), arg.id())).await.unwrap(),
 		};
-		let val = Expr::from_api(&ret, PathSetBuilder::new(), ctx.clone()).await;
+		let val = Expr::from_api(ret, PathSetBuilder::new(), ExprFromApiCtx {
+			sys: owner.id(),
+			ctx: ctx.clone(),
+		})
+		.await;
 		ctx.exprs.take_expr(arg.id());
 		val
 	}

orchid-host/src/ctx.rs

@@ -1,6 +1,7 @@
 use std::cell::RefCell;
 use std::num::{NonZero, NonZeroU16};
 use std::rc::{Rc, Weak};
+use std::time::Duration;
 use std::{fmt, ops};
 
 use futures::future::LocalBoxFuture;
@@ -14,12 +15,21 @@ use crate::system::{System, WeakSystem};
 use crate::tree::WeakRoot;
 
 pub trait JoinHandle {
+	/// It is guaranteed that the future will never be polled after this is called
 	fn abort(&self);
+	/// take the future out of the task. If the return value
+	/// is dropped, the spawned future is also dropped
 	fn join(self: Box<Self>) -> LocalBoxFuture<'static, ()>;
 }
 
 pub trait Spawner {
-	fn spawn_obj(&self, fut: LocalBoxFuture<'static, ()>) -> Box<dyn JoinHandle>;
+	/// Spawn a future off-task, but in the same thread. If all objects associated
+	/// with the owning [Ctx] are dropped (eg. expressions, extensions, systems)
+	/// and no extensions create permanent ref loops, then all tasks will
+	/// eventually settle, therefore the implementor of this interface should not
+	/// exit while there are pending tasks to allow external communication
+	/// channels to cleanly shut down.
+	fn spawn_obj(&self, delay: Duration, fut: LocalBoxFuture<'static, ()>) -> Box<dyn JoinHandle>;
 }
 
 pub struct CtxData {
@@ -58,12 +68,16 @@ impl Ctx {
 	}
 	/// Spawn a parallel future that you can join at any later time.
 	///
-	/// Don't use this for async Drop, use [orchid_base::stash::stash] instead.
+	/// Don't use this for async Drop, use [orchid_base::stash] instead.
 	/// If you use this for an actor object, make sure to actually join the
 	/// handle.
 	#[must_use]
-	pub fn spawn(&self, fut: impl Future<Output = ()> + 'static) -> Box<dyn JoinHandle> {
-		self.spawner.spawn_obj(Box::pin(fut))
+	pub fn spawn(
+		&self,
+		delay: Duration,
+		fut: impl Future<Output = ()> + 'static,
+	) -> Box<dyn JoinHandle> {
+		self.spawner.spawn_obj(delay, Box::pin(fut))
 	}
 	#[must_use]
 	pub(crate) async fn system_inst(&self, id: api::SysId) -> Option<System> {

orchid-host/src/dealias.rs

@@ -1,9 +1,6 @@
 use hashbrown::HashSet;
 use itertools::Itertools;
-use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
-use orchid_base::interner::{IStr, is};
-use orchid_base::location::Pos;
-use orchid_base::name::VName;
+use orchid_base::{IStr, OrcErrv, OrcRes, Pos, VName, is, mk_errv};
 
 /// Errors produced by absolute_path
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]

orchid-host/src/dylib.rs

@@ -1,18 +1,19 @@
 use std::io;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 
 use futures::io::BufReader;
 use futures::{AsyncBufReadExt, StreamExt};
 use hashbrown::HashMap;
 use libloading::{Library, Symbol};
-use orchid_base::binary::vt_to_future;
-use orchid_base::logging::log;
+use orchid_base::{log, on_drop, vt_to_future};
 use unsync_pipe::pipe;
 
 use crate::api;
 use crate::ctx::Ctx;
 use crate::extension::ExtPort;
+use crate::task_set::TaskSet;
 
 static DYNAMIC_LIBRARIES: Mutex<Option<HashMap<PathBuf, Arc<Library>>>> = Mutex::new(None);
 fn load_dylib(path: &Path) -> Result<Arc<Library>, libloading::Error> {
@@ -32,7 +33,7 @@ pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Err
 	let (output, read_output) = pipe(1024);
 	let (write_log, read_log) = pipe(1024);
 	let log_path = path.to_string_lossy().to_string();
-	let _ = ctx.spawn(async move {
+	let _ = ctx.spawn(Duration::ZERO, async move {
 		let mut lines = BufReader::new(read_log).lines();
 		while let Some(line) = lines.next().await {
 			match line {
@@ -44,16 +45,43 @@ pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Err
 			}
 		}
 	});
+	let tasks = TaskSet::default();
 	let library = load_dylib(path)?;
 	let entrypoint: Symbol<unsafe extern "C" fn(api::binary::ExtensionContext)> =
 		unsafe { library.get("orchid_extension_main") }?;
-	let data = Box::into_raw(Box::new(ctx)) as *const ();
-	extern "C" fn drop(data: *const ()) { std::mem::drop(unsafe { Box::from_raw(data as *mut Ctx) }) }
-	extern "C" fn spawn(data: *const (), vt: api::binary::FutureBin) {
-		let _ = unsafe { (data as *mut Ctx).as_mut().unwrap().spawn(vt_to_future(vt)) };
-	}
+
+	let data = Box::into_raw(Box::new(SpawnerState { ctx, tasks: tasks.clone() })) as *const ();
 	let spawner = api::binary::SpawnerBin { data, drop, spawn };
 	let cx = api::binary::ExtensionContext { input, output, log: write_log, spawner };
 	unsafe { (entrypoint)(cx) };
-	Ok(ExtPort { input: Box::pin(write_input), output: Box::pin(read_output) })
+	Ok(ExtPort {
+		input: Box::pin(write_input),
+		output: Box::pin(read_output),
+		drop_trigger: Box::new(on_drop(move || tasks.abort_all())),
+	})
+}
+
+struct SpawnerState {
+	ctx: Ctx,
+	tasks: TaskSet,
+}
+extern "C" fn drop(data: *const ()) {
+	let state = unsafe { Box::from_raw(data as *mut SpawnerState) };
+	state.tasks.abort_all();
+}
+extern "C" fn spawn(data: *const (), delay: u64, vt: api::binary::FutureBin) {
+	let future = vt_to_future(vt);
+	// SAFETY: this is technically a Box but it can be used directly as a &mut
+	let state = unsafe { (data as *mut SpawnerState).as_mut() }.unwrap();
+	state.tasks.with(|store| {
+		store.add_with(|id| {
+			state.ctx.spawn(Duration::from_millis(delay), async move {
+				future.await;
+				// SAFETY: We know this is live because the drop handle that frees it also
+				// aborts the future
+				let state = unsafe { (data as *mut SpawnerState).as_mut() }.unwrap();
+				state.tasks.with(|store| store.remove(id));
+			})
+		});
+	});
 }

orchid-host/src/execute.rs

@@ -3,10 +3,7 @@ use std::mem;
 use bound::Bound;
 use futures::FutureExt;
 use futures_locks::{RwLockWriteGuard, TryLockError};
-use orchid_base::error::OrcErrv;
-use orchid_base::format::fmt;
-use orchid_base::location::Pos;
-use orchid_base::logging::log;
+use orchid_base::{OrcErrv, Pos, fmt, log};
 
 use crate::expr::{Expr, ExprKind, PathSet, Step};
 use crate::tree::Root;

orchid-host/src/expr.rs

@@ -7,12 +7,10 @@ use std::{fmt, mem};
 use futures::FutureExt;
 use futures_locks::RwLock;
 use itertools::Itertools;
-use orchid_base::error::OrcErrv;
-use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
-use orchid_base::location::{Pos, SrcRange};
-use orchid_base::name::Sym;
-use orchid_base::tl_cache;
-use orchid_base::tree::{AtomRepr, TokenVariant, indent};
+use orchid_base::{
+	AtomRepr, FmtCtx, FmtUnit, Format, OrcErrv, Pos, SrcRange, Sym, TokenVariant, Variants, indent,
+	tl_cache,
+};
 use substack::Substack;
 
 use crate::api;
@@ -26,6 +24,12 @@ pub struct ExprData {
 	kind: RwLock<ExprKind>,
 }
 
+#[derive(Clone, Debug)]
+pub struct ExprFromApiCtx {
+	pub sys: api::SysId,
+	pub ctx: Ctx,
+}
+
 #[derive(Clone, Debug)]
 pub struct Expr(Rc<ExprData>);
 impl Expr {
@@ -54,35 +58,40 @@ impl Expr {
 		)
 	}
 	#[must_use]
-	pub async fn from_api(api: &api::Expression, psb: PathSetBuilder<'_, u64>, ctx: Ctx) -> Self {
+	pub async fn from_api(
+		api: api::Expression,
+		psb: PathSetBuilder<'_, u64>,
+		ctx: ExprFromApiCtx,
+	) -> Self {
 		let pos = Pos::from_api(&api.location).await;
-		let kind = match &api.kind {
+		let kind = match api.kind {
 			api::ExpressionKind::Arg(n) => {
-				assert!(psb.register_arg(n), "Arguments must be enclosed in a matching lambda");
+				assert!(psb.register_arg(&n), "Arguments must be enclosed in a matching lambda");
 				ExprKind::Arg
 			},
 			api::ExpressionKind::Bottom(bot) => ExprKind::Bottom(OrcErrv::from_api(bot).await),
 			api::ExpressionKind::Call(f, x) => {
 				let (lpsb, rpsb) = psb.split();
 				ExprKind::Call(
-					Expr::from_api(f, lpsb, ctx.clone()).boxed_local().await,
-					Expr::from_api(x, rpsb, ctx).boxed_local().await,
+					Expr::from_api(*f, lpsb, ctx.clone()).boxed_local().await,
+					Expr::from_api(*x, rpsb, ctx).boxed_local().await,
 				)
 			},
-			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(*name).await),
+			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(name).await),
 			api::ExpressionKind::Lambda(x, body) => {
-				let lbuilder = psb.lambda(x);
-				let body = Expr::from_api(body, lbuilder.stack(), ctx).boxed_local().await;
+				let lbuilder = psb.lambda(&x);
+				let body = Expr::from_api(*body, lbuilder.stack(), ctx).boxed_local().await;
 				ExprKind::Lambda(lbuilder.collect(), body)
 			},
-			api::ExpressionKind::NewAtom(a) =>
-				ExprKind::Atom(AtomHand::from_api(a, pos.clone(), &mut ctx.clone()).await),
-			api::ExpressionKind::Slot(tk) => return ctx.exprs.take_expr(*tk).expect("Invalid slot"),
+			api::ExpressionKind::NewAtom(a) => ExprKind::Atom(
+				AtomHand::from_api(&a.associate(ctx.sys), pos.clone(), &mut ctx.ctx.clone()).await,
+			),
+			api::ExpressionKind::Slot(tk) => return ctx.ctx.exprs.take_expr(tk).expect("Invalid slot"),
 			api::ExpressionKind::Seq(a, b) => {
 				let (apsb, bpsb) = psb.split();
 				ExprKind::Seq(
-					Expr::from_api(a, apsb, ctx.clone()).boxed_local().await,
-					Expr::from_api(b, bpsb, ctx).boxed_local().await,
+					Expr::from_api(*a, apsb, ctx.clone()).boxed_local().await,
+					Expr::from_api(*b, bpsb, ctx).boxed_local().await,
 				)
 			},
 		};
@@ -159,8 +168,10 @@ async fn print_exprkind<'a>(
 			panic!("This variant is swapped into write guards, so a read can never see it")
 		},
 		ExprKind::Atom(a) => a.print(c).await,
-		ExprKind::Bottom(e) if e.len() == 1 => format!("Bottom({e})").into(),
-		ExprKind::Bottom(e) => format!("Bottom(\n\t{}\n)", indent(&e.to_string())).into(),
+		ExprKind::Bottom(e) => match e.one() {
+			Some(e) => format!("Bottom({e})").into(),
+			None => format!("Bottom(\n\t{}\n)", indent(&e.to_string())).into(),
+		},
 		ExprKind::Call(f, x) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
 			.unbounded("{0b} {1l}")
 			.bounded("({0b} {1})")))
@@ -333,8 +344,8 @@ impl WeakExpr {
 
 impl TokenVariant<api::ExprTicket> for Expr {
 	type FromApiCtx<'a> = ExprStore;
-	async fn from_api(api: &api::ExprTicket, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
-		ctx.get_expr(*api).expect("Invalid ticket")
+	async fn from_api(api: api::ExprTicket, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
+		ctx.get_expr(api).expect("Invalid ticket")
 	}
 	type ToApiCtx<'a> = ExprStore;
 	async fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> api::ExprTicket {
@@ -349,8 +360,8 @@ impl TokenVariant<api::ExprTicket> for Expr {
 pub struct ExprWillPanic;
 
 impl TokenVariant<api::Expression> for Expr {
-	type FromApiCtx<'a> = Ctx;
-	async fn from_api(api: &api::Expression, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
+	type FromApiCtx<'a> = ExprFromApiCtx;
+	async fn from_api(api: api::Expression, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
 		Self::from_api(api, PathSetBuilder::new(), ctx.clone()).await
 	}
 	type ToApiCtx<'a> = ExprWillPanic;

orchid-host/src/extension.rs

@@ -1,9 +1,11 @@
+use std::any::Any;
 use std::cell::RefCell;
 use std::future::Future;
 use std::io;
 use std::num::NonZeroU64;
 use std::pin::Pin;
 use std::rc::{Rc, Weak};
+use std::time::Duration;
 
 use async_fn_stream::stream;
 use derive_destructure::destructure;
@@ -14,28 +16,23 @@ use futures::{AsyncRead, AsyncWrite, AsyncWriteExt, SinkExt, StreamExt};
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use orchid_api_traits::{Decode, Encode, Request};
-use orchid_base::format::{FmtCtxImpl, Format};
-use orchid_base::interner::{IStr, IStrv, es, ev, is, iv};
-use orchid_base::location::Pos;
-use orchid_base::logging::log;
-use orchid_base::name::Sym;
-use orchid_base::reqnot::{
-	Client, ClientExt, CommCtx, MsgReaderExt, ReqHandleExt, ReqReaderExt, io_comm,
+use orchid_base::{
+	AtomRepr, Client, ClientExt, CommCtx, FmtCtxImpl, Format, IStr, IStrv, MsgReaderExt, Pos,
+	ReqHandleExt, ReqReaderExt, Sym, Witness, es, ev, io_comm, is, iv, log, stash, with_stash,
 };
-use orchid_base::stash::{stash, with_stash};
-use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::atom::AtomHand;
 use crate::ctx::{Ctx, JoinHandle};
 use crate::dealias::{ChildError, ChildErrorKind, walk};
-use crate::expr::{Expr, PathSetBuilder};
+use crate::expr::{Expr, ExprFromApiCtx, PathSetBuilder};
 use crate::system::SystemCtor;
 use crate::tree::MemberKind;
 
 pub struct ExtPort {
 	pub input: Pin<Box<dyn AsyncWrite>>,
 	pub output: Pin<Box<dyn AsyncRead>>,
+	pub drop_trigger: Box<dyn Any>,
 }
 
 pub struct ReqPair<R: Request>(R, Sender<R::Response>);
@@ -56,6 +53,8 @@ pub struct ExtensionData {
 	lex_recur: Mutex<HashMap<api::ParsId, Sender<ReqPair<api::SubLex>>>>,
 	strings: RefCell<HashSet<IStr>>,
 	string_vecs: RefCell<HashSet<IStrv>>,
+	/// Moved over from [ExtPort] to allow hooking to the extension's drop
+	_drop_trigger: Box<dyn Any>,
 }
 impl Drop for ExtensionData {
 	fn drop(&mut self) {
@@ -85,7 +84,7 @@ impl Extension {
 			let weak2 = weak;
 			let weak = weak.clone();
 			let ctx2 = ctx.clone();
-			let join_ext = ctx.clone().spawn(async move {
+			let join_ext = ctx.clone().spawn(Duration::ZERO, async move {
 				comm
 					.listen(
 						async |reader| {
@@ -192,11 +191,17 @@ impl Extension {
 												})
 												.await
 										},
-										api::ExprReq::Create(ref cre @ api::Create(ref expr)) => {
-											let expr = Expr::from_api(expr, PathSetBuilder::new(), ctx.clone()).await;
+										api::ExprReq::Create(cre) => {
+											let req = Witness::of(&cre);
+											let api::Create(sys, expr) = cre;
+											let expr = Expr::from_api(expr, PathSetBuilder::new(), ExprFromApiCtx {
+												ctx: ctx.clone(),
+												sys,
+											})
+											.await;
 											let expr_id = expr.id();
 											ctx.exprs.give_expr(expr);
-											handle.reply(cre, &expr_id).await
+											handle.reply(req, &expr_id).await
 										},
 									},
 									api::ExtHostReq::LsModule(ref ls @ api::LsModule(_sys, path)) => {
@@ -286,6 +291,7 @@ impl Extension {
 				client: Rc::new(client),
 				strings: RefCell::default(),
 				string_vecs: RefCell::default(),
+				_drop_trigger: init.drop_trigger,
 			}
 		})))
 	}
@@ -336,10 +342,13 @@ impl Extension {
 	}
 	pub fn system_drop(&self, id: api::SysId) {
 		let rc = self.clone();
-		let _ = self.ctx().spawn(with_stash(async move {
-			rc.client().request(api::SystemDrop(id)).await.unwrap();
-			rc.ctx().systems.write().await.remove(&id);
-		}));
+		let _ = self.ctx().spawn(
+			Duration::ZERO,
+			with_stash(async move {
+				rc.client().request(api::SystemDrop(id)).await.unwrap();
+				rc.ctx().systems.write().await.remove(&id);
+			}),
+		);
 	}
 	#[must_use]
 	pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }

orchid-host/src/inline.rs

@@ -1,10 +1,17 @@
 #[cfg(feature = "orchid-extension")]
+use std::time::Duration;
+
+#[cfg(feature = "orchid-extension")]
+use orchid_base::on_drop;
+#[cfg(feature = "orchid-extension")]
 use orchid_extension as ox;
 
 #[cfg(feature = "orchid-extension")]
 use crate::ctx::Ctx;
 #[cfg(feature = "orchid-extension")]
 use crate::extension::ExtPort;
+#[cfg(feature = "orchid-extension")]
+use crate::task_set::TaskSet;
 
 #[cfg(feature = "orchid-extension")]
 pub async fn ext_inline(builder: ox::entrypoint::ExtensionBuilder, ctx: Ctx) -> ExtPort {
@@ -13,7 +20,7 @@ pub async fn ext_inline(builder: ox::entrypoint::ExtensionBuilder, ctx: Ctx) ->
 
 	use futures::io::BufReader;
 	use futures::{AsyncBufReadExt, StreamExt};
-	use orchid_base::logging::log;
+	use orchid_base::log;
 	use unsync_pipe::pipe;
 
 	let (in_stdin, out_stdin) = pipe(1024);
@@ -22,7 +29,7 @@ pub async fn ext_inline(builder: ox::entrypoint::ExtensionBuilder, ctx: Ctx) ->
 
 	let name = builder.name;
 
-	std::mem::drop(ctx.spawn(async move {
+	std::mem::drop(ctx.spawn(Duration::ZERO, async move {
 		let mut lines = BufReader::new(out_stderr).lines();
 		while let Some(line) = lines.next().await {
 			match line {
@@ -35,11 +42,35 @@ pub async fn ext_inline(builder: ox::entrypoint::ExtensionBuilder, ctx: Ctx) ->
 		}
 	}));
 
-	builder.build(ox::ext_port::ExtPort {
-		input: Box::pin(out_stdin),
-		output: Box::pin(in_stdout),
-		log: Box::pin(in_stderr),
-		spawn: Rc::new(move |fut| std::mem::drop(ctx.spawn(fut))),
-	});
-	ExtPort { input: Box::pin(in_stdin), output: Box::pin(out_stdout) }
+	let task_set = TaskSet::default();
+	let task_set1 = task_set.clone();
+
+	std::mem::drop(ctx.clone().spawn(Duration::ZERO, async move {
+		let task_set2 = task_set1.clone();
+		builder
+			.run(ox::ext_port::ExtPort {
+				input: Box::pin(out_stdin),
+				output: Box::pin(in_stdout),
+				log: Box::pin(in_stderr),
+				spawn: Rc::new(move |delay, fut| {
+					let ctx1 = ctx.clone();
+					let task_set2 = task_set1.clone();
+					task_set1.with(move |store| {
+						store.add_with(move |id| {
+							ctx1.spawn(delay, async move {
+								fut.await;
+								task_set2.with(|store| store.remove(id));
+							})
+						})
+					});
+				}),
+			})
+			.await;
+		task_set2.abort_all();
+	}));
+	ExtPort {
+		input: Box::pin(in_stdin),
+		output: Box::pin(out_stdout),
+		drop_trigger: Box::new(on_drop(move || task_set.abort_all())),
+	}
 }

orchid-host/src/lex.rs

@@ -3,18 +3,14 @@ use std::ops::Range;
 
 use futures::FutureExt;
 use futures::lock::Mutex;
-use orchid_base::clone;
-use orchid_base::error::{OrcErrv, OrcRes, mk_errv, report};
-use orchid_base::interner::{IStr, is};
-use orchid_base::location::SrcRange;
-use orchid_base::name::Sym;
-use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
-use orchid_base::tokens::PARENS;
-use orchid_base::tree::recur;
+use orchid_base::{
+	IStr, OrcErrv, OrcRes, PARENS, SrcRange, Sym, clone, is, mk_errv, name_char, name_start, op_char,
+	report, unrep_space,
+};
 
 use crate::api;
 use crate::ctx::Ctx;
-use crate::expr::Expr;
+use crate::expr::{Expr, ExprFromApiCtx};
 use crate::expr_store::ExprStore;
 use crate::parsed::{ParsTok, ParsTokTree, tt_to_api};
 use crate::system::System;
@@ -63,8 +59,14 @@ impl<'a> LexCtx<'a> {
 		tt_to_api(&mut { exprs }, subtree).await
 	}
 	#[must_use]
-	pub async fn des_subtree(&mut self, tree: &api::TokenTree, exprs: ExprStore) -> ParsTokTree {
-		ParsTokTree::from_api(tree, &mut { exprs }, &mut self.ctx.clone(), self.path).await
+	pub async fn des_subtree(
+		&mut self,
+		tree: api::TokenTree,
+		sys: api::SysId,
+		exprs: ExprStore,
+	) -> ParsTokTree {
+		let mut cx = ExprFromApiCtx { ctx: self.ctx.clone(), sys };
+		ParsTokTree::from_api(tree, &mut { exprs }, &mut cx, self.path).await
 	}
 	#[must_use]
 	pub fn strip_char(&mut self, tgt: char) -> bool {
@@ -261,12 +263,13 @@ pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
 				.await;
 			match lx {
 				Err(e) =>
-					return Some(Err(errors.into_iter().fold(OrcErrv::from_api(&e).await, |a, b| a + b))),
+					return Some(Err(errors.into_iter().fold(OrcErrv::from_api(e).await, |a, b| a + b))),
 				Ok(Some(lexed)) => {
 					ctx.set_pos(lexed.pos);
 					let mut stable_trees = Vec::new();
 					for tok in lexed.expr {
-						stable_trees.push(recur(ctx.des_subtree(&tok, temp_store.clone()).await, &|tt, r| {
+						let tree = ctx.des_subtree(tok, sys.id(), temp_store.clone()).await;
+						stable_trees.push(tree.recur(&|tt, r| {
 							if let ParsTok::NewExpr(expr) = tt.tok {
 								return ParsTok::Handle(expr).at(tt.sr);
 							}

orchid-host/src/lib.rs

@@ -18,4 +18,5 @@ pub mod parsed;
 pub mod subprocess;
 mod sys_parser;
 pub mod system;
+mod task_set;
 pub mod tree;

orchid-host/src/logger.rs

@@ -6,7 +6,7 @@ use std::rc::Rc;
 use futures::future::LocalBoxFuture;
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::logging::{LogWriter, Logger};
+use orchid_base::{LogWriter, Logger};
 
 use crate::api;
 

orchid-host/src/parse.rs

@@ -1,13 +1,9 @@
 use futures::FutureExt;
 use itertools::Itertools;
-use orchid_base::error::{OrcRes, mk_errv, report};
-use orchid_base::format::fmt;
-use orchid_base::interner::{IStr, is};
-use orchid_base::name::Sym;
-use orchid_base::parse::{
-	Comment, Import, Parsed, Snippet, expect_end, line_items, parse_multiname, try_pop_no_fluff,
+use orchid_base::{
+	Comment, IStr, Import, OrcRes, Paren, Parsed, Snippet, Sym, TokTree, Token, expect_end, fmt, is,
+	line_items, mk_errv, parse_multiname, report, try_pop_no_fluff,
 };
-use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
 
 use crate::ctx::Ctx;
@@ -60,7 +56,7 @@ pub async fn parse_item(
 	comments: Vec<Comment>,
 	item: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	match item.pop_front() {
+	match item.split_first() {
 		Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
 			n if *n == is("export").await => match try_pop_no_fluff(postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>

orchid-host/src/parsed.rs

@@ -5,12 +5,10 @@ use futures::FutureExt;
 use futures::future::{LocalBoxFuture, join_all};
 use hashbrown::HashSet;
 use itertools::Itertools;
-use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
-use orchid_base::interner::{IStr, IStrv};
-use orchid_base::location::SrcRange;
-use orchid_base::parse::{Comment, Import};
-use orchid_base::tl_cache;
-use orchid_base::tree::{TokTree, Token, recur};
+use orchid_base::{
+	Comment, FmtCtx, FmtUnit, Format, IStr, IStrv, Import, SrcRange, TokTree, Token, Variants,
+	tl_cache,
+};
 
 use crate::api;
 use crate::dealias::{ChildErrorKind, ChildResult, Tree};
@@ -182,7 +180,7 @@ impl Tree for ParsedModule {
 impl Format for ParsedModule {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		let head_str = format!("export ::({})\n", self.exports.iter().join(", "));
-		Variants::default().sequence(self.items.len() + 1, "", "\n", "", None).units_own(
+		Variants::default().sequence(self.items.len() + 1, "", "\n", "", false).units_own(
 			[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
 		)
 	}
@@ -202,7 +200,7 @@ impl ConstPath {
 }
 
 pub async fn tt_to_api(exprs: &mut ExprStore, subtree: ParsTokTree) -> api::TokenTree {
-	let without_new_expr = recur(subtree, &|tt, r| {
+	let without_new_expr = subtree.recur(&|tt, r| {
 		if let ParsTok::NewExpr(expr) = tt.tok {
 			return ParsTok::Handle(expr).at(tt.sr);
 		}

orchid-host/src/subprocess.rs

@@ -1,36 +1,40 @@
-use std::{io, process};
+use std::time::Duration;
 
 use futures::io::BufReader;
 use futures::{self, AsyncBufReadExt, StreamExt};
-use orchid_base::logging::log;
+use orchid_base::log;
 use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
 
 use crate::ctx::Ctx;
 use crate::extension::ExtPort;
 
-pub async fn ext_command(cmd: process::Command, ctx: Ctx) -> io::Result<ExtPort> {
+pub async fn ext_command(cmd: std::process::Command, ctx: Ctx) -> std::io::Result<ExtPort> {
 	let name = cmd.get_program().to_string_lossy().to_string();
 	let mut child = tokio::process::Command::from(cmd)
-		.stdin(process::Stdio::piped())
-		.stdout(process::Stdio::piped())
-		.stderr(process::Stdio::piped())
+		.stdin(std::process::Stdio::piped())
+		.stdout(std::process::Stdio::piped())
+		.stderr(std::process::Stdio::piped())
 		.spawn()?;
 	std::thread::spawn(|| {});
 	let stdin = child.stdin.take().unwrap();
 	let stdout = child.stdout.take().unwrap();
 	let child_stderr = child.stderr.take().unwrap();
-	let _ = ctx.spawn(Box::pin(async move {
+	std::mem::drop(ctx.spawn(Duration::ZERO, async move {
 		let _ = child;
 		let mut lines = BufReader::new(child_stderr.compat()).lines();
 		while let Some(line) = lines.next().await {
 			match line {
 				Ok(line) => writeln!(log("stderr"), "subproc {name} err> {line}").await,
 				Err(e) => match e.kind() {
-					io::ErrorKind::BrokenPipe | io::ErrorKind::UnexpectedEof => break,
+					std::io::ErrorKind::BrokenPipe | std::io::ErrorKind::UnexpectedEof => break,
 					_ => panic!("Error while reading stderr {e}"),
 				},
 			}
 		}
 	}));
-	Ok(ExtPort { input: Box::pin(stdin.compat_write()), output: Box::pin(stdout.compat()) })
+	Ok(ExtPort {
+		input: Box::pin(stdin.compat_write()),
+		output: Box::pin(stdout.compat()),
+		drop_trigger: Box::new(()),
+	})
 }

orchid-host/src/sys_parser.rs

@@ -1,16 +1,11 @@
 use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
-use orchid_base::error::{OrcErrv, OrcRes};
-use orchid_base::interner::{IStr, es};
-use orchid_base::location::SrcRange;
-use orchid_base::name::Sym;
-use orchid_base::parse::Comment;
-use orchid_base::reqnot::ClientExt;
-use orchid_base::tree::ttv_from_api;
+use orchid_base::{ClientExt, Comment, IStr, OrcErrv, OrcRes, SrcRange, Sym, es, ttv_from_api};
 use substack::Substack;
 
 use crate::api;
+use crate::expr::ExprFromApiCtx;
 use crate::expr_store::ExprStore;
 use crate::parse::HostParseCtx;
 use crate::parsed::{
@@ -59,7 +54,7 @@ impl Parser {
 					sys: &self.system,
 				})
 				.await,
-			Err(e) => Err(OrcErrv::from_api(&e).await),
+			Err(e) => Err(OrcErrv::from_api(e).await),
 		}
 	}
 }
@@ -82,8 +77,8 @@ async fn conv(
 			api::ParsedLineKind::Member(api::ParsedMember { name, exported, kind }) =>
 				(name, exported, kind),
 			api::ParsedLineKind::Recursive(rec) => {
-				let tokens =
-					ttv_from_api(rec, ctx.ext_exprs, &mut ctx.sys.ctx().clone(), ctx.src_path).await;
+				let mut cx = ExprFromApiCtx { ctx: ctx.sys.ctx().clone(), sys: ctx.sys.id() };
+				let tokens = ttv_from_api(rec, ctx.ext_exprs, &mut cx, ctx.src_path).await;
 				items.extend(callback(module.clone(), tokens).await?);
 				continue;
 			},

orchid-host/src/system.rs

@@ -2,6 +2,7 @@ use std::collections::VecDeque;
 use std::fmt;
 use std::future::Future;
 use std::rc::{Rc, Weak};
+use std::time::Duration;
 
 use derive_destructure::destructure;
 use futures::future::join_all;
@@ -9,15 +10,10 @@ use futures_locks::RwLock;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use memo_map::MemoMap;
-use orchid_base::char_filter::char_filter_match;
-use orchid_base::error::{OrcRes, mk_errv_floating};
-use orchid_base::format::{FmtCtx, FmtUnit, Format};
-use orchid_base::interner::{IStr, es, is};
-use orchid_base::iter_utils::IteratorPrint;
-use orchid_base::logging::log;
-use orchid_base::name::{NameLike, Sym, VName, VPath};
-use orchid_base::reqnot::{Client, ClientExt};
-use orchid_base::stash::stash;
+use orchid_base::{
+	Client, ClientExt, FmtCtx, FmtUnit, Format, IStr, IteratorPrint, NameLike, OrcRes, Sym, VName,
+	VPath, char_filter_match, es, is, log, mk_errv_floating, stash,
+};
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
@@ -129,10 +125,10 @@ impl System {
 	}
 	pub(crate) fn drop_atom(&self, dropped_atom_id: api::AtomId) {
 		let this = self.0.clone();
-		let _ = self.0.ctx.spawn(Box::pin(async move {
+		let _ = self.0.ctx.spawn(Duration::ZERO, async move {
 			this.ext.client().request(api::AtomDrop(this.id, dropped_atom_id)).await.unwrap();
 			this.owned_atoms.write().await.remove(&dropped_atom_id);
-		}));
+		});
 	}
 	#[must_use]
 	pub fn downgrade(&self) -> WeakSystem {

orchid-host/src/task_set.rs

@@ -0,0 +1,23 @@
+use std::cell::RefCell;
+use std::rc::Rc;
+
+use orchid_base::IdStore;
+
+use crate::ctx::JoinHandle;
+
+#[derive(Clone)]
+pub struct TaskSet {
+	pub tasks: Rc<RefCell<Option<IdStore<Box<dyn JoinHandle>>>>>,
+}
+impl TaskSet {
+	pub fn with<R>(&self, f: impl FnOnce(&mut IdStore<Box<dyn JoinHandle>>) -> R) -> Option<R> {
+		self.tasks.borrow_mut().as_mut().map(f)
+	}
+	pub fn abort_all(&self) {
+		let Some(tasks) = self.tasks.borrow_mut().take() else { return };
+		tasks.into_iter().for_each(|(_, task)| task.abort());
+	}
+}
+impl Default for TaskSet {
+	fn default() -> Self { Self { tasks: Rc::new(RefCell::new(Some(IdStore::default()))) } }
+}

orchid-host/src/tree.rs

@@ -12,17 +12,15 @@ use hashbrown::HashMap;
 use hashbrown::hash_map::Entry;
 use itertools::Itertools;
 use memo_map::MemoMap;
-use orchid_base::clone;
-use orchid_base::error::{OrcRes, mk_errv, report};
-use orchid_base::interner::{IStr, IStrv, es, is, iv};
-use orchid_base::location::{CodeGenInfo, Pos};
-use orchid_base::name::{NameLike, Sym, VPath};
-use orchid_base::reqnot::ClientExt;
+use orchid_base::{
+	ClientExt, CodeGenInfo, IStr, IStrv, NameLike, OrcRes, Pos, Sym, VPath, clone, es, is, iv,
+	mk_errv, report,
+};
 
 use crate::api;
 use crate::ctx::Ctx;
 use crate::dealias::{ChildErrorKind, Tree, absolute_path, resolv_glob, walk};
-use crate::expr::{Expr, PathSetBuilder};
+use crate::expr::{Expr, ExprFromApiCtx, PathSetBuilder};
 use crate::parsed::{ItemKind, ParsedMemberKind, ParsedModule};
 use crate::system::System;
 
@@ -89,7 +87,8 @@ impl Root {
 		for (path, sys_id, pc_id) in deferred_consts {
 			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
 			let api_expr = sys.client().request(api::FetchParsedConst(sys.id(), pc_id)).await.unwrap();
-			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), this.ctx.clone()).await;
+			let cx = ExprFromApiCtx { sys: sys_id, ctx: this.ctx.clone() };
+			let expr = Expr::from_api(api_expr, PathSetBuilder::new(), cx).await;
 			new.0.write().await.consts.insert(path, expr);
 		}
 		new
@@ -176,7 +175,8 @@ impl Module {
 				api::MemberKind::Lazy(id) =>
 					(Some(LazyMemberHandle { id, sys: ctx.sys.id(), path: name.clone() }), None),
 				api::MemberKind::Const(val) => {
-					let expr = Expr::from_api(&val, PathSetBuilder::new(), ctx.sys.ctx().clone()).await;
+					let cx = ExprFromApiCtx { ctx: ctx.sys.ctx().clone(), sys: ctx.sys.id() };
+					let expr = Expr::from_api(val, PathSetBuilder::new(), cx).await;
 					ctx.consts.insert(name.clone(), expr);
 					(None, Some(MemberKind::Const))
 				},
@@ -456,7 +456,8 @@ impl LazyMemberHandle {
 		let sys = ctx.system_inst(self.sys).await.expect("Missing system for lazy member");
 		match sys.get_tree(self.id).await {
 			api::MemberKind::Const(c) => {
-				let expr = Expr::from_api(&c, PathSetBuilder::new(), ctx.clone()).await;
+				let ctx = ExprFromApiCtx { sys: sys.id(), ctx: ctx.clone() };
+				let expr = Expr::from_api(c, PathSetBuilder::new(), ctx).await;
 				let (.., path) = self.destructure();
 				consts.insert(path, expr);
 				MemberKind::Const