Traditional route appears to work

Beginnings of dylib extensions, entirely untestted

orchid-extension/Cargo.toml

@@ -11,15 +11,15 @@ async-once-cell = "0.5.4"
 bound = "0.6.0"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.20"
-futures = { version = "0.3.31", features = [
+futures = { version = "0.3.31", default-features = false, features = [
 	"std",
 	"async-await",
-], default-features = false }
+] }
 futures-locks = "0.7.1"
-hashbrown = "0.16.0"
+hashbrown = "0.16.1"
 include_dir = { version = "0.7.4", optional = true }
 itertools = "0.14.0"
-konst = "0.4.2"
+konst = "0.4.3"
 lazy_static = "1.5.0"
 memo-map = "0.3.3"
 never = "0.1.0"
@@ -28,12 +28,12 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-ordered-float = "5.0.0"
-pastey = "0.1.1"
+ordered-float = "5.1.0"
+pastey = "0.2.1"
 substack = "1.1.1"
 task-local = "0.1.0"
-tokio = { version = "1.47.1", optional = true, features = [] }
-tokio-util = { version = "0.7.16", optional = true, features = ["compat"] }
+tokio = { version = "1.49.0", optional = true, features = [] }
+tokio-util = { version = "0.7.17", optional = true, features = ["compat"] }
 
 trait-set = "0.3.0"
 

orchid-extension/src/atom_owned.rs

@@ -19,6 +19,7 @@ use never::Never;
 use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, take_first};
+use orchid_base::logging::log;
 use orchid_base::name::Sym;
 use task_local::task_local;
 
@@ -338,5 +339,5 @@ pub async fn debug_print_obj_store(show_atoms: bool) {
 			message += &format!("\n{k:?} -> {}", take_first(&atom.dyn_print().await, true));
 		}
 	}
-	eprintln!("{message}")
+	writeln!(log("debug"), "{message}").await
 }

orchid-extension/src/atom_thin.rs

@@ -8,7 +8,7 @@ use futures::{AsyncRead, AsyncWrite, FutureExt};
 use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
-use orchid_base::logging::logger;
+use orchid_base::logging::log;
 use orchid_base::name::Sym;
 
 use crate::api;
@@ -89,7 +89,7 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn drop<'a>(&'a self, AtomCtx(buf, _): AtomCtx<'a>) -> LocalBoxFuture<'a, ()> {
 		Box::pin(async move {
 			let string_self = T::decode_slice(&mut &buf[..]).print().await;
-			writeln!(logger(), "Received drop signal for non-drop atom {string_self:?}");
+			writeln!(log("warn"), "Received drop signal for non-drop atom {string_self:?}").await;
 		})
 	}
 }

orchid-extension/src/binary.rs

@@ -0,0 +1,30 @@
+use std::rc::Rc;
+
+use futures::future::LocalBoxFuture;
+use orchid_base::binary::future_to_vt;
+
+use crate::api;
+use crate::entrypoint::ExtensionBuilder;
+use crate::ext_port::ExtPort;
+
+pub type ExtCx = api::binary::ExtensionContext;
+
+struct Spawner(api::binary::Spawner);
+impl Drop for Spawner {
+	fn drop(&mut self) { (self.0.drop)(self.0.data) }
+}
+impl Spawner {
+	pub fn spawn(&self, fut: LocalBoxFuture<'static, ()>) {
+		(self.0.spawn)(self.0.data, future_to_vt(fut))
+	}
+}
+
+pub fn orchid_extension_main_body(cx: ExtCx, builder: ExtensionBuilder) {
+	let spawner = Spawner(cx.spawner);
+	builder.build(ExtPort {
+		input: Box::pin(cx.input),
+		output: Box::pin(cx.output),
+		log: Box::pin(cx.log),
+		spawn: Rc::new(move |fut| spawner.spawn(fut)),
+	});
+}

orchid-extension/src/entrypoint.rs

@@ -15,7 +15,7 @@ use orchid_api_traits::{Decode, Encode, Request, UnderRoot, enc_vec};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
 use orchid_base::error::try_with_reporter;
 use orchid_base::interner::{es, is, with_interner};
-use orchid_base::logging::{Logger, with_logger};
+use orchid_base::logging::{log, with_logger};
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Snippet};
 use orchid_base::reqnot::{
@@ -35,6 +35,7 @@ use crate::ext_port::ExtPort;
 use crate::func_atom::with_funs_ctx;
 use crate::interner::new_interner;
 use crate::lexer::{LexContext, ekey_cascade, ekey_not_applicable};
+use crate::logger::LoggerImpl;
 use crate::parser::{PTokTree, ParsCtx, get_const, linev_into_api, with_parsed_const_ctx};
 use crate::reflection::with_refl_roots;
 use crate::system::{SysCtx, atom_by_idx, cted, with_sys};
@@ -58,9 +59,17 @@ pub async fn with_comm<F: Future>(c: Rc<dyn Client>, ctx: CommCtx, fut: F) -> F:
 	CLIENT.scope(c, CTX.scope(Rc::new(RefCell::new(Some(ctx))), fut)).await
 }
 
+task_local! {
+	pub static MUTE_REPLY: ();
+}
+
 /// Send a request through the global client's [ClientExt::request]
 pub async fn request<T: Request + UnderRoot<Root = ExtHostReq>>(t: T) -> T::Response {
-	get_client().request(t).await.unwrap()
+	let response = get_client().request(t).await.unwrap();
+	if MUTE_REPLY.try_with(|b| *b).is_err() {
+		writeln!(log("msg"), "Got response {response:?}").await;
+	}
+	response
 }
 
 /// Send a notification through the global client's [ClientExt::notify]
@@ -79,15 +88,7 @@ task_local! {
 }
 
 async fn with_sys_record<F: Future>(id: api::SysId, fut: F) -> F::Output {
-	let cted = SYSTEM_TABLE.with(|tbl| {
-		eprintln!(
-			"Existing systems are {}",
-			tbl.borrow().iter().map(|(k, v)| format!("{k:?}={:?}", v.cted)).join(";")
-		);
-		let sys = tbl.borrow().get(&id).expect("Invalid sys ID").cted.clone();
-		eprintln!("Selected {:?}", sys);
-		sys
-	});
+	let cted = SYSTEM_TABLE.with(|tbl| tbl.borrow().get(&id).expect("Invalid sys ID").cted.clone());
 	with_sys(SysCtx(id, cted), fut).await
 }
 
@@ -126,8 +127,7 @@ impl ExtensionBuilder {
 		self.add_context(with_lazy_member_store);
 		self.add_context(with_refl_roots);
 		(ctx.spawn)(Box::pin(async move {
-			let api::HostHeader { log_strategy, msg_logs } =
-				api::HostHeader::decode(ctx.input.as_mut()).await.unwrap();
+			let host_header = api::HostHeader::decode(ctx.input.as_mut()).await.unwrap();
 			let decls = (self.systems.iter().enumerate())
 				.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
 				.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
@@ -137,24 +137,26 @@ impl ExtensionBuilder {
 				.await
 				.unwrap();
 			ctx.output.as_mut().flush().await.unwrap();
-			let logger = Logger::new(log_strategy);
-			let logger2 = logger.clone();
-			let msg_logger = Logger::new(msg_logs);
-			let (client, ctx, run_extension) = io_comm(
-				Rc::new(Mutex::new(ctx.output)),
-				Mutex::new(ctx.input),
-				async move |n: Box<dyn MsgReader<'_>>| {
-					match n.read().await.unwrap() {
+			let logger1 = LoggerImpl::from_api(&host_header.logger);
+			let logger2 = logger1.clone();
+			let (client, comm_ctx, extension_srv) =
+				io_comm(Rc::new(Mutex::new(ctx.output)), Mutex::new(ctx.input));
+			let extension_fut = extension_srv.listen(
+				async |n: Box<dyn MsgReader<'_>>| {
+					let notif = n.read().await.unwrap();
+					match notif {
 						api::HostExtNotif::Exit => exit().await,
 					}
 					Ok(())
 				},
-				async move |mut reader| {
+				async |mut reader| {
 					with_stash(async {
 						let req = reader.read_req().await.unwrap();
 						let handle = reader.finish().await;
+						// Atom printing is never reported because it generates too much
+						// noise
 						if !matches!(req, api::HostExtReq::AtomReq(api::AtomReq::AtomPrint(_))) {
-							writeln!(msg_logger, "{} extension received request {req:?}", self.name);
+							writeln!(log("msg"), "{} extension received request {req:?}", self.name).await;
 						}
 						match req {
 							api::HostExtReq::SystemDrop(sys_drop) => {
@@ -266,11 +268,6 @@ impl ExtensionBuilder {
 									let ekey_na = ekey_not_applicable().await;
 									let ekey_cascade = ekey_cascade().await;
 									let lexers = cted().inst().dyn_lexers();
-									writeln!(
-										logger,
-										"sys={sys:?}, tc={trigger_char}, lexers={}",
-										lexers.iter().map(|l| format!("{l:?}")).join(",")
-									);
 									for lx in
 										lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char))
 									{
@@ -290,7 +287,7 @@ impl ExtensionBuilder {
 											},
 										}
 									}
-									writeln!(logger, "Got notified about n/a character '{trigger_char}'");
+									writeln!(log("warn"), "Got notified about n/a character '{trigger_char}'").await;
 									expr_store.dispose().await;
 									handle.reply(&lex, &None).await
 								})
@@ -423,16 +420,16 @@ impl ExtensionBuilder {
 							logger2,
 							with_comm(
 								Rc::new(client),
-								ctx,
+								comm_ctx,
 								(self.context.into_iter()).fold(
-									Box::pin(async move { run_extension.await.unwrap() }) as LocalBoxFuture<()>,
+									Box::pin(async { extension_fut.await.unwrap() }) as LocalBoxFuture<()>,
 									|fut, cx| cx.apply(fut),
 								),
 							),
 						),
 					),
 				)
-				.await
+				.await;
 		}) as Pin<Box<_>>);
 	}
 }

orchid-extension/src/interner.rs

@@ -6,7 +6,7 @@ use orchid_base::interner::local_interner::{Int, StrBranch, StrvBranch};
 use orchid_base::interner::{IStr, IStrv, InternerSrv};
 
 use crate::api;
-use crate::entrypoint::request;
+use crate::entrypoint::{MUTE_REPLY, request};
 
 #[derive(Default)]
 struct ExtInterner {
@@ -17,7 +17,9 @@ impl InternerSrv for ExtInterner {
 	fn is<'a>(&'a self, v: &'a str) -> LocalBoxFuture<'a, IStr> {
 		match self.str.i(v) {
 			Ok(i) => Box::pin(ready(i)),
-			Err(e) => Box::pin(async { e.set_if_empty(request(api::InternStr(v.to_owned())).await) }),
+			Err(e) => Box::pin(async {
+				e.set_if_empty(MUTE_REPLY.scope((), request(api::InternStr(v.to_owned()))).await)
+			}),
 		}
 	}
 	fn es(&self, t: api::TStr) -> LocalBoxFuture<'_, IStr> {

orchid-extension/src/lib.rs

@@ -12,6 +12,7 @@ pub mod func_atom;
 pub mod gen_expr;
 pub mod interner;
 pub mod lexer;
+pub mod logger;
 pub mod other_system;
 pub mod parser;
 pub mod reflection;
@@ -19,3 +20,4 @@ pub mod system;
 pub mod system_ctor;
 pub mod tokio;
 pub mod tree;
+pub mod binary;

orchid-extension/src/logger.rs

@@ -0,0 +1,57 @@
+use std::fmt::Arguments;
+use std::fs::File;
+use std::io::Write;
+use std::rc::Rc;
+
+use futures::future::LocalBoxFuture;
+use hashbrown::HashMap;
+use orchid_base::interner::is;
+use orchid_base::logging::{LogWriter, Logger};
+
+use crate::api;
+use crate::entrypoint::notify;
+
+pub struct LogWriterImpl {
+	category: String,
+	strat: api::LogStrategy,
+}
+impl LogWriter for LogWriterImpl {
+	fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()> {
+		Box::pin(async move {
+			match &self.strat {
+				api::LogStrategy::Discard => (),
+				api::LogStrategy::Default =>
+					notify(api::Log { category: is(&self.category).await.to_api(), message: fmt.to_string() })
+						.await,
+				api::LogStrategy::File { path, .. } => {
+					let mut file = (File::options().write(true).create(true).truncate(false).open(path))
+						.unwrap_or_else(|e| panic!("Could not open {path}: {e}"));
+					file.write_fmt(fmt).unwrap_or_else(|e| panic!("Could not write to {path}: {e}"));
+				},
+			}
+		})
+	}
+}
+
+#[derive(Clone)]
+pub struct LoggerImpl {
+	default: Option<api::LogStrategy>,
+	routing: HashMap<String, api::LogStrategy>,
+}
+impl LoggerImpl {
+	pub fn from_api(api: &api::Logger) -> Self {
+		Self {
+			default: api.default.clone(),
+			routing: api.routing.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
+		}
+	}
+}
+impl Logger for LoggerImpl {
+	fn writer(&self, category: &str) -> Rc<dyn LogWriter> {
+		Rc::new(LogWriterImpl { category: category.to_string(), strat: self.strat(category) })
+	}
+	fn strat(&self, category: &str) -> orchid_api::LogStrategy {
+		(self.routing.get(category).cloned().or(self.default.clone()))
+			.expect("Unrecognized log category with no default strategy")
+	}
+}

orchid-extension/src/system_ctor.rs

@@ -64,8 +64,7 @@ pub trait SystemCtor: Debug + Send + Sync + 'static {
 	type Instance: System;
 	const NAME: &'static str;
 	const VERSION: f64;
-	/// Create a system instance. When this function is called, a context object
-	/// isn't yet available
+	/// Create a system instance.
 	fn inst(&self, deps: <Self::Deps as DepDef>::Sat) -> Self::Instance;
 }
 

orchid-extension/src/tokio.rs

@@ -2,9 +2,15 @@ use std::rc::Rc;
 
 use crate::entrypoint::ExtensionBuilder;
 use crate::ext_port::ExtPort;
-
+/// Run an extension inside a Tokio localset. Since the extension API does not
+/// provide a forking mechanism, it can safely abort once the localset is
+/// exhausted. If an extension absolutely needs a parallel thread, it can import
+/// and call [tokio::task::spawn_local] which will keep alive the localset and
+/// postpone the aggressive shutdown, and listen for the [Drop::drop] of the
+/// value returned by [crate::system_ctor::SystemCtor::inst] to initiate
+/// shutdown.
 #[cfg(feature = "tokio")]
-pub async fn tokio_main(builder: ExtensionBuilder) {
+pub async fn tokio_main(builder: ExtensionBuilder) -> ! {
 	use tokio::io::{stderr, stdin, stdout};
 	use tokio::task::{LocalSet, spawn_local};
 	use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
@@ -21,4 +27,5 @@ pub async fn tokio_main(builder: ExtensionBuilder) {
 		});
 	});
 	local_set.await;
+	std::process::exit(0)
 }