Temporary commit to try fix halting

clean shutdown doesn't for some reason

.cargo/config.toml

@@ -6,6 +6,9 @@ orcxdb = "xtask orcxdb"
 [env]
 CARGO_WORKSPACE_DIR = { value = "", relative = true }
 ORCHID_EXTENSIONS = "target/debug/orchid-std"
-ORCHID_DEFAULT_SYSTEMS = "orchid::std"
+ORCHID_DEFAULT_SYSTEMS = "orchid::std;orchid::macros"
 ORCHID_LOG_BUFFERS = "true"
-RUSTBACKTRACE = "1"
+RUST_BACKTRACE = "1"
+
+[build]
+# rustflags = ["-Znext-solver"]

Cargo.toml

@@ -10,5 +10,6 @@ members = [
   "orchid-api",
   "orchid-api-derive",
   "orchid-api-traits",
-  "stdio-perftest", "xtask", "orchid-macros",
+  "stdio-perftest",
+  "xtask", "async-fn-stream",
 ]

SWAP.md

@@ -1,6 +1,8 @@
-Since the macro AST is built as a custom tokenizer inside the system, it needs access to the import set. On the other hand, import sets aren't available until after parsing. Need a way to place this order in a lexer without restricting the expression value of the lexer.
+Decide whether we need patterns at runtime. Maybe macros aren't obligated to return MacTree so destructuring can be done in a safer and easier way?
 
-The daft option of accepting import resolution queries at runtime is available but consider better options.
+Double-check type and templating logic in the note, it's a bit fishy.
+
+Consider whether all macros need to be loaded or the const references could be used to pre-filter for a given let line.
 
 ## alternate extension mechanism
 

async-fn-stream/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "async-fn-stream"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+futures = { version = "0.3.31", features = ["std"], default-features = false }
+
+[dev-dependencies]
+test_executors = "0.3.5"

async-fn-stream/src/lib.rs

@@ -0,0 +1,207 @@
+use std::cell::Cell;
+use std::future::poll_fn;
+use std::marker::PhantomData;
+use std::pin::Pin;
+use std::ptr;
+use std::task::{Context, Poll};
+
+use futures::future::LocalBoxFuture;
+use futures::{FutureExt, Stream};
+
+type YieldSlot<'a, T> = &'a Cell<Option<T>>;
+
+/// Handle that allows you to emit values on a stream. If you drop
+/// this, the stream will end and you will not be polled again.
+pub struct StreamCtx<'a, T>(&'a Cell<Option<T>>, PhantomData<&'a ()>);
+impl<T> StreamCtx<'_, T> {
+	pub fn emit(&mut self, value: T) -> impl Future<Output = ()> {
+		assert!(self.0.replace(Some(value)).is_none(), "Leftover value in stream");
+		let mut state = Poll::Pending;
+		poll_fn(move |_| std::mem::replace(&mut state, Poll::Ready(())))
+	}
+}
+
+enum FnOrFut<'a, T, O> {
+	Fn(Option<Box<dyn FnOnce(YieldSlot<'a, T>) -> LocalBoxFuture<'a, O> + 'a>>),
+	Fut(LocalBoxFuture<'a, O>),
+}
+
+struct AsyncFnStream<'a, T> {
+	driver: FnOrFut<'a, T, ()>,
+	output: Cell<Option<T>>,
+}
+impl<'a, T> Stream for AsyncFnStream<'a, T> {
+	type Item = T;
+	fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		unsafe {
+			let self_mut = self.get_unchecked_mut();
+			let fut = match &mut self_mut.driver {
+				FnOrFut::Fut(fut) => fut,
+				FnOrFut::Fn(f) => {
+					// safety: the cell is held inline in self, which is pinned.
+					let cell = ptr::from_ref(&self_mut.output).as_ref().unwrap();
+					let fut = f.take().unwrap()(cell);
+					self_mut.driver = FnOrFut::Fut(fut);
+					return Pin::new_unchecked(self_mut).poll_next(cx);
+				},
+			};
+			match fut.as_mut().poll(cx) {
+				Poll::Ready(()) => Poll::Ready(None),
+				Poll::Pending => match self_mut.output.replace(None) {
+					None => Poll::Pending,
+					Some(t) => Poll::Ready(Some(t)),
+				},
+			}
+		}
+	}
+}
+
+struct AsyncFnTryStream<'a, T, E> {
+	driver: FnOrFut<'a, T, Result<StreamCtx<'a, T>, E>>,
+	output: Cell<Option<T>>,
+}
+impl<'a, T, E> Stream for AsyncFnTryStream<'a, T, E> {
+	type Item = Result<T, E>;
+	fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		unsafe {
+			let self_mut = self.get_unchecked_mut();
+			let fut = match &mut self_mut.driver {
+				FnOrFut::Fut(fut) => fut,
+				FnOrFut::Fn(f) => {
+					// safety: the cell is held inline in self, which is pinned.
+					let cell = ptr::from_ref(&self_mut.output).as_ref().unwrap();
+					let fut = f.take().unwrap()(cell);
+					self_mut.driver = FnOrFut::Fut(fut);
+					return Pin::new_unchecked(self_mut).poll_next(cx);
+				},
+			};
+			match fut.as_mut().poll(cx) {
+				Poll::Ready(Ok(_)) => Poll::Ready(None),
+				Poll::Ready(Err(ex)) => Poll::Ready(Some(Err(ex))),
+				Poll::Pending => match self_mut.output.replace(None) {
+					None => Poll::Pending,
+					Some(t) => Poll::Ready(Some(Ok(t))),
+				},
+			}
+		}
+	}
+}
+
+/// Create a stream from an async function acting as a coroutine
+pub fn stream<'a, T: 'a>(
+	f: impl for<'b> AsyncFnOnce(StreamCtx<'b, T>) + 'a,
+) -> impl Stream<Item = T> + 'a {
+	AsyncFnStream {
+		output: Cell::new(None),
+		driver: FnOrFut::Fn(Some(Box::new(|t| {
+			async { f(StreamCtx(t, PhantomData)).await }.boxed_local()
+		}))),
+	}
+}
+
+/// Create a stream of result from a fallible function.
+pub fn try_stream<'a, T: 'a, E: 'a>(
+	f: impl for<'b> AsyncFnOnce(StreamCtx<'b, T>) -> Result<StreamCtx<'b, T>, E> + 'a,
+) -> impl Stream<Item = Result<T, E>> + 'a {
+	AsyncFnTryStream {
+		output: Cell::new(None),
+		driver: FnOrFut::Fn(Some(Box::new(|t| {
+			async { f(StreamCtx(t, PhantomData)).await }.boxed_local()
+		}))),
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use std::task::Poll;
+	use std::{future, pin};
+
+	use futures::channel::mpsc::channel;
+	use futures::{Stream, StreamExt, TryStreamExt};
+	use test_executors::spin_on;
+
+	use crate::{stream, try_stream};
+
+	#[test]
+	fn sync() {
+		spin_on(async {
+			let v = stream(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+			})
+			.collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, [0, 1, 2, 3, 4])
+		})
+	}
+
+	#[test]
+	/// The exact behaviour of the poll function under blocked use
+	fn with_delay() {
+		spin_on(async {
+			let (mut send, mut recv) = channel(0);
+			let mut s = pin::pin!(stream(async |mut cx| {
+				for i in 0..2 {
+					cx.emit(i).await
+				}
+				recv.next().await;
+				for i in 2..5 {
+					cx.emit(i).await
+				}
+			}));
+			let mut log = String::new();
+			let log = future::poll_fn(|cx| {
+				match s.as_mut().poll_next(cx) {
+					Poll::Ready(Some(r)) => log += &format!("Found {r}\n"),
+					Poll::Ready(None) => return Poll::Ready(format!("{log}Ended")),
+					Poll::Pending => match send.try_send(()) {
+						Ok(()) => log += "Unblocked\n",
+						Err(err) => return Poll::Ready(format!("{log}Unblock err: {err}")),
+					},
+				}
+				Poll::Pending
+			})
+			.await;
+			const EXPECTED: &str = "\
+        Found 0\n\
+        Found 1\n\
+        Unblocked\n\
+        Found 2\n\
+        Found 3\n\
+        Found 4\n\
+        Ended";
+			assert_eq!(log, EXPECTED)
+		})
+	}
+
+	#[test]
+	fn sync_try_all_ok() {
+		spin_on(async {
+			let v = try_stream::<_, ()>(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+				Ok(cx)
+			})
+			.try_collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, Ok(vec![0, 1, 2, 3, 4]))
+		})
+	}
+
+	#[test]
+	fn sync_try_err() {
+		spin_on(async {
+			let v = try_stream::<_, ()>(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+				Err(())
+			})
+			.try_collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, Err(()))
+		})
+	}
+}

examples/hello-world/main.orc

@@ -1,2 +1,2 @@
-const user = "dave"
+let user = "dave"
-const main = println "Hello $user!" exit_status::success
+let main = println "Hello $user!" exit_status::success

notes/expr_refs.md

@@ -6,13 +6,13 @@ Reference loops are resource leaks. There are two primary ways to avoid referenc
 - Constants reference their constituent Expressions
 - Expressions reference Atoms
 - During evaluation, Constants replace their unbound names with Constants
-  - There is a reference cycle here, but it always goes through a Constant.
+	- There is a reference cycle here, but it always goes through a Constant.
-    > **todo** A potential fix may be to update all Constants to point to a dummy value before freeing Trees
+		> **todo** A potential fix may be to update all Constants to point to a dummy value before freeing Trees
 - Atoms reference the Systems that implement them
 - Atoms may reference Expressions that are not younger than them
-  - This link is managed by the System but tied to Atom and not System lifecycle
+	- This link is managed by the System but tied to Atom and not System lifecycle
-  - Atoms can technically be applied to themselves, but it's a copying apply so it probably isn't a risk factor
+	- Atoms can technically be applied to themselves, but it's a copying apply so it probably isn't a risk factor
 - Systems reference the Extension that contains them
 - Extensions reference the Port that connects them
-  - The Extension signals the remote peer to disconnect on drop
+	- The Extension signals the remote peer to disconnect on drop
-  - The port is also referenced in a loose receiver thread, which always eventually tries to find the Extension or polls for ingress so it always eventually exits after the Extension's drop handler is called
+	- The port is also referenced in a loose receiver thread, which always eventually tries to find the Extension or polls for ingress so it always eventually exits after the Extension's drop handler is called

notes/macros.md

@@ -17,11 +17,11 @@ Priority numbers are written in hexadecimal normal form to avoid precision bugs,
 - **32-39**: Binary operators, in inverse priority order
 - **80-87**: Expression-like structures such as if/then/else
 - **128-135**: Anything that creates lambdas
-  Programs triggered by a lower priority pattern than this can assume that all names are correctly bound
+	Programs triggered by a lower priority pattern than this can assume that all names are correctly bound
 - **200**: Aliases extracted for readability
-  The user-accessible entry points of all macro programs must be lower priority than this, so any arbitrary syntax can be extracted into an alias with no side effects
+	The user-accessible entry points of all macro programs must be lower priority than this, so any arbitrary syntax can be extracted into an alias with no side effects
 - **224-231**: Integration; documented hooks exposed by a macro package to allow third party packages to extend its functionality
-  The `statement` pattern produced by `do{}` blocks and matched by `let` and `cps` is a good example of this. When any of these are triggered, all macro programs are in a documented state.
+	The `statement` pattern produced by `do{}` blocks and matched by `let` and `cps` is a good example of this. When any of these are triggered, all macro programs are in a documented state.
 - **248-255**: Transitional states within macro programs get the highest priority
 
 The numbers are arbitrary and up for debate. These are just the ones I came up with when writing the examples.

notes/new_macro_model.md

@@ -42,9 +42,9 @@ Prioritised macro patterns must start and end with a vectorial placeholder. They
 Macros are checked from the outermost block inwards.
 
 1. For every name token, test all named macros starting with that name
-   1. If the tail is implicit, continue iterating
+	 1. If the tail is implicit, continue iterating
 2. Test all prioritized macros
-   1. Take the first rule that matches in the highest prioritized block
+	 1. Take the first rule that matches in the highest prioritized block
 
 Test all in a set of macros
 1. Take the first rule that matches in each block
@@ -75,26 +75,26 @@ Recursion has to happen through the interpreter itself, so the macro system is d
 - line parser `macro` parses a macro with the existing logic
 - atom `MacRecurState` holds the recursion state
 - function `resolve_recur` finds all matches on a MacTree
-  - type: `MacRecurState -> MacTree -> MacTree`
+	- type: `MacRecurState -> MacTree -> MacTree`
-  - use all relevant macros to find all matches in the tree
+	- use all relevant macros to find all matches in the tree
-    - since macros must contain a locally defined token, it can be assumed that at the point that a constant is evaluated and all imports in the parent module have been resolved, necessarily all relevant macro rules must have been loaded
+		- since macros must contain a locally defined token, it can be assumed that at the point that a constant is evaluated and all imports in the parent module have been resolved, necessarily all relevant macro rules must have been loaded
-  - for each match
+	- for each match
-    - check for recursion violations
+		- check for recursion violations
-    - wrap the body in iife-s corresponding to the named values in the match state
+		- wrap the body in iife-s corresponding to the named values in the match state
-    - emit a recursive call to process and run the body, and pass the same recursive call as argument for the macro to use
+		- emit a recursive call to process and run the body, and pass the same recursive call as argument for the macro to use
 		```
 		(\recur. lower (recur $body) recur)
 			(resolve_recur $mac_recur_state)
 		```
-  - emit a single call to `instantiate_tpl` which receives all of these
+	- emit a single call to `instantiate_tpl` which receives all of these
 - function `instantiate_tpl` inserts `MacTree` values into a `MacTree(tpl)`
-  - type: `MacTree(tpl) [-> MacTree] -> MacTree`  
+	- type: `MacTree(tpl) [-> MacTree] -> MacTree`  
 		_this function deduces the number of arguments from the first argument. This combines poorly with autocurry, but it's an easy way to avoid representing standalone tree lists_
-  - walks the tree to find max template slot number, reads and type checks as many template values
+	- walks the tree to find max template slot number, reads and type checks as many template values
-  - returns the populated tree
+	- returns the populated tree
 - function `resolve` is the main entry point of the code
-  - type: `MacTree -> MacTree`
+	- type: `MacTree -> MacTree`
-  - invokes `resolve_recur` with an empty `MacRecurState`
+	- invokes `resolve_recur` with an empty `MacRecurState`
 - function `lower` is the main exit point of the code
-  - type: `MacTree -> any`
+	- type: `MacTree -> any`
-  - Lowers `MacTree` into the equivalent `Expr`.
+	- Lowers `MacTree` into the equivalent `Expr`.

orchid-api-derive/Cargo.toml

@@ -9,9 +9,8 @@ proc-macro = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-quote = "1.0.38"
+quote = "1.0.40"
-syn = { version = "2.0.95" }
+syn = { version = "2.0.106" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-proc-macro2 = "1.0.92"
+proc-macro2 = "1.0.101"
-darling = "0.20.10"
 itertools = "0.14.0"

orchid-api-derive/src/decode.rs

@@ -12,7 +12,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
 	let decode = decode_body(&input.data);
 	let expanded = quote! {
 		impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause {
-			async fn decode<R: orchid_api_traits::async_std::io::Read + ?Sized>(
+			async fn decode<R: orchid_api_traits::AsyncRead + ?Sized>(
 				mut read: std::pin::Pin<&mut R>
 			) -> Self {
 				#decode

orchid-api-derive/src/encode.rs

@@ -14,7 +14,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
 	let encode = encode_body(&input.data);
 	let expanded = quote! {
 		impl #e_impl_generics orchid_api_traits::Encode for #name #e_ty_generics #e_where_clause {
-			async fn encode<W: orchid_api_traits::async_std::io::Write + ?Sized>(
+			async fn encode<W: orchid_api_traits::AsyncWrite + ?Sized>(
 				&self,
 				mut write: std::pin::Pin<&mut W>
 			) {

orchid-api-traits/Cargo.toml

@@ -6,9 +6,8 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-std = "1.13.0"
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
-async-stream = "0.3.6"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-futures = "0.3.31"
 itertools = "0.14.0"
 never = "0.1.0"
 ordered-float = "5.0.0"

orchid-api-traits/src/coding.rs

@@ -7,9 +7,8 @@ use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use async_fn_stream::stream;
-use async_stream::stream;
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, StreamExt};
-use futures::StreamExt;
 use never::Never;
 use ordered_float::NotNan;
 
@@ -18,16 +17,16 @@ use crate::encode_enum;
 pub trait Decode: 'static {
 	/// Decode an instance from the beginning of the buffer. Return the decoded
 	/// data and the remaining buffer.
-	fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
+	fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
 }
 pub trait Encode {
 	/// Append an instance of the struct to the buffer
-	fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
+	fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
 }
 pub trait Coding: Encode + Decode + Clone {
 	fn get_decoder<T: 'static, F: Future<Output = T> + 'static>(
 		map: impl Fn(Self) -> F + Clone + 'static,
-	) -> impl AsyncFn(Pin<&mut dyn Read>) -> T {
+	) -> impl AsyncFn(Pin<&mut dyn AsyncRead>) -> T {
 		async move |r| map(Self::decode(r).await).await
 	}
 }
@@ -36,14 +35,14 @@ impl<T: Encode + Decode + Clone> Coding for T {}
 macro_rules! num_impl {
 	($number:ty) => {
 		impl Decode for $number {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; (<$number>::BITS / 8) as usize];
 				read.read_exact(&mut bytes).await.unwrap();
 				<$number>::from_be_bytes(bytes)
 			}
 		}
 		impl Encode for $number {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 				write.write_all(&self.to_be_bytes()).await.expect("Could not write number")
 			}
 		}
@@ -63,12 +62,12 @@ num_impl!(i8);
 macro_rules! nonzero_impl {
 	($name:ty) => {
 		impl Decode for NonZero<$name> {
-			async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 				Self::new(<$name as Decode>::decode(read).await).unwrap()
 			}
 		}
 		impl Encode for NonZero<$name> {
-			async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 				self.get().encode(write).await
 			}
 		}
@@ -87,19 +86,21 @@ nonzero_impl!(i64);
 nonzero_impl!(i128);
 
 impl<T: Encode + ?Sized> Encode for &T {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) { (**self).encode(write).await }
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+		(**self).encode(write).await
+	}
 }
 macro_rules! float_impl {
 	($t:ty, $size:expr) => {
 		impl Decode for NotNan<$t> {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; $size];
 				read.read_exact(&mut bytes).await.unwrap();
 				NotNan::new(<$t>::from_be_bytes(bytes)).expect("Float was NaN")
 			}
 		}
 		impl Encode for NotNan<$t> {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 				write.write_all(&self.as_ref().to_be_bytes()).await.expect("Could not write number")
 			}
 		}
@@ -110,7 +111,7 @@ float_impl!(f64, 8);
 float_impl!(f32, 4);
 
 impl Decode for String {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
 		let mut data = vec![0u8; len];
 		read.read_exact(&mut data).await.unwrap();
@@ -118,30 +119,36 @@ impl Decode for String {
 	}
 }
 impl Encode for String {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl Encode for str {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl<T: Decode> Decode for Vec<T> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		stream! { loop { yield T::decode(read.as_mut()).await } }.take(len).collect().await
+		stream(async |mut cx| {
+			for _ in 0..len {
+				cx.emit(T::decode(read.as_mut()).await).await
+			}
+		})
+		.collect()
+		.await
 	}
 }
 impl<T: Encode> Encode for Vec<T> {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		self.as_slice().encode(write).await
 	}
 }
 impl<T: Encode> Encode for [T] {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		for t in self.iter() {
 			t.encode(write.as_mut()).await
@@ -149,7 +156,7 @@ impl<T: Encode> Encode for [T] {
 	}
 }
 impl<T: Decode> Decode for Option<T> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		match u8::decode(read.as_mut()).await {
 			0 => None,
 			1 => Some(T::decode(read).await),
@@ -158,14 +165,14 @@ impl<T: Decode> Decode for Option<T> {
 	}
 }
 impl<T: Encode> Encode for Option<T> {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		let t = if let Some(t) = self { t } else { return 0u8.encode(write.as_mut()).await };
 		1u8.encode(write.as_mut()).await;
 		t.encode(write).await;
 	}
 }
 impl<T: Decode, E: Decode> Decode for Result<T, E> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		match u8::decode(read.as_mut()).await {
 			0 => Self::Ok(T::decode(read).await),
 			1 => Self::Err(E::decode(read).await),
@@ -175,7 +182,7 @@ impl<T: Decode, E: Decode> Decode for Result<T, E> {
 }
 
 impl<T: Encode, E: Encode> Encode for Result<T, E> {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		match self {
 			Ok(t) => encode_enum(write, 0, |w| t.encode(w)).await,
 			Err(e) => encode_enum(write, 1, |w| e.encode(w)).await,
@@ -183,13 +190,19 @@ impl<T: Encode, E: Encode> Encode for Result<T, E> {
 	}
 }
 impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		stream! { loop { yield <(K, V)>::decode(read.as_mut()).await } }.take(len).collect().await
+		stream(async |mut cx| {
+			for _ in 0..len {
+				cx.emit(<(K, V)>::decode(read.as_mut()).await).await
+			}
+		})
+		.collect()
+		.await
 	}
 }
 impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		for pair in self.iter() {
 			pair.encode(write.as_mut()).await
@@ -199,12 +212,12 @@ impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
 macro_rules! tuple {
   (($($t:ident)*) ($($T:ident)*)) => {
     impl<$($T: Decode),*> Decode for ($($T,)*) {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				($($T::decode(read.as_mut()).await,)*)
 			}
     }
     impl<$($T: Encode),*> Encode for ($($T,)*) {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
         let ($($t,)*) = self;
         $( $t.encode(write.as_mut()).await; )*
       }
@@ -230,41 +243,45 @@ tuple!((t u v x y z a b c d e f g h i) (T U V X Y Z A B C D E F G H I));
 tuple!((t u v x y z a b c d e f g h i j) (T U V X Y Z A B C D E F G H I J)); // 16
 
 impl Decode for () {
-	async fn decode<R: Read + ?Sized>(_: Pin<&mut R>) -> Self {}
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {}
 }
 impl Encode for () {
-	async fn encode<W: Write + ?Sized>(&self, _: Pin<&mut W>) {}
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) {}
 }
 impl Decode for Never {
-	async fn decode<R: Read + ?Sized>(_: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {
 		unreachable!("A value of Never cannot exist so it can't have been serialized");
 	}
 }
 impl Encode for Never {
-	async fn encode<W: Write + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
 }
 impl Decode for bool {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let mut buf = [0];
 		read.read_exact(&mut buf).await.unwrap();
 		buf[0] != 0
 	}
 }
 impl Encode for bool {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await.unwrap()
 	}
 }
 impl<T: Decode, const N: usize> Decode for [T; N] {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		// TODO: figure out how to do this in safe rust on the stack
+		let v = stream(async |mut cx| {
-		let v =
+			for _ in 0..N {
-			stream! { loop { yield T::decode(read.as_mut()).await } }.take(N).collect::<Vec<_>>().await;
+				cx.emit(T::decode(read.as_mut()).await).await
+			}
+		})
+		.collect::<Vec<_>>()
+		.await;
 		v.try_into().unwrap_or_else(|_| unreachable!("The length of this stream is statically known"))
 	}
 }
 impl<T: Encode, const N: usize> Encode for [T; N] {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		for t in self.iter() {
 			t.encode(write.as_mut()).await
 		}
@@ -274,12 +291,12 @@ impl<T: Encode, const N: usize> Encode for [T; N] {
 macro_rules! two_end_range {
   ($this:ident, $name:tt, $op:tt, $start:expr, $end:expr) => {
     impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				T::decode(read.as_mut()).await $op T::decode(read).await
 			}
     }
     impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
         let $this = self;
         ($start).encode(write.as_mut()).await;
         ($end).encode(write).await;
@@ -294,12 +311,14 @@ two_end_range!(x, RangeInclusive, ..=, x.start(), x.end());
 macro_rules! smart_ptr {
 	($name:tt) => {
 		impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 				$name::new(T::decode(read).await)
 			}
 		}
 		impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) { (**self).encode(write).await }
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+				(**self).encode(write).await
+			}
 		}
 	};
 }
@@ -309,12 +328,12 @@ smart_ptr!(Rc);
 smart_ptr!(Box);
 
 impl Decode for char {
-	async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 		char::from_u32(u32::decode(read).await).unwrap()
 	}
 }
 impl Encode for char {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		(*self as u32).encode(write).await
 	}
 }

orchid-api-traits/src/helpers.rs

@@ -1,12 +1,12 @@
 use std::future::Future;
 use std::pin::Pin;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use itertools::{Chunk, Itertools};
 
 use crate::Encode;
 
-pub async fn encode_enum<'a, W: Write + ?Sized, F: Future<Output = ()>>(
+pub async fn encode_enum<'a, W: AsyncWrite + ?Sized, F: Future<Output = ()>>(
 	mut write: Pin<&'a mut W>,
 	id: u8,
 	f: impl FnOnce(Pin<&'a mut W>) -> F,
@@ -15,7 +15,7 @@ pub async fn encode_enum<'a, W: Write + ?Sized, F: Future<Output = ()>>(
 	f(write).await
 }
 
-pub async fn write_exact<W: Write + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
+pub async fn write_exact<W: AsyncWrite + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
 	write.write_all(bytes).await.expect("Failed to write exact bytes")
 }
 
@@ -27,7 +27,7 @@ pub fn print_bytes(b: &[u8]) -> String {
 		.join("  ")
 }
 
-pub async fn read_exact<R: Read + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
+pub async fn read_exact<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
 	let mut data = vec![0u8; bytes.len()];
 	read.read_exact(&mut data).await.expect("Failed to read bytes");
 	if data != bytes {

orchid-api-traits/src/lib.rs

@@ -3,8 +3,8 @@ mod helpers;
 mod hierarchy;
 mod relations;
 
-pub use async_std;
 pub use coding::*;
+pub use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 pub use helpers::*;
 pub use hierarchy::*;
 pub use relations::*;

orchid-api/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2024"
 ordered-float = "5.0.0"
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
-async-std = "1.13.0"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
 
 [dev-dependencies]
 test_executors = "0.3.2"

orchid-api/src/atom.rs

@@ -5,7 +5,7 @@ use orchid_api_traits::Request;
 
 use crate::{
 	ExprTicket, Expression, ExtHostReq, FormattingUnit, HostExtNotif, HostExtReq, OrcResult, SysId,
-	TStrv,
+	TVec,
 };
 
 pub type AtomData = Vec<u8>;
@@ -83,14 +83,14 @@ impl Request for DeserAtom {
 /// A request blindly routed to the system that provides an atom.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
-pub struct Fwded(pub Atom, pub TStrv, pub Vec<u8>);
+pub struct Fwded(pub Atom, pub TVec, pub Vec<u8>);
 impl Request for Fwded {
 	type Response = Option<Vec<u8>>;
 }
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
-pub struct Fwd(pub Atom, pub TStrv, pub Vec<u8>);
+pub struct Fwd(pub Atom, pub TVec, pub Vec<u8>);
 impl Request for Fwd {
 	type Response = Option<Vec<u8>>;
 }

orchid-api/src/expr.rs

@@ -3,7 +3,7 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{Atom, ExtHostNotif, ExtHostReq, Location, OrcError, SysId, TStrv};
+use crate::{Atom, ExtHostNotif, ExtHostReq, Location, OrcError, SysId, TVec};
 
 /// An arbitrary ID associated with an expression on the host side. Incoming
 /// tickets always come with some lifetime guarantee, which can be extended with
@@ -72,7 +72,7 @@ pub enum ExpressionKind {
 	/// Because the atom is newly constructed, it also must belong to this system.
 	NewAtom(Atom),
 	/// A reference to a constant
-	Const(TStrv),
+	Const(TVec),
 	/// A static runtime error.
 	Bottom(Vec<OrcError>),
 }

orchid-api/src/interner.rs

@@ -47,7 +47,7 @@ impl Request for ExternStr {
 #[extends(IntReq, ExtHostReq)]
 pub struct InternStrv(pub Vec<TStr>);
 impl Request for InternStrv {
-	type Response = TStrv;
+	type Response = TVec;
 }
 /// replica -> master to find the vector of interned strings corresponding to a
 /// token
@@ -57,7 +57,7 @@ impl Request for InternStrv {
 /// See [IntReq]
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(IntReq, ExtHostReq)]
-pub struct ExternStrv(pub TStrv);
+pub struct ExternStrv(pub TVec);
 impl Request for ExternStrv {
 	type Response = Vec<TStr>;
 }
@@ -68,7 +68,7 @@ pub struct TStr(pub NonZeroU64);
 
 /// A substitute for an interned string sequence in serialized datastructures.
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-pub struct TStrv(pub NonZeroU64);
+pub struct TVec(pub NonZeroU64);
 
 /// A request to sweep the replica. The master will not be sweeped until all
 /// replicas respond, as it must retain everything the replicas retained
@@ -84,5 +84,5 @@ impl Request for Sweep {
 #[derive(Clone, Debug, Coding)]
 pub struct Retained {
 	pub strings: Vec<TStr>,
-	pub vecs: Vec<TStrv>,
+	pub vecs: Vec<TVec>,
 }

orchid-api/src/lexer.rs

@@ -3,7 +3,7 @@ use std::ops::RangeInclusive;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TStrv, TokenTree};
+use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TVec, TokenTree};
 
 /// - All ranges contain at least one character
 /// - All ranges are in increasing characeter order
@@ -19,7 +19,7 @@ pub struct LexExpr {
 	pub text: TStr,
 	pub pos: u32,
 	/// Source root module path
-	pub src: TStrv,
+	pub src: TVec,
 }
 impl Request for LexExpr {
 	type Response = Option<OrcResult<LexedExpr>>;

orchid-api/src/location.rs

@@ -2,7 +2,7 @@ use std::ops::Range;
 
 use orchid_api_derive::Coding;
 
-use crate::{TStr, TStrv};
+use crate::{TStr, TVec};
 
 #[derive(Clone, Debug, Coding)]
 pub enum Location {
@@ -21,12 +21,12 @@ pub enum Location {
 
 #[derive(Clone, Debug, Coding)]
 pub struct SourceRange {
-	pub path: TStrv,
+	pub path: TVec,
 	pub range: Range<u32>,
 }
 
 #[derive(Clone, Debug, Coding)]
 pub struct CodeGenInfo {
-	pub generator: TStrv,
+	pub generator: TVec,
 	pub details: TStr,
 }

orchid-api/src/parser.rs

@@ -5,7 +5,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::{
-	Expression, ExtHostReq, HostExtReq, OrcResult, SourceRange, SysId, TStr, TStrv, TokenTree,
+	Expression, ExtHostReq, HostExtReq, OrcResult, SourceRange, SysId, TStr, TVec, TokenTree,
 };
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
@@ -24,12 +24,13 @@ pub struct ParsedConstId(pub NonZeroU64);
 pub struct ParseLine {
 	pub sys: SysId,
 	/// The immediately enclosing module path
-	pub module: TStrv,
+	pub module: TVec,
 	/// The root module path for the snipppet of source code, prefix of
 	/// [ParseLine#module]
-	pub src: TStrv,
+	pub src: TVec,
 	pub comments: Vec<Comment>,
 	pub exported: bool,
+	pub idx: u16,
 	pub line: Vec<TokenTree>,
 }
 impl Request for ParseLine {
@@ -59,7 +60,7 @@ pub struct ParsedMember {
 #[derive(Clone, Debug, Coding)]
 pub enum ParsedMemberKind {
 	Constant(ParsedConstId),
-	Module(Vec<ParsedLine>),
+	Module { lines: Vec<ParsedLine>, use_prelude: bool },
 }
 
 /// Obtain the value of a parsed constant. This is guaranteed to be called after
@@ -85,8 +86,8 @@ pub struct Comment {
 /// called during a [FetchParsedConst] call, but it can be called for a
 /// different [ParsedConstId] from the one in [FetchParsedConst].
 ///
-/// Each name is either resolved to an alias or existing constant `Some(TStrv)`
+/// Each name is either resolved to a valid name or a potential error error.
-/// or not resolved `None`. An error is never raised, as names may have a
+/// The error is not raised by the interpreter itself, as names may have a
 /// primary meaning such as a local binding which can be overridden by specific
 /// true names such as those triggering macro keywords. It is not recommended to
 /// define syntax that can break by defining arbitrary constants, as line
@@ -96,9 +97,9 @@ pub struct Comment {
 pub struct ResolveNames {
 	pub sys: SysId,
 	pub constid: ParsedConstId,
-	pub names: Vec<TStrv>,
+	pub names: Vec<TVec>,
 }
 
 impl Request for ResolveNames {
-	type Response = Vec<Option<TStrv>>;
+	type Response = Vec<OrcResult<TVec>>;
 }

orchid-api/src/proto.rs

@@ -24,7 +24,7 @@
 
 use std::pin::Pin;
 
-use async_std::io::{Read, Write};
+use futures::{AsyncRead, AsyncWrite};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact};
 
@@ -36,7 +36,7 @@ pub struct HostHeader {
 	pub msg_logs: logging::LogStrategy,
 }
 impl Decode for HostHeader {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		read_exact(read.as_mut(), HOST_INTRO).await;
 		Self {
 			log_strategy: logging::LogStrategy::decode(read.as_mut()).await,
@@ -45,7 +45,7 @@ impl Decode for HostHeader {
 	}
 }
 impl Encode for HostHeader {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write_exact(write.as_mut(), HOST_INTRO).await;
 		self.log_strategy.encode(write.as_mut()).await;
 		self.msg_logs.encode(write.as_mut()).await
@@ -58,13 +58,13 @@ pub struct ExtensionHeader {
 	pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		read_exact(read.as_mut(), EXT_INTRO).await;
 		Self { name: String::decode(read.as_mut()).await, systems: Vec::decode(read).await }
 	}
 }
 impl Encode for ExtensionHeader {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write_exact(write.as_mut(), EXT_INTRO).await;
 		self.name.encode(write.as_mut()).await;
 		self.systems.encode(write).await

orchid-api/src/system.rs

@@ -5,7 +5,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::{CharFilter, ExtHostReq, HostExtNotif, HostExtReq, MemberKind, TStr};
+use crate::{CharFilter, ExtHostReq, HostExtNotif, HostExtReq, MemberKind, TStr, TVec};
 
 /// ID of a system type
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
@@ -63,6 +63,7 @@ pub struct NewSystemResponse {
 	pub lex_filter: CharFilter,
 	pub line_types: Vec<TStr>,
 	pub const_root: HashMap<TStr, MemberKind>,
+	pub prelude: Vec<TVec>,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/tree.rs

@@ -6,7 +6,7 @@ use std::rc::Rc;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{ExprTicket, Expression, ExtHostReq, HostExtReq, OrcError, SysId, TStr, TStrv};
+use crate::{ExprTicket, Expression, ExtHostReq, HostExtReq, OrcError, SysId, TStr, TVec};
 
 /// A token tree from a lexer recursion request. Its lifetime is the lex call,
 /// the lexer can include it in its output or discard it by implication.
@@ -27,7 +27,7 @@ pub struct TokenTree {
 #[derive(Clone, Debug, Coding)]
 pub enum Token {
 	/// Lambda function head, from the opening \ until the beginning of the body.
-	LambdaHead(Vec<TokenTree>),
+	LambdaHead(Box<TokenTree>),
 	/// A name segment or an operator.
 	Name(TStr),
 	/// A newly generated expression. The last place this is supposed to happen is
@@ -92,7 +92,7 @@ impl Request for GetMember {
 /// an atom call.
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
-pub struct LsModule(pub SysId, pub TStrv);
+pub struct LsModule(pub SysId, pub TVec);
 impl Request for LsModule {
 	type Response = Result<ModuleInfo, LsModuleError>;
 }

orchid-base/Cargo.toml

@@ -3,16 +3,18 @@ name = "orchid-base"
 version = "0.1.0"
 edition = "2024"
 
+[features]
+mocks = []
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
-async-stream = "0.3.6"
 derive_destructure = "1.0.0"
-dyn-clone = "1.0.17"
+dyn-clone = "1.0.20"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.15.2"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
 never = "0.1.0"
@@ -21,9 +23,9 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 ordered-float = "5.0.0"
-regex = "1.11.1"
+regex = "1.11.2"
-rust-embed = "8.5.0"
+rust-embed = "8.7.2"
-some_executor = "0.4.0"
+some_executor = "0.6.1"
 substack = "1.1.1"
-test_executors = "0.3.2"
+test_executors = "0.3.5"
 trait-set = "0.3.0"

orchid-base/src/clone.rs

@@ -1,12 +1,12 @@
 #[macro_export]
 macro_rules! clone {
-  ($($n:ident),+; $body:expr) => (
+  ($($n:ident $($mut:ident)?),+; $body:expr) => (
     {
-      $( let $n = $n.clone(); )+
+      $( let $($mut)? $n = $n.clone(); )+
       $body
     }
   );
-  ($($n:ident),+) => {
+  ($($n:ident $($mut:ident)?),+) => {
-    $( let $n = $n.clone(); )+
+    $( let $($mut)? $n = $n.clone(); )+
   }
 }

orchid-base/src/ctx.rs

@@ -0,0 +1,28 @@
+use std::any::{TypeId, type_name};
+use std::marker::PhantomData;
+use std::rc::Rc;
+
+use orchid_api_traits::MsgSet;
+
+use crate::error::Reporter;
+use crate::interner::Interner;
+use crate::reqnot::{Client, DynClient};
+
+pub trait CtxDyn {
+	fn i(&self) -> Interner;
+	fn rep(&self) -> &Reporter;
+	fn client(&self, msg_set: TypeId) -> Option<Rc<dyn DynClient>>;
+	fn msg_set_type(&self) -> TypeId;
+}
+
+pub struct Ctx(Rc<dyn CtxDyn>);
+impl Ctx {
+	pub fn i(&self) -> Interner { self.0.i() }
+	pub fn rep(&self) -> &Reporter { self.0.rep() }
+	pub fn client<T: MsgSet>(&self) -> Client<T> {
+		let Some(dyn_client) = self.0.client(TypeId::of::<T>()) else {
+			panic!("Incorrect message set {} passed", type_name::<T>());
+		};
+		Client(dyn_client, PhantomData)
+	}
+}

orchid-base/src/error.rs

@@ -1,14 +1,22 @@
 use std::cell::RefCell;
 use std::ffi::OsStr;
-use std::fmt;
+use std::fmt::{self, Display};
-use std::ops::Add;
+use std::ops::{Add, AddAssign, Deref};
+use std::pin::Pin;
+use std::rc::Rc;
 use std::sync::Arc;
 
-use futures::future::join_all;
+use async_fn_stream::stream;
+use async_once_cell::{Lazy, OnceCell};
+use futures::future::{join_all, ready};
+use futures::lock::Mutex;
+use futures::{Stream, StreamExt, stream};
 use itertools::Itertools;
 
 use crate::api;
-use crate::interner::{Interner, Tok};
+use crate::ctx::Ctx;
+use crate::format::{FmtCtx, FmtUnit, Format};
+use crate::interner::{IStr, Interner};
 use crate::location::Pos;
 
 /// A point of interest in resolving the error, such as the point where
@@ -50,41 +58,126 @@ impl fmt::Display for ErrPos {
 }
 
 #[derive(Clone, Debug)]
-pub struct OrcErr {
+struct SingleError {
-	pub description: Tok<String>,
+	pub description: IStr,
 	pub message: Arc<String>,
 	pub positions: Vec<ErrPos>,
 }
-impl OrcErr {
+impl fmt::Display for SingleError {
-	fn to_api(&self) -> api::OrcError {
-		api::OrcError {
-			description: self.description.to_api(),
-			message: self.message.clone(),
-			locations: self.positions.iter().map(ErrPos::to_api).collect(),
-		}
-	}
-	async fn from_api(api: &api::OrcError, i: &Interner) -> Self {
-		Self {
-			description: Tok::from_api(api.description, i).await,
-			message: api.message.clone(),
-			positions: join_all(api.locations.iter().map(|e| ErrPos::from_api(e, i))).await,
-		}
-	}
-}
-impl Eq for OrcErr {}
-impl PartialEq for OrcErr {
-	fn eq(&self, other: &Self) -> bool { self.description == other.description }
-}
-impl From<OrcErr> for Vec<OrcErr> {
-	fn from(value: OrcErr) -> Self { vec![value] }
-}
-impl fmt::Display for OrcErr {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 		let pstr = self.positions.iter().map(|p| format!("{p}")).join("; ");
 		write!(f, "{}: {} @ {}", self.description, self.message, pstr)
 	}
 }
 
+pub struct OrcErr {
+	singles: OnceCell<OwnedOrcErr>,
+	futures: Mutex<Vec<Pin<Box<dyn Future<Output = OwnedOrcErr>>>>>,
+}
+impl OrcErr {
+	pub async fn into_owned(self) -> OwnedOrcErr {
+		self.to_owned().await;
+		self.singles.into_inner().expect("Initialized above")
+	}
+	pub async fn to_owned(&self) -> &OwnedOrcErr {
+		self
+			.singles
+			.get_or_init(async {
+				let results = join_all(self.futures.lock().await.drain(..)).await;
+				OwnedOrcErr(results.iter().flat_map(|err| err.0.iter()).cloned().collect())
+			})
+			.await
+	}
+	fn into_futures(self) -> Vec<Pin<Box<dyn Future<Output = OwnedOrcErr>>>> {
+		match self.singles.into_inner() {
+			Some(val) => vec![Box::pin(ready(val))],
+			None => self.futures.into_inner(),
+		}
+	}
+}
+impl From<OwnedOrcErr> for OrcErr {
+	fn from(value: OwnedOrcErr) -> Self {
+		Self { singles: OnceCell::from(value), futures: Mutex::new(vec![]) }
+	}
+}
+impl<T: Future<Output = OrcErr> + 'static> From<T> for OrcErr {
+	fn from(value: T) -> Self {
+		Self {
+			singles: OnceCell::new(),
+			futures: Mutex::new(vec![Box::pin(async { value.await.into_owned().await })]),
+		}
+	}
+}
+impl Add for OrcErr {
+	type Output = Self;
+	fn add(mut self, mut rhs: Self) -> Self::Output {
+		if let (Some(l), Some(r)) = (self.singles.get_mut(), rhs.singles.get_mut()) {
+			l.0.extend(r.0.drain(..));
+			return self;
+		}
+		Self {
+			singles: OnceCell::new(),
+			futures: Mutex::new(self.into_futures().into_iter().chain(rhs.into_futures()).collect()),
+		}
+	}
+}
+impl AddAssign for OrcErr {
+	fn add_assign(&mut self, mut rhs: Self) {
+		if let (Some(l), Some(r)) = (self.singles.get_mut(), rhs.singles.get_mut()) {
+			l.0.extend(r.0.drain(..));
+		} else {
+			let mut temp = Self { futures: Mutex::default(), singles: OnceCell::new() };
+			std::mem::swap(self, &mut temp);
+			self.futures.get_mut().extend(temp.into_futures().into_iter().chain(rhs.into_futures()));
+		}
+	}
+}
+impl Format for OrcErr {
+	async fn print<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		format!("{}", self.to_owned().await).into()
+	}
+}
+
+#[derive(Clone)]
+pub struct OwnedOrcErr(Vec<SingleError>);
+impl OwnedOrcErr {
+	pub fn to_api(&self) -> Vec<api::OrcError> {
+		self
+			.0
+			.iter()
+			.map(|err| api::OrcError {
+				description: err.description.to_api(),
+				message: err.message.clone(),
+				locations: err.positions.iter().map(|pos| pos.to_api()).collect(),
+			})
+			.collect_vec()
+	}
+	pub async fn from_api(api: impl IntoIterator<Item = &api::OrcError>, i: &Interner) -> Self {
+		Self(
+			join_all(api.into_iter().map(|e| async {
+				SingleError {
+					description: i.es(e.description).await,
+					message: e.message.clone(),
+					positions: join_all(e.locations.iter().map(|pos| ErrPos::from_api(pos, i))).await,
+				}
+			}))
+			.await,
+		)
+	}
+}
+impl Display for OwnedOrcErr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "{}", self.0.iter().join("\n"))
+	}
+}
+impl fmt::Debug for OwnedOrcErr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "OwnedOrcErr({self}") }
+}
+impl Add for OwnedOrcErr {
+	type Output = Self;
+	fn add(self, rhs: Self) -> Self::Output { Self(self.0.into_iter().chain(rhs.0).collect()) }
+}
+
 #[derive(Clone, Debug)]
 pub struct EmptyErrv;
 impl fmt::Display for EmptyErrv {
@@ -93,130 +186,116 @@ impl fmt::Display for EmptyErrv {
 	}
 }
 
-#[derive(Clone, Debug)]
+pub type OrcRes<T> = Result<T, OrcErr>;
-pub struct OrcErrv(Vec<OrcErr>);
+
-impl OrcErrv {
+pub fn join_ok<T, U>(left: OrcRes<T>, right: OrcRes<U>) -> OrcRes<(T, U)> {
-	pub fn new(errors: impl IntoIterator<Item = OrcErr>) -> Result<Self, EmptyErrv> {
+	match (left, right) {
-		let v = errors.into_iter().collect_vec();
+		(Ok(t), Ok(u)) => Ok((t, u)),
-		if v.is_empty() { Err(EmptyErrv) } else { Ok(Self(v)) }
+		(Err(e), Ok(_)) | (Ok(_), Err(e)) => Err(e),
-	}
+		(Err(e1), Err(e2)) => Err(e1 + e2),
-	#[must_use]
-	pub fn extended<T>(mut self, errors: impl IntoIterator<Item = T>) -> Self
-	where Self: Extend<T> {
-		self.extend(errors);
-		self
-	}
-	#[must_use]
-	pub fn len(&self) -> usize { self.0.len() }
-	#[must_use]
-	pub fn is_empty(&self) -> bool { self.len() == 0 }
-	#[must_use]
-	pub fn any(&self, f: impl FnMut(&OrcErr) -> bool) -> bool { self.0.iter().any(f) }
-	#[must_use]
-	pub fn keep_only(self, f: impl FnMut(&OrcErr) -> bool) -> Option<Self> {
-		let v = self.0.into_iter().filter(f).collect_vec();
-		if v.is_empty() { None } else { Some(Self(v)) }
-	}
-	#[must_use]
-	pub fn one(&self) -> Option<&OrcErr> { (self.0.len() == 1).then(|| &self.0[9]) }
-	pub fn pos_iter(&self) -> impl Iterator<Item = ErrPos> + '_ {
-		self.0.iter().flat_map(|e| e.positions.iter().cloned())
-	}
-	pub fn to_api(&self) -> Vec<api::OrcError> { self.0.iter().map(OrcErr::to_api).collect() }
-	pub async fn from_api<'a>(
-		api: impl IntoIterator<Item = &'a api::OrcError>,
-		i: &Interner,
-	) -> Self {
-		Self(join_all(api.into_iter().map(|e| OrcErr::from_api(e, i))).await)
-	}
-}
-impl From<OrcErr> for OrcErrv {
-	fn from(value: OrcErr) -> Self { Self(vec![value]) }
-}
-impl Add for OrcErrv {
-	type Output = Self;
-	fn add(self, rhs: Self) -> Self::Output { Self(self.0.into_iter().chain(rhs.0).collect_vec()) }
-}
-impl Extend<OrcErr> for OrcErrv {
-	fn extend<T: IntoIterator<Item = OrcErr>>(&mut self, iter: T) { self.0.extend(iter) }
-}
-impl Extend<OrcErrv> for OrcErrv {
-	fn extend<T: IntoIterator<Item = OrcErrv>>(&mut self, iter: T) {
-		self.0.extend(iter.into_iter().flatten())
-	}
-}
-impl IntoIterator for OrcErrv {
-	type IntoIter = std::vec::IntoIter<OrcErr>;
-	type Item = OrcErr;
-	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
-}
-impl fmt::Display for OrcErrv {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "{}", self.0.iter().join("\n"))
 	}
 }
 
-pub type OrcRes<T> = Result<T, OrcErrv>;
+#[macro_export]
+macro_rules! join_ok {
+	($($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		let $crate::join_ok!(@NAMES $($names $(: $tys)? = $vals;)*)
+		:
+		$crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*)
+		 =
+		$crate::join_ok!(@VALUES $($names $(: $tys)? = $vals;)*)?;
+	};
+	(@NAMES $name:ident $(: $ty:ty)? = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		($name, $crate::join_ok!(@NAMES $($names $(: $tys)? = $vals;)*))
+	};
+	(@NAMES) => { _ };
+	(@TYPES $name:ident : $ty:ty = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		($ty, $crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*))
+	};
+	(@TYPES $name:ident = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		(_, $crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*))
+	};
+	(@TYPES) => { () };
+	(@VALUES $name:ident $(: $ty:ty)? = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		$crate::error::join_ok($val, $crate::join_ok!(@VALUES $($names $(: $tys)? = $vals;)*))
+	};
+	(@VALUES) => { Ok(()) };
+}
 
-pub fn mk_err(
+impl Ctx {
-	description: Tok<String>,
+	pub fn mk_err_floating(
-	message: impl AsRef<str>,
+		&self,
-	posv: impl IntoIterator<Item = ErrPos>,
+		description: impl AsRef<str> + 'static,
-) -> OrcErr {
+		message: impl AsRef<str> + 'static,
-	OrcErr {
+	) -> OrcErr {
-		description,
+		self.mk_err::<Pos>(description, message, [])
-		message: Arc::new(message.as_ref().to_string()),
-		positions: posv.into_iter().collect(),
 	}
-}
+	pub fn mk_err<I: Into<ErrPos>>(
-
+		&self,
-pub fn mk_errv<I: Into<ErrPos>>(
+		description: impl AsRef<str> + 'static,
-	description: Tok<String>,
+		message: impl AsRef<str> + 'static,
-	message: impl AsRef<str>,
+		posv: impl IntoIterator<Item = I> + 'static,
-	posv: impl IntoIterator<Item = I>,
+	) -> OrcErr {
-) -> OrcErrv {
+		let i = self.i();
-	mk_err(description, message, posv.into_iter().map_into()).into()
+		async move {
-}
+			OwnedOrcErr(vec![SingleError {
-
+				description: i.is(description.as_ref()).await,
-pub async fn async_io_err<I: Into<ErrPos>>(
+				message: Arc::new(message.as_ref().to_string()),
-	err: async_std::io::Error,
+				positions: posv.into_iter().map_into().collect(),
-	i: &Interner,
+			}])
-	posv: impl IntoIterator<Item = I>,
+			.into()
-) -> OrcErrv {
+		}
-	mk_errv(i.i(&err.kind().to_string()).await, err.to_string(), posv)
+		.into()
-}
+	}
-
+	pub async fn async_io_err<I: Into<ErrPos>>(
-pub async fn os_str_to_string<'a, I: Into<ErrPos>>(
+		&self,
-	str: &'a OsStr,
+		err: std::io::Error,
-	i: &Interner,
+		posv: impl IntoIterator<Item = I> + 'static,
-	posv: impl IntoIterator<Item = I>,
+	) -> OrcErr {
-) -> OrcRes<&'a str> {
+		self.mk_err(err.kind().to_string(), err.to_string(), posv)
-	match str.to_str() {
+	}
-		Some(str) => Ok(str),
+	pub fn os_str_to_string<'a, I: Into<ErrPos>>(
-		None => Err(mk_errv(
+		&self,
-			i.i("Non-unicode string").await,
+		str: &'a OsStr,
-			format!("{str:?} is not representable as unicode"),
+		posv: impl IntoIterator<Item = I> + 'static,
-			posv,
+	) -> OrcRes<&'a str> {
-		)),
+		match str.to_str() {
+			Some(str) => Ok(str),
+			None => Err(self.mk_err(
+				"Non-unicode string",
+				format!("{str:?} is not representable as unicode"),
+				posv,
+			)),
+		}
 	}
 }
 
 pub struct Reporter {
-	errors: RefCell<Vec<OrcErr>>,
+	errors: RefCell<Option<OrcErr>>,
 }
 
 impl Reporter {
-	pub fn report(&self, e: impl Into<OrcErrv>) { self.errors.borrow_mut().extend(e.into()) }
+	pub fn report(&self, e: impl Into<OrcErr>) {
-	pub fn new() -> Self { Self { errors: RefCell::new(vec![]) } }
+		match &mut *self.errors.borrow_mut() {
-	pub fn errv(self) -> Option<OrcErrv> { OrcErrv::new(self.errors.into_inner()).ok() }
+			slot @ None => *slot = Some(e.into()),
-	pub fn merge<T>(self, res: OrcRes<T>) -> OrcRes<T> {
+			Some(err) => *err += e.into(),
-		match (res, self.errv()) {
-			(res, None) => res,
-			(Ok(_), Some(errv)) => Err(errv),
-			(Err(e), Some(errv)) => Err(e + errv),
 		}
 	}
-	pub fn is_empty(&self) -> bool { self.errors.borrow().is_empty() }
+	pub fn new() -> Self { Self { errors: RefCell::new(None) } }
+	pub fn res(self) -> Result<(), OrcErr> {
+		match self.errors.into_inner() {
+			Some(e) => Err(e),
+			None => Ok(()),
+		}
+	}
+	pub fn merge<T>(self, res: OrcRes<T>) -> OrcRes<T> {
+		match (res, self.res()) {
+			(res, Ok(())) => res,
+			(Ok(_), Err(e)) => Err(e),
+			(Err(e), Err(e2)) => Err(e + e2),
+		}
+	}
+	pub fn is_empty(&self) -> bool { self.errors.borrow().is_none() }
 }
 
 impl Default for Reporter {

orchid-base/src/interner.rs

@@ -1,13 +1,15 @@
+use std::any::Any;
 use std::borrow::Borrow;
 use std::future::Future;
-use std::hash::BuildHasher as _;
+use std::hash::{BuildHasher as _, Hash};
 use std::num::NonZeroU64;
 use std::ops::Deref;
 use std::rc::Rc;
 use std::sync::atomic;
 use std::{fmt, hash};
 
-use async_std::sync::Mutex;
+use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools as _;
 use orchid_api_traits::Request;
@@ -15,296 +17,460 @@ use orchid_api_traits::Request;
 use crate::api;
 use crate::reqnot::{DynRequester, Requester};
 
-/// Clippy crashes while verifying `Tok: Sized` without this and I cba to create
+// /// Clippy crashes while verifying `Tok: Sized` without this and I cba to
-/// a minimal example
+// create /// a minimal example
-#[derive(Clone)]
+// #[derive(Clone)]
-struct ForceSized<T>(T);
+// struct ForceSized<T>(T);
 
-#[derive(Clone)]
+// #[derive(Clone)]
-pub struct Tok<T: Interned> {
+// pub struct Tok<T: Interned> {
-	data: Rc<T>,
+// 	data: Rc<T>,
-	marker: ForceSized<T::Marker>,
+// 	marker: ForceSized<T::Marker>,
-}
+// }
-impl<T: Interned> Tok<T> {
+// impl<T: Interned> Tok<T> {
-	pub fn new(data: Rc<T>, marker: T::Marker) -> Self { Self { data, marker: ForceSized(marker) } }
+// 	pub fn new(data: Rc<T>, marker: T::Marker) -> Self { Self { data, marker:
-	pub fn to_api(&self) -> T::Marker { self.marker.0 }
+// ForceSized(marker) } } 	pub fn to_api(&self) -> T::Marker { self.marker.0 }
-	pub async fn from_api<M>(marker: M, i: &Interner) -> Self
+// 	pub async fn from_api<M>(marker: M, i: &Interner) -> Self
-	where M: InternMarker<Interned = T> {
+// 	where M: InternMarker<Interned = T> {
-		i.ex(marker).await
+// 		i.ex(marker).await
-	}
+// 	}
-	pub fn rc(&self) -> Rc<T> { self.data.clone() }
+// 	pub fn rc(&self) -> Rc<T> { self.data.clone() }
-}
+// }
-impl<T: Interned> Deref for Tok<T> {
+// impl<T: Interned> Deref for Tok<T> {
-	type Target = T;
+// 	type Target = T;
 
-	fn deref(&self) -> &Self::Target { self.data.as_ref() }
+// 	fn deref(&self) -> &Self::Target { self.data.as_ref() }
-}
+// }
-impl<T: Interned> Ord for Tok<T> {
+// impl<T: Interned> Ord for Tok<T> {
-	fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.to_api().cmp(&other.to_api()) }
+// 	fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-}
+// self.to_api().cmp(&other.to_api()) } }
-impl<T: Interned> PartialOrd for Tok<T> {
+// impl<T: Interned> PartialOrd for Tok<T> {
-	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
+// 	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-}
+// Some(self.cmp(other)) } }
-impl<T: Interned> Eq for Tok<T> {}
+// impl<T: Interned> Eq for Tok<T> {}
-impl<T: Interned> PartialEq for Tok<T> {
+// impl<T: Interned> PartialEq for Tok<T> {
-	fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
+// 	fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
-}
+// }
-impl<T: Interned> hash::Hash for Tok<T> {
+// impl<T: Interned> hash::Hash for Tok<T> {
-	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.to_api().hash(state) }
+// 	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.to_api().hash(state) }
-}
+// }
-impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
+// impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+// 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{}", &*self.data)
+// 		write!(f, "{}", &*self.data)
-	}
+// 	}
-}
+// }
-impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
+// impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+// 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "Token({} -> {:?})", self.to_api().get_id(), self.data.as_ref())
+// 		write!(f, "Token({} -> {:?})", self.to_api().get_id(), self.data.as_ref())
-	}
+// 	}
-}
+// }
 
-pub trait Interned: Eq + hash::Hash + Clone + fmt::Debug + Internable<Interned = Self> {
+// pub trait Interned: Eq + hash::Hash + Clone + fmt::Debug +
-	type Marker: InternMarker<Interned = Self> + Sized;
+// Internable<Interned = Self> { 	type Marker: InternMarker<Interned = Self> +
-	fn intern(
+// Sized; 	fn intern(
-		self: Rc<Self>,
+// 		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+// 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> impl Future<Output = Self::Marker>;
+// 	) -> impl Future<Output = Self::Marker>;
-	fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
+// 	fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
-}
+// }
 
-pub trait Internable: fmt::Debug {
+// pub trait Internable: fmt::Debug {
-	type Interned: Interned;
+// 	type Interned: Interned;
-	fn get_owned(&self) -> Rc<Self::Interned>;
+// 	fn get_owned(&self) -> Rc<Self::Interned>;
-}
+// }
 
-pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash + Sized {
+// pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash
-	type Interned: Interned<Marker = Self>;
+// + Sized { 	type Interned: Interned<Marker = Self>;
-	/// Only called on replicas
+// 	/// Only called on replicas
-	fn resolve(self, i: &Interner) -> impl Future<Output = Tok<Self::Interned>>;
+// 	fn resolve(self, i: &Interner) -> impl Future<Output = Tok<Self::Interned>>;
-	fn get_id(self) -> NonZeroU64;
+// 	fn get_id(self) -> NonZeroU64;
-	fn from_id(id: NonZeroU64) -> Self;
+// 	fn from_id(id: NonZeroU64) -> Self;
-}
+// }
 
-impl Interned for String {
+// impl Interned for String {
-	type Marker = api::TStr;
+// 	type Marker = api::TStr;
-	async fn intern(
+// 	async fn intern(
-		self: Rc<Self>,
+// 		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+// 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> Self::Marker {
+// 	) -> Self::Marker {
-		req.request(api::InternStr(self.to_string())).await
+// 		req.request(api::InternStr(self.to_string())).await
-	}
+// 	}
-	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.strings }
+// 	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut
-}
+// interners.strings } }
-impl InternMarker for api::TStr {
+// impl InternMarker for api::TStr {
-	type Interned = String;
+// 	type Interned = String;
-	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
+// 	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
-		Tok::new(Rc::new(i.0.master.as_ref().unwrap().request(api::ExternStr(self)).await), self)
+// 		Tok::new(Rc::new(i.0.master.as_ref().unwrap().
-	}
+// request(api::ExternStr(self)).await), self) 	}
-	fn get_id(self) -> NonZeroU64 { self.0 }
+// 	fn get_id(self) -> NonZeroU64 { self.0 }
-	fn from_id(id: NonZeroU64) -> Self { Self(id) }
+// 	fn from_id(id: NonZeroU64) -> Self { Self(id) }
-}
+// }
-impl Internable for str {
+// impl Internable for str {
-	type Interned = String;
+// 	type Interned = String;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
-}
+// }
-impl Internable for String {
+// impl Internable for String {
-	type Interned = String;
+// 	type Interned = String;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
-}
+// }
 
-impl Interned for Vec<Tok<String>> {
+// impl Interned for Vec<IStr> {
-	type Marker = api::TStrv;
+// 	type Marker = api::TStrv;
-	async fn intern(
+// 	async fn intern(
-		self: Rc<Self>,
+// 		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+// 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> Self::Marker {
+// 	) -> Self::Marker {
-		req.request(api::InternStrv(self.iter().map(|t| t.to_api()).collect())).await
+// 		req.request(api::InternStrv(self.iter().map(|t|
-	}
+// t.to_api()).collect())).await 	}
-	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.vecs }
+// 	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut
-}
+// interners.vecs } }
-impl InternMarker for api::TStrv {
+// impl InternMarker for api::TStrv {
-	type Interned = Vec<Tok<String>>;
+// 	type Interned = Vec<IStr>;
-	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
+// 	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
-		let rep = i.0.master.as_ref().unwrap().request(api::ExternStrv(self)).await;
+// 		let rep =
-		let data = futures::future::join_all(rep.into_iter().map(|m| i.ex(m))).await;
+// i.0.master.as_ref().unwrap().request(api::ExternStrv(self)).await; 		let data
-		Tok::new(Rc::new(data), self)
+// = futures::future::join_all(rep.into_iter().map(|m| i.ex(m))).await;
-	}
+// 		Tok::new(Rc::new(data), self)
-	fn get_id(self) -> NonZeroU64 { self.0 }
+// 	}
-	fn from_id(id: NonZeroU64) -> Self { Self(id) }
+// 	fn get_id(self) -> NonZeroU64 { self.0 }
-}
+// 	fn from_id(id: NonZeroU64) -> Self { Self(id) }
-impl Internable for [Tok<String>] {
+// }
-	type Interned = Vec<Tok<String>>;
+// impl Internable for [IStr] {
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+// 	type Interned = Vec<IStr>;
-}
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
-impl<const N: usize> Internable for [Tok<String>; N] {
+// }
-	type Interned = Vec<Tok<String>>;
+// impl<const N: usize> Internable for [IStr; N] {
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+// 	type Interned = Vec<IStr>;
-}
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
-impl Internable for Vec<Tok<String>> {
+// }
-	type Interned = Vec<Tok<String>>;
+// impl Internable for Vec<IStr> {
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+// 	type Interned = Vec<IStr>;
-}
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+// }
 // impl Internable for Vec<api::TStr> {
-// 	type Interned = Vec<Tok<String>>;
+// 	type Interned = Vec<IStr>;
 // 	fn get_owned(&self) -> Arc<Self::Interned> {
 // 		Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
 // 	}
 // }
 // impl Internable for [api::TStr] {
-// 	type Interned = Vec<Tok<String>>;
+// 	type Interned = Vec<IStr>;
 // 	fn get_owned(&self) -> Arc<Self::Interned> {
 // 		Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
 // 	}
 // }
 
-/// The number of references held to any token by the interner.
+macro_rules! token_def {
-const BASE_RC: usize = 3;
+	($type:ident, $trait:ident, $deref:ty, $api_repr:ty, $type_name:expr) => {
-
+		#[derive(Clone)]
-#[test]
+		pub struct $type(Rc<dyn $trait>);
-fn base_rc_correct() {
+		impl $type {
-	let tok = Tok::new(Rc::new("foo".to_string()), api::TStr(1.try_into().unwrap()));
+			pub fn new<T: $trait + 'static>(t: T) -> Self { Self(Rc::new(t) as _) }
-	let mut bimap = Bimap::default();
+			pub fn to_api(&self) -> $api_repr { self.0.to_api() }
-	bimap.insert(tok.clone());
+			pub fn inner(&self) -> &dyn Any { self.0.as_ref() }
-	assert_eq!(Rc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the current instance");
+			fn addr(&self) -> usize { Rc::as_ptr(&self.0).addr() }
+		}
+		pub trait $trait: Deref<Target = $deref> + Any {
+			fn to_api(&self) -> $api_repr;
+		}
+		impl Deref for $type {
+			type Target = $deref;
+			fn deref(&self) -> &Self::Target { self.0.deref() }
+		}
+		impl Eq for $type {}
+		impl PartialEq for $type {
+			fn eq(&self, other: &Self) -> bool { Rc::ptr_eq(&self.0, &other.0) }
+		}
+		impl Ord for $type {
+			fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.addr().cmp(&other.addr()) }
+		}
+		impl PartialOrd for $type {
+			fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
+		}
+		impl Hash for $type {
+			fn hash<H: hash::Hasher>(&self, state: &mut H) { self.addr().hash(state) }
+		}
+		impl std::fmt::Debug for $type {
+			fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+				f.debug_tuple($type_name).field(&self.to_api().0).field(&self.deref()).finish()
+			}
+		}
+	};
 }
 
-pub struct Bimap<T: Interned> {
+token_def!(IStr, IStrDyn, str, api::TStr, "IStr");
-	intern: HashMap<Rc<T>, Tok<T>>,
+impl std::fmt::Display for IStr {
-	by_id: HashMap<T::Marker, Tok<T>>,
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.deref()) }
-}
-impl<T: Interned> Bimap<T> {
-	pub fn insert(&mut self, token: Tok<T>) {
-		self.intern.insert(token.data.clone(), token.clone());
-		self.by_id.insert(token.to_api(), token);
-	}
-
-	pub fn by_marker(&self, marker: T::Marker) -> Option<Tok<T>> { self.by_id.get(&marker).cloned() }
-
-	pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Tok<T>>
-	where T: Borrow<Q> {
-		(self.intern.raw_entry())
-			.from_hash(self.intern.hasher().hash_one(q), |k| k.as_ref().borrow() == q)
-			.map(|p| p.1.clone())
-	}
-
-	pub fn sweep_replica(&mut self) -> Vec<T::Marker> {
-		(self.intern)
-			.extract_if(|k, _| Rc::strong_count(k) == BASE_RC)
-			.map(|(_, v)| {
-				self.by_id.remove(&v.to_api());
-				v.to_api()
-			})
-			.collect()
-	}
-
-	pub fn sweep_master(&mut self, retained: HashSet<T::Marker>) {
-		self.intern.retain(|k, v| BASE_RC < Rc::strong_count(k) || retained.contains(&v.to_api()))
-	}
 }
 
-impl<T: Interned> Default for Bimap<T> {
+token_def!(IVec, IVecDyn, [IStr], api::TVec, "IVec");
-	fn default() -> Self { Self { by_id: HashMap::new(), intern: HashMap::new() } }
-}
 
-pub trait UpComm {
+#[derive(Clone)]
-	fn up<R: Request>(&self, req: R) -> R::Response;
+pub struct Interner(Rc<dyn InternerDyn>);
-}
-
-#[derive(Default)]
-pub struct TypedInterners {
-	strings: Bimap<String>,
-	vecs: Bimap<Vec<Tok<String>>>,
-}
-
-#[derive(Default)]
-pub struct InternerData {
-	interners: Mutex<TypedInterners>,
-	master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
-}
-#[derive(Clone, Default)]
-pub struct Interner(Rc<InternerData>);
 impl Interner {
-	pub fn new_master() -> Self { Self::default() }
+	pub fn new<T: InternerDyn + 'static>(t: T) -> Self { Self(Rc::new(t) as _) }
-	pub fn new_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static) -> Self {
+	pub async fn is(&self, s: &(impl Borrow<str> + ?Sized)) -> IStr {
-		Self(Rc::new(InternerData { master: Some(Box::new(req)), interners: Mutex::default() }))
+		IStr(self.0.is(s.borrow()).await)
 	}
-	/// Intern some data; query its identifier if not known locally
+	pub async fn iv(&self, s: &(impl Borrow<[IStr]> + ?Sized)) -> IVec {
-	pub async fn i<T: Interned>(&self, t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
+		IVec(self.0.iv(s.borrow()).await)
-		let data = t.get_owned();
-		let mut g = self.0.interners.lock().await;
-		let typed = T::bimap(&mut g);
-		if let Some(tok) = typed.by_value(&data) {
-			return tok;
-		}
-		let marker = match &self.0.master {
-			Some(c) => data.clone().intern(&**c).await,
-			None =>
-				T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
-		};
-		let tok = Tok::new(data, marker);
-		T::bimap(&mut g).insert(tok.clone());
-		tok
-	}
-	/// Extern an identifier; query the data it represents if not known locally
-	pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
-		if let Some(tok) = M::Interned::bimap(&mut *self.0.interners.lock().await).by_marker(marker) {
-			return tok;
-		}
-		assert!(self.0.master.is_some(), "ID not in local interner and this is master");
-		let token = marker.resolve(self).await;
-		M::Interned::bimap(&mut *self.0.interners.lock().await).insert(token.clone());
-		token
-	}
-	pub async fn sweep_replica(&self) -> api::Retained {
-		assert!(self.0.master.is_some(), "Not a replica");
-		let mut g = self.0.interners.lock().await;
-		api::Retained { strings: g.strings.sweep_replica(), vecs: g.vecs.sweep_replica() }
-	}
-	pub async fn sweep_master(&self, retained: api::Retained) {
-		assert!(self.0.master.is_none(), "Not master");
-		let mut g = self.0.interners.lock().await;
-		g.strings.sweep_master(retained.strings.into_iter().collect());
-		g.vecs.sweep_master(retained.vecs.into_iter().collect());
 	}
+	pub async fn es(&self, m: api::TStr) -> IStr { IStr(self.0.es(m).await) }
+	pub async fn ev(&self, m: api::TVec) -> IVec { IVec(self.0.ev(m).await) }
 }
-impl fmt::Debug for Interner {
+pub trait InternerDyn {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+	fn is(&self, s: &str) -> LocalBoxFuture<Rc<dyn IStrDyn>>;
-		write!(f, "Interner{{ replica: {} }}", self.0.master.is_none())
+	fn iv(&self, v: &[IStr]) -> LocalBoxFuture<Rc<dyn IVecDyn>>;
-	}
+	fn es(&self, m: api::TStr) -> LocalBoxFuture<Rc<dyn IStrDyn>>;
+	fn ev(&self, m: api::TVec) -> LocalBoxFuture<Rc<dyn IVecDyn>>;
 }
 
-static ID: atomic::AtomicU64 = atomic::AtomicU64::new(1);
+#[cfg(any(feature = "mocks", test))]
+pub mod test {
 
-pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) {
+	use std::cell::RefCell;
-	into.strings = into.strings.iter().chain(&from.strings).copied().unique().collect();
-	into.vecs = into.vecs.iter().chain(&from.vecs).copied().unique().collect();
-}
-
-#[cfg(test)]
-mod test {
 	use std::num::NonZero;
-	use std::pin::Pin;
+	use std::sync::atomic::AtomicU64;
-
-	use orchid_api_traits::{Decode, enc_vec};
-	use test_executors::spin_on;
 
 	use super::*;
-	use crate::api;
+	use crate::testing::AsyncMonitor;
 
-	#[test]
+	pub(crate) struct DummyIStr(NonZeroU64, String);
-	fn test_i() {
+	impl Deref for DummyIStr {
-		let i = Interner::new_master();
+		type Target = str;
-		let _: Tok<String> = spin_on(i.i("foo"));
+		fn deref(&self) -> &Self::Target { &self.1 }
-		let _: Tok<Vec<Tok<String>>> = spin_on(i.i(&[spin_on(i.i("bar")), spin_on(i.i("baz"))]));
+	}
+	impl IStrDyn for DummyIStr {
+		fn to_api(&self) -> api::TStr { api::TStr(self.0) }
 	}
 
-	#[test]
+	pub(crate) struct DummyIStrv(NonZeroU64, Vec<IStr>);
-	fn test_coding() {
+	impl Deref for DummyIStrv {
-		spin_on(async {
+		type Target = [IStr];
-			let coded = api::TStr(NonZero::new(3u64).unwrap());
+		fn deref(&self) -> &Self::Target { &self.1 }
-			let mut enc = &enc_vec(&coded).await[..];
-			api::TStr::decode(Pin::new(&mut enc)).await;
-			assert_eq!(enc, [], "Did not consume all of {enc:?}")
-		})
 	}
+	impl IVecDyn for DummyIStrv {
+		fn to_api(&self) -> api::TVec { api::TVec(self.0) }
+	}
+
+	pub(crate) struct DummyInterner(
+		RefCell<(
+			HashMap<String, NonZeroU64>,
+			HashMap<NonZeroU64, Rc<DummyIStr>>,
+			HashMap<NonZeroU64, Rc<DummyIStrv>>,
+		)>,
+		AsyncMonitor<InternerEvent>,
+	);
+	pub enum InternerEvent {
+		ExternStr(Rc<DummyIStr>),
+		ExternVec(Rc<DummyIStrv>),
+		InternStr { token: Rc<DummyIStr>, new: bool },
+		InternVec { token: Rc<DummyIStrv>, new: bool },
+	}
+	impl DummyInterner {
+		pub fn new(monitor: AsyncMonitor<InternerEvent>) -> Interner {
+			Interner(Rc::new(Self(RefCell::default(), monitor)))
+		}
+	}
+	impl InternerDyn for DummyInterner {
+		fn es(&self, m: api::TStr) -> LocalBoxFuture<Rc<dyn IStrDyn>> {
+			let state = self.0.borrow();
+			let istr = state.1.get(&m.0).unwrap_or_else(|| panic!("Externed nonexistent {m:?}")).clone();
+			Box::pin(async {
+				self.1.notify(InternerEvent::ExternStr(istr.clone())).await;
+				istr as Rc<dyn IStrDyn>
+			})
+		}
+		fn ev(&self, m: api::TVec) -> LocalBoxFuture<Rc<dyn IVecDyn>> {
+			let state = self.0.borrow();
+			let ivec = state.2.get(&m.0).unwrap_or_else(|| panic!("Externed nonexistent {m:?}")).clone();
+			Box::pin(async {
+				self.1.notify(InternerEvent::ExternVec(ivec.clone())).await;
+				ivec as Rc<dyn IVecDyn>
+			})
+		}
+		fn is(&self, s: &str) -> LocalBoxFuture<Rc<dyn IStrDyn>> {
+			let mut this = self.0.borrow_mut();
+			let id = *(this.0.entry(format!("{s:?}")))
+				.or_insert_with(|| NonZero::new(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)).unwrap());
+			let (tok, new) = match this.1.entry(id) {
+				hashbrown::hash_map::Entry::Occupied(ent) => (ent.get().clone(), false),
+				hashbrown::hash_map::Entry::Vacant(ent) =>
+					(ent.insert(Rc::new(DummyIStr(id, s.to_string()))).clone(), true),
+			};
+			Box::pin(async move {
+				self.1.notify(InternerEvent::InternStr { token: tok.clone(), new }).await;
+				tok as _
+			})
+		}
+		fn iv(&self, s: &[IStr]) -> LocalBoxFuture<Rc<dyn IVecDyn>> {
+			let mut this = self.0.borrow_mut();
+			let id = *(this.0.entry(format!("{s:?}")))
+				.or_insert_with(|| NonZero::new(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)).unwrap());
+			let (tok, new) = match this.2.entry(id) {
+				hashbrown::hash_map::Entry::Occupied(ent) => (ent.get().clone(), false),
+				hashbrown::hash_map::Entry::Vacant(ent) =>
+					(ent.insert(Rc::new(DummyIStrv(id, s.to_vec()))).clone(), true),
+			};
+			Box::pin(async move {
+				self.1.notify(InternerEvent::InternVec { token: tok.clone(), new }).await;
+				tok as _
+			})
+		}
+	}
+	static COUNTER: AtomicU64 = AtomicU64::new(1);
 }
+
+// /// The number of references held to any token by the interner.
+// const BASE_RC: usize = 3;
+
+// #[test]
+// fn base_rc_correct() {
+// 	let tok = Tok::new(Rc::new("foo".to_string()),
+// api::TStr(1.try_into().unwrap())); 	let mut bimap = Bimap::default();
+// 	bimap.insert(tok.clone());
+// 	assert_eq!(Rc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the
+// current instance"); }
+
+// pub struct Bimap<K, V, Tok> {
+// 	intern: HashMap<V, Tok>,
+// 	by_id: HashMap<K, Tok>,
+// }
+// impl<K> Bimap<T> {
+// 	pub fn insert(&mut self, token: Tok<T>) {
+// 		self.intern.insert(token.data.clone(), token.clone());
+// 		self.by_id.insert(token.to_api(), token);
+// 	}
+
+// 	pub fn by_marker(&self, marker: T::Marker) -> Option<Tok<T>> {
+// self.by_id.get(&marker).cloned() }
+
+// 	pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Tok<T>>
+// 	where T: Borrow<Q> {
+// 		(self.intern.raw_entry())
+// 			.from_hash(self.intern.hasher().hash_one(q), |k| k.as_ref().borrow() == q)
+// 			.map(|p| p.1.clone())
+// 	}
+
+// 	pub fn sweep_replica(&mut self) -> Vec<T::Marker> {
+// 		(self.intern)
+// 			.extract_if(|k, _| Rc::strong_count(k) == BASE_RC)
+// 			.map(|(_, v)| {
+// 				self.by_id.remove(&v.to_api());
+// 				v.to_api()
+// 			})
+// 			.collect()
+// 	}
+
+// 	pub fn sweep_master(&mut self, retained: HashSet<T::Marker>) {
+// 		self.intern.retain(|k, v| BASE_RC < Rc::strong_count(k) ||
+// retained.contains(&v.to_api())) 	}
+// }
+
+// impl<T: Interned> Default for Bimap<T> {
+// 	fn default() -> Self { Self { by_id: HashMap::new(), intern: HashMap::new()
+// } } }
+
+// pub trait UpComm {
+// 	fn up<R: Request>(&self, req: R) -> R::Response;
+// }
+
+// #[derive(Default)]
+// pub struct TypedInterners {
+// 	strings: Bimap<String>,
+// 	vecs: Bimap<Vec<IStr>>,
+// }
+
+// #[derive(Default)]
+// pub struct InternerData {
+// 	interners: Mutex<TypedInterners>,
+// 	master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
+// }
+// #[derive(Clone, Default)]
+// pub struct Interner(Rc<InternerData>);
+// impl Interner {
+// 	pub fn new_master() -> Self { Self::default() }
+// 	pub fn new_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static)
+// -> Self { 		Self(Rc::new(InternerData { master: Some(Box::new(req)),
+// interners: Mutex::default() })) 	}
+// 	/// Intern some data; query its identifier if not known locally
+// 	pub async fn i<T: Interned>(&self, t: &(impl Internable<Interned = T> +
+// ?Sized)) -> Tok<T> { 		let data = t.get_owned();
+// 		let mut g = self.0.interners.lock().await;
+// 		let typed = T::bimap(&mut g);
+// 		if let Some(tok) = typed.by_value(&data) {
+// 			return tok;
+// 		}
+// 		let marker = match &self.0.master {
+// 			Some(c) => data.clone().intern(&**c).await,
+// 			None =>
+// 				T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1,
+// atomic::Ordering::Relaxed)).unwrap()), 		};
+// 		let tok = Tok::new(data, marker);
+// 		T::bimap(&mut g).insert(tok.clone());
+// 		tok
+// 	}
+// 	/// Extern an identifier; query the data it represents if not known locally
+// 	pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
+// 		if let Some(tok) = M::Interned::bimap(&mut
+// *self.0.interners.lock().await).by_marker(marker) { 			return tok;
+// 		}
+// 		assert!(self.0.master.is_some(), "ID not in local interner and this is
+// master"); 		let token = marker.resolve(self).await;
+// 		M::Interned::bimap(&mut
+// *self.0.interners.lock().await).insert(token.clone()); 		token
+// 	}
+// 	pub async fn sweep_replica(&self) -> api::Retained {
+// 		assert!(self.0.master.is_some(), "Not a replica");
+// 		let mut g = self.0.interners.lock().await;
+// 		api::Retained { strings: g.strings.sweep_replica(), vecs:
+// g.vecs.sweep_replica() } 	}
+// 	pub async fn sweep_master(&self, retained: api::Retained) {
+// 		assert!(self.0.master.is_none(), "Not master");
+// 		let mut g = self.0.interners.lock().await;
+// 		g.strings.sweep_master(retained.strings.into_iter().collect());
+// 		g.vecs.sweep_master(retained.vecs.into_iter().collect());
+// 	}
+// }
+// impl fmt::Debug for Interner {
+// 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+// 		write!(f, "Interner{{ replica: {} }}", self.0.master.is_none())
+// 	}
+// }
+
+// static ID: atomic::AtomicU64 = atomic::AtomicU64::new(1);
+
+// pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) {
+// 	into.strings =
+// into.strings.iter().chain(&from.strings).copied().unique().collect();
+// 	into.vecs = into.vecs.iter().chain(&from.vecs).copied().unique().collect();
+// }
+
+// #[cfg(test)]
+// mod test {
+// 	use std::num::NonZero;
+// 	use std::pin::Pin;
+
+// 	use orchid_api_traits::{Decode, enc_vec};
+// 	use test_executors::spin_on;
+
+// 	use super::*;
+// 	use crate::api;
+
+// 	#[test]
+// 	fn test_i() {
+// 		let i = Interner::new_master();
+// 		let _: IStr = spin_on(i.i("foo"));
+// 		let _: Tok<Vec<IStr>> = spin_on(i.i(&[spin_on(i.i("bar")),
+// spin_on(i.i("baz"))])); 	}
+
+// 	#[test]
+// 	fn test_coding() {
+// 		spin_on(async {
+// 			let coded = api::TStr(NonZero::new(3u64).unwrap());
+// 			let mut enc = &enc_vec(&coded).await[..];
+// 			api::TStr::decode(Pin::new(&mut enc)).await;
+// 			assert_eq!(enc, [], "Did not consume all of {enc:?}")
+// 		})
+// 	}
+// }

orchid-base/src/iter_utils.rs

@@ -0,0 +1,24 @@
+use std::fmt;
+
+use itertools::{Itertools, Position};
+
+pub struct PrintList<'a, I: Iterator<Item = E> + Clone, E: fmt::Display>(pub I, pub &'a str);
+impl<'a, I: Iterator<Item = E> + Clone, E: fmt::Display> fmt::Display for PrintList<'a, I, E> {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		for (pos, item) in self.0.clone().with_position() {
+			match pos {
+				Position::First | Position::Only => write!(f, "{item}")?,
+				Position::Middle => write!(f, ", {item}")?,
+				Position::Last => write!(f, ", {} {item}", self.1)?,
+			}
+		}
+		Ok(())
+	}
+}
+
+pub trait IteratorPrint: Iterator<Item: fmt::Display> + Clone {
+	fn display<'a>(self, operator: &'a str) -> PrintList<'a, Self, Self::Item> {
+		PrintList(self, operator)
+	}
+}
+impl<T: Iterator<Item: fmt::Display> + Clone> IteratorPrint for T {}

orchid-base/src/lib.rs

@@ -7,11 +7,13 @@ pub mod builtin;
 pub mod char_filter;
 pub mod clone;
 pub mod combine;
+pub mod ctx;
 pub mod error;
 pub mod event;
 pub mod format;
 pub mod id_store;
 pub mod interner;
+pub mod iter_utils;
 pub mod join;
 pub mod location;
 pub mod logging;
@@ -24,6 +26,7 @@ pub mod pure_seq;
 pub mod reqnot;
 pub mod sequence;
 pub mod side;
+pub mod testing;
 mod tl_cache;
 pub mod tokens;
 pub mod tree;

orchid-base/src/location.rs

@@ -7,15 +7,15 @@ use std::ops::Range;
 use trait_set::trait_set;
 
 use crate::error::ErrPos;
-use crate::interner::{Interner, Tok};
+use crate::interner::{IStr, Interner};
 use crate::name::Sym;
 use crate::{api, match_mapping, sym};
 
 trait_set! {
-	pub trait GetSrc = FnMut(&Sym) -> Tok<String>;
+	pub trait GetSrc = FnMut(&Sym) -> IStr;
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Pos {
 	None,
 	SlotTarget,
@@ -108,6 +108,10 @@ impl SrcRange {
 	pub fn to_api(&self) -> api::SourceRange {
 		api::SourceRange { path: self.path.to_api(), range: self.range.clone() }
 	}
+	pub fn to(&self, rhs: &Self) -> Self {
+		assert_eq!(self.path, rhs.path, "Range continues across files");
+		Self { path: self.path(), range: self.start().min(rhs.start())..self.end().max(rhs.end()) }
+	}
 }
 impl From<SrcRange> for ErrPos {
 	fn from(val: SrcRange) -> Self { val.pos().into() }
@@ -127,24 +131,21 @@ pub struct CodeGenInfo {
 	/// formatted like a Rust namespace
 	pub generator: Sym,
 	/// Unformatted user message with relevant circumstances and parameters
-	pub details: Tok<String>,
+	pub details: IStr,
 }
 impl CodeGenInfo {
 	/// A codegen marker with no user message and parameters
 	pub async fn new_short(generator: Sym, i: &Interner) -> Self {
-		Self { generator, details: i.i("").await }
+		Self { generator, details: i.is("").await }
 	}
 	/// A codegen marker with a user message or parameters
 	pub async fn new_details(generator: Sym, details: impl AsRef<str>, i: &Interner) -> Self {
-		Self { generator, details: i.i(details.as_ref()).await }
+		Self { generator, details: i.is(details.as_ref()).await }
 	}
 	/// Syntactic location
 	pub fn pos(&self) -> Pos { Pos::Gen(self.clone()) }
 	pub async fn from_api(api: &api::CodeGenInfo, i: &Interner) -> Self {
-		Self {
+		Self { generator: Sym::from_api(api.generator, i).await, details: i.es(api.details).await }
-			generator: Sym::from_api(api.generator, i).await,
-			details: Tok::from_api(api.details, i).await,
-		}
 	}
 	pub fn to_api(&self) -> api::CodeGenInfo {
 		api::CodeGenInfo { generator: self.generator.to_api(), details: self.details.to_api() }

orchid-base/src/match_mapping.rs

@@ -1,4 +1,4 @@
-/// A shorthand for mapping over enums with identical structure. Used for
+/// A shorthand for mapping over enums with similar structure. Used for
 /// converting between owned enums and the corresponding API enums that only
 /// differ in the type of their fields.
 ///
@@ -7,7 +7,11 @@
 /// match_mapping!(self, ThisType => OtherType {
 ///   EmptyVariant,
 ///   TupleVariant(foo => intern(foo), bar.clone()),
-///   StructVariant{ a.to_api(), b }
+///   StructVariant{ a.to_api(), b },
+///   DedicatedConverter(value () convert)
+/// } {
+///   ThisType::DimorphicVariant(c) => OtherType::CorrespondingVariant(c.left(), c.right()),
+///   ThisType::UnexpectedVariant => panic!(),
 /// })
 /// ```
 #[macro_export]

orchid-base/src/msg.rs

@@ -1,10 +1,10 @@
 use std::io;
 use std::pin::Pin;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use orchid_api_traits::{Decode, Encode};
 
-pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result<()> {
+pub async fn send_msg(mut write: Pin<&mut impl AsyncWrite>, msg: &[u8]) -> io::Result<()> {
 	let mut len_buf = vec![];
 	u32::try_from(msg.len()).unwrap().encode(Pin::new(&mut len_buf)).await;
 	write.write_all(&len_buf).await?;
@@ -12,7 +12,7 @@ pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result
 	write.flush().await
 }
 
-pub async fn recv_msg(mut read: Pin<&mut impl Read>) -> io::Result<Vec<u8>> {
+pub async fn recv_msg(mut read: Pin<&mut impl AsyncRead>) -> io::Result<Vec<u8>> {
 	let mut len_buf = [0u8; (u32::BITS / 8) as usize];
 	read.read_exact(&mut len_buf).await?;
 	let len = u32::decode(Pin::new(&mut &len_buf[..])).await;

orchid-base/src/name.rs

@@ -12,53 +12,48 @@ use itertools::Itertools;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::interner::{InternMarker, Interner, Tok};
+use crate::interner::{IStr, IVec, Interner};
 
 trait_set! {
 	/// Traits that all name iterators should implement
-	pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
+	pub trait NameIter = Iterator<Item = IStr> + DoubleEndedIterator + ExactSizeIterator;
 }
 
-/// A token path which may be empty. [VName] is the non-empty,
+/// A token path which may be empty. [VName] is the non-empty version
-/// [PathSlice] is the borrowed version
 #[derive(Clone, Default, Hash, PartialEq, Eq)]
-pub struct VPath(pub Vec<Tok<String>>);
+pub struct VPath(Vec<IStr>);
 impl VPath {
 	/// Collect segments into a vector
-	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn new(items: impl IntoIterator<Item = IStr>) -> Self { Self(items.into_iter().collect()) }
-		Self(items.into_iter().collect())
-	}
 	/// Number of path segments
 	pub fn len(&self) -> usize { self.0.len() }
 	/// Whether there are any path segments. In other words, whether this is a
 	/// valid name
 	pub fn is_empty(&self) -> bool { self.len() == 0 }
 	/// Prepend some tokens to the path
-	pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn prefix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(items.into_iter().chain(self.0).collect())
 	}
 	/// Append some tokens to the path
-	pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn suffix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Partition the string by `::` namespace separators
 	pub async fn parse(s: &str, i: &Interner) -> Self {
-		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(|s| i.i(s))).await })
+		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(|s| i.is(s))).await })
 	}
 	/// Walk over the segments
-	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
+	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> { self.0.iter().map(|s| s.as_ref()) }
-		Box::new(self.0.iter().map(|s| s.as_str()))
-	}
 	/// Try to convert into non-empty version
 	pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
 	/// Add a token to the path. Since now we know that it can't be empty, turn it
 	/// into a name.
-	pub fn name_with_suffix(self, name: Tok<String>) -> VName {
+	pub fn name_with_suffix(self, name: IStr) -> VName {
 		VName(self.into_iter().chain([name]).collect())
 	}
 	/// Add a token to the beginning of the. Since now we know that it can't be
 	/// empty, turn it into a name.
-	pub fn name_with_prefix(self, name: Tok<String>) -> VName {
+	pub fn name_with_prefix(self, name: IStr) -> VName {
 		VName([name].into_iter().chain(self).collect())
 	}
 
@@ -66,7 +61,7 @@ impl VPath {
 	pub async fn from_path(path: &Path, ext: &str, i: &Interner) -> Option<(Self, bool)> {
 		async fn to_vpath(p: &Path, i: &Interner) -> Option<VPath> {
 			let tok_opt_v =
-				join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(|s| i.i(s))))).await;
+				join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(|s| i.is(s))))).await;
 			tok_opt_v.into_iter().collect::<Option<_>>().map(VPath)
 		}
 		match path.extension().map(|s| s.to_str()) {
@@ -84,30 +79,28 @@ impl fmt::Display for VPath {
 		write!(f, "{}", self.str_iter().join("::"))
 	}
 }
-impl FromIterator<Tok<String>> for VPath {
+impl FromIterator<IStr> for VPath {
-	fn from_iter<T: IntoIterator<Item = Tok<String>>>(iter: T) -> Self {
+	fn from_iter<T: IntoIterator<Item = IStr>>(iter: T) -> Self { Self(iter.into_iter().collect()) }
-		Self(iter.into_iter().collect())
-	}
 }
 impl IntoIterator for VPath {
-	type Item = Tok<String>;
+	type Item = IStr;
 	type IntoIter = vec::IntoIter<Self::Item>;
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
-impl Borrow<[Tok<String>]> for VPath {
+impl Borrow<[IStr]> for VPath {
-	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
+	fn borrow(&self) -> &[IStr] { &self.0[..] }
 }
 impl Deref for VPath {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
 impl<T> Index<T> for VPath
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
-	fn index(&self, index: T) -> &Self::Output { &Borrow::<[Tok<String>]>::borrow(self)[index] }
+	fn index(&self, index: T) -> &Self::Output { &Borrow::<[IStr]>::borrow(self)[index] }
 }
 
 /// A mutable representation of a namespaced identifier of at least one segment.
@@ -117,11 +110,11 @@ where [Tok<String>]: Index<T>
 /// See also [Sym] for the immutable representation, and [VPath] for possibly
 /// empty values
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct VName(Vec<Tok<String>>);
+pub struct VName(Vec<IStr>);
 impl VName {
 	/// Assert that the sequence isn't empty and wrap it in [VName] to represent
 	/// this invariant
-	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
+	pub fn new(items: impl IntoIterator<Item = IStr>) -> Result<Self, EmptyNameError> {
 		let data: Vec<_> = items.into_iter().collect();
 		if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
 	}
@@ -129,27 +122,27 @@ impl VName {
 		name: impl IntoIterator<Item = api::TStr>,
 		i: &Interner,
 	) -> Result<Self, EmptyNameError> {
-		Self::new(join_all(name.into_iter().map(|m| Tok::from_api(m, i))).await)
+		Self::new(join_all(name.into_iter().map(|m| i.es(m))).await)
 	}
 	/// Unwrap the enclosed vector
-	pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
+	pub fn into_vec(self) -> Vec<IStr> { self.0 }
 	/// Get a reference to the enclosed vector
-	pub fn vec(&self) -> &Vec<Tok<String>> { &self.0 }
+	pub fn vec(&self) -> &Vec<IStr> { &self.0 }
 	/// Mutable access to the underlying vector. To ensure correct results, this
 	/// must never be empty.
-	pub fn vec_mut(&mut self) -> &mut Vec<Tok<String>> { &mut self.0 }
+	pub fn vec_mut(&mut self) -> &mut Vec<IStr> { &mut self.0 }
 	/// Intern the name and return a [Sym]
-	pub async fn to_sym(&self, i: &Interner) -> Sym { Sym(i.i(&self.0[..]).await) }
+	pub async fn to_sym(&self, i: &Interner) -> Sym { Sym(i.iv(&self.0[..]).await) }
 	/// If this name has only one segment, return it
-	pub fn as_root(&self) -> Option<Tok<String>> { self.0.iter().exactly_one().ok().cloned() }
+	pub fn as_root(&self) -> Option<IStr> { self.0.iter().exactly_one().ok().cloned() }
 	/// Prepend the segments to this name
 	#[must_use = "This is a pure function"]
-	pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn prefix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(items.into_iter().chain(self.0).collect())
 	}
 	/// Append the segments to this name
 	#[must_use = "This is a pure function"]
-	pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn suffix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Read a `::` separated namespaced name
@@ -160,7 +153,7 @@ impl VName {
 		Self::parse(s, i).await.expect("empty literal !?")
 	}
 	/// Obtain an iterator over the segments of the name
-	pub fn iter(&self) -> impl Iterator<Item = Tok<String>> + '_ { self.0.iter().cloned() }
+	pub fn iter(&self) -> impl Iterator<Item = IStr> + '_ { self.0.iter().cloned() }
 }
 impl fmt::Debug for VName {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
@@ -171,22 +164,22 @@ impl fmt::Display for VName {
 	}
 }
 impl IntoIterator for VName {
-	type Item = Tok<String>;
+	type Item = IStr;
 	type IntoIter = vec::IntoIter<Self::Item>;
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
 impl<T> Index<T> for VName
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[Tok<String>]> for VName {
+impl Borrow<[IStr]> for VName {
-	fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
+	fn borrow(&self) -> &[IStr] { self.0.borrow() }
 }
 impl Deref for VName {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -194,11 +187,9 @@ impl Deref for VName {
 /// empty sequence
 #[derive(Debug, Copy, Clone, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct EmptyNameError;
-impl TryFrom<&[Tok<String>]> for VName {
+impl TryFrom<&[IStr]> for VName {
 	type Error = EmptyNameError;
-	fn try_from(value: &[Tok<String>]) -> Result<Self, Self::Error> {
+	fn try_from(value: &[IStr]) -> Result<Self, Self::Error> { Self::new(value.iter().cloned()) }
-		Self::new(value.iter().cloned())
-	}
 }
 
 /// An interned representation of a namespaced identifier.
@@ -207,37 +198,37 @@ impl TryFrom<&[Tok<String>]> for VName {
 ///
 /// See also [VName]
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct Sym(Tok<Vec<Tok<String>>>);
+pub struct Sym(IVec);
 impl Sym {
 	/// Assert that the sequence isn't empty, intern it and wrap it in a [Sym] to
 	/// represent this invariant
 	pub async fn new(
-		v: impl IntoIterator<Item = Tok<String>>,
+		v: impl IntoIterator<Item = IStr>,
 		i: &Interner,
 	) -> Result<Self, EmptyNameError> {
 		let items = v.into_iter().collect_vec();
-		Self::from_tok(i.i(&items).await)
+		Self::from_tok(i.iv(&items).await)
 	}
 	/// Read a `::` separated namespaced name.
 	pub async fn parse(s: &str, i: &Interner) -> Result<Self, EmptyNameError> {
-		Ok(Sym(i.i(&VName::parse(s, i).await?.into_vec()).await))
+		Ok(Sym(i.iv(&VName::parse(s, i).await?.into_vec()).await))
 	}
 	/// Assert that a token isn't empty, and wrap it in a [Sym]
-	pub fn from_tok(t: Tok<Vec<Tok<String>>>) -> Result<Self, EmptyNameError> {
+	pub fn from_tok(t: IVec) -> Result<Self, EmptyNameError> {
 		if t.is_empty() { Err(EmptyNameError) } else { Ok(Self(t)) }
 	}
 	/// Grab the interner token
-	pub fn tok(&self) -> Tok<Vec<Tok<String>>> { self.0.clone() }
+	pub fn tok(&self) -> IVec { self.0.clone() }
 	/// Get a number unique to this name suitable for arbitrary ordering.
-	pub fn id(&self) -> NonZeroU64 { self.0.to_api().get_id() }
+	pub fn id(&self) -> NonZeroU64 { self.0.to_api().0 }
 	/// Extern the sym for editing
 	pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
-	pub async fn from_api(marker: api::TStrv, i: &Interner) -> Sym {
+	pub async fn from_api(marker: api::TVec, i: &Interner) -> Sym {
-		Self::from_tok(Tok::from_api(marker, i).await).expect("Empty sequence found for serialized Sym")
+		Self::from_tok(i.ev(marker).await).expect("Empty sequence found for serialized Sym")
 	}
-	pub fn to_api(&self) -> api::TStrv { self.tok().to_api() }
+	pub fn to_api(&self) -> api::TVec { self.tok().to_api() }
-	pub async fn push(&self, tok: Tok<String>, i: &Interner) -> Sym {
+	pub async fn suffix(&self, tokv: impl IntoIterator<Item = IStr>, i: &Interner) -> Sym {
-		Self::new(self.0.iter().cloned().chain([tok]), i).await.unwrap()
+		Self::new(self.0.iter().cloned().chain(tokv), i).await.unwrap()
 	}
 }
 impl fmt::Debug for Sym {
@@ -249,17 +240,17 @@ impl fmt::Display for Sym {
 	}
 }
 impl<T> Index<T> for Sym
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[Tok<String>]> for Sym {
+impl Borrow<[IStr]> for Sym {
-	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
+	fn borrow(&self) -> &[IStr] { &self.0[..] }
 }
 impl Deref for Sym {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -267,39 +258,39 @@ impl Deref for Sym {
 /// handled together in datastructures. The names can never be empty
 #[allow(clippy::len_without_is_empty)] // never empty
 pub trait NameLike:
-	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[Tok<String>]>
+	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[IStr]>
 {
 	/// Convert into held slice
-	fn as_slice(&self) -> &[Tok<String>] { Borrow::<[Tok<String>]>::borrow(self) }
+	fn as_slice(&self) -> &[IStr] { Borrow::<[IStr]>::borrow(self) }
 	/// Get iterator over tokens
-	fn iter(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
+	fn segs(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
 	/// Get iterator over string segments
 	fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ {
-		self.as_slice().iter().map(|t| t.as_str())
+		self.as_slice().iter().map(|t| t.as_ref())
 	}
 	/// Fully resolve the name for printing
 	#[must_use]
-	fn to_strv(&self) -> Vec<String> { self.iter().map(|s| s.to_string()).collect() }
+	fn to_strv(&self) -> Vec<String> { self.segs().map(|s| s.to_string()).collect() }
 	/// Format the name as an approximate filename
-	fn as_src_path(&self) -> String { format!("{}.orc", self.iter().join("/")) }
+	fn as_src_path(&self) -> String { format!("{}.orc", self.segs().join("/")) }
 	/// Return the number of segments in the name
-	fn len(&self) -> NonZeroUsize {
+	fn len_nz(&self) -> NonZeroUsize {
-		NonZeroUsize::try_from(self.iter().count()).expect("NameLike never empty")
+		NonZeroUsize::try_from(self.segs().count()).expect("NameLike never empty")
 	}
 	/// Like slice's `split_first` except we know that it always returns Some
-	fn split_first(&self) -> (Tok<String>, &[Tok<String>]) {
+	fn split_first_seg(&self) -> (IStr, &[IStr]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Like slice's `split_last` except we know that it always returns Some
-	fn split_last(&self) -> (Tok<String>, &[Tok<String>]) {
+	fn split_last_seg(&self) -> (IStr, &[IStr]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Get the first element
-	fn first(&self) -> Tok<String> { self.split_first().0 }
+	fn first_seg(&self) -> IStr { self.split_first_seg().0 }
 	/// Get the last element
-	fn last(&self) -> Tok<String> { self.split_last().0 }
+	fn last_seg(&self) -> IStr { self.split_last_seg().0 }
 }
 
 impl NameLike for Sym {}
@@ -314,9 +305,9 @@ impl NameLike for VName {}
 macro_rules! sym {
   ($seg1:tt $( :: $seg:tt)* ; $i:expr) => { async {
 		$crate::name::Sym::from_tok(
-			$i.i(&[
+			$i.iv(&[
-				$i.i(stringify!($seg1)).await
+				$i.is(stringify!($seg1)).await
-				$( , $i.i(stringify!($seg)).await )*
+				$( , $i.is(stringify!($seg)).await )*
 			])
 			.await
 		).unwrap()
@@ -332,8 +323,8 @@ macro_rules! sym {
 macro_rules! vname {
   ($seg1:tt $( :: $seg:tt)* ; $i:expr) => { async {
     $crate::name::VName::new([
-      $i.i(stringify!($seg1)).await
+      $i.is(stringify!($seg1)).await
-      $( , $i.i(stringify!($seg)).await )*
+      $( , $i.is(stringify!($seg)).await )*
     ]).unwrap()
 	} };
 }
@@ -345,8 +336,8 @@ macro_rules! vname {
 macro_rules! vpath {
   ($seg1:tt $( :: $seg:tt)+ ; $i:expr) => { async {
     $crate::name::VPath(vec![
-      $i.i(stringify!($seg1)).await
+      $i.is(stringify!($seg1)).await
-      $( , $i.i(stringify!($seg)).await )+
+      $( , $i.is(stringify!($seg)).await )+
 		])
 	} };
   () => {
@@ -361,35 +352,37 @@ mod test {
 	use test_executors::spin_on;
 
 	use super::{NameLike, Sym, VName};
-	use crate::interner::{Interner, Tok};
+	use crate::interner::test::DummyInterner;
+	use crate::interner::{IStr, Interner};
 	use crate::name::VPath;
+	use crate::testing::AsyncMonitor;
 
 	#[test]
 	fn recur() {
 		spin_on(async {
-			let i = Interner::new_master();
+			let i = DummyInterner::new(AsyncMonitor::default());
 			let myname = vname!(foo::bar; i).await;
-			let _borrowed_slice: &[Tok<String>] = myname.borrow();
+			let _borrowed_slice: &[IStr] = myname.borrow();
-			let _deref_pathslice: &[Tok<String>] = &myname;
+			let _deref_pathslice: &[IStr] = &myname;
-			let _as_slice_out: &[Tok<String>] = myname.as_slice();
+			let _as_slice_out: &[IStr] = myname.as_slice();
 		})
 	}
 
 	#[test]
 	fn literals() {
 		spin_on(async {
-			let i = Interner::new_master();
+			let i = DummyInterner::new(AsyncMonitor::default());
 			assert_eq!(
 				sym!(foo::bar::baz; i).await,
-				Sym::new([i.i("foo").await, i.i("bar").await, i.i("baz").await], &i).await.unwrap()
+				Sym::new([i.is("foo").await, i.is("bar").await, i.is("baz").await], &i).await.unwrap()
 			);
 			assert_eq!(
 				vname!(foo::bar::baz; i).await,
-				VName::new([i.i("foo").await, i.i("bar").await, i.i("baz").await]).unwrap()
+				VName::new([i.is("foo").await, i.is("bar").await, i.is("baz").await]).unwrap()
 			);
 			assert_eq!(
 				vpath!(foo::bar::baz; i).await,
-				VPath::new([i.i("foo").await, i.i("bar").await, i.i("baz").await])
+				VPath::new([i.is("foo").await, i.is("bar").await, i.is("baz").await])
 			);
 		})
 	}

orchid-base/src/number.rs

@@ -3,8 +3,8 @@ use std::ops::Range;
 
 use ordered_float::NotNan;
 
-use crate::error::{OrcErr, mk_err};
+use crate::ctx::Ctx;
-use crate::interner::Interner;
+use crate::error::OrcErr;
 use crate::location::SrcRange;
 use crate::name::Sym;
 
@@ -55,20 +55,20 @@ pub struct NumError {
 	pub kind: NumErrorKind,
 }
 
-pub async fn num_to_err(
+pub fn num_to_errv(
 	NumError { kind, range }: NumError,
 	offset: u32,
 	source: &Sym,
-	i: &Interner,
+	ctx: &Ctx,
 ) -> OrcErr {
-	mk_err(
+	ctx.mk_err(
-		i.i("Failed to parse number").await,
+		"Failed to parse number",
 		match kind {
 			NumErrorKind::NaN => "NaN emerged during parsing",
 			NumErrorKind::InvalidDigit => "non-digit character encountered",
 			NumErrorKind::Overflow => "The number being described is too large or too accurate",
 		},
-		[SrcRange::new(offset + range.start as u32..offset + range.end as u32, source).pos().into()],
+		[SrcRange::new(offset + range.start as u32..offset + range.end as u32, source)],
 	)
 }
 
@@ -79,7 +79,6 @@ pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
 		.or_else(|| string.strip_prefix("0b").map(|s| (2u8, s, 2)))
 		.or_else(|| string.strip_prefix("0o").map(|s| (8u8, s, 2)))
 		.unwrap_or((10u8, string, 0));
-	eprintln!("({radix}, {noprefix}, {pos})");
 	// identity
 	let (base_s, exponent) = match noprefix.split_once('p') {
 		Some((b, e)) => {
@@ -88,15 +87,14 @@ pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
 		},
 		None => (noprefix, 0),
 	};
-	eprintln!("({base_s},{exponent})");
 	match base_s.split_once('.') {
 		None => {
 			let base = int_parse(base_s, radix, pos)?;
-			if let Ok(pos_exp) = u32::try_from(exponent) {
+			if let Ok(pos_exp) = u32::try_from(exponent)
-				if let Some(radical) = u64::from(radix).checked_pow(pos_exp) {
+				&& let Some(radical) = u64::from(radix).checked_pow(pos_exp)
-					let num = base.checked_mul(radical).and_then(|m| m.try_into().ok()).ok_or(overflow_e)?;
+			{
-					return Ok(Numeric::Int(num));
+				let num = base.checked_mul(radical).and_then(|m| m.try_into().ok()).ok_or(overflow_e)?;
-				}
+				return Ok(Numeric::Int(num));
 			}
 			let f = (base as f64) * (radix as f64).powi(exponent);
 			let err = NumError { range: 0..string.len(), kind: NumErrorKind::NaN };

orchid-base/src/parse.rs

@@ -1,33 +1,20 @@
 use std::fmt::{self, Display};
 use std::iter;
 use std::ops::Deref;
+use std::rc::Rc;
 
 use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
 
 use crate::api;
-use crate::error::{OrcRes, Reporter, mk_err, mk_errv};
+use crate::ctx::Ctx;
-use crate::format::fmt;
+use crate::error::{OrcErr, OrcRes, Reporter};
-use crate::interner::{Interner, Tok};
+use crate::format::{FmtCtx, FmtUnit, Format, fmt};
+use crate::interner::{IStr, Interner};
 use crate::location::SrcRange;
 use crate::name::{Sym, VName, VPath};
-use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_range};
+use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_fmt, ttv_range};
-
-pub trait ParseCtx {
-	#[must_use]
-	fn i(&self) -> &Interner;
-	#[must_use]
-	fn reporter(&self) -> &Reporter;
-}
-pub struct ParseCtxImpl<'a> {
-	pub i: &'a Interner,
-	pub r: &'a Reporter,
-}
-impl ParseCtx for ParseCtxImpl<'_> {
-	fn i(&self) -> &Interner { self.i }
-	fn reporter(&self) -> &Reporter { self.r }
-}
 
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
 pub fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
@@ -95,38 +82,30 @@ impl<A: ExprRepr, X: ExtraTok> Deref for Snippet<'_, A, X> {
 	type Target = [TokTree<A, X>];
 	fn deref(&self) -> &Self::Target { self.cur }
 }
-
+impl<A: ExprRepr, X: ExtraTok> Format for Snippet<'_, A, X> {
-/// Remove tokens that aren't meaningful in expression context, such as comments
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-/// or line breaks
+		ttv_fmt(&**self, c).await
-pub fn strip_fluff<A: ExprRepr, X: ExtraTok>(tt: &TokTree<A, X>) -> Option<TokTree<A, X>> {
+	}
-	let tok = match &tt.tok {
-		Token::BR => return None,
-		Token::Comment(_) => return None,
-		Token::LambdaHead(arg) => Token::LambdaHead(arg.iter().filter_map(strip_fluff).collect()),
-		Token::S(p, b) => Token::S(*p, b.iter().filter_map(strip_fluff).collect()),
-		t => t.clone(),
-	};
-	Some(TokTree { tok, sr: tt.sr.clone() })
 }
 
 #[derive(Clone, Debug)]
 pub struct Comment {
-	pub text: Tok<String>,
+	pub text: IStr,
 	pub sr: SrcRange,
 }
 impl Comment {
 	// XXX: which of these four are actually used?
-	pub async fn from_api(c: &api::Comment, src: Sym, i: &Interner) -> Self {
+	pub async fn from_api(c: &api::Comment, src: Sym, cx: &Ctx) -> Self {
-		Self { text: i.ex(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
+		Self { text: cx.i().es(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
 	}
-	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>, i: &Interner) -> Option<Self> {
+	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>, cx: &Ctx) -> Option<Self> {
 		match &tk.tok {
-			Token::Comment(text) => Some(Self { text: i.i(&**text).await, sr: tk.sr.clone() }),
+			Token::Comment(text) => Some(Self { text: cx.i().is(&**text).await, sr: tk.sr.clone() }),
 			_ => None,
 		}
 	}
-	pub fn to_tk<R: ExprRepr, X: ExtraTok>(&self) -> TokTree<R, X> {
+	pub fn to_tk<A: ExprRepr, X: ExtraTok>(&self) -> TokTree<A, X> {
-		TokTree { tok: Token::Comment(self.text.rc().clone()), sr: self.sr.clone() }
+		TokTree { tok: Token::Comment(Rc::new(self.text.to_string())), sr: self.sr.clone() }
 	}
 	pub fn to_api(&self) -> api::Comment {
 		api::Comment { range: self.sr.range(), text: self.text.to_api() }
@@ -138,7 +117,7 @@ impl fmt::Display for Comment {
 }
 
 pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
+	ctx: &Ctx,
 	snip: Snippet<'a, A, X>,
 ) -> Vec<Parsed<'a, Vec<Comment>, A, X>> {
 	let mut items = Vec::new();
@@ -154,7 +133,7 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
 			Some(i) => {
 				let (cmts, tail) = line.split_at(i);
 				let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async {
-					Comment::from_tk(t, ctx.i()).await.expect("All are comments checked above")
+					Comment::from_tk(t, ctx).await.expect("All are comments checked above")
 				}))
 				.await;
 				items.push(Parsed { output: comments, tail });
@@ -165,49 +144,52 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
 }
 
 pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
+	ctx: &Ctx,
 	snip: Snippet<'a, A, X>,
 ) -> ParseRes<'a, &'a TokTree<A, X>, A, X> {
 	match snip.skip_fluff().pop_front() {
 		Some((output, tail)) => Ok(Parsed { output, tail }),
-		None => Err(mk_errv(
+		None => Err(
-			ctx.i().i("Unexpected end").await,
+			ctx.mk_err("Unexpected end", "Line ends abruptly; more tokens were expected", [snip.sr()]),
-			"Line ends abruptly; more tokens were expected",
+		),
-			[snip.sr()],
-		)),
 	}
 }
 
-pub async fn expect_end(
+pub async fn expect_end(ctx: &Ctx, snip: Snippet<'_, impl ExprRepr, impl ExtraTok>) -> OrcRes<()> {
-	ctx: &impl ParseCtx,
-	snip: Snippet<'_, impl ExprRepr, impl ExtraTok>,
-) -> OrcRes<()> {
 	match snip.skip_fluff().get(0) {
-		Some(surplus) => Err(mk_errv(
+		Some(surplus) =>
-			ctx.i().i("Extra code after end of line").await,
+			Err(ctx.mk_err("Extra code after end of line", "Code found after the end of the line", [
-			"Code found after the end of the line",
+				surplus.sr.pos(),
-			[surplus.sr.pos()],
+			])),
-		)),
 		None => Ok(()),
 	}
 }
 
 pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
+	ctx: &Ctx,
 	snip: Snippet<'a, A, X>,
-	tok: Tok<String>,
+	tok: IStr,
 ) -> ParseRes<'a, (), A, X> {
 	let Parsed { output: head, tail } = try_pop_no_fluff(ctx, snip).await?;
 	match &head.tok {
 		Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
-		t => Err(mk_errv(
+		t => Err(ctx.mk_err(
-			ctx.i().i("Expected specific keyword").await,
+			"Expected specific keyword",
-			format!("Expected {tok} but found {:?}", fmt(t, ctx.i()).await),
+			format!("Expected {tok} but found {:?}", fmt(t, &ctx.i()).await),
 			[head.sr()],
 		)),
 	}
 }
 
+pub async fn token_errv<A: ExprRepr, X: ExtraTok>(
+	ctx: &Ctx,
+	tok: &TokTree<A, X>,
+	description: &'static str,
+	message: impl FnOnce(&str) -> String,
+) -> OrcErr {
+	ctx.mk_err(description, message(&fmt(tok, &ctx.i()).await), [tok.sr.pos()])
+}
+
 pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
 	pub output: T,
 	pub tail: Snippet<'a, H, X>,
@@ -216,12 +198,12 @@ pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
 pub type ParseRes<'a, T, H, X> = OrcRes<Parsed<'a, T, H, X>>;
 
 pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
+	ctx: &Ctx,
 	tail: Snippet<'a, A, X>,
 ) -> ParseRes<'a, Vec<Import>, A, X> {
 	let Some((tt, tail)) = tail.skip_fluff().pop_front() else {
-		return Err(mk_errv(
+		return Err(ctx.mk_err(
-			ctx.i().i("Expected token").await,
+			"Expected token",
 			"Expected a name, a parenthesized list of names, or a globstar.",
 			[tail.sr().pos()],
 		));
@@ -230,17 +212,14 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
 	#[allow(clippy::type_complexity)] // it's an internal function
 	pub async fn rec<A: ExprRepr, X: ExtraTok>(
 		tt: &TokTree<A, X>,
-		ctx: &impl ParseCtx,
+		ctx: &Ctx,
-	) -> OrcRes<Vec<(Vec<Tok<String>>, Option<Tok<String>>, SrcRange)>> {
+	) -> OrcRes<Vec<(Vec<IStr>, Option<IStr>, SrcRange)>> {
 		let ttpos = tt.sr.pos();
 		match &tt.tok {
 			Token::NS(ns, body) => {
 				if !ns.starts_with(name_start) {
-					ctx.reporter().report(mk_err(
+					let err = ctx.mk_err("Unexpected name prefix", "Only names can precede ::", [ttpos]);
-						ctx.i().i("Unexpected name prefix").await,
+					ctx.rep().report(err)
-						"Only names can precede ::",
-						[ttpos.into()],
-					))
 				};
 				let out = Box::pin(rec(body, ctx)).await?;
 				Ok(out.into_iter().update(|i| i.0.push(ns.clone())).collect_vec())
@@ -256,16 +235,16 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
 				while let Some((output, tail)) = body.pop_front() {
 					match rec(output, ctx).boxed_local().await {
 						Ok(names) => o.extend(names),
-						Err(e) => ctx.reporter().report(e),
+						Err(e) => ctx.rep().report(e),
 					}
 					body = tail;
 				}
 				Ok(o)
 			},
 			t => {
-				return Err(mk_errv(
+				return Err(ctx.mk_err(
-					ctx.i().i("Unrecognized name end").await,
+					"Unrecognized name end",
-					format!("Names cannot end with {:?} tokens", fmt(t, ctx.i()).await),
+					format!("Names cannot end with {:?} tokens", fmt(t, &ctx.i()).await),
 					[ttpos],
 				));
 			},
@@ -284,7 +263,7 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
 #[derive(Debug, Clone)]
 pub struct Import {
 	pub path: VPath,
-	pub name: Option<Tok<String>>,
+	pub name: Option<IStr>,
 	pub sr: SrcRange,
 }
 impl Import {
@@ -295,10 +274,14 @@ impl Import {
 			None => self.path.into_name().expect("Import cannot be empty"),
 		}
 	}
+	pub fn new(sr: SrcRange, path: VPath, name: IStr) -> Self {
+		Import { path, name: Some(name), sr }
+	}
+	pub fn new_glob(sr: SrcRange, path: VPath) -> Self { Import { path, name: None, sr } }
 }
 impl Display for Import {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| t.as_str()))
+		write!(f, "{}::{}", self.path, self.name.as_deref().unwrap_or("*"))
 	}
 }
 

orchid-base/src/reqnot.rs

@@ -5,14 +5,18 @@ use std::marker::PhantomData;
 use std::mem;
 use std::ops::{BitAnd, Deref};
 use std::pin::Pin;
+use std::rc::Rc;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
 
-use async_std::channel;
-use async_std::sync::Mutex;
 use derive_destructure::destructure;
 use dyn_clone::{DynClone, clone_box};
+use futures::channel::mpsc::{self, Sender};
+use futures::channel::oneshot;
 use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
+use futures::{AsyncBufRead, AsyncWrite, SinkExt, Stream, StreamExt};
 use hashbrown::HashMap;
 use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
 use trait_set::trait_set;
@@ -22,6 +26,71 @@ use crate::logging::Logger;
 
 pub struct Receipt<'a>(PhantomData<&'a mut ()>);
 
+/// This object holds an exclusive lock on the outbound pipe.
+pub trait DynRequestWriter {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
+	/// Release the outbound pipe and wait for the response to begin.
+	fn get_response(self: Box<Self>) -> Pin<Box<dyn Future<Output = Box<dyn DynResponseHandle>>>>;
+}
+/// This object holds an exclusive lock on the inbound pipe.
+pub trait DynResponseHandle {
+	fn reader(&mut self) -> Pin<&mut dyn AsyncBufRead>;
+	fn finish(self: Box<Self>) -> Pin<Box<dyn Future<Output = ()>>>;
+}
+/// This object holds an exclusive lock on the outbound pipe.
+pub trait DynNotifWriter {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
+	fn finish(self: Box<Self>) -> Pin<Box<dyn Future<Output = ()>>>;
+}
+
+pub trait DynClient {
+	fn request(&self) -> Pin<Box<dyn Future<Output = Box<dyn DynRequestWriter>>>>;
+	fn notif(&self) -> Pin<Box<dyn Future<Output = Box<dyn DynNotifWriter>>>>;
+}
+
+pub struct Client<T: MsgSet>(pub(crate) Rc<dyn DynClient>, pub(crate) PhantomData<T>);
+impl<T: MsgSet> Client<T> {
+	pub async fn notify<Notif: Into<<T::Out as Channel>::Notif>>(&self, notif: Notif) {
+		let mut notif_writer = self.0.notif().await;
+		notif.into().encode(notif_writer.writer()).await;
+		notif_writer.finish().await;
+	}
+	pub async fn request<Req: Request + Into<<T::Out as Channel>::Req>>(
+		&self,
+		req: Req,
+	) -> Req::Response {
+		let root_req = req.into();
+		let mut req_writer = self.0.request().await;
+		root_req.encode(req_writer.writer()).await;
+		let mut req_hand = req_writer.get_response().await;
+		let res = Req::Response::decode(req_hand.reader()).await;
+		req_hand.finish().await;
+		res
+	}
+}
+
+pub struct DuplexServerState {
+	pending_outbound: HashMap<u64, Box<dyn FnOnce(&mut dyn AsyncBufRead)>>,
+	sender: Pin<Box<dyn AsyncWrite>>,
+	receiver: Pin<Box<dyn AsyncBufRead>>,
+}
+pub enum ServerEvent<T: MsgSet> {
+	Notif(<T::In as Channel>::Notif),
+	Req(RequestHandle<T>, <T::In as Channel>::Req),
+}
+pub async fn run_duplex_server<T: MsgSet>(
+	sender: Pin<Box<dyn AsyncWrite>>,
+	receiver: Pin<Box<dyn AsyncBufRead>>,
+) -> (impl Stream<Item = ServerEvent<T>>, Client<T>) {
+	let sender = Rc::new(Mutex::new(sender));
+	let receiver = Rc::new(Mutex::new(receiver));
+	let pending_outbound = Rc::new(Mutex::new(HashMap::new()));
+}
+pub struct DuplexServer(Rc<Mutex<DuplexServerState>>);
+impl DuplexServer {
+	pub fn receive(msg: )
+}
+
 trait_set! {
 	pub trait SendFn<T: MsgSet> =
 		for<'a> FnMut(&'a [u8], ReqNot<T>) -> LocalBoxFuture<'a, ()>
@@ -51,11 +120,10 @@ impl ReqHandlish for &'_ dyn ReqHandlish {
 }
 
 #[derive(destructure)]
-pub struct RequestHandle<'a, MS: MsgSet> {
+pub struct RequestHandle<MS: MsgSet> {
 	defer_drop: RefCell<Vec<Box<dyn Any>>>,
 	fulfilled: AtomicBool,
 	id: u64,
-	_reqlt: PhantomData<&'a mut ()>,
 	parent: ReqNot<MS>,
 }
 impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
@@ -88,7 +156,7 @@ impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
 impl<MS: MsgSet> ReqHandlish for RequestHandle<'_, MS> {
 	fn defer_drop_objsafe(&self, val: Box<dyn Any>) { self.defer_drop.borrow_mut().push(val); }
 }
-impl<MS: MsgSet> Drop for RequestHandle<'_, MS> {
+impl<MS: MsgSet> Drop for RequestHandle<MS> {
 	fn drop(&mut self) {
 		let done = self.fulfilled.load(Ordering::Relaxed);
 		debug_assert!(done, "Request {} dropped without response", self.id)
@@ -102,7 +170,7 @@ pub struct ReqNotData<T: MsgSet> {
 	send: Box<dyn SendFn<T>>,
 	notif: Box<dyn NotifFn<T>>,
 	req: Box<dyn ReqFn<T>>,
-	responses: HashMap<u64, channel::Sender<Vec<u8>>>,
+	responses: HashMap<u64, mpsc::Sender<Vec<u8>>>,
 }
 
 /// Wraps a raw message buffer to save on copying.
@@ -122,7 +190,7 @@ impl<T: MsgSet> ReqNot<T> {
 		notif: impl NotifFn<T>,
 		req: impl ReqFn<T>,
 	) -> Self {
-		Self(
+		let this = Self(
 			Arc::new(Mutex::new(ReqNotData {
 				id: 1,
 				send: Box::new(send),
@@ -131,7 +199,13 @@ impl<T: MsgSet> ReqNot<T> {
 				responses: HashMap::new(),
 			})),
 			logger,
-		)
+		);
+		let (sig_send, sig_recv) = std::sync::mpsc::sync_channel(0);
+		std::thread::spawn(move || {
+			std::thread::sleep(Duration::from_secs(10));
+			sig_send.send(()).expect("Crash!");
+		});
+		this
 	}
 
 	/// Can be called from a polling thread or dispatched in any other way
@@ -144,7 +218,7 @@ impl<T: MsgSet> ReqNot<T> {
 			let notif_val = <T::In as Channel>::Notif::decode(Pin::new(&mut &payload[..])).await;
 			notif_cb(notif_val, self.clone()).await
 		} else if 0 < id.bitand(1 << 63) {
-			let sender = g.responses.remove(&!id).expect("Received response for invalid message");
+			let mut sender = g.responses.remove(&!id).expect("Received response for invalid message");
 			sender.send(message.to_vec()).await.unwrap()
 		} else {
 			let message = <T::In as Channel>::Req::decode(Pin::new(&mut &payload[..])).await;
@@ -205,13 +279,13 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
 			g.id += 1;
 			let mut buf = id.to_be_bytes().to_vec();
 			req.encode(Pin::new(&mut buf)).await;
-			let (send, recv) = channel::bounded(1);
+			let (send, mut recv) = mpsc::channel(1);
 			g.responses.insert(id, send);
 			let mut send = clone_box(&*g.send);
 			mem::drop(g);
 			let rn = self.clone();
 			send(&buf, rn).await;
-			let items = recv.recv().await;
+			let items = recv.next().await;
 			RawReply(items.unwrap())
 		})
 	}
@@ -249,17 +323,16 @@ mod test {
 	use std::rc::Rc;
 	use std::sync::Arc;
 
-	use async_std::sync::Mutex;
 	use futures::FutureExt;
-	use orchid_api::LogStrategy;
+	use futures::lock::Mutex;
 	use orchid_api_derive::Coding;
 	use orchid_api_traits::{Channel, Request};
 	use test_executors::spin_on;
 
 	use super::{MsgSet, ReqNot};
-	use crate::clone;
 	use crate::logging::Logger;
 	use crate::reqnot::Requester as _;
+	use crate::{api, clone};
 
 	#[derive(Clone, Debug, Coding, PartialEq)]
 	pub struct TestReq(u8);
@@ -282,7 +355,7 @@ mod test {
 	#[test]
 	fn notification() {
 		spin_on(async {
-			let logger = Logger::new(LogStrategy::StdErr);
+			let logger = Logger::new(api::LogStrategy::StdErr);
 			let received = Arc::new(Mutex::new(None));
 			let receiver = ReqNot::<TestMsgSet>::new(
 				logger.clone(),
@@ -310,7 +383,7 @@ mod test {
 	#[test]
 	fn request() {
 		spin_on(async {
-			let logger = Logger::new(LogStrategy::StdErr);
+			let logger = Logger::new(api::LogStrategy::StdErr);
 			let receiver = Rc::new(Mutex::<Option<ReqNot<TestMsgSet>>>::new(None));
 			let sender = Rc::new(ReqNot::<TestMsgSet>::new(
 				logger.clone(),

orchid-base/src/testing.rs

@@ -0,0 +1,23 @@
+#![cfg(any(feature = "mocks", test))]
+
+use std::future::ready;
+use std::pin::Pin;
+use std::rc::Rc;
+
+pub struct AsyncMonitor<E: 'static>(Rc<dyn Fn(E) -> Pin<Box<dyn Future<Output = ()>>>>);
+impl<E: 'static> AsyncMonitor<E> {
+	pub fn new<F: AsyncFn(E) -> () + 'static>(f: F) -> Self {
+		let f_rc = Rc::new(f);
+		AsyncMonitor(Rc::new(move |e| {
+			let f_rc = f_rc.clone();
+			Box::pin(async move { f_rc(e).await })
+		}))
+	}
+	pub async fn notify(&self, e: E) -> () { (self.0)(e).await }
+}
+impl<E: 'static> Default for AsyncMonitor<E> {
+	fn default() -> Self { Self(Rc::new(|_| Box::pin(ready(())))) }
+}
+impl<E: 'static> Clone for AsyncMonitor<E> {
+	fn clone(&self) -> Self { Self(self.0.clone()) }
+}

orchid-base/src/tree.rs

@@ -4,7 +4,7 @@ use std::future::Future;
 use std::marker::PhantomData;
 use std::rc::Rc;
 
-use async_stream::stream;
+use async_fn_stream::stream;
 use futures::future::join_all;
 use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
@@ -12,11 +12,11 @@ use never::Never;
 use orchid_api_traits::Coding;
 use trait_set::trait_set;
 
-use crate::error::OrcErrv;
+use crate::error::OwnedOrcErr;
 use crate::format::{FmtCtx, FmtUnit, Format, Variants};
-use crate::interner::{Interner, Tok};
+use crate::interner::{IStr, Interner};
 use crate::location::{Pos, SrcRange};
-use crate::name::Sym;
+use crate::name::{Sym, VName, VPath};
 use crate::parse::Snippet;
 use crate::{api, match_mapping, tl_cache};
 
@@ -62,8 +62,7 @@ pub fn recur<H: ExprRepr, X: ExtraTok>(
 			tok @ (Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::Name(_)) => tok,
 			tok @ (Token::Handle(_) | Token::NewExpr(_)) => tok,
 			Token::NS(n, b) => Token::NS(n, Box::new(recur(*b, f))),
-			Token::LambdaHead(arg) =>
+			Token::LambdaHead(arg) => Token::LambdaHead(Box::new(recur(*arg, f))),
-				Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
 			Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
 		};
 		TokTree { sr: range, tok }
@@ -114,11 +113,11 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 		let pos = SrcRange::new(tt.range.clone(), src);
 		let tok = match_mapping!(&tt.token, api::Token => Token::<H, X> {
 			BR,
-			NS(n => Tok::from_api(*n, i).await,
+			NS(n => i.es(*n).await,
 				b => Box::new(Self::from_api(b, hctx, xctx, src, i).boxed_local().await)),
-			Bottom(e => OrcErrv::from_api(e, i).await),
+			Bottom(e => OwnedOrcErr::from_api(e, i).await),
-			LambdaHead(arg => ttv_from_api(arg, hctx, xctx, src, i).await),
+			LambdaHead(arg => Box::new(Self::from_api(arg, hctx, xctx, src, i).boxed_local().await)),
-			Name(n => Tok::from_api(*n, i).await),
+			Name(n => i.es(*n).await),
 			S(*par, b => ttv_from_api(b, hctx, xctx, src, i).await),
 			Comment(c.clone()),
 			NewExpr(expr => X::from_api(expr, xctx, pos.clone(), i).await),
@@ -137,7 +136,7 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 			NS(n.to_api(), b => Box::new(b.into_api(hctx, xctx).boxed_local().await)),
 			Bottom(e.to_api()),
 			Comment(c.clone()),
-			LambdaHead(arg => ttv_into_api(arg, hctx, xctx).boxed_local().await),
+			LambdaHead(arg => Box::new(arg.into_api(hctx, xctx).boxed_local().await)),
 			Name(nn.to_api()),
 			S(p, b => ttv_into_api(b, hctx, xctx).boxed_local().await),
 			Handle(hand.into_api(hctx).await),
@@ -146,25 +145,39 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 		api::TokenTree { range: self.sr.range.clone(), token }
 	}
 
-	pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) }
+	pub fn is_kw(&self, tk: IStr) -> bool { self.tok.is_kw(tk) }
-	pub fn as_name(&self) -> Option<Tok<String>> {
+	pub fn as_name(&self) -> Option<IStr> {
 		if let Token::Name(n) = &self.tok { Some(n.clone()) } else { None }
 	}
+	pub fn as_multiname(&self) -> Result<VName, &TokTree<H, X>> {
+		let mut segs = VPath::new([]);
+		let mut cur = self;
+		loop {
+			match &cur.tok {
+				Token::Name(last) => return Ok(segs.name_with_suffix(last.clone())),
+				Token::NS(seg, inner) => {
+					segs = segs.suffix([seg.clone()]);
+					cur = inner;
+				},
+				_ => return Err(cur),
+			}
+		}
+	}
 	pub fn as_s(&self, par: Paren) -> Option<Snippet<'_, H, X>> {
 		self.tok.as_s(par).map(|slc| Snippet::new(self, slc))
 	}
-	pub fn as_lambda(&self) -> Option<Snippet<'_, H, X>> {
+	pub fn as_lambda(&self) -> Option<&Self> {
 		match &self.tok {
-			Token::LambdaHead(arg) => Some(Snippet::new(self, arg)),
+			Token::LambdaHead(arg) => Some(&**arg),
 			_ => None,
 		}
 	}
 	pub fn is_fluff(&self) -> bool { matches!(self.tok, Token::Comment(_) | Token::BR) }
-	pub fn lambda(arg: Vec<Self>, mut body: Vec<Self>) -> Self {
+	pub fn lambda(arg: Self, mut body: Vec<Self>) -> Self {
-		let arg_range = ttv_range(&arg).expect("Lambda with empty arg!");
+		let arg_range = arg.sr();
 		let mut s_range = arg_range.clone();
 		s_range.range.end = body.last().expect("Lambda with empty body!").sr.range.end;
-		body.insert(0, Token::LambdaHead(arg).at(arg_range));
+		body.insert(0, Token::LambdaHead(Box::new(arg)).at(arg_range));
 		Token::S(Paren::Round, body).at(s_range)
 	}
 	pub fn sr(&self) -> SrcRange { self.sr.clone() }
@@ -182,11 +195,11 @@ pub async fn ttv_from_api<H: ExprRepr, X: ExtraTok>(
 	src: &Sym,
 	i: &Interner,
 ) -> Vec<TokTree<H, X>> {
-	stream! {
+	stream(async |mut cx| {
 		for tok in tokv {
-			yield TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src, i).boxed_local().await
+			cx.emit(TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src, i).boxed_local().await).await
 		}
-	}
+	})
 	.collect()
 	.await
 }
@@ -196,11 +209,11 @@ pub async fn ttv_into_api<H: ExprRepr, X: ExtraTok>(
 	hctx: &mut H::ToApiCtx<'_>,
 	xctx: &mut X::ToApiCtx<'_>,
 ) -> Vec<api::TokenTree> {
-	stream! {
+	stream(async |mut cx| {
 		for tok in tokv {
-			yield tok.into_api(hctx, xctx).await
+			cx.emit(tok.into_api(hctx, xctx).await).await
 		}
-	}
+	})
 	.collect()
 	.await
 }
@@ -230,11 +243,11 @@ pub enum Token<H: ExprRepr, X: ExtraTok> {
 	Comment(Rc<String>),
 	/// The part of a lambda between `\` and `.` enclosing the argument. The body
 	/// stretches to the end of the enclosing parens or the end of the const line
-	LambdaHead(Vec<TokTree<H, X>>),
+	LambdaHead(Box<TokTree<H, X>>),
 	/// A binding, operator, or a segment of a namespaced::name
-	Name(Tok<String>),
+	Name(IStr),
 	/// A namespace prefix, like `my_ns::` followed by a token
-	NS(Tok<String>, Box<TokTree<H, X>>),
+	NS(IStr, Box<TokTree<H, X>>),
 	/// A line break
 	BR,
 	/// `()`, `[]`, or `{}`
@@ -246,11 +259,11 @@ pub enum Token<H: ExprRepr, X: ExtraTok> {
 	/// A grammar error emitted by a lexer plugin if it was possible to continue
 	/// reading. Parsers should treat it as an atom unless it prevents parsing,
 	/// in which case both this and a relevant error should be returned.
-	Bottom(OrcErrv),
+	Bottom(OwnedOrcErr),
 }
 impl<H: ExprRepr, X: ExtraTok> Token<H, X> {
 	pub fn at(self, sr: SrcRange) -> TokTree<H, X> { TokTree { sr, tok: self } }
-	pub fn is_kw(&self, tk: Tok<String>) -> bool { matches!(self, Token::Name(n) if *n == tk) }
+	pub fn is_kw(&self, tk: IStr) -> bool { matches!(self, Token::Name(n) if *n == tk) }
 	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> {
 		match self {
 			Self::S(p, b) if *p == par => Some(b),
@@ -262,12 +275,11 @@ impl<H: ExprRepr, X: ExtraTok> Format for Token<H, X> {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		match self {
 			Self::BR => "\n".to_string().into(),
-			Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()).into(),
+			Self::Bottom(err) => format!("Botttom({}) ", indent(&err.to_string())).into(),
-			Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),
 			Self::Comment(c) => format!("--[{c}]--").into(),
 			Self::LambdaHead(arg) =>
 				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0b}.")))
-					.units([ttv_fmt(arg, c).await]),
+					.units([arg.print(c).boxed_local().await]),
 			Self::NS(n, b) => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}::{1l}")))
 				.units([n.to_string().into(), b.print(c).boxed_local().await]),
 			Self::Name(n) => format!("{n}").into(),

orchid-base/src/virt_fs/common.rs

@@ -13,13 +13,13 @@ pub enum Loaded {
   Code(Arc<String>),
   /// Conceptually equivalent to the list of *.orc files in a folder, without
   /// the extension
-  Collection(Arc<Vec<Tok<String>>>),
+  Collection(Arc<Vec<IStr>>),
 }
 impl Loaded {
   /// Is the loaded item source code (not a collection)?
   pub fn is_code(&self) -> bool { matches!(self, Loaded::Code(_)) }
   /// Collect the elements in a collection rreport
-  pub fn collection(items: impl IntoIterator<Item = Tok<String>>) -> Self {
+  pub fn collection(items: impl IntoIterator<Item = IStr>) -> Self {
     Self::Collection(Arc::new(items.into_iter().collect()))
   }
 }
@@ -55,7 +55,7 @@ impl ErrorSansOrigin for CodeNotFound {
 /// formats and other sources for libraries and dependencies.
 pub trait VirtFS {
   /// Implementation of [VirtFS::read]
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult;
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult;
   /// Discover information about a path without reading it.
   ///
   /// Implement this if your vfs backend can do expensive operations
@@ -68,7 +68,7 @@ pub trait VirtFS {
   }
   /// Convert a path into a human-readable string that is meaningful in the
   /// target context.
-  fn display(&self, path: &[Tok<String>]) -> Option<String>;
+  fn display(&self, path: &[IStr]) -> Option<String>;
   /// Convert the FS handler into a type-erased version of itself for packing in
   /// a tree.
   fn rc(self) -> Rc<dyn VirtFS>
@@ -81,15 +81,15 @@ pub trait VirtFS {
 }
 
 impl VirtFS for &dyn VirtFS {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (*self).get(path, full_path)
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> { (*self).display(path) }
+  fn display(&self, path: &[IStr]) -> Option<String> { (*self).display(path) }
 }
 
 impl<T: VirtFS + ?Sized> VirtFS for Rc<T> {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (**self).get(path, full_path)
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> { (**self).display(path) }
+  fn display(&self, path: &[IStr]) -> Option<String> { (**self).display(path) }
 }

orchid-base/src/virt_fs/decl.rs

@@ -32,7 +32,7 @@ impl<'a> Combine for &'a dyn VirtFS {
 pub type DeclTree = ModEntry<Rc<dyn VirtFS>, (), ()>;
 
 impl VirtFS for DeclTree {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     match &self.member {
       ModMember::Item(it) => it.get(path, full_path),
       ModMember::Sub(module) => match path.split_first() {
@@ -44,7 +44,7 @@ impl VirtFS for DeclTree {
     }
   }
 
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let (head, tail) = path.split_first()?;
     match &self.member {
       ModMember::Item(it) => it.display(path),
@@ -54,16 +54,16 @@ impl VirtFS for DeclTree {
 }
 
 impl VirtFS for String {
-  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
+  fn display(&self, _: &[IStr]) -> Option<String> { None }
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.as_str().to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }
 }
 
 impl<'a> VirtFS for &'a str {
-  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
+  fn display(&self, _: &[IStr]) -> Option<String> { None }
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }

orchid-base/src/virt_fs/dir.rs

@@ -99,14 +99,14 @@ impl DirNode {
     }
   }
 
-  fn mk_pathbuf(&self, path: &[Tok<String>]) -> PathBuf {
+  fn mk_pathbuf(&self, path: &[IStr]) -> PathBuf {
     let mut fpath = self.root.clone();
     path.iter().for_each(|seg| fpath.push(seg.as_str()));
     fpath
   }
 }
 impl VirtFS for DirNode {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     let fpath = self.mk_pathbuf(path);
     let mut binding = self.cached.borrow_mut();
     let (_, res) = (binding.raw_entry_mut().from_key(&fpath))
@@ -114,7 +114,7 @@ impl VirtFS for DirNode {
     res.clone()
   }
 
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let pathbuf = self.mk_pathbuf(path).with_extension(self.ext());
     Some(pathbuf.to_string_lossy().to_string())
   }

orchid-base/src/virt_fs/embed.rs

@@ -56,7 +56,7 @@ impl EmbeddedFS {
 }
 
 impl VirtFS for EmbeddedFS {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     if path.is_empty() {
       return Ok(Loaded::collection(self.tree.keys(|_| true)));
     }
@@ -67,7 +67,7 @@ impl VirtFS for EmbeddedFS {
       ModMember::Sub(sub) => Loaded::collection(sub.keys(|_| true)),
     })
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let Self { gen, suffix, .. } = self;
     Some(format!("{}{suffix} in {gen}", path.iter().join("/")))
   }

orchid-base/src/virt_fs/prefix.rs

@@ -21,18 +21,18 @@ impl<'a> PrefixFS<'a> {
       add: VPath::parse(add.as_ref()),
     }
   }
-  fn proc_path(&self, path: &[Tok<String>]) -> Option<Vec<Tok<String>>> {
+  fn proc_path(&self, path: &[IStr]) -> Option<Vec<IStr>> {
     let path = path.strip_prefix(self.remove.as_slice())?;
     Some(self.add.0.iter().chain(path).cloned().collect_vec())
   }
 }
 impl<'a> VirtFS for PrefixFS<'a> {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> super::FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> super::FSResult {
     let path =
       self.proc_path(path).ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())?;
     self.wrapped.get(&path, full_path)
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     self.wrapped.display(&self.proc_path(path)?)
   }
 }

orchid-extension/Cargo.toml

@@ -6,28 +6,36 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ahash = "0.8.11"
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
+async-lock = "3.4.1"
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
-async-stream = "0.3.6"
 derive_destructure = "1.0.0"
-dyn-clone = "1.0.17"
+dyn-clone = "1.0.20"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = [
-hashbrown = "0.15.2"
+  "std",
+  "async-await",
+], default-features = false }
+hashbrown = "0.16.0"
 include_dir = { version = "0.7.4", optional = true }
 itertools = "0.14.0"
-konst = "0.3.16"
+konst = "0.4.1"
 lazy_static = "1.5.0"
 memo-map = "0.3.3"
 never = "0.1.0"
-once_cell = "1.20.2"
+once_cell = "1.21.3"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 ordered-float = "5.0.0"
-pastey = "0.1.0"
+pastey = "0.1.1"
-some_executor = "0.5.1"
+some_executor = "0.6.1"
 substack = "1.1.1"
-tokio = { version = "1.46.1", optional = true }
+tokio = { version = "1.47.1", optional = true, features = [] }
+tokio-util = { version = "0.7.16", optional = true, features = ["compat"] }
+
 trait-set = "0.3.0"
+
+[features]
+tokio = ["dep:tokio", "dep:tokio-util"]
+default = ["tokio"]

orchid-extension/src/atom.rs

@@ -1,4 +1,5 @@
 use std::any::{Any, TypeId, type_name};
+use std::collections::HashMap;
 use std::fmt;
 use std::future::Future;
 use std::num::NonZeroU32;
@@ -6,16 +7,14 @@ use std::ops::Deref;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use ahash::HashMap;
-use async_std::io::{Read, Write};
-use async_std::stream;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::LocalBoxFuture;
-use futures::{FutureExt, StreamExt};
+use futures::{AsyncRead, AsyncWrite, FutureExt, StreamExt, stream};
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Coding, Decode, Encode, Request, enc_vec};
 use orchid_base::clone;
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use orchid_base::ctx::Ctx;
+use orchid_base::error::{OrcErr, OrcRes};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
@@ -24,6 +23,7 @@ use orchid_base::reqnot::Requester;
 use trait_set::trait_set;
 
 use crate::api;
+use crate::conv::ToExpr;
 // use crate::error::{ProjectError, ProjectResult};
 use crate::expr::{Expr, ExprData, ExprHandle, ExprKind};
 use crate::gen_expr::GExpr;
@@ -92,7 +92,7 @@ pub struct ForeignAtom {
 }
 impl ForeignAtom {
 	pub fn pos(&self) -> Pos { self.pos.clone() }
-	pub fn ctx(&self) -> SysCtx { self.expr.ctx.clone() }
+	pub fn ctx(&self) -> &SysCtx { &self.expr.ctx }
 	pub fn ex(self) -> Expr {
 		let (handle, pos) = (self.expr.clone(), self.pos.clone());
 		let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { ..self }) };
@@ -110,6 +110,9 @@ impl ForeignAtom {
 		.await?;
 		Some(M::Response::decode(Pin::new(&mut &rep[..])).await)
 	}
+	pub async fn downcast<T: AtomicFeatures>(self) -> Result<TypAtom<T>, NotTypAtom> {
+		TypAtom::downcast(self.ex().handle()).await
+	}
 }
 impl fmt::Display for ForeignAtom {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Atom::{:?}", self.atom) }
@@ -122,6 +125,9 @@ impl Format for ForeignAtom {
 		FmtUnit::from_api(&self.ctx().reqnot().request(api::ExtAtomPrint(self.atom.clone())).await)
 	}
 }
+impl ToExpr for ForeignAtom {
+	async fn to_expr(self) -> GExpr { self.ex().to_expr().await }
+}
 
 pub struct NotTypAtom {
 	pub pos: Pos,
@@ -130,12 +136,10 @@ pub struct NotTypAtom {
 	pub ctx: SysCtx,
 }
 impl NotTypAtom {
-	pub async fn mk_err(&self) -> OrcErr {
+	pub fn mk_err(&self, ctx: &Ctx) -> OrcErr {
-		mk_err(
+		ctx.mk_err("Not the expected type", format!("This expression is not a {}", self.typ.name()), [
-			self.ctx.i().i("Not the expected type").await,
+			self.pos.clone(),
-			format!("This expression is not a {}", self.typ.name()),
+		])
-			[self.pos.clone().into()],
-		)
 	}
 }
 
@@ -150,8 +154,8 @@ trait_set! {
 	trait AtomReqCb<A> = for<'a> Fn(
 		&'a A,
 		SysCtx,
-		Pin<&'a mut dyn Read>,
+		Pin<&'a mut dyn AsyncRead>,
-		Pin<&'a mut dyn Write>,
+		Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, ()>
 }
 
@@ -166,17 +170,19 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 		assert!(!M::NAME.is_empty(), "AtomMethod::NAME cannoot be empty");
 		self.handlers.push((
 			M::NAME,
-			Rc::new(move |a: &A, ctx: SysCtx, req: Pin<&mut dyn Read>, rep: Pin<&mut dyn Write>| {
+			Rc::new(
-				async { Supports::<M>::handle(a, ctx, M::decode(req).await).await.encode(rep).await }
+				move |a: &A, ctx: SysCtx, req: Pin<&mut dyn AsyncRead>, rep: Pin<&mut dyn AsyncWrite>| {
-					.boxed_local()
+					async { Supports::<M>::handle(a, ctx, M::decode(req).await).await.encode(rep).await }
-			}),
+						.boxed_local()
+				},
+			),
 		));
 		self
 	}
 
 	pub async fn pack(&self, ctx: SysCtx) -> MethodSet<A> {
 		MethodSet {
-			handlers: stream::from_iter(self.handlers.iter())
+			handlers: stream::iter(self.handlers.iter())
 				.then(|(k, v)| {
 					clone!(ctx; async move {
 						(Sym::parse(k, ctx.i()).await.unwrap(), v.clone())
@@ -197,8 +203,8 @@ impl<A: AtomCard> MethodSet<A> {
 		atom: &'a A,
 		ctx: SysCtx,
 		key: Sym,
-		req: Pin<&'a mut dyn Read>,
+		req: Pin<&'a mut dyn AsyncRead>,
-		rep: Pin<&'a mut dyn Write>,
+		rep: Pin<&'a mut dyn AsyncWrite>,
 	) -> bool {
 		match self.handlers.get(&key) {
 			None => false,
@@ -216,10 +222,12 @@ impl<A: AtomCard> Default for MethodSetBuilder<A> {
 
 #[derive(Clone)]
 pub struct TypAtom<A: AtomicFeatures> {
-	pub data: ForeignAtom,
+	pub untyped: ForeignAtom,
 	pub value: A::Data,
 }
 impl<A: AtomicFeatures> TypAtom<A> {
+	pub fn ctx(&self) -> &SysCtx { self.untyped.ctx() }
+	pub fn i(&self) -> &Interner { self.ctx().i() }
 	pub async fn downcast(expr: Rc<ExprHandle>) -> Result<Self, NotTypAtom> {
 		match Expr::from_handle(expr).atom().await {
 			Err(expr) => Err(NotTypAtom {
@@ -242,9 +250,9 @@ impl<A: AtomicFeatures> TypAtom<A> {
 	pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
 	where A: Supports<M> {
 		M::Response::decode(Pin::new(
-			&mut &(self.data.ctx().reqnot().request(api::Fwd(
+			&mut &(self.untyped.ctx().reqnot().request(api::Fwd(
-				self.data.atom.clone(),
+				self.untyped.atom.clone(),
-				Sym::parse(M::NAME, self.data.ctx().i()).await.unwrap().tok().to_api(),
+				Sym::parse(M::NAME, self.untyped.ctx().i()).await.unwrap().tok().to_api(),
 				enc_vec(&req).await,
 			)))
 			.await
@@ -257,6 +265,9 @@ impl<A: AtomicFeatures> Deref for TypAtom<A> {
 	type Target = A::Data;
 	fn deref(&self) -> &Self::Target { &self.value }
 }
+impl<A: AtomicFeatures> ToExpr for TypAtom<A> {
+	async fn to_expr(self) -> GExpr { self.untyped.to_expr().await }
+}
 
 pub struct AtomCtx<'a>(pub &'a [u8], pub Option<api::AtomId>, pub SysCtx);
 impl FmtCtx for AtomCtx<'_> {
@@ -274,14 +285,14 @@ pub trait AtomDynfo: 'static {
 		&'a self,
 		ctx: AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'b mut dyn Read>,
+		req: Pin<&'b mut dyn AsyncRead>,
-		rep: Pin<&'c mut dyn Write>,
+		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool>;
 	fn command<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>>;
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
-		write: Pin<&'b mut dyn Write>,
+		write: Pin<&'b mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>>;
 	fn deserialize<'a>(
 		&'a self,
@@ -317,10 +328,10 @@ impl Format for AtomFactory {
 	}
 }
 
-pub async fn err_not_callable(i: &Interner) -> OrcErr {
+pub fn err_not_callable(cx: &Ctx) -> OrcErr {
-	mk_err(i.i("This atom is not callable").await, "Attempted to apply value as function", [])
+	cx.mk_err_floating("This atom is not callable", "Attempted to apply value as function")
 }
 
-pub async fn err_not_command(i: &Interner) -> OrcErr {
+pub fn err_not_command(cx: &Ctx) -> OrcErr {
-	mk_err(i.i("This atom is not a command").await, "Settled on an inactionable value", [])
+	cx.mk_err_floating("This atom is not a command", "Settled on an inactionable value")
 }

orchid-extension/src/atom_owned.rs

@@ -6,15 +6,13 @@ use std::ops::Deref;
 use std::pin::Pin;
 use std::sync::atomic::AtomicU64;
 
+use async_lock::{RwLock, RwLockReadGuard};
 use async_once_cell::OnceCell;
-use async_std::io::{Read, Write};
-use async_std::sync::{RwLock, RwLockReadGuard};
-use futures::FutureExt;
 use futures::future::{LocalBoxFuture, ready};
+use futures::{AsyncRead, AsyncWrite, FutureExt};
 use itertools::Itertools;
 use memo_map::MemoMap;
 use never::Never;
-use orchid_api::AtomId;
 use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit};
@@ -23,7 +21,7 @@ use orchid_base::name::Sym;
 use crate::api;
 use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
-	MethodSetBuilder, err_not_callable, err_not_command, get_info,
+	MethodSetBuilder, TypAtom, err_not_callable, err_not_command, get_info,
 };
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
@@ -42,7 +40,6 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
 			let mut data = enc_vec(&typ_id).await;
 			self.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
 			ctx.get_or_default::<ObjStore>().objects.read().await.insert(atom_id, Box::new(self));
-			eprintln!("Created atom {:?} of type {}", atom_id, type_name::<A>());
 			api::Atom { drop: Some(atom_id), data, owner: ctx.sys_id() }
 		})
 	}
@@ -53,7 +50,7 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
 /// While an atom read guard is held, no atom can be removed.
 pub(crate) struct AtomReadGuard<'a> {
 	id: api::AtomId,
-	guard: RwLockReadGuard<'a, MemoMap<AtomId, Box<dyn DynOwnedAtom>>>,
+	guard: RwLockReadGuard<'a, MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
 }
 impl<'a> AtomReadGuard<'a> {
 	async fn new(id: api::AtomId, ctx: &'a SysCtx) -> Self {
@@ -68,6 +65,7 @@ impl Deref for AtomReadGuard<'_> {
 	fn deref(&self) -> &Self::Target { &**self.guard.get(&self.id).unwrap() }
 }
 
+/// Remove an atom from the store
 pub(crate) async fn take_atom(id: api::AtomId, ctx: &SysCtx) -> Box<dyn DynOwnedAtom> {
 	let mut g = ctx.get_or_default::<ObjStore>().objects.write().await;
 	g.remove(&id).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0))
@@ -104,8 +102,8 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx,
 		key: Sym,
-		req: Pin<&'b mut dyn Read>,
+		req: Pin<&'b mut dyn AsyncRead>,
-		rep: Pin<&'c mut dyn Write>,
+		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let a = AtomReadGuard::new(id.unwrap(), &ctx).await;
@@ -125,7 +123,7 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
-		mut write: Pin<&'b mut dyn Write>,
+		mut write: Pin<&'b mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		Box::pin(async move {
 			let id = id.unwrap();
@@ -216,7 +214,7 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	fn val(&self) -> impl Future<Output = Cow<'_, Self::Data>>;
 	#[allow(unused_variables)]
 	fn call_ref(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot([err_not_callable(arg.ctx().i()).await]) }
+		async move { bot(err_not_callable(arg.ctx().i()).await) }
 	}
 	fn call(self, arg: Expr) -> impl Future<Output = GExpr> {
 		async {
@@ -228,19 +226,19 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	}
 	#[allow(unused_variables)]
 	fn command(self, ctx: SysCtx) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command(ctx.i()).await.into()) }
+		async move { Err(err_not_command(ctx.i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn free(self, ctx: SysCtx) -> impl Future<Output = ()> { async {} }
 	#[allow(unused_variables)]
-	fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> {
+	fn print_atom<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> {
 		async { format!("OwnedAtom({})", type_name::<Self>()).into() }
 	}
 	#[allow(unused_variables)]
 	fn serialize(
 		&self,
 		ctx: SysCtx,
-		write: Pin<&mut (impl Write + ?Sized)>,
+		write: Pin<&mut (impl AsyncWrite + ?Sized)>,
 	) -> impl Future<Output = Self::Refs> {
 		assert_serializable::<Self>();
 		async { panic!("Either implement serialize or set Refs to Never for {}", type_name::<Self>()) }
@@ -262,7 +260,7 @@ fn assert_serializable<T: OwnedAtom>() {
 pub trait DynOwnedAtom: 'static {
 	fn atom_tid(&self) -> TypeId;
 	fn as_any_ref(&self) -> &dyn Any;
-	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()>;
+	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()>;
 	fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr>;
 	fn dyn_call(self: Box<Self>, arg: Expr) -> LocalBoxFuture<'static, GExpr>;
 	fn dyn_command(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, OrcRes<Option<GExpr>>>;
@@ -271,13 +269,13 @@ pub trait DynOwnedAtom: 'static {
 	fn dyn_serialize<'a>(
 		&'a self,
 		ctx: SysCtx,
-		sink: Pin<&'a mut dyn Write>,
+		sink: Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>>;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn as_any_ref(&self) -> &dyn Any { self }
-	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()> {
+	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()> {
 		async { self.val().await.as_ref().encode(buffer).await }.boxed_local()
 	}
 	fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
@@ -293,12 +291,12 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 		self.free(ctx).boxed_local()
 	}
 	fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit> {
-		async move { self.print(&FmtCtxImpl { i: ctx.i() }).await }.boxed_local()
+		async move { self.print_atom(&FmtCtxImpl { i: ctx.i() }).await }.boxed_local()
 	}
 	fn dyn_serialize<'a>(
 		&'a self,
 		ctx: SysCtx,
-		sink: Pin<&'a mut dyn Write>,
+		sink: Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		match TypeId::of::<Never>() == TypeId::of::<<Self as OwnedAtom>::Refs>() {
 			true => ready(None).boxed_local(),
@@ -313,3 +311,11 @@ struct ObjStore {
 	objects: RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
 }
 impl SysCtxEntry for ObjStore {}
+
+pub async fn own<A: OwnedAtom>(typ: TypAtom<A>) -> A {
+	let ctx = typ.untyped.ctx();
+	let g = ctx.get_or_default::<ObjStore>().objects.read().await;
+	let dyn_atom = (g.get(&typ.untyped.atom.drop.expect("Owned atoms always have a drop ID")))
+		.expect("Atom ID invalid; atom type probably not owned by this crate");
+	dyn_atom.as_any_ref().downcast_ref().cloned().expect("The ID should imply a type as well")
+}

orchid-extension/src/atom_thin.rs

@@ -3,9 +3,8 @@ use std::future::Future;
 use std::pin::Pin;
 
 use async_once_cell::OnceCell;
-use async_std::io::{Read, Write};
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
+use futures::{AsyncRead, AsyncWrite, FutureExt};
 use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
@@ -59,8 +58,8 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 		&'a self,
 		AtomCtx(buf, _, sys): AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'m1 mut dyn Read>,
+		req: Pin<&'m1 mut dyn AsyncRead>,
-		rep: Pin<&'m2 mut dyn Write>,
+		rep: Pin<&'m2 mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let ms = self.ms.get_or_init(self.msbuild.pack(sys.clone())).await;
@@ -76,7 +75,7 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
-		write: Pin<&'b mut dyn Write>,
+		write: Pin<&'b mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		Box::pin(async {
 			T::decode(Pin::new(&mut &ctx.0[..])).await.encode(write).await;
@@ -105,11 +104,11 @@ pub trait ThinAtom:
 {
 	#[allow(unused_variables)]
 	fn call(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot([err_not_callable(arg.ctx().i()).await]) }
+		async move { bot(err_not_callable(arg.ctx().i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn command(&self, ctx: SysCtx) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command(ctx.i()).await.into()) }
+		async move { Err(err_not_command(ctx.i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn print(&self, ctx: SysCtx) -> impl Future<Output = FmtUnit> {

orchid-extension/src/conv.rs

@@ -1,13 +1,14 @@
 use std::future::Future;
 
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use never::Never;
+use orchid_base::error::{OrcErrv, OrcRes, mk_err};
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
 
-use crate::atom::{AtomicFeatures, ToAtom, TypAtom};
+use crate::atom::{AtomicFeatures, ForeignAtom, ToAtom, TypAtom};
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, atom, bot};
-use crate::system::downcast_atom;
+use crate::system::{SysCtx, downcast_atom};
 
 pub trait TryFromExpr: Sized {
 	fn try_from_expr(expr: Expr) -> impl Future<Output = OrcRes<Self>>;
@@ -23,46 +24,61 @@ impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
 	}
 }
 
-async fn err_not_atom(pos: Pos, i: &Interner) -> OrcErr {
+async fn err_not_atom(pos: Pos, i: &Interner) -> OrcErrv {
-	mk_err(i.i("Expected an atom").await, "This expression is not an atom", [pos.into()])
+	mk_err(i.i("Expected an atom").await, "This expression is not an atom", [pos])
 }
 
-async fn err_type(pos: Pos, i: &Interner) -> OrcErr {
+async fn err_type(pos: Pos, i: &Interner) -> OrcErrv {
-	mk_err(i.i("Type error").await, "The atom is a different type than expected", [pos.into()])
+	mk_err(i.i("Type error").await, "The atom is a different type than expected", [pos])
 }
 
-impl<A: AtomicFeatures> TryFromExpr for TypAtom<A> {
+impl TryFromExpr for ForeignAtom {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
 		match expr.atom().await {
-			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), ex.ctx().i()).await.into()),
+			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), ex.ctx().i()).await),
-			Ok(f) => match downcast_atom::<A>(f).await {
+			Ok(f) => Ok(f),
-				Ok(a) => Ok(a),
-				Err(f) => Err(err_type(f.pos(), f.ctx().i()).await.into()),
-			},
 		}
 	}
 }
 
+impl<A: AtomicFeatures> TryFromExpr for TypAtom<A> {
+	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
+		let f = ForeignAtom::try_from_expr(expr).await?;
+		match downcast_atom::<A>(f).await {
+			Ok(a) => Ok(a),
+			Err(f) => Err(err_type(f.pos(), f.ctx().i()).await),
+		}
+	}
+}
+
+impl TryFromExpr for SysCtx {
+	async fn try_from_expr(expr: Expr) -> OrcRes<Self> { Ok(expr.ctx()) }
+}
+
 pub trait ToExpr {
-	fn to_expr(self) -> GExpr;
+	fn to_expr(self) -> impl Future<Output = GExpr>;
 }
 
 impl ToExpr for GExpr {
-	fn to_expr(self) -> GExpr { self }
+	async fn to_expr(self) -> GExpr { self }
 }
 impl ToExpr for Expr {
-	fn to_expr(self) -> GExpr { self.slot() }
+	async fn to_expr(self) -> GExpr { self.slot() }
 }
 
 impl<T: ToExpr> ToExpr for OrcRes<T> {
-	fn to_expr(self) -> GExpr {
+	async fn to_expr(self) -> GExpr {
 		match self {
 			Err(e) => bot(e),
-			Ok(t) => t.to_expr(),
+			Ok(t) => t.to_expr().await,
 		}
 	}
 }
 
 impl<A: ToAtom> ToExpr for A {
-	fn to_expr(self) -> GExpr { atom(self) }
+	async fn to_expr(self) -> GExpr { atom(self) }
+}
+
+impl ToExpr for Never {
+	async fn to_expr(self) -> GExpr { match self {} }
 }

orchid-extension/src/coroutine_exec.rs

@@ -0,0 +1,104 @@
+use std::borrow::Cow;
+use std::marker::PhantomData;
+use std::rc::Rc;
+
+use futures::channel::mpsc::{Sender, channel};
+use futures::lock::Mutex;
+use futures::stream::{self, LocalBoxStream};
+use futures::{FutureExt, SinkExt, StreamExt};
+use never::Never;
+use orchid_base::error::OrcRes;
+use orchid_base::format::{FmtCtx, FmtUnit};
+
+use crate::atom::Atomic;
+use crate::atom_owned::{OwnedAtom, OwnedVariant};
+use crate::conv::{ToExpr, TryFromExpr};
+use crate::expr::Expr;
+use crate::gen_expr::{GExpr, arg, call, lambda, seq};
+
+enum Command {
+	Execute(GExpr, Sender<Expr>),
+	Register(GExpr, Sender<Expr>),
+	Halt(GExpr),
+}
+
+struct BuilderCoroutineData {
+	name: Option<String>,
+	receiver: Mutex<LocalBoxStream<'static, Command>>,
+}
+
+#[derive(Clone)]
+struct BuilderCoroutine(Rc<BuilderCoroutineData>);
+impl BuilderCoroutine {
+	pub async fn run(self) -> GExpr {
+		let cmd = self.0.receiver.lock().await.next().await;
+		match cmd {
+			None => panic!("Before the stream ends, we should have gotten a Halt"),
+			Some(Command::Halt(expr)) => expr,
+			Some(Command::Execute(expr, reply)) => call([
+				lambda(0, [seq([
+					arg(0),
+					call([Replier { reply, builder: self }.to_expr().await, arg(0)]),
+				])]),
+				expr,
+			]),
+			Some(Command::Register(expr, reply)) =>
+				call([Replier { reply, builder: self }.to_expr().await, expr]),
+		}
+	}
+}
+
+#[derive(Clone)]
+pub struct Replier {
+	reply: Sender<Expr>,
+	builder: BuilderCoroutine,
+}
+impl Atomic for Replier {
+	type Data = ();
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for Replier {
+	type Refs = Never;
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	async fn call(mut self, arg: Expr) -> GExpr {
+		self.reply.send(arg).await.expect("What the heck");
+		std::mem::drop(self.reply);
+		self.builder.run().await
+	}
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		match &self.builder.0.name {
+			None => "BuilderCoroutine".into(),
+			Some(name) => format!("BuilderCoroutine({name})").into(),
+		}
+	}
+}
+
+pub async fn exec<R: ToExpr>(
+	debug: impl AsRef<str>,
+	f: impl for<'a> AsyncFnOnce(ExecHandle<'a>) -> R + 'static,
+) -> GExpr {
+	let (cmd_snd, cmd_recv) = channel(0);
+	let halt = async { Command::Halt(f(ExecHandle(cmd_snd, PhantomData)).await.to_expr().await) }
+		.into_stream();
+	let coro = BuilderCoroutine(Rc::new(BuilderCoroutineData {
+		name: Some(debug.as_ref().to_string()),
+		receiver: Mutex::new(stream::select(halt, cmd_recv).boxed_local()),
+	}));
+	coro.run().await
+}
+
+static WEIRD_DROP_ERR: &str = "Coroutine dropped while we are being polled somehow";
+
+pub struct ExecHandle<'a>(Sender<Command>, PhantomData<&'a ()>);
+impl ExecHandle<'_> {
+	pub async fn exec<T: TryFromExpr>(&mut self, val: impl ToExpr) -> OrcRes<T> {
+		let (reply_snd, mut reply_recv) = channel(1);
+		self.0.send(Command::Execute(val.to_expr().await, reply_snd)).await.expect(WEIRD_DROP_ERR);
+		T::try_from_expr(reply_recv.next().await.expect(WEIRD_DROP_ERR)).await
+	}
+	pub async fn register(&mut self, val: impl ToExpr) -> Expr {
+		let (reply_snd, mut reply_recv) = channel(1);
+		self.0.send(Command::Register(val.to_expr().await, reply_snd)).await.expect(WEIRD_DROP_ERR);
+		reply_recv.next().await.expect(WEIRD_DROP_ERR)
+	}
+}

orchid-extension/src/entrypoint.rs

@@ -5,18 +5,17 @@ use std::num::NonZero;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use async_std::channel::{self, Receiver, Sender};
+use futures::channel::mpsc::{Receiver, Sender, channel};
-use async_std::stream;
-use async_std::sync::Mutex;
 use futures::future::{LocalBoxFuture, join_all};
-use futures::{FutureExt, StreamExt, stream_select};
+use futures::lock::Mutex;
+use futures::{FutureExt, SinkExt, StreamExt, stream, stream_select};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::{ExtMsgSet, IntReq};
 use orchid_api_traits::{Decode, UnderRoot, enc_vec};
 use orchid_base::builtin::{ExtInit, ExtPort, Spawner};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
 use orchid_base::clone;
+use orchid_base::error::Reporter;
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::logging::Logger;
 use orchid_base::name::Sym;
@@ -29,12 +28,12 @@ use trait_set::trait_set;
 use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId};
 use crate::atom_owned::take_atom;
-use crate::expr::{Expr, ExprHandle};
+use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
-use crate::lexer::{LexContext, err_cascade, err_not_applicable};
+use crate::lexer::{LexContext, ekey_cascade, ekey_not_applicable};
-use crate::parser::{ParsCtx, get_const, linev_into_api};
+use crate::parser::{PTokTree, ParsCtx, get_const, linev_into_api};
 use crate::system::{SysCtx, atom_by_idx};
 use crate::system_ctor::{CtedObj, DynSystemCtor};
-use crate::tree::{GenTok, GenTokTree, LazyMemberFactory, TreeIntoApiCtxImpl};
+use crate::tree::{LazyMemberFactory, TreeIntoApiCtxImpl};
 
 pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>;
 pub type ExtReqNot = ReqNot<api::ExtMsgSet>;
@@ -50,7 +49,7 @@ impl ExtensionData {
 }
 
 pub enum MemberRecord {
-	Gen(Vec<Tok<String>>, LazyMemberFactory),
+	Gen(Vec<IStr>, LazyMemberFactory),
 	Res,
 }
 
@@ -84,16 +83,16 @@ pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
 pub struct ExtensionOwner {
 	_interner_cell: Rc<RefCell<Option<Interner>>>,
 	_systems_lock: Rc<Mutex<HashMap<api::SysId, SystemRecord>>>,
-	out_recv: Receiver<Vec<u8>>,
+	out_recv: Mutex<Receiver<Vec<u8>>>,
 	out_send: Sender<Vec<u8>>,
 }
 
 impl ExtPort for ExtensionOwner {
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		Box::pin(async { self.out_send.send(msg.to_vec()).boxed_local().await.unwrap() })
+		Box::pin(async { self.out_send.clone().send(msg.to_vec()).boxed_local().await.unwrap() })
 	}
 	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
-		Box::pin(async { (self.out_recv.recv().await).ok() })
+		Box::pin(async { self.out_recv.lock().await.next().await })
 	}
 }
 
@@ -109,9 +108,9 @@ pub fn extension_init(
 		.collect_vec();
 	let systems_lock = Rc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
 	let ext_header = api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() };
-	let (out_send, in_recv) = channel::bounded::<Vec<u8>>(1);
+	let (out_send, in_recv) = channel::<Vec<u8>>(1);
-	let (in_send, out_recv) = channel::bounded::<Vec<u8>>(1);
+	let (in_send, out_recv) = channel::<Vec<u8>>(1);
-	let (exit_send, exit_recv) = channel::bounded(1);
+	let (exit_send, exit_recv) = channel(1);
 	let logger = Logger::new(log_strategy);
 	let msg_logger = Logger::new(msg_logs);
 	let interner_cell = Rc::new(RefCell::new(None::<Interner>));
@@ -124,7 +123,7 @@ pub fn extension_init(
 	}));
 	let init_ctx = {
 		clone!(interner_weak, spawner, logger);
-		move |id: api::SysId, cted: CtedObj, reqnot: ReqNot<ExtMsgSet>| {
+		move |id: api::SysId, cted: CtedObj, reqnot: ReqNot<api::ExtMsgSet>| {
 			clone!(interner_weak, spawner, logger; async move {
 				let interner_rc =
 					interner_weak.upgrade().expect("System construction order while shutting down");
@@ -135,22 +134,30 @@ pub fn extension_init(
 	};
 	let rn = ReqNot::<api::ExtMsgSet>::new(
 		msg_logger.clone(),
-		move |a, _| clone!(in_send; Box::pin(async move { in_send.send(a.to_vec()).await.unwrap() })),
+		move |a, _| {
-		clone!(systems_weak, exit_send, get_ctx; move |n, _| {
+			clone!(in_send mut);
-			clone!(systems_weak, exit_send, get_ctx; async move {
+			Box::pin(async move { in_send.send(a.to_vec()).await.unwrap() })
-				match n {
+		},
-					api::HostExtNotif::Exit => exit_send.send(()).await.unwrap(),
+		{
-					api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
+			clone!(systems_weak, exit_send, get_ctx);
-						if let Some(rc) = systems_weak.upgrade() {
+			move |n, _| {
-							mem::drop(rc.lock().await.remove(&sys_id))
+				clone!(systems_weak, exit_send mut, get_ctx);
+				async move {
+					match n {
+						api::HostExtNotif::Exit => exit_send.send(()).await.unwrap(),
+						api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
+							if let Some(rc) = systems_weak.upgrade() {
+								mem::drop(rc.lock().await.remove(&sys_id))
+							},
+						api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) => {
+							let ctx = get_ctx(sys_id).await;
+							take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await
 						},
-					api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) => {
-						let ctx = get_ctx(sys_id).await;
-						take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await
 					}
 				}
-			}.boxed_local())
+				.boxed_local()
-		}),
+			}
+		},
 		{
 			clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
 			move |hand, req| {
@@ -173,7 +180,7 @@ pub fn extension_init(
 								});
 							let lazy_mems = Mutex::new(HashMap::new());
 							let ctx = init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await;
-							let const_root = stream::from_iter(cted.inst().dyn_env())
+							let const_root = stream::iter(cted.inst().dyn_env())
 								.then(|mem| {
 									let (req, lazy_mems) = (&hand, &lazy_mems);
 									clone!(i, ctx; async move {
@@ -189,10 +196,17 @@ pub fn extension_init(
 								})
 								.collect()
 								.await;
+							let prelude =
+								cted.inst().dyn_prelude(&i).await.iter().map(|sym| sym.to_api()).collect();
 							let record = SystemRecord { ctx, lazy_members: lazy_mems.into_inner() };
 							let systems = systems_weak.upgrade().expect("System constructed during shutdown");
 							systems.lock().await.insert(new_sys.id, record);
-							let response = api::NewSystemResponse { lex_filter, const_root, line_types: vec![] };
+							let line_types = join_all(
+								(cted.inst().dyn_parsers().iter())
+									.map(|p| async { i.i(p.line_head()).await.to_api() }),
+							)
+							.await;
+							let response = api::NewSystemResponse { lex_filter, const_root, line_types, prelude };
 							hand.handle(&new_sys, &response).await
 						},
 						api::HostExtReq::GetMember(get_tree @ api::GetMember(sys_id, tree_id)) => {
@@ -226,47 +240,63 @@ pub fn extension_init(
 							let sys_ctx = get_ctx(sys).await;
 							let text = Tok::from_api(text, &i).await;
 							let src = Sym::from_api(src, sys_ctx.i()).await;
-							let ctx = LexContext { id, pos, text: &text, src, ctx: sys_ctx.clone() };
+							let rep = Reporter::new();
+							let expr_store = BorrowedExprStore::new();
 							let trigger_char = text.chars().nth(pos as usize).unwrap();
-							let err_na = err_not_applicable(&i).await;
+							let ekey_na = ekey_not_applicable(&i).await;
-							let err_cascade = err_cascade(&i).await;
+							let ekey_cascade = ekey_cascade(&i).await;
 							let lexers = sys_ctx.cted().inst().dyn_lexers();
 							for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char)) {
+								let ctx = LexContext {
+									id,
+									pos,
+									text: &text,
+									src: src.clone(),
+									ctx: sys_ctx.clone(),
+									rep: &rep,
+									exprs: &expr_store,
+								};
 								match lx.lex(&text[pos as usize..], &ctx).await {
-									Err(e) if e.any(|e| *e == err_na) => continue,
+									Err(e) if e.any(|e| *e == ekey_na) => continue,
 									Err(e) => {
-										let eopt = e.keep_only(|e| *e != err_cascade).map(|e| Err(e.to_api()));
+										let eopt = e.keep_only(|e| *e != ekey_cascade).map(|e| Err(e.to_api()));
+										expr_store.dispose().await;
 										return hand.handle(&lex, &eopt).await;
 									},
 									Ok((s, expr)) => {
 										let expr = expr.into_api(&mut (), &mut (sys_ctx, &hand)).await;
 										let pos = (text.len() - s.len()) as u32;
+										expr_store.dispose().await;
 										return hand.handle(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
 									},
 								}
 							}
 							writeln!(logger, "Got notified about n/a character '{trigger_char}'");
+							expr_store.dispose().await;
 							hand.handle(&lex, &None).await
 						},
 						api::HostExtReq::ParseLine(pline) => {
-							let api::ParseLine { module, src, exported, comments, sys, line } = &pline;
+							let api::ParseLine { module, src, exported, comments, sys, line, idx } = &pline;
-							let mut ctx = get_ctx(*sys).await;
+							let ctx = get_ctx(*sys).await;
 							let parsers = ctx.cted().inst().dyn_parsers();
 							let src = Sym::from_api(*src, ctx.i()).await;
 							let comments =
 								join_all(comments.iter().map(|c| Comment::from_api(c, src.clone(), &i))).await;
-							let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx, &mut (), &src, &i).await;
+							let expr_store = BorrowedExprStore::new();
+							let mut from_api_ctx = (ctx.clone(), &expr_store);
+							let line: Vec<PTokTree> =
+								ttv_from_api(line, &mut from_api_ctx, &mut (), &src, &i).await;
 							let snip = Snippet::new(line.first().expect("Empty line"), &line);
-							let (head, tail) = snip.pop_front().unwrap();
+							let parser = parsers[*idx as usize];
-							let name = if let GenTok::Name(n) = &head.tok { n } else { panic!("No line head") };
-							let parser =
-								parsers.iter().find(|p| p.line_head() == **name).expect("No parser candidate");
 							let module = Sym::from_api(*module, ctx.i()).await;
-							let pctx = ParsCtx::new(ctx.clone(), module);
+							let reporter = Reporter::new();
-							let o_line = match parser.parse(pctx, *exported, comments, tail).await {
+							let pctx = ParsCtx::new(ctx.clone(), module, &reporter);
+							let parse_res = parser.parse(pctx, *exported, comments, snip).await;
+							let o_line = match reporter.merge(parse_res) {
 								Err(e) => Err(e.to_api()),
 								Ok(t) => Ok(linev_into_api(t, ctx.clone(), &hand).await),
 							};
+							expr_store.dispose().await;
 							hand.handle(&pline, &o_line).await
 						},
 						api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst { id, sys }) => {
@@ -279,6 +309,7 @@ pub fn extension_init(
 							let atom_req = atom_req.clone();
 							with_atom_record(&get_ctx, atom, async move |nfo, ctx, id, buf| {
 								let actx = AtomCtx(buf, atom.drop, ctx.clone());
+
 								match &atom_req {
 									api::AtomReq::SerializeAtom(ser) => {
 										let mut buf = enc_vec(&id).await;
@@ -309,16 +340,23 @@ pub fn extension_init(
 										hand.handle(fwded, &some.then_some(reply)).await
 									},
 									api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
-										// SAFETY: function calls own their argument implicitly
+										// SAFETY: function calls borrow their argument implicitly
-										let expr_handle = unsafe { ExprHandle::from_args(ctx.clone(), *arg) };
+										let expr_store = BorrowedExprStore::new();
-										let ret = nfo.call_ref(actx, Expr::from_handle(Rc::new(expr_handle))).await;
+										let expr_handle = ExprHandle::borrowed(ctx.clone(), *arg, &expr_store);
-										hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
+										let ret = nfo.call_ref(actx, Expr::from_handle(expr_handle.clone())).await;
+										expr_handle.drop_one().await;
+										let api_expr = ret.api_return(ctx.clone(), &hand).await;
+										expr_store.dispose().await;
+										hand.handle(call, &api_expr).await
 									},
 									api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
-										// SAFETY: function calls own their argument implicitly
+										// SAFETY: function calls borrow their argument implicitly
-										let expr_handle = unsafe { ExprHandle::from_args(ctx.clone(), *arg) };
+										let expr_store = BorrowedExprStore::new();
-										let ret = nfo.call(actx, Expr::from_handle(Rc::new(expr_handle))).await;
+										let expr_handle = ExprHandle::borrowed(ctx.clone(), *arg, &expr_store);
-										hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
+										let ret = nfo.call(actx, Expr::from_handle(expr_handle.clone())).await;
+										let api_expr = ret.api_return(ctx.clone(), &hand).await;
+										expr_store.dispose().await;
+										hand.handle(call, &api_expr).await
 									},
 									api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
 										Err(e) => hand.handle(cmd, &Err(e.to_api())).await,
@@ -340,8 +378,7 @@ pub fn extension_init(
 							let ctx = get_ctx(*sys).await;
 							// SAFETY: deserialization implicitly grants ownership to previously owned exprs
 							let refs = (refs.iter())
-								.map(|tk| unsafe { ExprHandle::from_args(ctx.clone(), *tk) })
+								.map(|tk| Expr::from_handle(ExprHandle::deserialize(ctx.clone(), *tk)))
-								.map(|handle| Expr::from_handle(Rc::new(handle)))
 								.collect_vec();
 							let id = AtomTypeId::decode(Pin::new(&mut read)).await;
 							let inst = ctx.cted().inst();
@@ -355,7 +392,7 @@ pub fn extension_init(
 		},
 	);
 	*interner_cell.borrow_mut() =
-		Some(Interner::new_replica(rn.clone().map(|ir: IntReq| ir.into_root())));
+		Some(Interner::new_replica(rn.clone().map(|ir: api::IntReq| ir.into_root())));
 	spawner(Box::pin(clone!(spawner; async move {
 		let mut streams = stream_select! { in_recv.map(Some), exit_recv.map(|_| None) };
 		while let Some(item) = streams.next().await {
@@ -368,7 +405,7 @@ pub fn extension_init(
 	ExtInit {
 		header: ext_header,
 		port: Box::new(ExtensionOwner {
-			out_recv,
+			out_recv: Mutex::new(out_recv),
 			out_send,
 			_interner_cell: interner_cell,
 			_systems_lock: systems_lock,

orchid-extension/src/expr.rs

@@ -1,9 +1,11 @@
+use std::cell::RefCell;
 use std::fmt;
+use std::hash::Hash;
 use std::rc::Rc;
 
 use async_once_cell::OnceCell;
 use derive_destructure::destructure;
-use orchid_api::ExtAtomPrint;
+use hashbrown::HashSet;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::location::Pos;
@@ -14,26 +16,61 @@ use crate::atom::ForeignAtom;
 use crate::gen_expr::{GExpr, GExprKind};
 use crate::system::SysCtx;
 
+pub struct BorrowedExprStore(RefCell<Option<HashSet<Rc<ExprHandle>>>>);
+impl BorrowedExprStore {
+	pub(crate) fn new() -> Self { Self(RefCell::new(Some(HashSet::new()))) }
+	pub async fn dispose(self) {
+		let elements = self.0.borrow_mut().take().unwrap();
+		for handle in elements {
+			handle.drop_one().await
+		}
+	}
+}
+impl Drop for BorrowedExprStore {
+	fn drop(&mut self) {
+		if self.0.borrow().is_some() {
+			panic!("This should always be explicitly disposed")
+		}
+	}
+}
+
 #[derive(destructure)]
 pub struct ExprHandle {
 	pub tk: api::ExprTicket,
 	pub ctx: SysCtx,
 }
 impl ExprHandle {
-	/// # Safety
+	/// This function does not signal to take ownership of the expr.
-	///
+	pub fn borrowed(ctx: SysCtx, tk: api::ExprTicket, store: &BorrowedExprStore) -> Rc<Self> {
-	/// This function does not signal to take ownership of the expr. It must only
+		let this = Rc::new(Self { ctx, tk });
-	/// be called on tickets that are already implicitly owned.
+		store.0.borrow_mut().as_mut().unwrap().insert(this.clone());
-	pub unsafe fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } }
+		this
+	}
+	pub fn deserialize(ctx: SysCtx, tk: api::ExprTicket) -> Rc<Self> { Rc::new(Self { ctx, tk }) }
 	pub fn get_ctx(&self) -> SysCtx { self.ctx.clone() }
-	pub async fn clone(&self) -> Self {
+	/// Drop one instance of the handle silently; if it's the last one, do
-		self.ctx.reqnot().notify(api::Acquire(self.ctx.sys_id(), self.tk)).await;
+	/// nothing, otherwise send an Acquire
-		Self { ctx: self.ctx.clone(), tk: self.tk }
+	pub async fn drop_one(self: Rc<Self>) {
+		if let Err(rc) = Rc::try_unwrap(self) {
+			rc.ctx.reqnot().notify(api::Acquire(rc.ctx.sys_id(), rc.tk)).await
+		}
 	}
 	/// Drop the handle and get the ticket without a release notification.
 	/// Use this with messages that imply ownership transfer. This function is
 	/// safe because abusing it is a memory leak.
-	pub fn into_tk(self) -> api::ExprTicket { self.destructure().0 }
+	pub fn serialize(self) -> api::ExprTicket { self.destructure().0 }
+}
+impl Eq for ExprHandle {}
+impl PartialEq for ExprHandle {
+	fn eq(&self, other: &Self) -> bool {
+		self.ctx.sys_id() == other.ctx.sys_id() && self.tk == other.tk
+	}
+}
+impl Hash for ExprHandle {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		self.ctx.sys_id().hash(state);
+		self.tk.hash(state);
+	}
 }
 impl fmt::Debug for ExprHandle {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -92,8 +129,9 @@ impl Format for Expr {
 		match &self.data().await.kind {
 			ExprKind::Opaque => "OPAQUE".to_string().into(),
 			ExprKind::Bottom(b) => format!("Bottom({b})").into(),
-			ExprKind::Atom(a) =>
+			ExprKind::Atom(a) => FmtUnit::from_api(
-				FmtUnit::from_api(&self.handle.ctx.reqnot().request(ExtAtomPrint(a.atom.clone())).await),
+				&self.handle.ctx.reqnot().request(api::ExtAtomPrint(a.atom.clone())).await,
+			),
 		}
 	}
 }

orchid-extension/src/func_atom.rs

@@ -1,13 +1,13 @@
+use std::any::TypeId;
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::future::Future;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use async_std::io::Write;
-use async_std::sync::Mutex;
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
+use futures::{AsyncWrite, FutureExt};
 use itertools::Itertools;
 use never::Never;
 use orchid_api_traits::Encode;
@@ -20,6 +20,7 @@ use trait_set::trait_set;
 use crate::atom::Atomic;
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use crate::conv::ToExpr;
+use crate::coroutine_exec::{ExecHandle, exec};
 use crate::expr::Expr;
 use crate::gen_expr::GExpr;
 use crate::system::{SysCtx, SysCtxEntry};
@@ -29,13 +30,40 @@ trait_set! {
 }
 
 pub trait ExprFunc<I, O>: Clone + 'static {
-	const ARITY: u8;
+	fn argtyps() -> &'static [TypeId];
-	fn apply(&self, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
+	fn apply<'a>(&self, hand: ExecHandle<'a>, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
 }
 
 #[derive(Default)]
-struct FunsCtx(Mutex<HashMap<Sym, (u8, Rc<dyn FunCB>)>>);
+struct FunsCtx(Mutex<HashMap<Sym, FunRecord>>);
 impl SysCtxEntry for FunsCtx {}
+#[derive(Clone)]
+struct FunRecord {
+	argtyps: &'static [TypeId],
+	fun: Rc<dyn FunCB>,
+}
+
+async fn process_args<I, O, F: ExprFunc<I, O>>(
+	debug: impl AsRef<str> + Clone + 'static,
+	f: F,
+) -> FunRecord {
+	let argtyps = F::argtyps();
+	let fun = Rc::new(move |v: Vec<Expr>| {
+		clone!(f, v mut);
+		exec(debug.clone(), async move |mut hand| {
+			let mut norm_args = Vec::with_capacity(v.len());
+			for (expr, typ) in v.into_iter().zip(argtyps) {
+				if *typ != TypeId::of::<Expr>() {
+					norm_args.push(hand.exec(expr).await?);
+				}
+			}
+			f.apply(hand, norm_args).await
+		})
+		.map(Ok)
+		.boxed_local()
+	});
+	FunRecord { argtyps, fun }
+}
 
 /// An Atom representing a partially applied named native function. These
 /// partial calls are serialized into the name of the native function and the
@@ -46,23 +74,22 @@ impl SysCtxEntry for FunsCtx {}
 pub(crate) struct Fun {
 	path: Sym,
 	args: Vec<Expr>,
-	arity: u8,
+	record: FunRecord,
-	fun: Rc<dyn FunCB>,
 }
 impl Fun {
 	pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, ctx: SysCtx, f: F) -> Self {
 		let funs: &FunsCtx = ctx.get_or_default();
 		let mut fung = funs.0.lock().await;
-		let fun = if let Some(x) = fung.get(&path) {
+		let record = if let Some(record) = fung.get(&path) {
-			x.1.clone()
+			record.clone()
 		} else {
-			let fun = Rc::new(move |v| clone!(f; async move { f.apply(v).await }.boxed_local()));
+			let record = process_args(path.to_string(), f).await;
-			fung.insert(path.clone(), (F::ARITY, fun.clone()));
+			fung.insert(path.clone(), record.clone());
-			fun
+			record
 		};
-		Self { args: vec![], arity: F::ARITY, path, fun }
+		Self { args: vec![], path, record }
 	}
-	pub fn arity(&self) -> u8 { self.arity }
+	pub fn arity(&self) -> u8 { self.record.argtyps.len() as u8 }
 }
 impl Atomic for Fun {
 	type Data = ();
@@ -72,28 +99,28 @@ impl OwnedAtom for Fun {
 	type Refs = Vec<Expr>;
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
 	async fn call_ref(&self, arg: Expr) -> GExpr {
-		std::io::Write::flush(&mut std::io::stderr()).unwrap();
 		let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
-		if new_args.len() == self.arity.into() {
+		if new_args.len() == self.record.argtyps.len() {
-			(self.fun)(new_args).await.to_expr()
+			(self.record.fun)(new_args).await.to_expr().await
 		} else {
-			Self { args: new_args, arity: self.arity, fun: self.fun.clone(), path: self.path.clone() }
+			Self { args: new_args, record: self.record.clone(), path: self.path.clone() }.to_expr().await
-				.to_expr()
 		}
 	}
 	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
-	async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs {
+	async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl AsyncWrite + ?Sized)>) -> Self::Refs {
 		self.path.to_api().encode(write).await;
 		self.args.clone()
 	}
 	async fn deserialize(mut ctx: impl DeserializeCtx, args: Self::Refs) -> Self {
 		let sys = ctx.sys();
 		let path = Sym::from_api(ctx.decode().await, sys.i()).await;
-		let (arity, fun) = sys.get_or_default::<FunsCtx>().0.lock().await.get(&path).unwrap().clone();
+		let record = (sys.get::<FunsCtx>().0.lock().await.get(&path))
-		Self { args, arity, path, fun }
+			.expect("Function missing during deserialization")
+			.clone();
+		Self { args, path, record }
 	}
-	async fn print<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		format!("{}:{}/{}", self.path, self.args.len(), self.arity).into()
+		format!("{}:{}/{}", self.path, self.args.len(), self.arity()).into()
 	}
 }
 
@@ -104,13 +131,14 @@ impl OwnedAtom for Fun {
 #[derive(Clone)]
 pub struct Lambda {
 	args: Vec<Expr>,
-	arity: u8,
+	record: FunRecord,
-	fun: Rc<dyn FunCB>,
 }
 impl Lambda {
-	pub fn new<I, O, F: ExprFunc<I, O>>(f: F) -> Self {
+	pub async fn new<I, O, F: ExprFunc<I, O>>(
-		let fun = Rc::new(move |v| clone!(f; async move { f.apply(v).await }.boxed_local()));
+		debug: impl AsRef<str> + Clone + 'static,
-		Self { args: vec![], arity: F::ARITY, fun }
+		f: F,
+	) -> Self {
+		Self { args: vec![], record: process_args(debug, f).await }
 	}
 }
 impl Atomic for Lambda {
@@ -122,53 +150,59 @@ impl OwnedAtom for Lambda {
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
 	async fn call_ref(&self, arg: Expr) -> GExpr {
 		let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
-		if new_args.len() == self.arity.into() {
+		if new_args.len() == self.record.argtyps.len() {
-			(self.fun)(new_args).await.to_expr()
+			(self.record.fun)(new_args).await.to_expr().await
 		} else {
-			Self { args: new_args, arity: self.arity, fun: self.fun.clone() }.to_expr()
+			Self { args: new_args, record: self.record.clone() }.to_expr().await
 		}
 	}
 	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
 }
 
 mod expr_func_derives {
+	use std::any::TypeId;
+	use std::sync::OnceLock;
+
 	use orchid_base::error::OrcRes;
 
 	use super::ExprFunc;
 	use crate::conv::{ToExpr, TryFromExpr};
-	use crate::func_atom::Expr;
+	use crate::func_atom::{ExecHandle, Expr};
 	use crate::gen_expr::GExpr;
 
 	macro_rules! expr_func_derive {
-    ($arity: tt, $($t:ident),*) => {
+    ($($t:ident),*) => {
       pastey::paste!{
         impl<
-          $($t: TryFromExpr, )*
+          $($t: TryFromExpr + 'static, )*
 					Out: ToExpr,
           Func: AsyncFn($($t,)*) -> Out + Clone + Send + Sync + 'static
         > ExprFunc<($($t,)*), Out> for Func {
-          const ARITY: u8 = $arity;
+          fn argtyps() -> &'static [TypeId] {
-          async fn apply(&self, v: Vec<Expr>) -> OrcRes<GExpr> {
+            static STORE: OnceLock<Vec<TypeId>> = OnceLock::new();
-            assert_eq!(v.len(), Self::ARITY.into(), "Arity mismatch");
+            &*STORE.get_or_init(|| vec![$(TypeId::of::<$t>()),*])
+          }
+          async fn apply<'a>(&self, _: ExecHandle<'a>, v: Vec<Expr>) -> OrcRes<GExpr> {
+            assert_eq!(v.len(), Self::argtyps().len(), "Arity mismatch");
             let [$([< $t:lower >],)*] = v.try_into().unwrap_or_else(|_| panic!("Checked above"));
-            Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_expr())
+            Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_expr().await)
           }
         }
       }
     };
   }
-	expr_func_derive!(1, A);
+	expr_func_derive!(A);
-	expr_func_derive!(2, A, B);
+	expr_func_derive!(A, B);
-	expr_func_derive!(3, A, B, C);
+	expr_func_derive!(A, B, C);
-	expr_func_derive!(4, A, B, C, D);
+	expr_func_derive!(A, B, C, D);
-	expr_func_derive!(5, A, B, C, D, E);
+	expr_func_derive!(A, B, C, D, E);
-	expr_func_derive!(6, A, B, C, D, E, F);
+	// expr_func_derive!(A, B, C, D, E, F);
-	expr_func_derive!(7, A, B, C, D, E, F, G);
+	// expr_func_derive!(A, B, C, D, E, F, G);
-	expr_func_derive!(8, A, B, C, D, E, F, G, H);
+	// expr_func_derive!(A, B, C, D, E, F, G, H);
-	expr_func_derive!(9, A, B, C, D, E, F, G, H, I);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I);
-	expr_func_derive!(10, A, B, C, D, E, F, G, H, I, J);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J);
-	expr_func_derive!(11, A, B, C, D, E, F, G, H, I, J, K);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K);
-	expr_func_derive!(12, A, B, C, D, E, F, G, H, I, J, K, L);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L);
-	expr_func_derive!(13, A, B, C, D, E, F, G, H, I, J, K, L, M);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L, M);
-	expr_func_derive!(14, A, B, C, D, E, F, G, H, I, J, K, L, M, N);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L, M, N);
 }

orchid-extension/src/gen_expr.rs

@@ -10,9 +10,7 @@ use orchid_base::{match_mapping, tl_cache};
 
 use crate::api;
 use crate::atom::{AtomFactory, ToAtom};
-use crate::conv::{ToExpr, TryFromExpr};
 use crate::expr::Expr;
-use crate::func_atom::Lambda;
 use crate::system::SysCtx;
 
 #[derive(Clone, Debug)]
@@ -35,6 +33,7 @@ impl GExpr {
 			}
 		}
 	}
+	pub fn at(self, pos: Pos) -> Self { GExpr { pos, kind: self.kind } }
 }
 impl Format for GExpr {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -67,7 +66,6 @@ impl GExprKind {
 			Lambda(arg, body => Box::new(body.api_return(ctx, hand).await)),
 			Arg(arg),
 			Const(name.to_api()),
-			Const(name.to_api()),
 			Bottom(err.to_api()),
 			NewAtom(fac.clone().build(ctx).await),
 		} {
@@ -129,10 +127,3 @@ pub fn call(v: impl IntoIterator<Item = GExpr>) -> GExpr {
 pub fn bot(ev: impl IntoIterator<Item = OrcErr>) -> GExpr {
 	inherit(GExprKind::Bottom(OrcErrv::new(ev).unwrap()))
 }
-
-pub fn with<I: TryFromExpr, O: ToExpr>(
-	expr: GExpr,
-	cont: impl AsyncFn(I) -> O + Clone + Send + Sync + 'static,
-) -> GExpr {
-	call([lambda(0, [seq([arg(0), call([Lambda::new(cont).to_expr(), arg(0)])])]), expr])
-}

orchid-extension/src/interner.rs

@@ -0,0 +1,7 @@
+use orchid_base::interner::{ApiStrTok, ApiStrvTok, IStr};
+
+pub struct ExtIStr(ApiStrTok, Rc<String>);
+impl Deref for ExtIStr {}
+pub struct ExtIStrv(ApiStrvTok, Rc<Vec<IStr>>);
+
+pub struct ExtInterner {}

orchid-extension/src/lexer.rs

@@ -4,48 +4,64 @@ use std::ops::RangeInclusive;
 
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_err};
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::{Pos, SrcRange};
 use orchid_base::name::Sym;
+use orchid_base::parse::ParseCtx;
 use orchid_base::reqnot::Requester;
 
 use crate::api;
+use crate::expr::BorrowedExprStore;
+use crate::parser::PTokTree;
 use crate::system::SysCtx;
 use crate::tree::GenTokTree;
 
-pub async fn err_cascade(i: &Interner) -> OrcErr {
+pub async fn ekey_cascade(i: &Interner) -> IStr {
-	mk_err(
+	i.i("An error cascading from a recursive call").await
-		i.i("An error cascading from a recursive call").await,
+}
-		"This error is a sentinel for the extension library.\
+pub async fn ekey_not_applicable(i: &Interner) -> IStr {
-		it should not be emitted by the extension.",
+	i.i("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
-		[Pos::None.into()],
+}
-	)
+const MSG_INTERNAL_ERROR: &str = "This error is a sentinel for the extension library.\
+	it should not be emitted by the extension.";
+
+pub async fn err_cascade(i: &Interner) -> OrcErrv {
+	mk_err(ekey_cascade(i).await, MSG_INTERNAL_ERROR, [Pos::None])
 }
 
-pub async fn err_not_applicable(i: &Interner) -> OrcErr {
+pub async fn err_not_applicable(i: &Interner) -> OrcErrv {
-	mk_err(
+	mk_err(ekey_not_applicable(i).await, MSG_INTERNAL_ERROR, [Pos::None])
-		i.i("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await,
-		&*err_cascade(i).await.message,
-		[Pos::None.into()],
-	)
 }
 
 pub struct LexContext<'a> {
+	pub(crate) exprs: &'a BorrowedExprStore,
 	pub ctx: SysCtx,
-	pub text: &'a Tok<String>,
+	pub text: &'a IStr,
 	pub id: api::ParsId,
 	pub pos: u32,
-	pub src: Sym,
+	pub(crate) src: Sym,
+	pub(crate) rep: &'a Reporter,
 }
 impl<'a> LexContext<'a> {
-	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree)> {
+	pub fn src(&self) -> &Sym { &self.src }
+	/// This function returns [PTokTree] because it can never return
+	/// [orchid_base::tree::Token::NewExpr]. You can use
+	/// [crate::parser::p_tree2gen] to convert this to [crate::tree::GenTokTree]
+	/// for embedding in the return value.
+	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, PTokTree)> {
 		let start = self.pos(tail);
 		let Some(lx) = self.ctx.reqnot().request(api::SubLex { pos: start, id: self.id }).await else {
-			return Err(err_cascade(self.ctx.i()).await.into());
+			return Err(err_cascade(self.ctx.i()).await);
 		};
-		let tree =
+		let tree = PTokTree::from_api(
-			GenTokTree::from_api(&lx.tree, &mut self.ctx.clone(), &mut (), &self.src, self.ctx.i()).await;
+			&lx.tree,
+			&mut (self.ctx.clone(), self.exprs),
+			&mut (),
+			&self.src,
+			self.ctx.i(),
+		)
+		.await;
 		Ok((&self.text[lx.pos as usize..], tree))
 	}
 
@@ -57,8 +73,10 @@ impl<'a> LexContext<'a> {
 	pub fn pos_lt(&self, len: impl TryInto<u32, Error: fmt::Debug>, tail: &'a str) -> SrcRange {
 		SrcRange::new(self.pos(tail) - len.try_into().unwrap()..self.pos(tail), &self.src)
 	}
-
+}
-	pub fn i(&self) -> &Interner { self.ctx.i() }
+impl ParseCtx for LexContext<'_> {
+	fn i(&self) -> &Interner { self.ctx.i() }
+	fn rep(&self) -> &Reporter { self.rep }
 }
 
 pub trait Lexer: Send + Sync + Sized + Default + 'static {

orchid-extension/src/lib.rs

@@ -4,14 +4,17 @@ pub mod atom;
 pub mod atom_owned;
 pub mod atom_thin;
 pub mod conv;
+pub mod coroutine_exec;
 pub mod entrypoint;
 pub mod expr;
 pub mod func_atom;
 pub mod gen_expr;
 pub mod lexer;
-pub mod msg;
+// pub mod msg;
+mod interner;
 pub mod other_system;
 pub mod parser;
+pub mod reflection;
 pub mod system;
 pub mod system_ctor;
 pub mod tokio;

orchid-extension/src/msg.rs

@@ -1,12 +1,9 @@
 use std::pin::pin;
 
 use async_once_cell::OnceCell;
-use async_std::io::{self, Stdout};
+use futures::lock::Mutex;
-use async_std::sync::Mutex;
 use orchid_base::msg::{recv_msg, send_msg};
 
-static STDOUT: OnceCell<Mutex<Stdout>> = OnceCell::new();
-
 pub async fn send_parent_msg(msg: &[u8]) -> io::Result<()> {
 	let stdout_lk = STDOUT.get_or_init(async { Mutex::new(io::stdout()) }).await;
 	let mut stdout_g = stdout_lk.lock().await;

orchid-extension/src/parser.rs

@@ -1,25 +1,47 @@
 use std::marker::PhantomData;
 
-use futures::FutureExt;
+use async_fn_stream::stream;
 use futures::future::{LocalBoxFuture, join_all};
+use futures::{FutureExt, Stream, StreamExt};
 use itertools::Itertools;
-use orchid_api::ResolveNames;
+use never::Never;
-use orchid_base::error::OrcRes;
+use orchid_base::error::{OrcErrv, OrcRes, Reporter};
 use orchid_base::id_store::IdStore;
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::SrcRange;
+use orchid_base::match_mapping;
 use orchid_base::name::Sym;
-use orchid_base::parse::{Comment, Snippet};
+use orchid_base::parse::{Comment, ParseCtx, Snippet};
 use orchid_base::reqnot::{ReqHandlish, Requester};
-use orchid_base::tree::ttv_into_api;
+use orchid_base::tree::{TokTree, Token, ttv_into_api};
 
 use crate::api;
+use crate::conv::ToExpr;
 use crate::expr::Expr;
 use crate::gen_expr::GExpr;
 use crate::system::{SysCtx, SysCtxEntry};
-use crate::tree::GenTokTree;
+use crate::tree::{GenTok, GenTokTree};
 
-pub type GenSnippet<'a> = Snippet<'a, Expr, GExpr>;
+pub type PTok = Token<Expr, Never>;
+pub type PTokTree = TokTree<Expr, Never>;
+pub type PSnippet<'a> = Snippet<'a, Expr, Never>;
+
+pub fn p_tok2gen(tok: PTok) -> GenTok {
+	match_mapping!(tok, PTok => GenTok {
+		Comment(s), Name(n), BR, Handle(ex), Bottom(err),
+		LambdaHead(arg => Box::new(p_tree2gen(*arg))),
+		NS(n, arg => Box::new(p_tree2gen(*arg))),
+		S(p, body () p_v2gen),
+	} {
+		PTok::NewExpr(never) => match never {}
+	})
+}
+pub fn p_tree2gen(tree: PTokTree) -> GenTokTree {
+	TokTree { tok: p_tok2gen(tree.tok), sr: tree.sr }
+}
+pub fn p_v2gen(v: impl IntoIterator<Item = PTokTree>) -> Vec<GenTokTree> {
+	v.into_iter().map(p_tree2gen).collect_vec()
+}
 
 pub trait Parser: Send + Sync + Sized + Default + 'static {
 	const LINE_HEAD: &'static str;
@@ -27,7 +49,7 @@ pub trait Parser: Send + Sync + Sized + Default + 'static {
 		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'a>,
+		line: PSnippet<'a>,
 	) -> impl Future<Output = OrcRes<Vec<ParsedLine>>> + 'a;
 }
 
@@ -38,7 +60,7 @@ pub trait DynParser: Send + Sync + 'static {
 		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'a>,
+		line: PSnippet<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>>;
 }
 
@@ -49,7 +71,7 @@ impl<T: Parser> DynParser for T {
 		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'a>,
+		line: PSnippet<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>> {
 		Box::pin(async move { Self::parse(ctx, exported, comments, line).await })
 	}
@@ -61,12 +83,19 @@ pub struct ParsCtx<'a> {
 	_parse: PhantomData<&'a mut ()>,
 	ctx: SysCtx,
 	module: Sym,
+	reporter: &'a Reporter,
 }
-impl ParsCtx<'_> {
+impl<'a> ParsCtx<'a> {
-	pub(crate) fn new(ctx: SysCtx, module: Sym) -> Self { Self { _parse: PhantomData, ctx, module } }
+	pub(crate) fn new(ctx: SysCtx, module: Sym, reporter: &'a Reporter) -> Self {
+		Self { _parse: PhantomData, ctx, module, reporter }
+	}
 	pub fn ctx(&self) -> &SysCtx { &self.ctx }
 	pub fn module(&self) -> Sym { self.module.clone() }
 }
+impl ParseCtx for ParsCtx<'_> {
+	fn i(&self) -> &Interner { self.ctx.i() }
+	fn rep(&self) -> &Reporter { self.reporter }
+}
 
 type BoxConstCallback = Box<dyn FnOnce(ConstCtx) -> LocalBoxFuture<'static, GExpr>>;
 
@@ -82,6 +111,31 @@ pub struct ParsedLine {
 	pub kind: ParsedLineKind,
 }
 impl ParsedLine {
+	pub fn cnst<'a, R: ToExpr + 'static, F: AsyncFnOnce(ConstCtx) -> R + 'static>(
+		sr: &SrcRange,
+		comments: impl IntoIterator<Item = &'a Comment>,
+		exported: bool,
+		name: IStr,
+		f: F,
+	) -> Self {
+		let cb = Box::new(|ctx| async move { f(ctx).await.to_expr().await }.boxed_local());
+		let kind = ParsedLineKind::Mem(ParsedMem { name, exported, kind: ParsedMemKind::Const(cb) });
+		let comments = comments.into_iter().cloned().collect();
+		ParsedLine { comments, sr: sr.clone(), kind }
+	}
+	pub fn module<'a>(
+		sr: &SrcRange,
+		comments: impl IntoIterator<Item = &'a Comment>,
+		exported: bool,
+		name: &IStr,
+		use_prelude: bool,
+		lines: impl IntoIterator<Item = ParsedLine>,
+	) -> Self {
+		let mem_kind = ParsedMemKind::Mod { lines: lines.into_iter().collect(), use_prelude };
+		let line_kind = ParsedLineKind::Mem(ParsedMem { name: name.clone(), exported, kind: mem_kind });
+		let comments = comments.into_iter().cloned().collect();
+		ParsedLine { comments, sr: sr.clone(), kind: line_kind }
+	}
 	pub async fn into_api(self, ctx: SysCtx, hand: &dyn ReqHandlish) -> api::ParsedLine {
 		api::ParsedLine {
 			comments: self.comments.into_iter().map(|c| c.to_api()).collect(),
@@ -94,8 +148,10 @@ impl ParsedLine {
 						ParsedMemKind::Const(cb) => api::ParsedMemberKind::Constant(api::ParsedConstId(
 							ctx.get_or_default::<ParsedConstCtxEntry>().consts.add(cb).id(),
 						)),
-						ParsedMemKind::Mod(plv) =>
+						ParsedMemKind::Mod { lines, use_prelude } => api::ParsedMemberKind::Module {
-							api::ParsedMemberKind::Module(linev_into_api(plv, ctx, hand).boxed_local().await),
+							lines: linev_into_api(lines, ctx, hand).boxed_local().await,
+							use_prelude,
+						},
 					},
 				}),
 				ParsedLineKind::Rec(tv) =>
@@ -119,55 +175,45 @@ pub enum ParsedLineKind {
 }
 
 pub struct ParsedMem {
-	name: Tok<String>,
+	pub name: IStr,
-	exported: bool,
+	pub exported: bool,
-	kind: ParsedMemKind,
+	pub kind: ParsedMemKind,
 }
 
 pub enum ParsedMemKind {
 	Const(BoxConstCallback),
-	Mod(Vec<ParsedLine>),
+	Mod { lines: Vec<ParsedLine>, use_prelude: bool },
 }
 
-impl ParsedMemKind {
+#[derive(Clone)]
-	pub fn cnst<F: AsyncFnOnce(ConstCtx) -> GExpr + 'static>(f: F) -> Self {
-		Self::Const(Box::new(|ctx| Box::pin(f(ctx))))
-	}
-}
-
-/* TODO: how the macro runner uses the multi-stage loader
-
-Since the macro runner actually has to invoke the interpreter,
-it'll run at const-time and not at postparse-time anyway.
-
-pasing stage establishes the role of every constant as a macro keyword
-postparse / const load links up constants with every macro they can directly invoke
-	the constants representing the keywords might not actually be postparsed,
-	\ the connection is instead made by detecting in the macro system that the
-	\ resolved name is owned by a macro
-	the returned constant from this call is always an entrypoint call to
-	\ the macro system
-	the constants representing the keywords resolve to panic
-execute relies on these links detected in the extension to dispatch relevant macros
-*/
-
 pub struct ConstCtx {
 	ctx: SysCtx,
 	constid: api::ParsedConstId,
 }
 impl ConstCtx {
-	pub async fn names<const N: usize>(&self, names: [&Sym; N]) -> [Option<Sym>; N] {
+	pub fn ctx(&self) -> &SysCtx { &self.ctx }
-		let resolve_names = ResolveNames {
+	pub fn i(&self) -> &Interner { self.ctx.i() }
+	pub fn names<'b>(
+		&'b self,
+		names: impl IntoIterator<Item = &'b Sym> + 'b,
+	) -> impl Stream<Item = OrcRes<Sym>> + 'b {
+		let resolve_names = api::ResolveNames {
 			constid: self.constid,
 			sys: self.ctx.sys_id(),
 			names: names.into_iter().map(|n| n.to_api()).collect_vec(),
 		};
-		let names = self.ctx.reqnot().request(resolve_names).await;
+		stream(async |mut cx| {
-		let mut results = [const { None }; N];
+			for name_opt in self.ctx.reqnot().request(resolve_names).await {
-		for (i, name) in names.into_iter().enumerate().filter_map(|(i, n)| Some((i, n?))) {
+				cx.emit(match name_opt {
-			results[i] = Some(Sym::from_api(name, self.ctx.i()).await);
+					Err(e) => Err(OrcErrv::from_api(&e, self.ctx.i()).await),
-		}
+					Ok(name) => Ok(Sym::from_api(name, self.ctx.i()).await),
-		results
+				})
+				.await
+			}
+		})
+	}
+	pub async fn names_n<const N: usize>(&self, names: [&Sym; N]) -> [OrcRes<Sym>; N] {
+		self.names(names).collect::<Vec<_>>().await.try_into().expect("Lengths must match")
 	}
 }
 

orchid-extension/src/reflection.rs

@@ -0,0 +1,163 @@
+use std::cell::OnceCell;
+use std::rc::Rc;
+
+use futures::FutureExt;
+use futures::lock::Mutex;
+use memo_map::MemoMap;
+use orchid_base::interner::Tok;
+use orchid_base::name::{NameLike, VPath};
+use orchid_base::reqnot::Requester;
+
+use crate::api;
+use crate::system::{SysCtx, SysCtxEntry, WeakSysCtx};
+
+pub struct ReflMemData {
+	// None for inferred steps
+	public: OnceCell<bool>,
+	kind: ReflMemKind,
+}
+#[derive(Clone)]
+pub struct ReflMem(Rc<ReflMemData>);
+impl ReflMem {
+	pub fn kind(&self) -> ReflMemKind { self.0.kind.clone() }
+}
+
+#[derive(Clone)]
+pub enum ReflMemKind {
+	Const,
+	Mod(ReflMod),
+}
+
+pub struct ReflModData {
+	inferred: Mutex<bool>,
+	path: VPath,
+	ctx: WeakSysCtx,
+	members: MemoMap<IStr, ReflMem>,
+}
+
+#[derive(Clone)]
+pub struct ReflMod(Rc<ReflModData>);
+impl ReflMod {
+	fn ctx(&self) -> SysCtx {
+		self.0.ctx.upgrade().expect("ReflectedModule accessed after context drop")
+	}
+	pub fn path(&self) -> &[IStr] { &self.0.path[..] }
+	pub fn is_root(&self) -> bool { self.0.path.is_empty() }
+	async fn try_populate(&self) -> Result<(), api::LsModuleError> {
+		let ctx = self.ctx();
+		let path_tok = ctx.i().i(&self.0.path[..]).await;
+		let reply = match ctx.reqnot().request(api::LsModule(ctx.sys_id(), path_tok.to_api())).await {
+			Err(api::LsModuleError::TreeUnavailable) =>
+				panic!("Reflected tree accessed outside an interpreter call. This extension is faulty."),
+			Err(err) => return Err(err),
+			Ok(details) => details,
+		};
+		for (k, v) in reply.members {
+			let k = ctx.i().ex(k).await;
+			let mem = match self.0.members.get(&k) {
+				Some(mem) => mem,
+				None => {
+					let path = self.0.path.clone().name_with_suffix(k.clone()).to_sym(ctx.i()).await;
+					let kind = match v.kind {
+						api::MemberInfoKind::Constant => ReflMemKind::Const,
+						api::MemberInfoKind::Module =>
+							ReflMemKind::Mod(default_module(&ctx, VPath::new(path.segs()))),
+					};
+					self.0.members.get_or_insert(&k, || default_member(self.is_root(), kind))
+				},
+			};
+			let _ = mem.0.public.set(v.public);
+		}
+		Ok(())
+	}
+	pub async fn get_child(&self, key: &IStr) -> Option<ReflMem> {
+		let inferred_g = self.0.inferred.lock().await;
+		if let Some(mem) = self.0.members.get(key) {
+			return Some(mem.clone());
+		}
+		if !*inferred_g {
+			return None;
+		}
+		match self.try_populate().await {
+			Err(api::LsModuleError::InvalidPath) =>
+				panic!("Path became invalid since module was created"),
+			Err(api::LsModuleError::IsConstant) =>
+				panic!("Path previously contained a module but now contains a constant"),
+			Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
+			Ok(()) => (),
+		}
+		self.0.members.get(key).cloned()
+	}
+	pub async fn get_by_path(&self, path: &[IStr]) -> Result<ReflMem, InvalidPathError> {
+		let ctx = self.ctx();
+		let (next, tail) = path.split_first().expect("Attempted to walk by empty path");
+		let inferred_g = self.0.inferred.lock().await;
+		if let Some(next) = self.0.members.get(next) {
+			return if tail.is_empty() {
+				Ok(next.clone())
+			} else {
+				match next.kind() {
+					ReflMemKind::Const => Err(InvalidPathError { keep_ancestry: true }),
+					ReflMemKind::Mod(m) => m.get_by_path(tail).boxed_local().await,
+				}
+			};
+		}
+		if !*inferred_g {
+			return Err(InvalidPathError { keep_ancestry: true });
+		}
+		let candidate = default_module(&ctx, self.0.path.clone().suffix([next.clone()]));
+		if tail.is_empty() {
+			return match candidate.try_populate().await {
+				Ok(()) => {
+					let tgt_mem = default_member(self.is_root(), ReflMemKind::Mod(candidate));
+					self.0.members.insert(next.clone(), tgt_mem.clone());
+					Ok(tgt_mem)
+				},
+				Err(api::LsModuleError::InvalidPath) => Err(InvalidPathError { keep_ancestry: false }),
+				Err(api::LsModuleError::IsConstant) => {
+					let const_mem = default_member(self.is_root(), ReflMemKind::Const);
+					self.0.members.insert(next.clone(), const_mem);
+					Err(InvalidPathError { keep_ancestry: true })
+				},
+				Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
+			};
+		}
+		match candidate.get_by_path(tail).boxed_local().await {
+			e @ Err(InvalidPathError { keep_ancestry: false }) => e,
+			res @ Err(InvalidPathError { keep_ancestry: true }) | res @ Ok(_) => {
+				let tgt_mem = default_member(self.is_root(), ReflMemKind::Mod(candidate));
+				self.0.members.insert(next.clone(), tgt_mem);
+				res
+			},
+		}
+	}
+}
+
+struct ReflRoot(ReflMod);
+impl SysCtxEntry for ReflRoot {}
+
+pub struct InvalidPathError {
+	keep_ancestry: bool,
+}
+
+fn default_module(ctx: &SysCtx, path: VPath) -> ReflMod {
+	ReflMod(Rc::new(ReflModData {
+		ctx: ctx.downgrade(),
+		inferred: Mutex::new(true),
+		path,
+		members: MemoMap::new(),
+	}))
+}
+
+fn default_member(is_root: bool, kind: ReflMemKind) -> ReflMem {
+	ReflMem(Rc::new(ReflMemData {
+		public: if is_root { true.into() } else { OnceCell::new() },
+		kind,
+	}))
+}
+
+fn get_root(ctx: &SysCtx) -> &ReflRoot {
+	ctx.get_or_insert(|| ReflRoot(default_module(ctx, VPath::new([]))))
+}
+
+pub fn refl(ctx: &SysCtx) -> ReflMod { get_root(ctx).0.clone() }

orchid-extension/src/system.rs

@@ -3,22 +3,23 @@ use std::fmt;
 use std::future::Future;
 use std::num::NonZero;
 use std::pin::Pin;
-use std::rc::Rc;
+use std::rc::{Rc, Weak};
 
 use futures::future::LocalBoxFuture;
 use memo_map::MemoMap;
-use orchid_api::ExtMsgSet;
 use orchid_api_traits::{Coding, Decode};
 use orchid_base::boxed_iter::BoxedIter;
 use orchid_base::builtin::Spawner;
 use orchid_base::interner::Interner;
 use orchid_base::logging::Logger;
+use orchid_base::name::Sym;
 use orchid_base::reqnot::{Receipt, ReqNot};
 
 use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TypAtom, get_info};
+use crate::coroutine_exec::Replier;
 use crate::entrypoint::ExtReq;
-use crate::func_atom::Fun;
+use crate::func_atom::{Fun, Lambda};
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
 use crate::system_ctor::{CtedObj, SystemCtor};
@@ -35,14 +36,14 @@ pub trait DynSystemCard: Send + Sync + 'static {
 	fn name(&self) -> &'static str;
 	/// Atoms explicitly defined by the system card. Do not rely on this for
 	/// querying atoms as it doesn't include the general atom types
-	fn atoms(&self) -> BoxedIter<Option<Box<dyn AtomDynfo>>>;
+	fn atoms(&'_ self) -> BoxedIter<'_, Option<Box<dyn AtomDynfo>>>;
 }
 
 /// Atoms supported by this package which may appear in all extensions.
 /// The indices of these are bitwise negated, such that the MSB of an atom index
 /// marks whether it belongs to this package (0) or the importer (1)
 fn general_atoms() -> impl Iterator<Item = Option<Box<dyn AtomDynfo>>> {
-	[Some(Fun::dynfo())].into_iter()
+	[Some(Fun::dynfo()), Some(Lambda::dynfo()), Some(Replier::dynfo())].into_iter()
 }
 
 pub fn atom_info_for(
@@ -76,11 +77,14 @@ pub async fn resolv_atom(
 
 impl<T: SystemCard> DynSystemCard for T {
 	fn name(&self) -> &'static str { T::Ctor::NAME }
-	fn atoms(&self) -> BoxedIter<Option<Box<dyn AtomDynfo>>> { Box::new(Self::atoms().into_iter()) }
+	fn atoms(&'_ self) -> BoxedIter<'_, Option<Box<dyn AtomDynfo>>> {
+		Box::new(Self::atoms().into_iter())
+	}
 }
 
 /// System as defined by author
 pub trait System: Send + Sync + SystemCard + 'static {
+	fn prelude(i: &Interner) -> impl Future<Output = Vec<Sym>>;
 	fn env() -> Vec<GenMember>;
 	fn lexers() -> Vec<LexerObj>;
 	fn parsers() -> Vec<ParserObj>;
@@ -88,7 +92,8 @@ pub trait System: Send + Sync + SystemCard + 'static {
 }
 
 pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
-	fn dyn_env(&self) -> Vec<GenMember>;
+	fn dyn_prelude<'a>(&'a self, i: &'a Interner) -> LocalBoxFuture<'a, Vec<Sym>>;
+	fn dyn_env(&'_ self) -> Vec<GenMember>;
 	fn dyn_lexers(&self) -> Vec<LexerObj>;
 	fn dyn_parsers(&self) -> Vec<ParserObj>;
 	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>>;
@@ -96,7 +101,10 @@ pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
 }
 
 impl<T: System> DynSystem for T {
-	fn dyn_env(&self) -> Vec<GenMember> { Self::env() }
+	fn dyn_prelude<'a>(&'a self, i: &'a Interner) -> LocalBoxFuture<'a, Vec<Sym>> {
+		Box::pin(Self::prelude(i))
+	}
+	fn dyn_env(&'_ self) -> Vec<GenMember> { Self::env() }
 	fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
 	fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
 	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>> {
@@ -126,24 +134,14 @@ where A: AtomicFeatures {
 	}
 	let val = dynfo.decode(AtomCtx(data, foreign.atom.drop, ctx)).await;
 	let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
-	Ok(TypAtom { value, data: foreign })
+	Ok(TypAtom { value, untyped: foreign })
 }
 
-// #[derive(Clone)]
+#[derive(Clone)]
-// pub struct SysCtx {
+pub struct WeakSysCtx(Weak<MemoMap<TypeId, Box<dyn Any>>>);
-// 	pub reqnot: ReqNot<api::ExtMsgSet>,
+impl WeakSysCtx {
-// 	pub spawner: Spawner,
+	pub fn upgrade(&self) -> Option<SysCtx> { Some(SysCtx(self.0.upgrade()?)) }
-// 	pub id: api::SysId,
+}
-// 	pub cted: CtedObj,
-// 	pub logger: Logger,
-// 	pub obj_store: ObjStore,
-// 	pub i: Rc<Interner>,
-// }
-// impl fmt::Debug for SysCtx {
-// 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-// 		write!(f, "SysCtx({:?})", self.id)
-// 	}
-// }
 
 #[derive(Clone)]
 pub struct SysCtx(Rc<MemoMap<TypeId, Box<dyn Any>>>);
@@ -151,7 +149,7 @@ impl SysCtx {
 	pub fn new(
 		id: api::SysId,
 		i: Interner,
-		reqnot: ReqNot<ExtMsgSet>,
+		reqnot: ReqNot<api::ExtMsgSet>,
 		spawner: Spawner,
 		logger: Logger,
 		cted: CtedObj,
@@ -160,6 +158,7 @@ impl SysCtx {
 		this.add(id).add(i).add(reqnot).add(spawner).add(logger).add(cted);
 		this
 	}
+	pub fn downgrade(&self) -> WeakSysCtx { WeakSysCtx(Rc::downgrade(&self.0)) }
 	pub fn add<T: SysCtxEntry>(&self, t: T) -> &Self {
 		assert!(self.0.insert(TypeId::of::<T>(), Box::new(t)), "Key already exists");
 		self
@@ -168,13 +167,7 @@ impl SysCtx {
 		(self.0.get_or_insert_owned(TypeId::of::<T>(), || Box::new(f())).downcast_ref())
 			.expect("Keyed by TypeId")
 	}
-	pub fn get_or_default<T: SysCtxEntry + Default>(&self) -> &T {
+	pub fn get_or_default<T: SysCtxEntry + Default>(&self) -> &T { self.get_or_insert(T::default) }
-		self.get_or_insert(|| {
-			let rc_id = self.0.as_ref() as *const _ as *const () as usize;
-			eprintln!("Default-initializing {} in {}", type_name::<T>(), rc_id);
-			T::default()
-		})
-	}
 	pub fn try_get<T: SysCtxEntry>(&self) -> Option<&T> {
 		Some(self.0.get(&TypeId::of::<T>())?.downcast_ref().expect("Keyed by TypeId"))
 	}
@@ -184,7 +177,7 @@ impl SysCtx {
 	/// Shorthand to get the [Interner] instance
 	pub fn i(&self) -> &Interner { self.get::<Interner>() }
 	/// Shorthand to get the messaging link
-	pub fn reqnot(&self) -> &ReqNot<ExtMsgSet> { self.get::<ReqNot<ExtMsgSet>>() }
+	pub fn reqnot(&self) -> &ReqNot<api::ExtMsgSet> { self.get::<ReqNot<api::ExtMsgSet>>() }
 	/// Shorthand to get the system ID
 	pub fn sys_id(&self) -> api::SysId { *self.get::<api::SysId>() }
 	/// Shorthand to get the task spawner callback

orchid-extension/src/system_ctor.rs

@@ -62,7 +62,7 @@ pub trait SystemCtor: Send + Sync + 'static {
 	type Instance: System;
 	const NAME: &'static str;
 	const VERSION: f64;
-	fn inst() -> Option<Self::Instance>;
+	fn inst(deps: <Self::Deps as DepDef>::Sat) -> Self::Instance;
 }
 
 pub trait DynSystemCtor: Send + Sync + 'static {
@@ -81,8 +81,8 @@ impl<T: SystemCtor> DynSystemCtor for T {
 	}
 	fn new_system(&self, api::NewSystem { system: _, id: _, depends }: &api::NewSystem) -> CtedObj {
 		let mut ids = depends.iter().copied();
-		let inst = Arc::new(T::inst().expect("Constructor did not create system"));
 		let deps = T::Deps::create(&mut || ids.next().unwrap());
+		let inst = Arc::new(T::inst(deps.clone()));
 		Arc::new(Cted::<T> { deps, inst })
 	}
 }

orchid-extension/src/tokio.rs

@@ -2,25 +2,28 @@ use crate::entrypoint::ExtensionData;
 
 #[cfg(feature = "tokio")]
 pub async fn tokio_main(data: ExtensionData) {
-	use std::io::Write;
+	use std::io::{ErrorKind, Write};
 	use std::mem;
-	use std::pin::Pin;
+	use std::pin::{Pin, pin};
 	use std::rc::Rc;
 
-	use async_std::io;
+	use async_once_cell::OnceCell;
 	use futures::StreamExt;
 	use futures::future::LocalBoxFuture;
+	use futures::lock::Mutex;
 	use futures::stream::FuturesUnordered;
 	use orchid_api_traits::{Decode, Encode};
+	use orchid_base::msg::{recv_msg, send_msg};
+	use tokio::io::{Stdout, stdin, stdout};
 	use tokio::task::{LocalSet, spawn_local};
+	use tokio_util::compat::{Compat, TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
 
 	use crate::api;
 	use crate::entrypoint::extension_init;
-	use crate::msg::{recv_parent_msg, send_parent_msg};
 
 	let local_set = LocalSet::new();
 	local_set.spawn_local(async {
-		let host_header = api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await;
+		let host_header = api::HostHeader::decode(Pin::new(&mut stdin().compat())).await;
 		let init =
 			Rc::new(extension_init(data, host_header, Rc::new(|fut| mem::drop(spawn_local(fut)))));
 		let mut buf = Vec::new();
@@ -32,17 +35,20 @@ pub async fn tokio_main(data: ExtensionData) {
 		let mut io = FuturesUnordered::<LocalBoxFuture<()>>::new();
 		io.push(Box::pin(async {
 			loop {
-				match recv_parent_msg().await {
+				match recv_msg(pin!(stdin().compat())).await {
 					Ok(msg) => init.send(&msg[..]).await,
-					Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break,
+					Err(e) if e.kind() == ErrorKind::BrokenPipe => break,
-					Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => break,
+					Err(e) if e.kind() == ErrorKind::UnexpectedEof => break,
 					Err(e) => panic!("{e}"),
 				}
 			}
 		}));
 		io.push(Box::pin(async {
 			while let Some(msg) = init.recv().await {
-				send_parent_msg(&msg[..]).await.unwrap();
+				static STDOUT: OnceCell<Mutex<Compat<Stdout>>> = OnceCell::new();
+				let stdout_lk = STDOUT.get_or_init(async { Mutex::new(stdout().compat_write()) }).await;
+				let mut stdout_g = stdout_lk.lock().await;
+				send_msg(pin!(&mut *stdout_g), &msg[..]).await.expect("Parent pipe broken");
 			}
 		}));
 		io.next().await;

orchid-extension/src/tree.rs

@@ -1,7 +1,6 @@
 use std::num::NonZero;
-use std::rc::Rc;
 
-use async_stream::stream;
+use async_fn_stream::stream;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, StreamExt};
@@ -18,9 +17,9 @@ use trait_set::trait_set;
 use crate::api;
 use crate::conv::ToExpr;
 use crate::entrypoint::MemberRecord;
-use crate::expr::{Expr, ExprHandle};
+use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
 use crate::func_atom::{ExprFunc, Fun};
-use crate::gen_expr::{GExpr, arg, call, lambda, seq, sym_ref};
+use crate::gen_expr::{GExpr, sym_ref};
 use crate::system::SysCtx;
 
 pub type GenTokTree = TokTree<Expr, GExpr>;
@@ -43,39 +42,38 @@ impl TokenVariant<api::Expression> for GExpr {
 }
 
 impl TokenVariant<api::ExprTicket> for Expr {
-	type FromApiCtx<'a> = SysCtx;
+	type FromApiCtx<'a> = (SysCtx, &'a BorrowedExprStore);
 	async fn from_api(
 		api: &api::ExprTicket,
-		ctx: &mut Self::FromApiCtx<'_>,
+		(ctx, exprs): &mut Self::FromApiCtx<'_>,
 		_: SrcRange,
 		_: &Interner,
 	) -> Self {
-		// SAFETY: receiving trees from sublexers implies ownership transfer
+		// SAFETY: receiving trees from sublexers implies borrowing
-		Expr::from_handle(Rc::new(unsafe { ExprHandle::from_args(ctx.clone(), *api) }))
+		Expr::from_handle(ExprHandle::borrowed(ctx.clone(), *api, exprs))
 	}
 	type ToApiCtx<'a> = ();
-	async fn into_api(self, (): &mut Self::ToApiCtx<'_>) -> api::ExprTicket {
+	async fn into_api(self, (): &mut Self::ToApiCtx<'_>) -> api::ExprTicket { self.handle().tk }
-		let hand = self.handle();
-		std::mem::drop(self);
-		let h = match Rc::try_unwrap(hand) {
-			Ok(h) => h,
-			Err(h) => h.as_ref().clone().await,
-		};
-		h.into_tk()
-	}
 }
 
-pub fn x_tok(x: impl ToExpr) -> GenTok { GenTok::NewExpr(x.to_expr()) }
+pub async fn x_tok(x: impl ToExpr) -> GenTok { GenTok::NewExpr(x.to_expr().await) }
-pub fn ref_tok(path: Sym) -> GenTok { GenTok::NewExpr(sym_ref(path)) }
+pub async fn ref_tok(path: Sym) -> GenTok { GenTok::NewExpr(sym_ref(path)) }
 
-pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenMember> {
+pub fn lazy(
+	public: bool,
+	name: &str,
+	cb: impl AsyncFnOnce(Sym, SysCtx) -> MemKind + Clone + 'static,
+) -> Vec<GenMember> {
 	vec![GenMember {
 		name: name.to_string(),
-		kind: MemKind::Const(value.to_expr()),
+		kind: MemKind::Lazy(LazyMemberFactory::new(cb)),
 		comments: vec![],
 		public,
 	}]
 }
+pub fn cnst(public: bool, name: &str, value: impl ToExpr + Clone + 'static) -> Vec<GenMember> {
+	lazy(public, name, async |_, _| MemKind::Const(value.to_expr().await))
+}
 pub fn module(
 	public: bool,
 	name: &str,
@@ -89,19 +87,8 @@ pub fn root_mod(name: &str, mems: impl IntoIterator<Item = Vec<GenMember>>) -> (
 	(name.to_string(), kind)
 }
 pub fn fun<I, O>(public: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenMember> {
-	let fac = LazyMemberFactory::new(move |sym, ctx| async {
+	let fac = LazyMemberFactory::new(async move |sym, ctx| {
-		return MemKind::Const(build_lambdas(Fun::new(sym, ctx, xf).await, 0));
+		MemKind::Const(Fun::new(sym, ctx, xf).await.to_expr().await)
-		fn build_lambdas(fun: Fun, i: u64) -> GExpr {
-			if i < fun.arity().into() {
-				return lambda(i, [build_lambdas(fun, i + 1)]);
-			}
-			let arity = fun.arity();
-			seq(
-				(0..arity)
-					.map(|i| arg(i as u64))
-					.chain([call([fun.to_expr()].into_iter().chain((0..arity).map(|i| arg(i as u64))))]),
-			)
-		}
 	});
 	vec![GenMember { name: name.to_string(), kind: MemKind::Lazy(fac), public, comments: vec![] }]
 }
@@ -203,8 +190,14 @@ impl MemKind {
 			Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
 			Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys(), ctx.req()).await),
 			Self::Mod { members } => api::MemberKind::Module(api::Module {
-				members: Box::pin(stream! { for m in members { yield m.into_api(ctx).await } }.collect())
+				members: stream(async |mut cx| {
-					.await,
+					for m in members {
+						cx.emit(m.into_api(ctx).await).await
+					}
+				})
+				.collect()
+				.boxed_local()
+				.await,
 			}),
 		}
 	}
@@ -213,21 +206,21 @@ impl MemKind {
 pub trait TreeIntoApiCtx {
 	fn sys(&self) -> SysCtx;
 	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
-	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
+	fn push_path(&mut self, seg: IStr) -> impl TreeIntoApiCtx;
 	fn req(&self) -> &impl ReqHandlish;
 }
 
 pub struct TreeIntoApiCtxImpl<'a, 'b, RH: ReqHandlish> {
 	pub sys: SysCtx,
-	pub basepath: &'a [Tok<String>],
+	pub basepath: &'a [IStr],
-	pub path: Substack<'a, Tok<String>>,
+	pub path: Substack<'a, IStr>,
 	pub lazy_members: &'b mut HashMap<api::TreeId, MemberRecord>,
 	pub req: &'a RH,
 }
 
 impl<RH: ReqHandlish> TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_, RH> {
 	fn sys(&self) -> SysCtx { self.sys.clone() }
-	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx {
+	fn push_path(&mut self, seg: IStr) -> impl TreeIntoApiCtx {
 		TreeIntoApiCtxImpl {
 			req: self.req,
 			lazy_members: self.lazy_members,

orchid-host/Cargo.toml

@@ -6,14 +6,14 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
+async-lock = "3.4.1"
 async-once-cell = "0.5.4"
-async-process = "2.3.0"
+async-process = "2.4.0"
-async-std = "1.13.0"
-async-stream = "0.3.6"
 bound = "0.6.0"
 derive_destructure = "1.0.0"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.15.2"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
 memo-map = "0.3.3"
@@ -23,7 +23,7 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 ordered-float = "5.0.0"
-paste = "1.0.15"
+pastey = "0.1.1"
 substack = "1.1.1"
-test_executors = "0.3.2"
+test_executors = "0.3.5"
 trait-set = "0.3.0"

orchid-host/src/atom.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 use std::rc::{Rc, Weak};
 
+use async_lock::OnceCell;
 use derive_destructure::destructure;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, take_first_fmt};
 use orchid_base::location::Pos;
@@ -9,7 +10,7 @@ use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::ctx::Ctx;
-use crate::expr::Expr;
+use crate::expr::{Expr, ExprParseCtx, PathSetBuilder};
 use crate::extension::Extension;
 use crate::system::System;
 
@@ -18,11 +19,12 @@ pub struct AtomData {
 	owner: System,
 	drop: Option<api::AtomId>,
 	data: Vec<u8>,
+	pub(crate) display: OnceCell<FmtUnit>,
 }
 impl AtomData {
 	#[must_use]
 	fn api(self) -> api::Atom {
-		let (owner, drop, data) = self.destructure();
+		let (owner, drop, data, _display) = self.destructure();
 		api::Atom { data, drop, owner: owner.id() }
 	}
 	#[must_use]
@@ -50,41 +52,28 @@ impl fmt::Debug for AtomData {
 #[derive(Clone, Debug)]
 pub struct AtomHand(Rc<AtomData>);
 impl AtomHand {
-	#[must_use]
+	pub(crate) fn new(data: Vec<u8>, owner: System, drop: Option<api::AtomId>) -> Self {
-	pub(crate) async fn new(api::Atom { data, drop, owner }: api::Atom, ctx: &Ctx) -> Self {
+		Self(Rc::new(AtomData { owner, drop, data, display: OnceCell::new() }))
-		let create = || async {
-			let owner = ctx.system_inst(owner).await.expect("Dropped system created atom");
-			AtomHand(Rc::new(AtomData { data, owner, drop }))
-		};
-		if let Some(id) = drop {
-			let mut owned_g = ctx.owned_atoms.write().await;
-			if let Some(data) = owned_g.get(&id) {
-				if let Some(atom) = data.upgrade() {
-					return atom;
-				}
-			}
-			let new = create().await;
-			owned_g.insert(id, new.downgrade());
-			new
-		} else {
-			create().await
-		}
 	}
 	#[must_use]
-	pub async fn call(self, arg: Expr) -> api::Expression {
+	pub async fn call(self, arg: Expr) -> Expr {
 		let owner_sys = self.0.owner.clone();
 		let reqnot = owner_sys.reqnot();
 		owner_sys.ext().exprs().give_expr(arg.clone());
-		match Rc::try_unwrap(self.0) {
+		let ret = match Rc::try_unwrap(self.0) {
 			Ok(data) => reqnot.request(api::FinalCall(data.api(), arg.id())).await,
 			Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await,
-		}
+		};
+		let mut parse_ctx = ExprParseCtx { ctx: owner_sys.ctx(), exprs: owner_sys.ext().exprs() };
+		let val = Expr::from_api(&ret, PathSetBuilder::new(), &mut parse_ctx).await;
+		owner_sys.ext().exprs().take_expr(arg.id());
+		val
 	}
 	#[must_use]
 	pub fn sys(&self) -> &System { &self.0.owner }
 	#[must_use]
 	pub fn ext(&self) -> &Extension { self.sys().ext() }
-	pub async fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
+	pub async fn req(&self, key: api::TVec, req: Vec<u8>) -> Option<Vec<u8>> {
 		self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await
 	}
 	#[must_use]
@@ -96,13 +85,19 @@ impl AtomHand {
 }
 impl Format for AtomHand {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		FmtUnit::from_api(&self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await)
+		(self.0.display.get_or_init(|| async {
+			FmtUnit::from_api(&self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await)
+		}))
+		.await
+		.clone()
 	}
 }
 impl AtomRepr for AtomHand {
 	type Ctx = Ctx;
-	async fn from_api(atom: &orchid_api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
+	async fn from_api(atom: &api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
-		Self::new(atom.clone(), ctx).await
+		let api::Atom { data, drop, owner } = atom.clone();
+		let sys = ctx.system_inst(owner).await.expect("Dropped system created atom");
+		if let Some(id) = drop { sys.new_atom(data, id).await } else { AtomHand::new(data, sys, drop) }
 	}
 	async fn to_api(&self) -> orchid_api::Atom { self.api_ref() }
 }

orchid-host/src/ctx.rs

@@ -3,14 +3,12 @@ use std::num::{NonZero, NonZeroU16};
 use std::rc::{Rc, Weak};
 use std::{fmt, ops};
 
-use async_std::sync::RwLock;
+use async_lock::RwLock;
 use hashbrown::HashMap;
-use orchid_api::SysId;
 use orchid_base::builtin::Spawner;
 use orchid_base::interner::Interner;
 
 use crate::api;
-use crate::atom::WeakAtomHand;
 use crate::expr_store::ExprStore;
 use crate::system::{System, WeakSystem};
 use crate::tree::WeakRoot;
@@ -20,7 +18,6 @@ pub struct CtxData {
 	pub spawn: Spawner,
 	pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
 	pub system_id: RefCell<NonZeroU16>,
-	pub owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
 	pub common_exprs: ExprStore,
 	pub root: RwLock<WeakRoot>,
 }
@@ -46,7 +43,6 @@ impl Ctx {
 			i: Interner::default(),
 			systems: RwLock::default(),
 			system_id: RefCell::new(NonZero::new(1).unwrap()),
-			owned_atoms: RwLock::default(),
 			common_exprs: ExprStore::default(),
 			root: RwLock::default(),
 		}))
@@ -59,7 +55,7 @@ impl Ctx {
 	pub(crate) fn next_sys_id(&self) -> api::SysId {
 		let mut g = self.system_id.borrow_mut();
 		*g = g.checked_add(1).unwrap_or(NonZeroU16::new(1).unwrap());
-		SysId(*g)
+		api::SysId(*g)
 	}
 	#[must_use]
 	pub fn downgrade(&self) -> WeakCtx { WeakCtx(Rc::downgrade(&self.0)) }

orchid-host/src/dealias.rs

@@ -1,6 +1,6 @@
 use hashbrown::HashSet;
 use itertools::Itertools;
-use orchid_base::error::{OrcErr, OrcRes, Reporter, mk_err, mk_errv};
+use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_err};
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::Pos;
 use orchid_base::name::VName;
@@ -16,7 +16,7 @@ pub enum AbsPathError {
 	RootPath,
 }
 impl AbsPathError {
-	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErr {
+	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErrv {
 		let (descr, msg) = match self {
 			AbsPathError::RootPath => (
 				i.i("Path ends on root module").await,
@@ -30,7 +30,7 @@ impl AbsPathError {
 				format!("{path} is leading outside the root."),
 			),
 		};
-		mk_err(descr, msg, [pos.into()])
+		mk_err(descr, msg, [pos])
 	}
 }
 
@@ -42,8 +42,8 @@ impl AbsPathError {
 /// if the relative path contains as many or more `super` segments than the
 /// length of the absolute path.
 pub async fn absolute_path(
-	mut cwd: &[Tok<String>],
+	mut cwd: &[IStr],
-	mut rel: &[Tok<String>],
+	mut rel: &[IStr],
 	i: &Interner,
 ) -> Result<VName, AbsPathError> {
 	let i_self = i.i("self").await;
@@ -67,13 +67,13 @@ pub struct DealiasCtx<'a> {
 }
 
 pub async fn resolv_glob<Mod: Tree>(
-	cwd: &[Tok<String>],
+	cwd: &[IStr],
 	root: &Mod,
-	abs_path: &[Tok<String>],
+	abs_path: &[IStr],
 	pos: Pos,
 	i: &Interner,
 	ctx: &mut Mod::Ctx<'_>,
-) -> OrcRes<HashSet<Tok<String>>> {
+) -> OrcRes<HashSet<IStr>> {
 	let coprefix_len = cwd.iter().zip(abs_path).take_while(|(a, b)| a == b).count();
 	let (co_prefix, diff_path) = abs_path.split_at(abs_path.len().min(coprefix_len + 1));
 	let fst_diff =
@@ -87,7 +87,7 @@ pub async fn resolv_glob<Mod: Tree>(
 				ChildErrorKind::Missing => ("Invalid import path", format!("{path} not found")),
 				ChildErrorKind::Private => ("Import inaccessible", format!("{path} is private")),
 			};
-			return Err(mk_errv(i.i(tk).await, msg, [pos]));
+			return Err(mk_err(i.i(tk).await, msg, [pos]));
 		},
 	};
 	Ok(target_module.children(coprefix_len < abs_path.len()))
@@ -98,11 +98,11 @@ pub type ChildResult<'a, T> = Result<&'a T, ChildErrorKind>;
 pub trait Tree {
 	type Ctx<'a>;
 	#[must_use]
-	fn children(&self, public_only: bool) -> HashSet<Tok<String>>;
+	fn children(&self, public_only: bool) -> HashSet<IStr>;
 	#[must_use]
 	fn child(
 		&self,
-		key: Tok<String>,
+		key: IStr,
 		public_only: bool,
 		ctx: &mut Self::Ctx<'_>,
 	) -> impl Future<Output = ChildResult<'_, Self>>;
@@ -133,7 +133,7 @@ pub struct ChildError {
 pub async fn walk<'a, T: Tree>(
 	root: &'a T,
 	public_only: bool,
-	path: impl IntoIterator<Item = Tok<String>>,
+	path: impl IntoIterator<Item = IStr>,
 	ctx: &mut T::Ctx<'_>,
 ) -> Result<&'a T, ChildError> {
 	let mut cur = root;

orchid-host/src/execute.rs

@@ -1,6 +1,6 @@
 use std::mem;
 
-use async_std::sync::RwLockWriteGuard;
+use async_lock::RwLockWriteGuard;
 use bound::Bound;
 use futures::FutureExt;
 use orchid_base::error::OrcErrv;
@@ -9,7 +9,7 @@ use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
 
 use crate::ctx::Ctx;
-use crate::expr::{Expr, ExprKind, ExprParseCtx, PathSet, PathSetBuilder, Step};
+use crate::expr::{Expr, ExprKind, PathSet, Step};
 use crate::tree::Root;
 
 type ExprGuard = Bound<RwLockWriteGuard<'static, ExprKind>, Expr>;
@@ -109,11 +109,8 @@ impl ExecCtx {
 				ExprKind::Call(f, x) if !self.did_pop => (ExprKind::Call(f.clone(), x), StackOp::Push(f)),
 				ExprKind::Call(f, x) => match f.try_into_owned_atom().await {
 					Ok(atom) => {
-						let ext = atom.sys().ext().clone();
 						let x_norm = self.unpack_ident(&x).await;
-						let mut parse_ctx = ExprParseCtx { ctx: &self.ctx, exprs: ext.exprs() };
+						let val = atom.call(x_norm).await;
-						let val =
-							Expr::from_api(&atom.call(x_norm).await, PathSetBuilder::new(), &mut parse_ctx).await;
 						(ExprKind::Identity(val.clone()), StackOp::Swap(val))
 					},
 					Err(f) => match &*f.kind().read().await {
@@ -121,15 +118,9 @@ impl ExecCtx {
 							panic!("This should not appear outside function bodies"),
 						ExprKind::Missing => panic!("Should have been replaced"),
 						ExprKind::Atom(a) => {
-							let ext = a.sys().ext().clone();
 							let x_norm = self.unpack_ident(&x).await;
-							let val = Expr::from_api(
+							let ret = a.clone().call(x_norm).await;
-								&a.clone().call(x_norm).await,
+							(ExprKind::Identity(ret.clone()), StackOp::Swap(ret))
-								PathSetBuilder::new(),
-								&mut ExprParseCtx { ctx: ext.ctx(), exprs: ext.exprs() },
-							)
-							.await;
-							(ExprKind::Identity(val.clone()), StackOp::Swap(val))
 						},
 						ExprKind::Bottom(exprv) => (ExprKind::Bottom(exprv.clone()), StackOp::Pop),
 						ExprKind::Lambda(None, body) =>

orchid-host/src/expr.rs

@@ -4,9 +4,8 @@ use std::num::NonZeroU64;
 use std::rc::{Rc, Weak};
 use std::{fmt, mem};
 
-use async_std::sync::RwLock;
+use async_lock::RwLock;
 use futures::FutureExt;
-use hashbrown::HashSet;
 use itertools::Itertools;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
@@ -116,19 +115,18 @@ impl Expr {
 }
 impl Format for Expr {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		return print_expr(self, c, &mut HashSet::new()).await;
+		return print_expr(self, c, Substack::Bottom).await;
 	}
 }
 async fn print_expr<'a>(
 	expr: &'a Expr,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
-	visited: &mut HashSet<api::ExprTicket>,
+	visited: Substack<'_, api::ExprTicket>,
 ) -> FmtUnit {
-	if visited.contains(&expr.id()) {
+	if visited.iter().any(|id| id == &expr.id()) {
 		return "CYCLIC_EXPR".to_string().into();
 	}
-	visited.insert(expr.id());
+	print_exprkind(&*expr.kind().read().await, c, visited.push(expr.id())).boxed_local().await
-	print_exprkind(&*expr.kind().read().await, c, visited).boxed_local().await
 }
 
 #[derive(Clone, Debug)]
@@ -152,13 +150,13 @@ impl ExprKind {
 }
 impl Format for ExprKind {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		print_exprkind(self, c, &mut HashSet::new()).await
+		print_exprkind(self, c, Substack::Bottom).await
 	}
 }
 async fn print_exprkind<'a>(
 	ek: &ExprKind,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
-	visited: &mut HashSet<api::ExprTicket>,
+	visited: Substack<'_, api::ExprTicket>,
 ) -> FmtUnit {
 	match &ek {
 		ExprKind::Arg => "Arg".to_string().into(),

orchid-host/src/expr_store.rs

@@ -28,9 +28,15 @@ impl ExprStore {
 			},
 		}
 	}
-	pub fn take_expr(&self, ticket: api::ExprTicket) {
+	pub fn take_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {
-		(self.0.exprs.borrow_mut().entry(ticket))
+		match self.0.exprs.borrow_mut().entry(ticket) {
-			.and_replace_entry_with(|_, (rc, rt)| (1 < rc).then_some((rc - 1, rt)));
+			Entry::Vacant(_) => None,
+			Entry::Occupied(oe) if oe.get().0 == 1 => Some(oe.remove().1),
+			Entry::Occupied(mut oe) => {
+				oe.get_mut().0 -= 1;
+				Some(oe.get().1.clone())
+			},
+		}
 	}
 	#[must_use]
 	pub fn get_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {

orchid-host/src/extension.rs

@@ -5,12 +5,12 @@ use std::num::NonZeroU64;
 use std::pin::pin;
 use std::rc::{Rc, Weak};
 
-use async_std::channel::{self, Sender};
+use async_fn_stream::stream;
-use async_std::sync::Mutex;
-use async_stream::stream;
 use derive_destructure::destructure;
+use futures::channel::mpsc::{Sender, channel};
 use futures::future::{join, join_all};
-use futures::{StreamExt, stream};
+use futures::lock::Mutex;
+use futures::{SinkExt, StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_api_traits::Request;
@@ -18,9 +18,11 @@ use orchid_base::builtin::ExtInit;
 use orchid_base::clone;
 use orchid_base::format::{FmtCtxImpl, Format};
 use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
 use orchid_base::name::Sym;
 use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _};
+use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::atom::AtomHand;
@@ -45,12 +47,12 @@ pub struct ExtensionData {
 	next_pars: RefCell<NonZeroU64>,
 	exprs: ExprStore,
 	exiting_snd: Sender<()>,
-	lex_recur: Mutex<HashMap<api::ParsId, channel::Sender<ReqPair<api::SubLex>>>>,
+	lex_recur: Mutex<HashMap<api::ParsId, Sender<ReqPair<api::SubLex>>>>,
 }
 impl Drop for ExtensionData {
 	fn drop(&mut self) {
 		let reqnot = self.reqnot.clone();
-		let exiting_snd = self.exiting_snd.clone();
+		let mut exiting_snd = self.exiting_snd.clone();
 		(self.ctx.spawn)(Box::pin(async move {
 			reqnot.notify(api::HostExtNotif::Exit).await;
 			exiting_snd.send(()).await.unwrap()
@@ -64,9 +66,9 @@ impl Extension {
 	pub fn new(init: ExtInit, logger: Logger, msg_logger: Logger, ctx: Ctx) -> io::Result<Self> {
 		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| {
 			let init = Rc::new(init);
-			let (exiting_snd, exiting_rcv) = channel::bounded::<()>(1);
+			let (exiting_snd, exiting_rcv) = channel::<()>(0);
 			(ctx.spawn)(clone!(init, weak, ctx; Box::pin(async move {
-				let rcv_stream = stream! { loop { yield init.recv().await } };
+				let rcv_stream = stream(async |mut cx| loop { cx.emit( init.recv().await).await });
 				let mut event_stream = pin!(stream::select(exiting_rcv.map(|()| None), rcv_stream));
 				while let Some(Some(msg)) = event_stream.next().await {
 					if let Some(reqnot) = weak.upgrade().map(|rc| rc.reqnot.clone()) {
@@ -100,7 +102,7 @@ impl Extension {
 							}
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
 								this.assert_own_sys(rel.0).await;
-								this.0.exprs.take_expr(rel.1)
+								this.0.exprs.take_expr(rel.1);
 							}
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
 								this.assert_own_sys(mov.dec).await;
@@ -147,13 +149,14 @@ impl Extension {
 										hand.handle(fw, &sys.request(body.clone()).await).await
 									},
 									api::ExtHostReq::SubLex(sl) => {
-										let (rep_in, rep_out) = channel::bounded(1);
+										let (rep_in, mut rep_out) = channel(0);
 										{
 											let lex_g = this.0.lex_recur.lock().await;
-											let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
+											let mut req_in =
+												lex_g.get(&sl.id).cloned().expect("Sublex for nonexistent lexid");
 											req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
 										}
-										hand.handle(&sl, &rep_out.recv().await.unwrap()).await
+										hand.handle(&sl, &rep_out.next().await.unwrap()).await
 									},
 									api::ExtHostReq::ExprReq(api::ExprReq::Inspect(
 										ins @ api::Inspect { target },
@@ -172,8 +175,8 @@ impl Extension {
 											let path = i.ex(path).await;
 											let root = (ctx.root.read().await.upgrade())
 												.expect("LSModule called when root isn't in context");
-											let root_data = &mut *root.0.write().await;
+											let root_data = &*root.0.read().await;
-											let mut walk_ctx = (ctx.clone(), &mut root_data.consts);
+											let mut walk_ctx = (ctx.clone(), &root_data.consts);
 											let module =
 												match walk(&root_data.root, false, path.iter().cloned(), &mut walk_ctx)
 													.await
@@ -188,7 +191,7 @@ impl Extension {
 												};
 											let mut members = std::collections::HashMap::new();
 											for (k, v) in &module.members {
-												let kind = match v.kind(ctx.clone(), &mut root_data.consts).await {
+												let kind = match v.kind(ctx.clone(), &root_data.consts).await {
 													MemberKind::Const => api::MemberInfoKind::Constant,
 													MemberKind::Module(_) => api::MemberInfoKind::Module,
 												};
@@ -206,16 +209,21 @@ impl Extension {
 											let sys = weak_sys.upgrade().expect("ResolveNames after sys drop");
 											sys.name_resolver(*constid).await
 										};
-										let mut responses = vec![const { None }; names.len()];
+										let responses = stream(async |mut cx| {
-										for (i, name) in names.iter().enumerate() {
+											for name in names {
-											if let Some(abs) = resolver(&ctx.i.ex(*name).await[..]).await {
+												cx.emit(match resolver(&ctx.i.ex(*name).await[..]).await {
-												responses[i] = Some(abs.to_sym(&ctx.i).await.to_api())
+													Ok(abs) => Ok(abs.to_sym(&ctx.i).await.to_api()),
+													Err(e) => Err(e.to_api()),
+												})
+												.await
 											}
-										}
+										})
+										.collect()
+										.await;
 										hand.handle(rn, &responses).await
 									},
 									api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
-										let atom = AtomHand::new(atom.clone(), &ctx).await;
+										let atom = AtomHand::from_api(atom, Pos::None, &mut ctx.clone()).await;
 										let unit = atom.print(&FmtCtxImpl { i: &this.ctx().i }).await;
 										hand.handle(eap, &unit.to_api()).await
 									},
@@ -252,7 +260,7 @@ impl Extension {
 	}
 	pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
-		source: Tok<String>,
+		source: IStr,
 		src: Sym,
 		pos: u32,
 		sys: api::SysId,
@@ -261,7 +269,7 @@ impl Extension {
 		// get unique lex ID
 		let id = api::ParsId(self.next_pars());
 		// create and register channel
-		let (req_in, req_out) = channel::bounded(1);
+		let (req_in, mut req_out) = channel(0);
 		self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released
 		let (ret, ()) = join(
 			async {
@@ -273,7 +281,7 @@ impl Extension {
 				res
 			},
 			async {
-				while let Ok(ReqPair(sublex, rep_in)) = req_out.recv().await {
+				while let Some(ReqPair(sublex, mut rep_in)) = req_out.next().await {
 					(rep_in.send(r(sublex.pos).await).await)
 						.expect("Response channel dropped while request pending")
 				}
@@ -293,6 +301,7 @@ impl Extension {
 	pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }
 }
 
+#[derive(Clone)]
 pub struct WeakExtension(Weak<ExtensionData>);
 impl WeakExtension {
 	#[must_use]

orchid-host/src/lex.rs

@@ -1,24 +1,23 @@
 use std::rc::Rc;
 
-use async_std::sync::Mutex;
 use futures::FutureExt;
-use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use futures::lock::Mutex;
+use orchid_base::error::{OrcErrv, OrcRes, mk_err};
 use orchid_base::interner::Tok;
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::PARENS;
-use orchid_base::tree::recur;
 
 use crate::api;
 use crate::ctx::Ctx;
-use crate::expr::{Expr, ExprParseCtx, ExprWillPanic};
+use crate::expr::{Expr, ExprParseCtx};
-use crate::parsed::{ParsTok, ParsTokTree};
+use crate::parsed::{ParsTok, ParsTokTree, tt_to_api};
 use crate::system::System;
 
 pub struct LexCtx<'a> {
 	pub systems: &'a [System],
-	pub source: &'a Tok<String>,
+	pub source: &'a IStr,
 	pub path: &'a Sym,
 	pub tail: &'a str,
 	pub sub_trees: &'a mut Vec<Expr>,
@@ -54,14 +53,7 @@ impl<'a> LexCtx<'a> {
 	}
 	#[must_use]
 	pub async fn ser_subtree(&mut self, subtree: ParsTokTree) -> api::TokenTree {
-		let mut exprs = self.ctx.common_exprs.clone();
+		tt_to_api(&mut self.ctx.common_exprs.clone(), subtree).await
-		let without_new_expr = recur(subtree, &|tt, r| {
-			if let ParsTok::NewExpr(expr) = tt.tok {
-				return ParsTok::Handle(expr).at(tt.sr);
-			}
-			r(tt)
-		});
-		without_new_expr.into_api(&mut exprs, &mut ExprWillPanic).await
 	}
 	#[must_use]
 	pub async fn des_subtree(&mut self, tree: &api::TokenTree) -> ParsTokTree {
@@ -113,7 +105,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 		ParsTok::NS(ctx.ctx.i.i(name).await, Box::new(body))
 	} else if ctx.strip_prefix("--[") {
 		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
-			return Err(mk_errv(
+			return Err(mk_err(
 				ctx.ctx.i.i("Unterminated block comment").await,
 				"This block comment has no ending ]--",
 				[SrcRange::new(start..start + 3, ctx.path)],
@@ -125,27 +117,18 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
 		ctx.push_pos(end as u32);
 		ParsTok::Comment(Rc::new(tail[2..end].to_string()))
-	} else if ctx.strip_char('\\') {
+	} else if let Some(tail) = ctx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
-		let mut arg = Vec::new();
+		// fanciness like \$placeh in templates is resolved in the macro engine.
+		ctx.set_tail(tail);
+		let arg = lex_once(ctx).boxed_local().await?;
 		ctx.trim_ws();
-		while !ctx.strip_char('.') {
+		ParsTok::LambdaHead(Box::new(arg))
-			if ctx.tail.is_empty() {
-				return Err(mk_errv(
-					ctx.ctx.i.i("Unclosed lambda").await,
-					"Lambdae started with \\ should separate arguments from body with .",
-					[SrcRange::new(start..start + 1, ctx.path)],
-				));
-			}
-			arg.push(lex_once(ctx).boxed_local().await?);
-			ctx.trim_ws();
-		}
-		ParsTok::LambdaHead(arg)
 	} else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
 		let mut body = Vec::new();
 		ctx.trim_ws();
 		while !ctx.strip_char(*rp) {
 			if ctx.tail.is_empty() {
-				return Err(mk_errv(
+				return Err(mk_err(
 					ctx.ctx.i.i("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
 					[SrcRange::new(start..start + 1, ctx.path)],
@@ -195,7 +178,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 		} else if ctx.tail.starts_with(op_char) {
 			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(op_char)).await)
 		} else {
-			return Err(mk_errv(
+			return Err(mk_err(
 				ctx.ctx.i.i("Unrecognized character").await,
 				"The following syntax is meaningless.",
 				[SrcRange::new(start..start + 1, ctx.path)],
@@ -205,12 +188,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 	Ok(ParsTokTree { tok, sr: SrcRange::new(start..ctx.get_pos(), ctx.path) })
 }
 
-pub async fn lex(
+pub async fn lex(text: IStr, path: Sym, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
-	text: Tok<String>,
-	path: Sym,
-	systems: &[System],
-	ctx: &Ctx,
-) -> OrcRes<Vec<ParsTokTree>> {
 	let mut sub_trees = Vec::new();
 	let mut ctx =
 		LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, path: &path, ctx };

orchid-host/src/lib.rs

@@ -11,5 +11,6 @@ pub mod lex;
 pub mod parse;
 pub mod parsed;
 pub mod subprocess;
+mod sys_parser;
 pub mod system;
 pub mod tree;

orchid-host/src/parse.rs

@@ -1,10 +1,9 @@
-use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
-use orchid_base::error::{OrcRes, Reporter, mk_errv};
+use orchid_base::error::{OrcRes, Reporter, mk_err};
 use orchid_base::format::fmt;
 use orchid_base::interner::{Interner, Tok};
-use orchid_base::name::{Sym, VPath};
+use orchid_base::name::Sym;
 use orchid_base::parse::{
 	Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname,
 	try_pop_no_fluff,
@@ -13,7 +12,7 @@ use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
 
 use crate::ctx::Ctx;
-use crate::expr::{Expr, ExprKind, PathSetBuilder};
+use crate::expr::Expr;
 use crate::parsed::{Item, ItemKind, ParsedMember, ParsedMemberKind, ParsedModule};
 use crate::system::System;
 
@@ -27,13 +26,14 @@ pub struct HostParseCtxImpl<'a> {
 }
 
 impl ParseCtx for HostParseCtxImpl<'_> {
-	fn reporter(&self) -> &Reporter { self.rep }
+	fn rep(&self) -> &Reporter { self.rep }
 	fn i(&self) -> &Interner { &self.ctx.i }
 }
 
 impl HostParseCtx for HostParseCtxImpl<'_> {
 	fn ctx(&self) -> &Ctx { &self.ctx }
 	fn systems(&self) -> impl Iterator<Item = &System> { self.systems.iter() }
+	fn src_path(&self) -> Sym { self.src.clone() }
 }
 
 pub trait HostParseCtx: ParseCtx {
@@ -41,11 +41,13 @@ pub trait HostParseCtx: ParseCtx {
 	fn ctx(&self) -> &Ctx;
 	#[must_use]
 	fn systems(&self) -> impl Iterator<Item = &System>;
+	#[must_use]
+	fn src_path(&self) -> Sym;
 }
 
 pub async fn parse_items(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	items: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
 	let lines = line_items(ctx, items).await;
@@ -56,7 +58,7 @@ pub async fn parse_items(
 
 pub async fn parse_item(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	comments: Vec<Comment>,
 	item: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
@@ -65,7 +67,7 @@ pub async fn parse_item(
 			n if *n == ctx.i().i("export").await => match try_pop_no_fluff(ctx, postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
 					parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
-				Parsed { output, tail: _ } => Err(mk_errv(
+				Parsed { output, tail: _ } => Err(mk_err(
 					ctx.i().i("Malformed export").await,
 					"`export` can either prefix other lines or list names inside ( )",
 					[output.sr()],
@@ -81,11 +83,10 @@ pub async fn parse_item(
 			},
 			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
 		},
-		Some(_) => Err(mk_errv(
+		Some(_) =>
-			ctx.i().i("Expected a line type").await,
+			Err(mk_err(ctx.i().i("Expected a line type").await, "All lines must begin with a keyword", [
-			"All lines must begin with a keyword",
+				item.sr(),
-			[item.sr()],
+			])),
-		)),
 		None => unreachable!("These lines are filtered and aggregated in earlier stages"),
 	}
 }
@@ -101,30 +102,27 @@ pub async fn parse_import<'a>(
 
 pub async fn parse_exportable_item<'a>(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	comments: Vec<Comment>,
 	exported: bool,
-	discr: Tok<String>,
+	discr: IStr,
 	tail: ParsSnippet<'a>,
 ) -> OrcRes<Vec<Item>> {
 	let kind = if discr == ctx.i().i("mod").await {
 		let (name, body) = parse_module(ctx, path, tail).await?;
 		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::Mod(body) })
-	} else if discr == ctx.i().i("const").await {
+	} else if let Some(parser) = ctx.systems().find_map(|s| s.get_parser(discr.clone())) {
-		let (name, expr) = parse_const(ctx, tail, path.clone()).await?;
+		return parser
-		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::ParsedConst(expr) })
+			.parse(ctx, path, tail.to_vec(), exported, comments, &mut async |stack, lines| {
-	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
-		return sys
-			.parse(path, tail.to_vec(), exported, comments, &mut async |stack, lines| {
 				let source = Snippet::new(lines.first().unwrap(), &lines);
 				parse_items(ctx, stack, source).await
 			})
 			.await;
 	} else {
 		let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
-		return Err(mk_errv(
+		return Err(mk_err(
 			ctx.i().i("Unrecognized line type").await,
-			format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
+			format!("Line types are: mod, {ext_lines}"),
 			[tail.prev().sr()],
 		));
 	};
@@ -133,13 +131,13 @@ pub async fn parse_exportable_item<'a>(
 
 pub async fn parse_module<'a>(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	tail: ParsSnippet<'a>,
-) -> OrcRes<(Tok<String>, ParsedModule)> {
+) -> OrcRes<(IStr, ParsedModule)> {
 	let (name, tail) = match try_pop_no_fluff(ctx, tail).await? {
 		Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
 		Parsed { output, .. } => {
-			return Err(mk_errv(
+			return Err(mk_err(
 				ctx.i().i("Missing module name").await,
 				format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
 				[output.sr()],
@@ -149,114 +147,12 @@ pub async fn parse_module<'a>(
 	let Parsed { output, tail: surplus } = try_pop_no_fluff(ctx, tail).await?;
 	expect_end(ctx, surplus).await?;
 	let Some(body) = output.as_s(Paren::Round) else {
-		return Err(mk_errv(
+		return Err(mk_err(
 			ctx.i().i("Expected module body").await,
 			format!("A ( block ) was expected, {} was found", fmt(output, ctx.i()).await),
 			[output.sr()],
 		));
 	};
 	let path = path.push(name.clone());
-	Ok((name, ParsedModule::new(parse_items(ctx, path, body).await?)))
+	Ok((name, ParsedModule::new(true, parse_items(ctx, path, body).await?)))
-}
-
-pub async fn parse_const<'a>(
-	ctx: &impl HostParseCtx,
-	tail: ParsSnippet<'a>,
-	path: Substack<'_, Tok<String>>,
-) -> OrcRes<(Tok<String>, Expr)> {
-	let Parsed { output, tail } = try_pop_no_fluff(ctx, tail).await?;
-	let Some(name) = output.as_name() else {
-		return Err(mk_errv(
-			ctx.i().i("Missing module name").await,
-			format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
-			[output.sr()],
-		));
-	};
-	let Parsed { output, tail } = try_pop_no_fluff(ctx, tail).await?;
-	if !output.is_kw(ctx.i().i("=").await) {
-		return Err(mk_errv(
-			ctx.i().i("Missing = separator").await,
-			format!("Expected = , found {}", fmt(output, ctx.i()).await),
-			[output.sr()],
-		));
-	}
-	try_pop_no_fluff(ctx, tail).await?;
-	// ctx.save_const(path, tail[..].to_vec()).await;
-	let final_path =
-		VPath::new(path.unreverse()).name_with_suffix(name.clone()).to_sym(ctx.i()).await;
-	let val = parse_expr(ctx, final_path, PathSetBuilder::new(), tail).await?;
-	Ok((name, val))
-}
-
-pub async fn parse_expr(
-	ctx: &impl HostParseCtx,
-	path: Sym,
-	psb: PathSetBuilder<'_, Tok<String>>,
-	tail: ParsSnippet<'_>,
-) -> OrcRes<Expr> {
-	let Some((last_idx, _)) = (tail.iter().enumerate().find(|(_, tt)| tt.as_lambda().is_some()))
-		.or_else(|| tail.iter().enumerate().rev().find(|(_, tt)| !tt.is_fluff()))
-	else {
-		return Err(mk_errv(ctx.i().i("Empty expression").await, "Expression ends abruptly here", [
-			tail.sr(),
-		]));
-	};
-	let (function, value) = tail.split_at(last_idx as u32);
-	let pos = tail.sr().pos();
-	if !function.iter().all(TokTree::is_fluff) {
-		let (f_psb, x_psb) = psb.split();
-		let x_expr = parse_expr(ctx, path.clone(), x_psb, value).boxed_local().await?;
-		let f_expr = parse_expr(ctx, path, f_psb, function).boxed_local().await?;
-		return Ok(ExprKind::Call(f_expr, x_expr).at(pos));
-	}
-	let Parsed { output: head, tail } = try_pop_no_fluff(ctx, value).await?;
-	match &head.tok {
-		Token::BR | Token::Comment(_) => panic!("Fluff skipped"),
-		Token::Bottom(b) => Ok(ExprKind::Bottom(b.clone()).at(pos.clone())),
-		Token::Handle(expr) => Ok(expr.clone()),
-		Token::NS(n, nametail) => {
-			let mut nametail = nametail;
-			let mut segments = vec![n.clone()];
-			while let Token::NS(n, newtail) = &nametail.tok {
-				segments.push(n.clone());
-				nametail = newtail;
-			}
-			let Token::Name(n) = &nametail.tok else {
-				return Err(mk_errv(
-					ctx.i().i("Loose namespace prefix in constant").await,
-					"Namespace prefixes in constants must be followed by names",
-					[pos],
-				));
-			};
-			segments.push(n.clone());
-			Ok(ExprKind::Const(Sym::new(segments, ctx.i()).await.unwrap()).at(pos.clone()))
-		},
-		Token::LambdaHead(h) => {
-			let [TokTree { tok: Token::Name(arg), .. }] = &h[..] else {
-				return Err(mk_errv(
-					ctx.i().i("Complex lambda binding in constant").await,
-					"Lambda args in constants must be identified by a single name",
-					[pos],
-				));
-			};
-			let lambda_builder = psb.lambda(arg);
-			let body = parse_expr(ctx, path.clone(), lambda_builder.stack(), tail).boxed_local().await?;
-			Ok(ExprKind::Lambda(lambda_builder.collect(), body).at(pos.clone()))
-		},
-		Token::S(Paren::Round, body) =>
-			parse_expr(ctx, path, psb, Snippet::new(head, body)).boxed_local().await,
-		Token::S(..) =>
-			return Err(mk_errv(
-				ctx.i().i("Constants may only contain (), not [] or {}").await,
-				"It seems like you are trying to call a macro. Consider a 'let' line",
-				[pos],
-			)),
-		Token::Name(n) =>
-			if psb.register_arg(n) {
-				Ok(ExprKind::Arg.at(pos))
-			} else {
-				Ok(ExprKind::Const(Sym::new([n.clone()], ctx.i()).await.unwrap()).at(pos))
-			},
-		Token::NewExpr(ex) => Ok(ex.clone()),
-	}
 }

orchid-host/src/parsed.rs

@@ -1,8 +1,8 @@
-use std::fmt::Debug;
+use std::fmt::{self, Debug};
 use std::rc::Rc;
 
 use futures::FutureExt;
-use futures::future::join_all;
+use futures::future::{LocalBoxFuture, join_all};
 use hashbrown::HashSet;
 use itertools::Itertools;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
@@ -10,11 +10,12 @@ use orchid_base::interner::Tok;
 use orchid_base::location::SrcRange;
 use orchid_base::parse::{Comment, Import};
 use orchid_base::tl_cache;
-use orchid_base::tree::{TokTree, Token};
+use orchid_base::tree::{TokTree, Token, recur};
 
 use crate::api;
 use crate::dealias::{ChildErrorKind, ChildResult, Tree};
-use crate::expr::Expr;
+use crate::expr::{Expr, ExprWillPanic};
+use crate::expr_store::ExprStore;
 use crate::system::System;
 
 pub type ParsTokTree = TokTree<Expr, Expr>;
@@ -26,6 +27,11 @@ pub struct Item {
 	pub comments: Vec<Comment>,
 	pub kind: ItemKind,
 }
+impl Item {
+	pub fn new(sr: SrcRange, kind: impl Into<ItemKind>) -> Self {
+		Self { sr, comments: Vec::new(), kind: kind.into() }
+	}
+}
 
 #[derive(Debug)]
 pub enum ItemKind {
@@ -36,6 +42,12 @@ impl ItemKind {
 	#[must_use]
 	pub fn at(self, sr: SrcRange) -> Item { Item { comments: vec![], sr, kind: self } }
 }
+impl From<ParsedMember> for ItemKind {
+	fn from(value: ParsedMember) -> Self { Self::Member(value) }
+}
+impl From<Import> for ItemKind {
+	fn from(value: Import) -> Self { Self::Import(value) }
+}
 
 impl Format for Item {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -43,10 +55,7 @@ impl Format for Item {
 		let item_text = match &self.kind {
 			ItemKind::Import(i) => format!("import {i}").into(),
 			ItemKind::Member(mem) => match &mem.kind {
-				ParsedMemberKind::ParsedConst(expr) =>
+				ParsedMemberKind::Const(_, sys) =>
-					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} = {1l}")))
-						.units([mem.name.rc().into(), expr.print(c).await]),
-				ParsedMemberKind::DeferredConst(_, sys) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} via {1}")))
 						.units([mem.name.rc().into(), sys.print(c).await]),
 				ParsedMemberKind::Mod(module) =>
@@ -60,13 +69,16 @@ impl Format for Item {
 }
 
 pub struct ParsedMember {
-	pub name: Tok<String>,
+	pub name: IStr,
 	pub exported: bool,
 	pub kind: ParsedMemberKind,
 }
 impl ParsedMember {
 	#[must_use]
-	pub fn name(&self) -> Tok<String> { self.name.clone() }
+	pub fn name(&self) -> IStr { self.name.clone() }
+	pub fn new(exported: bool, name: IStr, kind: impl Into<ParsedMemberKind>) -> Self {
+		Self { exported, name, kind: kind.into() }
+	}
 }
 impl Debug for ParsedMember {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -77,43 +89,60 @@ impl Debug for ParsedMember {
 	}
 }
 
-#[derive(Debug)]
+pub(crate) type ParsedExprCallback =
-pub enum ParsedMemberKind {
+	Rc<dyn for<'a> Fn(&'a [IStr]) -> LocalBoxFuture<'a, Expr>>;
-	DeferredConst(api::ParsedConstId, System),
+
-	ParsedConst(Expr),
+pub struct ParsedExpr {
-	Mod(ParsedModule),
+	pub(crate) debug: String,
+	pub(crate) callback: ParsedExprCallback,
+}
+impl ParsedExpr {
+	pub async fn run(self, imported_names: &[IStr]) -> Expr {
+		(self.callback)(imported_names).await
+	}
+}
+impl fmt::Debug for ParsedExpr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.debug) }
 }
 
-// TODO: cannot determine alias origin at this stage; parsed tree is never
+#[derive(Debug)]
-// walkable!
+pub enum ParsedMemberKind {
+	Const(api::ParsedConstId, System),
+	Mod(ParsedModule),
+}
+impl From<ParsedModule> for ParsedMemberKind {
+	fn from(value: ParsedModule) -> Self { Self::Mod(value) }
+}
 
 #[derive(Debug, Default)]
 pub struct ParsedModule {
-	pub exports: Vec<Tok<String>>,
+	pub exports: Vec<IStr>,
 	pub items: Vec<Item>,
+	pub use_prelude: bool,
 }
 impl ParsedModule {
 	#[must_use]
-	pub fn new(items: impl IntoIterator<Item = Item>) -> Self {
+	pub fn new(use_prelude: bool, items: impl IntoIterator<Item = Item>) -> Self {
 		let items = items.into_iter().collect_vec();
 		let exports = (items.iter())
 			.filter_map(|i| if let ItemKind::Member(m) = &i.kind { Some(m) } else { None })
 			.filter(|m| m.exported)
 			.map(|m| m.name.clone())
 			.collect_vec();
-		Self { exports, items }
+		Self { exports, items, use_prelude }
 	}
 	pub fn merge(&mut self, other: ParsedModule) {
 		let mut swap = ParsedModule::default();
 		std::mem::swap(self, &mut swap);
-		*self = ParsedModule::new(swap.items.into_iter().chain(other.items))
+		assert_eq!(self.use_prelude, other.use_prelude, "merging modules that disagree on prelude");
+		*self = ParsedModule::new(self.use_prelude, swap.items.into_iter().chain(other.items))
 	}
 	#[must_use]
 	pub fn get_imports(&self) -> impl IntoIterator<Item = &Import> {
 		(self.items.iter())
 			.filter_map(|it| if let ItemKind::Import(i) = &it.kind { Some(i) } else { None })
 	}
-	pub fn default_item(self, name: Tok<String>, sr: SrcRange) -> Item {
+	pub fn default_item(self, name: IStr, sr: SrcRange) -> Item {
 		let mem = ParsedMember { exported: true, name, kind: ParsedMemberKind::Mod(self) };
 		Item { comments: vec![], sr, kind: ItemKind::Member(mem) }
 	}
@@ -122,7 +151,7 @@ impl Tree for ParsedModule {
 	type Ctx<'a> = ();
 	async fn child(
 		&self,
-		key: Tok<String>,
+		key: IStr,
 		public_only: bool,
 		(): &mut Self::Ctx<'_>,
 	) -> ChildResult<'_, Self> {
@@ -134,14 +163,13 @@ impl Tree for ParsedModule {
 			.find(|m| m.name == key)
 		{
 			match &member.kind {
-				ParsedMemberKind::DeferredConst(..) | ParsedMemberKind::ParsedConst(_) =>
+				ParsedMemberKind::Const(..) => return ChildResult::Err(ChildErrorKind::Constant),
-					return ChildResult::Err(ChildErrorKind::Constant),
 				ParsedMemberKind::Mod(m) => return ChildResult::Ok(m),
 			}
 		}
 		ChildResult::Err(ChildErrorKind::Missing)
 	}
-	fn children(&self, public_only: bool) -> HashSet<Tok<String>> {
+	fn children(&self, public_only: bool) -> HashSet<IStr> {
 		let mut public: HashSet<_> = self.exports.iter().cloned().collect();
 		if !public_only {
 			public.extend(
@@ -164,24 +192,25 @@ impl Format for ParsedModule {
 	}
 }
 
-/// TODO:
-///
-/// idea, does the host need an IR here or can we figure out a way to transcribe
-/// these? Should we spin off a new stage for value parsing so that ParsTokTree
-/// doesn't appear in the interpreter's ingress?
-pub struct Const {
-	pub source: Option<Vec<ParsTokTree>>,
-}
-
 /// Selects a code element
 ///
 /// Either the steps point to a constant and rule_loc is None, or the steps
 /// point to a module and rule_loc selects a macro rule within that module
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct ConstPath {
-	steps: Tok<Vec<Tok<String>>>,
+	steps: Tok<Vec<IStr>>,
 }
 impl ConstPath {
 	#[must_use]
-	pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps } }
+	pub fn to_const(steps: Tok<Vec<IStr>>) -> Self { Self { steps } }
+}
+
+pub async fn tt_to_api(exprs: &mut ExprStore, subtree: ParsTokTree) -> api::TokenTree {
+	let without_new_expr = recur(subtree, &|tt, r| {
+		if let ParsTok::NewExpr(expr) = tt.tok {
+			return ParsTok::Handle(expr).at(tt.sr);
+		}
+		r(tt)
+	});
+	without_new_expr.into_api(exprs, &mut ExprWillPanic).await
 }

orchid-host/src/subprocess.rs

@@ -1,12 +1,12 @@
 use std::cell::RefCell;
-use std::io::Write;
+use std::io::{self, Write};
 use std::pin::Pin;
 
 use async_process::{self, Child, ChildStdin, ChildStdout};
-use async_std::io::{self, BufReadExt, BufReader};
-use async_std::sync::Mutex;
-use futures::AsyncWriteExt;
 use futures::future::LocalBoxFuture;
+use futures::io::BufReader;
+use futures::lock::Mutex;
+use futures::{self, AsyncBufReadExt, AsyncWriteExt};
 use orchid_api_traits::{Decode, Encode};
 use orchid_base::builtin::{ExtInit, ExtPort};
 use orchid_base::logging::Logger;

orchid-host/src/sys_parser.rs

@@ -0,0 +1,119 @@
+use futures::FutureExt;
+use futures::future::join_all;
+use itertools::Itertools;
+use orchid_base::error::{OrcErrv, OrcRes};
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::SrcRange;
+use orchid_base::name::Sym;
+use orchid_base::parse::Comment;
+use orchid_base::reqnot::Requester;
+use orchid_base::tree::ttv_from_api;
+use substack::Substack;
+
+use crate::api;
+use crate::expr::ExprParseCtx;
+use crate::expr_store::ExprStore;
+use crate::parse::HostParseCtx;
+use crate::parsed::{
+	Item, ItemKind, ParsTokTree, ParsedMember, ParsedMemberKind, ParsedModule, tt_to_api,
+};
+use crate::system::System;
+
+pub struct Parser {
+	pub(crate) system: System,
+	pub(crate) idx: u16,
+}
+type ModPath<'a> = Substack<'a, IStr>;
+
+impl Parser {
+	pub async fn parse(
+		&self,
+		ctx: &impl HostParseCtx,
+		path: ModPath<'_>,
+		line: Vec<ParsTokTree>,
+		exported: bool,
+		comments: Vec<Comment>,
+		callback: &mut impl AsyncFnMut(ModPath<'_>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
+	) -> OrcRes<Vec<Item>> {
+		let src_path = line.first().expect("cannot be empty").sr.path();
+		let line = join_all(
+			(line.into_iter())
+				.map(|t| async { tt_to_api(&mut self.system.ext().exprs().clone(), t).await }),
+		)
+		.await;
+		let mod_path = ctx.src_path().suffix(path.unreverse(), self.system.i()).await;
+		let comments = comments.iter().map(Comment::to_api).collect_vec();
+		let req = api::ParseLine {
+			idx: self.idx,
+			module: mod_path.to_api(),
+			src: src_path.to_api(),
+			exported,
+			sys: self.system.id(),
+			comments,
+			line,
+		};
+		match self.system.reqnot().request(req).await {
+			Ok(parsed_v) => {
+				let mut ext_exprs = self.system.ext().exprs().clone();
+				conv(parsed_v, path, callback, &mut ConvCtx {
+					i: self.system.i(),
+					mod_path: &mod_path,
+					ext_exprs: &mut ext_exprs,
+					pctx: &mut ExprParseCtx { ctx: self.system.ctx(), exprs: self.system.ext().exprs() },
+					src_path: &src_path,
+					sys: &self.system,
+				})
+				.await
+			},
+			Err(e) => Err(OrcErrv::from_api(&e, &self.system.ctx().i).await),
+		}
+	}
+}
+
+struct ConvCtx<'a> {
+	sys: &'a System,
+	mod_path: &'a Sym,
+	src_path: &'a Sym,
+	i: &'a Interner,
+	ext_exprs: &'a mut ExprStore,
+	pctx: &'a mut ExprParseCtx<'a>,
+}
+async fn conv(
+	parsed_v: Vec<api::ParsedLine>,
+	module: Substack<'_, IStr>,
+	callback: &'_ mut impl AsyncFnMut(Substack<'_, IStr>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
+	ctx: &mut ConvCtx<'_>,
+) -> OrcRes<Vec<Item>> {
+	let mut items = Vec::new();
+	for parsed in parsed_v {
+		let (name, exported, kind) = match parsed.kind {
+			api::ParsedLineKind::Member(api::ParsedMember { name, exported, kind }) =>
+				(name, exported, kind),
+			api::ParsedLineKind::Recursive(rec) => {
+				let tokens = ttv_from_api(rec, ctx.ext_exprs, ctx.pctx, ctx.src_path, ctx.i).await;
+				items.extend(callback(module.clone(), tokens).await?);
+				continue;
+			},
+		};
+		let name = ctx.i.ex(name).await;
+		let mkind = match kind {
+			api::ParsedMemberKind::Module { lines, use_prelude } => {
+				let items = conv(lines, module.push(name.clone()), callback, ctx).boxed_local().await?;
+				ParsedMemberKind::Mod(ParsedModule::new(use_prelude, items))
+			},
+			api::ParsedMemberKind::Constant(cid) => {
+				ctx.sys.0.const_paths.insert(cid, ctx.mod_path.suffix(module.unreverse(), ctx.i).await);
+				ParsedMemberKind::Const(cid, ctx.sys.clone())
+			},
+		};
+		items.push(Item {
+			comments: join_all(
+				parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone(), ctx.i)),
+			)
+			.await,
+			sr: SrcRange::from_api(&parsed.source_range, ctx.i).await,
+			kind: ItemKind::Member(ParsedMember { name, exported, kind: mkind }),
+		})
+	}
+	Ok(items)
+}

orchid-host/src/system.rs

@@ -3,42 +3,41 @@ use std::fmt;
 use std::future::Future;
 use std::rc::{Rc, Weak};
 
+use async_lock::RwLock;
 use derive_destructure::destructure;
-use futures::FutureExt;
 use futures::future::join_all;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use memo_map::MemoMap;
 use orchid_base::char_filter::char_filter_match;
-use orchid_base::error::{OrcErrv, OrcRes};
+use orchid_base::error::{OrcRes, mk_err_floating};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::interner::{Interner, Tok};
-use orchid_base::location::SrcRange;
+use orchid_base::iter_utils::IteratorPrint;
-use orchid_base::name::{NameLike, Sym, VName};
+use orchid_base::name::{NameLike, Sym, VName, VPath};
-use orchid_base::parse::Comment;
 use orchid_base::reqnot::{ReqNot, Requester};
-use orchid_base::tree::ttv_from_api;
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
 use crate::api;
+use crate::atom::{AtomHand, WeakAtomHand};
 use crate::ctx::Ctx;
-use crate::dealias::{absolute_path, walk};
+use crate::dealias::walk;
-use crate::expr::{ExprParseCtx, ExprWillPanic};
-use crate::expr_store::ExprStore;
 use crate::extension::{Extension, WeakExtension};
-use crate::parsed::{Item, ItemKind, ParsTokTree, ParsedMember, ParsedMemberKind, ParsedModule};
+use crate::sys_parser::Parser;
 use crate::tree::Root;
 
 #[derive(destructure)]
-struct SystemInstData {
+pub(crate) struct SystemInstData {
 	deps: Vec<System>,
 	ctx: Ctx,
 	ext: Extension,
 	decl_id: api::SysDeclId,
 	lex_filter: api::CharFilter,
 	id: api::SysId,
-	line_types: Vec<Tok<String>>,
+	line_types: Vec<IStr>,
+	prelude: Vec<Sym>,
+	owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
 	pub(crate) const_paths: MemoMap<api::ParsedConstId, Sym>,
 }
 impl Drop for SystemInstData {
@@ -56,7 +55,7 @@ impl fmt::Debug for SystemInstData {
 }
 
 #[derive(Clone, Debug)]
-pub struct System(Rc<SystemInstData>);
+pub struct System(pub(crate) Rc<SystemInstData>);
 impl System {
 	#[must_use]
 	pub fn id(&self) -> api::SysId { self.0.id }
@@ -69,6 +68,11 @@ impl System {
 	#[must_use]
 	pub fn deps(&self) -> &[System] { &self.0.deps }
 	#[must_use]
+	pub fn ctor(&self) -> SystemCtor {
+		(self.0.ext.system_ctors().find(|c| c.decl.id == self.0.decl_id).cloned())
+			.expect("Ctor was used to create ext")
+	}
+	#[must_use]
 	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { self.0.ext.reqnot() }
 	#[must_use]
 	pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
@@ -78,11 +82,13 @@ impl System {
 	pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
 	#[must_use]
 	pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
+	#[must_use]
+	pub fn prelude(&self) -> Vec<Sym> { self.0.prelude.clone() }
 	/// Have this system lex a part of the source. It is assumed that
 	/// [Self::can_lex] was called and returned true.
 	pub async fn lex<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
-		source: Tok<String>,
+		source: IStr,
 		src: Sym,
 		pos: u32,
 		r: impl FnMut(u32) -> F,
@@ -90,102 +96,32 @@ impl System {
 		self.0.ext.lex_req(source, src, pos, self.id(), r).await
 	}
 	#[must_use]
-	pub fn can_parse(&self, ltyp: Tok<String>) -> bool { self.0.line_types.contains(&ltyp) }
+	pub fn get_parser(&self, ltyp: IStr) -> Option<Parser> {
-	pub fn line_types(&self) -> impl Iterator<Item = &Tok<String>> + '_ { self.0.line_types.iter() }
+		(self.0.line_types.iter().enumerate())
-	pub async fn parse(
+			.find(|(_, txt)| *txt == &ltyp)
-		&self,
+			.map(|(idx, _)| Parser { idx: idx as u16, system: self.clone() })
-		path: Substack<'_, Tok<String>>,
-		line: Vec<ParsTokTree>,
-		exported: bool,
-		comments: Vec<Comment>,
-		callback: &mut impl AsyncFnMut(Substack<'_, Tok<String>>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
-	) -> OrcRes<Vec<Item>> {
-		let src_path = line.first().expect("cannot be empty").sr.path();
-		let line = join_all(line.into_iter().map(|t| async {
-			let mut expr_store = self.0.ext.exprs().clone();
-			t.into_api(&mut expr_store, &mut ExprWillPanic).await
-		}))
-		.await;
-		let comments = comments.iter().map(Comment::to_api).collect_vec();
-		let req = api::ParseLine {
-			module: self.i().i(&path.unreverse()).await.to_api(),
-			src: src_path.to_api(),
-			exported,
-			sys: self.id(),
-			comments,
-			line,
-		};
-		match self.reqnot().request(req).await {
-			Ok(parsed_v) => {
-				let mut ext_exprs = self.ext().exprs().clone();
-				struct ConvCtx<'a> {
-					sys: &'a System,
-					src_path: &'a Sym,
-					i: &'a Interner,
-					ext_exprs: &'a mut ExprStore,
-					pctx: &'a mut ExprParseCtx<'a>,
-				}
-				async fn conv(
-					parsed_v: Vec<api::ParsedLine>,
-					module: Substack<'_, Tok<String>>,
-					callback: &'_ mut impl AsyncFnMut(
-						Substack<'_, Tok<String>>,
-						Vec<ParsTokTree>,
-					) -> OrcRes<Vec<Item>>,
-					ctx: &mut ConvCtx<'_>,
-				) -> OrcRes<Vec<Item>> {
-					let mut items = Vec::new();
-					for parsed in parsed_v {
-						let (name, exported, kind) = match parsed.kind {
-							api::ParsedLineKind::Member(api::ParsedMember { name, exported, kind }) =>
-								(name, exported, kind),
-							api::ParsedLineKind::Recursive(rec) => {
-								let tokens = ttv_from_api(rec, ctx.ext_exprs, ctx.pctx, ctx.src_path, ctx.i).await;
-								items.extend(callback(module.clone(), tokens).await?);
-								continue;
-							},
-						};
-						let name = ctx.i.ex(name).await;
-						let mkind = match kind {
-							api::ParsedMemberKind::Module(items) => {
-								let items =
-									conv(items, module.push(name.clone()), callback, ctx).boxed_local().await?;
-								ParsedMemberKind::Mod(ParsedModule::new(items))
-							},
-							api::ParsedMemberKind::Constant(cid) =>
-								ParsedMemberKind::DeferredConst(cid, ctx.sys.clone()),
-						};
-						items.push(Item {
-							comments: join_all(
-								parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone(), ctx.i)),
-							)
-							.await,
-							sr: SrcRange::from_api(&parsed.source_range, ctx.i).await,
-							kind: ItemKind::Member(ParsedMember { name, exported, kind: mkind }),
-						})
-					}
-					Ok(items)
-				}
-				conv(parsed_v, path, callback, &mut ConvCtx {
-					i: self.i(),
-					ext_exprs: &mut ext_exprs,
-					pctx: &mut ExprParseCtx { ctx: self.ctx(), exprs: self.ext().exprs() },
-					src_path: &src_path,
-					sys: self,
-				})
-				.await
-			},
-			Err(e) => Err(OrcErrv::from_api(&e, &self.ctx().i).await),
-		}
 	}
+	pub fn line_types(&self) -> impl Iterator<Item = &IStr> + '_ { self.0.line_types.iter() }
+
 	#[must_use]
 	pub async fn request(&self, req: Vec<u8>) -> Vec<u8> {
 		self.reqnot().request(api::SysFwded(self.id(), req)).await
 	}
+	pub(crate) async fn new_atom(&self, data: Vec<u8>, id: api::AtomId) -> AtomHand {
+		let mut owned_g = self.0.owned_atoms.write().await;
+		if let Some(data) = owned_g.get(&id)
+			&& let Some(atom) = data.upgrade()
+		{
+			return atom;
+		}
+		let new = AtomHand::new(data, self.clone(), Some(id));
+		owned_g.insert(id, new.downgrade());
+		new
+	}
 	pub(crate) fn drop_atom(&self, drop: api::AtomId) {
 		let this = self.0.clone();
 		(self.0.ctx.spawn)(Box::pin(async move {
-			this.ctx.owned_atoms.write().await.remove(&drop);
+			this.owned_atoms.write().await.remove(&drop);
 		}))
 	}
 	#[must_use]
@@ -194,16 +130,39 @@ impl System {
 	pub(crate) async fn name_resolver(
 		&self,
 		orig: api::ParsedConstId,
-	) -> impl AsyncFnMut(&[Tok<String>]) -> Option<VName> + use<> {
+	) -> impl AsyncFnMut(&[IStr]) -> OrcRes<VName> + use<> {
 		let root = self.0.ctx.root.read().await.upgrade().expect("find_names when root not in context");
 		let orig = self.0.const_paths.get(&orig).expect("origin for find_names invalid").clone();
 		let ctx = self.0.ctx.clone();
 		async move |rel| {
-			let cwd = orig.split_last().1;
+			let cwd = orig.split_last_seg().1;
-			let abs = absolute_path(cwd, rel, &ctx.i).await.ok()?;
 			let root_data = &mut *root.0.write().await;
-			let walk_ctx = &mut (ctx.clone(), &mut root_data.consts);
+			let walk_ctx = &mut (ctx.clone(), &root_data.consts);
-			walk(&root_data.root, false, abs.iter(), walk_ctx).await.is_ok().then_some(abs)
+			let cmod = (walk(&root_data.root, false, cwd.iter().cloned(), walk_ctx).await)
+				.expect("the parent module of a constant should exist");
+			let (selector, tail) = rel.split_first().expect("Names cannot be empty");
+			if cmod.members.get(selector).is_some() {
+				return Ok(VName::new(cwd.iter().chain(rel).cloned()).unwrap());
+			}
+			match cmod.imports.get(selector) {
+				Some(Ok(dest)) => return Ok(dest.target.to_vname().suffix(tail.iter().cloned())),
+				Some(Err(dests)) =>
+					return Err(mk_err_floating(
+						ctx.i.i("Ambiguous name").await,
+						format!(
+							"{selector} could refer to {}",
+							dests.iter().map(|ri| &ri.target).display("or")
+						),
+					)),
+				None => (),
+			}
+			if tail.is_empty() {
+				return Ok(VPath::new(cwd.iter().cloned()).name_with_suffix(selector.clone()));
+			}
+			Err(mk_err_floating(
+				ctx.i.i("Invalid name").await,
+				format!("{selector} doesn't refer to a module"),
+			))
 		}
 	}
 }
@@ -221,6 +180,7 @@ impl WeakSystem {
 	pub fn upgrade(&self) -> Option<System> { self.0.upgrade().map(System) }
 }
 
+#[derive(Clone)]
 pub struct SystemCtor {
 	pub(crate) decl: api::SystemDecl,
 	pub(crate) ext: WeakExtension,
@@ -228,6 +188,10 @@ pub struct SystemCtor {
 impl SystemCtor {
 	#[must_use]
 	pub fn name(&self) -> &str { &self.decl.name }
+	pub async fn name_tok(&self) -> Sym {
+		(Sym::parse(&self.decl.name, &self.ext.upgrade().expect("ext dropped early").ctx().i).await)
+			.expect("System cannot have empty name")
+	}
 	#[must_use]
 	pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
 	#[must_use]
@@ -252,6 +216,8 @@ impl SystemCtor {
 			line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
 				.await,
 			id,
+			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok, &ext.ctx().i))).await,
+			owned_atoms: RwLock::new(HashMap::new()),
 			const_paths: MemoMap::new(),
 		}));
 		let api_module_root = api::Module {

orchid-host/src/tree.rs

@@ -2,19 +2,20 @@
 //! once
 use std::cell::RefCell;
 use std::rc::{Rc, Weak};
+use std::slice;
 
+use async_lock::RwLock;
 use async_once_cell::OnceCell;
-use async_std::sync::RwLock;
 use futures::{FutureExt, StreamExt, stream};
 use hashbrown::HashMap;
 use hashbrown::hash_map::Entry;
 use itertools::Itertools;
-use orchid_api::FetchParsedConst;
+use memo_map::MemoMap;
 use orchid_base::clone;
-use orchid_base::error::{OrcRes, Reporter, mk_err, mk_errv};
+use orchid_base::error::{OrcRes, Reporter, mk_err};
 use orchid_base::interner::Tok;
-use orchid_base::location::{Pos, SrcRange};
+use orchid_base::location::{CodeGenInfo, Pos};
-use orchid_base::name::{Sym, VPath};
+use orchid_base::name::{NameLike, Sym, VPath};
 use orchid_base::reqnot::Requester;
 
 use crate::api;
@@ -26,7 +27,7 @@ use crate::system::System;
 
 pub struct RootData {
 	pub root: Module,
-	pub consts: HashMap<Sym, Expr>,
+	pub consts: MemoMap<Sym, Expr>,
 	pub ctx: Ctx,
 }
 #[derive(Clone)]
@@ -36,23 +37,25 @@ impl Root {
 	pub fn new(ctx: Ctx) -> Self {
 		Root(Rc::new(RwLock::new(RootData {
 			root: Module::default(),
-			consts: HashMap::default(),
+			consts: MemoMap::default(),
 			ctx,
 		})))
 	}
 	#[must_use]
 	pub async fn from_api(api: api::Module, sys: &System) -> Self {
-		let mut consts = HashMap::new();
+		let consts = MemoMap::new();
-		let mut tfac = TreeFromApiCtx { consts: &mut consts, path: sys.i().i(&[][..]).await, sys };
+		let mut tfac = TreeFromApiCtx { consts: &consts, path: sys.i().i(&[][..]).await, sys };
 		let root = Module::from_api(api, &mut tfac).await;
 		Root(Rc::new(RwLock::new(RootData { root, consts, ctx: sys.ctx().clone() })))
 	}
 	pub async fn merge(&self, new: &Root) -> Result<Self, MergeErr> {
-		let this = self.0.read().await;
+		let this = self.0.write().await;
-		let that = new.0.read().await;
+		let that = new.0.write().await;
-		let mut consts =
+		let consts = MemoMap::new();
-			this.consts.iter().chain(&that.consts).map(|(k, v)| (k.clone(), v.clone())).collect();
+		for (k, v) in this.consts.iter().chain(that.consts.iter()) {
-		let root = this.root.merge(&that.root, this.ctx.clone(), &mut consts).await?;
+			consts.insert(k.clone(), v.clone());
+		}
+		let root = this.root.merge(&that.root, this.ctx.clone(), &consts).await?;
 		Ok(Self(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() }))))
 	}
 	#[must_use]
@@ -60,11 +63,12 @@ impl Root {
 		let mut ref_this = self.0.write().await;
 		let this = &mut *ref_this;
 		let mut deferred_consts = HashMap::new();
+		let consts = this.consts.clone();
 		let mut tfpctx = FromParsedCtx {
 			pars_root: parsed,
 			deferred_consts: &mut deferred_consts,
+			consts: &consts,
 			pars_prefix: pars_prefix.clone(),
-			consts: &mut this.consts,
 			root: &this.root,
 			ctx: &this.ctx,
 			rep,
@@ -78,14 +82,13 @@ impl Root {
 			)]);
 			module = Module { imports: HashMap::new(), members }
 		}
-		let mut consts = this.consts.clone();
+		let root = (this.root.merge(&module, this.ctx.clone(), &consts).await)
-		let root = (this.root.merge(&module, this.ctx.clone(), &mut consts).await)
 			.expect("Merge conflict between parsed and existing module");
 		let new = Root(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() })));
 		*this.ctx.root.write().await = new.downgrade();
 		for (path, (sys_id, pc_id)) in deferred_consts {
 			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
-			let api_expr = sys.reqnot().request(FetchParsedConst { id: pc_id, sys: sys.id() }).await;
+			let api_expr = sys.reqnot().request(api::FetchParsedConst { id: pc_id, sys: sys.id() }).await;
 			let mut xp_ctx = ExprParseCtx { ctx: &this.ctx, exprs: sys.ext().exprs() };
 			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), &mut xp_ctx).await;
 			new.0.write().await.consts.insert(path, expr);
@@ -93,7 +96,7 @@ impl Root {
 		new
 	}
 	pub async fn get_const_value(&self, name: Sym, pos: Pos) -> OrcRes<Expr> {
-		let this = &mut *self.0.write().await;
+		let this = &*self.0.read().await;
 		// shortcut for previously visited
 		if let Some(val) = this.consts.get(&name) {
 			return Ok(val.clone());
@@ -101,12 +104,12 @@ impl Root {
 		// load the node, then check if this "walk" call added it to the map
 		let ctx = this.ctx.clone();
 		let module =
-			walk(&this.root, false, name.iter().cloned(), &mut (ctx.clone(), &mut this.consts)).await;
+			walk(&this.root, false, name.iter().cloned(), &mut (ctx.clone(), &this.consts)).await;
 		if let Some(val) = this.consts.get(&name) {
 			return Ok(val.clone());
 		}
 		match module {
-			Ok(_) => Err(mk_errv(
+			Ok(_) => Err(mk_err(
 				ctx.i.i("module used as constant").await,
 				format!("{name} is a module, not a constant"),
 				[pos],
@@ -114,7 +117,7 @@ impl Root {
 			Err(e) => match e.kind {
 				ChildErrorKind::Private => panic!("public_only is false"),
 				ChildErrorKind::Constant => panic!("Tree refers to constant not in table"),
-				ChildErrorKind::Missing => Err(mk_errv(
+				ChildErrorKind::Missing => Err(mk_err(
 					ctx.i.i("Constant does not exist").await,
 					format!("{name} does not refer to a constant"),
 					[pos],
@@ -140,27 +143,27 @@ impl Default for WeakRoot {
 
 pub struct TreeFromApiCtx<'a> {
 	pub sys: &'a System,
-	pub consts: &'a mut HashMap<Sym, Expr>,
+	pub consts: &'a MemoMap<Sym, Expr>,
-	pub path: Tok<Vec<Tok<String>>>,
+	pub path: Tok<Vec<IStr>>,
 }
 impl<'a> TreeFromApiCtx<'a> {
 	#[must_use]
-	pub async fn push<'c>(&'c mut self, name: Tok<String>) -> TreeFromApiCtx<'c> {
+	pub async fn push<'c>(&'c self, name: IStr) -> TreeFromApiCtx<'c> {
 		let path = self.sys.ctx().i.i(&self.path.iter().cloned().chain([name]).collect_vec()).await;
-		TreeFromApiCtx { path, consts: &mut *self.consts, sys: self.sys }
+		TreeFromApiCtx { path, consts: self.consts, sys: self.sys }
 	}
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ResolvedImport {
-	target: Sym,
+	pub target: Sym,
-	sr: SrcRange,
+	pub pos: Pos,
 }
 
 #[derive(Clone, Default)]
 pub struct Module {
-	pub imports: HashMap<Tok<String>, Result<ResolvedImport, Vec<ResolvedImport>>>,
+	pub imports: HashMap<IStr, Result<ResolvedImport, Vec<ResolvedImport>>>,
-	pub members: HashMap<Tok<String>, Rc<Member>>,
+	pub members: HashMap<IStr, Rc<Member>>,
 }
 impl Module {
 	#[must_use]
@@ -208,7 +211,7 @@ impl Module {
 				Ok(abs_path) => {
 					let names_res = match abs_path.strip_prefix(&ctx.pars_prefix[..]) {
 						None => {
-							let mut tree_ctx = (ctx.ctx.clone(), &mut *ctx.consts);
+							let mut tree_ctx = (ctx.ctx.clone(), ctx.consts);
 							resolv_glob(&path, ctx.root, &abs_path, pos, &ctx.ctx.i, &mut tree_ctx).await
 						},
 						Some(sub_tgt) => {
@@ -234,6 +237,22 @@ impl Module {
 			}
 		}
 		let mut imports = HashMap::new();
+		if parsed.use_prelude {
+			let systems = ctx.ctx.systems.read().await;
+			for sys in systems.values().flat_map(|weak| weak.upgrade()) {
+				for prelude_item in sys.prelude() {
+					imports.insert(
+						prelude_item.last_seg(),
+						Ok(ResolvedImport {
+							target: prelude_item,
+							pos: CodeGenInfo::new_details(sys.ctor().name_tok().await, "In prelude", &ctx.ctx.i)
+								.await
+								.pos(),
+						}),
+					);
+				}
+			}
+		}
 		let conflicting_imports_msg = ctx.ctx.i.i("Conflicting imports").await;
 		for (key, values) in imports_by_name {
 			if values.len() == 1 {
@@ -243,8 +262,8 @@ impl Module {
 				match abs_path_res {
 					Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, sr.pos(), &import.to_string()).await),
 					Ok(abs_path) => {
-						imports
+						let target = abs_path.to_sym(&ctx.ctx.i).await;
-							.insert(key, Ok(ResolvedImport { target: abs_path.to_sym(&ctx.ctx.i).await, sr }));
+						imports.insert(key, Ok(ResolvedImport { target, pos: sr.pos() }));
 					},
 				}
 			} else {
@@ -252,7 +271,7 @@ impl Module {
 					ctx.rep.report(mk_err(
 						conflicting_imports_msg.clone(),
 						format!("{key} is imported multiple times from different modules"),
-						[item.sr.pos().into()],
+						[item.sr.pos()],
 					));
 				}
 			}
@@ -263,7 +282,10 @@ impl Module {
 				let values = stream::iter(values)
 					.then(|(n, sr)| {
 						clone!(key; async move {
-							ResolvedImport { target: n.to_vname().suffix([key.clone()]).to_sym(i).await, sr }
+							ResolvedImport {
+								target: n.to_vname().suffix([key.clone()]).to_sym(i).await,
+								pos: sr.pos(),
+							}
 						})
 					})
 					.collect::<Vec<_>>()
@@ -274,11 +296,12 @@ impl Module {
 		let self_referential_msg = ctx.ctx.i.i("Self-referential import").await;
 		for (key, value) in imports.iter() {
 			let Ok(import) = value else { continue };
-			if import.target.strip_prefix(&path[..]).is_some_and(|t| t.starts_with(&[key.clone()])) {
+			if import.target.strip_prefix(&path[..]).is_some_and(|t| t.starts_with(slice::from_ref(key)))
+			{
 				ctx.rep.report(mk_err(
 					self_referential_msg.clone(),
 					format!("import {} points to itself or a path within itself", &import.target),
-					[import.sr.pos().into()],
+					[import.pos.clone()],
 				));
 			}
 		}
@@ -302,7 +325,7 @@ impl Module {
 		&self,
 		other: &Module,
 		ctx: Ctx,
-		consts: &mut HashMap<Sym, Expr>,
+		consts: &MemoMap<Sym, Expr>,
 	) -> Result<Module, MergeErr> {
 		if !self.imports.is_empty() || !other.imports.is_empty() {
 			return Err(MergeErr { path: VPath::new([]), kind: MergeErrKind::Imports });
@@ -363,17 +386,17 @@ pub struct FromParsedCtx<'a> {
 	pars_prefix: Sym,
 	pars_root: &'a ParsedModule,
 	root: &'a Module,
-	consts: &'a mut HashMap<Sym, Expr>,
 	rep: &'a Reporter,
 	ctx: &'a Ctx,
+	consts: &'a MemoMap<Sym, Expr>,
 	deferred_consts: &'a mut HashMap<Sym, (api::SysId, api::ParsedConstId)>,
 }
 
 impl Tree for Module {
-	type Ctx<'a> = (Ctx, &'a mut HashMap<Sym, Expr>);
+	type Ctx<'a> = (Ctx, &'a MemoMap<Sym, Expr>);
 	async fn child(
 		&self,
-		key: Tok<String>,
+		key: IStr,
 		public_only: bool,
 		(ctx, consts): &mut Self::Ctx<'_>,
 	) -> crate::dealias::ChildResult<'_, Self> {
@@ -388,7 +411,7 @@ impl Tree for Module {
 			MemberKind::Const => Err(ChildErrorKind::Constant),
 		}
 	}
-	fn children(&self, public_only: bool) -> hashbrown::HashSet<Tok<String>> {
+	fn children(&self, public_only: bool) -> hashbrown::HashSet<IStr> {
 		self.members.iter().filter(|(_, v)| !public_only || v.public).map(|(k, _)| k.clone()).collect()
 	}
 }
@@ -400,7 +423,7 @@ pub struct Member {
 }
 impl Member {
 	#[must_use]
-	pub async fn kind<'a>(&'a self, ctx: Ctx, consts: &mut HashMap<Sym, Expr>) -> &'a MemberKind {
+	pub async fn kind<'a>(&'a self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> &'a MemberKind {
 		(self.kind.get_or_init(async {
 			let handle = self.lazy.borrow_mut().take().expect("If kind is uninit, lazy must be Some");
 			handle.run(ctx, consts).await
@@ -417,11 +440,7 @@ impl MemberKind {
 	#[must_use]
 	async fn from_parsed(parsed: &ParsedMemberKind, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
 		match parsed {
-			ParsedMemberKind::ParsedConst(expr) => {
+			ParsedMemberKind::Const(id, sys) => {
-				ctx.consts.insert(path, expr.clone());
-				MemberKind::Const
-			},
-			ParsedMemberKind::DeferredConst(id, sys) => {
 				ctx.deferred_consts.insert(path, (sys.id(), *id));
 				MemberKind::Const
 			},
@@ -438,12 +457,13 @@ pub struct LazyMemberHandle {
 }
 impl LazyMemberHandle {
 	#[must_use]
-	pub async fn run(self, ctx: Ctx, consts: &mut HashMap<Sym, Expr>) -> MemberKind {
+	pub async fn run(self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> MemberKind {
 		let sys = ctx.system_inst(self.sys).await.expect("Missing system for lazy member");
 		match sys.get_tree(self.id).await {
 			api::MemberKind::Const(c) => {
 				let mut pctx = ExprParseCtx { ctx: &ctx, exprs: sys.ext().exprs() };
-				consts.insert(self.path, Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await);
+				let expr = Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await;
+				consts.insert(self.path, expr);
 				MemberKind::Const
 			},
 			api::MemberKind::Module(m) => MemberKind::Module(

orchid-macros/Cargo.toml

@@ -1,6 +0,0 @@
-[package]
-name = "orchid-macros"
-version = "0.1.0"
-edition = "2024"
-
-[dependencies]

orchid-macros/src/main.rs

@@ -1,3 +0,0 @@
-fn main() {
-	println!("Hello, world!");
-}

orchid-std/Cargo.toml

@@ -5,13 +5,11 @@ edition = "2024"
 
 [dependencies]
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-async-stream = "0.3.6"
+hashbrown = "0.16.0"
-futures = "0.3.31"
-hashbrown = "0.15.2"
 itertools = "0.14.0"
 never = "0.1.0"
-once_cell = "1.20.2"
+once_cell = "1.21.3"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
@@ -20,8 +18,9 @@ orchid-extension = { version = "0.1.0", path = "../orchid-extension", features =
   "tokio",
 ] }
 ordered-float = "5.0.0"
-rust_decimal = "1.36.0"
+rust_decimal = "1.37.2"
-tokio = { version = "1.43.0", features = ["full"] }
+substack = "1.1.1"
+tokio = { version = "1.47.1", features = ["full"] }
 
 [dev-dependencies]
-test_executors = "0.3.2"
+test_executors = "0.3.5"

orchid-std/src/lib.rs

@@ -1,6 +1,9 @@
-// mod macros;
+mod macros;
 mod std;
 
 pub use std::number::num_atom::{Float, HomoArray, Int, Num};
 pub use std::std_system::StdSystem;
 pub use std::string::str_atom::OrcString;
+
+pub use macros::macro_system::MacroSystem;
+pub use macros::mactree::{MacTok, MacTree};

orchid-std/src/macros/instantiate_tpl.rs

@@ -0,0 +1,49 @@
+use std::borrow::Cow;
+
+use never::Never;
+use orchid_extension::atom::{Atomic, TypAtom};
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::GExpr;
+
+use crate::macros::mactree::{MacTok, MacTree, map_mactree};
+
+#[derive(Clone)]
+pub struct InstantiateTplCall {
+	pub(crate) tpl: MacTree,
+	pub(crate) argc: usize,
+	pub(crate) argv: Vec<MacTree>,
+}
+impl Atomic for InstantiateTplCall {
+	type Variant = OwnedVariant;
+	type Data = ();
+}
+impl OwnedAtom for InstantiateTplCall {
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	type Refs = Never;
+	// Technically must be supported but shouldn't actually ever be called
+	async fn call_ref(&self, arg: Expr) -> GExpr {
+		eprintln!(
+			"Copying partially applied instantiate_tpl call. This is an internal value.\
+      \nIt should be fully consumed within generated code."
+		);
+		self.clone().call(arg).await
+	}
+	async fn call(mut self, arg: Expr) -> GExpr {
+		match TypAtom::<MacTree>::try_from_expr(arg).await {
+			Err(e) => return Err::<Never, _>(e).to_expr().await,
+			Ok(t) => self.argv.push(own(t).await),
+		};
+		if self.argv.len() < self.argc {
+			return self.to_expr().await;
+		}
+		let mut args = self.argv.into_iter();
+		let ret = map_mactree(&self.tpl, &mut false, &mut |mt| match mt.tok() {
+			MacTok::Slot => Some(args.next().expect("Not enough arguments to fill all slots")),
+			_ => None,
+		});
+		assert!(args.next().is_none(), "Too many arguments for all slots");
+		ret.to_expr().await
+	}
+}

orchid-std/src/macros/let_line.rs

@@ -0,0 +1,120 @@
+use std::pin::pin;
+
+use futures::{FutureExt, StreamExt, stream};
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::{OrcRes, Reporter};
+use orchid_base::name::Sym;
+use orchid_base::parse::{
+	Comment, ParseCtx, Parsed, Snippet, expect_tok, token_errv, try_pop_no_fluff,
+};
+use orchid_base::sym;
+use orchid_base::tree::Paren;
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::parser::{ConstCtx, PSnippet, PTok, PTokTree, ParsCtx, ParsedLine, Parser};
+
+use crate::macros::mactree::{MacTok, MacTree, glossary_v, map_mactree_v};
+
+#[derive(Default)]
+pub struct LetLine;
+impl Parser for LetLine {
+	const LINE_HEAD: &'static str = "let";
+	async fn parse<'a>(
+		ctx: ParsCtx<'a>,
+		exported: bool,
+		comments: Vec<Comment>,
+		line: PSnippet<'a>,
+	) -> OrcRes<Vec<ParsedLine>> {
+		let sr = line.sr();
+		let Parsed { output: name_tok, tail } = try_pop_no_fluff(&ctx, line).await?;
+		let Some(name) = name_tok.as_name() else {
+			let err = token_errv(&ctx, name_tok, "Constant must have a name", |t| {
+				format!("Expected a name but found {t}")
+			});
+			return Err(err.await);
+		};
+		let Parsed { tail, .. } = expect_tok(&ctx, tail, ctx.i().i("=").await).await?;
+		let aliased = parse_tokv(tail, &ctx).await;
+		Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name, async move |ctx| {
+			let rep = Reporter::new();
+			let dealiased = dealias_mac_v(aliased, &ctx, &rep).await;
+			let macro_input = MacTok::S(Paren::Round, dealiased).at(sr.pos());
+			if let Some(e) = rep.res() {
+				return Err(e);
+			}
+			Ok(call([
+				sym_ref(sym!(macros::lower; ctx.i()).await),
+				call([sym_ref(sym!(macros::resolve; ctx.i()).await), atom(macro_input)]),
+			]))
+		})])
+	}
+}
+
+pub async fn dealias_mac_v(aliased: Vec<MacTree>, ctx: &ConstCtx, rep: &Reporter) -> Vec<MacTree> {
+	let keys = glossary_v(&aliased).collect_vec();
+	let mut names: HashMap<_, _> = HashMap::new();
+	let mut stream = pin!(ctx.names(&keys).zip(stream::iter(&keys)));
+	while let Some((canonical, local)) = stream.next().await {
+		match canonical {
+			Err(e) => rep.report(e),
+			Ok(name) => {
+				names.insert(local.clone(), name);
+			},
+		}
+	}
+	map_mactree_v(&aliased, &mut false, &mut |tree| match &*tree.tok {
+		MacTok::Name(n) => names.get(n).map(|new_n| MacTok::Name(new_n.clone()).at(tree.pos())),
+		_ => None,
+	})
+}
+
+pub async fn parse_tokv(line: PSnippet<'_>, ctx: &impl ParseCtx) -> Vec<MacTree> {
+	if let Some((idx, arg)) = line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?))) {
+		let (head, lambda) = line.split_at(idx as u32);
+		let (_, body) = lambda.pop_front().unwrap();
+		let body = parse_tokv(body, ctx).boxed_local().await;
+		let mut all = parse_tokv_no_lambdas(&head, ctx).await;
+		match parse_tok(arg, ctx).await {
+			Some(arg) => all.push(MacTok::Lambda(arg, body).at(lambda.sr().pos())),
+			None => ctx.rep().report(
+				token_errv(ctx, arg, "Lambda argument fluff", |arg| {
+					format!("Lambda arguments must be a valid token, found meaningless fragment {arg}")
+				})
+				.await,
+			),
+		};
+		all
+	} else {
+		parse_tokv_no_lambdas(&line, ctx).await
+	}
+}
+
+async fn parse_tokv_no_lambdas(line: &[PTokTree], ctx: &impl ParseCtx) -> Vec<MacTree> {
+	stream::iter(line).filter_map(|tt| parse_tok(tt, ctx)).collect().await
+}
+
+pub async fn parse_tok(tree: &PTokTree, ctx: &impl ParseCtx) -> Option<MacTree> {
+	let tok = match &tree.tok {
+		PTok::Bottom(errv) => MacTok::Bottom(errv.clone()),
+		PTok::BR | PTok::Comment(_) => return None,
+		PTok::Name(n) => MacTok::Name(Sym::new([n.clone()], ctx.i()).await.unwrap()),
+		PTok::NS(..) => match tree.as_multiname() {
+			Ok(mn) => MacTok::Name(mn.to_sym(ctx.i()).await),
+			Err(nested) => {
+				ctx.rep().report(
+					token_errv(ctx, tree, ":: can only be followed by a name in an expression", |tok| {
+						format!("Expected name, found {tok}")
+					})
+					.await,
+				);
+				return parse_tok(nested, ctx).boxed_local().await;
+			},
+		},
+		PTok::Handle(expr) => MacTok::Value(expr.clone()),
+		PTok::NewExpr(never) => match *never {},
+		PTok::LambdaHead(_) => panic!("Lambda-head handled in the sequence parser"),
+		PTok::S(p, body) =>
+			MacTok::S(*p, parse_tokv(Snippet::new(tree, body), ctx).boxed_local().await),
+	};
+	Some(tok.at(tree.sr().pos()))
+}

orchid-std/src/macros/macro_lib.rs

@@ -0,0 +1,83 @@
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::Reporter;
+use orchid_base::sym;
+use orchid_extension::atom::TypAtom;
+use orchid_extension::atom_owned::own;
+use orchid_extension::conv::ToExpr;
+use orchid_extension::coroutine_exec::exec;
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::reflection::{ReflMemKind, refl};
+use orchid_extension::tree::{GenMember, comments, fun, prefix};
+use substack::Substack;
+
+use crate::Int;
+use crate::macros::instantiate_tpl::InstantiateTplCall;
+use crate::macros::macro_line::{Macro, Matcher};
+use crate::macros::mactree::{LowerCtx, MacTree};
+use crate::macros::recur_state::RecurState;
+use crate::macros::resolve::{ResolveCtx, resolve};
+
+pub fn gen_macro_lib() -> Vec<GenMember> {
+	prefix("macros", [
+		comments(
+			["This is an internal function, you can't obtain a value of its argument type.", "hidden"],
+			fun(true, "instantiate_tpl", |tpl: TypAtom<MacTree>, right: Int| async move {
+				InstantiateTplCall {
+					tpl: own(tpl).await,
+					argc: right.0.try_into().unwrap(),
+					argv: Vec::new(),
+				}
+			}),
+		),
+		fun(true, "resolve", |tpl: TypAtom<MacTree>| async move {
+			call([
+				sym_ref(sym!(macros::resolve_recur; tpl.untyped.ctx().i()).await),
+				atom(RecurState::Bottom),
+				tpl.untyped.ex().to_expr().await,
+			])
+		}),
+		fun(true, "lower", |tpl: TypAtom<MacTree>| async move {
+			let ctx = LowerCtx { sys: tpl.untyped.ctx().clone(), rep: &Reporter::new() };
+			let res = own(tpl).await.lower(ctx, Substack::Bottom).await;
+			if let Some(e) = Reporter::new().res() { Err(e) } else { Ok(res) }
+		}),
+		fun(true, "resolve_recur", |state: TypAtom<RecurState>, tpl: TypAtom<MacTree>| async move {
+			exec("macros::resolve_recur", async move |mut h| {
+				let ctx = tpl.ctx().clone();
+				let root = refl(&ctx);
+				let tpl = own(tpl.clone()).await;
+				let mut macros = HashMap::new();
+				for n in tpl.glossary() {
+					if let Ok(ReflMemKind::Const) = root.get_by_path(n).await.map(|m| m.kind()) {
+						let Ok(mac) = h.exec::<TypAtom<Macro>>(sym_ref(n.clone())).await else { continue };
+						let mac = own(mac).await;
+						macros.entry(mac.0.own_kws[0].clone()).or_insert(mac);
+					}
+				}
+				let mut named = HashMap::new();
+				let mut priod = Vec::new();
+				for (_, mac) in macros.iter() {
+					for rule in mac.0.rules.iter() {
+						if rule.glossary.is_subset(tpl.glossary()) {
+							match &rule.pattern {
+								Matcher::Named(m) =>
+									named.entry(m.head()).or_insert(Vec::new()).push((m, mac, rule)),
+								Matcher::Priod(p) => priod.push((mac.0.prio, (p, mac, rule))),
+							}
+						}
+					}
+				}
+				let priod = priod.into_iter().sorted_unstable_by_key(|(p, _)| *p).map(|(_, r)| r).collect();
+				let mut rctx = ResolveCtx { h, recur: own(state).await, ctx: ctx.clone(), named, priod };
+				let resolve_res = resolve(&mut rctx, &tpl).await;
+				std::mem::drop(rctx);
+				match resolve_res {
+					Some(out_tree) => out_tree.to_expr().await,
+					None => tpl.to_expr().await,
+				}
+			})
+			.await
+		}),
+	])
+}

orchid-std/src/macros/macro_line.rs

@@ -0,0 +1,228 @@
+use std::borrow::Cow;
+use std::cell::RefCell;
+use std::rc::Rc;
+
+use async_once_cell::OnceCell;
+use futures::{StreamExt, stream};
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools;
+use never::Never;
+use orchid_base::error::{OrcRes, Reporter, mk_err};
+use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::parse::{
+	Comment, ParseCtx, Parsed, Snippet, expect_end, expect_tok, line_items, token_errv,
+	try_pop_no_fluff,
+};
+use orchid_base::tree::{Paren, Token};
+use orchid_base::{clone, sym};
+use orchid_extension::atom::{Atomic, TypAtom};
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::parser::{PSnippet, ParsCtx, ParsedLine, Parser};
+
+use crate::macros::let_line::{dealias_mac_v, parse_tokv};
+use crate::macros::mactree::{glossary_v, map_mactree_v};
+use crate::macros::recur_state::{RecurState, RulePath};
+use crate::macros::rule::matcher::{NamedMatcher, PriodMatcher};
+use crate::{Int, MacTok};
+
+#[derive(Default)]
+pub struct MacroLine;
+impl Parser for MacroLine {
+	const LINE_HEAD: &'static str = "macro";
+	async fn parse<'a>(
+		ctx: ParsCtx<'a>,
+		exported: bool,
+		comments: Vec<Comment>,
+		line: PSnippet<'a>,
+	) -> OrcRes<Vec<ParsedLine>> {
+		if exported {
+			return Err(mk_err(
+				ctx.i().i("macros are always exported").await,
+				"The export keyword is forbidden here to avoid confusion\n\
+					because macros are exported by default",
+				[line.sr()],
+			));
+		}
+		let module = ctx.module();
+		let Parsed { output, tail } = try_pop_no_fluff(&ctx, line).await?;
+		let bad_first_item_err = || {
+			token_errv(&ctx, output, "Expected priority or block", |s| {
+				format!("Expected a priority number or a () block, found {s}")
+			})
+		};
+		let (prio, body) = match &output.tok {
+			Token::S(Paren::Round, body) => (None, body),
+			Token::Handle(expr) => match TypAtom::<Int>::try_from_expr(expr.clone()).await {
+				Err(e) => {
+					return Err(e + bad_first_item_err().await);
+				},
+				Ok(prio) => {
+					let Token::S(Paren::Round, block) = &output.tok else {
+						return Err(
+							token_errv(&ctx, output, "Expected () block", |s| {
+								format!("Expected a () block, found {s}")
+							})
+							.await,
+						);
+					};
+					(Some(prio), block)
+				},
+			},
+			_ => return Err(bad_first_item_err().await),
+		};
+		expect_end(&ctx, tail).await?;
+		let lines = line_items(&ctx, Snippet::new(output, body)).await;
+		let Some((kw_line, rule_lines)) = lines.split_first() else { return Ok(Vec::new()) };
+		let mut keywords = HashMap::new();
+		let Parsed { tail: kw_tail, .. } =
+			expect_tok(&ctx, kw_line.tail, ctx.i().i("keywords").await).await?;
+		for kw_tok in kw_tail.iter().filter(|kw| !kw.is_fluff()) {
+			match kw_tok.as_name() {
+				Some(kw) => {
+					keywords.insert(kw, kw_tok.sr());
+				},
+				None => ctx.rep().report(
+					token_errv(&ctx, kw_tok, "invalid macro keywords list", |tok| {
+						format!("The keywords list must be a sequence of names; received {tok}")
+					})
+					.await,
+				),
+			}
+		}
+		let Some(macro_name) = keywords.keys().next().cloned() else {
+			return Err(mk_err(
+				ctx.i().i("macro with no keywords").await,
+				"Macros must define at least one macro of their own.",
+				[kw_line.tail.sr()],
+			));
+		};
+		let mut rules = Vec::new();
+		let mut lines = Vec::new();
+		for (idx, line) in rule_lines.iter().enumerate().map(|(n, v)| (n as u32, v)) {
+			let path = RulePath { module: module.clone(), main_kw: macro_name.clone(), rule: idx };
+			let sr = line.tail.sr();
+			let name = ctx.i().i(&path.name()).await;
+			let Parsed { tail, .. } = expect_tok(&ctx, line.tail, ctx.i().i("rule").await).await?;
+			let arrow_token = ctx.i().i("=>").await;
+			let Some((pattern, body)) = tail.split_once(|tok| tok.is_kw(arrow_token.clone())) else {
+				ctx.rep().report(mk_err(
+					ctx.i().i("Missing => in rule").await,
+					"Rule lines are of the form `rule ...pattern => ...body`",
+					[line.tail.sr()],
+				));
+				continue;
+			};
+			let pattern = parse_tokv(pattern, &ctx).await;
+			let mut placeholders = Vec::new();
+			map_mactree_v(&pattern, &mut false, &mut |tok| {
+				if let MacTok::Ph(ph) = tok.tok() {
+					placeholders.push((ph.clone(), tok.pos()))
+				}
+				None
+			});
+			let mut body_mactree = parse_tokv(body, &ctx).await;
+			for (ph, ph_pos) in placeholders.iter().rev() {
+				let name = ctx.module().suffix([ph.name.clone()], ctx.i()).await;
+				body_mactree = vec![
+					MacTok::Lambda(MacTok::Name(name).at(ph_pos.clone()), body_mactree).at(ph_pos.clone()),
+				]
+			}
+			let body_sr = body.sr();
+			rules.push((name.clone(), placeholders, rules.len() as u32, sr.pos(), pattern));
+			lines.push(ParsedLine::cnst(&sr, &line.output, true, name, async move |ctx| {
+				let rep = Reporter::new();
+				let body = dealias_mac_v(body_mactree, &ctx, &rep).await;
+				let macro_input = MacTok::S(Paren::Round, body).at(body_sr.pos());
+				if let Some(e) = rep.res() {
+					return Err(e);
+				}
+				Ok(call([
+					sym_ref(sym!(macros::resolve_recur; ctx.i()).await),
+					atom(RecurState::base(path)),
+					macro_input.to_expr().await,
+				]))
+			}))
+		}
+		let mac_cell = Rc::new(OnceCell::new());
+		let keywords = Rc::new(keywords);
+		let rules = Rc::new(RefCell::new(Some(rules)));
+		for (kw, sr) in &*keywords {
+			clone!(mac_cell, keywords, rules, module, prio);
+			lines.push(ParsedLine::cnst(&sr.clone(), &comments, true, kw.clone(), async move |cctx| {
+				let mac = mac_cell
+					.get_or_init(async {
+						let rep = Reporter::new();
+						let rules = rules.borrow_mut().take().expect("once cell initializer runs");
+						let rules = stream::iter(rules)
+							.then(|(body_name, placeholders, index, pos, pattern_macv)| {
+								let cctx = &cctx;
+								let rep = &rep;
+								let prio = &prio;
+								async move {
+									let pattern_abs = dealias_mac_v(pattern_macv, cctx, rep).await;
+									let glossary = glossary_v(&pattern_abs).collect();
+									let pattern_res = match prio {
+										None => NamedMatcher::new(&pattern_abs, cctx.i()).await.map(Matcher::Named),
+										Some(_) => PriodMatcher::new(&pattern_abs, cctx.i()).await.map(Matcher::Priod),
+									};
+									let placeholders = placeholders.into_iter().map(|(ph, _)| ph.name).collect_vec();
+									match pattern_res {
+										Ok(pattern) =>
+											Some(Rule { index, pos, body_name, pattern, glossary, placeholders }),
+										Err(e) => {
+											rep.report(e);
+											None
+										},
+									}
+								}
+							})
+							.flat_map(stream::iter)
+							.collect::<Vec<_>>()
+							.await;
+						let own_kws = keywords.keys().cloned().collect_vec();
+						Macro(Rc::new(MacroData { module, prio: prio.map(|i| i.0 as u64), rules, own_kws }))
+					})
+					.await;
+				atom(mac.clone())
+			}))
+		}
+		Ok(lines)
+	}
+}
+
+#[derive(Debug)]
+pub struct MacroData {
+	pub module: Sym,
+	pub prio: Option<u64>,
+	pub rules: Vec<Rule>,
+	pub own_kws: Vec<IStr>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Macro(pub Rc<MacroData>);
+#[derive(Debug)]
+pub struct Rule {
+	pub index: u32,
+	pub pos: Pos,
+	pub pattern: Matcher,
+	pub glossary: HashSet<Sym>,
+	pub placeholders: Vec<IStr>,
+	pub body_name: IStr,
+}
+#[derive(Debug)]
+pub enum Matcher {
+	Named(NamedMatcher),
+	Priod(PriodMatcher),
+}
+impl Atomic for Macro {
+	type Data = ();
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for Macro {
+	type Refs = Never;
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+}

orchid-std/src/macros/macro_system.rs

@@ -1,34 +1,49 @@
 use never::Never;
+use orchid_base::interner::Interner;
+use orchid_base::name::Sym;
 use orchid_base::reqnot::Receipt;
-use orchid_extension::atom::AtomDynfo;
+use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
 use orchid_extension::entrypoint::ExtReq;
-use orchid_extension::fs::DeclFs;
 use orchid_extension::lexer::LexerObj;
+use orchid_extension::other_system::SystemHandle;
 use orchid_extension::parser::ParserObj;
 use orchid_extension::system::{System, SystemCard};
 use orchid_extension::system_ctor::SystemCtor;
 use orchid_extension::tree::GenMember;
 
+use crate::macros::instantiate_tpl::InstantiateTplCall;
+use crate::macros::let_line::LetLine;
+use crate::macros::macro_lib::gen_macro_lib;
+use crate::macros::macro_line::{Macro, MacroLine};
 use crate::macros::mactree_lexer::MacTreeLexer;
+use crate::macros::recur_state::RecurState;
+use crate::{MacTree, StdSystem};
 
 #[derive(Default)]
 pub struct MacroSystem;
 impl SystemCtor for MacroSystem {
-	type Deps = ();
+	type Deps = StdSystem;
 	type Instance = Self;
-	const NAME: &'static str = "macros";
+	const NAME: &'static str = "orchid::macros";
 	const VERSION: f64 = 0.00_01;
-	fn inst() -> Option<Self::Instance> { Some(Self) }
+	fn inst(_: SystemHandle<StdSystem>) -> Self::Instance { Self }
 }
 impl SystemCard for MacroSystem {
 	type Ctor = Self;
 	type Req = Never;
-	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> { [] }
+	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> {
+		[
+			Some(InstantiateTplCall::dynfo()),
+			Some(MacTree::dynfo()),
+			Some(RecurState::dynfo()),
+			Some(Macro::dynfo()),
+		]
+	}
 }
 impl System for MacroSystem {
 	async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
-	fn vfs() -> orchid_extension::fs::DeclFs { DeclFs::Mod(&[]) }
+	async fn prelude(_: &Interner) -> Vec<Sym> { vec![] }
 	fn lexers() -> Vec<LexerObj> { vec![&MacTreeLexer] }
-	fn parsers() -> Vec<ParserObj> { vec![] }
+	fn parsers() -> Vec<ParserObj> { vec![&LetLine, &MacroLine] }
-	fn env() -> Vec<GenMember> { vec![] }
+	fn env() -> Vec<GenMember> { gen_macro_lib() }
 }

orchid-std/src/macros/mactree.rs

@@ -2,23 +2,85 @@ use std::borrow::Cow;
 use std::fmt::Display;
 use std::rc::Rc;
 
+use futures::FutureExt;
 use futures::future::join_all;
-use orchid_api::Paren;
+use hashbrown::HashSet;
-use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
+use itertools::Itertools;
+use orchid_base::error::{OrcErrv, Reporter, mk_err};
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants, fmt};
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::tl_cache;
+use orchid_base::tree::{Paren, indent};
 use orchid_extension::atom::Atomic;
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+use orchid_extension::conv::ToExpr;
 use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::{GExpr, arg, bot, call, lambda, sym_ref};
+use orchid_extension::system::SysCtx;
+use substack::Substack;
+
+#[derive(Clone)]
+pub struct LowerCtx<'a> {
+	pub sys: SysCtx,
+	pub rep: &'a Reporter,
+}
 
 #[derive(Debug, Clone)]
 pub struct MacTree {
 	pub pos: Pos,
 	pub tok: Rc<MacTok>,
+	pub glossary: Rc<HashSet<Sym>>,
+}
+impl MacTree {
+	pub fn tok(&self) -> &MacTok { &self.tok }
+	pub fn pos(&self) -> Pos { self.pos.clone() }
+	pub fn glossary(&self) -> &HashSet<Sym> { &self.glossary }
+	pub async fn lower(&self, ctx: LowerCtx<'_>, args: Substack<'_, Sym>) -> GExpr {
+		let expr = match self.tok() {
+			MacTok::Bottom(e) => bot(e.clone()),
+			MacTok::Lambda(arg, body) => {
+				let MacTok::Name(name) = &*arg.tok else {
+					let err = mk_err(
+						ctx.sys.i().i("Syntax error after macros").await,
+						"This token ends up as a binding, consider replacing it with a name",
+						[arg.pos()],
+					);
+					ctx.rep.report(err.clone());
+					return bot(err);
+				};
+				lambda(args.len() as u64, lower_v(body, ctx, args.push(name.clone())).await)
+			},
+			MacTok::Name(name) => match args.iter().enumerate().find(|(_, n)| *n == name) {
+				None => sym_ref(name.clone()),
+				Some((i, _)) => arg((args.len() - i) as u64),
+			},
+			MacTok::Ph(ph) => {
+				let err = mk_err(
+					ctx.sys.i().i("Placeholder in value").await,
+					format!("Placeholder {ph} is only supported in macro patterns"),
+					[self.pos()],
+				);
+				ctx.rep.report(err.clone());
+				return bot(err);
+			},
+			MacTok::S(Paren::Round, body) => call(lower_v(body, ctx, args).await),
+			MacTok::S(..) => {
+				let err = mk_err(
+					ctx.sys.i().i("[] or {} after macros").await,
+					format!("{} didn't match any macro", fmt(self, ctx.sys.i()).await),
+					[self.pos()],
+				);
+				ctx.rep.report(err.clone());
+				return bot(err);
+			},
+			MacTok::Slot => panic!("Uninstantiated template should never be exposed"),
+			MacTok::Value(v) => v.clone().to_expr().await,
+		};
+		expr.at(self.pos())
+	}
 }
-impl MacTree {}
 impl Atomic for MacTree {
 	type Data = ();
 	type Variant = OwnedVariant;
@@ -27,11 +89,20 @@ impl OwnedAtom for MacTree {
 	type Refs = ();
 
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	async fn print_atom<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		self.tok.print(c).await
+	}
+}
+impl Format for MacTree {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		self.tok.print(c).await
 	}
 }
 
+pub async fn lower_v(v: &[MacTree], ctx: LowerCtx<'_>, args: Substack<'_, Sym>) -> Vec<GExpr> {
+	join_all(v.iter().map(|t| t.lower(ctx.clone(), args.clone())).collect::<Vec<_>>()).await
+}
+
 #[derive(Debug, Clone)]
 pub enum MacTok {
 	S(Paren, Vec<MacTree>),
@@ -39,8 +110,25 @@ pub enum MacTok {
 	/// Only permitted in arguments to `instantiate_tpl`
 	Slot,
 	Value(Expr),
-	Lambda(Vec<MacTree>, Vec<MacTree>),
+	Lambda(MacTree, Vec<MacTree>),
+	/// Only permitted in "pattern" values produced by macro blocks, which are
+	/// never accessed as variables by usercode
 	Ph(Ph),
+	Bottom(OrcErrv),
+}
+impl MacTok {
+	pub fn build_glossary(&self) -> HashSet<Sym> {
+		match self {
+			MacTok::Bottom(_) | MacTok::Ph(_) | MacTok::Slot | MacTok::Value(_) => HashSet::new(),
+			MacTok::Name(sym) => HashSet::from([sym.clone()]),
+			MacTok::S(_, body) => body.iter().flat_map(|mt| &*mt.glossary).cloned().collect(),
+			MacTok::Lambda(arg, body) =>
+				body.iter().chain([arg]).flat_map(|mt| &*mt.glossary).cloned().collect(),
+		}
+	}
+	pub fn at(self, pos: impl Into<Pos>) -> MacTree {
+		MacTree { pos: pos.into(), glossary: Rc::new(self.build_glossary()), tok: Rc::new(self) }
+	}
 }
 impl Format for MacTok {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -50,7 +138,7 @@ impl Format for MacTok {
 				tl_cache!(Rc<Variants>: Rc::new(Variants::default()
 					.unbounded("\\{0b}.{1l}")
 					.bounded("(\\{0b}.{1b})"))),
-				[mtreev_fmt(arg, c).await, mtreev_fmt(b, c).await],
+				[arg.print(c).boxed_local().await, mtreev_fmt(b, c).await],
 			),
 			Self::Name(n) => format!("{n}").into(),
 			Self::Ph(ph) => format!("{ph}").into(),
@@ -62,7 +150,9 @@ impl Format for MacTok {
 				},
 				[mtreev_fmt(body, c).await],
 			),
-			Self::Slot => "SLOT".into(),
+			Self::Slot => "$SLOT".into(),
+			Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()).into(),
+			Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),
 		}
 	}
 }
@@ -76,7 +166,7 @@ pub async fn mtreev_fmt<'b>(
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct Ph {
-	pub name: Tok<String>,
+	pub name: IStr,
 	pub kind: PhKind,
 }
 impl Display for Ph {
@@ -96,3 +186,46 @@ pub enum PhKind {
 	Scalar,
 	Vector { at_least_one: bool, priority: u8 },
 }
+
+pub fn map_mactree<F: FnMut(MacTree) -> Option<MacTree>>(
+	src: &MacTree,
+	changed: &mut bool,
+	map: &mut F,
+) -> MacTree {
+	let tok = match map(src.clone()) {
+		Some(new_tok) => {
+			*changed = true;
+			return new_tok;
+		},
+		None => match &*src.tok {
+			MacTok::Lambda(arg, body) => MacTok::Lambda(
+				ro(changed, |changed| map_mactree(arg, changed, map)),
+				map_mactree_v(body, changed, map),
+			),
+			MacTok::Name(_) | MacTok::Value(_) => return src.clone(),
+			MacTok::Slot | MacTok::Ph(_) | MacTok::Bottom(_) => return src.clone(),
+			MacTok::S(p, body) => MacTok::S(*p, map_mactree_v(body, changed, map)),
+		},
+	};
+	if *changed { tok.at(src.pos()) } else { src.clone() }
+}
+pub fn map_mactree_v<F: FnMut(MacTree) -> Option<MacTree>>(
+	src: &[MacTree],
+	changed: &mut bool,
+	map: &mut F,
+) -> Vec<MacTree> {
+	src.iter().map(|tree| ro(changed, |changed| map_mactree(tree, changed, map))).collect_vec()
+}
+
+/// reverse "or". Inside, the flag is always false, but raising it will raise
+/// the outside flag too.
+fn ro<T>(flag: &mut bool, cb: impl FnOnce(&mut bool) -> T) -> T {
+	let mut new_flag = false;
+	let val = cb(&mut new_flag);
+	*flag |= new_flag;
+	val
+}
+
+pub fn glossary_v(src: &[MacTree]) -> impl Iterator<Item = Sym> {
+	src.iter().flat_map(|mt| mt.glossary()).cloned()
+}

orchid-std/src/macros/mactree_lexer.rs

@@ -1,12 +1,16 @@
 use std::ops::RangeInclusive;
-use std::rc::Rc;
 
 use futures::FutureExt;
-use orchid_base::error::{OrcRes, mk_errv};
+use orchid_base::error::{OrcRes, mk_err};
+use orchid_base::parse::ParseCtx;
+use orchid_base::sym;
 use orchid_base::tokens::PARENS;
+use orchid_base::tree::Paren;
 use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
-use orchid_extension::tree::{GenTok, GenTokTree, x_tok};
+use orchid_extension::parser::p_tree2gen;
+use orchid_extension::tree::{GenTok, GenTokTree, ref_tok, x_tok};
 
+use crate::macros::let_line::parse_tok;
 use crate::macros::mactree::{MacTok, MacTree};
 
 #[derive(Default)]
@@ -15,47 +19,78 @@ impl Lexer for MacTreeLexer {
 	const CHAR_FILTER: &'static [RangeInclusive<char>] = &['\''..='\''];
 	async fn lex<'a>(tail: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree)> {
 		let Some(tail2) = tail.strip_prefix('\'') else {
-			return Err(err_not_applicable(ctx.i()).await.into());
+			return Err(err_not_applicable(ctx.i()).await);
 		};
 		let tail3 = tail2.trim_start();
-		return match mac_tree(tail3, ctx).await {
+		let mut args = Vec::new();
-			Ok((tail4, mactree)) => Ok((tail4, x_tok(mactree).at(ctx.pos_tt(tail, tail4)))),
+		return match mac_tree(tail3, &mut args, ctx).await {
+			Ok((tail4, mactree)) => {
+				let range = ctx.pos_tt(tail, tail4);
+				let tok = match &args[..] {
+					[] => x_tok(mactree).await,
+					_ => {
+						let call = ([
+							ref_tok(sym!(macros::instantiate_tpl; ctx.i()).await).await.at(range.clone()),
+							x_tok(mactree).await.at(range.clone()),
+						]
+						.into_iter())
+						.chain(args.into_iter());
+						GenTok::S(Paren::Round, call.collect())
+					},
+				};
+				Ok((tail4, tok.at(range)))
+			},
 			Err(e) => Ok((tail2, GenTok::Bottom(e).at(ctx.pos_lt(1, tail2)))),
 		};
-		async fn mac_tree<'a>(tail: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, MacTree)> {
+		async fn mac_tree<'a>(
+			tail: &'a str,
+			args: &mut Vec<GenTokTree>,
+			ctx: &'a LexContext<'a>,
+		) -> OrcRes<(&'a str, MacTree)> {
 			for (lp, rp, paren) in PARENS {
 				let Some(mut body_tail) = tail.strip_prefix(*lp) else { continue };
 				let mut items = Vec::new();
 				return loop {
-					let tail2 = body_tail.trim();
+					let tail2 = body_tail.trim_start();
 					if let Some(tail3) = tail2.strip_prefix(*rp) {
-						break Ok((tail3, MacTree {
+						break Ok((tail3, MacTok::S(*paren, items).at(ctx.pos_tt(tail, tail3).pos())));
-							pos: ctx.pos_tt(tail, tail3).pos(),
-							tok: Rc::new(MacTok::S(*paren, items)),
-						}));
 					} else if tail2.is_empty() {
-						return Err(mk_errv(
+						return Err(mk_err(
 							ctx.i().i("Unclosed block").await,
 							format!("Expected closing {rp}"),
 							[ctx.pos_lt(1, tail)],
 						));
 					}
-					let (new_tail, new_item) = mac_tree(tail2, ctx).boxed_local().await?;
+					let (new_tail, new_item) = mac_tree(tail2, args, ctx).boxed_local().await?;
 					body_tail = new_tail;
 					items.push(new_item);
 				};
 			}
-			const INTERPOL: &[&str] = &["$", "..$"];
+			if let Some(tail2) = tail.strip_prefix("$") {
-			for pref in INTERPOL {
+				let (tail3, sub) = ctx.recurse(tail2).await?;
-				let Some(code) = tail.strip_prefix(pref) else { continue };
+				let sr = ctx.pos_tt(tail, tail3);
-				todo!("Register parameter, and push this onto the argument stack held in the atom")
+				args.push(p_tree2gen(sub));
+				return Ok((tail3, MacTok::Slot.at(sr.pos())));
+			}
+			if let Some(tail2) = tail.strip_prefix("\\") {
+				let tail2 = tail2.trim_start();
+				let (mut tail3, param) = mac_tree(tail2, args, ctx).boxed_local().await?;
+				let mut body = Vec::new();
+				loop {
+					let tail4 = tail3.trim_start();
+					if tail4.is_empty() || tail4.starts_with(|c| ")]}".contains(c)) {
+						break;
+					};
+					let (tail5, body_tok) = mac_tree(tail4, args, ctx).boxed_local().await?;
+					body.push(body_tok);
+					tail3 = tail5;
+				}
+				Ok((tail3, MacTok::Lambda(param, body).at(ctx.pos_tt(tail, tail3).pos())))
+			} else {
+				let (tail2, sub) = ctx.recurse(tail).await?;
+				let parsed = parse_tok(&sub, ctx).await.expect("Unexpected invalid token");
+				Ok((tail2, parsed))
 			}
-			todo!("recursive lexer call");
-			return Err(mk_errv(
-				ctx.i().i("Expected token after '").await,
-				format!("Expected a token after ', found {tail:?}"),
-				[ctx.pos_lt(1, tail)],
-			));
 		}
 	}
 }

orchid-std/src/macros/mod.rs

@@ -1,6 +1,12 @@
-mod macro_system;
+mod instantiate_tpl;
+mod let_line;
+mod macro_lib;
+mod macro_line;
+pub mod macro_system;
 pub mod mactree;
 mod mactree_lexer;
+pub mod recur_state;
+mod resolve;
 mod rule;
 
 use mactree::{MacTok, MacTree};

orchid-std/src/macros/recur_state.rs

@@ -0,0 +1,59 @@
+use std::borrow::Cow;
+use std::fmt;
+use std::rc::Rc;
+
+use never::Never;
+use orchid_base::interner::Tok;
+use orchid_base::name::Sym;
+use orchid_extension::atom::Atomic;
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct RulePath {
+	pub module: Sym,
+	pub main_kw: IStr,
+	pub rule: u32,
+}
+impl RulePath {
+	pub fn name(&self) -> String { format!("rule::{}::{}", self.main_kw, self.rule) }
+}
+impl fmt::Display for RulePath {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "Rule {}::({})::{}", self.module, self.main_kw, self.rule)
+	}
+}
+
+#[derive(Clone)]
+pub enum RecurState {
+	Bottom,
+	Recursive { path: RulePath, prev: Rc<RecurState> },
+}
+impl RecurState {
+	pub fn base(path: RulePath) -> Self {
+		RecurState::Recursive { path, prev: Rc::new(RecurState::Bottom) }
+	}
+	pub fn push(&self, new: RulePath) -> Option<Self> {
+		let mut cur = self;
+		while let Self::Recursive { path, prev } = cur {
+			if &new == path {
+				return None;
+			}
+			cur = prev;
+		}
+		Some(Self::Recursive { path: new, prev: Rc::new(self.clone()) })
+	}
+}
+impl Atomic for RecurState {
+	type Data = Option<()>;
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for RecurState {
+	type Refs = Never;
+
+	async fn val(&self) -> Cow<'_, Self::Data> {
+		Cow::Owned(match self {
+			Self::Bottom => None,
+			Self::Recursive { .. } => Some(()),
+		})
+	}
+}

orchid-std/src/macros/resolve.rs

@@ -0,0 +1,110 @@
+use futures::FutureExt;
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::mk_err;
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::sym;
+use orchid_base::tree::Paren;
+use orchid_extension::conv::ToExpr;
+use orchid_extension::coroutine_exec::ExecHandle;
+use orchid_extension::gen_expr::{GExpr, bot, call, sym_ref};
+use orchid_extension::system::SysCtx;
+
+use crate::macros::macro_line::{Macro, Rule};
+use crate::macros::recur_state::{RecurState, RulePath};
+use crate::macros::rule::matcher::{NamedMatcher, PriodMatcher};
+use crate::macros::rule::state::{MatchState, StateEntry};
+use crate::{MacTok, MacTree};
+
+pub struct ResolveCtx<'a> {
+	pub ctx: SysCtx,
+	pub recur: RecurState,
+	pub h: ExecHandle<'a>,
+	pub named: HashMap<Sym, Vec<(&'a NamedMatcher, &'a Macro, &'a Rule)>>,
+	pub priod: Vec<(&'a PriodMatcher, &'a Macro, &'a Rule)>,
+}
+
+pub async fn resolve(ctx: &mut ResolveCtx<'_>, value: &MacTree) -> Option<MacTree> {
+	match value.tok() {
+		MacTok::Ph(_) | MacTok::Slot => panic!("Forbidden element in value mactree"),
+		MacTok::Bottom(_) | MacTok::Value(_) | MacTok::Name(_) => None,
+		MacTok::Lambda(arg, body) =>
+			Some(MacTok::Lambda(arg.clone(), resolve_seq(ctx, body).await?).at(value.pos())),
+		MacTok::S(ptyp, body) => Some(MacTok::S(*ptyp, resolve_seq(ctx, body).await?).at(value.pos())),
+	}
+}
+
+pub async fn resolve_seq(ctx: &mut ResolveCtx<'_>, val: &[MacTree]) -> Option<Vec<MacTree>> {
+	let mut any_changed = false;
+	let mut i = 0;
+	let mut val = val.to_vec();
+	'all_named: while i < val.len() {
+		'one_named: {
+			let MacTok::Name(key) = val[i].tok() else { break 'one_named };
+			let Some(options) = ctx.named.get(key) else { break 'one_named };
+			let matches = (options.iter())
+				.filter_map(|r| Some((r.1, r.2, r.0.apply(&val[i..], |_| false)?)))
+				.collect_vec();
+			match matches.len() {
+				0 => break 'one_named,
+				1 => {
+					any_changed = true;
+					let (mac, rule, (state, tail)) = matches.into_iter().exactly_one().unwrap();
+					let end = val.len() - tail.len();
+					let body_call = mk_body_call(mac, rule, &state, &ctx.ctx, ctx.recur.clone()).await;
+					std::mem::drop(state);
+					val.splice(i..end, [MacTok::Value(ctx.h.register(body_call).await).at(Pos::None)]);
+					i = end;
+				},
+				2.. => todo!("Named macros conflict!"),
+			}
+			continue 'all_named;
+		}
+		i += 1;
+	}
+	for (matcher, mac, rule) in &ctx.priod {
+		let Some(state) = matcher.apply(&val, |_| false) else { continue };
+		return Some(vec![
+			MacTok::Value(
+				ctx.h.register(mk_body_call(mac, rule, &state, &ctx.ctx, ctx.recur.clone()).await).await,
+			)
+			.at(Pos::None),
+		]);
+	}
+	for expr in val.iter_mut() {
+		if let Some(new) = resolve(ctx, expr).boxed_local().await {
+			*expr = new;
+			any_changed = true;
+		}
+	}
+	if any_changed { Some(val) } else { None }
+}
+
+async fn mk_body_call(
+	mac: &Macro,
+	rule: &Rule,
+	state: &MatchState<'_>,
+	ctx: &SysCtx,
+	recur: RecurState,
+) -> GExpr {
+	let rule_path =
+		RulePath { module: mac.0.module.clone(), main_kw: mac.0.own_kws[0].clone(), rule: rule.index };
+	let Some(new_recur) = recur.push(rule_path.clone()) else {
+		return bot(mk_err(
+			ctx.i().i("Circular macro dependency").await,
+			format!("The definition of {rule_path} is circular"),
+			[rule.pos.clone()],
+		));
+	};
+	let mut call_args = vec![sym_ref(mac.0.module.suffix([rule.body_name.clone()], ctx.i()).await)];
+	for name in rule.placeholders.iter() {
+		call_args.push(match state.get(name).expect("Missing state entry for placeholder") {
+			StateEntry::Scalar(scal) => (**scal).clone().to_expr().await,
+			StateEntry::Vec(vec) => MacTok::S(Paren::Round, vec.to_vec()).at(Pos::None).to_expr().await,
+		});
+	}
+	call_args
+		.push(call([sym_ref(sym!(macros::resolve_recur; ctx.i()).await), new_recur.to_expr().await]));
+	call(call_args)
+}

orchid-std/src/macros/rule/build.rs

@@ -1,5 +1,9 @@
+use futures::FutureExt;
+use futures::future::join_all;
 use itertools::Itertools;
-use orchid_base::interner::Tok;
+use orchid_base::error::{OrcRes, mk_err};
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::join_ok;
 use orchid_base::side::Side;
 
 use super::shared::{AnyMatcher, ScalMatcher, VecMatcher};
@@ -7,12 +11,12 @@ use super::vec_attrs::vec_attrs;
 use crate::macros::mactree::{Ph, PhKind};
 use crate::macros::{MacTok, MacTree};
 
-pub type MaxVecSplit<'a> = (&'a [MacTree], (Tok<String>, u8, bool), &'a [MacTree]);
+pub type MaxVecSplit<'a> = (&'a [MacTree], (IStr, u8, bool), &'a [MacTree]);
 
 /// Derive the details of the central vectorial and the two sides from a
 /// slice of Expr's
 #[must_use]
-fn split_at_max_vec(pattern: &[MacTree]) -> Option<MaxVecSplit> {
+fn split_at_max_vec(pattern: &'_ [MacTree]) -> Option<MaxVecSplit<'_>> {
 	let rngidx = pattern
 		.iter()
 		.position_max_by_key(|expr| vec_attrs(expr).map(|attrs| attrs.1 as i64).unwrap_or(-1))?;
@@ -27,25 +31,29 @@ fn scal_cnt<'a>(iter: impl Iterator<Item = &'a MacTree>) -> usize {
 	iter.take_while(|expr| vec_attrs(expr).is_none()).count()
 }
 
-#[must_use]
+pub async fn mk_any(pattern: &[MacTree], i: &Interner) -> OrcRes<AnyMatcher> {
-pub fn mk_any(pattern: &[MacTree]) -> AnyMatcher {
 	let left_split = scal_cnt(pattern.iter());
 	if pattern.len() <= left_split {
-		return AnyMatcher::Scalar(mk_scalv(pattern));
+		return Ok(AnyMatcher::Scalar(mk_scalv(pattern, i).await?));
 	}
 	let (left, not_left) = pattern.split_at(left_split);
 	let right_split = not_left.len() - scal_cnt(pattern.iter().rev());
 	let (mid, right) = not_left.split_at(right_split);
-	AnyMatcher::Vec { left: mk_scalv(left), mid: mk_vec(mid), right: mk_scalv(right) }
+	join_ok! {
+		left = mk_scalv(left, i).await;
+		mid = mk_vec(mid, i).await;
+		right = mk_scalv(right, i).await;
+	}
+	Ok(AnyMatcher::Vec { left, mid, right })
 }
 
 /// Pattern MUST NOT contain vectorial placeholders
-#[must_use]
+async fn mk_scalv(pattern: &[MacTree], i: &Interner) -> OrcRes<Vec<ScalMatcher>> {
-fn mk_scalv(pattern: &[MacTree]) -> Vec<ScalMatcher> { pattern.iter().map(mk_scalar).collect() }
+	join_all(pattern.iter().map(|pat| mk_scalar(pat, i))).await.into_iter().collect()
+}
 
 /// Pattern MUST start and end with a vectorial placeholder
-#[must_use]
+pub async fn mk_vec(pattern: &[MacTree], i: &Interner) -> OrcRes<VecMatcher> {
-pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
 	debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
 	debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
 	debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
@@ -57,39 +65,56 @@ pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
 	let (l_side, l_sep) = left.split_at(left.len() - l_sep_size);
 	let main = VecMatcher::Placeh { key: key.clone(), nonzero };
 	match (left, right) {
-		(&[], &[]) => VecMatcher::Placeh { key, nonzero },
+		(&[], &[]) => Ok(VecMatcher::Placeh { key, nonzero }),
-		(&[], _) => VecMatcher::Scan {
+		(&[], _) => {
-			direction: Side::Left,
+			join_ok! {
-			left: Box::new(main),
+				sep = mk_scalv(r_sep, i).await;
-			sep: mk_scalv(r_sep),
+				right = mk_vec(r_side, i).boxed_local().await;
-			right: Box::new(mk_vec(r_side)),
+			}
+			Ok(VecMatcher::Scan {
+				direction: Side::Left,
+				left: Box::new(main),
+				sep,
+				right: Box::new(right),
+			})
 		},
-		(_, &[]) => VecMatcher::Scan {
+		(_, &[]) => {
-			direction: Side::Right,
+			join_ok! {
-			left: Box::new(mk_vec(l_side)),
+				left = mk_vec(l_side, i).boxed_local().await;
-			sep: mk_scalv(l_sep),
+				sep = mk_scalv(l_sep, i).await;
-			right: Box::new(main),
+			}
+			Ok(VecMatcher::Scan {
+				direction: Side::Right,
+				left: Box::new(left),
+				sep,
+				right: Box::new(main),
+			})
 		},
 		(..) => {
 			let mut key_order =
 				l_side.iter().chain(r_side.iter()).filter_map(vec_attrs).collect::<Vec<_>>();
 			key_order.sort_by_key(|(_, prio, _)| -(*prio as i64));
-			VecMatcher::Middle {
+			join_ok! {
-				left: Box::new(mk_vec(l_side)),
+				left = mk_vec(l_side, i).boxed_local().await;
-				left_sep: mk_scalv(l_sep),
+				left_sep = mk_scalv(l_sep, i).await;
-				mid: Box::new(main),
+				right_sep = mk_scalv(r_sep, i).await;
-				right_sep: mk_scalv(r_sep),
+				right = mk_vec(r_side, i).boxed_local().await;
-				right: Box::new(mk_vec(r_side)),
-				key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
 			}
+			Ok(VecMatcher::Middle {
+				left: Box::new(left),
+				left_sep,
+				mid: Box::new(main),
+				right_sep,
+				right: Box::new(right),
+				key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
+			})
 		},
 	}
 }
 
 /// Pattern MUST NOT be a vectorial placeholder
-#[must_use]
+async fn mk_scalar(pattern: &MacTree, i: &Interner) -> OrcRes<ScalMatcher> {
-fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
+	Ok(match &*pattern.tok {
-	match &*pattern.tok {
 		MacTok::Name(n) => ScalMatcher::Name(n.clone()),
 		MacTok::Ph(Ph { name, kind }) => match kind {
 			PhKind::Vector { .. } => {
@@ -97,16 +122,20 @@ fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
 			},
 			PhKind::Scalar => ScalMatcher::Placeh { key: name.clone() },
 		},
-		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
+		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body, i).boxed_local().await?)),
-		MacTok::Lambda(arg, body) => ScalMatcher::Lambda(Box::new(mk_any(arg)), Box::new(mk_any(body))),
+		MacTok::Lambda(..) =>
+			return Err(mk_err(
+				i.i("Lambda in matcher").await,
+				"Lambdas can't be matched for, only generated in templates",
+				[pattern.pos()],
+			)),
 		MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
-	}
+		MacTok::Bottom(errv) => return Err(errv.clone()),
+	})
 }
 
 #[cfg(test)]
 mod test {
-	use std::rc::Rc;
-
 	use orchid_base::interner::Interner;
 	use orchid_base::location::SrcRange;
 	use orchid_base::sym;
@@ -114,15 +143,14 @@ mod test {
 	use test_executors::spin_on;
 
 	use super::mk_any;
+	use crate::macros::MacTok;
 	use crate::macros::mactree::{Ph, PhKind};
-	use crate::macros::{MacTok, MacTree};
 
 	#[test]
 	fn test_scan() {
 		spin_on(async {
 			let i = Interner::new_master();
-			let ex =
+			let ex = |tok: MacTok| async { tok.at(SrcRange::mock(&i).await.pos()) };
-				|tok: MacTok| async { MacTree { tok: Rc::new(tok), pos: SrcRange::mock(&i).await.pos() } };
 			let pattern = vec![
 				ex(MacTok::Ph(Ph {
 					kind: PhKind::Vector { priority: 0, at_least_one: false },
@@ -150,7 +178,7 @@ mod test {
 				}))
 				.await,
 			];
-			let matcher = mk_any(&pattern);
+			let matcher = mk_any(&pattern, &i).await.expect("This matcher isn't broken");
 			println!("{matcher}");
 		})
 	}

orchid-std/src/macros/rule/matcher.rs

@@ -1,8 +1,8 @@
 use std::fmt;
-use std::rc::Rc;
 
 use itertools::Itertools;
-use orchid_base::interner::Interner;
+use orchid_base::error::OrcRes;
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 
@@ -15,46 +15,47 @@ use super::vec_match::vec_match;
 use crate::macros::mactree::{Ph, PhKind};
 use crate::macros::{MacTok, MacTree};
 
-pub fn first_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.first().unwrap()).is_some() }
+pub struct NamedMatcher {
-pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwrap()).is_some() }
+	inner: AnyMatcher,
-
+	head: Sym,
-pub struct NamedMatcher(AnyMatcher);
+	after_tok: IStr,
+}
 impl NamedMatcher {
-	pub async fn new(pattern: &[MacTree], i: &Interner) -> Self {
+	pub async fn new(pattern: &[MacTree], i: &Interner) -> OrcRes<Self> {
-		assert!(
+		let head = match pattern.first().map(|tree| tree.tok()) {
-			matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))),
+			Some(MacTok::Name(name)) => name.clone(),
-			"Named matchers must begin with a name"
+			_ => panic!("Named matchers must begin with a name"),
-		);
+		};
-
+		let after_tok = i.i("::after").await;
-		match last_is_vec(pattern) {
+		let inner = match pattern.last().and_then(vec_attrs).is_some() {
-			true => Self(mk_any(pattern)),
+			true => mk_any(pattern, i).await?,
 			false => {
 				let kind = PhKind::Vector { priority: 0, at_least_one: false };
-				let tok = MacTok::Ph(Ph { name: i.i("::after").await, kind });
+				let suffix = [MacTok::Ph(Ph { name: after_tok.clone(), kind }).at(Pos::None)];
-				let suffix = [MacTree { pos: Pos::None, tok: Rc::new(tok) }];
+				mk_any(&pattern.iter().cloned().chain(suffix).collect_vec(), i).await?
-				Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
 			},
-		}
+		};
+		Ok(Self { after_tok, inner, head })
 	}
+	pub fn head(&self) -> Sym { self.head.clone() }
 	/// Also returns the tail, if any, which should be matched further
 	/// Note that due to how priod works below, the main usable information from
 	/// the tail is its length
-	pub async fn apply<'a>(
+	pub fn apply<'a>(
 		&self,
 		seq: &'a [MacTree],
-		i: &Interner,
 		save_loc: impl Fn(Sym) -> bool,
 	) -> Option<(MatchState<'a>, &'a [MacTree])> {
-		let mut state = any_match(&self.0, seq, &save_loc)?;
+		let mut state = any_match(&self.inner, seq, &save_loc)?;
-		match state.remove(i.i("::after").await) {
+		match state.remove(self.after_tok.clone()) {
-			Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"),
+			Some(StateEntry::Scalar(_)) => panic!("{} can never be a scalar entry!", self.after_tok),
 			Some(StateEntry::Vec(v)) => Some((state, v)),
 			None => Some((state, &[][..])),
 		}
 	}
 }
 impl fmt::Display for NamedMatcher {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.inner.fmt(f) }
 }
 impl fmt::Debug for NamedMatcher {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "NamedMatcher({self})") }
@@ -62,12 +63,12 @@ impl fmt::Debug for NamedMatcher {
 
 pub struct PriodMatcher(VecMatcher);
 impl PriodMatcher {
-	pub fn new(pattern: &[MacTree]) -> Self {
+	pub async fn new(pattern: &[MacTree], i: &Interner) -> OrcRes<Self> {
 		assert!(
 			pattern.first().and_then(vec_attrs).is_some() && pattern.last().and_then(vec_attrs).is_some(),
 			"Prioritized matchers must start and end with a vectorial",
 		);
-		Self(mk_vec(pattern))
+		Ok(Self(mk_vec(pattern, i).await?))
 	}
 	/// tokens before the offset always match the prefix
 	pub fn apply<'a>(