Correctly halts

clean shutdown doesn't for some reason

.cargo/config.toml

@@ -1,9 +1,14 @@
 [alias]
 xtask = "run --quiet --package xtask --"
 orcx = "xtask orcx"
+orcxdb = "xtask orcxdb"
 
 [env]
 CARGO_WORKSPACE_DIR = { value = "", relative = true }
 ORCHID_EXTENSIONS = "target/debug/orchid-std"
-ORCHID_DEFAULT_SYSTEMS = "orchid::std"
+ORCHID_DEFAULT_SYSTEMS = "orchid::std;orchid::macros"
 ORCHID_LOG_BUFFERS = "true"
+RUST_BACKTRACE = "1"
+
+[build]
+# rustflags = ["-Znext-solver"]

Cargo.toml

@@ -10,5 +10,6 @@ members = [
   "orchid-api",
   "orchid-api-derive",
   "orchid-api-traits",
-  "stdio-perftest", "xtask",
+  "stdio-perftest",
+  "xtask", "async-fn-stream",
 ]

README.md

@@ -7,7 +7,7 @@ An experimental lazy, pure functional programming language designed to be embedd
 
 ## Usage
 
-The standalone interpreter can be built as the binary target from this package. The language tutorial and standard library documentation is at [www.lbfalvy.com/orchid-reference](https://www.lbfalvy.com/orchid-reference/). Embedder guide and Rust API documentation are coming soon.
+The standalone interpreter can be built as the binary target from this package. The language tutorial and standard library documentation is at [www.lbfalvy.com/orchid-reference](https://lbfalvy.github.io/orchid-reference/). Embedder guide and Rust API documentation are coming soon.
 
 ## Design
 
@@ -43,4 +43,4 @@ Identifying marks include the Orchid logo, the ribbon image above, and the names
 
 Contact information includes email addresses, links to the source code and issue tracker.
 
-Words listed as identifying marks are explicltly not considered as such when they apear in technical interfaces or APIs. For example, shell commands, identifiers within the language,
+Words listed as identifying marks are explicltly not considered as such when they appear in technical interfaces or APIs. For example, shell commands, identifiers within Orchid or Rust code, and names in package registries are not considered as identifying marks.

SWAP.md

@@ -1,16 +1,12 @@
-## Async conversion
+Decide whether we need patterns at runtime. Maybe macros aren't obligated to return MacTree so destructuring can be done in a safer and easier way?
 
-convert host to async non-send
+Double-check type and templating logic in the note, it's a bit fishy.
 
-demonstrate operation with existing lex-hello example
+Consider whether all macros need to be loaded or the const references could be used to pre-filter for a given let line.
-
-consider converting extension's SysCtx to a typed context bag
-
-align fn atom and macros on both sides with new design. No global state.
 
 ## alternate extension mechanism
 
-The Macro extension needs to be in the same compilation unit as the interpreter because the interpreter needs to proactively access its datastructures (in particular, it needs to generate MacTree from TokTree)
+The STD system will have a lot of traffic for trivial operations like algebra, stream IO will likely not be fast enough. A faster system is in order.
 
 Ideally, it should reuse `orchid-extension` for message routing and decoding.
 
@@ -18,10 +14,8 @@ Ideally, it should reuse `orchid-extension` for message routing and decoding.
 
 ## Preprocessor extension
 
-Must figure out how preprocessor can both be a System and referenced in the interpreter
+The macro system will not be privileged, it can take control from the interpreter via a custom top-level "let" line type.
 
 Must actually write macro system as recorded in note
 
-At this point swappable preprocessors aren't a target because interaction with module system sounds complicated
-
 Check if any of this needs interpreter, if so, start with that

async-fn-stream/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "async-fn-stream"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+futures = { version = "0.3.31", features = ["std"], default-features = false }
+
+[dev-dependencies]
+test_executors = "0.3.5"

async-fn-stream/src/lib.rs

@@ -0,0 +1,207 @@
+use std::cell::Cell;
+use std::future::poll_fn;
+use std::marker::PhantomData;
+use std::pin::Pin;
+use std::ptr;
+use std::task::{Context, Poll};
+
+use futures::future::LocalBoxFuture;
+use futures::{FutureExt, Stream};
+
+type YieldSlot<'a, T> = &'a Cell<Option<T>>;
+
+/// Handle that allows you to emit values on a stream. If you drop
+/// this, the stream will end and you will not be polled again.
+pub struct StreamCtx<'a, T>(&'a Cell<Option<T>>, PhantomData<&'a ()>);
+impl<T> StreamCtx<'_, T> {
+	pub fn emit(&mut self, value: T) -> impl Future<Output = ()> {
+		assert!(self.0.replace(Some(value)).is_none(), "Leftover value in stream");
+		let mut state = Poll::Pending;
+		poll_fn(move |_| std::mem::replace(&mut state, Poll::Ready(())))
+	}
+}
+
+enum FnOrFut<'a, T, O> {
+	Fn(Option<Box<dyn FnOnce(YieldSlot<'a, T>) -> LocalBoxFuture<'a, O> + 'a>>),
+	Fut(LocalBoxFuture<'a, O>),
+}
+
+struct AsyncFnStream<'a, T> {
+	driver: FnOrFut<'a, T, ()>,
+	output: Cell<Option<T>>,
+}
+impl<'a, T> Stream for AsyncFnStream<'a, T> {
+	type Item = T;
+	fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		unsafe {
+			let self_mut = self.get_unchecked_mut();
+			let fut = match &mut self_mut.driver {
+				FnOrFut::Fut(fut) => fut,
+				FnOrFut::Fn(f) => {
+					// safety: the cell is held inline in self, which is pinned.
+					let cell = ptr::from_ref(&self_mut.output).as_ref().unwrap();
+					let fut = f.take().unwrap()(cell);
+					self_mut.driver = FnOrFut::Fut(fut);
+					return Pin::new_unchecked(self_mut).poll_next(cx);
+				},
+			};
+			match fut.as_mut().poll(cx) {
+				Poll::Ready(()) => Poll::Ready(None),
+				Poll::Pending => match self_mut.output.replace(None) {
+					None => Poll::Pending,
+					Some(t) => Poll::Ready(Some(t)),
+				},
+			}
+		}
+	}
+}
+
+struct AsyncFnTryStream<'a, T, E> {
+	driver: FnOrFut<'a, T, Result<StreamCtx<'a, T>, E>>,
+	output: Cell<Option<T>>,
+}
+impl<'a, T, E> Stream for AsyncFnTryStream<'a, T, E> {
+	type Item = Result<T, E>;
+	fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		unsafe {
+			let self_mut = self.get_unchecked_mut();
+			let fut = match &mut self_mut.driver {
+				FnOrFut::Fut(fut) => fut,
+				FnOrFut::Fn(f) => {
+					// safety: the cell is held inline in self, which is pinned.
+					let cell = ptr::from_ref(&self_mut.output).as_ref().unwrap();
+					let fut = f.take().unwrap()(cell);
+					self_mut.driver = FnOrFut::Fut(fut);
+					return Pin::new_unchecked(self_mut).poll_next(cx);
+				},
+			};
+			match fut.as_mut().poll(cx) {
+				Poll::Ready(Ok(_)) => Poll::Ready(None),
+				Poll::Ready(Err(ex)) => Poll::Ready(Some(Err(ex))),
+				Poll::Pending => match self_mut.output.replace(None) {
+					None => Poll::Pending,
+					Some(t) => Poll::Ready(Some(Ok(t))),
+				},
+			}
+		}
+	}
+}
+
+/// Create a stream from an async function acting as a coroutine
+pub fn stream<'a, T: 'a>(
+	f: impl for<'b> AsyncFnOnce(StreamCtx<'b, T>) + 'a,
+) -> impl Stream<Item = T> + 'a {
+	AsyncFnStream {
+		output: Cell::new(None),
+		driver: FnOrFut::Fn(Some(Box::new(|t| {
+			async { f(StreamCtx(t, PhantomData)).await }.boxed_local()
+		}))),
+	}
+}
+
+/// Create a stream of result from a fallible function.
+pub fn try_stream<'a, T: 'a, E: 'a>(
+	f: impl for<'b> AsyncFnOnce(StreamCtx<'b, T>) -> Result<StreamCtx<'b, T>, E> + 'a,
+) -> impl Stream<Item = Result<T, E>> + 'a {
+	AsyncFnTryStream {
+		output: Cell::new(None),
+		driver: FnOrFut::Fn(Some(Box::new(|t| {
+			async { f(StreamCtx(t, PhantomData)).await }.boxed_local()
+		}))),
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use std::task::Poll;
+	use std::{future, pin};
+
+	use futures::channel::mpsc::channel;
+	use futures::{Stream, StreamExt, TryStreamExt};
+	use test_executors::spin_on;
+
+	use crate::{stream, try_stream};
+
+	#[test]
+	fn sync() {
+		spin_on(async {
+			let v = stream(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+			})
+			.collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, [0, 1, 2, 3, 4])
+		})
+	}
+
+	#[test]
+	/// The exact behaviour of the poll function under blocked use
+	fn with_delay() {
+		spin_on(async {
+			let (mut send, mut recv) = channel(0);
+			let mut s = pin::pin!(stream(async |mut cx| {
+				for i in 0..2 {
+					cx.emit(i).await
+				}
+				recv.next().await;
+				for i in 2..5 {
+					cx.emit(i).await
+				}
+			}));
+			let mut log = String::new();
+			let log = future::poll_fn(|cx| {
+				match s.as_mut().poll_next(cx) {
+					Poll::Ready(Some(r)) => log += &format!("Found {r}\n"),
+					Poll::Ready(None) => return Poll::Ready(format!("{log}Ended")),
+					Poll::Pending => match send.try_send(()) {
+						Ok(()) => log += "Unblocked\n",
+						Err(err) => return Poll::Ready(format!("{log}Unblock err: {err}")),
+					},
+				}
+				Poll::Pending
+			})
+			.await;
+			const EXPECTED: &str = "\
+        Found 0\n\
+        Found 1\n\
+        Unblocked\n\
+        Found 2\n\
+        Found 3\n\
+        Found 4\n\
+        Ended";
+			assert_eq!(log, EXPECTED)
+		})
+	}
+
+	#[test]
+	fn sync_try_all_ok() {
+		spin_on(async {
+			let v = try_stream::<_, ()>(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+				Ok(cx)
+			})
+			.try_collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, Ok(vec![0, 1, 2, 3, 4]))
+		})
+	}
+
+	#[test]
+	fn sync_try_err() {
+		spin_on(async {
+			let v = try_stream::<_, ()>(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+				Err(())
+			})
+			.try_collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, Err(()))
+		})
+	}
+}

examples/hello-world/main.orc

@@ -1,2 +1,2 @@
-const user = "dave"
+let user = "dave"
-const main = println "Hello $user!" exit_status::success
+let main = println "Hello $user!" exit_status::success

notes/new_macro_model.md

@@ -42,9 +42,7 @@ Prioritised macro patterns must start and end with a vectorial placeholder. They
 Macros are checked from the outermost block inwards.
 
 1. For every name token, test all named macros starting with that name
-   1. Take the first rule that matches in each block
+	 1. If the tail is implicit, continue iterating
-   2. If there are multiple matches across blocks, raise an ambiguity error
-   3. If the tail is implicit, recurse on it
 2. Test all prioritized macros
 	 1. Take the first rule that matches in the highest prioritized block
 
@@ -78,7 +76,8 @@ Recursion has to happen through the interpreter itself, so the macro system is d
 - atom `MacRecurState` holds the recursion state
 - function `resolve_recur` finds all matches on a MacTree
 	- type: `MacRecurState -> MacTree -> MacTree`
-  - use all macros to find all matches in the tree
+	- use all relevant macros to find all matches in the tree
+		- since macros must contain a locally defined token, it can be assumed that at the point that a constant is evaluated and all imports in the parent module have been resolved, necessarily all relevant macro rules must have been loaded
 	- for each match
 		- check for recursion violations
 		- wrap the body in iife-s corresponding to the named values in the match state

orchid-api-derive/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "orchid-api-derive"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 [lib]
 proc-macro = true
@@ -9,9 +9,8 @@ proc-macro = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-quote = "1.0.38"
+quote = "1.0.40"
-syn = { version = "2.0.95" }
+syn = { version = "2.0.106" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-proc-macro2 = "1.0.92"
+proc-macro2 = "1.0.101"
-darling = "0.20.10"
 itertools = "0.14.0"

orchid-api-derive/src/decode.rs

@@ -12,7 +12,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
 	let decode = decode_body(&input.data);
 	let expanded = quote! {
 		impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause {
-			async fn decode<R: orchid_api_traits::async_std::io::Read + ?Sized>(
+			async fn decode<R: orchid_api_traits::AsyncRead + ?Sized>(
 				mut read: std::pin::Pin<&mut R>
 			) -> Self {
 				#decode

orchid-api-derive/src/encode.rs

@@ -14,7 +14,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
 	let encode = encode_body(&input.data);
 	let expanded = quote! {
 		impl #e_impl_generics orchid_api_traits::Encode for #name #e_ty_generics #e_where_clause {
-			async fn encode<W: orchid_api_traits::async_std::io::Write + ?Sized>(
+			async fn encode<W: orchid_api_traits::AsyncWrite + ?Sized>(
 				&self,
 				mut write: std::pin::Pin<&mut W>
 			) {

orchid-api-traits/Cargo.toml

@@ -1,14 +1,13 @@
 [package]
 name = "orchid-api-traits"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-std = "1.13.0"
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
-async-stream = "0.3.6"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-futures = "0.3.31"
 itertools = "0.14.0"
 never = "0.1.0"
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"

orchid-api-traits/src/coding.rs

@@ -7,10 +7,8 @@ use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use async_fn_stream::stream;
-use async_stream::stream;
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, StreamExt};
-use futures::future::LocalBoxFuture;
-use futures::{FutureExt, StreamExt};
 use never::Never;
 use ordered_float::NotNan;
 
@@ -19,20 +17,17 @@ use crate::encode_enum;
 pub trait Decode: 'static {
 	/// Decode an instance from the beginning of the buffer. Return the decoded
 	/// data and the remaining buffer.
-	fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
+	fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
 }
 pub trait Encode {
 	/// Append an instance of the struct to the buffer
-	fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
+	fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
 }
 pub trait Coding: Encode + Decode + Clone {
 	fn get_decoder<T: 'static, F: Future<Output = T> + 'static>(
 		map: impl Fn(Self) -> F + Clone + 'static,
-	) -> impl for<'a> Fn(Pin<&'a mut dyn Read>) -> LocalBoxFuture<'a, T> {
+	) -> impl AsyncFn(Pin<&mut dyn AsyncRead>) -> T {
-		move |r| {
+		async move |r| map(Self::decode(r).await).await
-			let map = map.clone();
-			async move { map(Self::decode(r).await).await }.boxed_local()
-		}
 	}
 }
 impl<T: Encode + Decode + Clone> Coding for T {}
@@ -40,14 +35,14 @@ impl<T: Encode + Decode + Clone> Coding for T {}
 macro_rules! num_impl {
 	($number:ty) => {
 		impl Decode for $number {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; (<$number>::BITS / 8) as usize];
 				read.read_exact(&mut bytes).await.unwrap();
 				<$number>::from_be_bytes(bytes)
 			}
 		}
 		impl Encode for $number {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 				write.write_all(&self.to_be_bytes()).await.expect("Could not write number")
 			}
 		}
@@ -67,12 +62,12 @@ num_impl!(i8);
 macro_rules! nonzero_impl {
 	($name:ty) => {
 		impl Decode for NonZero<$name> {
-			async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 				Self::new(<$name as Decode>::decode(read).await).unwrap()
 			}
 		}
 		impl Encode for NonZero<$name> {
-			async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 				self.get().encode(write).await
 			}
 		}
@@ -91,19 +86,21 @@ nonzero_impl!(i64);
 nonzero_impl!(i128);
 
 impl<T: Encode + ?Sized> Encode for &T {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) { (**self).encode(write).await }
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+		(**self).encode(write).await
+	}
 }
 macro_rules! float_impl {
 	($t:ty, $size:expr) => {
 		impl Decode for NotNan<$t> {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; $size];
 				read.read_exact(&mut bytes).await.unwrap();
 				NotNan::new(<$t>::from_be_bytes(bytes)).expect("Float was NaN")
 			}
 		}
 		impl Encode for NotNan<$t> {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 				write.write_all(&self.as_ref().to_be_bytes()).await.expect("Could not write number")
 			}
 		}
@@ -114,7 +111,7 @@ float_impl!(f64, 8);
 float_impl!(f32, 4);
 
 impl Decode for String {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
 		let mut data = vec![0u8; len];
 		read.read_exact(&mut data).await.unwrap();
@@ -122,30 +119,36 @@ impl Decode for String {
 	}
 }
 impl Encode for String {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl Encode for str {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl<T: Decode> Decode for Vec<T> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		stream! { loop { yield T::decode(read.as_mut()).await } }.take(len).collect().await
+		stream(async |mut cx| {
+			for _ in 0..len {
+				cx.emit(T::decode(read.as_mut()).await).await
+			}
+		})
+		.collect()
+		.await
 	}
 }
 impl<T: Encode> Encode for Vec<T> {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		self.as_slice().encode(write).await
 	}
 }
 impl<T: Encode> Encode for [T] {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		for t in self.iter() {
 			t.encode(write.as_mut()).await
@@ -153,7 +156,7 @@ impl<T: Encode> Encode for [T] {
 	}
 }
 impl<T: Decode> Decode for Option<T> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		match u8::decode(read.as_mut()).await {
 			0 => None,
 			1 => Some(T::decode(read).await),
@@ -162,14 +165,14 @@ impl<T: Decode> Decode for Option<T> {
 	}
 }
 impl<T: Encode> Encode for Option<T> {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		let t = if let Some(t) = self { t } else { return 0u8.encode(write.as_mut()).await };
 		1u8.encode(write.as_mut()).await;
 		t.encode(write).await;
 	}
 }
 impl<T: Decode, E: Decode> Decode for Result<T, E> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		match u8::decode(read.as_mut()).await {
 			0 => Self::Ok(T::decode(read).await),
 			1 => Self::Err(E::decode(read).await),
@@ -179,7 +182,7 @@ impl<T: Decode, E: Decode> Decode for Result<T, E> {
 }
 
 impl<T: Encode, E: Encode> Encode for Result<T, E> {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		match self {
 			Ok(t) => encode_enum(write, 0, |w| t.encode(w)).await,
 			Err(e) => encode_enum(write, 1, |w| e.encode(w)).await,
@@ -187,13 +190,19 @@ impl<T: Encode, E: Encode> Encode for Result<T, E> {
 	}
 }
 impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		stream! { loop { yield <(K, V)>::decode(read.as_mut()).await } }.take(len).collect().await
+		stream(async |mut cx| {
+			for _ in 0..len {
+				cx.emit(<(K, V)>::decode(read.as_mut()).await).await
+			}
+		})
+		.collect()
+		.await
 	}
 }
 impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		for pair in self.iter() {
 			pair.encode(write.as_mut()).await
@@ -203,12 +212,12 @@ impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
 macro_rules! tuple {
   (($($t:ident)*) ($($T:ident)*)) => {
     impl<$($T: Decode),*> Decode for ($($T,)*) {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				($($T::decode(read.as_mut()).await,)*)
 			}
     }
     impl<$($T: Encode),*> Encode for ($($T,)*) {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
         let ($($t,)*) = self;
         $( $t.encode(write.as_mut()).await; )*
       }
@@ -234,41 +243,45 @@ tuple!((t u v x y z a b c d e f g h i) (T U V X Y Z A B C D E F G H I));
 tuple!((t u v x y z a b c d e f g h i j) (T U V X Y Z A B C D E F G H I J)); // 16
 
 impl Decode for () {
-	async fn decode<R: Read + ?Sized>(_: Pin<&mut R>) -> Self {}
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {}
 }
 impl Encode for () {
-	async fn encode<W: Write + ?Sized>(&self, _: Pin<&mut W>) {}
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) {}
 }
 impl Decode for Never {
-	async fn decode<R: Read + ?Sized>(_: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {
 		unreachable!("A value of Never cannot exist so it can't have been serialized");
 	}
 }
 impl Encode for Never {
-	async fn encode<W: Write + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
 }
 impl Decode for bool {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let mut buf = [0];
 		read.read_exact(&mut buf).await.unwrap();
 		buf[0] != 0
 	}
 }
 impl Encode for bool {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await.unwrap()
 	}
 }
 impl<T: Decode, const N: usize> Decode for [T; N] {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		// TODO: figure out how to do this in safe rust on the stack
+		let v = stream(async |mut cx| {
-		let v =
+			for _ in 0..N {
-			stream! { loop { yield T::decode(read.as_mut()).await } }.take(N).collect::<Vec<_>>().await;
+				cx.emit(T::decode(read.as_mut()).await).await
+			}
+		})
+		.collect::<Vec<_>>()
+		.await;
 		v.try_into().unwrap_or_else(|_| unreachable!("The length of this stream is statically known"))
 	}
 }
 impl<T: Encode, const N: usize> Encode for [T; N] {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		for t in self.iter() {
 			t.encode(write.as_mut()).await
 		}
@@ -278,12 +291,12 @@ impl<T: Encode, const N: usize> Encode for [T; N] {
 macro_rules! two_end_range {
   ($this:ident, $name:tt, $op:tt, $start:expr, $end:expr) => {
     impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				T::decode(read.as_mut()).await $op T::decode(read).await
 			}
     }
     impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
         let $this = self;
         ($start).encode(write.as_mut()).await;
         ($end).encode(write).await;
@@ -298,12 +311,14 @@ two_end_range!(x, RangeInclusive, ..=, x.start(), x.end());
 macro_rules! smart_ptr {
 	($name:tt) => {
 		impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 				$name::new(T::decode(read).await)
 			}
 		}
 		impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) { (**self).encode(write).await }
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+				(**self).encode(write).await
+			}
 		}
 	};
 }
@@ -313,12 +328,12 @@ smart_ptr!(Rc);
 smart_ptr!(Box);
 
 impl Decode for char {
-	async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 		char::from_u32(u32::decode(read).await).unwrap()
 	}
 }
 impl Encode for char {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		(*self as u32).encode(write).await
 	}
 }

orchid-api-traits/src/helpers.rs

@@ -1,12 +1,12 @@
 use std::future::Future;
 use std::pin::Pin;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use itertools::{Chunk, Itertools};
 
 use crate::Encode;
 
-pub async fn encode_enum<'a, W: Write + ?Sized, F: Future<Output = ()>>(
+pub async fn encode_enum<'a, W: AsyncWrite + ?Sized, F: Future<Output = ()>>(
 	mut write: Pin<&'a mut W>,
 	id: u8,
 	f: impl FnOnce(Pin<&'a mut W>) -> F,
@@ -15,7 +15,7 @@ pub async fn encode_enum<'a, W: Write + ?Sized, F: Future<Output = ()>>(
 	f(write).await
 }
 
-pub async fn write_exact<W: Write + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
+pub async fn write_exact<W: AsyncWrite + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
 	write.write_all(bytes).await.expect("Failed to write exact bytes")
 }
 
@@ -27,7 +27,7 @@ pub fn print_bytes(b: &[u8]) -> String {
 		.join("  ")
 }
 
-pub async fn read_exact<R: Read + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
+pub async fn read_exact<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
 	let mut data = vec![0u8; bytes.len()];
 	read.read_exact(&mut data).await.expect("Failed to read bytes");
 	if data != bytes {

orchid-api-traits/src/lib.rs

@@ -3,8 +3,8 @@ mod helpers;
 mod hierarchy;
 mod relations;
 
-pub use async_std;
 pub use coding::*;
+pub use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 pub use helpers::*;
 pub use hierarchy::*;
 pub use relations::*;

orchid-api/Cargo.toml

@@ -1,15 +1,16 @@
 [package]
 name = "orchid-api"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
-async-std = "1.13.0"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
+itertools = "0.14.0"
 
 [dev-dependencies]
 test_executors = "0.3.2"

orchid-api/src/atom.rs

@@ -1,14 +1,28 @@
+use std::fmt;
 use std::num::NonZeroU64;
 
+use itertools::Itertools;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::{
-	ExprTicket, Expression, ExtHostReq, FormattingUnit, HostExtNotif, HostExtReq, OrcResult, SysId,
+	ExprTicket, Expression, ExtHostReq, FormattingUnit, HostExtReq, OrcResult, SysId, TStrv,
-	TStrv,
 };
 
-pub type AtomData = Vec<u8>;
+#[derive(Clone, Coding)]
+pub struct AtomData(pub Vec<u8>);
+impl fmt::Debug for AtomData {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		let mut byte_strings = self.0.iter().map(|b| format!("{b:02x}"));
+		if self.0.len() < 32 {
+			write!(f, "AtomData({})", byte_strings.join(" "))
+		} else {
+			let data_table =
+				byte_strings.chunks(32).into_iter().map(|mut chunk| chunk.join(" ")).join("\n");
+			write!(f, "AtomData(\n{}\n)", data_table)
+		}
+	}
+}
 
 /// Unique ID associated with atoms that have an identity
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
@@ -16,7 +30,7 @@ pub struct AtomId(pub NonZeroU64);
 
 /// An atom owned by an implied system. Usually used in responses from a system.
 /// This has the same semantics as [Atom] except in that the owner is implied.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct LocalAtom {
 	pub drop: Option<AtomId>,
 	pub data: AtomData,
@@ -27,7 +41,7 @@ impl LocalAtom {
 
 /// An atom representation that can be serialized and sent around. Atoms
 /// represent the smallest increment of work.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct Atom {
 	/// Instance ID of the system that created the atom
 	pub owner: SysId,
@@ -49,7 +63,7 @@ pub struct Atom {
 }
 
 /// Attempt to apply an atom as a function to an expression
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct CallRef(pub Atom, pub ExprTicket);
 impl Request for CallRef {
@@ -59,14 +73,14 @@ impl Request for CallRef {
 /// Attempt to apply an atom as a function, consuming the atom and enabling the
 /// library to reuse its datastructures rather than duplicating them. This is an
 /// optimization over [CallRef] followed by [AtomDrop].
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct FinalCall(pub Atom, pub ExprTicket);
 impl Request for FinalCall {
 	type Response = Expression;
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct SerializeAtom(pub Atom);
 impl Request for SerializeAtom {
@@ -81,14 +95,14 @@ impl Request for DeserAtom {
 }
 
 /// A request blindly routed to the system that provides an atom.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct Fwded(pub Atom, pub TStrv, pub Vec<u8>);
 impl Request for Fwded {
 	type Response = Option<Vec<u8>>;
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct Fwd(pub Atom, pub TStrv, pub Vec<u8>);
 impl Request for Fwd {
@@ -100,7 +114,7 @@ pub enum NextStep {
 	Continue(Expression),
 	Halt,
 }
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct Command(pub Atom);
 impl Request for Command {
@@ -111,17 +125,20 @@ impl Request for Command {
 /// isn't referenced anywhere. This should have no effect if the atom's `drop`
 /// flag is false.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
-#[extends(HostExtNotif)]
+#[extends(HostExtReq)]
 pub struct AtomDrop(pub SysId, pub AtomId);
+impl Request for AtomDrop {
+	type Response = ();
+}
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
 pub struct AtomPrint(pub Atom);
 impl Request for AtomPrint {
 	type Response = FormattingUnit;
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct ExtAtomPrint(pub Atom);
 impl Request for ExtAtomPrint {
@@ -129,7 +146,7 @@ impl Request for ExtAtomPrint {
 }
 
 /// Requests that apply to an existing atom instance
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 #[extendable]
 pub enum AtomReq {

orchid-api/src/expr.rs

@@ -1,3 +1,4 @@
+use std::fmt;
 use std::num::NonZeroU64;
 
 use orchid_api_derive::{Coding, Hierarchy};
@@ -10,8 +11,13 @@ use crate::{Atom, ExtHostNotif, ExtHostReq, Location, OrcError, SysId, TStrv};
 /// [Acquire].
 ///
 /// The ID is globally unique within its lifetime, but may be reused.
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+#[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ExprTicket(pub NonZeroU64);
+impl fmt::Debug for ExprTicket {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "ExprTicket({:x})", self.0.get())
+	}
+}
 
 /// Acquire a strong reference to an expression. This keeps it alive until a
 /// corresponding [Release] is emitted. The number of times a system has
@@ -62,7 +68,7 @@ pub enum ExpressionKind {
 	Arg(u64),
 	/// Insert the specified host-expression in the template here. When the clause
 	/// is used in the const tree, this variant is forbidden.
-	Slot(ExprTicket),
+	Slot { tk: ExprTicket, by_value: bool },
 	/// The lhs must be fully processed before the rhs can be processed.
 	/// Equivalent to Haskell's function of the same name
 	Seq(Box<Expression>, Box<Expression>),

orchid-api/src/lexer.rs

@@ -3,7 +3,7 @@ use std::ops::RangeInclusive;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TokenTree, TreeTicket};
+use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TStrv, TokenTree};
 
 /// - All ranges contain at least one character
 /// - All ranges are in increasing characeter order
@@ -18,6 +18,8 @@ pub struct LexExpr {
 	pub id: ParsId,
 	pub text: TStr,
 	pub pos: u32,
+	/// Source root module path
+	pub src: TStrv,
 }
 impl Request for LexExpr {
 	type Response = Option<OrcResult<LexedExpr>>;
@@ -42,5 +44,5 @@ impl Request for SubLex {
 #[derive(Clone, Debug, Coding)]
 pub struct SubLexed {
 	pub pos: u32,
-	pub ticket: TreeTicket,
+	pub tree: TokenTree,
 }

orchid-api/src/lib.rs

@@ -2,8 +2,6 @@ mod lexer;
 pub use lexer::*;
 mod format;
 pub use format::*;
-mod macros;
-pub use macros::*;
 mod atom;
 pub use atom::*;
 mod error;
@@ -24,5 +22,3 @@ mod system;
 pub use system::*;
 mod tree;
 pub use tree::*;
-mod vfs;
-pub use vfs::*;

orchid-api/src/location.rs

@@ -17,8 +17,6 @@ pub enum Location {
 	Gen(CodeGenInfo),
 	/// Range and file
 	SourceRange(SourceRange),
-	/// Range only, file implied. Most notably used by parsers
-	Range(Range<u32>),
 }
 
 #[derive(Clone, Debug, Coding)]

orchid-api/src/macros.rs

@@ -1,86 +0,0 @@
-use std::collections::HashMap;
-use std::num::NonZeroU64;
-
-use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::Request;
-use ordered_float::NotNan;
-
-use crate::{
-	Atom, Comment, ExtHostReq, HostExtReq, Location, OrcResult, Paren, ParsId, SysId, TStr, TStrv,
-};
-
-#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct MacroTreeId(pub NonZeroU64);
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroTree {
-	pub location: Location,
-	pub token: MacroToken,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub enum MacroToken {
-	S(Paren, Vec<MacroTree>),
-	Name(TStrv),
-	Slot(MacroTreeId),
-	Lambda(Vec<MacroTree>, Vec<MacroTree>),
-	Ph(Placeholder),
-	Atom(Atom),
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroBlock {
-	pub priority: Option<NotNan<f64>>,
-	pub rules: Vec<MacroRule>,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroRule {
-	pub location: Location,
-	pub comments: Vec<Comment>,
-	pub pattern: Vec<MacroTree>,
-	pub id: MacroId,
-}
-
-/// A specific macro rule with a specific pattern across invocations
-#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, Hash)]
-pub struct MacroId(pub NonZeroU64);
-
-/// After a pattern matches, this call executes the body of the macro. This
-/// request returns None if an inner nested request raised an exception
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtReq)]
-pub struct ApplyMacro {
-	pub sys: SysId,
-	pub id: MacroId,
-	/// Recursion token
-	pub run_id: ParsId,
-	/// Must contain exactly the keys that were specified as placeholders in the
-	/// pattern
-	pub params: HashMap<TStr, Vec<MacroTree>>,
-}
-impl Request for ApplyMacro {
-	type Response = Option<OrcResult<Vec<MacroTree>>>;
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostReq)]
-pub struct RunMacros {
-	pub run_id: ParsId,
-	pub query: Vec<MacroTree>,
-}
-impl Request for RunMacros {
-	type Response = Option<Vec<MacroTree>>;
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct Placeholder {
-	pub name: TStr,
-	pub kind: PhKind,
-}
-
-#[derive(Clone, Copy, Debug, Coding)]
-pub enum PhKind {
-	Scalar,
-	Vector { priority: u8, at_least_one: bool },
-}

orchid-api/src/parser.rs

@@ -1,28 +1,102 @@
+use core::ops::Range;
 use std::num::NonZeroU64;
 
-use orchid_api_derive::{Coding, Decode, Encode, Hierarchy};
+use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{Comment, HostExtReq, OrcResult, SysId, TokenTree};
+use crate::{
+	Expression, ExtHostReq, HostExtReq, OrcResult, SourceRange, SysId, TStr, TStrv, TokenTree,
+};
 
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ParsId(pub NonZeroU64);
 
-// impl orchid_api_traits::Decode for ParsId {
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-// 	async fn decode<R: async_std::io::Read + ?Sized>(mut read:
+pub struct ParsedConstId(pub NonZeroU64);
-// std::pin::Pin<&mut R>) -> Self {
-// 		Self(orchid_api_traits::Decode::decode(read.as_mut()).await)
-// 	}
-// }
 
+/// Parse a single source line. Return values can be modules, constants, or
+/// token sequences for re-parsing. These re-parsed token sequences can also
+/// represent raw language items such as modules, imports, and const. This is
+/// how we enable generating imports without forcing import syntax to affect API
+/// versioning
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct ParseLine {
 	pub sys: SysId,
+	/// The immediately enclosing module path
+	pub module: TStrv,
+	/// The root module path for the snipppet of source code, prefix of
+	/// [ParseLine#module]
+	pub src: TStrv,
 	pub comments: Vec<Comment>,
 	pub exported: bool,
+	pub idx: u16,
 	pub line: Vec<TokenTree>,
 }
 impl Request for ParseLine {
-	type Response = OrcResult<Vec<TokenTree>>;
+	type Response = OrcResult<Vec<ParsedLine>>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct ParsedLine {
+	pub comments: Vec<Comment>,
+	pub source_range: SourceRange,
+	pub kind: ParsedLineKind,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum ParsedLineKind {
+	Recursive(Vec<TokenTree>),
+	Member(ParsedMember),
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct ParsedMember {
+	pub name: TStr,
+	pub exported: bool,
+	pub kind: ParsedMemberKind,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum ParsedMemberKind {
+	Constant(ParsedConstId),
+	Module { lines: Vec<ParsedLine>, use_prelude: bool },
+}
+
+/// Obtain the value of a parsed constant. This is guaranteed to be called after
+/// the last [ParseLine] but before any [crate::AtomReq]. As such, in principle
+/// the macro engine could run here.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct FetchParsedConst(pub SysId, pub ParsedConstId);
+impl Request for FetchParsedConst {
+	type Response = Expression;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Comment {
+	pub text: TStr,
+	pub range: Range<u32>,
+}
+
+/// Resolve relative names from the perspective of a constant. This can only be
+/// called during a [FetchParsedConst] call, but it can be called for a
+/// different [ParsedConstId] from the one in [FetchParsedConst].
+///
+/// Each name is either resolved to a valid name or a potential error error.
+/// The error is not raised by the interpreter itself, as names may have a
+/// primary meaning such as a local binding which can be overridden by specific
+/// true names such as those triggering macro keywords. It is not recommended to
+/// define syntax that can break by defining arbitrary constants, as line
+/// parsers can define new ones at will.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct ResolveNames {
+	pub sys: SysId,
+	pub constid: ParsedConstId,
+	pub names: Vec<TStrv>,
+}
+
+impl Request for ResolveNames {
+	type Response = Vec<OrcResult<TStrv>>;
 }

orchid-api/src/proto.rs

@@ -24,11 +24,11 @@
 
 use std::pin::Pin;
 
-use async_std::io::{Read, Write};
+use futures::{AsyncRead, AsyncWrite};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact};
 
-use crate::{atom, expr, interner, lexer, logging, macros, parser, system, tree, vfs};
+use crate::{atom, expr, interner, lexer, logging, parser, system, tree};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader {
@@ -36,7 +36,7 @@ pub struct HostHeader {
 	pub msg_logs: logging::LogStrategy,
 }
 impl Decode for HostHeader {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		read_exact(read.as_mut(), HOST_INTRO).await;
 		Self {
 			log_strategy: logging::LogStrategy::decode(read.as_mut()).await,
@@ -45,7 +45,7 @@ impl Decode for HostHeader {
 	}
 }
 impl Encode for HostHeader {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write_exact(write.as_mut(), HOST_INTRO).await;
 		self.log_strategy.encode(write.as_mut()).await;
 		self.msg_logs.encode(write.as_mut()).await
@@ -58,13 +58,13 @@ pub struct ExtensionHeader {
 	pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		read_exact(read.as_mut(), EXT_INTRO).await;
 		Self { name: String::decode(read.as_mut()).await, systems: Vec::decode(read).await }
 	}
 }
 impl Encode for ExtensionHeader {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write_exact(write.as_mut(), EXT_INTRO).await;
 		self.name.encode(write.as_mut()).await;
 		self.systems.encode(write).await
@@ -88,7 +88,8 @@ pub enum ExtHostReq {
 	SysFwd(system::SysFwd),
 	ExprReq(expr::ExprReq),
 	SubLex(lexer::SubLex),
-	RunMacros(macros::RunMacros),
+	LsModule(tree::LsModule),
+	ResolveNames(parser::ResolveNames),
 }
 
 /// Notifications sent from the extension to the host
@@ -117,17 +118,16 @@ pub enum HostExtReq {
 	DeserAtom(atom::DeserAtom),
 	LexExpr(lexer::LexExpr),
 	ParseLine(parser::ParseLine),
+	FetchParsedConst(parser::FetchParsedConst),
 	GetMember(tree::GetMember),
-	VfsReq(vfs::VfsReq),
+	SystemDrop(system::SystemDrop),
-	ApplyMacro(macros::ApplyMacro),
+	AtomDrop(atom::AtomDrop),
 }
 
 /// Notifications sent from the host to the extension
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extendable]
 pub enum HostExtNotif {
-	SystemDrop(system::SystemDrop),
-	AtomDrop(atom::AtomDrop),
 	/// The host can assume that after this notif is sent, a correctly written
 	/// extension will eventually exit.
 	Exit,

orchid-api/src/system.rs

@@ -5,7 +5,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::{CharFilter, ExtHostReq, HostExtNotif, HostExtReq, MemberKind, TStr};
+use crate::{CharFilter, ExtHostReq, HostExtReq, MemberKind, TStr, TStrv};
 
 /// ID of a system type
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
@@ -52,22 +52,26 @@ pub struct NewSystem {
 	pub depends: Vec<SysId>,
 }
 impl Request for NewSystem {
-	type Response = SystemInst;
+	type Response = NewSystemResponse;
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct SystemInst {
+pub struct NewSystemResponse {
 	/// The set of possible starting characters of tokens the lexer of this system
 	/// can process. The lexer will notify this system if it encounters one of
 	/// these characters.9
 	pub lex_filter: CharFilter,
 	pub line_types: Vec<TStr>,
 	pub const_root: HashMap<TStr, MemberKind>,
+	pub prelude: Vec<TStrv>,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtNotif)]
+#[extends(HostExtReq)]
 pub struct SystemDrop(pub SysId);
+impl Request for SystemDrop {
+	type Response = ();
+}
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(SysReq, HostExtReq)]

orchid-api/src/tree.rs

@@ -1,14 +1,12 @@
+use std::collections::HashMap;
 use std::num::NonZeroU64;
 use std::ops::Range;
-use std::sync::Arc;
+use std::rc::Rc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
-use ordered_float::NotNan;
 
-use crate::{
+use crate::{ExprTicket, Expression, ExtHostReq, HostExtReq, OrcError, SysId, TStr, TStrv};
-	Atom, Expression, HostExtReq, Location, MacroBlock, OrcError, Placeholder, SysId, TStr, TStrv,
-};
 
 /// A token tree from a lexer recursion request. Its lifetime is the lex call,
 /// the lexer can include it in its output or discard it by implication.
@@ -29,30 +27,27 @@ pub struct TokenTree {
 #[derive(Clone, Debug, Coding)]
 pub enum Token {
 	/// Lambda function head, from the opening \ until the beginning of the body.
-	LambdaHead(Vec<TokenTree>),
+	LambdaHead(Box<TokenTree>),
 	/// A name segment or an operator.
 	Name(TStr),
-	/// An absolute name
+	/// A newly generated expression. The last place this is supposed to happen is
-	Reference(TStrv),
+	/// in lexers, parsers and macros should have enumerable many outputs
+	/// expressed as function calls.
+	NewExpr(Expression),
+	/// A pre-existing expression
+	Handle(ExprTicket),
 	/// ::
-	NS,
+	NS(TStr, Box<TokenTree>),
 	/// Line break.
 	BR,
 	/// ( Round parens ), [ Square brackets ] or { Curly braces }
 	S(Paren, Vec<TokenTree>),
-	/// A new atom
-	Atom(Atom),
-	/// Anchor to insert a subtree
-	Slot(TreeTicket),
 	/// A static compile-time error returned by failing lexers if
-	/// the rest of the source is likely still meaningful
+	/// the rest of the source is likely still meaningful. This is distinct from
+	/// NewExpr(Bottom) because it fails in dead branches too.
 	Bottom(Vec<OrcError>),
 	/// A comment
-	Comment(Arc<String>),
+	Comment(Rc<String>),
-	/// Placeholder
-	Ph(Placeholder),
-	/// Macro block head
-	Macro(Option<NotNan<f64>>),
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]
@@ -65,31 +60,12 @@ pub enum Paren {
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct TreeId(pub NonZeroU64);
 
-#[derive(Clone, Debug, Coding)]
-pub struct Item {
-	pub location: Location,
-	pub comments: Vec<Comment>,
-	pub kind: ItemKind,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub enum ItemKind {
-	Member(Member),
-	Macro(MacroBlock),
-	Export(TStr),
-	Import(TStrv),
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct Comment {
-	pub text: TStr,
-	pub location: Location,
-}
-
 #[derive(Clone, Debug, Coding)]
 pub struct Member {
 	pub name: TStr,
+	pub exported: bool,
 	pub kind: MemberKind,
+	pub comments: Vec<TStr>,
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -101,12 +77,54 @@ pub enum MemberKind {
 
 #[derive(Clone, Debug, Coding)]
 pub struct Module {
-	pub items: Vec<Item>,
+	pub members: Vec<Member>,
 }
 
+/// Evaluate a lazy member. This call will only be issued to each system once.
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct GetMember(pub SysId, pub TreeId);
 impl Request for GetMember {
 	type Response = MemberKind;
 }
+
+/// This request can only be issued while the interpreter is running, so during
+/// an atom call.
+#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct LsModule(pub SysId, pub TStrv);
+impl Request for LsModule {
+	type Response = Result<ModuleInfo, LsModuleError>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum LsModuleError {
+	InvalidPath,
+	IsConstant,
+	TreeUnavailable,
+}
+
+/// Information about a module sent from the host to an extension. By necessity,
+/// members and imports are non-overlapping.
+#[derive(Clone, Debug, Coding)]
+pub struct ModuleInfo {
+	/// List the names defined in this module
+	pub members: HashMap<TStr, MemberInfo>,
+}
+
+#[derive(Clone, Copy, Debug, Coding)]
+pub struct MemberInfo {
+	/// true if the name is exported
+	pub public: bool,
+	/// Whether the tree item is a constant value or a module
+	pub kind: MemberInfoKind,
+}
+
+/// Indicates what kind of node a name refers to
+#[derive(Clone, Copy, Debug, Coding)]
+pub enum MemberInfoKind {
+	/// has children obtained with [crate::LsModule]
+	Module,
+	/// has a value retrievable in [crate::ExpressionKind::Const]
+	Constant,
+}

orchid-api/src/vfs.rs

@@ -1,47 +0,0 @@
-use std::collections::HashMap;
-use std::num::NonZeroU16;
-
-use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::Request;
-
-use crate::error::OrcResult;
-use crate::interner::TStr;
-use crate::proto::HostExtReq;
-use crate::system::SysId;
-
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-pub struct VfsId(pub NonZeroU16);
-
-#[derive(Clone, Debug, Coding)]
-pub enum Loaded {
-	Code(String),
-	Collection(Vec<TStr>),
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(VfsReq, HostExtReq)]
-pub struct VfsRead(pub SysId, pub VfsId, pub Vec<TStr>);
-impl Request for VfsRead {
-	type Response = OrcResult<Loaded>;
-}
-
-#[derive(Clone, Debug, Coding)]
-pub enum EagerVfs {
-	Lazy(VfsId),
-	Eager(HashMap<TStr, EagerVfs>),
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(VfsReq, HostExtReq)]
-pub struct GetVfs(pub SysId);
-impl Request for GetVfs {
-	type Response = EagerVfs;
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtReq)]
-#[extendable]
-pub enum VfsReq {
-	GetVfs(GetVfs),
-	VfsRead(VfsRead),
-}

orchid-base/Cargo.toml

@@ -1,18 +1,17 @@
 [package]
 name = "orchid-base"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
-async-stream = "0.3.6"
 derive_destructure = "1.0.0"
-dyn-clone = "1.0.17"
+dyn-clone = "1.0.20"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.15.2"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
 never = "0.1.0"
@@ -20,11 +19,10 @@ num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"
-regex = "1.11.1"
+regex = "1.11.2"
-rust-embed = "8.5.0"
+rust-embed = "8.7.2"
-rust_decimal = "1.36.0"
+some_executor = "0.6.1"
-some_executor = "0.4.0"
 substack = "1.1.1"
-test_executors = "0.3.2"
+test_executors = "0.3.5"
 trait-set = "0.3.0"

orchid-base/src/builtin.rs

@@ -6,7 +6,6 @@ use futures::future::LocalBoxFuture;
 use crate::api;
 
 pub type Spawner = Rc<dyn Fn(LocalBoxFuture<'static, ()>)>;
-pub type RecvCB<'a> = Box<dyn for<'b> FnOnce(&'b [u8]) -> LocalBoxFuture<'b, ()> + 'a>;
 
 /// The 3 primary contact points with an extension are
 /// - send a message
@@ -15,8 +14,10 @@ pub type RecvCB<'a> = Box<dyn for<'b> FnOnce(&'b [u8]) -> LocalBoxFuture<'b, ()>
 ///
 /// There are no ordering guarantees about these
 pub trait ExtPort {
+	#[must_use]
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()>;
-	fn recv<'a>(&'a self, cb: RecvCB<'a>) -> LocalBoxFuture<'a, ()>;
+	#[must_use]
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>>;
 }
 
 pub struct ExtInit {
@@ -25,7 +26,7 @@ pub struct ExtInit {
 }
 impl ExtInit {
 	pub async fn send(&self, msg: &[u8]) { self.port.send(msg).await }
-	pub async fn recv<'a, 's: 'a>(&'s self, cb: RecvCB<'a>) { self.port.recv(Box::new(cb)).await }
+	pub async fn recv(&self) -> Option<Vec<u8>> { self.port.recv().await }
 }
 impl Deref for ExtInit {
 	type Target = api::ExtensionHeader;

orchid-base/src/clone.rs

@@ -1,12 +1,12 @@
 #[macro_export]
 macro_rules! clone {
-  ($($n:ident),+; $body:expr) => (
+  ($($n:ident $($mut:ident)?),+; $body:expr) => (
     {
-      $( let $n = $n.clone(); )+
+      $( let $($mut)? $n = $n.clone(); )+
       $body
     }
   );
-  ($($n:ident),+) => {
+  ($($n:ident $($mut:ident)?),+) => {
-    $( let $n = $n.clone(); )+
+    $( let $($mut)? $n = $n.clone(); )+
   }
 }

orchid-base/src/error.rs

@@ -1,4 +1,5 @@
 use std::cell::RefCell;
+use std::ffi::OsStr;
 use std::fmt;
 use std::ops::Add;
 use std::sync::Arc;
@@ -39,6 +40,14 @@ impl ErrPos {
 impl From<Pos> for ErrPos {
 	fn from(origin: Pos) -> Self { Self { position: origin, message: None } }
 }
+impl fmt::Display for ErrPos {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match &self.message {
+			Some(msg) => write!(f, "{}: {}", self.position, msg),
+			None => write!(f, "{}", self.position),
+		}
+	}
+}
 
 #[derive(Clone, Debug)]
 pub struct OrcErr {
@@ -62,16 +71,15 @@ impl OrcErr {
 		}
 	}
 }
-impl Eq for OrcErr {}
+impl PartialEq<Tok<String>> for OrcErr {
-impl PartialEq for OrcErr {
+	fn eq(&self, other: &Tok<String>) -> bool { self.description == *other }
-	fn eq(&self, other: &Self) -> bool { self.description == other.description }
 }
 impl From<OrcErr> for Vec<OrcErr> {
 	fn from(value: OrcErr) -> Self { vec![value] }
 }
 impl fmt::Display for OrcErr {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let pstr = self.positions.iter().map(|p| format!("{p:?}")).join("; ");
+		let pstr = self.positions.iter().map(|p| format!("{p}")).join("; ");
 		write!(f, "{}: {} @ {}", self.description, self.message, pstr)
 	}
 }
@@ -149,40 +157,98 @@ impl fmt::Display for OrcErrv {
 
 pub type OrcRes<T> = Result<T, OrcErrv>;
 
-pub fn mk_err(
+pub fn join_ok<T, U>(left: OrcRes<T>, right: OrcRes<U>) -> OrcRes<(T, U)> {
-	description: Tok<String>,
+	match (left, right) {
-	message: impl AsRef<str>,
+		(Ok(t), Ok(u)) => Ok((t, u)),
-	posv: impl IntoIterator<Item = ErrPos>,
+		(Err(e), Ok(_)) | (Ok(_), Err(e)) => Err(e),
-) -> OrcErr {
+		(Err(e1), Err(e2)) => Err(e1 + e2),
-	OrcErr {
-		description,
-		message: Arc::new(message.as_ref().to_string()),
-		positions: posv.into_iter().collect(),
 	}
 }
 
-pub fn mk_errv(
+#[macro_export]
+macro_rules! join_ok {
+	($($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		let $crate::join_ok!(@NAMES $($names $(: $tys)? = $vals;)*)
+		:
+		$crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*)
+		 =
+		$crate::join_ok!(@VALUES $($names $(: $tys)? = $vals;)*)?;
+	};
+	(@NAMES $name:ident $(: $ty:ty)? = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		($name, $crate::join_ok!(@NAMES $($names $(: $tys)? = $vals;)*))
+	};
+	(@NAMES) => { _ };
+	(@TYPES $name:ident : $ty:ty = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		($ty, $crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*))
+	};
+	(@TYPES $name:ident = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		(_, $crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*))
+	};
+	(@TYPES) => { () };
+	(@VALUES $name:ident $(: $ty:ty)? = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		$crate::error::join_ok($val, $crate::join_ok!(@VALUES $($names $(: $tys)? = $vals;)*))
+	};
+	(@VALUES) => { Ok(()) };
+}
+
+pub fn mk_errv_floating(description: Tok<String>, message: impl AsRef<str>) -> OrcErrv {
+	mk_errv::<Pos>(description, message, [])
+}
+
+pub fn mk_errv<I: Into<ErrPos>>(
 	description: Tok<String>,
 	message: impl AsRef<str>,
-	posv: impl IntoIterator<Item = ErrPos>,
+	posv: impl IntoIterator<Item = I>,
 ) -> OrcErrv {
-	mk_err(description, message, posv).into()
+	OrcErr {
+		description,
+		message: Arc::new(message.as_ref().to_string()),
+		positions: posv.into_iter().map_into().collect(),
+	}
+	.into()
 }
 
-pub trait Reporter {
+pub async fn async_io_err<I: Into<ErrPos>>(
-	fn report(&self, e: impl Into<OrcErrv>);
+	err: std::io::Error,
+	i: &Interner,
+	posv: impl IntoIterator<Item = I>,
+) -> OrcErrv {
+	mk_errv(i.i(&err.kind().to_string()).await, err.to_string(), posv)
 }
 
-pub struct ReporterImpl {
+pub async fn os_str_to_string<'a, I: Into<ErrPos>>(
+	str: &'a OsStr,
+	i: &Interner,
+	posv: impl IntoIterator<Item = I>,
+) -> OrcRes<&'a str> {
+	match str.to_str() {
+		Some(str) => Ok(str),
+		None => Err(mk_errv(
+			i.i("Non-unicode string").await,
+			format!("{str:?} is not representable as unicode"),
+			posv,
+		)),
+	}
+}
+
+pub struct Reporter {
 	errors: RefCell<Vec<OrcErr>>,
 }
-impl ReporterImpl {
+
+impl Reporter {
+	pub fn report(&self, e: impl Into<OrcErrv>) { self.errors.borrow_mut().extend(e.into()) }
 	pub fn new() -> Self { Self { errors: RefCell::new(vec![]) } }
 	pub fn errv(self) -> Option<OrcErrv> { OrcErrv::new(self.errors.into_inner()).ok() }
+	pub fn merge<T>(self, res: OrcRes<T>) -> OrcRes<T> {
+		match (res, self.errv()) {
+			(res, None) => res,
+			(Ok(_), Some(errv)) => Err(errv),
+			(Err(e), Some(errv)) => Err(e + errv),
+		}
+	}
+	pub fn is_empty(&self) -> bool { self.errors.borrow().is_empty() }
 }
-impl Reporter for ReporterImpl {
+
-	fn report(&self, e: impl Into<OrcErrv>) { self.errors.borrow_mut().extend(e.into()) }
+impl Default for Reporter {
-}
-impl Default for ReporterImpl {
 	fn default() -> Self { Self::new() }
 }

orchid-base/src/format.rs

@@ -13,6 +13,7 @@ use crate::interner::Interner;
 use crate::{api, match_mapping};
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
+#[must_use]
 pub struct FmtUnit {
 	pub subs: Vec<FmtUnit>,
 	pub variants: Rc<Variants>,
@@ -209,6 +210,9 @@ impl From<Rc<String>> for Variants {
 impl From<String> for Variants {
 	fn from(value: String) -> Self { Self::from(Rc::new(value)) }
 }
+impl From<&str> for Variants {
+	fn from(value: &str) -> Self { Self::from(value.to_string()) }
+}
 impl FromStr for Variants {
 	type Err = Infallible;
 	fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(Self::default().bounded(s)) }
@@ -265,8 +269,12 @@ impl FmtCtx for FmtCtxImpl<'_> {
 }
 
 pub trait Format {
+	#[must_use]
 	fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> + 'a;
 }
 impl Format for Never {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit { match *self {} }
 }
+
+/// Format with default strategy. Currently equal to [take_first_fmt]
+pub async fn fmt(v: &(impl Format + ?Sized), i: &Interner) -> String { take_first_fmt(v, i).await }

orchid-base/src/interner.rs

@@ -7,7 +7,7 @@ use std::rc::Rc;
 use std::sync::atomic;
 use std::{fmt, hash};
 
-use async_std::sync::Mutex;
+use futures::lock::Mutex;
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools as _;
 use orchid_api_traits::Request;
@@ -98,7 +98,7 @@ impl Interned for String {
 impl InternMarker for api::TStr {
 	type Interned = String;
 	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
-		Tok::new(Rc::new(i.master.as_ref().unwrap().request(api::ExternStr(self)).await), self)
+		Tok::new(Rc::new(i.0.master.as_ref().unwrap().request(api::ExternStr(self)).await), self)
 	}
 	fn get_id(self) -> NonZeroU64 { self.0 }
 	fn from_id(id: NonZeroU64) -> Self { Self(id) }
@@ -125,7 +125,7 @@ impl Interned for Vec<Tok<String>> {
 impl InternMarker for api::TStrv {
 	type Interned = Vec<Tok<String>>;
 	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
-		let rep = i.master.as_ref().unwrap().request(api::ExternStrv(self)).await;
+		let rep = i.0.master.as_ref().unwrap().request(api::ExternStrv(self)).await;
 		let data = futures::future::join_all(rep.into_iter().map(|m| i.ex(m))).await;
 		Tok::new(Rc::new(data), self)
 	}
@@ -217,24 +217,26 @@ pub struct TypedInterners {
 }
 
 #[derive(Default)]
-pub struct Interner {
+pub struct InternerData {
 	interners: Mutex<TypedInterners>,
 	master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
 }
+#[derive(Clone, Default)]
+pub struct Interner(Rc<InternerData>);
 impl Interner {
 	pub fn new_master() -> Self { Self::default() }
 	pub fn new_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static) -> Self {
-		Self { master: Some(Box::new(req)), interners: Mutex::default() }
+		Self(Rc::new(InternerData { master: Some(Box::new(req)), interners: Mutex::default() }))
 	}
 	/// Intern some data; query its identifier if not known locally
 	pub async fn i<T: Interned>(&self, t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
 		let data = t.get_owned();
-		let mut g = self.interners.lock().await;
+		let mut g = self.0.interners.lock().await;
 		let typed = T::bimap(&mut g);
 		if let Some(tok) = typed.by_value(&data) {
 			return tok;
 		}
-		let marker = match &self.master {
+		let marker = match &self.0.master {
 			Some(c) => data.clone().intern(&**c).await,
 			None =>
 				T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
@@ -245,29 +247,29 @@ impl Interner {
 	}
 	/// Extern an identifier; query the data it represents if not known locally
 	pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
-		if let Some(tok) = M::Interned::bimap(&mut *self.interners.lock().await).by_marker(marker) {
+		if let Some(tok) = M::Interned::bimap(&mut *self.0.interners.lock().await).by_marker(marker) {
 			return tok;
 		}
-		assert!(self.master.is_some(), "ID not in local interner and this is master");
+		assert!(self.0.master.is_some(), "ID not in local interner and this is master");
 		let token = marker.resolve(self).await;
-		M::Interned::bimap(&mut *self.interners.lock().await).insert(token.clone());
+		M::Interned::bimap(&mut *self.0.interners.lock().await).insert(token.clone());
 		token
 	}
 	pub async fn sweep_replica(&self) -> api::Retained {
-		assert!(self.master.is_some(), "Not a replica");
+		assert!(self.0.master.is_some(), "Not a replica");
-		let mut g = self.interners.lock().await;
+		let mut g = self.0.interners.lock().await;
 		api::Retained { strings: g.strings.sweep_replica(), vecs: g.vecs.sweep_replica() }
 	}
 	pub async fn sweep_master(&self, retained: api::Retained) {
-		assert!(self.master.is_none(), "Not master");
+		assert!(self.0.master.is_none(), "Not master");
-		let mut g = self.interners.lock().await;
+		let mut g = self.0.interners.lock().await;
 		g.strings.sweep_master(retained.strings.into_iter().collect());
 		g.vecs.sweep_master(retained.vecs.into_iter().collect());
 	}
 }
 impl fmt::Debug for Interner {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "Interner{{ replica: {} }}", self.master.is_none())
+		write!(f, "Interner{{ replica: {} }}", self.0.master.is_none())
 	}
 }
 

orchid-base/src/iter_utils.rs

@@ -0,0 +1,24 @@
+use std::fmt;
+
+use itertools::{Itertools, Position};
+
+pub struct PrintList<'a, I: Iterator<Item = E> + Clone, E: fmt::Display>(pub I, pub &'a str);
+impl<'a, I: Iterator<Item = E> + Clone, E: fmt::Display> fmt::Display for PrintList<'a, I, E> {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		for (pos, item) in self.0.clone().with_position() {
+			match pos {
+				Position::First | Position::Only => write!(f, "{item}")?,
+				Position::Middle => write!(f, ", {item}")?,
+				Position::Last => write!(f, ", {} {item}", self.1)?,
+			}
+		}
+		Ok(())
+	}
+}
+
+pub trait IteratorPrint: Iterator<Item: fmt::Display> + Clone {
+	fn display<'a>(self, operator: &'a str) -> PrintList<'a, Self, Self::Item> {
+		PrintList(self, operator)
+	}
+}
+impl<T: Iterator<Item: fmt::Display> + Clone> IteratorPrint for T {}

orchid-base/src/lib.rs

@@ -12,10 +12,10 @@ pub mod event;
 pub mod format;
 pub mod id_store;
 pub mod interner;
+pub mod iter_utils;
 pub mod join;
 pub mod location;
 pub mod logging;
-pub mod macros;
 mod match_mapping;
 pub mod msg;
 pub mod name;

orchid-base/src/location.rs

@@ -6,6 +6,7 @@ use std::ops::Range;
 
 use trait_set::trait_set;
 
+use crate::error::ErrPos;
 use crate::interner::{Interner, Tok};
 use crate::name::Sym;
 use crate::{api, match_mapping, sym};
@@ -14,7 +15,7 @@ trait_set! {
 	pub trait GetSrc = FnMut(&Sym) -> Tok<String>;
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Pos {
 	None,
 	SlotTarget,
@@ -23,15 +24,13 @@ pub enum Pos {
 	Inherit,
 	Gen(CodeGenInfo),
 	/// Range and file
-	SourceRange(SourceRange),
+	SrcRange(SrcRange),
-	/// Range only, file implied. Most notably used by parsers
-	Range(Range<u32>),
 }
 impl Pos {
 	pub fn pretty_print(&self, get_src: &mut impl GetSrc) -> String {
 		match self {
 			Self::Gen(g) => g.to_string(),
-			Self::SourceRange(sr) => sr.pretty_print(&get_src(&sr.path)),
+			Self::SrcRange(sr) => sr.pretty_print(&get_src(&sr.path)),
 			// Can't pretty print partial and meta-location
 			other => format!("{other:?}"),
 		}
@@ -39,30 +38,41 @@ impl Pos {
 	pub async fn from_api(api: &api::Location, i: &Interner) -> Self {
 		match_mapping!(api, api::Location => Pos {
 			None, Inherit, SlotTarget,
-			Range(r.clone()),
 			Gen(cgi => CodeGenInfo::from_api(cgi, i).await),
-			SourceRange(sr => SourceRange::from_api(sr, i).await)
+		} {
+			api::Location::SourceRange(sr) => Self::SrcRange(SrcRange::from_api(sr, i).await)
 		})
 	}
 	pub fn to_api(&self) -> api::Location {
 		match_mapping!(self, Pos => api::Location {
 			None, Inherit, SlotTarget,
-			Range(r.clone()),
 			Gen(cgi.to_api()),
-			SourceRange(sr.to_api()),
+		} {
+			Self::SrcRange(sr) => api::Location::SourceRange(sr.to_api()),
 		})
 	}
 }
+impl fmt::Display for Pos {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Pos::Inherit => f.write_str("Unresolved inherited position"),
+			Pos::SlotTarget => f.write_str("Unresolved slot target position"),
+			Pos::None => f.write_str("N/A"),
+			Pos::Gen(g) => write!(f, "{g}"),
+			Pos::SrcRange(sr) => write!(f, "{sr}"),
+		}
+	}
+}
 
 /// Exact source code location. Includes where the code was loaded from, what
 /// the original source code was, and a byte range.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct SourceRange {
+pub struct SrcRange {
 	pub(crate) path: Sym,
 	pub(crate) range: Range<u32>,
 }
-impl SourceRange {
+impl SrcRange {
-	pub fn new(range: &Range<u32>, path: &Sym) -> Self {
+	pub fn new(range: Range<u32>, path: &Sym) -> Self {
 		Self { range: range.clone(), path: path.clone() }
 	}
 	/// Create a dud [SourceRange] for testing. Its value is unspecified and
@@ -77,7 +87,7 @@ impl SourceRange {
 	/// 0-based index of last byte + 1
 	pub fn end(&self) -> u32 { self.range.end }
 	/// Syntactic location
-	pub fn pos(&self) -> Pos { Pos::SourceRange(self.clone()) }
+	pub fn pos(&self) -> Pos { Pos::SrcRange(self.clone()) }
 	/// Transform the numeric byte range
 	pub fn map_range(&self, map: impl FnOnce(Range<u32>) -> Range<u32>) -> Self {
 		Self { range: map(self.range()), path: self.path() }
@@ -92,12 +102,27 @@ impl SourceRange {
 		}
 	}
 	pub fn zw(path: Sym, pos: u32) -> Self { Self { path, range: pos..pos } }
-	async fn from_api(api: &api::SourceRange, i: &Interner) -> Self {
+	pub async fn from_api(api: &api::SourceRange, i: &Interner) -> Self {
 		Self { path: Sym::from_api(api.path, i).await, range: api.range.clone() }
 	}
-	fn to_api(&self) -> api::SourceRange {
+	pub fn to_api(&self) -> api::SourceRange {
 		api::SourceRange { path: self.path.to_api(), range: self.range.clone() }
 	}
+	pub fn to(&self, rhs: &Self) -> Self {
+		assert_eq!(self.path, rhs.path, "Range continues across files");
+		Self { path: self.path(), range: self.start().min(rhs.start())..self.end().max(rhs.end()) }
+	}
+}
+impl From<SrcRange> for ErrPos {
+	fn from(val: SrcRange) -> Self { val.pos().into() }
+}
+impl fmt::Display for SrcRange {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self.range.len() {
+			0 => write!(f, "{}:{}", self.path(), self.range.start),
+			n => write!(f, "{}:{}+{}", self.path(), self.range.start, n),
+		}
+	}
 }
 
 /// Information about a code generator attached to the generated code

orchid-base/src/macros.rs

@@ -1,168 +0,0 @@
-use std::marker::PhantomData;
-use std::rc::Rc;
-use std::sync::Arc;
-
-use async_stream::stream;
-use futures::future::{LocalBoxFuture, join_all};
-use futures::{FutureExt, StreamExt};
-use never::Never;
-use trait_set::trait_set;
-
-use crate::format::{FmtCtx, FmtUnit, Format, Variants};
-use crate::interner::Interner;
-use crate::location::Pos;
-use crate::name::Sym;
-use crate::tree::{Paren, Ph};
-use crate::{api, match_mapping, tl_cache};
-
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
-pub struct MacroSlot<'a>(api::MacroTreeId, PhantomData<&'a ()>);
-impl MacroSlot<'_> {
-	pub fn id(self) -> api::MacroTreeId { self.0 }
-}
-
-trait_set! {
-	pub trait MacroAtomToApi<A> = for<'a> FnMut(&'a A) -> LocalBoxFuture<'a, api::MacroToken>;
-	pub trait MacroAtomFromApi<'a, A> =
-		for<'b> FnMut(&'b api::Atom) -> LocalBoxFuture<'b, MTok<'a, A>>;
-}
-
-#[derive(Clone, Debug)]
-pub struct MTree<'a, A> {
-	pub pos: Pos,
-	pub tok: Rc<MTok<'a, A>>,
-}
-impl<'a, A> MTree<'a, A> {
-	pub(crate) async fn from_api(
-		api: &api::MacroTree,
-		do_atom: &mut impl MacroAtomFromApi<'a, A>,
-		i: &Interner,
-	) -> Self {
-		Self {
-			pos: Pos::from_api(&api.location, i).await,
-			tok: Rc::new(MTok::from_api(&api.token, i, do_atom).await),
-		}
-	}
-	pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroTree {
-		api::MacroTree {
-			location: self.pos.to_api(),
-			token: self.tok.to_api(do_atom).boxed_local().await,
-		}
-	}
-}
-impl<A: Format> Format for MTree<'_, A> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		self.tok.print(c).boxed_local().await
-	}
-}
-
-#[derive(Clone, Debug)]
-pub enum MTok<'a, A> {
-	S(Paren, Vec<MTree<'a, A>>),
-	Name(Sym),
-	Slot(MacroSlot<'a>),
-	Lambda(Vec<MTree<'a, A>>, Vec<MTree<'a, A>>),
-	Ph(Ph),
-	Atom(A),
-	/// Used in extensions to directly return input
-	Ref(Arc<MTok<'a, Never>>),
-	/// Used in the matcher to skip previous macro output which can only go in
-	/// vectorial placeholders
-	Done(Rc<MTok<'a, A>>),
-}
-impl<'a, A> MTok<'a, A> {
-	pub(crate) async fn from_api(
-		api: &api::MacroToken,
-		i: &Interner,
-		do_atom: &mut impl MacroAtomFromApi<'a, A>,
-	) -> Self {
-		match_mapping!(&api, api::MacroToken => MTok::<'a, A> {
-			Lambda(x => mtreev_from_api(x, i, do_atom).await, b => mtreev_from_api(b, i, do_atom).await),
-			Name(t => Sym::from_api(*t, i).await),
-			Slot(tk => MacroSlot(*tk, PhantomData)),
-			S(p.clone(), b => mtreev_from_api(b, i, do_atom).await),
-			Ph(ph => Ph::from_api(ph, i).await),
-		} {
-			api::MacroToken::Atom(a) => do_atom(a).await
-		})
-	}
-	pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroToken {
-		fn sink<T>(n: &Never) -> LocalBoxFuture<'_, T> { match *n {} }
-		match_mapping!(&self, MTok => api::MacroToken {
-			Lambda(x => mtreev_to_api(x, do_atom).await, b => mtreev_to_api(b, do_atom).await),
-			Name(t.tok().to_api()),
-			Ph(ph.to_api()),
-			S(p.clone(), b => mtreev_to_api(b, do_atom).await),
-			Slot(tk.0.clone()),
-		} {
-			MTok::Ref(r) => r.to_api(&mut sink).boxed_local().await,
-			MTok::Done(t) => t.to_api(do_atom).boxed_local().await,
-			MTok::Atom(a) => do_atom(a).await,
-		})
-	}
-	pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Rc::new(self) } }
-}
-impl<A: Format> Format for MTok<'_, A> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		match self {
-			Self::Atom(a) => a.print(c).await,
-			Self::Done(d) =>
-				FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("(Done){0l}"))), [
-					d.print(c).boxed_local().await,
-				]),
-			Self::Lambda(arg, b) => FmtUnit::new(
-				tl_cache!(Rc<Variants>: Rc::new(Variants::default()
-					.unbounded("\\{0b}.{1l}")
-					.bounded("(\\{0b}.{1b})"))),
-				[mtreev_fmt(arg, c).await, mtreev_fmt(b, c).await],
-			),
-			Self::Name(n) => format!("{n}").into(),
-			Self::Ph(ph) => format!("{ph}").into(),
-			Self::Ref(r) =>
-				FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("(ref){0l}"))), [
-					r.print(c).boxed_local().await,
-				]),
-			Self::S(p, body) => FmtUnit::new(
-				match *p {
-					Paren::Round => Rc::new(Variants::default().bounded("({0b})")),
-					Paren::Curly => Rc::new(Variants::default().bounded("{{0b}}")),
-					Paren::Square => Rc::new(Variants::default().bounded("[{0b}]")),
-				},
-				[mtreev_fmt(body, c).await],
-			),
-			Self::Slot(slot) => format!("{:?}", slot.0).into(),
-		}
-	}
-}
-
-pub async fn mtreev_from_api<'a, 'b, A>(
-	apiv: impl IntoIterator<Item = &'b api::MacroTree>,
-	i: &Interner,
-	do_atom: &'b mut (impl MacroAtomFromApi<'a, A> + 'b),
-) -> Vec<MTree<'a, A>> {
-	stream! {
-		for api in apiv {
-			yield MTree::from_api(api, do_atom, i).boxed_local().await
-		}
-	}
-	.collect()
-	.await
-}
-
-pub async fn mtreev_to_api<'a: 'b, 'b, A: 'b>(
-	v: impl IntoIterator<Item = &'b MTree<'a, A>>,
-	do_atom: &mut impl MacroAtomToApi<A>,
-) -> Vec<api::MacroTree> {
-	let mut out = Vec::new();
-	for t in v {
-		out.push(t.to_api(do_atom).await);
-	}
-	out
-}
-
-pub async fn mtreev_fmt<'a: 'b, 'b, A: 'b + Format>(
-	v: impl IntoIterator<Item = &'b MTree<'a, A>>,
-	c: &(impl FmtCtx + ?Sized),
-) -> FmtUnit {
-	FmtUnit::sequence(" ", None, join_all(v.into_iter().map(|t| t.print(c))).await)
-}

orchid-base/src/match_mapping.rs

@@ -1,4 +1,4 @@
-/// A shorthand for mapping over enums with identical structure. Used for
+/// A shorthand for mapping over enums with similar structure. Used for
 /// converting between owned enums and the corresponding API enums that only
 /// differ in the type of their fields.
 ///
@@ -7,7 +7,11 @@
 /// match_mapping!(self, ThisType => OtherType {
 ///   EmptyVariant,
 ///   TupleVariant(foo => intern(foo), bar.clone()),
-///   StructVariant{ a.to_api(), b }
+///   StructVariant{ a.to_api(), b },
+///   DedicatedConverter(value () convert)
+/// } {
+///   ThisType::DimorphicVariant(c) => OtherType::CorrespondingVariant(c.left(), c.right()),
+///   ThisType::UnexpectedVariant => panic!(),
 /// })
 /// ```
 #[macro_export]

orchid-base/src/msg.rs

@@ -1,10 +1,10 @@
 use std::io;
 use std::pin::Pin;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use orchid_api_traits::{Decode, Encode};
 
-pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result<()> {
+pub async fn send_msg(mut write: Pin<&mut impl AsyncWrite>, msg: &[u8]) -> io::Result<()> {
 	let mut len_buf = vec![];
 	u32::try_from(msg.len()).unwrap().encode(Pin::new(&mut len_buf)).await;
 	write.write_all(&len_buf).await?;
@@ -12,7 +12,7 @@ pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result
 	write.flush().await
 }
 
-pub async fn recv_msg(mut read: Pin<&mut impl Read>) -> io::Result<Vec<u8>> {
+pub async fn recv_msg(mut read: Pin<&mut impl AsyncRead>) -> io::Result<Vec<u8>> {
 	let mut len_buf = [0u8; (u32::BITS / 8) as usize];
 	read.read_exact(&mut len_buf).await?;
 	let len = u32::decode(Pin::new(&mut &len_buf[..])).await;

orchid-base/src/name.rs

@@ -2,11 +2,10 @@
 
 use std::borrow::Borrow;
 use std::hash::Hash;
-use std::iter::Cloned;
 use std::num::{NonZeroU64, NonZeroUsize};
-use std::ops::{Bound, Deref, Index, RangeBounds};
+use std::ops::{Deref, Index};
 use std::path::Path;
-use std::{fmt, slice, vec};
+use std::{fmt, vec};
 
 use futures::future::{OptionFuture, join_all};
 use itertools::Itertools;
@@ -20,10 +19,9 @@ trait_set! {
 	pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
 }
 
-/// A token path which may be empty. [VName] is the non-empty,
+/// A token path which may be empty. [VName] is the non-empty version
-/// [PathSlice] is the borrowed version
 #[derive(Clone, Default, Hash, PartialEq, Eq)]
-pub struct VPath(pub Vec<Tok<String>>);
+pub struct VPath(Vec<Tok<String>>);
 impl VPath {
 	/// Collect segments into a vector
 	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Self {
@@ -54,12 +52,12 @@ impl VPath {
 	pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
 	/// Add a token to the path. Since now we know that it can't be empty, turn it
 	/// into a name.
-	pub fn name_with_prefix(self, name: Tok<String>) -> VName {
+	pub fn name_with_suffix(self, name: Tok<String>) -> VName {
 		VName(self.into_iter().chain([name]).collect())
 	}
 	/// Add a token to the beginning of the. Since now we know that it can't be
 	/// empty, turn it into a name.
-	pub fn name_with_suffix(self, name: Tok<String>) -> VName {
+	pub fn name_with_prefix(self, name: Tok<String>) -> VName {
 		VName([name].into_iter().chain(self).collect())
 	}
 
@@ -237,6 +235,9 @@ impl Sym {
 		Self::from_tok(Tok::from_api(marker, i).await).expect("Empty sequence found for serialized Sym")
 	}
 	pub fn to_api(&self) -> api::TStrv { self.tok().to_api() }
+	pub async fn suffix(&self, tokv: impl IntoIterator<Item = Tok<String>>, i: &Interner) -> Sym {
+		Self::new(self.0.iter().cloned().chain(tokv), i).await.unwrap()
+	}
 }
 impl fmt::Debug for Sym {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Sym({self})") }
@@ -270,34 +271,34 @@ pub trait NameLike:
 	/// Convert into held slice
 	fn as_slice(&self) -> &[Tok<String>] { Borrow::<[Tok<String>]>::borrow(self) }
 	/// Get iterator over tokens
-	fn iter(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
+	fn segs(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
 	/// Get iterator over string segments
 	fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ {
 		self.as_slice().iter().map(|t| t.as_str())
 	}
 	/// Fully resolve the name for printing
 	#[must_use]
-	fn to_strv(&self) -> Vec<String> { self.iter().map(|s| s.to_string()).collect() }
+	fn to_strv(&self) -> Vec<String> { self.segs().map(|s| s.to_string()).collect() }
 	/// Format the name as an approximate filename
-	fn as_src_path(&self) -> String { format!("{}.orc", self.iter().join("/")) }
+	fn as_src_path(&self) -> String { format!("{}.orc", self.segs().join("/")) }
 	/// Return the number of segments in the name
-	fn len(&self) -> NonZeroUsize {
+	fn len_nz(&self) -> NonZeroUsize {
-		NonZeroUsize::try_from(self.iter().count()).expect("NameLike never empty")
+		NonZeroUsize::try_from(self.segs().count()).expect("NameLike never empty")
 	}
 	/// Like slice's `split_first` except we know that it always returns Some
-	fn split_first(&self) -> (Tok<String>, &[Tok<String>]) {
+	fn split_first_seg(&self) -> (Tok<String>, &[Tok<String>]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Like slice's `split_last` except we know that it always returns Some
-	fn split_last(&self) -> (Tok<String>, &[Tok<String>]) {
+	fn split_last_seg(&self) -> (Tok<String>, &[Tok<String>]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Get the first element
-	fn first(&self) -> Tok<String> { self.split_first().0 }
+	fn first_seg(&self) -> Tok<String> { self.split_first_seg().0 }
 	/// Get the last element
-	fn last(&self) -> Tok<String> { self.split_last().0 }
+	fn last_seg(&self) -> Tok<String> { self.split_last_seg().0 }
 }
 
 impl NameLike for Sym {}

orchid-base/src/number.rs

@@ -1,35 +1,27 @@
 use std::num::IntErrorKind;
 use std::ops::Range;
 
-use num_traits::ToPrimitive;
 use ordered_float::NotNan;
-use rust_decimal::Decimal;
 
-use crate::error::{OrcErr, mk_err};
+use crate::error::{OrcErrv, mk_errv};
 use crate::interner::Interner;
-use crate::location::Pos;
+use crate::location::SrcRange;
+use crate::name::Sym;
 
 /// A number, either floating point or unsigned int, parsed by Orchid.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Numeric {
-	/// A nonnegative integer
+	/// An integer
-	Uint(u64),
+	Int(i64),
 	/// A binary float other than NaN
 	Float(NotNan<f64>),
-	/// A decimal number
-	Decimal(Decimal),
 }
 impl Numeric {
-	pub fn decimal(num: i64, scale: u32) -> Self { Self::Decimal(Decimal::new(num, scale)) }
 	pub fn float(value: f64) -> Self { Self::Float(NotNan::new(value).unwrap()) }
 	pub fn to_f64(self) -> NotNan<f64> {
 		match self {
 			Self::Float(f) => f,
-			Self::Decimal(d) => {
+			Self::Int(i) => NotNan::new(i as f64).expect("int cannot be NaN"),
-				let f = d.to_f64().expect("This is apparently always possible");
-				NotNan::new(f).expect("decimal was nan")
-			},
-			Self::Uint(i) => NotNan::new(i as f64).expect("int cannot be NaN"),
 		}
 	}
 }
@@ -63,45 +55,48 @@ pub struct NumError {
 	pub kind: NumErrorKind,
 }
 
-pub async fn num_to_err(NumError { kind, range }: NumError, offset: u32, i: &Interner) -> OrcErr {
+pub async fn num_to_errv(
-	mk_err(
+	NumError { kind, range }: NumError,
+	offset: u32,
+	source: &Sym,
+	i: &Interner,
+) -> OrcErrv {
+	mk_errv(
 		i.i("Failed to parse number").await,
 		match kind {
 			NumErrorKind::NaN => "NaN emerged during parsing",
 			NumErrorKind::InvalidDigit => "non-digit character encountered",
 			NumErrorKind::Overflow => "The number being described is too large or too accurate",
 		},
-		[Pos::Range(offset + range.start as u32..offset + range.end as u32).into()],
+		[SrcRange::new(offset + range.start as u32..offset + range.end as u32, source)],
 	)
 }
 
 /// Parse a numbre literal out of text
 pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
-	let overflow_err = NumError { range: 0..string.len(), kind: NumErrorKind::Overflow };
+	let overflow_e = NumError { range: 0..string.len(), kind: NumErrorKind::Overflow };
 	let (radix, noprefix, pos) = (string.strip_prefix("0x").map(|s| (16u8, s, 2)))
 		.or_else(|| string.strip_prefix("0b").map(|s| (2u8, s, 2)))
 		.or_else(|| string.strip_prefix("0o").map(|s| (8u8, s, 2)))
 		.unwrap_or((10u8, string, 0));
-	eprintln!("({radix}, {noprefix}, {pos})");
 	// identity
-	let (base, exponent) = match noprefix.split_once('p') {
+	let (base_s, exponent) = match noprefix.split_once('p') {
 		Some((b, e)) => {
 			let (s, d, len) = e.strip_prefix('-').map_or((1, e, 0), |ue| (-1, ue, 1));
 			(b, s * int_parse(d, 10, pos + b.len() + 1 + len)? as i32)
 		},
 		None => (noprefix, 0),
 	};
-	eprintln!("({base},{exponent})");
+	match base_s.split_once('.') {
-	match base.split_once('.') {
 		None => {
-			let base_usize = int_parse(base, radix, pos)?;
+			let base = int_parse(base_s, radix, pos)?;
-			if let Ok(pos_exp) = u32::try_from(exponent) {
+			if let Ok(pos_exp) = u32::try_from(exponent)
-				if let Some(radical) = u64::from(radix).checked_pow(pos_exp) {
+				&& let Some(radical) = u64::from(radix).checked_pow(pos_exp)
-					let number = base_usize.checked_mul(radical).ok_or(overflow_err)?;
+			{
-					return Ok(Numeric::Uint(number));
+				let num = base.checked_mul(radical).and_then(|m| m.try_into().ok()).ok_or(overflow_e)?;
+				return Ok(Numeric::Int(num));
 			}
-			}
+			let f = (base as f64) * (radix as f64).powi(exponent);
-			let f = (base_usize as f64) * (radix as f64).powi(exponent);
 			let err = NumError { range: 0..string.len(), kind: NumErrorKind::NaN };
 			Ok(Numeric::Float(NotNan::new(f).map_err(|_| err)?))
 		},
@@ -109,25 +104,9 @@ pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
 			let whole_n = int_parse(whole, radix, pos)?;
 			let part_n = int_parse(part, radix, pos + whole.len() + 1)?;
 			let scale = part.chars().filter(|c| *c != '_').count() as u32;
-			if radix == 10 {
-				let scaled_unit = 10u64.checked_pow(scale).ok_or(overflow_err.clone())?;
-				let scaled_n = i128::from(whole_n) * i128::from(scaled_unit) + i128::from(part_n);
-				let decimal = Decimal::from_i128_with_scale(scaled_n, scale);
-				let p = if let Ok(uexp) = u32::try_from(exponent) {
-					let e_multiplier = 10i64.checked_pow(uexp).ok_or(overflow_err)?;
-					Decimal::new(e_multiplier, 0)
-				} else {
-					let inv_oom = u32::try_from(-exponent).map_err(|_| overflow_err)?;
-					eprintln!("inv_oom: {inv_oom}");
-					Decimal::new(1, inv_oom)
-				};
-				eprintln!("({scaled_n}, {scale}, {p})");
-				Ok(Numeric::Decimal(decimal * p))
-			} else {
 			let real_val = whole_n as f64 + (part_n as f64 / (radix as f64).powi(scale as i32));
 			let f = real_val * (radix as f64).powi(exponent);
 			Ok(Numeric::Float(NotNan::new(f).expect("None of the inputs are NaN")))
-			}
 		},
 	}
 }
@@ -168,7 +147,7 @@ mod test {
 
 	#[test]
 	fn just_ints() {
-		let test = |s, n| assert_eq!(parse_num(s), Ok(Numeric::Uint(n)));
+		let test = |s, n| assert_eq!(parse_num(s), Ok(Numeric::Int(n)));
 		test("12345", 12345);
 		test("0xcafebabe", 0xcafebabe);
 		test("0o751", 0o751);
@@ -178,11 +157,11 @@ mod test {
 	#[test]
 	fn decimals() {
 		let test = |s, n| assert_eq!(parse_num(s), Ok(n));
-		test("3.1417", Numeric::decimal(31417, 4));
+		test("3.1417", Numeric::float(3.1417));
 		test("0xf.cafe", Numeric::float(0xf as f64 + 0xcafe as f64 / 0x10000 as f64));
-		test("34p3", Numeric::Uint(34000));
+		test("34p3", Numeric::Int(34000));
-		test("0x2p3", Numeric::Uint(0x2 * 0x1000));
+		test("0x2p3", Numeric::Int(0x2 * 0x1000));
-		test("1.5p3", Numeric::decimal(1500, 0));
+		test("1.5p3", Numeric::float(1500.0));
 		test("0x2.5p3", Numeric::float((0x25 * 0x100) as f64));
 	}
 }

orchid-base/src/parse.rs

@@ -1,17 +1,33 @@
 use std::fmt::{self, Display};
 use std::iter;
-use std::ops::{Deref, Range};
+use std::ops::Deref;
 
+use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
 
 use crate::api;
-use crate::error::{OrcRes, Reporter, mk_err, mk_errv};
+use crate::error::{OrcErrv, OrcRes, Reporter, mk_errv};
-use crate::format::{Format, take_first_fmt};
+use crate::format::{FmtCtx, FmtUnit, Format, fmt};
 use crate::interner::{Interner, Tok};
-use crate::location::Pos;
+use crate::location::SrcRange;
-use crate::name::{VName, VPath};
+use crate::name::{Sym, VName, VPath};
-use crate::tree::{AtomRepr, ExtraTok, Paren, TokTree, Token};
+use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_fmt, ttv_range};
+
+pub trait ParseCtx {
+	#[must_use]
+	fn i(&self) -> &Interner;
+	#[must_use]
+	fn rep(&self) -> &Reporter;
+}
+pub struct ParseCtxImpl<'a> {
+	pub i: &'a Interner,
+	pub r: &'a Reporter,
+}
+impl ParseCtx for ParseCtxImpl<'_> {
+	fn i(&self) -> &Interner { self.i }
+	fn rep(&self) -> &Reporter { self.r }
+}
 
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
 pub fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
@@ -21,51 +37,41 @@ pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
 /// A cheaply copiable subsection of a document that holds onto context data and
 /// one token for error reporting on empty subsections.
 #[derive(Debug)]
-pub struct Snippet<'a, 'b, A: AtomRepr, X: ExtraTok> {
+pub struct Snippet<'a, A: ExprRepr, X: ExtraTok> {
-	prev: &'a TokTree<'b, A, X>,
+	prev: &'a TokTree<A, X>,
-	cur: &'a [TokTree<'b, A, X>],
+	cur: &'a [TokTree<A, X>],
-	interner: &'a Interner,
 }
-impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
+impl<'a, A, X> Snippet<'a, A, X>
-	pub fn new(
+where
-		prev: &'a TokTree<'b, A, X>,
+	A: ExprRepr,
-		cur: &'a [TokTree<'b, A, X>],
+	X: ExtraTok,
-		interner: &'a Interner,
+{
-	) -> Self {
+	pub fn new(prev: &'a TokTree<A, X>, cur: &'a [TokTree<A, X>]) -> Self { Self { prev, cur } }
-		Self { prev, cur, interner }
-	}
-	pub fn i(&self) -> &'a Interner { self.interner }
 	pub fn split_at(self, pos: u32) -> (Self, Self) {
-		let Self { prev, cur, interner } = self;
+		let Self { prev, cur } = self;
-		let fst = Self { prev, cur: &cur[..pos as usize], interner };
+		let fst = Self { prev, cur: &cur[..pos as usize] };
 		let new_prev = if pos == 0 { self.prev } else { &self.cur[pos as usize - 1] };
-		let snd = Self { prev: new_prev, cur: &self.cur[pos as usize..], interner };
+		let snd = Self { prev: new_prev, cur: &self.cur[pos as usize..] };
 		(fst, snd)
 	}
-	pub fn find_idx(self, mut f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<u32> {
+	pub fn find_idx(self, mut f: impl FnMut(&Token<A, X>) -> bool) -> Option<u32> {
 		self.cur.iter().position(|t| f(&t.tok)).map(|t| t as u32)
 	}
-	pub fn get(self, idx: u32) -> Option<&'a TokTree<'b, A, X>> { self.cur.get(idx as usize) }
+	pub fn get(self, idx: u32) -> Option<&'a TokTree<A, X>> { self.cur.get(idx as usize) }
 	pub fn len(self) -> u32 { self.cur.len() as u32 }
-	pub fn prev(self) -> &'a TokTree<'b, A, X> { self.prev }
+	pub fn prev(self) -> &'a TokTree<A, X> { self.prev }
-	pub fn pos(self) -> Range<u32> {
+	pub fn sr(self) -> SrcRange { ttv_range(self.cur).unwrap_or_else(|| self.prev.sr.clone()) }
-		(self.cur.first().map(|f| f.range.start..self.cur.last().unwrap().range.end))
+	pub fn pop_front(self) -> Option<(&'a TokTree<A, X>, Self)> {
-			.unwrap_or(self.prev.range.clone())
-	}
-	pub fn pop_front(self) -> Option<(&'a TokTree<'b, A, X>, Self)> {
 		self.cur.first().map(|r| (r, self.split_at(1).1))
 	}
-	pub fn pop_back(self) -> Option<(Self, &'a TokTree<'b, A, X>)> {
+	pub fn pop_back(self) -> Option<(Self, &'a TokTree<A, X>)> {
 		self.cur.last().map(|r| (self.split_at(self.len() - 1).0, r))
 	}
-	pub fn split_once(self, f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<(Self, Self)> {
+	pub fn split_once(self, f: impl FnMut(&Token<A, X>) -> bool) -> Option<(Self, Self)> {
 		let idx = self.find_idx(f)?;
 		Some((self.split_at(idx).0, self.split_at(idx + 1).1))
 	}
-	pub fn split(
+	pub fn split(mut self, mut f: impl FnMut(&Token<A, X>) -> bool) -> impl Iterator<Item = Self> {
-		mut self,
-		mut f: impl FnMut(&Token<'b, A, X>) -> bool,
-	) -> impl Iterator<Item = Self> {
 		iter::from_fn(move || {
 			if self.is_empty() {
 				return None;
@@ -77,47 +83,45 @@ impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
 	}
 	pub fn is_empty(self) -> bool { self.len() == 0 }
 	pub fn skip_fluff(self) -> Self {
-		let non_fluff_start = self.find_idx(|t| !matches!(t, Token::NS | Token::Comment(_)));
+		let non_fluff_start = self.find_idx(|t| !matches!(t, Token::BR | Token::Comment(_)));
 		self.split_at(non_fluff_start.unwrap_or(self.len())).1
 	}
-	/// Format the argument using the context held in this snippet
-	pub async fn fmt(self, v: &(impl Format + ?Sized)) -> String { take_first_fmt(v, self.i()).await }
 }
-impl<A: AtomRepr, X: ExtraTok> Copy for Snippet<'_, '_, A, X> {}
+impl<A: ExprRepr, X: ExtraTok> Copy for Snippet<'_, A, X> {}
-impl<A: AtomRepr, X: ExtraTok> Clone for Snippet<'_, '_, A, X> {
+impl<A: ExprRepr, X: ExtraTok> Clone for Snippet<'_, A, X> {
 	fn clone(&self) -> Self { *self }
 }
-impl<'b, A: AtomRepr, X: ExtraTok> Deref for Snippet<'_, 'b, A, X> {
+impl<A: ExprRepr, X: ExtraTok> Deref for Snippet<'_, A, X> {
-	type Target = [TokTree<'b, A, X>];
+	type Target = [TokTree<A, X>];
 	fn deref(&self) -> &Self::Target { self.cur }
 }
-
+impl<A: ExprRepr, X: ExtraTok> Format for Snippet<'_, A, X> {
-/// Remove tokens that aren't meaningful in expression context, such as comments
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-/// or line breaks
+		ttv_fmt(&**self, c).await
-pub fn strip_fluff<'a, A: AtomRepr, X: ExtraTok>(
+	}
-	tt: &TokTree<'a, A, X>,
-) -> Option<TokTree<'a, A, X>> {
-	let tok = match &tt.tok {
-		Token::BR => return None,
-		Token::Comment(_) => return None,
-		Token::LambdaHead(arg) => Token::LambdaHead(arg.iter().filter_map(strip_fluff).collect()),
-		Token::S(p, b) => Token::S(*p, b.iter().filter_map(strip_fluff).collect()),
-		t => t.clone(),
-	};
-	Some(TokTree { tok, range: tt.range.clone() })
 }
 
 #[derive(Clone, Debug)]
 pub struct Comment {
 	pub text: Tok<String>,
-	pub pos: Pos,
+	pub sr: SrcRange,
 }
 impl Comment {
-	pub fn to_api(&self) -> api::Comment {
+	// XXX: which of these four are actually used?
-		api::Comment { location: self.pos.to_api(), text: self.text.to_api() }
+	pub async fn from_api(c: &api::Comment, src: Sym, i: &Interner) -> Self {
+		Self { text: i.ex(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
 	}
-	pub async fn from_api(api: &api::Comment, i: &Interner) -> Self {
+	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>, i: &Interner) -> Option<Self> {
-		Self { pos: Pos::from_api(&api.location, i).await, text: Tok::from_api(api.text, i).await }
+		match &tk.tok {
+			Token::Comment(text) => Some(Self { text: i.i(&**text).await, sr: tk.sr.clone() }),
+			_ => None,
+		}
+	}
+	pub fn to_tk<R: ExprRepr, X: ExtraTok>(&self) -> TokTree<R, X> {
+		TokTree { tok: Token::Comment(self.text.rc().clone()), sr: self.sr.clone() }
+	}
+	pub fn to_api(&self) -> api::Comment {
+		api::Comment { range: self.sr.range(), text: self.text.to_api() }
 	}
 }
 
@@ -125,9 +129,10 @@ impl fmt::Display for Comment {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "--[{}]--", self.text) }
 }
 
-pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, 'b, A, X>,
+	ctx: &impl ParseCtx,
-) -> Vec<Parsed<'a, 'b, Vec<Comment>, A, X>> {
+	snip: Snippet<'a, A, X>,
+) -> Vec<Parsed<'a, Vec<Comment>, A, X>> {
 	let mut items = Vec::new();
 	let mut comments = Vec::new();
 	for mut line in snip.split(|t| matches!(t, Token::BR)) {
@@ -141,11 +146,7 @@ pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
 			Some(i) => {
 				let (cmts, tail) = line.split_at(i);
 				let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async {
-					match &t.tok {
+					Comment::from_tk(t, ctx.i()).await.expect("All are comments checked above")
-						Token::Comment(c) =>
-							Comment { text: tail.i().i(&**c).await, pos: Pos::Range(t.range.clone()) },
-						_ => unreachable!("All are comments checked above"),
-					}
 				}))
 				.await;
 				items.push(Parsed { output: comments, tail });
@@ -155,166 +156,151 @@ pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
 	items
 }
 
-pub async fn try_pop_no_fluff<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, 'b, A, X>,
+	ctx: &impl ParseCtx,
-) -> ParseRes<'a, 'b, &'a TokTree<'b, A, X>, A, X> {
+	snip: Snippet<'a, A, X>,
+) -> ParseRes<'a, &'a TokTree<A, X>, A, X> {
 	match snip.skip_fluff().pop_front() {
 		Some((output, tail)) => Ok(Parsed { output, tail }),
-		None =>
+		None => Err(mk_errv(
-			Err(mk_errv(snip.i().i("Unexpected end").await, "Pattern ends abruptly", [Pos::Range(
+			ctx.i().i("Unexpected end").await,
-				snip.pos(),
+			"Line ends abruptly; more tokens were expected",
-			)
+			[snip.sr()],
-			.into()])),
+		)),
 	}
 }
 
-pub async fn expect_end(snip: Snippet<'_, '_, impl AtomRepr, impl ExtraTok>) -> OrcRes<()> {
+pub async fn expect_end(
+	ctx: &impl ParseCtx,
+	snip: Snippet<'_, impl ExprRepr, impl ExtraTok>,
+) -> OrcRes<()> {
 	match snip.skip_fluff().get(0) {
 		Some(surplus) => Err(mk_errv(
-			snip.i().i("Extra code after end of line").await,
+			ctx.i().i("Extra code after end of line").await,
 			"Code found after the end of the line",
-			[Pos::Range(surplus.range.clone()).into()],
+			[surplus.sr.pos()],
 		)),
 		None => Ok(()),
 	}
 }
 
-pub async fn expect_tok<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, 'b, A, X>,
+	ctx: &impl ParseCtx,
+	snip: Snippet<'a, A, X>,
 	tok: Tok<String>,
-) -> ParseRes<'a, 'b, (), A, X> {
+) -> ParseRes<'a, (), A, X> {
-	let Parsed { output: head, tail } = try_pop_no_fluff(snip).await?;
+	let Parsed { output: head, tail } = try_pop_no_fluff(ctx, snip).await?;
 	match &head.tok {
 		Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
 		t => Err(mk_errv(
-			snip.i().i("Expected specific keyword").await,
+			ctx.i().i("Expected specific keyword").await,
-			format!("Expected {tok} but found {:?}", snip.fmt(t).await),
+			format!("Expected {tok} but found {:?}", fmt(t, ctx.i()).await),
-			[Pos::Range(head.range.clone()).into()],
+			[head.sr()],
 		)),
 	}
 }
 
-pub struct Parsed<'a, 'b, T, A: AtomRepr, X: ExtraTok> {
+pub async fn token_errv<A: ExprRepr, X: ExtraTok>(
+	ctx: &impl ParseCtx,
+	tok: &TokTree<A, X>,
+	description: &'static str,
+	message: impl FnOnce(&str) -> String,
+) -> OrcErrv {
+	mk_errv(ctx.i().i(description).await, message(&fmt(tok, ctx.i()).await), [tok.sr.pos()])
+}
+
+pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
 	pub output: T,
-	pub tail: Snippet<'a, 'b, A, X>,
+	pub tail: Snippet<'a, H, X>,
 }
 
-pub type ParseRes<'a, 'b, T, A, X> = OrcRes<Parsed<'a, 'b, T, A, X>>;
+pub type ParseRes<'a, T, H, X> = OrcRes<Parsed<'a, T, H, X>>;
 
-pub async fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &(impl Reporter + ?Sized),
+	ctx: &impl ParseCtx,
-	tail: Snippet<'a, 'b, A, X>,
+	tail: Snippet<'a, A, X>,
-) -> ParseRes<'a, 'b, Vec<(Import, Pos)>, A, X> {
+) -> ParseRes<'a, Vec<Import>, A, X> {
-	let ret = rec(ctx, tail).await;
+	let Some((tt, tail)) = tail.skip_fluff().pop_front() else {
+		return Err(mk_errv(
+			ctx.i().i("Expected token").await,
+			"Expected a name, a parenthesized list of names, or a globstar.",
+			[tail.sr().pos()],
+		));
+	};
+	let ret = rec(tt, ctx).await;
 	#[allow(clippy::type_complexity)] // it's an internal function
-	pub async fn rec<'a, 'b, A: AtomRepr, X: ExtraTok>(
+	pub async fn rec<A: ExprRepr, X: ExtraTok>(
-		ctx: &(impl Reporter + ?Sized),
+		tt: &TokTree<A, X>,
-		tail: Snippet<'a, 'b, A, X>,
+		ctx: &impl ParseCtx,
-	) -> ParseRes<'a, 'b, Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>, A, X> {
+	) -> OrcRes<Vec<(Vec<Tok<String>>, Option<Tok<String>>, SrcRange)>> {
-		let comma = tail.i().i(",").await;
+		let ttpos = tt.sr.pos();
-		let globstar = tail.i().i("*").await;
-		let Some((name, tail)) = tail.skip_fluff().pop_front() else {
-			return Err(mk_errv(
-				tail.i().i("Expected name").await,
-				"Expected a name, a list of names, or a globstar.",
-				[Pos::Range(tail.pos()).into()],
-			));
-		};
-		if let Some((Token::NS, tail)) = tail.skip_fluff().pop_front().map(|(tt, s)| (&tt.tok, s)) {
-			let n = match &name.tok {
-				Token::Name(n) if n.starts_with(name_start) => Ok(n),
-				_ =>
-					Err(mk_err(tail.i().i("Unexpected name prefix").await, "Only names can precede ::", [
-						Pos::Range(name.range.clone()).into(),
-					])),
-			};
-			match (Box::pin(rec(ctx, tail)).await, n) {
-				(Err(ev), n) => Err(ev.extended(n.err())),
-				(Ok(Parsed { tail, .. }), Err(e)) => {
-					ctx.report(e);
-					Ok(Parsed { output: vec![], tail })
-				},
-				(Ok(Parsed { tail, output }), Ok(pre)) => Ok(Parsed {
-					output: output.into_iter().update(|i| i.0.push(pre.clone())).collect_vec(),
-					tail,
-				}),
-			}
-		} else {
-			let output = match &name.tok {
-				Token::Name(ntok) => {
-					let nopt = match ntok {
-						n if *n == globstar => None,
-						n if n.starts_with(op_char) => {
-							return Err(mk_errv(
-								tail.i().i("Unescaped operator in multiname").await,
-								"Operators in multinames should be enclosed in []",
-								[Pos::Range(name.range.clone()).into()],
-							));
-						},
-						n => Some(n.clone()),
-					};
-					vec![(vec![], nopt, Pos::Range(name.range.clone()))]
-				},
-				Token::S(Paren::Square, b) => {
-					let mut ok = Vec::new();
-					for tt in b.iter() {
 		match &tt.tok {
-							Token::Name(n) if n.starts_with(op_char) =>
+			Token::NS(ns, body) => {
-								ok.push((vec![], Some(n.clone()), Pos::Range(tt.range.clone()))),
+				if !ns.starts_with(name_start) {
-							Token::BR | Token::Comment(_) => (),
+					ctx.rep().report(mk_errv(
-							_ => ctx.report(mk_err(
+						ctx.i().i("Unexpected name prefix").await,
-								tail.i().i("Non-operator in escapement in multiname").await,
+						"Only names can precede ::",
-								"In multinames, [] functions as a literal name list reserved for operators",
+						[ttpos],
-								[Pos::Range(name.range.clone()).into()],
+					))
-							)),
+				};
-						}
+				let out = Box::pin(rec(body, ctx)).await?;
-					}
+				Ok(out.into_iter().update(|i| i.0.push(ns.clone())).collect_vec())
-					ok
+			},
+			Token::Name(ntok) => {
+				let n = ntok;
+				let nopt = Some(n.clone());
+				Ok(vec![(vec![], nopt, tt.sr.clone())])
 			},
 			Token::S(Paren::Round, b) => {
-					let mut ok = Vec::new();
+				let mut o = Vec::new();
-					let body = Snippet::new(name, b, tail.interner);
+				let mut body = Snippet::new(tt, b);
-					for csent in body.split(|n| matches!(n, Token::Name(n) if *n == comma)) {
+				while let Some((output, tail)) = body.pop_front() {
-						match Box::pin(rec(ctx, csent)).await {
+					match rec(output, ctx).boxed_local().await {
-							Err(e) => ctx.report(e),
+						Ok(names) => o.extend(names),
-							Ok(Parsed { output, tail }) => match tail.get(0) {
+						Err(e) => ctx.rep().report(e),
-								None => ok.extend(output),
-								Some(t) => ctx.report(mk_err(
-									tail.i().i("Unexpected token in multiname group").await,
-									"Unexpected token. Likely missing a :: or , or wanted [] instead of ()",
-									[Pos::Range(t.range.clone()).into()],
-								)),
-							},
 					}
+					body = tail;
 				}
-					ok
+				Ok(o)
 			},
 			t => {
 				return Err(mk_errv(
-						tail.i().i("Unrecognized name end").await,
+					ctx.i().i("Unrecognized name end").await,
-						format!("Names cannot end with {:?} tokens", tail.fmt(t).await),
+					format!("Names cannot end with {:?} tokens", fmt(t, ctx.i()).await),
-						[Pos::Range(name.range.clone()).into()],
+					[ttpos],
 				));
 			},
-			};
-			Ok(Parsed { output, tail })
 		}
 	}
-	ret.map(|Parsed { output, tail }| {
+	ret.map(|output| {
 		let output = (output.into_iter())
-			.map(|(p, name, pos)| (Import { path: VPath::new(p.into_iter().rev()), name }, pos))
+			.map(|(p, name, sr)| Import { path: VPath::new(p.into_iter().rev()), name, sr })
 			.collect_vec();
 		Parsed { output, tail }
 	})
 }
 
-/// A compound name, possibly ending with a globstar
+/// A compound name, possibly ending with a globstar. It cannot be just a
+/// globstar; either the name has to be known or the path has to be non-empty.
 #[derive(Debug, Clone)]
 pub struct Import {
 	pub path: VPath,
 	pub name: Option<Tok<String>>,
+	pub sr: SrcRange,
+}
+impl Import {
+	/// Most specific concrete path
+	pub fn mspath(self) -> VName {
+		match self.name {
+			Some(n) => self.path.name_with_suffix(n),
+			None => self.path.into_name().expect("Import cannot be empty"),
+		}
+	}
+	pub fn new(sr: SrcRange, path: VPath, name: Tok<String>) -> Self {
+		Import { path, name: Some(name), sr }
+	}
+	pub fn new_glob(sr: SrcRange, path: VPath) -> Self { Import { path, name: None, sr } }
 }
-
 impl Display for Import {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| t.as_str()))
@@ -327,14 +313,5 @@ mod test {
 
 	use super::Snippet;
 
-	fn _covary_snip_a<'a, 'b>(
+	fn _covary_snip_a<'a>(x: Snippet<'static, Never, Never>) -> Snippet<'a, Never, Never> { x }
-		x: Snippet<'static, 'b, Never, Never>,
-	) -> Snippet<'a, 'b, Never, Never> {
-		x
-	}
-	fn _covary_snip_b<'a, 'b>(
-		x: Snippet<'a, 'static, Never, Never>,
-	) -> Snippet<'a, 'b, Never, Never> {
-		x
-	}
 }

orchid-base/src/reqnot.rs

@@ -1,4 +1,3 @@
-use std::any::{Any, TypeId};
 use std::cell::RefCell;
 use std::future::Future;
 use std::marker::PhantomData;
@@ -8,17 +7,18 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 
-use async_std::channel;
-use async_std::sync::Mutex;
 use derive_destructure::destructure;
 use dyn_clone::{DynClone, clone_box};
+use futures::channel::mpsc;
 use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
+use futures::{SinkExt, StreamExt};
 use hashbrown::HashMap;
 use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
 use trait_set::trait_set;
 
+use crate::clone;
 use crate::logging::Logger;
-use crate::{api, clone};
 
 pub struct Receipt<'a>(PhantomData<&'a mut ()>);
 
@@ -40,16 +40,19 @@ fn get_id(message: &[u8]) -> (u64, &[u8]) {
 }
 
 pub trait ReqHandlish {
-	fn defer_drop(&self, val: impl Any + 'static)
+	fn defer(&self, cb: impl Future<Output = ()> + 'static)
 	where Self: Sized {
-		self.defer_drop_objsafe(Box::new(val));
+		self.defer_objsafe(Box::pin(cb));
 	}
-	fn defer_drop_objsafe(&self, val: Box<dyn Any>);
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>);
+}
+impl ReqHandlish for &'_ dyn ReqHandlish {
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>) { (**self).defer_objsafe(val) }
 }
 
 #[derive(destructure)]
 pub struct RequestHandle<'a, MS: MsgSet> {
-	defer_drop: RefCell<Vec<Box<dyn Any>>>,
+	defer: RefCell<Vec<Pin<Box<dyn Future<Output = ()>>>>>,
 	fulfilled: AtomicBool,
 	id: u64,
 	_reqlt: PhantomData<&'a mut ()>,
@@ -57,13 +60,7 @@ pub struct RequestHandle<'a, MS: MsgSet> {
 }
 impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
 	fn new(parent: ReqNot<MS>, id: u64) -> Self {
-		Self {
+		Self { defer: RefCell::default(), fulfilled: false.into(), _reqlt: PhantomData, parent, id }
-			defer_drop: RefCell::default(),
-			fulfilled: false.into(),
-			_reqlt: PhantomData,
-			parent,
-			id,
-		}
 	}
 	pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
 	pub async fn handle<U: Request>(&self, _: &U, rep: &U::Response) -> Receipt<'a> {
@@ -79,11 +76,17 @@ impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
 		response.encode(Pin::new(&mut buf)).await;
 		let mut send = clone_box(&*self.reqnot().0.lock().await.send);
 		(send)(&buf, self.parent.clone()).await;
+		let deferred = mem::take(&mut *self.defer.borrow_mut());
+		for item in deferred {
+			item.await
+		}
 		Receipt(PhantomData)
 	}
 }
 impl<MS: MsgSet> ReqHandlish for RequestHandle<'_, MS> {
-	fn defer_drop_objsafe(&self, val: Box<dyn Any>) { self.defer_drop.borrow_mut().push(val); }
+	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>) {
+		self.defer.borrow_mut().push(val)
+	}
 }
 impl<MS: MsgSet> Drop for RequestHandle<'_, MS> {
 	fn drop(&mut self) {
@@ -99,7 +102,7 @@ pub struct ReqNotData<T: MsgSet> {
 	send: Box<dyn SendFn<T>>,
 	notif: Box<dyn NotifFn<T>>,
 	req: Box<dyn ReqFn<T>>,
-	responses: HashMap<u64, channel::Sender<Vec<u8>>>,
+	responses: HashMap<u64, mpsc::Sender<Vec<u8>>>,
 }
 
 /// Wraps a raw message buffer to save on copying.
@@ -141,7 +144,7 @@ impl<T: MsgSet> ReqNot<T> {
 			let notif_val = <T::In as Channel>::Notif::decode(Pin::new(&mut &payload[..])).await;
 			notif_cb(notif_val, self.clone()).await
 		} else if 0 < id.bitand(1 << 63) {
-			let sender = g.responses.remove(&!id).expect("Received response for invalid message");
+			let mut sender = g.responses.remove(&!id).expect("Received response for invalid message");
 			sender.send(message.to_vec()).await.unwrap()
 		} else {
 			let message = <T::In as Channel>::Req::decode(Pin::new(&mut &payload[..])).await;
@@ -170,12 +173,16 @@ pub trait DynRequester {
 
 pub struct MappedRequester<'a, T: 'a>(Box<dyn Fn(T) -> LocalBoxFuture<'a, RawReply> + 'a>, Logger);
 impl<'a, T> MappedRequester<'a, T> {
-	fn new<U: DynRequester + 'a>(req: U, logger: Logger) -> Self
+	fn new<U: DynRequester + 'a, F: Fn(T) -> U::Transfer + 'a>(
-	where T: Into<U::Transfer> {
+		req: U,
+		cb: F,
+		logger: Logger,
+	) -> Self {
 		let req_arc = Arc::new(req);
+		let cb_arc = Arc::new(cb);
 		MappedRequester(
 			Box::new(move |t| {
-				Box::pin(clone!(req_arc; async move { req_arc.raw_request(t.into()).await}))
+				Box::pin(clone!(req_arc, cb_arc; async move { req_arc.raw_request(cb_arc(t)).await}))
 			}),
 			logger,
 		)
@@ -198,13 +205,13 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
 			g.id += 1;
 			let mut buf = id.to_be_bytes().to_vec();
 			req.encode(Pin::new(&mut buf)).await;
-			let (send, recv) = channel::bounded(1);
+			let (send, mut recv) = mpsc::channel(1);
 			g.responses.insert(id, send);
 			let mut send = clone_box(&*g.send);
 			mem::drop(g);
 			let rn = self.clone();
 			send(&buf, rn).await;
-			let items = recv.recv().await;
+			let items = recv.next().await;
 			RawReply(items.unwrap())
 		})
 	}
@@ -217,10 +224,10 @@ pub trait Requester: DynRequester {
 		&self,
 		data: R,
 	) -> impl Future<Output = R::Response>;
-	fn map<'a, U: Into<Self::Transfer>>(self) -> MappedRequester<'a, U>
+	fn map<'a, U>(self, cb: impl Fn(U) -> Self::Transfer + 'a) -> MappedRequester<'a, U>
 	where Self: Sized + 'a {
 		let logger = self.logger().clone();
-		MappedRequester::new(self, logger)
+		MappedRequester::new(self, cb, logger)
 	}
 }
 
@@ -228,7 +235,10 @@ impl<This: DynRequester + ?Sized> Requester for This {
 	async fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
 		let req = format!("{data:?}");
 		let rep = R::Response::decode(Pin::new(&mut &self.raw_request(data.into()).await[..])).await;
+		let req_str = req.to_string();
+		if !req_str.starts_with("AtomPrint") && !req_str.starts_with("ExtAtomPrint") {
 			writeln!(self.logger(), "Request {req} got response {rep:?}");
+		}
 		rep
 	}
 }
@@ -242,17 +252,16 @@ mod test {
 	use std::rc::Rc;
 	use std::sync::Arc;
 
-	use async_std::sync::Mutex;
 	use futures::FutureExt;
-	use orchid_api::LogStrategy;
+	use futures::lock::Mutex;
 	use orchid_api_derive::Coding;
 	use orchid_api_traits::{Channel, Request};
 	use test_executors::spin_on;
 
 	use super::{MsgSet, ReqNot};
-	use crate::clone;
 	use crate::logging::Logger;
 	use crate::reqnot::Requester as _;
+	use crate::{api, clone};
 
 	#[derive(Clone, Debug, Coding, PartialEq)]
 	pub struct TestReq(u8);
@@ -275,7 +284,7 @@ mod test {
 	#[test]
 	fn notification() {
 		spin_on(async {
-			let logger = Logger::new(LogStrategy::StdErr);
+			let logger = Logger::new(api::LogStrategy::StdErr);
 			let received = Arc::new(Mutex::new(None));
 			let receiver = ReqNot::<TestMsgSet>::new(
 				logger.clone(),
@@ -303,7 +312,7 @@ mod test {
 	#[test]
 	fn request() {
 		spin_on(async {
-			let logger = Logger::new(LogStrategy::StdErr);
+			let logger = Logger::new(api::LogStrategy::StdErr);
 			let receiver = Rc::new(Mutex::<Option<ReqNot<TestMsgSet>>>::new(None));
 			let sender = Rc::new(ReqNot::<TestMsgSet>::new(
 				logger.clone(),

orchid-base/src/tree.rs

@@ -2,54 +2,78 @@ use std::borrow::Borrow;
 use std::fmt::{self, Debug, Display};
 use std::future::Future;
 use std::marker::PhantomData;
-use std::ops::Range;
 use std::rc::Rc;
-use std::sync::Arc;
 
-pub use api::PhKind;
+use async_fn_stream::stream;
-use async_stream::stream;
+use futures::future::join_all;
-use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
 use never::Never;
-use ordered_float::NotNan;
+use orchid_api_traits::Coding;
 use trait_set::trait_set;
 
 use crate::error::OrcErrv;
 use crate::format::{FmtCtx, FmtUnit, Format, Variants};
 use crate::interner::{Interner, Tok};
-use crate::location::Pos;
+use crate::location::{Pos, SrcRange};
-use crate::name::Sym;
+use crate::name::{Sym, VName, VPath};
 use crate::parse::Snippet;
 use crate::{api, match_mapping, tl_cache};
 
-trait_set! {
+pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::Debug {
-	pub trait RecurCB<'a, A: AtomRepr, X: ExtraTok> = Fn(TokTree<'a, A, X>) -> TokTree<'a, A, X>;
+	type FromApiCtx<'a>;
-	pub trait ExtraTok = Format + Clone + fmt::Debug;
+	type ToApiCtx<'a>;
-	pub trait RefDoExtra<X> =
+	#[must_use]
-		for<'b> FnMut(&'b X, Range<u32>) -> LocalBoxFuture<'b, api::TokenTree>;
+	fn from_api(
+		api: &ApiEquiv,
+		ctx: &mut Self::FromApiCtx<'_>,
+		pos: SrcRange,
+		i: &Interner,
+	) -> impl Future<Output = Self>;
+	#[must_use]
+	fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> impl Future<Output = ApiEquiv>;
+}
+impl<T: Clone + Debug + Coding> TokenVariant<T> for Never {
+	type FromApiCtx<'a> = ();
+	type ToApiCtx<'a> = ();
+	async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: SrcRange, _: &Interner) -> Self {
+		panic!("Cannot deserialize Never")
+	}
+	async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> T { match self {} }
 }
 
-pub fn recur<'a, A: AtomRepr, X: ExtraTok>(
+trait_set! {
-	tt: TokTree<'a, A, X>,
+	// TokenHandle
-	f: &impl Fn(TokTree<'a, A, X>, &dyn RecurCB<'a, A, X>) -> TokTree<'a, A, X>,
+	pub trait ExprRepr = TokenVariant<api::ExprTicket>;
-) -> TokTree<'a, A, X> {
+	// TokenExpr
-	f(tt, &|TokTree { range, tok }| {
+	pub trait ExtraTok = TokenVariant<api::Expression>;
+}
+
+trait_set! {
+	pub trait RecurCB<H: ExprRepr, X: ExtraTok> = Fn(TokTree<H, X>) -> TokTree<H, X>;
+}
+
+pub fn recur<H: ExprRepr, X: ExtraTok>(
+	tt: TokTree<H, X>,
+	f: &impl Fn(TokTree<H, X>, &dyn RecurCB<H, X>) -> TokTree<H, X>,
+) -> TokTree<H, X> {
+	f(tt, &|TokTree { sr: range, tok }| {
 		let tok = match tok {
-			tok @ (Token::Atom(_) | Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::NS) => tok,
+			tok @ (Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::Name(_)) => tok,
-			tok @ (Token::Name(_) | Token::Slot(_) | Token::X(_) | Token::Ph(_) | Token::Macro(_)) => tok,
+			tok @ (Token::Handle(_) | Token::NewExpr(_)) => tok,
-			tok @ Token::Reference(_) => tok,
+			Token::NS(n, b) => Token::NS(n, Box::new(recur(*b, f))),
-			Token::LambdaHead(arg) =>
+			Token::LambdaHead(arg) => Token::LambdaHead(Box::new(recur(*arg, f))),
-				Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
 			Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
 		};
-		TokTree { range, tok }
+		TokTree { sr: range, tok }
 	})
 }
 
 pub trait AtomRepr: Clone + Format {
 	type Ctx: ?Sized;
+	#[must_use]
 	fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> impl Future<Output = Self>;
+	#[must_use]
 	fn to_api(&self) -> impl Future<Output = orchid_api::Atom> + '_;
 }
 impl AtomRepr for Never {
@@ -71,139 +95,139 @@ impl Display for TokHandle<'_> {
 }
 
 #[derive(Clone, Debug)]
-pub struct TokTree<'a, A: AtomRepr, X: ExtraTok> {
+pub struct TokTree<H: ExprRepr, X: ExtraTok> {
-	pub tok: Token<'a, A, X>,
+	pub tok: Token<H, X>,
-	pub range: Range<u32>,
+	/// The protocol has a Range<u32> because these are always transmitted in the
+	/// context of a given snippet, but internal logic and error reporting is
+	/// easier if the in-memory representation also includes the snippet path.
+	pub sr: SrcRange,
 }
-impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
+impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
-	pub async fn from_api(tt: &api::TokenTree, ctx: &mut A::Ctx, i: &Interner) -> Self {
+	pub async fn from_api(
-		let tok = match_mapping!(&tt.token, api::Token => Token::<'b, A, X> {
+		tt: &api::TokenTree,
-			BR, NS,
+		hctx: &mut H::FromApiCtx<'_>,
-			Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx).await),
+		xctx: &mut X::FromApiCtx<'_>,
-			Bottom(e => OrcErrv::from_api(e, i).await),
+		src: &Sym,
-			LambdaHead(arg => ttv_from_api(arg, ctx, i).await),
+		i: &Interner,
-			Name(n => Tok::from_api(*n, i).await),
+	) -> Self {
-			S(*par, b => ttv_from_api(b, ctx, i).await),
+		let pos = SrcRange::new(tt.range.clone(), src);
-			Comment(c.clone()),
+		let tok = match_mapping!(&tt.token, api::Token => Token::<H, X> {
-			Slot(id => TokHandle::new(*id)),
-			Ph(ph => Ph::from_api(ph, i).await),
-			Macro(*prio),
-			Reference(tok => Sym::from_api(*tok, i).await)
-		});
-		Self { range: tt.range.clone(), tok }
-	}
-
-	pub async fn to_api(&self, do_extra: &mut impl RefDoExtra<X>) -> api::TokenTree {
-		let token = match_mapping!(&self.tok, Token => api::Token {
-			Atom(a.to_api().await),
 			BR,
-			NS,
+			NS(n => Tok::from_api(*n, i).await,
-			Bottom(e.to_api()),
+				b => Box::new(Self::from_api(b, hctx, xctx, src, i).boxed_local().await)),
+			Bottom(e => OrcErrv::from_api(e, i).await),
+			LambdaHead(arg => Box::new(Self::from_api(arg, hctx, xctx, src, i).boxed_local().await)),
+			Name(n => Tok::from_api(*n, i).await),
+			S(*par, b => ttv_from_api(b, hctx, xctx, src, i).await),
 			Comment(c.clone()),
-			LambdaHead(arg => ttv_to_api(arg, do_extra).boxed_local().await),
+			NewExpr(expr => X::from_api(expr, xctx, pos.clone(), i).await),
-			Name(n.to_api()),
+			Handle(tk => H::from_api(tk, hctx, pos.clone(), i).await)
-			Slot(tt.ticket()),
-			S(*p, b => ttv_to_api(b, do_extra).boxed_local().await),
-			Ph(ph.to_api()),
-			Macro(*prio),
-			Reference(sym.to_api()),
-		} {
-			Token::X(x) => return do_extra(x, self.range.clone()).await
 		});
-		api::TokenTree { range: self.range.clone(), token }
+		Self { sr: pos, tok }
 	}
 
 	pub async fn into_api(
 		self,
-		do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
+		hctx: &mut H::ToApiCtx<'_>,
+		xctx: &mut X::ToApiCtx<'_>,
 	) -> api::TokenTree {
-		let token = match self.tok {
+		let token = match_mapping!(self.tok, Token => api::Token {
-			Token::Atom(a) => api::Token::Atom(a.to_api().await),
+			BR,
-			Token::Reference(sym) => api::Token::Reference(sym.to_api()),
+			NS(n.to_api(), b => Box::new(b.into_api(hctx, xctx).boxed_local().await)),
-			Token::BR => api::Token::BR,
+			Bottom(e.to_api()),
-			Token::NS => api::Token::NS,
+			Comment(c.clone()),
-			Token::Bottom(e) => api::Token::Bottom(e.to_api()),
+			LambdaHead(arg => Box::new(arg.into_api(hctx, xctx).boxed_local().await)),
-			Token::Comment(c) => api::Token::Comment(c.clone()),
+			Name(nn.to_api()),
-			Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra).await),
+			S(p, b => ttv_into_api(b, hctx, xctx).boxed_local().await),
-			Token::Name(n) => api::Token::Name(n.to_api()),
+			Handle(hand.into_api(hctx).await),
-			Token::Slot(tt) => api::Token::Slot(tt.ticket()),
+			NewExpr(expr.into_api(xctx).await),
-			Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra).await),
+		});
-			Token::Ph(Ph { kind, name }) =>
+		api::TokenTree { range: self.sr.range.clone(), token }
-				api::Token::Ph(api::Placeholder { name: name.to_api(), kind }),
-			Token::X(x) => return do_extra(x, self.range.clone()),
-			Token::Macro(prio) => api::Token::Macro(prio),
-		};
-		api::TokenTree { range: self.range.clone(), token }
 	}
 
 	pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) }
 	pub fn as_name(&self) -> Option<Tok<String>> {
 		if let Token::Name(n) = &self.tok { Some(n.clone()) } else { None }
 	}
-	pub fn as_s<'a>(&'a self, par: Paren, i: &'a Interner) -> Option<Snippet<'a, 'b, A, X>> {
+	pub fn as_multiname(&self) -> Result<VName, &TokTree<H, X>> {
-		self.tok.as_s(par).map(|slc| Snippet::new(self, slc, i))
+		let mut segs = VPath::new([]);
+		let mut cur = self;
+		loop {
+			match &cur.tok {
+				Token::Name(last) => return Ok(segs.name_with_suffix(last.clone())),
+				Token::NS(seg, inner) => {
+					segs = segs.suffix([seg.clone()]);
+					cur = inner;
+				},
+				_ => return Err(cur),
 			}
-	pub fn lambda(arg: Vec<Self>, mut body: Vec<Self>) -> Self {
+		}
-		let arg_range = ttv_range(&arg);
+	}
-		let s_range = arg_range.start..body.last().expect("Lambda with empty body!").range.end;
+	pub fn as_s(&self, par: Paren) -> Option<Snippet<'_, H, X>> {
-		body.insert(0, Token::LambdaHead(arg).at(arg_range));
+		self.tok.as_s(par).map(|slc| Snippet::new(self, slc))
+	}
+	pub fn as_lambda(&self) -> Option<&Self> {
+		match &self.tok {
+			Token::LambdaHead(arg) => Some(&**arg),
+			_ => None,
+		}
+	}
+	pub fn is_fluff(&self) -> bool { matches!(self.tok, Token::Comment(_) | Token::BR) }
+	pub fn lambda(arg: Self, mut body: Vec<Self>) -> Self {
+		let arg_range = arg.sr();
+		let mut s_range = arg_range.clone();
+		s_range.range.end = body.last().expect("Lambda with empty body!").sr.range.end;
+		body.insert(0, Token::LambdaHead(Box::new(arg)).at(arg_range));
 		Token::S(Paren::Round, body).at(s_range)
 	}
+	pub fn sr(&self) -> SrcRange { self.sr.clone() }
 }
-impl<A: AtomRepr, X: ExtraTok> Format for TokTree<'_, A, X> {
+impl<H: ExprRepr, X: ExtraTok> Format for TokTree<H, X> {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		self.tok.print(c).await
 	}
 }
 
-pub async fn ttv_from_api<A: AtomRepr, X: ExtraTok>(
+pub async fn ttv_from_api<H: ExprRepr, X: ExtraTok>(
 	tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>,
-	ctx: &mut A::Ctx,
+	hctx: &mut H::FromApiCtx<'_>,
+	xctx: &mut X::FromApiCtx<'_>,
+	src: &Sym,
 	i: &Interner,
-) -> Vec<TokTree<'static, A, X>> {
+) -> Vec<TokTree<H, X>> {
-	stream! {
+	stream(async |mut cx| {
 		for tok in tokv {
-			yield TokTree::<A, X>::from_api(tok.borrow(), ctx, i).boxed_local().await
+			cx.emit(TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src, i).boxed_local().await).await
-		}
 		}
+	})
 	.collect()
 	.await
 }
 
-pub async fn ttv_to_api<'a, A: AtomRepr, X: ExtraTok>(
+pub async fn ttv_into_api<H: ExprRepr, X: ExtraTok>(
-	tokv: impl IntoIterator<Item: Borrow<TokTree<'a, A, X>>>,
+	tokv: impl IntoIterator<Item = TokTree<H, X>>,
-	do_extra: &mut impl RefDoExtra<X>,
+	hctx: &mut H::ToApiCtx<'_>,
+	xctx: &mut X::ToApiCtx<'_>,
 ) -> Vec<api::TokenTree> {
-	let mut output = Vec::new();
+	stream(async |mut cx| {
 		for tok in tokv {
-		output.push(Borrow::<TokTree<A, X>>::borrow(&tok).to_api(do_extra).await)
+			cx.emit(tok.into_api(hctx, xctx).await).await
-	}
-	output
-}
-
-pub async fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>(
-	tokv: impl IntoIterator<Item = TokTree<'a, A, X>>,
-	do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
-) -> Vec<api::TokenTree> {
-	stream! {
-		for tok in tokv {
-			yield tok.into_api(do_extra).await
-		}
 		}
+	})
 	.collect()
 	.await
 }
 
-pub fn wrap_tokv<'a, A: AtomRepr, X: ExtraTok>(
+pub fn wrap_tokv<H: ExprRepr, X: ExtraTok>(
-	items: impl IntoIterator<Item = TokTree<'a, A, X>>,
+	items: impl IntoIterator<Item = TokTree<H, X>>,
-) -> TokTree<'a, A, X> {
+) -> TokTree<H, X> {
 	let items_v = items.into_iter().collect_vec();
 	match items_v.len() {
 		0 => panic!("A tokv with no elements is illegal"),
 		1 => items_v.into_iter().next().unwrap(),
 		_ => {
-			let range = items_v.first().unwrap().range.start..items_v.last().unwrap().range.end;
+			let sr = ttv_range(&items_v).expect("empty handled above");
-			Token::S(api::Paren::Round, items_v).at(range)
+			Token::S(api::Paren::Round, items_v).at(sr)
 		},
 	}
 }
@@ -212,70 +236,54 @@ pub use api::Paren;
 
 /// Lexer output variant
 #[derive(Clone, Debug)]
-pub enum Token<'a, A: AtomRepr, X: ExtraTok> {
+pub enum Token<H: ExprRepr, X: ExtraTok> {
 	/// Information about the code addressed to the human reader or dev tooling
 	/// It has no effect on the behaviour of the program unless it's explicitly
 	/// read via reflection
-	Comment(Arc<String>),
+	Comment(Rc<String>),
 	/// The part of a lambda between `\` and `.` enclosing the argument. The body
 	/// stretches to the end of the enclosing parens or the end of the const line
-	LambdaHead(Vec<TokTree<'a, A, X>>),
+	LambdaHead(Box<TokTree<H, X>>),
 	/// A binding, operator, or a segment of a namespaced::name
 	Name(Tok<String>),
-	/// The namespace separator ::
+	/// A namespace prefix, like `my_ns::` followed by a token
-	NS,
+	NS(Tok<String>, Box<TokTree<H, X>>),
 	/// A line break
 	BR,
 	/// `()`, `[]`, or `{}`
-	S(Paren, Vec<TokTree<'a, A, X>>),
+	S(Paren, Vec<TokTree<H, X>>),
-	/// A fully formed reference to external code emitted by a lexer plugin
+	/// A newly instantiated expression
-	Reference(Sym),
+	NewExpr(X),
-	/// A value emitted by a lexer plugin
+	/// An existing expr from a nested lexer
-	Atom(A),
+	Handle(H),
 	/// A grammar error emitted by a lexer plugin if it was possible to continue
 	/// reading. Parsers should treat it as an atom unless it prevents parsing,
 	/// in which case both this and a relevant error should be returned.
 	Bottom(OrcErrv),
-	/// An instruction from a plugin for the lexer to embed a subexpression
-	/// without retransmitting it. It should not appear anywhere outside lexer
-	/// plugin responses.
-	Slot(TokHandle<'a>),
-	/// Additional domain-specific token types
-	X(X),
-	/// A placeholder for metaprogramming, either $name, ..$name, ..$name:N,
-	/// ...$name, or ...$name:N
-	Ph(Ph),
-	/// `macro` or `macro(`X`)` where X is any valid floating point number
-	/// expression. `macro` is not a valid name in Orchid for this reason.
-	Macro(Option<NotNan<f64>>),
 }
-impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
+impl<H: ExprRepr, X: ExtraTok> Token<H, X> {
-	pub fn at(self, range: Range<u32>) -> TokTree<'a, A, X> { TokTree { range, tok: self } }
+	pub fn at(self, sr: SrcRange) -> TokTree<H, X> { TokTree { sr, tok: self } }
 	pub fn is_kw(&self, tk: Tok<String>) -> bool { matches!(self, Token::Name(n) if *n == tk) }
-	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<'a, A, X>]> {
+	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> {
 		match self {
 			Self::S(p, b) if *p == par => Some(b),
 			_ => None,
 		}
 	}
 }
-impl<A: AtomRepr, X: ExtraTok> Format for Token<'_, A, X> {
+impl<H: ExprRepr, X: ExtraTok> Format for Token<H, X> {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		match self {
-			Self::Atom(a) => a.print(c).await,
 			Self::BR => "\n".to_string().into(),
 			Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()).into(),
 			Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),
 			Self::Comment(c) => format!("--[{c}]--").into(),
 			Self::LambdaHead(arg) =>
-				FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0b}."))), [
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0b}.")))
-					ttv_fmt(arg, c).await,
+					.units([arg.print(c).boxed_local().await]),
-				]),
+			Self::NS(n, b) => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}::{1l}")))
-			Self::NS => "::".to_string().into(),
+				.units([n.to_string().into(), b.print(c).boxed_local().await]),
 			Self::Name(n) => format!("{n}").into(),
-			Self::Reference(sym) => format!("{sym}").into(),
-			Self::Slot(th) => format!("{th}").into(),
-			Self::Ph(ph) => format!("{ph}").into(),
 			Self::S(p, b) => FmtUnit::new(
 				match *p {
 					Paren::Round => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("({0b})"))),
@@ -284,67 +292,22 @@ impl<A: AtomRepr, X: ExtraTok> Format for Token<'_, A, X> {
 				},
 				[ttv_fmt(b, c).await],
 			),
-			Self::X(x) => x.print(c).await,
+			Self::Handle(h) => h.print(c).await,
-			Self::Macro(None) => "macro".to_string().into(),
+			Self::NewExpr(ex) => ex.print(c).await,
-			Self::Macro(Some(prio)) => format!("macro({prio})").into(),
 		}
 	}
 }
 
-pub fn ttv_range(ttv: &[TokTree<'_, impl AtomRepr, impl ExtraTok>]) -> Range<u32> {
+pub fn ttv_range<'a>(ttv: &[TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>]) -> Option<SrcRange> {
-	assert!(!ttv.is_empty(), "Empty slice has no range");
+	let range = ttv.first()?.sr.range.start..ttv.last().unwrap().sr.range.end;
-	ttv.first().unwrap().range.start..ttv.last().unwrap().range.end
+	Some(SrcRange { path: ttv.first().unwrap().sr.path(), range })
 }
 
 pub async fn ttv_fmt<'a: 'b, 'b>(
-	ttv: impl IntoIterator<Item = &'b TokTree<'a, impl AtomRepr + 'b, impl ExtraTok + 'b>>,
+	ttv: impl IntoIterator<Item = &'b TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>>,
 	c: &(impl FmtCtx + ?Sized),
 ) -> FmtUnit {
 	FmtUnit::sequence(" ", None, join_all(ttv.into_iter().map(|t| t.print(c))).await)
 }
 
 pub fn indent(s: &str) -> String { s.replace("\n", "\n  ") }
-
-#[derive(Clone, Debug)]
-pub struct Ph {
-	pub name: Tok<String>,
-	pub kind: PhKind,
-}
-impl Ph {
-	pub async fn from_api(api: &api::Placeholder, i: &Interner) -> Self {
-		Self { name: Tok::from_api(api.name, i).await, kind: api.kind }
-	}
-	pub fn to_api(&self) -> api::Placeholder {
-		api::Placeholder { name: self.name.to_api(), kind: self.kind }
-	}
-}
-impl Display for Ph {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let n = &self.name;
-		match self.kind {
-			PhKind::Scalar => write!(f, "${n}"),
-			PhKind::Vector { priority: 0, at_least_one: true } => write!(f, "...${}", self.name),
-			PhKind::Vector { priority: p, at_least_one: true } => write!(f, "...${}:{}", self.name, p),
-			PhKind::Vector { priority: 0, at_least_one: false } => write!(f, "..${}", self.name),
-			PhKind::Vector { priority: p, at_least_one: false } => write!(f, "..${}:{}", self.name, p),
-		}
-	}
-}
-
-#[cfg(test)]
-mod test {
-	use super::*;
-
-	#[test]
-	fn test_covariance() {
-		fn _f<'a>(x: Token<'static, Never, Never>) -> Token<'a, Never, Never> { x }
-	}
-
-	#[test]
-	fn fail_covariance() {
-		// this fails to compile
-		// fn _f<'a, 'b>(x: &'a mut &'static ()) -> &'a mut &'b () { x }
-		// this passes because it's covariant
-		fn _f<'a, 'b>(x: &'a fn() -> &'static ()) -> &'a fn() -> &'b () { x }
-	}
-}

orchid-extension/Cargo.toml

@@ -1,30 +1,41 @@
 [package]
 name = "orchid-extension"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ahash = "0.8.11"
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
+async-lock = "3.4.1"
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
 derive_destructure = "1.0.0"
-dyn-clone = "1.0.17"
+dyn-clone = "1.0.20"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = [
-hashbrown = "0.15.2"
+  "std",
+  "async-await",
+], default-features = false }
+hashbrown = "0.16.0"
+include_dir = { version = "0.7.4", optional = true }
 itertools = "0.14.0"
-konst = "0.3.16"
+konst = "0.4.1"
 lazy_static = "1.5.0"
 memo-map = "0.3.3"
 never = "0.1.0"
-once_cell = "1.20.2"
+once_cell = "1.21.3"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"
-paste = "1.0.15"
+pastey = "0.1.1"
-some_executor = "0.4.0"
+some_executor = "0.6.1"
 substack = "1.1.1"
+tokio = { version = "1.47.1", optional = true, features = [] }
+tokio-util = { version = "0.7.16", optional = true, features = ["compat"] }
+
 trait-set = "0.3.0"
+
+[features]
+tokio = ["dep:tokio", "dep:tokio-util"]
+default = ["tokio"]

orchid-extension/src/atom.rs

@@ -1,31 +1,28 @@
 use std::any::{Any, TypeId, type_name};
+use std::collections::HashMap;
 use std::fmt;
 use std::future::Future;
-use std::marker::PhantomData;
 use std::num::NonZeroU32;
 use std::ops::Deref;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use ahash::HashMap;
-use async_std::io::{Read, Write};
-use async_std::stream;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::LocalBoxFuture;
-use futures::{FutureExt, StreamExt};
+use futures::{AsyncRead, AsyncWrite, FutureExt, StreamExt, stream};
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Coding, Decode, Encode, Request, enc_vec};
 use orchid_base::clone;
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use orchid_base::error::{OrcErrv, OrcRes, mk_errv, mk_errv_floating};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::reqnot::Requester;
-use orchid_base::tree::AtomRepr;
 use trait_set::trait_set;
 
 use crate::api;
+use crate::conv::ToExpr;
 // use crate::error::{ProjectError, ProjectResult};
 use crate::expr::{Expr, ExprData, ExprHandle, ExprKind};
 use crate::gen_expr::GExpr;
@@ -42,7 +39,10 @@ pub trait AtomicVariant {}
 pub trait Atomic: 'static + Sized {
 	type Variant: AtomicVariant;
 	type Data: Clone + Coding + Sized + 'static;
-	fn reg_reqs() -> MethodSetBuilder<Self>;
+	/// Register handlers for IPC calls. If this atom implements [Supports], you
+	/// should register your implementations here. If this atom doesn't
+	/// participate in IPC at all, the default implementation is fine
+	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl<A: Atomic> AtomCard for A {
 	type Data = <Self as Atomic>::Data;
@@ -84,56 +84,48 @@ pub fn get_info<A: AtomCard>(
 }
 
 #[derive(Clone)]
-pub struct ForeignAtom<'a> {
+pub struct ForeignAtom {
-	pub(crate) expr: Option<Rc<ExprHandle>>,
+	pub(crate) expr: Rc<ExprHandle>,
-	pub(crate) _life: PhantomData<&'a ()>,
-	pub(crate) ctx: SysCtx,
 	pub(crate) atom: api::Atom,
 	pub(crate) pos: Pos,
 }
-impl ForeignAtom<'_> {
+impl ForeignAtom {
-	pub fn ex_opt(self) -> Option<Expr> {
-		let (handle, pos) = (self.expr.as_ref()?.clone(), self.pos.clone());
-		let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { _life: PhantomData, ..self }) };
-		Some(Expr::new(handle, data))
-	}
 	pub fn pos(&self) -> Pos { self.pos.clone() }
-	pub fn ctx(&self) -> SysCtx { self.ctx.clone() }
+	pub fn ctx(&self) -> &SysCtx { &self.expr.ctx }
-}
+	pub fn ex(self) -> Expr {
-impl ForeignAtom<'static> {
+		let (handle, pos) = (self.expr.clone(), self.pos.clone());
-	pub fn ex(self) -> Expr { self.ex_opt().unwrap() }
+		let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { ..self }) };
+		Expr::new(handle, data)
+	}
 	pub(crate) fn new(handle: Rc<ExprHandle>, atom: api::Atom, pos: Pos) -> Self {
-		ForeignAtom { _life: PhantomData, atom, ctx: handle.ctx.clone(), expr: Some(handle), pos }
+		ForeignAtom { atom, expr: handle, pos }
 	}
 	pub async fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
-		let rep = (self.ctx.reqnot.request(api::Fwd(
+		let rep = (self.ctx().reqnot().request(api::Fwd(
 			self.atom.clone(),
-			Sym::parse(M::NAME, &self.ctx.i).await.unwrap().tok().to_api(),
+			Sym::parse(M::NAME, self.ctx().i()).await.unwrap().tok().to_api(),
 			enc_vec(&m).await,
 		)))
 		.await?;
 		Some(M::Response::decode(Pin::new(&mut &rep[..])).await)
 	}
-}
+	pub async fn downcast<T: AtomicFeatures>(self) -> Result<TypAtom<T>, NotTypAtom> {
-impl fmt::Display for ForeignAtom<'_> {
+		TypAtom::downcast(self.ex().handle()).await
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "{}::{:?}", if self.expr.is_some() { "Clause" } else { "Tok" }, self.atom)
 	}
 }
-impl fmt::Debug for ForeignAtom<'_> {
+impl fmt::Display for ForeignAtom {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Atom::{:?}", self.atom) }
+}
+impl fmt::Debug for ForeignAtom {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ForeignAtom({self})") }
 }
-impl Format for ForeignAtom<'_> {
+impl Format for ForeignAtom {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		FmtUnit::from_api(&self.ctx.reqnot.request(api::ExtAtomPrint(self.atom.clone())).await)
+		FmtUnit::from_api(&self.ctx().reqnot().request(api::ExtAtomPrint(self.atom.clone())).await)
 	}
 }
-impl AtomRepr for ForeignAtom<'_> {
+impl ToExpr for ForeignAtom {
-	type Ctx = SysCtx;
+	async fn to_expr(self) -> GExpr { self.ex().to_expr().await }
-	async fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self {
-		Self { atom: atom.clone(), _life: PhantomData, ctx: ctx.clone(), expr: None, pos }
-	}
-	async fn to_api(&self) -> orchid_api::Atom { self.atom.clone() }
 }
 
 pub struct NotTypAtom {
@@ -143,11 +135,11 @@ pub struct NotTypAtom {
 	pub ctx: SysCtx,
 }
 impl NotTypAtom {
-	pub async fn mk_err(&self) -> OrcErr {
+	pub async fn mk_err(&self) -> OrcErrv {
-		mk_err(
+		mk_errv(
-			self.ctx.i.i("Not the expected type").await,
+			self.ctx.i().i("Not the expected type").await,
 			format!("This expression is not a {}", self.typ.name()),
-			[self.pos.clone().into()],
+			[self.pos.clone()],
 		)
 	}
 }
@@ -163,8 +155,8 @@ trait_set! {
 	trait AtomReqCb<A> = for<'a> Fn(
 		&'a A,
 		SysCtx,
-		Pin<&'a mut dyn Read>,
+		Pin<&'a mut dyn AsyncRead>,
-		Pin<&'a mut dyn Write>,
+		Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, ()>
 }
 
@@ -179,20 +171,22 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 		assert!(!M::NAME.is_empty(), "AtomMethod::NAME cannoot be empty");
 		self.handlers.push((
 			M::NAME,
-			Rc::new(move |a: &A, ctx: SysCtx, req: Pin<&mut dyn Read>, rep: Pin<&mut dyn Write>| {
+			Rc::new(
+				move |a: &A, ctx: SysCtx, req: Pin<&mut dyn AsyncRead>, rep: Pin<&mut dyn AsyncWrite>| {
 					async { Supports::<M>::handle(a, ctx, M::decode(req).await).await.encode(rep).await }
 						.boxed_local()
-			}),
+				},
+			),
 		));
 		self
 	}
 
 	pub async fn pack(&self, ctx: SysCtx) -> MethodSet<A> {
 		MethodSet {
-			handlers: stream::from_iter(self.handlers.iter())
+			handlers: stream::iter(self.handlers.iter())
 				.then(|(k, v)| {
 					clone!(ctx; async move {
-						(Sym::parse(k, &ctx.i).await.unwrap(), v.clone())
+						(Sym::parse(k, ctx.i()).await.unwrap(), v.clone())
 					})
 				})
 				.collect()
@@ -210,8 +204,8 @@ impl<A: AtomCard> MethodSet<A> {
 		atom: &'a A,
 		ctx: SysCtx,
 		key: Sym,
-		req: Pin<&'a mut dyn Read>,
+		req: Pin<&'a mut dyn AsyncRead>,
-		rep: Pin<&'a mut dyn Write>,
+		rep: Pin<&'a mut dyn AsyncWrite>,
 	) -> bool {
 		match self.handlers.get(&key) {
 			None => false,
@@ -228,11 +222,13 @@ impl<A: AtomCard> Default for MethodSetBuilder<A> {
 }
 
 #[derive(Clone)]
-pub struct TypAtom<'a, A: AtomicFeatures> {
+pub struct TypAtom<A: AtomicFeatures> {
-	pub data: ForeignAtom<'a>,
+	pub untyped: ForeignAtom,
 	pub value: A::Data,
 }
-impl<A: AtomicFeatures> TypAtom<'static, A> {
+impl<A: AtomicFeatures> TypAtom<A> {
+	pub fn ctx(&self) -> &SysCtx { self.untyped.ctx() }
+	pub fn i(&self) -> &Interner { self.ctx().i() }
 	pub async fn downcast(expr: Rc<ExprHandle>) -> Result<Self, NotTypAtom> {
 		match Expr::from_handle(expr).atom().await {
 			Err(expr) => Err(NotTypAtom {
@@ -245,21 +241,19 @@ impl<A: AtomicFeatures> TypAtom<'static, A> {
 				Ok(tatom) => Ok(tatom),
 				Err(fa) => Err(NotTypAtom {
 					pos: fa.pos.clone(),
-					ctx: fa.ctx.clone(),
+					ctx: fa.ctx().clone(),
 					expr: fa.ex(),
 					typ: Box::new(A::info()),
 				}),
 			},
 		}
 	}
-}
-impl<A: AtomicFeatures> TypAtom<'_, A> {
 	pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
 	where A: Supports<M> {
 		M::Response::decode(Pin::new(
-			&mut &(self.data.ctx.reqnot.request(api::Fwd(
+			&mut &(self.untyped.ctx().reqnot().request(api::Fwd(
-				self.data.atom.clone(),
+				self.untyped.atom.clone(),
-				Sym::parse(M::NAME, &self.data.ctx.i).await.unwrap().tok().to_api(),
+				Sym::parse(M::NAME, self.untyped.ctx().i()).await.unwrap().tok().to_api(),
 				enc_vec(&req).await,
 			)))
 			.await
@@ -268,41 +262,44 @@ impl<A: AtomicFeatures> TypAtom<'_, A> {
 		.await
 	}
 }
-impl<A: AtomicFeatures> Deref for TypAtom<'_, A> {
+impl<A: AtomicFeatures> Deref for TypAtom<A> {
 	type Target = A::Data;
 	fn deref(&self) -> &Self::Target { &self.value }
 }
+impl<A: AtomicFeatures> ToExpr for TypAtom<A> {
+	async fn to_expr(self) -> GExpr { self.untyped.to_expr().await }
+}
 
 pub struct AtomCtx<'a>(pub &'a [u8], pub Option<api::AtomId>, pub SysCtx);
 impl FmtCtx for AtomCtx<'_> {
-	fn i(&self) -> &Interner { &self.2.i }
+	fn i(&self) -> &Interner { self.2.i() }
 }
 
 pub trait AtomDynfo: 'static {
 	fn tid(&self) -> TypeId;
 	fn name(&self) -> &'static str;
 	fn decode<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>>;
-	fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>;
+	fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
-	fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>;
+	fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
 	fn print<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit>;
 	fn handle_req<'a, 'b: 'a, 'c: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'b mut dyn Read>,
+		req: Pin<&'b mut dyn AsyncRead>,
-		rep: Pin<&'c mut dyn Write>,
+		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool>;
 	fn command<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>>;
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
-		write: Pin<&'b mut dyn Write>,
+		write: Pin<&'b mut dyn AsyncWrite>,
-	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>>;
+	) -> LocalBoxFuture<'a, Option<Vec<Expr>>>;
 	fn deserialize<'a>(
 		&'a self,
 		ctx: SysCtx,
 		data: &'a [u8],
-		refs: &'a [api::ExprTicket],
+		refs: &'a [Expr],
 	) -> LocalBoxFuture<'a, api::Atom>;
 	fn drop<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, ()>;
 }
@@ -312,9 +309,7 @@ trait_set! {
 }
 pub struct AtomFactory(Box<dyn AtomFactoryFn>);
 impl AtomFactory {
-	pub fn new<F: Future<Output = api::Atom> + 'static>(
+	pub fn new(f: impl AsyncFnOnce(SysCtx) -> api::Atom + Clone + 'static) -> Self {
-		f: impl FnOnce(SysCtx) -> F + Clone + 'static,
-	) -> Self {
 		Self(Box::new(|ctx| f(ctx).boxed_local()))
 	}
 	pub async fn build(self, ctx: SysCtx) -> api::Atom { (self.0)(ctx).await }
@@ -334,10 +329,10 @@ impl Format for AtomFactory {
 	}
 }
 
-pub async fn err_not_callable(i: &Interner) -> OrcErr {
+pub async fn err_not_callable(i: &Interner) -> OrcErrv {
-	mk_err(i.i("This atom is not callable").await, "Attempted to apply value as function", [])
+	mk_errv_floating(i.i("This atom is not callable").await, "Attempted to apply value as function")
 }
 
-pub async fn err_not_command(i: &Interner) -> OrcErr {
+pub async fn err_not_command(i: &Interner) -> OrcErrv {
-	mk_err(i.i("This atom is not a command").await, "Settled on an inactionable value", [])
+	mk_errv_floating(i.i("This atom is not a command").await, "Settled on an inactionable value")
 }

orchid-extension/src/atom_owned.rs

@@ -4,44 +4,46 @@ use std::future::Future;
 use std::num::NonZero;
 use std::ops::Deref;
 use std::pin::Pin;
-use std::rc::Rc;
 use std::sync::atomic::AtomicU64;
 
+use async_lock::{RwLock, RwLockReadGuard};
 use async_once_cell::OnceCell;
-use async_std::io::{Read, Write};
+use dyn_clone::{DynClone, clone_box};
-use async_std::sync::{RwLock, RwLockReadGuard};
-use futures::FutureExt;
 use futures::future::{LocalBoxFuture, ready};
+use futures::{AsyncRead, AsyncWrite, FutureExt};
 use itertools::Itertools;
 use memo_map::MemoMap;
 use never::Never;
-use orchid_api::AtomId;
 use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
-use orchid_base::format::FmtUnit;
+use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, take_first};
 use orchid_base::name::Sym;
 
 use crate::api;
 use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
-	MethodSetBuilder, err_not_callable, err_not_command, get_info,
+	MethodSetBuilder, TypAtom, err_not_callable, err_not_command, get_info,
 };
-use crate::expr::{Expr, ExprHandle};
+use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
-use crate::system::SysCtx;
+use crate::system::{SysCtx, SysCtxEntry};
+use crate::system_ctor::CtedObj;
 
 pub struct OwnedVariant;
 impl AtomicVariant for OwnedVariant {}
 impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVariant> for A {
 	fn _factory(self) -> AtomFactory {
-		AtomFactory::new(move |ctx| async move {
+		AtomFactory::new(async move |ctx| {
-			let serial = ctx.obj_store.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+			let serial =
+				ctx.get_or_default::<ObjStore>().next_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
 			let atom_id = api::AtomId(NonZero::new(serial + 1).unwrap());
-			let (typ_id, _) = get_info::<A>(ctx.cted.inst().card());
+			let (typ_id, _) = get_info::<A>(ctx.get::<CtedObj>().inst().card());
 			let mut data = enc_vec(&typ_id).await;
 			self.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
-			ctx.obj_store.1.read().await.insert(atom_id, Box::new(self));
+			let g = ctx.get_or_default::<ObjStore>().objects.read().await;
-			api::Atom { drop: Some(atom_id), data, owner: ctx.id }
+			g.insert(atom_id, Box::new(self));
+			std::mem::drop(g);
+			api::Atom { drop: Some(atom_id), data: api::AtomData(data), owner: ctx.sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { OwnedAtomDynfo { msbuild: A::reg_reqs(), ms: OnceCell::new() } }
@@ -51,12 +53,15 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
 /// While an atom read guard is held, no atom can be removed.
 pub(crate) struct AtomReadGuard<'a> {
 	id: api::AtomId,
-	guard: RwLockReadGuard<'a, MemoMap<AtomId, Box<dyn DynOwnedAtom>>>,
+	guard: RwLockReadGuard<'a, MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
 }
 impl<'a> AtomReadGuard<'a> {
 	async fn new(id: api::AtomId, ctx: &'a SysCtx) -> Self {
-		let guard = ctx.obj_store.1.read().await;
+		let guard = ctx.get_or_default::<ObjStore>().objects.read().await;
-		assert!(guard.get(&id).is_some(), "Received invalid atom ID: {}", id.0);
+		if guard.get(&id).is_none() {
+			let valid = guard.iter().map(|i| i.0).collect_vec();
+			panic!("Received invalid atom ID: {id:?} not in {valid:?}");
+		}
 		Self { id, guard }
 	}
 }
@@ -65,8 +70,9 @@ impl Deref for AtomReadGuard<'_> {
 	fn deref(&self) -> &Self::Target { &**self.guard.get(&self.id).unwrap() }
 }
 
+/// Remove an atom from the store
 pub(crate) async fn take_atom(id: api::AtomId, ctx: &SysCtx) -> Box<dyn DynOwnedAtom> {
-	let mut g = ctx.obj_store.1.write().await;
+	let mut g = ctx.get_or_default::<ObjStore>().objects.write().await;
 	g.remove(&id).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0))
 }
 
@@ -78,75 +84,69 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	fn tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn name(&self) -> &'static str { type_name::<T>() }
 	fn decode<'a>(&'a self, AtomCtx(data, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
-		async {
+		Box::pin(async {
 			Box::new(<T as AtomCard>::Data::decode(Pin::new(&mut &data[..])).await) as Box<dyn Any>
+		})
 	}
-		.boxed_local()
+	fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
-	}
+		Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_call(arg).await })
-	fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
-		async move { take_atom(id.unwrap(), &ctx).await.dyn_call(ctx.clone(), arg).await }.boxed_local()
 	}
 	fn call_ref<'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
-		arg: api::ExprTicket,
+		arg: Expr,
 	) -> LocalBoxFuture<'a, GExpr> {
-		async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_call_ref(ctx.clone(), arg).await }
+		Box::pin(async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_call_ref(arg).await })
-			.boxed_local()
 	}
 	fn print(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> LocalBoxFuture<'_, FmtUnit> {
-		async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_print(ctx.clone()).await }
+		Box::pin(
-			.boxed_local()
+			async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_print(ctx.clone()).await },
+		)
 	}
 	fn handle_req<'a, 'b: 'a, 'c: 'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx,
 		key: Sym,
-		req: Pin<&'b mut dyn Read>,
+		req: Pin<&'b mut dyn AsyncRead>,
-		rep: Pin<&'c mut dyn Write>,
+		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
-		async move {
+		Box::pin(async move {
 			let a = AtomReadGuard::new(id.unwrap(), &ctx).await;
 			let ms = self.ms.get_or_init(self.msbuild.pack(ctx.clone())).await;
 			ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), ctx.clone(), key, req, rep).await
-		}
+		})
-		.boxed_local()
 	}
 	fn command<'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>> {
-		async move { take_atom(id.unwrap(), &ctx).await.dyn_command(ctx.clone()).await }.boxed_local()
+		Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_command(ctx.clone()).await })
 	}
 	fn drop(&self, AtomCtx(_, id, ctx): AtomCtx) -> LocalBoxFuture<'_, ()> {
-		async move { take_atom(id.unwrap(), &ctx).await.dyn_free(ctx.clone()).await }.boxed_local()
+		Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_free(ctx.clone()).await })
 	}
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
-		mut write: Pin<&'b mut dyn Write>,
+		mut write: Pin<&'b mut dyn AsyncWrite>,
-	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>> {
+	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
-		async move {
+		Box::pin(async move {
 			let id = id.unwrap();
 			id.encode(write.as_mut()).await;
-			let refs = AtomReadGuard::new(id, &ctx).await.dyn_serialize(ctx.clone(), write).await;
+			AtomReadGuard::new(id, &ctx).await.dyn_serialize(ctx.clone(), write).await
-			refs.map(|v| v.into_iter().map(|t| t.handle().tk).collect_vec())
+		})
-		}
-		.boxed_local()
 	}
 	fn deserialize<'a>(
 		&'a self,
 		ctx: SysCtx,
 		data: &'a [u8],
-		refs: &'a [api::ExprTicket],
+		refs: &'a [Expr],
 	) -> LocalBoxFuture<'a, api::Atom> {
-		async move {
+		Box::pin(async move {
-			let refs =
+			let refs = T::Refs::from_iter(refs.iter().cloned());
-				refs.iter().map(|tk| Expr::from_handle(Rc::new(ExprHandle::from_args(ctx.clone(), *tk))));
+			let obj = T::deserialize(DeserCtxImpl(data, &ctx), refs).await;
-			let obj = T::deserialize(DeserCtxImpl(data, &ctx), T::Refs::from_iter(refs)).await;
 			obj._factory().build(ctx).await
-		}
+		})
-		.boxed_local()
 	}
 }
 
@@ -218,12 +218,12 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	type Refs: RefSet;
 	fn val(&self) -> impl Future<Output = Cow<'_, Self::Data>>;
 	#[allow(unused_variables)]
-	fn call_ref(&self, arg: ExprHandle) -> impl Future<Output = GExpr> {
+	fn call_ref(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot([err_not_callable(&arg.ctx.i).await]) }
+		async move { bot(err_not_callable(arg.ctx().i()).await) }
 	}
-	fn call(self, arg: ExprHandle) -> impl Future<Output = GExpr> {
+	fn call(self, arg: Expr) -> impl Future<Output = GExpr> {
 		async {
-			let ctx = arg.get_ctx();
+			let ctx = arg.ctx();
 			let gcl = self.call_ref(arg).await;
 			self.free(ctx).await;
 			gcl
@@ -231,19 +231,19 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	}
 	#[allow(unused_variables)]
 	fn command(self, ctx: SysCtx) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command(&ctx.i).await.into()) }
+		async move { Err(err_not_command(ctx.i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn free(self, ctx: SysCtx) -> impl Future<Output = ()> { async {} }
 	#[allow(unused_variables)]
-	fn print(&self, ctx: SysCtx) -> impl Future<Output = FmtUnit> {
+	fn print_atom<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> {
 		async { format!("OwnedAtom({})", type_name::<Self>()).into() }
 	}
 	#[allow(unused_variables)]
 	fn serialize(
 		&self,
 		ctx: SysCtx,
-		write: Pin<&mut (impl Write + ?Sized)>,
+		write: Pin<&mut (impl AsyncWrite + ?Sized)>,
 	) -> impl Future<Output = Self::Refs> {
 		assert_serializable::<Self>();
 		async { panic!("Either implement serialize or set Refs to Never for {}", type_name::<Self>()) }
@@ -262,37 +262,32 @@ fn assert_serializable<T: OwnedAtom>() {
 	assert_ne!(TypeId::of::<T::Refs>(), TypeId::of::<Never>(), "{MSG}");
 }
 
-pub trait DynOwnedAtom: 'static {
+pub trait DynOwnedAtom: DynClone + 'static {
 	fn atom_tid(&self) -> TypeId;
 	fn as_any_ref(&self) -> &dyn Any;
-	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()>;
+	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()>;
-	fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr>;
+	fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr>;
-	fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket)
+	fn dyn_call(self: Box<Self>, arg: Expr) -> LocalBoxFuture<'static, GExpr>;
-	-> LocalBoxFuture<'static, GExpr>;
 	fn dyn_command(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, OrcRes<Option<GExpr>>>;
 	fn dyn_free(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, ()>;
 	fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit>;
 	fn dyn_serialize<'a>(
 		&'a self,
 		ctx: SysCtx,
-		sink: Pin<&'a mut dyn Write>,
+		sink: Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>>;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn as_any_ref(&self) -> &dyn Any { self }
-	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()> {
+	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()> {
 		async { self.val().await.as_ref().encode(buffer).await }.boxed_local()
 	}
-	fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
+	fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
-		self.call_ref(ExprHandle::from_args(ctx, arg)).boxed_local()
+		self.call_ref(arg).boxed_local()
 	}
-	fn dyn_call(
+	fn dyn_call(self: Box<Self>, arg: Expr) -> LocalBoxFuture<'static, GExpr> {
-		self: Box<Self>,
+		self.call(arg).boxed_local()
-		ctx: SysCtx,
-		arg: api::ExprTicket,
-	) -> LocalBoxFuture<'static, GExpr> {
-		self.call(ExprHandle::from_args(ctx, arg)).boxed_local()
 	}
 	fn dyn_command(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, OrcRes<Option<GExpr>>> {
 		self.command(ctx).boxed_local()
@@ -300,11 +295,13 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn dyn_free(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, ()> {
 		self.free(ctx).boxed_local()
 	}
-	fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit> { self.print(ctx).boxed_local() }
+	fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit> {
+		async move { self.print_atom(&FmtCtxImpl { i: ctx.i() }).await }.boxed_local()
+	}
 	fn dyn_serialize<'a>(
 		&'a self,
 		ctx: SysCtx,
-		sink: Pin<&'a mut dyn Write>,
+		sink: Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		match TypeId::of::<Never>() == TypeId::of::<<Self as OwnedAtom>::Refs>() {
 			true => ready(None).boxed_local(),
@@ -313,4 +310,39 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	}
 }
 
-pub type ObjStore = Rc<(AtomicU64, RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>)>;
+#[derive(Default)]
+pub(crate) struct ObjStore {
+	pub(crate) next_id: AtomicU64,
+	pub(crate) objects: RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
+}
+impl SysCtxEntry for ObjStore {}
+
+pub async fn own<A: OwnedAtom>(typ: TypAtom<A>) -> A {
+	let ctx = typ.untyped.ctx();
+	let g = ctx.get_or_default::<ObjStore>().objects.read().await;
+	let atom_id = typ.untyped.atom.drop.expect("Owned atoms always have a drop ID");
+	let dyn_atom =
+		g.get(&atom_id).expect("Atom ID invalid; atom type probably not owned by this crate");
+	dyn_atom.as_any_ref().downcast_ref().cloned().expect("The ID should imply a type as well")
+}
+
+pub async fn debug_print_obj_store(ctx: &SysCtx, show_atoms: bool) {
+	let store = ctx.get_or_default::<ObjStore>();
+	let keys = store.objects.read().await.keys().cloned().collect_vec();
+	let mut message = "Atoms in store:".to_string();
+	if !show_atoms {
+		message += &keys.iter().map(|k| format!(" {:?}", k)).join("");
+	} else {
+		for k in keys {
+			let g = store.objects.read().await;
+			let Some(atom) = g.get(&k) else {
+				message += &format!("\n{k:?} has since been deleted");
+				continue;
+			};
+			let atom = clone_box(&**atom);
+			std::mem::drop(g);
+			message += &format!("\n{k:?} -> {}", take_first(&atom.dyn_print(ctx.clone()).await, true));
+		}
+	}
+	eprintln!("{message}")
+}

orchid-extension/src/atom_thin.rs

@@ -3,9 +3,8 @@ use std::future::Future;
 use std::pin::Pin;
 
 use async_once_cell::OnceCell;
-use async_std::io::{Read, Write};
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
+use futures::{AsyncRead, AsyncWrite, FutureExt};
 use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
@@ -16,19 +15,20 @@ use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
 	MethodSetBuilder, err_not_callable, err_not_command, get_info,
 };
-use crate::expr::ExprHandle;
+use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
 use crate::system::SysCtx;
+use crate::system_ctor::CtedObj;
 
 pub struct ThinVariant;
 impl AtomicVariant for ThinVariant {}
 impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant> for A {
 	fn _factory(self) -> AtomFactory {
-		AtomFactory::new(move |ctx| async move {
+		AtomFactory::new(async move |ctx| {
-			let (id, _) = get_info::<A>(ctx.cted.inst().card());
+			let (id, _) = get_info::<A>(ctx.get::<CtedObj>().inst().card());
 			let mut buf = enc_vec(&id).await;
 			self.encode(Pin::new(&mut buf)).await;
-			api::Atom { drop: None, data: buf, owner: ctx.id }
+			api::Atom { drop: None, data: api::AtomData(buf), owner: ctx.sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { ThinAtomDynfo { msbuild: Self::reg_reqs(), ms: OnceCell::new() } }
@@ -41,37 +41,25 @@ pub struct ThinAtomDynfo<T: ThinAtom> {
 }
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn print<'a>(&self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit> {
-		async move { T::decode(Pin::new(&mut &buf[..])).await.print(ctx).await }.boxed_local()
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.print(ctx).await })
 	}
 	fn tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn name(&self) -> &'static str { type_name::<T>() }
 	fn decode<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
-		async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> }.boxed_local()
+		Box::pin(async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> })
 	}
-	fn call<'a>(
+	fn call<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		&'a self,
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
-		AtomCtx(buf, _, ctx): AtomCtx<'a>,
-		arg: api::ExprTicket,
-	) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move {
-			T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await
-		})
 	}
-	fn call_ref<'a>(
+	fn call_ref<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		&'a self,
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
-		AtomCtx(buf, _, ctx): AtomCtx<'a>,
-		arg: api::ExprTicket,
-	) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move {
-			T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await
-		})
 	}
 	fn handle_req<'a, 'm1: 'a, 'm2: 'a>(
 		&'a self,
 		AtomCtx(buf, _, sys): AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'m1 mut dyn Read>,
+		req: Pin<&'m1 mut dyn AsyncRead>,
-		rep: Pin<&'m2 mut dyn Write>,
+		rep: Pin<&'m2 mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let ms = self.ms.get_or_init(self.msbuild.pack(sys.clone())).await;
@@ -87,8 +75,8 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
-		write: Pin<&'b mut dyn Write>,
+		write: Pin<&'b mut dyn AsyncWrite>,
-	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>> {
+	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		Box::pin(async {
 			T::decode(Pin::new(&mut &ctx.0[..])).await.encode(write).await;
 			Some(Vec::new())
@@ -98,17 +86,16 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 		&'a self,
 		ctx: SysCtx,
 		data: &'a [u8],
-		refs: &'a [api::ExprTicket],
+		refs: &'a [Expr],
 	) -> LocalBoxFuture<'a, api::Atom> {
 		assert!(refs.is_empty(), "Refs found when deserializing thin atom");
-		async { T::decode(Pin::new(&mut &data[..])).await._factory().build(ctx).await }.boxed_local()
+		Box::pin(async { T::decode(Pin::new(&mut &data[..])).await._factory().build(ctx).await })
 	}
 	fn drop<'a>(&'a self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, ()> {
-		async move {
+		Box::pin(async move {
 			let string_self = T::decode(Pin::new(&mut &buf[..])).await.print(ctx.clone()).await;
-			writeln!(ctx.logger, "Received drop signal for non-drop atom {string_self:?}");
+			writeln!(ctx.logger(), "Received drop signal for non-drop atom {string_self:?}");
-		}
+		})
-		.boxed_local()
 	}
 }
 
@@ -116,12 +103,12 @@ pub trait ThinAtom:
 	AtomCard<Data = Self> + Atomic<Variant = ThinVariant> + Coding + Send + Sync + 'static
 {
 	#[allow(unused_variables)]
-	fn call(&self, arg: ExprHandle) -> impl Future<Output = GExpr> {
+	fn call(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot([err_not_callable(&arg.ctx.i).await]) }
+		async move { bot(err_not_callable(arg.ctx().i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn command(&self, ctx: SysCtx) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command(&ctx.i).await.into()) }
+		async move { Err(err_not_command(ctx.i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn print(&self, ctx: SysCtx) -> impl Future<Output = FmtUnit> {

orchid-extension/src/conv.rs

@@ -1,13 +1,14 @@
 use std::future::Future;
 
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use never::Never;
+use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
 
-use crate::atom::{AtomicFeatures, ToAtom, TypAtom};
+use crate::atom::{AtomicFeatures, ForeignAtom, ToAtom, TypAtom};
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, atom, bot};
-use crate::system::downcast_atom;
+use crate::system::{SysCtx, downcast_atom};
 
 pub trait TryFromExpr: Sized {
 	fn try_from_expr(expr: Expr) -> impl Future<Output = OrcRes<Self>>;
@@ -23,46 +24,61 @@ impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
 	}
 }
 
-async fn err_not_atom(pos: Pos, i: &Interner) -> OrcErr {
+async fn err_not_atom(pos: Pos, i: &Interner) -> OrcErrv {
-	mk_err(i.i("Expected an atom").await, "This expression is not an atom", [pos.into()])
+	mk_errv(i.i("Expected an atom").await, "This expression is not an atom", [pos])
 }
 
-async fn err_type(pos: Pos, i: &Interner) -> OrcErr {
+async fn err_type(pos: Pos, i: &Interner) -> OrcErrv {
-	mk_err(i.i("Type error").await, "The atom is a different type than expected", [pos.into()])
+	mk_errv(i.i("Type error").await, "The atom is a different type than expected", [pos])
 }
 
-impl<A: AtomicFeatures> TryFromExpr for TypAtom<'_, A> {
+impl TryFromExpr for ForeignAtom {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
 		match expr.atom().await {
-			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), &ex.ctx().i).await.into()),
+			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), ex.ctx().i()).await),
-			Ok(f) => match downcast_atom::<A>(f).await {
+			Ok(f) => Ok(f),
+		}
+	}
+}
+
+impl<A: AtomicFeatures> TryFromExpr for TypAtom<A> {
+	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
+		let f = ForeignAtom::try_from_expr(expr).await?;
+		match downcast_atom::<A>(f).await {
 			Ok(a) => Ok(a),
-				Err(f) => Err(err_type(f.pos(), &f.ctx().i).await.into()),
+			Err(f) => Err(err_type(f.pos(), f.ctx().i()).await),
-			},
 		}
 	}
 }
 
+impl TryFromExpr for SysCtx {
+	async fn try_from_expr(expr: Expr) -> OrcRes<Self> { Ok(expr.ctx()) }
+}
+
 pub trait ToExpr {
-	fn to_expr(self) -> GExpr;
+	fn to_expr(self) -> impl Future<Output = GExpr>;
 }
 
 impl ToExpr for GExpr {
-	fn to_expr(self) -> GExpr { self }
+	async fn to_expr(self) -> GExpr { self }
 }
 impl ToExpr for Expr {
-	fn to_expr(self) -> GExpr { self.gen() }
+	async fn to_expr(self) -> GExpr { self.slot() }
 }
 
 impl<T: ToExpr> ToExpr for OrcRes<T> {
-	fn to_expr(self) -> GExpr {
+	async fn to_expr(self) -> GExpr {
 		match self {
 			Err(e) => bot(e),
-			Ok(t) => t.to_expr(),
+			Ok(t) => t.to_expr().await,
 		}
 	}
 }
 
 impl<A: ToAtom> ToExpr for A {
-	fn to_expr(self) -> GExpr { atom(self) }
+	async fn to_expr(self) -> GExpr { atom(self) }
+}
+
+impl ToExpr for Never {
+	async fn to_expr(self) -> GExpr { match self {} }
 }

orchid-extension/src/coroutine_exec.rs

@@ -0,0 +1,104 @@
+use std::borrow::Cow;
+use std::marker::PhantomData;
+use std::rc::Rc;
+
+use futures::channel::mpsc::{Sender, channel};
+use futures::lock::Mutex;
+use futures::stream::{self, LocalBoxStream};
+use futures::{FutureExt, SinkExt, StreamExt};
+use never::Never;
+use orchid_base::error::OrcRes;
+use orchid_base::format::{FmtCtx, FmtUnit};
+
+use crate::atom::Atomic;
+use crate::atom_owned::{OwnedAtom, OwnedVariant};
+use crate::conv::{ToExpr, TryFromExpr};
+use crate::expr::Expr;
+use crate::gen_expr::{GExpr, arg, call, lambda, seq};
+
+enum Command {
+	Execute(GExpr, Sender<Expr>),
+	Register(GExpr, Sender<Expr>),
+	Halt(GExpr),
+}
+
+struct BuilderCoroutineData {
+	name: Option<String>,
+	receiver: Mutex<LocalBoxStream<'static, Command>>,
+}
+
+#[derive(Clone)]
+struct BuilderCoroutine(Rc<BuilderCoroutineData>);
+impl BuilderCoroutine {
+	pub async fn run(self) -> GExpr {
+		let cmd = self.0.receiver.lock().await.next().await;
+		match cmd {
+			None => panic!("Before the stream ends, we should have gotten a Halt"),
+			Some(Command::Halt(expr)) => expr,
+			Some(Command::Execute(expr, reply)) => call([
+				lambda(0, [seq([
+					arg(0),
+					call([Replier { reply, builder: self }.to_expr().await, arg(0)]),
+				])]),
+				expr,
+			]),
+			Some(Command::Register(expr, reply)) =>
+				call([Replier { reply, builder: self }.to_expr().await, expr]),
+		}
+	}
+}
+
+#[derive(Clone)]
+pub struct Replier {
+	reply: Sender<Expr>,
+	builder: BuilderCoroutine,
+}
+impl Atomic for Replier {
+	type Data = ();
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for Replier {
+	type Refs = Never;
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	async fn call(mut self, arg: Expr) -> GExpr {
+		self.reply.send(arg).await.expect("What the heck");
+		std::mem::drop(self.reply);
+		self.builder.run().await
+	}
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		match &self.builder.0.name {
+			None => "BuilderCoroutine".into(),
+			Some(name) => format!("BuilderCoroutine({name})").into(),
+		}
+	}
+}
+
+pub async fn exec<R: ToExpr>(
+	debug: impl AsRef<str>,
+	f: impl for<'a> AsyncFnOnce(ExecHandle<'a>) -> R + 'static,
+) -> GExpr {
+	let (cmd_snd, cmd_recv) = channel(0);
+	let halt = async { Command::Halt(f(ExecHandle(cmd_snd, PhantomData)).await.to_expr().await) }
+		.into_stream();
+	let coro = BuilderCoroutine(Rc::new(BuilderCoroutineData {
+		name: Some(debug.as_ref().to_string()),
+		receiver: Mutex::new(stream::select(halt, cmd_recv).boxed_local()),
+	}));
+	coro.run().await
+}
+
+static WEIRD_DROP_ERR: &str = "Coroutine dropped while we are being polled somehow";
+
+pub struct ExecHandle<'a>(Sender<Command>, PhantomData<&'a ()>);
+impl ExecHandle<'_> {
+	pub async fn exec<T: TryFromExpr>(&mut self, val: impl ToExpr) -> OrcRes<T> {
+		let (reply_snd, mut reply_recv) = channel(1);
+		self.0.send(Command::Execute(val.to_expr().await, reply_snd)).await.expect(WEIRD_DROP_ERR);
+		T::try_from_expr(reply_recv.next().await.expect(WEIRD_DROP_ERR)).await
+	}
+	pub async fn register(&mut self, val: impl ToExpr) -> Expr {
+		let (reply_snd, mut reply_recv) = channel(1);
+		self.0.send(Command::Register(val.to_expr().await, reply_snd)).await.expect(WEIRD_DROP_ERR);
+		reply_recv.next().await.expect(WEIRD_DROP_ERR)
+	}
+}

orchid-extension/src/entrypoint.rs

@@ -1,45 +1,40 @@
 use std::cell::RefCell;
 use std::future::Future;
-use std::io::Write;
 use std::mem;
 use std::num::NonZero;
 use std::pin::Pin;
 use std::rc::Rc;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicBool, Ordering};
 
-use async_std::channel::{Receiver, Sender};
+use async_lock::RwLock;
-use async_std::stream;
+use futures::channel::mpsc::{Receiver, Sender, channel};
-use async_std::sync::Mutex;
 use futures::future::{LocalBoxFuture, join_all};
-use futures::{FutureExt, StreamExt};
+use futures::lock::Mutex;
+use futures::{FutureExt, SinkExt, StreamExt, stream, stream_select};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::ApplyMacro;
+use orchid_api_traits::{Decode, UnderRoot, enc_vec};
-use orchid_api_traits::{Decode, Encode, enc_vec};
+use orchid_base::builtin::{ExtInit, ExtPort, Spawner};
-use orchid_base::builtin::{ExtPort, Spawner};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
 use orchid_base::clone;
+use orchid_base::error::Reporter;
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::logging::Logger;
-use orchid_base::macros::{mtreev_from_api, mtreev_to_api};
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Snippet};
 use orchid_base::reqnot::{ReqNot, RequestHandle, Requester};
-use orchid_base::tree::{ttv_from_api, ttv_to_api};
+use orchid_base::tree::{TokenVariant, ttv_from_api};
 use substack::Substack;
 use trait_set::trait_set;
 
 use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId};
-use crate::atom_owned::{ObjStore, take_atom};
+use crate::atom_owned::take_atom;
-use crate::fs::VirtFS;
+use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
-use crate::lexer::{LexContext, err_cascade, err_not_applicable};
+use crate::lexer::{LexContext, ekey_cascade, ekey_not_applicable};
-use crate::macros::{Rule, RuleCtx};
+use crate::parser::{PTokTree, ParsCtx, get_const, linev_into_api};
-use crate::msg::{recv_parent_msg, send_parent_msg};
 use crate::system::{SysCtx, atom_by_idx};
 use crate::system_ctor::{CtedObj, DynSystemCtor};
-use crate::tree::{GenTok, GenTokTree, LazyMemberFactory, TIACtxImpl, do_extra};
+use crate::tree::{LazyMemberFactory, TreeIntoApiCtxImpl};
 
 pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>;
 pub type ExtReqNot = ReqNot<api::ExtMsgSet>;
@@ -52,7 +47,6 @@ impl ExtensionData {
 	pub fn new(name: &'static str, systems: &'static [&'static dyn DynSystemCtor]) -> Self {
 		Self { name, systems }
 	}
-	// pub fn main(self) { extension_main(self) }
 }
 
 pub enum MemberRecord {
@@ -61,128 +55,128 @@ pub enum MemberRecord {
 }
 
 pub struct SystemRecord {
-	cted: CtedObj,
+	lazy_members: Mutex<HashMap<api::TreeId, MemberRecord>>,
-	vfses: HashMap<api::VfsId, &'static dyn VirtFS>,
+	ctx: SysCtx,
-	declfs: api::EagerVfs,
-	lazy_members: HashMap<api::TreeId, MemberRecord>,
-	rules: HashMap<api::MacroId, Rc<Rule>>,
 }
 
 trait_set! {
-	pub trait WARCallback<'a, T> = FnOnce(
+	pub trait WithAtomRecordCallback<'a, T> = AsyncFnOnce(
 		Box<dyn AtomDynfo>,
 		SysCtx,
 		AtomTypeId,
 		&'a [u8]
-	) -> LocalBoxFuture<'a, T>
+	) -> T
 }
 
 pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
-	get_sys_ctx: &impl Fn(api::SysId, ReqNot<api::ExtMsgSet>) -> F,
+	get_sys_ctx: &impl Fn(api::SysId) -> F,
-	reqnot: ReqNot<api::ExtMsgSet>,
 	atom: &'a api::Atom,
-	cb: impl WARCallback<'a, T>,
+	cb: impl WithAtomRecordCallback<'a, T>,
 ) -> T {
-	let mut data = &atom.data[..];
+	let mut data = &atom.data.0[..];
-	let ctx = get_sys_ctx(atom.owner, reqnot).await;
+	let ctx = get_sys_ctx(atom.owner).await;
-	let inst = ctx.cted.inst();
+	let inst = ctx.get::<CtedObj>().inst();
 	let id = AtomTypeId::decode(Pin::new(&mut data)).await;
 	let atom_record = atom_by_idx(inst.card(), id.clone()).expect("Atom ID reserved");
 	cb(atom_record, ctx, id, data).await
 }
 
-// pub fn extension_main(data: ExtensionData) {
-
-// 	if thread::Builder::new()
-// 		.name(format!("ext-main:{}", data.name))
-// 		.spawn(|| extension_main_logic(data))
-// 		.unwrap()
-// 		.join()
-// 		.is_err()
-// 	{
-// 		process::exit(-1)
-// 	}
-// }
-
 pub struct ExtensionOwner {
-	rn: ReqNot<api::ExtMsgSet>,
+	_interner_cell: Rc<RefCell<Option<Interner>>>,
-	out_recv: Receiver<Vec<u8>>,
+	_systems_lock: Rc<RwLock<HashMap<api::SysId, SystemRecord>>>,
+	out_recv: Mutex<Receiver<Vec<u8>>>,
 	out_send: Sender<Vec<u8>>,
 }
 
 impl ExtPort for ExtensionOwner {
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		self.rn.receive(msg).boxed_local()
+		Box::pin(async { self.out_send.clone().send(msg.to_vec()).boxed_local().await.unwrap() })
 	}
-	fn recv<'a>(
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
-		&'a self,
+		Box::pin(async { self.out_recv.lock().await.next().await })
-		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
-	) -> LocalBoxFuture<'a, ()> {
-		async {
-			let msg = self.out_recv.recv().await.unwrap();
-			cb(&msg[..]).await
-		}
-		.boxed_local()
 	}
 }
 
-pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
+pub fn extension_init(
-	let api::HostHeader { log_strategy, msg_logs } =
+	data: ExtensionData,
-		api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await;
+	host_header: api::HostHeader,
-	let mut buf = Vec::new();
+	spawner: Spawner,
+) -> ExtInit {
+	let api::HostHeader { log_strategy, msg_logs } = host_header;
 	let decls = (data.systems.iter().enumerate())
 		.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
 		.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
 		.collect_vec();
-	let systems = Rc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
+	let systems_lock = Rc::new(RwLock::new(HashMap::<api::SysId, SystemRecord>::new()));
-	api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() }
+	let ext_header = api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() };
-		.encode(Pin::new(&mut buf))
+	let (out_send, in_recv) = channel::<Vec<u8>>(1);
-		.await;
+	let (in_send, out_recv) = channel::<Vec<u8>>(1);
-	std::io::stdout().write_all(&buf).unwrap();
+	let (exit_send, exit_recv) = channel(1);
-	std::io::stdout().flush().unwrap();
-	let exiting = Arc::new(AtomicBool::new(false));
 	let logger = Logger::new(log_strategy);
 	let msg_logger = Logger::new(msg_logs);
-	let interner_cell = Rc::new(RefCell::new(None::<Rc<Interner>>));
+	let interner_cell = Rc::new(RefCell::new(None::<Interner>));
 	let interner_weak = Rc::downgrade(&interner_cell);
-	let obj_store = ObjStore::default();
+	let systems_weak = Rc::downgrade(&systems_lock);
-	let mk_ctx = clone!(
+	let get_ctx = clone!(systems_weak; move |id: api::SysId| clone!(systems_weak; async move {
-		logger, systems, spawner, obj_store, interner_weak;
+		let systems =
-		move |id: api::SysId, reqnot: ReqNot<api::ExtMsgSet>| {
+			systems_weak.upgrade().expect("System table dropped before request processing done");
-			clone!(logger, systems, spawner, obj_store, interner_weak; async move {
+		systems.read().await.get(&id).expect("System not found").ctx.clone()
-				let cted = systems.lock().await[&id].cted.clone();
+	}));
-				let interner_cell = (interner_weak.upgrade())
+	let init_ctx = {
-					.expect("mk_ctx called after Interner rc dropped");
+		clone!(interner_weak, spawner, logger);
-				let i = (interner_cell.borrow().clone())
+		move |id: api::SysId, cted: CtedObj, reqnot: ReqNot<api::ExtMsgSet>| {
-					.expect("mk_ctx called before interner initialized");
+			clone!(interner_weak, spawner, logger; async move {
-				SysCtx { id, cted, logger, reqnot, spawner, obj_store, i: i.clone() }
+				let interner_rc =
-			}.boxed_local())
+					interner_weak.upgrade().expect("System construction order while shutting down");
-	});
+				let i = interner_rc.borrow().clone().expect("mk_ctx called very early, no interner!");
+				SysCtx::new(id, i, reqnot, spawner, logger, cted)
+			})
+		}
+	};
 	let rn = ReqNot::<api::ExtMsgSet>::new(
 		msg_logger.clone(),
-		move |a, _| async move { send_parent_msg(a).await.unwrap() }.boxed_local(),
+		move |a, _| {
-		clone!(systems, exiting, mk_ctx; move |n, reqnot| {
+			clone!(in_send mut);
-			clone!(systems, exiting, mk_ctx; async move {
+			Box::pin(async move { in_send.send(a.to_vec()).await.unwrap() })
-				match n {
+		},
-					api::HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
-					api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
-						mem::drop(systems.lock().await.remove(&sys_id)),
-					api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) => {
-						let ctx = mk_ctx(sys_id, reqnot).await;
-						take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await
-					}
-				}
-			}.boxed_local())
-		}),
 		{
-			clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls, msg_logger);
+			clone!(exit_send);
+			move |n, _| {
+				clone!(exit_send mut);
+				async move {
+					match n {
+						api::HostExtNotif::Exit => {
+							eprintln!("Exit received");
+							exit_send.send(()).await.unwrap()
+						},
+					}
+				}
+				.boxed_local()
+			}
+		},
+		{
+			clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
 			move |hand, req| {
-				clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls, msg_logger);
+				clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
 				async move {
 					let interner_cell = interner_weak.upgrade().expect("Interner dropped before request");
 					let i = interner_cell.borrow().clone().expect("Request arrived before interner set");
+					if !matches!(req, api::HostExtReq::AtomReq(api::AtomReq::AtomPrint(_))) {
 						writeln!(msg_logger, "{} extension received request {req:?}", data.name);
+					}
+
 					match req {
+						api::HostExtReq::SystemDrop(sys_drop) => {
+							if let Some(rc) = systems_weak.upgrade() {
+								mem::drop(rc.write().await.remove(&sys_drop.0))
+							}
+							hand.handle(&sys_drop, &()).await
+						},
+						api::HostExtReq::AtomDrop(atom_drop @ api::AtomDrop(sys_id, atom)) => {
+							let ctx = get_ctx(sys_id).await;
+							take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await;
+							hand.handle(&atom_drop, &()).await
+						},
 						api::HostExtReq::Ping(ping @ api::Ping) => hand.handle(&ping, &()).await,
 						api::HostExtReq::Sweep(sweep @ api::Sweep) =>
 							hand.handle(&sweep, &i.sweep_replica().await).await,
@@ -190,161 +184,152 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 							let (sys_id, _) = (decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system))
 								.expect("NewSystem call received for invalid system");
 							let cted = data.systems[sys_id].new_system(&new_sys);
-							let mut vfses = HashMap::new();
 							let lex_filter =
 								cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
 									char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
 								});
-							let lazy_mems = Mutex::new(HashMap::new());
+							let lazy_members = Mutex::new(HashMap::new());
-							let rules = Mutex::new(HashMap::new());
+							let ctx = init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await;
-							let ctx = SysCtx {
+							let const_root = stream::iter(cted.inst().dyn_env())
-								cted: cted.clone(),
+								.then(|mem| {
-								id: new_sys.id,
+									let lazy_mems = &lazy_members;
-								logger: logger.clone(),
-								reqnot: hand.reqnot(),
-								i: i.clone(),
-								obj_store: obj_store.clone(),
-								spawner: spawner.clone(),
-							};
-							let const_root = stream::from_iter(cted.inst().dyn_env())
-								.then(|(k, v)| {
-									let (req, lazy_mems, rules) = (&hand, &lazy_mems, &rules);
 									clone!(i, ctx; async move {
-										let name = i.i(&k).await.to_api();
+										let mut tia_ctx = TreeIntoApiCtxImpl {
-										let value = v.into_api(&mut TIACtxImpl {
 											lazy_members: &mut *lazy_mems.lock().await,
-											rules: &mut *rules.lock().await,
 											sys: ctx,
 											basepath: &[],
 											path: Substack::Bottom,
-											req
+										};
-										})
+										(i.i(&mem.name).await.to_api(), mem.kind.into_api(&mut tia_ctx).await)
-										.await;
-										(name, value)
 									})
 								})
 								.collect()
 								.await;
-							let declfs = cted.inst().dyn_vfs().to_api_rec(&mut vfses, &i).await;
+							let prelude =
-							let record = SystemRecord {
+								cted.inst().dyn_prelude(&i).await.iter().map(|sym| sym.to_api()).collect();
-								declfs,
+							let record = SystemRecord { ctx, lazy_members };
-								vfses,
+							let systems = systems_weak.upgrade().expect("System constructed during shutdown");
-								cted,
+							systems.write().await.insert(new_sys.id, record);
-								lazy_members: lazy_mems.into_inner(),
+							let line_types = join_all(
-								rules: rules.into_inner(),
+								(cted.inst().dyn_parsers().iter())
-							};
+									.map(|p| async { i.i(p.line_head()).await.to_api() }),
-							systems.lock().await.insert(new_sys.id, record);
+							)
-							hand
+							.await;
-								.handle(&new_sys, &api::SystemInst { lex_filter, const_root, line_types: vec![] })
+							let response = api::NewSystemResponse { lex_filter, const_root, line_types, prelude };
-								.await
+							hand.handle(&new_sys, &response).await
 						},
 						api::HostExtReq::GetMember(get_tree @ api::GetMember(sys_id, tree_id)) => {
-							let sys_ctx = mk_ctx(sys_id, hand.reqnot()).await;
+							let sys_ctx = get_ctx(sys_id).await;
-							let mut systems_g = systems.lock().await;
+							let systems = systems_weak.upgrade().expect("Member queried during shutdown");
-							let SystemRecord { lazy_members, rules, .. } =
+							let systems_g = systems.read().await;
-								systems_g.get_mut(&sys_id).expect("System not found");
+							let mut lazy_members =
+								systems_g.get(&sys_id).expect("System not found").lazy_members.lock().await;
 							let (path, cb) = match lazy_members.insert(tree_id, MemberRecord::Res) {
 								None => panic!("Tree for ID not found"),
 								Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
 								Some(MemberRecord::Gen(path, cb)) => (path, cb),
 							};
-							let tree = cb.build(Sym::new(path.clone(), &i).await.unwrap()).await;
+							let tree = cb.build(Sym::new(path.clone(), &i).await.unwrap(), sys_ctx.clone()).await;
-							let mut tia_ctx = TIACtxImpl {
+							let mut tia_ctx = TreeIntoApiCtxImpl {
 								sys: sys_ctx,
 								path: Substack::Bottom,
 								basepath: &path,
-								lazy_members,
+								lazy_members: &mut lazy_members,
-								rules,
-								req: &hand,
 							};
 							hand.handle(&get_tree, &tree.into_api(&mut tia_ctx).await).await
 						},
-						api::HostExtReq::VfsReq(api::VfsReq::GetVfs(get_vfs @ api::GetVfs(sys_id))) => {
-							let systems_g = systems.lock().await;
-							hand.handle(&get_vfs, &systems_g[&sys_id].declfs).await
-						},
 						api::HostExtReq::SysReq(api::SysReq::SysFwded(fwd)) => {
 							let api::SysFwded(sys_id, payload) = fwd;
-							let ctx = mk_ctx(sys_id, hand.reqnot()).await;
+							let ctx = get_ctx(sys_id).await;
-							let sys = ctx.cted.inst();
+							let sys = ctx.cted().inst();
 							sys.dyn_request(hand, payload).await
 						},
-						api::HostExtReq::VfsReq(api::VfsReq::VfsRead(vfs_read)) => {
+						api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, src, text, pos, id }) => {
-							let api::VfsRead(sys_id, vfs_id, path) = &vfs_read;
+							let mut sys_ctx = get_ctx(sys).await;
-							let ctx = mk_ctx(*sys_id, hand.reqnot()).await;
-							let systems_g = systems.lock().await;
-							let path = join_all(path.iter().map(|t| Tok::from_api(*t, &i))).await;
-							let vfs = systems_g[sys_id].vfses[vfs_id].load(&path, ctx).await;
-							hand.handle(&vfs_read, &vfs).await
-						},
-						api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, text, pos, id }) => {
-							let systems_g = systems.lock().await;
-							let lexers = systems_g[&sys].cted.inst().dyn_lexers();
-							mem::drop(systems_g);
 							let text = Tok::from_api(text, &i).await;
-							let ctx = LexContext { sys, id, pos, reqnot: hand.reqnot(), text: &text, i: &i };
+							let src = Sym::from_api(src, sys_ctx.i()).await;
+							let rep = Reporter::new();
+							let expr_store = BorrowedExprStore::new();
 							let trigger_char = text.chars().nth(pos as usize).unwrap();
-							let err_na = err_not_applicable(&i).await;
+							let ekey_na = ekey_not_applicable(&i).await;
-							let err_cascade = err_cascade(&i).await;
+							let ekey_cascade = ekey_cascade(&i).await;
+							let lexers = sys_ctx.cted().inst().dyn_lexers();
 							for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char)) {
+								let ctx = LexContext {
+									id,
+									pos,
+									text: &text,
+									src: src.clone(),
+									ctx: sys_ctx.clone(),
+									rep: &rep,
+									exprs: &expr_store,
+								};
 								match lx.lex(&text[pos as usize..], &ctx).await {
-									Err(e) if e.any(|e| *e == err_na) => continue,
+									Err(e) if e.any(|e| *e == ekey_na) => continue,
 									Err(e) => {
-										let eopt = e.keep_only(|e| *e != err_cascade).map(|e| Err(e.to_api()));
+										let eopt = e.keep_only(|e| *e != ekey_cascade).map(|e| Err(e.to_api()));
+										expr_store.dispose().await;
 										return hand.handle(&lex, &eopt).await;
 									},
 									Ok((s, expr)) => {
-										let ctx = mk_ctx(sys, hand.reqnot()).await;
+										let expr = expr.into_api(&mut (), &mut sys_ctx).await;
-										let expr = expr
-											.to_api(&mut |f, r| {
-												clone!(ctx; async move { do_extra(f, r, ctx).await }).boxed_local()
-											})
-											.await;
 										let pos = (text.len() - s.len()) as u32;
+										expr_store.dispose().await;
 										return hand.handle(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
 									},
 								}
 							}
 							writeln!(logger, "Got notified about n/a character '{trigger_char}'");
+							expr_store.dispose().await;
 							hand.handle(&lex, &None).await
 						},
 						api::HostExtReq::ParseLine(pline) => {
-							let api::ParseLine { exported, comments, sys, line } = &pline;
+							let api::ParseLine { module, src, exported, comments, sys, line, idx } = &pline;
-							let mut ctx = mk_ctx(*sys, hand.reqnot()).await;
+							let ctx = get_ctx(*sys).await;
-							let parsers = ctx.cted.inst().dyn_parsers();
+							let parsers = ctx.cted().inst().dyn_parsers();
-							let comments = join_all(comments.iter().map(|c| Comment::from_api(c, &i))).await;
+							let src = Sym::from_api(*src, ctx.i()).await;
-							let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx, &i).await;
+							let comments =
-							let snip = Snippet::new(line.first().expect("Empty line"), &line, &i);
+								join_all(comments.iter().map(|c| Comment::from_api(c, src.clone(), &i))).await;
-							let (head, tail) = snip.pop_front().unwrap();
+							let expr_store = BorrowedExprStore::new();
-							let name = if let GenTok::Name(n) = &head.tok { n } else { panic!("No line head") };
+							let mut from_api_ctx = (ctx.clone(), &expr_store);
-							let parser =
+							let line: Vec<PTokTree> =
-								parsers.iter().find(|p| p.line_head() == **name).expect("No parser candidate");
+								ttv_from_api(line, &mut from_api_ctx, &mut (), &src, &i).await;
-							let o_line = match parser.parse(*exported, comments, tail) {
+							let snip = Snippet::new(line.first().expect("Empty line"), &line);
+							let parser = parsers[*idx as usize];
+							let module = Sym::from_api(*module, ctx.i()).await;
+							let reporter = Reporter::new();
+							let pctx = ParsCtx::new(ctx.clone(), module, &reporter);
+							let parse_res = parser.parse(pctx, *exported, comments, snip).await;
+							let o_line = match reporter.merge(parse_res) {
 								Err(e) => Err(e.to_api()),
-								Ok(t) => Ok(
+								Ok(t) => Ok(linev_into_api(t, ctx.clone()).await),
-									ttv_to_api(t, &mut |f, range| {
-										clone!(ctx);
-										async move {
-											api::TokenTree { range, token: api::Token::Atom(f.clone().build(ctx).await) }
-										}
-										.boxed_local()
-									})
-									.await,
-								),
 							};
+							mem::drop(line);
+							expr_store.dispose().await;
 							hand.handle(&pline, &o_line).await
 						},
+						api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst(sys, id)) => {
+							let ctx = get_ctx(sys).await;
+							let cnst = get_const(id, ctx.clone()).await;
+							hand.handle(fpc, &cnst.api_return(ctx).await).await
+						},
 						api::HostExtReq::AtomReq(atom_req) => {
 							let atom = atom_req.get_atom();
 							let atom_req = atom_req.clone();
-							with_atom_record(&mk_ctx, hand.reqnot(), atom, move |nfo, ctx, id, buf| {
+							with_atom_record(&get_ctx, atom, async move |nfo, ctx, id, buf| {
-								async move {
 								let actx = AtomCtx(buf, atom.drop, ctx.clone());
 								match &atom_req {
 									api::AtomReq::SerializeAtom(ser) => {
 										let mut buf = enc_vec(&id).await;
-											let refs_opt = nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await;
+										match nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await {
-											hand.handle(ser, &refs_opt.map(|refs| (buf, refs))).await
+											None => hand.handle(ser, &None).await,
+											Some(refs) => {
+												let refs =
+													join_all(refs.into_iter().map(|ex| async { ex.into_api(&mut ()).await }))
+														.await;
+												hand.handle(ser, &Some((buf, refs))).await
+											},
+										}
 									},
 									api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
 										hand.handle(print, &nfo.print(actx).await.to_api()).await,
@@ -363,68 +348,49 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 										hand.handle(fwded, &some.then_some(reply)).await
 									},
 									api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
-											let ret = nfo.call_ref(actx, *arg).await;
+										let expr_store = BorrowedExprStore::new();
-											hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
+										let expr_handle = ExprHandle::borrowed(ctx.clone(), *arg, &expr_store);
+										let ret = nfo.call_ref(actx, Expr::from_handle(expr_handle.clone())).await;
+										let api_expr = ret.api_return(ctx.clone()).await;
+										mem::drop(expr_handle);
+										expr_store.dispose().await;
+										hand.handle(call, &api_expr).await
 									},
 									api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
-											let ret = nfo.call(actx, *arg).await;
+										let expr_store = BorrowedExprStore::new();
-											hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
+										let expr_handle = ExprHandle::borrowed(ctx.clone(), *arg, &expr_store);
+										let ret = nfo.call(actx, Expr::from_handle(expr_handle.clone())).await;
+										let api_expr = ret.api_return(ctx.clone()).await;
+										mem::drop(expr_handle);
+										expr_store.dispose().await;
+										hand.handle(call, &api_expr).await
 									},
 									api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
 										Err(e) => hand.handle(cmd, &Err(e.to_api())).await,
 										Ok(opt) => match opt {
 											None => hand.handle(cmd, &Ok(api::NextStep::Halt)).await,
 											Some(cont) => {
-													let cont = cont.api_return(ctx.clone(), &hand).await;
+												let cont = cont.api_return(ctx.clone()).await;
 												hand.handle(cmd, &Ok(api::NextStep::Continue(cont))).await
 											},
 										},
 									},
 								}
-								}
-								.boxed_local()
 							})
 							.await
 						},
 						api::HostExtReq::DeserAtom(deser) => {
 							let api::DeserAtom(sys, buf, refs) = &deser;
 							let mut read = &mut &buf[..];
-							let ctx = mk_ctx(*sys, hand.reqnot()).await;
+							let ctx = get_ctx(*sys).await;
+							// SAFETY: deserialization implicitly grants ownership to previously owned exprs
+							let refs = (refs.iter())
+								.map(|tk| Expr::from_handle(ExprHandle::deserialize(ctx.clone(), *tk)))
+								.collect_vec();
 							let id = AtomTypeId::decode(Pin::new(&mut read)).await;
-							let inst = ctx.cted.inst();
+							let inst = ctx.cted().inst();
 							let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
-							hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, refs).await).await
+							hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, &refs).await).await
-						},
-						orchid_api::HostExtReq::ApplyMacro(am) => {
-							let tok = hand.will_handle_as(&am);
-							let ApplyMacro { id, params, run_id, sys } = am;
-							let sys_ctx = mk_ctx(sys, hand.reqnot()).await;
-							let mut ctx =
-								RuleCtx { args: ahash::HashMap::default(), run_id, sys: sys_ctx.clone() };
-							for (k, v) in params {
-								ctx.args.insert(
-									Tok::from_api(k, &i).await,
-									mtreev_from_api(&v, &i, &mut |_| panic!("No atom in macro prompt!")).await,
-								);
-							}
-							let err_cascade = err_cascade(&i).await;
-							let systems_g = systems.lock().await;
-							let rule = &systems_g[&sys].rules[&id];
-							match (rule.apply)(ctx).await {
-								Err(e) => {
-									let new_errors = e.keep_only(|e| *e != err_cascade);
-									hand.handle_as(tok, &new_errors.map(|e| Err(e.to_api()))).await
-								},
-								Ok(t) => {
-									let result = mtreev_to_api(&t, &mut |a| {
-										clone!(sys_ctx; async move {
-											api::MacroToken::Atom(a.clone().build(sys_ctx.clone()).await)
-										}.boxed_local())
-									})
-									.await;
-									hand.handle_as(tok, &Some(Ok(result))).await
-								},
-							}
 						},
 					}
 				}
@@ -432,9 +398,24 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 			}
 		},
 	);
-	*interner_cell.borrow_mut() = Some(Rc::new(Interner::new_replica(rn.clone().map())));
+	*interner_cell.borrow_mut() =
-	while !exiting.load(Ordering::Relaxed) {
+		Some(Interner::new_replica(rn.clone().map(|ir: api::IntReq| ir.into_root())));
-		let rcvd = recv_parent_msg().await.unwrap();
+	spawner(Box::pin(clone!(spawner; async move {
-		spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd).await })))
+		let mut streams = stream_select! { in_recv.map(Some), exit_recv.map(|_| None) };
+		while let Some(item) = streams.next().await {
+			match item {
+				Some(rcvd) => spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd[..]).await }))),
+				None => break,
+			}
+		}
+	})));
+	ExtInit {
+		header: ext_header,
+		port: Box::new(ExtensionOwner {
+			out_recv: Mutex::new(out_recv),
+			out_send,
+			_interner_cell: interner_cell,
+			_systems_lock: systems_lock,
+		}),
 	}
 }

orchid-extension/src/expr.rs

@@ -1,9 +1,11 @@
+use std::cell::RefCell;
 use std::fmt;
+use std::hash::Hash;
 use std::rc::Rc;
 
 use async_once_cell::OnceCell;
 use derive_destructure::destructure;
-use orchid_api::ExtAtomPrint;
+use hashbrown::HashSet;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::location::Pos;
@@ -14,17 +16,70 @@ use crate::atom::ForeignAtom;
 use crate::gen_expr::{GExpr, GExprKind};
 use crate::system::SysCtx;
 
+pub struct BorrowedExprStore(RefCell<Option<HashSet<Rc<ExprHandle>>>>);
+impl BorrowedExprStore {
+	pub(crate) fn new() -> Self { Self(RefCell::new(Some(HashSet::new()))) }
+	pub async fn dispose(self) {
+		let elements = self.0.borrow_mut().take().unwrap();
+		for handle in elements {
+			handle.drop_one().await
+		}
+	}
+}
+impl Drop for BorrowedExprStore {
+	fn drop(&mut self) {
+		if self.0.borrow().is_some() {
+			panic!("This should always be explicitly disposed")
+		}
+	}
+}
+
 #[derive(destructure)]
 pub struct ExprHandle {
 	pub tk: api::ExprTicket,
 	pub ctx: SysCtx,
 }
 impl ExprHandle {
-	pub(crate) fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } }
+	/// This function does not signal to take ownership of the expr.
+	pub fn borrowed(ctx: SysCtx, tk: api::ExprTicket, store: &BorrowedExprStore) -> Rc<Self> {
+		let this = Rc::new(Self { ctx, tk });
+		store.0.borrow_mut().as_mut().unwrap().insert(this.clone());
+		this
+	}
+	pub fn deserialize(ctx: SysCtx, tk: api::ExprTicket) -> Rc<Self> { Rc::new(Self { ctx, tk }) }
 	pub fn get_ctx(&self) -> SysCtx { self.ctx.clone() }
-	pub async fn clone(&self) -> Self {
+	/// Drop one instance of the handle silently; if it's the last one, do
-		self.ctx.reqnot.notify(api::Acquire(self.ctx.id, self.tk)).await;
+	/// nothing, otherwise send an Acquire
-		Self { ctx: self.ctx.clone(), tk: self.tk }
+	pub async fn drop_one(self: Rc<Self>) {
+		match Rc::try_unwrap(self) {
+			Err(rc) => {
+				eprintln!("Extending lifetime for {:?}", rc.tk);
+				rc.ctx.reqnot().notify(api::Acquire(rc.ctx.sys_id(), rc.tk)).await
+			},
+			Ok(hand) => {
+				// avoid calling destructor
+				hand.destructure();
+			},
+		}
+	}
+	/// Drop the handle and get the ticket without a release notification.
+	/// Use this with messages that imply ownership transfer. This function is
+	/// safe because abusing it is a memory leak.
+	pub fn serialize(self) -> api::ExprTicket {
+		eprintln!("Skipping destructor for {:?}", self.tk);
+		self.destructure().0
+	}
+}
+impl Eq for ExprHandle {}
+impl PartialEq for ExprHandle {
+	fn eq(&self, other: &Self) -> bool {
+		self.ctx.sys_id() == other.ctx.sys_id() && self.tk == other.tk
+	}
+}
+impl Hash for ExprHandle {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		self.ctx.sys_id().hash(state);
+		self.tk.hash(state);
 	}
 }
 impl fmt::Debug for ExprHandle {
@@ -34,9 +89,9 @@ impl fmt::Debug for ExprHandle {
 }
 impl Drop for ExprHandle {
 	fn drop(&mut self) {
-		let notif = api::Release(self.ctx.id, self.tk);
+		let notif = api::Release(self.ctx.sys_id(), self.tk);
-		let SysCtx { reqnot, spawner, .. } = self.ctx.clone();
+		let reqnot = self.ctx.reqnot().clone();
-		spawner(Box::pin(async move { reqnot.notify(notif).await }))
+		self.ctx.spawner()(Box::pin(async move { reqnot.notify(notif).await }))
 	}
 }
 
@@ -53,20 +108,20 @@ impl Expr {
 
 	pub async fn data(&self) -> &ExprData {
 		(self.data.get_or_init(async {
-			let details = self.handle.ctx.reqnot.request(api::Inspect { target: self.handle.tk }).await;
+			let details = self.handle.ctx.reqnot().request(api::Inspect { target: self.handle.tk }).await;
-			let pos = Pos::from_api(&details.location, &self.handle.ctx.i).await;
+			let pos = Pos::from_api(&details.location, self.handle.ctx.i()).await;
 			let kind = match details.kind {
 				api::InspectedKind::Atom(a) =>
 					ExprKind::Atom(ForeignAtom::new(self.handle.clone(), a, pos.clone())),
 				api::InspectedKind::Bottom(b) =>
-					ExprKind::Bottom(OrcErrv::from_api(&b, &self.handle.ctx.i).await),
+					ExprKind::Bottom(OrcErrv::from_api(&b, self.handle.ctx.i()).await),
 				api::InspectedKind::Opaque => ExprKind::Opaque,
 			};
 			ExprData { pos, kind }
 		}))
 		.await
 	}
-	pub async fn atom(self) -> Result<ForeignAtom<'static>, Self> {
+	pub async fn atom(self) -> Result<ForeignAtom, Self> {
 		match self.data().await {
 			ExprData { kind: ExprKind::Atom(atom), .. } => Ok(atom.clone()),
 			_ => Err(self),
@@ -75,15 +130,18 @@ impl Expr {
 	pub fn handle(&self) -> Rc<ExprHandle> { self.handle.clone() }
 	pub fn ctx(&self) -> SysCtx { self.handle.ctx.clone() }
 
-	pub fn gen(&self) -> GExpr { GExpr { pos: Pos::SlotTarget, kind: GExprKind::Slot(self.clone()) } }
+	pub fn slot(&self) -> GExpr {
+		GExpr { pos: Pos::SlotTarget, kind: GExprKind::Slot(self.clone()) }
+	}
 }
 impl Format for Expr {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		match &self.data().await.kind {
 			ExprKind::Opaque => "OPAQUE".to_string().into(),
 			ExprKind::Bottom(b) => format!("Bottom({b})").into(),
-			ExprKind::Atom(a) =>
+			ExprKind::Atom(a) => FmtUnit::from_api(
-				FmtUnit::from_api(&self.handle.ctx.reqnot.request(ExtAtomPrint(a.atom.clone())).await),
+				&self.handle.ctx.reqnot().request(api::ExtAtomPrint(a.atom.clone())).await,
+			),
 		}
 	}
 }
@@ -96,7 +154,7 @@ pub struct ExprData {
 
 #[derive(Clone, Debug)]
 pub enum ExprKind {
-	Atom(ForeignAtom<'static>),
+	Atom(ForeignAtom),
 	Bottom(OrcErrv),
 	Opaque,
 }

orchid-extension/src/fs.rs

@@ -1,46 +0,0 @@
-use std::num::NonZero;
-
-use futures::FutureExt;
-use futures::future::LocalBoxFuture;
-use hashbrown::HashMap;
-use orchid_base::interner::{Interner, Tok};
-
-use crate::api;
-use crate::system::SysCtx;
-
-pub trait VirtFS: Send + Sync + 'static {
-	fn load<'a>(
-		&'a self,
-		path: &'a [Tok<String>],
-		ctx: SysCtx,
-	) -> LocalBoxFuture<'a, api::OrcResult<api::Loaded>>;
-}
-
-pub enum DeclFs {
-	Lazy(&'static dyn VirtFS),
-	Mod(&'static [(&'static str, DeclFs)]),
-}
-impl DeclFs {
-	pub async fn to_api_rec(
-		&self,
-		vfses: &mut HashMap<api::VfsId, &'static dyn VirtFS>,
-		i: &Interner,
-	) -> api::EagerVfs {
-		match self {
-			DeclFs::Lazy(fs) => {
-				let vfsc: u16 = vfses.len().try_into().expect("too many vfses (more than u16::MAX)");
-				let id = api::VfsId(NonZero::new(vfsc + 1).unwrap());
-				vfses.insert(id, *fs);
-				api::EagerVfs::Lazy(id)
-			},
-			DeclFs::Mod(children) => {
-				let mut output = std::collections::HashMap::new();
-				for (k, v) in children.iter() {
-					output
-						.insert(i.i::<String>(*k).await.to_api(), v.to_api_rec(vfses, i).boxed_local().await);
-				}
-				api::EagerVfs::Eager(output)
-			},
-		}
-	}
-}

orchid-extension/src/func_atom.rs

@@ -1,40 +1,68 @@
+use std::any::TypeId;
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::future::Future;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use async_std::io::Write;
-use async_std::sync::Mutex;
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
+use futures::{AsyncWrite, FutureExt};
 use itertools::Itertools;
 use never::Never;
 use orchid_api_traits::Encode;
 use orchid_base::clone;
 use orchid_base::error::OrcRes;
-use orchid_base::format::{FmtCtxImpl, Format, take_first};
+use orchid_base::format::{FmtCtx, FmtUnit};
 use orchid_base::name::Sym;
 use trait_set::trait_set;
 
-use crate::atom::{Atomic, MethodSetBuilder};
+use crate::atom::Atomic;
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use crate::conv::ToExpr;
-use crate::expr::{Expr, ExprHandle};
+use crate::coroutine_exec::{ExecHandle, exec};
+use crate::expr::Expr;
 use crate::gen_expr::GExpr;
-use crate::system::SysCtx;
+use crate::system::{SysCtx, SysCtxEntry};
 
 trait_set! {
 	trait FunCB = Fn(Vec<Expr>) -> LocalBoxFuture<'static, OrcRes<GExpr>> + 'static;
 }
 
 pub trait ExprFunc<I, O>: Clone + 'static {
-	const ARITY: u8;
+	fn argtyps() -> &'static [TypeId];
-	fn apply(&self, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
+	fn apply<'a>(&self, hand: ExecHandle<'a>, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
 }
 
-thread_local! {
+#[derive(Default)]
-	static FUNS: Rc<Mutex<HashMap<Sym, (u8, Rc<dyn FunCB>)>>> = Rc::default();
+struct FunsCtx(Mutex<HashMap<Sym, FunRecord>>);
+impl SysCtxEntry for FunsCtx {}
+#[derive(Clone)]
+struct FunRecord {
+	argtyps: &'static [TypeId],
+	fun: Rc<dyn FunCB>,
+}
+
+async fn process_args<I, O, F: ExprFunc<I, O>>(
+	debug: impl AsRef<str> + Clone + 'static,
+	f: F,
+) -> FunRecord {
+	let argtyps = F::argtyps();
+	let fun = Rc::new(move |v: Vec<Expr>| {
+		clone!(f, v mut);
+		exec(debug.clone(), async move |mut hand| {
+			let mut norm_args = Vec::with_capacity(v.len());
+			for (expr, typ) in v.into_iter().zip(argtyps) {
+				if *typ != TypeId::of::<Expr>() {
+					norm_args.push(hand.exec(expr).await?);
+				}
+			}
+			f.apply(hand, norm_args).await
+		})
+		.map(Ok)
+		.boxed_local()
+	});
+	FunRecord { argtyps, fun }
 }
 
 /// An Atom representing a partially applied named native function. These
@@ -46,55 +74,53 @@ thread_local! {
 pub(crate) struct Fun {
 	path: Sym,
 	args: Vec<Expr>,
-	arity: u8,
+	record: FunRecord,
-	fun: Rc<dyn FunCB>,
 }
 impl Fun {
-	pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, f: F) -> Self {
+	pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, ctx: SysCtx, f: F) -> Self {
-		let funs = FUNS.with(|funs| funs.clone());
+		let funs: &FunsCtx = ctx.get_or_default();
-		let mut fung = funs.lock().await;
+		let mut fung = funs.0.lock().await;
-		let fun = if let Some(x) = fung.get(&path) {
+		let record = if let Some(record) = fung.get(&path) {
-			x.1.clone()
+			record.clone()
 		} else {
-			let fun = Rc::new(move |v| clone!(f; async move { f.apply(v).await }.boxed_local()));
+			let record = process_args(path.to_string(), f).await;
-			fung.insert(path.clone(), (F::ARITY, fun.clone()));
+			fung.insert(path.clone(), record.clone());
-			fun
+			record
 		};
-		Self { args: vec![], arity: F::ARITY, path, fun }
+		Self { args: vec![], path, record }
 	}
-	pub fn arity(&self) -> u8 { self.arity }
+	pub fn arity(&self) -> u8 { self.record.argtyps.len() as u8 }
 }
 impl Atomic for Fun {
 	type Data = ();
 	type Variant = OwnedVariant;
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl OwnedAtom for Fun {
 	type Refs = Vec<Expr>;
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-	async fn call_ref(&self, arg: ExprHandle) -> GExpr {
+	async fn call_ref(&self, arg: Expr) -> GExpr {
-		std::io::Write::flush(&mut std::io::stderr()).unwrap();
+		let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
-		let new_args = self.args.iter().cloned().chain([Expr::from_handle(Rc::new(arg))]).collect_vec();
+		if new_args.len() == self.record.argtyps.len() {
-		if new_args.len() == self.arity.into() {
+			(self.record.fun)(new_args).await.to_expr().await
-			(self.fun)(new_args).await.to_expr()
 		} else {
-			Self { args: new_args, arity: self.arity, fun: self.fun.clone(), path: self.path.clone() }
+			Self { args: new_args, record: self.record.clone(), path: self.path.clone() }.to_expr().await
-				.to_expr()
 		}
 	}
-	async fn call(self, arg: ExprHandle) -> GExpr { self.call_ref(arg).await }
+	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
-	async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs {
+	async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl AsyncWrite + ?Sized)>) -> Self::Refs {
 		self.path.to_api().encode(write).await;
 		self.args.clone()
 	}
 	async fn deserialize(mut ctx: impl DeserializeCtx, args: Self::Refs) -> Self {
 		let sys = ctx.sys();
-		let path = Sym::from_api(ctx.decode().await, &sys.i).await;
+		let path = Sym::from_api(ctx.decode().await, sys.i()).await;
-		let (arity, fun) = FUNS.with(|f| f.clone()).lock().await.get(&path).unwrap().clone();
+		let record = (sys.get::<FunsCtx>().0.lock().await.get(&path))
-		Self { args, arity, path, fun }
+			.expect("Function missing during deserialization")
+			.clone();
+		Self { args, path, record }
 	}
-	async fn print(&self, _: SysCtx) -> orchid_base::format::FmtUnit {
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		format!("{}:{}/{}", self.path, self.args.len(), self.arity).into()
+		format!("{}:{}/{}", self.path, self.args.len(), self.arity()).into()
 	}
 }
 
@@ -105,74 +131,78 @@ impl OwnedAtom for Fun {
 #[derive(Clone)]
 pub struct Lambda {
 	args: Vec<Expr>,
-	arity: u8,
+	record: FunRecord,
-	fun: Rc<dyn FunCB>,
 }
 impl Lambda {
-	pub fn new<I, O, F: ExprFunc<I, O>>(f: F) -> Self {
+	pub async fn new<I, O, F: ExprFunc<I, O>>(
-		let fun = Rc::new(move |v| clone!(f; async move { f.apply(v).await }.boxed_local()));
+		debug: impl AsRef<str> + Clone + 'static,
-		Self { args: vec![], arity: F::ARITY, fun }
+		f: F,
+	) -> Self {
+		Self { args: vec![], record: process_args(debug, f).await }
 	}
 }
 impl Atomic for Lambda {
 	type Data = ();
 	type Variant = OwnedVariant;
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl OwnedAtom for Lambda {
 	type Refs = Never;
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-	async fn call_ref(&self, arg: ExprHandle) -> GExpr {
+	async fn call_ref(&self, arg: Expr) -> GExpr {
-		let new_args = self.args.iter().cloned().chain([Expr::from_handle(Rc::new(arg))]).collect_vec();
+		let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
-		if new_args.len() == self.arity.into() {
+		if new_args.len() == self.record.argtyps.len() {
-			(self.fun)(new_args).await.to_expr()
+			(self.record.fun)(new_args).await.to_expr().await
 		} else {
-			Self { args: new_args, arity: self.arity, fun: self.fun.clone() }.to_expr()
+			Self { args: new_args, record: self.record.clone() }.to_expr().await
 		}
 	}
-	async fn call(self, arg: ExprHandle) -> GExpr { self.call_ref(arg).await }
+	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
 }
 
 mod expr_func_derives {
-	use std::future::Future;
+	use std::any::TypeId;
+	use std::sync::OnceLock;
 
 	use orchid_base::error::OrcRes;
 
 	use super::ExprFunc;
 	use crate::conv::{ToExpr, TryFromExpr};
-	use crate::func_atom::Expr;
+	use crate::func_atom::{ExecHandle, Expr};
 	use crate::gen_expr::GExpr;
 
 	macro_rules! expr_func_derive {
-    ($arity: tt, $($t:ident),*) => {
+    ($($t:ident),*) => {
-      paste::paste!{
+      pastey::paste!{
         impl<
-          $($t: TryFromExpr, )*
+          $($t: TryFromExpr + 'static, )*
-					Fut: Future<Output: ToExpr>,
+					Out: ToExpr,
-          Func: Fn($($t,)*) -> Fut + Clone + Send + Sync + 'static
+          Func: AsyncFn($($t,)*) -> Out + Clone + Send + Sync + 'static
-        > ExprFunc<($($t,)*), Fut::Output> for Func {
+        > ExprFunc<($($t,)*), Out> for Func {
-          const ARITY: u8 = $arity;
+          fn argtyps() -> &'static [TypeId] {
-          async fn apply(&self, v: Vec<Expr>) -> OrcRes<GExpr> {
+            static STORE: OnceLock<Vec<TypeId>> = OnceLock::new();
-            assert_eq!(v.len(), Self::ARITY.into(), "Arity mismatch");
+            &*STORE.get_or_init(|| vec![$(TypeId::of::<$t>()),*])
+          }
+          async fn apply<'a>(&self, _: ExecHandle<'a>, v: Vec<Expr>) -> OrcRes<GExpr> {
+            assert_eq!(v.len(), Self::argtyps().len(), "Arity mismatch");
             let [$([< $t:lower >],)*] = v.try_into().unwrap_or_else(|_| panic!("Checked above"));
-            Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_expr())
+            Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_expr().await)
           }
         }
       }
     };
   }
-	expr_func_derive!(1, A);
+	expr_func_derive!(A);
-	expr_func_derive!(2, A, B);
+	expr_func_derive!(A, B);
-	expr_func_derive!(3, A, B, C);
+	expr_func_derive!(A, B, C);
-	expr_func_derive!(4, A, B, C, D);
+	expr_func_derive!(A, B, C, D);
-	expr_func_derive!(5, A, B, C, D, E);
+	expr_func_derive!(A, B, C, D, E);
-	expr_func_derive!(6, A, B, C, D, E, F);
+	// expr_func_derive!(A, B, C, D, E, F);
-	expr_func_derive!(7, A, B, C, D, E, F, G);
+	// expr_func_derive!(A, B, C, D, E, F, G);
-	expr_func_derive!(8, A, B, C, D, E, F, G, H);
+	// expr_func_derive!(A, B, C, D, E, F, G, H);
-	expr_func_derive!(9, A, B, C, D, E, F, G, H, I);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I);
-	expr_func_derive!(10, A, B, C, D, E, F, G, H, I, J);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J);
-	expr_func_derive!(11, A, B, C, D, E, F, G, H, I, J, K);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K);
-	expr_func_derive!(12, A, B, C, D, E, F, G, H, I, J, K, L);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L);
-	expr_func_derive!(13, A, B, C, D, E, F, G, H, I, J, K, L, M);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L, M);
-	expr_func_derive!(14, A, B, C, D, E, F, G, H, I, J, K, L, M, N);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L, M, N);
 }

orchid-extension/src/gen_expr.rs

@@ -1,40 +1,51 @@
-use std::future::Future;
+use std::mem;
+use std::rc::Rc;
 
 use futures::FutureExt;
 use orchid_base::error::{OrcErr, OrcErrv};
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::location::Pos;
-use orchid_base::match_mapping;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::ReqHandlish;
+use orchid_base::{match_mapping, tl_cache};
 
 use crate::api;
 use crate::atom::{AtomFactory, ToAtom};
-use crate::conv::{ToExpr, TryFromExpr};
 use crate::expr::Expr;
-use crate::func_atom::Lambda;
 use crate::system::SysCtx;
 
+#[derive(Clone, Debug)]
 pub struct GExpr {
 	pub kind: GExprKind,
 	pub pos: Pos,
 }
 impl GExpr {
-	pub async fn api_return(self, ctx: SysCtx, hand: &impl ReqHandlish) -> api::Expression {
+	pub async fn api_return(self, ctx: SysCtx) -> api::Expression {
 		if let GExprKind::Slot(ex) = self.kind {
-			hand.defer_drop(ex.handle());
+			let hand = ex.handle();
+			mem::drop(ex);
 			api::Expression {
 				location: api::Location::SlotTarget,
-				kind: api::ExpressionKind::Slot(ex.handle().tk),
+				kind: match Rc::try_unwrap(hand) {
+					Ok(h) => api::ExpressionKind::Slot { tk: h.serialize(), by_value: true },
+					Err(rc) => api::ExpressionKind::Slot { tk: rc.tk, by_value: false },
+				},
 			}
 		} else {
 			api::Expression {
 				location: api::Location::Inherit,
-				kind: self.kind.api_return(ctx, hand).boxed_local().await,
+				kind: self.kind.api_return(ctx).boxed_local().await,
 			}
 		}
 	}
+	pub fn at(self, pos: Pos) -> Self { GExpr { pos, kind: self.kind } }
+}
+impl Format for GExpr {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		self.kind.print(c).await
+	}
 }
 
+#[derive(Clone, Debug)]
 pub enum GExprKind {
 	Call(Box<GExpr>, Box<GExpr>),
 	Lambda(u64, Box<GExpr>),
@@ -46,20 +57,19 @@ pub enum GExprKind {
 	Bottom(OrcErrv),
 }
 impl GExprKind {
-	pub async fn api_return(self, ctx: SysCtx, hand: &impl ReqHandlish) -> api::ExpressionKind {
+	pub async fn api_return(self, ctx: SysCtx) -> api::ExpressionKind {
 		match_mapping!(self, Self => api::ExpressionKind {
 			Call(
-				f => Box::new(f.api_return(ctx.clone(), hand).await),
+				f => Box::new(f.api_return(ctx.clone()).await),
-				x => Box::new(x.api_return(ctx, hand).await)
+				x => Box::new(x.api_return(ctx).await)
 			),
 			Seq(
-				a => Box::new(a.api_return(ctx.clone(), hand).await),
+				a => Box::new(a.api_return(ctx.clone()).await),
-				b => Box::new(b.api_return(ctx, hand).await)
+				b => Box::new(b.api_return(ctx).await)
 			),
-			Lambda(arg, body => Box::new(body.api_return(ctx, hand).await)),
+			Lambda(arg, body => Box::new(body.api_return(ctx).await)),
 			Arg(arg),
 			Const(name.to_api()),
-			Const(name.to_api()),
 			Bottom(err.to_api()),
 			NewAtom(fac.clone().build(ctx).await),
 		} {
@@ -67,6 +77,28 @@ impl GExprKind {
 		})
 	}
 }
+impl Format for GExprKind {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		match self {
+			GExprKind::Call(f, x) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0} ({1})")))
+					.units([f.print(c).await, x.print(c).await]),
+			GExprKind::Lambda(arg, body) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0}.{1}")))
+					.units([arg.to_string().into(), body.print(c).await]),
+			GExprKind::Arg(arg) => arg.to_string().into(),
+			GExprKind::Seq(a, b) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("[{0}] {1}")))
+					.units([a.print(c).await, b.print(c).await]),
+			GExprKind::Const(sym) => sym.to_string().into(),
+			GExprKind::NewAtom(atom_factory) => atom_factory.to_string().into(),
+			GExprKind::Slot(expr) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{{{0}}}")))
+					.units([expr.print(c).await]),
+			GExprKind::Bottom(orc_errv) => orc_errv.to_string().into(),
+		}
+	}
+}
 
 fn inherit(kind: GExprKind) -> GExpr { GExpr { pos: Pos::Inherit, kind } }
 
@@ -99,10 +131,3 @@ pub fn call(v: impl IntoIterator<Item = GExpr>) -> GExpr {
 pub fn bot(ev: impl IntoIterator<Item = OrcErr>) -> GExpr {
 	inherit(GExprKind::Bottom(OrcErrv::new(ev).unwrap()))
 }
-
-pub fn with<I: TryFromExpr, Fut: Future<Output: ToExpr>>(
-	expr: GExpr,
-	cont: impl Fn(I) -> Fut + Clone + Send + Sync + 'static,
-) -> GExpr {
-	call([lambda(0, [seq([arg(0), call([Lambda::new(cont).to_expr(), arg(0)])])]), expr])
-}

orchid-extension/src/lexer.rs

@@ -1,56 +1,82 @@
+use std::fmt;
 use std::future::Future;
-use std::ops::{Range, RangeInclusive};
+use std::ops::RangeInclusive;
 
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_errv};
 use orchid_base::interner::{Interner, Tok};
-use orchid_base::location::Pos;
+use orchid_base::location::{Pos, SrcRange};
-use orchid_base::reqnot::{ReqNot, Requester};
+use orchid_base::name::Sym;
-use orchid_base::tree::TokHandle;
+use orchid_base::parse::ParseCtx;
+use orchid_base::reqnot::Requester;
 
 use crate::api;
-use crate::tree::{GenTok, GenTokTree};
+use crate::expr::BorrowedExprStore;
+use crate::parser::PTokTree;
+use crate::system::SysCtx;
+use crate::tree::GenTokTree;
 
-pub async fn err_cascade(i: &Interner) -> OrcErr {
+pub async fn ekey_cascade(i: &Interner) -> Tok<String> {
-	mk_err(
+	i.i("An error cascading from a recursive call").await
-		i.i("An error cascading from a recursive call").await,
+}
-		"This error is a sentinel for the extension library.\
+pub async fn ekey_not_applicable(i: &Interner) -> Tok<String> {
-		it should not be emitted by the extension.",
+	i.i("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
-		[Pos::None.into()],
+}
-	)
+const MSG_INTERNAL_ERROR: &str = "This error is a sentinel for the extension library.\
+	it should not be emitted by the extension.";
+
+pub async fn err_cascade(i: &Interner) -> OrcErrv {
+	mk_errv(ekey_cascade(i).await, MSG_INTERNAL_ERROR, [Pos::None])
 }
 
-pub async fn err_not_applicable(i: &Interner) -> OrcErr {
+pub async fn err_not_applicable(i: &Interner) -> OrcErrv {
-	mk_err(
+	mk_errv(ekey_not_applicable(i).await, MSG_INTERNAL_ERROR, [Pos::None])
-		i.i("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await,
-		&*err_cascade(i).await.message,
-		[Pos::None.into()],
-	)
 }
 
 pub struct LexContext<'a> {
+	pub(crate) exprs: &'a BorrowedExprStore,
+	pub ctx: SysCtx,
 	pub text: &'a Tok<String>,
-	pub sys: api::SysId,
 	pub id: api::ParsId,
 	pub pos: u32,
-	pub reqnot: ReqNot<api::ExtMsgSet>,
+	pub(crate) src: Sym,
-	pub i: &'a Interner,
+	pub(crate) rep: &'a Reporter,
 }
 impl<'a> LexContext<'a> {
-	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree<'a>)> {
+	pub fn src(&self) -> &Sym { &self.src }
+	/// This function returns [PTokTree] because it can never return
+	/// [orchid_base::tree::Token::NewExpr]. You can use
+	/// [crate::parser::p_tree2gen] to convert this to [crate::tree::GenTokTree]
+	/// for embedding in the return value.
+	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, PTokTree)> {
 		let start = self.pos(tail);
-		let Some(lx) = self.reqnot.request(api::SubLex { pos: start, id: self.id }).await else {
+		let Some(lx) = self.ctx.reqnot().request(api::SubLex { pos: start, id: self.id }).await else {
-			return Err(err_cascade(self.i).await.into());
+			return Err(err_cascade(self.ctx.i()).await);
 		};
-		Ok((&self.text[lx.pos as usize..], GenTok::Slot(TokHandle::new(lx.ticket)).at(start..lx.pos)))
+		let tree = PTokTree::from_api(
+			&lx.tree,
+			&mut (self.ctx.clone(), self.exprs),
+			&mut (),
+			&self.src,
+			self.ctx.i(),
+		)
+		.await;
+		Ok((&self.text[lx.pos as usize..], tree))
 	}
 
 	pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }
-
+	pub fn pos_tt(&self, tail_with: &'a str, tail_without: &'a str) -> SrcRange {
-	pub fn tok_ran(&self, len: u32, tail: &'a str) -> Range<u32> {
+		SrcRange::new(self.pos(tail_with)..self.pos(tail_without), &self.src)
-		self.pos(tail) - len..self.pos(tail)
 	}
+
+	pub fn pos_lt(&self, len: impl TryInto<u32, Error: fmt::Debug>, tail: &'a str) -> SrcRange {
+		SrcRange::new(self.pos(tail) - len.try_into().unwrap()..self.pos(tail), &self.src)
+	}
+}
+impl ParseCtx for LexContext<'_> {
+	fn i(&self) -> &Interner { self.ctx.i() }
+	fn rep(&self) -> &Reporter { self.rep }
 }
 
 pub trait Lexer: Send + Sync + Sized + Default + 'static {
@@ -58,7 +84,7 @@ pub trait Lexer: Send + Sync + Sized + Default + 'static {
 	fn lex<'a>(
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> impl Future<Output = OrcRes<(&'a str, GenTokTree<'a>)>>;
+	) -> impl Future<Output = OrcRes<(&'a str, GenTokTree)>>;
 }
 
 pub trait DynLexer: Send + Sync + 'static {
@@ -67,7 +93,7 @@ pub trait DynLexer: Send + Sync + 'static {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree<'a>)>>;
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>>;
 }
 
 impl<T: Lexer> DynLexer for T {
@@ -76,7 +102,7 @@ impl<T: Lexer> DynLexer for T {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree<'a>)>> {
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>> {
 		T::lex(tail, ctx).boxed_local()
 	}
 }

orchid-extension/src/lib.rs

@@ -4,16 +4,17 @@ pub mod atom;
 pub mod atom_owned;
 pub mod atom_thin;
 pub mod conv;
+pub mod coroutine_exec;
 pub mod entrypoint;
 pub mod expr;
-pub mod fs;
 pub mod func_atom;
 pub mod gen_expr;
 pub mod lexer;
-pub mod macros;
+// pub mod msg;
-pub mod msg;
 pub mod other_system;
 pub mod parser;
+pub mod reflection;
 pub mod system;
 pub mod system_ctor;
+pub mod tokio;
 pub mod tree;

orchid-extension/src/macros.rs

@@ -1,102 +0,0 @@
-use std::rc::Rc;
-
-use ahash::HashMap;
-use futures::future::{LocalBoxFuture, join_all};
-use itertools::Itertools;
-use never::Never;
-use orchid_base::error::OrcRes;
-use orchid_base::interner::Tok;
-use orchid_base::macros::{MTree, mtreev_from_api, mtreev_to_api};
-use orchid_base::reqnot::Requester;
-use trait_set::trait_set;
-
-use crate::api;
-use crate::atom::AtomFactory;
-use crate::lexer::err_cascade;
-use crate::system::SysCtx;
-use crate::tree::TreeIntoApiCtx;
-
-pub trait Macro {
-	fn pattern() -> MTree<'static, Never>;
-	fn apply(binds: HashMap<Tok<String>, MTree<'_, Never>>) -> MTree<'_, AtomFactory>;
-}
-
-pub trait DynMacro {
-	fn pattern(&self) -> MTree<'static, Never>;
-	fn apply<'a>(&self, binds: HashMap<Tok<String>, MTree<'a, Never>>) -> MTree<'a, AtomFactory>;
-}
-
-impl<T: Macro> DynMacro for T {
-	fn pattern(&self) -> MTree<'static, Never> { Self::pattern() }
-	fn apply<'a>(&self, binds: HashMap<Tok<String>, MTree<'a, Never>>) -> MTree<'a, AtomFactory> {
-		Self::apply(binds)
-	}
-}
-
-pub struct RuleCtx<'a> {
-	pub(crate) args: HashMap<Tok<String>, Vec<MTree<'a, Never>>>,
-	pub(crate) run_id: api::ParsId,
-	pub(crate) sys: SysCtx,
-}
-impl<'a> RuleCtx<'a> {
-	pub async fn recurse(&mut self, tree: &[MTree<'a, Never>]) -> OrcRes<Vec<MTree<'a, Never>>> {
-		let req = api::RunMacros {
-			run_id: self.run_id,
-			query: mtreev_to_api(tree, &mut |b| match *b {}).await,
-		};
-		let Some(treev) = self.sys.reqnot.request(req).await else {
-			return Err(err_cascade(&self.sys.i).await.into());
-		};
-		static ATOM_MSG: &str = "Returned atom from Rule recursion";
-		Ok(mtreev_from_api(&treev, &self.sys.i, &mut |_| panic!("{ATOM_MSG}")).await)
-	}
-	pub fn getv(&mut self, key: &Tok<String>) -> Vec<MTree<'a, Never>> {
-		self.args.remove(key).expect("Key not found")
-	}
-	pub fn gets(&mut self, key: &Tok<String>) -> MTree<'a, Never> {
-		let v = self.getv(key);
-		assert!(v.len() == 1, "Not a scalar");
-		v.into_iter().next().unwrap()
-	}
-	pub fn unused_arg<'b>(&mut self, keys: impl IntoIterator<Item = &'b Tok<String>>) {
-		keys.into_iter().for_each(|k| {
-			self.getv(k);
-		});
-	}
-}
-
-trait_set! {
-	pub trait RuleCB = for<'a> Fn(RuleCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Vec<MTree<'a, AtomFactory>>>>;
-}
-
-pub struct Rule {
-	pub(crate) comments: Vec<String>,
-	pub(crate) pattern: Vec<MTree<'static, Never>>,
-	pub(crate) apply: Rc<dyn RuleCB>,
-}
-impl Rule {
-	pub(crate) async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MacroRule {
-		api::MacroRule {
-			comments: join_all(self.comments.iter().map(|c| async {
-				api::Comment { text: ctx.sys().i.i(c).await.to_api(), location: api::Location::Inherit }
-			}))
-			.await,
-			location: api::Location::Inherit,
-			pattern: mtreev_to_api(&self.pattern, &mut |b| match *b {}).await,
-			id: ctx.with_rule(Rc::new(self)),
-		}
-	}
-}
-
-pub fn rule_cmt<'a>(
-	cmt: impl IntoIterator<Item = &'a str>,
-	pattern: Vec<MTree<'static, Never>>,
-	apply: impl RuleCB + 'static,
-) -> Rule {
-	let comments = cmt.into_iter().map(|s| s.to_string()).collect_vec();
-	Rule { comments, pattern, apply: Rc::new(apply) }
-}
-
-pub fn rule(pattern: Vec<MTree<'static, Never>>, apply: impl RuleCB + 'static) -> Rule {
-	rule_cmt([], pattern, apply)
-}

orchid-extension/src/msg.rs

@@ -1,12 +1,9 @@
 use std::pin::pin;
 
 use async_once_cell::OnceCell;
-use async_std::io::{self, Stdout};
+use futures::lock::Mutex;
-use async_std::sync::Mutex;
 use orchid_base::msg::{recv_msg, send_msg};
 
-static STDOUT: OnceCell<Mutex<Stdout>> = OnceCell::new();
-
 pub async fn send_parent_msg(msg: &[u8]) -> io::Result<()> {
 	let stdout_lk = STDOUT.get_or_init(async { Mutex::new(io::stdout()) }).await;
 	let mut stdout_g = stdout_lk.lock().await;

orchid-extension/src/parser.rs

@@ -1,40 +1,221 @@
-use orchid_base::error::OrcRes;
+use std::marker::PhantomData;
-use orchid_base::parse::{Comment, Snippet};
 
-use crate::atom::{AtomFactory, ForeignAtom};
+use async_fn_stream::stream;
-use crate::tree::GenTokTree;
+use futures::future::{LocalBoxFuture, join_all};
+use futures::{FutureExt, Stream, StreamExt};
+use itertools::Itertools;
+use never::Never;
+use orchid_base::error::{OrcErrv, OrcRes, Reporter};
+use orchid_base::id_store::IdStore;
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::SrcRange;
+use orchid_base::match_mapping;
+use orchid_base::name::Sym;
+use orchid_base::parse::{Comment, ParseCtx, Snippet};
+use orchid_base::reqnot::Requester;
+use orchid_base::tree::{TokTree, Token, ttv_into_api};
 
-pub type GenSnippet<'a> = Snippet<'a, 'a, ForeignAtom<'a>, AtomFactory>;
+use crate::api;
+use crate::conv::ToExpr;
+use crate::expr::Expr;
+use crate::gen_expr::GExpr;
+use crate::system::{SysCtx, SysCtxEntry};
+use crate::tree::{GenTok, GenTokTree};
+
+pub type PTok = Token<Expr, Never>;
+pub type PTokTree = TokTree<Expr, Never>;
+pub type PSnippet<'a> = Snippet<'a, Expr, Never>;
+
+pub fn p_tok2gen(tok: PTok) -> GenTok {
+	match_mapping!(tok, PTok => GenTok {
+		Comment(s), Name(n), BR, Handle(ex), Bottom(err),
+		LambdaHead(arg => Box::new(p_tree2gen(*arg))),
+		NS(n, arg => Box::new(p_tree2gen(*arg))),
+		S(p, body () p_v2gen),
+	} {
+		PTok::NewExpr(never) => match never {}
+	})
+}
+pub fn p_tree2gen(tree: PTokTree) -> GenTokTree {
+	TokTree { tok: p_tok2gen(tree.tok), sr: tree.sr }
+}
+pub fn p_v2gen(v: impl IntoIterator<Item = PTokTree>) -> Vec<GenTokTree> {
+	v.into_iter().map(p_tree2gen).collect_vec()
+}
 
 pub trait Parser: Send + Sync + Sized + Default + 'static {
 	const LINE_HEAD: &'static str;
-	fn parse(
+	fn parse<'a>(
+		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'_>,
+		line: PSnippet<'a>,
-	) -> OrcRes<Vec<GenTokTree<'_>>>;
+	) -> impl Future<Output = OrcRes<Vec<ParsedLine>>> + 'a;
 }
 
 pub trait DynParser: Send + Sync + 'static {
 	fn line_head(&self) -> &'static str;
 	fn parse<'a>(
 		&self,
+		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'a>,
+		line: PSnippet<'a>,
-	) -> OrcRes<Vec<GenTokTree<'a>>>;
+	) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>>;
 }
 
 impl<T: Parser> DynParser for T {
 	fn line_head(&self) -> &'static str { Self::LINE_HEAD }
 	fn parse<'a>(
 		&self,
+		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'a>,
+		line: PSnippet<'a>,
-	) -> OrcRes<Vec<GenTokTree<'a>>> {
+	) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>> {
-		Self::parse(exported, comments, line)
+		Box::pin(async move { Self::parse(ctx, exported, comments, line).await })
 	}
 }
 
 pub type ParserObj = &'static dyn DynParser;
+
+pub struct ParsCtx<'a> {
+	_parse: PhantomData<&'a mut ()>,
+	ctx: SysCtx,
+	module: Sym,
+	reporter: &'a Reporter,
+}
+impl<'a> ParsCtx<'a> {
+	pub(crate) fn new(ctx: SysCtx, module: Sym, reporter: &'a Reporter) -> Self {
+		Self { _parse: PhantomData, ctx, module, reporter }
+	}
+	pub fn ctx(&self) -> &SysCtx { &self.ctx }
+	pub fn module(&self) -> Sym { self.module.clone() }
+}
+impl ParseCtx for ParsCtx<'_> {
+	fn i(&self) -> &Interner { self.ctx.i() }
+	fn rep(&self) -> &Reporter { self.reporter }
+}
+
+type BoxConstCallback = Box<dyn FnOnce(ConstCtx) -> LocalBoxFuture<'static, GExpr>>;
+
+#[derive(Default)]
+pub(crate) struct ParsedConstCtxEntry {
+	pub(crate) consts: IdStore<BoxConstCallback>,
+}
+impl SysCtxEntry for ParsedConstCtxEntry {}
+
+pub struct ParsedLine {
+	pub sr: SrcRange,
+	pub comments: Vec<Comment>,
+	pub kind: ParsedLineKind,
+}
+impl ParsedLine {
+	pub fn cnst<'a, R: ToExpr + 'static, F: AsyncFnOnce(ConstCtx) -> R + 'static>(
+		sr: &SrcRange,
+		comments: impl IntoIterator<Item = &'a Comment>,
+		exported: bool,
+		name: Tok<String>,
+		f: F,
+	) -> Self {
+		let cb = Box::new(|ctx| async move { f(ctx).await.to_expr().await }.boxed_local());
+		let kind = ParsedLineKind::Mem(ParsedMem { name, exported, kind: ParsedMemKind::Const(cb) });
+		let comments = comments.into_iter().cloned().collect();
+		ParsedLine { comments, sr: sr.clone(), kind }
+	}
+	pub fn module<'a>(
+		sr: &SrcRange,
+		comments: impl IntoIterator<Item = &'a Comment>,
+		exported: bool,
+		name: &Tok<String>,
+		use_prelude: bool,
+		lines: impl IntoIterator<Item = ParsedLine>,
+	) -> Self {
+		let mem_kind = ParsedMemKind::Mod { lines: lines.into_iter().collect(), use_prelude };
+		let line_kind = ParsedLineKind::Mem(ParsedMem { name: name.clone(), exported, kind: mem_kind });
+		let comments = comments.into_iter().cloned().collect();
+		ParsedLine { comments, sr: sr.clone(), kind: line_kind }
+	}
+	pub async fn into_api(self, mut ctx: SysCtx) -> api::ParsedLine {
+		api::ParsedLine {
+			comments: self.comments.into_iter().map(|c| c.to_api()).collect(),
+			source_range: self.sr.to_api(),
+			kind: match self.kind {
+				ParsedLineKind::Mem(mem) => api::ParsedLineKind::Member(api::ParsedMember {
+					name: mem.name.to_api(),
+					exported: mem.exported,
+					kind: match mem.kind {
+						ParsedMemKind::Const(cb) => api::ParsedMemberKind::Constant(api::ParsedConstId(
+							ctx.get_or_default::<ParsedConstCtxEntry>().consts.add(cb).id(),
+						)),
+						ParsedMemKind::Mod { lines, use_prelude } => api::ParsedMemberKind::Module {
+							lines: linev_into_api(lines, ctx).boxed_local().await,
+							use_prelude,
+						},
+					},
+				}),
+				ParsedLineKind::Rec(tv) =>
+					api::ParsedLineKind::Recursive(ttv_into_api(tv, &mut (), &mut ctx).await),
+			},
+		}
+	}
+}
+
+pub(crate) async fn linev_into_api(v: Vec<ParsedLine>, ctx: SysCtx) -> Vec<api::ParsedLine> {
+	join_all(v.into_iter().map(|l| l.into_api(ctx.clone()))).await
+}
+
+pub enum ParsedLineKind {
+	Mem(ParsedMem),
+	Rec(Vec<GenTokTree>),
+}
+
+pub struct ParsedMem {
+	pub name: Tok<String>,
+	pub exported: bool,
+	pub kind: ParsedMemKind,
+}
+
+pub enum ParsedMemKind {
+	Const(BoxConstCallback),
+	Mod { lines: Vec<ParsedLine>, use_prelude: bool },
+}
+
+#[derive(Clone)]
+pub struct ConstCtx {
+	ctx: SysCtx,
+	constid: api::ParsedConstId,
+}
+impl ConstCtx {
+	pub fn ctx(&self) -> &SysCtx { &self.ctx }
+	pub fn i(&self) -> &Interner { self.ctx.i() }
+	pub fn names<'b>(
+		&'b self,
+		names: impl IntoIterator<Item = &'b Sym> + 'b,
+	) -> impl Stream<Item = OrcRes<Sym>> + 'b {
+		let resolve_names = api::ResolveNames {
+			constid: self.constid,
+			sys: self.ctx.sys_id(),
+			names: names.into_iter().map(|n| n.to_api()).collect_vec(),
+		};
+		stream(async |mut cx| {
+			for name_opt in self.ctx.reqnot().request(resolve_names).await {
+				cx.emit(match name_opt {
+					Err(e) => Err(OrcErrv::from_api(&e, self.ctx.i()).await),
+					Ok(name) => Ok(Sym::from_api(name, self.ctx.i()).await),
+				})
+				.await
+			}
+		})
+	}
+	pub async fn names_n<const N: usize>(&self, names: [&Sym; N]) -> [OrcRes<Sym>; N] {
+		self.names(names).collect::<Vec<_>>().await.try_into().expect("Lengths must match")
+	}
+}
+
+pub(crate) async fn get_const(id: api::ParsedConstId, ctx: SysCtx) -> GExpr {
+	let ent = ctx.get_or_default::<ParsedConstCtxEntry>();
+	let rec = ent.consts.get(id.0).expect("Bad ID or double read of parsed const");
+	let ctx = ConstCtx { constid: id, ctx: ctx.clone() };
+	rec.remove()(ctx).await
+}

orchid-extension/src/reflection.rs

@@ -0,0 +1,163 @@
+use std::cell::OnceCell;
+use std::rc::Rc;
+
+use futures::FutureExt;
+use futures::lock::Mutex;
+use memo_map::MemoMap;
+use orchid_base::interner::Tok;
+use orchid_base::name::{NameLike, VPath};
+use orchid_base::reqnot::Requester;
+
+use crate::api;
+use crate::system::{SysCtx, SysCtxEntry, WeakSysCtx};
+
+pub struct ReflMemData {
+	// None for inferred steps
+	public: OnceCell<bool>,
+	kind: ReflMemKind,
+}
+#[derive(Clone)]
+pub struct ReflMem(Rc<ReflMemData>);
+impl ReflMem {
+	pub fn kind(&self) -> ReflMemKind { self.0.kind.clone() }
+}
+
+#[derive(Clone)]
+pub enum ReflMemKind {
+	Const,
+	Mod(ReflMod),
+}
+
+pub struct ReflModData {
+	inferred: Mutex<bool>,
+	path: VPath,
+	ctx: WeakSysCtx,
+	members: MemoMap<Tok<String>, ReflMem>,
+}
+
+#[derive(Clone)]
+pub struct ReflMod(Rc<ReflModData>);
+impl ReflMod {
+	fn ctx(&self) -> SysCtx {
+		self.0.ctx.upgrade().expect("ReflectedModule accessed after context drop")
+	}
+	pub fn path(&self) -> &[Tok<String>] { &self.0.path[..] }
+	pub fn is_root(&self) -> bool { self.0.path.is_empty() }
+	async fn try_populate(&self) -> Result<(), api::LsModuleError> {
+		let ctx = self.ctx();
+		let path_tok = ctx.i().i(&self.0.path[..]).await;
+		let reply = match ctx.reqnot().request(api::LsModule(ctx.sys_id(), path_tok.to_api())).await {
+			Err(api::LsModuleError::TreeUnavailable) =>
+				panic!("Reflected tree accessed outside an interpreter call. This extension is faulty."),
+			Err(err) => return Err(err),
+			Ok(details) => details,
+		};
+		for (k, v) in reply.members {
+			let k = ctx.i().ex(k).await;
+			let mem = match self.0.members.get(&k) {
+				Some(mem) => mem,
+				None => {
+					let path = self.0.path.clone().name_with_suffix(k.clone()).to_sym(ctx.i()).await;
+					let kind = match v.kind {
+						api::MemberInfoKind::Constant => ReflMemKind::Const,
+						api::MemberInfoKind::Module =>
+							ReflMemKind::Mod(default_module(&ctx, VPath::new(path.segs()))),
+					};
+					self.0.members.get_or_insert(&k, || default_member(self.is_root(), kind))
+				},
+			};
+			let _ = mem.0.public.set(v.public);
+		}
+		Ok(())
+	}
+	pub async fn get_child(&self, key: &Tok<String>) -> Option<ReflMem> {
+		let inferred_g = self.0.inferred.lock().await;
+		if let Some(mem) = self.0.members.get(key) {
+			return Some(mem.clone());
+		}
+		if !*inferred_g {
+			return None;
+		}
+		match self.try_populate().await {
+			Err(api::LsModuleError::InvalidPath) =>
+				panic!("Path became invalid since module was created"),
+			Err(api::LsModuleError::IsConstant) =>
+				panic!("Path previously contained a module but now contains a constant"),
+			Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
+			Ok(()) => (),
+		}
+		self.0.members.get(key).cloned()
+	}
+	pub async fn get_by_path(&self, path: &[Tok<String>]) -> Result<ReflMem, InvalidPathError> {
+		let ctx = self.ctx();
+		let (next, tail) = path.split_first().expect("Attempted to walk by empty path");
+		let inferred_g = self.0.inferred.lock().await;
+		if let Some(next) = self.0.members.get(next) {
+			return if tail.is_empty() {
+				Ok(next.clone())
+			} else {
+				match next.kind() {
+					ReflMemKind::Const => Err(InvalidPathError { keep_ancestry: true }),
+					ReflMemKind::Mod(m) => m.get_by_path(tail).boxed_local().await,
+				}
+			};
+		}
+		if !*inferred_g {
+			return Err(InvalidPathError { keep_ancestry: true });
+		}
+		let candidate = default_module(&ctx, self.0.path.clone().suffix([next.clone()]));
+		if tail.is_empty() {
+			return match candidate.try_populate().await {
+				Ok(()) => {
+					let tgt_mem = default_member(self.is_root(), ReflMemKind::Mod(candidate));
+					self.0.members.insert(next.clone(), tgt_mem.clone());
+					Ok(tgt_mem)
+				},
+				Err(api::LsModuleError::InvalidPath) => Err(InvalidPathError { keep_ancestry: false }),
+				Err(api::LsModuleError::IsConstant) => {
+					let const_mem = default_member(self.is_root(), ReflMemKind::Const);
+					self.0.members.insert(next.clone(), const_mem);
+					Err(InvalidPathError { keep_ancestry: true })
+				},
+				Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
+			};
+		}
+		match candidate.get_by_path(tail).boxed_local().await {
+			e @ Err(InvalidPathError { keep_ancestry: false }) => e,
+			res @ Err(InvalidPathError { keep_ancestry: true }) | res @ Ok(_) => {
+				let tgt_mem = default_member(self.is_root(), ReflMemKind::Mod(candidate));
+				self.0.members.insert(next.clone(), tgt_mem);
+				res
+			},
+		}
+	}
+}
+
+struct ReflRoot(ReflMod);
+impl SysCtxEntry for ReflRoot {}
+
+pub struct InvalidPathError {
+	keep_ancestry: bool,
+}
+
+fn default_module(ctx: &SysCtx, path: VPath) -> ReflMod {
+	ReflMod(Rc::new(ReflModData {
+		ctx: ctx.downgrade(),
+		inferred: Mutex::new(true),
+		path,
+		members: MemoMap::new(),
+	}))
+}
+
+fn default_member(is_root: bool, kind: ReflMemKind) -> ReflMem {
+	ReflMem(Rc::new(ReflMemData {
+		public: if is_root { true.into() } else { OnceCell::new() },
+		kind,
+	}))
+}
+
+fn get_root(ctx: &SysCtx) -> &ReflRoot {
+	ctx.get_or_insert(|| ReflRoot(default_module(ctx, VPath::new([]))))
+}
+
+pub fn refl(ctx: &SysCtx) -> ReflMod { get_root(ctx).0.clone() }

orchid-extension/src/system.rs

@@ -1,29 +1,29 @@
-use core::fmt;
+use std::any::{Any, TypeId, type_name};
-use std::any::{TypeId, type_name};
+use std::fmt;
 use std::future::Future;
 use std::num::NonZero;
 use std::pin::Pin;
-use std::rc::Rc;
+use std::rc::{Rc, Weak};
 
 use futures::future::LocalBoxFuture;
-use hashbrown::HashMap;
+use memo_map::MemoMap;
 use orchid_api_traits::{Coding, Decode};
 use orchid_base::boxed_iter::BoxedIter;
 use orchid_base::builtin::Spawner;
 use orchid_base::interner::Interner;
 use orchid_base::logging::Logger;
+use orchid_base::name::Sym;
 use orchid_base::reqnot::{Receipt, ReqNot};
 
 use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TypAtom, get_info};
-use crate::atom_owned::ObjStore;
+use crate::coroutine_exec::Replier;
 use crate::entrypoint::ExtReq;
-use crate::fs::DeclFs;
+use crate::func_atom::{Fun, Lambda};
-use crate::func_atom::Fun;
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
 use crate::system_ctor::{CtedObj, SystemCtor};
-use crate::tree::MemKind;
+use crate::tree::GenMember;
 
 /// System as consumed by foreign code
 pub trait SystemCard: Default + Send + Sync + 'static {
@@ -36,14 +36,14 @@ pub trait DynSystemCard: Send + Sync + 'static {
 	fn name(&self) -> &'static str;
 	/// Atoms explicitly defined by the system card. Do not rely on this for
 	/// querying atoms as it doesn't include the general atom types
-	fn atoms(&self) -> BoxedIter<Option<Box<dyn AtomDynfo>>>;
+	fn atoms(&'_ self) -> BoxedIter<'_, Option<Box<dyn AtomDynfo>>>;
 }
 
 /// Atoms supported by this package which may appear in all extensions.
 /// The indices of these are bitwise negated, such that the MSB of an atom index
 /// marks whether it belongs to this package (0) or the importer (1)
 fn general_atoms() -> impl Iterator<Item = Option<Box<dyn AtomDynfo>>> {
-	[Some(Fun::dynfo())].into_iter()
+	[Some(Fun::dynfo()), Some(Lambda::dynfo()), Some(Replier::dynfo())].into_iter()
 }
 
 pub fn atom_info_for(
@@ -71,27 +71,29 @@ pub async fn resolv_atom(
 	sys: &(impl DynSystemCard + ?Sized),
 	atom: &api::Atom,
 ) -> Box<dyn AtomDynfo> {
-	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data[..])).await;
+	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data.0[..])).await;
 	atom_by_idx(sys, tid).expect("Value of nonexistent type found")
 }
 
 impl<T: SystemCard> DynSystemCard for T {
 	fn name(&self) -> &'static str { T::Ctor::NAME }
-	fn atoms(&self) -> BoxedIter<Option<Box<dyn AtomDynfo>>> { Box::new(Self::atoms().into_iter()) }
+	fn atoms(&'_ self) -> BoxedIter<'_, Option<Box<dyn AtomDynfo>>> {
+		Box::new(Self::atoms().into_iter())
+	}
 }
 
 /// System as defined by author
 pub trait System: Send + Sync + SystemCard + 'static {
-	fn env() -> Vec<(String, MemKind)>;
+	fn prelude(i: &Interner) -> impl Future<Output = Vec<Sym>>;
-	fn vfs() -> DeclFs;
+	fn env() -> Vec<GenMember>;
 	fn lexers() -> Vec<LexerObj>;
 	fn parsers() -> Vec<ParserObj>;
 	fn request(hand: ExtReq<'_>, req: Self::Req) -> impl Future<Output = Receipt<'_>>;
 }
 
 pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
-	fn dyn_env(&self) -> HashMap<String, MemKind>;
+	fn dyn_prelude<'a>(&'a self, i: &'a Interner) -> LocalBoxFuture<'a, Vec<Sym>>;
-	fn dyn_vfs(&self) -> DeclFs;
+	fn dyn_env(&'_ self) -> Vec<GenMember>;
 	fn dyn_lexers(&self) -> Vec<LexerObj>;
 	fn dyn_parsers(&self) -> Vec<ParserObj>;
 	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>>;
@@ -99,8 +101,10 @@ pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
 }
 
 impl<T: System> DynSystem for T {
-	fn dyn_env(&self) -> HashMap<String, MemKind> { Self::env().into_iter().collect() }
+	fn dyn_prelude<'a>(&'a self, i: &'a Interner) -> LocalBoxFuture<'a, Vec<Sym>> {
-	fn dyn_vfs(&self) -> DeclFs { Self::vfs() }
+		Box::pin(Self::prelude(i))
+	}
+	fn dyn_env(&'_ self) -> Vec<GenMember> { Self::env() }
 	fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
 	fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
 	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>> {
@@ -111,16 +115,16 @@ impl<T: System> DynSystem for T {
 	fn card(&self) -> &dyn DynSystemCard { self }
 }
 
-pub async fn downcast_atom<A>(foreign: ForeignAtom<'_>) -> Result<TypAtom<'_, A>, ForeignAtom<'_>>
+pub async fn downcast_atom<A>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom>
 where A: AtomicFeatures {
-	let mut data = &foreign.atom.data[..];
+	let mut data = &foreign.atom.data.0[..];
-	let ctx = foreign.ctx.clone();
+	let ctx = foreign.ctx().clone();
 	let value = AtomTypeId::decode(Pin::new(&mut data)).await;
-	let own_inst = ctx.cted.inst();
+	let own_inst = ctx.get::<CtedObj>().inst();
-	let owner = if ctx.id == foreign.atom.owner {
+	let owner = if *ctx.get::<api::SysId>() == foreign.atom.owner {
 		own_inst.card()
 	} else {
-		(ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
+		(ctx.get::<CtedObj>().deps().find(|s| s.id() == foreign.atom.owner))
 			.ok_or_else(|| foreign.clone())?
 			.get_card()
 	};
@@ -130,21 +134,68 @@ where A: AtomicFeatures {
 	}
 	let val = dynfo.decode(AtomCtx(data, foreign.atom.drop, ctx)).await;
 	let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
-	Ok(TypAtom { value, data: foreign })
+	Ok(TypAtom { value, untyped: foreign })
 }
 
 #[derive(Clone)]
-pub struct SysCtx {
+pub struct WeakSysCtx(Weak<MemoMap<TypeId, Box<dyn Any>>>);
-	pub reqnot: ReqNot<api::ExtMsgSet>,
+impl WeakSysCtx {
-	pub spawner: Spawner,
+	pub fn upgrade(&self) -> Option<SysCtx> { Some(SysCtx(self.0.upgrade()?)) }
-	pub id: api::SysId,
+}
-	pub cted: CtedObj,
+
-	pub logger: Logger,
+#[derive(Clone)]
-	pub obj_store: ObjStore,
+pub struct SysCtx(Rc<MemoMap<TypeId, Box<dyn Any>>>);
-	pub i: Rc<Interner>,
+impl SysCtx {
+	pub fn new(
+		id: api::SysId,
+		i: Interner,
+		reqnot: ReqNot<api::ExtMsgSet>,
+		spawner: Spawner,
+		logger: Logger,
+		cted: CtedObj,
+	) -> Self {
+		let this = Self(Rc::new(MemoMap::new()));
+		this.add(id).add(i).add(reqnot).add(spawner).add(logger).add(cted);
+		this
+	}
+	pub fn downgrade(&self) -> WeakSysCtx { WeakSysCtx(Rc::downgrade(&self.0)) }
+	pub fn add<T: SysCtxEntry>(&self, t: T) -> &Self {
+		assert!(self.0.insert(TypeId::of::<T>(), Box::new(t)), "Key already exists");
+		self
+	}
+	pub fn get_or_insert<T: SysCtxEntry>(&self, f: impl FnOnce() -> T) -> &T {
+		(self.0.get_or_insert_owned(TypeId::of::<T>(), || Box::new(f())).downcast_ref())
+			.expect("Keyed by TypeId")
+	}
+	pub fn get_or_default<T: SysCtxEntry + Default>(&self) -> &T { self.get_or_insert(T::default) }
+	pub fn try_get<T: SysCtxEntry>(&self) -> Option<&T> {
+		Some(self.0.get(&TypeId::of::<T>())?.downcast_ref().expect("Keyed by TypeId"))
+	}
+	pub fn get<T: SysCtxEntry>(&self) -> &T {
+		self.try_get().unwrap_or_else(|| panic!("Context {} missing", type_name::<T>()))
+	}
+	/// Shorthand to get the [Interner] instance
+	pub fn i(&self) -> &Interner { self.get::<Interner>() }
+	/// Shorthand to get the messaging link
+	pub fn reqnot(&self) -> &ReqNot<api::ExtMsgSet> { self.get::<ReqNot<api::ExtMsgSet>>() }
+	/// Shorthand to get the system ID
+	pub fn sys_id(&self) -> api::SysId { *self.get::<api::SysId>() }
+	/// Shorthand to get the task spawner callback
+	pub fn spawner(&self) -> &Spawner { self.get::<Spawner>() }
+	/// Shorthand to get the logger
+	pub fn logger(&self) -> &Logger { self.get::<Logger>() }
+	/// Shorthand to get the constructed system object
+	pub fn cted(&self) -> &CtedObj { self.get::<CtedObj>() }
 }
 impl fmt::Debug for SysCtx {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "SysCtx({:?})", self.id)
+		write!(f, "SysCtx({:?})", self.sys_id())
 	}
 }
+pub trait SysCtxEntry: 'static + Sized {}
+impl SysCtxEntry for api::SysId {}
+impl SysCtxEntry for ReqNot<api::ExtMsgSet> {}
+impl SysCtxEntry for Spawner {}
+impl SysCtxEntry for CtedObj {}
+impl SysCtxEntry for Logger {}
+impl SysCtxEntry for Interner {}

orchid-extension/src/system_ctor.rs

@@ -62,7 +62,7 @@ pub trait SystemCtor: Send + Sync + 'static {
 	type Instance: System;
 	const NAME: &'static str;
 	const VERSION: f64;
-	fn inst() -> Option<Self::Instance>;
+	fn inst(deps: <Self::Deps as DepDef>::Sat) -> Self::Instance;
 }
 
 pub trait DynSystemCtor: Send + Sync + 'static {
@@ -81,8 +81,8 @@ impl<T: SystemCtor> DynSystemCtor for T {
 	}
 	fn new_system(&self, api::NewSystem { system: _, id: _, depends }: &api::NewSystem) -> CtedObj {
 		let mut ids = depends.iter().copied();
-		let inst = Arc::new(T::inst().expect("Constructor did not create system"));
 		let deps = T::Deps::create(&mut || ids.next().unwrap());
+		let inst = Arc::new(T::inst(deps.clone()));
 		Arc::new(Cted::<T> { deps, inst })
 	}
 }
@@ -90,7 +90,7 @@ impl<T: SystemCtor> DynSystemCtor for T {
 mod dep_set_tuple_impls {
 	use orchid_base::box_chain;
 	use orchid_base::boxed_iter::BoxedIter;
-	use paste::paste;
+	use pastey::paste;
 
 	use super::{DepDef, DepSat};
 	use crate::api;

orchid-extension/src/tokio.rs

@@ -0,0 +1,57 @@
+use crate::entrypoint::ExtensionData;
+
+#[cfg(feature = "tokio")]
+pub async fn tokio_main(data: ExtensionData) {
+	use std::io::{ErrorKind, Write};
+	use std::mem;
+	use std::pin::{Pin, pin};
+	use std::rc::Rc;
+
+	use async_once_cell::OnceCell;
+	use futures::StreamExt;
+	use futures::future::LocalBoxFuture;
+	use futures::lock::Mutex;
+	use futures::stream::FuturesUnordered;
+	use orchid_api_traits::{Decode, Encode};
+	use orchid_base::msg::{recv_msg, send_msg};
+	use tokio::io::{Stdout, stdin, stdout};
+	use tokio::task::{LocalSet, spawn_local};
+	use tokio_util::compat::{Compat, TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
+
+	use crate::api;
+	use crate::entrypoint::extension_init;
+
+	let local_set = LocalSet::new();
+	local_set.spawn_local(async {
+		let host_header = api::HostHeader::decode(Pin::new(&mut stdin().compat())).await;
+		let init =
+			Rc::new(extension_init(data, host_header, Rc::new(|fut| mem::drop(spawn_local(fut)))));
+		let mut buf = Vec::new();
+		init.header.encode(Pin::new(&mut buf)).await;
+		std::io::stdout().write_all(&buf).unwrap();
+		std::io::stdout().flush().unwrap();
+		// These are concurrent processes that never exit, so if the FuturesUnordered
+		// produces any result the extension should exit
+		let mut io = FuturesUnordered::<LocalBoxFuture<()>>::new();
+		io.push(Box::pin(async {
+			loop {
+				match recv_msg(pin!(stdin().compat())).await {
+					Ok(msg) => init.send(&msg[..]).await,
+					Err(e) if e.kind() == ErrorKind::BrokenPipe => break,
+					Err(e) if e.kind() == ErrorKind::UnexpectedEof => break,
+					Err(e) => panic!("{e}"),
+				}
+			}
+		}));
+		io.push(Box::pin(async {
+			while let Some(msg) = init.recv().await {
+				static STDOUT: OnceCell<Mutex<Compat<Stdout>>> = OnceCell::new();
+				let stdout_lk = STDOUT.get_or_init(async { Mutex::new(stdout().compat_write()) }).await;
+				let mut stdout_g = stdout_lk.lock().await;
+				send_msg(pin!(&mut *stdout_g), &msg[..]).await.expect("Parent pipe broken");
+			}
+		}));
+		io.next().await;
+	});
+	local_set.await;
+}

orchid-extension/src/tree.rs

@@ -1,125 +1,108 @@
-use std::future::Future;
 use std::num::NonZero;
-use std::ops::Range;
-use std::rc::Rc;
 
+use async_fn_stream::stream;
 use dyn_clone::{DynClone, clone_box};
-use futures::FutureExt;
 use futures::future::{LocalBoxFuture, join_all};
+use futures::{FutureExt, StreamExt};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::ReqHandlish;
+use orchid_base::tree::{TokTree, Token, TokenVariant};
-use orchid_base::tree::{TokTree, Token};
-use ordered_float::NotNan;
 use substack::Substack;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::atom::{AtomFactory, ForeignAtom};
 use crate::conv::ToExpr;
 use crate::entrypoint::MemberRecord;
+use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
 use crate::func_atom::{ExprFunc, Fun};
-use crate::gen_expr::{GExpr, arg, call, lambda, seq};
+use crate::gen_expr::{GExpr, sym_ref};
-use crate::macros::Rule;
 use crate::system::SysCtx;
 
-pub type GenTokTree<'a> = TokTree<'a, ForeignAtom<'a>, AtomFactory>;
+pub type GenTokTree = TokTree<Expr, GExpr>;
-pub type GenTok<'a> = Token<'a, ForeignAtom<'a>, AtomFactory>;
+pub type GenTok = Token<Expr, GExpr>;
 
-pub async fn do_extra(f: &AtomFactory, r: Range<u32>, ctx: SysCtx) -> api::TokenTree {
+impl TokenVariant<api::Expression> for GExpr {
-	api::TokenTree { range: r, token: api::Token::Atom(f.clone().build(ctx).await) }
+	type FromApiCtx<'a> = ();
-}
+	type ToApiCtx<'a> = SysCtx;
-
+	async fn from_api(
-fn with_export(mem: GenMember, public: bool) -> Vec<GenItem> {
+		_: &api::Expression,
-	(public.then(|| GenItemKind::Export(mem.name.clone())).into_iter())
+		_: &mut Self::FromApiCtx<'_>,
-		.chain([GenItemKind::Member(mem)])
+		_: SrcRange,
-		.map(|kind| GenItem { comments: vec![], kind })
+		_: &Interner,
-		.collect()
+	) -> Self {
-}
+		panic!("Received new expression from host")
-
-pub struct GenItem {
-	pub kind: GenItemKind,
-	pub comments: Vec<String>,
-}
-impl GenItem {
-	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Item {
-		let kind = match self.kind {
-			GenItemKind::Export(n) => api::ItemKind::Export(ctx.sys().i.i::<String>(&n).await.to_api()),
-			GenItemKind::Member(mem) => api::ItemKind::Member(mem.into_api(ctx).await),
-			GenItemKind::Import(cn) => api::ItemKind::Import(cn.tok().to_api()),
-			GenItemKind::Macro(priority, gen_rules) => {
-				let mut rules = Vec::with_capacity(gen_rules.len());
-				for rule in gen_rules {
-					rules.push(rule.into_api(ctx).await)
 	}
-				api::ItemKind::Macro(api::MacroBlock { priority, rules })
+	async fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> api::Expression {
-			},
+		self.api_return(ctx.clone()).await
-		};
-		let comments = join_all(self.comments.iter().map(|c| async {
-			api::Comment {
-				location: api::Location::Inherit,
-				text: ctx.sys().i.i::<String>(c).await.to_api(),
-			}
-		}))
-		.await;
-		api::Item { location: api::Location::Inherit, comments, kind }
 	}
 }
 
-pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenItem> {
+impl TokenVariant<api::ExprTicket> for Expr {
-	with_export(GenMember { name: name.to_string(), kind: MemKind::Const(value.to_expr()) }, public)
+	type FromApiCtx<'a> = (SysCtx, &'a BorrowedExprStore);
+	async fn from_api(
+		api: &api::ExprTicket,
+		(ctx, exprs): &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
+		// SAFETY: receiving trees from sublexers implies borrowing
+		Expr::from_handle(ExprHandle::borrowed(ctx.clone(), *api, exprs))
+	}
+	type ToApiCtx<'a> = ();
+	async fn into_api(self, (): &mut Self::ToApiCtx<'_>) -> api::ExprTicket { self.handle().tk }
+}
+
+pub async fn x_tok(x: impl ToExpr) -> GenTok { GenTok::NewExpr(x.to_expr().await) }
+pub async fn ref_tok(path: Sym) -> GenTok { GenTok::NewExpr(sym_ref(path)) }
+
+pub fn lazy(
+	public: bool,
+	name: &str,
+	cb: impl AsyncFnOnce(Sym, SysCtx) -> MemKind + Clone + 'static,
+) -> Vec<GenMember> {
+	vec![GenMember {
+		name: name.to_string(),
+		kind: MemKind::Lazy(LazyMemberFactory::new(cb)),
+		comments: vec![],
+		public,
+	}]
+}
+pub fn cnst(public: bool, name: &str, value: impl ToExpr + Clone + 'static) -> Vec<GenMember> {
+	lazy(public, name, async |_, _| MemKind::Const(value.to_expr().await))
 }
 pub fn module(
 	public: bool,
 	name: &str,
-	imports: impl IntoIterator<Item = Sym>,
+	mems: impl IntoIterator<Item = Vec<GenMember>>,
-	items: impl IntoIterator<Item = Vec<GenItem>>,
+) -> Vec<GenMember> {
-) -> Vec<GenItem> {
+	let (name, kind) = root_mod(name, mems);
-	let (name, kind) = root_mod(name, imports, items);
+	vec![GenMember { name, kind, public, comments: vec![] }]
-	with_export(GenMember { name, kind }, public)
 }
-pub fn root_mod(
+pub fn root_mod(name: &str, mems: impl IntoIterator<Item = Vec<GenMember>>) -> (String, MemKind) {
-	name: &str,
+	let kind = MemKind::Mod { members: mems.into_iter().flatten().collect() };
-	imports: impl IntoIterator<Item = Sym>,
-	items: impl IntoIterator<Item = Vec<GenItem>>,
-) -> (String, MemKind) {
-	let kind = MemKind::Mod {
-		imports: imports.into_iter().collect(),
-		items: items.into_iter().flatten().collect(),
-	};
 	(name.to_string(), kind)
 }
-pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenItem> {
+pub fn fun<I, O>(public: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenMember> {
-	let fac = LazyMemberFactory::new(move |sym| async {
+	let fac = LazyMemberFactory::new(async move |sym, ctx| {
-		return MemKind::Const(build_lambdas(Fun::new(sym, xf).await, 0));
+		MemKind::Const(Fun::new(sym, ctx, xf).await.to_expr().await)
-		fn build_lambdas(fun: Fun, i: u64) -> GExpr {
-			if i < fun.arity().into() {
-				return lambda(i, [build_lambdas(fun, i + 1)]);
-			}
-			let arity = fun.arity();
-			seq(
-				(0..arity)
-					.map(|i| arg(i as u64))
-					.chain([call([fun.to_expr()].into_iter().chain((0..arity).map(|i| arg(i as u64))))]),
-			)
-		}
 	});
-	with_export(GenMember { name: name.to_string(), kind: MemKind::Lazy(fac) }, exported)
+	vec![GenMember { name: name.to_string(), kind: MemKind::Lazy(fac), public, comments: vec![] }]
 }
-pub fn macro_block(prio: Option<f64>, rules: impl IntoIterator<Item = Rule>) -> Vec<GenItem> {
+pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
-	let prio = prio.map(|p| NotNan::new(p).unwrap());
+	let mut items = items.into_iter().flatten().collect_vec();
-	vec![GenItem {
+	for step in path.split("::").collect_vec().into_iter().rev() {
-		kind: GenItemKind::Macro(prio, rules.into_iter().collect_vec()),
+		items = module(true, step, [items]);
-		comments: vec![],
+	}
-	}]
+	items
 }
 
 pub fn comments<'a>(
 	cmts: impl IntoIterator<Item = &'a str>,
-	mut val: Vec<GenItem>,
+	mut val: Vec<GenMember>,
-) -> Vec<GenItem> {
+) -> Vec<GenMember> {
 	let cmts = cmts.into_iter().map(|c| c.to_string()).collect_vec();
 	for v in val.iter_mut() {
 		v.comments.extend(cmts.iter().cloned());
@@ -127,64 +110,94 @@ pub fn comments<'a>(
 	val
 }
 
+/// Trivially merge a gen tree. Behaviours were chosen to make this simple.
+///
+/// - Comments on imports are discarded
+/// - Comments on exports and submodules are combined
+/// - Duplicate constants result in an error
+/// - A combination of lazy and anything results in an error
+pub fn merge_trivial(trees: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
+	let mut all_members = HashMap::<String, (MemKind, Vec<String>)>::new();
+	for mem in trees.into_iter().flatten() {
+		assert!(mem.public, "Non-trivial merge in {}", mem.name);
+		match mem.kind {
+			unit @ (MemKind::Const(_) | MemKind::Lazy(_)) => {
+				let prev = all_members.insert(mem.name.clone(), (unit, mem.comments.into_iter().collect()));
+				assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name);
+			},
+			MemKind::Mod { members } => match all_members.entry(mem.name.clone()) {
+				hashbrown::hash_map::Entry::Vacant(slot) => {
+					slot.insert((MemKind::Mod { members }, mem.comments.into_iter().collect()));
+				},
+				hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() {
+					(MemKind::Mod { members: old_items, .. }, old_cmts) => {
+						let mut swap = vec![];
+						std::mem::swap(&mut swap, old_items);
+						*old_items = merge_trivial([swap, members]);
+						old_cmts.extend(mem.comments);
+					},
+					_ => panic!("non-trivial merge on {}", mem.name),
+				},
+			},
+		}
+	}
+
+	(all_members.into_iter())
+		.map(|(name, (kind, comments))| GenMember { comments, kind, name, public: true })
+		.collect_vec()
+}
+
 trait_set! {
 	trait LazyMemberCallback =
-		FnOnce(Sym) -> LocalBoxFuture<'static, MemKind> + DynClone
+		FnOnce(Sym, SysCtx) -> LocalBoxFuture<'static, MemKind> + DynClone
 }
 pub struct LazyMemberFactory(Box<dyn LazyMemberCallback>);
 impl LazyMemberFactory {
-	pub fn new<F: Future<Output = MemKind> + 'static>(
+	pub fn new(cb: impl AsyncFnOnce(Sym, SysCtx) -> MemKind + Clone + 'static) -> Self {
-		cb: impl FnOnce(Sym) -> F + Clone + 'static,
+		Self(Box::new(|s, ctx| cb(s, ctx).boxed_local()))
-	) -> Self {
-		Self(Box::new(|s| cb(s).boxed_local()))
 	}
-	pub async fn build(self, path: Sym) -> MemKind { (self.0)(path).await }
+	pub async fn build(self, path: Sym, ctx: SysCtx) -> MemKind { (self.0)(path, ctx).await }
 }
 impl Clone for LazyMemberFactory {
 	fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
 }
 
-pub enum GenItemKind {
-	Member(GenMember),
-	Export(String),
-	Import(Sym),
-	Macro(Option<NotNan<f64>>, Vec<Rule>),
-}
-
 pub struct GenMember {
-	name: String,
+	pub name: String,
-	kind: MemKind,
+	pub kind: MemKind,
+	pub public: bool,
+	pub comments: Vec<String>,
 }
 impl GenMember {
 	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Member {
-		let name = ctx.sys().i.i::<String>(&self.name).await;
+		let name = ctx.sys().i().i::<String>(&self.name).await;
-		api::Member {
+		let kind = self.kind.into_api(&mut ctx.push_path(name.clone())).await;
-			kind: self.kind.into_api(&mut ctx.push_path(name.clone())).await,
+		let comments =
-			name: name.to_api(),
+			join_all(self.comments.iter().map(|cmt| async { ctx.sys().i().i(cmt).await.to_api() })).await;
-		}
+		api::Member { kind, name: name.to_api(), comments, exported: self.public }
 	}
 }
 
 pub enum MemKind {
 	Const(GExpr),
-	Mod { imports: Vec<Sym>, items: Vec<GenItem> },
+	Mod { members: Vec<GenMember> },
 	Lazy(LazyMemberFactory),
 }
 impl MemKind {
 	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
 		match self {
 			Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
-			Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys(), ctx.req()).await),
+			Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys()).await),
-			Self::Mod { imports, items } => {
+			Self::Mod { members } => api::MemberKind::Module(api::Module {
-				let all_items = (imports.into_iter())
+				members: stream(async |mut cx| {
-					.map(|t| GenItem { comments: vec![], kind: GenItemKind::Import(t) })
+					for m in members {
-					.chain(items);
+						cx.emit(m.into_api(ctx).await).await
-				let mut items = Vec::new();
-				for i in all_items {
-					items.push(i.into_api(ctx).boxed_local().await)
 					}
-				api::MemberKind::Module(api::Module { items })
+				})
-			},
+				.collect()
+				.boxed_local()
+				.await,
+			}),
 		}
 	}
 }
@@ -192,27 +205,21 @@ impl MemKind {
 pub trait TreeIntoApiCtx {
 	fn sys(&self) -> SysCtx;
 	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
-	fn with_rule(&mut self, rule: Rc<Rule>) -> api::MacroId;
 	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
-	fn req(&self) -> &impl ReqHandlish;
 }
 
-pub struct TIACtxImpl<'a, 'b, RH: ReqHandlish> {
+pub struct TreeIntoApiCtxImpl<'a, 'b> {
 	pub sys: SysCtx,
 	pub basepath: &'a [Tok<String>],
 	pub path: Substack<'a, Tok<String>>,
 	pub lazy_members: &'b mut HashMap<api::TreeId, MemberRecord>,
-	pub rules: &'b mut HashMap<api::MacroId, Rc<Rule>>,
-	pub req: &'a RH,
 }
 
-impl<RH: ReqHandlish> TreeIntoApiCtx for TIACtxImpl<'_, '_, RH> {
+impl TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_> {
 	fn sys(&self) -> SysCtx { self.sys.clone() }
 	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx {
-		TIACtxImpl {
+		TreeIntoApiCtxImpl {
-			req: self.req,
 			lazy_members: self.lazy_members,
-			rules: self.rules,
 			sys: self.sys.clone(),
 			basepath: self.basepath,
 			path: self.path.push(seg),
@@ -224,10 +231,4 @@ impl<RH: ReqHandlish> TreeIntoApiCtx for TIACtxImpl<'_, '_, RH> {
 		self.lazy_members.insert(id, MemberRecord::Gen(path, fac));
 		id
 	}
-	fn with_rule(&mut self, rule: Rc<Rule>) -> orchid_api::MacroId {
-		let id = api::MacroId(NonZero::new((self.lazy_members.len() + 1) as u64).unwrap());
-		self.rules.insert(id, rule);
-		id
-	}
-	fn req(&self) -> &impl ReqHandlish { self.req }
 }

orchid-host/Cargo.toml

@@ -1,28 +1,29 @@
 [package]
 name = "orchid-host"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
+async-lock = "3.4.1"
 async-once-cell = "0.5.4"
-async-process = "2.3.0"
+async-process = "2.4.0"
-async-std = "1.13.0"
-async-stream = "0.3.6"
 bound = "0.6.0"
 derive_destructure = "1.0.0"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.15.2"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
+memo-map = "0.3.3"
 never = "0.1.0"
 num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"
-paste = "1.0.15"
+pastey = "0.1.1"
 substack = "1.1.1"
-test_executors = "0.3.2"
+test_executors = "0.3.5"
 trait-set = "0.3.0"

orchid-host/src/atom.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 use std::rc::{Rc, Weak};
 
+use async_lock::OnceCell;
 use derive_destructure::destructure;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, take_first_fmt};
 use orchid_base::location::Pos;
@@ -9,7 +10,7 @@ use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::ctx::Ctx;
-use crate::expr::Expr;
+use crate::expr::{Expr, ExprParseCtx, PathSetBuilder};
 use crate::extension::Extension;
 use crate::system::System;
 
@@ -18,14 +19,17 @@ pub struct AtomData {
 	owner: System,
 	drop: Option<api::AtomId>,
 	data: Vec<u8>,
+	pub(crate) display: OnceCell<FmtUnit>,
 }
 impl AtomData {
+	#[must_use]
 	fn api(self) -> api::Atom {
-		let (owner, drop, data) = self.destructure();
+		let (owner, drop, data, _display) = self.destructure();
-		api::Atom { data, drop, owner: owner.id() }
+		api::Atom { data: api::AtomData(data), drop, owner: owner.id() }
 	}
+	#[must_use]
 	fn api_ref(&self) -> api::Atom {
-		api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
+		api::Atom { data: api::AtomData(self.data.clone()), drop: self.drop, owner: self.owner.id() }
 	}
 }
 impl Drop for AtomData {
@@ -48,57 +52,62 @@ impl fmt::Debug for AtomData {
 #[derive(Clone, Debug)]
 pub struct AtomHand(Rc<AtomData>);
 impl AtomHand {
-	pub(crate) async fn new(api::Atom { data, drop, owner }: api::Atom, ctx: &Ctx) -> Self {
+	pub(crate) fn new(data: Vec<u8>, owner: System, drop: Option<api::AtomId>) -> Self {
-		let create = || async {
+		Self(Rc::new(AtomData { owner, drop, data, display: OnceCell::new() }))
-			let owner = ctx.system_inst(owner).await.expect("Dropped system created atom");
-			AtomHand(Rc::new(AtomData { data, owner, drop }))
-		};
-		if let Some(id) = drop {
-			let mut owned_g = ctx.owned_atoms.write().await;
-			if let Some(data) = owned_g.get(&id) {
-				if let Some(atom) = data.upgrade() {
-					return atom;
 	}
-			}
+	#[must_use]
-			let new = create().await;
+	pub async fn call(self, arg: Expr) -> Expr {
-			owned_g.insert(id, new.downgrade());
-			new
-		} else {
-			create().await
-		}
-	}
-	pub async fn call(self, arg: Expr) -> api::Expression {
 		let owner_sys = self.0.owner.clone();
 		let reqnot = owner_sys.reqnot();
 		owner_sys.ext().exprs().give_expr(arg.clone());
-		match Rc::try_unwrap(self.0) {
+		let ret = match Rc::try_unwrap(self.0) {
 			Ok(data) => reqnot.request(api::FinalCall(data.api(), arg.id())).await,
 			Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await,
+		};
+		let mut parse_ctx = ExprParseCtx { ctx: owner_sys.ctx(), exprs: owner_sys.ext().exprs() };
+		let val = Expr::from_api(&ret, PathSetBuilder::new(), &mut parse_ctx).await;
+		owner_sys.ext().exprs().take_expr(arg.id());
+		val
 	}
-	}
+	#[must_use]
 	pub fn sys(&self) -> &System { &self.0.owner }
+	#[must_use]
 	pub fn ext(&self) -> &Extension { self.sys().ext() }
 	pub async fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
 		self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await
 	}
+	#[must_use]
 	pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
+	#[must_use]
 	pub async fn to_string(&self) -> String { take_first_fmt(self, &self.0.owner.ctx().i).await }
+	#[must_use]
 	pub fn downgrade(&self) -> WeakAtomHand { WeakAtomHand(Rc::downgrade(&self.0)) }
 }
 impl Format for AtomHand {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		(self.0.display.get_or_init(|| async {
 			FmtUnit::from_api(&self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await)
+		}))
+		.await
+		.clone()
 	}
 }
 impl AtomRepr for AtomHand {
 	type Ctx = Ctx;
-	async fn from_api(atom: &orchid_api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
+	async fn from_api(atom: &api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
-		Self::new(atom.clone(), ctx).await
+		let api::Atom { data, drop, owner } = atom.clone();
+		let sys = ctx.system_inst(owner).await.expect("Dropped system created atom");
+		if let Some(id) = drop {
+			sys.new_atom(data.0, id).await
+		} else {
+			AtomHand::new(data.0, sys, drop)
+		}
 	}
 	async fn to_api(&self) -> orchid_api::Atom { self.api_ref() }
 }
 
 pub struct WeakAtomHand(Weak<AtomData>);
 impl WeakAtomHand {
+	#[must_use]
 	pub fn upgrade(&self) -> Option<AtomHand> { self.0.upgrade().map(AtomHand) }
 }

orchid-host/src/ctx.rs

@@ -1,26 +1,25 @@
 use std::cell::RefCell;
 use std::num::{NonZero, NonZeroU16};
-use std::rc::Rc;
+use std::rc::{Rc, Weak};
 use std::{fmt, ops};
 
-use async_std::sync::RwLock;
+use async_lock::RwLock;
 use hashbrown::HashMap;
-use orchid_api::SysId;
 use orchid_base::builtin::Spawner;
 use orchid_base::interner::Interner;
 
 use crate::api;
-use crate::atom::WeakAtomHand;
+use crate::expr_store::ExprStore;
 use crate::system::{System, WeakSystem};
-use crate::tree::Module;
+use crate::tree::WeakRoot;
 
 pub struct CtxData {
-	pub i: Rc<Interner>,
+	pub i: Interner,
 	pub spawn: Spawner,
 	pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
 	pub system_id: RefCell<NonZeroU16>,
-	pub owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
+	pub common_exprs: ExprStore,
-	pub root: RwLock<Module>,
+	pub root: RwLock<WeakRoot>,
 }
 #[derive(Clone)]
 pub struct Ctx(Rc<CtxData>);
@@ -28,25 +27,38 @@ impl ops::Deref for Ctx {
 	type Target = CtxData;
 	fn deref(&self) -> &Self::Target { &self.0 }
 }
+#[derive(Clone)]
+pub struct WeakCtx(Weak<CtxData>);
+impl WeakCtx {
+	#[must_use]
+	pub fn try_upgrade(&self) -> Option<Ctx> { Some(Ctx(self.0.upgrade()?)) }
+	#[must_use]
+	pub fn upgrade(&self) -> Ctx { self.try_upgrade().expect("Ctx manually kept alive until exit") }
+}
 impl Ctx {
+	#[must_use]
 	pub fn new(spawn: Spawner) -> Self {
 		Self(Rc::new(CtxData {
 			spawn,
-			i: Rc::default(),
+			i: Interner::default(),
 			systems: RwLock::default(),
 			system_id: RefCell::new(NonZero::new(1).unwrap()),
-			owned_atoms: RwLock::default(),
+			common_exprs: ExprStore::default(),
-			root: RwLock::new(Module::default()),
+			root: RwLock::default(),
 		}))
 	}
+	#[must_use]
 	pub(crate) async fn system_inst(&self, id: api::SysId) -> Option<System> {
 		self.systems.read().await.get(&id).and_then(WeakSystem::upgrade)
 	}
+	#[must_use]
 	pub(crate) fn next_sys_id(&self) -> api::SysId {
 		let mut g = self.system_id.borrow_mut();
 		*g = g.checked_add(1).unwrap_or(NonZeroU16::new(1).unwrap());
-		SysId(*g)
+		api::SysId(*g)
 	}
+	#[must_use]
+	pub fn downgrade(&self) -> WeakCtx { WeakCtx(Rc::downgrade(&self.0)) }
 }
 impl fmt::Debug for Ctx {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

orchid-host/src/dealias.rs

@@ -1,16 +1,9 @@
-use std::rc::Rc;
+use hashbrown::HashSet;
-
+use itertools::Itertools;
-use futures::FutureExt;
+use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_errv};
-use hashbrown::{HashMap, HashSet};
-use itertools::{Either, Itertools};
-use orchid_base::error::{OrcErr, Reporter, mk_err};
-use orchid_base::format::{FmtCtxImpl, Format, take_first};
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::Pos;
-use orchid_base::name::{NameLike, Sym, VName};
+use orchid_base::name::VName;
-
-use crate::macros::{MacTok, MacTree};
-use crate::tree::{ItemKind, MemberKind, Module, RuleKind, WalkErrorKind};
 
 /// Errors produced by absolute_path
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
@@ -23,7 +16,7 @@ pub enum AbsPathError {
 	RootPath,
 }
 impl AbsPathError {
-	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErr {
+	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErrv {
 		let (descr, msg) = match self {
 			AbsPathError::RootPath => (
 				i.i("Path ends on root module").await,
@@ -37,7 +30,7 @@ impl AbsPathError {
 				format!("{path} is leading outside the root."),
 			),
 		};
-		mk_err(descr, msg, [pos.into()])
+		mk_errv(descr, msg, [pos])
 	}
 }
 
@@ -48,193 +41,107 @@ impl AbsPathError {
 ///
 /// if the relative path contains as many or more `super` segments than the
 /// length of the absolute path.
-pub fn absolute_path(
+pub async fn absolute_path(
 	mut cwd: &[Tok<String>],
 	mut rel: &[Tok<String>],
+	i: &Interner,
 ) -> Result<VName, AbsPathError> {
-	let mut relative = false;
+	let i_self = i.i("self").await;
-	if rel.first().map(|t| t.as_str()) == Some("self") {
+	let i_super = i.i("super").await;
-		relative = true;
+	let relative = rel.first().is_some_and(|s| *s != i_self && *s != i_super);
-		rel = rel.split_first().expect("checked above").1;
+	if let Some((_, tail)) = rel.split_first().filter(|(h, _)| **h != i_self) {
+		rel = tail;
 	} else {
-		while rel.first().map(|t| t.as_str()) == Some("super") {
+		while let Some((_, tail)) = rel.split_first().filter(|(h, _)| **h == i_super) {
-			match cwd.split_last() {
+			cwd = cwd.split_last().ok_or(AbsPathError::TooManySupers)?.1;
-				Some((_, torso)) => cwd = torso,
+			rel = tail;
-				None => return Err(AbsPathError::TooManySupers),
-			};
-			rel = rel.split_first().expect("checked above").1;
-			relative = true;
 		}
 	}
-	match relative {
+	if relative { VName::new(cwd.iter().chain(rel).cloned()) } else { VName::new(rel.to_vec()) }
-		true => VName::new(cwd.iter().chain(rel).cloned()),
-		false => VName::new(rel.to_vec()),
-	}
 		.map_err(|_| AbsPathError::RootPath)
 }
 
-pub async fn resolv_glob(
+pub struct DealiasCtx<'a> {
+	pub i: &'a Interner,
+	pub rep: &'a Reporter,
+}
+
+pub async fn resolv_glob<Mod: Tree>(
 	cwd: &[Tok<String>],
-	root: &Module,
+	root: &Mod,
 	abs_path: &[Tok<String>],
 	pos: Pos,
 	i: &Interner,
-	r: &impl Reporter,
+	ctx: &mut Mod::Ctx<'_>,
-) -> Vec<Tok<String>> {
+) -> OrcRes<HashSet<Tok<String>>> {
 	let coprefix_len = cwd.iter().zip(abs_path).take_while(|(a, b)| a == b).count();
-	let (co_prefix, diff_path) = abs_path.split_at(coprefix_len);
+	let (co_prefix, diff_path) = abs_path.split_at(abs_path.len().min(coprefix_len + 1));
-	let co_parent = root.walk(false, co_prefix.iter().cloned()).await.expect("Invalid step in cwd");
+	let fst_diff =
-	let target_module = match co_parent.walk(true, diff_path.iter().cloned()).await {
+		walk(root, false, co_prefix.iter().cloned(), ctx).await.expect("Invalid step in cwd");
+	let target_module = match walk(fst_diff, true, diff_path.iter().cloned(), ctx).await {
 		Ok(t) => t,
 		Err(e) => {
 			let path = abs_path[..=coprefix_len + e.pos].iter().join("::");
 			let (tk, msg) = match e.kind {
-				WalkErrorKind::Constant =>
+				ChildErrorKind::Constant => ("Invalid import path", format!("{path} is a const")),
-					(i.i("Invalid import path").await, format!("{path} is a constant")),
+				ChildErrorKind::Missing => ("Invalid import path", format!("{path} not found")),
-				WalkErrorKind::Missing => (i.i("Invalid import path").await, format!("{path} not found")),
+				ChildErrorKind::Private => ("Import inaccessible", format!("{path} is private")),
-				WalkErrorKind::Private => (i.i("Import inaccessible").await, format!("{path} is private")),
 			};
-			r.report(mk_err(tk, msg, [pos.into()]));
+			return Err(mk_errv(i.i(tk).await, msg, [pos]));
-			return vec![];
 		},
 	};
-	target_module.exports.clone()
+	Ok(target_module.children(coprefix_len < abs_path.len()))
 }
 
-/// Read import statements and convert them into aliases, rasising any import
+pub type ChildResult<'a, T> = Result<&'a T, ChildErrorKind>;
-/// errors in the process
+
-pub async fn imports_to_aliases(
+pub trait Tree {
-	module: &Module,
+	type Ctx<'a>;
-	cwd: &mut Vec<Tok<String>>,
+	#[must_use]
-	root: &Module,
+	fn children(&self, public_only: bool) -> HashSet<Tok<String>>;
-	alias_map: &mut HashMap<Sym, Sym>,
+	#[must_use]
-	alias_rev_map: &mut HashMap<Sym, HashSet<Sym>>,
+	fn child(
-	i: &Interner,
+		&self,
-	rep: &impl Reporter,
+		key: Tok<String>,
-) {
+		public_only: bool,
-	let mut import_locs = HashMap::<Sym, Vec<Pos>>::new();
+		ctx: &mut Self::Ctx<'_>,
-	for item in &module.items {
+	) -> impl Future<Output = ChildResult<'_, Self>>;
-		match &item.kind {
+}
-			ItemKind::Import(imp) => match absolute_path(cwd, &imp.path) {
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-				Err(e) => rep.report(e.err_obj(i, item.pos.clone(), &imp.path.iter().join("::")).await),
+pub enum ChildErrorKind {
-				Ok(abs_path) => {
+	Missing,
-					let names = match imp.name.as_ref() {
+	/// Only thrown if public_only is true
-						Some(n) => Either::Right([n.clone()].into_iter()),
+	Private,
-						None => Either::Left(
+	Constant,
-							resolv_glob(cwd, root, &abs_path, item.pos.clone(), i, rep).await.into_iter(),
+}
-						),
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-					};
+pub struct ChildError {
-					for name in names {
+	pub pos: usize,
-						let mut tgt = abs_path.clone().suffix([name.clone()]).to_sym(i).await;
+	pub kind: ChildErrorKind,
-						let src = Sym::new(cwd.iter().cloned().chain([name]), i).await.unwrap();
-						import_locs.entry(src.clone()).or_insert(vec![]).push(item.pos.clone());
-						if let Some(tgt2) = alias_map.get(&tgt) {
-							tgt = tgt2.clone();
-						}
-						if src == tgt {
-							rep.report(mk_err(
-								i.i("Circular references").await,
-								format!("{src} circularly refers to itself"),
-								[item.pos.clone().into()],
-							));
-							continue;
-						}
-						if let Some(fst_val) = alias_map.get(&src) {
-							let locations = (import_locs.get(&src))
-								.expect("The same name could only have appeared in the same module");
-							rep.report(mk_err(
-								i.i("Conflicting imports").await,
-								if fst_val == &src {
-									format!("{src} is imported multiple times")
-								} else {
-									format!("{} could refer to both {fst_val} and {src}", src.last())
-								},
-								locations.iter().map(|p| p.clone().into()).collect_vec(),
-							))
-						}
-						let mut srcv = vec![src.clone()];
-						if let Some(src_extra) = alias_rev_map.remove(&src) {
-							srcv.extend(src_extra);
-						}
-						for src in srcv {
-							alias_map.insert(src.clone(), tgt.clone());
-							alias_rev_map.entry(tgt.clone()).or_insert(HashSet::new()).insert(src);
-						}
-					}
-				},
-			},
-			ItemKind::Member(mem) => match mem.kind().await {
-				MemberKind::Const(_) => (),
-				MemberKind::Mod(m) => {
-					cwd.push(mem.name());
-					imports_to_aliases(m, cwd, root, alias_map, alias_rev_map, i, rep).boxed_local().await;
-					cwd.pop();
-				},
-			},
-			ItemKind::Export(_) | ItemKind::Macro(..) => (),
-		}
-	}
 }
 
-pub async fn dealias(module: &mut Module, alias_map: &HashMap<Sym, Sym>, i: &Interner) {
+// Problem: walk should take into account aliases and visibility
-	for item in &mut module.items {
+//
-		match &mut item.kind {
+// help: since every alias is also its own import, visibility only has to be
-			ItemKind::Export(_) | ItemKind::Import(_) => (),
+// checked on the top level
-			ItemKind::Member(mem) => match mem.kind_mut().await {
+//
-				MemberKind::Const(c) => {
+// idea: do a simple stack machine like below with no visibility for aliases and
-					let Some(source) = c.source() else { continue };
+// call it from an access-checking implementation for just the top level
-					let Some(new_source) = dealias_mactreev(source, alias_map, i).await else { continue };
+//
-					c.set_source(new_source);
+// caveat: we need to check EVERY IMPORT to ensure that all
-				},
+// errors are raised
-				MemberKind::Mod(m) => dealias(m, alias_map, i).boxed_local().await,
+pub async fn walk<'a, T: Tree>(
-			},
+	root: &'a T,
-			ItemKind::Macro(_, rules) =>
+	public_only: bool,
-				for rule in rules.iter_mut() {
+	path: impl IntoIterator<Item = Tok<String>>,
-					let RuleKind::Native(c) = &mut rule.kind else { continue };
+	ctx: &mut T::Ctx<'_>,
-					let Some(source) = c.source() else { continue };
+) -> Result<&'a T, ChildError> {
-					let Some(new_source) = dealias_mactreev(source, alias_map, i).await else { continue };
+	let mut cur = root;
-					c.set_source(new_source);
+	for (i, item) in path.into_iter().enumerate() {
-				},
+		match cur.child(item, public_only, ctx).await {
+			ChildResult::Ok(v) => cur = v,
+			ChildResult::Err(kind) => return Err(ChildError { pos: i, kind }),
 		}
 	}
-}
+	Ok(cur)
-
-async fn dealias_mactree(
-	mtree: &MacTree,
-	aliases: &HashMap<Sym, Sym>,
-	i: &Interner,
-) -> Option<MacTree> {
-	let new_tok = match &*mtree.tok {
-		MacTok::Atom(_) | MacTok::Ph(_) => return None,
-		tok @ (MacTok::Done(_) | MacTok::Ref(_) | MacTok::Slot(_)) => panic!(
-			"{} should not appear in retained pre-macro source",
-			take_first(&tok.print(&FmtCtxImpl { i }).await, true)
-		),
-		MacTok::Name(n) => MacTok::Name(aliases.get(n).unwrap_or(n).clone()),
-		MacTok::Lambda(arg, body) => {
-			match (dealias_mactreev(arg, aliases, i).await, dealias_mactreev(body, aliases, i).await) {
-				(None, None) => return None,
-				(Some(arg), None) => MacTok::Lambda(arg, body.clone()),
-				(None, Some(body)) => MacTok::Lambda(arg.clone(), body),
-				(Some(arg), Some(body)) => MacTok::Lambda(arg, body),
-			}
-		},
-		MacTok::S(p, b) => MacTok::S(*p, dealias_mactreev(b, aliases, i).await?),
-	};
-	Some(MacTree { pos: mtree.pos.clone(), tok: Rc::new(new_tok) })
-}
-
-async fn dealias_mactreev(
-	mtreev: &[MacTree],
-	aliases: &HashMap<Sym, Sym>,
-	i: &Interner,
-) -> Option<Vec<MacTree>> {
-	let mut results = Vec::with_capacity(mtreev.len());
-	let mut any_some = false;
-	for item in mtreev {
-		let out = dealias_mactree(item, aliases, i).boxed_local().await;
-		any_some |= out.is_some();
-		results.push(out.unwrap_or(item.clone()));
-	}
-	any_some.then_some(results)
 }

orchid-host/src/execute.rs

@@ -1,17 +1,16 @@
 use std::mem;
 
-use async_std::sync::RwLockWriteGuard;
+use async_lock::RwLockWriteGuard;
 use bound::Bound;
 use futures::FutureExt;
-use orchid_base::error::{OrcErrv, mk_errv};
+use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtxImpl, Format, take_first};
 use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
-use orchid_base::name::NameLike;
 
 use crate::ctx::Ctx;
 use crate::expr::{Expr, ExprKind, PathSet, Step};
-use crate::tree::{ItemKind, MemberKind};
+use crate::tree::Root;
 
 type ExprGuard = Bound<RwLockWriteGuard<'static, ExprKind>, Expr>;
 
@@ -38,16 +37,21 @@ pub struct ExecCtx {
 	cur_pos: Pos,
 	did_pop: bool,
 	logger: Logger,
+	root: Root,
 }
 impl ExecCtx {
-	pub async fn new(ctx: Ctx, logger: Logger, init: Expr) -> Self {
+	#[must_use]
+	pub async fn new(ctx: Ctx, logger: Logger, root: Root, init: Expr) -> Self {
 		let cur_pos = init.pos();
 		let cur = Bound::async_new(init, |init| init.kind().write()).await;
-		Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger }
+		Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger, root }
 	}
+	#[must_use]
 	pub fn remaining_gas(&self) -> u64 { self.gas.expect("queried remaining_gas but no gas was set") }
 	pub fn set_gas(&mut self, gas: Option<u64>) { self.gas = gas }
+	#[must_use]
 	pub fn idle(&self) -> bool { self.did_pop }
+	#[must_use]
 	pub fn result(self) -> ExecResult {
 		if self.idle() {
 			match &*self.cur {
@@ -58,15 +62,18 @@ impl ExecCtx {
 			ExecResult::Gas(self)
 		}
 	}
+	#[must_use]
 	pub fn use_gas(&mut self, amount: u64) -> bool {
 		if let Some(gas) = &mut self.gas {
 			*gas -= amount;
 		}
 		self.gas != Some(0)
 	}
+	#[must_use]
 	pub async fn try_lock(&self, ex: &Expr) -> ExprGuard {
 		Bound::async_new(ex.clone(), |ex| ex.kind().write()).await
 	}
+	#[must_use]
 	pub async fn unpack_ident(&self, ex: &Expr) -> Expr {
 		match ex.kind().try_write().as_deref_mut() {
 			Some(ExprKind::Identity(ex)) => {
@@ -91,37 +98,10 @@ impl ExecCtx {
 				},
 				ExprKind::Seq(a, b) if !self.did_pop => (ExprKind::Seq(a.clone(), b), StackOp::Push(a)),
 				ExprKind::Seq(_, b) => (ExprKind::Identity(b), StackOp::Nop),
-				ExprKind::Const(name) => {
+				ExprKind::Const(name) =>
-					let (cn, mp) = name.split_last();
+					match self.root.get_const_value(name, self.cur_pos.clone()).await {
-					let root_lock = self.ctx.root.read().await;
+						Err(e) => (ExprKind::Bottom(e), StackOp::Pop),
-					let module = root_lock.walk(true, mp.iter().cloned()).await.unwrap();
+						Ok(v) => (ExprKind::Identity(v), StackOp::Nop),
-					let member = (module.items.iter())
-						.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
-						.find(|m| m.name() == cn);
-					match member {
-						None => (
-							ExprKind::Bottom(mk_errv(
-								self.ctx.i.i("Constant does not exist").await,
-								format!("{name} does not refer to a constant"),
-								[self.cur_pos.clone().into()],
-							)),
-							StackOp::Pop,
-						),
-						Some(mem) => match mem.kind().await {
-							MemberKind::Mod(_) => (
-								ExprKind::Bottom(mk_errv(
-									self.ctx.i.i("module used as constant").await,
-									format!("{name} is a module"),
-									[self.cur_pos.clone().into()],
-								)),
-								StackOp::Pop,
-							),
-							MemberKind::Const(c) => {
-								let value = c.get_bytecode(&self.ctx).await;
-								(ExprKind::Identity(value.clone()), StackOp::Nop)
-							},
-						},
-					}
 					},
 				ExprKind::Arg => panic!("This should not appear outside function bodies"),
 				ek @ ExprKind::Atom(_) => (ek, StackOp::Pop),
@@ -129,9 +109,8 @@ impl ExecCtx {
 				ExprKind::Call(f, x) if !self.did_pop => (ExprKind::Call(f.clone(), x), StackOp::Push(f)),
 				ExprKind::Call(f, x) => match f.try_into_owned_atom().await {
 					Ok(atom) => {
-						let mut ext = atom.sys().ext().clone();
 						let x_norm = self.unpack_ident(&x).await;
-						let val = Expr::from_api(&atom.call(x_norm).await, &mut ext).await;
+						let val = atom.call(x_norm).await;
 						(ExprKind::Identity(val.clone()), StackOp::Swap(val))
 					},
 					Err(f) => match &*f.kind().read().await {
@@ -139,10 +118,9 @@ impl ExecCtx {
 							panic!("This should not appear outside function bodies"),
 						ExprKind::Missing => panic!("Should have been replaced"),
 						ExprKind::Atom(a) => {
-							let mut ext = a.sys().ext().clone();
 							let x_norm = self.unpack_ident(&x).await;
-							let val = Expr::from_api(&a.clone().call(x_norm).await, &mut ext).await;
+							let ret = a.clone().call(x_norm).await;
-							(ExprKind::Identity(val.clone()), StackOp::Swap(val))
+							(ExprKind::Identity(ret.clone()), StackOp::Swap(ret))
 						},
 						ExprKind::Bottom(exprv) => (ExprKind::Bottom(exprv.clone()), StackOp::Pop),
 						ExprKind::Lambda(None, body) =>
@@ -185,6 +163,7 @@ impl ExecCtx {
 	}
 }
 
+#[must_use]
 async fn substitute(
 	src: &Expr,
 	path: &[Step],

orchid-host/src/expr.rs

@@ -4,27 +4,28 @@ use std::num::NonZeroU64;
 use std::rc::{Rc, Weak};
 use std::{fmt, mem};
 
-use async_std::sync::RwLock;
+use async_lock::RwLock;
 use futures::FutureExt;
-use hashbrown::HashSet;
 use itertools::Itertools;
-use orchid_base::error::{OrcErrv, mk_errv};
+use orchid_base::error::OrcErrv;
-use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, Format, Variants, take_first};
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
-use orchid_base::location::Pos;
+use orchid_base::interner::Interner;
-use orchid_base::macros::mtreev_fmt;
+use orchid_base::location::{Pos, SrcRange};
 use orchid_base::name::Sym;
-use orchid_base::tokens::Paren;
+use orchid_base::tl_cache;
-use orchid_base::tree::{AtomRepr, indent};
+use orchid_base::tree::{AtomRepr, TokenVariant, indent};
-use orchid_base::{match_mapping, tl_cache};
 use substack::Substack;
 
 use crate::api;
 use crate::atom::AtomHand;
 use crate::ctx::Ctx;
-use crate::extension::Extension;
+use crate::expr_store::ExprStore;
-use crate::macros::{MacTok, MacTree};
 
-pub type ExprParseCtx = Extension;
+#[derive(Clone)]
+pub struct ExprParseCtx<'a> {
+	pub ctx: &'a Ctx,
+	pub exprs: &'a ExprStore,
+}
 
 #[derive(Debug)]
 pub struct ExprData {
@@ -35,6 +36,7 @@ pub struct ExprData {
 #[derive(Clone, Debug)]
 pub struct Expr(Rc<ExprData>);
 impl Expr {
+	#[must_use]
 	pub fn pos(&self) -> Pos { self.0.pos.clone() }
 	pub async fn try_into_owned_atom(self) -> Result<AtomHand, Self> {
 		match Rc::try_unwrap(self.0) {
@@ -45,24 +47,63 @@ impl Expr {
 			},
 		}
 	}
+	#[must_use]
 	pub async fn as_atom(&self) -> Option<AtomHand> {
 		if let ExprKind::Atom(a) = &*self.kind().read().await { Some(a.clone()) } else { None }
 	}
+	#[must_use]
 	pub fn strong_count(&self) -> usize { Rc::strong_count(&self.0) }
+	#[must_use]
 	pub fn id(&self) -> api::ExprTicket {
 		api::ExprTicket(
 			NonZeroU64::new(self.0.as_ref() as *const ExprData as usize as u64)
 				.expect("this is a ref, it cannot be null"),
 		)
 	}
-	pub async fn from_api(api: &api::Expression, ctx: &mut ExprParseCtx) -> Self {
+	#[must_use]
-		if let api::ExpressionKind::Slot(tk) = &api.kind {
+	pub async fn from_api(
-			return ctx.exprs().get_expr(*tk).expect("Invalid slot");
+		api: &api::Expression,
-		}
+		psb: PathSetBuilder<'_, u64>,
-		let pos = Pos::from_api(&api.location, &ctx.ctx().i).await;
+		ctx: &mut ExprParseCtx<'_>,
-		let kind = RwLock::new(ExprKind::from_api(&api.kind, pos.clone(), ctx).boxed_local().await);
+	) -> Self {
-		Self(Rc::new(ExprData { pos, kind }))
+		let pos = Pos::from_api(&api.location, &ctx.ctx.i).await;
+		let kind = match &api.kind {
+			api::ExpressionKind::Arg(n) => {
+				assert!(psb.register_arg(n), "Arguments must be enclosed in a matching lambda");
+				ExprKind::Arg
+			},
+			api::ExpressionKind::Bottom(bot) =>
+				ExprKind::Bottom(OrcErrv::from_api(bot, &ctx.ctx.i).await),
+			api::ExpressionKind::Call(f, x) => {
+				let (lpsb, rpsb) = psb.split();
+				ExprKind::Call(
+					Expr::from_api(f, lpsb, ctx).boxed_local().await,
+					Expr::from_api(x, rpsb, ctx).boxed_local().await,
+				)
+			},
+			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(*name, &ctx.ctx.i).await),
+			api::ExpressionKind::Lambda(x, body) => {
+				let lbuilder = psb.lambda(x);
+				let body = Expr::from_api(body, lbuilder.stack(), ctx).boxed_local().await;
+				ExprKind::Lambda(lbuilder.collect(), body)
+			},
+			api::ExpressionKind::NewAtom(a) =>
+				ExprKind::Atom(AtomHand::from_api(a, pos.clone(), &mut ctx.ctx.clone()).await),
+			api::ExpressionKind::Slot { tk, by_value: false } =>
+				return ctx.exprs.get_expr(*tk).expect("Invalid slot"),
+			api::ExpressionKind::Slot { tk, by_value: true } =>
+				return ctx.exprs.take_expr(*tk).expect("Invalid slot"),
+			api::ExpressionKind::Seq(a, b) => {
+				let (apsb, bpsb) = psb.split();
+				ExprKind::Seq(
+					Expr::from_api(a, apsb, ctx).boxed_local().await,
+					Expr::from_api(b, bpsb, ctx).boxed_local().await,
+				)
+			},
+		};
+		Self(Rc::new(ExprData { pos, kind: RwLock::new(kind) }))
 	}
+	#[must_use]
 	pub async fn to_api(&self) -> api::InspectedKind {
 		use api::InspectedKind as K;
 		match &*self.0.kind.read().await {
@@ -72,23 +113,23 @@ impl Expr {
 			_ => K::Opaque,
 		}
 	}
+	#[must_use]
 	pub fn kind(&self) -> &RwLock<ExprKind> { &self.0.kind }
 }
 impl Format for Expr {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		return print_expr(self, c, &mut HashSet::new()).await;
+		return print_expr(self, c, Substack::Bottom).await;
 	}
 }
 async fn print_expr<'a>(
 	expr: &'a Expr,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
-	visited: &mut HashSet<api::ExprTicket>,
+	visited: Substack<'_, api::ExprTicket>,
 ) -> FmtUnit {
-	if visited.contains(&expr.id()) {
+	if visited.iter().any(|id| id == &expr.id()) {
 		return "CYCLIC_EXPR".to_string().into();
 	}
-	visited.insert(expr.id());
+	print_exprkind(&*expr.kind().read().await, c, visited.push(expr.id())).boxed_local().await
-	print_exprkind(&*expr.kind().read().await, c, visited).boxed_local().await
 }
 
 #[derive(Clone, Debug)]
@@ -107,34 +148,18 @@ pub enum ExprKind {
 	Missing,
 }
 impl ExprKind {
-	pub async fn from_api(api: &api::ExpressionKind, pos: Pos, ctx: &mut ExprParseCtx) -> Self {
+	#[must_use]
-		match_mapping!(api, api::ExpressionKind => ExprKind {
-			Lambda(id => PathSet::from_api(*id, api), b => Expr::from_api(b, ctx).await),
-			Bottom(b => OrcErrv::from_api(b, &ctx.ctx().i).await),
-			Call(f => Expr::from_api(f, ctx).await, x => Expr::from_api(x, ctx).await),
-			Const(c => Sym::from_api(*c, &ctx.ctx().i).await),
-			Seq(a => Expr::from_api(a, ctx).await, b => Expr::from_api(b, ctx).await),
-		} {
-			api::ExpressionKind::Arg(_) => ExprKind::Arg,
-			api::ExpressionKind::NewAtom(a) => ExprKind::Atom(AtomHand::from_api(
-				a,
-				pos,
-				&mut ctx.ctx().clone()
-			).await),
-			api::ExpressionKind::Slot(_) => panic!("Handled in Expr"),
-		})
-	}
 	pub fn at(self, pos: Pos) -> Expr { Expr(Rc::new(ExprData { pos, kind: RwLock::new(self) })) }
 }
 impl Format for ExprKind {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		print_exprkind(self, c, &mut HashSet::new()).await
+		print_exprkind(self, c, Substack::Bottom).await
 	}
 }
 async fn print_exprkind<'a>(
 	ek: &ExprKind,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
-	visited: &mut HashSet<api::ExprTicket>,
+	visited: Substack<'_, api::ExprTicket>,
 ) -> FmtUnit {
 	match &ek {
 		ExprKind::Arg => "Arg".to_string().into(),
@@ -174,103 +199,53 @@ pub enum Step {
 	Right,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone)]
-pub struct PathSet {
+pub enum PathSetFrame<'a, T: PartialEq> {
-	/// The single steps through [super::nort::Clause::Apply]
+	Lambda(&'a T, &'a RefCell<Option<PathSet>>),
-	pub steps: Vec<Step>,
+	Step(Step),
-	/// if Some, it splits at a [super::nort::Clause::Apply]. If None, it ends in
-	/// a [super::nort::Clause::LambdaArg]
-	pub next: Option<(Box<PathSet>, Box<PathSet>)>,
-}
-impl PathSet {
-	pub fn next(&self) -> Option<(&PathSet, &PathSet)> {
-		self.next.as_ref().map(|(l, r)| (&**l, &**r))
-	}
-	pub fn from_api(id: u64, api: &api::ExpressionKind) -> Option<Self> {
-		use api::ExpressionKind as K;
-		struct Suffix(VecDeque<Step>, Option<(Box<PathSet>, Box<PathSet>)>);
-		fn seal(Suffix(steps, next): Suffix) -> PathSet { PathSet { steps: steps.into(), next } }
-		fn after(step: Step, mut suf: Suffix) -> Suffix {
-			suf.0.push_front(step);
-			suf
-		}
-		return from_api_inner(id, api).map(seal);
-		fn from_api_inner(id: u64, api: &api::ExpressionKind) -> Option<Suffix> {
-			match &api {
-				K::Arg(id2) => (id == *id2).then_some(Suffix(VecDeque::new(), None)),
-				K::Bottom(_) | K::Const(_) | K::NewAtom(_) | K::Slot(_) => None,
-				K::Lambda(_, b) => from_api_inner(id, &b.kind),
-				K::Call(l, r) | K::Seq(l, r) => {
-					match (from_api_inner(id, &l.kind), from_api_inner(id, &r.kind)) {
-						(Some(a), Some(b)) =>
-							Some(Suffix(VecDeque::new(), Some((Box::new(seal(a)), Box::new(seal(b)))))),
-						(Some(l), None) => Some(after(Step::Left, l)),
-						(None, Some(r)) => Some(after(Step::Right, r)),
-						(None, None) => None,
-					}
-				},
-			}
-		}
-	}
-}
-impl fmt::Display for PathSet {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		fn print_step(step: Step) -> &'static str { if step == Step::Left { "l" } else { "r" } }
-		let step_s = self.steps.iter().copied().map(print_step).join("");
-		match &self.next {
-			Some((left, right)) => {
-				if !step_s.is_empty() {
-					write!(f, "{step_s}>")?;
-				}
-				write!(f, "({left}|{right})")
-			},
-			None => write!(f, "{step_s}x"),
-		}
-	}
-}
-
-pub fn bot_expr(err: impl Into<OrcErrv>) -> Expr {
-	let errv: OrcErrv = err.into();
-	let pos = errv.pos_iter().next().map_or(Pos::None, |ep| ep.position.clone());
-	ExprKind::Bottom(errv).at(pos)
-}
-
-pub struct WeakExpr(Weak<ExprData>);
-impl WeakExpr {
-	pub fn upgrade(&self) -> Option<Expr> { self.0.upgrade().map(Expr) }
 }
 
 #[derive(Clone)]
-pub enum SrcToExprStep<'a> {
+pub struct PathSetBuilder<'a, T: PartialEq>(Substack<'a, PathSetFrame<'a, T>>);
-	Left,
+impl<'a, T: PartialEq> PathSetBuilder<'a, T> {
-	Right,
+	#[must_use]
-	Lambda(Sym, &'a RefCell<Option<PathSet>>),
+	pub fn new() -> Self { Self(Substack::Bottom) }
-}
+	#[must_use]
-
+	pub fn split(&'a self) -> (Self, Self) {
-pub async fn mtreev_to_expr(
+		(
-	src: &[MacTree],
+			Self(self.0.push(PathSetFrame::Step(Step::Left))),
-	stack: Substack<'_, SrcToExprStep<'_>>,
+			Self(self.0.push(PathSetFrame::Step(Step::Right))),
-	ctx: &Ctx,
+		)
-) -> ExprKind {
+	}
-	let Some((x, f)) = src.split_last() else { panic!("Empty expression cannot be evaluated") };
+	#[must_use]
-	let x_stack = if f.is_empty() { stack.clone() } else { stack.push(SrcToExprStep::Right) };
+	pub fn lambda<'b>(self, arg: &'b T) -> LambdaBuilder<'b, T>
-	let x_kind = match &*x.tok {
+	where 'a: 'b {
-		MacTok::Atom(a) => ExprKind::Atom(a.clone()),
+		LambdaBuilder { arg, path: RefCell::default(), stack: self }
-		MacTok::Name(n) => 'name: {
+	}
+	/// Register an argument with the corresponding lambda and return true if one
+	/// was found. (if false is returned, the name is unbound and may refer to a
+	/// global)
+	pub fn register_arg(self, t: &T) -> bool {
 		let mut steps = VecDeque::new();
-			for step in x_stack.iter() {
+		for step in self.0.iter() {
 			match step {
-					SrcToExprStep::Left => steps.push_front(Step::Left),
+				PathSetFrame::Step(step) => steps.push_front(*step),
-					SrcToExprStep::Right => steps.push_front(Step::Right),
+				PathSetFrame::Lambda(name, _) if **name != *t => (),
-					SrcToExprStep::Lambda(name, _) if name != n => continue,
+				PathSetFrame::Lambda(_, cell) => {
-					SrcToExprStep::Lambda(_, cell) => {
+					let mut ps_opt = cell.borrow_mut();
-						let mut ps = cell.borrow_mut();
+					match &mut *ps_opt {
-						match &mut *ps {
 						val @ None => *val = Some(PathSet { steps: steps.into(), next: None }),
 						Some(val) => {
 							let mut swap = PathSet { steps: Vec::new(), next: None };
 							mem::swap(&mut swap, val);
 							*val = merge(swap, &Vec::from(steps));
+						},
+					}
+					return true;
+				},
+			};
+		}
+		return false;
 		fn merge(ps: PathSet, steps: &[Step]) -> PathSet {
 			let diff_idx = ps.steps.iter().zip(steps).take_while(|(l, r)| l == r).count();
 			if diff_idx == ps.steps.len() {
@@ -280,9 +255,7 @@ pub async fn mtreev_to_expr(
 						None => panic!("New path same as old path"),
 					}
 				}
-										let Some((left, right)) = ps.next else {
+				let Some((left, right)) = ps.next else { panic!("Old path ends where new path continues") };
-											panic!("Old path ends where new path continues")
-										};
 				let next = match steps[diff_idx] {
 					Step::Left => Some((Box::new(merge(*left, &steps[diff_idx + 1..])), right)),
 					Step::Right => Some((left, Box::new(merge(*right, &steps[diff_idx + 1..])))),
@@ -301,59 +274,104 @@ pub async fn mtreev_to_expr(
 				PathSet { steps: shared_steps, next: Some((Box::new(left), Box::new(right))) }
 			}
 		}
-							},
 	}
-						break 'name ExprKind::Arg;
+}
-					},
+
-				}
+impl<'a, T: PartialEq> Default for PathSetBuilder<'a, T> {
-			}
+	fn default() -> Self { Self::new() }
-			ExprKind::Const(n.clone())
+}
-		},
+
-		MacTok::Ph(_) | MacTok::Done(_) | MacTok::Ref(_) | MacTok::Slot(_) =>
+pub struct LambdaBuilder<'a, T: PartialEq> {
-			ExprKind::Bottom(mk_errv(
+	arg: &'a T,
-				ctx.i.i("placeholder in value").await,
+	path: RefCell<Option<PathSet>>,
-				"Placeholders cannot appear anywhere outside macro patterns",
+	stack: PathSetBuilder<'a, T>,
-				[x.pos.clone().into()],
+}
-			)),
+impl<'a, T: PartialEq> LambdaBuilder<'a, T> {
-		MacTok::S(Paren::Round, b) if b.is_empty() =>
+	#[must_use]
-			return ExprKind::Bottom(mk_errv(
+	pub fn stack(&'a self) -> PathSetBuilder<'a, T> {
-				ctx.i.i("Empty expression").await,
+		PathSetBuilder(self.stack.0.push(PathSetFrame::Lambda(self.arg, &self.path)))
-				"Empty parens () are illegal",
+	}
-				[x.pos.clone().into()],
+	#[must_use]
-			)),
+	pub fn collect(self) -> Option<PathSet> { self.path.into_inner() }
-		MacTok::S(Paren::Round, b) => mtreev_to_expr(b, x_stack, ctx).boxed_local().await,
+}
-		MacTok::S(..) => ExprKind::Bottom(mk_errv(
+
-			ctx.i.i("non-round parentheses after macros").await,
+#[derive(Clone, Debug)]
-			"[] or {} block was not consumed by macros; expressions may only contain ()",
+pub struct PathSet {
-			[x.pos.clone().into()],
+	/// The single steps through [super::nort::Clause::Apply]
-		)),
+	pub steps: Vec<Step>,
-		MacTok::Lambda(_, b) if b.is_empty() =>
+	/// if Some, it splits at a [super::nort::Clause::Apply]. If None, it ends in
-			return ExprKind::Bottom(mk_errv(
+	/// a [super::nort::Clause::LambdaArg]
-				ctx.i.i("Empty lambda").await,
+	pub next: Option<(Box<PathSet>, Box<PathSet>)>,
-				"Lambdas must have a body",
+}
-				[x.pos.clone().into()],
+impl PathSet {
-			)),
+	#[must_use]
-		MacTok::Lambda(arg, b) => 'lambda_converter: {
+	pub fn next(&self) -> Option<(&PathSet, &PathSet)> {
-			if let [MacTree { tok, .. }] = &**arg {
+		self.next.as_ref().map(|(l, r)| (&**l, &**r))
-				if let MacTok::Name(n) = &**tok {
+	}
-					let path = RefCell::new(None);
+}
-					let b = mtreev_to_expr(b, x_stack.push(SrcToExprStep::Lambda(n.clone(), &path)), ctx)
+impl fmt::Display for PathSet {
-						.boxed_local()
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-						.await;
+		fn print_step(step: Step) -> &'static str { if step == Step::Left { "l" } else { "r" } }
-					break 'lambda_converter ExprKind::Lambda(path.into_inner(), b.at(x.pos.clone()));
+		let step_s = self.steps.iter().copied().map(print_step).join("");
-				}
+		match &self.next {
-			}
+			Some((left, right)) => {
-			let argstr = take_first(&mtreev_fmt(arg, &FmtCtxImpl { i: &ctx.i }).await, true);
+				if !step_s.is_empty() {
-			ExprKind::Bottom(mk_errv(
+					write!(f, "{step_s}>")?;
-				ctx.i.i("Malformeed lambda").await,
+				}
-				format!("Lambda argument should be single name, found {argstr}"),
+				write!(f, "({left}|{right})")
-				[x.pos.clone().into()],
+			},
-			))
+			None => write!(f, "{step_s}x"),
-		},
+		}
-	};
+	}
-	if f.is_empty() {
+}
-		return x_kind;
+
-	}
+#[must_use]
-	let f = mtreev_to_expr(f, stack.push(SrcToExprStep::Left), ctx).boxed_local().await;
+pub fn bot_expr(err: impl Into<OrcErrv>) -> Expr {
-	ExprKind::Call(f.at(Pos::None), x_kind.at(x.pos.clone()))
+	let errv: OrcErrv = err.into();
+	let pos = errv.pos_iter().next().map_or(Pos::None, |ep| ep.position.clone());
+	ExprKind::Bottom(errv).at(pos)
+}
+
+pub struct WeakExpr(Weak<ExprData>);
+impl WeakExpr {
+	#[must_use]
+	pub fn upgrade(&self) -> Option<Expr> { self.0.upgrade().map(Expr) }
+}
+
+impl TokenVariant<api::ExprTicket> for Expr {
+	type FromApiCtx<'a> = ExprStore;
+	async fn from_api(
+		api: &api::ExprTicket,
+		ctx: &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
+		ctx.get_expr(*api).expect("Invalid ticket")
+	}
+	type ToApiCtx<'a> = ExprStore;
+	async fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> api::ExprTicket {
+		let id = self.id();
+		ctx.give_expr(self);
+		id
+	}
+}
+
+/// Acknowledgment that expr serialization is impossible and thus will panic.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct ExprWillPanic;
+
+impl TokenVariant<api::Expression> for Expr {
+	type FromApiCtx<'a> = ExprParseCtx<'a>;
+	async fn from_api(
+		api: &api::Expression,
+		ctx: &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
+		Self::from_api(api, PathSetBuilder::new(), ctx).await
+	}
+	type ToApiCtx<'a> = ExprWillPanic;
+	async fn into_api(self, ExprWillPanic: &mut Self::ToApiCtx<'_>) -> api::Expression {
+		panic!("Cannot serialize expr!")
+	}
 }

orchid-host/src/expr_store.rs

@@ -1,6 +1,8 @@
 use std::cell::RefCell;
 use std::fmt;
+use std::rc::Rc;
 
+use bound::Bound;
 use hashbrown::HashMap;
 use hashbrown::hash_map::Entry;
 
@@ -8,28 +10,83 @@ use crate::api;
 use crate::expr::Expr;
 
 #[derive(Default)]
-pub struct ExprStore(RefCell<HashMap<api::ExprTicket, (u32, Expr)>>);
+pub struct ExprStoreData {
+	exprs: RefCell<HashMap<api::ExprTicket, (u32, Expr)>>,
+	parent: Option<ExprStore>,
+	tracking_parent: bool,
+}
+#[derive(Clone, Default)]
+pub struct ExprStore(Rc<ExprStoreData>);
 impl ExprStore {
+	/// If tracking_parent is false, get_expr can fall back to the parent if none
+	/// is found here.
+	///
+	/// If tracking_parent is true, get_expr can still fall back to the parent,
+	/// but operations on the parent can access the child exprs too until this
+	/// store is dropped.
+	#[must_use]
+	pub fn derive(&self, tracking_parent: bool) -> Self {
+		Self(Rc::new(ExprStoreData {
+			exprs: RefCell::default(),
+			parent: Some(self.clone()),
+			tracking_parent,
+		}))
+	}
 	pub fn give_expr(&self, expr: Expr) {
-		match self.0.borrow_mut().entry(expr.id()) {
+		if self.0.tracking_parent {
+			self.0.parent.as_ref().unwrap().give_expr(expr.clone());
+		}
+		match self.0.exprs.borrow_mut().entry(expr.id()) {
 			Entry::Occupied(mut oe) => oe.get_mut().0 += 1,
 			Entry::Vacant(v) => {
 				v.insert((1, expr));
 			},
 		}
 	}
-	pub fn take_expr(&self, ticket: api::ExprTicket) {
+	pub fn take_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {
-		(self.0.borrow_mut().entry(ticket))
+		if self.0.tracking_parent {
-			.and_replace_entry_with(|_, (rc, rt)| (1 < rc).then_some((rc - 1, rt)));
+			self.0.parent.as_ref().unwrap().take_expr(ticket);
 		}
+		match self.0.exprs.borrow_mut().entry(ticket) {
+			Entry::Vacant(_) => panic!("Attempted to double-take expression"),
+			Entry::Occupied(oe) if oe.get().0 == 1 => Some(oe.remove().1),
+			Entry::Occupied(mut oe) => {
+				oe.get_mut().0 -= 1;
+				Some(oe.get().1.clone())
+			},
+		}
+	}
+	#[must_use]
 	pub fn get_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {
-		self.0.borrow().get(&ticket).map(|(_, expr)| expr.clone())
+		(self.0.exprs.borrow().get(&ticket).map(|(_, expr)| expr.clone()))
+			.or_else(|| self.0.parent.as_ref()?.get_expr(ticket))
+	}
+	pub fn iter(&self) -> impl Iterator<Item = (u32, Expr)> {
+		let r = Bound::new(self.clone(), |this| this.0.exprs.borrow());
+		let mut iter = Bound::new(r, |r| r.values());
+		std::iter::from_fn(move || iter.wrapped_mut().next().cloned())
 	}
 }
 impl fmt::Display for ExprStore {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let r = self.0.borrow();
+		let r = self.0.exprs.borrow();
 		let rc: u32 = r.values().map(|v| v.0).sum();
 		write!(f, "Store holding {rc} refs to {} exprs", r.len())
 	}
 }
+impl Drop for ExprStore {
+	fn drop(&mut self) {
+		if 1 < Rc::strong_count(&self.0) {
+			return;
+		}
+		if !self.0.tracking_parent {
+			return;
+		}
+		let parent = self.0.parent.as_ref().unwrap();
+		for (id, (count, _)) in self.0.exprs.borrow().iter() {
+			for _ in 0..*count {
+				parent.take_expr(*id);
+			}
+		}
+	}
+}

orchid-host/src/extension.rs

@@ -2,29 +2,35 @@ use std::cell::RefCell;
 use std::future::Future;
 use std::io;
 use std::num::NonZeroU64;
+use std::pin::pin;
 use std::rc::{Rc, Weak};
 
-use async_std::channel::{self, Sender};
+use async_fn_stream::stream;
-use async_std::sync::Mutex;
 use derive_destructure::destructure;
-use futures::FutureExt;
+use futures::channel::mpsc::{Sender, channel};
 use futures::future::{join, join_all};
+use futures::lock::Mutex;
+use futures::{SinkExt, StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::HostMsgSet;
 use orchid_api_traits::Request;
 use orchid_base::builtin::ExtInit;
 use orchid_base::clone;
 use orchid_base::format::{FmtCtxImpl, Format};
 use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
+use orchid_base::name::Sym;
 use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _};
+use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::atom::AtomHand;
 use crate::ctx::Ctx;
+use crate::dealias::{ChildError, ChildErrorKind, walk};
 use crate::expr_store::ExprStore;
 use crate::system::SystemCtor;
+use crate::tree::MemberKind;
 
 pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 
@@ -34,20 +40,24 @@ pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 /// upgrading fails.
 #[derive(destructure)]
 pub struct ExtensionData {
+	name: String,
 	ctx: Ctx,
-	init: ExtInit,
 	reqnot: ReqNot<api::HostMsgSet>,
 	systems: Vec<SystemCtor>,
 	logger: Logger,
 	next_pars: RefCell<NonZeroU64>,
 	exprs: ExprStore,
-	lex_recur: Mutex<HashMap<api::ParsId, channel::Sender<ReqPair<api::SubLex>>>>,
+	exiting_snd: Sender<()>,
-	mac_recur: Mutex<HashMap<api::ParsId, channel::Sender<ReqPair<api::RunMacros>>>>,
+	lex_recur: Mutex<HashMap<api::ParsId, Sender<ReqPair<api::SubLex>>>>,
 }
 impl Drop for ExtensionData {
 	fn drop(&mut self) {
 		let reqnot = self.reqnot.clone();
-		(self.ctx.spawn)(Box::pin(async move { reqnot.notify(api::HostExtNotif::Exit).await }))
+		let mut exiting_snd = self.exiting_snd.clone();
+		(self.ctx.spawn)(Box::pin(async move {
+			reqnot.notify(api::HostExtNotif::Exit).await;
+			exiting_snd.send(()).await.unwrap()
+		}))
 	}
 }
 
@@ -55,37 +65,65 @@ impl Drop for ExtensionData {
 pub struct Extension(Rc<ExtensionData>);
 impl Extension {
 	pub fn new(init: ExtInit, logger: Logger, msg_logger: Logger, ctx: Ctx) -> io::Result<Self> {
-		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
+		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| {
-			exprs: ExprStore::default(),
+			let init = Rc::new(init);
+			let (exiting_snd, exiting_rcv) = channel::<()>(0);
+			(ctx.spawn)({
+				clone!(init, weak, ctx);
+				Box::pin(async move {
+					let rcv_stream = stream(async |mut cx| {
+						loop {
+							cx.emit(init.recv().await).await
+						}
+					});
+					let mut event_stream = pin!(stream::select(exiting_rcv.map(|()| None), rcv_stream));
+					while let Some(Some(msg)) = event_stream.next().await {
+						if let Some(reqnot) = weak.upgrade().map(|rc| rc.reqnot.clone()) {
+							let reqnot = reqnot.clone();
+							(ctx.spawn)(Box::pin(async move {
+								reqnot.receive(&msg).await;
+							}))
+						}
+					}
+				})
+			});
+			ExtensionData {
+				name: init.name.clone(),
+				exiting_snd,
+				exprs: ctx.common_exprs.derive(false),
 				ctx: ctx.clone(),
 				systems: (init.systems.iter().cloned())
 					.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
 					.collect(),
 				logger: logger.clone(),
-			init,
 				next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
 				lex_recur: Mutex::default(),
-			mac_recur: Mutex::default(),
 				reqnot: ReqNot::new(
 					msg_logger,
-				clone!(weak; move |sfn, _| clone!(weak; async move {
+					move |sfn, _| clone!(init; Box::pin(async move { init.send(sfn).await })),
-					let data = weak.upgrade().unwrap();
-					data.init.send(sfn).await
-				}.boxed_local())),
 					clone!(weak; move |notif, _| {
 						clone!(weak; Box::pin(async move {
 						let this = Extension(weak.upgrade().unwrap());
+						if !matches!(notif, api::ExtHostNotif::Log(_)) {
+							writeln!(this.reqnot().logger(), "Host received notif {notif:?}");
+						}
 						match notif {
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
 								let target = this.0.exprs.get_expr(acq.1).expect("Invalid ticket");
 								this.0.exprs.give_expr(target)
 							}
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
-							this.assert_own_sys(rel.0).await;
+								if this.is_own_sys(rel.0).await {
-							this.0.exprs.take_expr(rel.1)
+									this.0.exprs.take_expr(rel.1);
+								} else {
+									writeln!(this.reqnot().logger(), "Not our system {:?}", rel.0)
+								}
 							}
 							api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
-							this.assert_own_sys(mov.dec).await;
+								if !this.is_own_sys(mov.dec).await {
+									writeln!(this.reqnot().logger(), "Not our system {:?}", mov.dec);
+									return;
+								}
 								let recp = this.ctx().system_inst(mov.inc).await.expect("invallid recipient sys id");
 								let expr = this.0.exprs.get_expr(mov.expr).expect("invalid ticket");
 								recp.ext().0.exprs.give_expr(expr);
@@ -100,7 +138,9 @@ impl Extension {
 							clone!(weak, ctx);
 							Box::pin(async move {
 								let this = Self(weak.upgrade().unwrap());
+								if !matches!(req, api::ExtHostReq::ExtAtomPrint(_)) {
 									writeln!(this.reqnot().logger(), "Host received request {req:?}");
+								}
 								let i = this.ctx().i.clone();
 								match req {
 									api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
@@ -118,7 +158,8 @@ impl Extension {
 										},
 									},
 									api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
-									let sys = ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
+										let sys =
+											ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
 										let reply =
 											sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await;
 										hand.handle(fw, &reply).await
@@ -128,15 +169,18 @@ impl Extension {
 										hand.handle(fw, &sys.request(body.clone()).await).await
 									},
 									api::ExtHostReq::SubLex(sl) => {
-									let (rep_in, rep_out) = channel::bounded(1);
+										let (rep_in, mut rep_out) = channel(0);
 										{
 											let lex_g = this.0.lex_recur.lock().await;
-										let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
+											let mut req_in =
+												lex_g.get(&sl.id).cloned().expect("Sublex for nonexistent lexid");
 											req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
 										}
-									hand.handle(&sl, &rep_out.recv().await.unwrap()).await
+										hand.handle(&sl, &rep_out.next().await.unwrap()).await
 									},
-								api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect { target })) => {
+									api::ExtHostReq::ExprReq(api::ExprReq::Inspect(
+										ins @ api::Inspect { target },
+									)) => {
 										let expr = this.exprs().get_expr(target).expect("Invalid ticket");
 										hand
 											.handle(&ins, &api::Inspected {
@@ -146,15 +190,60 @@ impl Extension {
 											})
 											.await
 									},
-								api::ExtHostReq::RunMacros(rm) => {
+									api::ExtHostReq::LsModule(ref ls @ api::LsModule(_sys, path)) => {
-									let (rep_in, rep_out) = channel::bounded(1);
+										let reply: <api::LsModule as Request>::Response = 'reply: {
-									let lex_g = this.0.mac_recur.lock().await;
+											let path = i.ex(path).await;
-									let req_in = lex_g.get(&rm.run_id).expect("Sublex for nonexistent lexid");
+											let root = (ctx.root.read().await.upgrade())
-									req_in.send(ReqPair(rm.clone(), rep_in)).await.unwrap();
+												.expect("LSModule called when root isn't in context");
-									hand.handle(&rm, &rep_out.recv().await.unwrap()).await
+											let root_data = &*root.0.read().await;
+											let mut walk_ctx = (ctx.clone(), &root_data.consts);
+											let module =
+												match walk(&root_data.root, false, path.iter().cloned(), &mut walk_ctx)
+													.await
+												{
+													Ok(module) => module,
+													Err(ChildError { kind, .. }) =>
+														break 'reply Err(match kind {
+															ChildErrorKind::Private => panic!("Access checking was disabled"),
+															ChildErrorKind::Constant => api::LsModuleError::IsConstant,
+															ChildErrorKind::Missing => api::LsModuleError::InvalidPath,
+														}),
+												};
+											let mut members = std::collections::HashMap::new();
+											for (k, v) in &module.members {
+												let kind = match v.kind(ctx.clone(), &root_data.consts).await {
+													MemberKind::Const => api::MemberInfoKind::Constant,
+													MemberKind::Module(_) => api::MemberInfoKind::Module,
+												};
+												members.insert(k.to_api(), api::MemberInfo { public: v.public, kind });
+											}
+											Ok(api::ModuleInfo { members })
+										};
+										hand.handle(ls, &reply).await
+									},
+									api::ExtHostReq::ResolveNames(ref rn) => {
+										let api::ResolveNames { constid, names, sys } = rn;
+										let mut resolver = {
+											let systems = ctx.systems.read().await;
+											let weak_sys = systems.get(sys).expect("ResolveNames for invalid sys");
+											let sys = weak_sys.upgrade().expect("ResolveNames after sys drop");
+											sys.name_resolver(*constid).await
+										};
+										let responses = stream(async |mut cx| {
+											for name in names {
+												cx.emit(match resolver(&ctx.i.ex(*name).await[..]).await {
+													Ok(abs) => Ok(abs.to_sym(&ctx.i).await.to_api()),
+													Err(e) => Err(e.to_api()),
+												})
+												.await
+											}
+										})
+										.collect()
+										.await;
+										hand.handle(rn, &responses).await
 									},
 									api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
-									let atom = AtomHand::new(atom.clone(), &ctx).await;
+										let atom = AtomHand::from_api(atom, Pos::None, &mut ctx.clone()).await;
 										let unit = atom.print(&FmtCtxImpl { i: &this.ctx().i }).await;
 										hand.handle(eap, &unit.to_api()).await
 									},
@@ -163,20 +252,28 @@ impl Extension {
 						}
 					},
 				),
+			}
 		})))
 	}
-	pub(crate) fn reqnot(&self) -> &ReqNot<HostMsgSet> { &self.0.reqnot }
+	pub fn name(&self) -> &String { &self.0.name }
+	#[must_use]
+	pub fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.reqnot }
+	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
+	#[must_use]
 	pub fn logger(&self) -> &Logger { &self.0.logger }
 	pub fn system_ctors(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
+	#[must_use]
 	pub fn exprs(&self) -> &ExprStore { &self.0.exprs }
+	#[must_use]
 	pub async fn is_own_sys(&self, id: api::SysId) -> bool {
-		let sys = self.ctx().system_inst(id).await.expect("invalid sender sys id");
+		let Some(sys) = self.ctx().system_inst(id).await else {
+			writeln!(self.logger(), "Invalid system ID {id:?}");
+			return false;
+		};
 		Rc::ptr_eq(&self.0, &sys.ext().0)
 	}
-	pub async fn assert_own_sys(&self, id: api::SysId) {
+	#[must_use]
-		assert!(self.is_own_sys(id).await, "Incoming message impersonates separate system");
-	}
 	pub fn next_pars(&self) -> NonZeroU64 {
 		let mut next_pars = self.0.next_pars.borrow_mut();
 		*next_pars = next_pars.checked_add(1).unwrap_or(NonZeroU64::new(1).unwrap());
@@ -185,6 +282,7 @@ impl Extension {
 	pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
 		source: Tok<String>,
+		src: Sym,
 		pos: u32,
 		sys: api::SysId,
 		mut r: impl FnMut(u32) -> F,
@@ -192,18 +290,19 @@ impl Extension {
 		// get unique lex ID
 		let id = api::ParsId(self.next_pars());
 		// create and register channel
-		let (req_in, req_out) = channel::bounded(1);
+		let (req_in, mut req_out) = channel(0);
 		self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released
 		let (ret, ()) = join(
 			async {
-				let res =
+				let res = (self.reqnot())
-					(self.reqnot()).request(api::LexExpr { id, pos, sys, text: source.to_api() }).await;
+					.request(api::LexExpr { id, pos, sys, src: src.to_api(), text: source.to_api() })
+					.await;
 				// collect sender to unblock recursion handler branch before returning
 				self.0.lex_recur.lock().await.remove(&id);
 				res
 			},
 			async {
-				while let Ok(ReqPair(sublex, rep_in)) = req_out.recv().await {
+				while let Some(ReqPair(sublex, mut rep_in)) = req_out.next().await {
 					(rep_in.send(r(sublex.pos).await).await)
 						.expect("Response channel dropped while request pending")
 				}
@@ -212,31 +311,20 @@ impl Extension {
 		.await;
 		ret.transpose()
 	}
-	pub async fn recv_one(&self) {
-		let reqnot = self.0.reqnot.clone();
-		let ctx = self.ctx().clone();
-		(self.0.init.recv(Box::new(move |msg| {
-			Box::pin(async move {
-				let msg = msg.to_vec();
-				let reqnot = reqnot.clone();
-				(ctx.spawn)(Box::pin(async move {
-					reqnot.receive(&msg).await;
-				}))
-			})
-		})))
-		.await;
-	}
 	pub fn system_drop(&self, id: api::SysId) {
 		let rc = self.clone();
 		(self.ctx().spawn)(Box::pin(async move {
-			rc.reqnot().notify(api::SystemDrop(id)).await;
+			rc.reqnot().request(api::SystemDrop(id)).await;
 			rc.ctx().systems.write().await.remove(&id);
 		}))
 	}
+	#[must_use]
 	pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }
 }
 
+#[derive(Clone)]
 pub struct WeakExtension(Weak<ExtensionData>);
 impl WeakExtension {
+	#[must_use]
 	pub fn upgrade(&self) -> Option<Extension> { self.0.upgrade().map(Extension) }
 }

orchid-host/src/lex.rs

@@ -1,48 +1,52 @@
-use std::num::NonZeroU64;
+use std::rc::Rc;
-use std::sync::Arc;
 
-use async_std::sync::Mutex;
 use futures::FutureExt;
-use hashbrown::HashMap;
+use futures::lock::Mutex;
+use orchid_base::clone;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
 use orchid_base::interner::Tok;
-use orchid_base::location::Pos;
+use orchid_base::location::SrcRange;
-use orchid_base::match_mapping;
 use orchid_base::name::Sym;
-use orchid_base::number::{num_to_err, parse_num};
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::PARENS;
-use orchid_base::tree::{AtomRepr, Ph};
+use orchid_base::tree::recur;
 
 use crate::api;
-use crate::atom::AtomHand;
 use crate::ctx::Ctx;
+use crate::expr::{Expr, ExprParseCtx};
+use crate::expr_store::ExprStore;
+use crate::parsed::{ParsTok, ParsTokTree, tt_to_api};
 use crate::system::System;
-use crate::tree::{ParsTok, ParsTokTree};
 
 pub struct LexCtx<'a> {
 	pub systems: &'a [System],
 	pub source: &'a Tok<String>,
+	pub path: &'a Sym,
 	pub tail: &'a str,
-	pub sub_trees: &'a mut HashMap<api::TreeTicket, ParsTokTree>,
+	pub sub_trees: &'a mut Vec<Expr>,
 	pub ctx: &'a Ctx,
 }
 impl<'a> LexCtx<'a> {
+	#[must_use]
 	pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
 	where 'a: 'b {
 		LexCtx {
 			source: self.source,
+			path: self.path,
 			tail: &self.source[pos as usize..],
 			systems: self.systems,
 			sub_trees: &mut *self.sub_trees,
 			ctx: self.ctx,
 		}
 	}
+	#[must_use]
 	pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 }
+	#[must_use]
 	pub fn end_pos(&self) -> u32 { self.source.len() as u32 }
 	pub fn set_pos(&mut self, pos: u32) { self.tail = &self.source[pos as usize..] }
 	pub fn push_pos(&mut self, delta: u32) { self.set_pos(self.get_pos() + delta) }
 	pub fn set_tail(&mut self, tail: &'a str) { self.tail = tail }
+	#[must_use]
 	pub fn strip_prefix(&mut self, tgt: &str) -> bool {
 		if let Some(src) = self.tail.strip_prefix(tgt) {
 			self.tail = src;
@@ -50,14 +54,22 @@ impl<'a> LexCtx<'a> {
 		}
 		false
 	}
-	pub fn add_subtree(&mut self, subtree: ParsTokTree) -> api::TreeTicket {
+	#[must_use]
-		let next_idx = api::TreeTicket(NonZeroU64::new(self.sub_trees.len() as u64 + 1).unwrap());
+	pub async fn ser_subtree(&mut self, subtree: ParsTokTree, exprs: ExprStore) -> api::TokenTree {
-		self.sub_trees.insert(next_idx, subtree);
+		tt_to_api(&mut { exprs }, subtree).await
-		next_idx
 	}
-	pub fn rm_subtree(&mut self, ticket: api::TreeTicket) -> ParsTokTree {
+	#[must_use]
-		self.sub_trees.remove(&ticket).unwrap()
+	pub async fn des_subtree(&mut self, tree: &api::TokenTree, exprs: ExprStore) -> ParsTokTree {
+		ParsTokTree::from_api(
+			tree,
+			&mut { exprs },
+			&mut ExprParseCtx { ctx: self.ctx, exprs: &self.ctx.common_exprs },
+			self.path,
+			&self.ctx.i,
+		)
+		.await
 	}
+	#[must_use]
 	pub fn strip_char(&mut self, tgt: char) -> bool {
 		if let Some(src) = self.tail.strip_prefix(tgt) {
 			self.tail = src;
@@ -69,6 +81,7 @@ impl<'a> LexCtx<'a> {
 		self.tail = self.tail.trim_start_matches(filter);
 	}
 	pub fn trim_ws(&mut self) { self.trim(|c| c.is_whitespace() && !"\r\n".contains(c)) }
+	#[must_use]
 	pub fn get_start_matches(&mut self, filter: impl Fn(char) -> bool) -> &'a str {
 		let rest = self.tail.trim_start_matches(filter);
 		let matches = &self.tail[..self.tail.len() - rest.len()];
@@ -86,37 +99,33 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 	);
 	let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
 		ParsTok::BR
-	} else if ctx.strip_prefix("::") {
+	} else if let Some(tail) = (ctx.tail.starts_with(name_start).then_some(ctx.tail))
-		ParsTok::NS
+		.and_then(|t| t.trim_start_matches(name_char).strip_prefix("::"))
+	{
+		let name = &ctx.tail[..ctx.tail.len() - tail.len() - "::".len()];
+		ctx.set_tail(tail);
+		let body = lex_once(ctx).boxed_local().await?;
+		ParsTok::NS(ctx.ctx.i.i(name).await, Box::new(body))
 	} else if ctx.strip_prefix("--[") {
 		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
 			return Err(mk_errv(
 				ctx.ctx.i.i("Unterminated block comment").await,
 				"This block comment has no ending ]--",
-				[Pos::Range(start..start + 3).into()],
+				[SrcRange::new(start..start + 3, ctx.path)],
 			));
 		};
 		ctx.set_tail(tail);
-		ParsTok::Comment(Arc::new(cmt.to_string()))
+		ParsTok::Comment(Rc::new(cmt.to_string()))
 	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
 		ctx.push_pos(end as u32);
-		ParsTok::Comment(Arc::new(tail[2..end].to_string()))
+		ParsTok::Comment(Rc::new(tail[2..end].to_string()))
-	} else if ctx.strip_char('\\') {
+	} else if let Some(tail) = ctx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
-		let mut arg = Vec::new();
+		// fanciness like \$placeh in templates is resolved in the macro engine.
+		ctx.set_tail(tail);
+		let arg = lex_once(ctx).boxed_local().await?;
 		ctx.trim_ws();
-		while !ctx.strip_char('.') {
+		ParsTok::LambdaHead(Box::new(arg))
-			if ctx.tail.is_empty() {
-				return Err(mk_errv(
-					ctx.ctx.i.i("Unclosed lambda").await,
-					"Lambdae started with \\ should separate arguments from body with .",
-					[Pos::Range(start..start + 1).into()],
-				));
-			}
-			arg.push(lex_once(ctx).boxed_local().await?);
-			ctx.trim_ws();
-		}
-		ParsTok::LambdaHead(arg)
 	} else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
 		let mut body = Vec::new();
 		ctx.trim_ws();
@@ -125,44 +134,38 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 				return Err(mk_errv(
 					ctx.ctx.i.i("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
-					[Pos::Range(start..start + 1).into()],
+					[SrcRange::new(start..start + 1, ctx.path)],
 				));
 			}
 			body.push(lex_once(ctx).boxed_local().await?);
 			ctx.trim_ws();
 		}
 		ParsTok::S(*paren, body)
-	} else if ctx.strip_prefix("macro")
-		&& !ctx.tail.chars().next().is_some_and(|x| x.is_ascii_alphabetic())
-	{
-		ctx.strip_prefix("macro");
-		if ctx.strip_char('(') {
-			let pos = ctx.get_pos();
-			let numstr = ctx.get_start_matches(|x| x != ')').trim();
-			match parse_num(numstr) {
-				Ok(num) => ParsTok::Macro(Some(num.to_f64())),
-				Err(e) => return Err(num_to_err(e, pos, &ctx.ctx.i).await.into()),
-			}
-		} else {
-			ParsTok::Macro(None)
-		}
 	} else {
 		for sys in ctx.systems {
 			let mut errors = Vec::new();
 			if ctx.tail.starts_with(|c| sys.can_lex(c)) {
-				let (source, pos) = (ctx.source.clone(), ctx.get_pos());
+				let (source, pos, path) = (ctx.source.clone(), ctx.get_pos(), ctx.path.clone());
 				let ctx_lck = &Mutex::new(&mut *ctx);
 				let errors_lck = &Mutex::new(&mut errors);
+				let temp_store = sys.ext().exprs().derive(true);
+				let temp_store_cb = temp_store.clone();
 				let lx = sys
-					.lex(source, pos, |pos| async move {
+					.lex(source, path, pos, |pos| {
+						clone!(temp_store_cb);
+						async move {
 							let mut ctx_g = ctx_lck.lock().await;
 							match lex_once(&mut ctx_g.push(pos)).boxed_local().await {
-							Ok(t) => Some(api::SubLexed { pos: t.range.end, ticket: ctx_g.add_subtree(t) }),
+								Ok(t) => Some(api::SubLexed {
+									pos: t.sr.end(),
+									tree: ctx_g.ser_subtree(t, temp_store_cb.clone()).await,
+								}),
 								Err(e) => {
 									errors_lck.lock().await.push(e);
 									None
 								},
 							}
+						}
 					})
 					.await;
 				match lx {
@@ -172,7 +175,14 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 						),
 					Ok(Some(lexed)) => {
 						ctx.set_pos(lexed.pos);
-						return Ok(tt_to_owned(&lexed.expr, ctx).await);
+						let lexed_tree = ctx.des_subtree(&lexed.expr, temp_store).await;
+						let stable_tree = recur(lexed_tree, &|tt, r| {
+							if let ParsTok::NewExpr(expr) = tt.tok {
+								return ParsTok::Handle(expr).at(tt.sr);
+							}
+							r(tt)
+						});
+						return Ok(stable_tree);
 					},
 					Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
 						Some(errors) => return Err(errors),
@@ -189,47 +199,22 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 			return Err(mk_errv(
 				ctx.ctx.i.i("Unrecognized character").await,
 				"The following syntax is meaningless.",
-				[Pos::Range(start..start + 1).into()],
+				[SrcRange::new(start..start + 1, ctx.path)],
 			));
 		}
 	};
-	Ok(ParsTokTree { tok, range: start..ctx.get_pos() })
+	Ok(ParsTokTree { tok, sr: SrcRange::new(start..ctx.get_pos(), ctx.path) })
 }
 
-async fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
+pub async fn lex(
-	let tok = match_mapping!(&api.token, api::Token => ParsTok {
+	text: Tok<String>,
-		Atom(atom =>
+	path: Sym,
-			AtomHand::from_api(atom, Pos::Range(api.range.clone()), &mut ctx.ctx.clone()).await
+	systems: &[System],
-		),
+	ctx: &Ctx,
-		Bottom(err => OrcErrv::from_api(err, &ctx.ctx.i).await),
+) -> OrcRes<Vec<ParsTokTree>> {
-		LambdaHead(arg => ttv_to_owned(arg, ctx).boxed_local().await),
+	let mut sub_trees = Vec::new();
-		Name(name => Tok::from_api(*name, &ctx.ctx.i).await),
+	let mut ctx =
-		Reference(tstrv => Sym::from_api(*tstrv, &ctx.ctx.i).await),
+		LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, path: &path, ctx };
-		S(p.clone(), b => ttv_to_owned(b, ctx).boxed_local().await),
-		BR, NS,
-		Comment(c.clone()),
-		Ph(ph => Ph::from_api(ph, &ctx.ctx.i).await),
-		Macro(*prio),
-	} {
-		api::Token::Slot(id) => return ctx.rm_subtree(*id),
-	});
-	ParsTokTree { range: api.range.clone(), tok }
-}
-
-async fn ttv_to_owned<'a>(
-	api: impl IntoIterator<Item = &'a api::TokenTree>,
-	ctx: &mut LexCtx<'_>,
-) -> Vec<ParsTokTree> {
-	let mut out = Vec::new();
-	for tt in api {
-		out.push(tt_to_owned(tt, ctx).await)
-	}
-	out
-}
-
-pub async fn lex(text: Tok<String>, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
-	let mut sub_trees = HashMap::new();
-	let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, ctx };
 	let mut tokv = Vec::new();
 	ctx.trim(unrep_space);
 	while !ctx.tail.is_empty() {

orchid-host/src/lib.rs

@@ -8,9 +8,9 @@ pub mod expr;
 pub mod expr_store;
 pub mod extension;
 pub mod lex;
-pub mod macros;
 pub mod parse;
-pub mod rule;
+pub mod parsed;
 pub mod subprocess;
+mod sys_parser;
 pub mod system;
 pub mod tree;

orchid-host/src/macros.rs

@@ -1,103 +0,0 @@
-use std::rc::Rc;
-
-use futures::FutureExt;
-use hashbrown::{HashMap, HashSet};
-use itertools::Itertools;
-use orchid_base::clone;
-use orchid_base::macros::{MTok, MTree, mtreev_from_api, mtreev_to_api};
-use orchid_base::name::Sym;
-use trait_set::trait_set;
-
-use crate::api;
-use crate::atom::AtomHand;
-use crate::ctx::Ctx;
-use crate::rule::state::MatchState;
-use crate::tree::Code;
-
-pub type MacTok = MTok<'static, AtomHand>;
-pub type MacTree = MTree<'static, AtomHand>;
-
-trait_set! {
-	trait MacroCB = Fn(Vec<MacTree>) -> Option<Vec<MacTree>>;
-}
-
-type Slots = HashMap<api::MacroTreeId, Rc<MacTok>>;
-
-pub async fn macro_treev_to_api(mtree: Vec<MacTree>, slots: &mut Slots) -> Vec<api::MacroTree> {
-	mtreev_to_api(&mtree, &mut |a: &AtomHand| {
-		let id = api::MacroTreeId((slots.len() as u64 + 1).try_into().unwrap());
-		slots.insert(id, Rc::new(MacTok::Atom(a.clone())));
-		async move { api::MacroToken::Slot(id) }.boxed_local()
-	})
-	.await
-}
-
-pub async fn macro_treev_from_api(api: Vec<api::MacroTree>, ctx: Ctx) -> Vec<MacTree> {
-	mtreev_from_api(&api, &ctx.clone().i, &mut move |atom| {
-		clone!(ctx);
-		Box::pin(async move { MacTok::Atom(AtomHand::new(atom.clone(), &ctx).await) })
-	})
-	.await
-}
-
-pub fn deslot_macro(tree: &[MacTree], slots: &mut Slots) -> Option<Vec<MacTree>> {
-	return work(slots, tree);
-	fn work(
-		slots: &mut HashMap<api::MacroTreeId, Rc<MacTok>>,
-		tree: &[MacTree],
-	) -> Option<Vec<MacTree>> {
-		let items = (tree.iter())
-			.map(|t| {
-				Some(MacTree {
-					tok: match &*t.tok {
-						MacTok::Atom(_) | MacTok::Name(_) | MacTok::Ph(_) => return None,
-						MacTok::Ref(_) => panic!("Ref is an extension-local optimization"),
-						MacTok::Done(_) => panic!("Created and removed by matcher"),
-						MacTok::Slot(slot) => slots.get(&slot.id()).expect("Slot not found").clone(),
-						MacTok::S(paren, b) => Rc::new(MacTok::S(*paren, work(slots, b)?)),
-						MacTok::Lambda(a, b) => Rc::new(match (work(slots, a), work(slots, b)) {
-							(None, None) => return None,
-							(Some(a), None) => MacTok::Lambda(a, b.clone()),
-							(None, Some(b)) => MacTok::Lambda(a.clone(), b),
-							(Some(a), Some(b)) => MacTok::Lambda(a, b),
-						}),
-					},
-					pos: t.pos.clone(),
-				})
-			})
-			.collect_vec();
-		let any_changed = items.iter().any(Option::is_some);
-		any_changed.then(|| {
-			(items.into_iter().enumerate())
-				.map(|(i, opt)| opt.unwrap_or_else(|| tree[i].clone()))
-				.collect_vec()
-		})
-	}
-}
-
-pub struct Macro<Matcher> {
-	deps: HashSet<Sym>,
-	cases: Vec<(Matcher, Code)>,
-}
-
-fn fill_lexicon(tgt: &MacTree, lexicon: &mut HashSet<Sym>) {
-	match &*tgt.tok {
-		MTok::Name(n) => {
-			lexicon.insert(n.clone());
-		},
-		MTok::Lambda(arg, body) => {
-			arg.iter().for_each(|t| fill_lexicon(t, lexicon));
-			body.iter().for_each(|t| fill_lexicon(t, lexicon))
-		},
-		MTok::S(_, body) => body.iter().for_each(|t| fill_lexicon(t, lexicon)),
-		_ => (),
-	}
-}
-
-fn run_body(body: &Code, mut state: MatchState<'_>) -> Vec<MacTree> {
-	let inject: Vec<MacTree> = todo!("Call the interpreter with bindings");
-	inject
-		.into_iter()
-		.map(|MTree { pos, tok }| MTree { pos, tok: Rc::new(MTok::Done(tok)) })
-		.collect_vec()
-}

orchid-host/src/parse.rs

@@ -1,317 +1,159 @@
-use std::rc::Rc;
-
-use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
-use never::Never;
+use orchid_base::error::{OrcRes, Reporter, mk_errv};
-use orchid_base::error::{OrcErrv, OrcRes, Reporter, ReporterImpl, mk_err, mk_errv};
+use orchid_base::format::fmt;
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
-use orchid_base::location::Pos;
-use orchid_base::macros::{MTok, MTree};
 use orchid_base::name::Sym;
 use orchid_base::parse::{
-	Comment, Import, Parsed, Snippet, expect_end, line_items, parse_multiname, try_pop_no_fluff,
+	Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname,
+	try_pop_no_fluff,
 };
 use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
 
-use crate::atom::AtomHand;
+use crate::ctx::Ctx;
-use crate::macros::MacTree;
+use crate::expr::Expr;
+use crate::parsed::{Item, ItemKind, ParsedMember, ParsedMemberKind, ParsedModule};
 use crate::system::System;
-use crate::tree::{Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, Rule, RuleKind};
 
-type ParsSnippet<'a> = Snippet<'a, 'static, AtomHand, Never>;
+type ParsSnippet<'a> = Snippet<'a, Expr, Expr>;
 
-pub struct ParseCtxImpl<'a> {
+pub struct HostParseCtxImpl<'a> {
+	pub ctx: Ctx,
+	pub src: Sym,
 	pub systems: &'a [System],
-	pub reporter: &'a ReporterImpl,
+	pub rep: &'a Reporter,
 }
 
-impl ParseCtx for ParseCtxImpl<'_> {
+impl ParseCtx for HostParseCtxImpl<'_> {
-	fn reporter(&self) -> &(impl Reporter + ?Sized) { self.reporter }
+	fn rep(&self) -> &Reporter { self.rep }
+	fn i(&self) -> &Interner { &self.ctx.i }
+}
+
+impl HostParseCtx for HostParseCtxImpl<'_> {
+	fn ctx(&self) -> &Ctx { &self.ctx }
 	fn systems(&self) -> impl Iterator<Item = &System> { self.systems.iter() }
+	fn src_path(&self) -> Sym { self.src.clone() }
 }
 
-pub trait ParseCtx {
+pub trait HostParseCtx: ParseCtx {
+	#[must_use]
+	fn ctx(&self) -> &Ctx;
+	#[must_use]
 	fn systems(&self) -> impl Iterator<Item = &System>;
-	fn reporter(&self) -> &(impl Reporter + ?Sized);
+	#[must_use]
+	fn src_path(&self) -> Sym;
 }
 
 pub async fn parse_items(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
 	path: Substack<'_, Tok<String>>,
 	items: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	let lines = line_items(items).await;
+	let lines = line_items(ctx, items).await;
 	let line_res =
 		join_all(lines.into_iter().map(|p| parse_item(ctx, path.clone(), p.output, p.tail))).await;
 	Ok(line_res.into_iter().flat_map(|l| l.ok().into_iter().flatten()).collect())
 }
 
 pub async fn parse_item(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
 	path: Substack<'_, Tok<String>>,
 	comments: Vec<Comment>,
 	item: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
 	match item.pop_front() {
 		Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
-			n if *n == item.i().i("export").await => match try_pop_no_fluff(postdisc).await? {
+			n if *n == ctx.i().i("export").await => match try_pop_no_fluff(ctx, postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
 					parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
-				Parsed { output: TokTree { tok: Token::NS, .. }, tail } => {
+				Parsed { output, tail: _ } => Err(mk_errv(
-					let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail).await?;
+					ctx.i().i("Malformed export").await,
-					let mut ok = Vec::new();
+					"`export` can either prefix other lines or list names inside ( )",
-					for (e, pos) in exports {
+					[output.sr()],
-						match (&e.path[..], e.name) {
-							([], Some(n)) =>
-								ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }),
-							(_, Some(_)) => ctx.reporter().report(mk_err(
-								tail.i().i("Compound export").await,
-								"Cannot export compound names (names containing the :: separator)",
-								[pos.into()],
-							)),
-							(_, None) => ctx.reporter().report(mk_err(
-								tail.i().i("Wildcard export").await,
-								"Exports cannot contain the globstar *",
-								[pos.into()],
-							)),
-						}
-					}
-					expect_end(tail).await?;
-					Ok(ok)
-				},
-				Parsed { output, tail } => Err(mk_errv(
-					tail.i().i("Malformed export").await,
-					"`export` can either prefix other lines or list names inside ::( ) or ::[ ]",
-					[Pos::Range(output.range.clone()).into()],
 				)),
 			},
-			n if *n == item.i().i("import").await => parse_import(ctx, postdisc).await.map(|v| {
+			n if *n == ctx.i().i("import").await => {
-				Vec::from_iter(v.into_iter().map(|(t, pos)| Item {
+				let imports = parse_import(ctx, postdisc).await?;
+				Ok(Vec::from_iter(imports.into_iter().map(|t| Item {
 					comments: comments.clone(),
-					pos,
+					sr: t.sr.clone(),
 					kind: ItemKind::Import(t),
-				}))
+				})))
-			}),
+			},
 			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
 		},
 		Some(_) => Err(mk_errv(
-			item.i().i("Expected a line type").await,
+			ctx.i().i("Expected a line type").await,
 			"All lines must begin with a keyword",
-			[Pos::Range(item.pos()).into()],
+			[item.sr()],
 		)),
 		None => unreachable!("These lines are filtered and aggregated in earlier stages"),
 	}
 }
 
-pub async fn parse_import(
+pub async fn parse_import<'a>(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
-	tail: ParsSnippet<'_>,
+	tail: ParsSnippet<'a>,
-) -> OrcRes<Vec<(Import, Pos)>> {
+) -> OrcRes<Vec<Import>> {
-	let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail).await?;
+	let Parsed { output: imports, tail } = parse_multiname(ctx, tail).await?;
-	expect_end(tail).await?;
+	expect_end(ctx, tail).await?;
 	Ok(imports)
 }
 
-pub async fn parse_exportable_item(
+pub async fn parse_exportable_item<'a>(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
 	path: Substack<'_, Tok<String>>,
 	comments: Vec<Comment>,
 	exported: bool,
 	discr: Tok<String>,
-	tail: ParsSnippet<'_>,
+	tail: ParsSnippet<'a>,
 ) -> OrcRes<Vec<Item>> {
-	let kind = if discr == tail.i().i("mod").await {
+	let kind = if discr == ctx.i().i("mod").await {
 		let (name, body) = parse_module(ctx, path, tail).await?;
-		ItemKind::Member(Member::new(name, MemberKind::Mod(body)))
+		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::Mod(body) })
-	} else if discr == tail.i().i("const").await {
+	} else if let Some(parser) = ctx.systems().find_map(|s| s.get_parser(discr.clone())) {
-		let (name, val) = parse_const(tail, path.clone()).await?;
+		return parser
-		let locator = CodeLocator::to_const(tail.i().i(&path.push(name.clone()).unreverse()).await);
+			.parse(ctx, path, tail.to_vec(), exported, comments, &mut async |stack, lines| {
-		ItemKind::Member(Member::new(name, MemberKind::Const(Code::from_code(locator, val))))
+				let source = Snippet::new(lines.first().unwrap(), &lines);
-	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
+				parse_items(ctx, stack, source).await
-		let line = sys.parse(tail.to_vec(), exported, comments).await?;
+			})
-		return parse_items(ctx, path, Snippet::new(tail.prev(), &line, tail.i())).await;
+			.await;
 	} else {
 		let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
 		return Err(mk_errv(
-			tail.i().i("Unrecognized line type").await,
+			ctx.i().i("Unrecognized line type").await,
-			format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
+			format!("Line types are: mod, {ext_lines}"),
-			[Pos::Range(tail.prev().range.clone()).into()],
+			[tail.prev().sr()],
 		));
 	};
-	Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }])
+	Ok(vec![Item { comments, sr: tail.sr(), kind }])
 }
 
-pub async fn parse_module(
+pub async fn parse_module<'a>(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
 	path: Substack<'_, Tok<String>>,
-	tail: ParsSnippet<'_>,
+	tail: ParsSnippet<'a>,
-) -> OrcRes<(Tok<String>, Module)> {
+) -> OrcRes<(Tok<String>, ParsedModule)> {
-	let (name, tail) = match try_pop_no_fluff(tail).await? {
+	let (name, tail) = match try_pop_no_fluff(ctx, tail).await? {
 		Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
 		Parsed { output, .. } => {
 			return Err(mk_errv(
-				tail.i().i("Missing module name").await,
+				ctx.i().i("Missing module name").await,
-				format!("A name was expected, {} was found", tail.fmt(output).await),
+				format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
-				[Pos::Range(output.range.clone()).into()],
+				[output.sr()],
 			));
 		},
 	};
-	let Parsed { output, tail: surplus } = try_pop_no_fluff(tail).await?;
+	let Parsed { output, tail: surplus } = try_pop_no_fluff(ctx, tail).await?;
-	expect_end(surplus).await?;
+	expect_end(ctx, surplus).await?;
-	let Some(body) = output.as_s(Paren::Round, tail.i()) else {
+	let Some(body) = output.as_s(Paren::Round) else {
 		return Err(mk_errv(
-			tail.i().i("Expected module body").await,
+			ctx.i().i("Expected module body").await,
-			format!("A ( block ) was expected, {} was found", tail.fmt(output).await),
+			format!("A ( block ) was expected, {} was found", fmt(output, ctx.i()).await),
-			[Pos::Range(output.range.clone()).into()],
+			[output.sr()],
 		));
 	};
 	let path = path.push(name.clone());
-	Ok((name, Module::new(parse_items(ctx, path, body).await?)))
+	Ok((name, ParsedModule::new(true, parse_items(ctx, path, body).await?)))
-}
-
-pub async fn parse_const(
-	tail: ParsSnippet<'_>,
-	path: Substack<'_, Tok<String>>,
-) -> OrcRes<(Tok<String>, Vec<MacTree>)> {
-	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
-	let Some(name) = output.as_name() else {
-		return Err(mk_errv(
-			tail.i().i("Missing module name").await,
-			format!("A name was expected, {} was found", tail.fmt(output).await),
-			[Pos::Range(output.range.clone()).into()],
-		));
-	};
-	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
-	if !output.is_kw(tail.i().i("=").await) {
-		return Err(mk_errv(
-			tail.i().i("Missing = separator").await,
-			format!("Expected = , found {}", tail.fmt(output).await),
-			[Pos::Range(output.range.clone()).into()],
-		));
-	}
-	try_pop_no_fluff(tail).await?;
-	Ok((name, parse_mtree(tail, path).await?))
-}
-
-pub async fn parse_mtree(
-	mut snip: ParsSnippet<'_>,
-	path: Substack<'_, Tok<String>>,
-) -> OrcRes<Vec<MacTree>> {
-	let mut mtreev = Vec::new();
-	while let Some((ttree, tail)) = snip.pop_front() {
-		snip = tail;
-		let (range, tok, tail) = match &ttree.tok {
-			Token::S(p, b) => {
-				let b = parse_mtree(Snippet::new(ttree, b, snip.i()), path.clone()).boxed_local().await?;
-				(ttree.range.clone(), MTok::S(*p, b), tail)
-			},
-			Token::Reference(name) => (ttree.range.clone(), MTok::Name(name.clone()), tail),
-			Token::Name(tok) => {
-				let mut segments = path.unreverse();
-				segments.push(tok.clone());
-				let mut end = ttree.range.end;
-				while let Some((TokTree { tok: Token::NS, .. }, tail)) = snip.pop_front() {
-					let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
-					let Some(seg) = output.as_name() else {
-						return Err(mk_errv(
-							tail.i().i("Namespaced name interrupted").await,
-							"In expression context, :: must always be followed by a name.\n\
-											::() is permitted only in import and export items",
-							[Pos::Range(output.range.clone()).into()],
-						));
-					};
-					segments.push(seg);
-					snip = tail;
-					end = output.range.end;
-				}
-				(ttree.range.start..end, MTok::Name(Sym::new(segments, snip.i()).await.unwrap()), snip)
-			},
-			Token::NS => {
-				return Err(mk_errv(
-					tail.i().i("Unexpected :: in expression").await,
-					":: can only follow a name",
-					[Pos::Range(ttree.range.clone()).into()],
-				));
-			},
-			Token::Ph(ph) => (ttree.range.clone(), MTok::Ph(ph.clone()), tail),
-			Token::Macro(_) => {
-				return Err(mk_errv(
-					tail.i().i("Invalid keyword in expression").await,
-					"Expressions cannot use `macro` as a name.",
-					[Pos::Range(ttree.range.clone()).into()],
-				));
-			},
-			Token::Atom(a) => (ttree.range.clone(), MTok::Atom(a.clone()), tail),
-			Token::BR | Token::Comment(_) => continue,
-			Token::Bottom(e) => return Err(e.clone()),
-			Token::LambdaHead(arg) => (
-				ttree.range.start..snip.pos().end,
-				MTok::Lambda(
-					parse_mtree(Snippet::new(ttree, arg, snip.i()), path.clone()).boxed_local().await?,
-					parse_mtree(tail, path.clone()).boxed_local().await?,
-				),
-				Snippet::new(ttree, &[], snip.i()),
-			),
-			Token::Slot(_) | Token::X(_) =>
-				panic!("Did not expect {} in parsed token tree", tail.fmt(ttree).await),
-		};
-		mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Rc::new(tok) });
-		snip = tail;
-	}
-	Ok(mtreev)
-}
-
-pub async fn parse_macro(
-	tail: ParsSnippet<'_>,
-	macro_i: u16,
-	path: Substack<'_, Tok<String>>,
-) -> OrcRes<Vec<Rule>> {
-	let (surplus, prev, block) = match try_pop_no_fluff(tail).await? {
-		Parsed { tail, output: o @ TokTree { tok: Token::S(Paren::Round, b), .. } } => (tail, o, b),
-		Parsed { output, .. } => {
-			return Err(mk_errv(
-				tail.i().i("m").await,
-				"Macro blocks must either start with a block or a ..$:number",
-				[Pos::Range(output.range.clone()).into()],
-			));
-		},
-	};
-	expect_end(surplus).await?;
-	let mut errors = Vec::new();
-	let mut rules = Vec::new();
-	for (i, item) in line_items(Snippet::new(prev, block, tail.i())).await.into_iter().enumerate() {
-		let Parsed { tail, output } = try_pop_no_fluff(item.tail).await?;
-		if !output.is_kw(tail.i().i("rule").await) {
-			errors.extend(mk_errv(
-				tail.i().i("non-rule in macro").await,
-				format!("Expected `rule`, got {}", tail.fmt(output).await),
-				[Pos::Range(output.range.clone()).into()],
-			));
-			continue;
-		};
-		let arrow = tail.i().i("=>").await;
-		let (pat, body) = match tail.split_once(|t| t.is_kw(arrow.clone())) {
-			Some((a, b)) => (a, b),
-			None => {
-				errors.extend(mk_errv(
-					tail.i().i("no => in macro rule").await,
-					"The pattern and body of a rule must be separated by a =>",
-					[Pos::Range(tail.pos()).into()],
-				));
-				continue;
-			},
-		};
-		rules.push(Rule {
-			comments: item.output,
-			pos: Pos::Range(tail.pos()),
-			pattern: parse_mtree(pat, path.clone()).await?,
-			kind: RuleKind::Native(Code::from_code(
-				CodeLocator::to_rule(tail.i().i(&path.unreverse()).await, macro_i, i as u16),
-				parse_mtree(body, path.clone()).await?,
-			)),
-		})
-	}
-	if let Ok(e) = OrcErrv::new(errors) { Err(e) } else { Ok(rules) }
 }

orchid-host/src/parsed.rs

@@ -0,0 +1,215 @@
+use std::fmt::{self, Debug};
+use std::rc::Rc;
+
+use futures::FutureExt;
+use futures::future::{LocalBoxFuture, join_all};
+use hashbrown::HashSet;
+use itertools::Itertools;
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
+use orchid_base::interner::Tok;
+use orchid_base::location::SrcRange;
+use orchid_base::parse::{Comment, Import};
+use orchid_base::tl_cache;
+use orchid_base::tree::{TokTree, Token, recur};
+
+use crate::api;
+use crate::dealias::{ChildErrorKind, ChildResult, Tree};
+use crate::expr::{Expr, ExprWillPanic};
+use crate::expr_store::ExprStore;
+use crate::system::System;
+
+pub type ParsTokTree = TokTree<Expr, Expr>;
+pub type ParsTok = Token<Expr, Expr>;
+
+#[derive(Debug)]
+pub struct Item {
+	pub sr: SrcRange,
+	pub comments: Vec<Comment>,
+	pub kind: ItemKind,
+}
+impl Item {
+	pub fn new(sr: SrcRange, kind: impl Into<ItemKind>) -> Self {
+		Self { sr, comments: Vec::new(), kind: kind.into() }
+	}
+}
+
+#[derive(Debug)]
+pub enum ItemKind {
+	Member(ParsedMember),
+	Import(Import),
+}
+impl ItemKind {
+	#[must_use]
+	pub fn at(self, sr: SrcRange) -> Item { Item { comments: vec![], sr, kind: self } }
+}
+impl From<ParsedMember> for ItemKind {
+	fn from(value: ParsedMember) -> Self { Self::Member(value) }
+}
+impl From<Import> for ItemKind {
+	fn from(value: Import) -> Self { Self::Import(value) }
+}
+
+impl Format for Item {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		let comment_text = self.comments.iter().join("\n");
+		let item_text = match &self.kind {
+			ItemKind::Import(i) => format!("import {i}").into(),
+			ItemKind::Member(mem) => match &mem.kind {
+				ParsedMemberKind::Const(_, sys) =>
+					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} via {1}")))
+						.units([mem.name.rc().into(), sys.print(c).await]),
+				ParsedMemberKind::Mod(module) =>
+					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
+						.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
+			},
+		};
+		tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}\n{1}")))
+			.units([comment_text.into(), item_text])
+	}
+}
+
+pub struct ParsedMember {
+	pub name: Tok<String>,
+	pub exported: bool,
+	pub kind: ParsedMemberKind,
+}
+impl ParsedMember {
+	#[must_use]
+	pub fn name(&self) -> Tok<String> { self.name.clone() }
+	pub fn new(exported: bool, name: Tok<String>, kind: impl Into<ParsedMemberKind>) -> Self {
+		Self { exported, name, kind: kind.into() }
+	}
+}
+impl Debug for ParsedMember {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Member")
+			.field("name", &self.name)
+			.field("kind", &self.kind)
+			.finish_non_exhaustive()
+	}
+}
+
+pub(crate) type ParsedExprCallback =
+	Rc<dyn for<'a> Fn(&'a [Tok<String>]) -> LocalBoxFuture<'a, Expr>>;
+
+pub struct ParsedExpr {
+	pub(crate) debug: String,
+	pub(crate) callback: ParsedExprCallback,
+}
+impl ParsedExpr {
+	pub async fn run(self, imported_names: &[Tok<String>]) -> Expr {
+		(self.callback)(imported_names).await
+	}
+}
+impl fmt::Debug for ParsedExpr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.debug) }
+}
+
+#[derive(Debug)]
+pub enum ParsedMemberKind {
+	Const(api::ParsedConstId, System),
+	Mod(ParsedModule),
+}
+impl From<ParsedModule> for ParsedMemberKind {
+	fn from(value: ParsedModule) -> Self { Self::Mod(value) }
+}
+#[derive(Debug, Default)]
+pub struct ParsedModule {
+	pub exports: Vec<Tok<String>>,
+	pub items: Vec<Item>,
+	pub use_prelude: bool,
+}
+impl ParsedModule {
+	#[must_use]
+	pub fn new(use_prelude: bool, items: impl IntoIterator<Item = Item>) -> Self {
+		let items = items.into_iter().collect_vec();
+		let exports = (items.iter())
+			.filter_map(|i| if let ItemKind::Member(m) = &i.kind { Some(m) } else { None })
+			.filter(|m| m.exported)
+			.map(|m| m.name.clone())
+			.collect_vec();
+		Self { exports, items, use_prelude }
+	}
+	pub fn merge(&mut self, other: ParsedModule) {
+		let mut swap = ParsedModule::default();
+		std::mem::swap(self, &mut swap);
+		assert_eq!(self.use_prelude, other.use_prelude, "merging modules that disagree on prelude");
+		*self = ParsedModule::new(self.use_prelude, swap.items.into_iter().chain(other.items))
+	}
+	#[must_use]
+	pub fn get_imports(&self) -> impl IntoIterator<Item = &Import> {
+		(self.items.iter())
+			.filter_map(|it| if let ItemKind::Import(i) = &it.kind { Some(i) } else { None })
+	}
+	pub fn default_item(self, name: Tok<String>, sr: SrcRange) -> Item {
+		let mem = ParsedMember { exported: true, name, kind: ParsedMemberKind::Mod(self) };
+		Item { comments: vec![], sr, kind: ItemKind::Member(mem) }
+	}
+}
+impl Tree for ParsedModule {
+	type Ctx<'a> = ();
+	async fn child(
+		&self,
+		key: Tok<String>,
+		public_only: bool,
+		(): &mut Self::Ctx<'_>,
+	) -> ChildResult<'_, Self> {
+		if public_only && !self.exports.contains(&key) {
+			return ChildResult::Err(ChildErrorKind::Private);
+		}
+		if let Some(member) = (self.items.iter())
+			.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
+			.find(|m| m.name == key)
+		{
+			match &member.kind {
+				ParsedMemberKind::Const(..) => return ChildResult::Err(ChildErrorKind::Constant),
+				ParsedMemberKind::Mod(m) => return ChildResult::Ok(m),
+			}
+		}
+		ChildResult::Err(ChildErrorKind::Missing)
+	}
+	fn children(&self, public_only: bool) -> HashSet<Tok<String>> {
+		let mut public: HashSet<_> = self.exports.iter().cloned().collect();
+		if !public_only {
+			public.extend(
+				(self.items.iter())
+					.filter_map(
+						|it| if let ItemKind::Member(mem) = &it.kind { Some(&mem.name) } else { None },
+					)
+					.cloned(),
+			)
+		}
+		public
+	}
+}
+impl Format for ParsedModule {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		let head_str = format!("export ::({})\n", self.exports.iter().join(", "));
+		Variants::sequence(self.items.len() + 1, "\n", None).units(
+			[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
+		)
+	}
+}
+
+/// Selects a code element
+///
+/// Either the steps point to a constant and rule_loc is None, or the steps
+/// point to a module and rule_loc selects a macro rule within that module
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct ConstPath {
+	steps: Tok<Vec<Tok<String>>>,
+}
+impl ConstPath {
+	#[must_use]
+	pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps } }
+}
+
+pub async fn tt_to_api(exprs: &mut ExprStore, subtree: ParsTokTree) -> api::TokenTree {
+	let without_new_expr = recur(subtree, &|tt, r| {
+		if let ParsTok::NewExpr(expr) = tt.tok {
+			return ParsTok::Handle(expr).at(tt.sr);
+		}
+		r(tt)
+	});
+	without_new_expr.into_api(exprs, &mut ExprWillPanic).await
+}

orchid-host/src/subprocess.rs

@@ -1,13 +1,12 @@
 use std::cell::RefCell;
-use std::path::PathBuf;
+use std::io::{self, Write};
 use std::pin::Pin;
-use std::thread;
 
 use async_process::{self, Child, ChildStdin, ChildStdout};
-use async_std::io::{self, BufReadExt, BufReader};
-use async_std::sync::Mutex;
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
+use futures::io::BufReader;
+use futures::lock::Mutex;
+use futures::{self, AsyncBufReadExt, AsyncWriteExt};
 use orchid_api_traits::{Decode, Encode};
 use orchid_base::builtin::{ExtInit, ExtPort};
 use orchid_base::logging::Logger;
@@ -22,8 +21,6 @@ pub async fn ext_command(
 	msg_logs: Logger,
 	ctx: Ctx,
 ) -> io::Result<ExtInit> {
-	let prog_pbuf = PathBuf::from(cmd.get_program());
-	let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy().to_string();
 	let mut child = async_process::Command::from(cmd)
 		.stdin(async_process::Stdio::piped())
 		.stdout(async_process::Stdio::piped())
@@ -36,8 +33,7 @@ pub async fn ext_command(
 	let mut stdout = child.stdout.take().unwrap();
 	let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await;
 	let child_stderr = child.stderr.take().unwrap();
-	thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || {
+	(ctx.spawn)(Box::pin(async move {
-		async_std::task::block_on(async move {
 		let mut reader = BufReader::new(child_stderr);
 		loop {
 			let mut buf = String::new();
@@ -46,47 +42,59 @@ pub async fn ext_command(
 			}
 			logger.log(buf.strip_suffix('\n').expect("Readline implies this"));
 		}
-		})
+	}));
-	})?;
 	Ok(ExtInit {
-		header,
 		port: Box::new(Subprocess {
+			name: header.name.clone(),
 			child: RefCell::new(Some(child)),
-			stdin: Mutex::new(Box::pin(stdin)),
+			stdin: Some(Mutex::new(Box::pin(stdin))),
 			stdout: Mutex::new(Box::pin(stdout)),
 			ctx,
 		}),
+		header,
 	})
 }
 
 pub struct Subprocess {
+	name: String,
 	child: RefCell<Option<Child>>,
-	stdin: Mutex<Pin<Box<ChildStdin>>>,
+	stdin: Option<Mutex<Pin<Box<ChildStdin>>>>,
 	stdout: Mutex<Pin<Box<ChildStdout>>>,
 	ctx: Ctx,
 }
 impl Drop for Subprocess {
 	fn drop(&mut self) {
 		let mut child = self.child.borrow_mut().take().unwrap();
+		let name = self.name.clone();
+		if std::thread::panicking() {
+			eprintln!("Killing extension {name}");
+			// we don't really care to handle errors here
+			let _: Result<_, _> = std::io::stderr().flush();
+			let _: Result<_, _> = child.kill();
+			return;
+		}
+		let stdin = self.stdin.take().unwrap();
 		(self.ctx.spawn)(Box::pin(async move {
-			let status = child.status().await.expect("Extension exited with error");
+			stdin.lock().await.close().await.unwrap();
-			assert!(status.success(), "Extension exited with error {status}");
+			let status = (child.status().await)
+				.unwrap_or_else(|e| panic!("{e}, extension {name} exited with error"));
+			assert!(status.success(), "Extension {name} exited with error {status}");
 		}))
 	}
 }
 impl ExtPort for Subprocess {
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		async { send_msg(Pin::new(&mut *self.stdin.lock().await), msg).await.unwrap() }.boxed_local()
+		Box::pin(async {
+			send_msg(Pin::new(&mut *self.stdin.as_ref().unwrap().lock().await), msg).await.unwrap()
+		})
 	}
-	fn recv<'a>(
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
-		&'a self,
-		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
-	) -> LocalBoxFuture<'a, ()> {
 		Box::pin(async {
 			std::io::Write::flush(&mut std::io::stderr()).unwrap();
 			match recv_msg(self.stdout.lock().await.as_mut()).await {
-				Ok(msg) => cb(&msg).await,
+				Ok(msg) => Some(msg),
-				Err(e) if e.kind() == io::ErrorKind::BrokenPipe => (),
+				Err(e) if e.kind() == io::ErrorKind::BrokenPipe => None,
+				Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => None,
 				Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
 			}
 		})

orchid-host/src/sys_parser.rs

@@ -0,0 +1,116 @@
+use futures::FutureExt;
+use futures::future::join_all;
+use itertools::Itertools;
+use orchid_base::error::{OrcErrv, OrcRes};
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::SrcRange;
+use orchid_base::name::Sym;
+use orchid_base::parse::Comment;
+use orchid_base::reqnot::Requester;
+use orchid_base::tree::ttv_from_api;
+use substack::Substack;
+
+use crate::api;
+use crate::expr::ExprParseCtx;
+use crate::expr_store::ExprStore;
+use crate::parse::HostParseCtx;
+use crate::parsed::{
+	Item, ItemKind, ParsTokTree, ParsedMember, ParsedMemberKind, ParsedModule, tt_to_api,
+};
+use crate::system::System;
+
+pub struct Parser {
+	pub(crate) system: System,
+	pub(crate) idx: u16,
+}
+type ModPath<'a> = Substack<'a, Tok<String>>;
+
+impl Parser {
+	pub async fn parse(
+		&self,
+		ctx: &impl HostParseCtx,
+		path: ModPath<'_>,
+		line: Vec<ParsTokTree>,
+		exported: bool,
+		comments: Vec<Comment>,
+		callback: &mut impl AsyncFnMut(ModPath<'_>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
+	) -> OrcRes<Vec<Item>> {
+		let mut temp_store = self.system.ext().exprs().derive(true);
+		let src_path = line.first().expect("cannot be empty").sr.path();
+		let line =
+			join_all((line.into_iter()).map(|t| async { tt_to_api(&mut temp_store.clone(), t).await }))
+				.await;
+		let mod_path = ctx.src_path().suffix(path.unreverse(), self.system.i()).await;
+		let comments = comments.iter().map(Comment::to_api).collect_vec();
+		let req = api::ParseLine {
+			idx: self.idx,
+			module: mod_path.to_api(),
+			src: src_path.to_api(),
+			exported,
+			sys: self.system.id(),
+			comments,
+			line,
+		};
+		match self.system.reqnot().request(req).await {
+			Ok(parsed_v) =>
+				conv(parsed_v, path, callback, &mut ConvCtx {
+					i: self.system.i(),
+					mod_path: &mod_path,
+					ext_exprs: &mut temp_store,
+					pctx: &mut ExprParseCtx { ctx: self.system.ctx(), exprs: self.system.ext().exprs() },
+					src_path: &src_path,
+					sys: &self.system,
+				})
+				.await,
+			Err(e) => Err(OrcErrv::from_api(&e, &self.system.ctx().i).await),
+		}
+	}
+}
+
+struct ConvCtx<'a> {
+	sys: &'a System,
+	mod_path: &'a Sym,
+	src_path: &'a Sym,
+	i: &'a Interner,
+	ext_exprs: &'a mut ExprStore,
+	pctx: &'a mut ExprParseCtx<'a>,
+}
+async fn conv(
+	parsed_v: Vec<api::ParsedLine>,
+	module: Substack<'_, Tok<String>>,
+	callback: &'_ mut impl AsyncFnMut(Substack<'_, Tok<String>>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
+	ctx: &mut ConvCtx<'_>,
+) -> OrcRes<Vec<Item>> {
+	let mut items = Vec::new();
+	for parsed in parsed_v {
+		let (name, exported, kind) = match parsed.kind {
+			api::ParsedLineKind::Member(api::ParsedMember { name, exported, kind }) =>
+				(name, exported, kind),
+			api::ParsedLineKind::Recursive(rec) => {
+				let tokens = ttv_from_api(rec, ctx.ext_exprs, ctx.pctx, ctx.src_path, ctx.i).await;
+				items.extend(callback(module.clone(), tokens).await?);
+				continue;
+			},
+		};
+		let name = ctx.i.ex(name).await;
+		let mkind = match kind {
+			api::ParsedMemberKind::Module { lines, use_prelude } => {
+				let items = conv(lines, module.push(name.clone()), callback, ctx).boxed_local().await?;
+				ParsedMemberKind::Mod(ParsedModule::new(use_prelude, items))
+			},
+			api::ParsedMemberKind::Constant(cid) => {
+				ctx.sys.0.const_paths.insert(cid, ctx.mod_path.suffix(module.unreverse(), ctx.i).await);
+				ParsedMemberKind::Const(cid, ctx.sys.clone())
+			},
+		};
+		items.push(Item {
+			comments: join_all(
+				parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone(), ctx.i)),
+			)
+			.await,
+			sr: SrcRange::from_api(&parsed.source_range, ctx.i).await,
+			kind: ItemKind::Member(ParsedMember { name, exported, kind: mkind }),
+		})
+	}
+	Ok(items)
+}

orchid-host/src/system.rs

@@ -1,39 +1,44 @@
 use std::collections::VecDeque;
+use std::fmt;
 use std::future::Future;
 use std::rc::{Rc, Weak};
-use std::{fmt, mem};
 
-use async_stream::stream;
+use async_lock::RwLock;
 use derive_destructure::destructure;
-use futures::StreamExt;
 use futures::future::join_all;
 use hashbrown::HashMap;
 use itertools::Itertools;
+use memo_map::MemoMap;
 use orchid_base::char_filter::char_filter_match;
-use orchid_base::clone;
+use orchid_base::error::{OrcRes, mk_errv_floating};
-use orchid_base::error::{OrcErrv, OrcRes};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
-use orchid_base::location::Pos;
+use orchid_base::iter_utils::IteratorPrint;
-use orchid_base::parse::Comment;
+use orchid_base::name::{NameLike, Sym, VName, VPath};
 use orchid_base::reqnot::{ReqNot, Requester};
-use orchid_base::tree::ttv_from_api;
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
 use crate::api;
+use crate::atom::{AtomHand, WeakAtomHand};
 use crate::ctx::Ctx;
+use crate::dealias::walk;
 use crate::extension::{Extension, WeakExtension};
-use crate::tree::{ItemKind, Member, Module, ParsTokTree};
+use crate::sys_parser::Parser;
+use crate::tree::Root;
 
 #[derive(destructure)]
-struct SystemInstData {
+pub(crate) struct SystemInstData {
+	deps: Vec<System>,
 	ctx: Ctx,
 	ext: Extension,
 	decl_id: api::SysDeclId,
 	lex_filter: api::CharFilter,
 	id: api::SysId,
 	line_types: Vec<Tok<String>>,
+	prelude: Vec<Sym>,
+	owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
+	pub(crate) const_paths: MemoMap<api::ParsedConstId, Sym>,
 }
 impl Drop for SystemInstData {
 	fn drop(&mut self) { self.ext.system_drop(self.id); }
@@ -50,53 +55,123 @@ impl fmt::Debug for SystemInstData {
 }
 
 #[derive(Clone, Debug)]
-pub struct System(Rc<SystemInstData>);
+pub struct System(pub(crate) Rc<SystemInstData>);
 impl System {
+	#[must_use]
+	pub async fn atoms(&self) -> impl std::ops::Deref<Target = HashMap<api::AtomId, WeakAtomHand>> {
+		self.0.owned_atoms.read().await
+	}
+	#[must_use]
 	pub fn id(&self) -> api::SysId { self.0.id }
+	#[must_use]
 	pub fn ext(&self) -> &Extension { &self.0.ext }
+	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
+	#[must_use]
+	pub fn i(&self) -> &Interner { &self.0.ctx.i }
+	#[must_use]
+	pub fn deps(&self) -> &[System] { &self.0.deps }
+	#[must_use]
+	pub fn ctor(&self) -> SystemCtor {
+		(self.0.ext.system_ctors().find(|c| c.decl.id == self.0.decl_id).cloned())
+			.expect("Ctor was used to create ext")
+	}
+	#[must_use]
 	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { self.0.ext.reqnot() }
+	#[must_use]
 	pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
 		self.reqnot().request(api::GetMember(self.0.id, id)).await
 	}
+	#[must_use]
 	pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
+	#[must_use]
 	pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
+	#[must_use]
+	pub fn prelude(&self) -> Vec<Sym> { self.0.prelude.clone() }
 	/// Have this system lex a part of the source. It is assumed that
 	/// [Self::can_lex] was called and returned true.
 	pub async fn lex<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
 		source: Tok<String>,
+		src: Sym,
 		pos: u32,
 		r: impl FnMut(u32) -> F,
 	) -> api::OrcResult<Option<api::LexedExpr>> {
-		self.0.ext.lex_req(source, pos, self.id(), r).await
+		self.0.ext.lex_req(source, src, pos, self.id(), r).await
+	}
+	#[must_use]
+	pub fn get_parser(&self, ltyp: Tok<String>) -> Option<Parser> {
+		(self.0.line_types.iter().enumerate())
+			.find(|(_, txt)| *txt == &ltyp)
+			.map(|(idx, _)| Parser { idx: idx as u16, system: self.clone() })
 	}
-	pub fn can_parse(&self, ltyp: Tok<String>) -> bool { self.0.line_types.contains(&ltyp) }
 	pub fn line_types(&self) -> impl Iterator<Item = &Tok<String>> + '_ { self.0.line_types.iter() }
-	pub async fn parse(
+
-		&self,
+	#[must_use]
-		line: Vec<ParsTokTree>,
-		exported: bool,
-		comments: Vec<Comment>,
-	) -> OrcRes<Vec<ParsTokTree>> {
-		let line =
-			join_all(line.iter().map(|t| async { t.to_api(&mut |n, _| match *n {}).await })).await;
-		let comments = comments.iter().map(Comment::to_api).collect_vec();
-		match self.reqnot().request(api::ParseLine { exported, sys: self.id(), comments, line }).await {
-			Ok(parsed) => Ok(ttv_from_api(parsed, &mut self.ctx().clone(), &self.ctx().i).await),
-			Err(e) => Err(OrcErrv::from_api(&e, &self.ctx().i).await),
-		}
-	}
 	pub async fn request(&self, req: Vec<u8>) -> Vec<u8> {
 		self.reqnot().request(api::SysFwded(self.id(), req)).await
 	}
-	pub(crate) fn drop_atom(&self, drop: api::AtomId) {
+	pub(crate) async fn new_atom(&self, data: Vec<u8>, id: api::AtomId) -> AtomHand {
+		let mut owned_g = self.0.owned_atoms.write().await;
+		if let Some(data) = owned_g.get(&id)
+			&& let Some(atom) = data.upgrade()
+		{
+			return atom;
+		}
+		let new = AtomHand::new(data, self.clone(), Some(id));
+		owned_g.insert(id, new.downgrade());
+		new
+	}
+	pub(crate) fn drop_atom(&self, dropped_atom_id: api::AtomId) {
 		let this = self.0.clone();
 		(self.0.ctx.spawn)(Box::pin(async move {
-			this.ctx.owned_atoms.write().await.remove(&drop);
+			this.ext.reqnot().request(api::AtomDrop(this.id, dropped_atom_id)).await;
+			this.owned_atoms.write().await.remove(&dropped_atom_id);
 		}))
 	}
-	pub fn downgrade(&self) -> WeakSystem { WeakSystem(Rc::downgrade(&self.0)) }
+	#[must_use]
+	pub fn downgrade(&self) -> WeakSystem {
+		WeakSystem(Rc::downgrade(&self.0), self.0.decl_id, self.ext().downgrade())
+	}
+	/// Implementation of [api::ResolveNames]
+	pub(crate) async fn name_resolver(
+		&self,
+		orig: api::ParsedConstId,
+	) -> impl AsyncFnMut(&[Tok<String>]) -> OrcRes<VName> + use<> {
+		let root = self.0.ctx.root.read().await.upgrade().expect("find_names when root not in context");
+		let orig = self.0.const_paths.get(&orig).expect("origin for find_names invalid").clone();
+		let ctx = self.0.ctx.clone();
+		async move |rel| {
+			let cwd = orig.split_last_seg().1;
+			let root_data = &mut *root.0.write().await;
+			let walk_ctx = &mut (ctx.clone(), &root_data.consts);
+			let cmod = (walk(&root_data.root, false, cwd.iter().cloned(), walk_ctx).await)
+				.expect("the parent module of a constant should exist");
+			let (selector, tail) = rel.split_first().expect("Names cannot be empty");
+			if cmod.members.get(selector).is_some() {
+				return Ok(VName::new(cwd.iter().chain(rel).cloned()).unwrap());
+			}
+			match cmod.imports.get(selector) {
+				Some(Ok(dest)) => return Ok(dest.target.to_vname().suffix(tail.iter().cloned())),
+				Some(Err(dests)) =>
+					return Err(mk_errv_floating(
+						ctx.i.i("Ambiguous name").await,
+						format!(
+							"{selector} could refer to {}",
+							dests.iter().map(|ri| &ri.target).display("or")
+						),
+					)),
+				None => (),
+			}
+			if tail.is_empty() {
+				return Ok(VPath::new(cwd.iter().cloned()).name_with_suffix(selector.clone()));
+			}
+			Err(mk_errv_floating(
+				ctx.i.i("Invalid name").await,
+				format!("{selector} doesn't refer to a module"),
+			))
+		}
+	}
 }
 impl Format for System {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -106,29 +181,45 @@ impl Format for System {
 	}
 }
 
-pub struct WeakSystem(Weak<SystemInstData>);
+pub struct WeakSystem(Weak<SystemInstData>, api::SysDeclId, WeakExtension);
 impl WeakSystem {
+	#[must_use]
 	pub fn upgrade(&self) -> Option<System> { self.0.upgrade().map(System) }
+	pub fn ext(&self) -> Option<Extension> { self.2.upgrade() }
+	pub fn ctor(&self) -> Option<SystemCtor> {
+		self.ext()?.system_ctors().find(|ctor| ctor.decl.id == self.1).cloned()
+	}
 }
 
+#[derive(Clone)]
 pub struct SystemCtor {
 	pub(crate) decl: api::SystemDecl,
 	pub(crate) ext: WeakExtension,
 }
 impl SystemCtor {
+	#[must_use]
 	pub fn name(&self) -> &str { &self.decl.name }
+	pub async fn name_tok(&self) -> Sym {
+		(Sym::parse(&self.decl.name, &self.ext.upgrade().expect("ext dropped early").ctx().i).await)
+			.expect("System cannot have empty name")
+	}
+	#[must_use]
 	pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
+	#[must_use]
 	pub fn depends(&self) -> impl ExactSizeIterator<Item = &str> {
 		self.decl.depends.iter().map(|s| &**s)
 	}
+	#[must_use]
 	pub fn id(&self) -> api::SysDeclId { self.decl.id }
-	pub async fn run<'a>(&self, depends: impl IntoIterator<Item = &'a System>) -> System {
+	#[must_use]
-		let depends = depends.into_iter().map(|si| si.id()).collect_vec();
+	pub async fn run(&self, deps: Vec<System>) -> (Root, System) {
+		let depends = deps.iter().map(|si| si.id()).collect_vec();
 		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
 		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
 		let id = ext.ctx().next_sys_id();
 		let sys_inst = ext.reqnot().request(api::NewSystem { depends, id, system: self.decl.id }).await;
 		let data = System(Rc::new(SystemInstData {
+			deps,
 			decl_id: self.decl.id,
 			ext: ext.clone(),
 			ctx: ext.ctx().clone(),
@@ -136,24 +227,18 @@ impl SystemCtor {
 			line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
 				.await,
 			id,
+			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok, &ext.ctx().i))).await,
+			owned_atoms: RwLock::new(HashMap::new()),
+			const_paths: MemoMap::new(),
 		}));
-		let const_root = clone!(data, ext; stream! {
+		let api_module_root = api::Module {
-			for (k, v) in sys_inst.const_root {
+			members: (sys_inst.const_root.into_iter())
-				yield Member::from_api(
+				.map(|(k, v)| api::Member { name: k, kind: v, comments: vec![], exported: true })
-					api::Member { name: k, kind: v },
+				.collect_vec(),
-					&mut vec![Tok::from_api(k, &ext.ctx().i).await],
+		};
-					&data,
+		let root = Root::from_api(api_module_root, &data).await;
-				).await;
-			}
-		})
-		.map(|mem| ItemKind::Member(mem).at(Pos::None))
-		.collect::<Vec<_>>()
-		.await;
 		ext.ctx().systems.write().await.insert(id, data.downgrade());
-		let mut swap = Module::default();
+		(root, data)
-		mem::swap(&mut swap, &mut *ext.ctx().root.write().await);
-		*ext.ctx().root.write().await = Module::new(swap.items.into_iter().chain(const_root));
-		data
 	}
 }
 
@@ -166,7 +251,7 @@ pub enum SysResolvErr {
 pub async fn init_systems(
 	tgts: &[String],
 	exts: &[Extension],
-) -> Result<Vec<System>, SysResolvErr> {
+) -> Result<(Root, Vec<System>), SysResolvErr> {
 	let mut to_load = HashMap::<&str, &SystemCtor>::new();
 	let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque<&str>>();
 	while let Some(target) = to_find.pop_front() {
@@ -205,9 +290,11 @@ pub async fn init_systems(
 		walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?;
 	}
 	let mut systems = HashMap::<&str, System>::new();
+	let mut root = Root::new(exts.first().unwrap().ctx().clone());
 	for ctor in to_load_ordered.iter() {
-		let sys = ctor.run(ctor.depends().map(|n| &systems[n])).await;
+		let (sys_root, sys) = ctor.run(ctor.depends().map(|n| systems[n].clone()).collect()).await;
 		systems.insert(ctor.name(), sys);
+		root = root.merge(&sys_root).await.expect("Conflicting roots");
 	}
-	Ok(systems.into_values().collect_vec())
+	Ok((root, systems.into_values().collect_vec()))
 }

orchid-host/src/tree.rs

@@ -1,342 +1,488 @@
-use std::fmt::Debug;
+//! This tree isn't Clone because lazy subtrees are guaranteed to only be loaded
-use std::rc::Rc;
+//! once
+use std::cell::RefCell;
+use std::rc::{Rc, Weak};
+use std::slice;
 
+use async_lock::RwLock;
 use async_once_cell::OnceCell;
-use async_std::sync::Mutex;
+use derive_destructure::destructure;
-use async_stream::stream;
+use futures::{FutureExt, StreamExt, stream};
-use futures::future::join_all;
+use hashbrown::HashMap;
-use futures::{FutureExt, StreamExt};
+use hashbrown::hash_map::Entry;
 use itertools::Itertools;
-use never::Never;
+use memo_map::MemoMap;
-use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
+use orchid_base::clone;
+use orchid_base::error::{OrcRes, Reporter, mk_errv};
 use orchid_base::interner::Tok;
-use orchid_base::location::Pos;
+use orchid_base::location::{CodeGenInfo, Pos};
-use orchid_base::macros::{mtreev_fmt, mtreev_from_api};
+use orchid_base::name::{NameLike, Sym, VPath};
-use orchid_base::name::Sym;
+use orchid_base::reqnot::Requester;
-use orchid_base::parse::{Comment, Import};
-use orchid_base::tree::{AtomRepr, TokTree, Token};
-use orchid_base::{clone, tl_cache};
-use ordered_float::NotNan;
-use substack::Substack;
 
 use crate::api;
-use crate::atom::AtomHand;
 use crate::ctx::Ctx;
-use crate::expr::{Expr, mtreev_to_expr};
+use crate::dealias::{ChildErrorKind, Tree, absolute_path, resolv_glob, walk};
-use crate::macros::{MacTok, MacTree};
+use crate::expr::{Expr, ExprParseCtx, PathSetBuilder};
+use crate::parsed::{ItemKind, ParsedMemberKind, ParsedModule};
 use crate::system::System;
 
-pub type ParsTokTree = TokTree<'static, AtomHand, Never>;
+pub struct RootData {
-pub type ParsTok = Token<'static, AtomHand, Never>;
+	pub root: Module,
+	pub consts: MemoMap<Sym, Expr>,
+	pub ctx: Ctx,
+}
+#[derive(Clone)]
+pub struct Root(pub Rc<RwLock<RootData>>);
+impl Root {
+	#[must_use]
+	pub fn new(ctx: Ctx) -> Self {
+		Root(Rc::new(RwLock::new(RootData {
+			root: Module::default(),
+			consts: MemoMap::default(),
+			ctx,
+		})))
+	}
+	#[must_use]
+	pub async fn from_api(api: api::Module, sys: &System) -> Self {
+		let consts = MemoMap::new();
+		let mut tfac = TreeFromApiCtx { consts: &consts, path: sys.i().i(&[][..]).await, sys };
+		let root = Module::from_api(api, &mut tfac).await;
+		Root(Rc::new(RwLock::new(RootData { root, consts, ctx: sys.ctx().clone() })))
+	}
+	pub async fn merge(&self, new: &Root) -> Result<Self, MergeErr> {
+		let this = self.0.write().await;
+		let that = new.0.write().await;
+		let consts = MemoMap::new();
+		for (k, v) in this.consts.iter().chain(that.consts.iter()) {
+			consts.insert(k.clone(), v.clone());
+		}
+		let root = this.root.merge(&that.root, this.ctx.clone(), &consts).await?;
+		Ok(Self(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() }))))
+	}
+	#[must_use]
+	pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym, rep: &Reporter) -> Self {
+		let mut ref_this = self.0.write().await;
+		let this = &mut *ref_this;
+		let mut deferred_consts = HashMap::new();
+		let consts = this.consts.clone();
+		let mut tfpctx = FromParsedCtx {
+			pars_root: parsed,
+			deferred_consts: &mut deferred_consts,
+			consts: &consts,
+			pars_prefix: pars_prefix.clone(),
+			root: &this.root,
+			ctx: &this.ctx,
+			rep,
+		};
+		let mut module = Module::from_parsed(parsed, pars_prefix.clone(), &mut tfpctx).await;
+		for step in pars_prefix.iter().rev() {
+			let kind = OnceCell::from(MemberKind::Module(module));
+			let members = HashMap::from([(
+				step.clone(),
+				Rc::new(Member { public: true, lazy: RefCell::new(None), kind }),
+			)]);
+			module = Module { imports: HashMap::new(), members }
+		}
+		let root = (this.root.merge(&module, this.ctx.clone(), &consts).await)
+			.expect("Merge conflict between parsed and existing module");
+		let new = Root(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() })));
+		*this.ctx.root.write().await = new.downgrade();
+		for (path, (sys_id, pc_id)) in deferred_consts {
+			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
+			let api_expr = sys.reqnot().request(api::FetchParsedConst(sys.id(), pc_id)).await;
+			let mut xp_ctx = ExprParseCtx { ctx: &this.ctx, exprs: sys.ext().exprs() };
+			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), &mut xp_ctx).await;
+			new.0.write().await.consts.insert(path, expr);
+		}
+		new
+	}
+	pub async fn get_const_value(&self, name: Sym, pos: Pos) -> OrcRes<Expr> {
+		let this = &*self.0.read().await;
+		// shortcut for previously visited
+		if let Some(val) = this.consts.get(&name) {
+			return Ok(val.clone());
+		}
+		// load the node, then check if this "walk" call added it to the map
+		let ctx = this.ctx.clone();
+		let module =
+			walk(&this.root, false, name.iter().cloned(), &mut (ctx.clone(), &this.consts)).await;
+		if let Some(val) = this.consts.get(&name) {
+			return Ok(val.clone());
+		}
+		match module {
+			Ok(_) => Err(mk_errv(
+				ctx.i.i("module used as constant").await,
+				format!("{name} is a module, not a constant"),
+				[pos],
+			)),
+			Err(e) => match e.kind {
+				ChildErrorKind::Private => panic!("public_only is false"),
+				ChildErrorKind::Constant => panic!("Tree refers to constant not in table"),
+				ChildErrorKind::Missing => Err(mk_errv(
+					ctx.i.i("Constant does not exist").await,
+					format!("{name} does not refer to a constant"),
+					[pos],
+				)),
+			},
+		}
+	}
+	#[must_use]
+	pub fn downgrade(&self) -> WeakRoot { WeakRoot(Rc::downgrade(&self.0)) }
+}
 
-#[derive(Debug)]
+#[derive(Clone)]
-pub struct Item {
+pub struct WeakRoot(Weak<RwLock<RootData>>);
+impl WeakRoot {
+	#[must_use]
+	pub fn new() -> Self { Self(Weak::new()) }
+	#[must_use]
+	pub fn upgrade(&self) -> Option<Root> { Some(Root(self.0.upgrade()?)) }
+}
+impl Default for WeakRoot {
+	fn default() -> Self { Self::new() }
+}
+
+pub struct TreeFromApiCtx<'a> {
+	pub sys: &'a System,
+	pub consts: &'a MemoMap<Sym, Expr>,
+	pub path: Tok<Vec<Tok<String>>>,
+}
+impl<'a> TreeFromApiCtx<'a> {
+	#[must_use]
+	pub async fn push<'c>(&'c self, name: Tok<String>) -> TreeFromApiCtx<'c> {
+		let path = self.sys.ctx().i.i(&self.path.iter().cloned().chain([name]).collect_vec()).await;
+		TreeFromApiCtx { path, consts: self.consts, sys: self.sys }
+	}
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ResolvedImport {
+	pub target: Sym,
 	pub pos: Pos,
-	pub comments: Vec<Comment>,
+}
-	pub kind: ItemKind,
+
+#[derive(Clone, Default)]
+pub struct Module {
+	pub imports: HashMap<Tok<String>, Result<ResolvedImport, Vec<ResolvedImport>>>,
+	pub members: HashMap<Tok<String>, Rc<Member>>,
+}
+impl Module {
+	#[must_use]
+	pub async fn from_api(api: api::Module, ctx: &mut TreeFromApiCtx<'_>) -> Self {
+		let mut members = HashMap::new();
+		for mem in api.members {
+			let mem_name = ctx.sys.i().ex(mem.name).await;
+			let vname = VPath::new(ctx.path.iter().cloned()).name_with_suffix(mem_name.clone());
+			let name = vname.to_sym(ctx.sys.i()).await;
+			let (lazy, kind) = match mem.kind {
+				api::MemberKind::Lazy(id) =>
+					(Some(LazyMemberHandle { id, sys: ctx.sys.id(), path: name.clone() }), None),
+				api::MemberKind::Const(val) => {
+					let mut expr_ctx = ExprParseCtx { ctx: ctx.sys.ctx(), exprs: ctx.sys.ext().exprs() };
+					let expr = Expr::from_api(&val, PathSetBuilder::new(), &mut expr_ctx).await;
+					ctx.consts.insert(name.clone(), expr);
+					(None, Some(MemberKind::Const))
+				},
+				api::MemberKind::Module(m) => {
+					let m = Self::from_api(m, &mut ctx.push(mem_name.clone()).await).boxed_local().await;
+					(None, Some(MemberKind::Module(m)))
+				},
+			};
+			members.insert(
+				mem_name.clone(),
+				Rc::new(Member {
+					public: mem.exported,
+					lazy: RefCell::new(lazy),
+					kind: kind.map_or_else(OnceCell::new, OnceCell::from),
+				}),
+			);
+		}
+		Self { members, imports: HashMap::new() }
+	}
+	#[must_use]
+	async fn from_parsed(parsed: &ParsedModule, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
+		let imports_by_name = (parsed.get_imports().into_iter())
+			.filter_map(|i| Some((i.name.clone()?, i)))
+			.into_group_map();
+		let mut glob_imports_by_name = HashMap::<_, Vec<_>>::new();
+		for import in parsed.get_imports().into_iter().filter(|i| i.name.is_none()) {
+			let pos = import.sr.pos();
+			match absolute_path(&path, &import.path, &ctx.ctx.i).await {
+				Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, pos, &import.path.to_string()).await),
+				Ok(abs_path) => {
+					let names_res = match abs_path.strip_prefix(&ctx.pars_prefix[..]) {
+						None => {
+							let mut tree_ctx = (ctx.ctx.clone(), ctx.consts);
+							resolv_glob(&path, ctx.root, &abs_path, pos, &ctx.ctx.i, &mut tree_ctx).await
+						},
+						Some(sub_tgt) => {
+							let sub_path = (path.strip_prefix(&ctx.pars_prefix[..]))
+								.expect("from_parsed called with path outside pars_prefix");
+							resolv_glob(sub_path, ctx.pars_root, sub_tgt, pos, &ctx.ctx.i, &mut ()).await
+						},
+					};
+					let abs_path = abs_path.to_sym(&ctx.ctx.i).await;
+					match names_res {
+						Err(e) => ctx.rep.report(e),
+						Ok(names) =>
+							for name in names {
+								match glob_imports_by_name.entry(name) {
+									Entry::Occupied(mut o) => o.get_mut().push((abs_path.clone(), import.sr.clone())),
+									Entry::Vacant(v) => {
+										v.insert_entry(vec![(abs_path.clone(), import.sr.clone())]);
+									},
+								}
+							},
+					}
+				},
+			}
+		}
+		let mut imports = HashMap::new();
+		if parsed.use_prelude {
+			let systems = ctx.ctx.systems.read().await;
+			for sys in systems.values().flat_map(|weak| weak.upgrade()) {
+				for prelude_item in sys.prelude() {
+					imports.insert(
+						prelude_item.last_seg(),
+						Ok(ResolvedImport {
+							target: prelude_item,
+							pos: CodeGenInfo::new_details(sys.ctor().name_tok().await, "In prelude", &ctx.ctx.i)
+								.await
+								.pos(),
+						}),
+					);
+				}
+			}
+		}
+		let conflicting_imports_msg = ctx.ctx.i.i("Conflicting imports").await;
+		for (key, values) in imports_by_name {
+			if values.len() == 1 {
+				let import = values.into_iter().next().unwrap();
+				let sr = import.sr.clone();
+				let abs_path_res = absolute_path(&path, &import.clone().mspath(), &ctx.ctx.i).await;
+				match abs_path_res {
+					Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, sr.pos(), &import.to_string()).await),
+					Ok(abs_path) => {
+						let target = abs_path.to_sym(&ctx.ctx.i).await;
+						imports.insert(key, Ok(ResolvedImport { target, pos: sr.pos() }));
+					},
+				}
+			} else {
+				for item in values {
+					ctx.rep.report(mk_errv(
+						conflicting_imports_msg.clone(),
+						format!("{key} is imported multiple times from different modules"),
+						[item.sr.pos()],
+					));
+				}
+			}
+		}
+		for (key, values) in glob_imports_by_name {
+			if !imports.contains_key(&key) {
+				let i = &ctx.ctx.i;
+				let values = stream::iter(values)
+					.then(|(n, sr)| {
+						clone!(key; async move {
+							ResolvedImport {
+								target: n.to_vname().suffix([key.clone()]).to_sym(i).await,
+								pos: sr.pos(),
+							}
+						})
+					})
+					.collect::<Vec<_>>()
+					.await;
+				imports.insert(key, if values.len() == 1 { Ok(values[0].clone()) } else { Err(values) });
+			}
+		}
+		let self_referential_msg = ctx.ctx.i.i("Self-referential import").await;
+		for (key, value) in imports.iter() {
+			let Ok(import) = value else { continue };
+			if import.target.strip_prefix(&path[..]).is_some_and(|t| t.starts_with(slice::from_ref(key)))
+			{
+				ctx.rep.report(mk_errv(
+					self_referential_msg.clone(),
+					format!("import {} points to itself or a path within itself", &import.target),
+					[import.pos.clone()],
+				));
+			}
+		}
+		let mut members = HashMap::new();
+		for item in &parsed.items {
+			match &item.kind {
+				ItemKind::Member(mem) => {
+					let path = path.to_vname().suffix([mem.name.clone()]).to_sym(&ctx.ctx.i).await;
+					let kind = OnceCell::from(MemberKind::from_parsed(&mem.kind, path.clone(), ctx).await);
+					members.insert(
+						mem.name.clone(),
+						Rc::new(Member { kind, lazy: RefCell::default(), public: mem.exported }),
+					);
+				},
+				ItemKind::Import(_) => (),
+			}
+		}
+		Module { imports, members }
+	}
+	pub async fn merge(
+		&self,
+		other: &Module,
+		ctx: Ctx,
+		consts: &MemoMap<Sym, Expr>,
+	) -> Result<Module, MergeErr> {
+		if !self.imports.is_empty() || !other.imports.is_empty() {
+			return Err(MergeErr { path: VPath::new([]), kind: MergeErrKind::Imports });
+		}
+		let mut members = HashMap::new();
+		for (key, mem) in &other.members {
+			let Some(own) = self.members.get(key) else {
+				members.insert(key.clone(), mem.clone());
+				continue;
+			};
+			if own.public != mem.public {
+				return Err(MergeErr { path: VPath::new([]), kind: MergeErrKind::Visibility });
+			}
+			match (own.kind(ctx.clone(), consts).await, mem.kind(ctx.clone(), consts).await) {
+				(MemberKind::Module(own_sub), MemberKind::Module(sub)) => {
+					match own_sub.merge(sub, ctx.clone(), consts).boxed_local().await {
+						Ok(module) => {
+							members.insert(
+								key.clone(),
+								Rc::new(Member {
+									lazy: RefCell::new(None),
+									public: own.public,
+									kind: OnceCell::from(MemberKind::Module(module)),
+								}),
+							);
+						},
+						Err(mut e) => {
+							e.path = e.path.prefix([key.clone()]);
+							return Err(e);
+						},
+					}
+				},
+				_ => return Err(MergeErr { path: VPath::new([key.clone()]), kind: MergeErrKind::Const }),
+			}
+		}
+		for (key, mem) in &self.members {
+			if let Entry::Vacant(slot) = members.entry(key.clone()) {
+				slot.insert(mem.clone());
+			}
+		}
+		Ok(Module { imports: HashMap::new(), members })
+	}
 }
 
 #[derive(Debug)]
-pub enum ItemKind {
+pub struct MergeErr {
-	Member(Member),
+	pub path: VPath,
-	Export(Tok<String>),
+	pub kind: MergeErrKind,
-	Import(Import),
-	Macro(Option<NotNan<f64>>, Vec<Rule>),
 }
-impl ItemKind {
+#[derive(Debug)]
-	pub fn at(self, pos: Pos) -> Item { Item { comments: vec![], pos, kind: self } }
+pub enum MergeErrKind {
+	Imports,
+	Visibility,
+	Const,
 }
 
-impl Item {
+pub struct FromParsedCtx<'a> {
-	pub async fn from_api(tree: api::Item, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
+	pars_prefix: Sym,
-		let kind = match tree.kind {
+	pars_root: &'a ParsedModule,
-			api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys).await),
+	root: &'a Module,
-			api::ItemKind::Import(name) => ItemKind::Import(Import {
+	rep: &'a Reporter,
-				path: Sym::from_api(name, &sys.ctx().i).await.iter().cloned().collect(),
+	ctx: &'a Ctx,
-				name: None,
+	consts: &'a MemoMap<Sym, Expr>,
-			}),
+	deferred_consts: &'a mut HashMap<Sym, (api::SysId, api::ParsedConstId)>,
-			api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &sys.ctx().i).await),
-			api::ItemKind::Macro(macro_block) => {
-				let mut rules = Vec::new();
-				for rule in macro_block.rules {
-					let mut comments = Vec::new();
-					for comment in rule.comments {
-						comments.push(Comment::from_api(&comment, &sys.ctx().i).await);
-					}
-					let pos = Pos::from_api(&rule.location, &sys.ctx().i).await;
-					let pattern = mtreev_from_api(&rule.pattern, &sys.ctx().i, &mut {
-						clone!(pos, sys);
-						move |a| {
-							clone!(pos, sys);
-							Box::pin(async move {
-								MacTok::Atom(AtomHand::from_api(a, pos.clone(), &mut sys.ctx().clone()).await)
-							})
-						}
-					})
-					.await;
-					rules.push(Rule { pos, pattern, kind: RuleKind::Remote(sys.clone(), rule.id), comments });
-				}
-				ItemKind::Macro(macro_block.priority, rules)
-			},
-		};
-		let mut comments = Vec::new();
-		for comment in tree.comments.iter() {
-			comments.push(Comment::from_api(comment, &sys.ctx().i).await)
-		}
-		Self { pos: Pos::from_api(&tree.location, &sys.ctx().i).await, comments, kind }
-	}
 }
-impl Format for Item {
+
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+impl Tree for Module {
-		let comment_text = self.comments.iter().join("\n");
+	type Ctx<'a> = (Ctx, &'a MemoMap<Sym, Expr>);
-		let item_text = match &self.kind {
+	async fn child(
-			ItemKind::Import(i) => format!("import {i}").into(),
+		&self,
-			ItemKind::Export(e) => format!("export {e}").into(),
+		key: Tok<String>,
-			ItemKind::Macro(None, rules) =>
+		public_only: bool,
-				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("macro {{\n\t{0}\n}}")))
+		(ctx, consts): &mut Self::Ctx<'_>,
-					.units([Variants::sequence(rules.len(), "\n", None)
+	) -> crate::dealias::ChildResult<'_, Self> {
-						.units(join_all(rules.iter().map(|r| r.print(c))).await)]),
+		let Some(member) = self.members.get(&key) else {
-			ItemKind::Member(mem) => match mem.kind.get() {
+			return Err(ChildErrorKind::Missing);
-				None => format!("lazy {}", mem.name).into(),
-				Some(MemberKind::Const(val)) =>
-					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} = {1}")))
-						.units([mem.name.rc().into(), val.print(c).await]),
-				Some(MemberKind::Mod(module)) =>
-					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
-						.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
-			},
-			_ => panic!(),
 		};
-		tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}\n{1}")))
+		if public_only && !member.public {
-			.units([comment_text.into(), item_text])
+			return Err(ChildErrorKind::Private);
+		}
+		match &member.kind(ctx.clone(), consts).await {
+			MemberKind::Module(m) => Ok(m),
+			MemberKind::Const => Err(ChildErrorKind::Constant),
+		}
+	}
+	fn children(&self, public_only: bool) -> hashbrown::HashSet<Tok<String>> {
+		self.members.iter().filter(|(_, v)| !public_only || v.public).map(|(k, _)| k.clone()).collect()
 	}
 }
 
 pub struct Member {
-	name: Tok<String>,
+	pub public: bool,
-	kind: OnceCell<MemberKind>,
+	pub lazy: RefCell<Option<LazyMemberHandle>>,
-	lazy: Mutex<Option<LazyMemberHandle>>,
+	pub kind: OnceCell<MemberKind>,
 }
 impl Member {
-	pub fn name(&self) -> Tok<String> { self.name.clone() }
+	#[must_use]
-	pub async fn kind(&self) -> &MemberKind {
+	pub async fn kind<'a>(&'a self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> &'a MemberKind {
 		(self.kind.get_or_init(async {
-			let handle = self.lazy.lock().await.take().expect("Neither known nor lazy");
+			let handle = self.lazy.borrow_mut().take().expect("If kind is uninit, lazy must be Some");
-			handle.run().await
+			handle.run(ctx, consts).await
 		}))
 		.await
 	}
-	pub async fn kind_mut(&mut self) -> &mut MemberKind {
-		self.kind().await;
-		self.kind.get_mut().expect("kind() already filled the cell")
-	}
-	pub async fn from_api(api: api::Member, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
-		path.push(Tok::from_api(api.name, &sys.ctx().i).await);
-		let kind = match api.kind {
-			api::MemberKind::Lazy(id) => {
-				let handle = LazyMemberHandle(id, sys.clone(), path.clone());
-				return handle.into_member(path.pop().unwrap());
-			},
-			api::MemberKind::Const(c) => MemberKind::Const(Code::from_expr(
-				CodeLocator::to_const(sys.ctx().i.i(&*path).await),
-				Expr::from_api(&c, &mut sys.ext().clone()).await,
-			)),
-			api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, path, sys).await),
-		};
-		let name = path.pop().unwrap();
-		Member { name, kind: OnceCell::from(kind), lazy: Mutex::default() }
-	}
-	pub fn new(name: Tok<String>, kind: MemberKind) -> Self {
-		Member { name, kind: OnceCell::from(kind), lazy: Mutex::default() }
-	}
-}
-impl Debug for Member {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		f.debug_struct("Member")
-			.field("name", &self.name)
-			.field("kind", &self.kind)
-			.finish_non_exhaustive()
-	}
 }
 
-#[derive(Debug)]
 pub enum MemberKind {
-	Const(Code),
+	Const,
-	Mod(Module),
+	Module(Module),
 }
-
+impl MemberKind {
-#[derive(Debug, Default)]
+	#[must_use]
-pub struct Module {
+	async fn from_parsed(parsed: &ParsedMemberKind, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
-	pub imports: Vec<Sym>,
+		match parsed {
-	pub exports: Vec<Tok<String>>,
+			ParsedMemberKind::Const(id, sys) => {
-	pub items: Vec<Item>,
+				ctx.deferred_consts.insert(path, (sys.id(), *id));
-}
+				MemberKind::Const
-impl Module {
-	pub fn new(items: impl IntoIterator<Item = Item>) -> Self {
-		let items = items.into_iter().collect_vec();
-		let exports = (items.iter())
-			.filter_map(|i| match &i.kind {
-				ItemKind::Export(e) => Some(e.clone()),
-				_ => None,
-			})
-			.collect_vec();
-		Self { imports: vec![], exports, items }
-	}
-	pub async fn from_api(m: api::Module, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
-		Self::new(
-			stream! { for item in m.items { yield Item::from_api(item, path, sys).boxed_local().await } }
-				.collect::<Vec<_>>()
-				.await,
-		)
-	}
-	pub async fn walk(
-		&self,
-		allow_private: bool,
-		path: impl IntoIterator<Item = Tok<String>>,
-	) -> Result<&Module, WalkError> {
-		let mut cur = self;
-		for (pos, step) in path.into_iter().enumerate() {
-			let Some(member) = (cur.items.iter())
-				.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
-				.find(|m| m.name == step)
-			else {
-				return Err(WalkError { pos, kind: WalkErrorKind::Missing });
-			};
-			if !allow_private && !cur.exports.contains(&step) {
-				return Err(WalkError { pos, kind: WalkErrorKind::Private });
-			}
-			match member.kind().await {
-				MemberKind::Const(_) => return Err(WalkError { pos, kind: WalkErrorKind::Constant }),
-				MemberKind::Mod(m) => cur = m,
-			}
-		}
-		Ok(cur)
-	}
-}
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-pub enum WalkErrorKind {
-	Missing,
-	Private,
-	Constant,
-}
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-pub struct WalkError {
-	pub pos: usize,
-	pub kind: WalkErrorKind,
-}
-impl Format for Module {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		let import_str = self.imports.iter().map(|i| format!("import {i}")).join("\n");
-		let head_str = format!("{import_str}\nexport ::({})\n", self.exports.iter().join(", "));
-		Variants::sequence(self.items.len() + 1, "\n", None).units(
-			[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
-		)
-	}
-}
-
-pub struct LazyMemberHandle(api::TreeId, System, Vec<Tok<String>>);
-impl LazyMemberHandle {
-	pub async fn run(self) -> MemberKind {
-		match self.1.get_tree(self.0).await {
-			api::MemberKind::Const(c) => MemberKind::Const(Code {
-				bytecode: Expr::from_api(&c, &mut self.1.ext().clone()).await.into(),
-				locator: CodeLocator { steps: self.1.ctx().i.i(&self.2).await, rule_loc: None },
-				source: None,
-			}),
-			api::MemberKind::Module(m) =>
-				MemberKind::Mod(Module::from_api(m, &mut { self.2 }, &self.1).await),
-			api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run().boxed_local().await,
-		}
-	}
-	pub fn into_member(self, name: Tok<String>) -> Member {
-		Member { name, kind: OnceCell::new(), lazy: Mutex::new(Some(self)) }
-	}
-}
-
-#[derive(Debug)]
-pub struct Rule {
-	pub pos: Pos,
-	pub comments: Vec<Comment>,
-	pub pattern: Vec<MacTree>,
-	pub kind: RuleKind,
-}
-impl Format for Rule {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		FmtUnit::new(
-			tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0b}\n{1} => {2b}"))),
-			[
-				self.comments.iter().join("\n").into(),
-				mtreev_fmt(&self.pattern, c).await,
-				match &self.kind {
-					RuleKind::Native(code) => code.print(c).await,
-					RuleKind::Remote(sys, id) => FmtUnit::new(
-						tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0} #{1}"))),
-						[sys.print(c).await, format!("{id:?}").into()],
-					),
 			},
-			],
+			ParsedMemberKind::Mod(m) =>
+				MemberKind::Module(Module::from_parsed(m, path, ctx).boxed_local().await),
+		}
+	}
+}
+
+#[derive(destructure)]
+pub struct LazyMemberHandle {
+	id: api::TreeId,
+	sys: api::SysId,
+	path: Sym,
+}
+impl LazyMemberHandle {
+	#[must_use]
+	pub async fn run(mut self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> MemberKind {
+		let sys = ctx.system_inst(self.sys).await.expect("Missing system for lazy member");
+		match sys.get_tree(self.id).await {
+			api::MemberKind::Const(c) => {
+				let mut pctx = ExprParseCtx { ctx: &ctx, exprs: sys.ext().exprs() };
+				let expr = Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await;
+				let (.., path) = self.destructure();
+				consts.insert(path, expr);
+				MemberKind::Const
+			},
+			api::MemberKind::Module(m) => {
+				let (.., path) = self.destructure();
+				MemberKind::Module(
+					Module::from_api(m, &mut TreeFromApiCtx { sys: &sys, consts, path: path.tok() }).await,
 				)
-	}
+			},
-}
+			api::MemberKind::Lazy(id) => {
-
+				self.id = id;
-#[derive(Debug)]
+				self.run(ctx, consts).boxed_local().await
-pub enum RuleKind {
+			},
-	Remote(System, api::MacroId),
+		}
-	Native(Code),
+	}
-}
+	#[must_use]
-
+	pub async fn into_member(self, public: bool) -> Member {
-#[derive(Debug)]
+		Member { public, kind: OnceCell::new(), lazy: RefCell::new(Some(self)) }
-pub struct Code {
-	locator: CodeLocator,
-	source: Option<Vec<MacTree>>,
-	bytecode: OnceCell<Expr>,
-}
-impl Code {
-	pub fn from_expr(locator: CodeLocator, expr: Expr) -> Self {
-		Self { locator, source: None, bytecode: expr.into() }
-	}
-	pub fn from_code(locator: CodeLocator, code: Vec<MacTree>) -> Self {
-		Self { locator, source: Some(code), bytecode: OnceCell::new() }
-	}
-	pub fn source(&self) -> Option<&Vec<MacTree>> { self.source.as_ref() }
-	pub fn set_source(&mut self, source: Vec<MacTree>) {
-		self.source = Some(source);
-		self.bytecode = OnceCell::new();
-	}
-	pub async fn get_bytecode(&self, ctx: &Ctx) -> &Expr {
-		(self.bytecode.get_or_init(async {
-			let src = self.source.as_ref().expect("no bytecode or source");
-			mtreev_to_expr(src, Substack::Bottom, ctx).await.at(Pos::None)
-		}))
-		.await
-	}
-}
-impl Format for Code {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		if let Some(bc) = self.bytecode.get() {
-			return bc.print(c).await;
-		}
-		if let Some(src) = &self.source {
-			return mtreev_fmt(src, c).await;
-		}
-		panic!("Code must be initialized with at least one state")
-	}
-}
-
-/// Selects a code element
-///
-/// Either the steps point to a constant and rule_loc is None, or the steps
-/// point to a module and rule_loc selects a macro rule within that module
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-pub struct CodeLocator {
-	steps: Tok<Vec<Tok<String>>>,
-	/// Index of a macro block in the module demarked by the steps, and a rule in
-	/// that macro
-	rule_loc: Option<(u16, u16)>,
-}
-impl CodeLocator {
-	pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps, rule_loc: None } }
-	pub fn to_rule(steps: Tok<Vec<Tok<String>>>, macro_i: u16, rule_i: u16) -> Self {
-		Self { steps, rule_loc: Some((macro_i, rule_i)) }
 	}
 }

orchid-std/Cargo.toml

@@ -1,19 +1,26 @@
 [package]
 name = "orchid-std"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 [dependencies]
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-futures = "0.3.31"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 never = "0.1.0"
-once_cell = "1.20.2"
+once_cell = "1.21.3"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-orchid-extension = { version = "0.1.0", path = "../orchid-extension" }
+orchid-extension = { version = "0.1.0", path = "../orchid-extension", features = [
-ordered-float = "4.6.0"
+  "tokio",
-tokio = { version = "1.43.0", features = ["full"] }
+] }
+ordered-float = "5.0.0"
+rust_decimal = "1.37.2"
+substack = "1.1.1"
+tokio = { version = "1.47.1", features = ["full"] }
+
+[dev-dependencies]
+test_executors = "0.3.5"

orchid-std/src/lib.rs

@@ -1,7 +1,9 @@
-mod number;
+mod macros;
 mod std;
-mod string;
 
-pub use std::StdSystem;
+pub use std::number::num_atom::{Float, HomoArray, Int, Num};
+pub use std::std_system::StdSystem;
+pub use std::string::str_atom::OrcString;
 
-pub use string::str_atom::OrcString;
+pub use macros::macro_system::MacroSystem;
+pub use macros::mactree::{MacTok, MacTree};

orchid-std/src/macros/instantiate_tpl.rs

@@ -0,0 +1,49 @@
+use std::borrow::Cow;
+
+use never::Never;
+use orchid_extension::atom::{Atomic, TypAtom};
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::GExpr;
+
+use crate::macros::mactree::{MacTok, MacTree, map_mactree};
+
+#[derive(Clone)]
+pub struct InstantiateTplCall {
+	pub(crate) tpl: MacTree,
+	pub(crate) argc: usize,
+	pub(crate) argv: Vec<MacTree>,
+}
+impl Atomic for InstantiateTplCall {
+	type Variant = OwnedVariant;
+	type Data = ();
+}
+impl OwnedAtom for InstantiateTplCall {
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	type Refs = Never;
+	// Technically must be supported but shouldn't actually ever be called
+	async fn call_ref(&self, arg: Expr) -> GExpr {
+		eprintln!(
+			"Copying partially applied instantiate_tpl call. This is an internal value.\
+      \nIt should be fully consumed within generated code."
+		);
+		self.clone().call(arg).await
+	}
+	async fn call(mut self, arg: Expr) -> GExpr {
+		match TypAtom::<MacTree>::try_from_expr(arg).await {
+			Err(e) => return Err::<Never, _>(e).to_expr().await,
+			Ok(t) => self.argv.push(own(t).await),
+		};
+		if self.argv.len() < self.argc {
+			return self.to_expr().await;
+		}
+		let mut args = self.argv.into_iter();
+		let ret = map_mactree(&self.tpl, &mut false, &mut |mt| match mt.tok() {
+			MacTok::Slot => Some(args.next().expect("Not enough arguments to fill all slots")),
+			_ => None,
+		});
+		assert!(args.next().is_none(), "Too many arguments for all slots");
+		ret.to_expr().await
+	}
+}

orchid-std/src/macros/let_line.rs

@@ -0,0 +1,120 @@
+use std::pin::pin;
+
+use futures::{FutureExt, StreamExt, stream};
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::{OrcRes, Reporter};
+use orchid_base::name::Sym;
+use orchid_base::parse::{
+	Comment, ParseCtx, Parsed, Snippet, expect_tok, token_errv, try_pop_no_fluff,
+};
+use orchid_base::sym;
+use orchid_base::tree::Paren;
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::parser::{ConstCtx, PSnippet, PTok, PTokTree, ParsCtx, ParsedLine, Parser};
+
+use crate::macros::mactree::{MacTok, MacTree, glossary_v, map_mactree_v};
+
+#[derive(Default)]
+pub struct LetLine;
+impl Parser for LetLine {
+	const LINE_HEAD: &'static str = "let";
+	async fn parse<'a>(
+		ctx: ParsCtx<'a>,
+		exported: bool,
+		comments: Vec<Comment>,
+		line: PSnippet<'a>,
+	) -> OrcRes<Vec<ParsedLine>> {
+		let sr = line.sr();
+		let Parsed { output: name_tok, tail } = try_pop_no_fluff(&ctx, line).await?;
+		let Some(name) = name_tok.as_name() else {
+			let err = token_errv(&ctx, name_tok, "Constant must have a name", |t| {
+				format!("Expected a name but found {t}")
+			});
+			return Err(err.await);
+		};
+		let Parsed { tail, .. } = expect_tok(&ctx, tail, ctx.i().i("=").await).await?;
+		let aliased = parse_tokv(tail, &ctx).await;
+		Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name, async move |ctx| {
+			let rep = Reporter::new();
+			let dealiased = dealias_mac_v(aliased, &ctx, &rep).await;
+			let macro_input = MacTok::S(Paren::Round, dealiased).at(sr.pos());
+			if let Some(e) = rep.errv() {
+				return Err(e);
+			}
+			Ok(call([
+				sym_ref(sym!(macros::lower; ctx.i()).await),
+				call([sym_ref(sym!(macros::resolve; ctx.i()).await), atom(macro_input)]),
+			]))
+		})])
+	}
+}
+
+pub async fn dealias_mac_v(aliased: Vec<MacTree>, ctx: &ConstCtx, rep: &Reporter) -> Vec<MacTree> {
+	let keys = glossary_v(&aliased).collect_vec();
+	let mut names: HashMap<_, _> = HashMap::new();
+	let mut stream = pin!(ctx.names(&keys).zip(stream::iter(&keys)));
+	while let Some((canonical, local)) = stream.next().await {
+		match canonical {
+			Err(e) => rep.report(e),
+			Ok(name) => {
+				names.insert(local.clone(), name);
+			},
+		}
+	}
+	map_mactree_v(&aliased, &mut false, &mut |tree| match &*tree.tok {
+		MacTok::Name(n) => names.get(n).map(|new_n| MacTok::Name(new_n.clone()).at(tree.pos())),
+		_ => None,
+	})
+}
+
+pub async fn parse_tokv(line: PSnippet<'_>, ctx: &impl ParseCtx) -> Vec<MacTree> {
+	if let Some((idx, arg)) = line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?))) {
+		let (head, lambda) = line.split_at(idx as u32);
+		let (_, body) = lambda.pop_front().unwrap();
+		let body = parse_tokv(body, ctx).boxed_local().await;
+		let mut all = parse_tokv_no_lambdas(&head, ctx).await;
+		match parse_tok(arg, ctx).await {
+			Some(arg) => all.push(MacTok::Lambda(arg, body).at(lambda.sr().pos())),
+			None => ctx.rep().report(
+				token_errv(ctx, arg, "Lambda argument fluff", |arg| {
+					format!("Lambda arguments must be a valid token, found meaningless fragment {arg}")
+				})
+				.await,
+			),
+		};
+		all
+	} else {
+		parse_tokv_no_lambdas(&line, ctx).await
+	}
+}
+
+async fn parse_tokv_no_lambdas(line: &[PTokTree], ctx: &impl ParseCtx) -> Vec<MacTree> {
+	stream::iter(line).filter_map(|tt| parse_tok(tt, ctx)).collect().await
+}
+
+pub async fn parse_tok(tree: &PTokTree, ctx: &impl ParseCtx) -> Option<MacTree> {
+	let tok = match &tree.tok {
+		PTok::Bottom(errv) => MacTok::Bottom(errv.clone()),
+		PTok::BR | PTok::Comment(_) => return None,
+		PTok::Name(n) => MacTok::Name(Sym::new([n.clone()], ctx.i()).await.unwrap()),
+		PTok::NS(..) => match tree.as_multiname() {
+			Ok(mn) => MacTok::Name(mn.to_sym(ctx.i()).await),
+			Err(nested) => {
+				ctx.rep().report(
+					token_errv(ctx, tree, ":: can only be followed by a name in an expression", |tok| {
+						format!("Expected name, found {tok}")
+					})
+					.await,
+				);
+				return parse_tok(nested, ctx).boxed_local().await;
+			},
+		},
+		PTok::Handle(expr) => MacTok::Value(expr.clone()),
+		PTok::NewExpr(never) => match *never {},
+		PTok::LambdaHead(_) => panic!("Lambda-head handled in the sequence parser"),
+		PTok::S(p, body) =>
+			MacTok::S(*p, parse_tokv(Snippet::new(tree, body), ctx).boxed_local().await),
+	};
+	Some(tok.at(tree.sr().pos()))
+}

orchid-std/src/macros/macro_lib.rs

@@ -0,0 +1,83 @@
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::Reporter;
+use orchid_base::sym;
+use orchid_extension::atom::TypAtom;
+use orchid_extension::atom_owned::own;
+use orchid_extension::conv::ToExpr;
+use orchid_extension::coroutine_exec::exec;
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::reflection::{ReflMemKind, refl};
+use orchid_extension::tree::{GenMember, comments, fun, prefix};
+use substack::Substack;
+
+use crate::Int;
+use crate::macros::instantiate_tpl::InstantiateTplCall;
+use crate::macros::macro_line::{Macro, Matcher};
+use crate::macros::mactree::{LowerCtx, MacTree};
+use crate::macros::recur_state::RecurState;
+use crate::macros::resolve::{ResolveCtx, resolve};
+
+pub fn gen_macro_lib() -> Vec<GenMember> {
+	prefix("macros", [
+		comments(
+			["This is an internal function, you can't obtain a value of its argument type.", "hidden"],
+			fun(true, "instantiate_tpl", |tpl: TypAtom<MacTree>, right: Int| async move {
+				InstantiateTplCall {
+					tpl: own(tpl).await,
+					argc: right.0.try_into().unwrap(),
+					argv: Vec::new(),
+				}
+			}),
+		),
+		fun(true, "resolve", |tpl: TypAtom<MacTree>| async move {
+			call([
+				sym_ref(sym!(macros::resolve_recur; tpl.untyped.ctx().i()).await),
+				atom(RecurState::Bottom),
+				tpl.untyped.ex().to_expr().await,
+			])
+		}),
+		fun(true, "lower", |tpl: TypAtom<MacTree>| async move {
+			let ctx = LowerCtx { sys: tpl.untyped.ctx().clone(), rep: &Reporter::new() };
+			let res = own(tpl).await.lower(ctx, Substack::Bottom).await;
+			if let Some(e) = Reporter::new().errv() { Err(e) } else { Ok(res) }
+		}),
+		fun(true, "resolve_recur", |state: TypAtom<RecurState>, tpl: TypAtom<MacTree>| async move {
+			exec("macros::resolve_recur", async move |mut h| {
+				let ctx = tpl.ctx().clone();
+				let root = refl(&ctx);
+				let tpl = own(tpl.clone()).await;
+				let mut macros = HashMap::new();
+				for n in tpl.glossary() {
+					if let Ok(ReflMemKind::Const) = root.get_by_path(n).await.map(|m| m.kind()) {
+						let Ok(mac) = h.exec::<TypAtom<Macro>>(sym_ref(n.clone())).await else { continue };
+						let mac = own(mac).await;
+						macros.entry(mac.0.own_kws[0].clone()).or_insert(mac);
+					}
+				}
+				let mut named = HashMap::new();
+				let mut priod = Vec::new();
+				for (_, mac) in macros.iter() {
+					for rule in mac.0.rules.iter() {
+						if rule.glossary.is_subset(tpl.glossary()) {
+							match &rule.pattern {
+								Matcher::Named(m) =>
+									named.entry(m.head()).or_insert(Vec::new()).push((m, mac, rule)),
+								Matcher::Priod(p) => priod.push((mac.0.prio, (p, mac, rule))),
+							}
+						}
+					}
+				}
+				let priod = priod.into_iter().sorted_unstable_by_key(|(p, _)| *p).map(|(_, r)| r).collect();
+				let mut rctx = ResolveCtx { h, recur: own(state).await, ctx: ctx.clone(), named, priod };
+				let resolve_res = resolve(&mut rctx, &tpl).await;
+				std::mem::drop(rctx);
+				match resolve_res {
+					Some(out_tree) => out_tree.to_expr().await,
+					None => tpl.to_expr().await,
+				}
+			})
+			.await
+		}),
+	])
+}

orchid-std/src/macros/macro_line.rs

@@ -0,0 +1,228 @@
+use std::borrow::Cow;
+use std::cell::RefCell;
+use std::rc::Rc;
+
+use async_once_cell::OnceCell;
+use futures::{StreamExt, stream};
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools;
+use never::Never;
+use orchid_base::error::{OrcRes, Reporter, mk_errv};
+use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::parse::{
+	Comment, ParseCtx, Parsed, Snippet, expect_end, expect_tok, line_items, token_errv,
+	try_pop_no_fluff,
+};
+use orchid_base::tree::{Paren, Token};
+use orchid_base::{clone, sym};
+use orchid_extension::atom::{Atomic, TypAtom};
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::parser::{PSnippet, ParsCtx, ParsedLine, Parser};
+
+use crate::macros::let_line::{dealias_mac_v, parse_tokv};
+use crate::macros::mactree::{glossary_v, map_mactree_v};
+use crate::macros::recur_state::{RecurState, RulePath};
+use crate::macros::rule::matcher::{NamedMatcher, PriodMatcher};
+use crate::{Int, MacTok};
+
+#[derive(Default)]
+pub struct MacroLine;
+impl Parser for MacroLine {
+	const LINE_HEAD: &'static str = "macro";
+	async fn parse<'a>(
+		ctx: ParsCtx<'a>,
+		exported: bool,
+		comments: Vec<Comment>,
+		line: PSnippet<'a>,
+	) -> OrcRes<Vec<ParsedLine>> {
+		if exported {
+			return Err(mk_errv(
+				ctx.i().i("macros are always exported").await,
+				"The export keyword is forbidden here to avoid confusion\n\
+					because macros are exported by default",
+				[line.sr()],
+			));
+		}
+		let module = ctx.module();
+		let Parsed { output, tail } = try_pop_no_fluff(&ctx, line).await?;
+		let bad_first_item_err = || {
+			token_errv(&ctx, output, "Expected priority or block", |s| {
+				format!("Expected a priority number or a () block, found {s}")
+			})
+		};
+		let (prio, body) = match &output.tok {
+			Token::S(Paren::Round, body) => (None, body),
+			Token::Handle(expr) => match TypAtom::<Int>::try_from_expr(expr.clone()).await {
+				Err(e) => {
+					return Err(e + bad_first_item_err().await);
+				},
+				Ok(prio) => {
+					let Token::S(Paren::Round, block) = &output.tok else {
+						return Err(
+							token_errv(&ctx, output, "Expected () block", |s| {
+								format!("Expected a () block, found {s}")
+							})
+							.await,
+						);
+					};
+					(Some(prio), block)
+				},
+			},
+			_ => return Err(bad_first_item_err().await),
+		};
+		expect_end(&ctx, tail).await?;
+		let lines = line_items(&ctx, Snippet::new(output, body)).await;
+		let Some((kw_line, rule_lines)) = lines.split_first() else { return Ok(Vec::new()) };
+		let mut keywords = HashMap::new();
+		let Parsed { tail: kw_tail, .. } =
+			expect_tok(&ctx, kw_line.tail, ctx.i().i("keywords").await).await?;
+		for kw_tok in kw_tail.iter().filter(|kw| !kw.is_fluff()) {
+			match kw_tok.as_name() {
+				Some(kw) => {
+					keywords.insert(kw, kw_tok.sr());
+				},
+				None => ctx.rep().report(
+					token_errv(&ctx, kw_tok, "invalid macro keywords list", |tok| {
+						format!("The keywords list must be a sequence of names; received {tok}")
+					})
+					.await,
+				),
+			}
+		}
+		let Some(macro_name) = keywords.keys().next().cloned() else {
+			return Err(mk_errv(
+				ctx.i().i("macro with no keywords").await,
+				"Macros must define at least one macro of their own.",
+				[kw_line.tail.sr()],
+			));
+		};
+		let mut rules = Vec::new();
+		let mut lines = Vec::new();
+		for (idx, line) in rule_lines.iter().enumerate().map(|(n, v)| (n as u32, v)) {
+			let path = RulePath { module: module.clone(), main_kw: macro_name.clone(), rule: idx };
+			let sr = line.tail.sr();
+			let name = ctx.i().i(&path.name()).await;
+			let Parsed { tail, .. } = expect_tok(&ctx, line.tail, ctx.i().i("rule").await).await?;
+			let arrow_token = ctx.i().i("=>").await;
+			let Some((pattern, body)) = tail.split_once(|tok| tok.is_kw(arrow_token.clone())) else {
+				ctx.rep().report(mk_errv(
+					ctx.i().i("Missing => in rule").await,
+					"Rule lines are of the form `rule ...pattern => ...body`",
+					[line.tail.sr()],
+				));
+				continue;
+			};
+			let pattern = parse_tokv(pattern, &ctx).await;
+			let mut placeholders = Vec::new();
+			map_mactree_v(&pattern, &mut false, &mut |tok| {
+				if let MacTok::Ph(ph) = tok.tok() {
+					placeholders.push((ph.clone(), tok.pos()))
+				}
+				None
+			});
+			let mut body_mactree = parse_tokv(body, &ctx).await;
+			for (ph, ph_pos) in placeholders.iter().rev() {
+				let name = ctx.module().suffix([ph.name.clone()], ctx.i()).await;
+				body_mactree = vec![
+					MacTok::Lambda(MacTok::Name(name).at(ph_pos.clone()), body_mactree).at(ph_pos.clone()),
+				]
+			}
+			let body_sr = body.sr();
+			rules.push((name.clone(), placeholders, rules.len() as u32, sr.pos(), pattern));
+			lines.push(ParsedLine::cnst(&sr, &line.output, true, name, async move |ctx| {
+				let rep = Reporter::new();
+				let body = dealias_mac_v(body_mactree, &ctx, &rep).await;
+				let macro_input = MacTok::S(Paren::Round, body).at(body_sr.pos());
+				if let Some(e) = rep.errv() {
+					return Err(e);
+				}
+				Ok(call([
+					sym_ref(sym!(macros::resolve_recur; ctx.i()).await),
+					atom(RecurState::base(path)),
+					macro_input.to_expr().await,
+				]))
+			}))
+		}
+		let mac_cell = Rc::new(OnceCell::new());
+		let keywords = Rc::new(keywords);
+		let rules = Rc::new(RefCell::new(Some(rules)));
+		for (kw, sr) in &*keywords {
+			clone!(mac_cell, keywords, rules, module, prio);
+			lines.push(ParsedLine::cnst(&sr.clone(), &comments, true, kw.clone(), async move |cctx| {
+				let mac = mac_cell
+					.get_or_init(async {
+						let rep = Reporter::new();
+						let rules = rules.borrow_mut().take().expect("once cell initializer runs");
+						let rules = stream::iter(rules)
+							.then(|(body_name, placeholders, index, pos, pattern_macv)| {
+								let cctx = &cctx;
+								let rep = &rep;
+								let prio = &prio;
+								async move {
+									let pattern_abs = dealias_mac_v(pattern_macv, cctx, rep).await;
+									let glossary = glossary_v(&pattern_abs).collect();
+									let pattern_res = match prio {
+										None => NamedMatcher::new(&pattern_abs, cctx.i()).await.map(Matcher::Named),
+										Some(_) => PriodMatcher::new(&pattern_abs, cctx.i()).await.map(Matcher::Priod),
+									};
+									let placeholders = placeholders.into_iter().map(|(ph, _)| ph.name).collect_vec();
+									match pattern_res {
+										Ok(pattern) =>
+											Some(Rule { index, pos, body_name, pattern, glossary, placeholders }),
+										Err(e) => {
+											rep.report(e);
+											None
+										},
+									}
+								}
+							})
+							.flat_map(stream::iter)
+							.collect::<Vec<_>>()
+							.await;
+						let own_kws = keywords.keys().cloned().collect_vec();
+						Macro(Rc::new(MacroData { module, prio: prio.map(|i| i.0 as u64), rules, own_kws }))
+					})
+					.await;
+				atom(mac.clone())
+			}))
+		}
+		Ok(lines)
+	}
+}
+
+#[derive(Debug)]
+pub struct MacroData {
+	pub module: Sym,
+	pub prio: Option<u64>,
+	pub rules: Vec<Rule>,
+	pub own_kws: Vec<Tok<String>>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Macro(pub Rc<MacroData>);
+#[derive(Debug)]
+pub struct Rule {
+	pub index: u32,
+	pub pos: Pos,
+	pub pattern: Matcher,
+	pub glossary: HashSet<Sym>,
+	pub placeholders: Vec<Tok<String>>,
+	pub body_name: Tok<String>,
+}
+#[derive(Debug)]
+pub enum Matcher {
+	Named(NamedMatcher),
+	Priod(PriodMatcher),
+}
+impl Atomic for Macro {
+	type Data = ();
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for Macro {
+	type Refs = Never;
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+}

orchid-std/src/macros/macro_system.rs

@@ -0,0 +1,49 @@
+use never::Never;
+use orchid_base::interner::Interner;
+use orchid_base::name::Sym;
+use orchid_base::reqnot::Receipt;
+use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
+use orchid_extension::entrypoint::ExtReq;
+use orchid_extension::lexer::LexerObj;
+use orchid_extension::other_system::SystemHandle;
+use orchid_extension::parser::ParserObj;
+use orchid_extension::system::{System, SystemCard};
+use orchid_extension::system_ctor::SystemCtor;
+use orchid_extension::tree::GenMember;
+
+use crate::macros::instantiate_tpl::InstantiateTplCall;
+use crate::macros::let_line::LetLine;
+use crate::macros::macro_lib::gen_macro_lib;
+use crate::macros::macro_line::{Macro, MacroLine};
+use crate::macros::mactree_lexer::MacTreeLexer;
+use crate::macros::recur_state::RecurState;
+use crate::{MacTree, StdSystem};
+
+#[derive(Default)]
+pub struct MacroSystem;
+impl SystemCtor for MacroSystem {
+	type Deps = StdSystem;
+	type Instance = Self;
+	const NAME: &'static str = "orchid::macros";
+	const VERSION: f64 = 0.00_01;
+	fn inst(_: SystemHandle<StdSystem>) -> Self::Instance { Self }
+}
+impl SystemCard for MacroSystem {
+	type Ctor = Self;
+	type Req = Never;
+	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> {
+		[
+			Some(InstantiateTplCall::dynfo()),
+			Some(MacTree::dynfo()),
+			Some(RecurState::dynfo()),
+			Some(Macro::dynfo()),
+		]
+	}
+}
+impl System for MacroSystem {
+	async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
+	async fn prelude(_: &Interner) -> Vec<Sym> { vec![] }
+	fn lexers() -> Vec<LexerObj> { vec![&MacTreeLexer] }
+	fn parsers() -> Vec<ParserObj> { vec![&LetLine, &MacroLine] }
+	fn env() -> Vec<GenMember> { gen_macro_lib() }
+}

orchid-std/src/macros/mactree.rs

@@ -0,0 +1,231 @@
+use std::borrow::Cow;
+use std::fmt::Display;
+use std::rc::Rc;
+
+use futures::FutureExt;
+use futures::future::join_all;
+use hashbrown::HashSet;
+use itertools::Itertools;
+use orchid_base::error::{OrcErrv, Reporter, mk_errv};
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants, fmt};
+use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::tl_cache;
+use orchid_base::tree::{Paren, indent};
+use orchid_extension::atom::Atomic;
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+use orchid_extension::conv::ToExpr;
+use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::{GExpr, arg, bot, call, lambda, sym_ref};
+use orchid_extension::system::SysCtx;
+use substack::Substack;
+
+#[derive(Clone)]
+pub struct LowerCtx<'a> {
+	pub sys: SysCtx,
+	pub rep: &'a Reporter,
+}
+
+#[derive(Debug, Clone)]
+pub struct MacTree {
+	pub pos: Pos,
+	pub tok: Rc<MacTok>,
+	pub glossary: Rc<HashSet<Sym>>,
+}
+impl MacTree {
+	pub fn tok(&self) -> &MacTok { &self.tok }
+	pub fn pos(&self) -> Pos { self.pos.clone() }
+	pub fn glossary(&self) -> &HashSet<Sym> { &self.glossary }
+	pub async fn lower(&self, ctx: LowerCtx<'_>, args: Substack<'_, Sym>) -> GExpr {
+		let expr = match self.tok() {
+			MacTok::Bottom(e) => bot(e.clone()),
+			MacTok::Lambda(arg, body) => {
+				let MacTok::Name(name) = &*arg.tok else {
+					let err = mk_errv(
+						ctx.sys.i().i("Syntax error after macros").await,
+						"This token ends up as a binding, consider replacing it with a name",
+						[arg.pos()],
+					);
+					ctx.rep.report(err.clone());
+					return bot(err);
+				};
+				lambda(args.len() as u64, lower_v(body, ctx, args.push(name.clone())).await)
+			},
+			MacTok::Name(name) => match args.iter().enumerate().find(|(_, n)| *n == name) {
+				None => sym_ref(name.clone()),
+				Some((i, _)) => arg((args.len() - i) as u64),
+			},
+			MacTok::Ph(ph) => {
+				let err = mk_errv(
+					ctx.sys.i().i("Placeholder in value").await,
+					format!("Placeholder {ph} is only supported in macro patterns"),
+					[self.pos()],
+				);
+				ctx.rep.report(err.clone());
+				return bot(err);
+			},
+			MacTok::S(Paren::Round, body) => call(lower_v(body, ctx, args).await),
+			MacTok::S(..) => {
+				let err = mk_errv(
+					ctx.sys.i().i("[] or {} after macros").await,
+					format!("{} didn't match any macro", fmt(self, ctx.sys.i()).await),
+					[self.pos()],
+				);
+				ctx.rep.report(err.clone());
+				return bot(err);
+			},
+			MacTok::Slot => panic!("Uninstantiated template should never be exposed"),
+			MacTok::Value(v) => v.clone().to_expr().await,
+		};
+		expr.at(self.pos())
+	}
+}
+impl Atomic for MacTree {
+	type Data = ();
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for MacTree {
+	type Refs = ();
+
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	async fn print_atom<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		self.tok.print(c).await
+	}
+}
+impl Format for MacTree {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		self.tok.print(c).await
+	}
+}
+
+pub async fn lower_v(v: &[MacTree], ctx: LowerCtx<'_>, args: Substack<'_, Sym>) -> Vec<GExpr> {
+	join_all(v.iter().map(|t| t.lower(ctx.clone(), args.clone())).collect::<Vec<_>>()).await
+}
+
+#[derive(Debug, Clone)]
+pub enum MacTok {
+	S(Paren, Vec<MacTree>),
+	Name(Sym),
+	/// Only permitted in arguments to `instantiate_tpl`
+	Slot,
+	Value(Expr),
+	Lambda(MacTree, Vec<MacTree>),
+	/// Only permitted in "pattern" values produced by macro blocks, which are
+	/// never accessed as variables by usercode
+	Ph(Ph),
+	Bottom(OrcErrv),
+}
+impl MacTok {
+	pub fn build_glossary(&self) -> HashSet<Sym> {
+		match self {
+			MacTok::Bottom(_) | MacTok::Ph(_) | MacTok::Slot | MacTok::Value(_) => HashSet::new(),
+			MacTok::Name(sym) => HashSet::from([sym.clone()]),
+			MacTok::S(_, body) => body.iter().flat_map(|mt| &*mt.glossary).cloned().collect(),
+			MacTok::Lambda(arg, body) =>
+				body.iter().chain([arg]).flat_map(|mt| &*mt.glossary).cloned().collect(),
+		}
+	}
+	pub fn at(self, pos: impl Into<Pos>) -> MacTree {
+		MacTree { pos: pos.into(), glossary: Rc::new(self.build_glossary()), tok: Rc::new(self) }
+	}
+}
+impl Format for MacTok {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		match self {
+			Self::Value(v) => v.print(c).await,
+			Self::Lambda(arg, b) => FmtUnit::new(
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default()
+					.unbounded("\\{0b}.{1l}")
+					.bounded("(\\{0b}.{1b})"))),
+				[arg.print(c).boxed_local().await, mtreev_fmt(b, c).await],
+			),
+			Self::Name(n) => format!("{n}").into(),
+			Self::Ph(ph) => format!("{ph}").into(),
+			Self::S(p, body) => FmtUnit::new(
+				match *p {
+					Paren::Round => Rc::new(Variants::default().bounded("({0b})")),
+					Paren::Curly => Rc::new(Variants::default().bounded("{{0b}}")),
+					Paren::Square => Rc::new(Variants::default().bounded("[{0b}]")),
+				},
+				[mtreev_fmt(body, c).await],
+			),
+			Self::Slot => "$SLOT".into(),
+			Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()).into(),
+			Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),
+		}
+	}
+}
+
+pub async fn mtreev_fmt<'b>(
+	v: impl IntoIterator<Item = &'b MacTree>,
+	c: &(impl FmtCtx + ?Sized),
+) -> FmtUnit {
+	FmtUnit::sequence(" ", None, join_all(v.into_iter().map(|t| t.print(c))).await)
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct Ph {
+	pub name: Tok<String>,
+	pub kind: PhKind,
+}
+impl Display for Ph {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self.kind {
+			PhKind::Scalar => write!(f, "${}", self.name),
+			PhKind::Vector { at_least_one: false, priority: 0 } => write!(f, "..${}", self.name),
+			PhKind::Vector { at_least_one: true, priority: 0 } => write!(f, "...${}", self.name),
+			PhKind::Vector { at_least_one: false, priority } => write!(f, "..${}:{priority}", self.name),
+			PhKind::Vector { at_least_one: true, priority } => write!(f, "...${}:{priority}", self.name),
+		}
+	}
+}
+
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]
+pub enum PhKind {
+	Scalar,
+	Vector { at_least_one: bool, priority: u8 },
+}
+
+pub fn map_mactree<F: FnMut(MacTree) -> Option<MacTree>>(
+	src: &MacTree,
+	changed: &mut bool,
+	map: &mut F,
+) -> MacTree {
+	let tok = match map(src.clone()) {
+		Some(new_tok) => {
+			*changed = true;
+			return new_tok;
+		},
+		None => match &*src.tok {
+			MacTok::Lambda(arg, body) => MacTok::Lambda(
+				ro(changed, |changed| map_mactree(arg, changed, map)),
+				map_mactree_v(body, changed, map),
+			),
+			MacTok::Name(_) | MacTok::Value(_) => return src.clone(),
+			MacTok::Slot | MacTok::Ph(_) | MacTok::Bottom(_) => return src.clone(),
+			MacTok::S(p, body) => MacTok::S(*p, map_mactree_v(body, changed, map)),
+		},
+	};
+	if *changed { tok.at(src.pos()) } else { src.clone() }
+}
+pub fn map_mactree_v<F: FnMut(MacTree) -> Option<MacTree>>(
+	src: &[MacTree],
+	changed: &mut bool,
+	map: &mut F,
+) -> Vec<MacTree> {
+	src.iter().map(|tree| ro(changed, |changed| map_mactree(tree, changed, map))).collect_vec()
+}
+
+/// reverse "or". Inside, the flag is always false, but raising it will raise
+/// the outside flag too.
+fn ro<T>(flag: &mut bool, cb: impl FnOnce(&mut bool) -> T) -> T {
+	let mut new_flag = false;
+	let val = cb(&mut new_flag);
+	*flag |= new_flag;
+	val
+}
+
+pub fn glossary_v(src: &[MacTree]) -> impl Iterator<Item = Sym> {
+	src.iter().flat_map(|mt| mt.glossary()).cloned()
+}

orchid-std/src/macros/mactree_lexer.rs

@@ -0,0 +1,96 @@
+use std::ops::RangeInclusive;
+
+use futures::FutureExt;
+use orchid_base::error::{OrcRes, mk_errv};
+use orchid_base::parse::ParseCtx;
+use orchid_base::sym;
+use orchid_base::tokens::PARENS;
+use orchid_base::tree::Paren;
+use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
+use orchid_extension::parser::p_tree2gen;
+use orchid_extension::tree::{GenTok, GenTokTree, ref_tok, x_tok};
+
+use crate::macros::let_line::parse_tok;
+use crate::macros::mactree::{MacTok, MacTree};
+
+#[derive(Default)]
+pub struct MacTreeLexer;
+impl Lexer for MacTreeLexer {
+	const CHAR_FILTER: &'static [RangeInclusive<char>] = &['\''..='\''];
+	async fn lex<'a>(tail: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree)> {
+		let Some(tail2) = tail.strip_prefix('\'') else {
+			return Err(err_not_applicable(ctx.i()).await);
+		};
+		let tail3 = tail2.trim_start();
+		let mut args = Vec::new();
+		return match mac_tree(tail3, &mut args, ctx).await {
+			Ok((tail4, mactree)) => {
+				let range = ctx.pos_tt(tail, tail4);
+				let tok = match &args[..] {
+					[] => x_tok(mactree).await,
+					_ => {
+						let call = ([
+							ref_tok(sym!(macros::instantiate_tpl; ctx.i()).await).await.at(range.clone()),
+							x_tok(mactree).await.at(range.clone()),
+						]
+						.into_iter())
+						.chain(args.into_iter());
+						GenTok::S(Paren::Round, call.collect())
+					},
+				};
+				Ok((tail4, tok.at(range)))
+			},
+			Err(e) => Ok((tail2, GenTok::Bottom(e).at(ctx.pos_lt(1, tail2)))),
+		};
+		async fn mac_tree<'a>(
+			tail: &'a str,
+			args: &mut Vec<GenTokTree>,
+			ctx: &'a LexContext<'a>,
+		) -> OrcRes<(&'a str, MacTree)> {
+			for (lp, rp, paren) in PARENS {
+				let Some(mut body_tail) = tail.strip_prefix(*lp) else { continue };
+				let mut items = Vec::new();
+				return loop {
+					let tail2 = body_tail.trim_start();
+					if let Some(tail3) = tail2.strip_prefix(*rp) {
+						break Ok((tail3, MacTok::S(*paren, items).at(ctx.pos_tt(tail, tail3).pos())));
+					} else if tail2.is_empty() {
+						return Err(mk_errv(
+							ctx.i().i("Unclosed block").await,
+							format!("Expected closing {rp}"),
+							[ctx.pos_lt(1, tail)],
+						));
+					}
+					let (new_tail, new_item) = mac_tree(tail2, args, ctx).boxed_local().await?;
+					body_tail = new_tail;
+					items.push(new_item);
+				};
+			}
+			if let Some(tail2) = tail.strip_prefix("$") {
+				let (tail3, sub) = ctx.recurse(tail2).await?;
+				let sr = ctx.pos_tt(tail, tail3);
+				args.push(p_tree2gen(sub));
+				return Ok((tail3, MacTok::Slot.at(sr.pos())));
+			}
+			if let Some(tail2) = tail.strip_prefix("\\") {
+				let tail2 = tail2.trim_start();
+				let (mut tail3, param) = mac_tree(tail2, args, ctx).boxed_local().await?;
+				let mut body = Vec::new();
+				loop {
+					let tail4 = tail3.trim_start();
+					if tail4.is_empty() || tail4.starts_with(|c| ")]}".contains(c)) {
+						break;
+					};
+					let (tail5, body_tok) = mac_tree(tail4, args, ctx).boxed_local().await?;
+					body.push(body_tok);
+					tail3 = tail5;
+				}
+				Ok((tail3, MacTok::Lambda(param, body).at(ctx.pos_tt(tail, tail3).pos())))
+			} else {
+				let (tail2, sub) = ctx.recurse(tail).await?;
+				let parsed = parse_tok(&sub, ctx).await.expect("Unexpected invalid token");
+				Ok((tail2, parsed))
+			}
+		}
+	}
+}

orchid-std/src/macros/mod.rs

@@ -0,0 +1,12 @@
+mod instantiate_tpl;
+mod let_line;
+mod macro_lib;
+mod macro_line;
+pub mod macro_system;
+pub mod mactree;
+mod mactree_lexer;
+pub mod recur_state;
+mod resolve;
+mod rule;
+
+use mactree::{MacTok, MacTree};

orchid-std/src/macros/recur_state.rs

@@ -0,0 +1,71 @@
+use std::borrow::Cow;
+use std::fmt;
+use std::rc::Rc;
+
+use never::Never;
+use orchid_base::format::{FmtCtx, FmtUnit};
+use orchid_base::interner::Tok;
+use orchid_base::name::Sym;
+use orchid_extension::atom::Atomic;
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct RulePath {
+	pub module: Sym,
+	pub main_kw: Tok<String>,
+	pub rule: u32,
+}
+impl RulePath {
+	pub fn name(&self) -> String { format!("rule::{}::{}", self.main_kw, self.rule) }
+}
+impl fmt::Display for RulePath {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "Rule {}::({})::{}", self.module, self.main_kw, self.rule)
+	}
+}
+
+#[derive(Clone)]
+pub enum RecurState {
+	Bottom,
+	Recursive { path: RulePath, prev: Rc<RecurState> },
+}
+impl RecurState {
+	pub fn base(path: RulePath) -> Self {
+		RecurState::Recursive { path, prev: Rc::new(RecurState::Bottom) }
+	}
+	pub fn push(&self, new: RulePath) -> Option<Self> {
+		let mut cur = self;
+		while let Self::Recursive { path, prev } = cur {
+			if &new == path {
+				return None;
+			}
+			cur = prev;
+		}
+		Some(Self::Recursive { path: new, prev: Rc::new(self.clone()) })
+	}
+}
+impl Atomic for RecurState {
+	type Data = Option<()>;
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for RecurState {
+	type Refs = Never;
+
+	async fn val(&self) -> Cow<'_, Self::Data> {
+		Cow::Owned(match self {
+			Self::Bottom => None,
+			Self::Recursive { .. } => Some(()),
+		})
+	}
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		self.to_string().into()
+	}
+}
+impl fmt::Display for RecurState {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Self::Bottom => write!(f, "RecurState::Bottom"),
+			Self::Recursive { path, prev } => write!(f, "{path}\n{prev}"),
+		}
+	}
+}

orchid-std/src/macros/resolve.rs

@@ -0,0 +1,110 @@
+use futures::FutureExt;
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::mk_errv;
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::sym;
+use orchid_base::tree::Paren;
+use orchid_extension::conv::ToExpr;
+use orchid_extension::coroutine_exec::ExecHandle;
+use orchid_extension::gen_expr::{GExpr, bot, call, sym_ref};
+use orchid_extension::system::SysCtx;
+
+use crate::macros::macro_line::{Macro, Rule};
+use crate::macros::recur_state::{RecurState, RulePath};
+use crate::macros::rule::matcher::{NamedMatcher, PriodMatcher};
+use crate::macros::rule::state::{MatchState, StateEntry};
+use crate::{MacTok, MacTree};
+
+pub struct ResolveCtx<'a> {
+	pub ctx: SysCtx,
+	pub recur: RecurState,
+	pub h: ExecHandle<'a>,
+	pub named: HashMap<Sym, Vec<(&'a NamedMatcher, &'a Macro, &'a Rule)>>,
+	pub priod: Vec<(&'a PriodMatcher, &'a Macro, &'a Rule)>,
+}
+
+pub async fn resolve(ctx: &mut ResolveCtx<'_>, value: &MacTree) -> Option<MacTree> {
+	match value.tok() {
+		MacTok::Ph(_) | MacTok::Slot => panic!("Forbidden element in value mactree"),
+		MacTok::Bottom(_) | MacTok::Value(_) | MacTok::Name(_) => None,
+		MacTok::Lambda(arg, body) =>
+			Some(MacTok::Lambda(arg.clone(), resolve_seq(ctx, body).await?).at(value.pos())),
+		MacTok::S(ptyp, body) => Some(MacTok::S(*ptyp, resolve_seq(ctx, body).await?).at(value.pos())),
+	}
+}
+
+pub async fn resolve_seq(ctx: &mut ResolveCtx<'_>, val: &[MacTree]) -> Option<Vec<MacTree>> {
+	let mut any_changed = false;
+	let mut i = 0;
+	let mut val = val.to_vec();
+	'all_named: while i < val.len() {
+		'one_named: {
+			let MacTok::Name(key) = val[i].tok() else { break 'one_named };
+			let Some(options) = ctx.named.get(key) else { break 'one_named };
+			let matches = (options.iter())
+				.filter_map(|r| Some((r.1, r.2, r.0.apply(&val[i..], |_| false)?)))
+				.collect_vec();
+			match matches.len() {
+				0 => break 'one_named,
+				1 => {
+					any_changed = true;
+					let (mac, rule, (state, tail)) = matches.into_iter().exactly_one().unwrap();
+					let end = val.len() - tail.len();
+					let body_call = mk_body_call(mac, rule, &state, &ctx.ctx, ctx.recur.clone()).await;
+					std::mem::drop(state);
+					val.splice(i..end, [MacTok::Value(ctx.h.register(body_call).await).at(Pos::None)]);
+					i = end;
+				},
+				2.. => todo!("Named macros conflict!"),
+			}
+			continue 'all_named;
+		}
+		i += 1;
+	}
+	for (matcher, mac, rule) in &ctx.priod {
+		let Some(state) = matcher.apply(&val, |_| false) else { continue };
+		return Some(vec![
+			MacTok::Value(
+				ctx.h.register(mk_body_call(mac, rule, &state, &ctx.ctx, ctx.recur.clone()).await).await,
+			)
+			.at(Pos::None),
+		]);
+	}
+	for expr in val.iter_mut() {
+		if let Some(new) = resolve(ctx, expr).boxed_local().await {
+			*expr = new;
+			any_changed = true;
+		}
+	}
+	if any_changed { Some(val) } else { None }
+}
+
+async fn mk_body_call(
+	mac: &Macro,
+	rule: &Rule,
+	state: &MatchState<'_>,
+	ctx: &SysCtx,
+	recur: RecurState,
+) -> GExpr {
+	let rule_path =
+		RulePath { module: mac.0.module.clone(), main_kw: mac.0.own_kws[0].clone(), rule: rule.index };
+	let Some(new_recur) = recur.push(rule_path.clone()) else {
+		return bot(mk_errv(
+			ctx.i().i("Circular macro dependency").await,
+			format!("The definition of {rule_path} is circular"),
+			[rule.pos.clone()],
+		));
+	};
+	let mut call_args = vec![sym_ref(mac.0.module.suffix([rule.body_name.clone()], ctx.i()).await)];
+	for name in rule.placeholders.iter() {
+		call_args.push(match state.get(name).expect("Missing state entry for placeholder") {
+			StateEntry::Scalar(scal) => (**scal).clone().to_expr().await,
+			StateEntry::Vec(vec) => MacTok::S(Paren::Round, vec.to_vec()).at(Pos::None).to_expr().await,
+		});
+	}
+	call_args
+		.push(call([sym_ref(sym!(macros::resolve_recur; ctx.i()).await), new_recur.to_expr().await]));
+	call(call_args)
+}

orchid-std/src/macros/rule/any_match.rs

@@ -2,9 +2,9 @@ use orchid_base::name::Sym;
 
 use super::scal_match::scalv_match;
 use super::shared::AnyMatcher;
+use super::state::MatchState;
 use super::vec_match::vec_match;
 use crate::macros::MacTree;
-use crate::rule::state::MatchState;
 
 #[must_use]
 pub fn any_match<'a>(

orchid-std/src/macros/rule/build.rs

@@ -1,19 +1,22 @@
+use futures::FutureExt;
+use futures::future::join_all;
 use itertools::Itertools;
-use orchid_api::PhKind;
+use orchid_base::error::{OrcRes, mk_errv};
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::join_ok;
 use orchid_base::side::Side;
-use orchid_base::tree::Ph;
 
 use super::shared::{AnyMatcher, ScalMatcher, VecMatcher};
+use super::vec_attrs::vec_attrs;
+use crate::macros::mactree::{Ph, PhKind};
 use crate::macros::{MacTok, MacTree};
-use crate::rule::vec_attrs::vec_attrs;
 
 pub type MaxVecSplit<'a> = (&'a [MacTree], (Tok<String>, u8, bool), &'a [MacTree]);
 
 /// Derive the details of the central vectorial and the two sides from a
 /// slice of Expr's
 #[must_use]
-fn split_at_max_vec(pattern: &[MacTree]) -> Option<MaxVecSplit> {
+fn split_at_max_vec(pattern: &'_ [MacTree]) -> Option<MaxVecSplit<'_>> {
 	let rngidx = pattern
 		.iter()
 		.position_max_by_key(|expr| vec_attrs(expr).map(|attrs| attrs.1 as i64).unwrap_or(-1))?;
@@ -28,25 +31,29 @@ fn scal_cnt<'a>(iter: impl Iterator<Item = &'a MacTree>) -> usize {
 	iter.take_while(|expr| vec_attrs(expr).is_none()).count()
 }
 
-#[must_use]
+pub async fn mk_any(pattern: &[MacTree], i: &Interner) -> OrcRes<AnyMatcher> {
-pub fn mk_any(pattern: &[MacTree]) -> AnyMatcher {
 	let left_split = scal_cnt(pattern.iter());
 	if pattern.len() <= left_split {
-		return AnyMatcher::Scalar(mk_scalv(pattern));
+		return Ok(AnyMatcher::Scalar(mk_scalv(pattern, i).await?));
 	}
 	let (left, not_left) = pattern.split_at(left_split);
 	let right_split = not_left.len() - scal_cnt(pattern.iter().rev());
 	let (mid, right) = not_left.split_at(right_split);
-	AnyMatcher::Vec { left: mk_scalv(left), mid: mk_vec(mid), right: mk_scalv(right) }
+	join_ok! {
+		left = mk_scalv(left, i).await;
+		mid = mk_vec(mid, i).await;
+		right = mk_scalv(right, i).await;
+	}
+	Ok(AnyMatcher::Vec { left, mid, right })
 }
 
 /// Pattern MUST NOT contain vectorial placeholders
-#[must_use]
+async fn mk_scalv(pattern: &[MacTree], i: &Interner) -> OrcRes<Vec<ScalMatcher>> {
-fn mk_scalv(pattern: &[MacTree]) -> Vec<ScalMatcher> { pattern.iter().map(mk_scalar).collect() }
+	join_all(pattern.iter().map(|pat| mk_scalar(pat, i))).await.into_iter().collect()
+}
 
 /// Pattern MUST start and end with a vectorial placeholder
-#[must_use]
+pub async fn mk_vec(pattern: &[MacTree], i: &Interner) -> OrcRes<VecMatcher> {
-pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
 	debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
 	debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
 	debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
@@ -58,40 +65,56 @@ pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
 	let (l_side, l_sep) = left.split_at(left.len() - l_sep_size);
 	let main = VecMatcher::Placeh { key: key.clone(), nonzero };
 	match (left, right) {
-		(&[], &[]) => VecMatcher::Placeh { key, nonzero },
+		(&[], &[]) => Ok(VecMatcher::Placeh { key, nonzero }),
-		(&[], _) => VecMatcher::Scan {
+		(&[], _) => {
+			join_ok! {
+				sep = mk_scalv(r_sep, i).await;
+				right = mk_vec(r_side, i).boxed_local().await;
+			}
+			Ok(VecMatcher::Scan {
 				direction: Side::Left,
 				left: Box::new(main),
-			sep: mk_scalv(r_sep),
+				sep,
-			right: Box::new(mk_vec(r_side)),
+				right: Box::new(right),
+			})
 		},
-		(_, &[]) => VecMatcher::Scan {
+		(_, &[]) => {
+			join_ok! {
+				left = mk_vec(l_side, i).boxed_local().await;
+				sep = mk_scalv(l_sep, i).await;
+			}
+			Ok(VecMatcher::Scan {
 				direction: Side::Right,
-			left: Box::new(mk_vec(l_side)),
+				left: Box::new(left),
-			sep: mk_scalv(l_sep),
+				sep,
 				right: Box::new(main),
+			})
 		},
 		(..) => {
 			let mut key_order =
 				l_side.iter().chain(r_side.iter()).filter_map(vec_attrs).collect::<Vec<_>>();
 			key_order.sort_by_key(|(_, prio, _)| -(*prio as i64));
-			VecMatcher::Middle {
+			join_ok! {
-				left: Box::new(mk_vec(l_side)),
+				left = mk_vec(l_side, i).boxed_local().await;
-				left_sep: mk_scalv(l_sep),
+				left_sep = mk_scalv(l_sep, i).await;
-				mid: Box::new(main),
+				right_sep = mk_scalv(r_sep, i).await;
-				right_sep: mk_scalv(r_sep),
+				right = mk_vec(r_side, i).boxed_local().await;
-				right: Box::new(mk_vec(r_side)),
-				key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
 			}
+			Ok(VecMatcher::Middle {
+				left: Box::new(left),
+				left_sep,
+				mid: Box::new(main),
+				right_sep,
+				right: Box::new(right),
+				key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
+			})
 		},
 	}
 }
 
 /// Pattern MUST NOT be a vectorial placeholder
-#[must_use]
+async fn mk_scalar(pattern: &MacTree, i: &Interner) -> OrcRes<ScalMatcher> {
-fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
+	Ok(match &*pattern.tok {
-	match &*pattern.tok {
-		MacTok::Atom(_) | MacTok::Done(_) => panic!("Atoms and Done aren't supported in matchers"),
 		MacTok::Name(n) => ScalMatcher::Name(n.clone()),
 		MacTok::Ph(Ph { name, kind }) => match kind {
 			PhKind::Vector { .. } => {
@@ -99,34 +122,35 @@ fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
 			},
 			PhKind::Scalar => ScalMatcher::Placeh { key: name.clone() },
 		},
-		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
+		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body, i).boxed_local().await?)),
-		MacTok::Lambda(arg, body) => ScalMatcher::Lambda(Box::new(mk_any(arg)), Box::new(mk_any(body))),
+		MacTok::Lambda(..) =>
-		MacTok::Ref(_) | MacTok::Slot(_) => panic!("Extension-only variants"),
+			return Err(mk_errv(
-	}
+				i.i("Lambda in matcher").await,
+				"Lambdas can't be matched for, only generated in templates",
+				[pattern.pos()],
+			)),
+		MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
+		MacTok::Bottom(errv) => return Err(errv.clone()),
+	})
 }
 
 #[cfg(test)]
 mod test {
-	use std::rc::Rc;
-
-	use orchid_api::PhKind;
 	use orchid_base::interner::Interner;
-	use orchid_base::location::SourceRange;
+	use orchid_base::location::SrcRange;
 	use orchid_base::sym;
 	use orchid_base::tokens::Paren;
-	use orchid_base::tree::Ph;
 	use test_executors::spin_on;
 
 	use super::mk_any;
-	use crate::macros::{MacTok, MacTree};
+	use crate::macros::MacTok;
+	use crate::macros::mactree::{Ph, PhKind};
 
 	#[test]
 	fn test_scan() {
 		spin_on(async {
 			let i = Interner::new_master();
-			let ex = |tok: MacTok| async {
+			let ex = |tok: MacTok| async { tok.at(SrcRange::mock(&i).await.pos()) };
-				MacTree { tok: Rc::new(tok), pos: SourceRange::mock(&i).await.pos() }
-			};
 			let pattern = vec![
 				ex(MacTok::Ph(Ph {
 					kind: PhKind::Vector { priority: 0, at_least_one: false },
@@ -154,7 +178,7 @@ mod test {
 				}))
 				.await,
 			];
-			let matcher = mk_any(&pattern);
+			let matcher = mk_any(&pattern, &i).await.expect("This matcher isn't broken");
 			println!("{matcher}");
 		})
 	}

orchid-std/src/macros/rule/matcher.rs

@@ -1,60 +1,61 @@
 use std::fmt;
 
 use itertools::Itertools;
-use orchid_api::PhKind;
+use orchid_base::error::OrcRes;
-use orchid_base::interner::Interner;
+use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
-use orchid_base::tree::Ph;
 
 use super::any_match::any_match;
-use super::build::mk_any;
+use super::build::{mk_any, mk_vec};
 use super::shared::{AnyMatcher, VecMatcher};
 use super::state::{MatchState, StateEntry};
 use super::vec_attrs::vec_attrs;
 use super::vec_match::vec_match;
+use crate::macros::mactree::{Ph, PhKind};
 use crate::macros::{MacTok, MacTree};
-use crate::rule::build::mk_vec;
 
-pub fn first_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.first().unwrap()).is_some() }
+pub struct NamedMatcher {
-pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwrap()).is_some() }
+	inner: AnyMatcher,
-
+	head: Sym,
-pub struct NamedMatcher(AnyMatcher);
+	after_tok: Tok<String>,
+}
 impl NamedMatcher {
-	pub async fn new(pattern: &[MacTree], i: &Interner) -> Self {
+	pub async fn new(pattern: &[MacTree], i: &Interner) -> OrcRes<Self> {
-		assert!(
+		let head = match pattern.first().map(|tree| tree.tok()) {
-			matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))),
+			Some(MacTok::Name(name)) => name.clone(),
-			"Named matchers must begin with a name"
+			_ => panic!("Named matchers must begin with a name"),
-		);
+		};
-
+		let after_tok = i.i("::after").await;
-		match last_is_vec(pattern) {
+		let inner = match pattern.last().and_then(vec_attrs).is_some() {
-			true => Self(mk_any(pattern)),
+			true => mk_any(pattern, i).await?,
 			false => {
-				let kind: PhKind = PhKind::Vector { priority: 0, at_least_one: false };
+				let kind = PhKind::Vector { priority: 0, at_least_one: false };
-				let suffix = [MacTok::Ph(Ph { name: i.i("::after").await, kind }).at(Pos::None)];
+				let suffix = [MacTok::Ph(Ph { name: after_tok.clone(), kind }).at(Pos::None)];
-				Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
+				mk_any(&pattern.iter().cloned().chain(suffix).collect_vec(), i).await?
 			},
+		};
+		Ok(Self { after_tok, inner, head })
 	}
-	}
+	pub fn head(&self) -> Sym { self.head.clone() }
 	/// Also returns the tail, if any, which should be matched further
 	/// Note that due to how priod works below, the main usable information from
 	/// the tail is its length
-	pub async fn apply<'a>(
+	pub fn apply<'a>(
 		&self,
 		seq: &'a [MacTree],
-		i: &Interner,
 		save_loc: impl Fn(Sym) -> bool,
 	) -> Option<(MatchState<'a>, &'a [MacTree])> {
-		let mut state = any_match(&self.0, seq, &save_loc)?;
+		let mut state = any_match(&self.inner, seq, &save_loc)?;
-		match state.remove(i.i("::after").await) {
+		match state.remove(self.after_tok.clone()) {
-			Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"),
+			Some(StateEntry::Scalar(_)) => panic!("{} can never be a scalar entry!", self.after_tok),
 			Some(StateEntry::Vec(v)) => Some((state, v)),
 			None => Some((state, &[][..])),
 		}
 	}
 }
 impl fmt::Display for NamedMatcher {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.inner.fmt(f) }
 }
 impl fmt::Debug for NamedMatcher {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "NamedMatcher({self})") }
@@ -62,12 +63,12 @@ impl fmt::Debug for NamedMatcher {
 
 pub struct PriodMatcher(VecMatcher);
 impl PriodMatcher {
-	pub fn new(pattern: &[MacTree]) -> Self {
+	pub async fn new(pattern: &[MacTree], i: &Interner) -> OrcRes<Self> {
 		assert!(
 			pattern.first().and_then(vec_attrs).is_some() && pattern.last().and_then(vec_attrs).is_some(),
 			"Prioritized matchers must start and end with a vectorial",
 		);
-		Self(mk_vec(pattern))
+		Ok(Self(mk_vec(pattern, i).await?))
 	}
 	/// tokens before the offset always match the prefix
 	pub fn apply<'a>(

orchid-std/src/macros/rule/scal_match.rs

@@ -2,8 +2,8 @@ use orchid_base::name::Sym;
 
 use super::any_match::any_match;
 use super::shared::ScalMatcher;
+use super::state::{MatchState, StateEntry};
 use crate::macros::{MacTok, MacTree};
-use crate::rule::state::{MatchState, StateEntry};
 
 #[must_use]
 pub fn scal_match<'a>(
@@ -16,13 +16,10 @@ pub fn scal_match<'a>(
 			true => MatchState::from_name(n1.clone(), expr.pos.clone()),
 			false => MatchState::default(),
 		}),
-		(ScalMatcher::Placeh { .. }, MacTok::Done(_)) => None,
 		(ScalMatcher::Placeh { key }, _) =>
 			Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
 		(ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>
 			any_match(b_mat, &body[..], save_loc),
-		(ScalMatcher::Lambda(arg_mat, b_mat), MacTok::Lambda(arg, body)) =>
-			Some(any_match(arg_mat, arg, save_loc)?.combine(any_match(b_mat, body, save_loc)?)),
 		_ => None,
 	}
 }