Temporary commit to try fix halting

clean shutdown doesn't for some reason

.cargo/config.toml

@@ -1,9 +1,14 @@
 [alias]
 xtask = "run --quiet --package xtask --"
 orcx = "xtask orcx"
+orcxdb = "xtask orcxdb"
 
 [env]
 CARGO_WORKSPACE_DIR = { value = "", relative = true }
 ORCHID_EXTENSIONS = "target/debug/orchid-std"
-ORCHID_DEFAULT_SYSTEMS = "orchid::std"
+ORCHID_DEFAULT_SYSTEMS = "orchid::std;orchid::macros"
 ORCHID_LOG_BUFFERS = "true"
+RUST_BACKTRACE = "1"
+
+[build]
+# rustflags = ["-Znext-solver"]

Cargo.toml

@@ -10,5 +10,6 @@ members = [
   "orchid-api",
   "orchid-api-derive",
   "orchid-api-traits",
-  "stdio-perftest", "xtask",
+  "stdio-perftest",
+  "xtask", "async-fn-stream",
 ]

README.md

@@ -7,7 +7,7 @@ An experimental lazy, pure functional programming language designed to be embedd
 
 ## Usage
 
-The standalone interpreter can be built as the binary target from this package. The language tutorial and standard library documentation is at [www.lbfalvy.com/orchid-reference](https://www.lbfalvy.com/orchid-reference/). Embedder guide and Rust API documentation are coming soon.
+The standalone interpreter can be built as the binary target from this package. The language tutorial and standard library documentation is at [www.lbfalvy.com/orchid-reference](https://lbfalvy.github.io/orchid-reference/). Embedder guide and Rust API documentation are coming soon.
 
 ## Design
 
@@ -43,4 +43,4 @@ Identifying marks include the Orchid logo, the ribbon image above, and the names
 
 Contact information includes email addresses, links to the source code and issue tracker.
 
-Words listed as identifying marks are explicltly not considered as such when they apear in technical interfaces or APIs. For example, shell commands, identifiers within the language,
+Words listed as identifying marks are explicltly not considered as such when they appear in technical interfaces or APIs. For example, shell commands, identifiers within Orchid or Rust code, and names in package registries are not considered as identifying marks.

SWAP.md

@@ -1,16 +1,12 @@
-## Async conversion
+Decide whether we need patterns at runtime. Maybe macros aren't obligated to return MacTree so destructuring can be done in a safer and easier way?
 
-convert host to async non-send
+Double-check type and templating logic in the note, it's a bit fishy.
 
-demonstrate operation with existing lex-hello example
+Consider whether all macros need to be loaded or the const references could be used to pre-filter for a given let line.
-
-consider converting extension's SysCtx to a typed context bag
-
-align fn atom and macros on both sides with new design. No global state.
 
 ## alternate extension mechanism
 
-The Macro extension needs to be in the same compilation unit as the interpreter because the interpreter needs to proactively access its datastructures (in particular, it needs to generate MacTree from TokTree)
+The STD system will have a lot of traffic for trivial operations like algebra, stream IO will likely not be fast enough. A faster system is in order.
 
 Ideally, it should reuse `orchid-extension` for message routing and decoding.
 
@@ -18,10 +14,8 @@ Ideally, it should reuse `orchid-extension` for message routing and decoding.
 
 ## Preprocessor extension
 
-Must figure out how preprocessor can both be a System and referenced in the interpreter
+The macro system will not be privileged, it can take control from the interpreter via a custom top-level "let" line type.
 
 Must actually write macro system as recorded in note
 
-At this point swappable preprocessors aren't a target because interaction with module system sounds complicated
-
 Check if any of this needs interpreter, if so, start with that

async-fn-stream/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "async-fn-stream"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+futures = { version = "0.3.31", features = ["std"], default-features = false }
+
+[dev-dependencies]
+test_executors = "0.3.5"

async-fn-stream/src/lib.rs

@@ -0,0 +1,207 @@
+use std::cell::Cell;
+use std::future::poll_fn;
+use std::marker::PhantomData;
+use std::pin::Pin;
+use std::ptr;
+use std::task::{Context, Poll};
+
+use futures::future::LocalBoxFuture;
+use futures::{FutureExt, Stream};
+
+type YieldSlot<'a, T> = &'a Cell<Option<T>>;
+
+/// Handle that allows you to emit values on a stream. If you drop
+/// this, the stream will end and you will not be polled again.
+pub struct StreamCtx<'a, T>(&'a Cell<Option<T>>, PhantomData<&'a ()>);
+impl<T> StreamCtx<'_, T> {
+	pub fn emit(&mut self, value: T) -> impl Future<Output = ()> {
+		assert!(self.0.replace(Some(value)).is_none(), "Leftover value in stream");
+		let mut state = Poll::Pending;
+		poll_fn(move |_| std::mem::replace(&mut state, Poll::Ready(())))
+	}
+}
+
+enum FnOrFut<'a, T, O> {
+	Fn(Option<Box<dyn FnOnce(YieldSlot<'a, T>) -> LocalBoxFuture<'a, O> + 'a>>),
+	Fut(LocalBoxFuture<'a, O>),
+}
+
+struct AsyncFnStream<'a, T> {
+	driver: FnOrFut<'a, T, ()>,
+	output: Cell<Option<T>>,
+}
+impl<'a, T> Stream for AsyncFnStream<'a, T> {
+	type Item = T;
+	fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		unsafe {
+			let self_mut = self.get_unchecked_mut();
+			let fut = match &mut self_mut.driver {
+				FnOrFut::Fut(fut) => fut,
+				FnOrFut::Fn(f) => {
+					// safety: the cell is held inline in self, which is pinned.
+					let cell = ptr::from_ref(&self_mut.output).as_ref().unwrap();
+					let fut = f.take().unwrap()(cell);
+					self_mut.driver = FnOrFut::Fut(fut);
+					return Pin::new_unchecked(self_mut).poll_next(cx);
+				},
+			};
+			match fut.as_mut().poll(cx) {
+				Poll::Ready(()) => Poll::Ready(None),
+				Poll::Pending => match self_mut.output.replace(None) {
+					None => Poll::Pending,
+					Some(t) => Poll::Ready(Some(t)),
+				},
+			}
+		}
+	}
+}
+
+struct AsyncFnTryStream<'a, T, E> {
+	driver: FnOrFut<'a, T, Result<StreamCtx<'a, T>, E>>,
+	output: Cell<Option<T>>,
+}
+impl<'a, T, E> Stream for AsyncFnTryStream<'a, T, E> {
+	type Item = Result<T, E>;
+	fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+		unsafe {
+			let self_mut = self.get_unchecked_mut();
+			let fut = match &mut self_mut.driver {
+				FnOrFut::Fut(fut) => fut,
+				FnOrFut::Fn(f) => {
+					// safety: the cell is held inline in self, which is pinned.
+					let cell = ptr::from_ref(&self_mut.output).as_ref().unwrap();
+					let fut = f.take().unwrap()(cell);
+					self_mut.driver = FnOrFut::Fut(fut);
+					return Pin::new_unchecked(self_mut).poll_next(cx);
+				},
+			};
+			match fut.as_mut().poll(cx) {
+				Poll::Ready(Ok(_)) => Poll::Ready(None),
+				Poll::Ready(Err(ex)) => Poll::Ready(Some(Err(ex))),
+				Poll::Pending => match self_mut.output.replace(None) {
+					None => Poll::Pending,
+					Some(t) => Poll::Ready(Some(Ok(t))),
+				},
+			}
+		}
+	}
+}
+
+/// Create a stream from an async function acting as a coroutine
+pub fn stream<'a, T: 'a>(
+	f: impl for<'b> AsyncFnOnce(StreamCtx<'b, T>) + 'a,
+) -> impl Stream<Item = T> + 'a {
+	AsyncFnStream {
+		output: Cell::new(None),
+		driver: FnOrFut::Fn(Some(Box::new(|t| {
+			async { f(StreamCtx(t, PhantomData)).await }.boxed_local()
+		}))),
+	}
+}
+
+/// Create a stream of result from a fallible function.
+pub fn try_stream<'a, T: 'a, E: 'a>(
+	f: impl for<'b> AsyncFnOnce(StreamCtx<'b, T>) -> Result<StreamCtx<'b, T>, E> + 'a,
+) -> impl Stream<Item = Result<T, E>> + 'a {
+	AsyncFnTryStream {
+		output: Cell::new(None),
+		driver: FnOrFut::Fn(Some(Box::new(|t| {
+			async { f(StreamCtx(t, PhantomData)).await }.boxed_local()
+		}))),
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use std::task::Poll;
+	use std::{future, pin};
+
+	use futures::channel::mpsc::channel;
+	use futures::{Stream, StreamExt, TryStreamExt};
+	use test_executors::spin_on;
+
+	use crate::{stream, try_stream};
+
+	#[test]
+	fn sync() {
+		spin_on(async {
+			let v = stream(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+			})
+			.collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, [0, 1, 2, 3, 4])
+		})
+	}
+
+	#[test]
+	/// The exact behaviour of the poll function under blocked use
+	fn with_delay() {
+		spin_on(async {
+			let (mut send, mut recv) = channel(0);
+			let mut s = pin::pin!(stream(async |mut cx| {
+				for i in 0..2 {
+					cx.emit(i).await
+				}
+				recv.next().await;
+				for i in 2..5 {
+					cx.emit(i).await
+				}
+			}));
+			let mut log = String::new();
+			let log = future::poll_fn(|cx| {
+				match s.as_mut().poll_next(cx) {
+					Poll::Ready(Some(r)) => log += &format!("Found {r}\n"),
+					Poll::Ready(None) => return Poll::Ready(format!("{log}Ended")),
+					Poll::Pending => match send.try_send(()) {
+						Ok(()) => log += "Unblocked\n",
+						Err(err) => return Poll::Ready(format!("{log}Unblock err: {err}")),
+					},
+				}
+				Poll::Pending
+			})
+			.await;
+			const EXPECTED: &str = "\
+        Found 0\n\
+        Found 1\n\
+        Unblocked\n\
+        Found 2\n\
+        Found 3\n\
+        Found 4\n\
+        Ended";
+			assert_eq!(log, EXPECTED)
+		})
+	}
+
+	#[test]
+	fn sync_try_all_ok() {
+		spin_on(async {
+			let v = try_stream::<_, ()>(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+				Ok(cx)
+			})
+			.try_collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, Ok(vec![0, 1, 2, 3, 4]))
+		})
+	}
+
+	#[test]
+	fn sync_try_err() {
+		spin_on(async {
+			let v = try_stream::<_, ()>(async |mut cx| {
+				for i in 0..5 {
+					cx.emit(i).await
+				}
+				Err(())
+			})
+			.try_collect::<Vec<_>>()
+			.await;
+			assert_eq!(v, Err(()))
+		})
+	}
+}

examples/hello-world/main.orc

@@ -1,2 +1,2 @@
-const user = "dave"
+let user = "dave"
-const main = println "Hello $user!" exit_status::success
+let main = println "Hello $user!" exit_status::success

notes/expr_refs.md

@@ -6,13 +6,13 @@ Reference loops are resource leaks. There are two primary ways to avoid referenc
 - Constants reference their constituent Expressions
 - Expressions reference Atoms
 - During evaluation, Constants replace their unbound names with Constants
-  - There is a reference cycle here, but it always goes through a Constant.
+	- There is a reference cycle here, but it always goes through a Constant.
-    > **todo** A potential fix may be to update all Constants to point to a dummy value before freeing Trees
+		> **todo** A potential fix may be to update all Constants to point to a dummy value before freeing Trees
 - Atoms reference the Systems that implement them
 - Atoms may reference Expressions that are not younger than them
-  - This link is managed by the System but tied to Atom and not System lifecycle
+	- This link is managed by the System but tied to Atom and not System lifecycle
-  - Atoms can technically be applied to themselves, but it's a copying apply so it probably isn't a risk factor
+	- Atoms can technically be applied to themselves, but it's a copying apply so it probably isn't a risk factor
 - Systems reference the Extension that contains them
 - Extensions reference the Port that connects them
-  - The Extension signals the remote peer to disconnect on drop
+	- The Extension signals the remote peer to disconnect on drop
-  - The port is also referenced in a loose receiver thread, which always eventually tries to find the Extension or polls for ingress so it always eventually exits after the Extension's drop handler is called
+	- The port is also referenced in a loose receiver thread, which always eventually tries to find the Extension or polls for ingress so it always eventually exits after the Extension's drop handler is called

notes/macros.md

@@ -17,11 +17,11 @@ Priority numbers are written in hexadecimal normal form to avoid precision bugs,
 - **32-39**: Binary operators, in inverse priority order
 - **80-87**: Expression-like structures such as if/then/else
 - **128-135**: Anything that creates lambdas
-  Programs triggered by a lower priority pattern than this can assume that all names are correctly bound
+	Programs triggered by a lower priority pattern than this can assume that all names are correctly bound
 - **200**: Aliases extracted for readability
-  The user-accessible entry points of all macro programs must be lower priority than this, so any arbitrary syntax can be extracted into an alias with no side effects
+	The user-accessible entry points of all macro programs must be lower priority than this, so any arbitrary syntax can be extracted into an alias with no side effects
 - **224-231**: Integration; documented hooks exposed by a macro package to allow third party packages to extend its functionality
-  The `statement` pattern produced by `do{}` blocks and matched by `let` and `cps` is a good example of this. When any of these are triggered, all macro programs are in a documented state.
+	The `statement` pattern produced by `do{}` blocks and matched by `let` and `cps` is a good example of this. When any of these are triggered, all macro programs are in a documented state.
 - **248-255**: Transitional states within macro programs get the highest priority
 
 The numbers are arbitrary and up for debate. These are just the ones I came up with when writing the examples.

notes/new_macro_model.md

@@ -42,11 +42,9 @@ Prioritised macro patterns must start and end with a vectorial placeholder. They
 Macros are checked from the outermost block inwards.
 
 1. For every name token, test all named macros starting with that name
-   1. Take the first rule that matches in each block
+	 1. If the tail is implicit, continue iterating
-   2. If there are multiple matches across blocks, raise an ambiguity error
-   3. If the tail is implicit, recurse on it
 2. Test all prioritized macros
-   1. Take the first rule that matches in the highest prioritized block
+	 1. Take the first rule that matches in the highest prioritized block
 
 Test all in a set of macros
 1. Take the first rule that matches in each block
@@ -77,25 +75,26 @@ Recursion has to happen through the interpreter itself, so the macro system is d
 - line parser `macro` parses a macro with the existing logic
 - atom `MacRecurState` holds the recursion state
 - function `resolve_recur` finds all matches on a MacTree
-  - type: `MacRecurState -> MacTree -> MacTree`
+	- type: `MacRecurState -> MacTree -> MacTree`
-  - use all macros to find all matches in the tree
+	- use all relevant macros to find all matches in the tree
-  - for each match
+		- since macros must contain a locally defined token, it can be assumed that at the point that a constant is evaluated and all imports in the parent module have been resolved, necessarily all relevant macro rules must have been loaded
-    - check for recursion violations
+	- for each match
-    - wrap the body in iife-s corresponding to the named values in the match state
+		- check for recursion violations
-    - emit a recursive call to process and run the body, and pass the same recursive call as argument for the macro to use
+		- wrap the body in iife-s corresponding to the named values in the match state
+		- emit a recursive call to process and run the body, and pass the same recursive call as argument for the macro to use
 		```
 		(\recur. lower (recur $body) recur)
 			(resolve_recur $mac_recur_state)
 		```
-  - emit a single call to `instantiate_tpl` which receives all of these
+	- emit a single call to `instantiate_tpl` which receives all of these
 - function `instantiate_tpl` inserts `MacTree` values into a `MacTree(tpl)`
-  - type: `MacTree(tpl) [-> MacTree] -> MacTree`  
+	- type: `MacTree(tpl) [-> MacTree] -> MacTree`  
 		_this function deduces the number of arguments from the first argument. This combines poorly with autocurry, but it's an easy way to avoid representing standalone tree lists_
-  - walks the tree to find max template slot number, reads and type checks as many template values
+	- walks the tree to find max template slot number, reads and type checks as many template values
-  - returns the populated tree
+	- returns the populated tree
 - function `resolve` is the main entry point of the code
-  - type: `MacTree -> MacTree`
+	- type: `MacTree -> MacTree`
-  - invokes `resolve_recur` with an empty `MacRecurState`
+	- invokes `resolve_recur` with an empty `MacRecurState`
 - function `lower` is the main exit point of the code
-  - type: `MacTree -> any`
+	- type: `MacTree -> any`
-  - Lowers `MacTree` into the equivalent `Expr`.
+	- Lowers `MacTree` into the equivalent `Expr`.

orchid-api-derive/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "orchid-api-derive"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 [lib]
 proc-macro = true
@@ -9,9 +9,8 @@ proc-macro = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-quote = "1.0.38"
+quote = "1.0.40"
-syn = { version = "2.0.95" }
+syn = { version = "2.0.106" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-proc-macro2 = "1.0.92"
+proc-macro2 = "1.0.101"
-darling = "0.20.10"
 itertools = "0.14.0"

orchid-api-derive/src/decode.rs

@@ -12,7 +12,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
 	let decode = decode_body(&input.data);
 	let expanded = quote! {
 		impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause {
-			async fn decode<R: orchid_api_traits::async_std::io::Read + ?Sized>(
+			async fn decode<R: orchid_api_traits::AsyncRead + ?Sized>(
 				mut read: std::pin::Pin<&mut R>
 			) -> Self {
 				#decode

orchid-api-derive/src/encode.rs

@@ -14,7 +14,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
 	let encode = encode_body(&input.data);
 	let expanded = quote! {
 		impl #e_impl_generics orchid_api_traits::Encode for #name #e_ty_generics #e_where_clause {
-			async fn encode<W: orchid_api_traits::async_std::io::Write + ?Sized>(
+			async fn encode<W: orchid_api_traits::AsyncWrite + ?Sized>(
 				&self,
 				mut write: std::pin::Pin<&mut W>
 			) {

orchid-api-traits/Cargo.toml

@@ -1,14 +1,13 @@
 [package]
 name = "orchid-api-traits"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-std = "1.13.0"
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
-async-stream = "0.3.6"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-futures = "0.3.31"
 itertools = "0.14.0"
 never = "0.1.0"
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"

orchid-api-traits/src/coding.rs

@@ -7,10 +7,8 @@ use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use async_fn_stream::stream;
-use async_stream::stream;
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, StreamExt};
-use futures::future::LocalBoxFuture;
-use futures::{FutureExt, StreamExt};
 use never::Never;
 use ordered_float::NotNan;
 
@@ -19,20 +17,17 @@ use crate::encode_enum;
 pub trait Decode: 'static {
 	/// Decode an instance from the beginning of the buffer. Return the decoded
 	/// data and the remaining buffer.
-	fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
+	fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
 }
 pub trait Encode {
 	/// Append an instance of the struct to the buffer
-	fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
+	fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
 }
 pub trait Coding: Encode + Decode + Clone {
 	fn get_decoder<T: 'static, F: Future<Output = T> + 'static>(
 		map: impl Fn(Self) -> F + Clone + 'static,
-	) -> impl for<'a> Fn(Pin<&'a mut dyn Read>) -> LocalBoxFuture<'a, T> {
+	) -> impl AsyncFn(Pin<&mut dyn AsyncRead>) -> T {
-		move |r| {
+		async move |r| map(Self::decode(r).await).await
-			let map = map.clone();
-			async move { map(Self::decode(r).await).await }.boxed_local()
-		}
 	}
 }
 impl<T: Encode + Decode + Clone> Coding for T {}
@@ -40,14 +35,14 @@ impl<T: Encode + Decode + Clone> Coding for T {}
 macro_rules! num_impl {
 	($number:ty) => {
 		impl Decode for $number {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; (<$number>::BITS / 8) as usize];
 				read.read_exact(&mut bytes).await.unwrap();
 				<$number>::from_be_bytes(bytes)
 			}
 		}
 		impl Encode for $number {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 				write.write_all(&self.to_be_bytes()).await.expect("Could not write number")
 			}
 		}
@@ -67,12 +62,12 @@ num_impl!(i8);
 macro_rules! nonzero_impl {
 	($name:ty) => {
 		impl Decode for NonZero<$name> {
-			async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 				Self::new(<$name as Decode>::decode(read).await).unwrap()
 			}
 		}
 		impl Encode for NonZero<$name> {
-			async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 				self.get().encode(write).await
 			}
 		}
@@ -91,19 +86,21 @@ nonzero_impl!(i64);
 nonzero_impl!(i128);
 
 impl<T: Encode + ?Sized> Encode for &T {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) { (**self).encode(write).await }
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+		(**self).encode(write).await
+	}
 }
 macro_rules! float_impl {
 	($t:ty, $size:expr) => {
 		impl Decode for NotNan<$t> {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; $size];
 				read.read_exact(&mut bytes).await.unwrap();
 				NotNan::new(<$t>::from_be_bytes(bytes)).expect("Float was NaN")
 			}
 		}
 		impl Encode for NotNan<$t> {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 				write.write_all(&self.as_ref().to_be_bytes()).await.expect("Could not write number")
 			}
 		}
@@ -114,7 +111,7 @@ float_impl!(f64, 8);
 float_impl!(f32, 4);
 
 impl Decode for String {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
 		let mut data = vec![0u8; len];
 		read.read_exact(&mut data).await.unwrap();
@@ -122,30 +119,36 @@ impl Decode for String {
 	}
 }
 impl Encode for String {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl Encode for str {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl<T: Decode> Decode for Vec<T> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		stream! { loop { yield T::decode(read.as_mut()).await } }.take(len).collect().await
+		stream(async |mut cx| {
+			for _ in 0..len {
+				cx.emit(T::decode(read.as_mut()).await).await
+			}
+		})
+		.collect()
+		.await
 	}
 }
 impl<T: Encode> Encode for Vec<T> {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		self.as_slice().encode(write).await
 	}
 }
 impl<T: Encode> Encode for [T] {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		for t in self.iter() {
 			t.encode(write.as_mut()).await
@@ -153,7 +156,7 @@ impl<T: Encode> Encode for [T] {
 	}
 }
 impl<T: Decode> Decode for Option<T> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		match u8::decode(read.as_mut()).await {
 			0 => None,
 			1 => Some(T::decode(read).await),
@@ -162,14 +165,14 @@ impl<T: Decode> Decode for Option<T> {
 	}
 }
 impl<T: Encode> Encode for Option<T> {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		let t = if let Some(t) = self { t } else { return 0u8.encode(write.as_mut()).await };
 		1u8.encode(write.as_mut()).await;
 		t.encode(write).await;
 	}
 }
 impl<T: Decode, E: Decode> Decode for Result<T, E> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		match u8::decode(read.as_mut()).await {
 			0 => Self::Ok(T::decode(read).await),
 			1 => Self::Err(E::decode(read).await),
@@ -179,7 +182,7 @@ impl<T: Decode, E: Decode> Decode for Result<T, E> {
 }
 
 impl<T: Encode, E: Encode> Encode for Result<T, E> {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		match self {
 			Ok(t) => encode_enum(write, 0, |w| t.encode(w)).await,
 			Err(e) => encode_enum(write, 1, |w| e.encode(w)).await,
@@ -187,13 +190,19 @@ impl<T: Encode, E: Encode> Encode for Result<T, E> {
 	}
 }
 impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		stream! { loop { yield <(K, V)>::decode(read.as_mut()).await } }.take(len).collect().await
+		stream(async |mut cx| {
+			for _ in 0..len {
+				cx.emit(<(K, V)>::decode(read.as_mut()).await).await
+			}
+		})
+		.collect()
+		.await
 	}
 }
 impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
 		for pair in self.iter() {
 			pair.encode(write.as_mut()).await
@@ -203,12 +212,12 @@ impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
 macro_rules! tuple {
   (($($t:ident)*) ($($T:ident)*)) => {
     impl<$($T: Decode),*> Decode for ($($T,)*) {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				($($T::decode(read.as_mut()).await,)*)
 			}
     }
     impl<$($T: Encode),*> Encode for ($($T,)*) {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
         let ($($t,)*) = self;
         $( $t.encode(write.as_mut()).await; )*
       }
@@ -234,41 +243,45 @@ tuple!((t u v x y z a b c d e f g h i) (T U V X Y Z A B C D E F G H I));
 tuple!((t u v x y z a b c d e f g h i j) (T U V X Y Z A B C D E F G H I J)); // 16
 
 impl Decode for () {
-	async fn decode<R: Read + ?Sized>(_: Pin<&mut R>) -> Self {}
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {}
 }
 impl Encode for () {
-	async fn encode<W: Write + ?Sized>(&self, _: Pin<&mut W>) {}
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) {}
 }
 impl Decode for Never {
-	async fn decode<R: Read + ?Sized>(_: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {
 		unreachable!("A value of Never cannot exist so it can't have been serialized");
 	}
 }
 impl Encode for Never {
-	async fn encode<W: Write + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
 }
 impl Decode for bool {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let mut buf = [0];
 		read.read_exact(&mut buf).await.unwrap();
 		buf[0] != 0
 	}
 }
 impl Encode for bool {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await.unwrap()
 	}
 }
 impl<T: Decode, const N: usize> Decode for [T; N] {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		// TODO: figure out how to do this in safe rust on the stack
+		let v = stream(async |mut cx| {
-		let v =
+			for _ in 0..N {
-			stream! { loop { yield T::decode(read.as_mut()).await } }.take(N).collect::<Vec<_>>().await;
+				cx.emit(T::decode(read.as_mut()).await).await
+			}
+		})
+		.collect::<Vec<_>>()
+		.await;
 		v.try_into().unwrap_or_else(|_| unreachable!("The length of this stream is statically known"))
 	}
 }
 impl<T: Encode, const N: usize> Encode for [T; N] {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		for t in self.iter() {
 			t.encode(write.as_mut()).await
 		}
@@ -278,12 +291,12 @@ impl<T: Encode, const N: usize> Encode for [T; N] {
 macro_rules! two_end_range {
   ($this:ident, $name:tt, $op:tt, $start:expr, $end:expr) => {
     impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				T::decode(read.as_mut()).await $op T::decode(read).await
 			}
     }
     impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
         let $this = self;
         ($start).encode(write.as_mut()).await;
         ($end).encode(write).await;
@@ -298,12 +311,14 @@ two_end_range!(x, RangeInclusive, ..=, x.start(), x.end());
 macro_rules! smart_ptr {
 	($name:tt) => {
 		impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 				$name::new(T::decode(read).await)
 			}
 		}
 		impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) { (**self).encode(write).await }
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+				(**self).encode(write).await
+			}
 		}
 	};
 }
@@ -313,12 +328,12 @@ smart_ptr!(Rc);
 smart_ptr!(Box);
 
 impl Decode for char {
-	async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
 		char::from_u32(u32::decode(read).await).unwrap()
 	}
 }
 impl Encode for char {
-	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
 		(*self as u32).encode(write).await
 	}
 }

orchid-api-traits/src/helpers.rs

@@ -1,12 +1,12 @@
 use std::future::Future;
 use std::pin::Pin;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use itertools::{Chunk, Itertools};
 
 use crate::Encode;
 
-pub async fn encode_enum<'a, W: Write + ?Sized, F: Future<Output = ()>>(
+pub async fn encode_enum<'a, W: AsyncWrite + ?Sized, F: Future<Output = ()>>(
 	mut write: Pin<&'a mut W>,
 	id: u8,
 	f: impl FnOnce(Pin<&'a mut W>) -> F,
@@ -15,7 +15,7 @@ pub async fn encode_enum<'a, W: Write + ?Sized, F: Future<Output = ()>>(
 	f(write).await
 }
 
-pub async fn write_exact<W: Write + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
+pub async fn write_exact<W: AsyncWrite + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
 	write.write_all(bytes).await.expect("Failed to write exact bytes")
 }
 
@@ -27,7 +27,7 @@ pub fn print_bytes(b: &[u8]) -> String {
 		.join("  ")
 }
 
-pub async fn read_exact<R: Read + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
+pub async fn read_exact<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
 	let mut data = vec![0u8; bytes.len()];
 	read.read_exact(&mut data).await.expect("Failed to read bytes");
 	if data != bytes {

orchid-api-traits/src/lib.rs

@@ -3,8 +3,8 @@ mod helpers;
 mod hierarchy;
 mod relations;
 
-pub use async_std;
 pub use coding::*;
+pub use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 pub use helpers::*;
 pub use hierarchy::*;
 pub use relations::*;

orchid-api/Cargo.toml

@@ -1,15 +1,15 @@
 [package]
 name = "orchid-api"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
-async-std = "1.13.0"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
 
 [dev-dependencies]
 test_executors = "0.3.2"

orchid-api/src/atom.rs

@@ -5,7 +5,7 @@ use orchid_api_traits::Request;
 
 use crate::{
 	ExprTicket, Expression, ExtHostReq, FormattingUnit, HostExtNotif, HostExtReq, OrcResult, SysId,
-	TStrv,
+	TVec,
 };
 
 pub type AtomData = Vec<u8>;
@@ -83,14 +83,14 @@ impl Request for DeserAtom {
 /// A request blindly routed to the system that provides an atom.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
-pub struct Fwded(pub Atom, pub TStrv, pub Vec<u8>);
+pub struct Fwded(pub Atom, pub TVec, pub Vec<u8>);
 impl Request for Fwded {
 	type Response = Option<Vec<u8>>;
 }
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
-pub struct Fwd(pub Atom, pub TStrv, pub Vec<u8>);
+pub struct Fwd(pub Atom, pub TVec, pub Vec<u8>);
 impl Request for Fwd {
 	type Response = Option<Vec<u8>>;
 }

orchid-api/src/expr.rs

@@ -3,7 +3,7 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{Atom, ExtHostNotif, ExtHostReq, Location, OrcError, SysId, TStrv};
+use crate::{Atom, ExtHostNotif, ExtHostReq, Location, OrcError, SysId, TVec};
 
 /// An arbitrary ID associated with an expression on the host side. Incoming
 /// tickets always come with some lifetime guarantee, which can be extended with
@@ -72,7 +72,7 @@ pub enum ExpressionKind {
 	/// Because the atom is newly constructed, it also must belong to this system.
 	NewAtom(Atom),
 	/// A reference to a constant
-	Const(TStrv),
+	Const(TVec),
 	/// A static runtime error.
 	Bottom(Vec<OrcError>),
 }

orchid-api/src/interner.rs

@@ -47,7 +47,7 @@ impl Request for ExternStr {
 #[extends(IntReq, ExtHostReq)]
 pub struct InternStrv(pub Vec<TStr>);
 impl Request for InternStrv {
-	type Response = TStrv;
+	type Response = TVec;
 }
 /// replica -> master to find the vector of interned strings corresponding to a
 /// token
@@ -57,7 +57,7 @@ impl Request for InternStrv {
 /// See [IntReq]
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(IntReq, ExtHostReq)]
-pub struct ExternStrv(pub TStrv);
+pub struct ExternStrv(pub TVec);
 impl Request for ExternStrv {
 	type Response = Vec<TStr>;
 }
@@ -68,7 +68,7 @@ pub struct TStr(pub NonZeroU64);
 
 /// A substitute for an interned string sequence in serialized datastructures.
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-pub struct TStrv(pub NonZeroU64);
+pub struct TVec(pub NonZeroU64);
 
 /// A request to sweep the replica. The master will not be sweeped until all
 /// replicas respond, as it must retain everything the replicas retained
@@ -84,5 +84,5 @@ impl Request for Sweep {
 #[derive(Clone, Debug, Coding)]
 pub struct Retained {
 	pub strings: Vec<TStr>,
-	pub vecs: Vec<TStrv>,
+	pub vecs: Vec<TVec>,
 }

orchid-api/src/lexer.rs

@@ -3,7 +3,7 @@ use std::ops::RangeInclusive;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TokenTree, TreeTicket};
+use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TVec, TokenTree};
 
 /// - All ranges contain at least one character
 /// - All ranges are in increasing characeter order
@@ -18,6 +18,8 @@ pub struct LexExpr {
 	pub id: ParsId,
 	pub text: TStr,
 	pub pos: u32,
+	/// Source root module path
+	pub src: TVec,
 }
 impl Request for LexExpr {
 	type Response = Option<OrcResult<LexedExpr>>;
@@ -42,5 +44,5 @@ impl Request for SubLex {
 #[derive(Clone, Debug, Coding)]
 pub struct SubLexed {
 	pub pos: u32,
-	pub ticket: TreeTicket,
+	pub tree: TokenTree,
 }

orchid-api/src/lib.rs

@@ -2,8 +2,6 @@ mod lexer;
 pub use lexer::*;
 mod format;
 pub use format::*;
-mod macros;
-pub use macros::*;
 mod atom;
 pub use atom::*;
 mod error;
@@ -24,5 +22,3 @@ mod system;
 pub use system::*;
 mod tree;
 pub use tree::*;
-mod vfs;
-pub use vfs::*;

orchid-api/src/location.rs

@@ -2,7 +2,7 @@ use std::ops::Range;
 
 use orchid_api_derive::Coding;
 
-use crate::{TStr, TStrv};
+use crate::{TStr, TVec};
 
 #[derive(Clone, Debug, Coding)]
 pub enum Location {
@@ -17,18 +17,16 @@ pub enum Location {
 	Gen(CodeGenInfo),
 	/// Range and file
 	SourceRange(SourceRange),
-	/// Range only, file implied. Most notably used by parsers
-	Range(Range<u32>),
 }
 
 #[derive(Clone, Debug, Coding)]
 pub struct SourceRange {
-	pub path: TStrv,
+	pub path: TVec,
 	pub range: Range<u32>,
 }
 
 #[derive(Clone, Debug, Coding)]
 pub struct CodeGenInfo {
-	pub generator: TStrv,
+	pub generator: TVec,
 	pub details: TStr,
 }

orchid-api/src/macros.rs

@@ -1,86 +0,0 @@
-use std::collections::HashMap;
-use std::num::NonZeroU64;
-
-use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::Request;
-use ordered_float::NotNan;
-
-use crate::{
-	Atom, Comment, ExtHostReq, HostExtReq, Location, OrcResult, Paren, ParsId, SysId, TStr, TStrv,
-};
-
-#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct MacroTreeId(pub NonZeroU64);
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroTree {
-	pub location: Location,
-	pub token: MacroToken,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub enum MacroToken {
-	S(Paren, Vec<MacroTree>),
-	Name(TStrv),
-	Slot(MacroTreeId),
-	Lambda(Vec<MacroTree>, Vec<MacroTree>),
-	Ph(Placeholder),
-	Atom(Atom),
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroBlock {
-	pub priority: Option<NotNan<f64>>,
-	pub rules: Vec<MacroRule>,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroRule {
-	pub location: Location,
-	pub comments: Vec<Comment>,
-	pub pattern: Vec<MacroTree>,
-	pub id: MacroId,
-}
-
-/// A specific macro rule with a specific pattern across invocations
-#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, Hash)]
-pub struct MacroId(pub NonZeroU64);
-
-/// After a pattern matches, this call executes the body of the macro. This
-/// request returns None if an inner nested request raised an exception
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtReq)]
-pub struct ApplyMacro {
-	pub sys: SysId,
-	pub id: MacroId,
-	/// Recursion token
-	pub run_id: ParsId,
-	/// Must contain exactly the keys that were specified as placeholders in the
-	/// pattern
-	pub params: HashMap<TStr, Vec<MacroTree>>,
-}
-impl Request for ApplyMacro {
-	type Response = Option<OrcResult<Vec<MacroTree>>>;
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostReq)]
-pub struct RunMacros {
-	pub run_id: ParsId,
-	pub query: Vec<MacroTree>,
-}
-impl Request for RunMacros {
-	type Response = Option<Vec<MacroTree>>;
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct Placeholder {
-	pub name: TStr,
-	pub kind: PhKind,
-}
-
-#[derive(Clone, Copy, Debug, Coding)]
-pub enum PhKind {
-	Scalar,
-	Vector { priority: u8, at_least_one: bool },
-}

orchid-api/src/parser.rs

@@ -1,28 +1,105 @@
+use core::ops::Range;
 use std::num::NonZeroU64;
 
-use orchid_api_derive::{Coding, Decode, Encode, Hierarchy};
+use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{Comment, HostExtReq, OrcResult, SysId, TokenTree};
+use crate::{
+	Expression, ExtHostReq, HostExtReq, OrcResult, SourceRange, SysId, TStr, TVec, TokenTree,
+};
 
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ParsId(pub NonZeroU64);
 
-// impl orchid_api_traits::Decode for ParsId {
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-// 	async fn decode<R: async_std::io::Read + ?Sized>(mut read:
+pub struct ParsedConstId(pub NonZeroU64);
-// std::pin::Pin<&mut R>) -> Self {
-// 		Self(orchid_api_traits::Decode::decode(read.as_mut()).await)
-// 	}
-// }
 
+/// Parse a single source line. Return values can be modules, constants, or
+/// token sequences for re-parsing. These re-parsed token sequences can also
+/// represent raw language items such as modules, imports, and const. This is
+/// how we enable generating imports without forcing import syntax to affect API
+/// versioning
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct ParseLine {
 	pub sys: SysId,
+	/// The immediately enclosing module path
+	pub module: TVec,
+	/// The root module path for the snipppet of source code, prefix of
+	/// [ParseLine#module]
+	pub src: TVec,
 	pub comments: Vec<Comment>,
 	pub exported: bool,
+	pub idx: u16,
 	pub line: Vec<TokenTree>,
 }
 impl Request for ParseLine {
-	type Response = OrcResult<Vec<TokenTree>>;
+	type Response = OrcResult<Vec<ParsedLine>>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct ParsedLine {
+	pub comments: Vec<Comment>,
+	pub source_range: SourceRange,
+	pub kind: ParsedLineKind,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum ParsedLineKind {
+	Recursive(Vec<TokenTree>),
+	Member(ParsedMember),
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct ParsedMember {
+	pub name: TStr,
+	pub exported: bool,
+	pub kind: ParsedMemberKind,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum ParsedMemberKind {
+	Constant(ParsedConstId),
+	Module { lines: Vec<ParsedLine>, use_prelude: bool },
+}
+
+/// Obtain the value of a parsed constant. This is guaranteed to be called after
+/// the last [ParseLine] but before any [crate::AtomReq]. As such, in principle
+/// the macro engine could run here.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct FetchParsedConst {
+	pub sys: SysId,
+	pub id: ParsedConstId,
+}
+impl Request for FetchParsedConst {
+	type Response = Expression;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Comment {
+	pub text: TStr,
+	pub range: Range<u32>,
+}
+
+/// Resolve relative names from the perspective of a constant. This can only be
+/// called during a [FetchParsedConst] call, but it can be called for a
+/// different [ParsedConstId] from the one in [FetchParsedConst].
+///
+/// Each name is either resolved to a valid name or a potential error error.
+/// The error is not raised by the interpreter itself, as names may have a
+/// primary meaning such as a local binding which can be overridden by specific
+/// true names such as those triggering macro keywords. It is not recommended to
+/// define syntax that can break by defining arbitrary constants, as line
+/// parsers can define new ones at will.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct ResolveNames {
+	pub sys: SysId,
+	pub constid: ParsedConstId,
+	pub names: Vec<TVec>,
+}
+
+impl Request for ResolveNames {
+	type Response = Vec<OrcResult<TVec>>;
 }

orchid-api/src/proto.rs

@@ -24,11 +24,11 @@
 
 use std::pin::Pin;
 
-use async_std::io::{Read, Write};
+use futures::{AsyncRead, AsyncWrite};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact};
 
-use crate::{atom, expr, interner, lexer, logging, macros, parser, system, tree, vfs};
+use crate::{atom, expr, interner, lexer, logging, parser, system, tree};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader {
@@ -36,7 +36,7 @@ pub struct HostHeader {
 	pub msg_logs: logging::LogStrategy,
 }
 impl Decode for HostHeader {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		read_exact(read.as_mut(), HOST_INTRO).await;
 		Self {
 			log_strategy: logging::LogStrategy::decode(read.as_mut()).await,
@@ -45,7 +45,7 @@ impl Decode for HostHeader {
 	}
 }
 impl Encode for HostHeader {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write_exact(write.as_mut(), HOST_INTRO).await;
 		self.log_strategy.encode(write.as_mut()).await;
 		self.msg_logs.encode(write.as_mut()).await
@@ -58,13 +58,13 @@ pub struct ExtensionHeader {
 	pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
-	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		read_exact(read.as_mut(), EXT_INTRO).await;
 		Self { name: String::decode(read.as_mut()).await, systems: Vec::decode(read).await }
 	}
 }
 impl Encode for ExtensionHeader {
-	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write_exact(write.as_mut(), EXT_INTRO).await;
 		self.name.encode(write.as_mut()).await;
 		self.systems.encode(write).await
@@ -88,7 +88,8 @@ pub enum ExtHostReq {
 	SysFwd(system::SysFwd),
 	ExprReq(expr::ExprReq),
 	SubLex(lexer::SubLex),
-	RunMacros(macros::RunMacros),
+	LsModule(tree::LsModule),
+	ResolveNames(parser::ResolveNames),
 }
 
 /// Notifications sent from the extension to the host
@@ -117,9 +118,8 @@ pub enum HostExtReq {
 	DeserAtom(atom::DeserAtom),
 	LexExpr(lexer::LexExpr),
 	ParseLine(parser::ParseLine),
+	FetchParsedConst(parser::FetchParsedConst),
 	GetMember(tree::GetMember),
-	VfsReq(vfs::VfsReq),
-	ApplyMacro(macros::ApplyMacro),
 }
 
 /// Notifications sent from the host to the extension

orchid-api/src/system.rs

@@ -5,7 +5,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::{CharFilter, ExtHostReq, HostExtNotif, HostExtReq, MemberKind, TStr};
+use crate::{CharFilter, ExtHostReq, HostExtNotif, HostExtReq, MemberKind, TStr, TVec};
 
 /// ID of a system type
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
@@ -52,17 +52,18 @@ pub struct NewSystem {
 	pub depends: Vec<SysId>,
 }
 impl Request for NewSystem {
-	type Response = SystemInst;
+	type Response = NewSystemResponse;
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct SystemInst {
+pub struct NewSystemResponse {
 	/// The set of possible starting characters of tokens the lexer of this system
 	/// can process. The lexer will notify this system if it encounters one of
 	/// these characters.9
 	pub lex_filter: CharFilter,
 	pub line_types: Vec<TStr>,
 	pub const_root: HashMap<TStr, MemberKind>,
+	pub prelude: Vec<TVec>,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/tree.rs

@@ -1,14 +1,12 @@
+use std::collections::HashMap;
 use std::num::NonZeroU64;
 use std::ops::Range;
-use std::sync::Arc;
+use std::rc::Rc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
-use ordered_float::NotNan;
 
-use crate::{
+use crate::{ExprTicket, Expression, ExtHostReq, HostExtReq, OrcError, SysId, TStr, TVec};
-	Atom, Expression, HostExtReq, Location, MacroBlock, OrcError, Placeholder, SysId, TStr, TStrv,
-};
 
 /// A token tree from a lexer recursion request. Its lifetime is the lex call,
 /// the lexer can include it in its output or discard it by implication.
@@ -29,30 +27,27 @@ pub struct TokenTree {
 #[derive(Clone, Debug, Coding)]
 pub enum Token {
 	/// Lambda function head, from the opening \ until the beginning of the body.
-	LambdaHead(Vec<TokenTree>),
+	LambdaHead(Box<TokenTree>),
 	/// A name segment or an operator.
 	Name(TStr),
-	/// An absolute name
+	/// A newly generated expression. The last place this is supposed to happen is
-	Reference(TStrv),
+	/// in lexers, parsers and macros should have enumerable many outputs
+	/// expressed as function calls.
+	NewExpr(Expression),
+	/// A pre-existing expression
+	Handle(ExprTicket),
 	/// ::
-	NS,
+	NS(TStr, Box<TokenTree>),
 	/// Line break.
 	BR,
 	/// ( Round parens ), [ Square brackets ] or { Curly braces }
 	S(Paren, Vec<TokenTree>),
-	/// A new atom
-	Atom(Atom),
-	/// Anchor to insert a subtree
-	Slot(TreeTicket),
 	/// A static compile-time error returned by failing lexers if
-	/// the rest of the source is likely still meaningful
+	/// the rest of the source is likely still meaningful. This is distinct from
+	/// NewExpr(Bottom) because it fails in dead branches too.
 	Bottom(Vec<OrcError>),
 	/// A comment
-	Comment(Arc<String>),
+	Comment(Rc<String>),
-	/// Placeholder
-	Ph(Placeholder),
-	/// Macro block head
-	Macro(Option<NotNan<f64>>),
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]
@@ -65,31 +60,12 @@ pub enum Paren {
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct TreeId(pub NonZeroU64);
 
-#[derive(Clone, Debug, Coding)]
-pub struct Item {
-	pub location: Location,
-	pub comments: Vec<Comment>,
-	pub kind: ItemKind,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub enum ItemKind {
-	Member(Member),
-	Macro(MacroBlock),
-	Export(TStr),
-	Import(TStrv),
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct Comment {
-	pub text: TStr,
-	pub location: Location,
-}
-
 #[derive(Clone, Debug, Coding)]
 pub struct Member {
 	pub name: TStr,
+	pub exported: bool,
 	pub kind: MemberKind,
+	pub comments: Vec<TStr>,
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -101,12 +77,54 @@ pub enum MemberKind {
 
 #[derive(Clone, Debug, Coding)]
 pub struct Module {
-	pub items: Vec<Item>,
+	pub members: Vec<Member>,
 }
 
+/// Evaluate a lazy member. This call will only be issued to each system once.
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct GetMember(pub SysId, pub TreeId);
 impl Request for GetMember {
 	type Response = MemberKind;
 }
+
+/// This request can only be issued while the interpreter is running, so during
+/// an atom call.
+#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct LsModule(pub SysId, pub TVec);
+impl Request for LsModule {
+	type Response = Result<ModuleInfo, LsModuleError>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum LsModuleError {
+	InvalidPath,
+	IsConstant,
+	TreeUnavailable,
+}
+
+/// Information about a module sent from the host to an extension. By necessity,
+/// members and imports are non-overlapping.
+#[derive(Clone, Debug, Coding)]
+pub struct ModuleInfo {
+	/// List the names defined in this module
+	pub members: HashMap<TStr, MemberInfo>,
+}
+
+#[derive(Clone, Copy, Debug, Coding)]
+pub struct MemberInfo {
+	/// true if the name is exported
+	pub public: bool,
+	/// Whether the tree item is a constant value or a module
+	pub kind: MemberInfoKind,
+}
+
+/// Indicates what kind of node a name refers to
+#[derive(Clone, Copy, Debug, Coding)]
+pub enum MemberInfoKind {
+	/// has children obtained with [crate::LsModule]
+	Module,
+	/// has a value retrievable in [crate::ExpressionKind::Const]
+	Constant,
+}

orchid-api/src/vfs.rs

@@ -1,47 +0,0 @@
-use std::collections::HashMap;
-use std::num::NonZeroU16;
-
-use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::Request;
-
-use crate::error::OrcResult;
-use crate::interner::TStr;
-use crate::proto::HostExtReq;
-use crate::system::SysId;
-
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-pub struct VfsId(pub NonZeroU16);
-
-#[derive(Clone, Debug, Coding)]
-pub enum Loaded {
-	Code(String),
-	Collection(Vec<TStr>),
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(VfsReq, HostExtReq)]
-pub struct VfsRead(pub SysId, pub VfsId, pub Vec<TStr>);
-impl Request for VfsRead {
-	type Response = OrcResult<Loaded>;
-}
-
-#[derive(Clone, Debug, Coding)]
-pub enum EagerVfs {
-	Lazy(VfsId),
-	Eager(HashMap<TStr, EagerVfs>),
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(VfsReq, HostExtReq)]
-pub struct GetVfs(pub SysId);
-impl Request for GetVfs {
-	type Response = EagerVfs;
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtReq)]
-#[extendable]
-pub enum VfsReq {
-	GetVfs(GetVfs),
-	VfsRead(VfsRead),
-}

orchid-base/Cargo.toml

@@ -1,18 +1,20 @@
 [package]
 name = "orchid-base"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
+
+[features]
+mocks = []
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
-async-stream = "0.3.6"
 derive_destructure = "1.0.0"
-dyn-clone = "1.0.17"
+dyn-clone = "1.0.20"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.15.2"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
 never = "0.1.0"
@@ -20,11 +22,10 @@ num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"
-regex = "1.11.1"
+regex = "1.11.2"
-rust-embed = "8.5.0"
+rust-embed = "8.7.2"
-rust_decimal = "1.36.0"
+some_executor = "0.6.1"
-some_executor = "0.4.0"
 substack = "1.1.1"
-test_executors = "0.3.2"
+test_executors = "0.3.5"
 trait-set = "0.3.0"

orchid-base/src/builtin.rs

@@ -6,7 +6,6 @@ use futures::future::LocalBoxFuture;
 use crate::api;
 
 pub type Spawner = Rc<dyn Fn(LocalBoxFuture<'static, ()>)>;
-pub type RecvCB<'a> = Box<dyn for<'b> FnOnce(&'b [u8]) -> LocalBoxFuture<'b, ()> + 'a>;
 
 /// The 3 primary contact points with an extension are
 /// - send a message
@@ -15,8 +14,10 @@ pub type RecvCB<'a> = Box<dyn for<'b> FnOnce(&'b [u8]) -> LocalBoxFuture<'b, ()>
 ///
 /// There are no ordering guarantees about these
 pub trait ExtPort {
+	#[must_use]
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()>;
-	fn recv<'a>(&'a self, cb: RecvCB<'a>) -> LocalBoxFuture<'a, ()>;
+	#[must_use]
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>>;
 }
 
 pub struct ExtInit {
@@ -25,7 +26,7 @@ pub struct ExtInit {
 }
 impl ExtInit {
 	pub async fn send(&self, msg: &[u8]) { self.port.send(msg).await }
-	pub async fn recv<'a, 's: 'a>(&'s self, cb: RecvCB<'a>) { self.port.recv(Box::new(cb)).await }
+	pub async fn recv(&self) -> Option<Vec<u8>> { self.port.recv().await }
 }
 impl Deref for ExtInit {
 	type Target = api::ExtensionHeader;

orchid-base/src/clone.rs

@@ -1,12 +1,12 @@
 #[macro_export]
 macro_rules! clone {
-  ($($n:ident),+; $body:expr) => (
+  ($($n:ident $($mut:ident)?),+; $body:expr) => (
     {
-      $( let $n = $n.clone(); )+
+      $( let $($mut)? $n = $n.clone(); )+
       $body
     }
   );
-  ($($n:ident),+) => {
+  ($($n:ident $($mut:ident)?),+) => {
-    $( let $n = $n.clone(); )+
+    $( let $($mut)? $n = $n.clone(); )+
   }
 }

orchid-base/src/ctx.rs

@@ -0,0 +1,28 @@
+use std::any::{TypeId, type_name};
+use std::marker::PhantomData;
+use std::rc::Rc;
+
+use orchid_api_traits::MsgSet;
+
+use crate::error::Reporter;
+use crate::interner::Interner;
+use crate::reqnot::{Client, DynClient};
+
+pub trait CtxDyn {
+	fn i(&self) -> Interner;
+	fn rep(&self) -> &Reporter;
+	fn client(&self, msg_set: TypeId) -> Option<Rc<dyn DynClient>>;
+	fn msg_set_type(&self) -> TypeId;
+}
+
+pub struct Ctx(Rc<dyn CtxDyn>);
+impl Ctx {
+	pub fn i(&self) -> Interner { self.0.i() }
+	pub fn rep(&self) -> &Reporter { self.0.rep() }
+	pub fn client<T: MsgSet>(&self) -> Client<T> {
+		let Some(dyn_client) = self.0.client(TypeId::of::<T>()) else {
+			panic!("Incorrect message set {} passed", type_name::<T>());
+		};
+		Client(dyn_client, PhantomData)
+	}
+}

orchid-base/src/error.rs

@@ -1,13 +1,22 @@
 use std::cell::RefCell;
-use std::fmt;
+use std::ffi::OsStr;
-use std::ops::Add;
+use std::fmt::{self, Display};
+use std::ops::{Add, AddAssign, Deref};
+use std::pin::Pin;
+use std::rc::Rc;
 use std::sync::Arc;
 
-use futures::future::join_all;
+use async_fn_stream::stream;
+use async_once_cell::{Lazy, OnceCell};
+use futures::future::{join_all, ready};
+use futures::lock::Mutex;
+use futures::{Stream, StreamExt, stream};
 use itertools::Itertools;
 
 use crate::api;
-use crate::interner::{Interner, Tok};
+use crate::ctx::Ctx;
+use crate::format::{FmtCtx, FmtUnit, Format};
+use crate::interner::{IStr, Interner};
 use crate::location::Pos;
 
 /// A point of interest in resolving the error, such as the point where
@@ -39,43 +48,136 @@ impl ErrPos {
 impl From<Pos> for ErrPos {
 	fn from(origin: Pos) -> Self { Self { position: origin, message: None } }
 }
+impl fmt::Display for ErrPos {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match &self.message {
+			Some(msg) => write!(f, "{}: {}", self.position, msg),
+			None => write!(f, "{}", self.position),
+		}
+	}
+}
 
 #[derive(Clone, Debug)]
-pub struct OrcErr {
+struct SingleError {
-	pub description: Tok<String>,
+	pub description: IStr,
 	pub message: Arc<String>,
 	pub positions: Vec<ErrPos>,
 }
-impl OrcErr {
+impl fmt::Display for SingleError {
-	fn to_api(&self) -> api::OrcError {
-		api::OrcError {
-			description: self.description.to_api(),
-			message: self.message.clone(),
-			locations: self.positions.iter().map(ErrPos::to_api).collect(),
-		}
-	}
-	async fn from_api(api: &api::OrcError, i: &Interner) -> Self {
-		Self {
-			description: Tok::from_api(api.description, i).await,
-			message: api.message.clone(),
-			positions: join_all(api.locations.iter().map(|e| ErrPos::from_api(e, i))).await,
-		}
-	}
-}
-impl Eq for OrcErr {}
-impl PartialEq for OrcErr {
-	fn eq(&self, other: &Self) -> bool { self.description == other.description }
-}
-impl From<OrcErr> for Vec<OrcErr> {
-	fn from(value: OrcErr) -> Self { vec![value] }
-}
-impl fmt::Display for OrcErr {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let pstr = self.positions.iter().map(|p| format!("{p:?}")).join("; ");
+		let pstr = self.positions.iter().map(|p| format!("{p}")).join("; ");
 		write!(f, "{}: {} @ {}", self.description, self.message, pstr)
 	}
 }
 
+pub struct OrcErr {
+	singles: OnceCell<OwnedOrcErr>,
+	futures: Mutex<Vec<Pin<Box<dyn Future<Output = OwnedOrcErr>>>>>,
+}
+impl OrcErr {
+	pub async fn into_owned(self) -> OwnedOrcErr {
+		self.to_owned().await;
+		self.singles.into_inner().expect("Initialized above")
+	}
+	pub async fn to_owned(&self) -> &OwnedOrcErr {
+		self
+			.singles
+			.get_or_init(async {
+				let results = join_all(self.futures.lock().await.drain(..)).await;
+				OwnedOrcErr(results.iter().flat_map(|err| err.0.iter()).cloned().collect())
+			})
+			.await
+	}
+	fn into_futures(self) -> Vec<Pin<Box<dyn Future<Output = OwnedOrcErr>>>> {
+		match self.singles.into_inner() {
+			Some(val) => vec![Box::pin(ready(val))],
+			None => self.futures.into_inner(),
+		}
+	}
+}
+impl From<OwnedOrcErr> for OrcErr {
+	fn from(value: OwnedOrcErr) -> Self {
+		Self { singles: OnceCell::from(value), futures: Mutex::new(vec![]) }
+	}
+}
+impl<T: Future<Output = OrcErr> + 'static> From<T> for OrcErr {
+	fn from(value: T) -> Self {
+		Self {
+			singles: OnceCell::new(),
+			futures: Mutex::new(vec![Box::pin(async { value.await.into_owned().await })]),
+		}
+	}
+}
+impl Add for OrcErr {
+	type Output = Self;
+	fn add(mut self, mut rhs: Self) -> Self::Output {
+		if let (Some(l), Some(r)) = (self.singles.get_mut(), rhs.singles.get_mut()) {
+			l.0.extend(r.0.drain(..));
+			return self;
+		}
+		Self {
+			singles: OnceCell::new(),
+			futures: Mutex::new(self.into_futures().into_iter().chain(rhs.into_futures()).collect()),
+		}
+	}
+}
+impl AddAssign for OrcErr {
+	fn add_assign(&mut self, mut rhs: Self) {
+		if let (Some(l), Some(r)) = (self.singles.get_mut(), rhs.singles.get_mut()) {
+			l.0.extend(r.0.drain(..));
+		} else {
+			let mut temp = Self { futures: Mutex::default(), singles: OnceCell::new() };
+			std::mem::swap(self, &mut temp);
+			self.futures.get_mut().extend(temp.into_futures().into_iter().chain(rhs.into_futures()));
+		}
+	}
+}
+impl Format for OrcErr {
+	async fn print<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		format!("{}", self.to_owned().await).into()
+	}
+}
+
+#[derive(Clone)]
+pub struct OwnedOrcErr(Vec<SingleError>);
+impl OwnedOrcErr {
+	pub fn to_api(&self) -> Vec<api::OrcError> {
+		self
+			.0
+			.iter()
+			.map(|err| api::OrcError {
+				description: err.description.to_api(),
+				message: err.message.clone(),
+				locations: err.positions.iter().map(|pos| pos.to_api()).collect(),
+			})
+			.collect_vec()
+	}
+	pub async fn from_api(api: impl IntoIterator<Item = &api::OrcError>, i: &Interner) -> Self {
+		Self(
+			join_all(api.into_iter().map(|e| async {
+				SingleError {
+					description: i.es(e.description).await,
+					message: e.message.clone(),
+					positions: join_all(e.locations.iter().map(|pos| ErrPos::from_api(pos, i))).await,
+				}
+			}))
+			.await,
+		)
+	}
+}
+impl Display for OwnedOrcErr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "{}", self.0.iter().join("\n"))
+	}
+}
+impl fmt::Debug for OwnedOrcErr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "OwnedOrcErr({self}") }
+}
+impl Add for OwnedOrcErr {
+	type Output = Self;
+	fn add(self, rhs: Self) -> Self::Output { Self(self.0.into_iter().chain(rhs.0).collect()) }
+}
+
 #[derive(Clone, Debug)]
 pub struct EmptyErrv;
 impl fmt::Display for EmptyErrv {
@@ -84,105 +186,118 @@ impl fmt::Display for EmptyErrv {
 	}
 }
 
-#[derive(Clone, Debug)]
+pub type OrcRes<T> = Result<T, OrcErr>;
-pub struct OrcErrv(Vec<OrcErr>);
+
-impl OrcErrv {
+pub fn join_ok<T, U>(left: OrcRes<T>, right: OrcRes<U>) -> OrcRes<(T, U)> {
-	pub fn new(errors: impl IntoIterator<Item = OrcErr>) -> Result<Self, EmptyErrv> {
+	match (left, right) {
-		let v = errors.into_iter().collect_vec();
+		(Ok(t), Ok(u)) => Ok((t, u)),
-		if v.is_empty() { Err(EmptyErrv) } else { Ok(Self(v)) }
+		(Err(e), Ok(_)) | (Ok(_), Err(e)) => Err(e),
-	}
+		(Err(e1), Err(e2)) => Err(e1 + e2),
-	#[must_use]
-	pub fn extended<T>(mut self, errors: impl IntoIterator<Item = T>) -> Self
-	where Self: Extend<T> {
-		self.extend(errors);
-		self
-	}
-	#[must_use]
-	pub fn len(&self) -> usize { self.0.len() }
-	#[must_use]
-	pub fn is_empty(&self) -> bool { self.len() == 0 }
-	#[must_use]
-	pub fn any(&self, f: impl FnMut(&OrcErr) -> bool) -> bool { self.0.iter().any(f) }
-	#[must_use]
-	pub fn keep_only(self, f: impl FnMut(&OrcErr) -> bool) -> Option<Self> {
-		let v = self.0.into_iter().filter(f).collect_vec();
-		if v.is_empty() { None } else { Some(Self(v)) }
-	}
-	#[must_use]
-	pub fn one(&self) -> Option<&OrcErr> { (self.0.len() == 1).then(|| &self.0[9]) }
-	pub fn pos_iter(&self) -> impl Iterator<Item = ErrPos> + '_ {
-		self.0.iter().flat_map(|e| e.positions.iter().cloned())
-	}
-	pub fn to_api(&self) -> Vec<api::OrcError> { self.0.iter().map(OrcErr::to_api).collect() }
-	pub async fn from_api<'a>(
-		api: impl IntoIterator<Item = &'a api::OrcError>,
-		i: &Interner,
-	) -> Self {
-		Self(join_all(api.into_iter().map(|e| OrcErr::from_api(e, i))).await)
-	}
-}
-impl From<OrcErr> for OrcErrv {
-	fn from(value: OrcErr) -> Self { Self(vec![value]) }
-}
-impl Add for OrcErrv {
-	type Output = Self;
-	fn add(self, rhs: Self) -> Self::Output { Self(self.0.into_iter().chain(rhs.0).collect_vec()) }
-}
-impl Extend<OrcErr> for OrcErrv {
-	fn extend<T: IntoIterator<Item = OrcErr>>(&mut self, iter: T) { self.0.extend(iter) }
-}
-impl Extend<OrcErrv> for OrcErrv {
-	fn extend<T: IntoIterator<Item = OrcErrv>>(&mut self, iter: T) {
-		self.0.extend(iter.into_iter().flatten())
-	}
-}
-impl IntoIterator for OrcErrv {
-	type IntoIter = std::vec::IntoIter<OrcErr>;
-	type Item = OrcErr;
-	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
-}
-impl fmt::Display for OrcErrv {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "{}", self.0.iter().join("\n"))
 	}
 }
 
-pub type OrcRes<T> = Result<T, OrcErrv>;
+#[macro_export]
+macro_rules! join_ok {
+	($($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		let $crate::join_ok!(@NAMES $($names $(: $tys)? = $vals;)*)
+		:
+		$crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*)
+		 =
+		$crate::join_ok!(@VALUES $($names $(: $tys)? = $vals;)*)?;
+	};
+	(@NAMES $name:ident $(: $ty:ty)? = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		($name, $crate::join_ok!(@NAMES $($names $(: $tys)? = $vals;)*))
+	};
+	(@NAMES) => { _ };
+	(@TYPES $name:ident : $ty:ty = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		($ty, $crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*))
+	};
+	(@TYPES $name:ident = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		(_, $crate::join_ok!(@TYPES $($names $(: $tys)? = $vals;)*))
+	};
+	(@TYPES) => { () };
+	(@VALUES $name:ident $(: $ty:ty)? = $val:expr ; $($names:ident $(: $tys:ty)? = $vals:expr;)*) => {
+		$crate::error::join_ok($val, $crate::join_ok!(@VALUES $($names $(: $tys)? = $vals;)*))
+	};
+	(@VALUES) => { Ok(()) };
+}
 
-pub fn mk_err(
+impl Ctx {
-	description: Tok<String>,
+	pub fn mk_err_floating(
-	message: impl AsRef<str>,
+		&self,
-	posv: impl IntoIterator<Item = ErrPos>,
+		description: impl AsRef<str> + 'static,
-) -> OrcErr {
+		message: impl AsRef<str> + 'static,
-	OrcErr {
+	) -> OrcErr {
-		description,
+		self.mk_err::<Pos>(description, message, [])
-		message: Arc::new(message.as_ref().to_string()),
+	}
-		positions: posv.into_iter().collect(),
+	pub fn mk_err<I: Into<ErrPos>>(
+		&self,
+		description: impl AsRef<str> + 'static,
+		message: impl AsRef<str> + 'static,
+		posv: impl IntoIterator<Item = I> + 'static,
+	) -> OrcErr {
+		let i = self.i();
+		async move {
+			OwnedOrcErr(vec![SingleError {
+				description: i.is(description.as_ref()).await,
+				message: Arc::new(message.as_ref().to_string()),
+				positions: posv.into_iter().map_into().collect(),
+			}])
+			.into()
+		}
+		.into()
+	}
+	pub async fn async_io_err<I: Into<ErrPos>>(
+		&self,
+		err: std::io::Error,
+		posv: impl IntoIterator<Item = I> + 'static,
+	) -> OrcErr {
+		self.mk_err(err.kind().to_string(), err.to_string(), posv)
+	}
+	pub fn os_str_to_string<'a, I: Into<ErrPos>>(
+		&self,
+		str: &'a OsStr,
+		posv: impl IntoIterator<Item = I> + 'static,
+	) -> OrcRes<&'a str> {
+		match str.to_str() {
+			Some(str) => Ok(str),
+			None => Err(self.mk_err(
+				"Non-unicode string",
+				format!("{str:?} is not representable as unicode"),
+				posv,
+			)),
+		}
 	}
 }
 
-pub fn mk_errv(
+pub struct Reporter {
-	description: Tok<String>,
+	errors: RefCell<Option<OrcErr>>,
-	message: impl AsRef<str>,
-	posv: impl IntoIterator<Item = ErrPos>,
-) -> OrcErrv {
-	mk_err(description, message, posv).into()
 }
 
-pub trait Reporter {
+impl Reporter {
-	fn report(&self, e: impl Into<OrcErrv>);
+	pub fn report(&self, e: impl Into<OrcErr>) {
+		match &mut *self.errors.borrow_mut() {
+			slot @ None => *slot = Some(e.into()),
+			Some(err) => *err += e.into(),
+		}
+	}
+	pub fn new() -> Self { Self { errors: RefCell::new(None) } }
+	pub fn res(self) -> Result<(), OrcErr> {
+		match self.errors.into_inner() {
+			Some(e) => Err(e),
+			None => Ok(()),
+		}
+	}
+	pub fn merge<T>(self, res: OrcRes<T>) -> OrcRes<T> {
+		match (res, self.res()) {
+			(res, Ok(())) => res,
+			(Ok(_), Err(e)) => Err(e),
+			(Err(e), Err(e2)) => Err(e + e2),
+		}
+	}
+	pub fn is_empty(&self) -> bool { self.errors.borrow().is_none() }
 }
 
-pub struct ReporterImpl {
+impl Default for Reporter {
-	errors: RefCell<Vec<OrcErr>>,
-}
-impl ReporterImpl {
-	pub fn new() -> Self { Self { errors: RefCell::new(vec![]) } }
-	pub fn errv(self) -> Option<OrcErrv> { OrcErrv::new(self.errors.into_inner()).ok() }
-}
-impl Reporter for ReporterImpl {
-	fn report(&self, e: impl Into<OrcErrv>) { self.errors.borrow_mut().extend(e.into()) }
-}
-impl Default for ReporterImpl {
 	fn default() -> Self { Self::new() }
 }

orchid-base/src/format.rs

@@ -13,6 +13,7 @@ use crate::interner::Interner;
 use crate::{api, match_mapping};
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
+#[must_use]
 pub struct FmtUnit {
 	pub subs: Vec<FmtUnit>,
 	pub variants: Rc<Variants>,
@@ -209,6 +210,9 @@ impl From<Rc<String>> for Variants {
 impl From<String> for Variants {
 	fn from(value: String) -> Self { Self::from(Rc::new(value)) }
 }
+impl From<&str> for Variants {
+	fn from(value: &str) -> Self { Self::from(value.to_string()) }
+}
 impl FromStr for Variants {
 	type Err = Infallible;
 	fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(Self::default().bounded(s)) }
@@ -265,8 +269,12 @@ impl FmtCtx for FmtCtxImpl<'_> {
 }
 
 pub trait Format {
+	#[must_use]
 	fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> + 'a;
 }
 impl Format for Never {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit { match *self {} }
 }
+
+/// Format with default strategy. Currently equal to [take_first_fmt]
+pub async fn fmt(v: &(impl Format + ?Sized), i: &Interner) -> String { take_first_fmt(v, i).await }

orchid-base/src/interner.rs

@@ -1,13 +1,15 @@
+use std::any::Any;
 use std::borrow::Borrow;
 use std::future::Future;
-use std::hash::BuildHasher as _;
+use std::hash::{BuildHasher as _, Hash};
 use std::num::NonZeroU64;
 use std::ops::Deref;
 use std::rc::Rc;
 use std::sync::atomic;
 use std::{fmt, hash};
 
-use async_std::sync::Mutex;
+use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools as _;
 use orchid_api_traits::Request;
@@ -15,294 +17,460 @@ use orchid_api_traits::Request;
 use crate::api;
 use crate::reqnot::{DynRequester, Requester};
 
-/// Clippy crashes while verifying `Tok: Sized` without this and I cba to create
+// /// Clippy crashes while verifying `Tok: Sized` without this and I cba to
-/// a minimal example
+// create /// a minimal example
-#[derive(Clone)]
+// #[derive(Clone)]
-struct ForceSized<T>(T);
+// struct ForceSized<T>(T);
 
-#[derive(Clone)]
+// #[derive(Clone)]
-pub struct Tok<T: Interned> {
+// pub struct Tok<T: Interned> {
-	data: Rc<T>,
+// 	data: Rc<T>,
-	marker: ForceSized<T::Marker>,
+// 	marker: ForceSized<T::Marker>,
-}
+// }
-impl<T: Interned> Tok<T> {
+// impl<T: Interned> Tok<T> {
-	pub fn new(data: Rc<T>, marker: T::Marker) -> Self { Self { data, marker: ForceSized(marker) } }
+// 	pub fn new(data: Rc<T>, marker: T::Marker) -> Self { Self { data, marker:
-	pub fn to_api(&self) -> T::Marker { self.marker.0 }
+// ForceSized(marker) } } 	pub fn to_api(&self) -> T::Marker { self.marker.0 }
-	pub async fn from_api<M>(marker: M, i: &Interner) -> Self
+// 	pub async fn from_api<M>(marker: M, i: &Interner) -> Self
-	where M: InternMarker<Interned = T> {
+// 	where M: InternMarker<Interned = T> {
-		i.ex(marker).await
+// 		i.ex(marker).await
-	}
+// 	}
-	pub fn rc(&self) -> Rc<T> { self.data.clone() }
+// 	pub fn rc(&self) -> Rc<T> { self.data.clone() }
-}
+// }
-impl<T: Interned> Deref for Tok<T> {
+// impl<T: Interned> Deref for Tok<T> {
-	type Target = T;
+// 	type Target = T;
 
-	fn deref(&self) -> &Self::Target { self.data.as_ref() }
+// 	fn deref(&self) -> &Self::Target { self.data.as_ref() }
-}
+// }
-impl<T: Interned> Ord for Tok<T> {
+// impl<T: Interned> Ord for Tok<T> {
-	fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.to_api().cmp(&other.to_api()) }
+// 	fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-}
+// self.to_api().cmp(&other.to_api()) } }
-impl<T: Interned> PartialOrd for Tok<T> {
+// impl<T: Interned> PartialOrd for Tok<T> {
-	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
+// 	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-}
+// Some(self.cmp(other)) } }
-impl<T: Interned> Eq for Tok<T> {}
+// impl<T: Interned> Eq for Tok<T> {}
-impl<T: Interned> PartialEq for Tok<T> {
+// impl<T: Interned> PartialEq for Tok<T> {
-	fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
+// 	fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
-}
+// }
-impl<T: Interned> hash::Hash for Tok<T> {
+// impl<T: Interned> hash::Hash for Tok<T> {
-	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.to_api().hash(state) }
+// 	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.to_api().hash(state) }
-}
+// }
-impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
+// impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+// 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{}", &*self.data)
+// 		write!(f, "{}", &*self.data)
-	}
+// 	}
-}
+// }
-impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
+// impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+// 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "Token({} -> {:?})", self.to_api().get_id(), self.data.as_ref())
+// 		write!(f, "Token({} -> {:?})", self.to_api().get_id(), self.data.as_ref())
-	}
+// 	}
-}
+// }
 
-pub trait Interned: Eq + hash::Hash + Clone + fmt::Debug + Internable<Interned = Self> {
+// pub trait Interned: Eq + hash::Hash + Clone + fmt::Debug +
-	type Marker: InternMarker<Interned = Self> + Sized;
+// Internable<Interned = Self> { 	type Marker: InternMarker<Interned = Self> +
-	fn intern(
+// Sized; 	fn intern(
-		self: Rc<Self>,
+// 		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+// 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> impl Future<Output = Self::Marker>;
+// 	) -> impl Future<Output = Self::Marker>;
-	fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
+// 	fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
-}
+// }
 
-pub trait Internable: fmt::Debug {
+// pub trait Internable: fmt::Debug {
-	type Interned: Interned;
+// 	type Interned: Interned;
-	fn get_owned(&self) -> Rc<Self::Interned>;
+// 	fn get_owned(&self) -> Rc<Self::Interned>;
-}
+// }
 
-pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash + Sized {
+// pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash
-	type Interned: Interned<Marker = Self>;
+// + Sized { 	type Interned: Interned<Marker = Self>;
-	/// Only called on replicas
+// 	/// Only called on replicas
-	fn resolve(self, i: &Interner) -> impl Future<Output = Tok<Self::Interned>>;
+// 	fn resolve(self, i: &Interner) -> impl Future<Output = Tok<Self::Interned>>;
-	fn get_id(self) -> NonZeroU64;
+// 	fn get_id(self) -> NonZeroU64;
-	fn from_id(id: NonZeroU64) -> Self;
+// 	fn from_id(id: NonZeroU64) -> Self;
-}
+// }
 
-impl Interned for String {
+// impl Interned for String {
-	type Marker = api::TStr;
+// 	type Marker = api::TStr;
-	async fn intern(
+// 	async fn intern(
-		self: Rc<Self>,
+// 		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+// 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> Self::Marker {
+// 	) -> Self::Marker {
-		req.request(api::InternStr(self.to_string())).await
+// 		req.request(api::InternStr(self.to_string())).await
-	}
+// 	}
-	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.strings }
+// 	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut
-}
+// interners.strings } }
-impl InternMarker for api::TStr {
+// impl InternMarker for api::TStr {
-	type Interned = String;
+// 	type Interned = String;
-	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
+// 	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
-		Tok::new(Rc::new(i.master.as_ref().unwrap().request(api::ExternStr(self)).await), self)
+// 		Tok::new(Rc::new(i.0.master.as_ref().unwrap().
-	}
+// request(api::ExternStr(self)).await), self) 	}
-	fn get_id(self) -> NonZeroU64 { self.0 }
+// 	fn get_id(self) -> NonZeroU64 { self.0 }
-	fn from_id(id: NonZeroU64) -> Self { Self(id) }
+// 	fn from_id(id: NonZeroU64) -> Self { Self(id) }
-}
+// }
-impl Internable for str {
+// impl Internable for str {
-	type Interned = String;
+// 	type Interned = String;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
-}
+// }
-impl Internable for String {
+// impl Internable for String {
-	type Interned = String;
+// 	type Interned = String;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
-}
+// }
 
-impl Interned for Vec<Tok<String>> {
+// impl Interned for Vec<IStr> {
-	type Marker = api::TStrv;
+// 	type Marker = api::TStrv;
-	async fn intern(
+// 	async fn intern(
-		self: Rc<Self>,
+// 		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+// 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> Self::Marker {
+// 	) -> Self::Marker {
-		req.request(api::InternStrv(self.iter().map(|t| t.to_api()).collect())).await
+// 		req.request(api::InternStrv(self.iter().map(|t|
-	}
+// t.to_api()).collect())).await 	}
-	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.vecs }
+// 	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut
-}
+// interners.vecs } }
-impl InternMarker for api::TStrv {
+// impl InternMarker for api::TStrv {
-	type Interned = Vec<Tok<String>>;
+// 	type Interned = Vec<IStr>;
-	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
+// 	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
-		let rep = i.master.as_ref().unwrap().request(api::ExternStrv(self)).await;
+// 		let rep =
-		let data = futures::future::join_all(rep.into_iter().map(|m| i.ex(m))).await;
+// i.0.master.as_ref().unwrap().request(api::ExternStrv(self)).await; 		let data
-		Tok::new(Rc::new(data), self)
+// = futures::future::join_all(rep.into_iter().map(|m| i.ex(m))).await;
-	}
+// 		Tok::new(Rc::new(data), self)
-	fn get_id(self) -> NonZeroU64 { self.0 }
+// 	}
-	fn from_id(id: NonZeroU64) -> Self { Self(id) }
+// 	fn get_id(self) -> NonZeroU64 { self.0 }
-}
+// 	fn from_id(id: NonZeroU64) -> Self { Self(id) }
-impl Internable for [Tok<String>] {
+// }
-	type Interned = Vec<Tok<String>>;
+// impl Internable for [IStr] {
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+// 	type Interned = Vec<IStr>;
-}
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
-impl<const N: usize> Internable for [Tok<String>; N] {
+// }
-	type Interned = Vec<Tok<String>>;
+// impl<const N: usize> Internable for [IStr; N] {
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+// 	type Interned = Vec<IStr>;
-}
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
-impl Internable for Vec<Tok<String>> {
+// }
-	type Interned = Vec<Tok<String>>;
+// impl Internable for Vec<IStr> {
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+// 	type Interned = Vec<IStr>;
-}
+// 	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
+// }
 // impl Internable for Vec<api::TStr> {
-// 	type Interned = Vec<Tok<String>>;
+// 	type Interned = Vec<IStr>;
 // 	fn get_owned(&self) -> Arc<Self::Interned> {
 // 		Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
 // 	}
 // }
 // impl Internable for [api::TStr] {
-// 	type Interned = Vec<Tok<String>>;
+// 	type Interned = Vec<IStr>;
 // 	fn get_owned(&self) -> Arc<Self::Interned> {
 // 		Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
 // 	}
 // }
 
-/// The number of references held to any token by the interner.
+macro_rules! token_def {
-const BASE_RC: usize = 3;
+	($type:ident, $trait:ident, $deref:ty, $api_repr:ty, $type_name:expr) => {
-
+		#[derive(Clone)]
-#[test]
+		pub struct $type(Rc<dyn $trait>);
-fn base_rc_correct() {
+		impl $type {
-	let tok = Tok::new(Rc::new("foo".to_string()), api::TStr(1.try_into().unwrap()));
+			pub fn new<T: $trait + 'static>(t: T) -> Self { Self(Rc::new(t) as _) }
-	let mut bimap = Bimap::default();
+			pub fn to_api(&self) -> $api_repr { self.0.to_api() }
-	bimap.insert(tok.clone());
+			pub fn inner(&self) -> &dyn Any { self.0.as_ref() }
-	assert_eq!(Rc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the current instance");
+			fn addr(&self) -> usize { Rc::as_ptr(&self.0).addr() }
+		}
+		pub trait $trait: Deref<Target = $deref> + Any {
+			fn to_api(&self) -> $api_repr;
+		}
+		impl Deref for $type {
+			type Target = $deref;
+			fn deref(&self) -> &Self::Target { self.0.deref() }
+		}
+		impl Eq for $type {}
+		impl PartialEq for $type {
+			fn eq(&self, other: &Self) -> bool { Rc::ptr_eq(&self.0, &other.0) }
+		}
+		impl Ord for $type {
+			fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.addr().cmp(&other.addr()) }
+		}
+		impl PartialOrd for $type {
+			fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
+		}
+		impl Hash for $type {
+			fn hash<H: hash::Hasher>(&self, state: &mut H) { self.addr().hash(state) }
+		}
+		impl std::fmt::Debug for $type {
+			fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+				f.debug_tuple($type_name).field(&self.to_api().0).field(&self.deref()).finish()
+			}
+		}
+	};
 }
 
-pub struct Bimap<T: Interned> {
+token_def!(IStr, IStrDyn, str, api::TStr, "IStr");
-	intern: HashMap<Rc<T>, Tok<T>>,
+impl std::fmt::Display for IStr {
-	by_id: HashMap<T::Marker, Tok<T>>,
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.deref()) }
-}
-impl<T: Interned> Bimap<T> {
-	pub fn insert(&mut self, token: Tok<T>) {
-		self.intern.insert(token.data.clone(), token.clone());
-		self.by_id.insert(token.to_api(), token);
-	}
-
-	pub fn by_marker(&self, marker: T::Marker) -> Option<Tok<T>> { self.by_id.get(&marker).cloned() }
-
-	pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Tok<T>>
-	where T: Borrow<Q> {
-		(self.intern.raw_entry())
-			.from_hash(self.intern.hasher().hash_one(q), |k| k.as_ref().borrow() == q)
-			.map(|p| p.1.clone())
-	}
-
-	pub fn sweep_replica(&mut self) -> Vec<T::Marker> {
-		(self.intern)
-			.extract_if(|k, _| Rc::strong_count(k) == BASE_RC)
-			.map(|(_, v)| {
-				self.by_id.remove(&v.to_api());
-				v.to_api()
-			})
-			.collect()
-	}
-
-	pub fn sweep_master(&mut self, retained: HashSet<T::Marker>) {
-		self.intern.retain(|k, v| BASE_RC < Rc::strong_count(k) || retained.contains(&v.to_api()))
-	}
 }
 
-impl<T: Interned> Default for Bimap<T> {
+token_def!(IVec, IVecDyn, [IStr], api::TVec, "IVec");
-	fn default() -> Self { Self { by_id: HashMap::new(), intern: HashMap::new() } }
-}
 
-pub trait UpComm {
+#[derive(Clone)]
-	fn up<R: Request>(&self, req: R) -> R::Response;
+pub struct Interner(Rc<dyn InternerDyn>);
-}
-
-#[derive(Default)]
-pub struct TypedInterners {
-	strings: Bimap<String>,
-	vecs: Bimap<Vec<Tok<String>>>,
-}
-
-#[derive(Default)]
-pub struct Interner {
-	interners: Mutex<TypedInterners>,
-	master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
-}
 impl Interner {
-	pub fn new_master() -> Self { Self::default() }
+	pub fn new<T: InternerDyn + 'static>(t: T) -> Self { Self(Rc::new(t) as _) }
-	pub fn new_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static) -> Self {
+	pub async fn is(&self, s: &(impl Borrow<str> + ?Sized)) -> IStr {
-		Self { master: Some(Box::new(req)), interners: Mutex::default() }
+		IStr(self.0.is(s.borrow()).await)
 	}
-	/// Intern some data; query its identifier if not known locally
+	pub async fn iv(&self, s: &(impl Borrow<[IStr]> + ?Sized)) -> IVec {
-	pub async fn i<T: Interned>(&self, t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
+		IVec(self.0.iv(s.borrow()).await)
-		let data = t.get_owned();
-		let mut g = self.interners.lock().await;
-		let typed = T::bimap(&mut g);
-		if let Some(tok) = typed.by_value(&data) {
-			return tok;
-		}
-		let marker = match &self.master {
-			Some(c) => data.clone().intern(&**c).await,
-			None =>
-				T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
-		};
-		let tok = Tok::new(data, marker);
-		T::bimap(&mut g).insert(tok.clone());
-		tok
-	}
-	/// Extern an identifier; query the data it represents if not known locally
-	pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
-		if let Some(tok) = M::Interned::bimap(&mut *self.interners.lock().await).by_marker(marker) {
-			return tok;
-		}
-		assert!(self.master.is_some(), "ID not in local interner and this is master");
-		let token = marker.resolve(self).await;
-		M::Interned::bimap(&mut *self.interners.lock().await).insert(token.clone());
-		token
-	}
-	pub async fn sweep_replica(&self) -> api::Retained {
-		assert!(self.master.is_some(), "Not a replica");
-		let mut g = self.interners.lock().await;
-		api::Retained { strings: g.strings.sweep_replica(), vecs: g.vecs.sweep_replica() }
-	}
-	pub async fn sweep_master(&self, retained: api::Retained) {
-		assert!(self.master.is_none(), "Not master");
-		let mut g = self.interners.lock().await;
-		g.strings.sweep_master(retained.strings.into_iter().collect());
-		g.vecs.sweep_master(retained.vecs.into_iter().collect());
 	}
+	pub async fn es(&self, m: api::TStr) -> IStr { IStr(self.0.es(m).await) }
+	pub async fn ev(&self, m: api::TVec) -> IVec { IVec(self.0.ev(m).await) }
 }
-impl fmt::Debug for Interner {
+pub trait InternerDyn {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+	fn is(&self, s: &str) -> LocalBoxFuture<Rc<dyn IStrDyn>>;
-		write!(f, "Interner{{ replica: {} }}", self.master.is_none())
+	fn iv(&self, v: &[IStr]) -> LocalBoxFuture<Rc<dyn IVecDyn>>;
-	}
+	fn es(&self, m: api::TStr) -> LocalBoxFuture<Rc<dyn IStrDyn>>;
+	fn ev(&self, m: api::TVec) -> LocalBoxFuture<Rc<dyn IVecDyn>>;
 }
 
-static ID: atomic::AtomicU64 = atomic::AtomicU64::new(1);
+#[cfg(any(feature = "mocks", test))]
+pub mod test {
 
-pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) {
+	use std::cell::RefCell;
-	into.strings = into.strings.iter().chain(&from.strings).copied().unique().collect();
-	into.vecs = into.vecs.iter().chain(&from.vecs).copied().unique().collect();
-}
-
-#[cfg(test)]
-mod test {
 	use std::num::NonZero;
-	use std::pin::Pin;
+	use std::sync::atomic::AtomicU64;
-
-	use orchid_api_traits::{Decode, enc_vec};
-	use test_executors::spin_on;
 
 	use super::*;
-	use crate::api;
+	use crate::testing::AsyncMonitor;
 
-	#[test]
+	pub(crate) struct DummyIStr(NonZeroU64, String);
-	fn test_i() {
+	impl Deref for DummyIStr {
-		let i = Interner::new_master();
+		type Target = str;
-		let _: Tok<String> = spin_on(i.i("foo"));
+		fn deref(&self) -> &Self::Target { &self.1 }
-		let _: Tok<Vec<Tok<String>>> = spin_on(i.i(&[spin_on(i.i("bar")), spin_on(i.i("baz"))]));
+	}
+	impl IStrDyn for DummyIStr {
+		fn to_api(&self) -> api::TStr { api::TStr(self.0) }
 	}
 
-	#[test]
+	pub(crate) struct DummyIStrv(NonZeroU64, Vec<IStr>);
-	fn test_coding() {
+	impl Deref for DummyIStrv {
-		spin_on(async {
+		type Target = [IStr];
-			let coded = api::TStr(NonZero::new(3u64).unwrap());
+		fn deref(&self) -> &Self::Target { &self.1 }
-			let mut enc = &enc_vec(&coded).await[..];
-			api::TStr::decode(Pin::new(&mut enc)).await;
-			assert_eq!(enc, [], "Did not consume all of {enc:?}")
-		})
 	}
+	impl IVecDyn for DummyIStrv {
+		fn to_api(&self) -> api::TVec { api::TVec(self.0) }
+	}
+
+	pub(crate) struct DummyInterner(
+		RefCell<(
+			HashMap<String, NonZeroU64>,
+			HashMap<NonZeroU64, Rc<DummyIStr>>,
+			HashMap<NonZeroU64, Rc<DummyIStrv>>,
+		)>,
+		AsyncMonitor<InternerEvent>,
+	);
+	pub enum InternerEvent {
+		ExternStr(Rc<DummyIStr>),
+		ExternVec(Rc<DummyIStrv>),
+		InternStr { token: Rc<DummyIStr>, new: bool },
+		InternVec { token: Rc<DummyIStrv>, new: bool },
+	}
+	impl DummyInterner {
+		pub fn new(monitor: AsyncMonitor<InternerEvent>) -> Interner {
+			Interner(Rc::new(Self(RefCell::default(), monitor)))
+		}
+	}
+	impl InternerDyn for DummyInterner {
+		fn es(&self, m: api::TStr) -> LocalBoxFuture<Rc<dyn IStrDyn>> {
+			let state = self.0.borrow();
+			let istr = state.1.get(&m.0).unwrap_or_else(|| panic!("Externed nonexistent {m:?}")).clone();
+			Box::pin(async {
+				self.1.notify(InternerEvent::ExternStr(istr.clone())).await;
+				istr as Rc<dyn IStrDyn>
+			})
+		}
+		fn ev(&self, m: api::TVec) -> LocalBoxFuture<Rc<dyn IVecDyn>> {
+			let state = self.0.borrow();
+			let ivec = state.2.get(&m.0).unwrap_or_else(|| panic!("Externed nonexistent {m:?}")).clone();
+			Box::pin(async {
+				self.1.notify(InternerEvent::ExternVec(ivec.clone())).await;
+				ivec as Rc<dyn IVecDyn>
+			})
+		}
+		fn is(&self, s: &str) -> LocalBoxFuture<Rc<dyn IStrDyn>> {
+			let mut this = self.0.borrow_mut();
+			let id = *(this.0.entry(format!("{s:?}")))
+				.or_insert_with(|| NonZero::new(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)).unwrap());
+			let (tok, new) = match this.1.entry(id) {
+				hashbrown::hash_map::Entry::Occupied(ent) => (ent.get().clone(), false),
+				hashbrown::hash_map::Entry::Vacant(ent) =>
+					(ent.insert(Rc::new(DummyIStr(id, s.to_string()))).clone(), true),
+			};
+			Box::pin(async move {
+				self.1.notify(InternerEvent::InternStr { token: tok.clone(), new }).await;
+				tok as _
+			})
+		}
+		fn iv(&self, s: &[IStr]) -> LocalBoxFuture<Rc<dyn IVecDyn>> {
+			let mut this = self.0.borrow_mut();
+			let id = *(this.0.entry(format!("{s:?}")))
+				.or_insert_with(|| NonZero::new(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)).unwrap());
+			let (tok, new) = match this.2.entry(id) {
+				hashbrown::hash_map::Entry::Occupied(ent) => (ent.get().clone(), false),
+				hashbrown::hash_map::Entry::Vacant(ent) =>
+					(ent.insert(Rc::new(DummyIStrv(id, s.to_vec()))).clone(), true),
+			};
+			Box::pin(async move {
+				self.1.notify(InternerEvent::InternVec { token: tok.clone(), new }).await;
+				tok as _
+			})
+		}
+	}
+	static COUNTER: AtomicU64 = AtomicU64::new(1);
 }
+
+// /// The number of references held to any token by the interner.
+// const BASE_RC: usize = 3;
+
+// #[test]
+// fn base_rc_correct() {
+// 	let tok = Tok::new(Rc::new("foo".to_string()),
+// api::TStr(1.try_into().unwrap())); 	let mut bimap = Bimap::default();
+// 	bimap.insert(tok.clone());
+// 	assert_eq!(Rc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the
+// current instance"); }
+
+// pub struct Bimap<K, V, Tok> {
+// 	intern: HashMap<V, Tok>,
+// 	by_id: HashMap<K, Tok>,
+// }
+// impl<K> Bimap<T> {
+// 	pub fn insert(&mut self, token: Tok<T>) {
+// 		self.intern.insert(token.data.clone(), token.clone());
+// 		self.by_id.insert(token.to_api(), token);
+// 	}
+
+// 	pub fn by_marker(&self, marker: T::Marker) -> Option<Tok<T>> {
+// self.by_id.get(&marker).cloned() }
+
+// 	pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Tok<T>>
+// 	where T: Borrow<Q> {
+// 		(self.intern.raw_entry())
+// 			.from_hash(self.intern.hasher().hash_one(q), |k| k.as_ref().borrow() == q)
+// 			.map(|p| p.1.clone())
+// 	}
+
+// 	pub fn sweep_replica(&mut self) -> Vec<T::Marker> {
+// 		(self.intern)
+// 			.extract_if(|k, _| Rc::strong_count(k) == BASE_RC)
+// 			.map(|(_, v)| {
+// 				self.by_id.remove(&v.to_api());
+// 				v.to_api()
+// 			})
+// 			.collect()
+// 	}
+
+// 	pub fn sweep_master(&mut self, retained: HashSet<T::Marker>) {
+// 		self.intern.retain(|k, v| BASE_RC < Rc::strong_count(k) ||
+// retained.contains(&v.to_api())) 	}
+// }
+
+// impl<T: Interned> Default for Bimap<T> {
+// 	fn default() -> Self { Self { by_id: HashMap::new(), intern: HashMap::new()
+// } } }
+
+// pub trait UpComm {
+// 	fn up<R: Request>(&self, req: R) -> R::Response;
+// }
+
+// #[derive(Default)]
+// pub struct TypedInterners {
+// 	strings: Bimap<String>,
+// 	vecs: Bimap<Vec<IStr>>,
+// }
+
+// #[derive(Default)]
+// pub struct InternerData {
+// 	interners: Mutex<TypedInterners>,
+// 	master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
+// }
+// #[derive(Clone, Default)]
+// pub struct Interner(Rc<InternerData>);
+// impl Interner {
+// 	pub fn new_master() -> Self { Self::default() }
+// 	pub fn new_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static)
+// -> Self { 		Self(Rc::new(InternerData { master: Some(Box::new(req)),
+// interners: Mutex::default() })) 	}
+// 	/// Intern some data; query its identifier if not known locally
+// 	pub async fn i<T: Interned>(&self, t: &(impl Internable<Interned = T> +
+// ?Sized)) -> Tok<T> { 		let data = t.get_owned();
+// 		let mut g = self.0.interners.lock().await;
+// 		let typed = T::bimap(&mut g);
+// 		if let Some(tok) = typed.by_value(&data) {
+// 			return tok;
+// 		}
+// 		let marker = match &self.0.master {
+// 			Some(c) => data.clone().intern(&**c).await,
+// 			None =>
+// 				T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1,
+// atomic::Ordering::Relaxed)).unwrap()), 		};
+// 		let tok = Tok::new(data, marker);
+// 		T::bimap(&mut g).insert(tok.clone());
+// 		tok
+// 	}
+// 	/// Extern an identifier; query the data it represents if not known locally
+// 	pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
+// 		if let Some(tok) = M::Interned::bimap(&mut
+// *self.0.interners.lock().await).by_marker(marker) { 			return tok;
+// 		}
+// 		assert!(self.0.master.is_some(), "ID not in local interner and this is
+// master"); 		let token = marker.resolve(self).await;
+// 		M::Interned::bimap(&mut
+// *self.0.interners.lock().await).insert(token.clone()); 		token
+// 	}
+// 	pub async fn sweep_replica(&self) -> api::Retained {
+// 		assert!(self.0.master.is_some(), "Not a replica");
+// 		let mut g = self.0.interners.lock().await;
+// 		api::Retained { strings: g.strings.sweep_replica(), vecs:
+// g.vecs.sweep_replica() } 	}
+// 	pub async fn sweep_master(&self, retained: api::Retained) {
+// 		assert!(self.0.master.is_none(), "Not master");
+// 		let mut g = self.0.interners.lock().await;
+// 		g.strings.sweep_master(retained.strings.into_iter().collect());
+// 		g.vecs.sweep_master(retained.vecs.into_iter().collect());
+// 	}
+// }
+// impl fmt::Debug for Interner {
+// 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+// 		write!(f, "Interner{{ replica: {} }}", self.0.master.is_none())
+// 	}
+// }
+
+// static ID: atomic::AtomicU64 = atomic::AtomicU64::new(1);
+
+// pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) {
+// 	into.strings =
+// into.strings.iter().chain(&from.strings).copied().unique().collect();
+// 	into.vecs = into.vecs.iter().chain(&from.vecs).copied().unique().collect();
+// }
+
+// #[cfg(test)]
+// mod test {
+// 	use std::num::NonZero;
+// 	use std::pin::Pin;
+
+// 	use orchid_api_traits::{Decode, enc_vec};
+// 	use test_executors::spin_on;
+
+// 	use super::*;
+// 	use crate::api;
+
+// 	#[test]
+// 	fn test_i() {
+// 		let i = Interner::new_master();
+// 		let _: IStr = spin_on(i.i("foo"));
+// 		let _: Tok<Vec<IStr>> = spin_on(i.i(&[spin_on(i.i("bar")),
+// spin_on(i.i("baz"))])); 	}
+
+// 	#[test]
+// 	fn test_coding() {
+// 		spin_on(async {
+// 			let coded = api::TStr(NonZero::new(3u64).unwrap());
+// 			let mut enc = &enc_vec(&coded).await[..];
+// 			api::TStr::decode(Pin::new(&mut enc)).await;
+// 			assert_eq!(enc, [], "Did not consume all of {enc:?}")
+// 		})
+// 	}
+// }

orchid-base/src/iter_utils.rs

@@ -0,0 +1,24 @@
+use std::fmt;
+
+use itertools::{Itertools, Position};
+
+pub struct PrintList<'a, I: Iterator<Item = E> + Clone, E: fmt::Display>(pub I, pub &'a str);
+impl<'a, I: Iterator<Item = E> + Clone, E: fmt::Display> fmt::Display for PrintList<'a, I, E> {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		for (pos, item) in self.0.clone().with_position() {
+			match pos {
+				Position::First | Position::Only => write!(f, "{item}")?,
+				Position::Middle => write!(f, ", {item}")?,
+				Position::Last => write!(f, ", {} {item}", self.1)?,
+			}
+		}
+		Ok(())
+	}
+}
+
+pub trait IteratorPrint: Iterator<Item: fmt::Display> + Clone {
+	fn display<'a>(self, operator: &'a str) -> PrintList<'a, Self, Self::Item> {
+		PrintList(self, operator)
+	}
+}
+impl<T: Iterator<Item: fmt::Display> + Clone> IteratorPrint for T {}

orchid-base/src/lib.rs

@@ -7,15 +7,16 @@ pub mod builtin;
 pub mod char_filter;
 pub mod clone;
 pub mod combine;
+pub mod ctx;
 pub mod error;
 pub mod event;
 pub mod format;
 pub mod id_store;
 pub mod interner;
+pub mod iter_utils;
 pub mod join;
 pub mod location;
 pub mod logging;
-pub mod macros;
 mod match_mapping;
 pub mod msg;
 pub mod name;
@@ -25,6 +26,7 @@ pub mod pure_seq;
 pub mod reqnot;
 pub mod sequence;
 pub mod side;
+pub mod testing;
 mod tl_cache;
 pub mod tokens;
 pub mod tree;

orchid-base/src/location.rs

@@ -6,15 +6,16 @@ use std::ops::Range;
 
 use trait_set::trait_set;
 
-use crate::interner::{Interner, Tok};
+use crate::error::ErrPos;
+use crate::interner::{IStr, Interner};
 use crate::name::Sym;
 use crate::{api, match_mapping, sym};
 
 trait_set! {
-	pub trait GetSrc = FnMut(&Sym) -> Tok<String>;
+	pub trait GetSrc = FnMut(&Sym) -> IStr;
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Pos {
 	None,
 	SlotTarget,
@@ -23,15 +24,13 @@ pub enum Pos {
 	Inherit,
 	Gen(CodeGenInfo),
 	/// Range and file
-	SourceRange(SourceRange),
+	SrcRange(SrcRange),
-	/// Range only, file implied. Most notably used by parsers
-	Range(Range<u32>),
 }
 impl Pos {
 	pub fn pretty_print(&self, get_src: &mut impl GetSrc) -> String {
 		match self {
 			Self::Gen(g) => g.to_string(),
-			Self::SourceRange(sr) => sr.pretty_print(&get_src(&sr.path)),
+			Self::SrcRange(sr) => sr.pretty_print(&get_src(&sr.path)),
 			// Can't pretty print partial and meta-location
 			other => format!("{other:?}"),
 		}
@@ -39,30 +38,41 @@ impl Pos {
 	pub async fn from_api(api: &api::Location, i: &Interner) -> Self {
 		match_mapping!(api, api::Location => Pos {
 			None, Inherit, SlotTarget,
-			Range(r.clone()),
 			Gen(cgi => CodeGenInfo::from_api(cgi, i).await),
-			SourceRange(sr => SourceRange::from_api(sr, i).await)
+		} {
+			api::Location::SourceRange(sr) => Self::SrcRange(SrcRange::from_api(sr, i).await)
 		})
 	}
 	pub fn to_api(&self) -> api::Location {
 		match_mapping!(self, Pos => api::Location {
 			None, Inherit, SlotTarget,
-			Range(r.clone()),
 			Gen(cgi.to_api()),
-			SourceRange(sr.to_api()),
+		} {
+			Self::SrcRange(sr) => api::Location::SourceRange(sr.to_api()),
 		})
 	}
 }
+impl fmt::Display for Pos {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Pos::Inherit => f.write_str("Unresolved inherited position"),
+			Pos::SlotTarget => f.write_str("Unresolved slot target position"),
+			Pos::None => f.write_str("N/A"),
+			Pos::Gen(g) => write!(f, "{g}"),
+			Pos::SrcRange(sr) => write!(f, "{sr}"),
+		}
+	}
+}
 
 /// Exact source code location. Includes where the code was loaded from, what
 /// the original source code was, and a byte range.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct SourceRange {
+pub struct SrcRange {
 	pub(crate) path: Sym,
 	pub(crate) range: Range<u32>,
 }
-impl SourceRange {
+impl SrcRange {
-	pub fn new(range: &Range<u32>, path: &Sym) -> Self {
+	pub fn new(range: Range<u32>, path: &Sym) -> Self {
 		Self { range: range.clone(), path: path.clone() }
 	}
 	/// Create a dud [SourceRange] for testing. Its value is unspecified and
@@ -77,7 +87,7 @@ impl SourceRange {
 	/// 0-based index of last byte + 1
 	pub fn end(&self) -> u32 { self.range.end }
 	/// Syntactic location
-	pub fn pos(&self) -> Pos { Pos::SourceRange(self.clone()) }
+	pub fn pos(&self) -> Pos { Pos::SrcRange(self.clone()) }
 	/// Transform the numeric byte range
 	pub fn map_range(&self, map: impl FnOnce(Range<u32>) -> Range<u32>) -> Self {
 		Self { range: map(self.range()), path: self.path() }
@@ -92,12 +102,27 @@ impl SourceRange {
 		}
 	}
 	pub fn zw(path: Sym, pos: u32) -> Self { Self { path, range: pos..pos } }
-	async fn from_api(api: &api::SourceRange, i: &Interner) -> Self {
+	pub async fn from_api(api: &api::SourceRange, i: &Interner) -> Self {
 		Self { path: Sym::from_api(api.path, i).await, range: api.range.clone() }
 	}
-	fn to_api(&self) -> api::SourceRange {
+	pub fn to_api(&self) -> api::SourceRange {
 		api::SourceRange { path: self.path.to_api(), range: self.range.clone() }
 	}
+	pub fn to(&self, rhs: &Self) -> Self {
+		assert_eq!(self.path, rhs.path, "Range continues across files");
+		Self { path: self.path(), range: self.start().min(rhs.start())..self.end().max(rhs.end()) }
+	}
+}
+impl From<SrcRange> for ErrPos {
+	fn from(val: SrcRange) -> Self { val.pos().into() }
+}
+impl fmt::Display for SrcRange {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self.range.len() {
+			0 => write!(f, "{}:{}", self.path(), self.range.start),
+			n => write!(f, "{}:{}+{}", self.path(), self.range.start, n),
+		}
+	}
 }
 
 /// Information about a code generator attached to the generated code
@@ -106,24 +131,21 @@ pub struct CodeGenInfo {
 	/// formatted like a Rust namespace
 	pub generator: Sym,
 	/// Unformatted user message with relevant circumstances and parameters
-	pub details: Tok<String>,
+	pub details: IStr,
 }
 impl CodeGenInfo {
 	/// A codegen marker with no user message and parameters
 	pub async fn new_short(generator: Sym, i: &Interner) -> Self {
-		Self { generator, details: i.i("").await }
+		Self { generator, details: i.is("").await }
 	}
 	/// A codegen marker with a user message or parameters
 	pub async fn new_details(generator: Sym, details: impl AsRef<str>, i: &Interner) -> Self {
-		Self { generator, details: i.i(details.as_ref()).await }
+		Self { generator, details: i.is(details.as_ref()).await }
 	}
 	/// Syntactic location
 	pub fn pos(&self) -> Pos { Pos::Gen(self.clone()) }
 	pub async fn from_api(api: &api::CodeGenInfo, i: &Interner) -> Self {
-		Self {
+		Self { generator: Sym::from_api(api.generator, i).await, details: i.es(api.details).await }
-			generator: Sym::from_api(api.generator, i).await,
-			details: Tok::from_api(api.details, i).await,
-		}
 	}
 	pub fn to_api(&self) -> api::CodeGenInfo {
 		api::CodeGenInfo { generator: self.generator.to_api(), details: self.details.to_api() }

orchid-base/src/macros.rs

@@ -1,168 +0,0 @@
-use std::marker::PhantomData;
-use std::rc::Rc;
-use std::sync::Arc;
-
-use async_stream::stream;
-use futures::future::{LocalBoxFuture, join_all};
-use futures::{FutureExt, StreamExt};
-use never::Never;
-use trait_set::trait_set;
-
-use crate::format::{FmtCtx, FmtUnit, Format, Variants};
-use crate::interner::Interner;
-use crate::location::Pos;
-use crate::name::Sym;
-use crate::tree::{Paren, Ph};
-use crate::{api, match_mapping, tl_cache};
-
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
-pub struct MacroSlot<'a>(api::MacroTreeId, PhantomData<&'a ()>);
-impl MacroSlot<'_> {
-	pub fn id(self) -> api::MacroTreeId { self.0 }
-}
-
-trait_set! {
-	pub trait MacroAtomToApi<A> = for<'a> FnMut(&'a A) -> LocalBoxFuture<'a, api::MacroToken>;
-	pub trait MacroAtomFromApi<'a, A> =
-		for<'b> FnMut(&'b api::Atom) -> LocalBoxFuture<'b, MTok<'a, A>>;
-}
-
-#[derive(Clone, Debug)]
-pub struct MTree<'a, A> {
-	pub pos: Pos,
-	pub tok: Rc<MTok<'a, A>>,
-}
-impl<'a, A> MTree<'a, A> {
-	pub(crate) async fn from_api(
-		api: &api::MacroTree,
-		do_atom: &mut impl MacroAtomFromApi<'a, A>,
-		i: &Interner,
-	) -> Self {
-		Self {
-			pos: Pos::from_api(&api.location, i).await,
-			tok: Rc::new(MTok::from_api(&api.token, i, do_atom).await),
-		}
-	}
-	pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroTree {
-		api::MacroTree {
-			location: self.pos.to_api(),
-			token: self.tok.to_api(do_atom).boxed_local().await,
-		}
-	}
-}
-impl<A: Format> Format for MTree<'_, A> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		self.tok.print(c).boxed_local().await
-	}
-}
-
-#[derive(Clone, Debug)]
-pub enum MTok<'a, A> {
-	S(Paren, Vec<MTree<'a, A>>),
-	Name(Sym),
-	Slot(MacroSlot<'a>),
-	Lambda(Vec<MTree<'a, A>>, Vec<MTree<'a, A>>),
-	Ph(Ph),
-	Atom(A),
-	/// Used in extensions to directly return input
-	Ref(Arc<MTok<'a, Never>>),
-	/// Used in the matcher to skip previous macro output which can only go in
-	/// vectorial placeholders
-	Done(Rc<MTok<'a, A>>),
-}
-impl<'a, A> MTok<'a, A> {
-	pub(crate) async fn from_api(
-		api: &api::MacroToken,
-		i: &Interner,
-		do_atom: &mut impl MacroAtomFromApi<'a, A>,
-	) -> Self {
-		match_mapping!(&api, api::MacroToken => MTok::<'a, A> {
-			Lambda(x => mtreev_from_api(x, i, do_atom).await, b => mtreev_from_api(b, i, do_atom).await),
-			Name(t => Sym::from_api(*t, i).await),
-			Slot(tk => MacroSlot(*tk, PhantomData)),
-			S(p.clone(), b => mtreev_from_api(b, i, do_atom).await),
-			Ph(ph => Ph::from_api(ph, i).await),
-		} {
-			api::MacroToken::Atom(a) => do_atom(a).await
-		})
-	}
-	pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroToken {
-		fn sink<T>(n: &Never) -> LocalBoxFuture<'_, T> { match *n {} }
-		match_mapping!(&self, MTok => api::MacroToken {
-			Lambda(x => mtreev_to_api(x, do_atom).await, b => mtreev_to_api(b, do_atom).await),
-			Name(t.tok().to_api()),
-			Ph(ph.to_api()),
-			S(p.clone(), b => mtreev_to_api(b, do_atom).await),
-			Slot(tk.0.clone()),
-		} {
-			MTok::Ref(r) => r.to_api(&mut sink).boxed_local().await,
-			MTok::Done(t) => t.to_api(do_atom).boxed_local().await,
-			MTok::Atom(a) => do_atom(a).await,
-		})
-	}
-	pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Rc::new(self) } }
-}
-impl<A: Format> Format for MTok<'_, A> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		match self {
-			Self::Atom(a) => a.print(c).await,
-			Self::Done(d) =>
-				FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("(Done){0l}"))), [
-					d.print(c).boxed_local().await,
-				]),
-			Self::Lambda(arg, b) => FmtUnit::new(
-				tl_cache!(Rc<Variants>: Rc::new(Variants::default()
-					.unbounded("\\{0b}.{1l}")
-					.bounded("(\\{0b}.{1b})"))),
-				[mtreev_fmt(arg, c).await, mtreev_fmt(b, c).await],
-			),
-			Self::Name(n) => format!("{n}").into(),
-			Self::Ph(ph) => format!("{ph}").into(),
-			Self::Ref(r) =>
-				FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("(ref){0l}"))), [
-					r.print(c).boxed_local().await,
-				]),
-			Self::S(p, body) => FmtUnit::new(
-				match *p {
-					Paren::Round => Rc::new(Variants::default().bounded("({0b})")),
-					Paren::Curly => Rc::new(Variants::default().bounded("{{0b}}")),
-					Paren::Square => Rc::new(Variants::default().bounded("[{0b}]")),
-				},
-				[mtreev_fmt(body, c).await],
-			),
-			Self::Slot(slot) => format!("{:?}", slot.0).into(),
-		}
-	}
-}
-
-pub async fn mtreev_from_api<'a, 'b, A>(
-	apiv: impl IntoIterator<Item = &'b api::MacroTree>,
-	i: &Interner,
-	do_atom: &'b mut (impl MacroAtomFromApi<'a, A> + 'b),
-) -> Vec<MTree<'a, A>> {
-	stream! {
-		for api in apiv {
-			yield MTree::from_api(api, do_atom, i).boxed_local().await
-		}
-	}
-	.collect()
-	.await
-}
-
-pub async fn mtreev_to_api<'a: 'b, 'b, A: 'b>(
-	v: impl IntoIterator<Item = &'b MTree<'a, A>>,
-	do_atom: &mut impl MacroAtomToApi<A>,
-) -> Vec<api::MacroTree> {
-	let mut out = Vec::new();
-	for t in v {
-		out.push(t.to_api(do_atom).await);
-	}
-	out
-}
-
-pub async fn mtreev_fmt<'a: 'b, 'b, A: 'b + Format>(
-	v: impl IntoIterator<Item = &'b MTree<'a, A>>,
-	c: &(impl FmtCtx + ?Sized),
-) -> FmtUnit {
-	FmtUnit::sequence(" ", None, join_all(v.into_iter().map(|t| t.print(c))).await)
-}

orchid-base/src/match_mapping.rs

@@ -1,4 +1,4 @@
-/// A shorthand for mapping over enums with identical structure. Used for
+/// A shorthand for mapping over enums with similar structure. Used for
 /// converting between owned enums and the corresponding API enums that only
 /// differ in the type of their fields.
 ///
@@ -7,7 +7,11 @@
 /// match_mapping!(self, ThisType => OtherType {
 ///   EmptyVariant,
 ///   TupleVariant(foo => intern(foo), bar.clone()),
-///   StructVariant{ a.to_api(), b }
+///   StructVariant{ a.to_api(), b },
+///   DedicatedConverter(value () convert)
+/// } {
+///   ThisType::DimorphicVariant(c) => OtherType::CorrespondingVariant(c.left(), c.right()),
+///   ThisType::UnexpectedVariant => panic!(),
 /// })
 /// ```
 #[macro_export]

orchid-base/src/msg.rs

@@ -1,10 +1,10 @@
 use std::io;
 use std::pin::Pin;
 
-use async_std::io::{Read, ReadExt, Write, WriteExt};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use orchid_api_traits::{Decode, Encode};
 
-pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result<()> {
+pub async fn send_msg(mut write: Pin<&mut impl AsyncWrite>, msg: &[u8]) -> io::Result<()> {
 	let mut len_buf = vec![];
 	u32::try_from(msg.len()).unwrap().encode(Pin::new(&mut len_buf)).await;
 	write.write_all(&len_buf).await?;
@@ -12,7 +12,7 @@ pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result
 	write.flush().await
 }
 
-pub async fn recv_msg(mut read: Pin<&mut impl Read>) -> io::Result<Vec<u8>> {
+pub async fn recv_msg(mut read: Pin<&mut impl AsyncRead>) -> io::Result<Vec<u8>> {
 	let mut len_buf = [0u8; (u32::BITS / 8) as usize];
 	read.read_exact(&mut len_buf).await?;
 	let len = u32::decode(Pin::new(&mut &len_buf[..])).await;

orchid-base/src/name.rs

@@ -2,64 +2,58 @@
 
 use std::borrow::Borrow;
 use std::hash::Hash;
-use std::iter::Cloned;
 use std::num::{NonZeroU64, NonZeroUsize};
-use std::ops::{Bound, Deref, Index, RangeBounds};
+use std::ops::{Deref, Index};
 use std::path::Path;
-use std::{fmt, slice, vec};
+use std::{fmt, vec};
 
 use futures::future::{OptionFuture, join_all};
 use itertools::Itertools;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::interner::{InternMarker, Interner, Tok};
+use crate::interner::{IStr, IVec, Interner};
 
 trait_set! {
 	/// Traits that all name iterators should implement
-	pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
+	pub trait NameIter = Iterator<Item = IStr> + DoubleEndedIterator + ExactSizeIterator;
 }
 
-/// A token path which may be empty. [VName] is the non-empty,
+/// A token path which may be empty. [VName] is the non-empty version
-/// [PathSlice] is the borrowed version
 #[derive(Clone, Default, Hash, PartialEq, Eq)]
-pub struct VPath(pub Vec<Tok<String>>);
+pub struct VPath(Vec<IStr>);
 impl VPath {
 	/// Collect segments into a vector
-	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn new(items: impl IntoIterator<Item = IStr>) -> Self { Self(items.into_iter().collect()) }
-		Self(items.into_iter().collect())
-	}
 	/// Number of path segments
 	pub fn len(&self) -> usize { self.0.len() }
 	/// Whether there are any path segments. In other words, whether this is a
 	/// valid name
 	pub fn is_empty(&self) -> bool { self.len() == 0 }
 	/// Prepend some tokens to the path
-	pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn prefix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(items.into_iter().chain(self.0).collect())
 	}
 	/// Append some tokens to the path
-	pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn suffix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Partition the string by `::` namespace separators
 	pub async fn parse(s: &str, i: &Interner) -> Self {
-		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(|s| i.i(s))).await })
+		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(|s| i.is(s))).await })
 	}
 	/// Walk over the segments
-	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
+	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> { self.0.iter().map(|s| s.as_ref()) }
-		Box::new(self.0.iter().map(|s| s.as_str()))
-	}
 	/// Try to convert into non-empty version
 	pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
 	/// Add a token to the path. Since now we know that it can't be empty, turn it
 	/// into a name.
-	pub fn name_with_prefix(self, name: Tok<String>) -> VName {
+	pub fn name_with_suffix(self, name: IStr) -> VName {
 		VName(self.into_iter().chain([name]).collect())
 	}
 	/// Add a token to the beginning of the. Since now we know that it can't be
 	/// empty, turn it into a name.
-	pub fn name_with_suffix(self, name: Tok<String>) -> VName {
+	pub fn name_with_prefix(self, name: IStr) -> VName {
 		VName([name].into_iter().chain(self).collect())
 	}
 
@@ -67,7 +61,7 @@ impl VPath {
 	pub async fn from_path(path: &Path, ext: &str, i: &Interner) -> Option<(Self, bool)> {
 		async fn to_vpath(p: &Path, i: &Interner) -> Option<VPath> {
 			let tok_opt_v =
-				join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(|s| i.i(s))))).await;
+				join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(|s| i.is(s))))).await;
 			tok_opt_v.into_iter().collect::<Option<_>>().map(VPath)
 		}
 		match path.extension().map(|s| s.to_str()) {
@@ -85,30 +79,28 @@ impl fmt::Display for VPath {
 		write!(f, "{}", self.str_iter().join("::"))
 	}
 }
-impl FromIterator<Tok<String>> for VPath {
+impl FromIterator<IStr> for VPath {
-	fn from_iter<T: IntoIterator<Item = Tok<String>>>(iter: T) -> Self {
+	fn from_iter<T: IntoIterator<Item = IStr>>(iter: T) -> Self { Self(iter.into_iter().collect()) }
-		Self(iter.into_iter().collect())
-	}
 }
 impl IntoIterator for VPath {
-	type Item = Tok<String>;
+	type Item = IStr;
 	type IntoIter = vec::IntoIter<Self::Item>;
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
-impl Borrow<[Tok<String>]> for VPath {
+impl Borrow<[IStr]> for VPath {
-	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
+	fn borrow(&self) -> &[IStr] { &self.0[..] }
 }
 impl Deref for VPath {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
 impl<T> Index<T> for VPath
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
-	fn index(&self, index: T) -> &Self::Output { &Borrow::<[Tok<String>]>::borrow(self)[index] }
+	fn index(&self, index: T) -> &Self::Output { &Borrow::<[IStr]>::borrow(self)[index] }
 }
 
 /// A mutable representation of a namespaced identifier of at least one segment.
@@ -118,11 +110,11 @@ where [Tok<String>]: Index<T>
 /// See also [Sym] for the immutable representation, and [VPath] for possibly
 /// empty values
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct VName(Vec<Tok<String>>);
+pub struct VName(Vec<IStr>);
 impl VName {
 	/// Assert that the sequence isn't empty and wrap it in [VName] to represent
 	/// this invariant
-	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
+	pub fn new(items: impl IntoIterator<Item = IStr>) -> Result<Self, EmptyNameError> {
 		let data: Vec<_> = items.into_iter().collect();
 		if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
 	}
@@ -130,27 +122,27 @@ impl VName {
 		name: impl IntoIterator<Item = api::TStr>,
 		i: &Interner,
 	) -> Result<Self, EmptyNameError> {
-		Self::new(join_all(name.into_iter().map(|m| Tok::from_api(m, i))).await)
+		Self::new(join_all(name.into_iter().map(|m| i.es(m))).await)
 	}
 	/// Unwrap the enclosed vector
-	pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
+	pub fn into_vec(self) -> Vec<IStr> { self.0 }
 	/// Get a reference to the enclosed vector
-	pub fn vec(&self) -> &Vec<Tok<String>> { &self.0 }
+	pub fn vec(&self) -> &Vec<IStr> { &self.0 }
 	/// Mutable access to the underlying vector. To ensure correct results, this
 	/// must never be empty.
-	pub fn vec_mut(&mut self) -> &mut Vec<Tok<String>> { &mut self.0 }
+	pub fn vec_mut(&mut self) -> &mut Vec<IStr> { &mut self.0 }
 	/// Intern the name and return a [Sym]
-	pub async fn to_sym(&self, i: &Interner) -> Sym { Sym(i.i(&self.0[..]).await) }
+	pub async fn to_sym(&self, i: &Interner) -> Sym { Sym(i.iv(&self.0[..]).await) }
 	/// If this name has only one segment, return it
-	pub fn as_root(&self) -> Option<Tok<String>> { self.0.iter().exactly_one().ok().cloned() }
+	pub fn as_root(&self) -> Option<IStr> { self.0.iter().exactly_one().ok().cloned() }
 	/// Prepend the segments to this name
 	#[must_use = "This is a pure function"]
-	pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn prefix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(items.into_iter().chain(self.0).collect())
 	}
 	/// Append the segments to this name
 	#[must_use = "This is a pure function"]
-	pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn suffix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Read a `::` separated namespaced name
@@ -161,7 +153,7 @@ impl VName {
 		Self::parse(s, i).await.expect("empty literal !?")
 	}
 	/// Obtain an iterator over the segments of the name
-	pub fn iter(&self) -> impl Iterator<Item = Tok<String>> + '_ { self.0.iter().cloned() }
+	pub fn iter(&self) -> impl Iterator<Item = IStr> + '_ { self.0.iter().cloned() }
 }
 impl fmt::Debug for VName {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
@@ -172,22 +164,22 @@ impl fmt::Display for VName {
 	}
 }
 impl IntoIterator for VName {
-	type Item = Tok<String>;
+	type Item = IStr;
 	type IntoIter = vec::IntoIter<Self::Item>;
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
 impl<T> Index<T> for VName
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[Tok<String>]> for VName {
+impl Borrow<[IStr]> for VName {
-	fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
+	fn borrow(&self) -> &[IStr] { self.0.borrow() }
 }
 impl Deref for VName {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -195,11 +187,9 @@ impl Deref for VName {
 /// empty sequence
 #[derive(Debug, Copy, Clone, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct EmptyNameError;
-impl TryFrom<&[Tok<String>]> for VName {
+impl TryFrom<&[IStr]> for VName {
 	type Error = EmptyNameError;
-	fn try_from(value: &[Tok<String>]) -> Result<Self, Self::Error> {
+	fn try_from(value: &[IStr]) -> Result<Self, Self::Error> { Self::new(value.iter().cloned()) }
-		Self::new(value.iter().cloned())
-	}
 }
 
 /// An interned representation of a namespaced identifier.
@@ -208,35 +198,38 @@ impl TryFrom<&[Tok<String>]> for VName {
 ///
 /// See also [VName]
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct Sym(Tok<Vec<Tok<String>>>);
+pub struct Sym(IVec);
 impl Sym {
 	/// Assert that the sequence isn't empty, intern it and wrap it in a [Sym] to
 	/// represent this invariant
 	pub async fn new(
-		v: impl IntoIterator<Item = Tok<String>>,
+		v: impl IntoIterator<Item = IStr>,
 		i: &Interner,
 	) -> Result<Self, EmptyNameError> {
 		let items = v.into_iter().collect_vec();
-		Self::from_tok(i.i(&items).await)
+		Self::from_tok(i.iv(&items).await)
 	}
 	/// Read a `::` separated namespaced name.
 	pub async fn parse(s: &str, i: &Interner) -> Result<Self, EmptyNameError> {
-		Ok(Sym(i.i(&VName::parse(s, i).await?.into_vec()).await))
+		Ok(Sym(i.iv(&VName::parse(s, i).await?.into_vec()).await))
 	}
 	/// Assert that a token isn't empty, and wrap it in a [Sym]
-	pub fn from_tok(t: Tok<Vec<Tok<String>>>) -> Result<Self, EmptyNameError> {
+	pub fn from_tok(t: IVec) -> Result<Self, EmptyNameError> {
 		if t.is_empty() { Err(EmptyNameError) } else { Ok(Self(t)) }
 	}
 	/// Grab the interner token
-	pub fn tok(&self) -> Tok<Vec<Tok<String>>> { self.0.clone() }
+	pub fn tok(&self) -> IVec { self.0.clone() }
 	/// Get a number unique to this name suitable for arbitrary ordering.
-	pub fn id(&self) -> NonZeroU64 { self.0.to_api().get_id() }
+	pub fn id(&self) -> NonZeroU64 { self.0.to_api().0 }
 	/// Extern the sym for editing
 	pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
-	pub async fn from_api(marker: api::TStrv, i: &Interner) -> Sym {
+	pub async fn from_api(marker: api::TVec, i: &Interner) -> Sym {
-		Self::from_tok(Tok::from_api(marker, i).await).expect("Empty sequence found for serialized Sym")
+		Self::from_tok(i.ev(marker).await).expect("Empty sequence found for serialized Sym")
+	}
+	pub fn to_api(&self) -> api::TVec { self.tok().to_api() }
+	pub async fn suffix(&self, tokv: impl IntoIterator<Item = IStr>, i: &Interner) -> Sym {
+		Self::new(self.0.iter().cloned().chain(tokv), i).await.unwrap()
 	}
-	pub fn to_api(&self) -> api::TStrv { self.tok().to_api() }
 }
 impl fmt::Debug for Sym {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Sym({self})") }
@@ -247,17 +240,17 @@ impl fmt::Display for Sym {
 	}
 }
 impl<T> Index<T> for Sym
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[Tok<String>]> for Sym {
+impl Borrow<[IStr]> for Sym {
-	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
+	fn borrow(&self) -> &[IStr] { &self.0[..] }
 }
 impl Deref for Sym {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -265,39 +258,39 @@ impl Deref for Sym {
 /// handled together in datastructures. The names can never be empty
 #[allow(clippy::len_without_is_empty)] // never empty
 pub trait NameLike:
-	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[Tok<String>]>
+	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[IStr]>
 {
 	/// Convert into held slice
-	fn as_slice(&self) -> &[Tok<String>] { Borrow::<[Tok<String>]>::borrow(self) }
+	fn as_slice(&self) -> &[IStr] { Borrow::<[IStr]>::borrow(self) }
 	/// Get iterator over tokens
-	fn iter(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
+	fn segs(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
 	/// Get iterator over string segments
 	fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ {
-		self.as_slice().iter().map(|t| t.as_str())
+		self.as_slice().iter().map(|t| t.as_ref())
 	}
 	/// Fully resolve the name for printing
 	#[must_use]
-	fn to_strv(&self) -> Vec<String> { self.iter().map(|s| s.to_string()).collect() }
+	fn to_strv(&self) -> Vec<String> { self.segs().map(|s| s.to_string()).collect() }
 	/// Format the name as an approximate filename
-	fn as_src_path(&self) -> String { format!("{}.orc", self.iter().join("/")) }
+	fn as_src_path(&self) -> String { format!("{}.orc", self.segs().join("/")) }
 	/// Return the number of segments in the name
-	fn len(&self) -> NonZeroUsize {
+	fn len_nz(&self) -> NonZeroUsize {
-		NonZeroUsize::try_from(self.iter().count()).expect("NameLike never empty")
+		NonZeroUsize::try_from(self.segs().count()).expect("NameLike never empty")
 	}
 	/// Like slice's `split_first` except we know that it always returns Some
-	fn split_first(&self) -> (Tok<String>, &[Tok<String>]) {
+	fn split_first_seg(&self) -> (IStr, &[IStr]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Like slice's `split_last` except we know that it always returns Some
-	fn split_last(&self) -> (Tok<String>, &[Tok<String>]) {
+	fn split_last_seg(&self) -> (IStr, &[IStr]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Get the first element
-	fn first(&self) -> Tok<String> { self.split_first().0 }
+	fn first_seg(&self) -> IStr { self.split_first_seg().0 }
 	/// Get the last element
-	fn last(&self) -> Tok<String> { self.split_last().0 }
+	fn last_seg(&self) -> IStr { self.split_last_seg().0 }
 }
 
 impl NameLike for Sym {}
@@ -312,9 +305,9 @@ impl NameLike for VName {}
 macro_rules! sym {
   ($seg1:tt $( :: $seg:tt)* ; $i:expr) => { async {
 		$crate::name::Sym::from_tok(
-			$i.i(&[
+			$i.iv(&[
-				$i.i(stringify!($seg1)).await
+				$i.is(stringify!($seg1)).await
-				$( , $i.i(stringify!($seg)).await )*
+				$( , $i.is(stringify!($seg)).await )*
 			])
 			.await
 		).unwrap()
@@ -330,8 +323,8 @@ macro_rules! sym {
 macro_rules! vname {
   ($seg1:tt $( :: $seg:tt)* ; $i:expr) => { async {
     $crate::name::VName::new([
-      $i.i(stringify!($seg1)).await
+      $i.is(stringify!($seg1)).await
-      $( , $i.i(stringify!($seg)).await )*
+      $( , $i.is(stringify!($seg)).await )*
     ]).unwrap()
 	} };
 }
@@ -343,8 +336,8 @@ macro_rules! vname {
 macro_rules! vpath {
   ($seg1:tt $( :: $seg:tt)+ ; $i:expr) => { async {
     $crate::name::VPath(vec![
-      $i.i(stringify!($seg1)).await
+      $i.is(stringify!($seg1)).await
-      $( , $i.i(stringify!($seg)).await )+
+      $( , $i.is(stringify!($seg)).await )+
 		])
 	} };
   () => {
@@ -359,35 +352,37 @@ mod test {
 	use test_executors::spin_on;
 
 	use super::{NameLike, Sym, VName};
-	use crate::interner::{Interner, Tok};
+	use crate::interner::test::DummyInterner;
+	use crate::interner::{IStr, Interner};
 	use crate::name::VPath;
+	use crate::testing::AsyncMonitor;
 
 	#[test]
 	fn recur() {
 		spin_on(async {
-			let i = Interner::new_master();
+			let i = DummyInterner::new(AsyncMonitor::default());
 			let myname = vname!(foo::bar; i).await;
-			let _borrowed_slice: &[Tok<String>] = myname.borrow();
+			let _borrowed_slice: &[IStr] = myname.borrow();
-			let _deref_pathslice: &[Tok<String>] = &myname;
+			let _deref_pathslice: &[IStr] = &myname;
-			let _as_slice_out: &[Tok<String>] = myname.as_slice();
+			let _as_slice_out: &[IStr] = myname.as_slice();
 		})
 	}
 
 	#[test]
 	fn literals() {
 		spin_on(async {
-			let i = Interner::new_master();
+			let i = DummyInterner::new(AsyncMonitor::default());
 			assert_eq!(
 				sym!(foo::bar::baz; i).await,
-				Sym::new([i.i("foo").await, i.i("bar").await, i.i("baz").await], &i).await.unwrap()
+				Sym::new([i.is("foo").await, i.is("bar").await, i.is("baz").await], &i).await.unwrap()
 			);
 			assert_eq!(
 				vname!(foo::bar::baz; i).await,
-				VName::new([i.i("foo").await, i.i("bar").await, i.i("baz").await]).unwrap()
+				VName::new([i.is("foo").await, i.is("bar").await, i.is("baz").await]).unwrap()
 			);
 			assert_eq!(
 				vpath!(foo::bar::baz; i).await,
-				VPath::new([i.i("foo").await, i.i("bar").await, i.i("baz").await])
+				VPath::new([i.is("foo").await, i.is("bar").await, i.is("baz").await])
 			);
 		})
 	}

orchid-base/src/number.rs

@@ -1,35 +1,27 @@
 use std::num::IntErrorKind;
 use std::ops::Range;
 
-use num_traits::ToPrimitive;
 use ordered_float::NotNan;
-use rust_decimal::Decimal;
 
-use crate::error::{OrcErr, mk_err};
+use crate::ctx::Ctx;
-use crate::interner::Interner;
+use crate::error::OrcErr;
-use crate::location::Pos;
+use crate::location::SrcRange;
+use crate::name::Sym;
 
 /// A number, either floating point or unsigned int, parsed by Orchid.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Numeric {
-	/// A nonnegative integer
+	/// An integer
-	Uint(u64),
+	Int(i64),
 	/// A binary float other than NaN
 	Float(NotNan<f64>),
-	/// A decimal number
-	Decimal(Decimal),
 }
 impl Numeric {
-	pub fn decimal(num: i64, scale: u32) -> Self { Self::Decimal(Decimal::new(num, scale)) }
 	pub fn float(value: f64) -> Self { Self::Float(NotNan::new(value).unwrap()) }
 	pub fn to_f64(self) -> NotNan<f64> {
 		match self {
 			Self::Float(f) => f,
-			Self::Decimal(d) => {
+			Self::Int(i) => NotNan::new(i as f64).expect("int cannot be NaN"),
-				let f = d.to_f64().expect("This is apparently always possible");
-				NotNan::new(f).expect("decimal was nan")
-			},
-			Self::Uint(i) => NotNan::new(i as f64).expect("int cannot be NaN"),
 		}
 	}
 }
@@ -63,45 +55,48 @@ pub struct NumError {
 	pub kind: NumErrorKind,
 }
 
-pub async fn num_to_err(NumError { kind, range }: NumError, offset: u32, i: &Interner) -> OrcErr {
+pub fn num_to_errv(
-	mk_err(
+	NumError { kind, range }: NumError,
-		i.i("Failed to parse number").await,
+	offset: u32,
+	source: &Sym,
+	ctx: &Ctx,
+) -> OrcErr {
+	ctx.mk_err(
+		"Failed to parse number",
 		match kind {
 			NumErrorKind::NaN => "NaN emerged during parsing",
 			NumErrorKind::InvalidDigit => "non-digit character encountered",
 			NumErrorKind::Overflow => "The number being described is too large or too accurate",
 		},
-		[Pos::Range(offset + range.start as u32..offset + range.end as u32).into()],
+		[SrcRange::new(offset + range.start as u32..offset + range.end as u32, source)],
 	)
 }
 
 /// Parse a numbre literal out of text
 pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
-	let overflow_err = NumError { range: 0..string.len(), kind: NumErrorKind::Overflow };
+	let overflow_e = NumError { range: 0..string.len(), kind: NumErrorKind::Overflow };
 	let (radix, noprefix, pos) = (string.strip_prefix("0x").map(|s| (16u8, s, 2)))
 		.or_else(|| string.strip_prefix("0b").map(|s| (2u8, s, 2)))
 		.or_else(|| string.strip_prefix("0o").map(|s| (8u8, s, 2)))
 		.unwrap_or((10u8, string, 0));
-	eprintln!("({radix}, {noprefix}, {pos})");
 	// identity
-	let (base, exponent) = match noprefix.split_once('p') {
+	let (base_s, exponent) = match noprefix.split_once('p') {
 		Some((b, e)) => {
 			let (s, d, len) = e.strip_prefix('-').map_or((1, e, 0), |ue| (-1, ue, 1));
 			(b, s * int_parse(d, 10, pos + b.len() + 1 + len)? as i32)
 		},
 		None => (noprefix, 0),
 	};
-	eprintln!("({base},{exponent})");
+	match base_s.split_once('.') {
-	match base.split_once('.') {
 		None => {
-			let base_usize = int_parse(base, radix, pos)?;
+			let base = int_parse(base_s, radix, pos)?;
-			if let Ok(pos_exp) = u32::try_from(exponent) {
+			if let Ok(pos_exp) = u32::try_from(exponent)
-				if let Some(radical) = u64::from(radix).checked_pow(pos_exp) {
+				&& let Some(radical) = u64::from(radix).checked_pow(pos_exp)
-					let number = base_usize.checked_mul(radical).ok_or(overflow_err)?;
+			{
-					return Ok(Numeric::Uint(number));
+				let num = base.checked_mul(radical).and_then(|m| m.try_into().ok()).ok_or(overflow_e)?;
-				}
+				return Ok(Numeric::Int(num));
 			}
-			let f = (base_usize as f64) * (radix as f64).powi(exponent);
+			let f = (base as f64) * (radix as f64).powi(exponent);
 			let err = NumError { range: 0..string.len(), kind: NumErrorKind::NaN };
 			Ok(Numeric::Float(NotNan::new(f).map_err(|_| err)?))
 		},
@@ -109,25 +104,9 @@ pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
 			let whole_n = int_parse(whole, radix, pos)?;
 			let part_n = int_parse(part, radix, pos + whole.len() + 1)?;
 			let scale = part.chars().filter(|c| *c != '_').count() as u32;
-			if radix == 10 {
+			let real_val = whole_n as f64 + (part_n as f64 / (radix as f64).powi(scale as i32));
-				let scaled_unit = 10u64.checked_pow(scale).ok_or(overflow_err.clone())?;
+			let f = real_val * (radix as f64).powi(exponent);
-				let scaled_n = i128::from(whole_n) * i128::from(scaled_unit) + i128::from(part_n);
+			Ok(Numeric::Float(NotNan::new(f).expect("None of the inputs are NaN")))
-				let decimal = Decimal::from_i128_with_scale(scaled_n, scale);
-				let p = if let Ok(uexp) = u32::try_from(exponent) {
-					let e_multiplier = 10i64.checked_pow(uexp).ok_or(overflow_err)?;
-					Decimal::new(e_multiplier, 0)
-				} else {
-					let inv_oom = u32::try_from(-exponent).map_err(|_| overflow_err)?;
-					eprintln!("inv_oom: {inv_oom}");
-					Decimal::new(1, inv_oom)
-				};
-				eprintln!("({scaled_n}, {scale}, {p})");
-				Ok(Numeric::Decimal(decimal * p))
-			} else {
-				let real_val = whole_n as f64 + (part_n as f64 / (radix as f64).powi(scale as i32));
-				let f = real_val * (radix as f64).powi(exponent);
-				Ok(Numeric::Float(NotNan::new(f).expect("None of the inputs are NaN")))
-			}
 		},
 	}
 }
@@ -168,7 +147,7 @@ mod test {
 
 	#[test]
 	fn just_ints() {
-		let test = |s, n| assert_eq!(parse_num(s), Ok(Numeric::Uint(n)));
+		let test = |s, n| assert_eq!(parse_num(s), Ok(Numeric::Int(n)));
 		test("12345", 12345);
 		test("0xcafebabe", 0xcafebabe);
 		test("0o751", 0o751);
@@ -178,11 +157,11 @@ mod test {
 	#[test]
 	fn decimals() {
 		let test = |s, n| assert_eq!(parse_num(s), Ok(n));
-		test("3.1417", Numeric::decimal(31417, 4));
+		test("3.1417", Numeric::float(3.1417));
 		test("0xf.cafe", Numeric::float(0xf as f64 + 0xcafe as f64 / 0x10000 as f64));
-		test("34p3", Numeric::Uint(34000));
+		test("34p3", Numeric::Int(34000));
-		test("0x2p3", Numeric::Uint(0x2 * 0x1000));
+		test("0x2p3", Numeric::Int(0x2 * 0x1000));
-		test("1.5p3", Numeric::decimal(1500, 0));
+		test("1.5p3", Numeric::float(1500.0));
 		test("0x2.5p3", Numeric::float((0x25 * 0x100) as f64));
 	}
 }

orchid-base/src/parse.rs

@@ -1,17 +1,20 @@
 use std::fmt::{self, Display};
 use std::iter;
-use std::ops::{Deref, Range};
+use std::ops::Deref;
+use std::rc::Rc;
 
+use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
 
 use crate::api;
-use crate::error::{OrcRes, Reporter, mk_err, mk_errv};
+use crate::ctx::Ctx;
-use crate::format::{Format, take_first_fmt};
+use crate::error::{OrcErr, OrcRes, Reporter};
-use crate::interner::{Interner, Tok};
+use crate::format::{FmtCtx, FmtUnit, Format, fmt};
-use crate::location::Pos;
+use crate::interner::{IStr, Interner};
-use crate::name::{VName, VPath};
+use crate::location::SrcRange;
-use crate::tree::{AtomRepr, ExtraTok, Paren, TokTree, Token};
+use crate::name::{Sym, VName, VPath};
+use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_fmt, ttv_range};
 
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
 pub fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
@@ -21,51 +24,41 @@ pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
 /// A cheaply copiable subsection of a document that holds onto context data and
 /// one token for error reporting on empty subsections.
 #[derive(Debug)]
-pub struct Snippet<'a, 'b, A: AtomRepr, X: ExtraTok> {
+pub struct Snippet<'a, A: ExprRepr, X: ExtraTok> {
-	prev: &'a TokTree<'b, A, X>,
+	prev: &'a TokTree<A, X>,
-	cur: &'a [TokTree<'b, A, X>],
+	cur: &'a [TokTree<A, X>],
-	interner: &'a Interner,
 }
-impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
+impl<'a, A, X> Snippet<'a, A, X>
-	pub fn new(
+where
-		prev: &'a TokTree<'b, A, X>,
+	A: ExprRepr,
-		cur: &'a [TokTree<'b, A, X>],
+	X: ExtraTok,
-		interner: &'a Interner,
+{
-	) -> Self {
+	pub fn new(prev: &'a TokTree<A, X>, cur: &'a [TokTree<A, X>]) -> Self { Self { prev, cur } }
-		Self { prev, cur, interner }
-	}
-	pub fn i(&self) -> &'a Interner { self.interner }
 	pub fn split_at(self, pos: u32) -> (Self, Self) {
-		let Self { prev, cur, interner } = self;
+		let Self { prev, cur } = self;
-		let fst = Self { prev, cur: &cur[..pos as usize], interner };
+		let fst = Self { prev, cur: &cur[..pos as usize] };
 		let new_prev = if pos == 0 { self.prev } else { &self.cur[pos as usize - 1] };
-		let snd = Self { prev: new_prev, cur: &self.cur[pos as usize..], interner };
+		let snd = Self { prev: new_prev, cur: &self.cur[pos as usize..] };
 		(fst, snd)
 	}
-	pub fn find_idx(self, mut f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<u32> {
+	pub fn find_idx(self, mut f: impl FnMut(&Token<A, X>) -> bool) -> Option<u32> {
 		self.cur.iter().position(|t| f(&t.tok)).map(|t| t as u32)
 	}
-	pub fn get(self, idx: u32) -> Option<&'a TokTree<'b, A, X>> { self.cur.get(idx as usize) }
+	pub fn get(self, idx: u32) -> Option<&'a TokTree<A, X>> { self.cur.get(idx as usize) }
 	pub fn len(self) -> u32 { self.cur.len() as u32 }
-	pub fn prev(self) -> &'a TokTree<'b, A, X> { self.prev }
+	pub fn prev(self) -> &'a TokTree<A, X> { self.prev }
-	pub fn pos(self) -> Range<u32> {
+	pub fn sr(self) -> SrcRange { ttv_range(self.cur).unwrap_or_else(|| self.prev.sr.clone()) }
-		(self.cur.first().map(|f| f.range.start..self.cur.last().unwrap().range.end))
+	pub fn pop_front(self) -> Option<(&'a TokTree<A, X>, Self)> {
-			.unwrap_or(self.prev.range.clone())
-	}
-	pub fn pop_front(self) -> Option<(&'a TokTree<'b, A, X>, Self)> {
 		self.cur.first().map(|r| (r, self.split_at(1).1))
 	}
-	pub fn pop_back(self) -> Option<(Self, &'a TokTree<'b, A, X>)> {
+	pub fn pop_back(self) -> Option<(Self, &'a TokTree<A, X>)> {
 		self.cur.last().map(|r| (self.split_at(self.len() - 1).0, r))
 	}
-	pub fn split_once(self, f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<(Self, Self)> {
+	pub fn split_once(self, f: impl FnMut(&Token<A, X>) -> bool) -> Option<(Self, Self)> {
 		let idx = self.find_idx(f)?;
 		Some((self.split_at(idx).0, self.split_at(idx + 1).1))
 	}
-	pub fn split(
+	pub fn split(mut self, mut f: impl FnMut(&Token<A, X>) -> bool) -> impl Iterator<Item = Self> {
-		mut self,
-		mut f: impl FnMut(&Token<'b, A, X>) -> bool,
-	) -> impl Iterator<Item = Self> {
 		iter::from_fn(move || {
 			if self.is_empty() {
 				return None;
@@ -77,47 +70,45 @@ impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
 	}
 	pub fn is_empty(self) -> bool { self.len() == 0 }
 	pub fn skip_fluff(self) -> Self {
-		let non_fluff_start = self.find_idx(|t| !matches!(t, Token::NS | Token::Comment(_)));
+		let non_fluff_start = self.find_idx(|t| !matches!(t, Token::BR | Token::Comment(_)));
 		self.split_at(non_fluff_start.unwrap_or(self.len())).1
 	}
-	/// Format the argument using the context held in this snippet
-	pub async fn fmt(self, v: &(impl Format + ?Sized)) -> String { take_first_fmt(v, self.i()).await }
 }
-impl<A: AtomRepr, X: ExtraTok> Copy for Snippet<'_, '_, A, X> {}
+impl<A: ExprRepr, X: ExtraTok> Copy for Snippet<'_, A, X> {}
-impl<A: AtomRepr, X: ExtraTok> Clone for Snippet<'_, '_, A, X> {
+impl<A: ExprRepr, X: ExtraTok> Clone for Snippet<'_, A, X> {
 	fn clone(&self) -> Self { *self }
 }
-impl<'b, A: AtomRepr, X: ExtraTok> Deref for Snippet<'_, 'b, A, X> {
+impl<A: ExprRepr, X: ExtraTok> Deref for Snippet<'_, A, X> {
-	type Target = [TokTree<'b, A, X>];
+	type Target = [TokTree<A, X>];
 	fn deref(&self) -> &Self::Target { self.cur }
 }
-
+impl<A: ExprRepr, X: ExtraTok> Format for Snippet<'_, A, X> {
-/// Remove tokens that aren't meaningful in expression context, such as comments
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-/// or line breaks
+		ttv_fmt(&**self, c).await
-pub fn strip_fluff<'a, A: AtomRepr, X: ExtraTok>(
+	}
-	tt: &TokTree<'a, A, X>,
-) -> Option<TokTree<'a, A, X>> {
-	let tok = match &tt.tok {
-		Token::BR => return None,
-		Token::Comment(_) => return None,
-		Token::LambdaHead(arg) => Token::LambdaHead(arg.iter().filter_map(strip_fluff).collect()),
-		Token::S(p, b) => Token::S(*p, b.iter().filter_map(strip_fluff).collect()),
-		t => t.clone(),
-	};
-	Some(TokTree { tok, range: tt.range.clone() })
 }
 
 #[derive(Clone, Debug)]
 pub struct Comment {
-	pub text: Tok<String>,
+	pub text: IStr,
-	pub pos: Pos,
+	pub sr: SrcRange,
 }
 impl Comment {
-	pub fn to_api(&self) -> api::Comment {
+	// XXX: which of these four are actually used?
-		api::Comment { location: self.pos.to_api(), text: self.text.to_api() }
+	pub async fn from_api(c: &api::Comment, src: Sym, cx: &Ctx) -> Self {
+		Self { text: cx.i().es(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
 	}
-	pub async fn from_api(api: &api::Comment, i: &Interner) -> Self {
+	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>, cx: &Ctx) -> Option<Self> {
-		Self { pos: Pos::from_api(&api.location, i).await, text: Tok::from_api(api.text, i).await }
+		match &tk.tok {
+			Token::Comment(text) => Some(Self { text: cx.i().is(&**text).await, sr: tk.sr.clone() }),
+			_ => None,
+		}
+	}
+	pub fn to_tk<A: ExprRepr, X: ExtraTok>(&self) -> TokTree<A, X> {
+		TokTree { tok: Token::Comment(Rc::new(self.text.to_string())), sr: self.sr.clone() }
+	}
+	pub fn to_api(&self) -> api::Comment {
+		api::Comment { range: self.sr.range(), text: self.text.to_api() }
 	}
 }
 
@@ -125,9 +116,10 @@ impl fmt::Display for Comment {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "--[{}]--", self.text) }
 }
 
-pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, 'b, A, X>,
+	ctx: &Ctx,
-) -> Vec<Parsed<'a, 'b, Vec<Comment>, A, X>> {
+	snip: Snippet<'a, A, X>,
+) -> Vec<Parsed<'a, Vec<Comment>, A, X>> {
 	let mut items = Vec::new();
 	let mut comments = Vec::new();
 	for mut line in snip.split(|t| matches!(t, Token::BR)) {
@@ -141,11 +133,7 @@ pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
 			Some(i) => {
 				let (cmts, tail) = line.split_at(i);
 				let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async {
-					match &t.tok {
+					Comment::from_tk(t, ctx).await.expect("All are comments checked above")
-						Token::Comment(c) =>
-							Comment { text: tail.i().i(&**c).await, pos: Pos::Range(t.range.clone()) },
-						_ => unreachable!("All are comments checked above"),
-					}
 				}))
 				.await;
 				items.push(Parsed { output: comments, tail });
@@ -155,169 +143,145 @@ pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
 	items
 }
 
-pub async fn try_pop_no_fluff<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, 'b, A, X>,
+	ctx: &Ctx,
-) -> ParseRes<'a, 'b, &'a TokTree<'b, A, X>, A, X> {
+	snip: Snippet<'a, A, X>,
+) -> ParseRes<'a, &'a TokTree<A, X>, A, X> {
 	match snip.skip_fluff().pop_front() {
 		Some((output, tail)) => Ok(Parsed { output, tail }),
-		None =>
+		None => Err(
-			Err(mk_errv(snip.i().i("Unexpected end").await, "Pattern ends abruptly", [Pos::Range(
+			ctx.mk_err("Unexpected end", "Line ends abruptly; more tokens were expected", [snip.sr()]),
-				snip.pos(),
+		),
-			)
-			.into()])),
 	}
 }
 
-pub async fn expect_end(snip: Snippet<'_, '_, impl AtomRepr, impl ExtraTok>) -> OrcRes<()> {
+pub async fn expect_end(ctx: &Ctx, snip: Snippet<'_, impl ExprRepr, impl ExtraTok>) -> OrcRes<()> {
 	match snip.skip_fluff().get(0) {
-		Some(surplus) => Err(mk_errv(
+		Some(surplus) =>
-			snip.i().i("Extra code after end of line").await,
+			Err(ctx.mk_err("Extra code after end of line", "Code found after the end of the line", [
-			"Code found after the end of the line",
+				surplus.sr.pos(),
-			[Pos::Range(surplus.range.clone()).into()],
+			])),
-		)),
 		None => Ok(()),
 	}
 }
 
-pub async fn expect_tok<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, 'b, A, X>,
+	ctx: &Ctx,
-	tok: Tok<String>,
+	snip: Snippet<'a, A, X>,
-) -> ParseRes<'a, 'b, (), A, X> {
+	tok: IStr,
-	let Parsed { output: head, tail } = try_pop_no_fluff(snip).await?;
+) -> ParseRes<'a, (), A, X> {
+	let Parsed { output: head, tail } = try_pop_no_fluff(ctx, snip).await?;
 	match &head.tok {
 		Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
-		t => Err(mk_errv(
+		t => Err(ctx.mk_err(
-			snip.i().i("Expected specific keyword").await,
+			"Expected specific keyword",
-			format!("Expected {tok} but found {:?}", snip.fmt(t).await),
+			format!("Expected {tok} but found {:?}", fmt(t, &ctx.i()).await),
-			[Pos::Range(head.range.clone()).into()],
+			[head.sr()],
 		)),
 	}
 }
 
-pub struct Parsed<'a, 'b, T, A: AtomRepr, X: ExtraTok> {
+pub async fn token_errv<A: ExprRepr, X: ExtraTok>(
-	pub output: T,
+	ctx: &Ctx,
-	pub tail: Snippet<'a, 'b, A, X>,
+	tok: &TokTree<A, X>,
+	description: &'static str,
+	message: impl FnOnce(&str) -> String,
+) -> OrcErr {
+	ctx.mk_err(description, message(&fmt(tok, &ctx.i()).await), [tok.sr.pos()])
 }
 
-pub type ParseRes<'a, 'b, T, A, X> = OrcRes<Parsed<'a, 'b, T, A, X>>;
+pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
+	pub output: T,
+	pub tail: Snippet<'a, H, X>,
+}
 
-pub async fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub type ParseRes<'a, T, H, X> = OrcRes<Parsed<'a, T, H, X>>;
-	ctx: &(impl Reporter + ?Sized),
+
-	tail: Snippet<'a, 'b, A, X>,
+pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
-) -> ParseRes<'a, 'b, Vec<(Import, Pos)>, A, X> {
+	ctx: &Ctx,
-	let ret = rec(ctx, tail).await;
+	tail: Snippet<'a, A, X>,
+) -> ParseRes<'a, Vec<Import>, A, X> {
+	let Some((tt, tail)) = tail.skip_fluff().pop_front() else {
+		return Err(ctx.mk_err(
+			"Expected token",
+			"Expected a name, a parenthesized list of names, or a globstar.",
+			[tail.sr().pos()],
+		));
+	};
+	let ret = rec(tt, ctx).await;
 	#[allow(clippy::type_complexity)] // it's an internal function
-	pub async fn rec<'a, 'b, A: AtomRepr, X: ExtraTok>(
+	pub async fn rec<A: ExprRepr, X: ExtraTok>(
-		ctx: &(impl Reporter + ?Sized),
+		tt: &TokTree<A, X>,
-		tail: Snippet<'a, 'b, A, X>,
+		ctx: &Ctx,
-	) -> ParseRes<'a, 'b, Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>, A, X> {
+	) -> OrcRes<Vec<(Vec<IStr>, Option<IStr>, SrcRange)>> {
-		let comma = tail.i().i(",").await;
+		let ttpos = tt.sr.pos();
-		let globstar = tail.i().i("*").await;
+		match &tt.tok {
-		let Some((name, tail)) = tail.skip_fluff().pop_front() else {
+			Token::NS(ns, body) => {
-			return Err(mk_errv(
+				if !ns.starts_with(name_start) {
-				tail.i().i("Expected name").await,
+					let err = ctx.mk_err("Unexpected name prefix", "Only names can precede ::", [ttpos]);
-				"Expected a name, a list of names, or a globstar.",
+					ctx.rep().report(err)
-				[Pos::Range(tail.pos()).into()],
+				};
-			));
+				let out = Box::pin(rec(body, ctx)).await?;
-		};
+				Ok(out.into_iter().update(|i| i.0.push(ns.clone())).collect_vec())
-		if let Some((Token::NS, tail)) = tail.skip_fluff().pop_front().map(|(tt, s)| (&tt.tok, s)) {
+			},
-			let n = match &name.tok {
+			Token::Name(ntok) => {
-				Token::Name(n) if n.starts_with(name_start) => Ok(n),
+				let n = ntok;
-				_ =>
+				let nopt = Some(n.clone());
-					Err(mk_err(tail.i().i("Unexpected name prefix").await, "Only names can precede ::", [
+				Ok(vec![(vec![], nopt, tt.sr.clone())])
-						Pos::Range(name.range.clone()).into(),
+			},
-					])),
+			Token::S(Paren::Round, b) => {
-			};
+				let mut o = Vec::new();
-			match (Box::pin(rec(ctx, tail)).await, n) {
+				let mut body = Snippet::new(tt, b);
-				(Err(ev), n) => Err(ev.extended(n.err())),
+				while let Some((output, tail)) = body.pop_front() {
-				(Ok(Parsed { tail, .. }), Err(e)) => {
+					match rec(output, ctx).boxed_local().await {
-					ctx.report(e);
+						Ok(names) => o.extend(names),
-					Ok(Parsed { output: vec![], tail })
+						Err(e) => ctx.rep().report(e),
-				},
-				(Ok(Parsed { tail, output }), Ok(pre)) => Ok(Parsed {
-					output: output.into_iter().update(|i| i.0.push(pre.clone())).collect_vec(),
-					tail,
-				}),
-			}
-		} else {
-			let output = match &name.tok {
-				Token::Name(ntok) => {
-					let nopt = match ntok {
-						n if *n == globstar => None,
-						n if n.starts_with(op_char) => {
-							return Err(mk_errv(
-								tail.i().i("Unescaped operator in multiname").await,
-								"Operators in multinames should be enclosed in []",
-								[Pos::Range(name.range.clone()).into()],
-							));
-						},
-						n => Some(n.clone()),
-					};
-					vec![(vec![], nopt, Pos::Range(name.range.clone()))]
-				},
-				Token::S(Paren::Square, b) => {
-					let mut ok = Vec::new();
-					for tt in b.iter() {
-						match &tt.tok {
-							Token::Name(n) if n.starts_with(op_char) =>
-								ok.push((vec![], Some(n.clone()), Pos::Range(tt.range.clone()))),
-							Token::BR | Token::Comment(_) => (),
-							_ => ctx.report(mk_err(
-								tail.i().i("Non-operator in escapement in multiname").await,
-								"In multinames, [] functions as a literal name list reserved for operators",
-								[Pos::Range(name.range.clone()).into()],
-							)),
-						}
 					}
-					ok
+					body = tail;
-				},
+				}
-				Token::S(Paren::Round, b) => {
+				Ok(o)
-					let mut ok = Vec::new();
+			},
-					let body = Snippet::new(name, b, tail.interner);
+			t => {
-					for csent in body.split(|n| matches!(n, Token::Name(n) if *n == comma)) {
+				return Err(ctx.mk_err(
-						match Box::pin(rec(ctx, csent)).await {
+					"Unrecognized name end",
-							Err(e) => ctx.report(e),
+					format!("Names cannot end with {:?} tokens", fmt(t, &ctx.i()).await),
-							Ok(Parsed { output, tail }) => match tail.get(0) {
+					[ttpos],
-								None => ok.extend(output),
+				));
-								Some(t) => ctx.report(mk_err(
+			},
-									tail.i().i("Unexpected token in multiname group").await,
-									"Unexpected token. Likely missing a :: or , or wanted [] instead of ()",
-									[Pos::Range(t.range.clone()).into()],
-								)),
-							},
-						}
-					}
-					ok
-				},
-				t => {
-					return Err(mk_errv(
-						tail.i().i("Unrecognized name end").await,
-						format!("Names cannot end with {:?} tokens", tail.fmt(t).await),
-						[Pos::Range(name.range.clone()).into()],
-					));
-				},
-			};
-			Ok(Parsed { output, tail })
 		}
 	}
-	ret.map(|Parsed { output, tail }| {
+	ret.map(|output| {
 		let output = (output.into_iter())
-			.map(|(p, name, pos)| (Import { path: VPath::new(p.into_iter().rev()), name }, pos))
+			.map(|(p, name, sr)| Import { path: VPath::new(p.into_iter().rev()), name, sr })
 			.collect_vec();
 		Parsed { output, tail }
 	})
 }
 
-/// A compound name, possibly ending with a globstar
+/// A compound name, possibly ending with a globstar. It cannot be just a
+/// globstar; either the name has to be known or the path has to be non-empty.
 #[derive(Debug, Clone)]
 pub struct Import {
 	pub path: VPath,
-	pub name: Option<Tok<String>>,
+	pub name: Option<IStr>,
+	pub sr: SrcRange,
+}
+impl Import {
+	/// Most specific concrete path
+	pub fn mspath(self) -> VName {
+		match self.name {
+			Some(n) => self.path.name_with_suffix(n),
+			None => self.path.into_name().expect("Import cannot be empty"),
+		}
+	}
+	pub fn new(sr: SrcRange, path: VPath, name: IStr) -> Self {
+		Import { path, name: Some(name), sr }
+	}
+	pub fn new_glob(sr: SrcRange, path: VPath) -> Self { Import { path, name: None, sr } }
 }
-
 impl Display for Import {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| t.as_str()))
+		write!(f, "{}::{}", self.path, self.name.as_deref().unwrap_or("*"))
 	}
 }
 
@@ -327,14 +291,5 @@ mod test {
 
 	use super::Snippet;
 
-	fn _covary_snip_a<'a, 'b>(
+	fn _covary_snip_a<'a>(x: Snippet<'static, Never, Never>) -> Snippet<'a, Never, Never> { x }
-		x: Snippet<'static, 'b, Never, Never>,
-	) -> Snippet<'a, 'b, Never, Never> {
-		x
-	}
-	fn _covary_snip_b<'a, 'b>(
-		x: Snippet<'a, 'static, Never, Never>,
-	) -> Snippet<'a, 'b, Never, Never> {
-		x
-	}
 }

orchid-base/src/reqnot.rs

@@ -1,27 +1,96 @@
-use std::any::{Any, TypeId};
+use std::any::Any;
 use std::cell::RefCell;
 use std::future::Future;
 use std::marker::PhantomData;
 use std::mem;
 use std::ops::{BitAnd, Deref};
 use std::pin::Pin;
+use std::rc::Rc;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
 
-use async_std::channel;
-use async_std::sync::Mutex;
 use derive_destructure::destructure;
 use dyn_clone::{DynClone, clone_box};
+use futures::channel::mpsc::{self, Sender};
+use futures::channel::oneshot;
 use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
+use futures::{AsyncBufRead, AsyncWrite, SinkExt, Stream, StreamExt};
 use hashbrown::HashMap;
 use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
 use trait_set::trait_set;
 
+use crate::clone;
 use crate::logging::Logger;
-use crate::{api, clone};
 
 pub struct Receipt<'a>(PhantomData<&'a mut ()>);
 
+/// This object holds an exclusive lock on the outbound pipe.
+pub trait DynRequestWriter {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
+	/// Release the outbound pipe and wait for the response to begin.
+	fn get_response(self: Box<Self>) -> Pin<Box<dyn Future<Output = Box<dyn DynResponseHandle>>>>;
+}
+/// This object holds an exclusive lock on the inbound pipe.
+pub trait DynResponseHandle {
+	fn reader(&mut self) -> Pin<&mut dyn AsyncBufRead>;
+	fn finish(self: Box<Self>) -> Pin<Box<dyn Future<Output = ()>>>;
+}
+/// This object holds an exclusive lock on the outbound pipe.
+pub trait DynNotifWriter {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
+	fn finish(self: Box<Self>) -> Pin<Box<dyn Future<Output = ()>>>;
+}
+
+pub trait DynClient {
+	fn request(&self) -> Pin<Box<dyn Future<Output = Box<dyn DynRequestWriter>>>>;
+	fn notif(&self) -> Pin<Box<dyn Future<Output = Box<dyn DynNotifWriter>>>>;
+}
+
+pub struct Client<T: MsgSet>(pub(crate) Rc<dyn DynClient>, pub(crate) PhantomData<T>);
+impl<T: MsgSet> Client<T> {
+	pub async fn notify<Notif: Into<<T::Out as Channel>::Notif>>(&self, notif: Notif) {
+		let mut notif_writer = self.0.notif().await;
+		notif.into().encode(notif_writer.writer()).await;
+		notif_writer.finish().await;
+	}
+	pub async fn request<Req: Request + Into<<T::Out as Channel>::Req>>(
+		&self,
+		req: Req,
+	) -> Req::Response {
+		let root_req = req.into();
+		let mut req_writer = self.0.request().await;
+		root_req.encode(req_writer.writer()).await;
+		let mut req_hand = req_writer.get_response().await;
+		let res = Req::Response::decode(req_hand.reader()).await;
+		req_hand.finish().await;
+		res
+	}
+}
+
+pub struct DuplexServerState {
+	pending_outbound: HashMap<u64, Box<dyn FnOnce(&mut dyn AsyncBufRead)>>,
+	sender: Pin<Box<dyn AsyncWrite>>,
+	receiver: Pin<Box<dyn AsyncBufRead>>,
+}
+pub enum ServerEvent<T: MsgSet> {
+	Notif(<T::In as Channel>::Notif),
+	Req(RequestHandle<T>, <T::In as Channel>::Req),
+}
+pub async fn run_duplex_server<T: MsgSet>(
+	sender: Pin<Box<dyn AsyncWrite>>,
+	receiver: Pin<Box<dyn AsyncBufRead>>,
+) -> (impl Stream<Item = ServerEvent<T>>, Client<T>) {
+	let sender = Rc::new(Mutex::new(sender));
+	let receiver = Rc::new(Mutex::new(receiver));
+	let pending_outbound = Rc::new(Mutex::new(HashMap::new()));
+}
+pub struct DuplexServer(Rc<Mutex<DuplexServerState>>);
+impl DuplexServer {
+	pub fn receive(msg: )
+}
+
 trait_set! {
 	pub trait SendFn<T: MsgSet> =
 		for<'a> FnMut(&'a [u8], ReqNot<T>) -> LocalBoxFuture<'a, ()>
@@ -46,13 +115,15 @@ pub trait ReqHandlish {
 	}
 	fn defer_drop_objsafe(&self, val: Box<dyn Any>);
 }
+impl ReqHandlish for &'_ dyn ReqHandlish {
+	fn defer_drop_objsafe(&self, val: Box<dyn Any>) { (**self).defer_drop_objsafe(val) }
+}
 
 #[derive(destructure)]
-pub struct RequestHandle<'a, MS: MsgSet> {
+pub struct RequestHandle<MS: MsgSet> {
 	defer_drop: RefCell<Vec<Box<dyn Any>>>,
 	fulfilled: AtomicBool,
 	id: u64,
-	_reqlt: PhantomData<&'a mut ()>,
 	parent: ReqNot<MS>,
 }
 impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
@@ -85,7 +156,7 @@ impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
 impl<MS: MsgSet> ReqHandlish for RequestHandle<'_, MS> {
 	fn defer_drop_objsafe(&self, val: Box<dyn Any>) { self.defer_drop.borrow_mut().push(val); }
 }
-impl<MS: MsgSet> Drop for RequestHandle<'_, MS> {
+impl<MS: MsgSet> Drop for RequestHandle<MS> {
 	fn drop(&mut self) {
 		let done = self.fulfilled.load(Ordering::Relaxed);
 		debug_assert!(done, "Request {} dropped without response", self.id)
@@ -99,7 +170,7 @@ pub struct ReqNotData<T: MsgSet> {
 	send: Box<dyn SendFn<T>>,
 	notif: Box<dyn NotifFn<T>>,
 	req: Box<dyn ReqFn<T>>,
-	responses: HashMap<u64, channel::Sender<Vec<u8>>>,
+	responses: HashMap<u64, mpsc::Sender<Vec<u8>>>,
 }
 
 /// Wraps a raw message buffer to save on copying.
@@ -119,7 +190,7 @@ impl<T: MsgSet> ReqNot<T> {
 		notif: impl NotifFn<T>,
 		req: impl ReqFn<T>,
 	) -> Self {
-		Self(
+		let this = Self(
 			Arc::new(Mutex::new(ReqNotData {
 				id: 1,
 				send: Box::new(send),
@@ -128,7 +199,13 @@ impl<T: MsgSet> ReqNot<T> {
 				responses: HashMap::new(),
 			})),
 			logger,
-		)
+		);
+		let (sig_send, sig_recv) = std::sync::mpsc::sync_channel(0);
+		std::thread::spawn(move || {
+			std::thread::sleep(Duration::from_secs(10));
+			sig_send.send(()).expect("Crash!");
+		});
+		this
 	}
 
 	/// Can be called from a polling thread or dispatched in any other way
@@ -141,7 +218,7 @@ impl<T: MsgSet> ReqNot<T> {
 			let notif_val = <T::In as Channel>::Notif::decode(Pin::new(&mut &payload[..])).await;
 			notif_cb(notif_val, self.clone()).await
 		} else if 0 < id.bitand(1 << 63) {
-			let sender = g.responses.remove(&!id).expect("Received response for invalid message");
+			let mut sender = g.responses.remove(&!id).expect("Received response for invalid message");
 			sender.send(message.to_vec()).await.unwrap()
 		} else {
 			let message = <T::In as Channel>::Req::decode(Pin::new(&mut &payload[..])).await;
@@ -170,12 +247,16 @@ pub trait DynRequester {
 
 pub struct MappedRequester<'a, T: 'a>(Box<dyn Fn(T) -> LocalBoxFuture<'a, RawReply> + 'a>, Logger);
 impl<'a, T> MappedRequester<'a, T> {
-	fn new<U: DynRequester + 'a>(req: U, logger: Logger) -> Self
+	fn new<U: DynRequester + 'a, F: Fn(T) -> U::Transfer + 'a>(
-	where T: Into<U::Transfer> {
+		req: U,
+		cb: F,
+		logger: Logger,
+	) -> Self {
 		let req_arc = Arc::new(req);
+		let cb_arc = Arc::new(cb);
 		MappedRequester(
 			Box::new(move |t| {
-				Box::pin(clone!(req_arc; async move { req_arc.raw_request(t.into()).await}))
+				Box::pin(clone!(req_arc, cb_arc; async move { req_arc.raw_request(cb_arc(t)).await}))
 			}),
 			logger,
 		)
@@ -198,13 +279,13 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
 			g.id += 1;
 			let mut buf = id.to_be_bytes().to_vec();
 			req.encode(Pin::new(&mut buf)).await;
-			let (send, recv) = channel::bounded(1);
+			let (send, mut recv) = mpsc::channel(1);
 			g.responses.insert(id, send);
 			let mut send = clone_box(&*g.send);
 			mem::drop(g);
 			let rn = self.clone();
 			send(&buf, rn).await;
-			let items = recv.recv().await;
+			let items = recv.next().await;
 			RawReply(items.unwrap())
 		})
 	}
@@ -217,10 +298,10 @@ pub trait Requester: DynRequester {
 		&self,
 		data: R,
 	) -> impl Future<Output = R::Response>;
-	fn map<'a, U: Into<Self::Transfer>>(self) -> MappedRequester<'a, U>
+	fn map<'a, U>(self, cb: impl Fn(U) -> Self::Transfer + 'a) -> MappedRequester<'a, U>
 	where Self: Sized + 'a {
 		let logger = self.logger().clone();
-		MappedRequester::new(self, logger)
+		MappedRequester::new(self, cb, logger)
 	}
 }
 
@@ -242,17 +323,16 @@ mod test {
 	use std::rc::Rc;
 	use std::sync::Arc;
 
-	use async_std::sync::Mutex;
 	use futures::FutureExt;
-	use orchid_api::LogStrategy;
+	use futures::lock::Mutex;
 	use orchid_api_derive::Coding;
 	use orchid_api_traits::{Channel, Request};
 	use test_executors::spin_on;
 
 	use super::{MsgSet, ReqNot};
-	use crate::clone;
 	use crate::logging::Logger;
 	use crate::reqnot::Requester as _;
+	use crate::{api, clone};
 
 	#[derive(Clone, Debug, Coding, PartialEq)]
 	pub struct TestReq(u8);
@@ -275,7 +355,7 @@ mod test {
 	#[test]
 	fn notification() {
 		spin_on(async {
-			let logger = Logger::new(LogStrategy::StdErr);
+			let logger = Logger::new(api::LogStrategy::StdErr);
 			let received = Arc::new(Mutex::new(None));
 			let receiver = ReqNot::<TestMsgSet>::new(
 				logger.clone(),
@@ -303,7 +383,7 @@ mod test {
 	#[test]
 	fn request() {
 		spin_on(async {
-			let logger = Logger::new(LogStrategy::StdErr);
+			let logger = Logger::new(api::LogStrategy::StdErr);
 			let receiver = Rc::new(Mutex::<Option<ReqNot<TestMsgSet>>>::new(None));
 			let sender = Rc::new(ReqNot::<TestMsgSet>::new(
 				logger.clone(),

orchid-base/src/testing.rs

@@ -0,0 +1,23 @@
+#![cfg(any(feature = "mocks", test))]
+
+use std::future::ready;
+use std::pin::Pin;
+use std::rc::Rc;
+
+pub struct AsyncMonitor<E: 'static>(Rc<dyn Fn(E) -> Pin<Box<dyn Future<Output = ()>>>>);
+impl<E: 'static> AsyncMonitor<E> {
+	pub fn new<F: AsyncFn(E) -> () + 'static>(f: F) -> Self {
+		let f_rc = Rc::new(f);
+		AsyncMonitor(Rc::new(move |e| {
+			let f_rc = f_rc.clone();
+			Box::pin(async move { f_rc(e).await })
+		}))
+	}
+	pub async fn notify(&self, e: E) -> () { (self.0)(e).await }
+}
+impl<E: 'static> Default for AsyncMonitor<E> {
+	fn default() -> Self { Self(Rc::new(|_| Box::pin(ready(())))) }
+}
+impl<E: 'static> Clone for AsyncMonitor<E> {
+	fn clone(&self) -> Self { Self(self.0.clone()) }
+}

orchid-base/src/tree.rs

@@ -2,54 +2,78 @@ use std::borrow::Borrow;
 use std::fmt::{self, Debug, Display};
 use std::future::Future;
 use std::marker::PhantomData;
-use std::ops::Range;
 use std::rc::Rc;
-use std::sync::Arc;
 
-pub use api::PhKind;
+use async_fn_stream::stream;
-use async_stream::stream;
+use futures::future::join_all;
-use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
 use never::Never;
-use ordered_float::NotNan;
+use orchid_api_traits::Coding;
 use trait_set::trait_set;
 
-use crate::error::OrcErrv;
+use crate::error::OwnedOrcErr;
 use crate::format::{FmtCtx, FmtUnit, Format, Variants};
-use crate::interner::{Interner, Tok};
+use crate::interner::{IStr, Interner};
-use crate::location::Pos;
+use crate::location::{Pos, SrcRange};
-use crate::name::Sym;
+use crate::name::{Sym, VName, VPath};
 use crate::parse::Snippet;
 use crate::{api, match_mapping, tl_cache};
 
-trait_set! {
+pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::Debug {
-	pub trait RecurCB<'a, A: AtomRepr, X: ExtraTok> = Fn(TokTree<'a, A, X>) -> TokTree<'a, A, X>;
+	type FromApiCtx<'a>;
-	pub trait ExtraTok = Format + Clone + fmt::Debug;
+	type ToApiCtx<'a>;
-	pub trait RefDoExtra<X> =
+	#[must_use]
-		for<'b> FnMut(&'b X, Range<u32>) -> LocalBoxFuture<'b, api::TokenTree>;
+	fn from_api(
+		api: &ApiEquiv,
+		ctx: &mut Self::FromApiCtx<'_>,
+		pos: SrcRange,
+		i: &Interner,
+	) -> impl Future<Output = Self>;
+	#[must_use]
+	fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> impl Future<Output = ApiEquiv>;
+}
+impl<T: Clone + Debug + Coding> TokenVariant<T> for Never {
+	type FromApiCtx<'a> = ();
+	type ToApiCtx<'a> = ();
+	async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: SrcRange, _: &Interner) -> Self {
+		panic!("Cannot deserialize Never")
+	}
+	async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> T { match self {} }
 }
 
-pub fn recur<'a, A: AtomRepr, X: ExtraTok>(
+trait_set! {
-	tt: TokTree<'a, A, X>,
+	// TokenHandle
-	f: &impl Fn(TokTree<'a, A, X>, &dyn RecurCB<'a, A, X>) -> TokTree<'a, A, X>,
+	pub trait ExprRepr = TokenVariant<api::ExprTicket>;
-) -> TokTree<'a, A, X> {
+	// TokenExpr
-	f(tt, &|TokTree { range, tok }| {
+	pub trait ExtraTok = TokenVariant<api::Expression>;
+}
+
+trait_set! {
+	pub trait RecurCB<H: ExprRepr, X: ExtraTok> = Fn(TokTree<H, X>) -> TokTree<H, X>;
+}
+
+pub fn recur<H: ExprRepr, X: ExtraTok>(
+	tt: TokTree<H, X>,
+	f: &impl Fn(TokTree<H, X>, &dyn RecurCB<H, X>) -> TokTree<H, X>,
+) -> TokTree<H, X> {
+	f(tt, &|TokTree { sr: range, tok }| {
 		let tok = match tok {
-			tok @ (Token::Atom(_) | Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::NS) => tok,
+			tok @ (Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::Name(_)) => tok,
-			tok @ (Token::Name(_) | Token::Slot(_) | Token::X(_) | Token::Ph(_) | Token::Macro(_)) => tok,
+			tok @ (Token::Handle(_) | Token::NewExpr(_)) => tok,
-			tok @ Token::Reference(_) => tok,
+			Token::NS(n, b) => Token::NS(n, Box::new(recur(*b, f))),
-			Token::LambdaHead(arg) =>
+			Token::LambdaHead(arg) => Token::LambdaHead(Box::new(recur(*arg, f))),
-				Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
 			Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
 		};
-		TokTree { range, tok }
+		TokTree { sr: range, tok }
 	})
 }
 
 pub trait AtomRepr: Clone + Format {
 	type Ctx: ?Sized;
+	#[must_use]
 	fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> impl Future<Output = Self>;
+	#[must_use]
 	fn to_api(&self) -> impl Future<Output = orchid_api::Atom> + '_;
 }
 impl AtomRepr for Never {
@@ -71,139 +95,139 @@ impl Display for TokHandle<'_> {
 }
 
 #[derive(Clone, Debug)]
-pub struct TokTree<'a, A: AtomRepr, X: ExtraTok> {
+pub struct TokTree<H: ExprRepr, X: ExtraTok> {
-	pub tok: Token<'a, A, X>,
+	pub tok: Token<H, X>,
-	pub range: Range<u32>,
+	/// The protocol has a Range<u32> because these are always transmitted in the
+	/// context of a given snippet, but internal logic and error reporting is
+	/// easier if the in-memory representation also includes the snippet path.
+	pub sr: SrcRange,
 }
-impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
+impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
-	pub async fn from_api(tt: &api::TokenTree, ctx: &mut A::Ctx, i: &Interner) -> Self {
+	pub async fn from_api(
-		let tok = match_mapping!(&tt.token, api::Token => Token::<'b, A, X> {
+		tt: &api::TokenTree,
-			BR, NS,
+		hctx: &mut H::FromApiCtx<'_>,
-			Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx).await),
+		xctx: &mut X::FromApiCtx<'_>,
-			Bottom(e => OrcErrv::from_api(e, i).await),
+		src: &Sym,
-			LambdaHead(arg => ttv_from_api(arg, ctx, i).await),
+		i: &Interner,
-			Name(n => Tok::from_api(*n, i).await),
+	) -> Self {
-			S(*par, b => ttv_from_api(b, ctx, i).await),
+		let pos = SrcRange::new(tt.range.clone(), src);
-			Comment(c.clone()),
+		let tok = match_mapping!(&tt.token, api::Token => Token::<H, X> {
-			Slot(id => TokHandle::new(*id)),
-			Ph(ph => Ph::from_api(ph, i).await),
-			Macro(*prio),
-			Reference(tok => Sym::from_api(*tok, i).await)
-		});
-		Self { range: tt.range.clone(), tok }
-	}
-
-	pub async fn to_api(&self, do_extra: &mut impl RefDoExtra<X>) -> api::TokenTree {
-		let token = match_mapping!(&self.tok, Token => api::Token {
-			Atom(a.to_api().await),
 			BR,
-			NS,
+			NS(n => i.es(*n).await,
-			Bottom(e.to_api()),
+				b => Box::new(Self::from_api(b, hctx, xctx, src, i).boxed_local().await)),
+			Bottom(e => OwnedOrcErr::from_api(e, i).await),
+			LambdaHead(arg => Box::new(Self::from_api(arg, hctx, xctx, src, i).boxed_local().await)),
+			Name(n => i.es(*n).await),
+			S(*par, b => ttv_from_api(b, hctx, xctx, src, i).await),
 			Comment(c.clone()),
-			LambdaHead(arg => ttv_to_api(arg, do_extra).boxed_local().await),
+			NewExpr(expr => X::from_api(expr, xctx, pos.clone(), i).await),
-			Name(n.to_api()),
+			Handle(tk => H::from_api(tk, hctx, pos.clone(), i).await)
-			Slot(tt.ticket()),
-			S(*p, b => ttv_to_api(b, do_extra).boxed_local().await),
-			Ph(ph.to_api()),
-			Macro(*prio),
-			Reference(sym.to_api()),
-		} {
-			Token::X(x) => return do_extra(x, self.range.clone()).await
 		});
-		api::TokenTree { range: self.range.clone(), token }
+		Self { sr: pos, tok }
 	}
 
 	pub async fn into_api(
 		self,
-		do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
+		hctx: &mut H::ToApiCtx<'_>,
+		xctx: &mut X::ToApiCtx<'_>,
 	) -> api::TokenTree {
-		let token = match self.tok {
+		let token = match_mapping!(self.tok, Token => api::Token {
-			Token::Atom(a) => api::Token::Atom(a.to_api().await),
+			BR,
-			Token::Reference(sym) => api::Token::Reference(sym.to_api()),
+			NS(n.to_api(), b => Box::new(b.into_api(hctx, xctx).boxed_local().await)),
-			Token::BR => api::Token::BR,
+			Bottom(e.to_api()),
-			Token::NS => api::Token::NS,
+			Comment(c.clone()),
-			Token::Bottom(e) => api::Token::Bottom(e.to_api()),
+			LambdaHead(arg => Box::new(arg.into_api(hctx, xctx).boxed_local().await)),
-			Token::Comment(c) => api::Token::Comment(c.clone()),
+			Name(nn.to_api()),
-			Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra).await),
+			S(p, b => ttv_into_api(b, hctx, xctx).boxed_local().await),
-			Token::Name(n) => api::Token::Name(n.to_api()),
+			Handle(hand.into_api(hctx).await),
-			Token::Slot(tt) => api::Token::Slot(tt.ticket()),
+			NewExpr(expr.into_api(xctx).await),
-			Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra).await),
+		});
-			Token::Ph(Ph { kind, name }) =>
+		api::TokenTree { range: self.sr.range.clone(), token }
-				api::Token::Ph(api::Placeholder { name: name.to_api(), kind }),
-			Token::X(x) => return do_extra(x, self.range.clone()),
-			Token::Macro(prio) => api::Token::Macro(prio),
-		};
-		api::TokenTree { range: self.range.clone(), token }
 	}
 
-	pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) }
+	pub fn is_kw(&self, tk: IStr) -> bool { self.tok.is_kw(tk) }
-	pub fn as_name(&self) -> Option<Tok<String>> {
+	pub fn as_name(&self) -> Option<IStr> {
 		if let Token::Name(n) = &self.tok { Some(n.clone()) } else { None }
 	}
-	pub fn as_s<'a>(&'a self, par: Paren, i: &'a Interner) -> Option<Snippet<'a, 'b, A, X>> {
+	pub fn as_multiname(&self) -> Result<VName, &TokTree<H, X>> {
-		self.tok.as_s(par).map(|slc| Snippet::new(self, slc, i))
+		let mut segs = VPath::new([]);
+		let mut cur = self;
+		loop {
+			match &cur.tok {
+				Token::Name(last) => return Ok(segs.name_with_suffix(last.clone())),
+				Token::NS(seg, inner) => {
+					segs = segs.suffix([seg.clone()]);
+					cur = inner;
+				},
+				_ => return Err(cur),
+			}
+		}
 	}
-	pub fn lambda(arg: Vec<Self>, mut body: Vec<Self>) -> Self {
+	pub fn as_s(&self, par: Paren) -> Option<Snippet<'_, H, X>> {
-		let arg_range = ttv_range(&arg);
+		self.tok.as_s(par).map(|slc| Snippet::new(self, slc))
-		let s_range = arg_range.start..body.last().expect("Lambda with empty body!").range.end;
+	}
-		body.insert(0, Token::LambdaHead(arg).at(arg_range));
+	pub fn as_lambda(&self) -> Option<&Self> {
+		match &self.tok {
+			Token::LambdaHead(arg) => Some(&**arg),
+			_ => None,
+		}
+	}
+	pub fn is_fluff(&self) -> bool { matches!(self.tok, Token::Comment(_) | Token::BR) }
+	pub fn lambda(arg: Self, mut body: Vec<Self>) -> Self {
+		let arg_range = arg.sr();
+		let mut s_range = arg_range.clone();
+		s_range.range.end = body.last().expect("Lambda with empty body!").sr.range.end;
+		body.insert(0, Token::LambdaHead(Box::new(arg)).at(arg_range));
 		Token::S(Paren::Round, body).at(s_range)
 	}
+	pub fn sr(&self) -> SrcRange { self.sr.clone() }
 }
-impl<A: AtomRepr, X: ExtraTok> Format for TokTree<'_, A, X> {
+impl<H: ExprRepr, X: ExtraTok> Format for TokTree<H, X> {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		self.tok.print(c).await
 	}
 }
 
-pub async fn ttv_from_api<A: AtomRepr, X: ExtraTok>(
+pub async fn ttv_from_api<H: ExprRepr, X: ExtraTok>(
 	tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>,
-	ctx: &mut A::Ctx,
+	hctx: &mut H::FromApiCtx<'_>,
+	xctx: &mut X::FromApiCtx<'_>,
+	src: &Sym,
 	i: &Interner,
-) -> Vec<TokTree<'static, A, X>> {
+) -> Vec<TokTree<H, X>> {
-	stream! {
+	stream(async |mut cx| {
 		for tok in tokv {
-			yield TokTree::<A, X>::from_api(tok.borrow(), ctx, i).boxed_local().await
+			cx.emit(TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src, i).boxed_local().await).await
 		}
-	}
+	})
 	.collect()
 	.await
 }
 
-pub async fn ttv_to_api<'a, A: AtomRepr, X: ExtraTok>(
+pub async fn ttv_into_api<H: ExprRepr, X: ExtraTok>(
-	tokv: impl IntoIterator<Item: Borrow<TokTree<'a, A, X>>>,
+	tokv: impl IntoIterator<Item = TokTree<H, X>>,
-	do_extra: &mut impl RefDoExtra<X>,
+	hctx: &mut H::ToApiCtx<'_>,
+	xctx: &mut X::ToApiCtx<'_>,
 ) -> Vec<api::TokenTree> {
-	let mut output = Vec::new();
+	stream(async |mut cx| {
-	for tok in tokv {
-		output.push(Borrow::<TokTree<A, X>>::borrow(&tok).to_api(do_extra).await)
-	}
-	output
-}
-
-pub async fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>(
-	tokv: impl IntoIterator<Item = TokTree<'a, A, X>>,
-	do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
-) -> Vec<api::TokenTree> {
-	stream! {
 		for tok in tokv {
-			yield tok.into_api(do_extra).await
+			cx.emit(tok.into_api(hctx, xctx).await).await
 		}
-	}
+	})
 	.collect()
 	.await
 }
 
-pub fn wrap_tokv<'a, A: AtomRepr, X: ExtraTok>(
+pub fn wrap_tokv<H: ExprRepr, X: ExtraTok>(
-	items: impl IntoIterator<Item = TokTree<'a, A, X>>,
+	items: impl IntoIterator<Item = TokTree<H, X>>,
-) -> TokTree<'a, A, X> {
+) -> TokTree<H, X> {
 	let items_v = items.into_iter().collect_vec();
 	match items_v.len() {
 		0 => panic!("A tokv with no elements is illegal"),
 		1 => items_v.into_iter().next().unwrap(),
 		_ => {
-			let range = items_v.first().unwrap().range.start..items_v.last().unwrap().range.end;
+			let sr = ttv_range(&items_v).expect("empty handled above");
-			Token::S(api::Paren::Round, items_v).at(range)
+			Token::S(api::Paren::Round, items_v).at(sr)
 		},
 	}
 }
@@ -212,70 +236,53 @@ pub use api::Paren;
 
 /// Lexer output variant
 #[derive(Clone, Debug)]
-pub enum Token<'a, A: AtomRepr, X: ExtraTok> {
+pub enum Token<H: ExprRepr, X: ExtraTok> {
 	/// Information about the code addressed to the human reader or dev tooling
 	/// It has no effect on the behaviour of the program unless it's explicitly
 	/// read via reflection
-	Comment(Arc<String>),
+	Comment(Rc<String>),
 	/// The part of a lambda between `\` and `.` enclosing the argument. The body
 	/// stretches to the end of the enclosing parens or the end of the const line
-	LambdaHead(Vec<TokTree<'a, A, X>>),
+	LambdaHead(Box<TokTree<H, X>>),
 	/// A binding, operator, or a segment of a namespaced::name
-	Name(Tok<String>),
+	Name(IStr),
-	/// The namespace separator ::
+	/// A namespace prefix, like `my_ns::` followed by a token
-	NS,
+	NS(IStr, Box<TokTree<H, X>>),
 	/// A line break
 	BR,
 	/// `()`, `[]`, or `{}`
-	S(Paren, Vec<TokTree<'a, A, X>>),
+	S(Paren, Vec<TokTree<H, X>>),
-	/// A fully formed reference to external code emitted by a lexer plugin
+	/// A newly instantiated expression
-	Reference(Sym),
+	NewExpr(X),
-	/// A value emitted by a lexer plugin
+	/// An existing expr from a nested lexer
-	Atom(A),
+	Handle(H),
 	/// A grammar error emitted by a lexer plugin if it was possible to continue
 	/// reading. Parsers should treat it as an atom unless it prevents parsing,
 	/// in which case both this and a relevant error should be returned.
-	Bottom(OrcErrv),
+	Bottom(OwnedOrcErr),
-	/// An instruction from a plugin for the lexer to embed a subexpression
-	/// without retransmitting it. It should not appear anywhere outside lexer
-	/// plugin responses.
-	Slot(TokHandle<'a>),
-	/// Additional domain-specific token types
-	X(X),
-	/// A placeholder for metaprogramming, either $name, ..$name, ..$name:N,
-	/// ...$name, or ...$name:N
-	Ph(Ph),
-	/// `macro` or `macro(`X`)` where X is any valid floating point number
-	/// expression. `macro` is not a valid name in Orchid for this reason.
-	Macro(Option<NotNan<f64>>),
 }
-impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
+impl<H: ExprRepr, X: ExtraTok> Token<H, X> {
-	pub fn at(self, range: Range<u32>) -> TokTree<'a, A, X> { TokTree { range, tok: self } }
+	pub fn at(self, sr: SrcRange) -> TokTree<H, X> { TokTree { sr, tok: self } }
-	pub fn is_kw(&self, tk: Tok<String>) -> bool { matches!(self, Token::Name(n) if *n == tk) }
+	pub fn is_kw(&self, tk: IStr) -> bool { matches!(self, Token::Name(n) if *n == tk) }
-	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<'a, A, X>]> {
+	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> {
 		match self {
 			Self::S(p, b) if *p == par => Some(b),
 			_ => None,
 		}
 	}
 }
-impl<A: AtomRepr, X: ExtraTok> Format for Token<'_, A, X> {
+impl<H: ExprRepr, X: ExtraTok> Format for Token<H, X> {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		match self {
-			Self::Atom(a) => a.print(c).await,
 			Self::BR => "\n".to_string().into(),
-			Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()).into(),
+			Self::Bottom(err) => format!("Botttom({}) ", indent(&err.to_string())).into(),
-			Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())).into(),
 			Self::Comment(c) => format!("--[{c}]--").into(),
 			Self::LambdaHead(arg) =>
-				FmtUnit::new(tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0b}."))), [
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0b}.")))
-					ttv_fmt(arg, c).await,
+					.units([arg.print(c).boxed_local().await]),
-				]),
+			Self::NS(n, b) => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}::{1l}")))
-			Self::NS => "::".to_string().into(),
+				.units([n.to_string().into(), b.print(c).boxed_local().await]),
 			Self::Name(n) => format!("{n}").into(),
-			Self::Reference(sym) => format!("{sym}").into(),
-			Self::Slot(th) => format!("{th}").into(),
-			Self::Ph(ph) => format!("{ph}").into(),
 			Self::S(p, b) => FmtUnit::new(
 				match *p {
 					Paren::Round => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("({0b})"))),
@@ -284,67 +291,22 @@ impl<A: AtomRepr, X: ExtraTok> Format for Token<'_, A, X> {
 				},
 				[ttv_fmt(b, c).await],
 			),
-			Self::X(x) => x.print(c).await,
+			Self::Handle(h) => h.print(c).await,
-			Self::Macro(None) => "macro".to_string().into(),
+			Self::NewExpr(ex) => ex.print(c).await,
-			Self::Macro(Some(prio)) => format!("macro({prio})").into(),
 		}
 	}
 }
 
-pub fn ttv_range(ttv: &[TokTree<'_, impl AtomRepr, impl ExtraTok>]) -> Range<u32> {
+pub fn ttv_range<'a>(ttv: &[TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>]) -> Option<SrcRange> {
-	assert!(!ttv.is_empty(), "Empty slice has no range");
+	let range = ttv.first()?.sr.range.start..ttv.last().unwrap().sr.range.end;
-	ttv.first().unwrap().range.start..ttv.last().unwrap().range.end
+	Some(SrcRange { path: ttv.first().unwrap().sr.path(), range })
 }
 
 pub async fn ttv_fmt<'a: 'b, 'b>(
-	ttv: impl IntoIterator<Item = &'b TokTree<'a, impl AtomRepr + 'b, impl ExtraTok + 'b>>,
+	ttv: impl IntoIterator<Item = &'b TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>>,
 	c: &(impl FmtCtx + ?Sized),
 ) -> FmtUnit {
 	FmtUnit::sequence(" ", None, join_all(ttv.into_iter().map(|t| t.print(c))).await)
 }
 
 pub fn indent(s: &str) -> String { s.replace("\n", "\n  ") }
-
-#[derive(Clone, Debug)]
-pub struct Ph {
-	pub name: Tok<String>,
-	pub kind: PhKind,
-}
-impl Ph {
-	pub async fn from_api(api: &api::Placeholder, i: &Interner) -> Self {
-		Self { name: Tok::from_api(api.name, i).await, kind: api.kind }
-	}
-	pub fn to_api(&self) -> api::Placeholder {
-		api::Placeholder { name: self.name.to_api(), kind: self.kind }
-	}
-}
-impl Display for Ph {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let n = &self.name;
-		match self.kind {
-			PhKind::Scalar => write!(f, "${n}"),
-			PhKind::Vector { priority: 0, at_least_one: true } => write!(f, "...${}", self.name),
-			PhKind::Vector { priority: p, at_least_one: true } => write!(f, "...${}:{}", self.name, p),
-			PhKind::Vector { priority: 0, at_least_one: false } => write!(f, "..${}", self.name),
-			PhKind::Vector { priority: p, at_least_one: false } => write!(f, "..${}:{}", self.name, p),
-		}
-	}
-}
-
-#[cfg(test)]
-mod test {
-	use super::*;
-
-	#[test]
-	fn test_covariance() {
-		fn _f<'a>(x: Token<'static, Never, Never>) -> Token<'a, Never, Never> { x }
-	}
-
-	#[test]
-	fn fail_covariance() {
-		// this fails to compile
-		// fn _f<'a, 'b>(x: &'a mut &'static ()) -> &'a mut &'b () { x }
-		// this passes because it's covariant
-		fn _f<'a, 'b>(x: &'a fn() -> &'static ()) -> &'a fn() -> &'b () { x }
-	}
-}

orchid-base/src/virt_fs/common.rs

@@ -13,13 +13,13 @@ pub enum Loaded {
   Code(Arc<String>),
   /// Conceptually equivalent to the list of *.orc files in a folder, without
   /// the extension
-  Collection(Arc<Vec<Tok<String>>>),
+  Collection(Arc<Vec<IStr>>),
 }
 impl Loaded {
   /// Is the loaded item source code (not a collection)?
   pub fn is_code(&self) -> bool { matches!(self, Loaded::Code(_)) }
   /// Collect the elements in a collection rreport
-  pub fn collection(items: impl IntoIterator<Item = Tok<String>>) -> Self {
+  pub fn collection(items: impl IntoIterator<Item = IStr>) -> Self {
     Self::Collection(Arc::new(items.into_iter().collect()))
   }
 }
@@ -55,7 +55,7 @@ impl ErrorSansOrigin for CodeNotFound {
 /// formats and other sources for libraries and dependencies.
 pub trait VirtFS {
   /// Implementation of [VirtFS::read]
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult;
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult;
   /// Discover information about a path without reading it.
   ///
   /// Implement this if your vfs backend can do expensive operations
@@ -68,7 +68,7 @@ pub trait VirtFS {
   }
   /// Convert a path into a human-readable string that is meaningful in the
   /// target context.
-  fn display(&self, path: &[Tok<String>]) -> Option<String>;
+  fn display(&self, path: &[IStr]) -> Option<String>;
   /// Convert the FS handler into a type-erased version of itself for packing in
   /// a tree.
   fn rc(self) -> Rc<dyn VirtFS>
@@ -81,15 +81,15 @@ pub trait VirtFS {
 }
 
 impl VirtFS for &dyn VirtFS {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (*self).get(path, full_path)
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> { (*self).display(path) }
+  fn display(&self, path: &[IStr]) -> Option<String> { (*self).display(path) }
 }
 
 impl<T: VirtFS + ?Sized> VirtFS for Rc<T> {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (**self).get(path, full_path)
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> { (**self).display(path) }
+  fn display(&self, path: &[IStr]) -> Option<String> { (**self).display(path) }
 }

orchid-base/src/virt_fs/decl.rs

@@ -32,7 +32,7 @@ impl<'a> Combine for &'a dyn VirtFS {
 pub type DeclTree = ModEntry<Rc<dyn VirtFS>, (), ()>;
 
 impl VirtFS for DeclTree {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     match &self.member {
       ModMember::Item(it) => it.get(path, full_path),
       ModMember::Sub(module) => match path.split_first() {
@@ -44,7 +44,7 @@ impl VirtFS for DeclTree {
     }
   }
 
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let (head, tail) = path.split_first()?;
     match &self.member {
       ModMember::Item(it) => it.display(path),
@@ -54,16 +54,16 @@ impl VirtFS for DeclTree {
 }
 
 impl VirtFS for String {
-  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
+  fn display(&self, _: &[IStr]) -> Option<String> { None }
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.as_str().to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }
 }
 
 impl<'a> VirtFS for &'a str {
-  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
+  fn display(&self, _: &[IStr]) -> Option<String> { None }
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }

orchid-base/src/virt_fs/dir.rs

@@ -99,14 +99,14 @@ impl DirNode {
     }
   }
 
-  fn mk_pathbuf(&self, path: &[Tok<String>]) -> PathBuf {
+  fn mk_pathbuf(&self, path: &[IStr]) -> PathBuf {
     let mut fpath = self.root.clone();
     path.iter().for_each(|seg| fpath.push(seg.as_str()));
     fpath
   }
 }
 impl VirtFS for DirNode {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     let fpath = self.mk_pathbuf(path);
     let mut binding = self.cached.borrow_mut();
     let (_, res) = (binding.raw_entry_mut().from_key(&fpath))
@@ -114,7 +114,7 @@ impl VirtFS for DirNode {
     res.clone()
   }
 
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let pathbuf = self.mk_pathbuf(path).with_extension(self.ext());
     Some(pathbuf.to_string_lossy().to_string())
   }

orchid-base/src/virt_fs/embed.rs

@@ -56,7 +56,7 @@ impl EmbeddedFS {
 }
 
 impl VirtFS for EmbeddedFS {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     if path.is_empty() {
       return Ok(Loaded::collection(self.tree.keys(|_| true)));
     }
@@ -67,7 +67,7 @@ impl VirtFS for EmbeddedFS {
       ModMember::Sub(sub) => Loaded::collection(sub.keys(|_| true)),
     })
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let Self { gen, suffix, .. } = self;
     Some(format!("{}{suffix} in {gen}", path.iter().join("/")))
   }

orchid-base/src/virt_fs/prefix.rs

@@ -21,18 +21,18 @@ impl<'a> PrefixFS<'a> {
       add: VPath::parse(add.as_ref()),
     }
   }
-  fn proc_path(&self, path: &[Tok<String>]) -> Option<Vec<Tok<String>>> {
+  fn proc_path(&self, path: &[IStr]) -> Option<Vec<IStr>> {
     let path = path.strip_prefix(self.remove.as_slice())?;
     Some(self.add.0.iter().chain(path).cloned().collect_vec())
   }
 }
 impl<'a> VirtFS for PrefixFS<'a> {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> super::FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> super::FSResult {
     let path =
       self.proc_path(path).ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())?;
     self.wrapped.get(&path, full_path)
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     self.wrapped.display(&self.proc_path(path)?)
   }
 }

orchid-extension/Cargo.toml

@@ -1,30 +1,41 @@
 [package]
 name = "orchid-extension"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ahash = "0.8.11"
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
+async-lock = "3.4.1"
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
 derive_destructure = "1.0.0"
-dyn-clone = "1.0.17"
+dyn-clone = "1.0.20"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = [
-hashbrown = "0.15.2"
+  "std",
+  "async-await",
+], default-features = false }
+hashbrown = "0.16.0"
+include_dir = { version = "0.7.4", optional = true }
 itertools = "0.14.0"
-konst = "0.3.16"
+konst = "0.4.1"
 lazy_static = "1.5.0"
 memo-map = "0.3.3"
 never = "0.1.0"
-once_cell = "1.20.2"
+once_cell = "1.21.3"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"
-paste = "1.0.15"
+pastey = "0.1.1"
-some_executor = "0.4.0"
+some_executor = "0.6.1"
 substack = "1.1.1"
+tokio = { version = "1.47.1", optional = true, features = [] }
+tokio-util = { version = "0.7.16", optional = true, features = ["compat"] }
+
 trait-set = "0.3.0"
+
+[features]
+tokio = ["dep:tokio", "dep:tokio-util"]
+default = ["tokio"]

orchid-extension/src/atom.rs

@@ -1,31 +1,29 @@
 use std::any::{Any, TypeId, type_name};
+use std::collections::HashMap;
 use std::fmt;
 use std::future::Future;
-use std::marker::PhantomData;
 use std::num::NonZeroU32;
 use std::ops::Deref;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use ahash::HashMap;
-use async_std::io::{Read, Write};
-use async_std::stream;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::LocalBoxFuture;
-use futures::{FutureExt, StreamExt};
+use futures::{AsyncRead, AsyncWrite, FutureExt, StreamExt, stream};
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Coding, Decode, Encode, Request, enc_vec};
 use orchid_base::clone;
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use orchid_base::ctx::Ctx;
+use orchid_base::error::{OrcErr, OrcRes};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::reqnot::Requester;
-use orchid_base::tree::AtomRepr;
 use trait_set::trait_set;
 
 use crate::api;
+use crate::conv::ToExpr;
 // use crate::error::{ProjectError, ProjectResult};
 use crate::expr::{Expr, ExprData, ExprHandle, ExprKind};
 use crate::gen_expr::GExpr;
@@ -42,7 +40,10 @@ pub trait AtomicVariant {}
 pub trait Atomic: 'static + Sized {
 	type Variant: AtomicVariant;
 	type Data: Clone + Coding + Sized + 'static;
-	fn reg_reqs() -> MethodSetBuilder<Self>;
+	/// Register handlers for IPC calls. If this atom implements [Supports], you
+	/// should register your implementations here. If this atom doesn't
+	/// participate in IPC at all, the default implementation is fine
+	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl<A: Atomic> AtomCard for A {
 	type Data = <Self as Atomic>::Data;
@@ -84,56 +85,48 @@ pub fn get_info<A: AtomCard>(
 }
 
 #[derive(Clone)]
-pub struct ForeignAtom<'a> {
+pub struct ForeignAtom {
-	pub(crate) expr: Option<Rc<ExprHandle>>,
+	pub(crate) expr: Rc<ExprHandle>,
-	pub(crate) _life: PhantomData<&'a ()>,
-	pub(crate) ctx: SysCtx,
 	pub(crate) atom: api::Atom,
 	pub(crate) pos: Pos,
 }
-impl ForeignAtom<'_> {
+impl ForeignAtom {
-	pub fn ex_opt(self) -> Option<Expr> {
-		let (handle, pos) = (self.expr.as_ref()?.clone(), self.pos.clone());
-		let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { _life: PhantomData, ..self }) };
-		Some(Expr::new(handle, data))
-	}
 	pub fn pos(&self) -> Pos { self.pos.clone() }
-	pub fn ctx(&self) -> SysCtx { self.ctx.clone() }
+	pub fn ctx(&self) -> &SysCtx { &self.expr.ctx }
-}
+	pub fn ex(self) -> Expr {
-impl ForeignAtom<'static> {
+		let (handle, pos) = (self.expr.clone(), self.pos.clone());
-	pub fn ex(self) -> Expr { self.ex_opt().unwrap() }
+		let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { ..self }) };
+		Expr::new(handle, data)
+	}
 	pub(crate) fn new(handle: Rc<ExprHandle>, atom: api::Atom, pos: Pos) -> Self {
-		ForeignAtom { _life: PhantomData, atom, ctx: handle.ctx.clone(), expr: Some(handle), pos }
+		ForeignAtom { atom, expr: handle, pos }
 	}
 	pub async fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
-		let rep = (self.ctx.reqnot.request(api::Fwd(
+		let rep = (self.ctx().reqnot().request(api::Fwd(
 			self.atom.clone(),
-			Sym::parse(M::NAME, &self.ctx.i).await.unwrap().tok().to_api(),
+			Sym::parse(M::NAME, self.ctx().i()).await.unwrap().tok().to_api(),
 			enc_vec(&m).await,
 		)))
 		.await?;
 		Some(M::Response::decode(Pin::new(&mut &rep[..])).await)
 	}
-}
+	pub async fn downcast<T: AtomicFeatures>(self) -> Result<TypAtom<T>, NotTypAtom> {
-impl fmt::Display for ForeignAtom<'_> {
+		TypAtom::downcast(self.ex().handle()).await
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "{}::{:?}", if self.expr.is_some() { "Clause" } else { "Tok" }, self.atom)
 	}
 }
-impl fmt::Debug for ForeignAtom<'_> {
+impl fmt::Display for ForeignAtom {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Atom::{:?}", self.atom) }
+}
+impl fmt::Debug for ForeignAtom {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ForeignAtom({self})") }
 }
-impl Format for ForeignAtom<'_> {
+impl Format for ForeignAtom {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		FmtUnit::from_api(&self.ctx.reqnot.request(api::ExtAtomPrint(self.atom.clone())).await)
+		FmtUnit::from_api(&self.ctx().reqnot().request(api::ExtAtomPrint(self.atom.clone())).await)
 	}
 }
-impl AtomRepr for ForeignAtom<'_> {
+impl ToExpr for ForeignAtom {
-	type Ctx = SysCtx;
+	async fn to_expr(self) -> GExpr { self.ex().to_expr().await }
-	async fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self {
-		Self { atom: atom.clone(), _life: PhantomData, ctx: ctx.clone(), expr: None, pos }
-	}
-	async fn to_api(&self) -> orchid_api::Atom { self.atom.clone() }
 }
 
 pub struct NotTypAtom {
@@ -143,12 +136,10 @@ pub struct NotTypAtom {
 	pub ctx: SysCtx,
 }
 impl NotTypAtom {
-	pub async fn mk_err(&self) -> OrcErr {
+	pub fn mk_err(&self, ctx: &Ctx) -> OrcErr {
-		mk_err(
+		ctx.mk_err("Not the expected type", format!("This expression is not a {}", self.typ.name()), [
-			self.ctx.i.i("Not the expected type").await,
+			self.pos.clone(),
-			format!("This expression is not a {}", self.typ.name()),
+		])
-			[self.pos.clone().into()],
-		)
 	}
 }
 
@@ -163,8 +154,8 @@ trait_set! {
 	trait AtomReqCb<A> = for<'a> Fn(
 		&'a A,
 		SysCtx,
-		Pin<&'a mut dyn Read>,
+		Pin<&'a mut dyn AsyncRead>,
-		Pin<&'a mut dyn Write>,
+		Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, ()>
 }
 
@@ -179,20 +170,22 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 		assert!(!M::NAME.is_empty(), "AtomMethod::NAME cannoot be empty");
 		self.handlers.push((
 			M::NAME,
-			Rc::new(move |a: &A, ctx: SysCtx, req: Pin<&mut dyn Read>, rep: Pin<&mut dyn Write>| {
+			Rc::new(
-				async { Supports::<M>::handle(a, ctx, M::decode(req).await).await.encode(rep).await }
+				move |a: &A, ctx: SysCtx, req: Pin<&mut dyn AsyncRead>, rep: Pin<&mut dyn AsyncWrite>| {
-					.boxed_local()
+					async { Supports::<M>::handle(a, ctx, M::decode(req).await).await.encode(rep).await }
-			}),
+						.boxed_local()
+				},
+			),
 		));
 		self
 	}
 
 	pub async fn pack(&self, ctx: SysCtx) -> MethodSet<A> {
 		MethodSet {
-			handlers: stream::from_iter(self.handlers.iter())
+			handlers: stream::iter(self.handlers.iter())
 				.then(|(k, v)| {
 					clone!(ctx; async move {
-						(Sym::parse(k, &ctx.i).await.unwrap(), v.clone())
+						(Sym::parse(k, ctx.i()).await.unwrap(), v.clone())
 					})
 				})
 				.collect()
@@ -210,8 +203,8 @@ impl<A: AtomCard> MethodSet<A> {
 		atom: &'a A,
 		ctx: SysCtx,
 		key: Sym,
-		req: Pin<&'a mut dyn Read>,
+		req: Pin<&'a mut dyn AsyncRead>,
-		rep: Pin<&'a mut dyn Write>,
+		rep: Pin<&'a mut dyn AsyncWrite>,
 	) -> bool {
 		match self.handlers.get(&key) {
 			None => false,
@@ -228,11 +221,13 @@ impl<A: AtomCard> Default for MethodSetBuilder<A> {
 }
 
 #[derive(Clone)]
-pub struct TypAtom<'a, A: AtomicFeatures> {
+pub struct TypAtom<A: AtomicFeatures> {
-	pub data: ForeignAtom<'a>,
+	pub untyped: ForeignAtom,
 	pub value: A::Data,
 }
-impl<A: AtomicFeatures> TypAtom<'static, A> {
+impl<A: AtomicFeatures> TypAtom<A> {
+	pub fn ctx(&self) -> &SysCtx { self.untyped.ctx() }
+	pub fn i(&self) -> &Interner { self.ctx().i() }
 	pub async fn downcast(expr: Rc<ExprHandle>) -> Result<Self, NotTypAtom> {
 		match Expr::from_handle(expr).atom().await {
 			Err(expr) => Err(NotTypAtom {
@@ -245,21 +240,19 @@ impl<A: AtomicFeatures> TypAtom<'static, A> {
 				Ok(tatom) => Ok(tatom),
 				Err(fa) => Err(NotTypAtom {
 					pos: fa.pos.clone(),
-					ctx: fa.ctx.clone(),
+					ctx: fa.ctx().clone(),
 					expr: fa.ex(),
 					typ: Box::new(A::info()),
 				}),
 			},
 		}
 	}
-}
-impl<A: AtomicFeatures> TypAtom<'_, A> {
 	pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
 	where A: Supports<M> {
 		M::Response::decode(Pin::new(
-			&mut &(self.data.ctx.reqnot.request(api::Fwd(
+			&mut &(self.untyped.ctx().reqnot().request(api::Fwd(
-				self.data.atom.clone(),
+				self.untyped.atom.clone(),
-				Sym::parse(M::NAME, &self.data.ctx.i).await.unwrap().tok().to_api(),
+				Sym::parse(M::NAME, self.untyped.ctx().i()).await.unwrap().tok().to_api(),
 				enc_vec(&req).await,
 			)))
 			.await
@@ -268,41 +261,44 @@ impl<A: AtomicFeatures> TypAtom<'_, A> {
 		.await
 	}
 }
-impl<A: AtomicFeatures> Deref for TypAtom<'_, A> {
+impl<A: AtomicFeatures> Deref for TypAtom<A> {
 	type Target = A::Data;
 	fn deref(&self) -> &Self::Target { &self.value }
 }
+impl<A: AtomicFeatures> ToExpr for TypAtom<A> {
+	async fn to_expr(self) -> GExpr { self.untyped.to_expr().await }
+}
 
 pub struct AtomCtx<'a>(pub &'a [u8], pub Option<api::AtomId>, pub SysCtx);
 impl FmtCtx for AtomCtx<'_> {
-	fn i(&self) -> &Interner { &self.2.i }
+	fn i(&self) -> &Interner { self.2.i() }
 }
 
 pub trait AtomDynfo: 'static {
 	fn tid(&self) -> TypeId;
 	fn name(&self) -> &'static str;
 	fn decode<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>>;
-	fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>;
+	fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
-	fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>;
+	fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr>;
 	fn print<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit>;
 	fn handle_req<'a, 'b: 'a, 'c: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'b mut dyn Read>,
+		req: Pin<&'b mut dyn AsyncRead>,
-		rep: Pin<&'c mut dyn Write>,
+		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool>;
 	fn command<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>>;
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
-		write: Pin<&'b mut dyn Write>,
+		write: Pin<&'b mut dyn AsyncWrite>,
-	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>>;
+	) -> LocalBoxFuture<'a, Option<Vec<Expr>>>;
 	fn deserialize<'a>(
 		&'a self,
 		ctx: SysCtx,
 		data: &'a [u8],
-		refs: &'a [api::ExprTicket],
+		refs: &'a [Expr],
 	) -> LocalBoxFuture<'a, api::Atom>;
 	fn drop<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, ()>;
 }
@@ -312,9 +308,7 @@ trait_set! {
 }
 pub struct AtomFactory(Box<dyn AtomFactoryFn>);
 impl AtomFactory {
-	pub fn new<F: Future<Output = api::Atom> + 'static>(
+	pub fn new(f: impl AsyncFnOnce(SysCtx) -> api::Atom + Clone + 'static) -> Self {
-		f: impl FnOnce(SysCtx) -> F + Clone + 'static,
-	) -> Self {
 		Self(Box::new(|ctx| f(ctx).boxed_local()))
 	}
 	pub async fn build(self, ctx: SysCtx) -> api::Atom { (self.0)(ctx).await }
@@ -334,10 +328,10 @@ impl Format for AtomFactory {
 	}
 }
 
-pub async fn err_not_callable(i: &Interner) -> OrcErr {
+pub fn err_not_callable(cx: &Ctx) -> OrcErr {
-	mk_err(i.i("This atom is not callable").await, "Attempted to apply value as function", [])
+	cx.mk_err_floating("This atom is not callable", "Attempted to apply value as function")
 }
 
-pub async fn err_not_command(i: &Interner) -> OrcErr {
+pub fn err_not_command(cx: &Ctx) -> OrcErr {
-	mk_err(i.i("This atom is not a command").await, "Settled on an inactionable value", [])
+	cx.mk_err_floating("This atom is not a command", "Settled on an inactionable value")
 }

orchid-extension/src/atom_owned.rs

@@ -4,44 +4,43 @@ use std::future::Future;
 use std::num::NonZero;
 use std::ops::Deref;
 use std::pin::Pin;
-use std::rc::Rc;
 use std::sync::atomic::AtomicU64;
 
+use async_lock::{RwLock, RwLockReadGuard};
 use async_once_cell::OnceCell;
-use async_std::io::{Read, Write};
-use async_std::sync::{RwLock, RwLockReadGuard};
-use futures::FutureExt;
 use futures::future::{LocalBoxFuture, ready};
+use futures::{AsyncRead, AsyncWrite, FutureExt};
 use itertools::Itertools;
 use memo_map::MemoMap;
 use never::Never;
-use orchid_api::AtomId;
 use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
-use orchid_base::format::FmtUnit;
+use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit};
 use orchid_base::name::Sym;
 
 use crate::api;
 use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
-	MethodSetBuilder, err_not_callable, err_not_command, get_info,
+	MethodSetBuilder, TypAtom, err_not_callable, err_not_command, get_info,
 };
-use crate::expr::{Expr, ExprHandle};
+use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
-use crate::system::SysCtx;
+use crate::system::{SysCtx, SysCtxEntry};
+use crate::system_ctor::CtedObj;
 
 pub struct OwnedVariant;
 impl AtomicVariant for OwnedVariant {}
 impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVariant> for A {
 	fn _factory(self) -> AtomFactory {
-		AtomFactory::new(move |ctx| async move {
+		AtomFactory::new(async move |ctx| {
-			let serial = ctx.obj_store.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+			let serial =
+				ctx.get_or_default::<ObjStore>().next_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
 			let atom_id = api::AtomId(NonZero::new(serial + 1).unwrap());
-			let (typ_id, _) = get_info::<A>(ctx.cted.inst().card());
+			let (typ_id, _) = get_info::<A>(ctx.get::<CtedObj>().inst().card());
 			let mut data = enc_vec(&typ_id).await;
 			self.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
-			ctx.obj_store.1.read().await.insert(atom_id, Box::new(self));
+			ctx.get_or_default::<ObjStore>().objects.read().await.insert(atom_id, Box::new(self));
-			api::Atom { drop: Some(atom_id), data, owner: ctx.id }
+			api::Atom { drop: Some(atom_id), data, owner: ctx.sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { OwnedAtomDynfo { msbuild: A::reg_reqs(), ms: OnceCell::new() } }
@@ -51,12 +50,13 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
 /// While an atom read guard is held, no atom can be removed.
 pub(crate) struct AtomReadGuard<'a> {
 	id: api::AtomId,
-	guard: RwLockReadGuard<'a, MemoMap<AtomId, Box<dyn DynOwnedAtom>>>,
+	guard: RwLockReadGuard<'a, MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
 }
 impl<'a> AtomReadGuard<'a> {
 	async fn new(id: api::AtomId, ctx: &'a SysCtx) -> Self {
-		let guard = ctx.obj_store.1.read().await;
+		let guard = ctx.get_or_default::<ObjStore>().objects.read().await;
-		assert!(guard.get(&id).is_some(), "Received invalid atom ID: {}", id.0);
+		let valid = guard.iter().map(|i| i.0).collect_vec();
+		assert!(guard.get(&id).is_some(), "Received invalid atom ID: {id:?} not in {valid:?}");
 		Self { id, guard }
 	}
 }
@@ -65,8 +65,9 @@ impl Deref for AtomReadGuard<'_> {
 	fn deref(&self) -> &Self::Target { &**self.guard.get(&self.id).unwrap() }
 }
 
+/// Remove an atom from the store
 pub(crate) async fn take_atom(id: api::AtomId, ctx: &SysCtx) -> Box<dyn DynOwnedAtom> {
-	let mut g = ctx.obj_store.1.write().await;
+	let mut g = ctx.get_or_default::<ObjStore>().objects.write().await;
 	g.remove(&id).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0))
 }
 
@@ -78,75 +79,69 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	fn tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn name(&self) -> &'static str { type_name::<T>() }
 	fn decode<'a>(&'a self, AtomCtx(data, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
-		async {
+		Box::pin(async {
 			Box::new(<T as AtomCard>::Data::decode(Pin::new(&mut &data[..])).await) as Box<dyn Any>
-		}
+		})
-		.boxed_local()
 	}
-	fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
+	fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
-		async move { take_atom(id.unwrap(), &ctx).await.dyn_call(ctx.clone(), arg).await }.boxed_local()
+		Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_call(arg).await })
 	}
 	fn call_ref<'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
-		arg: api::ExprTicket,
+		arg: Expr,
 	) -> LocalBoxFuture<'a, GExpr> {
-		async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_call_ref(ctx.clone(), arg).await }
+		Box::pin(async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_call_ref(arg).await })
-			.boxed_local()
 	}
 	fn print(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> LocalBoxFuture<'_, FmtUnit> {
-		async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_print(ctx.clone()).await }
+		Box::pin(
-			.boxed_local()
+			async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_print(ctx.clone()).await },
+		)
 	}
 	fn handle_req<'a, 'b: 'a, 'c: 'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx,
 		key: Sym,
-		req: Pin<&'b mut dyn Read>,
+		req: Pin<&'b mut dyn AsyncRead>,
-		rep: Pin<&'c mut dyn Write>,
+		rep: Pin<&'c mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
-		async move {
+		Box::pin(async move {
 			let a = AtomReadGuard::new(id.unwrap(), &ctx).await;
 			let ms = self.ms.get_or_init(self.msbuild.pack(ctx.clone())).await;
 			ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), ctx.clone(), key, req, rep).await
-		}
+		})
-		.boxed_local()
 	}
 	fn command<'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>> {
-		async move { take_atom(id.unwrap(), &ctx).await.dyn_command(ctx.clone()).await }.boxed_local()
+		Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_command(ctx.clone()).await })
 	}
 	fn drop(&self, AtomCtx(_, id, ctx): AtomCtx) -> LocalBoxFuture<'_, ()> {
-		async move { take_atom(id.unwrap(), &ctx).await.dyn_free(ctx.clone()).await }.boxed_local()
+		Box::pin(async move { take_atom(id.unwrap(), &ctx).await.dyn_free(ctx.clone()).await })
 	}
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
-		mut write: Pin<&'b mut dyn Write>,
+		mut write: Pin<&'b mut dyn AsyncWrite>,
-	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>> {
+	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
-		async move {
+		Box::pin(async move {
 			let id = id.unwrap();
 			id.encode(write.as_mut()).await;
-			let refs = AtomReadGuard::new(id, &ctx).await.dyn_serialize(ctx.clone(), write).await;
+			AtomReadGuard::new(id, &ctx).await.dyn_serialize(ctx.clone(), write).await
-			refs.map(|v| v.into_iter().map(|t| t.handle().tk).collect_vec())
+		})
-		}
-		.boxed_local()
 	}
 	fn deserialize<'a>(
 		&'a self,
 		ctx: SysCtx,
 		data: &'a [u8],
-		refs: &'a [api::ExprTicket],
+		refs: &'a [Expr],
 	) -> LocalBoxFuture<'a, api::Atom> {
-		async move {
+		Box::pin(async move {
-			let refs =
+			let refs = T::Refs::from_iter(refs.iter().cloned());
-				refs.iter().map(|tk| Expr::from_handle(Rc::new(ExprHandle::from_args(ctx.clone(), *tk))));
+			let obj = T::deserialize(DeserCtxImpl(data, &ctx), refs).await;
-			let obj = T::deserialize(DeserCtxImpl(data, &ctx), T::Refs::from_iter(refs)).await;
 			obj._factory().build(ctx).await
-		}
+		})
-		.boxed_local()
 	}
 }
 
@@ -218,12 +213,12 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	type Refs: RefSet;
 	fn val(&self) -> impl Future<Output = Cow<'_, Self::Data>>;
 	#[allow(unused_variables)]
-	fn call_ref(&self, arg: ExprHandle) -> impl Future<Output = GExpr> {
+	fn call_ref(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot([err_not_callable(&arg.ctx.i).await]) }
+		async move { bot(err_not_callable(arg.ctx().i()).await) }
 	}
-	fn call(self, arg: ExprHandle) -> impl Future<Output = GExpr> {
+	fn call(self, arg: Expr) -> impl Future<Output = GExpr> {
 		async {
-			let ctx = arg.get_ctx();
+			let ctx = arg.ctx();
 			let gcl = self.call_ref(arg).await;
 			self.free(ctx).await;
 			gcl
@@ -231,19 +226,19 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	}
 	#[allow(unused_variables)]
 	fn command(self, ctx: SysCtx) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command(&ctx.i).await.into()) }
+		async move { Err(err_not_command(ctx.i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn free(self, ctx: SysCtx) -> impl Future<Output = ()> { async {} }
 	#[allow(unused_variables)]
-	fn print(&self, ctx: SysCtx) -> impl Future<Output = FmtUnit> {
+	fn print_atom<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> {
 		async { format!("OwnedAtom({})", type_name::<Self>()).into() }
 	}
 	#[allow(unused_variables)]
 	fn serialize(
 		&self,
 		ctx: SysCtx,
-		write: Pin<&mut (impl Write + ?Sized)>,
+		write: Pin<&mut (impl AsyncWrite + ?Sized)>,
 	) -> impl Future<Output = Self::Refs> {
 		assert_serializable::<Self>();
 		async { panic!("Either implement serialize or set Refs to Never for {}", type_name::<Self>()) }
@@ -265,34 +260,29 @@ fn assert_serializable<T: OwnedAtom>() {
 pub trait DynOwnedAtom: 'static {
 	fn atom_tid(&self) -> TypeId;
 	fn as_any_ref(&self) -> &dyn Any;
-	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()>;
+	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()>;
-	fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr>;
+	fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr>;
-	fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket)
+	fn dyn_call(self: Box<Self>, arg: Expr) -> LocalBoxFuture<'static, GExpr>;
-	-> LocalBoxFuture<'static, GExpr>;
 	fn dyn_command(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, OrcRes<Option<GExpr>>>;
 	fn dyn_free(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, ()>;
 	fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit>;
 	fn dyn_serialize<'a>(
 		&'a self,
 		ctx: SysCtx,
-		sink: Pin<&'a mut dyn Write>,
+		sink: Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>>;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn as_any_ref(&self) -> &dyn Any { self }
-	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()> {
+	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()> {
 		async { self.val().await.as_ref().encode(buffer).await }.boxed_local()
 	}
-	fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
+	fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
-		self.call_ref(ExprHandle::from_args(ctx, arg)).boxed_local()
+		self.call_ref(arg).boxed_local()
 	}
-	fn dyn_call(
+	fn dyn_call(self: Box<Self>, arg: Expr) -> LocalBoxFuture<'static, GExpr> {
-		self: Box<Self>,
+		self.call(arg).boxed_local()
-		ctx: SysCtx,
-		arg: api::ExprTicket,
-	) -> LocalBoxFuture<'static, GExpr> {
-		self.call(ExprHandle::from_args(ctx, arg)).boxed_local()
 	}
 	fn dyn_command(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, OrcRes<Option<GExpr>>> {
 		self.command(ctx).boxed_local()
@@ -300,11 +290,13 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn dyn_free(self: Box<Self>, ctx: SysCtx) -> LocalBoxFuture<'static, ()> {
 		self.free(ctx).boxed_local()
 	}
-	fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit> { self.print(ctx).boxed_local() }
+	fn dyn_print(&self, ctx: SysCtx) -> LocalBoxFuture<'_, FmtUnit> {
+		async move { self.print_atom(&FmtCtxImpl { i: ctx.i() }).await }.boxed_local()
+	}
 	fn dyn_serialize<'a>(
 		&'a self,
 		ctx: SysCtx,
-		sink: Pin<&'a mut dyn Write>,
+		sink: Pin<&'a mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		match TypeId::of::<Never>() == TypeId::of::<<Self as OwnedAtom>::Refs>() {
 			true => ready(None).boxed_local(),
@@ -313,4 +305,17 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	}
 }
 
-pub type ObjStore = Rc<(AtomicU64, RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>)>;
+#[derive(Default)]
+struct ObjStore {
+	next_id: AtomicU64,
+	objects: RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
+}
+impl SysCtxEntry for ObjStore {}
+
+pub async fn own<A: OwnedAtom>(typ: TypAtom<A>) -> A {
+	let ctx = typ.untyped.ctx();
+	let g = ctx.get_or_default::<ObjStore>().objects.read().await;
+	let dyn_atom = (g.get(&typ.untyped.atom.drop.expect("Owned atoms always have a drop ID")))
+		.expect("Atom ID invalid; atom type probably not owned by this crate");
+	dyn_atom.as_any_ref().downcast_ref().cloned().expect("The ID should imply a type as well")
+}

orchid-extension/src/atom_thin.rs

@@ -3,9 +3,8 @@ use std::future::Future;
 use std::pin::Pin;
 
 use async_once_cell::OnceCell;
-use async_std::io::{Read, Write};
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
+use futures::{AsyncRead, AsyncWrite, FutureExt};
 use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
@@ -16,19 +15,20 @@ use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
 	MethodSetBuilder, err_not_callable, err_not_command, get_info,
 };
-use crate::expr::ExprHandle;
+use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
 use crate::system::SysCtx;
+use crate::system_ctor::CtedObj;
 
 pub struct ThinVariant;
 impl AtomicVariant for ThinVariant {}
 impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant> for A {
 	fn _factory(self) -> AtomFactory {
-		AtomFactory::new(move |ctx| async move {
+		AtomFactory::new(async move |ctx| {
-			let (id, _) = get_info::<A>(ctx.cted.inst().card());
+			let (id, _) = get_info::<A>(ctx.get::<CtedObj>().inst().card());
 			let mut buf = enc_vec(&id).await;
 			self.encode(Pin::new(&mut buf)).await;
-			api::Atom { drop: None, data: buf, owner: ctx.id }
+			api::Atom { drop: None, data: buf, owner: ctx.sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { ThinAtomDynfo { msbuild: Self::reg_reqs(), ms: OnceCell::new() } }
@@ -41,37 +41,25 @@ pub struct ThinAtomDynfo<T: ThinAtom> {
 }
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn print<'a>(&self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit> {
-		async move { T::decode(Pin::new(&mut &buf[..])).await.print(ctx).await }.boxed_local()
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.print(ctx).await })
 	}
 	fn tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn name(&self) -> &'static str { type_name::<T>() }
 	fn decode<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
-		async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> }.boxed_local()
+		Box::pin(async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> })
 	}
-	fn call<'a>(
+	fn call<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		&'a self,
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
-		AtomCtx(buf, _, ctx): AtomCtx<'a>,
-		arg: api::ExprTicket,
-	) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move {
-			T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await
-		})
 	}
-	fn call_ref<'a>(
+	fn call_ref<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		&'a self,
+		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
-		AtomCtx(buf, _, ctx): AtomCtx<'a>,
-		arg: api::ExprTicket,
-	) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move {
-			T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await
-		})
 	}
 	fn handle_req<'a, 'm1: 'a, 'm2: 'a>(
 		&'a self,
 		AtomCtx(buf, _, sys): AtomCtx<'a>,
 		key: Sym,
-		req: Pin<&'m1 mut dyn Read>,
+		req: Pin<&'m1 mut dyn AsyncRead>,
-		rep: Pin<&'m2 mut dyn Write>,
+		rep: Pin<&'m2 mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let ms = self.ms.get_or_init(self.msbuild.pack(sys.clone())).await;
@@ -87,8 +75,8 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
-		write: Pin<&'b mut dyn Write>,
+		write: Pin<&'b mut dyn AsyncWrite>,
-	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>> {
+	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		Box::pin(async {
 			T::decode(Pin::new(&mut &ctx.0[..])).await.encode(write).await;
 			Some(Vec::new())
@@ -98,17 +86,16 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 		&'a self,
 		ctx: SysCtx,
 		data: &'a [u8],
-		refs: &'a [api::ExprTicket],
+		refs: &'a [Expr],
 	) -> LocalBoxFuture<'a, api::Atom> {
 		assert!(refs.is_empty(), "Refs found when deserializing thin atom");
-		async { T::decode(Pin::new(&mut &data[..])).await._factory().build(ctx).await }.boxed_local()
+		Box::pin(async { T::decode(Pin::new(&mut &data[..])).await._factory().build(ctx).await })
 	}
 	fn drop<'a>(&'a self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, ()> {
-		async move {
+		Box::pin(async move {
 			let string_self = T::decode(Pin::new(&mut &buf[..])).await.print(ctx.clone()).await;
-			writeln!(ctx.logger, "Received drop signal for non-drop atom {string_self:?}");
+			writeln!(ctx.logger(), "Received drop signal for non-drop atom {string_self:?}");
-		}
+		})
-		.boxed_local()
 	}
 }
 
@@ -116,12 +103,12 @@ pub trait ThinAtom:
 	AtomCard<Data = Self> + Atomic<Variant = ThinVariant> + Coding + Send + Sync + 'static
 {
 	#[allow(unused_variables)]
-	fn call(&self, arg: ExprHandle) -> impl Future<Output = GExpr> {
+	fn call(&self, arg: Expr) -> impl Future<Output = GExpr> {
-		async move { bot([err_not_callable(&arg.ctx.i).await]) }
+		async move { bot(err_not_callable(arg.ctx().i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn command(&self, ctx: SysCtx) -> impl Future<Output = OrcRes<Option<GExpr>>> {
-		async move { Err(err_not_command(&ctx.i).await.into()) }
+		async move { Err(err_not_command(ctx.i()).await) }
 	}
 	#[allow(unused_variables)]
 	fn print(&self, ctx: SysCtx) -> impl Future<Output = FmtUnit> {

orchid-extension/src/conv.rs

@@ -1,13 +1,14 @@
 use std::future::Future;
 
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use never::Never;
+use orchid_base::error::{OrcErrv, OrcRes, mk_err};
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
 
-use crate::atom::{AtomicFeatures, ToAtom, TypAtom};
+use crate::atom::{AtomicFeatures, ForeignAtom, ToAtom, TypAtom};
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, atom, bot};
-use crate::system::downcast_atom;
+use crate::system::{SysCtx, downcast_atom};
 
 pub trait TryFromExpr: Sized {
 	fn try_from_expr(expr: Expr) -> impl Future<Output = OrcRes<Self>>;
@@ -23,46 +24,61 @@ impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
 	}
 }
 
-async fn err_not_atom(pos: Pos, i: &Interner) -> OrcErr {
+async fn err_not_atom(pos: Pos, i: &Interner) -> OrcErrv {
-	mk_err(i.i("Expected an atom").await, "This expression is not an atom", [pos.into()])
+	mk_err(i.i("Expected an atom").await, "This expression is not an atom", [pos])
 }
 
-async fn err_type(pos: Pos, i: &Interner) -> OrcErr {
+async fn err_type(pos: Pos, i: &Interner) -> OrcErrv {
-	mk_err(i.i("Type error").await, "The atom is a different type than expected", [pos.into()])
+	mk_err(i.i("Type error").await, "The atom is a different type than expected", [pos])
 }
 
-impl<A: AtomicFeatures> TryFromExpr for TypAtom<'_, A> {
+impl TryFromExpr for ForeignAtom {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
 		match expr.atom().await {
-			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), &ex.ctx().i).await.into()),
+			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), ex.ctx().i()).await),
-			Ok(f) => match downcast_atom::<A>(f).await {
+			Ok(f) => Ok(f),
-				Ok(a) => Ok(a),
-				Err(f) => Err(err_type(f.pos(), &f.ctx().i).await.into()),
-			},
 		}
 	}
 }
 
+impl<A: AtomicFeatures> TryFromExpr for TypAtom<A> {
+	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
+		let f = ForeignAtom::try_from_expr(expr).await?;
+		match downcast_atom::<A>(f).await {
+			Ok(a) => Ok(a),
+			Err(f) => Err(err_type(f.pos(), f.ctx().i()).await),
+		}
+	}
+}
+
+impl TryFromExpr for SysCtx {
+	async fn try_from_expr(expr: Expr) -> OrcRes<Self> { Ok(expr.ctx()) }
+}
+
 pub trait ToExpr {
-	fn to_expr(self) -> GExpr;
+	fn to_expr(self) -> impl Future<Output = GExpr>;
 }
 
 impl ToExpr for GExpr {
-	fn to_expr(self) -> GExpr { self }
+	async fn to_expr(self) -> GExpr { self }
 }
 impl ToExpr for Expr {
-	fn to_expr(self) -> GExpr { self.gen() }
+	async fn to_expr(self) -> GExpr { self.slot() }
 }
 
 impl<T: ToExpr> ToExpr for OrcRes<T> {
-	fn to_expr(self) -> GExpr {
+	async fn to_expr(self) -> GExpr {
 		match self {
 			Err(e) => bot(e),
-			Ok(t) => t.to_expr(),
+			Ok(t) => t.to_expr().await,
 		}
 	}
 }
 
 impl<A: ToAtom> ToExpr for A {
-	fn to_expr(self) -> GExpr { atom(self) }
+	async fn to_expr(self) -> GExpr { atom(self) }
+}
+
+impl ToExpr for Never {
+	async fn to_expr(self) -> GExpr { match self {} }
 }

orchid-extension/src/coroutine_exec.rs

@@ -0,0 +1,104 @@
+use std::borrow::Cow;
+use std::marker::PhantomData;
+use std::rc::Rc;
+
+use futures::channel::mpsc::{Sender, channel};
+use futures::lock::Mutex;
+use futures::stream::{self, LocalBoxStream};
+use futures::{FutureExt, SinkExt, StreamExt};
+use never::Never;
+use orchid_base::error::OrcRes;
+use orchid_base::format::{FmtCtx, FmtUnit};
+
+use crate::atom::Atomic;
+use crate::atom_owned::{OwnedAtom, OwnedVariant};
+use crate::conv::{ToExpr, TryFromExpr};
+use crate::expr::Expr;
+use crate::gen_expr::{GExpr, arg, call, lambda, seq};
+
+enum Command {
+	Execute(GExpr, Sender<Expr>),
+	Register(GExpr, Sender<Expr>),
+	Halt(GExpr),
+}
+
+struct BuilderCoroutineData {
+	name: Option<String>,
+	receiver: Mutex<LocalBoxStream<'static, Command>>,
+}
+
+#[derive(Clone)]
+struct BuilderCoroutine(Rc<BuilderCoroutineData>);
+impl BuilderCoroutine {
+	pub async fn run(self) -> GExpr {
+		let cmd = self.0.receiver.lock().await.next().await;
+		match cmd {
+			None => panic!("Before the stream ends, we should have gotten a Halt"),
+			Some(Command::Halt(expr)) => expr,
+			Some(Command::Execute(expr, reply)) => call([
+				lambda(0, [seq([
+					arg(0),
+					call([Replier { reply, builder: self }.to_expr().await, arg(0)]),
+				])]),
+				expr,
+			]),
+			Some(Command::Register(expr, reply)) =>
+				call([Replier { reply, builder: self }.to_expr().await, expr]),
+		}
+	}
+}
+
+#[derive(Clone)]
+pub struct Replier {
+	reply: Sender<Expr>,
+	builder: BuilderCoroutine,
+}
+impl Atomic for Replier {
+	type Data = ();
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for Replier {
+	type Refs = Never;
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	async fn call(mut self, arg: Expr) -> GExpr {
+		self.reply.send(arg).await.expect("What the heck");
+		std::mem::drop(self.reply);
+		self.builder.run().await
+	}
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		match &self.builder.0.name {
+			None => "BuilderCoroutine".into(),
+			Some(name) => format!("BuilderCoroutine({name})").into(),
+		}
+	}
+}
+
+pub async fn exec<R: ToExpr>(
+	debug: impl AsRef<str>,
+	f: impl for<'a> AsyncFnOnce(ExecHandle<'a>) -> R + 'static,
+) -> GExpr {
+	let (cmd_snd, cmd_recv) = channel(0);
+	let halt = async { Command::Halt(f(ExecHandle(cmd_snd, PhantomData)).await.to_expr().await) }
+		.into_stream();
+	let coro = BuilderCoroutine(Rc::new(BuilderCoroutineData {
+		name: Some(debug.as_ref().to_string()),
+		receiver: Mutex::new(stream::select(halt, cmd_recv).boxed_local()),
+	}));
+	coro.run().await
+}
+
+static WEIRD_DROP_ERR: &str = "Coroutine dropped while we are being polled somehow";
+
+pub struct ExecHandle<'a>(Sender<Command>, PhantomData<&'a ()>);
+impl ExecHandle<'_> {
+	pub async fn exec<T: TryFromExpr>(&mut self, val: impl ToExpr) -> OrcRes<T> {
+		let (reply_snd, mut reply_recv) = channel(1);
+		self.0.send(Command::Execute(val.to_expr().await, reply_snd)).await.expect(WEIRD_DROP_ERR);
+		T::try_from_expr(reply_recv.next().await.expect(WEIRD_DROP_ERR)).await
+	}
+	pub async fn register(&mut self, val: impl ToExpr) -> Expr {
+		let (reply_snd, mut reply_recv) = channel(1);
+		self.0.send(Command::Register(val.to_expr().await, reply_snd)).await.expect(WEIRD_DROP_ERR);
+		reply_recv.next().await.expect(WEIRD_DROP_ERR)
+	}
+}

orchid-extension/src/entrypoint.rs

@@ -1,45 +1,39 @@
 use std::cell::RefCell;
 use std::future::Future;
-use std::io::Write;
 use std::mem;
 use std::num::NonZero;
 use std::pin::Pin;
 use std::rc::Rc;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicBool, Ordering};
 
-use async_std::channel::{Receiver, Sender};
+use futures::channel::mpsc::{Receiver, Sender, channel};
-use async_std::stream;
-use async_std::sync::Mutex;
 use futures::future::{LocalBoxFuture, join_all};
-use futures::{FutureExt, StreamExt};
+use futures::lock::Mutex;
+use futures::{FutureExt, SinkExt, StreamExt, stream, stream_select};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::ApplyMacro;
+use orchid_api_traits::{Decode, UnderRoot, enc_vec};
-use orchid_api_traits::{Decode, Encode, enc_vec};
+use orchid_base::builtin::{ExtInit, ExtPort, Spawner};
-use orchid_base::builtin::{ExtPort, Spawner};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
 use orchid_base::clone;
+use orchid_base::error::Reporter;
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::logging::Logger;
-use orchid_base::macros::{mtreev_from_api, mtreev_to_api};
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Snippet};
 use orchid_base::reqnot::{ReqNot, RequestHandle, Requester};
-use orchid_base::tree::{ttv_from_api, ttv_to_api};
+use orchid_base::tree::{TokenVariant, ttv_from_api};
 use substack::Substack;
 use trait_set::trait_set;
 
 use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId};
-use crate::atom_owned::{ObjStore, take_atom};
+use crate::atom_owned::take_atom;
-use crate::fs::VirtFS;
+use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
-use crate::lexer::{LexContext, err_cascade, err_not_applicable};
+use crate::lexer::{LexContext, ekey_cascade, ekey_not_applicable};
-use crate::macros::{Rule, RuleCtx};
+use crate::parser::{PTokTree, ParsCtx, get_const, linev_into_api};
-use crate::msg::{recv_parent_msg, send_parent_msg};
 use crate::system::{SysCtx, atom_by_idx};
 use crate::system_ctor::{CtedObj, DynSystemCtor};
-use crate::tree::{GenTok, GenTokTree, LazyMemberFactory, TIACtxImpl, do_extra};
+use crate::tree::{LazyMemberFactory, TreeIntoApiCtxImpl};
 
 pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>;
 pub type ExtReqNot = ReqNot<api::ExtMsgSet>;
@@ -52,132 +46,122 @@ impl ExtensionData {
 	pub fn new(name: &'static str, systems: &'static [&'static dyn DynSystemCtor]) -> Self {
 		Self { name, systems }
 	}
-	// pub fn main(self) { extension_main(self) }
 }
 
 pub enum MemberRecord {
-	Gen(Vec<Tok<String>>, LazyMemberFactory),
+	Gen(Vec<IStr>, LazyMemberFactory),
 	Res,
 }
 
 pub struct SystemRecord {
-	cted: CtedObj,
-	vfses: HashMap<api::VfsId, &'static dyn VirtFS>,
-	declfs: api::EagerVfs,
 	lazy_members: HashMap<api::TreeId, MemberRecord>,
-	rules: HashMap<api::MacroId, Rc<Rule>>,
+	ctx: SysCtx,
 }
 
 trait_set! {
-	pub trait WARCallback<'a, T> = FnOnce(
+	pub trait WithAtomRecordCallback<'a, T> = AsyncFnOnce(
 		Box<dyn AtomDynfo>,
 		SysCtx,
 		AtomTypeId,
 		&'a [u8]
-	) -> LocalBoxFuture<'a, T>
+	) -> T
 }
 
 pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
-	get_sys_ctx: &impl Fn(api::SysId, ReqNot<api::ExtMsgSet>) -> F,
+	get_sys_ctx: &impl Fn(api::SysId) -> F,
-	reqnot: ReqNot<api::ExtMsgSet>,
 	atom: &'a api::Atom,
-	cb: impl WARCallback<'a, T>,
+	cb: impl WithAtomRecordCallback<'a, T>,
 ) -> T {
 	let mut data = &atom.data[..];
-	let ctx = get_sys_ctx(atom.owner, reqnot).await;
+	let ctx = get_sys_ctx(atom.owner).await;
-	let inst = ctx.cted.inst();
+	let inst = ctx.get::<CtedObj>().inst();
 	let id = AtomTypeId::decode(Pin::new(&mut data)).await;
 	let atom_record = atom_by_idx(inst.card(), id.clone()).expect("Atom ID reserved");
 	cb(atom_record, ctx, id, data).await
 }
 
-// pub fn extension_main(data: ExtensionData) {
-
-// 	if thread::Builder::new()
-// 		.name(format!("ext-main:{}", data.name))
-// 		.spawn(|| extension_main_logic(data))
-// 		.unwrap()
-// 		.join()
-// 		.is_err()
-// 	{
-// 		process::exit(-1)
-// 	}
-// }
-
 pub struct ExtensionOwner {
-	rn: ReqNot<api::ExtMsgSet>,
+	_interner_cell: Rc<RefCell<Option<Interner>>>,
-	out_recv: Receiver<Vec<u8>>,
+	_systems_lock: Rc<Mutex<HashMap<api::SysId, SystemRecord>>>,
+	out_recv: Mutex<Receiver<Vec<u8>>>,
 	out_send: Sender<Vec<u8>>,
 }
 
 impl ExtPort for ExtensionOwner {
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		self.rn.receive(msg).boxed_local()
+		Box::pin(async { self.out_send.clone().send(msg.to_vec()).boxed_local().await.unwrap() })
 	}
-	fn recv<'a>(
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
-		&'a self,
+		Box::pin(async { self.out_recv.lock().await.next().await })
-		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
-	) -> LocalBoxFuture<'a, ()> {
-		async {
-			let msg = self.out_recv.recv().await.unwrap();
-			cb(&msg[..]).await
-		}
-		.boxed_local()
 	}
 }
 
-pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
+pub fn extension_init(
-	let api::HostHeader { log_strategy, msg_logs } =
+	data: ExtensionData,
-		api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await;
+	host_header: api::HostHeader,
-	let mut buf = Vec::new();
+	spawner: Spawner,
+) -> ExtInit {
+	let api::HostHeader { log_strategy, msg_logs } = host_header;
 	let decls = (data.systems.iter().enumerate())
 		.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
 		.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
 		.collect_vec();
-	let systems = Rc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
+	let systems_lock = Rc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
-	api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() }
+	let ext_header = api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() };
-		.encode(Pin::new(&mut buf))
+	let (out_send, in_recv) = channel::<Vec<u8>>(1);
-		.await;
+	let (in_send, out_recv) = channel::<Vec<u8>>(1);
-	std::io::stdout().write_all(&buf).unwrap();
+	let (exit_send, exit_recv) = channel(1);
-	std::io::stdout().flush().unwrap();
-	let exiting = Arc::new(AtomicBool::new(false));
 	let logger = Logger::new(log_strategy);
 	let msg_logger = Logger::new(msg_logs);
-	let interner_cell = Rc::new(RefCell::new(None::<Rc<Interner>>));
+	let interner_cell = Rc::new(RefCell::new(None::<Interner>));
 	let interner_weak = Rc::downgrade(&interner_cell);
-	let obj_store = ObjStore::default();
+	let systems_weak = Rc::downgrade(&systems_lock);
-	let mk_ctx = clone!(
+	let get_ctx = clone!(systems_weak; move |id: api::SysId| clone!(systems_weak; async move {
-		logger, systems, spawner, obj_store, interner_weak;
+		let systems =
-		move |id: api::SysId, reqnot: ReqNot<api::ExtMsgSet>| {
+			systems_weak.upgrade().expect("System table dropped before request processing done");
-			clone!(logger, systems, spawner, obj_store, interner_weak; async move {
+		systems.lock().await.get(&id).expect("System not found").ctx.clone()
-				let cted = systems.lock().await[&id].cted.clone();
+	}));
-				let interner_cell = (interner_weak.upgrade())
+	let init_ctx = {
-					.expect("mk_ctx called after Interner rc dropped");
+		clone!(interner_weak, spawner, logger);
-				let i = (interner_cell.borrow().clone())
+		move |id: api::SysId, cted: CtedObj, reqnot: ReqNot<api::ExtMsgSet>| {
-					.expect("mk_ctx called before interner initialized");
+			clone!(interner_weak, spawner, logger; async move {
-				SysCtx { id, cted, logger, reqnot, spawner, obj_store, i: i.clone() }
+				let interner_rc =
-			}.boxed_local())
+					interner_weak.upgrade().expect("System construction order while shutting down");
-	});
+				let i = interner_rc.borrow().clone().expect("mk_ctx called very early, no interner!");
+				SysCtx::new(id, i, reqnot, spawner, logger, cted)
+			})
+		}
+	};
 	let rn = ReqNot::<api::ExtMsgSet>::new(
 		msg_logger.clone(),
-		move |a, _| async move { send_parent_msg(a).await.unwrap() }.boxed_local(),
+		move |a, _| {
-		clone!(systems, exiting, mk_ctx; move |n, reqnot| {
+			clone!(in_send mut);
-			clone!(systems, exiting, mk_ctx; async move {
+			Box::pin(async move { in_send.send(a.to_vec()).await.unwrap() })
-				match n {
+		},
-					api::HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
+		{
-					api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
+			clone!(systems_weak, exit_send, get_ctx);
-						mem::drop(systems.lock().await.remove(&sys_id)),
+			move |n, _| {
-					api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) => {
+				clone!(systems_weak, exit_send mut, get_ctx);
-						let ctx = mk_ctx(sys_id, reqnot).await;
+				async move {
-						take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await
+					match n {
+						api::HostExtNotif::Exit => exit_send.send(()).await.unwrap(),
+						api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
+							if let Some(rc) = systems_weak.upgrade() {
+								mem::drop(rc.lock().await.remove(&sys_id))
+							},
+						api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) => {
+							let ctx = get_ctx(sys_id).await;
+							take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await
+						},
 					}
 				}
-			}.boxed_local())
+				.boxed_local()
-		}),
+			}
+		},
 		{
-			clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls, msg_logger);
+			clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
 			move |hand, req| {
-				clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls, msg_logger);
+				clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
 				async move {
 					let interner_cell = interner_weak.upgrade().expect("Interner dropped before request");
 					let i = interner_cell.borrow().clone().expect("Request arrived before interner set");
@@ -190,241 +174,216 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 							let (sys_id, _) = (decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system))
 								.expect("NewSystem call received for invalid system");
 							let cted = data.systems[sys_id].new_system(&new_sys);
-							let mut vfses = HashMap::new();
 							let lex_filter =
 								cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
 									char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
 								});
 							let lazy_mems = Mutex::new(HashMap::new());
-							let rules = Mutex::new(HashMap::new());
+							let ctx = init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await;
-							let ctx = SysCtx {
+							let const_root = stream::iter(cted.inst().dyn_env())
-								cted: cted.clone(),
+								.then(|mem| {
-								id: new_sys.id,
+									let (req, lazy_mems) = (&hand, &lazy_mems);
-								logger: logger.clone(),
-								reqnot: hand.reqnot(),
-								i: i.clone(),
-								obj_store: obj_store.clone(),
-								spawner: spawner.clone(),
-							};
-							let const_root = stream::from_iter(cted.inst().dyn_env())
-								.then(|(k, v)| {
-									let (req, lazy_mems, rules) = (&hand, &lazy_mems, &rules);
 									clone!(i, ctx; async move {
-										let name = i.i(&k).await.to_api();
+										let mut tia_ctx = TreeIntoApiCtxImpl {
-										let value = v.into_api(&mut TIACtxImpl {
 											lazy_members: &mut *lazy_mems.lock().await,
-											rules: &mut *rules.lock().await,
 											sys: ctx,
 											basepath: &[],
 											path: Substack::Bottom,
 											req
-										})
+										};
-										.await;
+										(i.i(&mem.name).await.to_api(), mem.kind.into_api(&mut tia_ctx).await)
-										(name, value)
 									})
 								})
 								.collect()
 								.await;
-							let declfs = cted.inst().dyn_vfs().to_api_rec(&mut vfses, &i).await;
+							let prelude =
-							let record = SystemRecord {
+								cted.inst().dyn_prelude(&i).await.iter().map(|sym| sym.to_api()).collect();
-								declfs,
+							let record = SystemRecord { ctx, lazy_members: lazy_mems.into_inner() };
-								vfses,
+							let systems = systems_weak.upgrade().expect("System constructed during shutdown");
-								cted,
-								lazy_members: lazy_mems.into_inner(),
-								rules: rules.into_inner(),
-							};
 							systems.lock().await.insert(new_sys.id, record);
-							hand
+							let line_types = join_all(
-								.handle(&new_sys, &api::SystemInst { lex_filter, const_root, line_types: vec![] })
+								(cted.inst().dyn_parsers().iter())
-								.await
+									.map(|p| async { i.i(p.line_head()).await.to_api() }),
+							)
+							.await;
+							let response = api::NewSystemResponse { lex_filter, const_root, line_types, prelude };
+							hand.handle(&new_sys, &response).await
 						},
 						api::HostExtReq::GetMember(get_tree @ api::GetMember(sys_id, tree_id)) => {
-							let sys_ctx = mk_ctx(sys_id, hand.reqnot()).await;
+							let sys_ctx = get_ctx(sys_id).await;
+							let systems = systems_weak.upgrade().expect("Member queried during shutdown");
 							let mut systems_g = systems.lock().await;
-							let SystemRecord { lazy_members, rules, .. } =
+							let SystemRecord { lazy_members, .. } =
 								systems_g.get_mut(&sys_id).expect("System not found");
 							let (path, cb) = match lazy_members.insert(tree_id, MemberRecord::Res) {
 								None => panic!("Tree for ID not found"),
 								Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
 								Some(MemberRecord::Gen(path, cb)) => (path, cb),
 							};
-							let tree = cb.build(Sym::new(path.clone(), &i).await.unwrap()).await;
+							let tree = cb.build(Sym::new(path.clone(), &i).await.unwrap(), sys_ctx.clone()).await;
-							let mut tia_ctx = TIACtxImpl {
+							let mut tia_ctx = TreeIntoApiCtxImpl {
 								sys: sys_ctx,
 								path: Substack::Bottom,
 								basepath: &path,
 								lazy_members,
-								rules,
 								req: &hand,
 							};
 							hand.handle(&get_tree, &tree.into_api(&mut tia_ctx).await).await
 						},
-						api::HostExtReq::VfsReq(api::VfsReq::GetVfs(get_vfs @ api::GetVfs(sys_id))) => {
-							let systems_g = systems.lock().await;
-							hand.handle(&get_vfs, &systems_g[&sys_id].declfs).await
-						},
 						api::HostExtReq::SysReq(api::SysReq::SysFwded(fwd)) => {
 							let api::SysFwded(sys_id, payload) = fwd;
-							let ctx = mk_ctx(sys_id, hand.reqnot()).await;
+							let ctx = get_ctx(sys_id).await;
-							let sys = ctx.cted.inst();
+							let sys = ctx.cted().inst();
 							sys.dyn_request(hand, payload).await
 						},
-						api::HostExtReq::VfsReq(api::VfsReq::VfsRead(vfs_read)) => {
+						api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, src, text, pos, id }) => {
-							let api::VfsRead(sys_id, vfs_id, path) = &vfs_read;
+							let sys_ctx = get_ctx(sys).await;
-							let ctx = mk_ctx(*sys_id, hand.reqnot()).await;
-							let systems_g = systems.lock().await;
-							let path = join_all(path.iter().map(|t| Tok::from_api(*t, &i))).await;
-							let vfs = systems_g[sys_id].vfses[vfs_id].load(&path, ctx).await;
-							hand.handle(&vfs_read, &vfs).await
-						},
-						api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, text, pos, id }) => {
-							let systems_g = systems.lock().await;
-							let lexers = systems_g[&sys].cted.inst().dyn_lexers();
-							mem::drop(systems_g);
 							let text = Tok::from_api(text, &i).await;
-							let ctx = LexContext { sys, id, pos, reqnot: hand.reqnot(), text: &text, i: &i };
+							let src = Sym::from_api(src, sys_ctx.i()).await;
+							let rep = Reporter::new();
+							let expr_store = BorrowedExprStore::new();
 							let trigger_char = text.chars().nth(pos as usize).unwrap();
-							let err_na = err_not_applicable(&i).await;
+							let ekey_na = ekey_not_applicable(&i).await;
-							let err_cascade = err_cascade(&i).await;
+							let ekey_cascade = ekey_cascade(&i).await;
+							let lexers = sys_ctx.cted().inst().dyn_lexers();
 							for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char)) {
+								let ctx = LexContext {
+									id,
+									pos,
+									text: &text,
+									src: src.clone(),
+									ctx: sys_ctx.clone(),
+									rep: &rep,
+									exprs: &expr_store,
+								};
 								match lx.lex(&text[pos as usize..], &ctx).await {
-									Err(e) if e.any(|e| *e == err_na) => continue,
+									Err(e) if e.any(|e| *e == ekey_na) => continue,
 									Err(e) => {
-										let eopt = e.keep_only(|e| *e != err_cascade).map(|e| Err(e.to_api()));
+										let eopt = e.keep_only(|e| *e != ekey_cascade).map(|e| Err(e.to_api()));
+										expr_store.dispose().await;
 										return hand.handle(&lex, &eopt).await;
 									},
 									Ok((s, expr)) => {
-										let ctx = mk_ctx(sys, hand.reqnot()).await;
+										let expr = expr.into_api(&mut (), &mut (sys_ctx, &hand)).await;
-										let expr = expr
-											.to_api(&mut |f, r| {
-												clone!(ctx; async move { do_extra(f, r, ctx).await }).boxed_local()
-											})
-											.await;
 										let pos = (text.len() - s.len()) as u32;
+										expr_store.dispose().await;
 										return hand.handle(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
 									},
 								}
 							}
 							writeln!(logger, "Got notified about n/a character '{trigger_char}'");
+							expr_store.dispose().await;
 							hand.handle(&lex, &None).await
 						},
 						api::HostExtReq::ParseLine(pline) => {
-							let api::ParseLine { exported, comments, sys, line } = &pline;
+							let api::ParseLine { module, src, exported, comments, sys, line, idx } = &pline;
-							let mut ctx = mk_ctx(*sys, hand.reqnot()).await;
+							let ctx = get_ctx(*sys).await;
-							let parsers = ctx.cted.inst().dyn_parsers();
+							let parsers = ctx.cted().inst().dyn_parsers();
-							let comments = join_all(comments.iter().map(|c| Comment::from_api(c, &i))).await;
+							let src = Sym::from_api(*src, ctx.i()).await;
-							let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx, &i).await;
+							let comments =
-							let snip = Snippet::new(line.first().expect("Empty line"), &line, &i);
+								join_all(comments.iter().map(|c| Comment::from_api(c, src.clone(), &i))).await;
-							let (head, tail) = snip.pop_front().unwrap();
+							let expr_store = BorrowedExprStore::new();
-							let name = if let GenTok::Name(n) = &head.tok { n } else { panic!("No line head") };
+							let mut from_api_ctx = (ctx.clone(), &expr_store);
-							let parser =
+							let line: Vec<PTokTree> =
-								parsers.iter().find(|p| p.line_head() == **name).expect("No parser candidate");
+								ttv_from_api(line, &mut from_api_ctx, &mut (), &src, &i).await;
-							let o_line = match parser.parse(*exported, comments, tail) {
+							let snip = Snippet::new(line.first().expect("Empty line"), &line);
+							let parser = parsers[*idx as usize];
+							let module = Sym::from_api(*module, ctx.i()).await;
+							let reporter = Reporter::new();
+							let pctx = ParsCtx::new(ctx.clone(), module, &reporter);
+							let parse_res = parser.parse(pctx, *exported, comments, snip).await;
+							let o_line = match reporter.merge(parse_res) {
 								Err(e) => Err(e.to_api()),
-								Ok(t) => Ok(
+								Ok(t) => Ok(linev_into_api(t, ctx.clone(), &hand).await),
-									ttv_to_api(t, &mut |f, range| {
-										clone!(ctx);
-										async move {
-											api::TokenTree { range, token: api::Token::Atom(f.clone().build(ctx).await) }
-										}
-										.boxed_local()
-									})
-									.await,
-								),
 							};
+							expr_store.dispose().await;
 							hand.handle(&pline, &o_line).await
 						},
+						api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst { id, sys }) => {
+							let ctx = get_ctx(sys).await;
+							let cnst = get_const(id, ctx.clone()).await;
+							hand.handle(fpc, &cnst.api_return(ctx, &hand).await).await
+						},
 						api::HostExtReq::AtomReq(atom_req) => {
 							let atom = atom_req.get_atom();
 							let atom_req = atom_req.clone();
-							with_atom_record(&mk_ctx, hand.reqnot(), atom, move |nfo, ctx, id, buf| {
+							with_atom_record(&get_ctx, atom, async move |nfo, ctx, id, buf| {
-								async move {
+								let actx = AtomCtx(buf, atom.drop, ctx.clone());
-									let actx = AtomCtx(buf, atom.drop, ctx.clone());
+
-									match &atom_req {
+								match &atom_req {
-										api::AtomReq::SerializeAtom(ser) => {
+									api::AtomReq::SerializeAtom(ser) => {
-											let mut buf = enc_vec(&id).await;
+										let mut buf = enc_vec(&id).await;
-											let refs_opt = nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await;
+										match nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await {
-											hand.handle(ser, &refs_opt.map(|refs| (buf, refs))).await
+											None => hand.handle(ser, &None).await,
-										},
+											Some(refs) => {
-										api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
+												let refs =
-											hand.handle(print, &nfo.print(actx).await.to_api()).await,
+													join_all(refs.into_iter().map(|ex| async { ex.into_api(&mut ()).await }))
-										api::AtomReq::Fwded(fwded) => {
+														.await;
-											let api::Fwded(_, key, payload) = &fwded;
+												hand.handle(ser, &Some((buf, refs))).await
-											let mut reply = Vec::new();
+											},
-											let key = Sym::from_api(*key, &i).await;
+										}
-											let some = nfo
+									},
-												.handle_req(
+									api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
-													actx,
+										hand.handle(print, &nfo.print(actx).await.to_api()).await,
-													key,
+									api::AtomReq::Fwded(fwded) => {
-													Pin::<&mut &[u8]>::new(&mut &payload[..]),
+										let api::Fwded(_, key, payload) = &fwded;
-													Pin::<&mut Vec<_>>::new(&mut reply),
+										let mut reply = Vec::new();
-												)
+										let key = Sym::from_api(*key, &i).await;
-												.await;
+										let some = nfo
-											hand.handle(fwded, &some.then_some(reply)).await
+											.handle_req(
-										},
+												actx,
-										api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
+												key,
-											let ret = nfo.call_ref(actx, *arg).await;
+												Pin::<&mut &[u8]>::new(&mut &payload[..]),
-											hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
+												Pin::<&mut Vec<_>>::new(&mut reply),
-										},
+											)
-										api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
+											.await;
-											let ret = nfo.call(actx, *arg).await;
+										hand.handle(fwded, &some.then_some(reply)).await
-											hand.handle(call, &ret.api_return(ctx.clone(), &hand).await).await
+									},
-										},
+									api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
-										api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
+										// SAFETY: function calls borrow their argument implicitly
-											Err(e) => hand.handle(cmd, &Err(e.to_api())).await,
+										let expr_store = BorrowedExprStore::new();
-											Ok(opt) => match opt {
+										let expr_handle = ExprHandle::borrowed(ctx.clone(), *arg, &expr_store);
-												None => hand.handle(cmd, &Ok(api::NextStep::Halt)).await,
+										let ret = nfo.call_ref(actx, Expr::from_handle(expr_handle.clone())).await;
-												Some(cont) => {
+										expr_handle.drop_one().await;
-													let cont = cont.api_return(ctx.clone(), &hand).await;
+										let api_expr = ret.api_return(ctx.clone(), &hand).await;
-													hand.handle(cmd, &Ok(api::NextStep::Continue(cont))).await
+										expr_store.dispose().await;
-												},
+										hand.handle(call, &api_expr).await
+									},
+									api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
+										// SAFETY: function calls borrow their argument implicitly
+										let expr_store = BorrowedExprStore::new();
+										let expr_handle = ExprHandle::borrowed(ctx.clone(), *arg, &expr_store);
+										let ret = nfo.call(actx, Expr::from_handle(expr_handle.clone())).await;
+										let api_expr = ret.api_return(ctx.clone(), &hand).await;
+										expr_store.dispose().await;
+										hand.handle(call, &api_expr).await
+									},
+									api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
+										Err(e) => hand.handle(cmd, &Err(e.to_api())).await,
+										Ok(opt) => match opt {
+											None => hand.handle(cmd, &Ok(api::NextStep::Halt)).await,
+											Some(cont) => {
+												let cont = cont.api_return(ctx.clone(), &hand).await;
+												hand.handle(cmd, &Ok(api::NextStep::Continue(cont))).await
 											},
 										},
-									}
+									},
 								}
-								.boxed_local()
 							})
 							.await
 						},
 						api::HostExtReq::DeserAtom(deser) => {
 							let api::DeserAtom(sys, buf, refs) = &deser;
 							let mut read = &mut &buf[..];
-							let ctx = mk_ctx(*sys, hand.reqnot()).await;
+							let ctx = get_ctx(*sys).await;
+							// SAFETY: deserialization implicitly grants ownership to previously owned exprs
+							let refs = (refs.iter())
+								.map(|tk| Expr::from_handle(ExprHandle::deserialize(ctx.clone(), *tk)))
+								.collect_vec();
 							let id = AtomTypeId::decode(Pin::new(&mut read)).await;
-							let inst = ctx.cted.inst();
+							let inst = ctx.cted().inst();
 							let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
-							hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, refs).await).await
+							hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, &refs).await).await
-						},
-						orchid_api::HostExtReq::ApplyMacro(am) => {
-							let tok = hand.will_handle_as(&am);
-							let ApplyMacro { id, params, run_id, sys } = am;
-							let sys_ctx = mk_ctx(sys, hand.reqnot()).await;
-							let mut ctx =
-								RuleCtx { args: ahash::HashMap::default(), run_id, sys: sys_ctx.clone() };
-							for (k, v) in params {
-								ctx.args.insert(
-									Tok::from_api(k, &i).await,
-									mtreev_from_api(&v, &i, &mut |_| panic!("No atom in macro prompt!")).await,
-								);
-							}
-							let err_cascade = err_cascade(&i).await;
-							let systems_g = systems.lock().await;
-							let rule = &systems_g[&sys].rules[&id];
-							match (rule.apply)(ctx).await {
-								Err(e) => {
-									let new_errors = e.keep_only(|e| *e != err_cascade);
-									hand.handle_as(tok, &new_errors.map(|e| Err(e.to_api()))).await
-								},
-								Ok(t) => {
-									let result = mtreev_to_api(&t, &mut |a| {
-										clone!(sys_ctx; async move {
-											api::MacroToken::Atom(a.clone().build(sys_ctx.clone()).await)
-										}.boxed_local())
-									})
-									.await;
-									hand.handle_as(tok, &Some(Ok(result))).await
-								},
-							}
 						},
 					}
 				}
@@ -432,9 +391,24 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 			}
 		},
 	);
-	*interner_cell.borrow_mut() = Some(Rc::new(Interner::new_replica(rn.clone().map())));
+	*interner_cell.borrow_mut() =
-	while !exiting.load(Ordering::Relaxed) {
+		Some(Interner::new_replica(rn.clone().map(|ir: api::IntReq| ir.into_root())));
-		let rcvd = recv_parent_msg().await.unwrap();
+	spawner(Box::pin(clone!(spawner; async move {
-		spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd).await })))
+		let mut streams = stream_select! { in_recv.map(Some), exit_recv.map(|_| None) };
+		while let Some(item) = streams.next().await {
+			match item {
+				Some(rcvd) => spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd[..]).await }))),
+				None => break,
+			}
+		}
+	})));
+	ExtInit {
+		header: ext_header,
+		port: Box::new(ExtensionOwner {
+			out_recv: Mutex::new(out_recv),
+			out_send,
+			_interner_cell: interner_cell,
+			_systems_lock: systems_lock,
+		}),
 	}
 }

orchid-extension/src/expr.rs

@@ -1,9 +1,11 @@
+use std::cell::RefCell;
 use std::fmt;
+use std::hash::Hash;
 use std::rc::Rc;
 
 use async_once_cell::OnceCell;
 use derive_destructure::destructure;
-use orchid_api::ExtAtomPrint;
+use hashbrown::HashSet;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::location::Pos;
@@ -14,17 +16,60 @@ use crate::atom::ForeignAtom;
 use crate::gen_expr::{GExpr, GExprKind};
 use crate::system::SysCtx;
 
+pub struct BorrowedExprStore(RefCell<Option<HashSet<Rc<ExprHandle>>>>);
+impl BorrowedExprStore {
+	pub(crate) fn new() -> Self { Self(RefCell::new(Some(HashSet::new()))) }
+	pub async fn dispose(self) {
+		let elements = self.0.borrow_mut().take().unwrap();
+		for handle in elements {
+			handle.drop_one().await
+		}
+	}
+}
+impl Drop for BorrowedExprStore {
+	fn drop(&mut self) {
+		if self.0.borrow().is_some() {
+			panic!("This should always be explicitly disposed")
+		}
+	}
+}
+
 #[derive(destructure)]
 pub struct ExprHandle {
 	pub tk: api::ExprTicket,
 	pub ctx: SysCtx,
 }
 impl ExprHandle {
-	pub(crate) fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } }
+	/// This function does not signal to take ownership of the expr.
+	pub fn borrowed(ctx: SysCtx, tk: api::ExprTicket, store: &BorrowedExprStore) -> Rc<Self> {
+		let this = Rc::new(Self { ctx, tk });
+		store.0.borrow_mut().as_mut().unwrap().insert(this.clone());
+		this
+	}
+	pub fn deserialize(ctx: SysCtx, tk: api::ExprTicket) -> Rc<Self> { Rc::new(Self { ctx, tk }) }
 	pub fn get_ctx(&self) -> SysCtx { self.ctx.clone() }
-	pub async fn clone(&self) -> Self {
+	/// Drop one instance of the handle silently; if it's the last one, do
-		self.ctx.reqnot.notify(api::Acquire(self.ctx.id, self.tk)).await;
+	/// nothing, otherwise send an Acquire
-		Self { ctx: self.ctx.clone(), tk: self.tk }
+	pub async fn drop_one(self: Rc<Self>) {
+		if let Err(rc) = Rc::try_unwrap(self) {
+			rc.ctx.reqnot().notify(api::Acquire(rc.ctx.sys_id(), rc.tk)).await
+		}
+	}
+	/// Drop the handle and get the ticket without a release notification.
+	/// Use this with messages that imply ownership transfer. This function is
+	/// safe because abusing it is a memory leak.
+	pub fn serialize(self) -> api::ExprTicket { self.destructure().0 }
+}
+impl Eq for ExprHandle {}
+impl PartialEq for ExprHandle {
+	fn eq(&self, other: &Self) -> bool {
+		self.ctx.sys_id() == other.ctx.sys_id() && self.tk == other.tk
+	}
+}
+impl Hash for ExprHandle {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		self.ctx.sys_id().hash(state);
+		self.tk.hash(state);
 	}
 }
 impl fmt::Debug for ExprHandle {
@@ -34,9 +79,9 @@ impl fmt::Debug for ExprHandle {
 }
 impl Drop for ExprHandle {
 	fn drop(&mut self) {
-		let notif = api::Release(self.ctx.id, self.tk);
+		let notif = api::Release(self.ctx.sys_id(), self.tk);
-		let SysCtx { reqnot, spawner, .. } = self.ctx.clone();
+		let reqnot = self.ctx.reqnot().clone();
-		spawner(Box::pin(async move { reqnot.notify(notif).await }))
+		self.ctx.spawner()(Box::pin(async move { reqnot.notify(notif).await }))
 	}
 }
 
@@ -53,20 +98,20 @@ impl Expr {
 
 	pub async fn data(&self) -> &ExprData {
 		(self.data.get_or_init(async {
-			let details = self.handle.ctx.reqnot.request(api::Inspect { target: self.handle.tk }).await;
+			let details = self.handle.ctx.reqnot().request(api::Inspect { target: self.handle.tk }).await;
-			let pos = Pos::from_api(&details.location, &self.handle.ctx.i).await;
+			let pos = Pos::from_api(&details.location, self.handle.ctx.i()).await;
 			let kind = match details.kind {
 				api::InspectedKind::Atom(a) =>
 					ExprKind::Atom(ForeignAtom::new(self.handle.clone(), a, pos.clone())),
 				api::InspectedKind::Bottom(b) =>
-					ExprKind::Bottom(OrcErrv::from_api(&b, &self.handle.ctx.i).await),
+					ExprKind::Bottom(OrcErrv::from_api(&b, self.handle.ctx.i()).await),
 				api::InspectedKind::Opaque => ExprKind::Opaque,
 			};
 			ExprData { pos, kind }
 		}))
 		.await
 	}
-	pub async fn atom(self) -> Result<ForeignAtom<'static>, Self> {
+	pub async fn atom(self) -> Result<ForeignAtom, Self> {
 		match self.data().await {
 			ExprData { kind: ExprKind::Atom(atom), .. } => Ok(atom.clone()),
 			_ => Err(self),
@@ -75,15 +120,18 @@ impl Expr {
 	pub fn handle(&self) -> Rc<ExprHandle> { self.handle.clone() }
 	pub fn ctx(&self) -> SysCtx { self.handle.ctx.clone() }
 
-	pub fn gen(&self) -> GExpr { GExpr { pos: Pos::SlotTarget, kind: GExprKind::Slot(self.clone()) } }
+	pub fn slot(&self) -> GExpr {
+		GExpr { pos: Pos::SlotTarget, kind: GExprKind::Slot(self.clone()) }
+	}
 }
 impl Format for Expr {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		match &self.data().await.kind {
 			ExprKind::Opaque => "OPAQUE".to_string().into(),
 			ExprKind::Bottom(b) => format!("Bottom({b})").into(),
-			ExprKind::Atom(a) =>
+			ExprKind::Atom(a) => FmtUnit::from_api(
-				FmtUnit::from_api(&self.handle.ctx.reqnot.request(ExtAtomPrint(a.atom.clone())).await),
+				&self.handle.ctx.reqnot().request(api::ExtAtomPrint(a.atom.clone())).await,
+			),
 		}
 	}
 }
@@ -96,7 +144,7 @@ pub struct ExprData {
 
 #[derive(Clone, Debug)]
 pub enum ExprKind {
-	Atom(ForeignAtom<'static>),
+	Atom(ForeignAtom),
 	Bottom(OrcErrv),
 	Opaque,
 }

orchid-extension/src/fs.rs

@@ -1,46 +0,0 @@
-use std::num::NonZero;
-
-use futures::FutureExt;
-use futures::future::LocalBoxFuture;
-use hashbrown::HashMap;
-use orchid_base::interner::{Interner, Tok};
-
-use crate::api;
-use crate::system::SysCtx;
-
-pub trait VirtFS: Send + Sync + 'static {
-	fn load<'a>(
-		&'a self,
-		path: &'a [Tok<String>],
-		ctx: SysCtx,
-	) -> LocalBoxFuture<'a, api::OrcResult<api::Loaded>>;
-}
-
-pub enum DeclFs {
-	Lazy(&'static dyn VirtFS),
-	Mod(&'static [(&'static str, DeclFs)]),
-}
-impl DeclFs {
-	pub async fn to_api_rec(
-		&self,
-		vfses: &mut HashMap<api::VfsId, &'static dyn VirtFS>,
-		i: &Interner,
-	) -> api::EagerVfs {
-		match self {
-			DeclFs::Lazy(fs) => {
-				let vfsc: u16 = vfses.len().try_into().expect("too many vfses (more than u16::MAX)");
-				let id = api::VfsId(NonZero::new(vfsc + 1).unwrap());
-				vfses.insert(id, *fs);
-				api::EagerVfs::Lazy(id)
-			},
-			DeclFs::Mod(children) => {
-				let mut output = std::collections::HashMap::new();
-				for (k, v) in children.iter() {
-					output
-						.insert(i.i::<String>(*k).await.to_api(), v.to_api_rec(vfses, i).boxed_local().await);
-				}
-				api::EagerVfs::Eager(output)
-			},
-		}
-	}
-}

orchid-extension/src/func_atom.rs

@@ -1,40 +1,68 @@
+use std::any::TypeId;
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::future::Future;
 use std::pin::Pin;
 use std::rc::Rc;
 
-use async_std::io::Write;
-use async_std::sync::Mutex;
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
+use futures::lock::Mutex;
+use futures::{AsyncWrite, FutureExt};
 use itertools::Itertools;
 use never::Never;
 use orchid_api_traits::Encode;
 use orchid_base::clone;
 use orchid_base::error::OrcRes;
-use orchid_base::format::{FmtCtxImpl, Format, take_first};
+use orchid_base::format::{FmtCtx, FmtUnit};
 use orchid_base::name::Sym;
 use trait_set::trait_set;
 
-use crate::atom::{Atomic, MethodSetBuilder};
+use crate::atom::Atomic;
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use crate::conv::ToExpr;
-use crate::expr::{Expr, ExprHandle};
+use crate::coroutine_exec::{ExecHandle, exec};
+use crate::expr::Expr;
 use crate::gen_expr::GExpr;
-use crate::system::SysCtx;
+use crate::system::{SysCtx, SysCtxEntry};
 
 trait_set! {
 	trait FunCB = Fn(Vec<Expr>) -> LocalBoxFuture<'static, OrcRes<GExpr>> + 'static;
 }
 
 pub trait ExprFunc<I, O>: Clone + 'static {
-	const ARITY: u8;
+	fn argtyps() -> &'static [TypeId];
-	fn apply(&self, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
+	fn apply<'a>(&self, hand: ExecHandle<'a>, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
 }
 
-thread_local! {
+#[derive(Default)]
-	static FUNS: Rc<Mutex<HashMap<Sym, (u8, Rc<dyn FunCB>)>>> = Rc::default();
+struct FunsCtx(Mutex<HashMap<Sym, FunRecord>>);
+impl SysCtxEntry for FunsCtx {}
+#[derive(Clone)]
+struct FunRecord {
+	argtyps: &'static [TypeId],
+	fun: Rc<dyn FunCB>,
+}
+
+async fn process_args<I, O, F: ExprFunc<I, O>>(
+	debug: impl AsRef<str> + Clone + 'static,
+	f: F,
+) -> FunRecord {
+	let argtyps = F::argtyps();
+	let fun = Rc::new(move |v: Vec<Expr>| {
+		clone!(f, v mut);
+		exec(debug.clone(), async move |mut hand| {
+			let mut norm_args = Vec::with_capacity(v.len());
+			for (expr, typ) in v.into_iter().zip(argtyps) {
+				if *typ != TypeId::of::<Expr>() {
+					norm_args.push(hand.exec(expr).await?);
+				}
+			}
+			f.apply(hand, norm_args).await
+		})
+		.map(Ok)
+		.boxed_local()
+	});
+	FunRecord { argtyps, fun }
 }
 
 /// An Atom representing a partially applied named native function. These
@@ -46,55 +74,53 @@ thread_local! {
 pub(crate) struct Fun {
 	path: Sym,
 	args: Vec<Expr>,
-	arity: u8,
+	record: FunRecord,
-	fun: Rc<dyn FunCB>,
 }
 impl Fun {
-	pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, f: F) -> Self {
+	pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, ctx: SysCtx, f: F) -> Self {
-		let funs = FUNS.with(|funs| funs.clone());
+		let funs: &FunsCtx = ctx.get_or_default();
-		let mut fung = funs.lock().await;
+		let mut fung = funs.0.lock().await;
-		let fun = if let Some(x) = fung.get(&path) {
+		let record = if let Some(record) = fung.get(&path) {
-			x.1.clone()
+			record.clone()
 		} else {
-			let fun = Rc::new(move |v| clone!(f; async move { f.apply(v).await }.boxed_local()));
+			let record = process_args(path.to_string(), f).await;
-			fung.insert(path.clone(), (F::ARITY, fun.clone()));
+			fung.insert(path.clone(), record.clone());
-			fun
+			record
 		};
-		Self { args: vec![], arity: F::ARITY, path, fun }
+		Self { args: vec![], path, record }
 	}
-	pub fn arity(&self) -> u8 { self.arity }
+	pub fn arity(&self) -> u8 { self.record.argtyps.len() as u8 }
 }
 impl Atomic for Fun {
 	type Data = ();
 	type Variant = OwnedVariant;
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl OwnedAtom for Fun {
 	type Refs = Vec<Expr>;
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-	async fn call_ref(&self, arg: ExprHandle) -> GExpr {
+	async fn call_ref(&self, arg: Expr) -> GExpr {
-		std::io::Write::flush(&mut std::io::stderr()).unwrap();
+		let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
-		let new_args = self.args.iter().cloned().chain([Expr::from_handle(Rc::new(arg))]).collect_vec();
+		if new_args.len() == self.record.argtyps.len() {
-		if new_args.len() == self.arity.into() {
+			(self.record.fun)(new_args).await.to_expr().await
-			(self.fun)(new_args).await.to_expr()
 		} else {
-			Self { args: new_args, arity: self.arity, fun: self.fun.clone(), path: self.path.clone() }
+			Self { args: new_args, record: self.record.clone(), path: self.path.clone() }.to_expr().await
-				.to_expr()
 		}
 	}
-	async fn call(self, arg: ExprHandle) -> GExpr { self.call_ref(arg).await }
+	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
-	async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs {
+	async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl AsyncWrite + ?Sized)>) -> Self::Refs {
 		self.path.to_api().encode(write).await;
 		self.args.clone()
 	}
 	async fn deserialize(mut ctx: impl DeserializeCtx, args: Self::Refs) -> Self {
 		let sys = ctx.sys();
-		let path = Sym::from_api(ctx.decode().await, &sys.i).await;
+		let path = Sym::from_api(ctx.decode().await, sys.i()).await;
-		let (arity, fun) = FUNS.with(|f| f.clone()).lock().await.get(&path).unwrap().clone();
+		let record = (sys.get::<FunsCtx>().0.lock().await.get(&path))
-		Self { args, arity, path, fun }
+			.expect("Function missing during deserialization")
+			.clone();
+		Self { args, path, record }
 	}
-	async fn print(&self, _: SysCtx) -> orchid_base::format::FmtUnit {
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		format!("{}:{}/{}", self.path, self.args.len(), self.arity).into()
+		format!("{}:{}/{}", self.path, self.args.len(), self.arity()).into()
 	}
 }
 
@@ -105,74 +131,78 @@ impl OwnedAtom for Fun {
 #[derive(Clone)]
 pub struct Lambda {
 	args: Vec<Expr>,
-	arity: u8,
+	record: FunRecord,
-	fun: Rc<dyn FunCB>,
 }
 impl Lambda {
-	pub fn new<I, O, F: ExprFunc<I, O>>(f: F) -> Self {
+	pub async fn new<I, O, F: ExprFunc<I, O>>(
-		let fun = Rc::new(move |v| clone!(f; async move { f.apply(v).await }.boxed_local()));
+		debug: impl AsRef<str> + Clone + 'static,
-		Self { args: vec![], arity: F::ARITY, fun }
+		f: F,
+	) -> Self {
+		Self { args: vec![], record: process_args(debug, f).await }
 	}
 }
 impl Atomic for Lambda {
 	type Data = ();
 	type Variant = OwnedVariant;
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl OwnedAtom for Lambda {
 	type Refs = Never;
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-	async fn call_ref(&self, arg: ExprHandle) -> GExpr {
+	async fn call_ref(&self, arg: Expr) -> GExpr {
-		let new_args = self.args.iter().cloned().chain([Expr::from_handle(Rc::new(arg))]).collect_vec();
+		let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
-		if new_args.len() == self.arity.into() {
+		if new_args.len() == self.record.argtyps.len() {
-			(self.fun)(new_args).await.to_expr()
+			(self.record.fun)(new_args).await.to_expr().await
 		} else {
-			Self { args: new_args, arity: self.arity, fun: self.fun.clone() }.to_expr()
+			Self { args: new_args, record: self.record.clone() }.to_expr().await
 		}
 	}
-	async fn call(self, arg: ExprHandle) -> GExpr { self.call_ref(arg).await }
+	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
 }
 
 mod expr_func_derives {
-	use std::future::Future;
+	use std::any::TypeId;
+	use std::sync::OnceLock;
 
 	use orchid_base::error::OrcRes;
 
 	use super::ExprFunc;
 	use crate::conv::{ToExpr, TryFromExpr};
-	use crate::func_atom::Expr;
+	use crate::func_atom::{ExecHandle, Expr};
 	use crate::gen_expr::GExpr;
 
 	macro_rules! expr_func_derive {
-    ($arity: tt, $($t:ident),*) => {
+    ($($t:ident),*) => {
-      paste::paste!{
+      pastey::paste!{
         impl<
-          $($t: TryFromExpr, )*
+          $($t: TryFromExpr + 'static, )*
-					Fut: Future<Output: ToExpr>,
+					Out: ToExpr,
-          Func: Fn($($t,)*) -> Fut + Clone + Send + Sync + 'static
+          Func: AsyncFn($($t,)*) -> Out + Clone + Send + Sync + 'static
-        > ExprFunc<($($t,)*), Fut::Output> for Func {
+        > ExprFunc<($($t,)*), Out> for Func {
-          const ARITY: u8 = $arity;
+          fn argtyps() -> &'static [TypeId] {
-          async fn apply(&self, v: Vec<Expr>) -> OrcRes<GExpr> {
+            static STORE: OnceLock<Vec<TypeId>> = OnceLock::new();
-            assert_eq!(v.len(), Self::ARITY.into(), "Arity mismatch");
+            &*STORE.get_or_init(|| vec![$(TypeId::of::<$t>()),*])
+          }
+          async fn apply<'a>(&self, _: ExecHandle<'a>, v: Vec<Expr>) -> OrcRes<GExpr> {
+            assert_eq!(v.len(), Self::argtyps().len(), "Arity mismatch");
             let [$([< $t:lower >],)*] = v.try_into().unwrap_or_else(|_| panic!("Checked above"));
-            Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_expr())
+            Ok(self($($t::try_from_expr([< $t:lower >]).await?,)*).await.to_expr().await)
           }
         }
       }
     };
   }
-	expr_func_derive!(1, A);
+	expr_func_derive!(A);
-	expr_func_derive!(2, A, B);
+	expr_func_derive!(A, B);
-	expr_func_derive!(3, A, B, C);
+	expr_func_derive!(A, B, C);
-	expr_func_derive!(4, A, B, C, D);
+	expr_func_derive!(A, B, C, D);
-	expr_func_derive!(5, A, B, C, D, E);
+	expr_func_derive!(A, B, C, D, E);
-	expr_func_derive!(6, A, B, C, D, E, F);
+	// expr_func_derive!(A, B, C, D, E, F);
-	expr_func_derive!(7, A, B, C, D, E, F, G);
+	// expr_func_derive!(A, B, C, D, E, F, G);
-	expr_func_derive!(8, A, B, C, D, E, F, G, H);
+	// expr_func_derive!(A, B, C, D, E, F, G, H);
-	expr_func_derive!(9, A, B, C, D, E, F, G, H, I);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I);
-	expr_func_derive!(10, A, B, C, D, E, F, G, H, I, J);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J);
-	expr_func_derive!(11, A, B, C, D, E, F, G, H, I, J, K);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K);
-	expr_func_derive!(12, A, B, C, D, E, F, G, H, I, J, K, L);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L);
-	expr_func_derive!(13, A, B, C, D, E, F, G, H, I, J, K, L, M);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L, M);
-	expr_func_derive!(14, A, B, C, D, E, F, G, H, I, J, K, L, M, N);
+	// expr_func_derive!(A, B, C, D, E, F, G, H, I, J, K, L, M, N);
 }

orchid-extension/src/gen_expr.rs

@@ -1,19 +1,19 @@
-use std::future::Future;
+use std::rc::Rc;
 
 use futures::FutureExt;
 use orchid_base::error::{OrcErr, OrcErrv};
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::location::Pos;
-use orchid_base::match_mapping;
 use orchid_base::name::Sym;
 use orchid_base::reqnot::ReqHandlish;
+use orchid_base::{match_mapping, tl_cache};
 
 use crate::api;
 use crate::atom::{AtomFactory, ToAtom};
-use crate::conv::{ToExpr, TryFromExpr};
 use crate::expr::Expr;
-use crate::func_atom::Lambda;
 use crate::system::SysCtx;
 
+#[derive(Clone, Debug)]
 pub struct GExpr {
 	pub kind: GExprKind,
 	pub pos: Pos,
@@ -33,8 +33,15 @@ impl GExpr {
 			}
 		}
 	}
+	pub fn at(self, pos: Pos) -> Self { GExpr { pos, kind: self.kind } }
+}
+impl Format for GExpr {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		self.kind.print(c).await
+	}
 }
 
+#[derive(Clone, Debug)]
 pub enum GExprKind {
 	Call(Box<GExpr>, Box<GExpr>),
 	Lambda(u64, Box<GExpr>),
@@ -59,7 +66,6 @@ impl GExprKind {
 			Lambda(arg, body => Box::new(body.api_return(ctx, hand).await)),
 			Arg(arg),
 			Const(name.to_api()),
-			Const(name.to_api()),
 			Bottom(err.to_api()),
 			NewAtom(fac.clone().build(ctx).await),
 		} {
@@ -67,6 +73,28 @@ impl GExprKind {
 		})
 	}
 }
+impl Format for GExprKind {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		match self {
+			GExprKind::Call(f, x) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0} ({1})")))
+					.units([f.print(c).await, x.print(c).await]),
+			GExprKind::Lambda(arg, body) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("\\{0}.{1}")))
+					.units([arg.to_string().into(), body.print(c).await]),
+			GExprKind::Arg(arg) => arg.to_string().into(),
+			GExprKind::Seq(a, b) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("[{0}] {1}")))
+					.units([a.print(c).await, b.print(c).await]),
+			GExprKind::Const(sym) => sym.to_string().into(),
+			GExprKind::NewAtom(atom_factory) => atom_factory.to_string().into(),
+			GExprKind::Slot(expr) =>
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{{{0}}}")))
+					.units([expr.print(c).await]),
+			GExprKind::Bottom(orc_errv) => orc_errv.to_string().into(),
+		}
+	}
+}
 
 fn inherit(kind: GExprKind) -> GExpr { GExpr { pos: Pos::Inherit, kind } }
 
@@ -99,10 +127,3 @@ pub fn call(v: impl IntoIterator<Item = GExpr>) -> GExpr {
 pub fn bot(ev: impl IntoIterator<Item = OrcErr>) -> GExpr {
 	inherit(GExprKind::Bottom(OrcErrv::new(ev).unwrap()))
 }
-
-pub fn with<I: TryFromExpr, Fut: Future<Output: ToExpr>>(
-	expr: GExpr,
-	cont: impl Fn(I) -> Fut + Clone + Send + Sync + 'static,
-) -> GExpr {
-	call([lambda(0, [seq([arg(0), call([Lambda::new(cont).to_expr(), arg(0)])])]), expr])
-}

orchid-extension/src/interner.rs

@@ -0,0 +1,7 @@
+use orchid_base::interner::{ApiStrTok, ApiStrvTok, IStr};
+
+pub struct ExtIStr(ApiStrTok, Rc<String>);
+impl Deref for ExtIStr {}
+pub struct ExtIStrv(ApiStrvTok, Rc<Vec<IStr>>);
+
+pub struct ExtInterner {}

orchid-extension/src/lexer.rs

@@ -1,56 +1,82 @@
+use std::fmt;
 use std::future::Future;
-use std::ops::{Range, RangeInclusive};
+use std::ops::RangeInclusive;
 
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
-use orchid_base::error::{OrcErr, OrcRes, mk_err};
+use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_err};
 use orchid_base::interner::{Interner, Tok};
-use orchid_base::location::Pos;
+use orchid_base::location::{Pos, SrcRange};
-use orchid_base::reqnot::{ReqNot, Requester};
+use orchid_base::name::Sym;
-use orchid_base::tree::TokHandle;
+use orchid_base::parse::ParseCtx;
+use orchid_base::reqnot::Requester;
 
 use crate::api;
-use crate::tree::{GenTok, GenTokTree};
+use crate::expr::BorrowedExprStore;
+use crate::parser::PTokTree;
+use crate::system::SysCtx;
+use crate::tree::GenTokTree;
 
-pub async fn err_cascade(i: &Interner) -> OrcErr {
+pub async fn ekey_cascade(i: &Interner) -> IStr {
-	mk_err(
+	i.i("An error cascading from a recursive call").await
-		i.i("An error cascading from a recursive call").await,
+}
-		"This error is a sentinel for the extension library.\
+pub async fn ekey_not_applicable(i: &Interner) -> IStr {
-		it should not be emitted by the extension.",
+	i.i("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
-		[Pos::None.into()],
+}
-	)
+const MSG_INTERNAL_ERROR: &str = "This error is a sentinel for the extension library.\
+	it should not be emitted by the extension.";
+
+pub async fn err_cascade(i: &Interner) -> OrcErrv {
+	mk_err(ekey_cascade(i).await, MSG_INTERNAL_ERROR, [Pos::None])
 }
 
-pub async fn err_not_applicable(i: &Interner) -> OrcErr {
+pub async fn err_not_applicable(i: &Interner) -> OrcErrv {
-	mk_err(
+	mk_err(ekey_not_applicable(i).await, MSG_INTERNAL_ERROR, [Pos::None])
-		i.i("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await,
-		&*err_cascade(i).await.message,
-		[Pos::None.into()],
-	)
 }
 
 pub struct LexContext<'a> {
-	pub text: &'a Tok<String>,
+	pub(crate) exprs: &'a BorrowedExprStore,
-	pub sys: api::SysId,
+	pub ctx: SysCtx,
+	pub text: &'a IStr,
 	pub id: api::ParsId,
 	pub pos: u32,
-	pub reqnot: ReqNot<api::ExtMsgSet>,
+	pub(crate) src: Sym,
-	pub i: &'a Interner,
+	pub(crate) rep: &'a Reporter,
 }
 impl<'a> LexContext<'a> {
-	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree<'a>)> {
+	pub fn src(&self) -> &Sym { &self.src }
+	/// This function returns [PTokTree] because it can never return
+	/// [orchid_base::tree::Token::NewExpr]. You can use
+	/// [crate::parser::p_tree2gen] to convert this to [crate::tree::GenTokTree]
+	/// for embedding in the return value.
+	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, PTokTree)> {
 		let start = self.pos(tail);
-		let Some(lx) = self.reqnot.request(api::SubLex { pos: start, id: self.id }).await else {
+		let Some(lx) = self.ctx.reqnot().request(api::SubLex { pos: start, id: self.id }).await else {
-			return Err(err_cascade(self.i).await.into());
+			return Err(err_cascade(self.ctx.i()).await);
 		};
-		Ok((&self.text[lx.pos as usize..], GenTok::Slot(TokHandle::new(lx.ticket)).at(start..lx.pos)))
+		let tree = PTokTree::from_api(
+			&lx.tree,
+			&mut (self.ctx.clone(), self.exprs),
+			&mut (),
+			&self.src,
+			self.ctx.i(),
+		)
+		.await;
+		Ok((&self.text[lx.pos as usize..], tree))
 	}
 
 	pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }
-
+	pub fn pos_tt(&self, tail_with: &'a str, tail_without: &'a str) -> SrcRange {
-	pub fn tok_ran(&self, len: u32, tail: &'a str) -> Range<u32> {
+		SrcRange::new(self.pos(tail_with)..self.pos(tail_without), &self.src)
-		self.pos(tail) - len..self.pos(tail)
 	}
+
+	pub fn pos_lt(&self, len: impl TryInto<u32, Error: fmt::Debug>, tail: &'a str) -> SrcRange {
+		SrcRange::new(self.pos(tail) - len.try_into().unwrap()..self.pos(tail), &self.src)
+	}
+}
+impl ParseCtx for LexContext<'_> {
+	fn i(&self) -> &Interner { self.ctx.i() }
+	fn rep(&self) -> &Reporter { self.rep }
 }
 
 pub trait Lexer: Send + Sync + Sized + Default + 'static {
@@ -58,7 +84,7 @@ pub trait Lexer: Send + Sync + Sized + Default + 'static {
 	fn lex<'a>(
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> impl Future<Output = OrcRes<(&'a str, GenTokTree<'a>)>>;
+	) -> impl Future<Output = OrcRes<(&'a str, GenTokTree)>>;
 }
 
 pub trait DynLexer: Send + Sync + 'static {
@@ -67,7 +93,7 @@ pub trait DynLexer: Send + Sync + 'static {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree<'a>)>>;
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>>;
 }
 
 impl<T: Lexer> DynLexer for T {
@@ -76,7 +102,7 @@ impl<T: Lexer> DynLexer for T {
 		&self,
 		tail: &'a str,
 		ctx: &'a LexContext<'a>,
-	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree<'a>)>> {
+	) -> LocalBoxFuture<'a, OrcRes<(&'a str, GenTokTree)>> {
 		T::lex(tail, ctx).boxed_local()
 	}
 }

orchid-extension/src/lib.rs

@@ -4,16 +4,18 @@ pub mod atom;
 pub mod atom_owned;
 pub mod atom_thin;
 pub mod conv;
+pub mod coroutine_exec;
 pub mod entrypoint;
 pub mod expr;
-pub mod fs;
 pub mod func_atom;
 pub mod gen_expr;
 pub mod lexer;
-pub mod macros;
+// pub mod msg;
-pub mod msg;
+mod interner;
 pub mod other_system;
 pub mod parser;
+pub mod reflection;
 pub mod system;
 pub mod system_ctor;
+pub mod tokio;
 pub mod tree;

orchid-extension/src/macros.rs

@@ -1,102 +0,0 @@
-use std::rc::Rc;
-
-use ahash::HashMap;
-use futures::future::{LocalBoxFuture, join_all};
-use itertools::Itertools;
-use never::Never;
-use orchid_base::error::OrcRes;
-use orchid_base::interner::Tok;
-use orchid_base::macros::{MTree, mtreev_from_api, mtreev_to_api};
-use orchid_base::reqnot::Requester;
-use trait_set::trait_set;
-
-use crate::api;
-use crate::atom::AtomFactory;
-use crate::lexer::err_cascade;
-use crate::system::SysCtx;
-use crate::tree::TreeIntoApiCtx;
-
-pub trait Macro {
-	fn pattern() -> MTree<'static, Never>;
-	fn apply(binds: HashMap<Tok<String>, MTree<'_, Never>>) -> MTree<'_, AtomFactory>;
-}
-
-pub trait DynMacro {
-	fn pattern(&self) -> MTree<'static, Never>;
-	fn apply<'a>(&self, binds: HashMap<Tok<String>, MTree<'a, Never>>) -> MTree<'a, AtomFactory>;
-}
-
-impl<T: Macro> DynMacro for T {
-	fn pattern(&self) -> MTree<'static, Never> { Self::pattern() }
-	fn apply<'a>(&self, binds: HashMap<Tok<String>, MTree<'a, Never>>) -> MTree<'a, AtomFactory> {
-		Self::apply(binds)
-	}
-}
-
-pub struct RuleCtx<'a> {
-	pub(crate) args: HashMap<Tok<String>, Vec<MTree<'a, Never>>>,
-	pub(crate) run_id: api::ParsId,
-	pub(crate) sys: SysCtx,
-}
-impl<'a> RuleCtx<'a> {
-	pub async fn recurse(&mut self, tree: &[MTree<'a, Never>]) -> OrcRes<Vec<MTree<'a, Never>>> {
-		let req = api::RunMacros {
-			run_id: self.run_id,
-			query: mtreev_to_api(tree, &mut |b| match *b {}).await,
-		};
-		let Some(treev) = self.sys.reqnot.request(req).await else {
-			return Err(err_cascade(&self.sys.i).await.into());
-		};
-		static ATOM_MSG: &str = "Returned atom from Rule recursion";
-		Ok(mtreev_from_api(&treev, &self.sys.i, &mut |_| panic!("{ATOM_MSG}")).await)
-	}
-	pub fn getv(&mut self, key: &Tok<String>) -> Vec<MTree<'a, Never>> {
-		self.args.remove(key).expect("Key not found")
-	}
-	pub fn gets(&mut self, key: &Tok<String>) -> MTree<'a, Never> {
-		let v = self.getv(key);
-		assert!(v.len() == 1, "Not a scalar");
-		v.into_iter().next().unwrap()
-	}
-	pub fn unused_arg<'b>(&mut self, keys: impl IntoIterator<Item = &'b Tok<String>>) {
-		keys.into_iter().for_each(|k| {
-			self.getv(k);
-		});
-	}
-}
-
-trait_set! {
-	pub trait RuleCB = for<'a> Fn(RuleCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Vec<MTree<'a, AtomFactory>>>>;
-}
-
-pub struct Rule {
-	pub(crate) comments: Vec<String>,
-	pub(crate) pattern: Vec<MTree<'static, Never>>,
-	pub(crate) apply: Rc<dyn RuleCB>,
-}
-impl Rule {
-	pub(crate) async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MacroRule {
-		api::MacroRule {
-			comments: join_all(self.comments.iter().map(|c| async {
-				api::Comment { text: ctx.sys().i.i(c).await.to_api(), location: api::Location::Inherit }
-			}))
-			.await,
-			location: api::Location::Inherit,
-			pattern: mtreev_to_api(&self.pattern, &mut |b| match *b {}).await,
-			id: ctx.with_rule(Rc::new(self)),
-		}
-	}
-}
-
-pub fn rule_cmt<'a>(
-	cmt: impl IntoIterator<Item = &'a str>,
-	pattern: Vec<MTree<'static, Never>>,
-	apply: impl RuleCB + 'static,
-) -> Rule {
-	let comments = cmt.into_iter().map(|s| s.to_string()).collect_vec();
-	Rule { comments, pattern, apply: Rc::new(apply) }
-}
-
-pub fn rule(pattern: Vec<MTree<'static, Never>>, apply: impl RuleCB + 'static) -> Rule {
-	rule_cmt([], pattern, apply)
-}

orchid-extension/src/msg.rs

@@ -1,12 +1,9 @@
 use std::pin::pin;
 
 use async_once_cell::OnceCell;
-use async_std::io::{self, Stdout};
+use futures::lock::Mutex;
-use async_std::sync::Mutex;
 use orchid_base::msg::{recv_msg, send_msg};
 
-static STDOUT: OnceCell<Mutex<Stdout>> = OnceCell::new();
-
 pub async fn send_parent_msg(msg: &[u8]) -> io::Result<()> {
 	let stdout_lk = STDOUT.get_or_init(async { Mutex::new(io::stdout()) }).await;
 	let mut stdout_g = stdout_lk.lock().await;

orchid-extension/src/parser.rs

@@ -1,40 +1,225 @@
-use orchid_base::error::OrcRes;
+use std::marker::PhantomData;
-use orchid_base::parse::{Comment, Snippet};
 
-use crate::atom::{AtomFactory, ForeignAtom};
+use async_fn_stream::stream;
-use crate::tree::GenTokTree;
+use futures::future::{LocalBoxFuture, join_all};
+use futures::{FutureExt, Stream, StreamExt};
+use itertools::Itertools;
+use never::Never;
+use orchid_base::error::{OrcErrv, OrcRes, Reporter};
+use orchid_base::id_store::IdStore;
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::SrcRange;
+use orchid_base::match_mapping;
+use orchid_base::name::Sym;
+use orchid_base::parse::{Comment, ParseCtx, Snippet};
+use orchid_base::reqnot::{ReqHandlish, Requester};
+use orchid_base::tree::{TokTree, Token, ttv_into_api};
 
-pub type GenSnippet<'a> = Snippet<'a, 'a, ForeignAtom<'a>, AtomFactory>;
+use crate::api;
+use crate::conv::ToExpr;
+use crate::expr::Expr;
+use crate::gen_expr::GExpr;
+use crate::system::{SysCtx, SysCtxEntry};
+use crate::tree::{GenTok, GenTokTree};
+
+pub type PTok = Token<Expr, Never>;
+pub type PTokTree = TokTree<Expr, Never>;
+pub type PSnippet<'a> = Snippet<'a, Expr, Never>;
+
+pub fn p_tok2gen(tok: PTok) -> GenTok {
+	match_mapping!(tok, PTok => GenTok {
+		Comment(s), Name(n), BR, Handle(ex), Bottom(err),
+		LambdaHead(arg => Box::new(p_tree2gen(*arg))),
+		NS(n, arg => Box::new(p_tree2gen(*arg))),
+		S(p, body () p_v2gen),
+	} {
+		PTok::NewExpr(never) => match never {}
+	})
+}
+pub fn p_tree2gen(tree: PTokTree) -> GenTokTree {
+	TokTree { tok: p_tok2gen(tree.tok), sr: tree.sr }
+}
+pub fn p_v2gen(v: impl IntoIterator<Item = PTokTree>) -> Vec<GenTokTree> {
+	v.into_iter().map(p_tree2gen).collect_vec()
+}
 
 pub trait Parser: Send + Sync + Sized + Default + 'static {
 	const LINE_HEAD: &'static str;
-	fn parse(
+	fn parse<'a>(
+		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'_>,
+		line: PSnippet<'a>,
-	) -> OrcRes<Vec<GenTokTree<'_>>>;
+	) -> impl Future<Output = OrcRes<Vec<ParsedLine>>> + 'a;
 }
 
 pub trait DynParser: Send + Sync + 'static {
 	fn line_head(&self) -> &'static str;
 	fn parse<'a>(
 		&self,
+		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'a>,
+		line: PSnippet<'a>,
-	) -> OrcRes<Vec<GenTokTree<'a>>>;
+	) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>>;
 }
 
 impl<T: Parser> DynParser for T {
 	fn line_head(&self) -> &'static str { Self::LINE_HEAD }
 	fn parse<'a>(
 		&self,
+		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: GenSnippet<'a>,
+		line: PSnippet<'a>,
-	) -> OrcRes<Vec<GenTokTree<'a>>> {
+	) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>> {
-		Self::parse(exported, comments, line)
+		Box::pin(async move { Self::parse(ctx, exported, comments, line).await })
 	}
 }
 
 pub type ParserObj = &'static dyn DynParser;
+
+pub struct ParsCtx<'a> {
+	_parse: PhantomData<&'a mut ()>,
+	ctx: SysCtx,
+	module: Sym,
+	reporter: &'a Reporter,
+}
+impl<'a> ParsCtx<'a> {
+	pub(crate) fn new(ctx: SysCtx, module: Sym, reporter: &'a Reporter) -> Self {
+		Self { _parse: PhantomData, ctx, module, reporter }
+	}
+	pub fn ctx(&self) -> &SysCtx { &self.ctx }
+	pub fn module(&self) -> Sym { self.module.clone() }
+}
+impl ParseCtx for ParsCtx<'_> {
+	fn i(&self) -> &Interner { self.ctx.i() }
+	fn rep(&self) -> &Reporter { self.reporter }
+}
+
+type BoxConstCallback = Box<dyn FnOnce(ConstCtx) -> LocalBoxFuture<'static, GExpr>>;
+
+#[derive(Default)]
+struct ParsedConstCtxEntry {
+	consts: IdStore<BoxConstCallback>,
+}
+impl SysCtxEntry for ParsedConstCtxEntry {}
+
+pub struct ParsedLine {
+	pub sr: SrcRange,
+	pub comments: Vec<Comment>,
+	pub kind: ParsedLineKind,
+}
+impl ParsedLine {
+	pub fn cnst<'a, R: ToExpr + 'static, F: AsyncFnOnce(ConstCtx) -> R + 'static>(
+		sr: &SrcRange,
+		comments: impl IntoIterator<Item = &'a Comment>,
+		exported: bool,
+		name: IStr,
+		f: F,
+	) -> Self {
+		let cb = Box::new(|ctx| async move { f(ctx).await.to_expr().await }.boxed_local());
+		let kind = ParsedLineKind::Mem(ParsedMem { name, exported, kind: ParsedMemKind::Const(cb) });
+		let comments = comments.into_iter().cloned().collect();
+		ParsedLine { comments, sr: sr.clone(), kind }
+	}
+	pub fn module<'a>(
+		sr: &SrcRange,
+		comments: impl IntoIterator<Item = &'a Comment>,
+		exported: bool,
+		name: &IStr,
+		use_prelude: bool,
+		lines: impl IntoIterator<Item = ParsedLine>,
+	) -> Self {
+		let mem_kind = ParsedMemKind::Mod { lines: lines.into_iter().collect(), use_prelude };
+		let line_kind = ParsedLineKind::Mem(ParsedMem { name: name.clone(), exported, kind: mem_kind });
+		let comments = comments.into_iter().cloned().collect();
+		ParsedLine { comments, sr: sr.clone(), kind: line_kind }
+	}
+	pub async fn into_api(self, ctx: SysCtx, hand: &dyn ReqHandlish) -> api::ParsedLine {
+		api::ParsedLine {
+			comments: self.comments.into_iter().map(|c| c.to_api()).collect(),
+			source_range: self.sr.to_api(),
+			kind: match self.kind {
+				ParsedLineKind::Mem(mem) => api::ParsedLineKind::Member(api::ParsedMember {
+					name: mem.name.to_api(),
+					exported: mem.exported,
+					kind: match mem.kind {
+						ParsedMemKind::Const(cb) => api::ParsedMemberKind::Constant(api::ParsedConstId(
+							ctx.get_or_default::<ParsedConstCtxEntry>().consts.add(cb).id(),
+						)),
+						ParsedMemKind::Mod { lines, use_prelude } => api::ParsedMemberKind::Module {
+							lines: linev_into_api(lines, ctx, hand).boxed_local().await,
+							use_prelude,
+						},
+					},
+				}),
+				ParsedLineKind::Rec(tv) =>
+					api::ParsedLineKind::Recursive(ttv_into_api(tv, &mut (), &mut (ctx, hand)).await),
+			},
+		}
+	}
+}
+
+pub(crate) async fn linev_into_api(
+	v: Vec<ParsedLine>,
+	ctx: SysCtx,
+	hand: &dyn ReqHandlish,
+) -> Vec<api::ParsedLine> {
+	join_all(v.into_iter().map(|l| l.into_api(ctx.clone(), hand))).await
+}
+
+pub enum ParsedLineKind {
+	Mem(ParsedMem),
+	Rec(Vec<GenTokTree>),
+}
+
+pub struct ParsedMem {
+	pub name: IStr,
+	pub exported: bool,
+	pub kind: ParsedMemKind,
+}
+
+pub enum ParsedMemKind {
+	Const(BoxConstCallback),
+	Mod { lines: Vec<ParsedLine>, use_prelude: bool },
+}
+
+#[derive(Clone)]
+pub struct ConstCtx {
+	ctx: SysCtx,
+	constid: api::ParsedConstId,
+}
+impl ConstCtx {
+	pub fn ctx(&self) -> &SysCtx { &self.ctx }
+	pub fn i(&self) -> &Interner { self.ctx.i() }
+	pub fn names<'b>(
+		&'b self,
+		names: impl IntoIterator<Item = &'b Sym> + 'b,
+	) -> impl Stream<Item = OrcRes<Sym>> + 'b {
+		let resolve_names = api::ResolveNames {
+			constid: self.constid,
+			sys: self.ctx.sys_id(),
+			names: names.into_iter().map(|n| n.to_api()).collect_vec(),
+		};
+		stream(async |mut cx| {
+			for name_opt in self.ctx.reqnot().request(resolve_names).await {
+				cx.emit(match name_opt {
+					Err(e) => Err(OrcErrv::from_api(&e, self.ctx.i()).await),
+					Ok(name) => Ok(Sym::from_api(name, self.ctx.i()).await),
+				})
+				.await
+			}
+		})
+	}
+	pub async fn names_n<const N: usize>(&self, names: [&Sym; N]) -> [OrcRes<Sym>; N] {
+		self.names(names).collect::<Vec<_>>().await.try_into().expect("Lengths must match")
+	}
+}
+
+pub(crate) async fn get_const(id: api::ParsedConstId, ctx: SysCtx) -> GExpr {
+	let ent = ctx.get::<ParsedConstCtxEntry>();
+	let rec = ent.consts.get(id.0).expect("Bad ID or double read of parsed const");
+	let ctx = ConstCtx { constid: id, ctx: ctx.clone() };
+	rec.remove()(ctx).await
+}

orchid-extension/src/reflection.rs

@@ -0,0 +1,163 @@
+use std::cell::OnceCell;
+use std::rc::Rc;
+
+use futures::FutureExt;
+use futures::lock::Mutex;
+use memo_map::MemoMap;
+use orchid_base::interner::Tok;
+use orchid_base::name::{NameLike, VPath};
+use orchid_base::reqnot::Requester;
+
+use crate::api;
+use crate::system::{SysCtx, SysCtxEntry, WeakSysCtx};
+
+pub struct ReflMemData {
+	// None for inferred steps
+	public: OnceCell<bool>,
+	kind: ReflMemKind,
+}
+#[derive(Clone)]
+pub struct ReflMem(Rc<ReflMemData>);
+impl ReflMem {
+	pub fn kind(&self) -> ReflMemKind { self.0.kind.clone() }
+}
+
+#[derive(Clone)]
+pub enum ReflMemKind {
+	Const,
+	Mod(ReflMod),
+}
+
+pub struct ReflModData {
+	inferred: Mutex<bool>,
+	path: VPath,
+	ctx: WeakSysCtx,
+	members: MemoMap<IStr, ReflMem>,
+}
+
+#[derive(Clone)]
+pub struct ReflMod(Rc<ReflModData>);
+impl ReflMod {
+	fn ctx(&self) -> SysCtx {
+		self.0.ctx.upgrade().expect("ReflectedModule accessed after context drop")
+	}
+	pub fn path(&self) -> &[IStr] { &self.0.path[..] }
+	pub fn is_root(&self) -> bool { self.0.path.is_empty() }
+	async fn try_populate(&self) -> Result<(), api::LsModuleError> {
+		let ctx = self.ctx();
+		let path_tok = ctx.i().i(&self.0.path[..]).await;
+		let reply = match ctx.reqnot().request(api::LsModule(ctx.sys_id(), path_tok.to_api())).await {
+			Err(api::LsModuleError::TreeUnavailable) =>
+				panic!("Reflected tree accessed outside an interpreter call. This extension is faulty."),
+			Err(err) => return Err(err),
+			Ok(details) => details,
+		};
+		for (k, v) in reply.members {
+			let k = ctx.i().ex(k).await;
+			let mem = match self.0.members.get(&k) {
+				Some(mem) => mem,
+				None => {
+					let path = self.0.path.clone().name_with_suffix(k.clone()).to_sym(ctx.i()).await;
+					let kind = match v.kind {
+						api::MemberInfoKind::Constant => ReflMemKind::Const,
+						api::MemberInfoKind::Module =>
+							ReflMemKind::Mod(default_module(&ctx, VPath::new(path.segs()))),
+					};
+					self.0.members.get_or_insert(&k, || default_member(self.is_root(), kind))
+				},
+			};
+			let _ = mem.0.public.set(v.public);
+		}
+		Ok(())
+	}
+	pub async fn get_child(&self, key: &IStr) -> Option<ReflMem> {
+		let inferred_g = self.0.inferred.lock().await;
+		if let Some(mem) = self.0.members.get(key) {
+			return Some(mem.clone());
+		}
+		if !*inferred_g {
+			return None;
+		}
+		match self.try_populate().await {
+			Err(api::LsModuleError::InvalidPath) =>
+				panic!("Path became invalid since module was created"),
+			Err(api::LsModuleError::IsConstant) =>
+				panic!("Path previously contained a module but now contains a constant"),
+			Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
+			Ok(()) => (),
+		}
+		self.0.members.get(key).cloned()
+	}
+	pub async fn get_by_path(&self, path: &[IStr]) -> Result<ReflMem, InvalidPathError> {
+		let ctx = self.ctx();
+		let (next, tail) = path.split_first().expect("Attempted to walk by empty path");
+		let inferred_g = self.0.inferred.lock().await;
+		if let Some(next) = self.0.members.get(next) {
+			return if tail.is_empty() {
+				Ok(next.clone())
+			} else {
+				match next.kind() {
+					ReflMemKind::Const => Err(InvalidPathError { keep_ancestry: true }),
+					ReflMemKind::Mod(m) => m.get_by_path(tail).boxed_local().await,
+				}
+			};
+		}
+		if !*inferred_g {
+			return Err(InvalidPathError { keep_ancestry: true });
+		}
+		let candidate = default_module(&ctx, self.0.path.clone().suffix([next.clone()]));
+		if tail.is_empty() {
+			return match candidate.try_populate().await {
+				Ok(()) => {
+					let tgt_mem = default_member(self.is_root(), ReflMemKind::Mod(candidate));
+					self.0.members.insert(next.clone(), tgt_mem.clone());
+					Ok(tgt_mem)
+				},
+				Err(api::LsModuleError::InvalidPath) => Err(InvalidPathError { keep_ancestry: false }),
+				Err(api::LsModuleError::IsConstant) => {
+					let const_mem = default_member(self.is_root(), ReflMemKind::Const);
+					self.0.members.insert(next.clone(), const_mem);
+					Err(InvalidPathError { keep_ancestry: true })
+				},
+				Err(api::LsModuleError::TreeUnavailable) => unreachable!(),
+			};
+		}
+		match candidate.get_by_path(tail).boxed_local().await {
+			e @ Err(InvalidPathError { keep_ancestry: false }) => e,
+			res @ Err(InvalidPathError { keep_ancestry: true }) | res @ Ok(_) => {
+				let tgt_mem = default_member(self.is_root(), ReflMemKind::Mod(candidate));
+				self.0.members.insert(next.clone(), tgt_mem);
+				res
+			},
+		}
+	}
+}
+
+struct ReflRoot(ReflMod);
+impl SysCtxEntry for ReflRoot {}
+
+pub struct InvalidPathError {
+	keep_ancestry: bool,
+}
+
+fn default_module(ctx: &SysCtx, path: VPath) -> ReflMod {
+	ReflMod(Rc::new(ReflModData {
+		ctx: ctx.downgrade(),
+		inferred: Mutex::new(true),
+		path,
+		members: MemoMap::new(),
+	}))
+}
+
+fn default_member(is_root: bool, kind: ReflMemKind) -> ReflMem {
+	ReflMem(Rc::new(ReflMemData {
+		public: if is_root { true.into() } else { OnceCell::new() },
+		kind,
+	}))
+}
+
+fn get_root(ctx: &SysCtx) -> &ReflRoot {
+	ctx.get_or_insert(|| ReflRoot(default_module(ctx, VPath::new([]))))
+}
+
+pub fn refl(ctx: &SysCtx) -> ReflMod { get_root(ctx).0.clone() }

orchid-extension/src/system.rs

@@ -1,29 +1,29 @@
-use core::fmt;
+use std::any::{Any, TypeId, type_name};
-use std::any::{TypeId, type_name};
+use std::fmt;
 use std::future::Future;
 use std::num::NonZero;
 use std::pin::Pin;
-use std::rc::Rc;
+use std::rc::{Rc, Weak};
 
 use futures::future::LocalBoxFuture;
-use hashbrown::HashMap;
+use memo_map::MemoMap;
 use orchid_api_traits::{Coding, Decode};
 use orchid_base::boxed_iter::BoxedIter;
 use orchid_base::builtin::Spawner;
 use orchid_base::interner::Interner;
 use orchid_base::logging::Logger;
+use orchid_base::name::Sym;
 use orchid_base::reqnot::{Receipt, ReqNot};
 
 use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TypAtom, get_info};
-use crate::atom_owned::ObjStore;
+use crate::coroutine_exec::Replier;
 use crate::entrypoint::ExtReq;
-use crate::fs::DeclFs;
+use crate::func_atom::{Fun, Lambda};
-use crate::func_atom::Fun;
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
 use crate::system_ctor::{CtedObj, SystemCtor};
-use crate::tree::MemKind;
+use crate::tree::GenMember;
 
 /// System as consumed by foreign code
 pub trait SystemCard: Default + Send + Sync + 'static {
@@ -36,14 +36,14 @@ pub trait DynSystemCard: Send + Sync + 'static {
 	fn name(&self) -> &'static str;
 	/// Atoms explicitly defined by the system card. Do not rely on this for
 	/// querying atoms as it doesn't include the general atom types
-	fn atoms(&self) -> BoxedIter<Option<Box<dyn AtomDynfo>>>;
+	fn atoms(&'_ self) -> BoxedIter<'_, Option<Box<dyn AtomDynfo>>>;
 }
 
 /// Atoms supported by this package which may appear in all extensions.
 /// The indices of these are bitwise negated, such that the MSB of an atom index
 /// marks whether it belongs to this package (0) or the importer (1)
 fn general_atoms() -> impl Iterator<Item = Option<Box<dyn AtomDynfo>>> {
-	[Some(Fun::dynfo())].into_iter()
+	[Some(Fun::dynfo()), Some(Lambda::dynfo()), Some(Replier::dynfo())].into_iter()
 }
 
 pub fn atom_info_for(
@@ -77,21 +77,23 @@ pub async fn resolv_atom(
 
 impl<T: SystemCard> DynSystemCard for T {
 	fn name(&self) -> &'static str { T::Ctor::NAME }
-	fn atoms(&self) -> BoxedIter<Option<Box<dyn AtomDynfo>>> { Box::new(Self::atoms().into_iter()) }
+	fn atoms(&'_ self) -> BoxedIter<'_, Option<Box<dyn AtomDynfo>>> {
+		Box::new(Self::atoms().into_iter())
+	}
 }
 
 /// System as defined by author
 pub trait System: Send + Sync + SystemCard + 'static {
-	fn env() -> Vec<(String, MemKind)>;
+	fn prelude(i: &Interner) -> impl Future<Output = Vec<Sym>>;
-	fn vfs() -> DeclFs;
+	fn env() -> Vec<GenMember>;
 	fn lexers() -> Vec<LexerObj>;
 	fn parsers() -> Vec<ParserObj>;
 	fn request(hand: ExtReq<'_>, req: Self::Req) -> impl Future<Output = Receipt<'_>>;
 }
 
 pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
-	fn dyn_env(&self) -> HashMap<String, MemKind>;
+	fn dyn_prelude<'a>(&'a self, i: &'a Interner) -> LocalBoxFuture<'a, Vec<Sym>>;
-	fn dyn_vfs(&self) -> DeclFs;
+	fn dyn_env(&'_ self) -> Vec<GenMember>;
 	fn dyn_lexers(&self) -> Vec<LexerObj>;
 	fn dyn_parsers(&self) -> Vec<ParserObj>;
 	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>>;
@@ -99,8 +101,10 @@ pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
 }
 
 impl<T: System> DynSystem for T {
-	fn dyn_env(&self) -> HashMap<String, MemKind> { Self::env().into_iter().collect() }
+	fn dyn_prelude<'a>(&'a self, i: &'a Interner) -> LocalBoxFuture<'a, Vec<Sym>> {
-	fn dyn_vfs(&self) -> DeclFs { Self::vfs() }
+		Box::pin(Self::prelude(i))
+	}
+	fn dyn_env(&'_ self) -> Vec<GenMember> { Self::env() }
 	fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
 	fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
 	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>> {
@@ -111,16 +115,16 @@ impl<T: System> DynSystem for T {
 	fn card(&self) -> &dyn DynSystemCard { self }
 }
 
-pub async fn downcast_atom<A>(foreign: ForeignAtom<'_>) -> Result<TypAtom<'_, A>, ForeignAtom<'_>>
+pub async fn downcast_atom<A>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom>
 where A: AtomicFeatures {
 	let mut data = &foreign.atom.data[..];
-	let ctx = foreign.ctx.clone();
+	let ctx = foreign.ctx().clone();
 	let value = AtomTypeId::decode(Pin::new(&mut data)).await;
-	let own_inst = ctx.cted.inst();
+	let own_inst = ctx.get::<CtedObj>().inst();
-	let owner = if ctx.id == foreign.atom.owner {
+	let owner = if *ctx.get::<api::SysId>() == foreign.atom.owner {
 		own_inst.card()
 	} else {
-		(ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
+		(ctx.get::<CtedObj>().deps().find(|s| s.id() == foreign.atom.owner))
 			.ok_or_else(|| foreign.clone())?
 			.get_card()
 	};
@@ -130,21 +134,68 @@ where A: AtomicFeatures {
 	}
 	let val = dynfo.decode(AtomCtx(data, foreign.atom.drop, ctx)).await;
 	let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
-	Ok(TypAtom { value, data: foreign })
+	Ok(TypAtom { value, untyped: foreign })
 }
 
 #[derive(Clone)]
-pub struct SysCtx {
+pub struct WeakSysCtx(Weak<MemoMap<TypeId, Box<dyn Any>>>);
-	pub reqnot: ReqNot<api::ExtMsgSet>,
+impl WeakSysCtx {
-	pub spawner: Spawner,
+	pub fn upgrade(&self) -> Option<SysCtx> { Some(SysCtx(self.0.upgrade()?)) }
-	pub id: api::SysId,
+}
-	pub cted: CtedObj,
+
-	pub logger: Logger,
+#[derive(Clone)]
-	pub obj_store: ObjStore,
+pub struct SysCtx(Rc<MemoMap<TypeId, Box<dyn Any>>>);
-	pub i: Rc<Interner>,
+impl SysCtx {
+	pub fn new(
+		id: api::SysId,
+		i: Interner,
+		reqnot: ReqNot<api::ExtMsgSet>,
+		spawner: Spawner,
+		logger: Logger,
+		cted: CtedObj,
+	) -> Self {
+		let this = Self(Rc::new(MemoMap::new()));
+		this.add(id).add(i).add(reqnot).add(spawner).add(logger).add(cted);
+		this
+	}
+	pub fn downgrade(&self) -> WeakSysCtx { WeakSysCtx(Rc::downgrade(&self.0)) }
+	pub fn add<T: SysCtxEntry>(&self, t: T) -> &Self {
+		assert!(self.0.insert(TypeId::of::<T>(), Box::new(t)), "Key already exists");
+		self
+	}
+	pub fn get_or_insert<T: SysCtxEntry>(&self, f: impl FnOnce() -> T) -> &T {
+		(self.0.get_or_insert_owned(TypeId::of::<T>(), || Box::new(f())).downcast_ref())
+			.expect("Keyed by TypeId")
+	}
+	pub fn get_or_default<T: SysCtxEntry + Default>(&self) -> &T { self.get_or_insert(T::default) }
+	pub fn try_get<T: SysCtxEntry>(&self) -> Option<&T> {
+		Some(self.0.get(&TypeId::of::<T>())?.downcast_ref().expect("Keyed by TypeId"))
+	}
+	pub fn get<T: SysCtxEntry>(&self) -> &T {
+		self.try_get().unwrap_or_else(|| panic!("Context {} missing", type_name::<T>()))
+	}
+	/// Shorthand to get the [Interner] instance
+	pub fn i(&self) -> &Interner { self.get::<Interner>() }
+	/// Shorthand to get the messaging link
+	pub fn reqnot(&self) -> &ReqNot<api::ExtMsgSet> { self.get::<ReqNot<api::ExtMsgSet>>() }
+	/// Shorthand to get the system ID
+	pub fn sys_id(&self) -> api::SysId { *self.get::<api::SysId>() }
+	/// Shorthand to get the task spawner callback
+	pub fn spawner(&self) -> &Spawner { self.get::<Spawner>() }
+	/// Shorthand to get the logger
+	pub fn logger(&self) -> &Logger { self.get::<Logger>() }
+	/// Shorthand to get the constructed system object
+	pub fn cted(&self) -> &CtedObj { self.get::<CtedObj>() }
 }
 impl fmt::Debug for SysCtx {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "SysCtx({:?})", self.id)
+		write!(f, "SysCtx({:?})", self.sys_id())
 	}
 }
+pub trait SysCtxEntry: 'static + Sized {}
+impl SysCtxEntry for api::SysId {}
+impl SysCtxEntry for ReqNot<api::ExtMsgSet> {}
+impl SysCtxEntry for Spawner {}
+impl SysCtxEntry for CtedObj {}
+impl SysCtxEntry for Logger {}
+impl SysCtxEntry for Interner {}

orchid-extension/src/system_ctor.rs

@@ -62,7 +62,7 @@ pub trait SystemCtor: Send + Sync + 'static {
 	type Instance: System;
 	const NAME: &'static str;
 	const VERSION: f64;
-	fn inst() -> Option<Self::Instance>;
+	fn inst(deps: <Self::Deps as DepDef>::Sat) -> Self::Instance;
 }
 
 pub trait DynSystemCtor: Send + Sync + 'static {
@@ -81,8 +81,8 @@ impl<T: SystemCtor> DynSystemCtor for T {
 	}
 	fn new_system(&self, api::NewSystem { system: _, id: _, depends }: &api::NewSystem) -> CtedObj {
 		let mut ids = depends.iter().copied();
-		let inst = Arc::new(T::inst().expect("Constructor did not create system"));
 		let deps = T::Deps::create(&mut || ids.next().unwrap());
+		let inst = Arc::new(T::inst(deps.clone()));
 		Arc::new(Cted::<T> { deps, inst })
 	}
 }
@@ -90,7 +90,7 @@ impl<T: SystemCtor> DynSystemCtor for T {
 mod dep_set_tuple_impls {
 	use orchid_base::box_chain;
 	use orchid_base::boxed_iter::BoxedIter;
-	use paste::paste;
+	use pastey::paste;
 
 	use super::{DepDef, DepSat};
 	use crate::api;

orchid-extension/src/tokio.rs

@@ -0,0 +1,57 @@
+use crate::entrypoint::ExtensionData;
+
+#[cfg(feature = "tokio")]
+pub async fn tokio_main(data: ExtensionData) {
+	use std::io::{ErrorKind, Write};
+	use std::mem;
+	use std::pin::{Pin, pin};
+	use std::rc::Rc;
+
+	use async_once_cell::OnceCell;
+	use futures::StreamExt;
+	use futures::future::LocalBoxFuture;
+	use futures::lock::Mutex;
+	use futures::stream::FuturesUnordered;
+	use orchid_api_traits::{Decode, Encode};
+	use orchid_base::msg::{recv_msg, send_msg};
+	use tokio::io::{Stdout, stdin, stdout};
+	use tokio::task::{LocalSet, spawn_local};
+	use tokio_util::compat::{Compat, TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
+
+	use crate::api;
+	use crate::entrypoint::extension_init;
+
+	let local_set = LocalSet::new();
+	local_set.spawn_local(async {
+		let host_header = api::HostHeader::decode(Pin::new(&mut stdin().compat())).await;
+		let init =
+			Rc::new(extension_init(data, host_header, Rc::new(|fut| mem::drop(spawn_local(fut)))));
+		let mut buf = Vec::new();
+		init.header.encode(Pin::new(&mut buf)).await;
+		std::io::stdout().write_all(&buf).unwrap();
+		std::io::stdout().flush().unwrap();
+		// These are concurrent processes that never exit, so if the FuturesUnordered
+		// produces any result the extension should exit
+		let mut io = FuturesUnordered::<LocalBoxFuture<()>>::new();
+		io.push(Box::pin(async {
+			loop {
+				match recv_msg(pin!(stdin().compat())).await {
+					Ok(msg) => init.send(&msg[..]).await,
+					Err(e) if e.kind() == ErrorKind::BrokenPipe => break,
+					Err(e) if e.kind() == ErrorKind::UnexpectedEof => break,
+					Err(e) => panic!("{e}"),
+				}
+			}
+		}));
+		io.push(Box::pin(async {
+			while let Some(msg) = init.recv().await {
+				static STDOUT: OnceCell<Mutex<Compat<Stdout>>> = OnceCell::new();
+				let stdout_lk = STDOUT.get_or_init(async { Mutex::new(stdout().compat_write()) }).await;
+				let mut stdout_g = stdout_lk.lock().await;
+				send_msg(pin!(&mut *stdout_g), &msg[..]).await.expect("Parent pipe broken");
+			}
+		}));
+		io.next().await;
+	});
+	local_set.await;
+}

orchid-extension/src/tree.rs

@@ -1,125 +1,109 @@
-use std::future::Future;
 use std::num::NonZero;
-use std::ops::Range;
-use std::rc::Rc;
 
+use async_fn_stream::stream;
 use dyn_clone::{DynClone, clone_box};
-use futures::FutureExt;
 use futures::future::{LocalBoxFuture, join_all};
+use futures::{FutureExt, StreamExt};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::reqnot::ReqHandlish;
-use orchid_base::tree::{TokTree, Token};
+use orchid_base::tree::{TokTree, Token, TokenVariant};
-use ordered_float::NotNan;
 use substack::Substack;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::atom::{AtomFactory, ForeignAtom};
 use crate::conv::ToExpr;
 use crate::entrypoint::MemberRecord;
+use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
 use crate::func_atom::{ExprFunc, Fun};
-use crate::gen_expr::{GExpr, arg, call, lambda, seq};
+use crate::gen_expr::{GExpr, sym_ref};
-use crate::macros::Rule;
 use crate::system::SysCtx;
 
-pub type GenTokTree<'a> = TokTree<'a, ForeignAtom<'a>, AtomFactory>;
+pub type GenTokTree = TokTree<Expr, GExpr>;
-pub type GenTok<'a> = Token<'a, ForeignAtom<'a>, AtomFactory>;
+pub type GenTok = Token<Expr, GExpr>;
 
-pub async fn do_extra(f: &AtomFactory, r: Range<u32>, ctx: SysCtx) -> api::TokenTree {
+impl TokenVariant<api::Expression> for GExpr {
-	api::TokenTree { range: r, token: api::Token::Atom(f.clone().build(ctx).await) }
+	type FromApiCtx<'a> = ();
-}
+	type ToApiCtx<'a> = (SysCtx, &'a dyn ReqHandlish);
-
+	async fn from_api(
-fn with_export(mem: GenMember, public: bool) -> Vec<GenItem> {
+		_: &api::Expression,
-	(public.then(|| GenItemKind::Export(mem.name.clone())).into_iter())
+		_: &mut Self::FromApiCtx<'_>,
-		.chain([GenItemKind::Member(mem)])
+		_: SrcRange,
-		.map(|kind| GenItem { comments: vec![], kind })
+		_: &Interner,
-		.collect()
+	) -> Self {
-}
+		panic!("Received new expression from host")
-
+	}
-pub struct GenItem {
+	async fn into_api(self, (ctx, hand): &mut Self::ToApiCtx<'_>) -> api::Expression {
-	pub kind: GenItemKind,
+		self.api_return(ctx.clone(), hand).await
-	pub comments: Vec<String>,
-}
-impl GenItem {
-	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Item {
-		let kind = match self.kind {
-			GenItemKind::Export(n) => api::ItemKind::Export(ctx.sys().i.i::<String>(&n).await.to_api()),
-			GenItemKind::Member(mem) => api::ItemKind::Member(mem.into_api(ctx).await),
-			GenItemKind::Import(cn) => api::ItemKind::Import(cn.tok().to_api()),
-			GenItemKind::Macro(priority, gen_rules) => {
-				let mut rules = Vec::with_capacity(gen_rules.len());
-				for rule in gen_rules {
-					rules.push(rule.into_api(ctx).await)
-				}
-				api::ItemKind::Macro(api::MacroBlock { priority, rules })
-			},
-		};
-		let comments = join_all(self.comments.iter().map(|c| async {
-			api::Comment {
-				location: api::Location::Inherit,
-				text: ctx.sys().i.i::<String>(c).await.to_api(),
-			}
-		}))
-		.await;
-		api::Item { location: api::Location::Inherit, comments, kind }
 	}
 }
 
-pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenItem> {
+impl TokenVariant<api::ExprTicket> for Expr {
-	with_export(GenMember { name: name.to_string(), kind: MemKind::Const(value.to_expr()) }, public)
+	type FromApiCtx<'a> = (SysCtx, &'a BorrowedExprStore);
+	async fn from_api(
+		api: &api::ExprTicket,
+		(ctx, exprs): &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
+		// SAFETY: receiving trees from sublexers implies borrowing
+		Expr::from_handle(ExprHandle::borrowed(ctx.clone(), *api, exprs))
+	}
+	type ToApiCtx<'a> = ();
+	async fn into_api(self, (): &mut Self::ToApiCtx<'_>) -> api::ExprTicket { self.handle().tk }
+}
+
+pub async fn x_tok(x: impl ToExpr) -> GenTok { GenTok::NewExpr(x.to_expr().await) }
+pub async fn ref_tok(path: Sym) -> GenTok { GenTok::NewExpr(sym_ref(path)) }
+
+pub fn lazy(
+	public: bool,
+	name: &str,
+	cb: impl AsyncFnOnce(Sym, SysCtx) -> MemKind + Clone + 'static,
+) -> Vec<GenMember> {
+	vec![GenMember {
+		name: name.to_string(),
+		kind: MemKind::Lazy(LazyMemberFactory::new(cb)),
+		comments: vec![],
+		public,
+	}]
+}
+pub fn cnst(public: bool, name: &str, value: impl ToExpr + Clone + 'static) -> Vec<GenMember> {
+	lazy(public, name, async |_, _| MemKind::Const(value.to_expr().await))
 }
 pub fn module(
 	public: bool,
 	name: &str,
-	imports: impl IntoIterator<Item = Sym>,
+	mems: impl IntoIterator<Item = Vec<GenMember>>,
-	items: impl IntoIterator<Item = Vec<GenItem>>,
+) -> Vec<GenMember> {
-) -> Vec<GenItem> {
+	let (name, kind) = root_mod(name, mems);
-	let (name, kind) = root_mod(name, imports, items);
+	vec![GenMember { name, kind, public, comments: vec![] }]
-	with_export(GenMember { name, kind }, public)
 }
-pub fn root_mod(
+pub fn root_mod(name: &str, mems: impl IntoIterator<Item = Vec<GenMember>>) -> (String, MemKind) {
-	name: &str,
+	let kind = MemKind::Mod { members: mems.into_iter().flatten().collect() };
-	imports: impl IntoIterator<Item = Sym>,
-	items: impl IntoIterator<Item = Vec<GenItem>>,
-) -> (String, MemKind) {
-	let kind = MemKind::Mod {
-		imports: imports.into_iter().collect(),
-		items: items.into_iter().flatten().collect(),
-	};
 	(name.to_string(), kind)
 }
-pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenItem> {
+pub fn fun<I, O>(public: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenMember> {
-	let fac = LazyMemberFactory::new(move |sym| async {
+	let fac = LazyMemberFactory::new(async move |sym, ctx| {
-		return MemKind::Const(build_lambdas(Fun::new(sym, xf).await, 0));
+		MemKind::Const(Fun::new(sym, ctx, xf).await.to_expr().await)
-		fn build_lambdas(fun: Fun, i: u64) -> GExpr {
-			if i < fun.arity().into() {
-				return lambda(i, [build_lambdas(fun, i + 1)]);
-			}
-			let arity = fun.arity();
-			seq(
-				(0..arity)
-					.map(|i| arg(i as u64))
-					.chain([call([fun.to_expr()].into_iter().chain((0..arity).map(|i| arg(i as u64))))]),
-			)
-		}
 	});
-	with_export(GenMember { name: name.to_string(), kind: MemKind::Lazy(fac) }, exported)
+	vec![GenMember { name: name.to_string(), kind: MemKind::Lazy(fac), public, comments: vec![] }]
 }
-pub fn macro_block(prio: Option<f64>, rules: impl IntoIterator<Item = Rule>) -> Vec<GenItem> {
+pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
-	let prio = prio.map(|p| NotNan::new(p).unwrap());
+	let mut items = items.into_iter().flatten().collect_vec();
-	vec![GenItem {
+	for step in path.split("::").collect_vec().into_iter().rev() {
-		kind: GenItemKind::Macro(prio, rules.into_iter().collect_vec()),
+		items = module(true, step, [items]);
-		comments: vec![],
+	}
-	}]
+	items
 }
 
 pub fn comments<'a>(
 	cmts: impl IntoIterator<Item = &'a str>,
-	mut val: Vec<GenItem>,
+	mut val: Vec<GenMember>,
-) -> Vec<GenItem> {
+) -> Vec<GenMember> {
 	let cmts = cmts.into_iter().map(|c| c.to_string()).collect_vec();
 	for v in val.iter_mut() {
 		v.comments.extend(cmts.iter().cloned());
@@ -127,47 +111,77 @@ pub fn comments<'a>(
 	val
 }
 
+/// Trivially merge a gen tree. Behaviours were chosen to make this simple.
+///
+/// - Comments on imports are discarded
+/// - Comments on exports and submodules are combined
+/// - Duplicate constants result in an error
+/// - A combination of lazy and anything results in an error
+pub fn merge_trivial(trees: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
+	let mut all_members = HashMap::<String, (MemKind, Vec<String>)>::new();
+	for mem in trees.into_iter().flatten() {
+		assert!(mem.public, "Non-trivial merge in {}", mem.name);
+		match mem.kind {
+			unit @ (MemKind::Const(_) | MemKind::Lazy(_)) => {
+				let prev = all_members.insert(mem.name.clone(), (unit, mem.comments.into_iter().collect()));
+				assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name);
+			},
+			MemKind::Mod { members } => match all_members.entry(mem.name.clone()) {
+				hashbrown::hash_map::Entry::Vacant(slot) => {
+					slot.insert((MemKind::Mod { members }, mem.comments.into_iter().collect()));
+				},
+				hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() {
+					(MemKind::Mod { members: old_items, .. }, old_cmts) => {
+						let mut swap = vec![];
+						std::mem::swap(&mut swap, old_items);
+						*old_items = merge_trivial([swap, members]);
+						old_cmts.extend(mem.comments);
+					},
+					_ => panic!("non-trivial merge on {}", mem.name),
+				},
+			},
+		}
+	}
+
+	(all_members.into_iter())
+		.map(|(name, (kind, comments))| GenMember { comments, kind, name, public: true })
+		.collect_vec()
+}
+
 trait_set! {
 	trait LazyMemberCallback =
-		FnOnce(Sym) -> LocalBoxFuture<'static, MemKind> + DynClone
+		FnOnce(Sym, SysCtx) -> LocalBoxFuture<'static, MemKind> + DynClone
 }
 pub struct LazyMemberFactory(Box<dyn LazyMemberCallback>);
 impl LazyMemberFactory {
-	pub fn new<F: Future<Output = MemKind> + 'static>(
+	pub fn new(cb: impl AsyncFnOnce(Sym, SysCtx) -> MemKind + Clone + 'static) -> Self {
-		cb: impl FnOnce(Sym) -> F + Clone + 'static,
+		Self(Box::new(|s, ctx| cb(s, ctx).boxed_local()))
-	) -> Self {
-		Self(Box::new(|s| cb(s).boxed_local()))
 	}
-	pub async fn build(self, path: Sym) -> MemKind { (self.0)(path).await }
+	pub async fn build(self, path: Sym, ctx: SysCtx) -> MemKind { (self.0)(path, ctx).await }
 }
 impl Clone for LazyMemberFactory {
 	fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
 }
 
-pub enum GenItemKind {
-	Member(GenMember),
-	Export(String),
-	Import(Sym),
-	Macro(Option<NotNan<f64>>, Vec<Rule>),
-}
-
 pub struct GenMember {
-	name: String,
+	pub name: String,
-	kind: MemKind,
+	pub kind: MemKind,
+	pub public: bool,
+	pub comments: Vec<String>,
 }
 impl GenMember {
 	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Member {
-		let name = ctx.sys().i.i::<String>(&self.name).await;
+		let name = ctx.sys().i().i::<String>(&self.name).await;
-		api::Member {
+		let kind = self.kind.into_api(&mut ctx.push_path(name.clone())).await;
-			kind: self.kind.into_api(&mut ctx.push_path(name.clone())).await,
+		let comments =
-			name: name.to_api(),
+			join_all(self.comments.iter().map(|cmt| async { ctx.sys().i().i(cmt).await.to_api() })).await;
-		}
+		api::Member { kind, name: name.to_api(), comments, exported: self.public }
 	}
 }
 
 pub enum MemKind {
 	Const(GExpr),
-	Mod { imports: Vec<Sym>, items: Vec<GenItem> },
+	Mod { members: Vec<GenMember> },
 	Lazy(LazyMemberFactory),
 }
 impl MemKind {
@@ -175,16 +189,16 @@ impl MemKind {
 		match self {
 			Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
 			Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys(), ctx.req()).await),
-			Self::Mod { imports, items } => {
+			Self::Mod { members } => api::MemberKind::Module(api::Module {
-				let all_items = (imports.into_iter())
+				members: stream(async |mut cx| {
-					.map(|t| GenItem { comments: vec![], kind: GenItemKind::Import(t) })
+					for m in members {
-					.chain(items);
+						cx.emit(m.into_api(ctx).await).await
-				let mut items = Vec::new();
+					}
-				for i in all_items {
+				})
-					items.push(i.into_api(ctx).boxed_local().await)
+				.collect()
-				}
+				.boxed_local()
-				api::MemberKind::Module(api::Module { items })
+				.await,
-			},
+			}),
 		}
 	}
 }
@@ -192,27 +206,24 @@ impl MemKind {
 pub trait TreeIntoApiCtx {
 	fn sys(&self) -> SysCtx;
 	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
-	fn with_rule(&mut self, rule: Rc<Rule>) -> api::MacroId;
+	fn push_path(&mut self, seg: IStr) -> impl TreeIntoApiCtx;
-	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
 	fn req(&self) -> &impl ReqHandlish;
 }
 
-pub struct TIACtxImpl<'a, 'b, RH: ReqHandlish> {
+pub struct TreeIntoApiCtxImpl<'a, 'b, RH: ReqHandlish> {
 	pub sys: SysCtx,
-	pub basepath: &'a [Tok<String>],
+	pub basepath: &'a [IStr],
-	pub path: Substack<'a, Tok<String>>,
+	pub path: Substack<'a, IStr>,
 	pub lazy_members: &'b mut HashMap<api::TreeId, MemberRecord>,
-	pub rules: &'b mut HashMap<api::MacroId, Rc<Rule>>,
 	pub req: &'a RH,
 }
 
-impl<RH: ReqHandlish> TreeIntoApiCtx for TIACtxImpl<'_, '_, RH> {
+impl<RH: ReqHandlish> TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_, RH> {
 	fn sys(&self) -> SysCtx { self.sys.clone() }
-	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx {
+	fn push_path(&mut self, seg: IStr) -> impl TreeIntoApiCtx {
-		TIACtxImpl {
+		TreeIntoApiCtxImpl {
 			req: self.req,
 			lazy_members: self.lazy_members,
-			rules: self.rules,
 			sys: self.sys.clone(),
 			basepath: self.basepath,
 			path: self.path.push(seg),
@@ -224,10 +235,5 @@ impl<RH: ReqHandlish> TreeIntoApiCtx for TIACtxImpl<'_, '_, RH> {
 		self.lazy_members.insert(id, MemberRecord::Gen(path, fac));
 		id
 	}
-	fn with_rule(&mut self, rule: Rc<Rule>) -> orchid_api::MacroId {
-		let id = api::MacroId(NonZero::new((self.lazy_members.len() + 1) as u64).unwrap());
-		self.rules.insert(id, rule);
-		id
-	}
 	fn req(&self) -> &impl ReqHandlish { self.req }
 }

orchid-host/Cargo.toml

@@ -1,28 +1,29 @@
 [package]
 name = "orchid-host"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
+async-lock = "3.4.1"
 async-once-cell = "0.5.4"
-async-process = "2.3.0"
+async-process = "2.4.0"
-async-std = "1.13.0"
-async-stream = "0.3.6"
 bound = "0.6.0"
 derive_destructure = "1.0.0"
-futures = "0.3.31"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.15.2"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
+memo-map = "0.3.3"
 never = "0.1.0"
 num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-ordered-float = "4.6.0"
+ordered-float = "5.0.0"
-paste = "1.0.15"
+pastey = "0.1.1"
 substack = "1.1.1"
-test_executors = "0.3.2"
+test_executors = "0.3.5"
 trait-set = "0.3.0"

orchid-host/src/atom.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 use std::rc::{Rc, Weak};
 
+use async_lock::OnceCell;
 use derive_destructure::destructure;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, take_first_fmt};
 use orchid_base::location::Pos;
@@ -9,7 +10,7 @@ use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::ctx::Ctx;
-use crate::expr::Expr;
+use crate::expr::{Expr, ExprParseCtx, PathSetBuilder};
 use crate::extension::Extension;
 use crate::system::System;
 
@@ -18,12 +19,15 @@ pub struct AtomData {
 	owner: System,
 	drop: Option<api::AtomId>,
 	data: Vec<u8>,
+	pub(crate) display: OnceCell<FmtUnit>,
 }
 impl AtomData {
+	#[must_use]
 	fn api(self) -> api::Atom {
-		let (owner, drop, data) = self.destructure();
+		let (owner, drop, data, _display) = self.destructure();
 		api::Atom { data, drop, owner: owner.id() }
 	}
+	#[must_use]
 	fn api_ref(&self) -> api::Atom {
 		api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
 	}
@@ -48,57 +52,58 @@ impl fmt::Debug for AtomData {
 #[derive(Clone, Debug)]
 pub struct AtomHand(Rc<AtomData>);
 impl AtomHand {
-	pub(crate) async fn new(api::Atom { data, drop, owner }: api::Atom, ctx: &Ctx) -> Self {
+	pub(crate) fn new(data: Vec<u8>, owner: System, drop: Option<api::AtomId>) -> Self {
-		let create = || async {
+		Self(Rc::new(AtomData { owner, drop, data, display: OnceCell::new() }))
-			let owner = ctx.system_inst(owner).await.expect("Dropped system created atom");
-			AtomHand(Rc::new(AtomData { data, owner, drop }))
-		};
-		if let Some(id) = drop {
-			let mut owned_g = ctx.owned_atoms.write().await;
-			if let Some(data) = owned_g.get(&id) {
-				if let Some(atom) = data.upgrade() {
-					return atom;
-				}
-			}
-			let new = create().await;
-			owned_g.insert(id, new.downgrade());
-			new
-		} else {
-			create().await
-		}
 	}
-	pub async fn call(self, arg: Expr) -> api::Expression {
+	#[must_use]
+	pub async fn call(self, arg: Expr) -> Expr {
 		let owner_sys = self.0.owner.clone();
 		let reqnot = owner_sys.reqnot();
 		owner_sys.ext().exprs().give_expr(arg.clone());
-		match Rc::try_unwrap(self.0) {
+		let ret = match Rc::try_unwrap(self.0) {
 			Ok(data) => reqnot.request(api::FinalCall(data.api(), arg.id())).await,
 			Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await,
-		}
+		};
+		let mut parse_ctx = ExprParseCtx { ctx: owner_sys.ctx(), exprs: owner_sys.ext().exprs() };
+		let val = Expr::from_api(&ret, PathSetBuilder::new(), &mut parse_ctx).await;
+		owner_sys.ext().exprs().take_expr(arg.id());
+		val
 	}
+	#[must_use]
 	pub fn sys(&self) -> &System { &self.0.owner }
+	#[must_use]
 	pub fn ext(&self) -> &Extension { self.sys().ext() }
-	pub async fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
+	pub async fn req(&self, key: api::TVec, req: Vec<u8>) -> Option<Vec<u8>> {
 		self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await
 	}
+	#[must_use]
 	pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
+	#[must_use]
 	pub async fn to_string(&self) -> String { take_first_fmt(self, &self.0.owner.ctx().i).await }
+	#[must_use]
 	pub fn downgrade(&self) -> WeakAtomHand { WeakAtomHand(Rc::downgrade(&self.0)) }
 }
 impl Format for AtomHand {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		FmtUnit::from_api(&self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await)
+		(self.0.display.get_or_init(|| async {
+			FmtUnit::from_api(&self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await)
+		}))
+		.await
+		.clone()
 	}
 }
 impl AtomRepr for AtomHand {
 	type Ctx = Ctx;
-	async fn from_api(atom: &orchid_api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
+	async fn from_api(atom: &api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
-		Self::new(atom.clone(), ctx).await
+		let api::Atom { data, drop, owner } = atom.clone();
+		let sys = ctx.system_inst(owner).await.expect("Dropped system created atom");
+		if let Some(id) = drop { sys.new_atom(data, id).await } else { AtomHand::new(data, sys, drop) }
 	}
 	async fn to_api(&self) -> orchid_api::Atom { self.api_ref() }
 }
 
 pub struct WeakAtomHand(Weak<AtomData>);
 impl WeakAtomHand {
+	#[must_use]
 	pub fn upgrade(&self) -> Option<AtomHand> { self.0.upgrade().map(AtomHand) }
 }

orchid-host/src/ctx.rs

@@ -1,26 +1,25 @@
 use std::cell::RefCell;
 use std::num::{NonZero, NonZeroU16};
-use std::rc::Rc;
+use std::rc::{Rc, Weak};
 use std::{fmt, ops};
 
-use async_std::sync::RwLock;
+use async_lock::RwLock;
 use hashbrown::HashMap;
-use orchid_api::SysId;
 use orchid_base::builtin::Spawner;
 use orchid_base::interner::Interner;
 
 use crate::api;
-use crate::atom::WeakAtomHand;
+use crate::expr_store::ExprStore;
 use crate::system::{System, WeakSystem};
-use crate::tree::Module;
+use crate::tree::WeakRoot;
 
 pub struct CtxData {
-	pub i: Rc<Interner>,
+	pub i: Interner,
 	pub spawn: Spawner,
 	pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
 	pub system_id: RefCell<NonZeroU16>,
-	pub owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
+	pub common_exprs: ExprStore,
-	pub root: RwLock<Module>,
+	pub root: RwLock<WeakRoot>,
 }
 #[derive(Clone)]
 pub struct Ctx(Rc<CtxData>);
@@ -28,25 +27,38 @@ impl ops::Deref for Ctx {
 	type Target = CtxData;
 	fn deref(&self) -> &Self::Target { &self.0 }
 }
+#[derive(Clone)]
+pub struct WeakCtx(Weak<CtxData>);
+impl WeakCtx {
+	#[must_use]
+	pub fn try_upgrade(&self) -> Option<Ctx> { Some(Ctx(self.0.upgrade()?)) }
+	#[must_use]
+	pub fn upgrade(&self) -> Ctx { self.try_upgrade().expect("Ctx manually kept alive until exit") }
+}
 impl Ctx {
+	#[must_use]
 	pub fn new(spawn: Spawner) -> Self {
 		Self(Rc::new(CtxData {
 			spawn,
-			i: Rc::default(),
+			i: Interner::default(),
 			systems: RwLock::default(),
 			system_id: RefCell::new(NonZero::new(1).unwrap()),
-			owned_atoms: RwLock::default(),
+			common_exprs: ExprStore::default(),
-			root: RwLock::new(Module::default()),
+			root: RwLock::default(),
 		}))
 	}
+	#[must_use]
 	pub(crate) async fn system_inst(&self, id: api::SysId) -> Option<System> {
 		self.systems.read().await.get(&id).and_then(WeakSystem::upgrade)
 	}
+	#[must_use]
 	pub(crate) fn next_sys_id(&self) -> api::SysId {
 		let mut g = self.system_id.borrow_mut();
 		*g = g.checked_add(1).unwrap_or(NonZeroU16::new(1).unwrap());
-		SysId(*g)
+		api::SysId(*g)
 	}
+	#[must_use]
+	pub fn downgrade(&self) -> WeakCtx { WeakCtx(Rc::downgrade(&self.0)) }
 }
 impl fmt::Debug for Ctx {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

orchid-host/src/dealias.rs

@@ -1,16 +1,9 @@
-use std::rc::Rc;
+use hashbrown::HashSet;
-
+use itertools::Itertools;
-use futures::FutureExt;
+use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_err};
-use hashbrown::{HashMap, HashSet};
-use itertools::{Either, Itertools};
-use orchid_base::error::{OrcErr, Reporter, mk_err};
-use orchid_base::format::{FmtCtxImpl, Format, take_first};
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::Pos;
-use orchid_base::name::{NameLike, Sym, VName};
+use orchid_base::name::VName;
-
-use crate::macros::{MacTok, MacTree};
-use crate::tree::{ItemKind, MemberKind, Module, RuleKind, WalkErrorKind};
 
 /// Errors produced by absolute_path
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
@@ -23,7 +16,7 @@ pub enum AbsPathError {
 	RootPath,
 }
 impl AbsPathError {
-	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErr {
+	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErrv {
 		let (descr, msg) = match self {
 			AbsPathError::RootPath => (
 				i.i("Path ends on root module").await,
@@ -37,7 +30,7 @@ impl AbsPathError {
 				format!("{path} is leading outside the root."),
 			),
 		};
-		mk_err(descr, msg, [pos.into()])
+		mk_err(descr, msg, [pos])
 	}
 }
 
@@ -48,193 +41,107 @@ impl AbsPathError {
 ///
 /// if the relative path contains as many or more `super` segments than the
 /// length of the absolute path.
-pub fn absolute_path(
+pub async fn absolute_path(
-	mut cwd: &[Tok<String>],
+	mut cwd: &[IStr],
-	mut rel: &[Tok<String>],
+	mut rel: &[IStr],
+	i: &Interner,
 ) -> Result<VName, AbsPathError> {
-	let mut relative = false;
+	let i_self = i.i("self").await;
-	if rel.first().map(|t| t.as_str()) == Some("self") {
+	let i_super = i.i("super").await;
-		relative = true;
+	let relative = rel.first().is_some_and(|s| *s != i_self && *s != i_super);
-		rel = rel.split_first().expect("checked above").1;
+	if let Some((_, tail)) = rel.split_first().filter(|(h, _)| **h != i_self) {
+		rel = tail;
 	} else {
-		while rel.first().map(|t| t.as_str()) == Some("super") {
+		while let Some((_, tail)) = rel.split_first().filter(|(h, _)| **h == i_super) {
-			match cwd.split_last() {
+			cwd = cwd.split_last().ok_or(AbsPathError::TooManySupers)?.1;
-				Some((_, torso)) => cwd = torso,
+			rel = tail;
-				None => return Err(AbsPathError::TooManySupers),
-			};
-			rel = rel.split_first().expect("checked above").1;
-			relative = true;
 		}
 	}
-	match relative {
+	if relative { VName::new(cwd.iter().chain(rel).cloned()) } else { VName::new(rel.to_vec()) }
-		true => VName::new(cwd.iter().chain(rel).cloned()),
+		.map_err(|_| AbsPathError::RootPath)
-		false => VName::new(rel.to_vec()),
-	}
-	.map_err(|_| AbsPathError::RootPath)
 }
 
-pub async fn resolv_glob(
+pub struct DealiasCtx<'a> {
-	cwd: &[Tok<String>],
+	pub i: &'a Interner,
-	root: &Module,
+	pub rep: &'a Reporter,
-	abs_path: &[Tok<String>],
+}
+
+pub async fn resolv_glob<Mod: Tree>(
+	cwd: &[IStr],
+	root: &Mod,
+	abs_path: &[IStr],
 	pos: Pos,
 	i: &Interner,
-	r: &impl Reporter,
+	ctx: &mut Mod::Ctx<'_>,
-) -> Vec<Tok<String>> {
+) -> OrcRes<HashSet<IStr>> {
 	let coprefix_len = cwd.iter().zip(abs_path).take_while(|(a, b)| a == b).count();
-	let (co_prefix, diff_path) = abs_path.split_at(coprefix_len);
+	let (co_prefix, diff_path) = abs_path.split_at(abs_path.len().min(coprefix_len + 1));
-	let co_parent = root.walk(false, co_prefix.iter().cloned()).await.expect("Invalid step in cwd");
+	let fst_diff =
-	let target_module = match co_parent.walk(true, diff_path.iter().cloned()).await {
+		walk(root, false, co_prefix.iter().cloned(), ctx).await.expect("Invalid step in cwd");
+	let target_module = match walk(fst_diff, true, diff_path.iter().cloned(), ctx).await {
 		Ok(t) => t,
 		Err(e) => {
 			let path = abs_path[..=coprefix_len + e.pos].iter().join("::");
 			let (tk, msg) = match e.kind {
-				WalkErrorKind::Constant =>
+				ChildErrorKind::Constant => ("Invalid import path", format!("{path} is a const")),
-					(i.i("Invalid import path").await, format!("{path} is a constant")),
+				ChildErrorKind::Missing => ("Invalid import path", format!("{path} not found")),
-				WalkErrorKind::Missing => (i.i("Invalid import path").await, format!("{path} not found")),
+				ChildErrorKind::Private => ("Import inaccessible", format!("{path} is private")),
-				WalkErrorKind::Private => (i.i("Import inaccessible").await, format!("{path} is private")),
 			};
-			r.report(mk_err(tk, msg, [pos.into()]));
+			return Err(mk_err(i.i(tk).await, msg, [pos]));
-			return vec![];
 		},
 	};
-	target_module.exports.clone()
+	Ok(target_module.children(coprefix_len < abs_path.len()))
 }
 
-/// Read import statements and convert them into aliases, rasising any import
+pub type ChildResult<'a, T> = Result<&'a T, ChildErrorKind>;
-/// errors in the process
+
-pub async fn imports_to_aliases(
+pub trait Tree {
-	module: &Module,
+	type Ctx<'a>;
-	cwd: &mut Vec<Tok<String>>,
+	#[must_use]
-	root: &Module,
+	fn children(&self, public_only: bool) -> HashSet<IStr>;
-	alias_map: &mut HashMap<Sym, Sym>,
+	#[must_use]
-	alias_rev_map: &mut HashMap<Sym, HashSet<Sym>>,
+	fn child(
-	i: &Interner,
+		&self,
-	rep: &impl Reporter,
+		key: IStr,
-) {
+		public_only: bool,
-	let mut import_locs = HashMap::<Sym, Vec<Pos>>::new();
+		ctx: &mut Self::Ctx<'_>,
-	for item in &module.items {
+	) -> impl Future<Output = ChildResult<'_, Self>>;
-		match &item.kind {
+}
-			ItemKind::Import(imp) => match absolute_path(cwd, &imp.path) {
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-				Err(e) => rep.report(e.err_obj(i, item.pos.clone(), &imp.path.iter().join("::")).await),
+pub enum ChildErrorKind {
-				Ok(abs_path) => {
+	Missing,
-					let names = match imp.name.as_ref() {
+	/// Only thrown if public_only is true
-						Some(n) => Either::Right([n.clone()].into_iter()),
+	Private,
-						None => Either::Left(
+	Constant,
-							resolv_glob(cwd, root, &abs_path, item.pos.clone(), i, rep).await.into_iter(),
+}
-						),
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-					};
+pub struct ChildError {
-					for name in names {
+	pub pos: usize,
-						let mut tgt = abs_path.clone().suffix([name.clone()]).to_sym(i).await;
+	pub kind: ChildErrorKind,
-						let src = Sym::new(cwd.iter().cloned().chain([name]), i).await.unwrap();
+}
-						import_locs.entry(src.clone()).or_insert(vec![]).push(item.pos.clone());
+
-						if let Some(tgt2) = alias_map.get(&tgt) {
+// Problem: walk should take into account aliases and visibility
-							tgt = tgt2.clone();
+//
-						}
+// help: since every alias is also its own import, visibility only has to be
-						if src == tgt {
+// checked on the top level
-							rep.report(mk_err(
+//
-								i.i("Circular references").await,
+// idea: do a simple stack machine like below with no visibility for aliases and
-								format!("{src} circularly refers to itself"),
+// call it from an access-checking implementation for just the top level
-								[item.pos.clone().into()],
+//
-							));
+// caveat: we need to check EVERY IMPORT to ensure that all
-							continue;
+// errors are raised
-						}
+pub async fn walk<'a, T: Tree>(
-						if let Some(fst_val) = alias_map.get(&src) {
+	root: &'a T,
-							let locations = (import_locs.get(&src))
+	public_only: bool,
-								.expect("The same name could only have appeared in the same module");
+	path: impl IntoIterator<Item = IStr>,
-							rep.report(mk_err(
+	ctx: &mut T::Ctx<'_>,
-								i.i("Conflicting imports").await,
+) -> Result<&'a T, ChildError> {
-								if fst_val == &src {
+	let mut cur = root;
-									format!("{src} is imported multiple times")
+	for (i, item) in path.into_iter().enumerate() {
-								} else {
+		match cur.child(item, public_only, ctx).await {
-									format!("{} could refer to both {fst_val} and {src}", src.last())
+			ChildResult::Ok(v) => cur = v,
-								},
+			ChildResult::Err(kind) => return Err(ChildError { pos: i, kind }),
-								locations.iter().map(|p| p.clone().into()).collect_vec(),
-							))
-						}
-						let mut srcv = vec![src.clone()];
-						if let Some(src_extra) = alias_rev_map.remove(&src) {
-							srcv.extend(src_extra);
-						}
-						for src in srcv {
-							alias_map.insert(src.clone(), tgt.clone());
-							alias_rev_map.entry(tgt.clone()).or_insert(HashSet::new()).insert(src);
-						}
-					}
-				},
-			},
-			ItemKind::Member(mem) => match mem.kind().await {
-				MemberKind::Const(_) => (),
-				MemberKind::Mod(m) => {
-					cwd.push(mem.name());
-					imports_to_aliases(m, cwd, root, alias_map, alias_rev_map, i, rep).boxed_local().await;
-					cwd.pop();
-				},
-			},
-			ItemKind::Export(_) | ItemKind::Macro(..) => (),
 		}
 	}
-}
+	Ok(cur)
-
-pub async fn dealias(module: &mut Module, alias_map: &HashMap<Sym, Sym>, i: &Interner) {
-	for item in &mut module.items {
-		match &mut item.kind {
-			ItemKind::Export(_) | ItemKind::Import(_) => (),
-			ItemKind::Member(mem) => match mem.kind_mut().await {
-				MemberKind::Const(c) => {
-					let Some(source) = c.source() else { continue };
-					let Some(new_source) = dealias_mactreev(source, alias_map, i).await else { continue };
-					c.set_source(new_source);
-				},
-				MemberKind::Mod(m) => dealias(m, alias_map, i).boxed_local().await,
-			},
-			ItemKind::Macro(_, rules) =>
-				for rule in rules.iter_mut() {
-					let RuleKind::Native(c) = &mut rule.kind else { continue };
-					let Some(source) = c.source() else { continue };
-					let Some(new_source) = dealias_mactreev(source, alias_map, i).await else { continue };
-					c.set_source(new_source);
-				},
-		}
-	}
-}
-
-async fn dealias_mactree(
-	mtree: &MacTree,
-	aliases: &HashMap<Sym, Sym>,
-	i: &Interner,
-) -> Option<MacTree> {
-	let new_tok = match &*mtree.tok {
-		MacTok::Atom(_) | MacTok::Ph(_) => return None,
-		tok @ (MacTok::Done(_) | MacTok::Ref(_) | MacTok::Slot(_)) => panic!(
-			"{} should not appear in retained pre-macro source",
-			take_first(&tok.print(&FmtCtxImpl { i }).await, true)
-		),
-		MacTok::Name(n) => MacTok::Name(aliases.get(n).unwrap_or(n).clone()),
-		MacTok::Lambda(arg, body) => {
-			match (dealias_mactreev(arg, aliases, i).await, dealias_mactreev(body, aliases, i).await) {
-				(None, None) => return None,
-				(Some(arg), None) => MacTok::Lambda(arg, body.clone()),
-				(None, Some(body)) => MacTok::Lambda(arg.clone(), body),
-				(Some(arg), Some(body)) => MacTok::Lambda(arg, body),
-			}
-		},
-		MacTok::S(p, b) => MacTok::S(*p, dealias_mactreev(b, aliases, i).await?),
-	};
-	Some(MacTree { pos: mtree.pos.clone(), tok: Rc::new(new_tok) })
-}
-
-async fn dealias_mactreev(
-	mtreev: &[MacTree],
-	aliases: &HashMap<Sym, Sym>,
-	i: &Interner,
-) -> Option<Vec<MacTree>> {
-	let mut results = Vec::with_capacity(mtreev.len());
-	let mut any_some = false;
-	for item in mtreev {
-		let out = dealias_mactree(item, aliases, i).boxed_local().await;
-		any_some |= out.is_some();
-		results.push(out.unwrap_or(item.clone()));
-	}
-	any_some.then_some(results)
 }

orchid-host/src/execute.rs

@@ -1,17 +1,16 @@
 use std::mem;
 
-use async_std::sync::RwLockWriteGuard;
+use async_lock::RwLockWriteGuard;
 use bound::Bound;
 use futures::FutureExt;
-use orchid_base::error::{OrcErrv, mk_errv};
+use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtxImpl, Format, take_first};
 use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
-use orchid_base::name::NameLike;
 
 use crate::ctx::Ctx;
 use crate::expr::{Expr, ExprKind, PathSet, Step};
-use crate::tree::{ItemKind, MemberKind};
+use crate::tree::Root;
 
 type ExprGuard = Bound<RwLockWriteGuard<'static, ExprKind>, Expr>;
 
@@ -38,16 +37,21 @@ pub struct ExecCtx {
 	cur_pos: Pos,
 	did_pop: bool,
 	logger: Logger,
+	root: Root,
 }
 impl ExecCtx {
-	pub async fn new(ctx: Ctx, logger: Logger, init: Expr) -> Self {
+	#[must_use]
+	pub async fn new(ctx: Ctx, logger: Logger, root: Root, init: Expr) -> Self {
 		let cur_pos = init.pos();
 		let cur = Bound::async_new(init, |init| init.kind().write()).await;
-		Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger }
+		Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger, root }
 	}
+	#[must_use]
 	pub fn remaining_gas(&self) -> u64 { self.gas.expect("queried remaining_gas but no gas was set") }
 	pub fn set_gas(&mut self, gas: Option<u64>) { self.gas = gas }
+	#[must_use]
 	pub fn idle(&self) -> bool { self.did_pop }
+	#[must_use]
 	pub fn result(self) -> ExecResult {
 		if self.idle() {
 			match &*self.cur {
@@ -58,15 +62,18 @@ impl ExecCtx {
 			ExecResult::Gas(self)
 		}
 	}
+	#[must_use]
 	pub fn use_gas(&mut self, amount: u64) -> bool {
 		if let Some(gas) = &mut self.gas {
 			*gas -= amount;
 		}
 		self.gas != Some(0)
 	}
+	#[must_use]
 	pub async fn try_lock(&self, ex: &Expr) -> ExprGuard {
 		Bound::async_new(ex.clone(), |ex| ex.kind().write()).await
 	}
+	#[must_use]
 	pub async fn unpack_ident(&self, ex: &Expr) -> Expr {
 		match ex.kind().try_write().as_deref_mut() {
 			Some(ExprKind::Identity(ex)) => {
@@ -91,47 +98,19 @@ impl ExecCtx {
 				},
 				ExprKind::Seq(a, b) if !self.did_pop => (ExprKind::Seq(a.clone(), b), StackOp::Push(a)),
 				ExprKind::Seq(_, b) => (ExprKind::Identity(b), StackOp::Nop),
-				ExprKind::Const(name) => {
+				ExprKind::Const(name) =>
-					let (cn, mp) = name.split_last();
+					match self.root.get_const_value(name, self.cur_pos.clone()).await {
-					let root_lock = self.ctx.root.read().await;
+						Err(e) => (ExprKind::Bottom(e), StackOp::Pop),
-					let module = root_lock.walk(true, mp.iter().cloned()).await.unwrap();
+						Ok(v) => (ExprKind::Identity(v), StackOp::Nop),
-					let member = (module.items.iter())
+					},
-						.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
-						.find(|m| m.name() == cn);
-					match member {
-						None => (
-							ExprKind::Bottom(mk_errv(
-								self.ctx.i.i("Constant does not exist").await,
-								format!("{name} does not refer to a constant"),
-								[self.cur_pos.clone().into()],
-							)),
-							StackOp::Pop,
-						),
-						Some(mem) => match mem.kind().await {
-							MemberKind::Mod(_) => (
-								ExprKind::Bottom(mk_errv(
-									self.ctx.i.i("module used as constant").await,
-									format!("{name} is a module"),
-									[self.cur_pos.clone().into()],
-								)),
-								StackOp::Pop,
-							),
-							MemberKind::Const(c) => {
-								let value = c.get_bytecode(&self.ctx).await;
-								(ExprKind::Identity(value.clone()), StackOp::Nop)
-							},
-						},
-					}
-				},
 				ExprKind::Arg => panic!("This should not appear outside function bodies"),
 				ek @ ExprKind::Atom(_) => (ek, StackOp::Pop),
 				ExprKind::Bottom(bot) => (ExprKind::Bottom(bot.clone()), StackOp::Unwind(bot)),
 				ExprKind::Call(f, x) if !self.did_pop => (ExprKind::Call(f.clone(), x), StackOp::Push(f)),
 				ExprKind::Call(f, x) => match f.try_into_owned_atom().await {
 					Ok(atom) => {
-						let mut ext = atom.sys().ext().clone();
 						let x_norm = self.unpack_ident(&x).await;
-						let val = Expr::from_api(&atom.call(x_norm).await, &mut ext).await;
+						let val = atom.call(x_norm).await;
 						(ExprKind::Identity(val.clone()), StackOp::Swap(val))
 					},
 					Err(f) => match &*f.kind().read().await {
@@ -139,10 +118,9 @@ impl ExecCtx {
 							panic!("This should not appear outside function bodies"),
 						ExprKind::Missing => panic!("Should have been replaced"),
 						ExprKind::Atom(a) => {
-							let mut ext = a.sys().ext().clone();
 							let x_norm = self.unpack_ident(&x).await;
-							let val = Expr::from_api(&a.clone().call(x_norm).await, &mut ext).await;
+							let ret = a.clone().call(x_norm).await;
-							(ExprKind::Identity(val.clone()), StackOp::Swap(val))
+							(ExprKind::Identity(ret.clone()), StackOp::Swap(ret))
 						},
 						ExprKind::Bottom(exprv) => (ExprKind::Bottom(exprv.clone()), StackOp::Pop),
 						ExprKind::Lambda(None, body) =>
@@ -185,6 +163,7 @@ impl ExecCtx {
 	}
 }
 
+#[must_use]
 async fn substitute(
 	src: &Expr,
 	path: &[Step],

orchid-host/src/expr.rs

@@ -4,27 +4,28 @@ use std::num::NonZeroU64;
 use std::rc::{Rc, Weak};
 use std::{fmt, mem};
 
-use async_std::sync::RwLock;
+use async_lock::RwLock;
 use futures::FutureExt;
-use hashbrown::HashSet;
 use itertools::Itertools;
-use orchid_base::error::{OrcErrv, mk_errv};
+use orchid_base::error::OrcErrv;
-use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, Format, Variants, take_first};
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
-use orchid_base::location::Pos;
+use orchid_base::interner::Interner;
-use orchid_base::macros::mtreev_fmt;
+use orchid_base::location::{Pos, SrcRange};
 use orchid_base::name::Sym;
-use orchid_base::tokens::Paren;
+use orchid_base::tl_cache;
-use orchid_base::tree::{AtomRepr, indent};
+use orchid_base::tree::{AtomRepr, TokenVariant, indent};
-use orchid_base::{match_mapping, tl_cache};
 use substack::Substack;
 
 use crate::api;
 use crate::atom::AtomHand;
 use crate::ctx::Ctx;
-use crate::extension::Extension;
+use crate::expr_store::ExprStore;
-use crate::macros::{MacTok, MacTree};
 
-pub type ExprParseCtx = Extension;
+#[derive(Clone)]
+pub struct ExprParseCtx<'a> {
+	pub ctx: &'a Ctx,
+	pub exprs: &'a ExprStore,
+}
 
 #[derive(Debug)]
 pub struct ExprData {
@@ -35,6 +36,7 @@ pub struct ExprData {
 #[derive(Clone, Debug)]
 pub struct Expr(Rc<ExprData>);
 impl Expr {
+	#[must_use]
 	pub fn pos(&self) -> Pos { self.0.pos.clone() }
 	pub async fn try_into_owned_atom(self) -> Result<AtomHand, Self> {
 		match Rc::try_unwrap(self.0) {
@@ -45,24 +47,60 @@ impl Expr {
 			},
 		}
 	}
+	#[must_use]
 	pub async fn as_atom(&self) -> Option<AtomHand> {
 		if let ExprKind::Atom(a) = &*self.kind().read().await { Some(a.clone()) } else { None }
 	}
+	#[must_use]
 	pub fn strong_count(&self) -> usize { Rc::strong_count(&self.0) }
+	#[must_use]
 	pub fn id(&self) -> api::ExprTicket {
 		api::ExprTicket(
 			NonZeroU64::new(self.0.as_ref() as *const ExprData as usize as u64)
 				.expect("this is a ref, it cannot be null"),
 		)
 	}
-	pub async fn from_api(api: &api::Expression, ctx: &mut ExprParseCtx) -> Self {
+	#[must_use]
-		if let api::ExpressionKind::Slot(tk) = &api.kind {
+	pub async fn from_api(
-			return ctx.exprs().get_expr(*tk).expect("Invalid slot");
+		api: &api::Expression,
-		}
+		psb: PathSetBuilder<'_, u64>,
-		let pos = Pos::from_api(&api.location, &ctx.ctx().i).await;
+		ctx: &mut ExprParseCtx<'_>,
-		let kind = RwLock::new(ExprKind::from_api(&api.kind, pos.clone(), ctx).boxed_local().await);
+	) -> Self {
-		Self(Rc::new(ExprData { pos, kind }))
+		let pos = Pos::from_api(&api.location, &ctx.ctx.i).await;
+		let kind = match &api.kind {
+			api::ExpressionKind::Arg(n) => {
+				assert!(psb.register_arg(n), "Arguments must be enclosed in a matching lambda");
+				ExprKind::Arg
+			},
+			api::ExpressionKind::Bottom(bot) =>
+				ExprKind::Bottom(OrcErrv::from_api(bot, &ctx.ctx.i).await),
+			api::ExpressionKind::Call(f, x) => {
+				let (lpsb, rpsb) = psb.split();
+				ExprKind::Call(
+					Expr::from_api(f, lpsb, ctx).boxed_local().await,
+					Expr::from_api(x, rpsb, ctx).boxed_local().await,
+				)
+			},
+			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(*name, &ctx.ctx.i).await),
+			api::ExpressionKind::Lambda(x, body) => {
+				let lbuilder = psb.lambda(x);
+				let body = Expr::from_api(body, lbuilder.stack(), ctx).boxed_local().await;
+				ExprKind::Lambda(lbuilder.collect(), body)
+			},
+			api::ExpressionKind::NewAtom(a) =>
+				ExprKind::Atom(AtomHand::from_api(a, pos.clone(), &mut ctx.ctx.clone()).await),
+			api::ExpressionKind::Slot(tk) => return ctx.exprs.get_expr(*tk).expect("Invalid slot"),
+			api::ExpressionKind::Seq(a, b) => {
+				let (apsb, bpsb) = psb.split();
+				ExprKind::Seq(
+					Expr::from_api(a, apsb, ctx).boxed_local().await,
+					Expr::from_api(b, bpsb, ctx).boxed_local().await,
+				)
+			},
+		};
+		Self(Rc::new(ExprData { pos, kind: RwLock::new(kind) }))
 	}
+	#[must_use]
 	pub async fn to_api(&self) -> api::InspectedKind {
 		use api::InspectedKind as K;
 		match &*self.0.kind.read().await {
@@ -72,23 +110,23 @@ impl Expr {
 			_ => K::Opaque,
 		}
 	}
+	#[must_use]
 	pub fn kind(&self) -> &RwLock<ExprKind> { &self.0.kind }
 }
 impl Format for Expr {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		return print_expr(self, c, &mut HashSet::new()).await;
+		return print_expr(self, c, Substack::Bottom).await;
 	}
 }
 async fn print_expr<'a>(
 	expr: &'a Expr,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
-	visited: &mut HashSet<api::ExprTicket>,
+	visited: Substack<'_, api::ExprTicket>,
 ) -> FmtUnit {
-	if visited.contains(&expr.id()) {
+	if visited.iter().any(|id| id == &expr.id()) {
 		return "CYCLIC_EXPR".to_string().into();
 	}
-	visited.insert(expr.id());
+	print_exprkind(&*expr.kind().read().await, c, visited.push(expr.id())).boxed_local().await
-	print_exprkind(&*expr.kind().read().await, c, visited).boxed_local().await
 }
 
 #[derive(Clone, Debug)]
@@ -107,34 +145,18 @@ pub enum ExprKind {
 	Missing,
 }
 impl ExprKind {
-	pub async fn from_api(api: &api::ExpressionKind, pos: Pos, ctx: &mut ExprParseCtx) -> Self {
+	#[must_use]
-		match_mapping!(api, api::ExpressionKind => ExprKind {
-			Lambda(id => PathSet::from_api(*id, api), b => Expr::from_api(b, ctx).await),
-			Bottom(b => OrcErrv::from_api(b, &ctx.ctx().i).await),
-			Call(f => Expr::from_api(f, ctx).await, x => Expr::from_api(x, ctx).await),
-			Const(c => Sym::from_api(*c, &ctx.ctx().i).await),
-			Seq(a => Expr::from_api(a, ctx).await, b => Expr::from_api(b, ctx).await),
-		} {
-			api::ExpressionKind::Arg(_) => ExprKind::Arg,
-			api::ExpressionKind::NewAtom(a) => ExprKind::Atom(AtomHand::from_api(
-				a,
-				pos,
-				&mut ctx.ctx().clone()
-			).await),
-			api::ExpressionKind::Slot(_) => panic!("Handled in Expr"),
-		})
-	}
 	pub fn at(self, pos: Pos) -> Expr { Expr(Rc::new(ExprData { pos, kind: RwLock::new(self) })) }
 }
 impl Format for ExprKind {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		print_exprkind(self, c, &mut HashSet::new()).await
+		print_exprkind(self, c, Substack::Bottom).await
 	}
 }
 async fn print_exprkind<'a>(
 	ek: &ExprKind,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
-	visited: &mut HashSet<api::ExprTicket>,
+	visited: Substack<'_, api::ExprTicket>,
 ) -> FmtUnit {
 	match &ek {
 		ExprKind::Arg => "Arg".to_string().into(),
@@ -174,6 +196,102 @@ pub enum Step {
 	Right,
 }
 
+#[derive(Clone)]
+pub enum PathSetFrame<'a, T: PartialEq> {
+	Lambda(&'a T, &'a RefCell<Option<PathSet>>),
+	Step(Step),
+}
+
+#[derive(Clone)]
+pub struct PathSetBuilder<'a, T: PartialEq>(Substack<'a, PathSetFrame<'a, T>>);
+impl<'a, T: PartialEq> PathSetBuilder<'a, T> {
+	#[must_use]
+	pub fn new() -> Self { Self(Substack::Bottom) }
+	#[must_use]
+	pub fn split(&'a self) -> (Self, Self) {
+		(
+			Self(self.0.push(PathSetFrame::Step(Step::Left))),
+			Self(self.0.push(PathSetFrame::Step(Step::Right))),
+		)
+	}
+	#[must_use]
+	pub fn lambda<'b>(self, arg: &'b T) -> LambdaBuilder<'b, T>
+	where 'a: 'b {
+		LambdaBuilder { arg, path: RefCell::default(), stack: self }
+	}
+	/// Register an argument with the corresponding lambda and return true if one
+	/// was found. (if false is returned, the name is unbound and may refer to a
+	/// global)
+	pub fn register_arg(self, t: &T) -> bool {
+		let mut steps = VecDeque::new();
+		for step in self.0.iter() {
+			match step {
+				PathSetFrame::Step(step) => steps.push_front(*step),
+				PathSetFrame::Lambda(name, _) if **name != *t => (),
+				PathSetFrame::Lambda(_, cell) => {
+					let mut ps_opt = cell.borrow_mut();
+					match &mut *ps_opt {
+						val @ None => *val = Some(PathSet { steps: steps.into(), next: None }),
+						Some(val) => {
+							let mut swap = PathSet { steps: Vec::new(), next: None };
+							mem::swap(&mut swap, val);
+							*val = merge(swap, &Vec::from(steps));
+						},
+					}
+					return true;
+				},
+			};
+		}
+		return false;
+		fn merge(ps: PathSet, steps: &[Step]) -> PathSet {
+			let diff_idx = ps.steps.iter().zip(steps).take_while(|(l, r)| l == r).count();
+			if diff_idx == ps.steps.len() {
+				if diff_idx == steps.len() {
+					match ps.next {
+						Some(_) => panic!("New path ends where old path forks"),
+						None => panic!("New path same as old path"),
+					}
+				}
+				let Some((left, right)) = ps.next else { panic!("Old path ends where new path continues") };
+				let next = match steps[diff_idx] {
+					Step::Left => Some((Box::new(merge(*left, &steps[diff_idx + 1..])), right)),
+					Step::Right => Some((left, Box::new(merge(*right, &steps[diff_idx + 1..])))),
+				};
+				PathSet { steps: ps.steps, next }
+			} else {
+				let shared_steps = ps.steps.iter().take(diff_idx).cloned().collect();
+				let main_steps = ps.steps.iter().skip(diff_idx + 1).cloned().collect();
+				let new_branch = steps[diff_idx + 1..].to_vec();
+				let main_side = PathSet { steps: main_steps, next: ps.next };
+				let new_side = PathSet { steps: new_branch, next: None };
+				let (left, right) = match steps[diff_idx] {
+					Step::Left => (new_side, main_side),
+					Step::Right => (main_side, new_side),
+				};
+				PathSet { steps: shared_steps, next: Some((Box::new(left), Box::new(right))) }
+			}
+		}
+	}
+}
+
+impl<'a, T: PartialEq> Default for PathSetBuilder<'a, T> {
+	fn default() -> Self { Self::new() }
+}
+
+pub struct LambdaBuilder<'a, T: PartialEq> {
+	arg: &'a T,
+	path: RefCell<Option<PathSet>>,
+	stack: PathSetBuilder<'a, T>,
+}
+impl<'a, T: PartialEq> LambdaBuilder<'a, T> {
+	#[must_use]
+	pub fn stack(&'a self) -> PathSetBuilder<'a, T> {
+		PathSetBuilder(self.stack.0.push(PathSetFrame::Lambda(self.arg, &self.path)))
+	}
+	#[must_use]
+	pub fn collect(self) -> Option<PathSet> { self.path.into_inner() }
+}
+
 #[derive(Clone, Debug)]
 pub struct PathSet {
 	/// The single steps through [super::nort::Clause::Apply]
@@ -183,35 +301,10 @@ pub struct PathSet {
 	pub next: Option<(Box<PathSet>, Box<PathSet>)>,
 }
 impl PathSet {
+	#[must_use]
 	pub fn next(&self) -> Option<(&PathSet, &PathSet)> {
 		self.next.as_ref().map(|(l, r)| (&**l, &**r))
 	}
-	pub fn from_api(id: u64, api: &api::ExpressionKind) -> Option<Self> {
-		use api::ExpressionKind as K;
-		struct Suffix(VecDeque<Step>, Option<(Box<PathSet>, Box<PathSet>)>);
-		fn seal(Suffix(steps, next): Suffix) -> PathSet { PathSet { steps: steps.into(), next } }
-		fn after(step: Step, mut suf: Suffix) -> Suffix {
-			suf.0.push_front(step);
-			suf
-		}
-		return from_api_inner(id, api).map(seal);
-		fn from_api_inner(id: u64, api: &api::ExpressionKind) -> Option<Suffix> {
-			match &api {
-				K::Arg(id2) => (id == *id2).then_some(Suffix(VecDeque::new(), None)),
-				K::Bottom(_) | K::Const(_) | K::NewAtom(_) | K::Slot(_) => None,
-				K::Lambda(_, b) => from_api_inner(id, &b.kind),
-				K::Call(l, r) | K::Seq(l, r) => {
-					match (from_api_inner(id, &l.kind), from_api_inner(id, &r.kind)) {
-						(Some(a), Some(b)) =>
-							Some(Suffix(VecDeque::new(), Some((Box::new(seal(a)), Box::new(seal(b)))))),
-						(Some(l), None) => Some(after(Step::Left, l)),
-						(None, Some(r)) => Some(after(Step::Right, r)),
-						(None, None) => None,
-					}
-				},
-			}
-		}
-	}
 }
 impl fmt::Display for PathSet {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -229,6 +322,7 @@ impl fmt::Display for PathSet {
 	}
 }
 
+#[must_use]
 pub fn bot_expr(err: impl Into<OrcErrv>) -> Expr {
 	let errv: OrcErrv = err.into();
 	let pos = errv.pos_iter().next().map_or(Pos::None, |ep| ep.position.clone());
@@ -237,123 +331,44 @@ pub fn bot_expr(err: impl Into<OrcErrv>) -> Expr {
 
 pub struct WeakExpr(Weak<ExprData>);
 impl WeakExpr {
+	#[must_use]
 	pub fn upgrade(&self) -> Option<Expr> { self.0.upgrade().map(Expr) }
 }
 
-#[derive(Clone)]
+impl TokenVariant<api::ExprTicket> for Expr {
-pub enum SrcToExprStep<'a> {
+	type FromApiCtx<'a> = ExprStore;
-	Left,
+	async fn from_api(
-	Right,
+		api: &api::ExprTicket,
-	Lambda(Sym, &'a RefCell<Option<PathSet>>),
+		ctx: &mut Self::FromApiCtx<'_>,
+		_: SrcRange,
+		_: &Interner,
+	) -> Self {
+		ctx.get_expr(*api).expect("Invalid ticket")
+	}
+	type ToApiCtx<'a> = ExprStore;
+	async fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> api::ExprTicket {
+		let id = self.id();
+		ctx.give_expr(self);
+		id
+	}
 }
 
-pub async fn mtreev_to_expr(
+/// Acknowledgment that expr serialization is impossible and thus will panic.
-	src: &[MacTree],
+#[derive(Debug, Clone, Copy, Default)]
-	stack: Substack<'_, SrcToExprStep<'_>>,
+pub struct ExprWillPanic;
-	ctx: &Ctx,
+
-) -> ExprKind {
+impl TokenVariant<api::Expression> for Expr {
-	let Some((x, f)) = src.split_last() else { panic!("Empty expression cannot be evaluated") };
+	type FromApiCtx<'a> = ExprParseCtx<'a>;
-	let x_stack = if f.is_empty() { stack.clone() } else { stack.push(SrcToExprStep::Right) };
+	async fn from_api(
-	let x_kind = match &*x.tok {
+		api: &api::Expression,
-		MacTok::Atom(a) => ExprKind::Atom(a.clone()),
+		ctx: &mut Self::FromApiCtx<'_>,
-		MacTok::Name(n) => 'name: {
+		_: SrcRange,
-			let mut steps = VecDeque::new();
+		_: &Interner,
-			for step in x_stack.iter() {
+	) -> Self {
-				match step {
+		Self::from_api(api, PathSetBuilder::new(), ctx).await
-					SrcToExprStep::Left => steps.push_front(Step::Left),
+	}
-					SrcToExprStep::Right => steps.push_front(Step::Right),
+	type ToApiCtx<'a> = ExprWillPanic;
-					SrcToExprStep::Lambda(name, _) if name != n => continue,
+	async fn into_api(self, ExprWillPanic: &mut Self::ToApiCtx<'_>) -> api::Expression {
-					SrcToExprStep::Lambda(_, cell) => {
+		panic!("Cannot serialize expr!")
-						let mut ps = cell.borrow_mut();
-						match &mut *ps {
-							val @ None => *val = Some(PathSet { steps: steps.into(), next: None }),
-							Some(val) => {
-								let mut swap = PathSet { steps: Vec::new(), next: None };
-								mem::swap(&mut swap, val);
-								*val = merge(swap, &Vec::from(steps));
-								fn merge(ps: PathSet, steps: &[Step]) -> PathSet {
-									let diff_idx = ps.steps.iter().zip(steps).take_while(|(l, r)| l == r).count();
-									if diff_idx == ps.steps.len() {
-										if diff_idx == steps.len() {
-											match ps.next {
-												Some(_) => panic!("New path ends where old path forks"),
-												None => panic!("New path same as old path"),
-											}
-										}
-										let Some((left, right)) = ps.next else {
-											panic!("Old path ends where new path continues")
-										};
-										let next = match steps[diff_idx] {
-											Step::Left => Some((Box::new(merge(*left, &steps[diff_idx + 1..])), right)),
-											Step::Right => Some((left, Box::new(merge(*right, &steps[diff_idx + 1..])))),
-										};
-										PathSet { steps: ps.steps, next }
-									} else {
-										let shared_steps = ps.steps.iter().take(diff_idx).cloned().collect();
-										let main_steps = ps.steps.iter().skip(diff_idx + 1).cloned().collect();
-										let new_branch = steps[diff_idx + 1..].to_vec();
-										let main_side = PathSet { steps: main_steps, next: ps.next };
-										let new_side = PathSet { steps: new_branch, next: None };
-										let (left, right) = match steps[diff_idx] {
-											Step::Left => (new_side, main_side),
-											Step::Right => (main_side, new_side),
-										};
-										PathSet { steps: shared_steps, next: Some((Box::new(left), Box::new(right))) }
-									}
-								}
-							},
-						}
-						break 'name ExprKind::Arg;
-					},
-				}
-			}
-			ExprKind::Const(n.clone())
-		},
-		MacTok::Ph(_) | MacTok::Done(_) | MacTok::Ref(_) | MacTok::Slot(_) =>
-			ExprKind::Bottom(mk_errv(
-				ctx.i.i("placeholder in value").await,
-				"Placeholders cannot appear anywhere outside macro patterns",
-				[x.pos.clone().into()],
-			)),
-		MacTok::S(Paren::Round, b) if b.is_empty() =>
-			return ExprKind::Bottom(mk_errv(
-				ctx.i.i("Empty expression").await,
-				"Empty parens () are illegal",
-				[x.pos.clone().into()],
-			)),
-		MacTok::S(Paren::Round, b) => mtreev_to_expr(b, x_stack, ctx).boxed_local().await,
-		MacTok::S(..) => ExprKind::Bottom(mk_errv(
-			ctx.i.i("non-round parentheses after macros").await,
-			"[] or {} block was not consumed by macros; expressions may only contain ()",
-			[x.pos.clone().into()],
-		)),
-		MacTok::Lambda(_, b) if b.is_empty() =>
-			return ExprKind::Bottom(mk_errv(
-				ctx.i.i("Empty lambda").await,
-				"Lambdas must have a body",
-				[x.pos.clone().into()],
-			)),
-		MacTok::Lambda(arg, b) => 'lambda_converter: {
-			if let [MacTree { tok, .. }] = &**arg {
-				if let MacTok::Name(n) = &**tok {
-					let path = RefCell::new(None);
-					let b = mtreev_to_expr(b, x_stack.push(SrcToExprStep::Lambda(n.clone(), &path)), ctx)
-						.boxed_local()
-						.await;
-					break 'lambda_converter ExprKind::Lambda(path.into_inner(), b.at(x.pos.clone()));
-				}
-			}
-			let argstr = take_first(&mtreev_fmt(arg, &FmtCtxImpl { i: &ctx.i }).await, true);
-			ExprKind::Bottom(mk_errv(
-				ctx.i.i("Malformeed lambda").await,
-				format!("Lambda argument should be single name, found {argstr}"),
-				[x.pos.clone().into()],
-			))
-		},
-	};
-	if f.is_empty() {
-		return x_kind;
 	}
-	let f = mtreev_to_expr(f, stack.push(SrcToExprStep::Left), ctx).boxed_local().await;
-	ExprKind::Call(f.at(Pos::None), x_kind.at(x.pos.clone()))
 }

orchid-host/src/expr_store.rs

@@ -1,5 +1,6 @@
 use std::cell::RefCell;
 use std::fmt;
+use std::rc::Rc;
 
 use hashbrown::HashMap;
 use hashbrown::hash_map::Entry;
@@ -8,27 +9,44 @@ use crate::api;
 use crate::expr::Expr;
 
 #[derive(Default)]
-pub struct ExprStore(RefCell<HashMap<api::ExprTicket, (u32, Expr)>>);
+pub struct ExprStoreData {
+	exprs: RefCell<HashMap<api::ExprTicket, (u32, Expr)>>,
+	parent: Option<ExprStore>,
+}
+#[derive(Clone, Default)]
+pub struct ExprStore(Rc<ExprStoreData>);
 impl ExprStore {
+	#[must_use]
+	pub fn derive(&self) -> Self {
+		Self(Rc::new(ExprStoreData { exprs: RefCell::default(), parent: Some(self.clone()) }))
+	}
 	pub fn give_expr(&self, expr: Expr) {
-		match self.0.borrow_mut().entry(expr.id()) {
+		match self.0.exprs.borrow_mut().entry(expr.id()) {
 			Entry::Occupied(mut oe) => oe.get_mut().0 += 1,
 			Entry::Vacant(v) => {
 				v.insert((1, expr));
 			},
 		}
 	}
-	pub fn take_expr(&self, ticket: api::ExprTicket) {
+	pub fn take_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {
-		(self.0.borrow_mut().entry(ticket))
+		match self.0.exprs.borrow_mut().entry(ticket) {
-			.and_replace_entry_with(|_, (rc, rt)| (1 < rc).then_some((rc - 1, rt)));
+			Entry::Vacant(_) => None,
+			Entry::Occupied(oe) if oe.get().0 == 1 => Some(oe.remove().1),
+			Entry::Occupied(mut oe) => {
+				oe.get_mut().0 -= 1;
+				Some(oe.get().1.clone())
+			},
+		}
 	}
+	#[must_use]
 	pub fn get_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {
-		self.0.borrow().get(&ticket).map(|(_, expr)| expr.clone())
+		(self.0.exprs.borrow().get(&ticket).map(|(_, expr)| expr.clone()))
+			.or_else(|| self.0.parent.as_ref()?.get_expr(ticket))
 	}
 }
 impl fmt::Display for ExprStore {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let r = self.0.borrow();
+		let r = self.0.exprs.borrow();
 		let rc: u32 = r.values().map(|v| v.0).sum();
 		write!(f, "Store holding {rc} refs to {} exprs", r.len())
 	}

orchid-host/src/extension.rs

@@ -2,29 +2,35 @@ use std::cell::RefCell;
 use std::future::Future;
 use std::io;
 use std::num::NonZeroU64;
+use std::pin::pin;
 use std::rc::{Rc, Weak};
 
-use async_std::channel::{self, Sender};
+use async_fn_stream::stream;
-use async_std::sync::Mutex;
 use derive_destructure::destructure;
-use futures::FutureExt;
+use futures::channel::mpsc::{Sender, channel};
 use futures::future::{join, join_all};
+use futures::lock::Mutex;
+use futures::{SinkExt, StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::HostMsgSet;
 use orchid_api_traits::Request;
 use orchid_base::builtin::ExtInit;
 use orchid_base::clone;
 use orchid_base::format::{FmtCtxImpl, Format};
 use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
+use orchid_base::name::Sym;
 use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _};
+use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::atom::AtomHand;
 use crate::ctx::Ctx;
+use crate::dealias::{ChildError, ChildErrorKind, walk};
 use crate::expr_store::ExprStore;
 use crate::system::SystemCtor;
+use crate::tree::MemberKind;
 
 pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 
@@ -35,19 +41,22 @@ pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 #[derive(destructure)]
 pub struct ExtensionData {
 	ctx: Ctx,
-	init: ExtInit,
 	reqnot: ReqNot<api::HostMsgSet>,
 	systems: Vec<SystemCtor>,
 	logger: Logger,
 	next_pars: RefCell<NonZeroU64>,
 	exprs: ExprStore,
-	lex_recur: Mutex<HashMap<api::ParsId, channel::Sender<ReqPair<api::SubLex>>>>,
+	exiting_snd: Sender<()>,
-	mac_recur: Mutex<HashMap<api::ParsId, channel::Sender<ReqPair<api::RunMacros>>>>,
+	lex_recur: Mutex<HashMap<api::ParsId, Sender<ReqPair<api::SubLex>>>>,
 }
 impl Drop for ExtensionData {
 	fn drop(&mut self) {
 		let reqnot = self.reqnot.clone();
-		(self.ctx.spawn)(Box::pin(async move { reqnot.notify(api::HostExtNotif::Exit).await }))
+		let mut exiting_snd = self.exiting_snd.clone();
+		(self.ctx.spawn)(Box::pin(async move {
+			reqnot.notify(api::HostExtNotif::Exit).await;
+			exiting_snd.send(()).await.unwrap()
+		}))
 	}
 }
 
@@ -55,121 +64,187 @@ impl Drop for ExtensionData {
 pub struct Extension(Rc<ExtensionData>);
 impl Extension {
 	pub fn new(init: ExtInit, logger: Logger, msg_logger: Logger, ctx: Ctx) -> io::Result<Self> {
-		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
+		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| {
-			exprs: ExprStore::default(),
+			let init = Rc::new(init);
-			ctx: ctx.clone(),
+			let (exiting_snd, exiting_rcv) = channel::<()>(0);
-			systems: (init.systems.iter().cloned())
+			(ctx.spawn)(clone!(init, weak, ctx; Box::pin(async move {
-				.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
+				let rcv_stream = stream(async |mut cx| loop { cx.emit( init.recv().await).await });
-				.collect(),
+				let mut event_stream = pin!(stream::select(exiting_rcv.map(|()| None), rcv_stream));
-			logger: logger.clone(),
+				while let Some(Some(msg)) = event_stream.next().await {
-			init,
+					if let Some(reqnot) = weak.upgrade().map(|rc| rc.reqnot.clone()) {
-			next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
+						let reqnot = reqnot.clone();
-			lex_recur: Mutex::default(),
+						(ctx.spawn)(Box::pin(async move {
-			mac_recur: Mutex::default(),
+							reqnot.receive(&msg).await;
-			reqnot: ReqNot::new(
+						}))
-				msg_logger,
-				clone!(weak; move |sfn, _| clone!(weak; async move {
-					let data = weak.upgrade().unwrap();
-					data.init.send(sfn).await
-				}.boxed_local())),
-				clone!(weak; move |notif, _| {
-					clone!(weak; Box::pin(async move {
-					let this = Extension(weak.upgrade().unwrap());
-					match notif {
-						api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
-							let target = this.0.exprs.get_expr(acq.1).expect("Invalid ticket");
-							this.0.exprs.give_expr(target)
-						}
-						api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
-							this.assert_own_sys(rel.0).await;
-							this.0.exprs.take_expr(rel.1)
-						}
-						api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
-							this.assert_own_sys(mov.dec).await;
-							let recp = this.ctx().system_inst(mov.inc).await.expect("invallid recipient sys id");
-							let expr = this.0.exprs.get_expr(mov.expr).expect("invalid ticket");
-							recp.ext().0.exprs.give_expr(expr);
-							this.0.exprs.take_expr(mov.expr);
-						},
-						api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str),
 					}
-				}))}),
+				}
-				{
+			})));
-					clone!(weak, ctx);
+			ExtensionData {
-					move |hand, req| {
+				exiting_snd,
-						clone!(weak, ctx);
+				exprs: ctx.common_exprs.derive(),
-						Box::pin(async move {
+				ctx: ctx.clone(),
-							let this = Self(weak.upgrade().unwrap());
+				systems: (init.systems.iter().cloned())
-							writeln!(this.reqnot().logger(), "Host received request {req:?}");
+					.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
-							let i = this.ctx().i.clone();
+					.collect(),
-							match req {
+				logger: logger.clone(),
-								api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
+				next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
-								api::ExtHostReq::IntReq(intreq) => match intreq {
+				lex_recur: Mutex::default(),
-									api::IntReq::InternStr(s) => hand.handle(&s, &i.i(&*s.0).await.to_api()).await,
+				reqnot: ReqNot::new(
-									api::IntReq::InternStrv(v) => {
+					msg_logger,
-										let tokens = join_all(v.0.iter().map(|m| i.ex(*m))).await;
+					move |sfn, _| clone!(init; Box::pin(async move { init.send(sfn).await })),
-										hand.handle(&v, &i.i(&tokens).await.to_api()).await
+					clone!(weak; move |notif, _| {
-									},
+						clone!(weak; Box::pin(async move {
-									api::IntReq::ExternStr(si) =>
+						let this = Extension(weak.upgrade().unwrap());
-										hand.handle(&si, &Tok::<String>::from_api(si.0, &i).await.rc()).await,
+						match notif {
-									api::IntReq::ExternStrv(vi) => {
+							api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
-										let markerv = (i.ex(vi.0).await.iter()).map(|t| t.to_api()).collect_vec();
+								let target = this.0.exprs.get_expr(acq.1).expect("Invalid ticket");
-										hand.handle(&vi, &markerv).await
+								this.0.exprs.give_expr(target)
-									},
-								},
-								api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
-									let sys = ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
-									let reply =
-										sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await;
-									hand.handle(fw, &reply).await
-								},
-								api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
-									let sys = ctx.system_inst(id).await.unwrap();
-									hand.handle(fw, &sys.request(body.clone()).await).await
-								},
-								api::ExtHostReq::SubLex(sl) => {
-									let (rep_in, rep_out) = channel::bounded(1);
-									{
-										let lex_g = this.0.lex_recur.lock().await;
-										let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
-										req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
-									}
-									hand.handle(&sl, &rep_out.recv().await.unwrap()).await
-								},
-								api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect { target })) => {
-									let expr = this.exprs().get_expr(target).expect("Invalid ticket");
-									hand
-										.handle(&ins, &api::Inspected {
-											refcount: expr.strong_count() as u32,
-											location: expr.pos().to_api(),
-											kind: expr.to_api().await,
-										})
-										.await
-								},
-								api::ExtHostReq::RunMacros(rm) => {
-									let (rep_in, rep_out) = channel::bounded(1);
-									let lex_g = this.0.mac_recur.lock().await;
-									let req_in = lex_g.get(&rm.run_id).expect("Sublex for nonexistent lexid");
-									req_in.send(ReqPair(rm.clone(), rep_in)).await.unwrap();
-									hand.handle(&rm, &rep_out.recv().await.unwrap()).await
-								},
-								api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
-									let atom = AtomHand::new(atom.clone(), &ctx).await;
-									let unit = atom.print(&FmtCtxImpl { i: &this.ctx().i }).await;
-									hand.handle(eap, &unit.to_api()).await
-								},
 							}
-						})
+							api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
-					}
+								this.assert_own_sys(rel.0).await;
-				},
+								this.0.exprs.take_expr(rel.1);
-			),
+							}
+							api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
+								this.assert_own_sys(mov.dec).await;
+								let recp = this.ctx().system_inst(mov.inc).await.expect("invallid recipient sys id");
+								let expr = this.0.exprs.get_expr(mov.expr).expect("invalid ticket");
+								recp.ext().0.exprs.give_expr(expr);
+								this.0.exprs.take_expr(mov.expr);
+							},
+							api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str),
+						}
+					}))}),
+					{
+						clone!(weak, ctx);
+						move |hand, req| {
+							clone!(weak, ctx);
+							Box::pin(async move {
+								let this = Self(weak.upgrade().unwrap());
+								writeln!(this.reqnot().logger(), "Host received request {req:?}");
+								let i = this.ctx().i.clone();
+								match req {
+									api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
+									api::ExtHostReq::IntReq(intreq) => match intreq {
+										api::IntReq::InternStr(s) => hand.handle(&s, &i.i(&*s.0).await.to_api()).await,
+										api::IntReq::InternStrv(v) => {
+											let tokens = join_all(v.0.iter().map(|m| i.ex(*m))).await;
+											hand.handle(&v, &i.i(&tokens).await.to_api()).await
+										},
+										api::IntReq::ExternStr(si) =>
+											hand.handle(&si, &Tok::<String>::from_api(si.0, &i).await.rc()).await,
+										api::IntReq::ExternStrv(vi) => {
+											let markerv = (i.ex(vi.0).await.iter()).map(|t| t.to_api()).collect_vec();
+											hand.handle(&vi, &markerv).await
+										},
+									},
+									api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
+										let sys =
+											ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
+										let reply =
+											sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await;
+										hand.handle(fw, &reply).await
+									},
+									api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
+										let sys = ctx.system_inst(id).await.unwrap();
+										hand.handle(fw, &sys.request(body.clone()).await).await
+									},
+									api::ExtHostReq::SubLex(sl) => {
+										let (rep_in, mut rep_out) = channel(0);
+										{
+											let lex_g = this.0.lex_recur.lock().await;
+											let mut req_in =
+												lex_g.get(&sl.id).cloned().expect("Sublex for nonexistent lexid");
+											req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
+										}
+										hand.handle(&sl, &rep_out.next().await.unwrap()).await
+									},
+									api::ExtHostReq::ExprReq(api::ExprReq::Inspect(
+										ins @ api::Inspect { target },
+									)) => {
+										let expr = this.exprs().get_expr(target).expect("Invalid ticket");
+										hand
+											.handle(&ins, &api::Inspected {
+												refcount: expr.strong_count() as u32,
+												location: expr.pos().to_api(),
+												kind: expr.to_api().await,
+											})
+											.await
+									},
+									api::ExtHostReq::LsModule(ref ls @ api::LsModule(_sys, path)) => {
+										let reply: <api::LsModule as Request>::Response = 'reply: {
+											let path = i.ex(path).await;
+											let root = (ctx.root.read().await.upgrade())
+												.expect("LSModule called when root isn't in context");
+											let root_data = &*root.0.read().await;
+											let mut walk_ctx = (ctx.clone(), &root_data.consts);
+											let module =
+												match walk(&root_data.root, false, path.iter().cloned(), &mut walk_ctx)
+													.await
+												{
+													Ok(module) => module,
+													Err(ChildError { kind, .. }) =>
+														break 'reply Err(match kind {
+															ChildErrorKind::Private => panic!("Access checking was disabled"),
+															ChildErrorKind::Constant => api::LsModuleError::IsConstant,
+															ChildErrorKind::Missing => api::LsModuleError::InvalidPath,
+														}),
+												};
+											let mut members = std::collections::HashMap::new();
+											for (k, v) in &module.members {
+												let kind = match v.kind(ctx.clone(), &root_data.consts).await {
+													MemberKind::Const => api::MemberInfoKind::Constant,
+													MemberKind::Module(_) => api::MemberInfoKind::Module,
+												};
+												members.insert(k.to_api(), api::MemberInfo { public: v.public, kind });
+											}
+											Ok(api::ModuleInfo { members })
+										};
+										hand.handle(ls, &reply).await
+									},
+									api::ExtHostReq::ResolveNames(ref rn) => {
+										let api::ResolveNames { constid, names, sys } = rn;
+										let mut resolver = {
+											let systems = ctx.systems.read().await;
+											let weak_sys = systems.get(sys).expect("ResolveNames for invalid sys");
+											let sys = weak_sys.upgrade().expect("ResolveNames after sys drop");
+											sys.name_resolver(*constid).await
+										};
+										let responses = stream(async |mut cx| {
+											for name in names {
+												cx.emit(match resolver(&ctx.i.ex(*name).await[..]).await {
+													Ok(abs) => Ok(abs.to_sym(&ctx.i).await.to_api()),
+													Err(e) => Err(e.to_api()),
+												})
+												.await
+											}
+										})
+										.collect()
+										.await;
+										hand.handle(rn, &responses).await
+									},
+									api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
+										let atom = AtomHand::from_api(atom, Pos::None, &mut ctx.clone()).await;
+										let unit = atom.print(&FmtCtxImpl { i: &this.ctx().i }).await;
+										hand.handle(eap, &unit.to_api()).await
+									},
+								}
+							})
+						}
+					},
+				),
+			}
 		})))
 	}
-	pub(crate) fn reqnot(&self) -> &ReqNot<HostMsgSet> { &self.0.reqnot }
+	#[must_use]
+	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.reqnot }
+	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
+	#[must_use]
 	pub fn logger(&self) -> &Logger { &self.0.logger }
 	pub fn system_ctors(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
+	#[must_use]
 	pub fn exprs(&self) -> &ExprStore { &self.0.exprs }
+	#[must_use]
 	pub async fn is_own_sys(&self, id: api::SysId) -> bool {
 		let sys = self.ctx().system_inst(id).await.expect("invalid sender sys id");
 		Rc::ptr_eq(&self.0, &sys.ext().0)
@@ -177,6 +252,7 @@ impl Extension {
 	pub async fn assert_own_sys(&self, id: api::SysId) {
 		assert!(self.is_own_sys(id).await, "Incoming message impersonates separate system");
 	}
+	#[must_use]
 	pub fn next_pars(&self) -> NonZeroU64 {
 		let mut next_pars = self.0.next_pars.borrow_mut();
 		*next_pars = next_pars.checked_add(1).unwrap_or(NonZeroU64::new(1).unwrap());
@@ -184,7 +260,8 @@ impl Extension {
 	}
 	pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
-		source: Tok<String>,
+		source: IStr,
+		src: Sym,
 		pos: u32,
 		sys: api::SysId,
 		mut r: impl FnMut(u32) -> F,
@@ -192,18 +269,19 @@ impl Extension {
 		// get unique lex ID
 		let id = api::ParsId(self.next_pars());
 		// create and register channel
-		let (req_in, req_out) = channel::bounded(1);
+		let (req_in, mut req_out) = channel(0);
 		self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released
 		let (ret, ()) = join(
 			async {
-				let res =
+				let res = (self.reqnot())
-					(self.reqnot()).request(api::LexExpr { id, pos, sys, text: source.to_api() }).await;
+					.request(api::LexExpr { id, pos, sys, src: src.to_api(), text: source.to_api() })
+					.await;
 				// collect sender to unblock recursion handler branch before returning
 				self.0.lex_recur.lock().await.remove(&id);
 				res
 			},
 			async {
-				while let Ok(ReqPair(sublex, rep_in)) = req_out.recv().await {
+				while let Some(ReqPair(sublex, mut rep_in)) = req_out.next().await {
 					(rep_in.send(r(sublex.pos).await).await)
 						.expect("Response channel dropped while request pending")
 				}
@@ -212,20 +290,6 @@ impl Extension {
 		.await;
 		ret.transpose()
 	}
-	pub async fn recv_one(&self) {
-		let reqnot = self.0.reqnot.clone();
-		let ctx = self.ctx().clone();
-		(self.0.init.recv(Box::new(move |msg| {
-			Box::pin(async move {
-				let msg = msg.to_vec();
-				let reqnot = reqnot.clone();
-				(ctx.spawn)(Box::pin(async move {
-					reqnot.receive(&msg).await;
-				}))
-			})
-		})))
-		.await;
-	}
 	pub fn system_drop(&self, id: api::SysId) {
 		let rc = self.clone();
 		(self.ctx().spawn)(Box::pin(async move {
@@ -233,10 +297,13 @@ impl Extension {
 			rc.ctx().systems.write().await.remove(&id);
 		}))
 	}
+	#[must_use]
 	pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }
 }
 
+#[derive(Clone)]
 pub struct WeakExtension(Weak<ExtensionData>);
 impl WeakExtension {
+	#[must_use]
 	pub fn upgrade(&self) -> Option<Extension> { self.0.upgrade().map(Extension) }
 }

orchid-host/src/lex.rs

@@ -1,48 +1,49 @@
-use std::num::NonZeroU64;
+use std::rc::Rc;
-use std::sync::Arc;
 
-use async_std::sync::Mutex;
 use futures::FutureExt;
-use hashbrown::HashMap;
+use futures::lock::Mutex;
-use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use orchid_base::error::{OrcErrv, OrcRes, mk_err};
 use orchid_base::interner::Tok;
-use orchid_base::location::Pos;
+use orchid_base::location::SrcRange;
-use orchid_base::match_mapping;
 use orchid_base::name::Sym;
-use orchid_base::number::{num_to_err, parse_num};
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::PARENS;
-use orchid_base::tree::{AtomRepr, Ph};
 
 use crate::api;
-use crate::atom::AtomHand;
 use crate::ctx::Ctx;
+use crate::expr::{Expr, ExprParseCtx};
+use crate::parsed::{ParsTok, ParsTokTree, tt_to_api};
 use crate::system::System;
-use crate::tree::{ParsTok, ParsTokTree};
 
 pub struct LexCtx<'a> {
 	pub systems: &'a [System],
-	pub source: &'a Tok<String>,
+	pub source: &'a IStr,
+	pub path: &'a Sym,
 	pub tail: &'a str,
-	pub sub_trees: &'a mut HashMap<api::TreeTicket, ParsTokTree>,
+	pub sub_trees: &'a mut Vec<Expr>,
 	pub ctx: &'a Ctx,
 }
 impl<'a> LexCtx<'a> {
+	#[must_use]
 	pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
 	where 'a: 'b {
 		LexCtx {
 			source: self.source,
+			path: self.path,
 			tail: &self.source[pos as usize..],
 			systems: self.systems,
 			sub_trees: &mut *self.sub_trees,
 			ctx: self.ctx,
 		}
 	}
+	#[must_use]
 	pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 }
+	#[must_use]
 	pub fn end_pos(&self) -> u32 { self.source.len() as u32 }
 	pub fn set_pos(&mut self, pos: u32) { self.tail = &self.source[pos as usize..] }
 	pub fn push_pos(&mut self, delta: u32) { self.set_pos(self.get_pos() + delta) }
 	pub fn set_tail(&mut self, tail: &'a str) { self.tail = tail }
+	#[must_use]
 	pub fn strip_prefix(&mut self, tgt: &str) -> bool {
 		if let Some(src) = self.tail.strip_prefix(tgt) {
 			self.tail = src;
@@ -50,14 +51,22 @@ impl<'a> LexCtx<'a> {
 		}
 		false
 	}
-	pub fn add_subtree(&mut self, subtree: ParsTokTree) -> api::TreeTicket {
+	#[must_use]
-		let next_idx = api::TreeTicket(NonZeroU64::new(self.sub_trees.len() as u64 + 1).unwrap());
+	pub async fn ser_subtree(&mut self, subtree: ParsTokTree) -> api::TokenTree {
-		self.sub_trees.insert(next_idx, subtree);
+		tt_to_api(&mut self.ctx.common_exprs.clone(), subtree).await
-		next_idx
 	}
-	pub fn rm_subtree(&mut self, ticket: api::TreeTicket) -> ParsTokTree {
+	#[must_use]
-		self.sub_trees.remove(&ticket).unwrap()
+	pub async fn des_subtree(&mut self, tree: &api::TokenTree) -> ParsTokTree {
+		ParsTokTree::from_api(
+			tree,
+			&mut self.ctx.common_exprs.clone(),
+			&mut ExprParseCtx { ctx: self.ctx, exprs: &self.ctx.common_exprs },
+			self.path,
+			&self.ctx.i,
+		)
+		.await
 	}
+	#[must_use]
 	pub fn strip_char(&mut self, tgt: char) -> bool {
 		if let Some(src) = self.tail.strip_prefix(tgt) {
 			self.tail = src;
@@ -69,6 +78,7 @@ impl<'a> LexCtx<'a> {
 		self.tail = self.tail.trim_start_matches(filter);
 	}
 	pub fn trim_ws(&mut self) { self.trim(|c| c.is_whitespace() && !"\r\n".contains(c)) }
+	#[must_use]
 	pub fn get_start_matches(&mut self, filter: impl Fn(char) -> bool) -> &'a str {
 		let rest = self.tail.trim_start_matches(filter);
 		let matches = &self.tail[..self.tail.len() - rest.len()];
@@ -86,78 +96,60 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 	);
 	let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
 		ParsTok::BR
-	} else if ctx.strip_prefix("::") {
+	} else if let Some(tail) = (ctx.tail.starts_with(name_start).then_some(ctx.tail))
-		ParsTok::NS
+		.and_then(|t| t.trim_start_matches(name_char).strip_prefix("::"))
+	{
+		let name = &ctx.tail[..ctx.tail.len() - tail.len() - "::".len()];
+		ctx.set_tail(tail);
+		let body = lex_once(ctx).boxed_local().await?;
+		ParsTok::NS(ctx.ctx.i.i(name).await, Box::new(body))
 	} else if ctx.strip_prefix("--[") {
 		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
-			return Err(mk_errv(
+			return Err(mk_err(
 				ctx.ctx.i.i("Unterminated block comment").await,
 				"This block comment has no ending ]--",
-				[Pos::Range(start..start + 3).into()],
+				[SrcRange::new(start..start + 3, ctx.path)],
 			));
 		};
 		ctx.set_tail(tail);
-		ParsTok::Comment(Arc::new(cmt.to_string()))
+		ParsTok::Comment(Rc::new(cmt.to_string()))
 	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
 		ctx.push_pos(end as u32);
-		ParsTok::Comment(Arc::new(tail[2..end].to_string()))
+		ParsTok::Comment(Rc::new(tail[2..end].to_string()))
-	} else if ctx.strip_char('\\') {
+	} else if let Some(tail) = ctx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
-		let mut arg = Vec::new();
+		// fanciness like \$placeh in templates is resolved in the macro engine.
+		ctx.set_tail(tail);
+		let arg = lex_once(ctx).boxed_local().await?;
 		ctx.trim_ws();
-		while !ctx.strip_char('.') {
+		ParsTok::LambdaHead(Box::new(arg))
-			if ctx.tail.is_empty() {
-				return Err(mk_errv(
-					ctx.ctx.i.i("Unclosed lambda").await,
-					"Lambdae started with \\ should separate arguments from body with .",
-					[Pos::Range(start..start + 1).into()],
-				));
-			}
-			arg.push(lex_once(ctx).boxed_local().await?);
-			ctx.trim_ws();
-		}
-		ParsTok::LambdaHead(arg)
 	} else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
 		let mut body = Vec::new();
 		ctx.trim_ws();
 		while !ctx.strip_char(*rp) {
 			if ctx.tail.is_empty() {
-				return Err(mk_errv(
+				return Err(mk_err(
 					ctx.ctx.i.i("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
-					[Pos::Range(start..start + 1).into()],
+					[SrcRange::new(start..start + 1, ctx.path)],
 				));
 			}
 			body.push(lex_once(ctx).boxed_local().await?);
 			ctx.trim_ws();
 		}
 		ParsTok::S(*paren, body)
-	} else if ctx.strip_prefix("macro")
-		&& !ctx.tail.chars().next().is_some_and(|x| x.is_ascii_alphabetic())
-	{
-		ctx.strip_prefix("macro");
-		if ctx.strip_char('(') {
-			let pos = ctx.get_pos();
-			let numstr = ctx.get_start_matches(|x| x != ')').trim();
-			match parse_num(numstr) {
-				Ok(num) => ParsTok::Macro(Some(num.to_f64())),
-				Err(e) => return Err(num_to_err(e, pos, &ctx.ctx.i).await.into()),
-			}
-		} else {
-			ParsTok::Macro(None)
-		}
 	} else {
 		for sys in ctx.systems {
 			let mut errors = Vec::new();
 			if ctx.tail.starts_with(|c| sys.can_lex(c)) {
-				let (source, pos) = (ctx.source.clone(), ctx.get_pos());
+				let (source, pos, path) = (ctx.source.clone(), ctx.get_pos(), ctx.path.clone());
 				let ctx_lck = &Mutex::new(&mut *ctx);
 				let errors_lck = &Mutex::new(&mut errors);
 				let lx = sys
-					.lex(source, pos, |pos| async move {
+					.lex(source, path, pos, |pos| async move {
 						let mut ctx_g = ctx_lck.lock().await;
 						match lex_once(&mut ctx_g.push(pos)).boxed_local().await {
-							Ok(t) => Some(api::SubLexed { pos: t.range.end, ticket: ctx_g.add_subtree(t) }),
+							Ok(t) => Some(api::SubLexed { pos: t.sr.end(), tree: ctx_g.ser_subtree(t).await }),
 							Err(e) => {
 								errors_lck.lock().await.push(e);
 								None
@@ -172,7 +164,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 						),
 					Ok(Some(lexed)) => {
 						ctx.set_pos(lexed.pos);
-						return Ok(tt_to_owned(&lexed.expr, ctx).await);
+						return Ok(ctx.des_subtree(&lexed.expr).await);
 					},
 					Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
 						Some(errors) => return Err(errors),
@@ -186,50 +178,20 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 		} else if ctx.tail.starts_with(op_char) {
 			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(op_char)).await)
 		} else {
-			return Err(mk_errv(
+			return Err(mk_err(
 				ctx.ctx.i.i("Unrecognized character").await,
 				"The following syntax is meaningless.",
-				[Pos::Range(start..start + 1).into()],
+				[SrcRange::new(start..start + 1, ctx.path)],
 			));
 		}
 	};
-	Ok(ParsTokTree { tok, range: start..ctx.get_pos() })
+	Ok(ParsTokTree { tok, sr: SrcRange::new(start..ctx.get_pos(), ctx.path) })
 }
 
-async fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
+pub async fn lex(text: IStr, path: Sym, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
-	let tok = match_mapping!(&api.token, api::Token => ParsTok {
+	let mut sub_trees = Vec::new();
-		Atom(atom =>
+	let mut ctx =
-			AtomHand::from_api(atom, Pos::Range(api.range.clone()), &mut ctx.ctx.clone()).await
+		LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, path: &path, ctx };
-		),
-		Bottom(err => OrcErrv::from_api(err, &ctx.ctx.i).await),
-		LambdaHead(arg => ttv_to_owned(arg, ctx).boxed_local().await),
-		Name(name => Tok::from_api(*name, &ctx.ctx.i).await),
-		Reference(tstrv => Sym::from_api(*tstrv, &ctx.ctx.i).await),
-		S(p.clone(), b => ttv_to_owned(b, ctx).boxed_local().await),
-		BR, NS,
-		Comment(c.clone()),
-		Ph(ph => Ph::from_api(ph, &ctx.ctx.i).await),
-		Macro(*prio),
-	} {
-		api::Token::Slot(id) => return ctx.rm_subtree(*id),
-	});
-	ParsTokTree { range: api.range.clone(), tok }
-}
-
-async fn ttv_to_owned<'a>(
-	api: impl IntoIterator<Item = &'a api::TokenTree>,
-	ctx: &mut LexCtx<'_>,
-) -> Vec<ParsTokTree> {
-	let mut out = Vec::new();
-	for tt in api {
-		out.push(tt_to_owned(tt, ctx).await)
-	}
-	out
-}
-
-pub async fn lex(text: Tok<String>, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
-	let mut sub_trees = HashMap::new();
-	let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, ctx };
 	let mut tokv = Vec::new();
 	ctx.trim(unrep_space);
 	while !ctx.tail.is_empty() {

orchid-host/src/lib.rs

@@ -8,9 +8,9 @@ pub mod expr;
 pub mod expr_store;
 pub mod extension;
 pub mod lex;
-pub mod macros;
 pub mod parse;
-pub mod rule;
+pub mod parsed;
 pub mod subprocess;
+mod sys_parser;
 pub mod system;
 pub mod tree;

orchid-host/src/macros.rs

@@ -1,103 +0,0 @@
-use std::rc::Rc;
-
-use futures::FutureExt;
-use hashbrown::{HashMap, HashSet};
-use itertools::Itertools;
-use orchid_base::clone;
-use orchid_base::macros::{MTok, MTree, mtreev_from_api, mtreev_to_api};
-use orchid_base::name::Sym;
-use trait_set::trait_set;
-
-use crate::api;
-use crate::atom::AtomHand;
-use crate::ctx::Ctx;
-use crate::rule::state::MatchState;
-use crate::tree::Code;
-
-pub type MacTok = MTok<'static, AtomHand>;
-pub type MacTree = MTree<'static, AtomHand>;
-
-trait_set! {
-	trait MacroCB = Fn(Vec<MacTree>) -> Option<Vec<MacTree>>;
-}
-
-type Slots = HashMap<api::MacroTreeId, Rc<MacTok>>;
-
-pub async fn macro_treev_to_api(mtree: Vec<MacTree>, slots: &mut Slots) -> Vec<api::MacroTree> {
-	mtreev_to_api(&mtree, &mut |a: &AtomHand| {
-		let id = api::MacroTreeId((slots.len() as u64 + 1).try_into().unwrap());
-		slots.insert(id, Rc::new(MacTok::Atom(a.clone())));
-		async move { api::MacroToken::Slot(id) }.boxed_local()
-	})
-	.await
-}
-
-pub async fn macro_treev_from_api(api: Vec<api::MacroTree>, ctx: Ctx) -> Vec<MacTree> {
-	mtreev_from_api(&api, &ctx.clone().i, &mut move |atom| {
-		clone!(ctx);
-		Box::pin(async move { MacTok::Atom(AtomHand::new(atom.clone(), &ctx).await) })
-	})
-	.await
-}
-
-pub fn deslot_macro(tree: &[MacTree], slots: &mut Slots) -> Option<Vec<MacTree>> {
-	return work(slots, tree);
-	fn work(
-		slots: &mut HashMap<api::MacroTreeId, Rc<MacTok>>,
-		tree: &[MacTree],
-	) -> Option<Vec<MacTree>> {
-		let items = (tree.iter())
-			.map(|t| {
-				Some(MacTree {
-					tok: match &*t.tok {
-						MacTok::Atom(_) | MacTok::Name(_) | MacTok::Ph(_) => return None,
-						MacTok::Ref(_) => panic!("Ref is an extension-local optimization"),
-						MacTok::Done(_) => panic!("Created and removed by matcher"),
-						MacTok::Slot(slot) => slots.get(&slot.id()).expect("Slot not found").clone(),
-						MacTok::S(paren, b) => Rc::new(MacTok::S(*paren, work(slots, b)?)),
-						MacTok::Lambda(a, b) => Rc::new(match (work(slots, a), work(slots, b)) {
-							(None, None) => return None,
-							(Some(a), None) => MacTok::Lambda(a, b.clone()),
-							(None, Some(b)) => MacTok::Lambda(a.clone(), b),
-							(Some(a), Some(b)) => MacTok::Lambda(a, b),
-						}),
-					},
-					pos: t.pos.clone(),
-				})
-			})
-			.collect_vec();
-		let any_changed = items.iter().any(Option::is_some);
-		any_changed.then(|| {
-			(items.into_iter().enumerate())
-				.map(|(i, opt)| opt.unwrap_or_else(|| tree[i].clone()))
-				.collect_vec()
-		})
-	}
-}
-
-pub struct Macro<Matcher> {
-	deps: HashSet<Sym>,
-	cases: Vec<(Matcher, Code)>,
-}
-
-fn fill_lexicon(tgt: &MacTree, lexicon: &mut HashSet<Sym>) {
-	match &*tgt.tok {
-		MTok::Name(n) => {
-			lexicon.insert(n.clone());
-		},
-		MTok::Lambda(arg, body) => {
-			arg.iter().for_each(|t| fill_lexicon(t, lexicon));
-			body.iter().for_each(|t| fill_lexicon(t, lexicon))
-		},
-		MTok::S(_, body) => body.iter().for_each(|t| fill_lexicon(t, lexicon)),
-		_ => (),
-	}
-}
-
-fn run_body(body: &Code, mut state: MatchState<'_>) -> Vec<MacTree> {
-	let inject: Vec<MacTree> = todo!("Call the interpreter with bindings");
-	inject
-		.into_iter()
-		.map(|MTree { pos, tok }| MTree { pos, tok: Rc::new(MTok::Done(tok)) })
-		.collect_vec()
-}

orchid-host/src/parse.rs

@@ -1,317 +1,158 @@
-use std::rc::Rc;
-
-use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
-use never::Never;
+use orchid_base::error::{OrcRes, Reporter, mk_err};
-use orchid_base::error::{OrcErrv, OrcRes, Reporter, ReporterImpl, mk_err, mk_errv};
+use orchid_base::format::fmt;
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
-use orchid_base::location::Pos;
-use orchid_base::macros::{MTok, MTree};
 use orchid_base::name::Sym;
 use orchid_base::parse::{
-	Comment, Import, Parsed, Snippet, expect_end, line_items, parse_multiname, try_pop_no_fluff,
+	Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname,
+	try_pop_no_fluff,
 };
 use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
 
-use crate::atom::AtomHand;
+use crate::ctx::Ctx;
-use crate::macros::MacTree;
+use crate::expr::Expr;
+use crate::parsed::{Item, ItemKind, ParsedMember, ParsedMemberKind, ParsedModule};
 use crate::system::System;
-use crate::tree::{Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, Rule, RuleKind};
 
-type ParsSnippet<'a> = Snippet<'a, 'static, AtomHand, Never>;
+type ParsSnippet<'a> = Snippet<'a, Expr, Expr>;
 
-pub struct ParseCtxImpl<'a> {
+pub struct HostParseCtxImpl<'a> {
+	pub ctx: Ctx,
+	pub src: Sym,
 	pub systems: &'a [System],
-	pub reporter: &'a ReporterImpl,
+	pub rep: &'a Reporter,
 }
 
-impl ParseCtx for ParseCtxImpl<'_> {
+impl ParseCtx for HostParseCtxImpl<'_> {
-	fn reporter(&self) -> &(impl Reporter + ?Sized) { self.reporter }
+	fn rep(&self) -> &Reporter { self.rep }
+	fn i(&self) -> &Interner { &self.ctx.i }
+}
+
+impl HostParseCtx for HostParseCtxImpl<'_> {
+	fn ctx(&self) -> &Ctx { &self.ctx }
 	fn systems(&self) -> impl Iterator<Item = &System> { self.systems.iter() }
+	fn src_path(&self) -> Sym { self.src.clone() }
 }
 
-pub trait ParseCtx {
+pub trait HostParseCtx: ParseCtx {
+	#[must_use]
+	fn ctx(&self) -> &Ctx;
+	#[must_use]
 	fn systems(&self) -> impl Iterator<Item = &System>;
-	fn reporter(&self) -> &(impl Reporter + ?Sized);
+	#[must_use]
+	fn src_path(&self) -> Sym;
 }
 
 pub async fn parse_items(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	items: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	let lines = line_items(items).await;
+	let lines = line_items(ctx, items).await;
 	let line_res =
 		join_all(lines.into_iter().map(|p| parse_item(ctx, path.clone(), p.output, p.tail))).await;
 	Ok(line_res.into_iter().flat_map(|l| l.ok().into_iter().flatten()).collect())
 }
 
 pub async fn parse_item(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	comments: Vec<Comment>,
 	item: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
 	match item.pop_front() {
 		Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
-			n if *n == item.i().i("export").await => match try_pop_no_fluff(postdisc).await? {
+			n if *n == ctx.i().i("export").await => match try_pop_no_fluff(ctx, postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
 					parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
-				Parsed { output: TokTree { tok: Token::NS, .. }, tail } => {
+				Parsed { output, tail: _ } => Err(mk_err(
-					let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail).await?;
+					ctx.i().i("Malformed export").await,
-					let mut ok = Vec::new();
+					"`export` can either prefix other lines or list names inside ( )",
-					for (e, pos) in exports {
+					[output.sr()],
-						match (&e.path[..], e.name) {
-							([], Some(n)) =>
-								ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }),
-							(_, Some(_)) => ctx.reporter().report(mk_err(
-								tail.i().i("Compound export").await,
-								"Cannot export compound names (names containing the :: separator)",
-								[pos.into()],
-							)),
-							(_, None) => ctx.reporter().report(mk_err(
-								tail.i().i("Wildcard export").await,
-								"Exports cannot contain the globstar *",
-								[pos.into()],
-							)),
-						}
-					}
-					expect_end(tail).await?;
-					Ok(ok)
-				},
-				Parsed { output, tail } => Err(mk_errv(
-					tail.i().i("Malformed export").await,
-					"`export` can either prefix other lines or list names inside ::( ) or ::[ ]",
-					[Pos::Range(output.range.clone()).into()],
 				)),
 			},
-			n if *n == item.i().i("import").await => parse_import(ctx, postdisc).await.map(|v| {
+			n if *n == ctx.i().i("import").await => {
-				Vec::from_iter(v.into_iter().map(|(t, pos)| Item {
+				let imports = parse_import(ctx, postdisc).await?;
+				Ok(Vec::from_iter(imports.into_iter().map(|t| Item {
 					comments: comments.clone(),
-					pos,
+					sr: t.sr.clone(),
 					kind: ItemKind::Import(t),
-				}))
+				})))
-			}),
+			},
 			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
 		},
-		Some(_) => Err(mk_errv(
+		Some(_) =>
-			item.i().i("Expected a line type").await,
+			Err(mk_err(ctx.i().i("Expected a line type").await, "All lines must begin with a keyword", [
-			"All lines must begin with a keyword",
+				item.sr(),
-			[Pos::Range(item.pos()).into()],
+			])),
-		)),
 		None => unreachable!("These lines are filtered and aggregated in earlier stages"),
 	}
 }
 
-pub async fn parse_import(
+pub async fn parse_import<'a>(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
-	tail: ParsSnippet<'_>,
+	tail: ParsSnippet<'a>,
-) -> OrcRes<Vec<(Import, Pos)>> {
+) -> OrcRes<Vec<Import>> {
-	let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail).await?;
+	let Parsed { output: imports, tail } = parse_multiname(ctx, tail).await?;
-	expect_end(tail).await?;
+	expect_end(ctx, tail).await?;
 	Ok(imports)
 }
 
-pub async fn parse_exportable_item(
+pub async fn parse_exportable_item<'a>(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	comments: Vec<Comment>,
 	exported: bool,
-	discr: Tok<String>,
+	discr: IStr,
-	tail: ParsSnippet<'_>,
+	tail: ParsSnippet<'a>,
 ) -> OrcRes<Vec<Item>> {
-	let kind = if discr == tail.i().i("mod").await {
+	let kind = if discr == ctx.i().i("mod").await {
 		let (name, body) = parse_module(ctx, path, tail).await?;
-		ItemKind::Member(Member::new(name, MemberKind::Mod(body)))
+		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::Mod(body) })
-	} else if discr == tail.i().i("const").await {
+	} else if let Some(parser) = ctx.systems().find_map(|s| s.get_parser(discr.clone())) {
-		let (name, val) = parse_const(tail, path.clone()).await?;
+		return parser
-		let locator = CodeLocator::to_const(tail.i().i(&path.push(name.clone()).unreverse()).await);
+			.parse(ctx, path, tail.to_vec(), exported, comments, &mut async |stack, lines| {
-		ItemKind::Member(Member::new(name, MemberKind::Const(Code::from_code(locator, val))))
+				let source = Snippet::new(lines.first().unwrap(), &lines);
-	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
+				parse_items(ctx, stack, source).await
-		let line = sys.parse(tail.to_vec(), exported, comments).await?;
+			})
-		return parse_items(ctx, path, Snippet::new(tail.prev(), &line, tail.i())).await;
+			.await;
 	} else {
 		let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
-		return Err(mk_errv(
+		return Err(mk_err(
-			tail.i().i("Unrecognized line type").await,
+			ctx.i().i("Unrecognized line type").await,
-			format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
+			format!("Line types are: mod, {ext_lines}"),
-			[Pos::Range(tail.prev().range.clone()).into()],
+			[tail.prev().sr()],
 		));
 	};
-	Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }])
+	Ok(vec![Item { comments, sr: tail.sr(), kind }])
 }
 
-pub async fn parse_module(
+pub async fn parse_module<'a>(
-	ctx: &impl ParseCtx,
+	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
-	tail: ParsSnippet<'_>,
+	tail: ParsSnippet<'a>,
-) -> OrcRes<(Tok<String>, Module)> {
+) -> OrcRes<(IStr, ParsedModule)> {
-	let (name, tail) = match try_pop_no_fluff(tail).await? {
+	let (name, tail) = match try_pop_no_fluff(ctx, tail).await? {
 		Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
 		Parsed { output, .. } => {
-			return Err(mk_errv(
+			return Err(mk_err(
-				tail.i().i("Missing module name").await,
+				ctx.i().i("Missing module name").await,
-				format!("A name was expected, {} was found", tail.fmt(output).await),
+				format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
-				[Pos::Range(output.range.clone()).into()],
+				[output.sr()],
 			));
 		},
 	};
-	let Parsed { output, tail: surplus } = try_pop_no_fluff(tail).await?;
+	let Parsed { output, tail: surplus } = try_pop_no_fluff(ctx, tail).await?;
-	expect_end(surplus).await?;
+	expect_end(ctx, surplus).await?;
-	let Some(body) = output.as_s(Paren::Round, tail.i()) else {
+	let Some(body) = output.as_s(Paren::Round) else {
-		return Err(mk_errv(
+		return Err(mk_err(
-			tail.i().i("Expected module body").await,
+			ctx.i().i("Expected module body").await,
-			format!("A ( block ) was expected, {} was found", tail.fmt(output).await),
+			format!("A ( block ) was expected, {} was found", fmt(output, ctx.i()).await),
-			[Pos::Range(output.range.clone()).into()],
+			[output.sr()],
 		));
 	};
 	let path = path.push(name.clone());
-	Ok((name, Module::new(parse_items(ctx, path, body).await?)))
+	Ok((name, ParsedModule::new(true, parse_items(ctx, path, body).await?)))
-}
-
-pub async fn parse_const(
-	tail: ParsSnippet<'_>,
-	path: Substack<'_, Tok<String>>,
-) -> OrcRes<(Tok<String>, Vec<MacTree>)> {
-	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
-	let Some(name) = output.as_name() else {
-		return Err(mk_errv(
-			tail.i().i("Missing module name").await,
-			format!("A name was expected, {} was found", tail.fmt(output).await),
-			[Pos::Range(output.range.clone()).into()],
-		));
-	};
-	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
-	if !output.is_kw(tail.i().i("=").await) {
-		return Err(mk_errv(
-			tail.i().i("Missing = separator").await,
-			format!("Expected = , found {}", tail.fmt(output).await),
-			[Pos::Range(output.range.clone()).into()],
-		));
-	}
-	try_pop_no_fluff(tail).await?;
-	Ok((name, parse_mtree(tail, path).await?))
-}
-
-pub async fn parse_mtree(
-	mut snip: ParsSnippet<'_>,
-	path: Substack<'_, Tok<String>>,
-) -> OrcRes<Vec<MacTree>> {
-	let mut mtreev = Vec::new();
-	while let Some((ttree, tail)) = snip.pop_front() {
-		snip = tail;
-		let (range, tok, tail) = match &ttree.tok {
-			Token::S(p, b) => {
-				let b = parse_mtree(Snippet::new(ttree, b, snip.i()), path.clone()).boxed_local().await?;
-				(ttree.range.clone(), MTok::S(*p, b), tail)
-			},
-			Token::Reference(name) => (ttree.range.clone(), MTok::Name(name.clone()), tail),
-			Token::Name(tok) => {
-				let mut segments = path.unreverse();
-				segments.push(tok.clone());
-				let mut end = ttree.range.end;
-				while let Some((TokTree { tok: Token::NS, .. }, tail)) = snip.pop_front() {
-					let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
-					let Some(seg) = output.as_name() else {
-						return Err(mk_errv(
-							tail.i().i("Namespaced name interrupted").await,
-							"In expression context, :: must always be followed by a name.\n\
-											::() is permitted only in import and export items",
-							[Pos::Range(output.range.clone()).into()],
-						));
-					};
-					segments.push(seg);
-					snip = tail;
-					end = output.range.end;
-				}
-				(ttree.range.start..end, MTok::Name(Sym::new(segments, snip.i()).await.unwrap()), snip)
-			},
-			Token::NS => {
-				return Err(mk_errv(
-					tail.i().i("Unexpected :: in expression").await,
-					":: can only follow a name",
-					[Pos::Range(ttree.range.clone()).into()],
-				));
-			},
-			Token::Ph(ph) => (ttree.range.clone(), MTok::Ph(ph.clone()), tail),
-			Token::Macro(_) => {
-				return Err(mk_errv(
-					tail.i().i("Invalid keyword in expression").await,
-					"Expressions cannot use `macro` as a name.",
-					[Pos::Range(ttree.range.clone()).into()],
-				));
-			},
-			Token::Atom(a) => (ttree.range.clone(), MTok::Atom(a.clone()), tail),
-			Token::BR | Token::Comment(_) => continue,
-			Token::Bottom(e) => return Err(e.clone()),
-			Token::LambdaHead(arg) => (
-				ttree.range.start..snip.pos().end,
-				MTok::Lambda(
-					parse_mtree(Snippet::new(ttree, arg, snip.i()), path.clone()).boxed_local().await?,
-					parse_mtree(tail, path.clone()).boxed_local().await?,
-				),
-				Snippet::new(ttree, &[], snip.i()),
-			),
-			Token::Slot(_) | Token::X(_) =>
-				panic!("Did not expect {} in parsed token tree", tail.fmt(ttree).await),
-		};
-		mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Rc::new(tok) });
-		snip = tail;
-	}
-	Ok(mtreev)
-}
-
-pub async fn parse_macro(
-	tail: ParsSnippet<'_>,
-	macro_i: u16,
-	path: Substack<'_, Tok<String>>,
-) -> OrcRes<Vec<Rule>> {
-	let (surplus, prev, block) = match try_pop_no_fluff(tail).await? {
-		Parsed { tail, output: o @ TokTree { tok: Token::S(Paren::Round, b), .. } } => (tail, o, b),
-		Parsed { output, .. } => {
-			return Err(mk_errv(
-				tail.i().i("m").await,
-				"Macro blocks must either start with a block or a ..$:number",
-				[Pos::Range(output.range.clone()).into()],
-			));
-		},
-	};
-	expect_end(surplus).await?;
-	let mut errors = Vec::new();
-	let mut rules = Vec::new();
-	for (i, item) in line_items(Snippet::new(prev, block, tail.i())).await.into_iter().enumerate() {
-		let Parsed { tail, output } = try_pop_no_fluff(item.tail).await?;
-		if !output.is_kw(tail.i().i("rule").await) {
-			errors.extend(mk_errv(
-				tail.i().i("non-rule in macro").await,
-				format!("Expected `rule`, got {}", tail.fmt(output).await),
-				[Pos::Range(output.range.clone()).into()],
-			));
-			continue;
-		};
-		let arrow = tail.i().i("=>").await;
-		let (pat, body) = match tail.split_once(|t| t.is_kw(arrow.clone())) {
-			Some((a, b)) => (a, b),
-			None => {
-				errors.extend(mk_errv(
-					tail.i().i("no => in macro rule").await,
-					"The pattern and body of a rule must be separated by a =>",
-					[Pos::Range(tail.pos()).into()],
-				));
-				continue;
-			},
-		};
-		rules.push(Rule {
-			comments: item.output,
-			pos: Pos::Range(tail.pos()),
-			pattern: parse_mtree(pat, path.clone()).await?,
-			kind: RuleKind::Native(Code::from_code(
-				CodeLocator::to_rule(tail.i().i(&path.unreverse()).await, macro_i, i as u16),
-				parse_mtree(body, path.clone()).await?,
-			)),
-		})
-	}
-	if let Ok(e) = OrcErrv::new(errors) { Err(e) } else { Ok(rules) }
 }

orchid-host/src/parsed.rs

@@ -0,0 +1,216 @@
+use std::fmt::{self, Debug};
+use std::rc::Rc;
+
+use futures::FutureExt;
+use futures::future::{LocalBoxFuture, join_all};
+use hashbrown::HashSet;
+use itertools::Itertools;
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
+use orchid_base::interner::Tok;
+use orchid_base::location::SrcRange;
+use orchid_base::parse::{Comment, Import};
+use orchid_base::tl_cache;
+use orchid_base::tree::{TokTree, Token, recur};
+
+use crate::api;
+use crate::dealias::{ChildErrorKind, ChildResult, Tree};
+use crate::expr::{Expr, ExprWillPanic};
+use crate::expr_store::ExprStore;
+use crate::system::System;
+
+pub type ParsTokTree = TokTree<Expr, Expr>;
+pub type ParsTok = Token<Expr, Expr>;
+
+#[derive(Debug)]
+pub struct Item {
+	pub sr: SrcRange,
+	pub comments: Vec<Comment>,
+	pub kind: ItemKind,
+}
+impl Item {
+	pub fn new(sr: SrcRange, kind: impl Into<ItemKind>) -> Self {
+		Self { sr, comments: Vec::new(), kind: kind.into() }
+	}
+}
+
+#[derive(Debug)]
+pub enum ItemKind {
+	Member(ParsedMember),
+	Import(Import),
+}
+impl ItemKind {
+	#[must_use]
+	pub fn at(self, sr: SrcRange) -> Item { Item { comments: vec![], sr, kind: self } }
+}
+impl From<ParsedMember> for ItemKind {
+	fn from(value: ParsedMember) -> Self { Self::Member(value) }
+}
+impl From<Import> for ItemKind {
+	fn from(value: Import) -> Self { Self::Import(value) }
+}
+
+impl Format for Item {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		let comment_text = self.comments.iter().join("\n");
+		let item_text = match &self.kind {
+			ItemKind::Import(i) => format!("import {i}").into(),
+			ItemKind::Member(mem) => match &mem.kind {
+				ParsedMemberKind::Const(_, sys) =>
+					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} via {1}")))
+						.units([mem.name.rc().into(), sys.print(c).await]),
+				ParsedMemberKind::Mod(module) =>
+					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
+						.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
+			},
+		};
+		tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}\n{1}")))
+			.units([comment_text.into(), item_text])
+	}
+}
+
+pub struct ParsedMember {
+	pub name: IStr,
+	pub exported: bool,
+	pub kind: ParsedMemberKind,
+}
+impl ParsedMember {
+	#[must_use]
+	pub fn name(&self) -> IStr { self.name.clone() }
+	pub fn new(exported: bool, name: IStr, kind: impl Into<ParsedMemberKind>) -> Self {
+		Self { exported, name, kind: kind.into() }
+	}
+}
+impl Debug for ParsedMember {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Member")
+			.field("name", &self.name)
+			.field("kind", &self.kind)
+			.finish_non_exhaustive()
+	}
+}
+
+pub(crate) type ParsedExprCallback =
+	Rc<dyn for<'a> Fn(&'a [IStr]) -> LocalBoxFuture<'a, Expr>>;
+
+pub struct ParsedExpr {
+	pub(crate) debug: String,
+	pub(crate) callback: ParsedExprCallback,
+}
+impl ParsedExpr {
+	pub async fn run(self, imported_names: &[IStr]) -> Expr {
+		(self.callback)(imported_names).await
+	}
+}
+impl fmt::Debug for ParsedExpr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.debug) }
+}
+
+#[derive(Debug)]
+pub enum ParsedMemberKind {
+	Const(api::ParsedConstId, System),
+	Mod(ParsedModule),
+}
+impl From<ParsedModule> for ParsedMemberKind {
+	fn from(value: ParsedModule) -> Self { Self::Mod(value) }
+}
+
+#[derive(Debug, Default)]
+pub struct ParsedModule {
+	pub exports: Vec<IStr>,
+	pub items: Vec<Item>,
+	pub use_prelude: bool,
+}
+impl ParsedModule {
+	#[must_use]
+	pub fn new(use_prelude: bool, items: impl IntoIterator<Item = Item>) -> Self {
+		let items = items.into_iter().collect_vec();
+		let exports = (items.iter())
+			.filter_map(|i| if let ItemKind::Member(m) = &i.kind { Some(m) } else { None })
+			.filter(|m| m.exported)
+			.map(|m| m.name.clone())
+			.collect_vec();
+		Self { exports, items, use_prelude }
+	}
+	pub fn merge(&mut self, other: ParsedModule) {
+		let mut swap = ParsedModule::default();
+		std::mem::swap(self, &mut swap);
+		assert_eq!(self.use_prelude, other.use_prelude, "merging modules that disagree on prelude");
+		*self = ParsedModule::new(self.use_prelude, swap.items.into_iter().chain(other.items))
+	}
+	#[must_use]
+	pub fn get_imports(&self) -> impl IntoIterator<Item = &Import> {
+		(self.items.iter())
+			.filter_map(|it| if let ItemKind::Import(i) = &it.kind { Some(i) } else { None })
+	}
+	pub fn default_item(self, name: IStr, sr: SrcRange) -> Item {
+		let mem = ParsedMember { exported: true, name, kind: ParsedMemberKind::Mod(self) };
+		Item { comments: vec![], sr, kind: ItemKind::Member(mem) }
+	}
+}
+impl Tree for ParsedModule {
+	type Ctx<'a> = ();
+	async fn child(
+		&self,
+		key: IStr,
+		public_only: bool,
+		(): &mut Self::Ctx<'_>,
+	) -> ChildResult<'_, Self> {
+		if public_only && !self.exports.contains(&key) {
+			return ChildResult::Err(ChildErrorKind::Private);
+		}
+		if let Some(member) = (self.items.iter())
+			.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
+			.find(|m| m.name == key)
+		{
+			match &member.kind {
+				ParsedMemberKind::Const(..) => return ChildResult::Err(ChildErrorKind::Constant),
+				ParsedMemberKind::Mod(m) => return ChildResult::Ok(m),
+			}
+		}
+		ChildResult::Err(ChildErrorKind::Missing)
+	}
+	fn children(&self, public_only: bool) -> HashSet<IStr> {
+		let mut public: HashSet<_> = self.exports.iter().cloned().collect();
+		if !public_only {
+			public.extend(
+				(self.items.iter())
+					.filter_map(
+						|it| if let ItemKind::Member(mem) = &it.kind { Some(&mem.name) } else { None },
+					)
+					.cloned(),
+			)
+		}
+		public
+	}
+}
+impl Format for ParsedModule {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		let head_str = format!("export ::({})\n", self.exports.iter().join(", "));
+		Variants::sequence(self.items.len() + 1, "\n", None).units(
+			[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
+		)
+	}
+}
+
+/// Selects a code element
+///
+/// Either the steps point to a constant and rule_loc is None, or the steps
+/// point to a module and rule_loc selects a macro rule within that module
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct ConstPath {
+	steps: Tok<Vec<IStr>>,
+}
+impl ConstPath {
+	#[must_use]
+	pub fn to_const(steps: Tok<Vec<IStr>>) -> Self { Self { steps } }
+}
+
+pub async fn tt_to_api(exprs: &mut ExprStore, subtree: ParsTokTree) -> api::TokenTree {
+	let without_new_expr = recur(subtree, &|tt, r| {
+		if let ParsTok::NewExpr(expr) = tt.tok {
+			return ParsTok::Handle(expr).at(tt.sr);
+		}
+		r(tt)
+	});
+	without_new_expr.into_api(exprs, &mut ExprWillPanic).await
+}

orchid-host/src/subprocess.rs

@@ -1,13 +1,12 @@
 use std::cell::RefCell;
-use std::path::PathBuf;
+use std::io::{self, Write};
 use std::pin::Pin;
-use std::thread;
 
 use async_process::{self, Child, ChildStdin, ChildStdout};
-use async_std::io::{self, BufReadExt, BufReader};
-use async_std::sync::Mutex;
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
+use futures::io::BufReader;
+use futures::lock::Mutex;
+use futures::{self, AsyncBufReadExt, AsyncWriteExt};
 use orchid_api_traits::{Decode, Encode};
 use orchid_base::builtin::{ExtInit, ExtPort};
 use orchid_base::logging::Logger;
@@ -22,8 +21,6 @@ pub async fn ext_command(
 	msg_logs: Logger,
 	ctx: Ctx,
 ) -> io::Result<ExtInit> {
-	let prog_pbuf = PathBuf::from(cmd.get_program());
-	let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy().to_string();
 	let mut child = async_process::Command::from(cmd)
 		.stdin(async_process::Stdio::piped())
 		.stdout(async_process::Stdio::piped())
@@ -36,57 +33,68 @@ pub async fn ext_command(
 	let mut stdout = child.stdout.take().unwrap();
 	let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await;
 	let child_stderr = child.stderr.take().unwrap();
-	thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || {
+	(ctx.spawn)(Box::pin(async move {
-		async_std::task::block_on(async move {
+		let mut reader = BufReader::new(child_stderr);
-			let mut reader = BufReader::new(child_stderr);
+		loop {
-			loop {
+			let mut buf = String::new();
-				let mut buf = String::new();
+			if 0 == reader.read_line(&mut buf).await.unwrap() {
-				if 0 == reader.read_line(&mut buf).await.unwrap() {
+				break;
-					break;
-				}
-				logger.log(buf.strip_suffix('\n').expect("Readline implies this"));
 			}
-		})
+			logger.log(buf.strip_suffix('\n').expect("Readline implies this"));
-	})?;
+		}
+	}));
 	Ok(ExtInit {
-		header,
 		port: Box::new(Subprocess {
+			name: header.name.clone(),
 			child: RefCell::new(Some(child)),
-			stdin: Mutex::new(Box::pin(stdin)),
+			stdin: Some(Mutex::new(Box::pin(stdin))),
 			stdout: Mutex::new(Box::pin(stdout)),
 			ctx,
 		}),
+		header,
 	})
 }
 
 pub struct Subprocess {
+	name: String,
 	child: RefCell<Option<Child>>,
-	stdin: Mutex<Pin<Box<ChildStdin>>>,
+	stdin: Option<Mutex<Pin<Box<ChildStdin>>>>,
 	stdout: Mutex<Pin<Box<ChildStdout>>>,
 	ctx: Ctx,
 }
 impl Drop for Subprocess {
 	fn drop(&mut self) {
 		let mut child = self.child.borrow_mut().take().unwrap();
+		let name = self.name.clone();
+		if std::thread::panicking() {
+			eprintln!("Killing extension {name}");
+			// we don't really care to handle errors here
+			let _: Result<_, _> = std::io::stderr().flush();
+			let _: Result<_, _> = child.kill();
+			return;
+		}
+		let stdin = self.stdin.take().unwrap();
 		(self.ctx.spawn)(Box::pin(async move {
-			let status = child.status().await.expect("Extension exited with error");
+			stdin.lock().await.close().await.unwrap();
-			assert!(status.success(), "Extension exited with error {status}");
+			let status = (child.status().await)
+				.unwrap_or_else(|e| panic!("{e}, extension {name} exited with error"));
+			assert!(status.success(), "Extension {name} exited with error {status}");
 		}))
 	}
 }
 impl ExtPort for Subprocess {
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		async { send_msg(Pin::new(&mut *self.stdin.lock().await), msg).await.unwrap() }.boxed_local()
+		Box::pin(async {
+			send_msg(Pin::new(&mut *self.stdin.as_ref().unwrap().lock().await), msg).await.unwrap()
+		})
 	}
-	fn recv<'a>(
+	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
-		&'a self,
-		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
-	) -> LocalBoxFuture<'a, ()> {
 		Box::pin(async {
 			std::io::Write::flush(&mut std::io::stderr()).unwrap();
 			match recv_msg(self.stdout.lock().await.as_mut()).await {
-				Ok(msg) => cb(&msg).await,
+				Ok(msg) => Some(msg),
-				Err(e) if e.kind() == io::ErrorKind::BrokenPipe => (),
+				Err(e) if e.kind() == io::ErrorKind::BrokenPipe => None,
+				Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => None,
 				Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
 			}
 		})

orchid-host/src/sys_parser.rs

@@ -0,0 +1,119 @@
+use futures::FutureExt;
+use futures::future::join_all;
+use itertools::Itertools;
+use orchid_base::error::{OrcErrv, OrcRes};
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::SrcRange;
+use orchid_base::name::Sym;
+use orchid_base::parse::Comment;
+use orchid_base::reqnot::Requester;
+use orchid_base::tree::ttv_from_api;
+use substack::Substack;
+
+use crate::api;
+use crate::expr::ExprParseCtx;
+use crate::expr_store::ExprStore;
+use crate::parse::HostParseCtx;
+use crate::parsed::{
+	Item, ItemKind, ParsTokTree, ParsedMember, ParsedMemberKind, ParsedModule, tt_to_api,
+};
+use crate::system::System;
+
+pub struct Parser {
+	pub(crate) system: System,
+	pub(crate) idx: u16,
+}
+type ModPath<'a> = Substack<'a, IStr>;
+
+impl Parser {
+	pub async fn parse(
+		&self,
+		ctx: &impl HostParseCtx,
+		path: ModPath<'_>,
+		line: Vec<ParsTokTree>,
+		exported: bool,
+		comments: Vec<Comment>,
+		callback: &mut impl AsyncFnMut(ModPath<'_>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
+	) -> OrcRes<Vec<Item>> {
+		let src_path = line.first().expect("cannot be empty").sr.path();
+		let line = join_all(
+			(line.into_iter())
+				.map(|t| async { tt_to_api(&mut self.system.ext().exprs().clone(), t).await }),
+		)
+		.await;
+		let mod_path = ctx.src_path().suffix(path.unreverse(), self.system.i()).await;
+		let comments = comments.iter().map(Comment::to_api).collect_vec();
+		let req = api::ParseLine {
+			idx: self.idx,
+			module: mod_path.to_api(),
+			src: src_path.to_api(),
+			exported,
+			sys: self.system.id(),
+			comments,
+			line,
+		};
+		match self.system.reqnot().request(req).await {
+			Ok(parsed_v) => {
+				let mut ext_exprs = self.system.ext().exprs().clone();
+				conv(parsed_v, path, callback, &mut ConvCtx {
+					i: self.system.i(),
+					mod_path: &mod_path,
+					ext_exprs: &mut ext_exprs,
+					pctx: &mut ExprParseCtx { ctx: self.system.ctx(), exprs: self.system.ext().exprs() },
+					src_path: &src_path,
+					sys: &self.system,
+				})
+				.await
+			},
+			Err(e) => Err(OrcErrv::from_api(&e, &self.system.ctx().i).await),
+		}
+	}
+}
+
+struct ConvCtx<'a> {
+	sys: &'a System,
+	mod_path: &'a Sym,
+	src_path: &'a Sym,
+	i: &'a Interner,
+	ext_exprs: &'a mut ExprStore,
+	pctx: &'a mut ExprParseCtx<'a>,
+}
+async fn conv(
+	parsed_v: Vec<api::ParsedLine>,
+	module: Substack<'_, IStr>,
+	callback: &'_ mut impl AsyncFnMut(Substack<'_, IStr>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
+	ctx: &mut ConvCtx<'_>,
+) -> OrcRes<Vec<Item>> {
+	let mut items = Vec::new();
+	for parsed in parsed_v {
+		let (name, exported, kind) = match parsed.kind {
+			api::ParsedLineKind::Member(api::ParsedMember { name, exported, kind }) =>
+				(name, exported, kind),
+			api::ParsedLineKind::Recursive(rec) => {
+				let tokens = ttv_from_api(rec, ctx.ext_exprs, ctx.pctx, ctx.src_path, ctx.i).await;
+				items.extend(callback(module.clone(), tokens).await?);
+				continue;
+			},
+		};
+		let name = ctx.i.ex(name).await;
+		let mkind = match kind {
+			api::ParsedMemberKind::Module { lines, use_prelude } => {
+				let items = conv(lines, module.push(name.clone()), callback, ctx).boxed_local().await?;
+				ParsedMemberKind::Mod(ParsedModule::new(use_prelude, items))
+			},
+			api::ParsedMemberKind::Constant(cid) => {
+				ctx.sys.0.const_paths.insert(cid, ctx.mod_path.suffix(module.unreverse(), ctx.i).await);
+				ParsedMemberKind::Const(cid, ctx.sys.clone())
+			},
+		};
+		items.push(Item {
+			comments: join_all(
+				parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone(), ctx.i)),
+			)
+			.await,
+			sr: SrcRange::from_api(&parsed.source_range, ctx.i).await,
+			kind: ItemKind::Member(ParsedMember { name, exported, kind: mkind }),
+		})
+	}
+	Ok(items)
+}

orchid-host/src/system.rs

@@ -1,39 +1,44 @@
 use std::collections::VecDeque;
+use std::fmt;
 use std::future::Future;
 use std::rc::{Rc, Weak};
-use std::{fmt, mem};
 
-use async_stream::stream;
+use async_lock::RwLock;
 use derive_destructure::destructure;
-use futures::StreamExt;
 use futures::future::join_all;
 use hashbrown::HashMap;
 use itertools::Itertools;
+use memo_map::MemoMap;
 use orchid_base::char_filter::char_filter_match;
-use orchid_base::clone;
+use orchid_base::error::{OrcRes, mk_err_floating};
-use orchid_base::error::{OrcErrv, OrcRes};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
-use orchid_base::location::Pos;
+use orchid_base::iter_utils::IteratorPrint;
-use orchid_base::parse::Comment;
+use orchid_base::name::{NameLike, Sym, VName, VPath};
 use orchid_base::reqnot::{ReqNot, Requester};
-use orchid_base::tree::ttv_from_api;
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
 use crate::api;
+use crate::atom::{AtomHand, WeakAtomHand};
 use crate::ctx::Ctx;
+use crate::dealias::walk;
 use crate::extension::{Extension, WeakExtension};
-use crate::tree::{ItemKind, Member, Module, ParsTokTree};
+use crate::sys_parser::Parser;
+use crate::tree::Root;
 
 #[derive(destructure)]
-struct SystemInstData {
+pub(crate) struct SystemInstData {
+	deps: Vec<System>,
 	ctx: Ctx,
 	ext: Extension,
 	decl_id: api::SysDeclId,
 	lex_filter: api::CharFilter,
 	id: api::SysId,
-	line_types: Vec<Tok<String>>,
+	line_types: Vec<IStr>,
+	prelude: Vec<Sym>,
+	owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
+	pub(crate) const_paths: MemoMap<api::ParsedConstId, Sym>,
 }
 impl Drop for SystemInstData {
 	fn drop(&mut self) { self.ext.system_drop(self.id); }
@@ -50,53 +55,116 @@ impl fmt::Debug for SystemInstData {
 }
 
 #[derive(Clone, Debug)]
-pub struct System(Rc<SystemInstData>);
+pub struct System(pub(crate) Rc<SystemInstData>);
 impl System {
+	#[must_use]
 	pub fn id(&self) -> api::SysId { self.0.id }
+	#[must_use]
 	pub fn ext(&self) -> &Extension { &self.0.ext }
+	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
+	#[must_use]
+	pub fn i(&self) -> &Interner { &self.0.ctx.i }
+	#[must_use]
+	pub fn deps(&self) -> &[System] { &self.0.deps }
+	#[must_use]
+	pub fn ctor(&self) -> SystemCtor {
+		(self.0.ext.system_ctors().find(|c| c.decl.id == self.0.decl_id).cloned())
+			.expect("Ctor was used to create ext")
+	}
+	#[must_use]
 	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { self.0.ext.reqnot() }
+	#[must_use]
 	pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
 		self.reqnot().request(api::GetMember(self.0.id, id)).await
 	}
+	#[must_use]
 	pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
+	#[must_use]
 	pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
+	#[must_use]
+	pub fn prelude(&self) -> Vec<Sym> { self.0.prelude.clone() }
 	/// Have this system lex a part of the source. It is assumed that
 	/// [Self::can_lex] was called and returned true.
 	pub async fn lex<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
-		source: Tok<String>,
+		source: IStr,
+		src: Sym,
 		pos: u32,
 		r: impl FnMut(u32) -> F,
 	) -> api::OrcResult<Option<api::LexedExpr>> {
-		self.0.ext.lex_req(source, pos, self.id(), r).await
+		self.0.ext.lex_req(source, src, pos, self.id(), r).await
 	}
-	pub fn can_parse(&self, ltyp: Tok<String>) -> bool { self.0.line_types.contains(&ltyp) }
+	#[must_use]
-	pub fn line_types(&self) -> impl Iterator<Item = &Tok<String>> + '_ { self.0.line_types.iter() }
+	pub fn get_parser(&self, ltyp: IStr) -> Option<Parser> {
-	pub async fn parse(
+		(self.0.line_types.iter().enumerate())
-		&self,
+			.find(|(_, txt)| *txt == &ltyp)
-		line: Vec<ParsTokTree>,
+			.map(|(idx, _)| Parser { idx: idx as u16, system: self.clone() })
-		exported: bool,
-		comments: Vec<Comment>,
-	) -> OrcRes<Vec<ParsTokTree>> {
-		let line =
-			join_all(line.iter().map(|t| async { t.to_api(&mut |n, _| match *n {}).await })).await;
-		let comments = comments.iter().map(Comment::to_api).collect_vec();
-		match self.reqnot().request(api::ParseLine { exported, sys: self.id(), comments, line }).await {
-			Ok(parsed) => Ok(ttv_from_api(parsed, &mut self.ctx().clone(), &self.ctx().i).await),
-			Err(e) => Err(OrcErrv::from_api(&e, &self.ctx().i).await),
-		}
 	}
+	pub fn line_types(&self) -> impl Iterator<Item = &IStr> + '_ { self.0.line_types.iter() }
+
+	#[must_use]
 	pub async fn request(&self, req: Vec<u8>) -> Vec<u8> {
 		self.reqnot().request(api::SysFwded(self.id(), req)).await
 	}
+	pub(crate) async fn new_atom(&self, data: Vec<u8>, id: api::AtomId) -> AtomHand {
+		let mut owned_g = self.0.owned_atoms.write().await;
+		if let Some(data) = owned_g.get(&id)
+			&& let Some(atom) = data.upgrade()
+		{
+			return atom;
+		}
+		let new = AtomHand::new(data, self.clone(), Some(id));
+		owned_g.insert(id, new.downgrade());
+		new
+	}
 	pub(crate) fn drop_atom(&self, drop: api::AtomId) {
 		let this = self.0.clone();
 		(self.0.ctx.spawn)(Box::pin(async move {
-			this.ctx.owned_atoms.write().await.remove(&drop);
+			this.owned_atoms.write().await.remove(&drop);
 		}))
 	}
+	#[must_use]
 	pub fn downgrade(&self) -> WeakSystem { WeakSystem(Rc::downgrade(&self.0)) }
+	/// Implementation of [api::ResolveNames]
+	pub(crate) async fn name_resolver(
+		&self,
+		orig: api::ParsedConstId,
+	) -> impl AsyncFnMut(&[IStr]) -> OrcRes<VName> + use<> {
+		let root = self.0.ctx.root.read().await.upgrade().expect("find_names when root not in context");
+		let orig = self.0.const_paths.get(&orig).expect("origin for find_names invalid").clone();
+		let ctx = self.0.ctx.clone();
+		async move |rel| {
+			let cwd = orig.split_last_seg().1;
+			let root_data = &mut *root.0.write().await;
+			let walk_ctx = &mut (ctx.clone(), &root_data.consts);
+			let cmod = (walk(&root_data.root, false, cwd.iter().cloned(), walk_ctx).await)
+				.expect("the parent module of a constant should exist");
+			let (selector, tail) = rel.split_first().expect("Names cannot be empty");
+			if cmod.members.get(selector).is_some() {
+				return Ok(VName::new(cwd.iter().chain(rel).cloned()).unwrap());
+			}
+			match cmod.imports.get(selector) {
+				Some(Ok(dest)) => return Ok(dest.target.to_vname().suffix(tail.iter().cloned())),
+				Some(Err(dests)) =>
+					return Err(mk_err_floating(
+						ctx.i.i("Ambiguous name").await,
+						format!(
+							"{selector} could refer to {}",
+							dests.iter().map(|ri| &ri.target).display("or")
+						),
+					)),
+				None => (),
+			}
+			if tail.is_empty() {
+				return Ok(VPath::new(cwd.iter().cloned()).name_with_suffix(selector.clone()));
+			}
+			Err(mk_err_floating(
+				ctx.i.i("Invalid name").await,
+				format!("{selector} doesn't refer to a module"),
+			))
+		}
+	}
 }
 impl Format for System {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -108,27 +176,39 @@ impl Format for System {
 
 pub struct WeakSystem(Weak<SystemInstData>);
 impl WeakSystem {
+	#[must_use]
 	pub fn upgrade(&self) -> Option<System> { self.0.upgrade().map(System) }
 }
 
+#[derive(Clone)]
 pub struct SystemCtor {
 	pub(crate) decl: api::SystemDecl,
 	pub(crate) ext: WeakExtension,
 }
 impl SystemCtor {
+	#[must_use]
 	pub fn name(&self) -> &str { &self.decl.name }
+	pub async fn name_tok(&self) -> Sym {
+		(Sym::parse(&self.decl.name, &self.ext.upgrade().expect("ext dropped early").ctx().i).await)
+			.expect("System cannot have empty name")
+	}
+	#[must_use]
 	pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
+	#[must_use]
 	pub fn depends(&self) -> impl ExactSizeIterator<Item = &str> {
 		self.decl.depends.iter().map(|s| &**s)
 	}
+	#[must_use]
 	pub fn id(&self) -> api::SysDeclId { self.decl.id }
-	pub async fn run<'a>(&self, depends: impl IntoIterator<Item = &'a System>) -> System {
+	#[must_use]
-		let depends = depends.into_iter().map(|si| si.id()).collect_vec();
+	pub async fn run(&self, deps: Vec<System>) -> (Root, System) {
+		let depends = deps.iter().map(|si| si.id()).collect_vec();
 		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
 		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
 		let id = ext.ctx().next_sys_id();
 		let sys_inst = ext.reqnot().request(api::NewSystem { depends, id, system: self.decl.id }).await;
 		let data = System(Rc::new(SystemInstData {
+			deps,
 			decl_id: self.decl.id,
 			ext: ext.clone(),
 			ctx: ext.ctx().clone(),
@@ -136,24 +216,18 @@ impl SystemCtor {
 			line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
 				.await,
 			id,
+			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok, &ext.ctx().i))).await,
+			owned_atoms: RwLock::new(HashMap::new()),
+			const_paths: MemoMap::new(),
 		}));
-		let const_root = clone!(data, ext; stream! {
+		let api_module_root = api::Module {
-			for (k, v) in sys_inst.const_root {
+			members: (sys_inst.const_root.into_iter())
-				yield Member::from_api(
+				.map(|(k, v)| api::Member { name: k, kind: v, comments: vec![], exported: true })
-					api::Member { name: k, kind: v },
+				.collect_vec(),
-					&mut vec![Tok::from_api(k, &ext.ctx().i).await],
+		};
-					&data,
+		let root = Root::from_api(api_module_root, &data).await;
-				).await;
-			}
-		})
-		.map(|mem| ItemKind::Member(mem).at(Pos::None))
-		.collect::<Vec<_>>()
-		.await;
 		ext.ctx().systems.write().await.insert(id, data.downgrade());
-		let mut swap = Module::default();
+		(root, data)
-		mem::swap(&mut swap, &mut *ext.ctx().root.write().await);
-		*ext.ctx().root.write().await = Module::new(swap.items.into_iter().chain(const_root));
-		data
 	}
 }
 
@@ -166,7 +240,7 @@ pub enum SysResolvErr {
 pub async fn init_systems(
 	tgts: &[String],
 	exts: &[Extension],
-) -> Result<Vec<System>, SysResolvErr> {
+) -> Result<(Root, Vec<System>), SysResolvErr> {
 	let mut to_load = HashMap::<&str, &SystemCtor>::new();
 	let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque<&str>>();
 	while let Some(target) = to_find.pop_front() {
@@ -205,9 +279,11 @@ pub async fn init_systems(
 		walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?;
 	}
 	let mut systems = HashMap::<&str, System>::new();
+	let mut root = Root::new(exts.first().unwrap().ctx().clone());
 	for ctor in to_load_ordered.iter() {
-		let sys = ctor.run(ctor.depends().map(|n| &systems[n])).await;
+		let (sys_root, sys) = ctor.run(ctor.depends().map(|n| systems[n].clone()).collect()).await;
 		systems.insert(ctor.name(), sys);
+		root = root.merge(&sys_root).await.expect("Conflicting roots");
 	}
-	Ok(systems.into_values().collect_vec())
+	Ok((root, systems.into_values().collect_vec()))
 }

orchid-host/src/tree.rs

@@ -1,342 +1,479 @@
-use std::fmt::Debug;
+//! This tree isn't Clone because lazy subtrees are guaranteed to only be loaded
-use std::rc::Rc;
+//! once
+use std::cell::RefCell;
+use std::rc::{Rc, Weak};
+use std::slice;
 
+use async_lock::RwLock;
 use async_once_cell::OnceCell;
-use async_std::sync::Mutex;
+use futures::{FutureExt, StreamExt, stream};
-use async_stream::stream;
+use hashbrown::HashMap;
-use futures::future::join_all;
+use hashbrown::hash_map::Entry;
-use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
-use never::Never;
+use memo_map::MemoMap;
-use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
+use orchid_base::clone;
+use orchid_base::error::{OrcRes, Reporter, mk_err};
 use orchid_base::interner::Tok;
-use orchid_base::location::Pos;
+use orchid_base::location::{CodeGenInfo, Pos};
-use orchid_base::macros::{mtreev_fmt, mtreev_from_api};
+use orchid_base::name::{NameLike, Sym, VPath};
-use orchid_base::name::Sym;
+use orchid_base::reqnot::Requester;
-use orchid_base::parse::{Comment, Import};
-use orchid_base::tree::{AtomRepr, TokTree, Token};
-use orchid_base::{clone, tl_cache};
-use ordered_float::NotNan;
-use substack::Substack;
 
 use crate::api;
-use crate::atom::AtomHand;
 use crate::ctx::Ctx;
-use crate::expr::{Expr, mtreev_to_expr};
+use crate::dealias::{ChildErrorKind, Tree, absolute_path, resolv_glob, walk};
-use crate::macros::{MacTok, MacTree};
+use crate::expr::{Expr, ExprParseCtx, PathSetBuilder};
+use crate::parsed::{ItemKind, ParsedMemberKind, ParsedModule};
 use crate::system::System;
 
-pub type ParsTokTree = TokTree<'static, AtomHand, Never>;
+pub struct RootData {
-pub type ParsTok = Token<'static, AtomHand, Never>;
+	pub root: Module,
-
+	pub consts: MemoMap<Sym, Expr>,
-#[derive(Debug)]
+	pub ctx: Ctx,
-pub struct Item {
-	pub pos: Pos,
-	pub comments: Vec<Comment>,
-	pub kind: ItemKind,
 }
-
+#[derive(Clone)]
-#[derive(Debug)]
+pub struct Root(pub Rc<RwLock<RootData>>);
-pub enum ItemKind {
+impl Root {
-	Member(Member),
+	#[must_use]
-	Export(Tok<String>),
+	pub fn new(ctx: Ctx) -> Self {
-	Import(Import),
+		Root(Rc::new(RwLock::new(RootData {
-	Macro(Option<NotNan<f64>>, Vec<Rule>),
+			root: Module::default(),
-}
+			consts: MemoMap::default(),
-impl ItemKind {
+			ctx,
-	pub fn at(self, pos: Pos) -> Item { Item { comments: vec![], pos, kind: self } }
+		})))
-}
+	}
-
+	#[must_use]
-impl Item {
+	pub async fn from_api(api: api::Module, sys: &System) -> Self {
-	pub async fn from_api(tree: api::Item, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
+		let consts = MemoMap::new();
-		let kind = match tree.kind {
+		let mut tfac = TreeFromApiCtx { consts: &consts, path: sys.i().i(&[][..]).await, sys };
-			api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys).await),
+		let root = Module::from_api(api, &mut tfac).await;
-			api::ItemKind::Import(name) => ItemKind::Import(Import {
+		Root(Rc::new(RwLock::new(RootData { root, consts, ctx: sys.ctx().clone() })))
-				path: Sym::from_api(name, &sys.ctx().i).await.iter().cloned().collect(),
+	}
-				name: None,
+	pub async fn merge(&self, new: &Root) -> Result<Self, MergeErr> {
-			}),
+		let this = self.0.write().await;
-			api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &sys.ctx().i).await),
+		let that = new.0.write().await;
-			api::ItemKind::Macro(macro_block) => {
+		let consts = MemoMap::new();
-				let mut rules = Vec::new();
+		for (k, v) in this.consts.iter().chain(that.consts.iter()) {
-				for rule in macro_block.rules {
+			consts.insert(k.clone(), v.clone());
-					let mut comments = Vec::new();
-					for comment in rule.comments {
-						comments.push(Comment::from_api(&comment, &sys.ctx().i).await);
-					}
-					let pos = Pos::from_api(&rule.location, &sys.ctx().i).await;
-					let pattern = mtreev_from_api(&rule.pattern, &sys.ctx().i, &mut {
-						clone!(pos, sys);
-						move |a| {
-							clone!(pos, sys);
-							Box::pin(async move {
-								MacTok::Atom(AtomHand::from_api(a, pos.clone(), &mut sys.ctx().clone()).await)
-							})
-						}
-					})
-					.await;
-					rules.push(Rule { pos, pattern, kind: RuleKind::Remote(sys.clone(), rule.id), comments });
-				}
-				ItemKind::Macro(macro_block.priority, rules)
-			},
-		};
-		let mut comments = Vec::new();
-		for comment in tree.comments.iter() {
-			comments.push(Comment::from_api(comment, &sys.ctx().i).await)
 		}
-		Self { pos: Pos::from_api(&tree.location, &sys.ctx().i).await, comments, kind }
+		let root = this.root.merge(&that.root, this.ctx.clone(), &consts).await?;
+		Ok(Self(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() }))))
+	}
+	#[must_use]
+	pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym, rep: &Reporter) -> Self {
+		let mut ref_this = self.0.write().await;
+		let this = &mut *ref_this;
+		let mut deferred_consts = HashMap::new();
+		let consts = this.consts.clone();
+		let mut tfpctx = FromParsedCtx {
+			pars_root: parsed,
+			deferred_consts: &mut deferred_consts,
+			consts: &consts,
+			pars_prefix: pars_prefix.clone(),
+			root: &this.root,
+			ctx: &this.ctx,
+			rep,
+		};
+		let mut module = Module::from_parsed(parsed, pars_prefix.clone(), &mut tfpctx).await;
+		for step in pars_prefix.iter().rev() {
+			let kind = OnceCell::from(MemberKind::Module(module));
+			let members = HashMap::from([(
+				step.clone(),
+				Rc::new(Member { public: true, lazy: RefCell::new(None), kind }),
+			)]);
+			module = Module { imports: HashMap::new(), members }
+		}
+		let root = (this.root.merge(&module, this.ctx.clone(), &consts).await)
+			.expect("Merge conflict between parsed and existing module");
+		let new = Root(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() })));
+		*this.ctx.root.write().await = new.downgrade();
+		for (path, (sys_id, pc_id)) in deferred_consts {
+			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
+			let api_expr = sys.reqnot().request(api::FetchParsedConst { id: pc_id, sys: sys.id() }).await;
+			let mut xp_ctx = ExprParseCtx { ctx: &this.ctx, exprs: sys.ext().exprs() };
+			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), &mut xp_ctx).await;
+			new.0.write().await.consts.insert(path, expr);
+		}
+		new
+	}
+	pub async fn get_const_value(&self, name: Sym, pos: Pos) -> OrcRes<Expr> {
+		let this = &*self.0.read().await;
+		// shortcut for previously visited
+		if let Some(val) = this.consts.get(&name) {
+			return Ok(val.clone());
+		}
+		// load the node, then check if this "walk" call added it to the map
+		let ctx = this.ctx.clone();
+		let module =
+			walk(&this.root, false, name.iter().cloned(), &mut (ctx.clone(), &this.consts)).await;
+		if let Some(val) = this.consts.get(&name) {
+			return Ok(val.clone());
+		}
+		match module {
+			Ok(_) => Err(mk_err(
+				ctx.i.i("module used as constant").await,
+				format!("{name} is a module, not a constant"),
+				[pos],
+			)),
+			Err(e) => match e.kind {
+				ChildErrorKind::Private => panic!("public_only is false"),
+				ChildErrorKind::Constant => panic!("Tree refers to constant not in table"),
+				ChildErrorKind::Missing => Err(mk_err(
+					ctx.i.i("Constant does not exist").await,
+					format!("{name} does not refer to a constant"),
+					[pos],
+				)),
+			},
+		}
+	}
+	#[must_use]
+	pub fn downgrade(&self) -> WeakRoot { WeakRoot(Rc::downgrade(&self.0)) }
+}
+
+#[derive(Clone)]
+pub struct WeakRoot(Weak<RwLock<RootData>>);
+impl WeakRoot {
+	#[must_use]
+	pub fn new() -> Self { Self(Weak::new()) }
+	#[must_use]
+	pub fn upgrade(&self) -> Option<Root> { Some(Root(self.0.upgrade()?)) }
+}
+impl Default for WeakRoot {
+	fn default() -> Self { Self::new() }
+}
+
+pub struct TreeFromApiCtx<'a> {
+	pub sys: &'a System,
+	pub consts: &'a MemoMap<Sym, Expr>,
+	pub path: Tok<Vec<IStr>>,
+}
+impl<'a> TreeFromApiCtx<'a> {
+	#[must_use]
+	pub async fn push<'c>(&'c self, name: IStr) -> TreeFromApiCtx<'c> {
+		let path = self.sys.ctx().i.i(&self.path.iter().cloned().chain([name]).collect_vec()).await;
+		TreeFromApiCtx { path, consts: self.consts, sys: self.sys }
 	}
 }
-impl Format for Item {
+
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+#[derive(Clone, Debug, PartialEq, Eq)]
-		let comment_text = self.comments.iter().join("\n");
+pub struct ResolvedImport {
-		let item_text = match &self.kind {
+	pub target: Sym,
-			ItemKind::Import(i) => format!("import {i}").into(),
+	pub pos: Pos,
-			ItemKind::Export(e) => format!("export {e}").into(),
+}
-			ItemKind::Macro(None, rules) =>
+
-				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("macro {{\n\t{0}\n}}")))
+#[derive(Clone, Default)]
-					.units([Variants::sequence(rules.len(), "\n", None)
+pub struct Module {
-						.units(join_all(rules.iter().map(|r| r.print(c))).await)]),
+	pub imports: HashMap<IStr, Result<ResolvedImport, Vec<ResolvedImport>>>,
-			ItemKind::Member(mem) => match mem.kind.get() {
+	pub members: HashMap<IStr, Rc<Member>>,
-				None => format!("lazy {}", mem.name).into(),
+}
-				Some(MemberKind::Const(val)) =>
+impl Module {
-					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} = {1}")))
+	#[must_use]
-						.units([mem.name.rc().into(), val.print(c).await]),
+	pub async fn from_api(api: api::Module, ctx: &mut TreeFromApiCtx<'_>) -> Self {
-				Some(MemberKind::Mod(module)) =>
+		let mut members = HashMap::new();
-					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
+		for mem in api.members {
-						.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
+			let mem_name = ctx.sys.i().ex(mem.name).await;
-			},
+			let vname = VPath::new(ctx.path.iter().cloned()).name_with_suffix(mem_name.clone());
-			_ => panic!(),
+			let name = vname.to_sym(ctx.sys.i()).await;
+			let (lazy, kind) = match mem.kind {
+				api::MemberKind::Lazy(id) =>
+					(Some(LazyMemberHandle { id, sys: ctx.sys.id(), path: name.clone() }), None),
+				api::MemberKind::Const(val) => {
+					let mut expr_ctx = ExprParseCtx { ctx: ctx.sys.ctx(), exprs: ctx.sys.ext().exprs() };
+					let expr = Expr::from_api(&val, PathSetBuilder::new(), &mut expr_ctx).await;
+					ctx.consts.insert(name.clone(), expr);
+					(None, Some(MemberKind::Const))
+				},
+				api::MemberKind::Module(m) => {
+					let m = Self::from_api(m, &mut ctx.push(mem_name.clone()).await).boxed_local().await;
+					(None, Some(MemberKind::Module(m)))
+				},
+			};
+			members.insert(
+				mem_name.clone(),
+				Rc::new(Member {
+					public: mem.exported,
+					lazy: RefCell::new(lazy),
+					kind: kind.map_or_else(OnceCell::new, OnceCell::from),
+				}),
+			);
+		}
+		Self { members, imports: HashMap::new() }
+	}
+	#[must_use]
+	async fn from_parsed(parsed: &ParsedModule, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
+		let imports_by_name = (parsed.get_imports().into_iter())
+			.filter_map(|i| Some((i.name.clone()?, i)))
+			.into_group_map();
+		let mut glob_imports_by_name = HashMap::<_, Vec<_>>::new();
+		for import in parsed.get_imports().into_iter().filter(|i| i.name.is_none()) {
+			let pos = import.sr.pos();
+			match absolute_path(&path, &import.path, &ctx.ctx.i).await {
+				Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, pos, &import.path.to_string()).await),
+				Ok(abs_path) => {
+					let names_res = match abs_path.strip_prefix(&ctx.pars_prefix[..]) {
+						None => {
+							let mut tree_ctx = (ctx.ctx.clone(), ctx.consts);
+							resolv_glob(&path, ctx.root, &abs_path, pos, &ctx.ctx.i, &mut tree_ctx).await
+						},
+						Some(sub_tgt) => {
+							let sub_path = (path.strip_prefix(&ctx.pars_prefix[..]))
+								.expect("from_parsed called with path outside pars_prefix");
+							resolv_glob(sub_path, ctx.pars_root, sub_tgt, pos, &ctx.ctx.i, &mut ()).await
+						},
+					};
+					let abs_path = abs_path.to_sym(&ctx.ctx.i).await;
+					match names_res {
+						Err(e) => ctx.rep.report(e),
+						Ok(names) =>
+							for name in names {
+								match glob_imports_by_name.entry(name) {
+									Entry::Occupied(mut o) => o.get_mut().push((abs_path.clone(), import.sr.clone())),
+									Entry::Vacant(v) => {
+										v.insert_entry(vec![(abs_path.clone(), import.sr.clone())]);
+									},
+								}
+							},
+					}
+				},
+			}
+		}
+		let mut imports = HashMap::new();
+		if parsed.use_prelude {
+			let systems = ctx.ctx.systems.read().await;
+			for sys in systems.values().flat_map(|weak| weak.upgrade()) {
+				for prelude_item in sys.prelude() {
+					imports.insert(
+						prelude_item.last_seg(),
+						Ok(ResolvedImport {
+							target: prelude_item,
+							pos: CodeGenInfo::new_details(sys.ctor().name_tok().await, "In prelude", &ctx.ctx.i)
+								.await
+								.pos(),
+						}),
+					);
+				}
+			}
+		}
+		let conflicting_imports_msg = ctx.ctx.i.i("Conflicting imports").await;
+		for (key, values) in imports_by_name {
+			if values.len() == 1 {
+				let import = values.into_iter().next().unwrap();
+				let sr = import.sr.clone();
+				let abs_path_res = absolute_path(&path, &import.clone().mspath(), &ctx.ctx.i).await;
+				match abs_path_res {
+					Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, sr.pos(), &import.to_string()).await),
+					Ok(abs_path) => {
+						let target = abs_path.to_sym(&ctx.ctx.i).await;
+						imports.insert(key, Ok(ResolvedImport { target, pos: sr.pos() }));
+					},
+				}
+			} else {
+				for item in values {
+					ctx.rep.report(mk_err(
+						conflicting_imports_msg.clone(),
+						format!("{key} is imported multiple times from different modules"),
+						[item.sr.pos()],
+					));
+				}
+			}
+		}
+		for (key, values) in glob_imports_by_name {
+			if !imports.contains_key(&key) {
+				let i = &ctx.ctx.i;
+				let values = stream::iter(values)
+					.then(|(n, sr)| {
+						clone!(key; async move {
+							ResolvedImport {
+								target: n.to_vname().suffix([key.clone()]).to_sym(i).await,
+								pos: sr.pos(),
+							}
+						})
+					})
+					.collect::<Vec<_>>()
+					.await;
+				imports.insert(key, if values.len() == 1 { Ok(values[0].clone()) } else { Err(values) });
+			}
+		}
+		let self_referential_msg = ctx.ctx.i.i("Self-referential import").await;
+		for (key, value) in imports.iter() {
+			let Ok(import) = value else { continue };
+			if import.target.strip_prefix(&path[..]).is_some_and(|t| t.starts_with(slice::from_ref(key)))
+			{
+				ctx.rep.report(mk_err(
+					self_referential_msg.clone(),
+					format!("import {} points to itself or a path within itself", &import.target),
+					[import.pos.clone()],
+				));
+			}
+		}
+		let mut members = HashMap::new();
+		for item in &parsed.items {
+			match &item.kind {
+				ItemKind::Member(mem) => {
+					let path = path.to_vname().suffix([mem.name.clone()]).to_sym(&ctx.ctx.i).await;
+					let kind = OnceCell::from(MemberKind::from_parsed(&mem.kind, path.clone(), ctx).await);
+					members.insert(
+						mem.name.clone(),
+						Rc::new(Member { kind, lazy: RefCell::default(), public: mem.exported }),
+					);
+				},
+				ItemKind::Import(_) => (),
+			}
+		}
+		Module { imports, members }
+	}
+	pub async fn merge(
+		&self,
+		other: &Module,
+		ctx: Ctx,
+		consts: &MemoMap<Sym, Expr>,
+	) -> Result<Module, MergeErr> {
+		if !self.imports.is_empty() || !other.imports.is_empty() {
+			return Err(MergeErr { path: VPath::new([]), kind: MergeErrKind::Imports });
+		}
+		let mut members = HashMap::new();
+		for (key, mem) in &other.members {
+			let Some(own) = self.members.get(key) else {
+				members.insert(key.clone(), mem.clone());
+				continue;
+			};
+			if own.public != mem.public {
+				return Err(MergeErr { path: VPath::new([]), kind: MergeErrKind::Visibility });
+			}
+			match (own.kind(ctx.clone(), consts).await, mem.kind(ctx.clone(), consts).await) {
+				(MemberKind::Module(own_sub), MemberKind::Module(sub)) => {
+					match own_sub.merge(sub, ctx.clone(), consts).boxed_local().await {
+						Ok(module) => {
+							members.insert(
+								key.clone(),
+								Rc::new(Member {
+									lazy: RefCell::new(None),
+									public: own.public,
+									kind: OnceCell::from(MemberKind::Module(module)),
+								}),
+							);
+						},
+						Err(mut e) => {
+							e.path = e.path.prefix([key.clone()]);
+							return Err(e);
+						},
+					}
+				},
+				_ => return Err(MergeErr { path: VPath::new([key.clone()]), kind: MergeErrKind::Const }),
+			}
+		}
+		for (key, mem) in &self.members {
+			if let Entry::Vacant(slot) = members.entry(key.clone()) {
+				slot.insert(mem.clone());
+			}
+		}
+		Ok(Module { imports: HashMap::new(), members })
+	}
+}
+
+#[derive(Debug)]
+pub struct MergeErr {
+	pub path: VPath,
+	pub kind: MergeErrKind,
+}
+#[derive(Debug)]
+pub enum MergeErrKind {
+	Imports,
+	Visibility,
+	Const,
+}
+
+pub struct FromParsedCtx<'a> {
+	pars_prefix: Sym,
+	pars_root: &'a ParsedModule,
+	root: &'a Module,
+	rep: &'a Reporter,
+	ctx: &'a Ctx,
+	consts: &'a MemoMap<Sym, Expr>,
+	deferred_consts: &'a mut HashMap<Sym, (api::SysId, api::ParsedConstId)>,
+}
+
+impl Tree for Module {
+	type Ctx<'a> = (Ctx, &'a MemoMap<Sym, Expr>);
+	async fn child(
+		&self,
+		key: IStr,
+		public_only: bool,
+		(ctx, consts): &mut Self::Ctx<'_>,
+	) -> crate::dealias::ChildResult<'_, Self> {
+		let Some(member) = self.members.get(&key) else {
+			return Err(ChildErrorKind::Missing);
 		};
-		tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}\n{1}")))
+		if public_only && !member.public {
-			.units([comment_text.into(), item_text])
+			return Err(ChildErrorKind::Private);
+		}
+		match &member.kind(ctx.clone(), consts).await {
+			MemberKind::Module(m) => Ok(m),
+			MemberKind::Const => Err(ChildErrorKind::Constant),
+		}
+	}
+	fn children(&self, public_only: bool) -> hashbrown::HashSet<IStr> {
+		self.members.iter().filter(|(_, v)| !public_only || v.public).map(|(k, _)| k.clone()).collect()
 	}
 }
 
 pub struct Member {
-	name: Tok<String>,
+	pub public: bool,
-	kind: OnceCell<MemberKind>,
+	pub lazy: RefCell<Option<LazyMemberHandle>>,
-	lazy: Mutex<Option<LazyMemberHandle>>,
+	pub kind: OnceCell<MemberKind>,
 }
 impl Member {
-	pub fn name(&self) -> Tok<String> { self.name.clone() }
+	#[must_use]
-	pub async fn kind(&self) -> &MemberKind {
+	pub async fn kind<'a>(&'a self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> &'a MemberKind {
 		(self.kind.get_or_init(async {
-			let handle = self.lazy.lock().await.take().expect("Neither known nor lazy");
+			let handle = self.lazy.borrow_mut().take().expect("If kind is uninit, lazy must be Some");
-			handle.run().await
+			handle.run(ctx, consts).await
 		}))
 		.await
 	}
-	pub async fn kind_mut(&mut self) -> &mut MemberKind {
-		self.kind().await;
-		self.kind.get_mut().expect("kind() already filled the cell")
-	}
-	pub async fn from_api(api: api::Member, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
-		path.push(Tok::from_api(api.name, &sys.ctx().i).await);
-		let kind = match api.kind {
-			api::MemberKind::Lazy(id) => {
-				let handle = LazyMemberHandle(id, sys.clone(), path.clone());
-				return handle.into_member(path.pop().unwrap());
-			},
-			api::MemberKind::Const(c) => MemberKind::Const(Code::from_expr(
-				CodeLocator::to_const(sys.ctx().i.i(&*path).await),
-				Expr::from_api(&c, &mut sys.ext().clone()).await,
-			)),
-			api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, path, sys).await),
-		};
-		let name = path.pop().unwrap();
-		Member { name, kind: OnceCell::from(kind), lazy: Mutex::default() }
-	}
-	pub fn new(name: Tok<String>, kind: MemberKind) -> Self {
-		Member { name, kind: OnceCell::from(kind), lazy: Mutex::default() }
-	}
-}
-impl Debug for Member {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		f.debug_struct("Member")
-			.field("name", &self.name)
-			.field("kind", &self.kind)
-			.finish_non_exhaustive()
-	}
 }
 
-#[derive(Debug)]
 pub enum MemberKind {
-	Const(Code),
+	Const,
-	Mod(Module),
+	Module(Module),
 }
-
+impl MemberKind {
-#[derive(Debug, Default)]
+	#[must_use]
-pub struct Module {
+	async fn from_parsed(parsed: &ParsedMemberKind, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
-	pub imports: Vec<Sym>,
+		match parsed {
-	pub exports: Vec<Tok<String>>,
+			ParsedMemberKind::Const(id, sys) => {
-	pub items: Vec<Item>,
+				ctx.deferred_consts.insert(path, (sys.id(), *id));
-}
+				MemberKind::Const
-impl Module {
+			},
-	pub fn new(items: impl IntoIterator<Item = Item>) -> Self {
+			ParsedMemberKind::Mod(m) =>
-		let items = items.into_iter().collect_vec();
+				MemberKind::Module(Module::from_parsed(m, path, ctx).boxed_local().await),
-		let exports = (items.iter())
-			.filter_map(|i| match &i.kind {
-				ItemKind::Export(e) => Some(e.clone()),
-				_ => None,
-			})
-			.collect_vec();
-		Self { imports: vec![], exports, items }
-	}
-	pub async fn from_api(m: api::Module, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
-		Self::new(
-			stream! { for item in m.items { yield Item::from_api(item, path, sys).boxed_local().await } }
-				.collect::<Vec<_>>()
-				.await,
-		)
-	}
-	pub async fn walk(
-		&self,
-		allow_private: bool,
-		path: impl IntoIterator<Item = Tok<String>>,
-	) -> Result<&Module, WalkError> {
-		let mut cur = self;
-		for (pos, step) in path.into_iter().enumerate() {
-			let Some(member) = (cur.items.iter())
-				.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
-				.find(|m| m.name == step)
-			else {
-				return Err(WalkError { pos, kind: WalkErrorKind::Missing });
-			};
-			if !allow_private && !cur.exports.contains(&step) {
-				return Err(WalkError { pos, kind: WalkErrorKind::Private });
-			}
-			match member.kind().await {
-				MemberKind::Const(_) => return Err(WalkError { pos, kind: WalkErrorKind::Constant }),
-				MemberKind::Mod(m) => cur = m,
-			}
 		}
-		Ok(cur)
-	}
-}
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-pub enum WalkErrorKind {
-	Missing,
-	Private,
-	Constant,
-}
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-pub struct WalkError {
-	pub pos: usize,
-	pub kind: WalkErrorKind,
-}
-impl Format for Module {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		let import_str = self.imports.iter().map(|i| format!("import {i}")).join("\n");
-		let head_str = format!("{import_str}\nexport ::({})\n", self.exports.iter().join(", "));
-		Variants::sequence(self.items.len() + 1, "\n", None).units(
-			[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
-		)
 	}
 }
 
-pub struct LazyMemberHandle(api::TreeId, System, Vec<Tok<String>>);
+pub struct LazyMemberHandle {
+	id: api::TreeId,
+	sys: api::SysId,
+	path: Sym,
+}
 impl LazyMemberHandle {
-	pub async fn run(self) -> MemberKind {
+	#[must_use]
-		match self.1.get_tree(self.0).await {
+	pub async fn run(self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> MemberKind {
-			api::MemberKind::Const(c) => MemberKind::Const(Code {
+		let sys = ctx.system_inst(self.sys).await.expect("Missing system for lazy member");
-				bytecode: Expr::from_api(&c, &mut self.1.ext().clone()).await.into(),
+		match sys.get_tree(self.id).await {
-				locator: CodeLocator { steps: self.1.ctx().i.i(&self.2).await, rule_loc: None },
+			api::MemberKind::Const(c) => {
-				source: None,
+				let mut pctx = ExprParseCtx { ctx: &ctx, exprs: sys.ext().exprs() };
-			}),
+				let expr = Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await;
-			api::MemberKind::Module(m) =>
+				consts.insert(self.path, expr);
-				MemberKind::Mod(Module::from_api(m, &mut { self.2 }, &self.1).await),
+				MemberKind::Const
-			api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run().boxed_local().await,
+			},
+			api::MemberKind::Module(m) => MemberKind::Module(
+				Module::from_api(m, &mut TreeFromApiCtx { sys: &sys, consts, path: self.path.tok() }).await,
+			),
+			api::MemberKind::Lazy(id) => Self { id, ..self }.run(ctx, consts).boxed_local().await,
 		}
 	}
-	pub fn into_member(self, name: Tok<String>) -> Member {
+	#[must_use]
-		Member { name, kind: OnceCell::new(), lazy: Mutex::new(Some(self)) }
+	pub async fn into_member(self, public: bool) -> Member {
-	}
+		Member { public, kind: OnceCell::new(), lazy: RefCell::new(Some(self)) }
-}
-
-#[derive(Debug)]
-pub struct Rule {
-	pub pos: Pos,
-	pub comments: Vec<Comment>,
-	pub pattern: Vec<MacTree>,
-	pub kind: RuleKind,
-}
-impl Format for Rule {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		FmtUnit::new(
-			tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0b}\n{1} => {2b}"))),
-			[
-				self.comments.iter().join("\n").into(),
-				mtreev_fmt(&self.pattern, c).await,
-				match &self.kind {
-					RuleKind::Native(code) => code.print(c).await,
-					RuleKind::Remote(sys, id) => FmtUnit::new(
-						tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0} #{1}"))),
-						[sys.print(c).await, format!("{id:?}").into()],
-					),
-				},
-			],
-		)
-	}
-}
-
-#[derive(Debug)]
-pub enum RuleKind {
-	Remote(System, api::MacroId),
-	Native(Code),
-}
-
-#[derive(Debug)]
-pub struct Code {
-	locator: CodeLocator,
-	source: Option<Vec<MacTree>>,
-	bytecode: OnceCell<Expr>,
-}
-impl Code {
-	pub fn from_expr(locator: CodeLocator, expr: Expr) -> Self {
-		Self { locator, source: None, bytecode: expr.into() }
-	}
-	pub fn from_code(locator: CodeLocator, code: Vec<MacTree>) -> Self {
-		Self { locator, source: Some(code), bytecode: OnceCell::new() }
-	}
-	pub fn source(&self) -> Option<&Vec<MacTree>> { self.source.as_ref() }
-	pub fn set_source(&mut self, source: Vec<MacTree>) {
-		self.source = Some(source);
-		self.bytecode = OnceCell::new();
-	}
-	pub async fn get_bytecode(&self, ctx: &Ctx) -> &Expr {
-		(self.bytecode.get_or_init(async {
-			let src = self.source.as_ref().expect("no bytecode or source");
-			mtreev_to_expr(src, Substack::Bottom, ctx).await.at(Pos::None)
-		}))
-		.await
-	}
-}
-impl Format for Code {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		if let Some(bc) = self.bytecode.get() {
-			return bc.print(c).await;
-		}
-		if let Some(src) = &self.source {
-			return mtreev_fmt(src, c).await;
-		}
-		panic!("Code must be initialized with at least one state")
-	}
-}
-
-/// Selects a code element
-///
-/// Either the steps point to a constant and rule_loc is None, or the steps
-/// point to a module and rule_loc selects a macro rule within that module
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-pub struct CodeLocator {
-	steps: Tok<Vec<Tok<String>>>,
-	/// Index of a macro block in the module demarked by the steps, and a rule in
-	/// that macro
-	rule_loc: Option<(u16, u16)>,
-}
-impl CodeLocator {
-	pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps, rule_loc: None } }
-	pub fn to_rule(steps: Tok<Vec<Tok<String>>>, macro_i: u16, rule_i: u16) -> Self {
-		Self { steps, rule_loc: Some((macro_i, rule_i)) }
 	}
 }

orchid-std/Cargo.toml

@@ -1,19 +1,26 @@
 [package]
 name = "orchid-std"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 [dependencies]
 async-once-cell = "0.5.4"
-async-std = "1.13.0"
+futures = { version = "0.3.31", features = ["std"], default-features = false }
-futures = "0.3.31"
+hashbrown = "0.16.0"
 itertools = "0.14.0"
 never = "0.1.0"
-once_cell = "1.20.2"
+once_cell = "1.21.3"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-orchid-extension = { version = "0.1.0", path = "../orchid-extension" }
+orchid-extension = { version = "0.1.0", path = "../orchid-extension", features = [
-ordered-float = "4.6.0"
+  "tokio",
-tokio = { version = "1.43.0", features = ["full"] }
+] }
+ordered-float = "5.0.0"
+rust_decimal = "1.37.2"
+substack = "1.1.1"
+tokio = { version = "1.47.1", features = ["full"] }
+
+[dev-dependencies]
+test_executors = "0.3.5"

orchid-std/src/lib.rs

@@ -1,7 +1,9 @@
-mod number;
+mod macros;
 mod std;
-mod string;
 
-pub use std::StdSystem;
+pub use std::number::num_atom::{Float, HomoArray, Int, Num};
+pub use std::std_system::StdSystem;
+pub use std::string::str_atom::OrcString;
 
-pub use string::str_atom::OrcString;
+pub use macros::macro_system::MacroSystem;
+pub use macros::mactree::{MacTok, MacTree};

orchid-std/src/macros/instantiate_tpl.rs

@@ -0,0 +1,49 @@
+use std::borrow::Cow;
+
+use never::Never;
+use orchid_extension::atom::{Atomic, TypAtom};
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::expr::Expr;
+use orchid_extension::gen_expr::GExpr;
+
+use crate::macros::mactree::{MacTok, MacTree, map_mactree};
+
+#[derive(Clone)]
+pub struct InstantiateTplCall {
+	pub(crate) tpl: MacTree,
+	pub(crate) argc: usize,
+	pub(crate) argv: Vec<MacTree>,
+}
+impl Atomic for InstantiateTplCall {
+	type Variant = OwnedVariant;
+	type Data = ();
+}
+impl OwnedAtom for InstantiateTplCall {
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	type Refs = Never;
+	// Technically must be supported but shouldn't actually ever be called
+	async fn call_ref(&self, arg: Expr) -> GExpr {
+		eprintln!(
+			"Copying partially applied instantiate_tpl call. This is an internal value.\
+      \nIt should be fully consumed within generated code."
+		);
+		self.clone().call(arg).await
+	}
+	async fn call(mut self, arg: Expr) -> GExpr {
+		match TypAtom::<MacTree>::try_from_expr(arg).await {
+			Err(e) => return Err::<Never, _>(e).to_expr().await,
+			Ok(t) => self.argv.push(own(t).await),
+		};
+		if self.argv.len() < self.argc {
+			return self.to_expr().await;
+		}
+		let mut args = self.argv.into_iter();
+		let ret = map_mactree(&self.tpl, &mut false, &mut |mt| match mt.tok() {
+			MacTok::Slot => Some(args.next().expect("Not enough arguments to fill all slots")),
+			_ => None,
+		});
+		assert!(args.next().is_none(), "Too many arguments for all slots");
+		ret.to_expr().await
+	}
+}

orchid-std/src/macros/let_line.rs

@@ -0,0 +1,120 @@
+use std::pin::pin;
+
+use futures::{FutureExt, StreamExt, stream};
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::{OrcRes, Reporter};
+use orchid_base::name::Sym;
+use orchid_base::parse::{
+	Comment, ParseCtx, Parsed, Snippet, expect_tok, token_errv, try_pop_no_fluff,
+};
+use orchid_base::sym;
+use orchid_base::tree::Paren;
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::parser::{ConstCtx, PSnippet, PTok, PTokTree, ParsCtx, ParsedLine, Parser};
+
+use crate::macros::mactree::{MacTok, MacTree, glossary_v, map_mactree_v};
+
+#[derive(Default)]
+pub struct LetLine;
+impl Parser for LetLine {
+	const LINE_HEAD: &'static str = "let";
+	async fn parse<'a>(
+		ctx: ParsCtx<'a>,
+		exported: bool,
+		comments: Vec<Comment>,
+		line: PSnippet<'a>,
+	) -> OrcRes<Vec<ParsedLine>> {
+		let sr = line.sr();
+		let Parsed { output: name_tok, tail } = try_pop_no_fluff(&ctx, line).await?;
+		let Some(name) = name_tok.as_name() else {
+			let err = token_errv(&ctx, name_tok, "Constant must have a name", |t| {
+				format!("Expected a name but found {t}")
+			});
+			return Err(err.await);
+		};
+		let Parsed { tail, .. } = expect_tok(&ctx, tail, ctx.i().i("=").await).await?;
+		let aliased = parse_tokv(tail, &ctx).await;
+		Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name, async move |ctx| {
+			let rep = Reporter::new();
+			let dealiased = dealias_mac_v(aliased, &ctx, &rep).await;
+			let macro_input = MacTok::S(Paren::Round, dealiased).at(sr.pos());
+			if let Some(e) = rep.res() {
+				return Err(e);
+			}
+			Ok(call([
+				sym_ref(sym!(macros::lower; ctx.i()).await),
+				call([sym_ref(sym!(macros::resolve; ctx.i()).await), atom(macro_input)]),
+			]))
+		})])
+	}
+}
+
+pub async fn dealias_mac_v(aliased: Vec<MacTree>, ctx: &ConstCtx, rep: &Reporter) -> Vec<MacTree> {
+	let keys = glossary_v(&aliased).collect_vec();
+	let mut names: HashMap<_, _> = HashMap::new();
+	let mut stream = pin!(ctx.names(&keys).zip(stream::iter(&keys)));
+	while let Some((canonical, local)) = stream.next().await {
+		match canonical {
+			Err(e) => rep.report(e),
+			Ok(name) => {
+				names.insert(local.clone(), name);
+			},
+		}
+	}
+	map_mactree_v(&aliased, &mut false, &mut |tree| match &*tree.tok {
+		MacTok::Name(n) => names.get(n).map(|new_n| MacTok::Name(new_n.clone()).at(tree.pos())),
+		_ => None,
+	})
+}
+
+pub async fn parse_tokv(line: PSnippet<'_>, ctx: &impl ParseCtx) -> Vec<MacTree> {
+	if let Some((idx, arg)) = line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?))) {
+		let (head, lambda) = line.split_at(idx as u32);
+		let (_, body) = lambda.pop_front().unwrap();
+		let body = parse_tokv(body, ctx).boxed_local().await;
+		let mut all = parse_tokv_no_lambdas(&head, ctx).await;
+		match parse_tok(arg, ctx).await {
+			Some(arg) => all.push(MacTok::Lambda(arg, body).at(lambda.sr().pos())),
+			None => ctx.rep().report(
+				token_errv(ctx, arg, "Lambda argument fluff", |arg| {
+					format!("Lambda arguments must be a valid token, found meaningless fragment {arg}")
+				})
+				.await,
+			),
+		};
+		all
+	} else {
+		parse_tokv_no_lambdas(&line, ctx).await
+	}
+}
+
+async fn parse_tokv_no_lambdas(line: &[PTokTree], ctx: &impl ParseCtx) -> Vec<MacTree> {
+	stream::iter(line).filter_map(|tt| parse_tok(tt, ctx)).collect().await
+}
+
+pub async fn parse_tok(tree: &PTokTree, ctx: &impl ParseCtx) -> Option<MacTree> {
+	let tok = match &tree.tok {
+		PTok::Bottom(errv) => MacTok::Bottom(errv.clone()),
+		PTok::BR | PTok::Comment(_) => return None,
+		PTok::Name(n) => MacTok::Name(Sym::new([n.clone()], ctx.i()).await.unwrap()),
+		PTok::NS(..) => match tree.as_multiname() {
+			Ok(mn) => MacTok::Name(mn.to_sym(ctx.i()).await),
+			Err(nested) => {
+				ctx.rep().report(
+					token_errv(ctx, tree, ":: can only be followed by a name in an expression", |tok| {
+						format!("Expected name, found {tok}")
+					})
+					.await,
+				);
+				return parse_tok(nested, ctx).boxed_local().await;
+			},
+		},
+		PTok::Handle(expr) => MacTok::Value(expr.clone()),
+		PTok::NewExpr(never) => match *never {},
+		PTok::LambdaHead(_) => panic!("Lambda-head handled in the sequence parser"),
+		PTok::S(p, body) =>
+			MacTok::S(*p, parse_tokv(Snippet::new(tree, body), ctx).boxed_local().await),
+	};
+	Some(tok.at(tree.sr().pos()))
+}

orchid-std/src/macros/macro_lib.rs

@@ -0,0 +1,83 @@
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::error::Reporter;
+use orchid_base::sym;
+use orchid_extension::atom::TypAtom;
+use orchid_extension::atom_owned::own;
+use orchid_extension::conv::ToExpr;
+use orchid_extension::coroutine_exec::exec;
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::reflection::{ReflMemKind, refl};
+use orchid_extension::tree::{GenMember, comments, fun, prefix};
+use substack::Substack;
+
+use crate::Int;
+use crate::macros::instantiate_tpl::InstantiateTplCall;
+use crate::macros::macro_line::{Macro, Matcher};
+use crate::macros::mactree::{LowerCtx, MacTree};
+use crate::macros::recur_state::RecurState;
+use crate::macros::resolve::{ResolveCtx, resolve};
+
+pub fn gen_macro_lib() -> Vec<GenMember> {
+	prefix("macros", [
+		comments(
+			["This is an internal function, you can't obtain a value of its argument type.", "hidden"],
+			fun(true, "instantiate_tpl", |tpl: TypAtom<MacTree>, right: Int| async move {
+				InstantiateTplCall {
+					tpl: own(tpl).await,
+					argc: right.0.try_into().unwrap(),
+					argv: Vec::new(),
+				}
+			}),
+		),
+		fun(true, "resolve", |tpl: TypAtom<MacTree>| async move {
+			call([
+				sym_ref(sym!(macros::resolve_recur; tpl.untyped.ctx().i()).await),
+				atom(RecurState::Bottom),
+				tpl.untyped.ex().to_expr().await,
+			])
+		}),
+		fun(true, "lower", |tpl: TypAtom<MacTree>| async move {
+			let ctx = LowerCtx { sys: tpl.untyped.ctx().clone(), rep: &Reporter::new() };
+			let res = own(tpl).await.lower(ctx, Substack::Bottom).await;
+			if let Some(e) = Reporter::new().res() { Err(e) } else { Ok(res) }
+		}),
+		fun(true, "resolve_recur", |state: TypAtom<RecurState>, tpl: TypAtom<MacTree>| async move {
+			exec("macros::resolve_recur", async move |mut h| {
+				let ctx = tpl.ctx().clone();
+				let root = refl(&ctx);
+				let tpl = own(tpl.clone()).await;
+				let mut macros = HashMap::new();
+				for n in tpl.glossary() {
+					if let Ok(ReflMemKind::Const) = root.get_by_path(n).await.map(|m| m.kind()) {
+						let Ok(mac) = h.exec::<TypAtom<Macro>>(sym_ref(n.clone())).await else { continue };
+						let mac = own(mac).await;
+						macros.entry(mac.0.own_kws[0].clone()).or_insert(mac);
+					}
+				}
+				let mut named = HashMap::new();
+				let mut priod = Vec::new();
+				for (_, mac) in macros.iter() {
+					for rule in mac.0.rules.iter() {
+						if rule.glossary.is_subset(tpl.glossary()) {
+							match &rule.pattern {
+								Matcher::Named(m) =>
+									named.entry(m.head()).or_insert(Vec::new()).push((m, mac, rule)),
+								Matcher::Priod(p) => priod.push((mac.0.prio, (p, mac, rule))),
+							}
+						}
+					}
+				}
+				let priod = priod.into_iter().sorted_unstable_by_key(|(p, _)| *p).map(|(_, r)| r).collect();
+				let mut rctx = ResolveCtx { h, recur: own(state).await, ctx: ctx.clone(), named, priod };
+				let resolve_res = resolve(&mut rctx, &tpl).await;
+				std::mem::drop(rctx);
+				match resolve_res {
+					Some(out_tree) => out_tree.to_expr().await,
+					None => tpl.to_expr().await,
+				}
+			})
+			.await
+		}),
+	])
+}

orchid-std/src/macros/macro_line.rs

@@ -0,0 +1,228 @@
+use std::borrow::Cow;
+use std::cell::RefCell;
+use std::rc::Rc;
+
+use async_once_cell::OnceCell;
+use futures::{StreamExt, stream};
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools;
+use never::Never;
+use orchid_base::error::{OrcRes, Reporter, mk_err};
+use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::parse::{
+	Comment, ParseCtx, Parsed, Snippet, expect_end, expect_tok, line_items, token_errv,
+	try_pop_no_fluff,
+};
+use orchid_base::tree::{Paren, Token};
+use orchid_base::{clone, sym};
+use orchid_extension::atom::{Atomic, TypAtom};
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::parser::{PSnippet, ParsCtx, ParsedLine, Parser};
+
+use crate::macros::let_line::{dealias_mac_v, parse_tokv};
+use crate::macros::mactree::{glossary_v, map_mactree_v};
+use crate::macros::recur_state::{RecurState, RulePath};
+use crate::macros::rule::matcher::{NamedMatcher, PriodMatcher};
+use crate::{Int, MacTok};
+
+#[derive(Default)]
+pub struct MacroLine;
+impl Parser for MacroLine {
+	const LINE_HEAD: &'static str = "macro";
+	async fn parse<'a>(
+		ctx: ParsCtx<'a>,
+		exported: bool,
+		comments: Vec<Comment>,
+		line: PSnippet<'a>,
+	) -> OrcRes<Vec<ParsedLine>> {
+		if exported {
+			return Err(mk_err(
+				ctx.i().i("macros are always exported").await,
+				"The export keyword is forbidden here to avoid confusion\n\
+					because macros are exported by default",
+				[line.sr()],
+			));
+		}
+		let module = ctx.module();
+		let Parsed { output, tail } = try_pop_no_fluff(&ctx, line).await?;
+		let bad_first_item_err = || {
+			token_errv(&ctx, output, "Expected priority or block", |s| {
+				format!("Expected a priority number or a () block, found {s}")
+			})
+		};
+		let (prio, body) = match &output.tok {
+			Token::S(Paren::Round, body) => (None, body),
+			Token::Handle(expr) => match TypAtom::<Int>::try_from_expr(expr.clone()).await {
+				Err(e) => {
+					return Err(e + bad_first_item_err().await);
+				},
+				Ok(prio) => {
+					let Token::S(Paren::Round, block) = &output.tok else {
+						return Err(
+							token_errv(&ctx, output, "Expected () block", |s| {
+								format!("Expected a () block, found {s}")
+							})
+							.await,
+						);
+					};
+					(Some(prio), block)
+				},
+			},
+			_ => return Err(bad_first_item_err().await),
+		};
+		expect_end(&ctx, tail).await?;
+		let lines = line_items(&ctx, Snippet::new(output, body)).await;
+		let Some((kw_line, rule_lines)) = lines.split_first() else { return Ok(Vec::new()) };
+		let mut keywords = HashMap::new();
+		let Parsed { tail: kw_tail, .. } =
+			expect_tok(&ctx, kw_line.tail, ctx.i().i("keywords").await).await?;
+		for kw_tok in kw_tail.iter().filter(|kw| !kw.is_fluff()) {
+			match kw_tok.as_name() {
+				Some(kw) => {
+					keywords.insert(kw, kw_tok.sr());
+				},
+				None => ctx.rep().report(
+					token_errv(&ctx, kw_tok, "invalid macro keywords list", |tok| {
+						format!("The keywords list must be a sequence of names; received {tok}")
+					})
+					.await,
+				),
+			}
+		}
+		let Some(macro_name) = keywords.keys().next().cloned() else {
+			return Err(mk_err(
+				ctx.i().i("macro with no keywords").await,
+				"Macros must define at least one macro of their own.",
+				[kw_line.tail.sr()],
+			));
+		};
+		let mut rules = Vec::new();
+		let mut lines = Vec::new();
+		for (idx, line) in rule_lines.iter().enumerate().map(|(n, v)| (n as u32, v)) {
+			let path = RulePath { module: module.clone(), main_kw: macro_name.clone(), rule: idx };
+			let sr = line.tail.sr();
+			let name = ctx.i().i(&path.name()).await;
+			let Parsed { tail, .. } = expect_tok(&ctx, line.tail, ctx.i().i("rule").await).await?;
+			let arrow_token = ctx.i().i("=>").await;
+			let Some((pattern, body)) = tail.split_once(|tok| tok.is_kw(arrow_token.clone())) else {
+				ctx.rep().report(mk_err(
+					ctx.i().i("Missing => in rule").await,
+					"Rule lines are of the form `rule ...pattern => ...body`",
+					[line.tail.sr()],
+				));
+				continue;
+			};
+			let pattern = parse_tokv(pattern, &ctx).await;
+			let mut placeholders = Vec::new();
+			map_mactree_v(&pattern, &mut false, &mut |tok| {
+				if let MacTok::Ph(ph) = tok.tok() {
+					placeholders.push((ph.clone(), tok.pos()))
+				}
+				None
+			});
+			let mut body_mactree = parse_tokv(body, &ctx).await;
+			for (ph, ph_pos) in placeholders.iter().rev() {
+				let name = ctx.module().suffix([ph.name.clone()], ctx.i()).await;
+				body_mactree = vec![
+					MacTok::Lambda(MacTok::Name(name).at(ph_pos.clone()), body_mactree).at(ph_pos.clone()),
+				]
+			}
+			let body_sr = body.sr();
+			rules.push((name.clone(), placeholders, rules.len() as u32, sr.pos(), pattern));
+			lines.push(ParsedLine::cnst(&sr, &line.output, true, name, async move |ctx| {
+				let rep = Reporter::new();
+				let body = dealias_mac_v(body_mactree, &ctx, &rep).await;
+				let macro_input = MacTok::S(Paren::Round, body).at(body_sr.pos());
+				if let Some(e) = rep.res() {
+					return Err(e);
+				}
+				Ok(call([
+					sym_ref(sym!(macros::resolve_recur; ctx.i()).await),
+					atom(RecurState::base(path)),
+					macro_input.to_expr().await,
+				]))
+			}))
+		}
+		let mac_cell = Rc::new(OnceCell::new());
+		let keywords = Rc::new(keywords);
+		let rules = Rc::new(RefCell::new(Some(rules)));
+		for (kw, sr) in &*keywords {
+			clone!(mac_cell, keywords, rules, module, prio);
+			lines.push(ParsedLine::cnst(&sr.clone(), &comments, true, kw.clone(), async move |cctx| {
+				let mac = mac_cell
+					.get_or_init(async {
+						let rep = Reporter::new();
+						let rules = rules.borrow_mut().take().expect("once cell initializer runs");
+						let rules = stream::iter(rules)
+							.then(|(body_name, placeholders, index, pos, pattern_macv)| {
+								let cctx = &cctx;
+								let rep = &rep;
+								let prio = &prio;
+								async move {
+									let pattern_abs = dealias_mac_v(pattern_macv, cctx, rep).await;
+									let glossary = glossary_v(&pattern_abs).collect();
+									let pattern_res = match prio {
+										None => NamedMatcher::new(&pattern_abs, cctx.i()).await.map(Matcher::Named),
+										Some(_) => PriodMatcher::new(&pattern_abs, cctx.i()).await.map(Matcher::Priod),
+									};
+									let placeholders = placeholders.into_iter().map(|(ph, _)| ph.name).collect_vec();
+									match pattern_res {
+										Ok(pattern) =>
+											Some(Rule { index, pos, body_name, pattern, glossary, placeholders }),
+										Err(e) => {
+											rep.report(e);
+											None
+										},
+									}
+								}
+							})
+							.flat_map(stream::iter)
+							.collect::<Vec<_>>()
+							.await;
+						let own_kws = keywords.keys().cloned().collect_vec();
+						Macro(Rc::new(MacroData { module, prio: prio.map(|i| i.0 as u64), rules, own_kws }))
+					})
+					.await;
+				atom(mac.clone())
+			}))
+		}
+		Ok(lines)
+	}
+}
+
+#[derive(Debug)]
+pub struct MacroData {
+	pub module: Sym,
+	pub prio: Option<u64>,
+	pub rules: Vec<Rule>,
+	pub own_kws: Vec<IStr>,
+}
+
+#[derive(Clone, Debug)]
+pub struct Macro(pub Rc<MacroData>);
+#[derive(Debug)]
+pub struct Rule {
+	pub index: u32,
+	pub pos: Pos,
+	pub pattern: Matcher,
+	pub glossary: HashSet<Sym>,
+	pub placeholders: Vec<IStr>,
+	pub body_name: IStr,
+}
+#[derive(Debug)]
+pub enum Matcher {
+	Named(NamedMatcher),
+	Priod(PriodMatcher),
+}
+impl Atomic for Macro {
+	type Data = ();
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for Macro {
+	type Refs = Never;
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+}