Traditional route appears to work

Beginnings of dylib extensions, entirely untestted

Cargo.toml

@@ -2,14 +2,16 @@
 resolver = "2"
 
 members = [
-  "orcx",
-  "orchid-std",
-  "orchid-host",
-  "orchid-extension",
-  "orchid-base",
-  "orchid-api",
-  "orchid-api-derive",
-  "orchid-api-traits",
-  "stdio-perftest",
-  "xtask", "async-fn-stream",
+	"orcx",
+	"orchid-std",
+	"orchid-host",
+	"orchid-extension",
+	"orchid-base",
+	"orchid-api",
+	"orchid-api-derive",
+	"orchid-api-traits",
+	"stdio-perftest",
+	"xtask",
+	"async-fn-stream",
+	"unsync-pipe",
 ]

async-fn-stream/Cargo.toml

@@ -7,4 +7,4 @@ edition = "2024"
 futures = { version = "0.3.31", features = ["std"], default-features = false }
 
 [dev-dependencies]
-test_executors = "0.3.5"
+test_executors = "0.4.1"

examples/hello-world/main.orc

@@ -1,6 +1,6 @@
 let my_tuple = option::some t[1, 2]
 
 let main = match my_tuple {
-	option::of t[ref head, ..] => head;
-	option::empty => "foo";
+	option::some t[ref head, ..] => head;
+	option::none => "foo";
 }

orchid-api-derive/Cargo.toml

@@ -9,8 +9,8 @@ proc-macro = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-quote = "1.0.40"
-syn = { version = "2.0.106" }
+quote = "1.0.42"
+syn = { version = "2.0.112" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-proc-macro2 = "1.0.101"
+proc-macro2 = "1.0.104"
 itertools = "0.14.0"

orchid-api-derive/src/decode.rs

@@ -14,8 +14,8 @@ pub fn derive(input: TokenStream) -> TokenStream {
 		impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause {
 			async fn decode<R: orchid_api_traits::AsyncRead + ?Sized>(
 				mut read: std::pin::Pin<&mut R>
-			) -> Self {
-				#decode
+			) -> std::io::Result<Self> {
+				Ok(#decode)
 			}
 		}
 	};
@@ -30,7 +30,7 @@ fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream {
 				let syn::Field { ty, ident, .. } = &f;
 				quote! {
 					#ident : (Box::pin(< #ty as orchid_api_traits::Decode>::decode(read.as_mut()))
-						as std::pin::Pin<Box<dyn std::future::Future<Output = _>>>).await
+						as std::pin::Pin<Box<dyn std::future::Future<Output = std::io::Result<_>>>>).await?
 				}
 			});
 			quote! { { #( #exprs, )* } }
@@ -40,7 +40,7 @@ fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream {
 				let ty = &field.ty;
 				quote! {
 					(Box::pin(< #ty as orchid_api_traits::Decode>::decode(read.as_mut()))
-						as std::pin::Pin<Box<dyn std::future::Future<Output = _>>>).await,
+						as std::pin::Pin<Box<dyn std::future::Future<Output = std::io::Result<_>>>>).await?,
 				}
 			});
 			quote! { ( #( #exprs )* ) }
@@ -62,7 +62,7 @@ fn decode_body(data: &syn::Data) -> proc_macro2::TokenStream {
 				quote! { #id => Self::#ident #fields, }
 			});
 			quote! {
-				match <u8 as orchid_api_traits::Decode>::decode(read.as_mut()).await {
+				match <u8 as orchid_api_traits::Decode>::decode(read.as_mut()).await? {
 					#(#opts)*
 					x => panic!("Unrecognized enum kind {x}")
 				}

orchid-api-derive/src/encode.rs

@@ -17,8 +17,9 @@ pub fn derive(input: TokenStream) -> TokenStream {
 			async fn encode<W: orchid_api_traits::AsyncWrite + ?Sized>(
 				&self,
 				mut write: std::pin::Pin<&mut W>
-			) {
-				#encode
+			) -> std::io::Result<()> {
+				#encode;
+				Ok(())
 			}
 		}
 	};
@@ -43,7 +44,7 @@ fn encode_body(data: &syn::Data) -> Option<pm2::TokenStream> {
 				quote! {
 					Self::#ident #dest => {
 						(Box::pin((#i as u8).encode(write.as_mut()))
-							as std::pin::Pin<Box<dyn std::future::Future<Output = _>>>).await;
+							as std::pin::Pin<Box<dyn std::future::Future<Output = std::io::Result<()>>>>).await?;
 						#body
 					}
 				}
@@ -61,7 +62,7 @@ fn encode_body(data: &syn::Data) -> Option<pm2::TokenStream> {
 fn encode_names<T: ToTokens>(names: impl Iterator<Item = T>) -> pm2::TokenStream {
 	quote! { #(
 		(Box::pin(#names .encode(write.as_mut()))
-			as std::pin::Pin<Box<dyn std::future::Future<Output = _>>>).await;
+			as std::pin::Pin<Box<dyn std::future::Future<Output = std::io::Result<()>>>>).await?;
 	)* }
 }
 

orchid-api-traits/Cargo.toml

@@ -6,8 +6,7 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 futures = { version = "0.3.31", features = ["std"], default-features = false }
 itertools = "0.14.0"
 never = "0.1.0"
-ordered-float = "5.0.0"
+ordered-float = "5.1.0"

orchid-api-traits/src/coding.rs

@@ -1,33 +1,44 @@
 use std::collections::HashMap;
 use std::future::Future;
 use std::hash::Hash;
+use std::io;
 use std::num::NonZero;
 use std::ops::{Range, RangeInclusive};
 use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
 
-use async_fn_stream::stream;
-use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, StreamExt};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 use never::Never;
 use ordered_float::NotNan;
 
-use crate::encode_enum;
+use crate::{decode_err, decode_err_for, encode_enum, spin_on};
 
-pub trait Decode: 'static {
+pub trait Decode: 'static + Sized {
 	/// Decode an instance from the beginning of the buffer. Return the decoded
 	/// data and the remaining buffer.
-	fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
+	fn decode<R: AsyncRead + ?Sized>(
+		read: Pin<&mut R>,
+	) -> impl Future<Output = io::Result<Self>> + '_;
+	fn decode_slice(slc: &mut &[u8]) -> Self {
+		spin_on(Self::decode(Pin::new(slc) as Pin<&mut _>)).expect("Decode from slice cannot fail")
+	}
 }
 pub trait Encode {
 	/// Append an instance of the struct to the buffer
-	fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
+	fn encode<W: AsyncWrite + ?Sized>(
+		&self,
+		write: Pin<&mut W>,
+	) -> impl Future<Output = io::Result<()>>;
+	fn encode_vec(&self, vec: &mut Vec<u8>) {
+		spin_on(self.encode(Pin::new(vec) as Pin<&mut _>)).expect("Encode to vector cannot fail")
+	}
 }
 pub trait Coding: Encode + Decode + Clone {
-	fn get_decoder<T: 'static, F: Future<Output = T> + 'static>(
-		map: impl Fn(Self) -> F + Clone + 'static,
-	) -> impl AsyncFn(Pin<&mut dyn AsyncRead>) -> T {
-		async move |r| map(Self::decode(r).await).await
+	fn get_decoder<T: 'static>(
+		map: impl AsyncFn(Self) -> T + Clone + 'static,
+	) -> impl AsyncFn(Pin<&mut dyn AsyncRead>) -> io::Result<T> {
+		async move |r| Ok(map(Self::decode(r).await?).await)
 	}
 }
 impl<T: Encode + Decode + Clone> Coding for T {}
@@ -35,15 +46,15 @@ impl<T: Encode + Decode + Clone> Coding for T {}
 macro_rules! num_impl {
 	($number:ty) => {
 		impl Decode for $number {
-			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
 				let mut bytes = [0u8; (<$number>::BITS / 8) as usize];
-				read.read_exact(&mut bytes).await.unwrap();
-				<$number>::from_be_bytes(bytes)
+				read.read_exact(&mut bytes).await?;
+				Ok(<$number>::from_be_bytes(bytes))
 			}
 		}
 		impl Encode for $number {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-				write.write_all(&self.to_be_bytes()).await.expect("Could not write number")
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+				write.write_all(&self.to_be_bytes()).await
 			}
 		}
 	};
@@ -62,12 +73,12 @@ num_impl!(i8);
 macro_rules! nonzero_impl {
 	($name:ty) => {
 		impl Decode for NonZero<$name> {
-			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
-				Self::new(<$name as Decode>::decode(read).await).unwrap()
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> io::Result<Self> {
+				Self::new(<$name as Decode>::decode(read).await?).ok_or_else(decode_err)
 			}
 		}
 		impl Encode for NonZero<$name> {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
 				self.get().encode(write).await
 			}
 		}
@@ -86,22 +97,22 @@ nonzero_impl!(i64);
 nonzero_impl!(i128);
 
 impl<T: Encode + ?Sized> Encode for &T {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
 		(**self).encode(write).await
 	}
 }
 macro_rules! float_impl {
 	($t:ty, $size:expr) => {
 		impl Decode for NotNan<$t> {
-			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
 				let mut bytes = [0u8; $size];
-				read.read_exact(&mut bytes).await.unwrap();
-				NotNan::new(<$t>::from_be_bytes(bytes)).expect("Float was NaN")
+				read.read_exact(&mut bytes).await?;
+				NotNan::new(<$t>::from_be_bytes(bytes)).map_err(|_| decode_err())
 			}
 		}
 		impl Encode for NotNan<$t> {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-				write.write_all(&self.as_ref().to_be_bytes()).await.expect("Could not write number")
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+				write.write_all(&self.as_ref().to_be_bytes()).await
 			}
 		}
 	};
@@ -111,78 +122,77 @@ float_impl!(f64, 8);
 float_impl!(f32, 4);
 
 impl Decode for String {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len: usize = u64::decode(read.as_mut()).await.try_into().unwrap();
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+		let len: usize = u64::decode(read.as_mut()).await?.try_into().map_err(decode_err_for)?;
 		let mut data = vec![0u8; len];
-		read.read_exact(&mut data).await.unwrap();
-		std::str::from_utf8(&data).expect("String invalid UTF-8").to_owned()
+		read.read_exact(&mut data).await?;
+		Ok(std::str::from_utf8(&data).map_err(decode_err_for)?.to_owned())
 	}
 }
 impl Encode for String {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		write.write_all(self.as_bytes()).await.unwrap()
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+		u64::try_from(self.len()).map_err(decode_err_for)?.encode(write.as_mut()).await?;
+		write.write_all(self.as_bytes()).await
 	}
 }
 impl Encode for str {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		write.write_all(self.as_bytes()).await.unwrap()
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+		u64::try_from(self.len()).map_err(decode_err_for)?.encode(write.as_mut()).await?;
+		write.write_all(self.as_bytes()).await
 	}
 }
 impl<T: Decode> Decode for Vec<T> {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len = u64::decode(read.as_mut()).await;
-		stream(async |mut cx| {
-			for _ in 0..len {
-				cx.emit(T::decode(read.as_mut()).await).await
-			}
-		})
-		.collect()
-		.await
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+		let len = u64::decode(read.as_mut()).await?;
+		let mut values = Vec::with_capacity(len.try_into().map_err(decode_err_for)?);
+		for _ in 0..len {
+			values.push(T::decode(read.as_mut()).await?);
+		}
+		Ok(values)
 	}
 }
 impl<T: Encode> Encode for Vec<T> {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
 		self.as_slice().encode(write).await
 	}
 }
 impl<T: Encode> Encode for [T] {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await?;
 		for t in self.iter() {
-			t.encode(write.as_mut()).await
+			t.encode(write.as_mut()).await?
 		}
+		Ok(())
 	}
 }
 impl<T: Decode> Decode for Option<T> {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		match u8::decode(read.as_mut()).await {
-			0 => None,
-			1 => Some(T::decode(read).await),
-			x => panic!("{x} is not a valid option value"),
-		}
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+		Ok(match bool::decode(read.as_mut()).await? {
+			false => None,
+			true => Some(T::decode(read).await?),
+		})
 	}
 }
 impl<T: Encode> Encode for Option<T> {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		let t = if let Some(t) = self { t } else { return 0u8.encode(write.as_mut()).await };
-		1u8.encode(write.as_mut()).await;
-		t.encode(write).await;
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+		self.is_some().encode(write.as_mut()).await?;
+		if let Some(t) = self {
+			t.encode(write).await?
+		}
+		Ok(())
 	}
 }
 impl<T: Decode, E: Decode> Decode for Result<T, E> {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		match u8::decode(read.as_mut()).await {
-			0 => Self::Ok(T::decode(read).await),
-			1 => Self::Err(E::decode(read).await),
-			x => panic!("Invalid Result tag {x}"),
-		}
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+		Ok(match bool::decode(read.as_mut()).await? {
+			false => Self::Ok(T::decode(read).await?),
+			true => Self::Err(E::decode(read).await?),
+		})
 	}
 }
 
 impl<T: Encode, E: Encode> Encode for Result<T, E> {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
 		match self {
 			Ok(t) => encode_enum(write, 0, |w| t.encode(w)).await,
 			Err(e) => encode_enum(write, 1, |w| e.encode(w)).await,
@@ -190,36 +200,37 @@ impl<T: Encode, E: Encode> Encode for Result<T, E> {
 	}
 }
 impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len = u64::decode(read.as_mut()).await;
-		stream(async |mut cx| {
-			for _ in 0..len {
-				cx.emit(<(K, V)>::decode(read.as_mut()).await).await
-			}
-		})
-		.collect()
-		.await
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+		let len = u64::decode(read.as_mut()).await?;
+		let mut map = HashMap::with_capacity(len.try_into().map_err(decode_err_for)?);
+		for _ in 0..len {
+			map.insert(K::decode(read.as_mut()).await?, V::decode(read.as_mut()).await?);
+		}
+		Ok(map)
 	}
 }
 impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		for pair in self.iter() {
-			pair.encode(write.as_mut()).await
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await?;
+		for (key, value) in self.iter() {
+			key.encode(write.as_mut()).await?;
+			value.encode(write.as_mut()).await?;
 		}
+		Ok(())
 	}
 }
 macro_rules! tuple {
   (($($t:ident)*) ($($T:ident)*)) => {
     impl<$($T: Decode),*> Decode for ($($T,)*) {
-			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-				($($T::decode(read.as_mut()).await,)*)
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+				Ok(($($T::decode(read.as_mut()).await?,)*))
 			}
     }
     impl<$($T: Encode),*> Encode for ($($T,)*) {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
         let ($($t,)*) = self;
-        $( $t.encode(write.as_mut()).await; )*
+        $( $t.encode(write.as_mut()).await?; )*
+				Ok(())
       }
     }
   };
@@ -243,63 +254,67 @@ tuple!((t u v x y z a b c d e f g h i) (T U V X Y Z A B C D E F G H I));
 tuple!((t u v x y z a b c d e f g h i j) (T U V X Y Z A B C D E F G H I J)); // 16
 
 impl Decode for () {
-	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {}
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> io::Result<Self> { Ok(()) }
 }
 impl Encode for () {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) {}
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) -> io::Result<()> { Ok(()) }
 }
 impl Decode for Never {
-	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(_: Pin<&mut R>) -> io::Result<Self> {
 		unreachable!("A value of Never cannot exist so it can't have been serialized");
 	}
 }
 impl Encode for Never {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
+	async fn encode<W: AsyncWrite + ?Sized>(&self, _: Pin<&mut W>) -> io::Result<()> {
+		match *self {}
+	}
 }
 impl Decode for bool {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
 		let mut buf = [0];
-		read.read_exact(&mut buf).await.unwrap();
-		buf[0] != 0
+		read.read_exact(&mut buf).await?;
+		Ok(buf[0] != 0)
 	}
 }
 impl Encode for bool {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await.unwrap()
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+		write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await
 	}
 }
 impl<T: Decode, const N: usize> Decode for [T; N] {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let v = stream(async |mut cx| {
-			for _ in 0..N {
-				cx.emit(T::decode(read.as_mut()).await).await
-			}
-		})
-		.collect::<Vec<_>>()
-		.await;
-		v.try_into().unwrap_or_else(|_| unreachable!("The length of this stream is statically known"))
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+		let mut v = Vec::with_capacity(N);
+		for _ in 0..N {
+			v.push(T::decode(read.as_mut()).await?);
+		}
+		match v.try_into() {
+			Err(_) => unreachable!("The length of this stream is statically known"),
+			Ok(arr) => Ok(arr),
+		}
 	}
 }
 impl<T: Encode, const N: usize> Encode for [T; N] {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
 		for t in self.iter() {
-			t.encode(write.as_mut()).await
+			t.encode(write.as_mut()).await?
 		}
+		Ok(())
 	}
 }
 
 macro_rules! two_end_range {
   ($this:ident, $name:tt, $op:tt, $start:expr, $end:expr) => {
     impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-				T::decode(read.as_mut()).await $op T::decode(read).await
+			async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+				Ok(T::decode(read.as_mut()).await? $op T::decode(read).await?)
 			}
     }
     impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
         let $this = self;
-        ($start).encode(write.as_mut()).await;
-        ($end).encode(write).await;
+        ($start).encode(write.as_mut()).await?;
+        ($end).encode(write).await?;
+				Ok(())
       }
     }
   }
@@ -311,12 +326,12 @@ two_end_range!(x, RangeInclusive, ..=, x.start(), x.end());
 macro_rules! smart_ptr {
 	($name:tt) => {
 		impl<T: Decode> Decode for $name<T> {
-			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
-				$name::new(T::decode(read).await)
+			async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> io::Result<Self> {
+				Ok($name::new(T::decode(read).await?))
 			}
 		}
 		impl<T: Encode> Encode for $name<T> {
-			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+			async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
 				(**self).encode(write).await
 			}
 		}
@@ -328,12 +343,12 @@ smart_ptr!(Rc);
 smart_ptr!(Box);
 
 impl Decode for char {
-	async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> Self {
-		char::from_u32(u32::decode(read).await).unwrap()
+	async fn decode<R: AsyncRead + ?Sized>(read: Pin<&mut R>) -> io::Result<Self> {
+		char::from_u32(u32::decode(read).await?).ok_or_else(decode_err)
 	}
 }
 impl Encode for char {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) {
+	async fn encode<W: AsyncWrite + ?Sized>(&self, write: Pin<&mut W>) -> io::Result<()> {
 		(*self as u32).encode(write).await
 	}
 }

orchid-api-traits/src/helpers.rs

@@ -1,24 +1,24 @@
-use std::future::Future;
-use std::pin::Pin;
+use std::error::Error;
+use std::io;
+use std::pin::{Pin, pin};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::task::{Context, Poll, Wake};
 
-use futures::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+use futures::{AsyncRead, AsyncReadExt, AsyncWrite};
 use itertools::{Chunk, Itertools};
 
 use crate::Encode;
 
-pub async fn encode_enum<'a, W: AsyncWrite + ?Sized, F: Future<Output = ()>>(
+pub async fn encode_enum<'a, W: AsyncWrite + ?Sized>(
 	mut write: Pin<&'a mut W>,
 	id: u8,
-	f: impl FnOnce(Pin<&'a mut W>) -> F,
-) {
-	id.encode(write.as_mut()).await;
+	f: impl AsyncFnOnce(Pin<&'a mut W>) -> io::Result<()>,
+) -> io::Result<()> {
+	id.encode(write.as_mut()).await?;
 	f(write).await
 }
 
-pub async fn write_exact<W: AsyncWrite + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
-	write.write_all(bytes).await.expect("Failed to write exact bytes")
-}
-
 pub fn print_bytes(b: &[u8]) -> String {
 	(b.iter().map(|b| format!("{b:02x}")))
 		.chunks(4)
@@ -27,16 +27,52 @@ pub fn print_bytes(b: &[u8]) -> String {
 		.join("  ")
 }
 
-pub async fn read_exact<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
+pub async fn read_exact<R: AsyncRead + ?Sized>(
+	mut read: Pin<&mut R>,
+	bytes: &'static [u8],
+) -> io::Result<()> {
 	let mut data = vec![0u8; bytes.len()];
-	read.read_exact(&mut data).await.expect("Failed to read bytes");
-	if data != bytes {
-		panic!("Wrong bytes!\nExpected: {}\nFound: {}", print_bytes(bytes), print_bytes(&data));
+	read.read_exact(&mut data).await?;
+	if data == bytes {
+		Ok(())
+	} else {
+		let msg =
+			format!("Wrong bytes!\nExpected: {}\nFound: {}", print_bytes(bytes), print_bytes(&data));
+		Err(io::Error::new(io::ErrorKind::InvalidData, msg))
 	}
 }
 
-pub async fn enc_vec(enc: &impl Encode) -> Vec<u8> {
+pub fn enc_vec(enc: &impl Encode) -> Vec<u8> {
 	let mut vec = Vec::new();
-	enc.encode(Pin::new(&mut vec)).await;
+	enc.encode_vec(&mut vec);
 	vec
 }
+
+/// Raises a bool flag when called
+struct FlagWaker(AtomicBool);
+impl Wake for FlagWaker {
+	fn wake(self: Arc<Self>) { self.0.store(true, Ordering::Relaxed) }
+}
+
+pub fn spin_on<F: Future>(fut: F) -> F::Output {
+	let flag = AtomicBool::new(false);
+	let flag_waker = Arc::new(FlagWaker(flag));
+	let mut future = pin!(fut);
+	loop {
+		let waker = flag_waker.clone().into();
+		let mut ctx = Context::from_waker(&waker);
+		match future.as_mut().poll(&mut ctx) {
+			// ideally the future should return synchronously
+			Poll::Ready(res) => break res,
+			// poorly written futures may yield and immediately wake
+			Poll::Pending if flag_waker.0.load(Ordering::Relaxed) => (),
+			// there is no external event to wait for, this has to be a deadlock
+			Poll::Pending => panic!("Future inside spin_on cannot block"),
+		};
+	}
+}
+
+pub fn decode_err() -> io::Error { io::Error::new(io::ErrorKind::InvalidData, "Unexpected zero") }
+pub fn decode_err_for(e: impl Error) -> io::Error {
+	io::Error::new(io::ErrorKind::InvalidData, e.to_string())
+}

orchid-api-traits/src/hierarchy.rs

@@ -29,25 +29,21 @@ pub trait Extends: InHierarchy<IsRoot = TLFalse> + Into<Self::Parent> {
 pub trait UnderRootImpl<IsRoot: TLBool>: Sized {
 	type __Root: UnderRoot<IsRoot = TLTrue, Root = Self::__Root>;
 	fn __into_root(self) -> Self::__Root;
-	fn __try_from_root(root: Self::__Root) -> Result<Self, Self::__Root>;
 }
 
 pub trait UnderRoot: InHierarchy {
 	type Root: UnderRoot<IsRoot = TLTrue, Root = Self::Root>;
 	fn into_root(self) -> Self::Root;
-	fn try_from_root(root: Self::Root) -> Result<Self, Self::Root>;
 }
 
 impl<T: InHierarchy + UnderRootImpl<T::IsRoot>> UnderRoot for T {
 	type Root = <Self as UnderRootImpl<<Self as InHierarchy>::IsRoot>>::__Root;
 	fn into_root(self) -> Self::Root { self.__into_root() }
-	fn try_from_root(root: Self::Root) -> Result<Self, Self::Root> { Self::__try_from_root(root) }
 }
 
 impl<T: InHierarchy<IsRoot = TLTrue>> UnderRootImpl<TLTrue> for T {
 	type __Root = Self;
 	fn __into_root(self) -> Self::__Root { self }
-	fn __try_from_root(root: Self::__Root) -> Result<Self, Self::__Root> { Ok(root) }
 }
 
 impl<T: InHierarchy<IsRoot = TLFalse> + Extends> UnderRootImpl<TLFalse> for T {
@@ -57,8 +53,4 @@ impl<T: InHierarchy<IsRoot = TLFalse> + Extends> UnderRootImpl<TLFalse> for T {
 	fn __into_root(self) -> Self::__Root {
 		<Self as Into<<Self as Extends>::Parent>>::into(self).into_root()
 	}
-	fn __try_from_root(root: Self::__Root) -> Result<Self, Self::__Root> {
-		let parent = <Self as Extends>::Parent::try_from_root(root)?;
-		parent.clone().try_into().map_err(|_| parent.into_root())
-	}
 }

orchid-api-traits/src/relations.rs

@@ -1,5 +1,6 @@
 use core::fmt;
-use std::future::Future;
+
+use never::Never;
 
 use super::coding::Coding;
 use crate::helpers::enc_vec;
@@ -8,20 +9,22 @@ pub trait Request: fmt::Debug + Sized + 'static {
 	type Response: fmt::Debug + Coding + 'static;
 }
 
-pub async fn respond<R: Request>(_: &R, rep: R::Response) -> Vec<u8> { enc_vec(&rep).await }
-pub async fn respond_with<R: Request, F: Future<Output = R::Response>>(
-	r: &R,
-	f: impl FnOnce(&R) -> F,
-) -> Vec<u8> {
-	respond(r, f(r).await).await
-}
+pub fn respond<R: Request>(_: &R, rep: R::Response) -> Vec<u8> { enc_vec(&rep) }
 
 pub trait Channel: 'static {
 	type Req: Coding + Sized + 'static;
 	type Notif: Coding + Sized + 'static;
 }
+impl Channel for Never {
+	type Notif = Never;
+	type Req = Never;
+}
 
 pub trait MsgSet: Sync + 'static {
 	type In: Channel;
 	type Out: Channel;
 }
+impl MsgSet for Never {
+	type In = Never;
+	type Out = Never;
+}

orchid-api/Cargo.toml

@@ -6,11 +6,12 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ordered-float = "5.0.0"
+ordered-float = "5.1.0"
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 futures = { version = "0.3.31", features = ["std"], default-features = false }
 itertools = "0.14.0"
+unsync-pipe = { version = "0.2.0", path = "../unsync-pipe" }
 
 [dev-dependencies]
-test_executors = "0.3.5"
+test_executors = "0.4.1"

orchid-api/src/binary.rs

@@ -0,0 +1,90 @@
+//! # Binary extension definition
+//!
+//! A binary extension is a DLL / shared object / dylib with a symbol called
+//! `orchid_extension_main` which accepts a single argument of type
+//! [ExtensionContext]. Once that is received, communication continuees through
+//! the channel with the same protocol outlined in [crate::proto]
+
+use unsync_pipe::{Reader, Writer};
+
+/// !Send !Sync owned waker
+///
+/// This object is [Clone] for convenience but it has `drop` and no `clone` so
+/// interactions must reflect a single logical owner
+#[derive(Clone, Copy)]
+#[repr(C)]
+pub struct OwnedWakerVT {
+	pub data: *const (),
+	/// `self`
+	pub drop: extern "C" fn(*const ()),
+	/// `self`
+	pub wake: extern "C" fn(*const ()),
+	/// `&self`
+	pub wake_ref: extern "C" fn(*const ()),
+}
+
+/// !Send !Sync, equivalent to `&mut Context<'a>`, hence no `drop`.
+/// When received in [FutureVT::poll], it must not outlive the call.
+///
+/// You cannot directly wake using this waker, because such a trampoline would
+/// pass through the binary interface twice for no reason. An efficient
+/// implementation should implement that trampoline action internally, whereas
+/// an inefficient but compliant implementation can clone a fresh waker and use
+/// it up.
+#[derive(Clone, Copy)]
+#[repr(C)]
+pub struct FutureContextVT {
+	pub data: *const (),
+	/// `&self`
+	pub waker: extern "C" fn(*const ()) -> OwnedWakerVT,
+}
+
+/// ABI-stable `Poll<()>`
+#[derive(Clone, Copy)]
+#[repr(C)]
+pub enum UnitPoll {
+	Pending,
+	Ready,
+}
+
+/// ABI-stable `Pin<Box<dyn Future<Output = ()>>>`
+///
+/// This object is [Clone] for convenience, but it has `drop` and no `clone` so
+/// interactions must reflect a single logical owner
+#[derive(Clone, Copy)]
+#[repr(C)]
+pub struct FutureVT {
+	pub data: *const (),
+	/// `self`
+	pub drop: extern "C" fn(*const ()),
+	/// `&mut self` Equivalent to [Future::poll]
+	pub poll: extern "C" fn(*const (), FutureContextVT) -> UnitPoll,
+}
+
+/// Handle for a runtime that allows its holder to spawn futures across dynamic
+/// library boundaries
+#[derive(Clone, Copy)]
+#[repr(C)]
+pub struct Spawner {
+	pub data: *const (),
+	/// `self`
+	pub drop: extern "C" fn(*const ()),
+	/// `&self` Add a future to this extension's task
+	pub spawn: extern "C" fn(*const (), FutureVT),
+}
+
+/// Extension context.
+///
+/// This struct is a plain old value, all of the contained values have a
+/// distinct `drop` member
+#[repr(C)]
+pub struct ExtensionContext {
+	/// Spawns tasks associated with this extension
+	pub spawner: Spawner,
+	/// serialized [crate::HostExtChannel]
+	pub input: Reader,
+	/// serialized [crate::ExtHostChannel]
+	pub output: Writer,
+	/// UTF-8 log stream directly to log service.
+	pub log: Writer,
+}

orchid-api/src/interner.rs

@@ -3,7 +3,7 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{ExtHostReq, HostExtReq};
+use crate::{ExtHostNotif, ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.
@@ -71,18 +71,21 @@ pub struct TStr(pub NonZeroU64);
 pub struct TStrv(pub NonZeroU64);
 
 /// A request to sweep the replica. The master will not be sweeped until all
-/// replicas respond, as it must retain everything the replicas retained
+/// replicas respond. For efficiency, replicas should make sure to send the
+/// [Sweeped] notif before returning.
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct Sweep;
 impl Request for Sweep {
-	type Response = Retained;
+	type Response = ();
 }
 
-/// List of keys in this replica that couldn't be sweeped because local
-/// datastructures reference their value.
-#[derive(Clone, Debug, Coding)]
-pub struct Retained {
+/// List of keys in this replica that were removed during a sweep. This may have
+/// been initiated via a [Sweep] request, but can also be triggered by the
+/// replica autonomously.
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostNotif)]
+pub struct Sweeped {
 	pub strings: Vec<TStr>,
 	pub vecs: Vec<TStrv>,
 }

orchid-api/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod binary;
 mod lexer;
 pub use lexer::*;
 mod format;

orchid-api/src/logging.rs

@@ -1,14 +1,30 @@
+use std::collections::HashMap;
+
 use orchid_api_derive::{Coding, Hierarchy};
 
-use crate::ExtHostNotif;
+use crate::{ExtHostNotif, TStr};
 
+/// Describes what to do with a log stream.
+/// Log streams are unstructured utf8 text unless otherwise stated.
 #[derive(Clone, Debug, Coding, PartialEq, Eq, Hash)]
 pub enum LogStrategy {
-	StdErr,
-	File(String),
+	/// Context-dependent default stream, often stderr
+	Default,
+	/// A file on the local filesystem
+	File { path: String, append: bool },
+	/// Discard any log output
 	Discard,
 }
 
+#[derive(Clone, Debug, Coding)]
+pub struct Logger {
+	pub routing: HashMap<String, LogStrategy>,
+	pub default: Option<LogStrategy>,
+}
+
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostNotif)]
-pub struct Log(pub String);
+pub struct Log {
+	pub category: TStr,
+	pub message: String,
+}

orchid-api/src/proto.rs

@@ -22,51 +22,49 @@
 //! be preserved. Toolkits must ensure that the client code is able to observe
 //! the ordering of messages.
 
+use std::io;
 use std::pin::Pin;
 
-use futures::{AsyncRead, AsyncWrite};
+use futures::{AsyncRead, AsyncWrite, AsyncWriteExt};
 use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact};
+use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact};
 
-use crate::{atom, expr, interner, lexer, logging, parser, system, tree};
+use crate::{Sweeped, atom, expr, interner, lexer, logging, parser, system, tree};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
+#[derive(Clone, Debug)]
 pub struct HostHeader {
-	pub log_strategy: logging::LogStrategy,
-	pub msg_logs: logging::LogStrategy,
+	pub logger: logging::Logger,
 }
 impl Decode for HostHeader {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		read_exact(read.as_mut(), HOST_INTRO).await;
-		Self {
-			log_strategy: logging::LogStrategy::decode(read.as_mut()).await,
-			msg_logs: logging::LogStrategy::decode(read.as_mut()).await,
-		}
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+		read_exact(read.as_mut(), HOST_INTRO).await?;
+		Ok(Self { logger: logging::Logger::decode(read).await? })
 	}
 }
 impl Encode for HostHeader {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write_exact(write.as_mut(), HOST_INTRO).await;
-		self.log_strategy.encode(write.as_mut()).await;
-		self.msg_logs.encode(write.as_mut()).await
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+		write.write_all(HOST_INTRO).await?;
+		self.logger.encode(write.as_mut()).await
 	}
 }
 
 static EXT_INTRO: &[u8] = b"Orchid extension, binary API v0\n";
+#[derive(Clone, Debug)]
 pub struct ExtensionHeader {
 	pub name: String,
 	pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
-	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		read_exact(read.as_mut(), EXT_INTRO).await;
-		Self { name: String::decode(read.as_mut()).await, systems: Vec::decode(read).await }
+	async fn decode<R: AsyncRead + ?Sized>(mut read: Pin<&mut R>) -> io::Result<Self> {
+		read_exact(read.as_mut(), EXT_INTRO).await?;
+		Ok(Self { name: String::decode(read.as_mut()).await?, systems: Vec::decode(read).await? })
 	}
 }
 impl Encode for ExtensionHeader {
-	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write_exact(write.as_mut(), EXT_INTRO).await;
-		self.name.encode(write.as_mut()).await;
+	async fn encode<W: AsyncWrite + ?Sized>(&self, mut write: Pin<&mut W>) -> io::Result<()> {
+		write.write_all(EXT_INTRO).await?;
+		self.name.encode(write.as_mut()).await?;
 		self.systems.encode(write).await
 	}
 }
@@ -99,6 +97,7 @@ pub enum ExtHostReq {
 pub enum ExtHostNotif {
 	ExprNotif(expr::ExprNotif),
 	Log(logging::Log),
+	Sweeped(Sweeped),
 }
 
 pub struct ExtHostChannel;
@@ -155,22 +154,22 @@ impl MsgSet for HostMsgSet {
 
 #[cfg(test)]
 mod tests {
+	use std::collections::HashMap;
+
 	use orchid_api_traits::enc_vec;
 	use ordered_float::NotNan;
 	use test_executors::spin_on;
 
 	use super::*;
+	use crate::Logger;
 
 	#[test]
 	fn host_header_enc() {
 		spin_on(async {
-			let hh = HostHeader {
-				log_strategy: logging::LogStrategy::File("SomeFile".to_string()),
-				msg_logs: logging::LogStrategy::File("SomeFile".to_string()),
-			};
-			let mut enc = &enc_vec(&hh).await[..];
+			let hh = HostHeader { logger: Logger { routing: HashMap::new(), default: None } };
+			let mut enc = &enc_vec(&hh)[..];
 			eprintln!("Encoded to {enc:?}");
-			HostHeader::decode(Pin::new(&mut enc)).await;
+			HostHeader::decode(Pin::new(&mut enc)).await.unwrap();
 			assert_eq!(enc, []);
 		})
 	}
@@ -187,9 +186,9 @@ mod tests {
 					priority: NotNan::new(1f64).unwrap(),
 				}],
 			};
-			let mut enc = &enc_vec(&eh).await[..];
+			let mut enc = &enc_vec(&eh)[..];
 			eprintln!("Encoded to {enc:?}");
-			ExtensionHeader::decode(Pin::new(&mut enc)).await;
+			ExtensionHeader::decode(Pin::new(&mut enc)).await.unwrap();
 			assert_eq!(enc, [])
 		})
 	}

orchid-api/src/tree.rs

@@ -2,7 +2,6 @@ use std::collections::HashMap;
 use std::fmt;
 use std::num::NonZeroU64;
 use std::ops::Range;
-use std::rc::Rc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
@@ -48,7 +47,7 @@ pub enum Token {
 	/// NewExpr(Bottom) because it fails in dead branches too.
 	Bottom(Vec<OrcError>),
 	/// A comment
-	Comment(Rc<String>),
+	Comment(TStr),
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]

orchid-base/Cargo.toml

@@ -6,12 +6,14 @@ edition = "2024"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+unsync-pipe = { version = "0.2.0", path = "../unsync-pipe" }
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
+bound = "0.6.0"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.20"
 futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.16.0"
+hashbrown = "0.16.1"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
 never = "0.1.0"
@@ -19,10 +21,13 @@ num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
-ordered-float = "5.0.0"
-regex = "1.11.2"
-rust-embed = "8.7.2"
-some_executor = "0.6.1"
+ordered-float = "5.1.0"
+regex = "1.12.2"
+rust-embed = "8.9.0"
 substack = "1.1.1"
-test_executors = "0.3.5"
 trait-set = "0.3.0"
+task-local = "0.1.0"
+
+[dev-dependencies]
+futures = "0.3.31"
+test_executors = "0.4.1"

orchid-base/src/binary.rs

@@ -0,0 +1,118 @@
+use std::pin::Pin;
+use std::rc::Rc;
+use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker};
+
+use orchid_api::binary::{FutureContextVT, FutureVT, OwnedWakerVT, UnitPoll};
+
+type WideBox = Box<dyn Future<Output = ()>>;
+
+static OWNED_VTABLE: RawWakerVTable = RawWakerVTable::new(
+	|data| {
+		let data = unsafe { Rc::<OwnedWakerVT>::from_raw(data as *const _) };
+		let val = RawWaker::new(Rc::into_raw(data.clone()) as *const (), &OWNED_VTABLE);
+		// Clone must create a duplicate of the Rc, so it has to be un-leaked, cloned,
+		// then leaked again.
+		let _ = Rc::into_raw(data);
+		val
+	},
+	|data| {
+		// Wake must awaken the task and then clean up the state, so the waker must be
+		// un-leaked
+		let data = unsafe { Rc::<OwnedWakerVT>::from_raw(data as *const _) };
+		(data.wake)(data.data);
+	},
+	|data| {
+		// Wake-by-ref must awaken the task while preserving the future, so the Rc is
+		// untouched
+		let data = unsafe { (data as *const OwnedWakerVT).as_ref() }.unwrap();
+		(data.wake_ref)(data.data);
+	},
+	|data| {
+		// Drop must clean up the state, so the waker must be un-leaked
+		let data = unsafe { Rc::<OwnedWakerVT>::from_raw(data as *const _) };
+		(data.drop)(data.data);
+	},
+);
+
+struct BorrowedWakerData<'a> {
+	go_around: &'a mut bool,
+	cx: FutureContextVT,
+}
+static BORROWED_VTABLE: RawWakerVTable = RawWakerVTable::new(
+	|data| {
+		let data = unsafe { (data as *mut BorrowedWakerData).as_mut() }.unwrap();
+		let owned_data = Rc::<OwnedWakerVT>::new((data.cx.waker)(data.cx.data));
+		RawWaker::new(Rc::into_raw(owned_data) as *const (), &OWNED_VTABLE)
+	},
+	|data| *unsafe { (data as *mut BorrowedWakerData).as_mut() }.unwrap().go_around = true,
+	|data| *unsafe { (data as *mut BorrowedWakerData).as_mut() }.unwrap().go_around = true,
+	|_data| {},
+);
+
+/// Convert a future to a binary-compatible format that can be sent across
+/// dynamic library boundaries
+pub fn future_to_vt<Fut: Future<Output = ()> + 'static>(fut: Fut) -> FutureVT {
+	let wide_box = Box::new(fut) as WideBox;
+	let data = Box::into_raw(Box::new(wide_box));
+	extern "C" fn drop(raw: *const ()) {
+		std::mem::drop(unsafe { Box::<WideBox>::from_raw(raw as *mut _) })
+	}
+	extern "C" fn poll(raw: *const (), cx: FutureContextVT) -> UnitPoll {
+		let mut this = unsafe { Pin::new_unchecked(&mut **(raw as *mut WideBox).as_mut().unwrap()) };
+		loop {
+			let mut go_around = false;
+			let borrowed_waker = unsafe {
+				Waker::from_raw(RawWaker::new(
+					&mut BorrowedWakerData { go_around: &mut go_around, cx } as *mut _ as *const (),
+					&BORROWED_VTABLE,
+				))
+			};
+			let mut ctx = Context::from_waker(&borrowed_waker);
+			let result = this.as_mut().poll(&mut ctx);
+			if matches!(result, Poll::Ready(())) {
+				break UnitPoll::Ready;
+			}
+			if !go_around {
+				break UnitPoll::Pending;
+			}
+		}
+	}
+	FutureVT { data: data as *const _, drop, poll }
+}
+
+struct VirtualFuture {
+	vt: FutureVT,
+}
+impl Unpin for VirtualFuture {}
+impl Future for VirtualFuture {
+	type Output = ();
+	fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+		extern "C" fn waker(raw: *const ()) -> OwnedWakerVT {
+			let waker = unsafe { (raw as *mut Context).as_mut() }.unwrap().waker().clone();
+			let data = Box::into_raw(Box::<Waker>::new(waker)) as *const ();
+			return OwnedWakerVT { data, drop, wake, wake_ref };
+			extern "C" fn drop(raw: *const ()) {
+				std::mem::drop(unsafe { Box::<Waker>::from_raw(raw as *mut Waker) })
+			}
+			extern "C" fn wake(raw: *const ()) {
+				unsafe { Box::<Waker>::from_raw(raw as *mut Waker) }.wake();
+			}
+			extern "C" fn wake_ref(raw: *const ()) {
+				unsafe { (raw as *mut Waker).as_mut() }.unwrap().wake_by_ref();
+			}
+		}
+		let cx = FutureContextVT { data: cx as *mut Context as *const (), waker };
+		let result = (self.vt.poll)(self.vt.data, cx);
+		match result {
+			UnitPoll::Pending => Poll::Pending,
+			UnitPoll::Ready => Poll::Ready(()),
+		}
+	}
+}
+impl Drop for VirtualFuture {
+	fn drop(&mut self) { (self.vt.drop)(self.vt.data) }
+}
+
+/// Receive a future sent across dynamic library boundaries and convert it into
+/// an owned object
+pub fn vt_to_future(vt: FutureVT) -> impl Future<Output = ()> { VirtualFuture { vt } }

orchid-base/src/builtin.rs

@@ -1,34 +0,0 @@
-use std::ops::Deref;
-use std::rc::Rc;
-
-use futures::future::LocalBoxFuture;
-
-use crate::api;
-
-pub type Spawner = Rc<dyn Fn(LocalBoxFuture<'static, ()>)>;
-
-/// The 3 primary contact points with an extension are
-/// - send a message
-/// - wait for a message to arrive
-/// - wait for the extension to stop after exit (this is the implicit Drop)
-///
-/// There are no ordering guarantees about these
-pub trait ExtPort {
-	#[must_use]
-	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()>;
-	#[must_use]
-	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>>;
-}
-
-pub struct ExtInit {
-	pub header: api::ExtensionHeader,
-	pub port: Box<dyn ExtPort>,
-}
-impl ExtInit {
-	pub async fn send(&self, msg: &[u8]) { self.port.send(msg).await }
-	pub async fn recv(&self) -> Option<Vec<u8>> { self.port.recv().await }
-}
-impl Deref for ExtInit {
-	type Target = api::ExtensionHeader;
-	fn deref(&self) -> &Self::Target { &self.header }
-}

orchid-base/src/error.rs

@@ -2,13 +2,16 @@ use std::cell::RefCell;
 use std::ffi::OsStr;
 use std::fmt;
 use std::ops::Add;
+use std::rc::Rc;
 use std::sync::Arc;
 
+use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
+use task_local::task_local;
 
 use crate::api;
-use crate::interner::{Interner, Tok};
+use crate::interner::{IStr, es, is};
 use crate::location::Pos;
 
 /// A point of interest in resolving the error, such as the point where
@@ -24,10 +27,10 @@ impl ErrPos {
 	pub fn new(msg: &str, position: Pos) -> Self {
 		Self { message: Some(Arc::new(msg.to_string())), position }
 	}
-	async fn from_api(api: &api::ErrLocation, i: &Interner) -> Self {
+	async fn from_api(api: &api::ErrLocation) -> Self {
 		Self {
 			message: Some(api.message.clone()).filter(|s| !s.is_empty()),
-			position: Pos::from_api(&api.location, i).await,
+			position: Pos::from_api(&api.location).await,
 		}
 	}
 	fn to_api(&self) -> api::ErrLocation {
@@ -51,7 +54,7 @@ impl fmt::Display for ErrPos {
 
 #[derive(Clone, Debug)]
 pub struct OrcErr {
-	pub description: Tok<String>,
+	pub description: IStr,
 	pub message: Arc<String>,
 	pub positions: Vec<ErrPos>,
 }
@@ -63,16 +66,16 @@ impl OrcErr {
 			locations: self.positions.iter().map(ErrPos::to_api).collect(),
 		}
 	}
-	async fn from_api(api: &api::OrcError, i: &Interner) -> Self {
+	async fn from_api(api: &api::OrcError) -> Self {
 		Self {
-			description: Tok::from_api(api.description, i).await,
+			description: es(api.description).await,
 			message: api.message.clone(),
-			positions: join_all(api.locations.iter().map(|e| ErrPos::from_api(e, i))).await,
+			positions: join_all(api.locations.iter().map(ErrPos::from_api)).await,
 		}
 	}
 }
-impl PartialEq<Tok<String>> for OrcErr {
-	fn eq(&self, other: &Tok<String>) -> bool { self.description == *other }
+impl PartialEq<IStr> for OrcErr {
+	fn eq(&self, other: &IStr) -> bool { self.description == *other }
 }
 impl From<OrcErr> for Vec<OrcErr> {
 	fn from(value: OrcErr) -> Self { vec![value] }
@@ -122,12 +125,10 @@ impl OrcErrv {
 		self.0.iter().flat_map(|e| e.positions.iter().cloned())
 	}
 	pub fn to_api(&self) -> Vec<api::OrcError> { self.0.iter().map(OrcErr::to_api).collect() }
-	pub async fn from_api<'a>(
-		api: impl IntoIterator<Item = &'a api::OrcError>,
-		i: &Interner,
-	) -> Self {
-		Self(join_all(api.into_iter().map(|e| OrcErr::from_api(e, i))).await)
+	pub async fn from_api<'a>(api: impl IntoIterator<Item = &'a api::OrcError>) -> Self {
+		Self(join_all(api.into_iter().map(OrcErr::from_api)).await)
 	}
+	pub fn iter(&self) -> impl Iterator<Item = OrcErr> + '_ { self.0.iter().cloned() }
 }
 impl From<OrcErr> for OrcErrv {
 	fn from(value: OrcErr) -> Self { Self(vec![value]) }
@@ -191,12 +192,12 @@ macro_rules! join_ok {
 	(@VALUES) => { Ok(()) };
 }
 
-pub fn mk_errv_floating(description: Tok<String>, message: impl AsRef<str>) -> OrcErrv {
+pub fn mk_errv_floating(description: IStr, message: impl AsRef<str>) -> OrcErrv {
 	mk_errv::<Pos>(description, message, [])
 }
 
 pub fn mk_errv<I: Into<ErrPos>>(
-	description: Tok<String>,
+	description: IStr,
 	message: impl AsRef<str>,
 	posv: impl IntoIterator<Item = I>,
 ) -> OrcErrv {
@@ -210,45 +211,71 @@ pub fn mk_errv<I: Into<ErrPos>>(
 
 pub async fn async_io_err<I: Into<ErrPos>>(
 	err: std::io::Error,
-	i: &Interner,
 	posv: impl IntoIterator<Item = I>,
 ) -> OrcErrv {
-	mk_errv(i.i(&err.kind().to_string()).await, err.to_string(), posv)
+	mk_errv(is(&err.kind().to_string()).await, err.to_string(), posv)
 }
 
-pub async fn os_str_to_string<'a, I: Into<ErrPos>>(
-	str: &'a OsStr,
-	i: &Interner,
+pub async fn os_str_to_string<I: Into<ErrPos>>(
+	str: &OsStr,
 	posv: impl IntoIterator<Item = I>,
-) -> OrcRes<&'a str> {
+) -> OrcRes<&str> {
 	match str.to_str() {
 		Some(str) => Ok(str),
 		None => Err(mk_errv(
-			i.i("Non-unicode string").await,
+			is("Non-unicode string").await,
 			format!("{str:?} is not representable as unicode"),
 			posv,
 		)),
 	}
 }
 
-pub struct Reporter {
-	errors: RefCell<Vec<OrcErr>>,
+#[derive(Clone, Default)]
+struct Reporter {
+	errors: Rc<RefCell<Vec<OrcErr>>>,
 }
 
-impl Reporter {
-	pub fn report(&self, e: impl Into<OrcErrv>) { self.errors.borrow_mut().extend(e.into()) }
-	pub fn new() -> Self { Self { errors: RefCell::new(vec![]) } }
-	pub fn errv(self) -> Option<OrcErrv> { OrcErrv::new(self.errors.into_inner()).ok() }
-	pub fn merge<T>(self, res: OrcRes<T>) -> OrcRes<T> {
-		match (res, self.errv()) {
-			(res, None) => res,
-			(Ok(_), Some(errv)) => Err(errv),
-			(Err(e), Some(errv)) => Err(e + errv),
-		}
+task_local! {
+	static REPORTER: Reporter;
+}
+
+/// Run the future with a new reporter, and return all errors reported within.
+///
+/// If your future returns [OrcRes], see [try_with_reporter]
+pub async fn with_reporter<T>(fut: impl Future<Output = T>) -> OrcRes<T> {
+	try_with_reporter(fut.map(Ok)).await
+}
+
+/// Run the future with a new reporter, and return all errors either returned or
+/// reported by it
+///
+/// If your future may report errors but always returns an approximate value,
+/// see [with_reporter]
+pub async fn try_with_reporter<T>(fut: impl Future<Output = OrcRes<T>>) -> OrcRes<T> {
+	let rep = Reporter::default();
+	let res = REPORTER.scope(rep.clone(), fut).await;
+	let errors = rep.errors.take();
+	match (res, &errors[..]) {
+		(Ok(t), []) => Ok(t),
+		(Ok(_), [_, ..]) => Err(OrcErrv::new(errors).unwrap()),
+		(Err(e), _) => Err(e.extended(errors)),
 	}
-	pub fn is_empty(&self) -> bool { self.errors.borrow().is_empty() }
 }
 
-impl Default for Reporter {
-	fn default() -> Self { Self::new() }
+pub async fn is_erroring() -> bool {
+	(REPORTER.try_with(|r| !r.errors.borrow().is_empty()))
+		.expect("Sidechannel errors must be caught by a reporter")
+}
+
+/// Report an error that is fatal and prevents a correct output, but
+/// still allows the current task to continue and produce an approximate output.
+/// This can be used for
+pub fn report(e: impl Into<OrcErrv>) {
+	let errv = e.into();
+	REPORTER.try_with(|r| r.errors.borrow_mut().extend(errv.clone())).unwrap_or_else(|_| {
+		panic!(
+			"Unhandled error! Sidechannel errors must be caught by an enclosing call to with_reporter.\n\
+			Error: {errv}"
+		)
+	})
 }

orchid-base/src/format.rs

@@ -3,6 +3,7 @@ use std::cmp::Ordering;
 use std::convert::Infallible;
 use std::future::Future;
 use std::iter;
+use std::marker::PhantomData;
 use std::rc::Rc;
 use std::str::FromStr;
 
@@ -11,7 +12,6 @@ use itertools::{Itertools, chain};
 use never::Never;
 use regex::Regex;
 
-use crate::interner::Interner;
 use crate::{api, match_mapping};
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
@@ -300,16 +300,16 @@ pub fn take_first(unit: &FmtUnit, bounded: bool) -> String {
 	fill_slots(&first.elements, &unit.subs, 0, bounded)
 }
 
-pub async fn take_first_fmt(v: &(impl Format + ?Sized), i: &Interner) -> String {
-	take_first(&v.print(&FmtCtxImpl { i }).await, false)
+pub async fn take_first_fmt(v: &(impl Format + ?Sized)) -> String {
+	take_first(&v.print(&FmtCtxImpl { _foo: PhantomData }).await, false)
 }
 
+#[derive(Default)]
 pub struct FmtCtxImpl<'a> {
-	pub i: &'a Interner,
+	_foo: PhantomData<&'a ()>,
 }
 
 pub trait FmtCtx {
-	fn i(&self) -> &Interner;
 	// fn print_as(&self, p: &(impl Format + ?Sized)) -> impl Future<Output =
 	// String> where Self: Sized {
 	// 	async {
@@ -319,9 +319,7 @@ pub trait FmtCtx {
 	// 	}
 	// }
 }
-impl FmtCtx for FmtCtxImpl<'_> {
-	fn i(&self) -> &Interner { self.i }
-}
+impl FmtCtx for FmtCtxImpl<'_> {}
 
 pub trait Format {
 	#[must_use]
@@ -332,13 +330,10 @@ impl Format for Never {
 }
 
 /// Format with default strategy. Currently equal to [take_first_fmt]
-pub async fn fmt(v: &(impl Format + ?Sized), i: &Interner) -> String { take_first_fmt(v, i).await }
+pub async fn fmt(v: &(impl Format + ?Sized)) -> String { take_first_fmt(v).await }
 /// Format a sequence with default strategy. Currently equal to [take_first_fmt]
-pub async fn fmt_v<F: Format + ?Sized, R: Borrow<F>>(
-	v: impl IntoIterator<Item = R>,
-	i: &Interner,
+pub async fn fmt_v<F: Format + ?Sized>(
+	v: impl IntoIterator<Item: Borrow<F>>,
 ) -> impl Iterator<Item = String> {
-	join_all(v.into_iter().map(|f| async move { take_first_fmt(f.borrow(), i).await }))
-		.await
-		.into_iter()
+	join_all(v.into_iter().map(|f| async move { take_first_fmt(f.borrow()).await })).await.into_iter()
 }

orchid-base/src/interner.rs

@@ -1,310 +1,382 @@
-use std::borrow::Borrow;
+use std::fmt::{Debug, Display};
 use std::future::Future;
-use std::hash::BuildHasher as _;
-use std::num::NonZeroU64;
+use std::hash::Hash;
 use std::ops::Deref;
 use std::rc::Rc;
-use std::sync::atomic;
 use std::{fmt, hash};
 
-use futures::lock::Mutex;
-use hashbrown::{HashMap, HashSet};
-use itertools::Itertools as _;
-use orchid_api_traits::Request;
+use futures::future::LocalBoxFuture;
+use task_local::task_local;
 
 use crate::api;
-use crate::reqnot::{DynRequester, Requester};
 
-/// Clippy crashes while verifying `Tok: Sized` without this and I cba to create
-/// a minimal example
-#[derive(Clone)]
-struct ForceSized<T>(T);
+pub trait IStrHandle: AsRef<str> {
+	fn rc(&self) -> Rc<String>;
+}
+pub trait IStrvHandle: AsRef<[IStr]> {
+	fn rc(&self) -> Rc<Vec<IStr>>;
+}
 
 #[derive(Clone)]
-pub struct Tok<T: Interned> {
-	data: Rc<T>,
-	marker: ForceSized<T::Marker>,
+pub struct IStr(pub api::TStr, pub Rc<dyn IStrHandle>);
+impl IStr {
+	/// Obtain a unique ID for this interned data.
+	///
+	/// NOTICE: the ID is guaranteed to be the same for any interned instance of
+	/// the same value only as long as at least one instance exists. If a value is
+	/// no longer interned, the interner is free to forget about it.
+	pub fn to_api(&self) -> api::TStr { self.0 }
+	pub fn rc(&self) -> Rc<String> { self.1.rc() }
 }
-impl<T: Interned> Tok<T> {
-	pub fn new(data: Rc<T>, marker: T::Marker) -> Self { Self { data, marker: ForceSized(marker) } }
-	pub fn to_api(&self) -> T::Marker { self.marker.0 }
-	pub async fn from_api<M>(marker: M, i: &Interner) -> Self
-	where M: InternMarker<Interned = T> {
-		i.ex(marker).await
-	}
-	pub fn rc(&self) -> Rc<T> { self.data.clone() }
+impl Deref for IStr {
+	type Target = str;
+	fn deref(&self) -> &Self::Target { self.1.as_ref().as_ref() }
 }
-impl<T: Interned> Deref for Tok<T> {
-	type Target = T;
-
-	fn deref(&self) -> &Self::Target { self.data.as_ref() }
+impl Eq for IStr {}
+impl PartialEq for IStr {
+	fn eq(&self, other: &Self) -> bool { self.0 == other.0 }
 }
-impl<T: Interned> Ord for Tok<T> {
-	fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.to_api().cmp(&other.to_api()) }
+impl Hash for IStr {
+	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.0.hash(state) }
 }
-impl<T: Interned> PartialOrd for Tok<T> {
-	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
+impl Display for IStr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.deref()) }
 }
-impl<T: Interned> Eq for Tok<T> {}
-impl<T: Interned> PartialEq for Tok<T> {
-	fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
+impl Debug for IStr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "IStr({self}") }
 }
-impl<T: Interned> hash::Hash for Tok<T> {
-	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.to_api().hash(state) }
+#[derive(Clone)]
+pub struct IStrv(pub api::TStrv, pub Rc<dyn IStrvHandle>);
+impl IStrv {
+	/// Obtain a unique ID for this interned data.
+	///
+	/// NOTICE: the ID is guaranteed to be the same for any interned instance of
+	/// the same value only as long as at least one instance exists. If a value is
+	/// no longer interned, the interner is free to forget about it.
+	pub fn to_api(&self) -> api::TStrv { self.0 }
+	pub fn rc(&self) -> Rc<Vec<IStr>> { self.1.rc() }
 }
-impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{}", &*self.data)
-	}
+impl Deref for IStrv {
+	type Target = [IStr];
+	fn deref(&self) -> &Self::Target { self.1.as_ref().as_ref() }
 }
-impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
+impl Eq for IStrv {}
+impl PartialEq for IStrv {
+	fn eq(&self, other: &Self) -> bool { self.0 == other.0 }
+}
+impl Hash for IStrv {
+	fn hash<H: hash::Hasher>(&self, state: &mut H) { self.0.0.hash(state) }
+}
+impl Display for IStrv {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "Token({} -> {:?})", self.to_api().get_id(), self.data.as_ref())
-	}
-}
-
-pub trait Interned: Eq + hash::Hash + Clone + fmt::Debug + Internable<Interned = Self> {
-	type Marker: InternMarker<Interned = Self> + Sized;
-	fn intern(
-		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> impl Future<Output = Self::Marker>;
-	fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
-}
-
-pub trait Internable: fmt::Debug {
-	type Interned: Interned;
-	fn get_owned(&self) -> Rc<Self::Interned>;
-}
-
-pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash + Sized {
-	type Interned: Interned<Marker = Self>;
-	/// Only called on replicas
-	fn resolve(self, i: &Interner) -> impl Future<Output = Tok<Self::Interned>>;
-	fn get_id(self) -> NonZeroU64;
-	fn from_id(id: NonZeroU64) -> Self;
-}
-
-impl Interned for String {
-	type Marker = api::TStr;
-	async fn intern(
-		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> Self::Marker {
-		req.request(api::InternStr(self.to_string())).await
-	}
-	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.strings }
-}
-impl InternMarker for api::TStr {
-	type Interned = String;
-	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
-		Tok::new(Rc::new(i.0.master.as_ref().unwrap().request(api::ExternStr(self)).await), self)
-	}
-	fn get_id(self) -> NonZeroU64 { self.0 }
-	fn from_id(id: NonZeroU64) -> Self { Self(id) }
-}
-impl Internable for str {
-	type Interned = String;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
-}
-impl Internable for String {
-	type Interned = String;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_string()) }
-}
-
-impl Interned for Vec<Tok<String>> {
-	type Marker = api::TStrv;
-	async fn intern(
-		self: Rc<Self>,
-		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> Self::Marker {
-		req.request(api::InternStrv(self.iter().map(|t| t.to_api()).collect())).await
-	}
-	fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.vecs }
-}
-impl InternMarker for api::TStrv {
-	type Interned = Vec<Tok<String>>;
-	async fn resolve(self, i: &Interner) -> Tok<Self::Interned> {
-		let rep = i.0.master.as_ref().unwrap().request(api::ExternStrv(self)).await;
-		let data = futures::future::join_all(rep.into_iter().map(|m| i.ex(m))).await;
-		Tok::new(Rc::new(data), self)
-	}
-	fn get_id(self) -> NonZeroU64 { self.0 }
-	fn from_id(id: NonZeroU64) -> Self { Self(id) }
-}
-impl Internable for [Tok<String>] {
-	type Interned = Vec<Tok<String>>;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
-}
-impl<const N: usize> Internable for [Tok<String>; N] {
-	type Interned = Vec<Tok<String>>;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
-}
-impl Internable for Vec<Tok<String>> {
-	type Interned = Vec<Tok<String>>;
-	fn get_owned(&self) -> Rc<Self::Interned> { Rc::new(self.to_vec()) }
-}
-// impl Internable for Vec<api::TStr> {
-// 	type Interned = Vec<Tok<String>>;
-// 	fn get_owned(&self) -> Arc<Self::Interned> {
-// 		Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
-// 	}
-// }
-// impl Internable for [api::TStr] {
-// 	type Interned = Vec<Tok<String>>;
-// 	fn get_owned(&self) -> Arc<Self::Interned> {
-// 		Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
-// 	}
-// }
-
-/// The number of references held to any token by the interner.
-const BASE_RC: usize = 3;
-
-#[test]
-fn base_rc_correct() {
-	let tok = Tok::new(Rc::new("foo".to_string()), api::TStr(1.try_into().unwrap()));
-	let mut bimap = Bimap::default();
-	bimap.insert(tok.clone());
-	assert_eq!(Rc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the current instance");
-}
-
-pub struct Bimap<T: Interned> {
-	intern: HashMap<Rc<T>, Tok<T>>,
-	by_id: HashMap<T::Marker, Tok<T>>,
-}
-impl<T: Interned> Bimap<T> {
-	pub fn insert(&mut self, token: Tok<T>) {
-		self.intern.insert(token.data.clone(), token.clone());
-		self.by_id.insert(token.to_api(), token);
-	}
-
-	pub fn by_marker(&self, marker: T::Marker) -> Option<Tok<T>> { self.by_id.get(&marker).cloned() }
-
-	pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Tok<T>>
-	where T: Borrow<Q> {
-		(self.intern.raw_entry())
-			.from_hash(self.intern.hasher().hash_one(q), |k| k.as_ref().borrow() == q)
-			.map(|p| p.1.clone())
-	}
-
-	pub fn sweep_replica(&mut self) -> Vec<T::Marker> {
-		(self.intern)
-			.extract_if(|k, _| Rc::strong_count(k) == BASE_RC)
-			.map(|(_, v)| {
-				self.by_id.remove(&v.to_api());
-				v.to_api()
-			})
-			.collect()
-	}
-
-	pub fn sweep_master(&mut self, retained: HashSet<T::Marker>) {
-		self.intern.retain(|k, v| BASE_RC < Rc::strong_count(k) || retained.contains(&v.to_api()))
-	}
-}
-
-impl<T: Interned> Default for Bimap<T> {
-	fn default() -> Self { Self { by_id: HashMap::new(), intern: HashMap::new() } }
-}
-
-pub trait UpComm {
-	fn up<R: Request>(&self, req: R) -> R::Response;
-}
-
-#[derive(Default)]
-pub struct TypedInterners {
-	strings: Bimap<String>,
-	vecs: Bimap<Vec<Tok<String>>>,
-}
-
-#[derive(Default)]
-pub struct InternerData {
-	interners: Mutex<TypedInterners>,
-	master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
-}
-#[derive(Clone, Default)]
-pub struct Interner(Rc<InternerData>);
-impl Interner {
-	pub fn new_master() -> Self { Self::default() }
-	pub fn new_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static) -> Self {
-		Self(Rc::new(InternerData { master: Some(Box::new(req)), interners: Mutex::default() }))
-	}
-	/// Intern some data; query its identifier if not known locally
-	pub async fn i<T: Interned>(&self, t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
-		let data = t.get_owned();
-		let mut g = self.0.interners.lock().await;
-		let typed = T::bimap(&mut g);
-		if let Some(tok) = typed.by_value(&data) {
-			return tok;
+		let mut iter = self.deref().iter();
+		match iter.next() {
+			None => return Ok(()),
+			Some(s) => write!(f, "{s}")?,
 		}
-		let marker = match &self.0.master {
-			Some(c) => data.clone().intern(&**c).await,
-			None =>
-				T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
-		};
-		let tok = Tok::new(data, marker);
-		T::bimap(&mut g).insert(tok.clone());
-		tok
-	}
-	/// Extern an identifier; query the data it represents if not known locally
-	pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
-		if let Some(tok) = M::Interned::bimap(&mut *self.0.interners.lock().await).by_marker(marker) {
-			return tok;
+		for s in iter {
+			write!(f, "::{s}")?
 		}
-		assert!(self.0.master.is_some(), "ID not in local interner and this is master");
-		let token = marker.resolve(self).await;
-		M::Interned::bimap(&mut *self.0.interners.lock().await).insert(token.clone());
-		token
-	}
-	pub async fn sweep_replica(&self) -> api::Retained {
-		assert!(self.0.master.is_some(), "Not a replica");
-		let mut g = self.0.interners.lock().await;
-		api::Retained { strings: g.strings.sweep_replica(), vecs: g.vecs.sweep_replica() }
-	}
-	pub async fn sweep_master(&self, retained: api::Retained) {
-		assert!(self.0.master.is_none(), "Not master");
-		let mut g = self.0.interners.lock().await;
-		g.strings.sweep_master(retained.strings.into_iter().collect());
-		g.vecs.sweep_master(retained.vecs.into_iter().collect());
+		Ok(())
 	}
 }
-impl fmt::Debug for Interner {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "Interner{{ replica: {} }}", self.0.master.is_none())
-	}
+impl Debug for IStrv {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "IStrv({self})") }
 }
 
-static ID: atomic::AtomicU64 = atomic::AtomicU64::new(1);
-
-pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) {
-	into.strings = into.strings.iter().chain(&from.strings).copied().unique().collect();
-	into.vecs = into.vecs.iter().chain(&from.vecs).copied().unique().collect();
+pub trait InternerSrv {
+	fn is<'a>(&'a self, v: &'a str) -> LocalBoxFuture<'a, IStr>;
+	fn es(&self, t: api::TStr) -> LocalBoxFuture<'_, IStr>;
+	fn iv<'a>(&'a self, v: &'a [IStr]) -> LocalBoxFuture<'a, IStrv>;
+	fn ev(&self, t: api::TStrv) -> LocalBoxFuture<'_, IStrv>;
 }
 
-#[cfg(test)]
-mod test {
-	use std::num::NonZero;
-	use std::pin::Pin;
+task_local! {
+	static INTERNER: Rc<dyn InternerSrv>;
+}
 
-	use orchid_api_traits::{Decode, enc_vec};
-	use test_executors::spin_on;
+pub async fn with_interner<F: Future>(val: Rc<dyn InternerSrv>, fut: F) -> F::Output {
+	INTERNER.scope(val, fut).await
+}
 
-	use super::*;
+fn get_interner() -> Rc<dyn InternerSrv> {
+	INTERNER.try_with(|i| i.clone()).expect("Interner not initialized")
+}
+
+pub async fn is(v: &str) -> IStr { get_interner().is(v).await }
+pub async fn iv(v: &[IStr]) -> IStrv { get_interner().iv(v).await }
+pub async fn es(v: api::TStr) -> IStr { get_interner().es(v).await }
+pub async fn ev(v: api::TStrv) -> IStrv { get_interner().ev(v).await }
+
+pub mod local_interner {
+	use std::borrow::Borrow;
+	use std::cell::RefCell;
+	use std::fmt::Debug;
+	use std::future;
+	use std::hash::{BuildHasher, Hash};
+	use std::num::NonZeroU64;
+	use std::rc::{Rc, Weak};
+
+	use futures::future::LocalBoxFuture;
+	use hashbrown::hash_table::{Entry, OccupiedEntry, VacantEntry};
+	use hashbrown::{DefaultHashBuilder, HashTable};
+	use orchid_api_traits::Coding;
+
+	use super::{IStr, IStrHandle, IStrv, IStrvHandle, InternerSrv};
 	use crate::api;
 
-	#[test]
-	fn test_i() {
-		let i = Interner::new_master();
-		let _: Tok<String> = spin_on(i.i("foo"));
-		let _: Tok<Vec<Tok<String>>> = spin_on(i.i(&[spin_on(i.i("bar")), spin_on(i.i("baz"))]));
+	/// Associated types and methods for parallel concepts between scalar and
+	/// vector interning
+	pub trait InternableCard: 'static + Sized + Default + Debug {
+		/// API representation of an interner key
+		type Token: Clone + Copy + Debug + Hash + Eq + PartialOrd + Ord + Coding + 'static;
+		/// Owned version of interned value physically held by `'static` interner
+		/// and token
+		type Data: 'static + Borrow<Self::Borrow> + Eq + Hash + Debug;
+		/// Borrowed version of interned value placed in intern queries to avoid a
+		/// copy
+		type Borrow: ToOwned<Owned = Self::Data> + ?Sized + Eq + Hash + Debug;
+		/// Smart object handed out by the interner for storage and comparison in
+		/// third party code. [IStr] or [IStrv]
+		type Interned: Clone + Debug;
+		/// Create smart object from token for fast comparison and a handle for
+		/// everything else incl. virtual drop
+		fn new_interned(token: Self::Token, handle: Rc<Handle<Self>>) -> Self::Interned;
 	}
 
-	#[test]
-	fn test_coding() {
-		spin_on(async {
-			let coded = api::TStr(NonZero::new(3u64).unwrap());
-			let mut enc = &enc_vec(&coded).await[..];
-			api::TStr::decode(Pin::new(&mut enc)).await;
-			assert_eq!(enc, [], "Did not consume all of {enc:?}")
-		})
+	#[derive(Default, Debug)]
+	pub struct StrBranch;
+	impl InternableCard for StrBranch {
+		type Data = String;
+		type Token = api::TStr;
+		type Borrow = str;
+		type Interned = IStr;
+		fn new_interned(t: Self::Token, h: Rc<Handle<Self>>) -> Self::Interned { IStr(t, h) }
 	}
+
+	#[derive(Default, Debug)]
+	pub struct StrvBranch;
+	impl InternableCard for StrvBranch {
+		type Data = Vec<IStr>;
+		type Token = api::TStrv;
+		type Borrow = [IStr];
+		type Interned = IStrv;
+		fn new_interned(t: Self::Token, h: Rc<Handle<Self>>) -> Self::Interned { IStrv(t, h) }
+	}
+
+	/// Pairs interned data with its internment key
+	#[derive(Debug)]
+	struct Data<B: InternableCard> {
+		token: B::Token,
+		data: Rc<B::Data>,
+	}
+	impl<B: InternableCard> Clone for Data<B> {
+		fn clone(&self) -> Self { Self { token: self.token, data: self.data.clone() } }
+	}
+
+	/// Implementor for the trait objects held by [IStr] and [IStrv]
+	pub struct Handle<B: InternableCard> {
+		data: Data<B>,
+		parent: Weak<RefCell<IntData<B>>>,
+	}
+	impl IStrHandle for Handle<StrBranch> {
+		fn rc(&self) -> Rc<String> { self.data.data.clone() }
+	}
+	impl AsRef<str> for Handle<StrBranch> {
+		fn as_ref(&self) -> &str { self.data.data.as_ref().as_ref() }
+	}
+	impl IStrvHandle for Handle<StrvBranch> {
+		fn rc(&self) -> Rc<Vec<IStr>> { self.data.data.clone() }
+	}
+	impl AsRef<[IStr]> for Handle<StrvBranch> {
+		fn as_ref(&self) -> &[IStr] { self.data.data.as_ref().as_ref() }
+	}
+	impl<B: InternableCard> Drop for Handle<B> {
+		fn drop(&mut self) {
+			let Some(parent) = self.parent.upgrade() else { return };
+			if let Entry::Occupied(ent) =
+				parent.borrow_mut().entry_by_data(self.data.data.as_ref().borrow())
+			{
+				ent.remove();
+			}
+			if let Entry::Occupied(ent) = parent.borrow_mut().entry_by_tok(self.data.token) {
+				ent.remove();
+			}
+		}
+	}
+
+	/// Information retained about an interned token indexed both by key and
+	/// value.
+	struct Rec<B: InternableCard> {
+		/// This reference is weak, but the [Drop] handler of [Handle] removes all
+		/// [Rec]s from the interner so it is guaranteed to be live.
+		handle: Weak<Handle<B>>,
+		/// Keys for indexing from either table
+		data: Data<B>,
+	}
+
+	/// Read data from an occupied entry in an interner. The equivalent insert
+	/// command is [insert]
+	fn read<B: InternableCard>(entry: OccupiedEntry<'_, Rec<B>>) -> B::Interned {
+		let hand = entry.get().handle.upgrade().expect("Found entry but handle already dropped");
+		B::new_interned(entry.get().data.token, hand)
+	}
+
+	/// Insert some data into an entry borrowed from this same interner.
+	/// The equivalent read command is [read]
+	fn insert<B: InternableCard>(entry: VacantEntry<'_, Rec<B>>, handle: Rc<Handle<B>>) {
+		entry.insert(Rec { data: handle.data.clone(), handle: Rc::downgrade(&handle) });
+	}
+
+	#[derive(Default)]
+	struct IntData<B: InternableCard> {
+		by_tok: HashTable<Rec<B>>,
+		by_data: HashTable<Rec<B>>,
+		hasher: DefaultHashBuilder,
+	}
+	impl<B: InternableCard> IntData<B> {
+		fn entry_by_data(&mut self, query: &B::Borrow) -> Entry<'_, Rec<B>> {
+			self.by_data.entry(
+				self.hasher.hash_one(query),
+				|rec| rec.data.data.as_ref().borrow() == query,
+				|rec| self.hasher.hash_one(rec.data.data.as_ref().borrow()),
+			)
+		}
+		fn entry_by_tok(&mut self, token: B::Token) -> Entry<'_, Rec<B>> {
+			self.by_tok.entry(
+				self.hasher.hash_one(token),
+				|rec| rec.data.token == token,
+				|rec| self.hasher.hash_one(rec.data.token),
+			)
+		}
+	}
+
+	/// Failing intern command that can be recovered if the value is found
+	/// elsewhere
+	pub struct InternError<'a, B: InternableCard> {
+		int: &'a Int<B>,
+		query: &'a B::Borrow,
+	}
+	impl<B: InternableCard> InternError<'_, B> {
+		/// If a racing write populates the entry, the continuation returns that
+		/// value and discards its argument
+		pub fn set_if_empty(self, token: B::Token) -> B::Interned {
+			let mut int_data = self.int.0.borrow_mut();
+			match int_data.entry_by_data(self.query) {
+				Entry::Occupied(ent) => read(ent),
+				Entry::Vacant(ent) => {
+					let hand = self.int.mk_handle(Data { token, data: Rc::new(self.query.to_owned()) });
+					insert(ent, hand.clone());
+					let Entry::Vacant(other_ent) = int_data.entry_by_tok(token) else {
+						panic!("Data and key tables out of sync")
+					};
+					insert(other_ent, hand.clone());
+					B::new_interned(token, hand)
+				},
+			}
+		}
+	}
+	impl<B: InternableCard> Debug for InternError<'_, B> {
+		fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+			f.debug_tuple("InternEntry").field(&self.query).finish()
+		}
+	}
+
+	/// Failing extern command that can be recovered if the value is found
+	/// elsewhere
+	pub struct ExternError<'a, B: InternableCard> {
+		int: &'a Int<B>,
+		token: B::Token,
+	}
+	impl<B: InternableCard> ExternError<'_, B> {
+		/// If a racing write populates the entry, the continuation returns that
+		/// value and discards its argument
+		pub fn set_if_empty(&self, data: Rc<B::Data>) -> B::Interned {
+			let mut int_data = self.int.0.borrow_mut();
+			match int_data.entry_by_tok(self.token) {
+				Entry::Occupied(ent) => read(ent),
+				Entry::Vacant(ent) => {
+					let hand = self.int.mk_handle(Data { token: self.token, data: data.clone() });
+					insert(ent, hand.clone());
+					let Entry::Vacant(other_ent) = int_data.entry_by_data(data.as_ref().borrow()) else {
+						panic!("Data and key tables out of sync")
+					};
+					insert(other_ent, hand.clone());
+					B::new_interned(self.token, hand)
+				},
+			}
+		}
+	}
+	impl<B: InternableCard> Debug for ExternError<'_, B> {
+		fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+			f.debug_tuple("ExternEntry").field(&self.token).finish()
+		}
+	}
+
+	#[derive(Default)]
+	pub struct Int<B: InternableCard>(Rc<RefCell<IntData<B>>>);
+	impl<B: InternableCard> Int<B> {
+		fn mk_handle(&self, data: Data<B>) -> Rc<Handle<B>> {
+			Rc::new(Handle { data: data.clone(), parent: Rc::downgrade(&self.0.clone()) })
+		}
+
+		/// Look up by value, or yield to figure out its ID from elsewhere
+		pub fn i<'a>(&'a self, query: &'a B::Borrow) -> Result<B::Interned, InternError<'a, B>> {
+			if let Entry::Occupied(val) = self.0.borrow_mut().entry_by_data(query) {
+				return Ok(read(val));
+			}
+			Err(InternError { int: self, query })
+		}
+
+		/// Look up by key or yield to figure out its value from elsewhere
+		pub fn e(&self, token: B::Token) -> Result<B::Interned, ExternError<'_, B>> {
+			if let Entry::Occupied(ent) = self.0.borrow_mut().entry_by_tok(token) {
+				return Ok(read(ent));
+			}
+			Err(ExternError { int: self, token })
+		}
+	}
+
+	thread_local! {
+		static NEXT_ID: RefCell<u64> = 0.into();
+	}
+
+	fn with_new_id<T>(fun: impl FnOnce(NonZeroU64) -> T) -> T {
+		fun(
+			NonZeroU64::new(NEXT_ID.with_borrow_mut(|id| {
+				*id += 1;
+				*id
+			}))
+			.unwrap(),
+		)
+	}
+
+	#[derive(Default)]
+	struct LocalInterner {
+		str: Int<StrBranch>,
+		strv: Int<StrvBranch>,
+	}
+	impl InternerSrv for LocalInterner {
+		fn is<'a>(&'a self, v: &'a str) -> LocalBoxFuture<'a, IStr> {
+			match self.str.i(v) {
+				Ok(int) => Box::pin(future::ready(int)),
+				Err(e) => with_new_id(|id| Box::pin(future::ready(e.set_if_empty(api::TStr(id))))),
+			}
+		}
+		fn es(&self, t: api::TStr) -> LocalBoxFuture<'_, IStr> {
+			Box::pin(future::ready(self.str.e(t).expect("Unrecognized token cannot be externed")))
+		}
+		fn iv<'a>(&'a self, v: &'a [IStr]) -> LocalBoxFuture<'a, IStrv> {
+			match self.strv.i(v) {
+				Ok(int) => Box::pin(future::ready(int)),
+				Err(e) => with_new_id(|id| Box::pin(future::ready(e.set_if_empty(api::TStrv(id))))),
+			}
+		}
+		fn ev(&self, t: orchid_api::TStrv) -> LocalBoxFuture<'_, IStrv> {
+			Box::pin(future::ready(self.strv.e(t).expect("Unrecognized token cannot be externed")))
+		}
+	}
+
+	/// Creates a basic thread-local interner for testing and root role.
+	pub fn local_interner() -> Rc<dyn InternerSrv> { Rc::<LocalInterner>::default() }
 }

orchid-base/src/lib.rs

@@ -1,9 +1,9 @@
 pub use async_once_cell;
 use orchid_api as api;
 
+pub mod binary;
 pub mod box_cow;
 pub mod boxed_iter;
-pub mod builtin;
 pub mod char_filter;
 pub mod clone;
 pub mod combine;
@@ -14,6 +14,7 @@ pub mod id_store;
 pub mod interner;
 pub mod iter_utils;
 pub mod join;
+mod localset;
 pub mod location;
 pub mod logging;
 mod match_mapping;
@@ -25,6 +26,7 @@ pub mod pure_seq;
 pub mod reqnot;
 pub mod sequence;
 pub mod side;
+pub mod stash;
 mod tl_cache;
 pub mod tokens;
 pub mod tree;

orchid-base/src/localset.rs

@@ -0,0 +1,48 @@
+use std::collections::VecDeque;
+use std::pin::Pin;
+use std::task::Poll;
+
+use futures::StreamExt;
+use futures::channel::mpsc::{UnboundedReceiver, UnboundedSender, unbounded};
+use futures::future::LocalBoxFuture;
+
+pub struct LocalSet<'a, E> {
+	receiver: UnboundedReceiver<LocalBoxFuture<'a, Result<(), E>>>,
+	pending: VecDeque<LocalBoxFuture<'a, Result<(), E>>>,
+}
+impl<'a, E> LocalSet<'a, E> {
+	pub fn new() -> (UnboundedSender<LocalBoxFuture<'a, Result<(), E>>>, Self) {
+		let (sender, receiver) = unbounded();
+		(sender, Self { receiver, pending: VecDeque::new() })
+	}
+}
+impl<E> Future for LocalSet<'_, E> {
+	type Output = Result<(), E>;
+	fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
+		let this = self.get_mut();
+		let mut any_pending = false;
+		loop {
+			match this.receiver.poll_next_unpin(cx) {
+				Poll::Pending => {
+					any_pending = true;
+					break;
+				},
+				Poll::Ready(None) => break,
+				Poll::Ready(Some(fut)) => this.pending.push_back(fut),
+			}
+		}
+		let count = this.pending.len();
+		for _ in 0..count {
+			let mut req = this.pending.pop_front().unwrap();
+			match req.as_mut().poll(cx) {
+				Poll::Ready(Ok(())) => (),
+				Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
+				Poll::Pending => {
+					any_pending = true;
+					this.pending.push_back(req)
+				},
+			}
+		}
+		if any_pending { Poll::Pending } else { Poll::Ready(Ok(())) }
+	}
+}

orchid-base/src/location.rs

@@ -8,12 +8,12 @@ use futures::future::join_all;
 use trait_set::trait_set;
 
 use crate::error::ErrPos;
-use crate::interner::{Interner, Tok};
+use crate::interner::{IStr, es, is};
 use crate::name::Sym;
 use crate::{api, match_mapping, sym};
 
 trait_set! {
-	pub trait GetSrc = FnMut(&Sym) -> Tok<String>;
+	pub trait GetSrc = FnMut(&Sym) -> IStr;
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -37,13 +37,13 @@ impl Pos {
 			other => format!("{other:?}"),
 		}
 	}
-	pub async fn from_api(api: &api::Location, i: &Interner) -> Self {
+	pub async fn from_api(api: &api::Location) -> Self {
 		match_mapping!(api, api::Location => Pos {
 			None, Inherit, SlotTarget,
-			Gen(cgi => CodeGenInfo::from_api(cgi, i).await),
-			Multi(v => join_all(v.iter().map(|l| Pos::from_api(l, i))).await)
+			Gen(cgi => CodeGenInfo::from_api(cgi).await),
+			Multi(v => join_all(v.iter().map(Pos::from_api)).await)
 		} {
-			api::Location::SourceRange(sr) => Self::SrcRange(SrcRange::from_api(sr, i).await)
+			api::Location::SourceRange(sr) => Self::SrcRange(SrcRange::from_api(sr).await)
 		})
 	}
 	pub fn to_api(&self) -> api::Location {
@@ -108,7 +108,7 @@ impl SrcRange {
 	}
 	/// Create a dud [SourceRange] for testing. Its value is unspecified and
 	/// volatile.
-	pub async fn mock(i: &Interner) -> Self { Self { range: 0..1, path: sym!(test; i) } }
+	pub async fn mock() -> Self { Self { range: 0..1, path: sym!(test) } }
 	/// Path the source text was loaded from
 	pub fn path(&self) -> Sym { self.path.clone() }
 	/// Byte range
@@ -133,8 +133,8 @@ impl SrcRange {
 		}
 	}
 	pub fn zw(path: Sym, pos: u32) -> Self { Self { path, range: pos..pos } }
-	pub async fn from_api(api: &api::SourceRange, i: &Interner) -> Self {
-		Self { path: Sym::from_api(api.path, i).await, range: api.range.clone() }
+	pub async fn from_api(api: &api::SourceRange) -> Self {
+		Self { path: Sym::from_api(api.path).await, range: api.range.clone() }
 	}
 	pub fn to_api(&self) -> api::SourceRange {
 		api::SourceRange { path: self.path.to_api(), range: self.range.clone() }
@@ -162,24 +162,19 @@ pub struct CodeGenInfo {
 	/// formatted like a Rust namespace
 	pub generator: Sym,
 	/// Unformatted user message with relevant circumstances and parameters
-	pub details: Tok<String>,
+	pub details: IStr,
 }
 impl CodeGenInfo {
 	/// A codegen marker with no user message and parameters
-	pub async fn new_short(generator: Sym, i: &Interner) -> Self {
-		Self { generator, details: i.i("").await }
-	}
+	pub async fn new_short(generator: Sym) -> Self { Self { generator, details: is("").await } }
 	/// A codegen marker with a user message or parameters
-	pub async fn new_details(generator: Sym, details: impl AsRef<str>, i: &Interner) -> Self {
-		Self { generator, details: i.i(details.as_ref()).await }
+	pub async fn new_details(generator: Sym, details: impl AsRef<str>) -> Self {
+		Self { generator, details: is(details.as_ref()).await }
 	}
 	/// Syntactic location
 	pub fn pos(&self) -> Pos { Pos::Gen(self.clone()) }
-	pub async fn from_api(api: &api::CodeGenInfo, i: &Interner) -> Self {
-		Self {
-			generator: Sym::from_api(api.generator, i).await,
-			details: Tok::from_api(api.details, i).await,
-		}
+	pub async fn from_api(api: &api::CodeGenInfo) -> Self {
+		Self { generator: Sym::from_api(api.generator).await, details: es(api.details).await }
 	}
 	pub fn to_api(&self) -> api::CodeGenInfo {
 		api::CodeGenInfo { generator: self.generator.to_api(), details: self.details.to_api() }

orchid-base/src/logging.rs

@@ -1,36 +1,74 @@
+use std::any::Any;
+use std::cell::RefCell;
 use std::fmt::Arguments;
-use std::fs::File;
-use std::io::{Write, stderr};
+use std::io::Write;
+use std::rc::Rc;
 
-pub use api::LogStrategy;
-use itertools::Itertools;
+use futures::future::LocalBoxFuture;
+use task_local::task_local;
 
 use crate::api;
 
-#[derive(Clone)]
-pub struct Logger(api::LogStrategy);
-impl Logger {
-	pub fn new(strat: api::LogStrategy) -> Self { Self(strat) }
-	pub fn log(&self, msg: impl AsRef<str>) { writeln!(self, "{}", msg.as_ref()) }
-	pub fn strat(&self) -> api::LogStrategy { self.0.clone() }
-	pub fn is_active(&self) -> bool { !matches!(self.0, api::LogStrategy::Discard) }
-	pub fn log_buf(&self, event: impl AsRef<str>, buf: &[u8]) {
-		if std::env::var("ORCHID_LOG_BUFFERS").is_ok_and(|v| !v.is_empty()) {
-			writeln!(self, "{}: [{}]", event.as_ref(), buf.iter().map(|b| format!("{b:02x}")).join(" "))
+task_local! {
+	static DEFAULT_WRITER: RefCell<Box<dyn Write>>
+}
+
+/// Set the stream used for [api::LogStrategy::Default]. If not set,
+/// [std::io::stderr] will be used.
+pub async fn with_default_stream<F: Future>(stderr: impl Write + 'static, fut: F) -> F::Output {
+	DEFAULT_WRITER.scope(RefCell::new(Box::new(stderr)), fut).await
+}
+
+pub trait LogWriter {
+	fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()>;
+}
+
+pub trait Logger: Any {
+	fn writer(&self, category: &str) -> Rc<dyn LogWriter>;
+	fn strat(&self, category: &str) -> api::LogStrategy;
+	fn is_active(&self, category: &str) -> bool {
+		!matches!(self.strat(category), api::LogStrategy::Discard)
+	}
+}
+
+task_local! {
+	static LOGGER: Rc<dyn Logger>;
+}
+
+pub async fn with_logger<F: Future>(logger: impl Logger + 'static, fut: F) -> F::Output {
+	LOGGER.scope(Rc::new(logger), fut).await
+}
+
+pub fn log(category: &str) -> Rc<dyn LogWriter> {
+	LOGGER.try_with(|l| l.writer(category)).expect("Logger not set!")
+}
+
+pub fn get_logger() -> Rc<dyn Logger> { LOGGER.try_with(|l| l.clone()).expect("Logger not set!") }
+
+pub mod test {
+	use std::fmt::Arguments;
+	use std::rc::Rc;
+
+	use futures::future::LocalBoxFuture;
+
+	use crate::clone;
+	use crate::logging::{LogWriter, Logger};
+
+	#[derive(Clone)]
+	pub struct TestLogger(Rc<dyn Fn(String) -> LocalBoxFuture<'static, ()>>);
+	impl LogWriter for TestLogger {
+		fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()> {
+			(self.0)(fmt.to_string())
 		}
 	}
-	pub fn write_fmt(&self, fmt: Arguments) {
-		match &self.0 {
-			api::LogStrategy::Discard => (),
-			api::LogStrategy::StdErr => {
-				stderr().write_fmt(fmt).expect("Could not write to stderr!");
-				stderr().flush().expect("Could not flush stderr")
-			},
-			api::LogStrategy::File(f) => {
-				let mut file = (File::options().write(true).create(true).truncate(true).open(f))
-					.expect("Could not open logfile");
-				file.write_fmt(fmt).expect("Could not write to logfile");
-			},
+	impl Logger for TestLogger {
+		fn strat(&self, _category: &str) -> orchid_api::LogStrategy { orchid_api::LogStrategy::Default }
+		fn writer(&self, _category: &str) -> std::rc::Rc<dyn LogWriter> { Rc::new(self.clone()) }
+	}
+	impl TestLogger {
+		pub fn new(f: impl AsyncFn(String) + 'static) -> Self {
+			let f = Rc::new(f);
+			Self(Rc::new(move |s| clone!(f; Box::pin(async move { f(s).await }))))
 		}
 	}
 }

orchid-base/src/msg.rs

@@ -6,7 +6,8 @@ use orchid_api_traits::{Decode, Encode};
 
 pub async fn send_msg(mut write: Pin<&mut impl AsyncWrite>, msg: &[u8]) -> io::Result<()> {
 	let mut len_buf = vec![];
-	u32::try_from(msg.len()).unwrap().encode(Pin::new(&mut len_buf)).await;
+	let len_prefix = u32::try_from(msg.len()).expect("Message over 4GB not permitted on channel");
+	len_prefix.encode_vec(&mut len_buf);
 	write.write_all(&len_buf).await?;
 	write.write_all(msg).await?;
 	write.flush().await
@@ -15,7 +16,7 @@ pub async fn send_msg(mut write: Pin<&mut impl AsyncWrite>, msg: &[u8]) -> io::R
 pub async fn recv_msg(mut read: Pin<&mut impl AsyncRead>) -> io::Result<Vec<u8>> {
 	let mut len_buf = [0u8; (u32::BITS / 8) as usize];
 	read.read_exact(&mut len_buf).await?;
-	let len = u32::decode(Pin::new(&mut &len_buf[..])).await;
+	let len = u32::decode(Pin::new(&mut &len_buf[..])).await?;
 	let mut msg = vec![0u8; len as usize];
 	read.read_exact(&mut msg).await?;
 	Ok(msg)

orchid-base/src/name.rs

@@ -12,65 +12,60 @@ use itertools::Itertools;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::interner::{InternMarker, Interner, Tok};
+use crate::interner::{IStr, IStrv, es, ev, is, iv};
 
 trait_set! {
 	/// Traits that all name iterators should implement
-	pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
+	pub trait NameIter = Iterator<Item = IStr> + DoubleEndedIterator + ExactSizeIterator;
 }
 
 /// A token path which may be empty. [VName] is the non-empty version
 #[derive(Clone, Default, Hash, PartialEq, Eq)]
-pub struct VPath(Vec<Tok<String>>);
+pub struct VPath(Vec<IStr>);
 impl VPath {
 	/// Collect segments into a vector
-	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Self {
-		Self(items.into_iter().collect())
-	}
+	pub fn new(items: impl IntoIterator<Item = IStr>) -> Self { Self(items.into_iter().collect()) }
 	/// Number of path segments
 	pub fn len(&self) -> usize { self.0.len() }
 	/// Whether there are any path segments. In other words, whether this is a
 	/// valid name
 	pub fn is_empty(&self) -> bool { self.len() == 0 }
 	/// Prepend some tokens to the path
-	pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn prefix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(items.into_iter().chain(self.0).collect())
 	}
 	/// Append some tokens to the path
-	pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn suffix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Partition the string by `::` namespace separators
-	pub async fn parse(s: &str, i: &Interner) -> Self {
-		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(|s| i.i(s))).await })
+	pub async fn parse(s: &str) -> Self {
+		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(is)).await })
 	}
 	/// Walk over the segments
-	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
-		Box::new(self.0.iter().map(|s| s.as_str()))
-	}
+	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> { Box::new(self.0.iter().map(|s| &**s)) }
 	/// Try to convert into non-empty version
 	pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
 	/// Add a token to the path. Since now we know that it can't be empty, turn it
 	/// into a name.
-	pub fn name_with_suffix(self, name: Tok<String>) -> VName {
+	pub fn name_with_suffix(self, name: IStr) -> VName {
 		VName(self.into_iter().chain([name]).collect())
 	}
 	/// Add a token to the beginning of the. Since now we know that it can't be
 	/// empty, turn it into a name.
-	pub fn name_with_prefix(self, name: Tok<String>) -> VName {
+	pub fn name_with_prefix(self, name: IStr) -> VName {
 		VName([name].into_iter().chain(self).collect())
 	}
 
 	/// Convert a fs path to a vpath
-	pub async fn from_path(path: &Path, ext: &str, i: &Interner) -> Option<(Self, bool)> {
-		async fn to_vpath(p: &Path, i: &Interner) -> Option<VPath> {
-			let tok_opt_v =
-				join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(|s| i.i(s))))).await;
+	pub async fn from_path(path: &Path, ext: &str) -> Option<(Self, bool)> {
+		async fn to_vpath(p: &Path) -> Option<VPath> {
+			let tok_opt_v = join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(is)))).await;
 			tok_opt_v.into_iter().collect::<Option<_>>().map(VPath)
 		}
 		match path.extension().map(|s| s.to_str()) {
-			Some(Some(s)) if s == ext => Some((to_vpath(&path.with_extension(""), i).await?, true)),
-			None => Some((to_vpath(path, i).await?, false)),
+			Some(Some(s)) if s == ext => Some((to_vpath(&path.with_extension("")).await?, true)),
+			None => Some((to_vpath(path).await?, false)),
 			Some(_) => None,
 		}
 	}
@@ -83,30 +78,28 @@ impl fmt::Display for VPath {
 		write!(f, "{}", self.str_iter().join("::"))
 	}
 }
-impl FromIterator<Tok<String>> for VPath {
-	fn from_iter<T: IntoIterator<Item = Tok<String>>>(iter: T) -> Self {
-		Self(iter.into_iter().collect())
-	}
+impl FromIterator<IStr> for VPath {
+	fn from_iter<T: IntoIterator<Item = IStr>>(iter: T) -> Self { Self(iter.into_iter().collect()) }
 }
 impl IntoIterator for VPath {
-	type Item = Tok<String>;
+	type Item = IStr;
 	type IntoIter = vec::IntoIter<Self::Item>;
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
-impl Borrow<[Tok<String>]> for VPath {
-	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
+impl Borrow<[IStr]> for VPath {
+	fn borrow(&self) -> &[IStr] { &self.0[..] }
 }
 impl Deref for VPath {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
 impl<T> Index<T> for VPath
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
-	fn index(&self, index: T) -> &Self::Output { &Borrow::<[Tok<String>]>::borrow(self)[index] }
+	fn index(&self, index: T) -> &Self::Output { &Borrow::<[IStr]>::borrow(self)[index] }
 }
 
 /// A mutable representation of a namespaced identifier of at least one segment.
@@ -116,50 +109,43 @@ where [Tok<String>]: Index<T>
 /// See also [Sym] for the immutable representation, and [VPath] for possibly
 /// empty values
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct VName(Vec<Tok<String>>);
+pub struct VName(Vec<IStr>);
 impl VName {
 	/// Assert that the sequence isn't empty and wrap it in [VName] to represent
 	/// this invariant
-	pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
+	pub fn new(items: impl IntoIterator<Item = IStr>) -> Result<Self, EmptyNameError> {
 		let data: Vec<_> = items.into_iter().collect();
 		if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
 	}
-	pub async fn deintern(
-		name: impl IntoIterator<Item = api::TStr>,
-		i: &Interner,
-	) -> Result<Self, EmptyNameError> {
-		Self::new(join_all(name.into_iter().map(|m| Tok::from_api(m, i))).await)
+	pub async fn deintern(name: impl IntoIterator<Item = api::TStr>) -> Result<Self, EmptyNameError> {
+		Self::new(join_all(name.into_iter().map(es)).await)
 	}
 	/// Unwrap the enclosed vector
-	pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
+	pub fn into_vec(self) -> Vec<IStr> { self.0 }
 	/// Get a reference to the enclosed vector
-	pub fn vec(&self) -> &Vec<Tok<String>> { &self.0 }
+	pub fn vec(&self) -> &Vec<IStr> { &self.0 }
 	/// Mutable access to the underlying vector. To ensure correct results, this
 	/// must never be empty.
-	pub fn vec_mut(&mut self) -> &mut Vec<Tok<String>> { &mut self.0 }
+	pub fn vec_mut(&mut self) -> &mut Vec<IStr> { &mut self.0 }
 	/// Intern the name and return a [Sym]
-	pub async fn to_sym(&self, i: &Interner) -> Sym { Sym(i.i(&self.0[..]).await) }
+	pub async fn to_sym(&self) -> Sym { Sym(iv(&self.0[..]).await) }
 	/// If this name has only one segment, return it
-	pub fn as_root(&self) -> Option<Tok<String>> { self.0.iter().exactly_one().ok().cloned() }
+	pub fn as_root(&self) -> Option<IStr> { self.0.iter().exactly_one().ok().cloned() }
 	/// Prepend the segments to this name
 	#[must_use = "This is a pure function"]
-	pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn prefix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(items.into_iter().chain(self.0).collect())
 	}
 	/// Append the segments to this name
 	#[must_use = "This is a pure function"]
-	pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn suffix(self, items: impl IntoIterator<Item = IStr>) -> Self {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Read a `::` separated namespaced name
-	pub async fn parse(s: &str, i: &Interner) -> Result<Self, EmptyNameError> {
-		Self::new(VPath::parse(s, i).await)
-	}
-	pub async fn literal(s: &'static str, i: &Interner) -> Self {
-		Self::parse(s, i).await.expect("empty literal !?")
-	}
+	pub async fn parse(s: &str) -> Result<Self, EmptyNameError> { Self::new(VPath::parse(s).await) }
+	pub async fn literal(s: &'static str) -> Self { Self::parse(s).await.expect("empty literal !?") }
 	/// Obtain an iterator over the segments of the name
-	pub fn iter(&self) -> impl Iterator<Item = Tok<String>> + '_ { self.0.iter().cloned() }
+	pub fn iter(&self) -> impl Iterator<Item = IStr> + '_ { self.0.iter().cloned() }
 }
 impl fmt::Debug for VName {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
@@ -170,22 +156,22 @@ impl fmt::Display for VName {
 	}
 }
 impl IntoIterator for VName {
-	type Item = Tok<String>;
+	type Item = IStr;
 	type IntoIter = vec::IntoIter<Self::Item>;
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
 impl<T> Index<T> for VName
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[Tok<String>]> for VName {
-	fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
+impl Borrow<[IStr]> for VName {
+	fn borrow(&self) -> &[IStr] { self.0.borrow() }
 }
 impl Deref for VName {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -193,11 +179,9 @@ impl Deref for VName {
 /// empty sequence
 #[derive(Debug, Copy, Clone, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct EmptyNameError;
-impl TryFrom<&[Tok<String>]> for VName {
+impl TryFrom<&[IStr]> for VName {
 	type Error = EmptyNameError;
-	fn try_from(value: &[Tok<String>]) -> Result<Self, Self::Error> {
-		Self::new(value.iter().cloned())
-	}
+	fn try_from(value: &[IStr]) -> Result<Self, Self::Error> { Self::new(value.iter().cloned()) }
 }
 
 /// An interned representation of a namespaced identifier.
@@ -206,37 +190,34 @@ impl TryFrom<&[Tok<String>]> for VName {
 ///
 /// See also [VName]
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct Sym(Tok<Vec<Tok<String>>>);
+pub struct Sym(IStrv);
 impl Sym {
 	/// Assert that the sequence isn't empty, intern it and wrap it in a [Sym] to
 	/// represent this invariant
-	pub async fn new(
-		v: impl IntoIterator<Item = Tok<String>>,
-		i: &Interner,
-	) -> Result<Self, EmptyNameError> {
+	pub async fn new(v: impl IntoIterator<Item = IStr>) -> Result<Self, EmptyNameError> {
 		let items = v.into_iter().collect_vec();
-		Self::from_tok(i.i(&items).await)
+		Self::from_tok(iv(&items).await)
 	}
 	/// Read a `::` separated namespaced name.
-	pub async fn parse(s: &str, i: &Interner) -> Result<Self, EmptyNameError> {
-		Ok(Sym(i.i(&VName::parse(s, i).await?.into_vec()).await))
+	pub async fn parse(s: &str) -> Result<Self, EmptyNameError> {
+		Ok(Sym(iv(&VName::parse(s).await?.into_vec()).await))
 	}
 	/// Assert that a token isn't empty, and wrap it in a [Sym]
-	pub fn from_tok(t: Tok<Vec<Tok<String>>>) -> Result<Self, EmptyNameError> {
+	pub fn from_tok(t: IStrv) -> Result<Self, EmptyNameError> {
 		if t.is_empty() { Err(EmptyNameError) } else { Ok(Self(t)) }
 	}
 	/// Grab the interner token
-	pub fn tok(&self) -> Tok<Vec<Tok<String>>> { self.0.clone() }
+	pub fn tok(&self) -> IStrv { self.0.clone() }
 	/// Get a number unique to this name suitable for arbitrary ordering.
-	pub fn id(&self) -> NonZeroU64 { self.0.to_api().get_id() }
+	pub fn id(&self) -> NonZeroU64 { self.0.to_api().0 }
 	/// Extern the sym for editing
 	pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
-	pub async fn from_api(marker: api::TStrv, i: &Interner) -> Sym {
-		Self::from_tok(Tok::from_api(marker, i).await).expect("Empty sequence found for serialized Sym")
+	pub async fn from_api(marker: api::TStrv) -> Sym {
+		Self::from_tok(ev(marker).await).expect("Empty sequence found for serialized Sym")
 	}
 	pub fn to_api(&self) -> api::TStrv { self.tok().to_api() }
-	pub async fn suffix(&self, tokv: impl IntoIterator<Item = Tok<String>>, i: &Interner) -> Sym {
-		Self::new(self.0.iter().cloned().chain(tokv), i).await.unwrap()
+	pub async fn suffix(&self, tokv: impl IntoIterator<Item = IStr>) -> Sym {
+		Self::new(self.0.iter().cloned().chain(tokv)).await.unwrap()
 	}
 }
 impl fmt::Debug for Sym {
@@ -248,17 +229,17 @@ impl fmt::Display for Sym {
 	}
 }
 impl<T> Index<T> for Sym
-where [Tok<String>]: Index<T>
+where [IStr]: Index<T>
 {
-	type Output = <[Tok<String>] as Index<T>>::Output;
+	type Output = <[IStr] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[Tok<String>]> for Sym {
-	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
+impl Borrow<[IStr]> for Sym {
+	fn borrow(&self) -> &[IStr] { &self.0[..] }
 }
 impl Deref for Sym {
-	type Target = [Tok<String>];
+	type Target = [IStr];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -266,16 +247,14 @@ impl Deref for Sym {
 /// handled together in datastructures. The names can never be empty
 #[allow(clippy::len_without_is_empty)] // never empty
 pub trait NameLike:
-	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[Tok<String>]>
+	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[IStr]>
 {
 	/// Convert into held slice
-	fn as_slice(&self) -> &[Tok<String>] { Borrow::<[Tok<String>]>::borrow(self) }
+	fn as_slice(&self) -> &[IStr] { Borrow::<[IStr]>::borrow(self) }
 	/// Get iterator over tokens
 	fn segs(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
 	/// Get iterator over string segments
-	fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ {
-		self.as_slice().iter().map(|t| t.as_str())
-	}
+	fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ { self.as_slice().iter().map(|t| &**t) }
 	/// Fully resolve the name for printing
 	#[must_use]
 	fn to_strv(&self) -> Vec<String> { self.segs().map(|s| s.to_string()).collect() }
@@ -286,19 +265,19 @@ pub trait NameLike:
 		NonZeroUsize::try_from(self.segs().count()).expect("NameLike never empty")
 	}
 	/// Like slice's `split_first` except we know that it always returns Some
-	fn split_first_seg(&self) -> (Tok<String>, &[Tok<String>]) {
+	fn split_first_seg(&self) -> (IStr, &[IStr]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Like slice's `split_last` except we know that it always returns Some
-	fn split_last_seg(&self) -> (Tok<String>, &[Tok<String>]) {
+	fn split_last_seg(&self) -> (IStr, &[IStr]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Get the first element
-	fn first_seg(&self) -> Tok<String> { self.split_first_seg().0 }
+	fn first_seg(&self) -> IStr { self.split_first_seg().0 }
 	/// Get the last element
-	fn last_seg(&self) -> Tok<String> { self.split_last_seg().0 }
+	fn last_seg(&self) -> IStr { self.split_last_seg().0 }
 }
 
 impl NameLike for Sym {}
@@ -311,11 +290,11 @@ impl NameLike for VName {}
 /// cloning the token.
 #[macro_export]
 macro_rules! sym {
-  ($seg1:tt $( :: $seg:tt)* ; $i:expr) => {
+  ($seg1:tt $( :: $seg:tt)*) => {
 		$crate::name::Sym::from_tok(
-			$i.i(&[
-				$i.i(stringify!($seg1)).await
-				$( , $i.i(stringify!($seg)).await )*
+			$crate::interner::iv(&[
+				$crate::interner::is(stringify!($seg1)).await
+				$( , $crate::interner::is(stringify!($seg)).await )*
 			])
 			.await
 		).unwrap()
@@ -327,10 +306,10 @@ macro_rules! sym {
 /// The components are interned much like in [sym].
 #[macro_export]
 macro_rules! vname {
-  ($seg1:tt $( :: $seg:tt)* ; $i:expr) => {
+  ($seg1:tt $( :: $seg:tt)*) => {
     $crate::name::VName::new([
-      $i.i(stringify!($seg1)).await
-      $( , $i.i(stringify!($seg)).await )*
+      $crate::interner::is(stringify!($seg1)).await
+      $( , $crate::interner::is(stringify!($seg)).await )*
     ]).unwrap()
 	};
 }
@@ -340,10 +319,10 @@ macro_rules! vname {
 /// The components are interned much like in [sym].
 #[macro_export]
 macro_rules! vpath {
-  ($seg1:tt $( :: $seg:tt)+ ; $i:expr) => {
+  ($seg1:tt $( :: $seg:tt)*) => {
     $crate::name::VPath(vec![
-      $i.i(stringify!($seg1)).await
-      $( , $i.i(stringify!($seg)).await )+
+      $crate::interner::is(stringify!($seg1)).await
+      $( , $crate::interner::is(stringify!($seg)).await )*
 		])
 	};
   () => {
@@ -352,42 +331,33 @@ macro_rules! vpath {
 }
 
 #[cfg(test)]
-mod test {
+pub mod test {
 	use std::borrow::Borrow;
 
-	use test_executors::spin_on;
-
 	use super::{NameLike, Sym, VName};
-	use crate::interner::{Interner, Tok};
+	use crate::interner::{IStr, is};
 	use crate::name::VPath;
 
-	#[test]
-	fn recur() {
-		spin_on(async {
-			let i = Interner::new_master();
-			let myname = vname!(foo::bar; i);
-			let _borrowed_slice: &[Tok<String>] = myname.borrow();
-			let _deref_pathslice: &[Tok<String>] = &myname;
-			let _as_slice_out: &[Tok<String>] = myname.as_slice();
-		})
+	pub async fn recur() {
+		let myname = vname!(foo::bar);
+		let _borrowed_slice: &[IStr] = myname.borrow();
+		let _deref_pathslice: &[IStr] = &myname;
+		let _as_slice_out: &[IStr] = myname.as_slice();
 	}
 
-	#[test]
-	fn literals() {
-		spin_on(async {
-			let i = Interner::new_master();
-			assert_eq!(
-				sym!(foo::bar::baz; i),
-				Sym::new([i.i("foo").await, i.i("bar").await, i.i("baz").await], &i).await.unwrap()
-			);
-			assert_eq!(
-				vname!(foo::bar::baz; i),
-				VName::new([i.i("foo").await, i.i("bar").await, i.i("baz").await]).unwrap()
-			);
-			assert_eq!(
-				vpath!(foo::bar::baz; i),
-				VPath::new([i.i("foo").await, i.i("bar").await, i.i("baz").await])
-			);
-		})
+	/// Tests that literals are correctly interned as equal
+	pub async fn literals() {
+		assert_eq!(
+			sym!(foo::bar::baz),
+			Sym::new([is("foo").await, is("bar").await, is("baz").await]).await.unwrap()
+		);
+		assert_eq!(
+			vname!(foo::bar::baz),
+			VName::new([is("foo").await, is("bar").await, is("baz").await]).unwrap()
+		);
+		assert_eq!(
+			vpath!(foo::bar::baz),
+			VPath::new([is("foo").await, is("bar").await, is("baz").await])
+		);
 	}
 }

orchid-base/src/number.rs

@@ -4,7 +4,7 @@ use std::ops::Range;
 use ordered_float::NotNan;
 
 use crate::error::{OrcErrv, mk_errv};
-use crate::interner::Interner;
+use crate::interner::is;
 use crate::location::SrcRange;
 use crate::name::Sym;
 
@@ -55,14 +55,9 @@ pub struct NumError {
 	pub kind: NumErrorKind,
 }
 
-pub async fn num_to_errv(
-	NumError { kind, range }: NumError,
-	offset: u32,
-	source: &Sym,
-	i: &Interner,
-) -> OrcErrv {
+pub async fn num_to_errv(NumError { kind, range }: NumError, offset: u32, source: &Sym) -> OrcErrv {
 	mk_errv(
-		i.i("Failed to parse number").await,
+		is("Failed to parse number").await,
 		match kind {
 			NumErrorKind::NaN => "NaN emerged during parsing",
 			NumErrorKind::InvalidDigit => "non-digit character encountered",

orchid-base/src/parse.rs

@@ -7,28 +7,13 @@ use futures::future::join_all;
 use itertools::Itertools;
 
 use crate::api;
-use crate::error::{OrcErrv, OrcRes, Reporter, mk_errv};
+use crate::error::{OrcErrv, OrcRes, mk_errv, report};
 use crate::format::{FmtCtx, FmtUnit, Format, fmt};
-use crate::interner::{Interner, Tok};
+use crate::interner::{IStr, es, is};
 use crate::location::SrcRange;
 use crate::name::{Sym, VName, VPath};
 use crate::tree::{ExprRepr, ExtraTok, Paren, TokTree, Token, ttv_fmt, ttv_range};
 
-pub trait ParseCtx {
-	#[must_use]
-	fn i(&self) -> &Interner;
-	#[must_use]
-	fn rep(&self) -> &Reporter;
-}
-pub struct ParseCtxImpl<'a> {
-	pub i: &'a Interner,
-	pub r: &'a Reporter,
-}
-impl ParseCtx for ParseCtxImpl<'_> {
-	fn i(&self) -> &Interner { self.i }
-	fn rep(&self) -> &Reporter { self.r }
-}
-
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
 pub fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
 pub fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}\\".contains(c) }
@@ -103,22 +88,22 @@ impl<A: ExprRepr, X: ExtraTok> Format for Snippet<'_, A, X> {
 
 #[derive(Clone, Debug)]
 pub struct Comment {
-	pub text: Tok<String>,
+	pub text: IStr,
 	pub sr: SrcRange,
 }
 impl Comment {
 	// XXX: which of these four are actually used?
-	pub async fn from_api(c: &api::Comment, src: Sym, i: &Interner) -> Self {
-		Self { text: i.ex(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
+	pub async fn from_api(c: &api::Comment, src: Sym) -> Self {
+		Self { text: es(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
 	}
-	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>, i: &Interner) -> Option<Self> {
+	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>) -> Option<Self> {
 		match &tk.tok {
-			Token::Comment(text) => Some(Self { text: i.i(&**text).await, sr: tk.sr.clone() }),
+			Token::Comment(text) => Some(Self { text: text.clone(), sr: tk.sr.clone() }),
 			_ => None,
 		}
 	}
 	pub fn to_tk<R: ExprRepr, X: ExtraTok>(&self) -> TokTree<R, X> {
-		TokTree { tok: Token::Comment(self.text.rc().clone()), sr: self.sr.clone() }
+		TokTree { tok: Token::Comment(self.text.clone()), sr: self.sr.clone() }
 	}
 	pub fn to_api(&self) -> api::Comment {
 		api::Comment { range: self.sr.range(), text: self.text.to_api() }
@@ -130,7 +115,6 @@ impl fmt::Display for Comment {
 }
 
 pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
 	snip: Snippet<'a, A, X>,
 ) -> Vec<Parsed<'a, Vec<Comment>, A, X>> {
 	let mut items = Vec::new();
@@ -145,9 +129,10 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
 			None => comments.extend(line.cur),
 			Some(i) => {
 				let (cmts, tail) = line.split_at(i);
-				let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async {
-					Comment::from_tk(t, ctx.i()).await.expect("All are comments checked above")
-				}))
+				let comments = join_all(
+					(comments.drain(..).chain(cmts.cur))
+						.map(|t| async { Comment::from_tk(t).await.expect("All are comments checked above") }),
+				)
 				.await;
 				items.push(Parsed { output: comments, tail });
 			},
@@ -157,26 +142,21 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
 }
 
 pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
 	snip: Snippet<'a, A, X>,
 ) -> ParseRes<'a, &'a TokTree<A, X>, A, X> {
 	match snip.skip_fluff().pop_front() {
 		Some((output, tail)) => Ok(Parsed { output, tail }),
-		None => Err(mk_errv(
-			ctx.i().i("Unexpected end").await,
-			"Line ends abruptly; more tokens were expected",
-			[snip.sr()],
-		)),
+		None =>
+			Err(mk_errv(is("Unexpected end").await, "Line ends abruptly; more tokens were expected", [
+				snip.sr(),
+			])),
 	}
 }
 
-pub async fn expect_end(
-	ctx: &impl ParseCtx,
-	snip: Snippet<'_, impl ExprRepr, impl ExtraTok>,
-) -> OrcRes<()> {
+pub async fn expect_end(snip: Snippet<'_, impl ExprRepr, impl ExtraTok>) -> OrcRes<()> {
 	match snip.skip_fluff().get(0) {
 		Some(surplus) => Err(mk_errv(
-			ctx.i().i("Extra code after end of line").await,
+			is("Extra code after end of line").await,
 			"Code found after the end of the line",
 			[surplus.sr.pos()],
 		)),
@@ -185,28 +165,26 @@ pub async fn expect_end(
 }
 
 pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
 	snip: Snippet<'a, A, X>,
-	tok: Tok<String>,
+	tok: IStr,
 ) -> ParseRes<'a, (), A, X> {
-	let Parsed { output: head, tail } = try_pop_no_fluff(ctx, snip).await?;
+	let Parsed { output: head, tail } = try_pop_no_fluff(snip).await?;
 	match &head.tok {
 		Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
 		t => Err(mk_errv(
-			ctx.i().i("Expected specific keyword").await,
-			format!("Expected {tok} but found {:?}", fmt(t, ctx.i()).await),
+			is("Expected specific keyword").await,
+			format!("Expected {tok} but found {:?}", fmt(t).await),
 			[head.sr()],
 		)),
 	}
 }
 
 pub async fn token_errv<A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
 	tok: &TokTree<A, X>,
 	description: &'static str,
 	message: impl FnOnce(&str) -> String,
 ) -> OrcErrv {
-	mk_errv(ctx.i().i(description).await, message(&fmt(tok, ctx.i()).await), [tok.sr.pos()])
+	mk_errv(is(description).await, message(&fmt(tok).await), [tok.sr.pos()])
 }
 
 pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
@@ -217,33 +195,27 @@ pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
 pub type ParseRes<'a, T, H, X> = OrcRes<Parsed<'a, T, H, X>>;
 
 pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
-	ctx: &impl ParseCtx,
 	tail: Snippet<'a, A, X>,
 ) -> ParseRes<'a, Vec<Import>, A, X> {
 	let Some((tt, tail)) = tail.skip_fluff().pop_front() else {
 		return Err(mk_errv(
-			ctx.i().i("Expected token").await,
+			is("Expected token").await,
 			"Expected a name, a parenthesized list of names, or a globstar.",
 			[tail.sr().pos()],
 		));
 	};
-	let ret = rec(tt, ctx).await;
+	let ret = rec(tt).await;
 	#[allow(clippy::type_complexity)] // it's an internal function
 	pub async fn rec<A: ExprRepr, X: ExtraTok>(
 		tt: &TokTree<A, X>,
-		ctx: &impl ParseCtx,
-	) -> OrcRes<Vec<(Vec<Tok<String>>, Option<Tok<String>>, SrcRange)>> {
+	) -> OrcRes<Vec<(Vec<IStr>, Option<IStr>, SrcRange)>> {
 		let ttpos = tt.sr.pos();
 		match &tt.tok {
 			Token::NS(ns, body) => {
 				if !ns.starts_with(name_start) {
-					ctx.rep().report(mk_errv(
-						ctx.i().i("Unexpected name prefix").await,
-						"Only names can precede ::",
-						[ttpos],
-					))
+					report(mk_errv(is("Unexpected name prefix").await, "Only names can precede ::", [ttpos]))
 				};
-				let out = Box::pin(rec(body, ctx)).await?;
+				let out = Box::pin(rec(body)).await?;
 				Ok(out.into_iter().update(|i| i.0.push(ns.clone())).collect_vec())
 			},
 			Token::Name(ntok) => {
@@ -255,21 +227,19 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
 				let mut o = Vec::new();
 				let mut body = Snippet::new(tt, b);
 				while let Some((output, tail)) = body.pop_front() {
-					match rec(output, ctx).boxed_local().await {
+					match rec(output).boxed_local().await {
 						Ok(names) => o.extend(names),
-						Err(e) => ctx.rep().report(e),
+						Err(e) => report(e),
 					}
 					body = tail;
 				}
 				Ok(o)
 			},
-			t => {
-				return Err(mk_errv(
-					ctx.i().i("Unrecognized name end").await,
-					format!("Names cannot end with {:?} tokens", fmt(t, ctx.i()).await),
-					[ttpos],
-				));
-			},
+			t => Err(mk_errv(
+				is("Unrecognized name end").await,
+				format!("Names cannot end with {:?} tokens", fmt(t).await),
+				[ttpos],
+			)),
 		}
 	}
 	ret.map(|output| {
@@ -285,7 +255,7 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
 #[derive(Debug, Clone)]
 pub struct Import {
 	pub path: VPath,
-	pub name: Option<Tok<String>>,
+	pub name: Option<IStr>,
 	pub sr: SrcRange,
 }
 impl Import {
@@ -296,14 +266,14 @@ impl Import {
 			None => self.path.into_name().expect("Import cannot be empty"),
 		}
 	}
-	pub fn new(sr: SrcRange, path: VPath, name: Tok<String>) -> Self {
+	pub fn new(sr: SrcRange, path: VPath, name: IStr) -> Self {
 		Import { path, name: Some(name), sr }
 	}
 	pub fn new_glob(sr: SrcRange, path: VPath) -> Self { Import { path, name: None, sr } }
 }
 impl Display for Import {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| t.as_str()))
+		write!(f, "{}::{}", self.path.iter().join("::"), self.name.as_ref().map_or("*", |t| &**t))
 	}
 }
 

orchid-base/src/reqnot.rs

@@ -1,357 +1,573 @@
 use std::cell::RefCell;
-use std::future::Future;
 use std::marker::PhantomData;
-use std::mem;
-use std::ops::{BitAnd, Deref};
-use std::pin::Pin;
-use std::sync::Arc;
-use std::thread::panicking;
+use std::pin::{Pin, pin};
+use std::rc::Rc;
+use std::{io, mem};
 
+use async_fn_stream::try_stream;
+use bound::Bound;
 use derive_destructure::destructure;
-use dyn_clone::{DynClone, clone_box};
-use futures::channel::mpsc;
+use futures::channel::mpsc::{self, Receiver, Sender, channel};
+use futures::channel::oneshot;
 use futures::future::LocalBoxFuture;
-use futures::lock::Mutex;
-use futures::{SinkExt, StreamExt};
+use futures::lock::{Mutex, MutexGuard};
+use futures::{
+	AsyncRead, AsyncWrite, AsyncWriteExt, FutureExt, SinkExt, Stream, StreamExt, stream_select,
+};
 use hashbrown::HashMap;
-use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request, enc_vec};
-use trait_set::trait_set;
+use orchid_api_traits::{Decode, Encode, Request, UnderRoot};
 
-use crate::clone;
-use crate::logging::Logger;
+use crate::localset::LocalSet;
 
+#[must_use = "Receipts indicate that a required action has been performed within a function. \
+	Most likely this should be returned somewhere."]
 pub struct Receipt<'a>(PhantomData<&'a mut ()>);
-
-trait_set! {
-	pub trait SendFn<T: MsgSet> =
-		for<'a> FnMut(&'a [u8], ReqNot<T>) -> LocalBoxFuture<'a, ()>
-		+ DynClone + 'static;
-	pub trait ReqFn<T: MsgSet> =
-		for<'a> FnMut(RequestHandle<'a, T>, <T::In as Channel>::Req)
-			-> LocalBoxFuture<'a, Receipt<'a>>
-		+ DynClone + 'static;
-	pub trait NotifFn<T: MsgSet> =
-		FnMut(<T::In as Channel>::Notif, ReqNot<T>) -> LocalBoxFuture<'static, ()>
-		+ DynClone + 'static;
+impl Receipt<'_> {
+	/// Only call this function from a custom implementation of [RepWriter]
+	pub fn _new() -> Self { Self(PhantomData) }
 }
 
-fn get_id(message: &[u8]) -> (u64, &[u8]) {
-	(u64::from_be_bytes(message[..8].to_vec().try_into().unwrap()), &message[8..])
+/// Write guard to outbound for the purpose of serializing a request. Only one
+/// can exist at a time. Dropping this object should panic.
+pub trait ReqWriter<'a> {
+	/// Access to the underlying channel. This may be buffered.
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
+	/// Finalize the request, release the outbound channel, then queue for the
+	/// reply on the inbound channel.
+	fn send(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepReader<'a> + 'a>>>;
 }
 
-pub trait ReqHandlish {
-	fn defer(&self, cb: impl Future<Output = ()> + 'static)
-	where Self: Sized {
-		self.defer_objsafe(Box::pin(cb));
-	}
-	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>);
-}
-impl ReqHandlish for &'_ dyn ReqHandlish {
-	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>) { (**self).defer_objsafe(val) }
+/// Write guard to inbound for the purpose of deserializing a reply. While held,
+/// no inbound requests or other replies can be processed.
+///
+/// Dropping this object should panic even if [RepReader::finish] returns
+/// synchronously, because the API isn't cancellation safe in general so it is a
+/// programmer error in all cases to drop an object related to it without proper
+/// cleanup.
+pub trait RepReader<'a> {
+	/// Access to the underlying channel. The length of the message is inferred
+	/// from the number of bytes read so this must not be buffered.
+	fn reader(&mut self) -> Pin<&mut dyn AsyncRead>;
+	/// Finish reading the request
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, ()>;
 }
 
-type LocalAsyncFnOnceBox = Box<dyn FnOnce(Vec<u8>) -> LocalBoxFuture<'static, ()>>;
+/// Write guard to outbound for the purpose of serializing a notification.
+///
+/// Dropping this object should panic for the same reason [RepReader] panics
+pub trait MsgWriter<'a> {
+	/// Access to the underlying channel. This may be buffered.
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
+	/// Send the notification
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<()>>;
+}
 
-#[derive(destructure)]
-pub struct RequestHandle<'a, MS: MsgSet> {
-	defer: RefCell<Vec<Pin<Box<dyn Future<Output = ()>>>>>,
-	_reqlt: PhantomData<&'a mut ()>,
-	parent: ReqNot<MS>,
-	raw_reply: RefCell<Option<LocalAsyncFnOnceBox>>,
+/// For initiating outbound requests and notifications
+pub trait Client {
+	fn start_request(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn ReqWriter<'_> + '_>>>;
+	fn start_notif(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn MsgWriter<'_> + '_>>>;
 }
-impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
-	pub fn new(parent: ReqNot<MS>, raw_reply: impl AsyncFnOnce(Vec<u8>) + 'static) -> Self {
-		Self {
-			defer: RefCell::default(),
-			_reqlt: PhantomData,
-			parent,
-			raw_reply: RefCell::new(Some(Box::new(|v| Box::pin(raw_reply(v))))),
-		}
+
+impl<T: Client + ?Sized> ClientExt for T {}
+/// Extension trait with convenience methods that handle outbound request and
+/// notif lifecycle and typing
+#[allow(async_fn_in_trait)]
+pub trait ClientExt: Client {
+	async fn request<T: Request + UnderRoot<Root: Encode>>(&self, t: T) -> io::Result<T::Response> {
+		let mut req = self.start_request().await?;
+		t.into_root().encode(req.writer().as_mut()).await?;
+		let mut rep = req.send().await?;
+		let response = T::Response::decode(rep.reader()).await;
+		rep.finish().await;
+		response
 	}
-	pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
-	pub async fn handle<U: Request>(&self, _: &U, rep: &U::Response) -> Receipt<'a> {
-		self.respond(rep).await
-	}
-	pub fn will_handle_as<U: Request>(&self, _: &U) -> ReqTypToken<U> { ReqTypToken(PhantomData) }
-	pub async fn handle_as<U: Request>(&self, _: ReqTypToken<U>, rep: &U::Response) -> Receipt<'a> {
-		self.respond(rep).await
-	}
-	pub async fn respond(&self, response: &impl Encode) -> Receipt<'a> {
-		let replier = self.raw_reply.borrow_mut().take().expect("Already responded to request");
-		let buf = enc_vec(response).await;
-		(replier)(buf).await;
-		let deferred = mem::take(&mut *self.defer.borrow_mut());
-		for item in deferred {
-			item.await
-		}
-		Receipt(PhantomData)
-	}
-}
-impl<MS: MsgSet> ReqHandlish for RequestHandle<'_, MS> {
-	fn defer_objsafe(&self, val: Pin<Box<dyn Future<Output = ()>>>) {
-		self.defer.borrow_mut().push(val)
-	}
-}
-impl<MS: MsgSet> Drop for RequestHandle<'_, MS> {
-	fn drop(&mut self) {
-		if !panicking() {
-			debug_assert!(self.raw_reply.borrow().is_none(), "Request dropped without response")
-		}
+	async fn notify<T: UnderRoot<Root: Encode>>(&self, t: T) -> io::Result<()> {
+		let mut notif = self.start_notif().await?;
+		t.into_root().encode(notif.writer().as_mut()).await?;
+		notif.finish().await?;
+		Ok(())
 	}
 }
 
-pub struct ReqTypToken<T>(PhantomData<T>);
-
-pub struct ReqNotData<T: MsgSet> {
-	id: u64,
-	send: Box<dyn SendFn<T>>,
-	notif: Box<dyn NotifFn<T>>,
-	req: Box<dyn ReqFn<T>>,
-	responses: HashMap<u64, mpsc::Sender<Vec<u8>>>,
+pub trait ReqReader<'a> {
+	fn reader(&mut self) -> Pin<&mut dyn AsyncRead>;
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, Box<dyn ReqHandle<'a> + 'a>>;
 }
-
-/// Wraps a raw message buffer to save on copying.
-/// Dereferences to the tail of the message buffer, cutting off the ID
-#[derive(Debug, Clone)]
-pub struct RawReply(Vec<u8>);
-impl Deref for RawReply {
-	type Target = [u8];
-	fn deref(&self) -> &Self::Target { get_id(&self.0[..]).1 }
-}
-
-pub struct ReqNot<T: MsgSet>(Arc<Mutex<ReqNotData<T>>>, Logger);
-impl<T: MsgSet> ReqNot<T> {
-	pub fn new(
-		logger: Logger,
-		send: impl SendFn<T>,
-		notif: impl NotifFn<T>,
-		req: impl ReqFn<T>,
-	) -> Self {
-		Self(
-			Arc::new(Mutex::new(ReqNotData {
-				id: 1,
-				send: Box::new(send),
-				notif: Box::new(notif),
-				req: Box::new(req),
-				responses: HashMap::new(),
-			})),
-			logger,
-		)
+impl<'a, T: ReqReader<'a> + ?Sized> ReqReaderExt<'a> for T {}
+#[allow(async_fn_in_trait)]
+pub trait ReqReaderExt<'a>: ReqReader<'a> {
+	async fn read_req<R: Decode>(&mut self) -> io::Result<R> { R::decode(self.reader()).await }
+	async fn reply<R: Request>(
+		self: Box<Self>,
+		req: impl Evidence<R>,
+		rep: &R::Response,
+	) -> io::Result<Receipt<'a>> {
+		self.finish().await.reply(req, rep).await
 	}
-
-	/// Can be called from a polling thread or dispatched in any other way
-	pub async fn receive(&self, message: &[u8]) {
-		let mut g = self.0.lock().await;
-		let (id, payload) = get_id(message);
-		if id == 0 {
-			let mut notif_cb = clone_box(&*g.notif);
-			mem::drop(g);
-			let notif_val = <T::In as Channel>::Notif::decode(Pin::new(&mut &payload[..])).await;
-			notif_cb(notif_val, self.clone()).await
-		} else if 0 < id.bitand(1 << 63) {
-			let mut sender = g.responses.remove(&!id).expect("Received response for invalid message");
-			let _ = sender.send(message.to_vec()).await;
-		} else {
-			let message = <T::In as Channel>::Req::decode(Pin::new(&mut &payload[..])).await;
-			let mut req_cb = clone_box(&*g.req);
-			mem::drop(g);
-			let rn = self.clone();
-			let rn2 = self.clone();
-			req_cb(
-				RequestHandle::new(rn, async move |vec| {
-					let mut buf = (!id).to_be_bytes().to_vec();
-					buf.extend(vec);
-					let mut send = clone_box(&*rn2.0.lock().await.send);
-					(send)(&buf, rn2.clone()).await;
-				}),
-				message,
-			)
-			.await;
-		}
-	}
-
-	pub async fn notify<N: Coding + Into<<T::Out as Channel>::Notif>>(&self, notif: N) {
-		let mut send = clone_box(&*self.0.lock().await.send);
-		let mut buf = vec![0; 8];
-		let msg: <T::Out as Channel>::Notif = notif.into();
-		msg.encode(Pin::new(&mut buf)).await;
-		send(&buf, self.clone()).await
+	async fn start_reply(self: Box<Self>) -> io::Result<Box<dyn RepWriter<'a> + 'a>> {
+		self.finish().await.start_reply().await
 	}
 }
 
-pub trait DynRequester {
-	type Transfer;
-	fn logger(&self) -> &Logger;
-	/// Encode and send a request, then receive the response buffer.
-	fn raw_request(&self, data: Self::Transfer) -> LocalBoxFuture<'_, RawReply>;
+pub trait ReqHandle<'a> {
+	fn start_reply(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepWriter<'a> + 'a>>>;
 }
-
-pub struct MappedRequester<'a, T: 'a>(Box<dyn Fn(T) -> LocalBoxFuture<'a, RawReply> + 'a>, Logger);
-impl<'a, T> MappedRequester<'a, T> {
-	fn new<U: DynRequester + 'a, F: Fn(T) -> U::Transfer + 'a>(
-		req: U,
-		cb: F,
-		logger: Logger,
-	) -> Self {
-		let req_arc = Arc::new(req);
-		let cb_arc = Arc::new(cb);
-		MappedRequester(
-			Box::new(move |t| {
-				Box::pin(clone!(req_arc, cb_arc; async move { req_arc.raw_request(cb_arc(t)).await}))
-			}),
-			logger,
-		)
+impl<'a, T: ReqHandle<'a> + ?Sized> ReqHandleExt<'a> for T {}
+#[allow(async_fn_in_trait)]
+pub trait ReqHandleExt<'a>: ReqHandle<'a> {
+	async fn reply<Req: Request>(
+		self: Box<Self>,
+		_: impl Evidence<Req>,
+		rep: &Req::Response,
+	) -> io::Result<Receipt<'a>> {
+		let mut reply = self.start_reply().await?;
+		rep.encode(reply.writer()).await?;
+		reply.finish().await
 	}
 }
 
-impl<T> DynRequester for MappedRequester<'_, T> {
-	type Transfer = T;
-	fn logger(&self) -> &Logger { &self.1 }
-	fn raw_request(&self, data: Self::Transfer) -> LocalBoxFuture<'_, RawReply> { self.0(data) }
+pub trait RepWriter<'a> {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite>;
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Receipt<'a>>>;
 }
 
-impl<T: MsgSet> DynRequester for ReqNot<T> {
-	type Transfer = <T::Out as Channel>::Req;
-	fn logger(&self) -> &Logger { &self.1 }
-	fn raw_request(&self, req: Self::Transfer) -> LocalBoxFuture<'_, RawReply> {
+pub trait MsgReader<'a> {
+	fn reader(&mut self) -> Pin<&mut dyn AsyncRead>;
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, ()>;
+}
+impl<'a, T: ?Sized + MsgReader<'a>> MsgReaderExt<'a> for T {}
+#[allow(async_fn_in_trait)]
+pub trait MsgReaderExt<'a>: MsgReader<'a> {
+	async fn read<N: Decode>(mut self: Box<Self>) -> io::Result<N> {
+		let n = N::decode(self.reader()).await;
+		self.finish().await;
+		n
+	}
+}
+
+/// A form of [Evidence] that doesn't require the value to be kept around
+pub struct Witness<T>(PhantomData<T>);
+impl<T> Witness<T> {
+	pub fn of(_: &T) -> Self { Self(PhantomData) }
+}
+impl<T> Copy for Witness<T> {}
+impl<T> Clone for Witness<T> {
+	fn clone(&self) -> Self { *self }
+}
+
+/// A proxy for the type of a value either previously saved into a [Witness] or
+/// still available.
+pub trait Evidence<T> {}
+impl<T> Evidence<T> for &'_ T {}
+impl<T> Evidence<T> for Witness<T> {}
+
+type IoRef<T> = Pin<Box<T>>;
+type IoLock<T> = Rc<Mutex<Pin<Box<T>>>>;
+type IoGuard<T> = Bound<MutexGuard<'static, Pin<Box<T>>>, IoLock<T>>;
+
+/// An incoming request. This holds a lock on the ingress channel.
+pub struct IoReqReader<'a> {
+	prefix: &'a [u8],
+	read: IoGuard<dyn AsyncRead>,
+	write: &'a Mutex<IoRef<dyn AsyncWrite>>,
+}
+impl<'a> ReqReader<'a> for IoReqReader<'a> {
+	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { self.read.as_mut() }
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'a, Box<dyn ReqHandle<'a> + 'a>> {
+		Box::pin(async {
+			Box::new(IoReqHandle { prefix: self.prefix, write: self.write }) as Box<dyn ReqHandle<'a>>
+		})
+	}
+}
+pub struct IoReqHandle<'a> {
+	prefix: &'a [u8],
+	write: &'a Mutex<IoRef<dyn AsyncWrite>>,
+}
+impl<'a> ReqHandle<'a> for IoReqHandle<'a> {
+	fn start_reply(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepWriter<'a> + 'a>>> {
 		Box::pin(async move {
-			let mut g = self.0.lock().await;
-			let id = g.id;
-			g.id += 1;
-			let mut buf = id.to_be_bytes().to_vec();
-			req.encode(Pin::new(&mut buf)).await;
-			let (send, mut recv) = mpsc::channel(1);
-			g.responses.insert(id, send);
-			let mut send = clone_box(&*g.send);
-			mem::drop(g);
-			let rn = self.clone();
-			send(&buf, rn).await;
-			let items = recv.next().await;
-			RawReply(items.unwrap())
+			let mut write = self.write.lock().await;
+			write.as_mut().write_all(self.prefix).await?;
+			Ok(Box::new(IoRepWriter { write }) as Box<dyn RepWriter<'a>>)
+		})
+	}
+}
+pub struct IoRepWriter<'a> {
+	write: MutexGuard<'a, IoRef<dyn AsyncWrite>>,
+}
+impl<'a> RepWriter<'a> for IoRepWriter<'a> {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { self.write.as_mut() }
+	fn finish(mut self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Receipt<'a>>> {
+		Box::pin(async move {
+			self.writer().flush().await?;
+			Ok(Receipt(PhantomData))
 		})
 	}
 }
 
-pub trait Requester: DynRequester {
-	#[must_use = "These types are subject to change with protocol versions. \
-    If you don't want to use the return value, At a minimum, force the type."]
-	fn request<R: Request + Into<Self::Transfer>>(
-		&self,
-		data: R,
-	) -> impl Future<Output = R::Response>;
-	fn map<'a, U>(self, cb: impl Fn(U) -> Self::Transfer + 'a) -> MappedRequester<'a, U>
-	where Self: Sized + 'a {
-		let logger = self.logger().clone();
-		MappedRequester::new(self, cb, logger)
+pub struct IoMsgReader<'a> {
+	_pd: PhantomData<&'a mut ()>,
+	read: IoGuard<dyn AsyncRead>,
+}
+impl<'a> MsgReader<'a> for IoMsgReader<'a> {
+	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { self.read.as_mut() }
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'static, ()> { Box::pin(async {}) }
+}
+
+#[derive(Debug)]
+struct ReplySub {
+	id: u64,
+	ack: oneshot::Sender<()>,
+	cb: oneshot::Sender<IoGuard<dyn AsyncRead>>,
+}
+
+struct IoClient {
+	output: IoLock<dyn AsyncWrite>,
+	id: Rc<RefCell<u64>>,
+	subscribe: Rc<Sender<ReplySub>>,
+}
+impl IoClient {
+	fn new(output: IoLock<dyn AsyncWrite>) -> (Receiver<ReplySub>, Self) {
+		let (req, rep) = mpsc::channel(0);
+		(rep, Self { output, id: Rc::new(RefCell::new(0)), subscribe: Rc::new(req) })
+	}
+	async fn lock_out(&self) -> IoGuard<dyn AsyncWrite> {
+		Bound::async_new(self.output.clone(), async |o| o.lock().await).await
+	}
+}
+impl Client for IoClient {
+	fn start_notif(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn MsgWriter<'_> + '_>>> {
+		Box::pin(async {
+			let mut o = self.lock_out().await;
+			0u64.encode(o.as_mut()).await?;
+			Ok(Box::new(IoNotifWriter { o }) as Box<dyn MsgWriter>)
+		})
+	}
+	fn start_request(&self) -> LocalBoxFuture<'_, io::Result<Box<dyn ReqWriter<'_> + '_>>> {
+		Box::pin(async {
+			let id = {
+				let mut id_g = self.id.borrow_mut();
+				*id_g += 1;
+				*id_g
+			};
+			let (cb, reply) = oneshot::channel();
+			let (ack, got_ack) = oneshot::channel();
+			self.subscribe.as_ref().clone().send(ReplySub { id, ack, cb }).await.unwrap();
+			got_ack.await.unwrap();
+			let mut w = self.lock_out().await;
+			id.encode(w.as_mut()).await?;
+			Ok(Box::new(IoReqWriter { reply, w }) as Box<dyn ReqWriter>)
+		})
 	}
 }
 
-impl<This: DynRequester + ?Sized> Requester for This {
-	async fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
-		let req = format!("{data:?}");
-		let rep = R::Response::decode(Pin::new(&mut &self.raw_request(data.into()).await[..])).await;
-		let req_str = req.to_string();
-		if !req_str.starts_with("AtomPrint") && !req_str.starts_with("ExtAtomPrint") {
-			writeln!(self.logger(), "Request {req} got response {rep:?}");
+struct IoReqWriter {
+	reply: oneshot::Receiver<IoGuard<dyn AsyncRead>>,
+	w: IoGuard<dyn AsyncWrite>,
+}
+impl<'a> ReqWriter<'a> for IoReqWriter {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { self.w.as_mut() }
+	fn send(self: Box<Self>) -> LocalBoxFuture<'a, io::Result<Box<dyn RepReader<'a> + 'a>>> {
+		Box::pin(async {
+			let Self { reply, mut w } = *self;
+			w.flush().await?;
+			mem::drop(w);
+			let i = reply.await.expect("Client dropped before reply received");
+			Ok(Box::new(IoRepReader { i }) as Box<dyn RepReader>)
+		})
+	}
+}
+
+struct IoRepReader {
+	i: IoGuard<dyn AsyncRead>,
+}
+impl<'a> RepReader<'a> for IoRepReader {
+	fn reader(&mut self) -> Pin<&mut dyn AsyncRead> { self.i.as_mut() }
+	fn finish(self: Box<Self>) -> LocalBoxFuture<'static, ()> { Box::pin(async {}) }
+}
+
+#[derive(destructure)]
+struct IoNotifWriter {
+	o: IoGuard<dyn AsyncWrite>,
+}
+impl<'a> MsgWriter<'a> for IoNotifWriter {
+	fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> { self.o.as_mut() }
+	fn finish(mut self: Box<Self>) -> LocalBoxFuture<'static, io::Result<()>> {
+		Box::pin(async move { self.o.flush().await })
+	}
+}
+
+pub struct CommCtx {
+	exit: Sender<()>,
+}
+
+impl CommCtx {
+	pub async fn exit(self) { self.exit.clone().send(()).await.expect("quit channel dropped"); }
+}
+
+/// Establish bidirectional request-notification communication over a duplex
+/// channel. The returned [IoClient] can be used for notifications immediately,
+/// but requests can only be received while the future is running. The future
+/// will only resolve when [CommCtx::quit] is called. The generic type
+/// parameters are associated with the client and serve to ensure with a runtime
+/// check that the correct message families are sent in the correct directions
+/// across the channel.
+pub fn io_comm(
+	o: Rc<Mutex<Pin<Box<dyn AsyncWrite>>>>,
+	i: Mutex<Pin<Box<dyn AsyncRead>>>,
+) -> (impl Client + 'static, CommCtx, IoCommServer) {
+	let i = Rc::new(i);
+	let (onsub, client) = IoClient::new(o.clone());
+	let (exit, onexit) = channel(1);
+	(client, CommCtx { exit }, IoCommServer { o, i, onsub, onexit })
+}
+pub struct IoCommServer {
+	o: Rc<Mutex<Pin<Box<dyn AsyncWrite>>>>,
+	i: Rc<Mutex<Pin<Box<dyn AsyncRead>>>>,
+	onsub: Receiver<ReplySub>,
+	onexit: Receiver<()>,
+}
+impl IoCommServer {
+	pub async fn listen(
+		self,
+		notif: impl for<'a> AsyncFn(Box<dyn MsgReader<'a> + 'a>) -> io::Result<()>,
+		req: impl for<'a> AsyncFn(Box<dyn ReqReader<'a> + 'a>) -> io::Result<Receipt<'a>>,
+	) -> io::Result<()> {
+		let Self { o, i, onexit, onsub } = self;
+		enum Event {
+			Input(u64, IoGuard<dyn AsyncRead>),
+			Sub(ReplySub),
+			Exit,
 		}
-		rep
+		let exiting = RefCell::new(false);
+		let input_stream = try_stream(async |mut h| {
+			loop {
+				let mut g = Bound::async_new(i.clone(), async |i| i.lock().await).await;
+				match u64::decode(g.as_mut()).await {
+					Ok(id) => h.emit(Event::Input(id, g)).await,
+					Err(e)
+						if matches!(
+							e.kind(),
+							io::ErrorKind::BrokenPipe
+								| io::ErrorKind::ConnectionAborted
+								| io::ErrorKind::UnexpectedEof
+						) =>
+						h.emit(Event::Exit).await,
+					Err(e) => return Err(e),
+				}
+			}
+		});
+		let (mut add_pending_req, fork_future) = LocalSet::new();
+		let mut fork_stream = pin!(fork_future.fuse().into_stream());
+		let mut pending_replies = HashMap::new();
+		'body: {
+			let mut shared = pin!(stream_select!(
+				pin!(input_stream) as Pin<&mut dyn Stream<Item = io::Result<Event>>>,
+				onsub.map(|sub| Ok(Event::Sub(sub))),
+				fork_stream.as_mut().map(|res| {
+					res.map(|()| panic!("this substream cannot exit while the loop is running"))
+				}),
+				onexit.map(|()| Ok(Event::Exit)),
+			));
+			while let Some(next) = shared.next().await {
+				match next {
+					Err(e) => break 'body Err(e),
+					Ok(Event::Exit) => {
+						*exiting.borrow_mut() = true;
+						let mut out = o.lock().await;
+						out.as_mut().flush().await?;
+						out.as_mut().close().await?;
+						break;
+					},
+					Ok(Event::Sub(ReplySub { id, ack, cb })) => {
+						pending_replies.insert(id, cb);
+						ack.send(()).unwrap();
+					},
+					Ok(Event::Input(0, read)) => {
+						let notif = &notif;
+						let notif_job =
+							async move { notif(Box::new(IoMsgReader { _pd: PhantomData, read })).await };
+						add_pending_req.send(Box::pin(notif_job)).await.unwrap();
+					},
+					// MSB == 0 is a request, !id where MSB == 1 is the corresponding response
+					Ok(Event::Input(id, read)) if (id & (1 << (u64::BITS - 1))) == 0 => {
+						let (o, req) = (o.clone(), &req);
+						let req_job = async move {
+							let mut prefix = Vec::new();
+							(!id).encode_vec(&mut prefix);
+							let _ = req(Box::new(IoReqReader { prefix: &pin!(prefix), read, write: &o })).await;
+							Ok(())
+						};
+						add_pending_req.send(Box::pin(req_job)).await.unwrap();
+					},
+					Ok(Event::Input(id, read)) => {
+						let cb = pending_replies.remove(&!id).expect("Reply to unrecognized request");
+						cb.send(read).unwrap_or_else(|_| panic!("Failed to send reply"));
+					},
+				}
+			}
+			Ok(())
+		}?;
+		mem::drop(add_pending_req);
+		while let Some(next) = fork_stream.next().await {
+			next?
+		}
+		Ok(())
 	}
 }
 
-impl<T: MsgSet> Clone for ReqNot<T> {
-	fn clone(&self) -> Self { Self(self.0.clone(), self.1.clone()) }
-}
-
 #[cfg(test)]
 mod test {
+	use std::cell::RefCell;
 	use std::rc::Rc;
-	use std::sync::Arc;
 
-	use futures::FutureExt;
+	use futures::channel::mpsc;
 	use futures::lock::Mutex;
-	use orchid_api_derive::Coding;
-	use orchid_api_traits::{Channel, Request};
+	use futures::{SinkExt, StreamExt, join};
+	use orchid_api_derive::{Coding, Hierarchy};
+	use orchid_api_traits::Request;
 	use test_executors::spin_on;
+	use unsync_pipe::pipe;
 
-	use super::{MsgSet, ReqNot};
-	use crate::logging::Logger;
-	use crate::reqnot::Requester as _;
-	use crate::{api, clone};
+	use crate::logging::test::TestLogger;
+	use crate::logging::with_logger;
+	use crate::reqnot::{ClientExt, MsgReaderExt, ReqReaderExt, io_comm};
 
-	#[derive(Clone, Debug, Coding, PartialEq)]
-	pub struct TestReq(u8);
-	impl Request for TestReq {
-		type Response = u8;
-	}
-
-	pub struct TestChan;
-	impl Channel for TestChan {
-		type Notif = u8;
-		type Req = TestReq;
-	}
-
-	pub struct TestMsgSet;
-	impl MsgSet for TestMsgSet {
-		type In = TestChan;
-		type Out = TestChan;
-	}
+	#[derive(Clone, Debug, PartialEq, Coding, Hierarchy)]
+	#[extendable]
+	struct TestNotif(u64);
 
 	#[test]
 	fn notification() {
-		spin_on(async {
-			let logger = Logger::new(api::LogStrategy::StdErr);
-			let received = Arc::new(Mutex::new(None));
-			let receiver = ReqNot::<TestMsgSet>::new(
-				logger.clone(),
-				|_, _| panic!("Should not send anything"),
-				clone!(received; move |notif, _| clone!(received; async move {
-					*received.lock().await = Some(notif);
-				}.boxed_local())),
-				|_, _| panic!("Not receiving a request"),
+		let logger = TestLogger::new(async |s| eprint!("{s}"));
+		spin_on(with_logger(logger, async {
+			let (in1, out2) = pipe(1024);
+			let (in2, out1) = pipe(1024);
+			let (received, mut on_receive) = mpsc::channel(2);
+			let (_, recv_ctx, recv_srv) =
+				io_comm(Rc::new(Mutex::new(Box::pin(in2))), Mutex::new(Box::pin(out2)));
+			let (sender, ..) = io_comm(Rc::new(Mutex::new(Box::pin(in1))), Mutex::new(Box::pin(out1)));
+			join!(
+				async {
+					recv_srv
+						.listen(
+							async |notif| {
+								received.clone().send(notif.read::<TestNotif>().await?).await.unwrap();
+								Ok(())
+							},
+							async |_| panic!("Should receive notif, not request"),
+						)
+						.await
+						.unwrap()
+				},
+				async {
+					sender.notify(TestNotif(3)).await.unwrap();
+					assert_eq!(on_receive.next().await, Some(TestNotif(3)));
+					sender.notify(TestNotif(4)).await.unwrap();
+					assert_eq!(on_receive.next().await, Some(TestNotif(4)));
+					recv_ctx.exit().await;
+				}
 			);
-			let sender = ReqNot::<TestMsgSet>::new(
-				logger,
-				clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move {
-					receiver.receive(d).await
-				}))),
-				|_, _| panic!("Should not receive notif"),
-				|_, _| panic!("Should not receive request"),
-			);
-			sender.notify(3).await;
-			assert_eq!(*received.lock().await, Some(3));
-			sender.notify(4).await;
-			assert_eq!(*received.lock().await, Some(4));
-		})
+		}))
+	}
+
+	#[derive(Clone, Debug, Coding, Hierarchy)]
+	#[extendable]
+	struct DummyRequest(u64);
+	impl Request for DummyRequest {
+		type Response = u64;
 	}
 
 	#[test]
 	fn request() {
-		spin_on(async {
-			let logger = Logger::new(api::LogStrategy::StdErr);
-			let receiver = Rc::new(Mutex::<Option<ReqNot<TestMsgSet>>>::new(None));
-			let sender = Rc::new(ReqNot::<TestMsgSet>::new(
-				logger.clone(),
-				clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move {
-					receiver.lock().await.as_ref().unwrap().receive(d).await
-				}))),
-				|_, _| panic!("Should not receive notif"),
-				|_, _| panic!("Should not receive request"),
-			));
-			*receiver.lock().await = Some(ReqNot::new(
-				logger,
-				clone!(sender; move |d, _| clone!(sender; Box::pin(async move {
-					sender.receive(d).await
-				}))),
-				|_, _| panic!("Not receiving notifs"),
-				|hand, req| {
-					Box::pin(async move {
-						assert_eq!(req, TestReq(5));
-						hand.respond(&6u8).await
-					})
+		let logger = TestLogger::new(async |s| eprint!("{s}"));
+		spin_on(with_logger(logger, async {
+			let (in1, out2) = pipe(1024);
+			let (in2, out1) = pipe(1024);
+			let (_, srv_ctx, srv) =
+				io_comm(Rc::new(Mutex::new(Box::pin(in2))), Mutex::new(Box::pin(out2)));
+			let (client, client_ctx, client_srv) =
+				io_comm(Rc::new(Mutex::new(Box::pin(in1))), Mutex::new(Box::pin(out1)));
+			join!(
+				async {
+					srv
+						.listen(
+							async |_| panic!("No notifs expected"),
+							async |mut req| {
+								let val = req.read_req::<DummyRequest>().await?;
+								req.reply(&val, &(val.0 + 1)).await
+							},
+						)
+						.await
+						.unwrap()
 				},
-			));
-			let response = sender.request(TestReq(5)).await;
-			assert_eq!(response, 6);
-		})
+				async {
+					client_srv
+						.listen(
+							async |_| panic!("Not expecting ingress notif"),
+							async |_| panic!("Not expecting ingress req"),
+						)
+						.await
+						.unwrap()
+				},
+				async {
+					let response = client.request(DummyRequest(5)).await.unwrap();
+					assert_eq!(response, 6);
+					srv_ctx.exit().await;
+					client_ctx.exit().await;
+				}
+			);
+		}))
+	}
+
+	#[test]
+	fn exit() {
+		let logger = TestLogger::new(async |s| eprint!("{s}"));
+		spin_on(with_logger(logger, async {
+			let (input1, output1) = pipe(1024);
+			let (input2, output2) = pipe(1024);
+			let (reply_client, reply_context, reply_server) =
+				io_comm(Rc::new(Mutex::new(Box::pin(input1))), Mutex::new(Box::pin(output2)));
+			let (req_client, req_context, req_server) =
+				io_comm(Rc::new(Mutex::new(Box::pin(input2))), Mutex::new(Box::pin(output1)));
+			let reply_context = RefCell::new(Some(reply_context));
+			let (exit, onexit) = futures::channel::oneshot::channel::<()>();
+			join!(
+				async move {
+					reply_server
+						.listen(
+							async |hand| {
+								let _notif = hand.read::<TestNotif>().await.unwrap();
+								let context = reply_context.borrow_mut().take().unwrap();
+								context.exit().await;
+								Ok(())
+							},
+							async |mut hand| {
+								let req = hand.read_req::<DummyRequest>().await?;
+								hand.reply(&req, &(req.0 + 1)).await
+							},
+						)
+						.await
+						.unwrap();
+					exit.send(()).unwrap();
+					let _client = reply_client;
+				},
+				async move {
+					req_server
+						.listen(
+							async |_| panic!("Only the other server expected notifs"),
+							async |_| panic!("Only the other server expected requests"),
+						)
+						.await
+						.unwrap();
+					let _ctx = req_context;
+				},
+				async move {
+					req_client.request(DummyRequest(0)).await.unwrap();
+					req_client.notify(TestNotif(0)).await.unwrap();
+					onexit.await.unwrap();
+				}
+			)
+		}));
 	}
 }

orchid-base/src/stash.rs

@@ -0,0 +1,44 @@
+//! A pattern for running async code from sync destructors and other
+//! unfortunately sync callbacks
+//!
+//! We create a task_local vecdeque which is moved into a thread_local whenever
+//! the task is being polled. A call to [stash] pushes the future onto this
+//! deque. Before [with_stash] returns, it pops everything from the deque
+//! individually and awaits each of them, pushing any additionally stashed
+//! futures onto the back of the same deque.
+
+use std::cell::RefCell;
+use std::collections::VecDeque;
+use std::pin::Pin;
+
+use task_local::task_local;
+
+#[derive(Default)]
+struct StashedFutures {
+	queue: RefCell<VecDeque<Pin<Box<dyn Future<Output = ()>>>>>,
+}
+
+task_local! {
+	static STASHED_FUTURES: StashedFutures;
+}
+
+/// Complete the argument future, and any futures spawned from it via [stash].
+/// This is useful mostly to guarantee that messaging destructors have run.
+pub async fn with_stash<F: Future>(fut: F) -> F::Output {
+	STASHED_FUTURES
+		.scope(StashedFutures::default(), async {
+			let val = fut.await;
+			while let Some(fut) = STASHED_FUTURES.with(|sf| sf.queue.borrow_mut().pop_front()) {
+				fut.await;
+			}
+			val
+		})
+		.await
+}
+
+/// Schedule a future to be run before the next [with_stash] guard ends. This is
+/// most useful for sending messages from destructors.
+pub fn stash<F: Future<Output = ()> + 'static>(fut: F) {
+	(STASHED_FUTURES.try_with(|sf| sf.queue.borrow_mut().push_back(Box::pin(fut))))
+		.expect("No stash! Timely completion cannot be guaranteed")
+}

orchid-base/src/tree.rs

@@ -14,7 +14,7 @@ use trait_set::trait_set;
 
 use crate::error::OrcErrv;
 use crate::format::{FmtCtx, FmtUnit, Format, Variants};
-use crate::interner::{Interner, Tok};
+use crate::interner::{IStr, es};
 use crate::location::{Pos, SrcRange};
 use crate::name::{Sym, VName, VPath};
 use crate::parse::Snippet;
@@ -28,7 +28,6 @@ pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::
 		api: &ApiEquiv,
 		ctx: &mut Self::FromApiCtx<'_>,
 		pos: SrcRange,
-		i: &Interner,
 	) -> impl Future<Output = Self>;
 	#[must_use]
 	fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> impl Future<Output = ApiEquiv>;
@@ -36,7 +35,7 @@ pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::
 impl<T: Clone + Debug + Coding> TokenVariant<T> for Never {
 	type FromApiCtx<'a> = ();
 	type ToApiCtx<'a> = ();
-	async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: SrcRange, _: &Interner) -> Self {
+	async fn from_api(_: &T, _: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
 		panic!("Cannot deserialize Never")
 	}
 	async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> T { match self {} }
@@ -108,20 +107,19 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 		hctx: &mut H::FromApiCtx<'_>,
 		xctx: &mut X::FromApiCtx<'_>,
 		src: &Sym,
-		i: &Interner,
 	) -> Self {
 		let pos = SrcRange::new(tt.range.clone(), src);
 		let tok = match_mapping!(&tt.token, api::Token => Token::<H, X> {
 			BR,
-			NS(n => Tok::from_api(*n, i).await,
-				b => Box::new(Self::from_api(b, hctx, xctx, src, i).boxed_local().await)),
-			Bottom(e => OrcErrv::from_api(e, i).await),
-			LambdaHead(arg => Box::new(Self::from_api(arg, hctx, xctx, src, i).boxed_local().await)),
-			Name(n => Tok::from_api(*n, i).await),
-			S(*par, b => ttv_from_api(b, hctx, xctx, src, i).await),
-			Comment(c.clone()),
-			NewExpr(expr => X::from_api(expr, xctx, pos.clone(), i).await),
-			Handle(tk => H::from_api(tk, hctx, pos.clone(), i).await)
+			NS(n => es(*n).await,
+				b => Box::new(Self::from_api(b, hctx, xctx, src).boxed_local().await)),
+			Bottom(e => OrcErrv::from_api(e).await),
+			LambdaHead(arg => Box::new(Self::from_api(arg, hctx, xctx, src).boxed_local().await)),
+			Name(n => es(*n).await),
+			S(*par, b => ttv_from_api(b, hctx, xctx, src).await),
+			Comment(c => es(*c).await),
+			NewExpr(expr => X::from_api(expr, xctx, pos.clone()).await),
+			Handle(tk => H::from_api(tk, hctx, pos.clone()).await)
 		});
 		Self { sr: pos, tok }
 	}
@@ -135,7 +133,7 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 			BR,
 			NS(n.to_api(), b => Box::new(b.into_api(hctx, xctx).boxed_local().await)),
 			Bottom(e.to_api()),
-			Comment(c.clone()),
+			Comment(c.to_api()),
 			LambdaHead(arg => Box::new(arg.into_api(hctx, xctx).boxed_local().await)),
 			Name(nn.to_api()),
 			S(p, b => ttv_into_api(b, hctx, xctx).boxed_local().await),
@@ -145,8 +143,8 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 		api::TokenTree { range: self.sr.range.clone(), token }
 	}
 
-	pub fn is_kw(&self, tk: Tok<String>) -> bool { self.tok.is_kw(tk) }
-	pub fn as_name(&self) -> Option<Tok<String>> {
+	pub fn is_kw(&self, tk: IStr) -> bool { self.tok.is_kw(tk) }
+	pub fn as_name(&self) -> Option<IStr> {
 		if let Token::Name(n) = &self.tok { Some(n.clone()) } else { None }
 	}
 	pub fn as_multiname(&self) -> Result<VName, &TokTree<H, X>> {
@@ -193,11 +191,10 @@ pub async fn ttv_from_api<H: ExprRepr, X: ExtraTok>(
 	hctx: &mut H::FromApiCtx<'_>,
 	xctx: &mut X::FromApiCtx<'_>,
 	src: &Sym,
-	i: &Interner,
 ) -> Vec<TokTree<H, X>> {
 	stream(async |mut cx| {
 		for tok in tokv {
-			cx.emit(TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src, i).boxed_local().await).await
+			cx.emit(TokTree::<H, X>::from_api(tok.borrow(), hctx, xctx, src).boxed_local().await).await
 		}
 	})
 	.collect()
@@ -240,14 +237,14 @@ pub enum Token<H: ExprRepr, X: ExtraTok> {
 	/// Information about the code addressed to the human reader or dev tooling
 	/// It has no effect on the behaviour of the program unless it's explicitly
 	/// read via reflection
-	Comment(Rc<String>),
+	Comment(IStr),
 	/// The part of a lambda between `\` and `.` enclosing the argument. The body
 	/// stretches to the end of the enclosing parens or the end of the const line
 	LambdaHead(Box<TokTree<H, X>>),
 	/// A binding, operator, or a segment of a namespaced::name
-	Name(Tok<String>),
+	Name(IStr),
 	/// A namespace prefix, like `my_ns::` followed by a token
-	NS(Tok<String>, Box<TokTree<H, X>>),
+	NS(IStr, Box<TokTree<H, X>>),
 	/// A line break
 	BR,
 	/// `()`, `[]`, or `{}`
@@ -263,7 +260,7 @@ pub enum Token<H: ExprRepr, X: ExtraTok> {
 }
 impl<H: ExprRepr, X: ExtraTok> Token<H, X> {
 	pub fn at(self, sr: SrcRange) -> TokTree<H, X> { TokTree { sr, tok: self } }
-	pub fn is_kw(&self, tk: Tok<String>) -> bool { matches!(self, Token::Name(n) if *n == tk) }
+	pub fn is_kw(&self, tk: IStr) -> bool { matches!(self, Token::Name(n) if *n == tk) }
 	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> {
 		match self {
 			Self::S(p, b) if *p == par => Some(b),

orchid-base/src/virt_fs/common.rs

@@ -9,19 +9,19 @@ use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
 /// as the file system. Cheap to clone.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum Loaded {
-  /// Conceptually equivalent to a sourcefile
-  Code(Arc<String>),
-  /// Conceptually equivalent to the list of *.orc files in a folder, without
-  /// the extension
-  Collection(Arc<Vec<Tok<String>>>),
+	/// Conceptually equivalent to a sourcefile
+	Code(Arc<String>),
+	/// Conceptually equivalent to the list of *.orc files in a folder, without
+	/// the extension
+	Collection(Arc<Vec<IStr>>),
 }
 impl Loaded {
-  /// Is the loaded item source code (not a collection)?
-  pub fn is_code(&self) -> bool { matches!(self, Loaded::Code(_)) }
-  /// Collect the elements in a collection rreport
-  pub fn collection(items: impl IntoIterator<Item = Tok<String>>) -> Self {
-    Self::Collection(Arc::new(items.into_iter().collect()))
-  }
+	/// Is the loaded item source code (not a collection)?
+	pub fn is_code(&self) -> bool { matches!(self, Loaded::Code(_)) }
+	/// Collect the elements in a collection rreport
+	pub fn collection(items: impl IntoIterator<Item = IStr>) -> Self {
+		Self::Collection(Arc::new(items.into_iter().collect()))
+	}
 }
 
 /// Returned by any source loading callback
@@ -30,66 +30,62 @@ pub type FSResult = Result<Loaded, ErrorSansOriginObj>;
 /// Type that indicates the type of an entry without reading the contents
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
 pub enum FSKind {
-  /// Invalid path or read error
-  None,
-  /// Source code
-  Code,
-  /// Internal tree node
-  Collection,
+	/// Invalid path or read error
+	None,
+	/// Source code
+	Code,
+	/// Internal tree node
+	Collection,
 }
 
 /// Distinguished error for missing code
 #[derive(Clone, PartialEq, Eq)]
 pub struct CodeNotFound(pub VPath);
 impl CodeNotFound {
-  /// Instantiate error
-  pub fn new(path: VPath) -> Self { Self(path) }
+	/// Instantiate error
+	pub fn new(path: VPath) -> Self { Self(path) }
 }
 impl ErrorSansOrigin for CodeNotFound {
-  const DESCRIPTION: &'static str = "No source code for path";
-  fn message(&self) -> String { format!("{} not found", self.0) }
+	const DESCRIPTION: &'static str = "No source code for path";
+	fn message(&self) -> String { format!("{} not found", self.0) }
 }
 
 /// A simplified view of a file system for the purposes of source code loading.
 /// This includes the real FS and source code, but also various in-memory
 /// formats and other sources for libraries and dependencies.
 pub trait VirtFS {
-  /// Implementation of [VirtFS::read]
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult;
-  /// Discover information about a path without reading it.
-  ///
-  /// Implement this if your vfs backend can do expensive operations
-  fn kind(&self, path: &PathSlice) -> FSKind {
-    match self.read(path) {
-      Err(_) => FSKind::None,
-      Ok(Loaded::Code(_)) => FSKind::Code,
-      Ok(Loaded::Collection(_)) => FSKind::Collection,
-    }
-  }
-  /// Convert a path into a human-readable string that is meaningful in the
-  /// target context.
-  fn display(&self, path: &[Tok<String>]) -> Option<String>;
-  /// Convert the FS handler into a type-erased version of itself for packing in
-  /// a tree.
-  fn rc(self) -> Rc<dyn VirtFS>
-  where Self: Sized + 'static {
-    Rc::new(self)
-  }
-  /// Read a path, returning either a text file, a directory listing or an
-  /// error. Wrapper for [VirtFS::get]
-  fn read(&self, path: &PathSlice) -> FSResult { self.get(path, path) }
+	/// Implementation of [VirtFS::read]
+	fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult;
+	/// Discover information about a path without reading it.
+	///
+	/// Implement this if your vfs backend can do expensive operations
+	fn kind(&self, path: &PathSlice) -> FSKind {
+		match self.read(path) {
+			Err(_) => FSKind::None,
+			Ok(Loaded::Code(_)) => FSKind::Code,
+			Ok(Loaded::Collection(_)) => FSKind::Collection,
+		}
+	}
+	/// Convert a path into a human-readable string that is meaningful in the
+	/// target context.
+	fn display(&self, path: &[IStr]) -> Option<String>;
+	/// Convert the FS handler into a type-erased version of itself for packing in
+	/// a tree.
+	fn rc(self) -> Rc<dyn VirtFS>
+	where Self: Sized + 'static {
+		Rc::new(self)
+	}
+	/// Read a path, returning either a text file, a directory listing or an
+	/// error. Wrapper for [VirtFS::get]
+	fn read(&self, path: &PathSlice) -> FSResult { self.get(path, path) }
 }
 
 impl VirtFS for &dyn VirtFS {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
-    (*self).get(path, full_path)
-  }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> { (*self).display(path) }
+	fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult { (*self).get(path, full_path) }
+	fn display(&self, path: &[IStr]) -> Option<String> { (*self).display(path) }
 }
 
 impl<T: VirtFS + ?Sized> VirtFS for Rc<T> {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
-    (**self).get(path, full_path)
-  }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> { (**self).display(path) }
+	fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult { (**self).get(path, full_path) }
+	fn display(&self, path: &[IStr]) -> Option<String> { (**self).display(path) }
 }

orchid-base/src/virt_fs/decl.rs

@@ -32,7 +32,7 @@ impl<'a> Combine for &'a dyn VirtFS {
 pub type DeclTree = ModEntry<Rc<dyn VirtFS>, (), ()>;
 
 impl VirtFS for DeclTree {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     match &self.member {
       ModMember::Item(it) => it.get(path, full_path),
       ModMember::Sub(module) => match path.split_first() {
@@ -44,7 +44,7 @@ impl VirtFS for DeclTree {
     }
   }
 
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let (head, tail) = path.split_first()?;
     match &self.member {
       ModMember::Item(it) => it.display(path),
@@ -54,16 +54,16 @@ impl VirtFS for DeclTree {
 }
 
 impl VirtFS for String {
-  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn display(&self, _: &[IStr]) -> Option<String> { None }
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.as_str().to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }
 }
 
 impl<'a> VirtFS for &'a str {
-  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn display(&self, _: &[IStr]) -> Option<String> { None }
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }

orchid-base/src/virt_fs/dir.rs

@@ -99,14 +99,14 @@ impl DirNode {
     }
   }
 
-  fn mk_pathbuf(&self, path: &[Tok<String>]) -> PathBuf {
+  fn mk_pathbuf(&self, path: &[IStr]) -> PathBuf {
     let mut fpath = self.root.clone();
     path.iter().for_each(|seg| fpath.push(seg.as_str()));
     fpath
   }
 }
 impl VirtFS for DirNode {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     let fpath = self.mk_pathbuf(path);
     let mut binding = self.cached.borrow_mut();
     let (_, res) = (binding.raw_entry_mut().from_key(&fpath))
@@ -114,7 +114,7 @@ impl VirtFS for DirNode {
     res.clone()
   }
 
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let pathbuf = self.mk_pathbuf(path).with_extension(self.ext());
     Some(pathbuf.to_string_lossy().to_string())
   }

orchid-base/src/virt_fs/embed.rs

@@ -56,7 +56,7 @@ impl EmbeddedFS {
 }
 
 impl VirtFS for EmbeddedFS {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> FSResult {
     if path.is_empty() {
       return Ok(Loaded::collection(self.tree.keys(|_| true)));
     }
@@ -67,7 +67,7 @@ impl VirtFS for EmbeddedFS {
       ModMember::Sub(sub) => Loaded::collection(sub.keys(|_| true)),
     })
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     let Self { gen, suffix, .. } = self;
     Some(format!("{}{suffix} in {gen}", path.iter().join("/")))
   }

orchid-base/src/virt_fs/prefix.rs

@@ -21,18 +21,18 @@ impl<'a> PrefixFS<'a> {
       add: VPath::parse(add.as_ref()),
     }
   }
-  fn proc_path(&self, path: &[Tok<String>]) -> Option<Vec<Tok<String>>> {
+  fn proc_path(&self, path: &[IStr]) -> Option<Vec<IStr>> {
     let path = path.strip_prefix(self.remove.as_slice())?;
     Some(self.add.0.iter().chain(path).cloned().collect_vec())
   }
 }
 impl<'a> VirtFS for PrefixFS<'a> {
-  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> super::FSResult {
+  fn get(&self, path: &[IStr], full_path: &PathSlice) -> super::FSResult {
     let path =
       self.proc_path(path).ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())?;
     self.wrapped.get(&path, full_path)
   }
-  fn display(&self, path: &[Tok<String>]) -> Option<String> {
+  fn display(&self, path: &[IStr]) -> Option<String> {
     self.wrapped.display(&self.proc_path(path)?)
   }
 }

orchid-extension/Cargo.toml

@@ -8,17 +8,18 @@ edition = "2024"
 [dependencies]
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
+bound = "0.6.0"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.20"
-futures = { version = "0.3.31", features = [
+futures = { version = "0.3.31", default-features = false, features = [
 	"std",
 	"async-await",
-], default-features = false }
+] }
 futures-locks = "0.7.1"
-hashbrown = "0.16.0"
+hashbrown = "0.16.1"
 include_dir = { version = "0.7.4", optional = true }
 itertools = "0.14.0"
-konst = "0.4.2"
+konst = "0.4.3"
 lazy_static = "1.5.0"
 memo-map = "0.3.3"
 never = "0.1.0"
@@ -27,13 +28,12 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-ordered-float = "5.0.0"
-pastey = "0.1.1"
-some_executor = "0.6.1"
+ordered-float = "5.1.0"
+pastey = "0.2.1"
 substack = "1.1.1"
 task-local = "0.1.0"
-tokio = { version = "1.47.1", optional = true, features = [] }
-tokio-util = { version = "0.7.16", optional = true, features = ["compat"] }
+tokio = { version = "1.49.0", optional = true, features = [] }
+tokio-util = { version = "0.7.17", optional = true, features = ["compat"] }
 
 trait-set = "0.3.0"
 

orchid-extension/src/atom.rs

@@ -14,20 +14,20 @@ use orchid_api_derive::Coding;
 use orchid_api_traits::{Coding, Decode, Encode, Request, enc_vec};
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv, mk_errv_floating};
 use orchid_base::format::{FmtCtx, FmtUnit, Format, fmt};
+use orchid_base::interner::is;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::Requester;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::context::{ctx, i};
 use crate::conv::ToExpr;
+use crate::entrypoint::request;
 // use crate::error::{ProjectError, ProjectResult};
 use crate::expr::{Expr, ExprData, ExprHandle, ExprKind};
 use crate::gen_expr::GExpr;
-use crate::system::{DynSystemCard, atom_info_for, downcast_atom};
+use crate::system::{DynSystemCard, atom_by_idx, atom_info_for, cted, downcast_atom};
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct AtomTypeId(pub NonZeroU32);
 
 pub trait AtomCard: 'static + Sized {
@@ -99,13 +99,13 @@ impl ForeignAtom {
 		ForeignAtom { atom, expr: handle, pos }
 	}
 	pub async fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
-		let rep = (ctx().reqnot().request(api::Fwd(
+		let rep = (request(api::Fwd(
 			self.atom.clone(),
-			Sym::parse(M::NAME, &i()).await.unwrap().tok().to_api(),
-			enc_vec(&m).await,
+			Sym::parse(M::NAME).await.unwrap().tok().to_api(),
+			enc_vec(&m),
 		)))
 		.await?;
-		Some(M::Response::decode(Pin::new(&mut &rep[..])).await)
+		Some(M::Response::decode_slice(&mut &rep[..]))
 	}
 	pub async fn downcast<T: AtomicFeatures>(self) -> Result<TAtom<T>, NotTypAtom> {
 		TAtom::downcast(self.ex().handle()).await
@@ -119,7 +119,7 @@ impl fmt::Debug for ForeignAtom {
 }
 impl Format for ForeignAtom {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
-		FmtUnit::from_api(&ctx().reqnot().request(api::ExtAtomPrint(self.atom.clone())).await)
+		FmtUnit::from_api(&request(api::ExtAtomPrint(self.atom.clone())).await)
 	}
 }
 impl ToExpr for ForeignAtom {
@@ -138,8 +138,8 @@ pub struct NotTypAtom {
 impl NotTypAtom {
 	pub async fn mk_err(&self) -> OrcErrv {
 		mk_errv(
-			i().i("Not the expected type").await,
-			format!("The expression {} is not a {}", fmt(&self.expr, &i()).await, self.typ.name()),
+			is("Not the expected type").await,
+			format!("The expression {} is not a {}", fmt(&self.expr).await, self.typ.name()),
 			[self.pos.clone()],
 		)
 	}
@@ -172,8 +172,10 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 		self.handlers.push((
 			M::NAME,
 			Rc::new(move |a: &A, req: Pin<&mut dyn AsyncRead>, rep: Pin<&mut dyn AsyncWrite>| {
-				async { Supports::<M>::handle(a, M::decode(req).await).await.encode(rep).await }
-					.boxed_local()
+				async {
+					Supports::<M>::handle(a, M::decode(req).await.unwrap()).await.encode(rep).await.unwrap()
+				}
+				.boxed_local()
 			}),
 		));
 		self
@@ -182,7 +184,7 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 	pub async fn pack(&self) -> MethodSet<A> {
 		MethodSet {
 			handlers: stream::iter(self.handlers.iter())
-				.then(async |(k, v)| (Sym::parse(k, &i()).await.unwrap(), v.clone()))
+				.then(async |(k, v)| (Sym::parse(k).await.unwrap(), v.clone()))
 				.collect()
 				.await,
 		}
@@ -234,16 +236,15 @@ impl<A: AtomicFeatures> TAtom<A> {
 	}
 	pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
 	where A: Supports<M> {
-		M::Response::decode(Pin::new(
-			&mut &(ctx().reqnot().request(api::Fwd(
+		M::Response::decode_slice(
+			&mut &(request(api::Fwd(
 				self.untyped.atom.clone(),
-				Sym::parse(M::NAME, &i()).await.unwrap().tok().to_api(),
-				enc_vec(&req).await,
+				Sym::parse(M::NAME).await.unwrap().tok().to_api(),
+				enc_vec(&req),
 			)))
 			.await
 			.unwrap()[..],
-		))
-		.await
+		)
 	}
 }
 impl<A: AtomicFeatures> Deref for TAtom<A> {
@@ -311,9 +312,18 @@ impl Format for AtomFactory {
 }
 
 pub async fn err_not_callable() -> OrcErrv {
-	mk_errv_floating(i().i("This atom is not callable").await, "Attempted to apply value as function")
+	mk_errv_floating(is("This atom is not callable").await, "Attempted to apply value as function")
 }
 
 pub async fn err_not_command() -> OrcErrv {
-	mk_errv_floating(i().i("This atom is not a command").await, "Settled on an inactionable value")
+	mk_errv_floating(is("This atom is not a command").await, "Settled on an inactionable value")
+}
+
+/// Read the type ID prefix from an atom, return type information and the rest
+/// of the data
+pub(crate) fn resolve_atom_type(atom: &api::Atom) -> (Box<dyn AtomDynfo>, AtomTypeId, &[u8]) {
+	let mut data = &atom.data.0[..];
+	let tid = AtomTypeId::decode_slice(&mut data);
+	let atom_record = atom_by_idx(cted().inst().card(), tid).expect("Unrecognized atom type ID");
+	(atom_record, tid, data)
 }

orchid-extension/src/atom_owned.rs

@@ -1,11 +1,12 @@
 use std::any::{Any, TypeId, type_name};
 use std::borrow::Cow;
+use std::cell::RefCell;
 use std::future::Future;
 use std::marker::PhantomData;
 use std::num::NonZero;
 use std::ops::Deref;
 use std::pin::Pin;
-use std::sync::atomic::AtomicU64;
+use std::rc::Rc;
 
 use async_once_cell::OnceCell;
 use dyn_clone::{DynClone, clone_box};
@@ -18,33 +19,35 @@ use never::Never;
 use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, take_first};
+use orchid_base::logging::log;
 use orchid_base::name::Sym;
+use task_local::task_local;
 
 use crate::api;
 use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
 	MethodSetBuilder, TAtom, err_not_callable, err_not_command, get_info,
 };
-use crate::context::{SysCtxEntry, ctx, i};
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
-use crate::system_ctor::CtedObj;
+use crate::system::{cted, sys_id};
 
 pub struct OwnedVariant;
 impl AtomicVariant for OwnedVariant {}
 impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVariant> for A {
 	fn _factory(self) -> AtomFactory {
 		AtomFactory::new(async move || {
-			let serial = ctx()
-				.get_or_default::<ObjStore>()
-				.next_id
-				.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-			let atom_id = api::AtomId(NonZero::new(serial + 1).unwrap());
-			let (typ_id, _) = get_info::<A>(ctx().get::<CtedObj>().inst().card());
-			let mut data = enc_vec(&typ_id).await;
+			let obj_store = get_obj_store();
+			let atom_id = {
+				let mut id = obj_store.next_id.borrow_mut();
+				*id += 1;
+				api::AtomId(NonZero::new(*id + 1).unwrap())
+			};
+			let (typ_id, _) = get_info::<A>(cted().inst().card());
+			let mut data = enc_vec(&typ_id);
 			self.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
-			ctx().get_or_default::<ObjStore>().objects.read().await.insert(atom_id, Box::new(self));
-			api::Atom { drop: Some(atom_id), data: api::AtomData(data), owner: ctx().sys_id() }
+			obj_store.objects.read().await.insert(atom_id, Box::new(self));
+			api::Atom { drop: Some(atom_id), data: api::AtomData(data), owner: sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { OwnedAtomDynfo { msbuild: A::reg_reqs(), ms: OnceCell::new() } }
@@ -59,7 +62,7 @@ pub(crate) struct AtomReadGuard<'a> {
 }
 impl<'a> AtomReadGuard<'a> {
 	async fn new(id: api::AtomId) -> Self {
-		let guard = ctx().get_or_default::<ObjStore>().objects.read().await;
+		let guard = get_obj_store().objects.read().await;
 		if guard.get(&id).is_none() {
 			panic!("Received invalid atom ID: {id:?}");
 		}
@@ -73,7 +76,7 @@ impl Deref for AtomReadGuard<'_> {
 
 /// Remove an atom from the store
 pub(crate) async fn take_atom(id: api::AtomId) -> Box<dyn DynOwnedAtom> {
-	let mut g = ctx().get_or_default::<ObjStore>().objects.write().await;
+	let mut g = get_obj_store().objects.write().await;
 	g.remove(&id).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0))
 }
 
@@ -86,7 +89,7 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	fn name(&self) -> &'static str { type_name::<T>() }
 	fn decode<'a>(&'a self, AtomCtx(data, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
 		Box::pin(async {
-			Box::new(<T as AtomCard>::Data::decode(Pin::new(&mut &data[..])).await) as Box<dyn Any>
+			Box::new(<T as AtomCard>::Data::decode_slice(&mut &data[..])) as Box<dyn Any>
 		})
 	}
 	fn call(&self, AtomCtx(_, id): AtomCtx, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
@@ -127,7 +130,7 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		Box::pin(async move {
 			let id = id.unwrap();
-			id.encode(write.as_mut()).await;
+			id.encode(write.as_mut()).await.unwrap();
 			AtomReadGuard::new(id).await.dyn_serialize(write).await
 		})
 	}
@@ -155,7 +158,7 @@ pub trait DeserializeCtx: Sized {
 
 struct DeserCtxImpl<'a>(&'a [u8]);
 impl DeserializeCtx for DeserCtxImpl<'_> {
-	async fn read<T: Decode>(&mut self) -> T { T::decode(Pin::new(&mut self.0)).await }
+	async fn read<T: Decode>(&mut self) -> T { T::decode(Pin::new(&mut self.0)).await.unwrap() }
 	fn is_empty(&self) -> bool { self.0.is_empty() }
 }
 
@@ -266,7 +269,7 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn as_any_ref(&self) -> &dyn Any { self }
 	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn AsyncWrite>) -> LocalBoxFuture<'a, ()> {
-		async { self.val().await.as_ref().encode(buffer).await }.boxed_local()
+		async { self.val().await.as_ref().encode(buffer).await.unwrap() }.boxed_local()
 	}
 	fn dyn_call_ref(&self, arg: Expr) -> LocalBoxFuture<'_, GExpr> {
 		self.call_ref(arg).boxed_local()
@@ -279,7 +282,7 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	}
 	fn dyn_free(self: Box<Self>) -> LocalBoxFuture<'static, ()> { self.free().boxed_local() }
 	fn dyn_print(&self) -> LocalBoxFuture<'_, FmtUnit> {
-		async move { self.print_atom(&FmtCtxImpl { i: &i() }).await }.boxed_local()
+		async move { self.print_atom(&FmtCtxImpl::default()).await }.boxed_local()
 	}
 	fn dyn_serialize<'a>(
 		&'a self,
@@ -294,13 +297,24 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 
 #[derive(Default)]
 pub(crate) struct ObjStore {
-	pub(crate) next_id: AtomicU64,
+	pub(crate) next_id: RefCell<u64>,
 	pub(crate) objects: RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>,
 }
-impl SysCtxEntry for ObjStore {}
+
+task_local! {
+	static OBJ_STORE: Rc<ObjStore>;
+}
+
+pub(crate) fn with_obj_store<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+	Box::pin(OBJ_STORE.scope(Rc::new(ObjStore::default()), fut))
+}
+
+pub(crate) fn get_obj_store() -> Rc<ObjStore> {
+	OBJ_STORE.try_with(|store| store.clone()).expect("Owned atom store not initialized")
+}
 
 pub async fn own<A: OwnedAtom>(typ: &TAtom<A>) -> A {
-	let g = ctx().get_or_default::<ObjStore>().objects.read().await;
+	let g = get_obj_store().objects.read().await;
 	let atom_id = typ.untyped.atom.drop.expect("Owned atoms always have a drop ID");
 	let dyn_atom =
 		g.get(&atom_id).expect("Atom ID invalid; atom type probably not owned by this crate");
@@ -308,8 +322,7 @@ pub async fn own<A: OwnedAtom>(typ: &TAtom<A>) -> A {
 }
 
 pub async fn debug_print_obj_store(show_atoms: bool) {
-	let ctx = ctx();
-	let store = ctx.get_or_default::<ObjStore>();
+	let store = get_obj_store();
 	let keys = store.objects.read().await.keys().cloned().collect_vec();
 	let mut message = "Atoms in store:".to_string();
 	if !show_atoms {
@@ -326,5 +339,5 @@ pub async fn debug_print_obj_store(show_atoms: bool) {
 			message += &format!("\n{k:?} -> {}", take_first(&atom.dyn_print().await, true));
 		}
 	}
-	eprintln!("{message}")
+	writeln!(log("debug"), "{message}").await
 }

orchid-extension/src/atom_thin.rs

@@ -8,6 +8,7 @@ use futures::{AsyncRead, AsyncWrite, FutureExt};
 use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
+use orchid_base::logging::log;
 use orchid_base::name::Sym;
 
 use crate::api;
@@ -15,20 +16,19 @@ use crate::atom::{
 	AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
 	MethodSetBuilder, err_not_callable, err_not_command, get_info,
 };
-use crate::context::ctx;
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, bot};
-use crate::system_ctor::CtedObj;
+use crate::system::{cted, sys_id};
 
 pub struct ThinVariant;
 impl AtomicVariant for ThinVariant {}
 impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant> for A {
 	fn _factory(self) -> AtomFactory {
 		AtomFactory::new(async move || {
-			let (id, _) = get_info::<A>(ctx().get::<CtedObj>().inst().card());
-			let mut buf = enc_vec(&id).await;
-			self.encode(Pin::new(&mut buf)).await;
-			api::Atom { drop: None, data: api::AtomData(buf), owner: ctx().sys_id() }
+			let (id, _) = get_info::<A>(cted().inst().card());
+			let mut buf = enc_vec(&id);
+			self.encode_vec(&mut buf);
+			api::Atom { drop: None, data: api::AtomData(buf), owner: sys_id() }
 		})
 	}
 	fn _info() -> Self::_Info { ThinAtomDynfo { msbuild: Self::reg_reqs(), ms: OnceCell::new() } }
@@ -41,18 +41,18 @@ pub struct ThinAtomDynfo<T: ThinAtom> {
 }
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn print<'a>(&self, AtomCtx(buf, _): AtomCtx<'a>) -> LocalBoxFuture<'a, FmtUnit> {
-		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.print().await })
+		Box::pin(async move { T::decode_slice(&mut &buf[..]).print().await })
 	}
 	fn tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn name(&self) -> &'static str { type_name::<T>() }
 	fn decode<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
-		Box::pin(async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> })
+		Box::pin(async { Box::new(T::decode_slice(&mut &buf[..])) as Box<dyn Any> })
 	}
 	fn call<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
+		Box::pin(async move { T::decode_slice(&mut &buf[..]).call(arg).await })
 	}
 	fn call_ref<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>, arg: Expr) -> LocalBoxFuture<'a, GExpr> {
-		Box::pin(async move { T::decode(Pin::new(&mut &buf[..])).await.call(arg).await })
+		Box::pin(async move { T::decode_slice(&mut &buf[..]).call(arg).await })
 	}
 	fn handle_req<'a, 'm1: 'a, 'm2: 'a>(
 		&'a self,
@@ -63,14 +63,14 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	) -> LocalBoxFuture<'a, bool> {
 		Box::pin(async move {
 			let ms = self.ms.get_or_init(self.msbuild.pack()).await;
-			ms.dispatch(&T::decode(Pin::new(&mut &buf[..])).await, key, req, rep).await
+			ms.dispatch(&T::decode_slice(&mut &buf[..]), key, req, rep).await
 		})
 	}
 	fn command<'a>(
 		&'a self,
 		AtomCtx(buf, _): AtomCtx<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>> {
-		async move { T::decode(Pin::new(&mut &buf[..])).await.command().await }.boxed_local()
+		async move { T::decode_slice(&mut &buf[..]).command().await }.boxed_local()
 	}
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
@@ -78,18 +78,18 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 		write: Pin<&'b mut dyn AsyncWrite>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		Box::pin(async {
-			T::decode(Pin::new(&mut &ctx.0[..])).await.encode(write).await;
+			T::decode_slice(&mut &ctx.0[..]).encode(write).await.unwrap();
 			Some(Vec::new())
 		})
 	}
 	fn deserialize<'a>(&'a self, data: &'a [u8], refs: &'a [Expr]) -> LocalBoxFuture<'a, api::Atom> {
 		assert!(refs.is_empty(), "Refs found when deserializing thin atom");
-		Box::pin(async { T::decode(Pin::new(&mut &data[..])).await._factory().build().await })
+		Box::pin(async { T::decode_slice(&mut &data[..])._factory().build().await })
 	}
 	fn drop<'a>(&'a self, AtomCtx(buf, _): AtomCtx<'a>) -> LocalBoxFuture<'a, ()> {
 		Box::pin(async move {
-			let string_self = T::decode(Pin::new(&mut &buf[..])).await.print().await;
-			writeln!(ctx().logger(), "Received drop signal for non-drop atom {string_self:?}");
+			let string_self = T::decode_slice(&mut &buf[..]).print().await;
+			writeln!(log("warn"), "Received drop signal for non-drop atom {string_self:?}").await;
 		})
 	}
 }

orchid-extension/src/binary.rs

@@ -0,0 +1,30 @@
+use std::rc::Rc;
+
+use futures::future::LocalBoxFuture;
+use orchid_base::binary::future_to_vt;
+
+use crate::api;
+use crate::entrypoint::ExtensionBuilder;
+use crate::ext_port::ExtPort;
+
+pub type ExtCx = api::binary::ExtensionContext;
+
+struct Spawner(api::binary::Spawner);
+impl Drop for Spawner {
+	fn drop(&mut self) { (self.0.drop)(self.0.data) }
+}
+impl Spawner {
+	pub fn spawn(&self, fut: LocalBoxFuture<'static, ()>) {
+		(self.0.spawn)(self.0.data, future_to_vt(fut))
+	}
+}
+
+pub fn orchid_extension_main_body(cx: ExtCx, builder: ExtensionBuilder) {
+	let spawner = Spawner(cx.spawner);
+	builder.build(ExtPort {
+		input: Box::pin(cx.input),
+		output: Box::pin(cx.output),
+		log: Box::pin(cx.log),
+		spawn: Rc::new(move |fut| spawner.spawn(fut)),
+	});
+}

orchid-extension/src/context.rs

@@ -1,90 +0,0 @@
-use std::any::{Any, TypeId, type_name};
-use std::fmt;
-use std::num::NonZero;
-use std::rc::Rc;
-
-use memo_map::MemoMap;
-use orchid_base::builtin::Spawner;
-use orchid_base::interner::Interner;
-use orchid_base::logging::Logger;
-use orchid_base::reqnot::ReqNot;
-use task_local::task_local;
-
-use crate::api;
-use crate::system_ctor::CtedObj;
-
-#[derive(Clone)]
-pub struct SysCtx(Rc<MemoMap<TypeId, Box<dyn Any>>>);
-impl SysCtx {
-	pub fn new(
-		id: api::SysId,
-		i: Interner,
-		reqnot: ReqNot<api::ExtMsgSet>,
-		spawner: Spawner,
-		logger: Logger,
-		cted: CtedObj,
-	) -> Self {
-		let this = Self(Rc::new(MemoMap::new()));
-		this.add(id).add(i).add(reqnot).add(spawner).add(logger).add(cted);
-		this
-	}
-	pub fn add<T: SysCtxEntry>(&self, t: T) -> &Self {
-		assert!(self.0.insert(TypeId::of::<T>(), Box::new(t)), "Key already exists");
-		self
-	}
-	pub fn get_or_insert<T: SysCtxEntry>(&self, f: impl FnOnce() -> T) -> &T {
-		(self.0.get_or_insert_owned(TypeId::of::<T>(), || Box::new(f())).downcast_ref())
-			.expect("Keyed by TypeId")
-	}
-	pub fn get_or_default<T: SysCtxEntry + Default>(&self) -> &T { self.get_or_insert(T::default) }
-	pub fn try_get<T: SysCtxEntry>(&self) -> Option<&T> {
-		Some(self.0.get(&TypeId::of::<T>())?.downcast_ref().expect("Keyed by TypeId"))
-	}
-	pub fn get<T: SysCtxEntry>(&self) -> &T {
-		self.try_get().unwrap_or_else(|| panic!("Context {} missing", type_name::<T>()))
-	}
-	/// Shorthand to get the messaging link
-	pub fn reqnot(&self) -> &ReqNot<api::ExtMsgSet> { self.get::<ReqNot<api::ExtMsgSet>>() }
-	/// Shorthand to get the system ID
-	pub fn sys_id(&self) -> api::SysId { *self.get::<api::SysId>() }
-	/// Spawn a task that will eventually be executed asynchronously
-	pub fn spawn(&self, f: impl Future<Output = ()> + 'static) {
-		(self.get::<Spawner>())(Box::pin(CTX.scope(self.clone(), f)))
-	}
-	/// Shorthand to get the logger
-	pub fn logger(&self) -> &Logger { self.get::<Logger>() }
-	/// Shorthand to get the constructed system object
-	pub fn cted(&self) -> &CtedObj { self.get::<CtedObj>() }
-}
-impl fmt::Debug for SysCtx {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "SysCtx({:?})", self.sys_id())
-	}
-}
-pub trait SysCtxEntry: 'static + Sized {}
-impl SysCtxEntry for api::SysId {}
-impl SysCtxEntry for ReqNot<api::ExtMsgSet> {}
-impl SysCtxEntry for Spawner {}
-impl SysCtxEntry for CtedObj {}
-impl SysCtxEntry for Logger {}
-impl SysCtxEntry for Interner {}
-
-task_local! {
-	static CTX: SysCtx;
-}
-
-pub async fn with_ctx<F: Future>(ctx: SysCtx, f: F) -> F::Output { CTX.scope(ctx, f).await }
-pub fn ctx() -> SysCtx { CTX.get() }
-
-/// Shorthand to get the [Interner] instance
-pub fn i() -> Interner { ctx().get::<Interner>().clone() }
-
-pub fn mock_ctx() -> SysCtx {
-	let ctx = SysCtx(Rc::default());
-	ctx
-		.add(Logger::new(api::LogStrategy::StdErr))
-		.add(Interner::new_master())
-		.add::<Spawner>(Rc::new(|_| panic!("Cannot fork in test environment")))
-		.add(api::SysId(NonZero::<u16>::MIN));
-	ctx
-}

orchid-extension/src/conv.rs

@@ -4,11 +4,11 @@ use std::pin::Pin;
 use dyn_clone::DynClone;
 use never::Never;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use orchid_base::interner::is;
 use orchid_base::location::Pos;
 use trait_set::trait_set;
 
 use crate::atom::{AtomicFeatures, ForeignAtom, TAtom, ToAtom};
-use crate::context::i;
 use crate::expr::Expr;
 use crate::gen_expr::{GExpr, atom, bot};
 
@@ -27,7 +27,7 @@ impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
 }
 
 async fn err_not_atom(pos: Pos) -> OrcErrv {
-	mk_errv(i().i("Expected an atom").await, "This expression is not an atom", [pos])
+	mk_errv(is("Expected an atom").await, "This expression is not an atom", [pos])
 }
 
 impl TryFromExpr for ForeignAtom {

orchid-extension/src/entrypoint.rs

@@ -1,432 +1,435 @@
 use std::cell::RefCell;
 use std::future::Future;
-use std::mem;
 use std::num::NonZero;
 use std::pin::Pin;
 use std::rc::Rc;
+use std::{io, mem};
 
-use futures::channel::mpsc::{Receiver, Sender, channel};
 use futures::future::{LocalBoxFuture, join_all};
 use futures::lock::Mutex;
-use futures::{FutureExt, SinkExt, StreamExt, stream, stream_select};
-use futures_locks::RwLock;
+use futures::{AsyncRead, AsyncWrite, AsyncWriteExt, StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api_traits::{Decode, UnderRoot, enc_vec};
-use orchid_base::builtin::{ExtInit, ExtPort, Spawner};
+use orchid_api::{ExtHostNotif, ExtHostReq};
+use orchid_api_traits::{Decode, Encode, Request, UnderRoot, enc_vec};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
-use orchid_base::clone;
-use orchid_base::error::Reporter;
-use orchid_base::interner::{Interner, Tok};
-use orchid_base::logging::Logger;
+use orchid_base::error::try_with_reporter;
+use orchid_base::interner::{es, is, with_interner};
+use orchid_base::logging::{log, with_logger};
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Snippet};
-use orchid_base::reqnot::{ReqNot, RequestHandle, Requester};
+use orchid_base::reqnot::{
+	Client, ClientExt, CommCtx, MsgReader, MsgReaderExt, Receipt, RepWriter, ReqHandle, ReqHandleExt,
+	ReqReader, ReqReaderExt, Witness, io_comm,
+};
+use orchid_base::stash::with_stash;
 use orchid_base::tree::{TokenVariant, ttv_from_api};
 use substack::Substack;
-use trait_set::trait_set;
+use task_local::task_local;
 
 use crate::api;
-use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId};
-use crate::atom_owned::take_atom;
-use crate::context::{SysCtx, ctx, i, with_ctx};
+use crate::atom::{AtomCtx, AtomTypeId, resolve_atom_type};
+use crate::atom_owned::{take_atom, with_obj_store};
 use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
+use crate::ext_port::ExtPort;
+use crate::func_atom::with_funs_ctx;
+use crate::interner::new_interner;
 use crate::lexer::{LexContext, ekey_cascade, ekey_not_applicable};
-use crate::parser::{PTokTree, ParsCtx, get_const, linev_into_api};
-use crate::system::atom_by_idx;
-use crate::system_ctor::{CtedObj, DynSystemCtor};
-use crate::tree::{LazyMemberFactory, TreeIntoApiCtxImpl};
+use crate::logger::LoggerImpl;
+use crate::parser::{PTokTree, ParsCtx, get_const, linev_into_api, with_parsed_const_ctx};
+use crate::reflection::with_refl_roots;
+use crate::system::{SysCtx, atom_by_idx, cted, with_sys};
+use crate::system_ctor::{CtedObj, DynSystemCtor, SystemCtor};
+use crate::tree::{TreeIntoApiCtxImpl, get_lazy, with_lazy_member_store};
 
-pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>;
-pub type ExtReqNot = ReqNot<api::ExtMsgSet>;
-
-pub struct ExtensionData {
-	pub name: &'static str,
-	pub systems: &'static [&'static dyn DynSystemCtor],
+task_local::task_local! {
+	static CLIENT: Rc<dyn Client>;
+	static CTX: Rc<RefCell<Option<CommCtx>>>;
 }
-impl ExtensionData {
-	pub fn new(name: &'static str, systems: &'static [&'static dyn DynSystemCtor]) -> Self {
-		Self { name, systems }
+
+fn get_client() -> Rc<dyn Client> { CLIENT.get() }
+pub async fn exit() {
+	let cx = CTX.get().borrow_mut().take();
+	cx.unwrap().exit().await
+}
+
+/// Sent the client used for global [request] and [notify] functions within the
+/// runtime of this future
+pub async fn with_comm<F: Future>(c: Rc<dyn Client>, ctx: CommCtx, fut: F) -> F::Output {
+	CLIENT.scope(c, CTX.scope(Rc::new(RefCell::new(Some(ctx))), fut)).await
+}
+
+task_local! {
+	pub static MUTE_REPLY: ();
+}
+
+/// Send a request through the global client's [ClientExt::request]
+pub async fn request<T: Request + UnderRoot<Root = ExtHostReq>>(t: T) -> T::Response {
+	let response = get_client().request(t).await.unwrap();
+	if MUTE_REPLY.try_with(|b| *b).is_err() {
+		writeln!(log("msg"), "Got response {response:?}").await;
 	}
+	response
 }
 
-pub enum MemberRecord {
-	Gen(Vec<Tok<String>>, LazyMemberFactory),
-	Res,
+/// Send a notification through the global client's [ClientExt::notify]
+pub async fn notify<T: UnderRoot<Root = ExtHostNotif>>(t: T) {
+	get_client().notify(t).await.unwrap()
 }
 
 pub struct SystemRecord {
-	lazy_members: Mutex<HashMap<api::TreeId, MemberRecord>>,
-	ctx: SysCtx,
+	cted: CtedObj,
 }
 
-trait_set! {
-	pub trait WithAtomRecordCallback<'a, T> = AsyncFnOnce(
-		Box<dyn AtomDynfo>,
-		AtomTypeId,
-		&'a [u8]
-	) -> T
+type SystemTable = RefCell<HashMap<api::SysId, Rc<SystemRecord>>>;
+
+task_local! {
+	static SYSTEM_TABLE: SystemTable;
 }
 
-pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
-	get_sys_ctx: &impl Fn(api::SysId) -> F,
-	atom: &'a api::Atom,
-	cb: impl WithAtomRecordCallback<'a, T>,
-) -> T {
-	let mut data = &atom.data.0[..];
-	let ctx = get_sys_ctx(atom.owner).await;
-	let inst = ctx.get::<CtedObj>().inst();
-	let id = AtomTypeId::decode(Pin::new(&mut data)).await;
-	let atom_record = atom_by_idx(inst.card(), id.clone()).expect("Atom ID reserved");
-	with_ctx(ctx, async move { cb(atom_record, id, data).await }).await
+async fn with_sys_record<F: Future>(id: api::SysId, fut: F) -> F::Output {
+	let cted = SYSTEM_TABLE.with(|tbl| tbl.borrow().get(&id).expect("Invalid sys ID").cted.clone());
+	with_sys(SysCtx(id, cted), fut).await
 }
 
-pub struct ExtensionOwner {
-	_interner_cell: Rc<RefCell<Option<Interner>>>,
-	_systems_lock: Rc<RwLock<HashMap<api::SysId, SystemRecord>>>,
-	out_recv: Mutex<Receiver<Vec<u8>>>,
-	out_send: Sender<Vec<u8>>,
+pub trait ContextModifier: 'static {
+	fn apply<'a>(self: Box<Self>, fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()>;
 }
 
-impl ExtPort for ExtensionOwner {
-	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		Box::pin(async { self.out_send.clone().send(msg.to_vec()).boxed_local().await.unwrap() })
-	}
-	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
-		Box::pin(async { self.out_recv.lock().await.next().await })
+impl<F: AsyncFnOnce(LocalBoxFuture<'_, ()>) + 'static> ContextModifier for F {
+	fn apply<'a>(self: Box<Self>, fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+		Box::pin((self)(fut))
 	}
 }
 
-pub fn extension_init(
-	data: ExtensionData,
-	host_header: api::HostHeader,
-	spawner: Spawner,
-) -> ExtInit {
-	let api::HostHeader { log_strategy, msg_logs } = host_header;
-	let decls = (data.systems.iter().enumerate())
-		.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
-		.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
-		.collect_vec();
-	let systems_lock = Rc::new(RwLock::new(HashMap::<api::SysId, SystemRecord>::new()));
-	let ext_header = api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() };
-	let (out_send, in_recv) = channel::<Vec<u8>>(1);
-	let (in_send, out_recv) = channel::<Vec<u8>>(1);
-	let (exit_send, exit_recv) = channel(1);
-	let logger = Logger::new(log_strategy);
-	let msg_logger = Logger::new(msg_logs);
-	let interner_cell = Rc::new(RefCell::new(None::<Interner>));
-	let interner_weak = Rc::downgrade(&interner_cell);
-	let systems_weak = Rc::downgrade(&systems_lock);
-	let get_ctx = clone!(systems_weak; move |id: api::SysId| clone!(systems_weak; async move {
-		let systems =
-			systems_weak.upgrade().expect("System table dropped before request processing done");
-		systems.read().await.get(&id).expect("System not found").ctx.clone()
-	}));
-	let init_ctx = {
-		clone!(interner_weak, spawner, logger);
-		move |id: api::SysId, cted: CtedObj, reqnot: ReqNot<api::ExtMsgSet>| {
-			clone!(interner_weak, spawner, logger; async move {
-				let interner_rc =
-					interner_weak.upgrade().expect("System construction order while shutting down");
-				let i = interner_rc.borrow().clone().expect("mk_ctx called very early, no interner!");
-				SysCtx::new(id, i, reqnot, spawner, logger, cted)
-			})
-		}
-	};
-	let rn = ReqNot::<api::ExtMsgSet>::new(
-		msg_logger.clone(),
-		move |a, _| {
-			clone!(in_send mut);
-			Box::pin(async move { in_send.send(a.to_vec()).await.unwrap() })
-		},
-		{
-			clone!(exit_send);
-			move |n, _| {
-				clone!(exit_send mut);
-				async move {
-					match n {
-						api::HostExtNotif::Exit => exit_send.send(()).await.unwrap(),
+pub struct ExtensionBuilder {
+	pub name: &'static str,
+	pub systems: Vec<Box<dyn DynSystemCtor>>,
+	pub context: Vec<Box<dyn ContextModifier>>,
+}
+impl ExtensionBuilder {
+	pub fn new(name: &'static str) -> Self { Self { name, systems: Vec::new(), context: Vec::new() } }
+	pub fn system(mut self, ctor: impl SystemCtor) -> Self {
+		self.systems.push(Box::new(ctor) as Box<_>);
+		self
+	}
+	pub fn add_context(&mut self, fun: impl ContextModifier) {
+		self.context.push(Box::new(fun) as Box<_>);
+	}
+	pub fn context(mut self, fun: impl ContextModifier) -> Self {
+		self.add_context(fun);
+		self
+	}
+	pub fn build(mut self, mut ctx: ExtPort) {
+		self.add_context(with_funs_ctx);
+		self.add_context(with_parsed_const_ctx);
+		self.add_context(with_obj_store);
+		self.add_context(with_lazy_member_store);
+		self.add_context(with_refl_roots);
+		(ctx.spawn)(Box::pin(async move {
+			let host_header = api::HostHeader::decode(ctx.input.as_mut()).await.unwrap();
+			let decls = (self.systems.iter().enumerate())
+				.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
+				.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
+				.collect_vec();
+			api::ExtensionHeader { name: self.name.to_string(), systems: decls.clone() }
+				.encode(ctx.output.as_mut())
+				.await
+				.unwrap();
+			ctx.output.as_mut().flush().await.unwrap();
+			let logger1 = LoggerImpl::from_api(&host_header.logger);
+			let logger2 = logger1.clone();
+			let (client, comm_ctx, extension_srv) =
+				io_comm(Rc::new(Mutex::new(ctx.output)), Mutex::new(ctx.input));
+			let extension_fut = extension_srv.listen(
+				async |n: Box<dyn MsgReader<'_>>| {
+					let notif = n.read().await.unwrap();
+					match notif {
+						api::HostExtNotif::Exit => exit().await,
 					}
-				}
-				.boxed_local()
-			}
-		},
-		{
-			clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
-			move |hand, req| {
-				clone!(logger, get_ctx, init_ctx, systems_weak, interner_weak, decls, msg_logger);
-				async move {
-					let interner_cell = interner_weak.upgrade().expect("Interner dropped before request");
-					let interner =
-						interner_cell.borrow().clone().expect("Request arrived before interner set");
-					if !matches!(req, api::HostExtReq::AtomReq(api::AtomReq::AtomPrint(_))) {
-						writeln!(msg_logger, "{} extension received request {req:?}", data.name);
-					}
-
-					match req {
-						api::HostExtReq::SystemDrop(sys_drop) => {
-							if let Some(rc) = systems_weak.upgrade() {
-								mem::drop(rc.write().await.remove(&sys_drop.0))
-							}
-							hand.handle(&sys_drop, &()).await
-						},
-						api::HostExtReq::AtomDrop(atom_drop @ api::AtomDrop(sys_id, atom)) =>
-							with_ctx(get_ctx(sys_id).await, async move {
-								take_atom(atom).await.dyn_free().await;
-								hand.handle(&atom_drop, &()).await
-							})
-							.await,
-						api::HostExtReq::Ping(ping @ api::Ping) => hand.handle(&ping, &()).await,
-						api::HostExtReq::Sweep(sweep @ api::Sweep) =>
-							hand.handle(&sweep, &interner.sweep_replica().await).await,
-						api::HostExtReq::SysReq(api::SysReq::NewSystem(new_sys)) => {
-							let (sys_id, _) = (decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system))
-								.expect("NewSystem call received for invalid system");
-							let cted = data.systems[sys_id].new_system(&new_sys);
-							with_ctx(init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await, async move {
-								let lex_filter =
-									cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
-										char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
-									});
-								let lazy_members = Mutex::new(HashMap::new());
-								let const_root = stream::iter(cted.inst().dyn_env().await)
-									.then(|mem| {
-										let lazy_mems = &lazy_members;
-										async move {
-											let name = i().i(&mem.name).await;
+					Ok(())
+				},
+				async |mut reader| {
+					with_stash(async {
+						let req = reader.read_req().await.unwrap();
+						let handle = reader.finish().await;
+						// Atom printing is never reported because it generates too much
+						// noise
+						if !matches!(req, api::HostExtReq::AtomReq(api::AtomReq::AtomPrint(_))) {
+							writeln!(log("msg"), "{} extension received request {req:?}", self.name).await;
+						}
+						match req {
+							api::HostExtReq::SystemDrop(sys_drop) => {
+								SYSTEM_TABLE.with(|l| l.borrow_mut().remove(&sys_drop.0));
+								handle.reply(&sys_drop, &()).await
+							},
+							api::HostExtReq::AtomDrop(atom_drop @ api::AtomDrop(sys_id, atom)) =>
+								with_sys_record(sys_id, async {
+									take_atom(atom).await.dyn_free().await;
+									handle.reply(&atom_drop, &()).await
+								})
+								.await,
+							api::HostExtReq::Ping(ping @ api::Ping) => handle.reply(&ping, &()).await,
+							api::HostExtReq::Sweep(api::Sweep) => todo!(),
+							api::HostExtReq::SysReq(api::SysReq::NewSystem(new_sys)) => {
+								let (ctor_idx, _) =
+									(decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system))
+										.expect("NewSystem call received for invalid system");
+								let cted = self.systems[ctor_idx].new_system(&new_sys);
+								let record = Rc::new(SystemRecord { cted: cted.clone() });
+								SYSTEM_TABLE.with(|tbl| {
+									let mut g = tbl.borrow_mut();
+									g.insert(new_sys.id, record);
+								});
+								with_sys_record(new_sys.id, async {
+									let lex_filter =
+										cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
+											char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
+										});
+									let const_root = stream::iter(cted.inst().dyn_env().await)
+										.then(async |mem| {
+											let name = is(&mem.name).await;
 											let mut tia_ctx = TreeIntoApiCtxImpl {
-												lazy_members: &mut *lazy_mems.lock().await,
 												basepath: &[],
 												path: Substack::Bottom.push(name.clone()),
 											};
 											(name.to_api(), mem.kind.into_api(&mut tia_ctx).await)
-										}
-									})
-									.collect()
+										})
+										.collect()
+										.await;
+									let prelude =
+										cted.inst().dyn_prelude().await.iter().map(|sym| sym.to_api()).collect();
+									let line_types = join_all(
+										(cted.inst().dyn_parsers().iter())
+											.map(async |p| is(p.line_head()).await.to_api()),
+									)
 									.await;
-								let prelude =
-									cted.inst().dyn_prelude().await.iter().map(|sym| sym.to_api()).collect();
-								let record = SystemRecord { ctx: ctx(), lazy_members };
-								let systems = systems_weak.upgrade().expect("System constructed during shutdown");
-								systems.write().await.insert(new_sys.id, record);
-								let line_types = join_all(
-									(cted.inst().dyn_parsers().iter())
-										.map(|p| async { interner.i(p.line_head()).await.to_api() }),
-								)
-								.await;
-								let response =
-									api::NewSystemResponse { lex_filter, const_root, line_types, prelude };
-								hand.handle(&new_sys, &response).await
-							})
-							.await
-						},
-						api::HostExtReq::GetMember(get_tree @ api::GetMember(sys_id, tree_id)) =>
-							with_ctx(get_ctx(sys_id).await, async move {
-								let systems = systems_weak.upgrade().expect("Member queried during shutdown");
-								let systems_g = systems.read().await;
-								let mut lazy_members =
-									systems_g.get(&sys_id).expect("System not found").lazy_members.lock().await;
-								let (path, cb) = match lazy_members.insert(tree_id, MemberRecord::Res) {
-									None => panic!("Tree for ID not found"),
-									Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
-									Some(MemberRecord::Gen(path, cb)) => (path, cb),
-								};
-								let tree = cb.build(Sym::new(path.clone(), &interner).await.unwrap()).await;
-								let mut tia_ctx = TreeIntoApiCtxImpl {
-									path: Substack::Bottom,
-									basepath: &path,
-									lazy_members: &mut lazy_members,
-								};
-								hand.handle(&get_tree, &tree.into_api(&mut tia_ctx).await).await
-							})
-							.await,
-						api::HostExtReq::SysReq(api::SysReq::SysFwded(fwd)) => {
-							let fwd_tok = hand.will_handle_as(&fwd);
-							let api::SysFwded(sys_id, payload) = fwd;
-							let ctx = get_ctx(sys_id).await;
-							with_ctx(ctx.clone(), async move {
-								let sys = ctx.cted().inst();
-								let reply = Rc::new(RefCell::new(None));
-								let reply2 = reply.clone();
-								let sub_hand = ExtReq::new(hand.reqnot(), async move |v| {
-									reply2.borrow_mut().replace(v);
-								});
-								sys.dyn_request(sub_hand, payload).await;
-								let reply_buf =
-									reply.borrow_mut().take().expect("Request discarded but did not throw");
-								hand.handle_as(fwd_tok, &reply_buf).await
-							})
-							.await
-						},
-						api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, src, text, pos, id }) =>
-							with_ctx(get_ctx(sys).await, async move {
-								let text = Tok::from_api(text, &i()).await;
-								let src = Sym::from_api(src, &i()).await;
-								let rep = Reporter::new();
-								let expr_store = BorrowedExprStore::new();
-								let trigger_char = text.chars().nth(pos as usize).unwrap();
-								let ekey_na = ekey_not_applicable().await;
-								let ekey_cascade = ekey_cascade().await;
-								let lexers = ctx().cted().inst().dyn_lexers();
-								for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char))
-								{
-									let ctx = LexContext::new(&expr_store, &text, id, pos, src.clone(), &rep);
-									match lx.lex(&text[pos as usize..], &ctx).await {
-										Err(e) if e.any(|e| *e == ekey_na) => continue,
-										Err(e) => {
-											let eopt = e.keep_only(|e| *e != ekey_cascade).map(|e| Err(e.to_api()));
-											expr_store.dispose().await;
-											return hand.handle(&lex, &eopt).await;
+									let response =
+										api::NewSystemResponse { lex_filter, const_root, line_types, prelude };
+									handle.reply(&new_sys, &response).await
+								})
+								.await
+							},
+							api::HostExtReq::GetMember(get_tree @ api::GetMember(sys_id, tree_id)) =>
+								with_sys_record(sys_id, async {
+									let (path, tree) = get_lazy(tree_id).await;
+									let mut tia_ctx =
+										TreeIntoApiCtxImpl { path: Substack::Bottom, basepath: &path[..] };
+									handle.reply(&get_tree, &tree.into_api(&mut tia_ctx).await).await
+								})
+								.await,
+							api::HostExtReq::SysReq(api::SysReq::SysFwded(fwd)) => {
+								let fwd_tok = Witness::of(&fwd);
+								let api::SysFwded(sys_id, payload) = fwd;
+								with_sys_record(sys_id, async {
+									struct TrivialReqCycle<'a> {
+										req: &'a [u8],
+										rep: &'a mut Vec<u8>,
+									}
+									impl<'a> ReqReader<'a> for TrivialReqCycle<'a> {
+										fn reader(&mut self) -> Pin<&mut dyn AsyncRead> {
+											Pin::new(&mut self.req) as Pin<&mut _>
+										}
+										fn finish(self: Box<Self>) -> LocalBoxFuture<'a, Box<dyn ReqHandle<'a> + 'a>> {
+											Box::pin(async { self as Box<_> })
+										}
+									}
+									impl<'a> ReqHandle<'a> for TrivialReqCycle<'a> {
+										fn start_reply(
+											self: Box<Self>,
+										) -> LocalBoxFuture<'a, io::Result<Box<dyn RepWriter<'a> + 'a>>> {
+											Box::pin(async { Ok(self as Box<_>) })
+										}
+									}
+									impl<'a> RepWriter<'a> for TrivialReqCycle<'a> {
+										fn writer(&mut self) -> Pin<&mut dyn AsyncWrite> {
+											Pin::new(&mut self.rep) as Pin<&mut _>
+										}
+										fn finish(
+											self: Box<Self>,
+										) -> LocalBoxFuture<'a, io::Result<orchid_base::reqnot::Receipt<'a>>>
+										{
+											Box::pin(async { Ok(Receipt::_new()) })
+										}
+									}
+									let mut reply = Vec::new();
+									let req = TrivialReqCycle { req: &payload, rep: &mut reply };
+									let _ = cted().inst().dyn_request(Box::new(req)).await;
+									handle.reply(fwd_tok, &reply).await
+								})
+								.await
+							},
+							api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, src, text, pos, id }) =>
+								with_sys_record(sys, async {
+									let text = es(text).await;
+									let src = Sym::from_api(src).await;
+									let expr_store = BorrowedExprStore::new();
+									let trigger_char = text.chars().nth(pos as usize).unwrap();
+									let ekey_na = ekey_not_applicable().await;
+									let ekey_cascade = ekey_cascade().await;
+									let lexers = cted().inst().dyn_lexers();
+									for lx in
+										lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char))
+									{
+										let ctx = LexContext::new(&expr_store, &text, id, pos, src.clone());
+										match try_with_reporter(lx.lex(&text[pos as usize..], &ctx)).await {
+											Err(e) if e.any(|e| *e == ekey_na) => continue,
+											Err(e) => {
+												let eopt = e.keep_only(|e| *e != ekey_cascade).map(|e| Err(e.to_api()));
+												expr_store.dispose().await;
+												return handle.reply(&lex, &eopt).await;
+											},
+											Ok((s, expr)) => {
+												let expr = expr.into_api(&mut (), &mut ()).await;
+												let pos = (text.len() - s.len()) as u32;
+												expr_store.dispose().await;
+												return handle.reply(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
+											},
+										}
+									}
+									writeln!(log("warn"), "Got notified about n/a character '{trigger_char}'").await;
+									expr_store.dispose().await;
+									handle.reply(&lex, &None).await
+								})
+								.await,
+							api::HostExtReq::ParseLine(pline) => {
+								let api::ParseLine { module, src, exported, comments, sys, line, idx } = &pline;
+								with_sys_record(*sys, async {
+									let parsers = cted().inst().dyn_parsers();
+									let src = Sym::from_api(*src).await;
+									let comments =
+										join_all(comments.iter().map(|c| Comment::from_api(c, src.clone()))).await;
+									let expr_store = BorrowedExprStore::new();
+									let line: Vec<PTokTree> =
+										ttv_from_api(line, &mut &expr_store, &mut (), &src).await;
+									let snip = Snippet::new(line.first().expect("Empty line"), &line);
+									let parser = parsers[*idx as usize];
+									let module = Sym::from_api(*module).await;
+									let pctx = ParsCtx::new(module);
+									let o_line =
+										match try_with_reporter(parser.parse(pctx, *exported, comments, snip)).await {
+											Err(e) => Err(e.to_api()),
+											Ok(t) => Ok(linev_into_api(t).await),
+										};
+									mem::drop(line);
+									expr_store.dispose().await;
+									handle.reply(&pline, &o_line).await
+								})
+								.await
+							},
+							api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst(sys, id)) =>
+								with_sys_record(sys, async {
+									let cnst = get_const(id).await;
+									handle.reply(fpc, &cnst.serialize().await).await
+								})
+								.await,
+							api::HostExtReq::AtomReq(atom_req) => {
+								let atom = atom_req.get_atom();
+								with_sys_record(atom.owner, async {
+									let (nfo, id, buf) = resolve_atom_type(atom);
+									let actx = AtomCtx(buf, atom.drop);
+									match &atom_req {
+										api::AtomReq::SerializeAtom(ser) => {
+											let mut buf = enc_vec(&id);
+											match nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await {
+												None => handle.reply(ser, &None).await,
+												Some(refs) => {
+													let refs =
+														join_all(refs.into_iter().map(async |ex| ex.into_api(&mut ()).await))
+															.await;
+													handle.reply(ser, &Some((buf, refs))).await
+												},
+											}
 										},
-										Ok((s, expr)) => {
-											let expr = expr.into_api(&mut (), &mut ()).await;
-											let pos = (text.len() - s.len()) as u32;
+										api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
+											handle.reply(print, &nfo.print(actx).await.to_api()).await,
+										api::AtomReq::Fwded(fwded) => {
+											let api::Fwded(_, key, payload) = &fwded;
+											let mut reply = Vec::new();
+											let key = Sym::from_api(*key).await;
+											let some = nfo
+												.handle_req(
+													actx,
+													key,
+													Pin::<&mut &[u8]>::new(&mut &payload[..]),
+													Pin::<&mut Vec<_>>::new(&mut reply),
+												)
+												.await;
+											handle.reply(fwded, &some.then_some(reply)).await
+										},
+										api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
+											let expr_store = BorrowedExprStore::new();
+											let expr_handle = ExprHandle::borrowed(*arg, &expr_store);
+											let ret = nfo.call_ref(actx, Expr::from_handle(expr_handle.clone())).await;
+											let api_expr = ret.serialize().await;
+											mem::drop(expr_handle);
 											expr_store.dispose().await;
-											return hand.handle(&lex, &Some(Ok(api::LexedExpr { pos, expr }))).await;
+											handle.reply(call, &api_expr).await
+										},
+										api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
+											let expr_store = BorrowedExprStore::new();
+											let expr_handle = ExprHandle::borrowed(*arg, &expr_store);
+											let ret = nfo.call(actx, Expr::from_handle(expr_handle.clone())).await;
+											let api_expr = ret.serialize().await;
+											mem::drop(expr_handle);
+											expr_store.dispose().await;
+											handle.reply(call, &api_expr).await
+										},
+										api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
+											Err(e) => handle.reply(cmd, &Err(e.to_api())).await,
+											Ok(opt) => match opt {
+												None => handle.reply(cmd, &Ok(api::NextStep::Halt)).await,
+												Some(cont) => {
+													let cont = cont.serialize().await;
+													handle.reply(cmd, &Ok(api::NextStep::Continue(cont))).await
+												},
+											},
 										},
 									}
-								}
-								writeln!(logger, "Got notified about n/a character '{trigger_char}'");
-								expr_store.dispose().await;
-								hand.handle(&lex, &None).await
-							})
-							.await,
-						api::HostExtReq::ParseLine(pline) => {
-							let api::ParseLine { module, src, exported, comments, sys, line, idx } = &pline;
-							with_ctx(get_ctx(*sys).await, async {
-								let parsers = ctx().cted().inst().dyn_parsers();
-								let src = Sym::from_api(*src, &i()).await;
-								let comments =
-									join_all(comments.iter().map(|c| Comment::from_api(c, src.clone(), &interner)))
-										.await;
-								let expr_store = BorrowedExprStore::new();
-								let line: Vec<PTokTree> =
-									ttv_from_api(line, &mut &expr_store, &mut (), &src, &i()).await;
-								let snip = Snippet::new(line.first().expect("Empty line"), &line);
-								let parser = parsers[*idx as usize];
-								let module = Sym::from_api(*module, &i()).await;
-								let reporter = Reporter::new();
-								let pctx = ParsCtx::new(module, &reporter);
-								let parse_res = parser.parse(pctx, *exported, comments, snip).await;
-								let o_line = match reporter.merge(parse_res) {
-									Err(e) => Err(e.to_api()),
-									Ok(t) => Ok(linev_into_api(t).await),
-								};
-								mem::drop(line);
-								expr_store.dispose().await;
-								hand.handle(&pline, &o_line).await
-							})
-							.await
-						},
-						api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst(sys, id)) =>
-							with_ctx(get_ctx(sys).await, async move {
-								let cnst = get_const(id).await;
-								hand.handle(fpc, &cnst.serialize().await).await
-							})
-							.await,
-						api::HostExtReq::AtomReq(atom_req) => {
-							let atom = atom_req.get_atom();
-							let atom_req = atom_req.clone();
-							with_atom_record(&get_ctx, atom, async move |nfo, id, buf| {
-								let actx = AtomCtx(buf, atom.drop);
-								match &atom_req {
-									api::AtomReq::SerializeAtom(ser) => {
-										let mut buf = enc_vec(&id).await;
-										match nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await {
-											None => hand.handle(ser, &None).await,
-											Some(refs) => {
-												let refs =
-													join_all(refs.into_iter().map(|ex| async { ex.into_api(&mut ()).await }))
-														.await;
-												hand.handle(ser, &Some((buf, refs))).await
-											},
-										}
-									},
-									api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
-										hand.handle(print, &nfo.print(actx).await.to_api()).await,
-									api::AtomReq::Fwded(fwded) => {
-										let api::Fwded(_, key, payload) = &fwded;
-										let mut reply = Vec::new();
-										let key = Sym::from_api(*key, &interner).await;
-										let some = nfo
-											.handle_req(
-												actx,
-												key,
-												Pin::<&mut &[u8]>::new(&mut &payload[..]),
-												Pin::<&mut Vec<_>>::new(&mut reply),
-											)
-											.await;
-										hand.handle(fwded, &some.then_some(reply)).await
-									},
-									api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
-										let expr_store = BorrowedExprStore::new();
-										let expr_handle = ExprHandle::borrowed(*arg, &expr_store);
-										let ret = nfo.call_ref(actx, Expr::from_handle(expr_handle.clone())).await;
-										let api_expr = ret.serialize().await;
-										mem::drop(expr_handle);
-										expr_store.dispose().await;
-										hand.handle(call, &api_expr).await
-									},
-									api::AtomReq::FinalCall(call @ api::FinalCall(_, arg)) => {
-										let expr_store = BorrowedExprStore::new();
-										let expr_handle = ExprHandle::borrowed(*arg, &expr_store);
-										let ret = nfo.call(actx, Expr::from_handle(expr_handle.clone())).await;
-										let api_expr = ret.serialize().await;
-										mem::drop(expr_handle);
-										expr_store.dispose().await;
-										hand.handle(call, &api_expr).await
-									},
-									api::AtomReq::Command(cmd @ api::Command(_)) => match nfo.command(actx).await {
-										Err(e) => hand.handle(cmd, &Err(e.to_api())).await,
-										Ok(opt) => match opt {
-											None => hand.handle(cmd, &Ok(api::NextStep::Halt)).await,
-											Some(cont) => {
-												let cont = cont.serialize().await;
-												hand.handle(cmd, &Ok(api::NextStep::Continue(cont))).await
-											},
-										},
-									},
-								}
-							})
-							.await
-						},
-						api::HostExtReq::DeserAtom(deser) => {
-							let api::DeserAtom(sys, buf, refs) = &deser;
-							let mut read = &mut &buf[..];
-							let ctx = get_ctx(*sys).await;
-							// SAFETY: deserialization implicitly grants ownership to previously owned exprs
-							let refs = (refs.iter())
-								.map(|tk| Expr::from_handle(ExprHandle::deserialize(*tk)))
-								.collect_vec();
-							let id = AtomTypeId::decode(Pin::new(&mut read)).await;
-							let inst = ctx.cted().inst();
-							let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
-							hand.handle(&deser, &nfo.deserialize(read, &refs).await).await
-						},
-					}
-				}
-				.boxed_local()
-			}
-		},
-	);
-	*interner_cell.borrow_mut() =
-		Some(Interner::new_replica(rn.clone().map(|ir: api::IntReq| ir.into_root())));
-	spawner(Box::pin(clone!(spawner; async move {
-		let mut streams = stream_select! { in_recv.map(Some), exit_recv.map(|_| None) };
-		while let Some(item) = streams.next().await {
-			match item {
-				Some(rcvd) => spawner(Box::pin(clone!(rn; async move { rn.receive(&rcvd[..]).await }))),
-				None => break,
-			}
-		}
-	})));
-	ExtInit {
-		header: ext_header,
-		port: Box::new(ExtensionOwner {
-			out_recv: Mutex::new(out_recv),
-			out_send,
-			_interner_cell: interner_cell,
-			_systems_lock: systems_lock,
-		}),
+								})
+								.await
+							},
+							api::HostExtReq::DeserAtom(deser) => {
+								let api::DeserAtom(sys, buf, refs) = &deser;
+								let read = &mut &buf[..];
+								with_sys_record(*sys, async {
+									// SAFETY: deserialization implicitly grants ownership to previously owned exprs
+									let refs = (refs.iter())
+										.map(|tk| Expr::from_handle(ExprHandle::deserialize(*tk)))
+										.collect_vec();
+									let id = AtomTypeId::decode_slice(read);
+									let nfo = atom_by_idx(cted().inst().card(), id)
+										.expect("Deserializing atom with invalid ID");
+									handle.reply(&deser, &nfo.deserialize(read, &refs).await).await
+								})
+								.await
+							},
+						}
+					})
+					.await
+				},
+			);
+			// add essential services to the very tail, then fold all context into the run
+			// future
+			SYSTEM_TABLE
+				.scope(
+					RefCell::default(),
+					with_interner(
+						new_interner(),
+						with_logger(
+							logger2,
+							with_comm(
+								Rc::new(client),
+								comm_ctx,
+								(self.context.into_iter()).fold(
+									Box::pin(async { extension_fut.await.unwrap() }) as LocalBoxFuture<()>,
+									|fut, cx| cx.apply(fut),
+								),
+							),
+						),
+					),
+				)
+				.await;
+		}) as Pin<Box<_>>);
 	}
 }

orchid-extension/src/expr.rs

@@ -10,12 +10,13 @@ use hashbrown::HashSet;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::location::Pos;
-use orchid_base::reqnot::Requester;
+use orchid_base::stash::stash;
 
 use crate::api;
 use crate::atom::ForeignAtom;
-use crate::context::{ctx, i};
+use crate::entrypoint::{notify, request};
 use crate::gen_expr::{GExpr, GExprKind};
+use crate::system::sys_id;
 
 pub struct BorrowedExprStore(RefCell<Option<HashSet<Rc<ExprHandle>>>>);
 impl BorrowedExprStore {
@@ -73,7 +74,7 @@ impl ExprHandle {
 	/// to lend the expr, and you expect the receiver to use
 	/// [ExprHandle::borrowed] or [ExprHandle::from_ticket]
 	pub fn ticket(&self) -> api::ExprTicket { self.0 }
-	async fn send_acq(&self) { ctx().reqnot().notify(api::Acquire(ctx().sys_id(), self.0)).await }
+	async fn send_acq(&self) { notify(api::Acquire(sys_id(), self.0)).await }
 	/// If this is the last one reference, do nothing, otherwise send an Acquire
 	pub async fn on_borrow_expire(self: Rc<Self>) { self.serialize().await; }
 	/// Drop the handle and get the ticket without a release notification.
@@ -94,8 +95,8 @@ impl fmt::Debug for ExprHandle {
 }
 impl Drop for ExprHandle {
 	fn drop(&mut self) {
-		let notif = api::Release(ctx().sys_id(), self.0);
-		ctx().spawn(async move { ctx().reqnot().clone().notify(notif).await })
+		let notif = api::Release(sys_id(), self.0);
+		stash(async move { notify(notif).await })
 	}
 }
 
@@ -117,12 +118,12 @@ impl Expr {
 	}
 	pub async fn data(&self) -> &ExprData {
 		(self.data.get_or_init(async {
-			let details = ctx().reqnot().request(api::Inspect { target: self.handle.ticket() }).await;
-			let pos = Pos::from_api(&details.location, &i()).await;
+			let details = request(api::Inspect { target: self.handle.ticket() }).await;
+			let pos = Pos::from_api(&details.location).await;
 			let kind = match details.kind {
 				api::InspectedKind::Atom(a) =>
 					ExprKind::Atom(ForeignAtom::new(self.handle.clone(), a, pos.clone())),
-				api::InspectedKind::Bottom(b) => ExprKind::Bottom(OrcErrv::from_api(&b, &i()).await),
+				api::InspectedKind::Bottom(b) => ExprKind::Bottom(OrcErrv::from_api(&b).await),
 				api::InspectedKind::Opaque => ExprKind::Opaque,
 			};
 			ExprData { pos, kind }
@@ -150,8 +151,7 @@ impl Format for Expr {
 		match &self.data().await.kind {
 			ExprKind::Opaque => "OPAQUE".to_string().into(),
 			ExprKind::Bottom(b) => format!("Bottom({b})").into(),
-			ExprKind::Atom(a) =>
-				FmtUnit::from_api(&ctx().reqnot().request(api::ExtAtomPrint(a.atom.clone())).await),
+			ExprKind::Atom(a) => FmtUnit::from_api(&request(api::ExtAtomPrint(a.atom.clone())).await),
 		}
 	}
 }

orchid-extension/src/ext_port.rs

@@ -0,0 +1,12 @@
+use std::pin::Pin;
+use std::rc::Rc;
+
+use futures::future::LocalBoxFuture;
+use futures::{AsyncRead, AsyncWrite};
+
+pub struct ExtPort {
+	pub input: Pin<Box<dyn AsyncRead>>,
+	pub output: Pin<Box<dyn AsyncWrite>>,
+	pub log: Pin<Box<dyn AsyncWrite>>,
+	pub spawn: Rc<dyn Fn(LocalBoxFuture<'static, ()>)>,
+}

orchid-extension/src/func_atom.rs

@@ -1,12 +1,12 @@
 use std::any::TypeId;
 use std::borrow::Cow;
+use std::cell::RefCell;
 use std::collections::HashMap;
 use std::future::Future;
 use std::pin::Pin;
 use std::rc::Rc;
 
 use futures::future::LocalBoxFuture;
-use futures::lock::Mutex;
 use futures::{AsyncWrite, FutureExt};
 use itertools::Itertools;
 use never::Never;
@@ -15,15 +15,17 @@ use orchid_base::clone;
 use orchid_base::error::OrcRes;
 use orchid_base::format::{FmtCtx, FmtUnit};
 use orchid_base::name::Sym;
+use task_local::task_local;
 use trait_set::trait_set;
 
+use crate::api;
 use crate::atom::Atomic;
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
-use crate::context::{SysCtxEntry, ctx, i};
 use crate::conv::ToExpr;
 use crate::coroutine_exec::{ExecHandle, exec};
 use crate::expr::Expr;
 use crate::gen_expr::GExpr;
+use crate::system::sys_id;
 
 trait_set! {
 	trait FunCB = Fn(Vec<Expr>) -> LocalBoxFuture<'static, OrcRes<GExpr>> + 'static;
@@ -34,9 +36,14 @@ pub trait ExprFunc<I, O>: Clone + 'static {
 	fn apply<'a>(&self, hand: ExecHandle<'a>, v: Vec<Expr>) -> impl Future<Output = OrcRes<GExpr>>;
 }
 
-#[derive(Default)]
-struct FunsCtx(Mutex<HashMap<Sym, FunRecord>>);
-impl SysCtxEntry for FunsCtx {}
+task_local! {
+	static FUNS_CTX: RefCell<HashMap<(api::SysId, Sym), FunRecord>>;
+}
+
+pub(crate) fn with_funs_ctx<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+	Box::pin(FUNS_CTX.scope(RefCell::default(), fut))
+}
+
 #[derive(Clone)]
 struct FunRecord {
 	argtyps: &'static [TypeId],
@@ -77,17 +84,17 @@ pub(crate) struct Fun {
 }
 impl Fun {
 	pub async fn new<I, O, F: ExprFunc<I, O>>(path: Sym, f: F) -> Self {
-		let ctx = ctx();
-		let funs: &FunsCtx = ctx.get_or_default();
-		let mut fung = funs.0.lock().await;
-		let record = if let Some(record) = fung.get(&path) {
-			record.clone()
-		} else {
-			let record = process_args(f);
-			fung.insert(path.clone(), record.clone());
-			record
-		};
-		Self { args: vec![], path, record }
+		FUNS_CTX.with(|cx| {
+			let mut fung = cx.borrow_mut();
+			let record = if let Some(record) = fung.get(&(sys_id(), path.clone())) {
+				record.clone()
+			} else {
+				let record = process_args(f);
+				fung.insert((sys_id(), path.clone()), record.clone());
+				record
+			};
+			Self { args: vec![], path, record }
+		})
 	}
 	pub fn arity(&self) -> u8 { self.record.argtyps.len() as u8 }
 }
@@ -108,12 +115,12 @@ impl OwnedAtom for Fun {
 	}
 	async fn call(self, arg: Expr) -> GExpr { self.call_ref(arg).await }
 	async fn serialize(&self, write: Pin<&mut (impl AsyncWrite + ?Sized)>) -> Self::Refs {
-		self.path.to_api().encode(write).await;
+		self.path.to_api().encode(write).await.unwrap();
 		self.args.clone()
 	}
 	async fn deserialize(mut ds_cx: impl DeserializeCtx, args: Self::Refs) -> Self {
-		let path = Sym::from_api(ds_cx.decode().await, &i()).await;
-		let record = (ctx().get::<FunsCtx>().0.lock().await.get(&path))
+		let path = Sym::from_api(ds_cx.decode().await).await;
+		let record = (FUNS_CTX.with(|funs| funs.borrow().get(&(sys_id(), path.clone())).cloned()))
 			.expect("Function missing during deserialization")
 			.clone();
 		Self { args, path, record }

orchid-extension/src/gen_expr.rs

@@ -6,12 +6,11 @@ use orchid_base::error::{OrcErr, OrcErrv};
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::Requester;
 use orchid_base::{match_mapping, tl_cache};
 
 use crate::api;
 use crate::atom::{AtomFactory, ToAtom};
-use crate::context::ctx;
+use crate::entrypoint::request;
 use crate::expr::Expr;
 
 #[derive(Clone, Debug)]
@@ -40,7 +39,7 @@ impl GExpr {
 	}
 	pub fn at(self, pos: Pos) -> Self { GExpr { pos, kind: self.kind } }
 	pub async fn create(self) -> Expr {
-		Expr::deserialize(ctx().reqnot().request(api::Create(self.serialize().await)).await).await
+		Expr::deserialize(request(api::Create(self.serialize().await)).await).await
 	}
 }
 impl Format for GExpr {

orchid-extension/src/interner.rs

@@ -0,0 +1,50 @@
+use std::rc::Rc;
+
+use futures::future::{LocalBoxFuture, join_all, ready};
+use itertools::Itertools;
+use orchid_base::interner::local_interner::{Int, StrBranch, StrvBranch};
+use orchid_base::interner::{IStr, IStrv, InternerSrv};
+
+use crate::api;
+use crate::entrypoint::{MUTE_REPLY, request};
+
+#[derive(Default)]
+struct ExtInterner {
+	str: Int<StrBranch>,
+	strv: Int<StrvBranch>,
+}
+impl InternerSrv for ExtInterner {
+	fn is<'a>(&'a self, v: &'a str) -> LocalBoxFuture<'a, IStr> {
+		match self.str.i(v) {
+			Ok(i) => Box::pin(ready(i)),
+			Err(e) => Box::pin(async {
+				e.set_if_empty(MUTE_REPLY.scope((), request(api::InternStr(v.to_owned()))).await)
+			}),
+		}
+	}
+	fn es(&self, t: api::TStr) -> LocalBoxFuture<'_, IStr> {
+		match self.str.e(t) {
+			Ok(i) => Box::pin(ready(i)),
+			Err(e) => Box::pin(async move { e.set_if_empty(Rc::new(request(api::ExternStr(t)).await)) }),
+		}
+	}
+	fn iv<'a>(&'a self, v: &'a [IStr]) -> LocalBoxFuture<'a, IStrv> {
+		match self.strv.i(v) {
+			Ok(i) => Box::pin(ready(i)),
+			Err(e) => Box::pin(async {
+				e.set_if_empty(request(api::InternStrv(v.iter().map(|is| is.to_api()).collect_vec())).await)
+			}),
+		}
+	}
+	fn ev(&self, t: orchid_api::TStrv) -> LocalBoxFuture<'_, IStrv> {
+		match self.strv.e(t) {
+			Ok(i) => Box::pin(ready(i)),
+			Err(e) => Box::pin(async move {
+				let tstr_v = request(api::ExternStrv(t)).await;
+				e.set_if_empty(Rc::new(join_all(tstr_v.into_iter().map(|t| self.es(t))).await))
+			}),
+		}
+	}
+}
+
+pub fn new_interner() -> Rc<dyn InternerSrv> { Rc::<ExtInterner>::default() }

orchid-extension/src/lexer.rs

@@ -1,27 +1,24 @@
 use std::fmt;
+use std::fmt::Debug;
 use std::future::Future;
 use std::ops::RangeInclusive;
 
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
-use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_errv};
-use orchid_base::interner::{Interner, Tok};
+use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use orchid_base::interner::{IStr, is};
 use orchid_base::location::{Pos, SrcRange};
 use orchid_base::name::Sym;
-use orchid_base::parse::ParseCtx;
-use orchid_base::reqnot::Requester;
 
 use crate::api;
-use crate::context::{ctx, i};
+use crate::entrypoint::request;
 use crate::expr::BorrowedExprStore;
 use crate::parser::PTokTree;
 use crate::tree::GenTokTree;
 
-pub async fn ekey_cascade() -> Tok<String> {
-	i().i("An error cascading from a recursive call").await
-}
-pub async fn ekey_not_applicable() -> Tok<String> {
-	i().i("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
+pub async fn ekey_cascade() -> IStr { is("An error cascading from a recursive call").await }
+pub async fn ekey_not_applicable() -> IStr {
+	is("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
 }
 const MSG_INTERNAL_ERROR: &str = "This error is a sentinel for the extension library.\
 	it should not be emitted by the extension.";
@@ -36,23 +33,20 @@ pub async fn err_not_applicable() -> OrcErrv {
 
 pub struct LexContext<'a> {
 	pub(crate) exprs: &'a BorrowedExprStore,
-	pub text: &'a Tok<String>,
+	pub text: &'a IStr,
 	pub id: api::ParsId,
 	pub pos: u32,
-	i: Interner,
 	pub(crate) src: Sym,
-	pub(crate) rep: &'a Reporter,
 }
 impl<'a> LexContext<'a> {
 	pub fn new(
 		exprs: &'a BorrowedExprStore,
-		text: &'a Tok<String>,
+		text: &'a IStr,
 		id: api::ParsId,
 		pos: u32,
 		src: Sym,
-		rep: &'a Reporter,
 	) -> Self {
-		Self { exprs, i: i(), id, pos, rep, src, text }
+		Self { exprs, id, pos, src, text }
 	}
 	pub fn src(&self) -> &Sym { &self.src }
 	/// This function returns [PTokTree] because it can never return
@@ -61,10 +55,10 @@ impl<'a> LexContext<'a> {
 	/// for embedding in the return value.
 	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, PTokTree)> {
 		let start = self.pos(tail);
-		let Some(lx) = ctx().reqnot().request(api::SubLex { pos: start, id: self.id }).await else {
+		let Some(lx) = request(api::SubLex { pos: start, id: self.id }).await else {
 			return Err(err_cascade().await);
 		};
-		let tree = PTokTree::from_api(&lx.tree, &mut { self.exprs }, &mut (), &self.src, &i()).await;
+		let tree = PTokTree::from_api(&lx.tree, &mut { self.exprs }, &mut (), &self.src).await;
 		Ok((&self.text[lx.pos as usize..], tree))
 	}
 
@@ -77,12 +71,8 @@ impl<'a> LexContext<'a> {
 		SrcRange::new(self.pos(tail) - len.try_into().unwrap()..self.pos(tail), &self.src)
 	}
 }
-impl ParseCtx for LexContext<'_> {
-	fn i(&self) -> &Interner { &self.i }
-	fn rep(&self) -> &Reporter { self.rep }
-}
 
-pub trait Lexer: Send + Sync + Sized + Default + 'static {
+pub trait Lexer: Debug + Send + Sync + Sized + Default + 'static {
 	const CHAR_FILTER: &'static [RangeInclusive<char>];
 	fn lex<'a>(
 		tail: &'a str,
@@ -90,7 +80,7 @@ pub trait Lexer: Send + Sync + Sized + Default + 'static {
 	) -> impl Future<Output = OrcRes<(&'a str, GenTokTree)>>;
 }
 
-pub trait DynLexer: Send + Sync + 'static {
+pub trait DynLexer: Debug + Send + Sync + 'static {
 	fn char_filter(&self) -> &'static [RangeInclusive<char>];
 	fn lex<'a>(
 		&self,

orchid-extension/src/lib.rs

@@ -7,11 +7,12 @@ pub mod conv;
 pub mod coroutine_exec;
 pub mod entrypoint;
 pub mod expr;
+pub mod ext_port;
 pub mod func_atom;
 pub mod gen_expr;
+pub mod interner;
 pub mod lexer;
-// pub mod msg;
-pub mod context;
+pub mod logger;
 pub mod other_system;
 pub mod parser;
 pub mod reflection;
@@ -19,3 +20,4 @@ pub mod system;
 pub mod system_ctor;
 pub mod tokio;
 pub mod tree;
+pub mod binary;

orchid-extension/src/logger.rs

@@ -0,0 +1,57 @@
+use std::fmt::Arguments;
+use std::fs::File;
+use std::io::Write;
+use std::rc::Rc;
+
+use futures::future::LocalBoxFuture;
+use hashbrown::HashMap;
+use orchid_base::interner::is;
+use orchid_base::logging::{LogWriter, Logger};
+
+use crate::api;
+use crate::entrypoint::notify;
+
+pub struct LogWriterImpl {
+	category: String,
+	strat: api::LogStrategy,
+}
+impl LogWriter for LogWriterImpl {
+	fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()> {
+		Box::pin(async move {
+			match &self.strat {
+				api::LogStrategy::Discard => (),
+				api::LogStrategy::Default =>
+					notify(api::Log { category: is(&self.category).await.to_api(), message: fmt.to_string() })
+						.await,
+				api::LogStrategy::File { path, .. } => {
+					let mut file = (File::options().write(true).create(true).truncate(false).open(path))
+						.unwrap_or_else(|e| panic!("Could not open {path}: {e}"));
+					file.write_fmt(fmt).unwrap_or_else(|e| panic!("Could not write to {path}: {e}"));
+				},
+			}
+		})
+	}
+}
+
+#[derive(Clone)]
+pub struct LoggerImpl {
+	default: Option<api::LogStrategy>,
+	routing: HashMap<String, api::LogStrategy>,
+}
+impl LoggerImpl {
+	pub fn from_api(api: &api::Logger) -> Self {
+		Self {
+			default: api.default.clone(),
+			routing: api.routing.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
+		}
+	}
+}
+impl Logger for LoggerImpl {
+	fn writer(&self, category: &str) -> Rc<dyn LogWriter> {
+		Rc::new(LogWriterImpl { category: category.to_string(), strat: self.strat(category) })
+	}
+	fn strat(&self, category: &str) -> orchid_api::LogStrategy {
+		(self.routing.get(category).cloned().or(self.default.clone()))
+			.expect("Unrecognized log category with no default strategy")
+	}
+}

orchid-extension/src/other_system.rs

@@ -1,6 +1,7 @@
 use crate::api;
 use crate::system::{DynSystemCard, SystemCard};
 
+#[derive(Debug)]
 pub struct SystemHandle<C: SystemCard> {
 	pub(crate) card: C,
 	pub(crate) id: api::SysId,

orchid-extension/src/parser.rs

@@ -5,21 +5,22 @@ use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, Stream, StreamExt};
 use itertools::Itertools;
 use never::Never;
-use orchid_base::error::{OrcErrv, OrcRes, Reporter};
+use orchid_base::error::{OrcErrv, OrcRes};
 use orchid_base::id_store::IdStore;
-use orchid_base::interner::{Interner, Tok};
+use orchid_base::interner::IStr;
 use orchid_base::location::SrcRange;
 use orchid_base::match_mapping;
 use orchid_base::name::Sym;
-use orchid_base::parse::{Comment, ParseCtx, Snippet};
-use orchid_base::reqnot::Requester;
+use orchid_base::parse::{Comment, Snippet};
 use orchid_base::tree::{TokTree, Token, ttv_into_api};
+use task_local::task_local;
 
 use crate::api;
-use crate::context::{SysCtxEntry, ctx, i};
 use crate::conv::ToExpr;
+use crate::entrypoint::request;
 use crate::expr::Expr;
 use crate::gen_expr::GExpr;
+use crate::system::sys_id;
 use crate::tree::{GenTok, GenTokTree};
 
 pub type PTok = Token<Expr, Never>;
@@ -82,27 +83,21 @@ pub type ParserObj = &'static dyn DynParser;
 pub struct ParsCtx<'a> {
 	_parse: PhantomData<&'a mut ()>,
 	module: Sym,
-	reporter: &'a Reporter,
-	i: Interner,
 }
 impl<'a> ParsCtx<'a> {
-	pub(crate) fn new(module: Sym, reporter: &'a Reporter) -> Self {
-		Self { _parse: PhantomData, module, reporter, i: i() }
-	}
+	pub(crate) fn new(module: Sym) -> Self { Self { _parse: PhantomData, module } }
 	pub fn module(&self) -> Sym { self.module.clone() }
 }
-impl ParseCtx for ParsCtx<'_> {
-	fn i(&self) -> &Interner { &self.i }
-	fn rep(&self) -> &Reporter { self.reporter }
-}
 
 type BoxConstCallback = Box<dyn FnOnce(ConstCtx) -> LocalBoxFuture<'static, GExpr>>;
 
-#[derive(Default)]
-pub(crate) struct ParsedConstCtxEntry {
-	pub(crate) consts: IdStore<BoxConstCallback>,
+task_local! {
+	static PARSED_CONST_CTX: IdStore<BoxConstCallback>
+}
+
+pub(crate) fn with_parsed_const_ctx<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+	Box::pin(PARSED_CONST_CTX.scope(IdStore::default(), fut))
 }
-impl SysCtxEntry for ParsedConstCtxEntry {}
 
 pub struct ParsedLine {
 	pub sr: SrcRange,
@@ -114,7 +109,7 @@ impl ParsedLine {
 		sr: &SrcRange,
 		comments: impl IntoIterator<Item = &'a Comment>,
 		exported: bool,
-		name: Tok<String>,
+		name: IStr,
 		f: F,
 	) -> Self {
 		let cb = Box::new(|ctx| async move { f(ctx).await.to_gen().await }.boxed_local());
@@ -126,7 +121,7 @@ impl ParsedLine {
 		sr: &SrcRange,
 		comments: impl IntoIterator<Item = &'a Comment>,
 		exported: bool,
-		name: &Tok<String>,
+		name: &IStr,
 		use_prelude: bool,
 		lines: impl IntoIterator<Item = ParsedLine>,
 	) -> Self {
@@ -145,7 +140,7 @@ impl ParsedLine {
 					exported: mem.exported,
 					kind: match mem.kind {
 						ParsedMemKind::Const(cb) => api::ParsedMemberKind::Constant(api::ParsedConstId(
-							ctx().get_or_default::<ParsedConstCtxEntry>().consts.add(cb).id(),
+							PARSED_CONST_CTX.with(|consts| consts.add(cb).id()),
 						)),
 						ParsedMemKind::Mod { lines, use_prelude } => api::ParsedMemberKind::Module {
 							lines: linev_into_api(lines).boxed_local().await,
@@ -170,7 +165,7 @@ pub enum ParsedLineKind {
 }
 
 pub struct ParsedMem {
-	pub name: Tok<String>,
+	pub name: IStr,
 	pub exported: bool,
 	pub kind: ParsedMemKind,
 }
@@ -191,14 +186,14 @@ impl ConstCtx {
 	) -> impl Stream<Item = OrcRes<Sym>> + 'b {
 		let resolve_names = api::ResolveNames {
 			constid: self.constid,
-			sys: ctx().sys_id(),
+			sys: sys_id(),
 			names: names.into_iter().map(|n| n.to_api()).collect_vec(),
 		};
 		stream(async |mut cx| {
-			for name_opt in ctx().reqnot().request(resolve_names).await {
+			for name_opt in request(resolve_names).await {
 				cx.emit(match name_opt {
-					Err(e) => Err(OrcErrv::from_api(&e, &i()).await),
-					Ok(name) => Ok(Sym::from_api(name, &i()).await),
+					Err(e) => Err(OrcErrv::from_api(&e).await),
+					Ok(name) => Ok(Sym::from_api(name).await),
 				})
 				.await
 			}
@@ -210,8 +205,7 @@ impl ConstCtx {
 }
 
 pub(crate) async fn get_const(id: api::ParsedConstId) -> GExpr {
-	let cb = (ctx().get_or_default::<ParsedConstCtxEntry>().consts.get(id.0))
-		.expect("Bad ID or double read of parsed const")
-		.remove();
+	let cb = PARSED_CONST_CTX
+		.with(|ent| ent.get(id.0).expect("Bad ID or double read of parsed const").remove());
 	cb(ConstCtx { constid: id }).await
 }

orchid-extension/src/reflection.rs

@@ -1,15 +1,18 @@
-use std::cell::OnceCell;
+use std::cell::{OnceCell, RefCell};
 use std::rc::Rc;
 
 use futures::FutureExt;
+use futures::future::LocalBoxFuture;
 use futures::lock::Mutex;
+use hashbrown::HashMap;
 use memo_map::MemoMap;
-use orchid_base::interner::Tok;
+use orchid_base::interner::{IStr, es, iv};
 use orchid_base::name::{NameLike, VPath};
-use orchid_base::reqnot::Requester;
+use task_local::task_local;
 
 use crate::api;
-use crate::context::{SysCtxEntry, ctx, i};
+use crate::entrypoint::request;
+use crate::system::sys_id;
 
 #[derive(Debug)]
 pub struct ReflMemData {
@@ -33,29 +36,28 @@ pub enum ReflMemKind {
 pub struct ReflModData {
 	inferred: Mutex<bool>,
 	path: VPath,
-	members: MemoMap<Tok<String>, ReflMem>,
+	members: MemoMap<IStr, ReflMem>,
 }
 
 #[derive(Clone, Debug)]
 pub struct ReflMod(Rc<ReflModData>);
 impl ReflMod {
-	pub fn path(&self) -> &[Tok<String>] { &self.0.path[..] }
+	pub fn path(&self) -> &[IStr] { &self.0.path[..] }
 	pub fn is_root(&self) -> bool { self.0.path.is_empty() }
 	async fn try_populate(&self) -> Result<(), api::LsModuleError> {
-		let path_tok = i().i(&self.0.path[..]).await;
-		let reply = match ctx().reqnot().request(api::LsModule(ctx().sys_id(), path_tok.to_api())).await
-		{
+		let path_tok = iv(&self.0.path[..]).await;
+		let reply = match request(api::LsModule(sys_id(), path_tok.to_api())).await {
 			Err(api::LsModuleError::TreeUnavailable) =>
 				panic!("Reflected tree accessed outside an interpreter call. This extension is faulty."),
 			Err(err) => return Err(err),
 			Ok(details) => details,
 		};
 		for (k, v) in reply.members {
-			let k = i().ex(k).await;
+			let k = es(k).await;
 			let mem = match self.0.members.get(&k) {
 				Some(mem) => mem,
 				None => {
-					let path = self.0.path.clone().name_with_suffix(k.clone()).to_sym(&i()).await;
+					let path = self.0.path.clone().name_with_suffix(k.clone()).to_sym().await;
 					let kind = match v.kind {
 						api::MemberInfoKind::Constant => ReflMemKind::Const,
 						api::MemberInfoKind::Module =>
@@ -68,7 +70,7 @@ impl ReflMod {
 		}
 		Ok(())
 	}
-	pub async fn get_child(&self, key: &Tok<String>) -> Option<ReflMem> {
+	pub async fn get_child(&self, key: &IStr) -> Option<ReflMem> {
 		let inferred_g = self.0.inferred.lock().await;
 		if let Some(mem) = self.0.members.get(key) {
 			return Some(mem.clone());
@@ -86,7 +88,7 @@ impl ReflMod {
 		}
 		self.0.members.get(key).cloned()
 	}
-	pub async fn get_by_path(&self, path: &[Tok<String>]) -> Result<ReflMem, InvalidPathError> {
+	pub async fn get_by_path(&self, path: &[IStr]) -> Result<ReflMem, InvalidPathError> {
 		let (next, tail) = path.split_first().expect("Attempted to walk by empty path");
 		let inferred_g = self.0.inferred.lock().await;
 		if let Some(next) = self.0.members.get(next) {
@@ -130,9 +132,9 @@ impl ReflMod {
 	}
 }
 
-#[derive(Clone)]
-struct ReflRoot(ReflMod);
-impl SysCtxEntry for ReflRoot {}
+task_local! {
+	static REFL_ROOTS: RefCell<HashMap<api::SysId, ReflMod>>
+}
 
 #[derive(Clone, Debug)]
 pub struct InvalidPathError {
@@ -150,8 +152,12 @@ fn default_member(is_root: bool, kind: ReflMemKind) -> ReflMem {
 	}))
 }
 
-fn get_root() -> ReflRoot {
-	ctx().get_or_insert(|| ReflRoot(default_module(VPath::new([])))).clone()
+pub fn refl() -> ReflMod {
+	REFL_ROOTS.with(|tbl| {
+		tbl.borrow_mut().entry(sys_id()).or_insert_with(|| default_module(VPath::new([]))).clone()
+	})
 }
 
-pub fn refl() -> ReflMod { get_root().0.clone() }
+pub fn with_refl_roots<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+	Box::pin(REFL_ROOTS.scope(RefCell::default(), fut))
+}

orchid-extension/src/system.rs

@@ -1,4 +1,5 @@
 use std::any::{Any, TypeId};
+use std::fmt::Debug;
 use std::future::Future;
 use std::num::NonZero;
 use std::pin::Pin;
@@ -8,13 +9,13 @@ use futures::future::LocalBoxFuture;
 use orchid_api_traits::{Coding, Decode, Encode, Request};
 use orchid_base::boxed_iter::BoxedIter;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::{Receipt, Requester};
+use orchid_base::reqnot::{Receipt, ReqHandle, ReqReader, ReqReaderExt};
+use task_local::task_local;
 
 use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TAtom, get_info};
-use crate::context::ctx;
 use crate::coroutine_exec::Replier;
-use crate::entrypoint::ExtReq;
+use crate::entrypoint::request;
 use crate::func_atom::{Fun, Lambda};
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
@@ -22,7 +23,7 @@ use crate::system_ctor::{CtedObj, SystemCtor};
 use crate::tree::GenMember;
 
 /// System as consumed by foreign code
-pub trait SystemCard: Default + Send + Sync + 'static {
+pub trait SystemCard: Debug + Default + Send + Sync + 'static {
 	type Ctor: SystemCtor;
 	type Req: Coding;
 	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>>;
@@ -67,7 +68,7 @@ pub async fn resolv_atom(
 	sys: &(impl DynSystemCard + ?Sized),
 	atom: &api::Atom,
 ) -> Box<dyn AtomDynfo> {
-	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data.0[..])).await;
+	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data.0[..])).await.unwrap();
 	atom_by_idx(sys, tid).expect("Value of nonexistent type found")
 }
 
@@ -84,7 +85,10 @@ pub trait System: Send + Sync + SystemCard + 'static {
 	fn env() -> impl Future<Output = Vec<GenMember>>;
 	fn lexers() -> Vec<LexerObj>;
 	fn parsers() -> Vec<ParserObj>;
-	fn request(hand: ExtReq<'_>, req: Self::Req) -> impl Future<Output = Receipt<'_>>;
+	fn request<'a>(
+		hand: Box<dyn ReqHandle<'a> + 'a>,
+		req: Self::Req,
+	) -> impl Future<Output = Receipt<'a>>;
 }
 
 pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
@@ -92,7 +96,7 @@ pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
 	fn dyn_env(&self) -> LocalBoxFuture<'_, Vec<GenMember>>;
 	fn dyn_lexers(&self) -> Vec<LexerObj>;
 	fn dyn_parsers(&self) -> Vec<ParserObj>;
-	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>>;
+	fn dyn_request<'a>(&self, hand: Box<dyn ReqReader<'a> + 'a>) -> LocalBoxFuture<'a, Receipt<'a>>;
 	fn card(&self) -> &dyn DynSystemCard;
 }
 
@@ -101,26 +105,41 @@ impl<T: System> DynSystem for T {
 	fn dyn_env(&self) -> LocalBoxFuture<'_, Vec<GenMember>> { Self::env().boxed_local() }
 	fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
 	fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
-	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>> {
+	fn dyn_request<'a>(
+		&self,
+		mut hand: Box<dyn ReqReader<'a> + 'a>,
+	) -> LocalBoxFuture<'a, Receipt<'a>> {
 		Box::pin(async move {
-			Self::request(hand, <Self as SystemCard>::Req::decode(Pin::new(&mut &req[..])).await).await
+			let value = hand.read_req::<<Self as SystemCard>::Req>().await.unwrap();
+			Self::request(hand.finish().await, value).await
 		})
 	}
 	fn card(&self) -> &dyn DynSystemCard { self }
 }
 
+#[derive(Clone)]
+pub(crate) struct SysCtx(pub api::SysId, pub CtedObj);
+
+task_local! {
+	static SYS_CTX: SysCtx;
+}
+
+pub(crate) async fn with_sys<F: Future>(sys: SysCtx, fut: F) -> F::Output {
+	SYS_CTX.scope(sys, fut).await
+}
+
+pub fn sys_id() -> api::SysId { SYS_CTX.with(|cx| cx.0) }
+pub fn cted() -> CtedObj { SYS_CTX.with(|cx| cx.1.clone()) }
 pub async fn downcast_atom<A>(foreign: ForeignAtom) -> Result<TAtom<A>, ForeignAtom>
 where A: AtomicFeatures {
 	let mut data = &foreign.atom.data.0[..];
-	let ctx = ctx();
-	let value = AtomTypeId::decode(Pin::new(&mut data)).await;
-	let own_inst = ctx.get::<CtedObj>().inst();
-	let owner = if *ctx.get::<api::SysId>() == foreign.atom.owner {
+	let value = AtomTypeId::decode_slice(&mut data);
+	let cted = cted();
+	let own_inst = cted.inst();
+	let owner = if sys_id() == foreign.atom.owner {
 		own_inst.card()
 	} else {
-		(ctx.get::<CtedObj>().deps().find(|s| s.id() == foreign.atom.owner))
-			.ok_or_else(|| foreign.clone())?
-			.get_card()
+		cted.deps().find(|s| s.id() == foreign.atom.owner).ok_or_else(|| foreign.clone())?.get_card()
 	};
 	if owner.atoms().flatten().all(|dynfo| dynfo.tid() != TypeId::of::<A>()) {
 		return Err(foreign);
@@ -130,22 +149,24 @@ where A: AtomicFeatures {
 		return Err(foreign);
 	}
 	let val = dynfo.decode(AtomCtx(data, foreign.atom.drop)).await;
-	let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
-	Ok(TAtom { value, untyped: foreign })
+	let Ok(value) = val.downcast::<A::Data>() else {
+		panic!("decode of {} returned wrong type.", dynfo.name());
+	};
+	Ok(TAtom { value: *value, untyped: foreign })
 }
 
 pub async fn dep_req<Sys: SystemCard, Req: Request + Into<Sys::Req>>(req: Req) -> Req::Response {
-	let ctx = ctx();
 	let mut msg = Vec::new();
-	req.into().encode(std::pin::pin!(&mut msg)).await;
-	let own_inst = ctx.get::<CtedObj>().inst();
+	req.into().encode_vec(&mut msg);
+	let cted = cted();
+	let own_inst = cted.inst();
 	let owner = if own_inst.card().type_id() == TypeId::of::<Sys>() {
-		ctx.sys_id()
+		sys_id()
 	} else {
-		(ctx.get::<CtedObj>().deps().find(|s| s.get_card().type_id() == TypeId::of::<Sys>()))
+		(cted.deps().find(|s| s.get_card().type_id() == TypeId::of::<Sys>()))
 			.expect("System not in dependency array")
 			.id()
 	};
-	let reply = ctx.reqnot().request(api::SysFwd(owner, msg)).await;
-	Req::Response::decode(std::pin::pin!(&reply[..])).await
+	let reply = request(api::SysFwd(owner, msg)).await;
+	Req::Response::decode(std::pin::pin!(&reply[..])).await.unwrap()
 }

orchid-extension/src/system_ctor.rs

@@ -1,4 +1,5 @@
 use std::any::Any;
+use std::fmt::Debug;
 use std::sync::Arc;
 
 use orchid_base::boxed_iter::{BoxedIter, box_empty, box_once};
@@ -8,6 +9,7 @@ use crate::api;
 use crate::other_system::{DynSystemHandle, SystemHandle};
 use crate::system::{DynSystem, System, SystemCard};
 
+#[derive(Debug)]
 pub struct Cted<Ctor: SystemCtor + ?Sized> {
 	pub deps: <Ctor::Deps as DepDef>::Sat,
 	pub inst: Arc<Ctor::Instance>,
@@ -15,7 +17,7 @@ pub struct Cted<Ctor: SystemCtor + ?Sized> {
 impl<C: SystemCtor + ?Sized> Clone for Cted<C> {
 	fn clone(&self) -> Self { Self { deps: self.deps.clone(), inst: self.inst.clone() } }
 }
-pub trait DynCted: Send + Sync + 'static {
+pub trait DynCted: Debug + Send + Sync + 'static {
 	fn as_any(&self) -> &dyn Any;
 	fn deps<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)>;
 	fn inst(&self) -> Arc<dyn DynSystem>;
@@ -27,11 +29,11 @@ impl<C: SystemCtor + ?Sized> DynCted for Cted<C> {
 }
 pub type CtedObj = Arc<dyn DynCted>;
 
-pub trait DepSat: Clone + Send + Sync + 'static {
+pub trait DepSat: Debug + Clone + Send + Sync + 'static {
 	fn iter<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)>;
 }
 
-pub trait DepDef {
+pub trait DepDef: Debug {
 	type Sat: DepSat;
 	fn report(names: &mut impl FnMut(&'static str));
 	fn create(take: &mut impl FnMut() -> api::SysId) -> Self::Sat;
@@ -57,17 +59,16 @@ impl DepDef for () {
 	fn report(_: &mut impl FnMut(&'static str)) {}
 }
 
-pub trait SystemCtor: Send + Sync + 'static {
+pub trait SystemCtor: Debug + Send + Sync + 'static {
 	type Deps: DepDef;
 	type Instance: System;
 	const NAME: &'static str;
 	const VERSION: f64;
-	/// Create a system instance. When this function is called, a context object
-	/// isn't yet available
-	fn inst(deps: <Self::Deps as DepDef>::Sat) -> Self::Instance;
+	/// Create a system instance.
+	fn inst(&self, deps: <Self::Deps as DepDef>::Sat) -> Self::Instance;
 }
 
-pub trait DynSystemCtor: Send + Sync + 'static {
+pub trait DynSystemCtor: Debug + Send + Sync + 'static {
 	fn decl(&self, id: api::SysDeclId) -> api::SystemDecl;
 	fn new_system(&self, new: &api::NewSystem) -> CtedObj;
 }
@@ -84,7 +85,7 @@ impl<T: SystemCtor> DynSystemCtor for T {
 	fn new_system(&self, api::NewSystem { system: _, id: _, depends }: &api::NewSystem) -> CtedObj {
 		let mut ids = depends.iter().copied();
 		let deps = T::Deps::create(&mut || ids.next().unwrap());
-		let inst = Arc::new(T::inst(deps.clone()));
+		let inst = Arc::new(self.inst(deps.clone()));
 		Arc::new(Cted::<T> { deps, inst })
 	}
 }
@@ -151,8 +152,4 @@ mod dep_set_tuple_impls {
 	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J);
 	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K);
 	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L); // 12
-	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M);
-	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M, N);
-	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O);
-	dep_set_tuple_impl!(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P); // 16
 }

orchid-extension/src/tokio.rs

@@ -1,57 +1,31 @@
-use crate::entrypoint::ExtensionData;
+use std::rc::Rc;
 
+use crate::entrypoint::ExtensionBuilder;
+use crate::ext_port::ExtPort;
+/// Run an extension inside a Tokio localset. Since the extension API does not
+/// provide a forking mechanism, it can safely abort once the localset is
+/// exhausted. If an extension absolutely needs a parallel thread, it can import
+/// and call [tokio::task::spawn_local] which will keep alive the localset and
+/// postpone the aggressive shutdown, and listen for the [Drop::drop] of the
+/// value returned by [crate::system_ctor::SystemCtor::inst] to initiate
+/// shutdown.
 #[cfg(feature = "tokio")]
-pub async fn tokio_main(data: ExtensionData) {
-	use std::io::{ErrorKind, Write};
-	use std::mem;
-	use std::pin::{Pin, pin};
-	use std::rc::Rc;
-
-	use async_once_cell::OnceCell;
-	use futures::StreamExt;
-	use futures::future::LocalBoxFuture;
-	use futures::lock::Mutex;
-	use futures::stream::FuturesUnordered;
-	use orchid_api_traits::{Decode, Encode};
-	use orchid_base::msg::{recv_msg, send_msg};
-	use tokio::io::{Stdout, stdin, stdout};
+pub async fn tokio_main(builder: ExtensionBuilder) -> ! {
+	use tokio::io::{stderr, stdin, stdout};
 	use tokio::task::{LocalSet, spawn_local};
-	use tokio_util::compat::{Compat, TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
-
-	use crate::api;
-	use crate::entrypoint::extension_init;
+	use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
 
 	let local_set = LocalSet::new();
 	local_set.spawn_local(async {
-		let host_header = api::HostHeader::decode(Pin::new(&mut stdin().compat())).await;
-		let init =
-			Rc::new(extension_init(data, host_header, Rc::new(|fut| mem::drop(spawn_local(fut)))));
-		let mut buf = Vec::new();
-		init.header.encode(Pin::new(&mut buf)).await;
-		std::io::stdout().write_all(&buf).unwrap();
-		std::io::stdout().flush().unwrap();
-		// These are concurrent processes that never exit, so if the FuturesUnordered
-		// produces any result the extension should exit
-		let mut io = FuturesUnordered::<LocalBoxFuture<()>>::new();
-		io.push(Box::pin(async {
-			loop {
-				match recv_msg(pin!(stdin().compat())).await {
-					Ok(msg) => init.send(&msg[..]).await,
-					Err(e) if e.kind() == ErrorKind::BrokenPipe => break,
-					Err(e) if e.kind() == ErrorKind::UnexpectedEof => break,
-					Err(e) => panic!("{e}"),
-				}
-			}
-		}));
-		io.push(Box::pin(async {
-			while let Some(msg) = init.recv().await {
-				static STDOUT: OnceCell<Mutex<Compat<Stdout>>> = OnceCell::new();
-				let stdout_lk = STDOUT.get_or_init(async { Mutex::new(stdout().compat_write()) }).await;
-				let mut stdout_g = stdout_lk.lock().await;
-				send_msg(pin!(&mut *stdout_g), &msg[..]).await.expect("Parent pipe broken");
-			}
-		}));
-		io.next().await;
+		builder.build(ExtPort {
+			input: Box::pin(stdin().compat()),
+			output: Box::pin(stdout().compat_write()),
+			log: Box::pin(stderr().compat_write()),
+			spawn: Rc::new(|fut| {
+				spawn_local(fut);
+			}),
+		});
 	});
 	local_set.await;
+	std::process::exit(0)
 }

orchid-extension/src/tree.rs

@@ -1,4 +1,6 @@
+use std::cell::RefCell;
 use std::num::NonZero;
+use std::rc::Rc;
 
 use async_fn_stream::stream;
 use dyn_clone::{DynClone, clone_box};
@@ -6,17 +8,16 @@ use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, StreamExt};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::interner::{Interner, Tok};
+use orchid_base::interner::{IStr, is};
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::tree::{TokTree, Token, TokenVariant};
 use substack::Substack;
+use task_local::task_local;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::context::i;
 use crate::conv::ToExpr;
-use crate::entrypoint::MemberRecord;
 use crate::expr::{BorrowedExprStore, Expr, ExprHandle};
 use crate::func_atom::{ExprFunc, Fun};
 use crate::gen_expr::{GExpr, sym_ref};
@@ -27,12 +28,7 @@ pub type GenTok = Token<Expr, GExpr>;
 impl TokenVariant<api::Expression> for GExpr {
 	type FromApiCtx<'a> = ();
 	type ToApiCtx<'a> = ();
-	async fn from_api(
-		_: &api::Expression,
-		_: &mut Self::FromApiCtx<'_>,
-		_: SrcRange,
-		_: &Interner,
-	) -> Self {
+	async fn from_api(_: &api::Expression, _: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
 		panic!("Received new expression from host")
 	}
 	async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> api::Expression { self.serialize().await }
@@ -40,12 +36,7 @@ impl TokenVariant<api::Expression> for GExpr {
 
 impl TokenVariant<api::ExprTicket> for Expr {
 	type FromApiCtx<'a> = &'a BorrowedExprStore;
-	async fn from_api(
-		api: &api::ExprTicket,
-		exprs: &mut Self::FromApiCtx<'_>,
-		_: SrcRange,
-		_: &Interner,
-	) -> Self {
+	async fn from_api(api: &api::ExprTicket, exprs: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
 		// SAFETY: receiving trees from sublexers implies borrowing
 		Expr::from_handle(ExprHandle::borrowed(*api, exprs))
 	}
@@ -84,9 +75,8 @@ pub fn root_mod(name: &str, mems: impl IntoIterator<Item = Vec<GenMember>>) -> (
 	(name.to_string(), kind)
 }
 pub fn fun<I, O>(public: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenMember> {
-	let fac = LazyMemberFactory::new(async move |sym| {
-		MemKind::Const(Fun::new(sym, xf).await.to_gen().await)
-	});
+	let fac =
+		LazyMemberFactory::new(async move |sym| MemKind::Const(Fun::new(sym, xf).await.to_gen().await));
 	vec![GenMember { name: name.to_string(), kind: MemKind::Lazy(fac), public, comments: vec![] }]
 }
 pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
@@ -167,10 +157,10 @@ pub struct GenMember {
 	pub comments: Vec<String>,
 }
 impl GenMember {
-	pub async fn into_api(self, tia_cx: &mut impl TreeIntoApiCtx) -> api::Member {
-		let name = i().i::<String>(&self.name).await;
+	pub(crate) async fn into_api(self, tia_cx: &mut impl TreeIntoApiCtx) -> api::Member {
+		let name = is(&self.name).await;
 		let kind = self.kind.into_api(&mut tia_cx.push_path(name.clone())).await;
-		let comments = join_all(self.comments.iter().map(async |cmt| i().i(cmt).await.to_api())).await;
+		let comments = join_all(self.comments.iter().map(async |cmt| is(cmt).await.to_api())).await;
 		api::Member { kind, name: name.to_api(), comments, exported: self.public }
 	}
 }
@@ -181,9 +171,9 @@ pub enum MemKind {
 	Lazy(LazyMemberFactory),
 }
 impl MemKind {
-	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
+	pub(crate) async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
 		match self {
-			Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
+			Self::Lazy(lazy) => api::MemberKind::Lazy(add_lazy(ctx, lazy)),
 			Self::Const(c) => api::MemberKind::Const(c.serialize().await),
 			Self::Mod { members } => api::MemberKind::Module(api::Module {
 				members: stream(async |mut cx| {
@@ -199,29 +189,58 @@ impl MemKind {
 	}
 }
 
-pub trait TreeIntoApiCtx {
-	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
-	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
+pub enum MemberRecord {
+	Gen(Vec<IStr>, LazyMemberFactory),
+	Res,
 }
 
-pub struct TreeIntoApiCtxImpl<'a, 'b> {
-	pub basepath: &'a [Tok<String>],
-	pub path: Substack<'a, Tok<String>>,
-	pub lazy_members: &'b mut HashMap<api::TreeId, MemberRecord>,
+#[derive(Clone, Default)]
+pub(crate) struct LazyMemberStore(Rc<RefCell<HashMap<api::TreeId, MemberRecord>>>);
+
+task_local! {
+	static LAZY_MEMBERS: LazyMemberStore;
 }
 
-impl TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_> {
-	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx {
-		TreeIntoApiCtxImpl {
-			lazy_members: self.lazy_members,
-			basepath: self.basepath,
-			path: self.path.push(seg),
-		}
-	}
-	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId {
-		let id = api::TreeId(NonZero::new((self.lazy_members.len() + 2) as u64).unwrap());
-		let path = self.basepath.iter().cloned().chain(self.path.unreverse()).collect_vec();
-		self.lazy_members.insert(id, MemberRecord::Gen(path, fac));
+pub fn with_lazy_member_store<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+	Box::pin(LAZY_MEMBERS.scope(LazyMemberStore::default(), fut))
+}
+
+fn add_lazy(cx: &impl TreeIntoApiCtx, fac: LazyMemberFactory) -> api::TreeId {
+	LAZY_MEMBERS.with(|lazy_members| {
+		let mut g = lazy_members.0.borrow_mut();
+		let id = api::TreeId(NonZero::new((g.len() + 2) as u64).unwrap());
+		let path = cx.path().collect_vec();
+		g.insert(id, MemberRecord::Gen(path, fac));
 		id
+	})
+}
+
+pub async fn get_lazy(id: api::TreeId) -> (Sym, MemKind) {
+	let (path, cb) =
+		LAZY_MEMBERS.with(|tbl| match tbl.0.borrow_mut().insert(id, MemberRecord::Res) {
+			None => panic!("Tree for ID not found"),
+			Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
+			Some(MemberRecord::Gen(path, cb)) => (path, cb),
+		});
+	let path = Sym::new(path).await.unwrap();
+	(path.clone(), cb.build(path).await)
+}
+
+pub(crate) trait TreeIntoApiCtx {
+	fn push_path(&mut self, seg: IStr) -> impl TreeIntoApiCtx;
+	fn path(&self) -> impl Iterator<Item = IStr>;
+}
+
+pub struct TreeIntoApiCtxImpl<'a> {
+	pub basepath: &'a [IStr],
+	pub path: Substack<'a, IStr>,
+}
+
+impl TreeIntoApiCtx for TreeIntoApiCtxImpl<'_> {
+	fn push_path(&mut self, seg: IStr) -> impl TreeIntoApiCtx {
+		TreeIntoApiCtxImpl { basepath: self.basepath, path: self.path.push(seg) }
+	}
+	fn path(&self) -> impl Iterator<Item = IStr> {
+		self.basepath.iter().cloned().chain(self.path.unreverse())
 	}
 }

orchid-host/Cargo.toml

@@ -8,22 +8,28 @@ edition = "2024"
 [dependencies]
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
-async-process = "2.4.0"
 bound = "0.6.0"
 derive_destructure = "1.0.0"
 futures = { version = "0.3.31", features = ["std"], default-features = false }
 futures-locks = "0.7.1"
-hashbrown = "0.16.0"
+hashbrown = "0.16.1"
 itertools = "0.14.0"
 lazy_static = "1.5.0"
+libloading = { version = "0.9.0", optional = true }
 memo-map = "0.3.3"
 never = "0.1.0"
 num-traits = "0.2.19"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
-ordered-float = "5.0.0"
-pastey = "0.1.1"
+ordered-float = "5.1.0"
+pastey = "0.2.1"
 substack = "1.1.1"
-test_executors = "0.3.5"
+test_executors = "0.4.1"
+tokio = { version = "1.49.0", features = ["process"], optional = true }
+tokio-util = { version = "0.7.18", features = ["compat"], optional = true }
 trait-set = "0.3.0"
+unsync-pipe = { version = "0.2.0", path = "../unsync-pipe" }
+
+[features]
+tokio = ["dep:tokio", "dep:tokio-util", "dep:libloading"]

orchid-host/src/atom.rs

@@ -5,7 +5,7 @@ use async_once_cell::OnceCell;
 use derive_destructure::destructure;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, take_first_fmt};
 use orchid_base::location::Pos;
-use orchid_base::reqnot::Requester;
+use orchid_base::reqnot::ClientExt;
 use orchid_base::tree::AtomRepr;
 
 use crate::api;
@@ -59,11 +59,11 @@ impl AtomHand {
 	pub async fn call(self, arg: Expr) -> Expr {
 		let owner_sys = self.0.owner.clone();
 		let ctx = owner_sys.ctx();
-		let reqnot = owner_sys.reqnot();
+		let client = owner_sys.client();
 		ctx.exprs.give_expr(arg.clone());
 		let ret = match Rc::try_unwrap(self.0) {
-			Ok(data) => reqnot.request(api::FinalCall(data.api(), arg.id())).await,
-			Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await,
+			Ok(data) => client.request(api::FinalCall(data.api(), arg.id())).await.unwrap(),
+			Err(hand) => client.request(api::CallRef(hand.api_ref(), arg.id())).await.unwrap(),
 		};
 		let val = Expr::from_api(&ret, PathSetBuilder::new(), ctx.clone()).await;
 		ctx.exprs.take_expr(arg.id());
@@ -74,19 +74,21 @@ impl AtomHand {
 	#[must_use]
 	pub fn ext(&self) -> &Extension { self.sys().ext() }
 	pub async fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
-		self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await
+		self.0.owner.client().request(api::Fwded(self.0.api_ref(), key, req)).await.unwrap()
 	}
 	#[must_use]
 	pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
 	#[must_use]
-	pub async fn to_string(&self) -> String { take_first_fmt(self, &self.0.owner.ctx().i).await }
+	pub async fn to_string(&self) -> String { take_first_fmt(self).await }
 	#[must_use]
 	pub fn downgrade(&self) -> WeakAtomHand { WeakAtomHand(Rc::downgrade(&self.0)) }
 }
 impl Format for AtomHand {
 	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		(self.0.display.get_or_init(async {
-			FmtUnit::from_api(&self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await)
+			FmtUnit::from_api(
+				&self.0.owner.client().request(api::AtomPrint(self.0.api_ref())).await.unwrap(),
+			)
 		}))
 		.await
 		.clone()

orchid-host/src/ctx.rs

@@ -3,23 +3,32 @@ use std::num::{NonZero, NonZeroU16};
 use std::rc::{Rc, Weak};
 use std::{fmt, ops};
 
+use futures::future::LocalBoxFuture;
 use futures_locks::RwLock;
 use hashbrown::HashMap;
-use orchid_base::builtin::Spawner;
-use orchid_base::interner::Interner;
 
 use crate::api;
 use crate::expr_store::ExprStore;
+use crate::logger::LoggerImpl;
 use crate::system::{System, WeakSystem};
 use crate::tree::WeakRoot;
 
+pub trait JoinHandle {
+	fn abort(&self);
+	fn join(self: Box<Self>) -> LocalBoxFuture<'static, ()>;
+}
+
+pub trait Spawner {
+	fn spawn_obj(&self, fut: LocalBoxFuture<'static, ()>) -> Box<dyn JoinHandle>;
+}
+
 pub struct CtxData {
-	pub i: Interner,
-	pub spawn: Spawner,
+	spawner: Rc<dyn Spawner>,
 	pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
 	pub system_id: RefCell<NonZeroU16>,
 	pub exprs: ExprStore,
 	pub root: RwLock<WeakRoot>,
+	pub logger: LoggerImpl,
 }
 #[derive(Clone)]
 pub struct Ctx(Rc<CtxData>);
@@ -37,16 +46,25 @@ impl WeakCtx {
 }
 impl Ctx {
 	#[must_use]
-	pub fn new(spawn: Spawner) -> Self {
+	pub fn new(spawner: impl Spawner + 'static, logger: LoggerImpl) -> Self {
 		Self(Rc::new(CtxData {
-			spawn,
-			i: Interner::default(),
+			spawner: Rc::new(spawner),
 			systems: RwLock::default(),
 			system_id: RefCell::new(NonZero::new(1).unwrap()),
 			exprs: ExprStore::default(),
 			root: RwLock::default(),
+			logger,
 		}))
 	}
+	/// Spawn a parallel future that you can join at any later time.
+	///
+	/// Don't use this for async Drop, use [orchid_base::stash::stash] instead.
+	/// If you use this for an actor object, make sure to actually join the
+	/// handle.
+	#[must_use]
+	pub fn spawn(&self, fut: impl Future<Output = ()> + 'static) -> Box<dyn JoinHandle> {
+		self.spawner.spawn_obj(Box::pin(fut))
+	}
 	#[must_use]
 	pub(crate) async fn system_inst(&self, id: api::SysId) -> Option<System> {
 		self.systems.read().await.get(&id).and_then(WeakSystem::upgrade)
@@ -62,9 +80,6 @@ impl Ctx {
 }
 impl fmt::Debug for Ctx {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		f.debug_struct("Ctx")
-			.field("i", &self.i)
-			.field("system_id", &self.system_id)
-			.finish_non_exhaustive()
+		f.debug_struct("Ctx").field("system_id", &self.system_id).finish_non_exhaustive()
 	}
 }

orchid-host/src/dealias.rs

@@ -1,7 +1,7 @@
 use hashbrown::HashSet;
 use itertools::Itertools;
-use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_errv};
-use orchid_base::interner::{Interner, Tok};
+use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
+use orchid_base::interner::{IStr, is};
 use orchid_base::location::Pos;
 use orchid_base::name::VName;
 
@@ -16,17 +16,17 @@ pub enum AbsPathError {
 	RootPath,
 }
 impl AbsPathError {
-	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErrv {
+	pub async fn err_obj(self, pos: Pos, path: &str) -> OrcErrv {
 		let (descr, msg) = match self {
 			AbsPathError::RootPath => (
-				i.i("Path ends on root module").await,
+				is("Path ends on root module").await,
 				format!(
 					"{path} is equal to the empty path. You cannot directly reference the root. \
 					Use one fewer 'super::' or add more segments to make it valid."
 				),
 			),
 			AbsPathError::TooManySupers => (
-				i.i("Too many 'super::' steps in path").await,
+				is("Too many 'super::' steps in path").await,
 				format!("{path} is leading outside the root."),
 			),
 		};
@@ -41,13 +41,9 @@ impl AbsPathError {
 ///
 /// if the relative path contains as many or more `super` segments than the
 /// length of the absolute path.
-pub async fn absolute_path(
-	mut cwd: &[Tok<String>],
-	mut rel: &[Tok<String>],
-	i: &Interner,
-) -> Result<VName, AbsPathError> {
-	let i_self = i.i("self").await;
-	let i_super = i.i("super").await;
+pub async fn absolute_path(mut cwd: &[IStr], mut rel: &[IStr]) -> Result<VName, AbsPathError> {
+	let i_self = is("self").await;
+	let i_super = is("super").await;
 	let mut relative = false;
 	if let Some((_, tail)) = rel.split_first().filter(|(h, _)| **h == i_self) {
 		rel = tail;
@@ -63,19 +59,13 @@ pub async fn absolute_path(
 		.map_err(|_| AbsPathError::RootPath)
 }
 
-pub struct DealiasCtx<'a> {
-	pub i: &'a Interner,
-	pub rep: &'a Reporter,
-}
-
 pub async fn resolv_glob<Mod: Tree>(
-	cwd: &[Tok<String>],
+	cwd: &[IStr],
 	root: &Mod,
-	abs_path: &[Tok<String>],
+	abs_path: &[IStr],
 	pos: Pos,
-	i: &Interner,
 	ctx: &mut Mod::Ctx<'_>,
-) -> OrcRes<HashSet<Tok<String>>> {
+) -> OrcRes<HashSet<IStr>> {
 	let coprefix_len = cwd.iter().zip(abs_path).take_while(|(a, b)| a == b).count();
 	let (co_prefix, diff_path) = abs_path.split_at(abs_path.len().min(coprefix_len + 1));
 	let fst_diff =
@@ -89,7 +79,7 @@ pub async fn resolv_glob<Mod: Tree>(
 				ChildErrorKind::Missing => ("Invalid import path", format!("{path} not found")),
 				ChildErrorKind::Private => ("Import inaccessible", format!("{path} is private")),
 			};
-			return Err(mk_errv(i.i(tk).await, msg, [pos]));
+			return Err(mk_errv(is(tk).await, msg, [pos]));
 		},
 	};
 	Ok(target_module.children(coprefix_len < abs_path.len()))
@@ -100,11 +90,11 @@ pub type ChildResult<'a, T> = Result<&'a T, ChildErrorKind>;
 pub trait Tree {
 	type Ctx<'a>;
 	#[must_use]
-	fn children(&self, public_only: bool) -> HashSet<Tok<String>>;
+	fn children(&self, public_only: bool) -> HashSet<IStr>;
 	#[must_use]
 	fn child(
 		&self,
-		key: Tok<String>,
+		key: IStr,
 		public_only: bool,
 		ctx: &mut Self::Ctx<'_>,
 	) -> impl Future<Output = ChildResult<'_, Self>>;
@@ -135,7 +125,7 @@ pub struct ChildError {
 pub async fn walk<'a, T: Tree>(
 	root: &'a T,
 	public_only: bool,
-	path: impl IntoIterator<Item = Tok<String>>,
+	path: impl IntoIterator<Item = IStr>,
 	ctx: &mut T::Ctx<'_>,
 ) -> Result<&'a T, ChildError> {
 	let mut cur = root;

orchid-host/src/dylib.rs

@@ -0,0 +1,55 @@
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex};
+
+use hashbrown::HashMap;
+use libloading::Library;
+use orchid_base::binary::vt_to_future;
+
+use crate::api;
+use crate::ctx::Ctx;
+use crate::extension::ExtPort;
+
+static DYNAMIC_LIBRARIES: Mutex<Option<HashMap<PathBuf, Arc<Library>>>> = Mutex::new(None);
+fn load_dylib(path: &Path) -> Result<Arc<Library>, libloading::Error> {
+	let mut g = DYNAMIC_LIBRARIES.lock().unwrap();
+	let map = g.get_or_insert_default();
+	if let Some(lib) = map.get(path) {
+		Ok(lib.clone())
+	} else {
+		let lib = Arc::new(unsafe { Library::new(path) }?);
+		map.insert(path.to_owned(), lib.clone());
+		Ok(lib)
+	}
+}
+
+#[cfg(feature = "tokio")]
+pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Error> {
+	use futures::io::BufReader;
+	use futures::{AsyncBufReadExt, StreamExt};
+	use libloading::Symbol;
+	use unsync_pipe::pipe;
+
+	let (write_input, input) = pipe(1024);
+	let (output, read_output) = pipe(1024);
+	let (log, read_log) = pipe(1024);
+	let log_path = path.to_string_lossy().to_string();
+	let _ = ctx.spawn(async move {
+		use orchid_base::logging::log;
+		let mut lines = BufReader::new(read_log).lines();
+		while let Some(line) = lines.next().await {
+			writeln!(log("stderr"), "{log_path} err> {}", line.expect("Readline implies this")).await;
+		}
+	});
+	let library = load_dylib(path)?;
+	let entrypoint: Symbol<unsafe extern "C" fn(api::binary::ExtensionContext)> =
+		unsafe { library.get("orchid_extension_main") }?;
+	let data = Box::into_raw(Box::new(ctx)) as *const ();
+	extern "C" fn drop(data: *const ()) { std::mem::drop(unsafe { Box::from_raw(data as *mut Ctx) }) }
+	extern "C" fn spawn(data: *const (), vt: api::binary::FutureVT) {
+		let _ = unsafe { (data as *mut Ctx).as_mut().unwrap().spawn(vt_to_future(vt)) };
+	}
+	let spawner = api::binary::Spawner { data, drop, spawn };
+	let cx = api::binary::ExtensionContext { input, output, log, spawner };
+	unsafe { (entrypoint)(cx) };
+	Ok(ExtPort { input: Box::pin(write_input), output: Box::pin(read_output) })
+}

orchid-host/src/execute.rs

@@ -4,11 +4,10 @@ use bound::Bound;
 use futures::FutureExt;
 use futures_locks::{RwLockWriteGuard, TryLockError};
 use orchid_base::error::OrcErrv;
-use orchid_base::format::{FmtCtxImpl, Format, take_first};
+use orchid_base::format::fmt;
 use orchid_base::location::Pos;
-use orchid_base::logging::Logger;
+use orchid_base::logging::log;
 
-use crate::ctx::Ctx;
 use crate::expr::{Expr, ExprKind, PathSet, Step};
 use crate::tree::Root;
 
@@ -30,21 +29,19 @@ pub enum ExecResult {
 }
 
 pub struct ExecCtx {
-	ctx: Ctx,
 	gas: Option<u64>,
 	stack: Vec<ExprGuard>,
 	cur: ExprGuard,
 	cur_pos: Pos,
 	did_pop: bool,
-	logger: Logger,
 	root: Root,
 }
 impl ExecCtx {
 	#[must_use]
-	pub async fn new(ctx: Ctx, logger: Logger, root: Root, init: Expr) -> Self {
+	pub async fn new(root: Root, init: Expr) -> Self {
 		let cur_pos = init.pos();
 		let cur = Bound::async_new(init, |init| init.kind().write()).await;
-		Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger, root }
+		Self { gas: None, stack: vec![], cur, cur_pos, did_pop: false, root }
 	}
 	#[must_use]
 	pub fn remaining_gas(&self) -> u64 { self.gas.expect("queried remaining_gas but no gas was set") }
@@ -89,8 +86,7 @@ impl ExecCtx {
 		while self.use_gas(1) {
 			let mut kind_swap = ExprKind::Missing;
 			mem::swap(&mut kind_swap, &mut self.cur);
-			let unit = kind_swap.print(&FmtCtxImpl { i: &self.ctx.i }).await;
-			writeln!(self.logger, "Exxecute lvl{} {}", self.stack.len(), take_first(&unit, true));
+			writeln!(log("debug"), "Exxecute lvl{} {}", self.stack.len(), fmt(&kind_swap).await).await;
 			let (kind, op) = match kind_swap {
 				ExprKind::Identity(target) => {
 					let inner = self.unpack_ident(&target).await;

orchid-host/src/expr.rs

@@ -9,7 +9,6 @@ use futures_locks::RwLock;
 use itertools::Itertools;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
-use orchid_base::interner::Interner;
 use orchid_base::location::{Pos, SrcRange};
 use orchid_base::name::Sym;
 use orchid_base::tl_cache;
@@ -56,13 +55,13 @@ impl Expr {
 	}
 	#[must_use]
 	pub async fn from_api(api: &api::Expression, psb: PathSetBuilder<'_, u64>, ctx: Ctx) -> Self {
-		let pos = Pos::from_api(&api.location, &ctx.i).await;
+		let pos = Pos::from_api(&api.location).await;
 		let kind = match &api.kind {
 			api::ExpressionKind::Arg(n) => {
 				assert!(psb.register_arg(n), "Arguments must be enclosed in a matching lambda");
 				ExprKind::Arg
 			},
-			api::ExpressionKind::Bottom(bot) => ExprKind::Bottom(OrcErrv::from_api(bot, &ctx.i).await),
+			api::ExpressionKind::Bottom(bot) => ExprKind::Bottom(OrcErrv::from_api(bot).await),
 			api::ExpressionKind::Call(f, x) => {
 				let (lpsb, rpsb) = psb.split();
 				ExprKind::Call(
@@ -70,7 +69,7 @@ impl Expr {
 					Expr::from_api(x, rpsb, ctx).boxed_local().await,
 				)
 			},
-			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(*name, &ctx.i).await),
+			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(*name).await),
 			api::ExpressionKind::Lambda(x, body) => {
 				let lbuilder = psb.lambda(x);
 				let body = Expr::from_api(body, lbuilder.stack(), ctx).boxed_local().await;
@@ -326,12 +325,7 @@ impl WeakExpr {
 
 impl TokenVariant<api::ExprTicket> for Expr {
 	type FromApiCtx<'a> = ExprStore;
-	async fn from_api(
-		api: &api::ExprTicket,
-		ctx: &mut Self::FromApiCtx<'_>,
-		_: SrcRange,
-		_: &Interner,
-	) -> Self {
+	async fn from_api(api: &api::ExprTicket, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
 		ctx.get_expr(*api).expect("Invalid ticket")
 	}
 	type ToApiCtx<'a> = ExprStore;
@@ -348,12 +342,7 @@ pub struct ExprWillPanic;
 
 impl TokenVariant<api::Expression> for Expr {
 	type FromApiCtx<'a> = Ctx;
-	async fn from_api(
-		api: &api::Expression,
-		ctx: &mut Self::FromApiCtx<'_>,
-		_: SrcRange,
-		_: &Interner,
-	) -> Self {
+	async fn from_api(api: &api::Expression, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
 		Self::from_api(api, PathSetBuilder::new(), ctx.clone()).await
 	}
 	type ToApiCtx<'a> = ExprWillPanic;

orchid-host/src/extension.rs

@@ -2,7 +2,7 @@ use std::cell::RefCell;
 use std::future::Future;
 use std::io;
 use std::num::NonZeroU64;
-use std::pin::pin;
+use std::pin::Pin;
 use std::rc::{Rc, Weak};
 
 use async_fn_stream::stream;
@@ -10,28 +10,32 @@ use derive_destructure::destructure;
 use futures::channel::mpsc::{Sender, channel};
 use futures::future::{join, join_all};
 use futures::lock::Mutex;
-use futures::{SinkExt, StreamExt, stream};
-use hashbrown::HashMap;
+use futures::{AsyncRead, AsyncWrite, AsyncWriteExt, SinkExt, StreamExt};
+use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
-use orchid_api_traits::Request;
-use orchid_base::builtin::ExtInit;
-use orchid_base::clone;
+use orchid_api_traits::{Decode, Encode, Request};
 use orchid_base::format::{FmtCtxImpl, Format};
-use orchid_base::interner::Tok;
+use orchid_base::interner::{IStr, IStrv, es, ev, is, iv};
 use orchid_base::location::Pos;
-use orchid_base::logging::Logger;
+use orchid_base::logging::log;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _};
+use orchid_base::reqnot::{Client, ClientExt, MsgReaderExt, ReqHandleExt, ReqReaderExt, io_comm};
+use orchid_base::stash::{stash, with_stash};
 use orchid_base::tree::AtomRepr;
 
 use crate::api;
 use crate::atom::AtomHand;
-use crate::ctx::Ctx;
+use crate::ctx::{Ctx, JoinHandle};
 use crate::dealias::{ChildError, ChildErrorKind, walk};
 use crate::expr::{Expr, PathSetBuilder};
 use crate::system::SystemCtor;
 use crate::tree::MemberKind;
 
+pub struct ExtPort {
+	pub input: Pin<Box<dyn AsyncWrite>>,
+	pub output: Pin<Box<dyn AsyncRead>>,
+}
+
 pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 
 /// Data held about an Extension. This is refcounted within [Extension]. It's
@@ -42,120 +46,129 @@ pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 pub struct ExtensionData {
 	name: String,
 	ctx: Ctx,
-	reqnot: ReqNot<api::HostMsgSet>,
+	join_ext: Option<Box<dyn JoinHandle>>,
+	client: Rc<dyn Client>,
 	systems: Vec<SystemCtor>,
-	logger: Logger,
 	next_pars: RefCell<NonZeroU64>,
-	exiting_snd: Sender<()>,
 	lex_recur: Mutex<HashMap<api::ParsId, Sender<ReqPair<api::SubLex>>>>,
+	strings: RefCell<HashSet<IStr>>,
+	string_vecs: RefCell<HashSet<IStrv>>,
 }
 impl Drop for ExtensionData {
 	fn drop(&mut self) {
-		let reqnot = self.reqnot.clone();
-		let mut exiting_snd = self.exiting_snd.clone();
-		(self.ctx.spawn)(Box::pin(async move {
-			reqnot.notify(api::HostExtNotif::Exit).await;
-
-			exiting_snd.send(()).await.unwrap()
-		}))
+		let client = self.client.clone();
+		let join_ext = self.join_ext.take().expect("Only called once in Drop");
+		stash(async move {
+			client.notify(api::HostExtNotif::Exit).await.unwrap();
+			join_ext.join().await;
+		})
 	}
 }
 
 #[derive(Clone)]
 pub struct Extension(Rc<ExtensionData>);
 impl Extension {
-	pub fn new(init: ExtInit, logger: Logger, msg_logger: Logger, ctx: Ctx) -> io::Result<Self> {
+	pub async fn new(mut init: ExtPort, ctx: Ctx) -> io::Result<Self> {
+		api::HostHeader { logger: ctx.logger.to_api() }.encode(init.input.as_mut()).await.unwrap();
+		init.input.flush().await.unwrap();
+
+		let header = api::ExtensionHeader::decode(init.output.as_mut()).await.unwrap();
+		let header2 = header.clone();
 		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| {
-			let init = Rc::new(init);
-			let (exiting_snd, exiting_rcv) = channel::<()>(0);
-			(ctx.spawn)({
-				clone!(init, weak, ctx);
-				Box::pin(async move {
-					let rcv_stream = stream(async |mut cx| {
-						loop {
-							cx.emit(init.recv().await).await
-						}
-					});
-					let mut event_stream = pin!(stream::select(exiting_rcv.map(|()| None), rcv_stream));
-					while let Some(Some(msg)) = event_stream.next().await {
-						if let Some(reqnot) = weak.upgrade().map(|rc| rc.reqnot.clone()) {
-							let reqnot = reqnot.clone();
-							(ctx.spawn)(Box::pin(async move {
-								reqnot.receive(&msg).await;
-							}))
-						}
-					}
-				})
-			});
-			ExtensionData {
-				name: init.name.clone(),
-				exiting_snd,
-				ctx: ctx.clone(),
-				systems: (init.systems.iter().cloned())
-					.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
-					.collect(),
-				logger: logger.clone(),
-				next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
-				lex_recur: Mutex::default(),
-				reqnot: ReqNot::new(
-					msg_logger,
-					move |sfn, _| clone!(init; Box::pin(async move { init.send(sfn).await })),
-					clone!(weak; move |notif, _| {
-						clone!(weak; Box::pin(async move {
-						let this = Extension(weak.upgrade().unwrap());
-						if !matches!(notif, api::ExtHostNotif::Log(_)) {
-							writeln!(this.reqnot().logger(), "Host received notif {notif:?}");
-						}
-						match notif {
-							api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
-								let target = this.0.ctx.exprs.get_expr(acq.1).expect("Invalid ticket");
-								this.0.ctx.exprs.give_expr(target)
-							}
-							api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
-								if this.is_own_sys(rel.0).await {
-									this.0.ctx.exprs.take_expr(rel.1);
-								} else {
-									writeln!(this.reqnot().logger(), "Not our system {:?}", rel.0)
+			// context not needed because exit is extension-initiated
+			let (client, _, comm) = io_comm(Rc::new(Mutex::new(init.input)), Mutex::new(init.output));
+			let weak2 = weak;
+			let weak = weak.clone();
+			let ctx2 = ctx.clone();
+			let join_ext = ctx.clone().spawn(async move {
+				comm
+					.listen(
+						async |reader| {
+							with_stash(async {
+								let this = Extension(weak.upgrade().unwrap());
+								let notif = reader.read::<api::ExtHostNotif>().await.unwrap();
+								// logging is never logged because its value will be logged anyway
+								if !matches!(notif, api::ExtHostNotif::Log(_)) {
+									writeln!(log("msg"), "Host received notif {notif:?}").await;
+								}
+								match notif {
+									api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
+										let target = this.0.ctx.exprs.get_expr(acq.1).expect("Invalid ticket");
+										this.0.ctx.exprs.give_expr(target)
+									},
+									api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
+										if this.is_own_sys(rel.0).await {
+											this.0.ctx.exprs.take_expr(rel.1);
+										} else {
+											writeln!(log("warn"), "Not our system {:?}", rel.0).await
+										}
+									},
+									api::ExtHostNotif::Log(api::Log { category, message }) =>
+										write!(log(&es(category).await), "{message}").await,
+									api::ExtHostNotif::Sweeped(data) => {
+										for i in join_all(data.strings.into_iter().map(es)).await {
+											this.0.strings.borrow_mut().remove(&i);
+										}
+										for i in join_all(data.vecs.into_iter().map(ev)).await {
+											this.0.string_vecs.borrow_mut().remove(&i);
+										}
+									},
+								}
+								Ok(())
+							})
+							.await
+						},
+						async |mut reader| {
+							with_stash(async {
+								let req = reader.read_req::<api::ExtHostReq>().await.unwrap();
+								let handle = reader.finish().await;
+								// Atom printing and interning is never reported because it generates too much
+								// noise
+								if !matches!(req, api::ExtHostReq::ExtAtomPrint(_))
+									|| matches!(req, api::ExtHostReq::IntReq(_))
+								{
+									writeln!(log("msg"), "Host received request {req:?}").await;
 								}
-							},
-							api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str),
-						}
-					}))}),
-					{
-						clone!(weak, ctx);
-						move |hand, req| {
-							clone!(weak, ctx);
-							Box::pin(async move {
 								let this = Self(weak.upgrade().unwrap());
-								if !matches!(req, api::ExtHostReq::ExtAtomPrint(_)) {
-									writeln!(this.reqnot().logger(), "Host received request {req:?}");
-								}
-								let i = this.ctx().i.clone();
 								match req {
-									api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
+									api::ExtHostReq::Ping(ping) => handle.reply(&ping, &()).await,
 									api::ExtHostReq::IntReq(intreq) => match intreq {
-										api::IntReq::InternStr(s) => hand.handle(&s, &i.i(&*s.0).await.to_api()).await,
-										api::IntReq::InternStrv(v) => {
-											let tokens = join_all(v.0.iter().map(|m| i.ex(*m))).await;
-											hand.handle(&v, &i.i(&tokens).await.to_api()).await
+										api::IntReq::InternStr(s) => {
+											let i = is(&s.0).await;
+											this.0.strings.borrow_mut().insert(i.clone());
+											handle.reply(&s, &i.to_api()).await
+										},
+										api::IntReq::InternStrv(v) => {
+											let tokens = join_all(v.0.iter().map(|m| es(*m))).await;
+											this.0.strings.borrow_mut().extend(tokens.iter().cloned());
+											let i = iv(&tokens).await;
+											this.0.string_vecs.borrow_mut().insert(i.clone());
+											handle.reply(&v, &i.to_api()).await
+										},
+										api::IntReq::ExternStr(si) => {
+											let i = es(si.0).await;
+											this.0.strings.borrow_mut().insert(i.clone());
+											handle.reply(&si, &i.to_string()).await
 										},
-										api::IntReq::ExternStr(si) =>
-											hand.handle(&si, &Tok::<String>::from_api(si.0, &i).await.rc()).await,
 										api::IntReq::ExternStrv(vi) => {
-											let markerv = (i.ex(vi.0).await.iter()).map(|t| t.to_api()).collect_vec();
-											hand.handle(&vi, &markerv).await
+											let i = ev(vi.0).await;
+											this.0.strings.borrow_mut().extend(i.iter().cloned());
+											this.0.string_vecs.borrow_mut().insert(i.clone());
+											let markerv = i.iter().map(|t| t.to_api()).collect_vec();
+											handle.reply(&vi, &markerv).await
 										},
 									},
 									api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
 										let sys =
 											ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
+										let client = sys.client();
 										let reply =
-											sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await;
-										hand.handle(fw, &reply).await
+											client.request(api::Fwded(fw.0.clone(), *key, body.clone())).await.unwrap();
+										handle.reply(fw, &reply).await
 									},
 									api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
 										let sys = ctx.system_inst(id).await.unwrap();
-										hand.handle(fw, &sys.request(body.clone()).await).await
+										handle.reply(fw, &sys.request(body.clone()).await).await
 									},
 									api::ExtHostReq::SubLex(sl) => {
 										let (rep_in, mut rep_out) = channel(0);
@@ -165,13 +178,13 @@ impl Extension {
 												lex_g.get(&sl.id).cloned().expect("Sublex for nonexistent lexid");
 											req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
 										}
-										hand.handle(&sl, &rep_out.next().await.unwrap()).await
+										handle.reply(&sl, &rep_out.next().await.unwrap()).await
 									},
 									api::ExtHostReq::ExprReq(expr_req) => match expr_req {
 										api::ExprReq::Inspect(ins @ api::Inspect { target }) => {
 											let expr = ctx.exprs.get_expr(target).expect("Invalid ticket");
-											hand
-												.handle(&ins, &api::Inspected {
+											handle
+												.reply(&ins, &api::Inspected {
 													refcount: expr.strong_count() as u32,
 													location: expr.pos().to_api(),
 													kind: expr.to_api().await,
@@ -182,12 +195,12 @@ impl Extension {
 											let expr = Expr::from_api(expr, PathSetBuilder::new(), ctx.clone()).await;
 											let expr_id = expr.id();
 											ctx.exprs.give_expr(expr);
-											hand.handle(cre, &expr_id).await
+											handle.reply(cre, &expr_id).await
 										},
 									},
 									api::ExtHostReq::LsModule(ref ls @ api::LsModule(_sys, path)) => {
 										let reply: <api::LsModule as Request>::Response = 'reply: {
-											let path = i.ex(path).await;
+											let path = ev(path).await;
 											let root = (ctx.root.read().await.upgrade())
 												.expect("LSModule called when root isn't in context");
 											let root_data = &*root.0.read().await;
@@ -214,7 +227,7 @@ impl Extension {
 											}
 											Ok(api::ModuleInfo { members })
 										};
-										hand.handle(ls, &reply).await
+										handle.reply(ls, &reply).await
 									},
 									api::ExtHostReq::ResolveNames(ref rn) => {
 										let api::ResolveNames { constid, names, sys } = rn;
@@ -226,42 +239,63 @@ impl Extension {
 										};
 										let responses = stream(async |mut cx| {
 											for name in names {
-												cx.emit(match resolver(&ctx.i.ex(*name).await[..]).await {
-													Ok(abs) => Ok(abs.to_sym(&ctx.i).await.to_api()),
-													Err(e) => Err(e.to_api()),
+												cx.emit(match resolver(&ev(*name).await[..]).await {
+													Ok(abs) => {
+														let sym = abs.to_sym().await;
+														this.0.string_vecs.borrow_mut().insert(sym.tok());
+														Ok(sym.to_api())
+													},
+													Err(e) => {
+														(this.0.strings.borrow_mut())
+															.extend(e.iter().map(|e| e.description.clone()));
+														Err(e.to_api())
+													},
 												})
 												.await
 											}
 										})
 										.collect()
 										.await;
-										hand.handle(rn, &responses).await
+										handle.reply(rn, &responses).await
 									},
 									api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
 										let atom = AtomHand::from_api(atom, Pos::None, &mut ctx.clone()).await;
-										let unit = atom.print(&FmtCtxImpl { i: &this.ctx().i }).await;
-										hand.handle(eap, &unit.to_api()).await
+										let unit = atom.print(&FmtCtxImpl::default()).await;
+										handle.reply(eap, &unit.to_api()).await
 									},
 								}
 							})
-						}
-					},
-				),
+							.await
+						},
+					)
+					.await
+					.unwrap();
+			});
+			ExtensionData {
+				name: header2.name.clone(),
+				ctx: ctx2,
+				systems: (header.systems.iter().cloned())
+					.map(|decl| SystemCtor { decl, ext: WeakExtension(weak2.clone()) })
+					.collect(),
+				join_ext: Some(join_ext),
+				next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
+				lex_recur: Mutex::default(),
+				client: Rc::new(client),
+				strings: RefCell::default(),
+				string_vecs: RefCell::default(),
 			}
 		})))
 	}
 	pub fn name(&self) -> &String { &self.0.name }
 	#[must_use]
-	pub fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.reqnot }
+	pub fn client(&self) -> &dyn Client { &*self.0.client }
 	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
-	#[must_use]
-	pub fn logger(&self) -> &Logger { &self.0.logger }
 	pub fn system_ctors(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
 	#[must_use]
 	pub async fn is_own_sys(&self, id: api::SysId) -> bool {
 		let Some(sys) = self.ctx().system_inst(id).await else {
-			writeln!(self.logger(), "Invalid system ID {id:?}");
+			writeln!(log("warn"), "Invalid system ID {id:?}").await;
 			return false;
 		};
 		Rc::ptr_eq(&self.0, &sys.ext().0)
@@ -274,7 +308,7 @@ impl Extension {
 	}
 	pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
-		source: Tok<String>,
+		source: IStr,
 		src: Sym,
 		pos: u32,
 		sys: api::SysId,
@@ -287,9 +321,10 @@ impl Extension {
 		self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released
 		let (ret, ()) = join(
 			async {
-				let res = (self.reqnot())
+				let res = (self.client())
 					.request(api::LexExpr { id, pos, sys, src: src.to_api(), text: source.to_api() })
-					.await;
+					.await
+					.unwrap();
 				// collect sender to unblock recursion handler branch before returning
 				self.0.lex_recur.lock().await.remove(&id);
 				res
@@ -306,10 +341,10 @@ impl Extension {
 	}
 	pub fn system_drop(&self, id: api::SysId) {
 		let rc = self.clone();
-		(self.ctx().spawn)(Box::pin(async move {
-			rc.reqnot().request(api::SystemDrop(id)).await;
+		let _ = self.ctx().spawn(with_stash(async move {
+			rc.client().request(api::SystemDrop(id)).await.unwrap();
 			rc.ctx().systems.write().await.remove(&id);
-		}))
+		}));
 	}
 	#[must_use]
 	pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }

orchid-host/src/lex.rs

@@ -1,10 +1,8 @@
-use std::rc::Rc;
-
 use futures::FutureExt;
 use futures::lock::Mutex;
 use orchid_base::clone;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
-use orchid_base::interner::Tok;
+use orchid_base::interner::{IStr, is};
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
@@ -20,7 +18,7 @@ use crate::system::System;
 
 pub struct LexCtx<'a> {
 	pub systems: &'a [System],
-	pub source: &'a Tok<String>,
+	pub source: &'a IStr,
 	pub path: &'a Sym,
 	pub tail: &'a str,
 	pub sub_trees: &'a mut Vec<Expr>,
@@ -60,7 +58,7 @@ impl<'a> LexCtx<'a> {
 	}
 	#[must_use]
 	pub async fn des_subtree(&mut self, tree: &api::TokenTree, exprs: ExprStore) -> ParsTokTree {
-		ParsTokTree::from_api(tree, &mut { exprs }, &mut self.ctx.clone(), self.path, &self.ctx.i).await
+		ParsTokTree::from_api(tree, &mut { exprs }, &mut self.ctx.clone(), self.path).await
 	}
 	#[must_use]
 	pub fn strip_char(&mut self, tgt: char) -> bool {
@@ -98,21 +96,21 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 		let name = &ctx.tail[..ctx.tail.len() - tail.len() - "::".len()];
 		ctx.set_tail(tail);
 		let body = lex_once(ctx).boxed_local().await?;
-		ParsTok::NS(ctx.ctx.i.i(name).await, Box::new(body))
+		ParsTok::NS(is(name).await, Box::new(body))
 	} else if ctx.strip_prefix("--[") {
 		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
 			return Err(mk_errv(
-				ctx.ctx.i.i("Unterminated block comment").await,
+				is("Unterminated block comment").await,
 				"This block comment has no ending ]--",
 				[SrcRange::new(start..start + 3, ctx.path)],
 			));
 		};
 		ctx.set_tail(tail);
-		ParsTok::Comment(Rc::new(cmt.to_string()))
+		ParsTok::Comment(is(cmt).await)
 	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
 		ctx.push_pos(end as u32);
-		ParsTok::Comment(Rc::new(tail[2..end].to_string()))
+		ParsTok::Comment(is(&tail[2..end]).await)
 	} else if let Some(tail) = ctx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
 		// fanciness like \$placeh in templates is resolved in the macro engine.
 		ctx.set_tail(tail);
@@ -125,7 +123,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 		while !ctx.strip_char(*rp) {
 			if ctx.tail.is_empty() {
 				return Err(mk_errv(
-					ctx.ctx.i.i("unclosed paren").await,
+					is("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
 					[SrcRange::new(start..start + 1, ctx.path)],
 				));
@@ -162,10 +160,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 					})
 					.await;
 				match lx {
-					Err(e) =>
-						return Err(
-							errors.into_iter().fold(OrcErrv::from_api(&e, &ctx.ctx.i).await, |a, b| a + b),
-						),
+					Err(e) => return Err(errors.into_iter().fold(OrcErrv::from_api(&e).await, |a, b| a + b)),
 					Ok(Some(lexed)) => {
 						ctx.set_pos(lexed.pos);
 						let lexed_tree = ctx.des_subtree(&lexed.expr, temp_store).await;
@@ -185,12 +180,12 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 			}
 		}
 		if ctx.tail.starts_with(name_start) {
-			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(name_char)).await)
+			ParsTok::Name(is(ctx.get_start_matches(name_char)).await)
 		} else if ctx.tail.starts_with(op_char) {
-			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(op_char)).await)
+			ParsTok::Name(is(ctx.get_start_matches(op_char)).await)
 		} else {
 			return Err(mk_errv(
-				ctx.ctx.i.i("Unrecognized character").await,
+				is("Unrecognized character").await,
 				"The following syntax is meaningless.",
 				[SrcRange::new(start..start + 1, ctx.path)],
 			));
@@ -199,12 +194,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 	Ok(ParsTokTree { tok, sr: SrcRange::new(start..ctx.get_pos(), ctx.path) })
 }
 
-pub async fn lex(
-	text: Tok<String>,
-	path: Sym,
-	systems: &[System],
-	ctx: &Ctx,
-) -> OrcRes<Vec<ParsTokTree>> {
+pub async fn lex(text: IStr, path: Sym, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
 	let mut sub_trees = Vec::new();
 	let mut ctx =
 		LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, path: &path, ctx };

orchid-host/src/lib.rs

@@ -3,11 +3,13 @@ use orchid_api as api;
 pub mod atom;
 pub mod ctx;
 pub mod dealias;
+pub mod dylib;
 pub mod execute;
 pub mod expr;
 pub mod expr_store;
 pub mod extension;
 pub mod lex;
+pub mod logger;
 pub mod parse;
 pub mod parsed;
 pub mod subprocess;

orchid-host/src/logger.rs

@@ -0,0 +1,84 @@
+use std::fmt::Arguments;
+use std::fs::File;
+use std::io::{Write, stderr};
+use std::rc::Rc;
+
+use futures::future::LocalBoxFuture;
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::logging::{LogWriter, Logger};
+
+use crate::api;
+
+pub struct LogWriterImpl(api::LogStrategy);
+impl LogWriter for LogWriterImpl {
+	fn write_fmt<'a>(&'a self, fmt: Arguments<'a>) -> LocalBoxFuture<'a, ()> {
+		Box::pin(async move {
+			match &self.0 {
+				api::LogStrategy::Discard => (),
+				api::LogStrategy::Default => {
+					stderr().write_fmt(fmt).expect("Could not write to stderr!");
+					stderr().flush().expect("Could not flush stderr")
+				},
+				api::LogStrategy::File { path, .. } => {
+					let mut file = (File::options().write(true).create(true).truncate(false).open(path))
+						.unwrap_or_else(|e| panic!("Could not open {path}: {e}"));
+					file.write_fmt(fmt).unwrap_or_else(|e| panic!("Could not write to {path}: {e}"));
+				},
+			}
+		})
+	}
+}
+
+#[derive(Clone, Default)]
+pub struct LoggerImpl {
+	routing: HashMap<String, api::LogStrategy>,
+	default: Option<api::LogStrategy>,
+}
+impl LoggerImpl {
+	pub fn to_api(&self) -> api::Logger {
+		api::Logger {
+			default: self.default.clone(),
+			routing: self.routing.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
+		}
+	}
+
+	pub fn new(
+		default: Option<api::LogStrategy>,
+		strats: impl IntoIterator<Item = (String, api::LogStrategy)>,
+	) -> Self {
+		Self { routing: strats.into_iter().collect(), default }
+	}
+	pub fn set_default(&mut self, strat: api::LogStrategy) { self.default = Some(strat) }
+	pub fn clear_default(&mut self) { self.default = None }
+	pub fn set_category(&mut self, category: &str, strat: api::LogStrategy) {
+		self.routing.insert(category.to_string(), strat);
+	}
+	pub fn with_default(mut self, strat: api::LogStrategy) -> Self {
+		self.set_default(strat);
+		self
+	}
+	pub fn with_category(mut self, category: &str, strat: api::LogStrategy) -> Self {
+		self.set_category(category, strat);
+		self
+	}
+	pub async fn log(&self, category: &str, msg: impl AsRef<str>) {
+		writeln!(self.writer(category), "{}", msg.as_ref()).await
+	}
+	pub fn has_category(&self, category: &str) -> bool { self.routing.contains_key(category) }
+	pub async fn log_buf(&self, category: &str, event: impl AsRef<str>, buf: &[u8]) {
+		if std::env::var("ORCHID_LOG_BUFFERS").is_ok_and(|v| !v.is_empty()) {
+			let data = buf.iter().map(|b| format!("{b:02x}")).join(" ");
+			writeln!(self.writer(category), "{}: [{data}]", event.as_ref()).await
+		}
+	}
+}
+impl Logger for LoggerImpl {
+	fn writer(&self, category: &str) -> Rc<dyn LogWriter> {
+		Rc::new(LogWriterImpl(self.strat(category).clone()))
+	}
+	fn strat(&self, category: &str) -> api::LogStrategy {
+		(self.routing.get(category).cloned().or(self.default.clone()))
+			.expect("Invalid category and catchall logger not set")
+	}
+}

orchid-host/src/parse.rs

@@ -1,12 +1,11 @@
 use futures::FutureExt;
 use itertools::Itertools;
-use orchid_base::error::{OrcRes, Reporter, mk_errv};
+use orchid_base::error::{OrcRes, mk_errv, report};
 use orchid_base::format::fmt;
-use orchid_base::interner::{Interner, Tok};
+use orchid_base::interner::{IStr, is};
 use orchid_base::name::Sym;
 use orchid_base::parse::{
-	Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname,
-	try_pop_no_fluff,
+	Comment, Import, Parsed, Snippet, expect_end, line_items, parse_multiname, try_pop_no_fluff,
 };
 use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
@@ -22,12 +21,6 @@ pub struct HostParseCtxImpl<'a> {
 	pub ctx: Ctx,
 	pub src: Sym,
 	pub systems: &'a [System],
-	pub rep: &'a Reporter,
-}
-
-impl ParseCtx for HostParseCtxImpl<'_> {
-	fn rep(&self) -> &Reporter { self.rep }
-	fn i(&self) -> &Interner { &self.ctx.i }
 }
 
 impl HostParseCtx for HostParseCtxImpl<'_> {
@@ -36,7 +29,7 @@ impl HostParseCtx for HostParseCtxImpl<'_> {
 	fn src_path(&self) -> Sym { self.src.clone() }
 }
 
-pub trait HostParseCtx: ParseCtx {
+pub trait HostParseCtx {
 	#[must_use]
 	fn ctx(&self) -> &Ctx;
 	#[must_use]
@@ -47,14 +40,14 @@ pub trait HostParseCtx: ParseCtx {
 
 pub async fn parse_items(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	items: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	let lines = line_items(ctx, items).await;
+	let lines = line_items(items).await;
 	let mut line_ok = Vec::new();
 	for Parsed { output: comments, tail } in lines {
 		match parse_item(ctx, path.clone(), comments, tail).boxed_local().await {
-			Err(e) => ctx.rep().report(e),
+			Err(e) => report(e),
 			Ok(l) => line_ok.extend(l),
 		}
 	}
@@ -63,23 +56,23 @@ pub async fn parse_items(
 
 pub async fn parse_item(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	comments: Vec<Comment>,
 	item: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
 	match item.pop_front() {
 		Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
-			n if *n == ctx.i().i("export").await => match try_pop_no_fluff(ctx, postdisc).await? {
+			n if *n == is("export").await => match try_pop_no_fluff(postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
 					parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
 				Parsed { output, tail: _ } => Err(mk_errv(
-					ctx.i().i("Malformed export").await,
+					is("Malformed export").await,
 					"`export` can either prefix other lines or list names inside ( )",
 					[output.sr()],
 				)),
 			},
-			n if *n == ctx.i().i("import").await => {
-				let imports = parse_import(ctx, postdisc).await?;
+			n if *n == is("import").await => {
+				let imports = parse_import(postdisc).await?;
 				Ok(Vec::from_iter(imports.into_iter().map(|t| Item {
 					comments: comments.clone(),
 					sr: t.sr.clone(),
@@ -88,33 +81,29 @@ pub async fn parse_item(
 			},
 			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
 		},
-		Some(_) => Err(mk_errv(
-			ctx.i().i("Expected a line type").await,
-			"All lines must begin with a keyword",
-			[item.sr()],
-		)),
+		Some(_) =>
+			Err(mk_errv(is("Expected a line type").await, "All lines must begin with a keyword", [
+				item.sr()
+			])),
 		None => unreachable!("These lines are filtered and aggregated in earlier stages"),
 	}
 }
 
-pub async fn parse_import<'a>(
-	ctx: &impl HostParseCtx,
-	tail: ParsSnippet<'a>,
-) -> OrcRes<Vec<Import>> {
-	let Parsed { output: imports, tail } = parse_multiname(ctx, tail).await?;
-	expect_end(ctx, tail).await?;
+pub async fn parse_import<'a>(tail: ParsSnippet<'a>) -> OrcRes<Vec<Import>> {
+	let Parsed { output: imports, tail } = parse_multiname(tail).await?;
+	expect_end(tail).await?;
 	Ok(imports)
 }
 
 pub async fn parse_exportable_item<'a>(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	comments: Vec<Comment>,
 	exported: bool,
-	discr: Tok<String>,
+	discr: IStr,
 	tail: ParsSnippet<'a>,
 ) -> OrcRes<Vec<Item>> {
-	let kind = if discr == ctx.i().i("mod").await {
+	let kind = if discr == is("mod").await {
 		let (name, body) = parse_module(ctx, path, tail).await?;
 		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::Mod(body) })
 	} else if let Some(parser) = ctx.systems().find_map(|s| s.get_parser(discr.clone())) {
@@ -127,7 +116,7 @@ pub async fn parse_exportable_item<'a>(
 	} else {
 		let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
 		return Err(mk_errv(
-			ctx.i().i("Unrecognized line type").await,
+			is("Unrecognized line type").await,
 			format!("Line types are: mod, {ext_lines}"),
 			[tail.prev().sr()],
 		));
@@ -137,25 +126,25 @@ pub async fn parse_exportable_item<'a>(
 
 pub async fn parse_module<'a>(
 	ctx: &impl HostParseCtx,
-	path: Substack<'_, Tok<String>>,
+	path: Substack<'_, IStr>,
 	tail: ParsSnippet<'a>,
-) -> OrcRes<(Tok<String>, ParsedModule)> {
-	let (name, tail) = match try_pop_no_fluff(ctx, tail).await? {
+) -> OrcRes<(IStr, ParsedModule)> {
+	let (name, tail) = match try_pop_no_fluff(tail).await? {
 		Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
 		Parsed { output, .. } => {
 			return Err(mk_errv(
-				ctx.i().i("Missing module name").await,
-				format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
+				is("Missing module name").await,
+				format!("A name was expected, {} was found", fmt(output).await),
 				[output.sr()],
 			));
 		},
 	};
-	let Parsed { output, tail: surplus } = try_pop_no_fluff(ctx, tail).await?;
-	expect_end(ctx, surplus).await?;
+	let Parsed { output, tail: surplus } = try_pop_no_fluff(tail).await?;
+	expect_end(surplus).await?;
 	let Some(body) = output.as_s(Paren::Round) else {
 		return Err(mk_errv(
-			ctx.i().i("Expected module body").await,
-			format!("A ( block ) was expected, {} was found", fmt(output, ctx.i()).await),
+			is("Expected module body").await,
+			format!("A ( block ) was expected, {} was found", fmt(output).await),
 			[output.sr()],
 		));
 	};

orchid-host/src/parsed.rs

@@ -6,7 +6,7 @@ use futures::future::{LocalBoxFuture, join_all};
 use hashbrown::HashSet;
 use itertools::Itertools;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
-use orchid_base::interner::Tok;
+use orchid_base::interner::{IStr, IStrv};
 use orchid_base::location::SrcRange;
 use orchid_base::parse::{Comment, Import};
 use orchid_base::tl_cache;
@@ -57,10 +57,10 @@ impl Format for Item {
 			ItemKind::Member(mem) => match &mem.kind {
 				ParsedMemberKind::Const(_, sys) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} via {1}")))
-						.units([mem.name.rc().into(), sys.print(c).await]),
+						.units([mem.name.to_string().into(), sys.print(c).await]),
 				ParsedMemberKind::Mod(module) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
-						.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
+						.units([mem.name.to_string().into(), module.print(c).boxed_local().await]),
 			},
 		};
 		tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0}\n{1}")))
@@ -69,14 +69,14 @@ impl Format for Item {
 }
 
 pub struct ParsedMember {
-	pub name: Tok<String>,
+	pub name: IStr,
 	pub exported: bool,
 	pub kind: ParsedMemberKind,
 }
 impl ParsedMember {
 	#[must_use]
-	pub fn name(&self) -> Tok<String> { self.name.clone() }
-	pub fn new(exported: bool, name: Tok<String>, kind: impl Into<ParsedMemberKind>) -> Self {
+	pub fn name(&self) -> IStr { self.name.clone() }
+	pub fn new(exported: bool, name: IStr, kind: impl Into<ParsedMemberKind>) -> Self {
 		Self { exported, name, kind: kind.into() }
 	}
 }
@@ -89,17 +89,14 @@ impl Debug for ParsedMember {
 	}
 }
 
-pub(crate) type ParsedExprCallback =
-	Rc<dyn for<'a> Fn(&'a [Tok<String>]) -> LocalBoxFuture<'a, Expr>>;
+pub(crate) type ParsedExprCallback = Rc<dyn for<'a> Fn(&'a [IStr]) -> LocalBoxFuture<'a, Expr>>;
 
 pub struct ParsedExpr {
 	pub(crate) debug: String,
 	pub(crate) callback: ParsedExprCallback,
 }
 impl ParsedExpr {
-	pub async fn run(self, imported_names: &[Tok<String>]) -> Expr {
-		(self.callback)(imported_names).await
-	}
+	pub async fn run(self, imported_names: &[IStr]) -> Expr { (self.callback)(imported_names).await }
 }
 impl fmt::Debug for ParsedExpr {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.debug) }
@@ -115,7 +112,7 @@ impl From<ParsedModule> for ParsedMemberKind {
 }
 #[derive(Debug, Default)]
 pub struct ParsedModule {
-	pub exports: Vec<Tok<String>>,
+	pub exports: Vec<IStr>,
 	pub items: Vec<Item>,
 	pub use_prelude: bool,
 }
@@ -141,7 +138,7 @@ impl ParsedModule {
 		(self.items.iter())
 			.filter_map(|it| if let ItemKind::Import(i) = &it.kind { Some(i) } else { None })
 	}
-	pub fn default_item(self, name: Tok<String>, sr: SrcRange) -> Item {
+	pub fn default_item(self, name: IStr, sr: SrcRange) -> Item {
 		let mem = ParsedMember { exported: true, name, kind: ParsedMemberKind::Mod(self) };
 		Item { comments: vec![], sr, kind: ItemKind::Member(mem) }
 	}
@@ -150,7 +147,7 @@ impl Tree for ParsedModule {
 	type Ctx<'a> = ();
 	async fn child(
 		&self,
-		key: Tok<String>,
+		key: IStr,
 		public_only: bool,
 		(): &mut Self::Ctx<'_>,
 	) -> ChildResult<'_, Self> {
@@ -168,7 +165,7 @@ impl Tree for ParsedModule {
 		}
 		ChildResult::Err(ChildErrorKind::Missing)
 	}
-	fn children(&self, public_only: bool) -> HashSet<Tok<String>> {
+	fn children(&self, public_only: bool) -> HashSet<IStr> {
 		let mut public: HashSet<_> = self.exports.iter().cloned().collect();
 		if !public_only {
 			public.extend(
@@ -197,11 +194,11 @@ impl Format for ParsedModule {
 /// point to a module and rule_loc selects a macro rule within that module
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct ConstPath {
-	steps: Tok<Vec<Tok<String>>>,
+	steps: IStrv,
 }
 impl ConstPath {
 	#[must_use]
-	pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps } }
+	pub fn to_const(steps: IStrv) -> Self { Self { steps } }
 }
 
 pub async fn tt_to_api(exprs: &mut ExprStore, subtree: ParsTokTree) -> api::TokenTree {

orchid-host/src/subprocess.rs

@@ -1,102 +1,34 @@
-use std::cell::RefCell;
-use std::io::{self, Write};
-use std::pin::Pin;
+use std::{io, process};
 
-use async_process::{self, Child, ChildStdin, ChildStdout};
-use futures::future::LocalBoxFuture;
 use futures::io::BufReader;
-use futures::lock::Mutex;
-use futures::{self, AsyncBufReadExt, AsyncWriteExt};
-use orchid_api_traits::{Decode, Encode};
-use orchid_base::builtin::{ExtInit, ExtPort};
-use orchid_base::logging::Logger;
-use orchid_base::msg::{recv_msg, send_msg};
+use futures::{self, AsyncBufReadExt, StreamExt};
+use orchid_base::logging::log;
+#[cfg(feature = "tokio")]
+use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
 
-use crate::api;
 use crate::ctx::Ctx;
+use crate::extension::ExtPort;
 
-pub async fn ext_command(
-	cmd: std::process::Command,
-	logger: Logger,
-	msg_logs: Logger,
-	ctx: Ctx,
-) -> io::Result<ExtInit> {
-	let mut child = async_process::Command::from(cmd)
-		.stdin(async_process::Stdio::piped())
-		.stdout(async_process::Stdio::piped())
-		.stderr(async_process::Stdio::piped())
+#[cfg(feature = "tokio")]
+pub async fn ext_command(cmd: std::process::Command, ctx: Ctx) -> io::Result<ExtPort> {
+	let name = cmd.get_program().to_string_lossy().to_string();
+	let mut child = tokio::process::Command::from(cmd)
+		.stdin(process::Stdio::piped())
+		.stdout(process::Stdio::piped())
+		.stderr(process::Stdio::piped())
 		.spawn()?;
-	let mut stdin = child.stdin.take().unwrap();
-	api::HostHeader { log_strategy: logger.strat(), msg_logs: msg_logs.strat() }
-		.encode(Pin::new(&mut stdin))
-		.await;
-	let mut stdout = child.stdout.take().unwrap();
-	let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await;
-	let mut child_stderr = child.stderr.take().unwrap();
-	(ctx.spawn)(Box::pin(async move {
-		let mut reader = BufReader::new(&mut child_stderr);
-		loop {
-			let mut buf = String::new();
-			if 0 == reader.read_line(&mut buf).await.unwrap() {
-				break;
-			}
-			logger.log(buf.strip_suffix('\n').expect("Readline implies this"));
+	std::thread::spawn(|| {});
+	let stdin = child.stdin.take().unwrap();
+	let stdout = child.stdout.take().unwrap();
+	let child_stderr = child.stderr.take().unwrap();
+	let _ = ctx.spawn(Box::pin(async move {
+		let _ = child;
+		let mut lines = BufReader::new(child_stderr.compat()).lines();
+		while let Some(line) = lines.next().await {
+			// route stderr with an empty category string. This is not the intended logging
+			// method
+			writeln!(log("stderr"), "{} err> {}", name, line.expect("Readline implies this")).await;
 		}
 	}));
-	Ok(ExtInit {
-		port: Box::new(Subprocess {
-			name: header.name.clone(),
-			child: RefCell::new(Some(child)),
-			stdin: Some(Mutex::new(Box::pin(stdin))),
-			stdout: Mutex::new(Box::pin(stdout)),
-			ctx,
-		}),
-		header,
-	})
-}
-
-pub struct Subprocess {
-	name: String,
-	child: RefCell<Option<Child>>,
-	stdin: Option<Mutex<Pin<Box<ChildStdin>>>>,
-	stdout: Mutex<Pin<Box<ChildStdout>>>,
-	ctx: Ctx,
-}
-impl Drop for Subprocess {
-	fn drop(&mut self) {
-		let mut child = self.child.borrow_mut().take().unwrap();
-		let name = self.name.clone();
-		if std::thread::panicking() {
-			eprintln!("Killing extension {name}");
-			// we don't really care to handle errors here
-			let _: Result<_, _> = std::io::stderr().flush();
-			let _: Result<_, _> = child.kill();
-			return;
-		}
-		let stdin = self.stdin.take().unwrap();
-		(self.ctx.spawn)(Box::pin(async move {
-			stdin.lock().await.close().await.unwrap();
-			let status = (child.status().await)
-				.unwrap_or_else(|e| panic!("{e}, extension {name} exited with error"));
-			assert!(status.success(), "Extension {name} exited with error {status}");
-		}))
-	}
-}
-impl ExtPort for Subprocess {
-	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		Box::pin(async {
-			send_msg(Pin::new(&mut *self.stdin.as_ref().unwrap().lock().await), msg).await.unwrap()
-		})
-	}
-	fn recv(&self) -> LocalBoxFuture<'_, Option<Vec<u8>>> {
-		Box::pin(async {
-			std::io::Write::flush(&mut std::io::stderr()).unwrap();
-			match recv_msg(self.stdout.lock().await.as_mut()).await {
-				Ok(msg) => Some(msg),
-				Err(e) if e.kind() == io::ErrorKind::BrokenPipe => None,
-				Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => None,
-				Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
-			}
-		})
-	}
+	Ok(ExtPort { input: Box::pin(stdin.compat_write()), output: Box::pin(stdout.compat()) })
 }

orchid-host/src/sys_parser.rs

@@ -2,11 +2,11 @@ use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
 use orchid_base::error::{OrcErrv, OrcRes};
-use orchid_base::interner::{Interner, Tok};
+use orchid_base::interner::{IStr, es};
 use orchid_base::location::SrcRange;
 use orchid_base::name::Sym;
 use orchid_base::parse::Comment;
-use orchid_base::reqnot::Requester;
+use orchid_base::reqnot::ClientExt;
 use orchid_base::tree::ttv_from_api;
 use substack::Substack;
 
@@ -22,7 +22,7 @@ pub struct Parser {
 	pub(crate) system: System,
 	pub(crate) idx: u16,
 }
-type ModPath<'a> = Substack<'a, Tok<String>>;
+type ModPath<'a> = Substack<'a, IStr>;
 
 impl Parser {
 	pub async fn parse(
@@ -39,7 +39,7 @@ impl Parser {
 		let line =
 			join_all((line.into_iter()).map(|t| async { tt_to_api(&mut temp_store.clone(), t).await }))
 				.await;
-		let mod_path = ctx.src_path().suffix(path.unreverse(), self.system.i()).await;
+		let mod_path = ctx.src_path().suffix(path.unreverse()).await;
 		let comments = comments.iter().map(Comment::to_api).collect_vec();
 		let req = api::ParseLine {
 			idx: self.idx,
@@ -50,17 +50,16 @@ impl Parser {
 			comments,
 			line,
 		};
-		match self.system.reqnot().request(req).await {
+		match self.system.client().request(req).await.unwrap() {
 			Ok(parsed_v) =>
 				conv(parsed_v, path, callback, &mut ConvCtx {
-					i: self.system.i(),
 					mod_path: &mod_path,
 					ext_exprs: &mut temp_store,
 					src_path: &src_path,
 					sys: &self.system,
 				})
 				.await,
-			Err(e) => Err(OrcErrv::from_api(&e, &self.system.ctx().i).await),
+			Err(e) => Err(OrcErrv::from_api(&e).await),
 		}
 	}
 }
@@ -69,13 +68,12 @@ struct ConvCtx<'a> {
 	sys: &'a System,
 	mod_path: &'a Sym,
 	src_path: &'a Sym,
-	i: &'a Interner,
 	ext_exprs: &'a mut ExprStore,
 }
 async fn conv(
 	parsed_v: Vec<api::ParsedLine>,
-	module: Substack<'_, Tok<String>>,
-	callback: &'_ mut impl AsyncFnMut(Substack<'_, Tok<String>>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
+	module: Substack<'_, IStr>,
+	callback: &'_ mut impl AsyncFnMut(Substack<'_, IStr>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
 	ctx: &mut ConvCtx<'_>,
 ) -> OrcRes<Vec<Item>> {
 	let mut items = Vec::new();
@@ -85,12 +83,12 @@ async fn conv(
 				(name, exported, kind),
 			api::ParsedLineKind::Recursive(rec) => {
 				let tokens =
-					ttv_from_api(rec, ctx.ext_exprs, &mut ctx.sys.ctx().clone(), ctx.src_path, ctx.i).await;
+					ttv_from_api(rec, ctx.ext_exprs, &mut ctx.sys.ctx().clone(), ctx.src_path).await;
 				items.extend(callback(module.clone(), tokens).await?);
 				continue;
 			},
 		};
-		let name = ctx.i.ex(name).await;
+		let name = es(name).await;
 		let mem_path = module.push(name.clone());
 		let mkind = match kind {
 			api::ParsedMemberKind::Module { lines, use_prelude } => {
@@ -98,16 +96,16 @@ async fn conv(
 				ParsedMemberKind::Mod(ParsedModule::new(use_prelude, items))
 			},
 			api::ParsedMemberKind::Constant(cid) => {
-				ctx.sys.0.const_paths.insert(cid, ctx.mod_path.suffix(mem_path.unreverse(), ctx.i).await);
+				ctx.sys.0.const_paths.insert(cid, ctx.mod_path.suffix(mem_path.unreverse()).await);
 				ParsedMemberKind::Const(cid, ctx.sys.clone())
 			},
 		};
 		items.push(Item {
 			comments: join_all(
-				parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone(), ctx.i)),
+				parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone())),
 			)
 			.await,
-			sr: SrcRange::from_api(&parsed.source_range, ctx.i).await,
+			sr: SrcRange::from_api(&parsed.source_range).await,
 			kind: ItemKind::Member(ParsedMember { name, exported, kind: mkind }),
 		})
 	}

orchid-host/src/system.rs

@@ -12,10 +12,12 @@ use memo_map::MemoMap;
 use orchid_base::char_filter::char_filter_match;
 use orchid_base::error::{OrcRes, mk_errv_floating};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
-use orchid_base::interner::{Interner, Tok};
+use orchid_base::interner::{IStr, es, is};
 use orchid_base::iter_utils::IteratorPrint;
+use orchid_base::logging::log;
 use orchid_base::name::{NameLike, Sym, VName, VPath};
-use orchid_base::reqnot::{ReqNot, Requester};
+use orchid_base::reqnot::{Client, ClientExt};
+use orchid_base::stash::stash;
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
@@ -35,7 +37,7 @@ pub(crate) struct SystemInstData {
 	decl_id: api::SysDeclId,
 	lex_filter: api::CharFilter,
 	id: api::SysId,
-	line_types: Vec<Tok<String>>,
+	line_types: Vec<IStr>,
 	prelude: Vec<Sym>,
 	owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
 	pub(crate) const_paths: MemoMap<api::ParsedConstId, Sym>,
@@ -68,8 +70,6 @@ impl System {
 	#[must_use]
 	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
 	#[must_use]
-	pub fn i(&self) -> &Interner { &self.0.ctx.i }
-	#[must_use]
 	pub fn deps(&self) -> &[System] { &self.0.deps }
 	#[must_use]
 	pub fn ctor(&self) -> SystemCtor {
@@ -77,22 +77,27 @@ impl System {
 			.expect("Ctor was used to create ext")
 	}
 	#[must_use]
-	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { self.0.ext.reqnot() }
+	pub(crate) fn client(&self) -> &dyn Client { self.0.ext.client() }
 	#[must_use]
 	pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
-		self.reqnot().request(api::GetMember(self.0.id, id)).await
+		self.client().request(api::GetMember(self.0.id, id)).await.unwrap()
 	}
 	#[must_use]
 	pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
 	#[must_use]
-	pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
+	pub fn can_lex(&self, c: char) -> bool {
+		let ret = char_filter_match(&self.0.lex_filter, c);
+		let ctor = self.ctor();
+		stash(async move { writeln!(log("debug"), "{} can lex {c}: {}", ctor.name(), ret).await });
+		ret
+	}
 	#[must_use]
 	pub fn prelude(&self) -> Vec<Sym> { self.0.prelude.clone() }
 	/// Have this system lex a part of the source. It is assumed that
 	/// [Self::can_lex] was called and returned true.
 	pub async fn lex<F: Future<Output = Option<api::SubLexed>>>(
 		&self,
-		source: Tok<String>,
+		source: IStr,
 		src: Sym,
 		pos: u32,
 		r: impl FnMut(u32) -> F,
@@ -100,16 +105,16 @@ impl System {
 		self.0.ext.lex_req(source, src, pos, self.id(), r).await
 	}
 	#[must_use]
-	pub fn get_parser(&self, ltyp: Tok<String>) -> Option<Parser> {
+	pub fn get_parser(&self, ltyp: IStr) -> Option<Parser> {
 		(self.0.line_types.iter().enumerate())
 			.find(|(_, txt)| *txt == &ltyp)
 			.map(|(idx, _)| Parser { idx: idx as u16, system: self.clone() })
 	}
-	pub fn line_types(&self) -> impl Iterator<Item = &Tok<String>> + '_ { self.0.line_types.iter() }
+	pub fn line_types(&self) -> impl Iterator<Item = &IStr> + '_ { self.0.line_types.iter() }
 
 	#[must_use]
 	pub async fn request(&self, req: Vec<u8>) -> Vec<u8> {
-		self.reqnot().request(api::SysFwded(self.id(), req)).await
+		self.client().request(api::SysFwded(self.id(), req)).await.unwrap()
 	}
 	pub(crate) async fn new_atom(&self, data: Vec<u8>, id: api::AtomId) -> AtomHand {
 		let mut owned_g = self.0.owned_atoms.write().await;
@@ -124,10 +129,10 @@ impl System {
 	}
 	pub(crate) fn drop_atom(&self, dropped_atom_id: api::AtomId) {
 		let this = self.0.clone();
-		(self.0.ctx.spawn)(Box::pin(async move {
-			this.ext.reqnot().request(api::AtomDrop(this.id, dropped_atom_id)).await;
+		let _ = self.0.ctx.spawn(Box::pin(async move {
+			this.ext.client().request(api::AtomDrop(this.id, dropped_atom_id)).await.unwrap();
 			this.owned_atoms.write().await.remove(&dropped_atom_id);
-		}))
+		}));
 	}
 	#[must_use]
 	pub fn downgrade(&self) -> WeakSystem {
@@ -137,7 +142,7 @@ impl System {
 	pub(crate) async fn name_resolver(
 		&self,
 		orig: api::ParsedConstId,
-	) -> impl AsyncFnMut(&[Tok<String>]) -> OrcRes<VName> + use<> {
+	) -> impl AsyncFnMut(&[IStr]) -> OrcRes<VName> + use<> {
 		let root = self.0.ctx.root.read().await.upgrade().expect("find_names when root not in context");
 		let orig = self.0.const_paths.get(&orig).expect("origin for find_names invalid").clone();
 		let ctx = self.0.ctx.clone();
@@ -155,7 +160,7 @@ impl System {
 				Some(Ok(dest)) => return Ok(dest.target.to_vname().suffix(tail.iter().cloned())),
 				Some(Err(dests)) =>
 					return Err(mk_errv_floating(
-						ctx.i.i("Ambiguous name").await,
+						is("Ambiguous name").await,
 						format!(
 							"{selector} could refer to {}",
 							dests.iter().map(|ri| &ri.target).display("or")
@@ -170,7 +175,7 @@ impl System {
 				return Ok(VPath::new(cwd.iter().cloned()).name_with_suffix(selector.clone()));
 			}
 			Err(mk_errv_floating(
-				ctx.i.i("Invalid name").await,
+				is("Invalid name").await,
 				format!("{selector} doesn't refer to a module"),
 			))
 		}
@@ -203,8 +208,7 @@ impl SystemCtor {
 	#[must_use]
 	pub fn name(&self) -> &str { &self.decl.name }
 	pub async fn name_tok(&self) -> Sym {
-		(Sym::parse(&self.decl.name, &self.ext.upgrade().expect("ext dropped early").ctx().i).await)
-			.expect("System cannot have empty name")
+		(Sym::parse(&self.decl.name).await).expect("System cannot have empty name")
 	}
 	#[must_use]
 	pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
@@ -220,17 +224,17 @@ impl SystemCtor {
 		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
 		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
 		let id = ext.ctx().next_sys_id();
-		let sys_inst = ext.reqnot().request(api::NewSystem { depends, id, system: self.decl.id }).await;
+		let sys_inst =
+			ext.client().request(api::NewSystem { depends, id, system: self.decl.id }).await.unwrap();
 		let data = System(Rc::new(SystemInstData {
 			deps,
 			decl_id: self.decl.id,
 			ext: ext.clone(),
 			ctx: ext.ctx().clone(),
 			lex_filter: sys_inst.lex_filter,
-			line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
-				.await,
+			line_types: join_all(sys_inst.line_types.iter().map(|m| es(*m))).await,
 			id,
-			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok, &ext.ctx().i))).await,
+			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok))).await,
 			owned_atoms: RwLock::new(HashMap::new()),
 			const_paths: MemoMap::new(),
 		}));

orchid-host/src/tree.rs

@@ -13,11 +13,11 @@ use hashbrown::hash_map::Entry;
 use itertools::Itertools;
 use memo_map::MemoMap;
 use orchid_base::clone;
-use orchid_base::error::{OrcRes, Reporter, mk_errv};
-use orchid_base::interner::Tok;
+use orchid_base::error::{OrcRes, mk_errv, report};
+use orchid_base::interner::{IStr, IStrv, es, is, iv};
 use orchid_base::location::{CodeGenInfo, Pos};
 use orchid_base::name::{NameLike, Sym, VPath};
-use orchid_base::reqnot::Requester;
+use orchid_base::reqnot::ClientExt;
 
 use crate::api;
 use crate::ctx::Ctx;
@@ -45,7 +45,7 @@ impl Root {
 	#[must_use]
 	pub async fn from_api(api: api::Module, sys: &System) -> Self {
 		let consts = MemoMap::new();
-		let mut tfac = TreeFromApiCtx { consts: &consts, path: sys.i().i(&[][..]).await, sys };
+		let mut tfac = TreeFromApiCtx { consts: &consts, path: iv(&[][..]).await, sys };
 		let root = Module::from_api(api, &mut tfac).await;
 		Root(Rc::new(RwLock::new(RootData { root, consts, ctx: sys.ctx().clone() })))
 	}
@@ -60,7 +60,7 @@ impl Root {
 		Ok(Self(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() }))))
 	}
 	#[must_use]
-	pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym, rep: &Reporter) -> Self {
+	pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym) -> Self {
 		let mut ref_this = self.0.write().await;
 		let this = &mut *ref_this;
 		let mut deferred_consts = HashMap::new();
@@ -72,7 +72,6 @@ impl Root {
 			pars_prefix: pars_prefix.clone(),
 			root: &this.root,
 			ctx: &this.ctx,
-			rep,
 		};
 		let mut module = Module::from_parsed(parsed, pars_prefix.clone(), &mut tfpctx).await;
 		for step in pars_prefix.iter().rev() {
@@ -89,7 +88,7 @@ impl Root {
 		*this.ctx.root.write().await = new.downgrade();
 		for (path, (sys_id, pc_id)) in deferred_consts {
 			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
-			let api_expr = sys.reqnot().request(api::FetchParsedConst(sys.id(), pc_id)).await;
+			let api_expr = sys.client().request(api::FetchParsedConst(sys.id(), pc_id)).await.unwrap();
 			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), this.ctx.clone()).await;
 			new.0.write().await.consts.insert(path, expr);
 		}
@@ -110,7 +109,7 @@ impl Root {
 		}
 		match module {
 			Ok(_) => Err(mk_errv(
-				ctx.i.i("module used as constant").await,
+				is("module used as constant").await,
 				format!("{name} is a module, not a constant"),
 				[pos],
 			)),
@@ -118,7 +117,7 @@ impl Root {
 				ChildErrorKind::Private => panic!("public_only is false"),
 				ChildErrorKind::Constant => panic!("Tree refers to constant not in table"),
 				ChildErrorKind::Missing => Err(mk_errv(
-					ctx.i.i("Constant does not exist").await,
+					is("Constant does not exist").await,
 					format!("{name} does not refer to a constant"),
 					[pos],
 				)),
@@ -144,12 +143,12 @@ impl Default for WeakRoot {
 pub struct TreeFromApiCtx<'a> {
 	pub sys: &'a System,
 	pub consts: &'a MemoMap<Sym, Expr>,
-	pub path: Tok<Vec<Tok<String>>>,
+	pub path: IStrv,
 }
 impl<'a> TreeFromApiCtx<'a> {
 	#[must_use]
-	pub async fn push<'c>(&'c self, name: Tok<String>) -> TreeFromApiCtx<'c> {
-		let path = self.sys.ctx().i.i(&self.path.iter().cloned().chain([name]).collect_vec()).await;
+	pub async fn push<'c>(&'c self, name: IStr) -> TreeFromApiCtx<'c> {
+		let path = iv(&self.path.iter().cloned().chain([name]).collect_vec()).await;
 		TreeFromApiCtx { path, consts: self.consts, sys: self.sys }
 	}
 }
@@ -162,17 +161,17 @@ pub struct ResolvedImport {
 
 #[derive(Clone, Default)]
 pub struct Module {
-	pub imports: HashMap<Tok<String>, Result<ResolvedImport, Vec<ResolvedImport>>>,
-	pub members: HashMap<Tok<String>, Rc<Member>>,
+	pub imports: HashMap<IStr, Result<ResolvedImport, Vec<ResolvedImport>>>,
+	pub members: HashMap<IStr, Rc<Member>>,
 }
 impl Module {
 	#[must_use]
 	pub async fn from_api(api: api::Module, ctx: &mut TreeFromApiCtx<'_>) -> Self {
 		let mut members = HashMap::new();
 		for mem in api.members {
-			let mem_name = ctx.sys.i().ex(mem.name).await;
+			let mem_name = es(mem.name).await;
 			let vname = VPath::new(ctx.path.iter().cloned()).name_with_suffix(mem_name.clone());
-			let name = vname.to_sym(ctx.sys.i()).await;
+			let name = vname.to_sym().await;
 			let (lazy, kind) = match mem.kind {
 				api::MemberKind::Lazy(id) =>
 					(Some(LazyMemberHandle { id, sys: ctx.sys.id(), path: name.clone() }), None),
@@ -205,23 +204,23 @@ impl Module {
 		let mut glob_imports_by_name = HashMap::<_, Vec<_>>::new();
 		for import in parsed.get_imports().into_iter().filter(|i| i.name.is_none()) {
 			let pos = import.sr.pos();
-			match absolute_path(&path, &import.path, &ctx.ctx.i).await {
-				Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, pos, &import.path.to_string()).await),
+			match absolute_path(&path, &import.path).await {
+				Err(e) => report(e.err_obj(pos, &import.path.to_string()).await),
 				Ok(abs_path) => {
 					let names_res = match abs_path.strip_prefix(&ctx.pars_prefix[..]) {
 						None => {
 							let mut tree_ctx = (ctx.ctx.clone(), ctx.consts);
-							resolv_glob(&path, ctx.root, &abs_path, pos, &ctx.ctx.i, &mut tree_ctx).await
+							resolv_glob(&path, ctx.root, &abs_path, pos, &mut tree_ctx).await
 						},
 						Some(sub_tgt) => {
 							let sub_path = (path.strip_prefix(&ctx.pars_prefix[..]))
 								.expect("from_parsed called with path outside pars_prefix");
-							resolv_glob(sub_path, ctx.pars_root, sub_tgt, pos, &ctx.ctx.i, &mut ()).await
+							resolv_glob(sub_path, ctx.pars_root, sub_tgt, pos, &mut ()).await
 						},
 					};
-					let abs_path = abs_path.to_sym(&ctx.ctx.i).await;
+					let abs_path = abs_path.to_sym().await;
 					match names_res {
-						Err(e) => ctx.rep.report(e),
+						Err(e) => report(e),
 						Ok(names) =>
 							for name in names {
 								match glob_imports_by_name.entry(name) {
@@ -244,30 +243,28 @@ impl Module {
 						prelude_item.last_seg(),
 						Ok(ResolvedImport {
 							target: prelude_item,
-							pos: CodeGenInfo::new_details(sys.ctor().name_tok().await, "In prelude", &ctx.ctx.i)
-								.await
-								.pos(),
+							pos: CodeGenInfo::new_details(sys.ctor().name_tok().await, "In prelude").await.pos(),
 						}),
 					);
 				}
 			}
 		}
-		let conflicting_imports_msg = ctx.ctx.i.i("Conflicting imports").await;
+		let conflicting_imports_msg = is("Conflicting imports").await;
 		for (key, values) in imports_by_name {
 			if values.len() == 1 {
 				let import = values.into_iter().next().unwrap();
 				let sr = import.sr.clone();
-				let abs_path_res = absolute_path(&path, &import.clone().mspath(), &ctx.ctx.i).await;
+				let abs_path_res = absolute_path(&path, &import.clone().mspath()).await;
 				match abs_path_res {
-					Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, sr.pos(), &import.to_string()).await),
+					Err(e) => report(e.err_obj(sr.pos(), &import.to_string()).await),
 					Ok(abs_path) => {
-						let target = abs_path.to_sym(&ctx.ctx.i).await;
+						let target = abs_path.to_sym().await;
 						imports.insert(key, Ok(ResolvedImport { target, pos: sr.pos() }));
 					},
 				}
 			} else {
 				for item in values {
-					ctx.rep.report(mk_errv(
+					report(mk_errv(
 						conflicting_imports_msg.clone(),
 						format!("{key} is imported multiple times from different modules"),
 						[item.sr.pos()],
@@ -277,12 +274,11 @@ impl Module {
 		}
 		for (key, values) in glob_imports_by_name {
 			if !imports.contains_key(&key) {
-				let i = &ctx.ctx.i;
 				let values = stream::iter(values)
 					.then(|(n, sr)| {
 						clone!(key; async move {
 							ResolvedImport {
-								target: n.to_vname().suffix([key.clone()]).to_sym(i).await,
+								target: n.to_vname().suffix([key.clone()]).to_sym().await,
 								pos: sr.pos(),
 							}
 						})
@@ -292,12 +288,12 @@ impl Module {
 				imports.insert(key, if values.len() == 1 { Ok(values[0].clone()) } else { Err(values) });
 			}
 		}
-		let self_referential_msg = ctx.ctx.i.i("Self-referential import").await;
+		let self_referential_msg = is("Self-referential import").await;
 		for (key, value) in imports.iter() {
 			let Ok(import) = value else { continue };
 			if import.target.strip_prefix(&path[..]).is_some_and(|t| t.starts_with(slice::from_ref(key)))
 			{
-				ctx.rep.report(mk_errv(
+				report(mk_errv(
 					self_referential_msg.clone(),
 					format!("import {} points to itself or a path within itself", &import.target),
 					[import.pos.clone()],
@@ -308,7 +304,7 @@ impl Module {
 		for item in &parsed.items {
 			match &item.kind {
 				ItemKind::Member(mem) => {
-					let path = path.to_vname().suffix([mem.name.clone()]).to_sym(&ctx.ctx.i).await;
+					let path = path.to_vname().suffix([mem.name.clone()]).to_sym().await;
 					let kind = OnceCell::from(MemberKind::from_parsed(&mem.kind, path.clone(), ctx).await);
 					members.insert(
 						mem.name.clone(),
@@ -385,7 +381,6 @@ pub struct FromParsedCtx<'a> {
 	pars_prefix: Sym,
 	pars_root: &'a ParsedModule,
 	root: &'a Module,
-	rep: &'a Reporter,
 	ctx: &'a Ctx,
 	consts: &'a MemoMap<Sym, Expr>,
 	deferred_consts: &'a mut HashMap<Sym, (api::SysId, api::ParsedConstId)>,
@@ -395,7 +390,7 @@ impl Tree for Module {
 	type Ctx<'a> = (Ctx, &'a MemoMap<Sym, Expr>);
 	async fn child(
 		&self,
-		key: Tok<String>,
+		key: IStr,
 		public_only: bool,
 		(ctx, consts): &mut Self::Ctx<'_>,
 	) -> crate::dealias::ChildResult<'_, Self> {
@@ -410,7 +405,7 @@ impl Tree for Module {
 			MemberKind::Const => Err(ChildErrorKind::Constant),
 		}
 	}
-	fn children(&self, public_only: bool) -> hashbrown::HashSet<Tok<String>> {
+	fn children(&self, public_only: bool) -> hashbrown::HashSet<IStr> {
 		self.members.iter().filter(|(_, v)| !public_only || v.public).map(|(k, _)| k.clone()).collect()
 	}
 }

orchid-std/Cargo.toml

@@ -3,11 +3,19 @@ name = "orchid-std"
 version = "0.1.0"
 edition = "2024"
 
+[[bin]]
+name = "orchid-std"
+path = "src/main.rs"
+
+[lib]
+crate-type = ["cdylib", "lib"]
+path = "src/lib.rs"
+
 [dependencies]
 async-fn-stream = { version = "0.1.0", path = "../async-fn-stream" }
 async-once-cell = "0.5.4"
 futures = { version = "0.3.31", features = ["std"], default-features = false }
-hashbrown = "0.16.0"
+hashbrown = "0.16.1"
 itertools = "0.14.0"
 never = "0.1.0"
 once_cell = "1.21.3"
@@ -18,12 +26,12 @@ orchid-base = { version = "0.1.0", path = "../orchid-base" }
 orchid-extension = { version = "0.1.0", path = "../orchid-extension", features = [
 	"tokio",
 ] }
-ordered-float = "5.0.0"
-pastey = "0.1.1"
-rust_decimal = "1.38.0"
+ordered-float = "5.1.0"
+pastey = "0.2.1"
+rust_decimal = "1.39.0"
 subslice-offset = "0.1.1"
 substack = "1.1.1"
-tokio = { version = "1.47.1", features = ["full"] }
+tokio = { version = "1.49.0", features = ["full"] }
 
 [dev-dependencies]
-test_executors = "0.3.5"
+test_executors = "0.4.1"

orchid-std/src/lib.rs

@@ -11,3 +11,12 @@ pub use std::tuple::{HomoTpl, Tpl, Tuple, UntypedTuple};
 pub use macros::macro_system::MacroSystem;
 pub use macros::mactree::{MacTok, MacTree};
 use orchid_api as api;
+use orchid_extension::binary::orchid_extension_main_body;
+use orchid_extension::entrypoint::ExtensionBuilder;
+
+pub extern "C" fn orchid_extension_main(cx: api::binary::ExtensionContext) {
+	orchid_extension_main_body(
+		cx,
+		ExtensionBuilder::new("orchid-std::main").system(StdSystem).system(MacroSystem),
+	);
+}

orchid-std/src/macros/instantiate_tpl.rs

@@ -4,7 +4,6 @@ use never::Never;
 use orchid_base::format::fmt;
 use orchid_extension::atom::{Atomic, TAtom};
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
-use orchid_extension::context::i;
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::exec;
 use orchid_extension::expr::Expr;
@@ -38,7 +37,7 @@ impl OwnedAtom for InstantiateTplCall {
 	async fn call(mut self, arg: Expr) -> GExpr {
 		exec(async move |mut h| {
 			match h.exec::<TAtom<MacTree>>(arg.clone()).await {
-				Err(_) => panic!("Expected a macro param, found {}", fmt(&arg, &i()).await),
+				Err(_) => panic!("Expected a macro param, found {}", fmt(&arg).await),
 				Ok(t) => self.argv.push(own(&t).await),
 			};
 			if self.argv.len() < self.argc {

orchid-std/src/macros/let_line.rs

@@ -3,15 +3,13 @@ use std::pin::pin;
 use futures::{FutureExt, StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::error::{OrcRes, Reporter};
+use orchid_base::error::{OrcRes, report, with_reporter};
+use orchid_base::interner::is;
 use orchid_base::name::Sym;
-use orchid_base::parse::{
-	Comment, ParseCtx, Parsed, Snippet, expect_tok, token_errv, try_pop_no_fluff,
-};
+use orchid_base::parse::{Comment, Parsed, Snippet, expect_tok, token_errv, try_pop_no_fluff};
 use orchid_base::sym;
 use orchid_base::tree::Paren;
 use orchid_extension::atom::TAtom;
-use orchid_extension::context::i;
 use orchid_extension::conv::TryFromExpr;
 use orchid_extension::gen_expr::{atom, call, sym_ref};
 use orchid_extension::parser::{ConstCtx, PSnippet, PTok, PTokTree, ParsCtx, ParsedLine, Parser};
@@ -24,40 +22,36 @@ pub struct LetLine;
 impl Parser for LetLine {
 	const LINE_HEAD: &'static str = "let";
 	async fn parse<'a>(
-		ctx: ParsCtx<'a>,
+		_: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
 		line: PSnippet<'a>,
 	) -> OrcRes<Vec<ParsedLine>> {
 		let sr = line.sr();
-		let Parsed { output: name_tok, tail } = try_pop_no_fluff(&ctx, line).await?;
+		let Parsed { output: name_tok, tail } = try_pop_no_fluff(line).await?;
 		let Some(name) = name_tok.as_name() else {
-			let err = token_errv(&ctx, name_tok, "Constant must have a name", |t| {
+			let err = token_errv(name_tok, "Constant must have a name", |t| {
 				format!("Expected a name but found {t}")
 			});
 			return Err(err.await);
 		};
-		let Parsed { tail, .. } = expect_tok(&ctx, tail, ctx.i().i("=").await).await?;
-		let aliased = parse_tokv(tail, &ctx).await;
+		let Parsed { tail, .. } = expect_tok(tail, is("=").await).await?;
+		let aliased = parse_tokv(tail).await;
 		Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name, async move |ctx| {
-			let rep = Reporter::new();
 			let macro_input =
-				MacTok::S(Paren::Round, dealias_mac_v(&aliased, &ctx, &rep).await).at(sr.pos());
-			if let Some(e) = rep.errv() {
-				return Err(e);
-			}
-			Ok(call(sym_ref(sym!(macros::resolve; i())), [atom(macro_input)]))
+				MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&aliased, &ctx)).await?).at(sr.pos());
+			Ok(call(sym_ref(sym!(macros::resolve)), [atom(macro_input)]))
 		})])
 	}
 }
 
-pub async fn dealias_mac_v(aliased: &MacTreeSeq, ctx: &ConstCtx, rep: &Reporter) -> MacTreeSeq {
+pub async fn dealias_mac_v(aliased: &MacTreeSeq, ctx: &ConstCtx) -> MacTreeSeq {
 	let keys = aliased.glossary().iter().cloned().collect_vec();
 	let mut names: HashMap<_, _> = HashMap::new();
 	let mut stream = pin!(ctx.names(&keys).zip(stream::iter(&keys)));
 	while let Some((canonical, local)) = stream.next().await {
 		match canonical {
-			Err(e) => rep.report(e),
+			Err(e) => report(e),
 			Ok(name) => {
 				names.insert(local.clone(), name);
 			},
@@ -69,16 +63,16 @@ pub async fn dealias_mac_v(aliased: &MacTreeSeq, ctx: &ConstCtx, rep: &Reporter)
 	})
 }
 
-pub async fn parse_tokv(line: PSnippet<'_>, ctx: &impl ParseCtx) -> MacTreeSeq {
+pub async fn parse_tokv(line: PSnippet<'_>) -> MacTreeSeq {
 	if let Some((idx, arg)) = line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?))) {
 		let (head, lambda) = line.split_at(idx as u32);
 		let (_, body) = lambda.pop_front().unwrap();
-		let body = parse_tokv(body, ctx).boxed_local().await;
-		let mut all = parse_tokv_no_lambdas(&head, ctx).await;
-		match parse_tok(arg, ctx).await {
+		let body = parse_tokv(body).boxed_local().await;
+		let mut all = parse_tokv_no_lambdas(&head).await;
+		match parse_tok(arg).await {
 			Some(arg) => all.push(MacTok::Lambda(arg, body).at(lambda.sr().pos())),
-			None => ctx.rep().report(
-				token_errv(ctx, arg, "Lambda argument fluff", |arg| {
+			None => report(
+				token_errv(arg, "Lambda argument fluff", |arg| {
 					format!("Lambda arguments must be a valid token, found meaningless fragment {arg}")
 				})
 				.await,
@@ -86,29 +80,29 @@ pub async fn parse_tokv(line: PSnippet<'_>, ctx: &impl ParseCtx) -> MacTreeSeq {
 		};
 		MacTreeSeq::new(all)
 	} else {
-		MacTreeSeq::new(parse_tokv_no_lambdas(&line, ctx).await)
+		MacTreeSeq::new(parse_tokv_no_lambdas(&line).await)
 	}
 }
 
-async fn parse_tokv_no_lambdas(line: &[PTokTree], ctx: &impl ParseCtx) -> Vec<MacTree> {
-	stream::iter(line).filter_map(|tt| parse_tok(tt, ctx)).collect::<Vec<_>>().await
+async fn parse_tokv_no_lambdas(line: &[PTokTree]) -> Vec<MacTree> {
+	stream::iter(line).filter_map(parse_tok).collect::<Vec<_>>().await
 }
 
-pub async fn parse_tok(tree: &PTokTree, ctx: &impl ParseCtx) -> Option<MacTree> {
+pub async fn parse_tok(tree: &PTokTree) -> Option<MacTree> {
 	let tok = match &tree.tok {
 		PTok::Bottom(errv) => MacTok::Bottom(errv.clone()),
 		PTok::BR | PTok::Comment(_) => return None,
-		PTok::Name(n) => MacTok::Name(Sym::new([n.clone()], ctx.i()).await.unwrap()),
+		PTok::Name(n) => MacTok::Name(Sym::new([n.clone()]).await.unwrap()),
 		PTok::NS(..) => match tree.as_multiname() {
-			Ok(mn) => MacTok::Name(mn.to_sym(ctx.i()).await),
+			Ok(mn) => MacTok::Name(mn.to_sym().await),
 			Err(nested) => {
-				ctx.rep().report(
-					token_errv(ctx, tree, ":: can only be followed by a name in an expression", |tok| {
+				report(
+					token_errv(tree, ":: can only be followed by a name in an expression", |tok| {
 						format!("Expected name, found {tok}")
 					})
 					.await,
 				);
-				return parse_tok(nested, ctx).boxed_local().await;
+				return parse_tok(nested).boxed_local().await;
 			},
 		},
 		PTok::Handle(expr) => match TAtom::<PhAtom>::try_from_expr(expr.clone()).await {
@@ -117,8 +111,7 @@ pub async fn parse_tok(tree: &PTokTree, ctx: &impl ParseCtx) -> Option<MacTree>
 		},
 		PTok::NewExpr(never) => match *never {},
 		PTok::LambdaHead(_) => panic!("Lambda-head handled in the sequence parser"),
-		PTok::S(p, body) =>
-			MacTok::S(*p, parse_tokv(Snippet::new(tree, body), ctx).boxed_local().await),
+		PTok::S(p, body) => MacTok::S(*p, parse_tokv(Snippet::new(tree, body)).boxed_local().await),
 	};
 	Some(tok.at(tree.sr().pos()))
 }

orchid-std/src/macros/macro_lib.rs

@@ -1,14 +1,15 @@
 use orchid_base::sym;
 use orchid_extension::atom::TAtom;
 use orchid_extension::atom_owned::own;
-use orchid_extension::context::i;
 use orchid_extension::conv::ToExpr;
+use orchid_extension::coroutine_exec::exec;
 use orchid_extension::gen_expr::{call, sym_ref};
 use orchid_extension::tree::{GenMember, fun, prefix};
 
 use crate::macros::mactree::MacTree;
 use crate::macros::resolve::resolve;
 use crate::macros::utils::{build_macro, mactree, mactreev};
+use crate::{HomoTpl, UntypedTuple};
 
 pub async fn gen_macro_lib() -> Vec<GenMember> {
 	prefix("macros", [
@@ -18,47 +19,49 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 			build_macro(None, ["..", "_"]).finish(),
 			build_macro(Some(1), ["+"])
 				.rule(mactreev!("...$" lhs 0 macros::common::+ "...$" rhs 1), [async |[lhs, rhs]| {
-					call(sym_ref(sym!(std::number::add; i())), [resolve(lhs).await, resolve(rhs).await])
+					call(sym_ref(sym!(std::number::add)), [resolve(lhs).await, resolve(rhs).await])
 				}])
 				.finish(),
 			build_macro(Some(2), ["*"])
 				.rule(mactreev!("...$" lhs 0 macros::common::* "...$" rhs 1), [async |[lhs, rhs]| {
-					call(sym_ref(sym!(std::number::mul; i())), [resolve(lhs).await, resolve(rhs).await])
+					call(sym_ref(sym!(std::number::mul)), [resolve(lhs).await, resolve(rhs).await])
 				}])
 				.finish(),
 			build_macro(None, ["comma_list", ","])
 				.rule(
 					mactreev!(macros::common::comma_list ( "...$" head 0  macros::common::, "...$" tail 1)),
 					[async |[head, tail]| {
-						call(sym_ref(sym!(std::tuple::cat; i())), [
-							call(sym_ref(sym!(std::tuple::one; i())), [head.to_gen().await]),
-							resolve(mactree!(macros::common::comma_list "push" tail ;)).await,
-						])
+						exec(async |mut h| {
+							let recur = resolve(mactree!(macros::common::comma_list "push" tail ;)).await;
+							let mut tail = h.exec::<HomoTpl<TAtom<MacTree>>>(recur).await?;
+							tail.0.insert(0, h.exec(head).await?);
+							Ok(tail)
+						})
+						.await
 					}],
 				)
 				.rule(mactreev!(macros::common::comma_list ( "...$" final_tail 0 )), [async |[tail]| {
-					call(sym_ref(sym!(std::tuple::one; i())), [tail.to_gen().await])
-				}])
-				.rule(mactreev!(macros::common::comma_list()), [async |[]| {
-					sym_ref(sym!(std::tuple::empty; i()))
+					HomoTpl(vec![tail.to_gen().await])
 				}])
+				.rule(mactreev!(macros::common::comma_list()), [async |[]| UntypedTuple(Vec::new())])
 				.finish(),
 			build_macro(None, ["semi_list", ";"])
 				.rule(
 					mactreev!(macros::common::semi_list ( "...$" head 0  macros::common::; "...$" tail 1)),
 					[async |[head, tail]| {
-						call(sym_ref(sym!(std::tuple::cat; i())), [
-							call(sym_ref(sym!(std::tuple::one; i())), [head.to_gen().await]),
-							resolve(mactree!(macros::common::semi_list "push" tail ;)).await,
-						])
+						exec(async |mut h| {
+							let recur = resolve(mactree!(macros::common::semi_list "push" tail ;)).await;
+							let mut tail = h.exec::<HomoTpl<TAtom<MacTree>>>(recur).await?;
+							tail.0.insert(0, h.exec(head).await?);
+							Ok(tail)
+						})
+						.await
 					}],
 				)
 				.rule(mactreev!(macros::common::semi_list ( "...$" final_tail 0 )), [async |[tail]| {
-					call(sym_ref(sym!(std::tuple::one; i())), [tail.to_gen().await])
-				}])
-				.rule(mactreev!(macros::common::semi_list()), [async |[]| {
-					sym_ref(sym!(std::tuple::empty; i()))
+					HomoTpl(vec![tail.to_gen().await])
 				}])
+				.rule(mactreev!(macros::common::semi_list()), [async |[]| UntypedTuple(Vec::new())])
 				.finish(),
 		]),
 	])

orchid-std/src/macros/macro_line.rs

@@ -1,20 +1,19 @@
-use std::cell::RefCell;
 use std::rc::Rc;
 
+use async_fn_stream::stream;
 use async_once_cell::OnceCell;
-use futures::{StreamExt, stream};
+use futures::StreamExt;
 use itertools::Itertools;
-use orchid_base::error::{OrcRes, Reporter, mk_errv};
+use orchid_base::error::{OrcRes, mk_errv, report, with_reporter};
+use orchid_base::interner::is;
 use orchid_base::parse::{
-	Comment, ParseCtx, Parsed, Snippet, expect_end, expect_tok, line_items, token_errv,
-	try_pop_no_fluff,
+	Comment, Parsed, Snippet, expect_end, expect_tok, line_items, token_errv, try_pop_no_fluff,
 };
 use orchid_base::tree::{Paren, Token};
 use orchid_base::{clone, sym};
 use orchid_extension::atom::TAtom;
-use orchid_extension::context::i;
 use orchid_extension::conv::{ToExpr, TryFromExpr};
-use orchid_extension::gen_expr::{atom, call, sym_ref};
+use orchid_extension::gen_expr::{call, sym_ref};
 use orchid_extension::parser::{PSnippet, ParsCtx, ParsedLine, Parser};
 
 use crate::macros::let_line::{dealias_mac_v, parse_tokv};
@@ -35,22 +34,22 @@ impl Parser for MacroLine {
 	) -> OrcRes<Vec<ParsedLine>> {
 		if exported {
 			return Err(mk_errv(
-				ctx.i().i("macros are always exported").await,
+				is("macros are always exported").await,
 				"The export keyword is forbidden here to avoid confusion\n\
 					because macros are exported by default",
 				[line.sr()],
 			));
 		}
 		let module = ctx.module();
-		let Parsed { output: prio_or_body, tail } = try_pop_no_fluff(&ctx, line).await?;
+		let Parsed { output: prio_or_body, tail } = try_pop_no_fluff(line).await?;
 		let bad_first_item_err = || {
-			token_errv(&ctx, prio_or_body, "Expected priority or block", |s| {
+			token_errv(prio_or_body, "Expected priority or block", |s| {
 				format!("Expected a priority number or a () block, found {s}")
 			})
 		};
 		let (prio, body) = match &prio_or_body.tok {
 			Token::S(Paren::Round, body) => {
-				expect_end(&ctx, tail).await?;
+				expect_end(tail).await?;
 				(None, body)
 			},
 			Token::Handle(expr) => match TAtom::<Int>::try_from_expr(expr.clone()).await {
@@ -58,33 +57,32 @@ impl Parser for MacroLine {
 					return Err(e + bad_first_item_err().await);
 				},
 				Ok(prio) => {
-					let Parsed { output: body, tail } = try_pop_no_fluff(&ctx, tail).await?;
+					let Parsed { output: body, tail } = try_pop_no_fluff(tail).await?;
 					let Token::S(Paren::Round, block) = &body.tok else {
 						return Err(
-							token_errv(&ctx, prio_or_body, "Expected () block", |s| {
+							token_errv(prio_or_body, "Expected () block", |s| {
 								format!("Expected a () block, found {s}")
 							})
 							.await,
 						);
 					};
-					expect_end(&ctx, tail).await?;
+					expect_end(tail).await?;
 					(Some(prio), block)
 				},
 			},
 			_ => return Err(bad_first_item_err().await),
 		};
-		let lines = line_items(&ctx, Snippet::new(prio_or_body, body)).await;
+		let lines = line_items(Snippet::new(prio_or_body, body)).await;
 		let Some((kw_line, rule_lines)) = lines.split_first() else { return Ok(Vec::new()) };
 		let mut keywords = Vec::new();
-		let Parsed { tail: kw_tail, .. } =
-			expect_tok(&ctx, kw_line.tail, ctx.i().i("keywords").await).await?;
+		let Parsed { tail: kw_tail, .. } = expect_tok(kw_line.tail, is("keywords").await).await?;
 		for kw_tok in kw_tail.iter().filter(|kw| !kw.is_fluff()) {
 			match kw_tok.as_name() {
 				Some(kw) => {
 					keywords.push((kw, kw_tok.sr()));
 				},
-				None => ctx.rep().report(
-					token_errv(&ctx, kw_tok, "invalid macro keywords list", |tok| {
+				None => report(
+					token_errv(kw_tok, "invalid macro keywords list", |tok| {
 						format!("The keywords list must be a sequence of names; received {tok}")
 					})
 					.await,
@@ -93,7 +91,7 @@ impl Parser for MacroLine {
 		}
 		let Some((macro_name, _)) = keywords.first().cloned() else {
 			return Err(mk_errv(
-				ctx.i().i("macro with no keywords").await,
+				is("macro with no keywords").await,
 				"Macros must define at least one macro of their own.",
 				[kw_line.tail.sr()],
 			));
@@ -102,18 +100,18 @@ impl Parser for MacroLine {
 		let mut lines = Vec::new();
 		for (idx, line) in rule_lines.iter().enumerate().map(|(n, v)| (n as u32, v)) {
 			let sr = line.tail.sr();
-			let name = ctx.i().i(&format!("rule::{}::{}", macro_name, idx)).await;
-			let Parsed { tail, .. } = expect_tok(&ctx, line.tail, ctx.i().i("rule").await).await?;
-			let arrow_token = ctx.i().i("=>").await;
+			let name = is(&format!("rule::{}::{}", macro_name, idx)).await;
+			let Parsed { tail, .. } = expect_tok(line.tail, is("rule").await).await?;
+			let arrow_token = is("=>").await;
 			let Some((pattern, body)) = tail.split_once(|tok| tok.is_kw(arrow_token.clone())) else {
-				ctx.rep().report(mk_errv(
-					ctx.i().i("Missing => in rule").await,
+				report(mk_errv(
+					is("Missing => in rule").await,
 					"Rule lines are of the form `rule ...pattern => ...body`",
 					[line.tail.sr()],
 				));
 				continue;
 			};
-			let pattern = parse_tokv(pattern, &ctx).await;
+			let pattern = parse_tokv(pattern).await;
 			let mut placeholders = Vec::new();
 			pattern.map(&mut false, &mut |tok| {
 				if let MacTok::Ph(ph) = tok.tok() {
@@ -121,9 +119,9 @@ impl Parser for MacroLine {
 				}
 				None
 			});
-			let mut body_mactree = parse_tokv(body, &ctx).await;
+			let mut body_mactree = parse_tokv(body).await;
 			for (ph, ph_pos) in placeholders.iter().rev() {
-				let name = ctx.module().suffix([ph.name.clone()], ctx.i()).await;
+				let name = ctx.module().suffix([ph.name.clone()]).await;
 				body_mactree =
 					MacTreeSeq::new([
 						MacTok::Lambda(MacTok::Name(name).at(ph_pos.clone()), body_mactree).at(ph_pos.clone())
@@ -132,53 +130,42 @@ impl Parser for MacroLine {
 			let body_sr = body.sr();
 			rules.push((name.clone(), placeholders, pattern));
 			lines.push(ParsedLine::cnst(&sr, &line.output, true, name, async move |ctx| {
-				let rep = Reporter::new();
-				let body = dealias_mac_v(&body_mactree, &ctx, &rep).await;
-				let macro_input = MacTok::S(Paren::Round, body).at(body_sr.pos());
-				if let Some(e) = rep.errv() {
-					return Err(e);
-				}
-				Ok(call(sym_ref(sym!(macros::resolve; i())), [macro_input.to_gen().await]))
+				let macro_input =
+					MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&body_mactree, &ctx)).await?)
+						.at(body_sr.pos());
+				Ok(call(sym_ref(sym!(macros::resolve)), [macro_input.to_gen().await]))
 			}))
 		}
 		let mac_cell = Rc::new(OnceCell::new());
-		let rules = Rc::new(RefCell::new(Some(rules)));
+		let rules = Rc::new(rules);
 		for (kw, sr) in &*keywords {
 			clone!(mac_cell, rules, module, macro_name, prio);
-			lines.push(ParsedLine::cnst(&sr.clone(), &comments, true, kw.clone(), async move |cctx| {
-				let mac = mac_cell
-					.get_or_init(async {
-						let rep = Reporter::new();
-						let rules = rules.borrow_mut().take().expect("once cell initializer runs");
-						let rules = stream::iter(rules)
-							.then(|(body_name, placeholders, pattern_rel)| {
-								let cctx = &cctx;
-								let rep = &rep;
-								async move {
-									let pattern = dealias_mac_v(&pattern_rel, cctx, rep).await;
-									let pattern_res = Matcher::new(pattern.clone()).await;
-									let placeholders = placeholders.into_iter().map(|(ph, _)| ph.name).collect_vec();
-									match pattern_res {
-										Ok(matcher) => Some(Rule { body_name, matcher, pattern, placeholders }),
-										Err(e) => {
-											rep.report(e);
-											None
-										},
-									}
+			let kw_key = is(&format!("__macro__{kw}")).await;
+			lines.push(ParsedLine::cnst(&sr.clone(), &comments, true, kw_key, async move |cctx| {
+				let mac_future = async {
+					let rules = with_reporter(
+						stream(async |mut h| {
+							for (body, ph_names, pattern_rel) in rules.iter() {
+								let pattern = dealias_mac_v(pattern_rel, &cctx).await;
+								let ph_names = ph_names.iter().map(|(ph, _)| ph.name.clone()).collect_vec();
+								match Matcher::new(pattern.clone()).await {
+									Ok(matcher) =>
+										h.emit(Rule { body: body.clone(), matcher, pattern, ph_names }).await,
+									Err(e) => report(e),
 								}
-							})
-							.flat_map(stream::iter)
-							.collect::<Vec<_>>()
-							.await;
-						Macro(Rc::new(MacroData {
-							canonical_name: module.suffix([macro_name], &i()).await,
-							module,
-							prio: prio.map(|i| i.0 as u64),
-							rules,
-						}))
-					})
-					.await;
-				atom(mac.clone())
+							}
+						})
+						.collect::<Vec<_>>(),
+					)
+					.await?;
+					Ok(Macro(Rc::new(MacroData {
+						canonical_name: module.suffix([macro_name]).await,
+						module,
+						prio: prio.map(|i| i.0 as u64),
+						rules,
+					})))
+				};
+				mac_cell.get_or_init(mac_future).await.clone().to_gen().await
 			}))
 		}
 		Ok(lines)

orchid-std/src/macros/macro_system.rs

@@ -1,10 +1,8 @@
 use never::Never;
 use orchid_base::name::Sym;
-use orchid_base::reqnot::Receipt;
+use orchid_base::reqnot::{Receipt, ReqHandle};
 use orchid_base::sym;
 use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
-use orchid_extension::context::i;
-use orchid_extension::entrypoint::ExtReq;
 use orchid_extension::lexer::LexerObj;
 use orchid_extension::other_system::SystemHandle;
 use orchid_extension::parser::ParserObj;
@@ -24,14 +22,14 @@ use crate::macros::std_macros::gen_std_macro_lib;
 use crate::macros::utils::MacroBodyArgCollector;
 use crate::{MacTree, StdSystem};
 
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub struct MacroSystem;
 impl SystemCtor for MacroSystem {
 	type Deps = StdSystem;
 	type Instance = Self;
 	const NAME: &'static str = "orchid::macros";
 	const VERSION: f64 = 0.00_01;
-	fn inst(_: SystemHandle<StdSystem>) -> Self::Instance { Self }
+	fn inst(&self, _: SystemHandle<StdSystem>) -> Self::Instance { Self }
 }
 impl SystemCard for MacroSystem {
 	type Ctor = Self;
@@ -48,19 +46,19 @@ impl SystemCard for MacroSystem {
 	}
 }
 impl System for MacroSystem {
-	async fn request(_: ExtReq<'_>, req: Never) -> Receipt<'_> { match req {} }
+	async fn request<'a>(_: Box<dyn ReqHandle<'a> + 'a>, req: Never) -> Receipt<'a> { match req {} }
 	async fn prelude() -> Vec<Sym> {
 		vec![
-			sym!(macros::common::+; i()),
-			sym!(macros::common::*; i()),
-			sym!(macros::common::,; i()),
-			sym!(macros::common::;; i()),
-			sym!(macros::common::..; i()),
-			sym!(macros::common::_; i()),
-			sym!(std::tuple::t; i()),
-			sym!(pattern::match; i()),
-			sym!(pattern::ref; i()),
-			sym!(pattern::=>; i()),
+			sym!(macros::common::+),
+			sym!(macros::common::*),
+			sym!(macros::common::,),
+			sym!(macros::common::;),
+			sym!(macros::common::..),
+			sym!(macros::common::_),
+			sym!(std::tuple::t),
+			sym!(pattern::match),
+			sym!(pattern::ref),
+			sym!(pattern::=>),
 		]
 	}
 	fn lexers() -> Vec<LexerObj> { vec![&MacTreeLexer, &PhLexer] }

orchid-std/src/macros/macro_value.rs

@@ -2,7 +2,7 @@ use std::borrow::Cow;
 use std::rc::Rc;
 
 use never::Never;
-use orchid_base::interner::Tok;
+use orchid_base::interner::IStr;
 use orchid_base::name::Sym;
 use orchid_extension::atom::Atomic;
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
@@ -25,8 +25,8 @@ pub struct Macro(pub Rc<MacroData>);
 pub struct Rule {
 	pub pattern: MacTreeSeq,
 	pub matcher: Matcher,
-	pub placeholders: Vec<Tok<String>>,
-	pub body_name: Tok<String>,
+	pub ph_names: Vec<IStr>,
+	pub body: IStr,
 }
 impl Atomic for Macro {
 	type Data = ();

orchid-std/src/macros/mactree.rs

@@ -8,7 +8,7 @@ use hashbrown::HashSet;
 use orchid_api_derive::Coding;
 use orchid_base::error::OrcErrv;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
-use orchid_base::interner::Tok;
+use orchid_base::interner::IStr;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::tl_cache;
@@ -205,7 +205,7 @@ pub async fn mtreev_fmt<'b>(
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct Ph {
-	pub name: Tok<String>,
+	pub name: IStr,
 	pub kind: PhKind,
 }
 impl Display for Ph {

orchid-std/src/macros/mactree_lexer.rs

@@ -3,7 +3,7 @@ use std::ops::RangeInclusive;
 use futures::FutureExt;
 use itertools::chain;
 use orchid_base::error::{OrcRes, mk_errv};
-use orchid_base::parse::ParseCtx;
+use orchid_base::interner::is;
 use orchid_base::tokens::PARENS;
 use orchid_base::tree::Paren;
 use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
@@ -14,7 +14,7 @@ use crate::macros::instantiate_tpl::InstantiateTplCall;
 use crate::macros::let_line::parse_tok;
 use crate::macros::mactree::{MacTok, MacTree, MacTreeSeq};
 
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub struct MacTreeLexer;
 impl Lexer for MacTreeLexer {
 	const CHAR_FILTER: &'static [RangeInclusive<char>] = &['\''..='\''];
@@ -54,11 +54,9 @@ impl Lexer for MacTreeLexer {
 						let tok = MacTok::S(*paren, MacTreeSeq::new(items));
 						break Ok((tail3, tok.at(ctx.pos_tt(tail, tail3).pos())));
 					} else if tail2.is_empty() {
-						return Err(mk_errv(
-							ctx.i().i("Unclosed block").await,
-							format!("Expected closing {rp}"),
-							[ctx.pos_lt(1, tail)],
-						));
+						return Err(mk_errv(is("Unclosed block").await, format!("Expected closing {rp}"), [
+							ctx.pos_lt(1, tail),
+						]));
 					}
 					let (new_tail, new_item) = mac_tree(tail2, args, ctx).boxed_local().await?;
 					body_tail = new_tail;
@@ -87,7 +85,7 @@ impl Lexer for MacTreeLexer {
 				Ok((tail3, MacTok::Lambda(param, MacTreeSeq::new(body)).at(ctx.pos_tt(tail, tail3).pos())))
 			} else {
 				let (tail2, sub) = ctx.recurse(tail).await?;
-				let parsed = parse_tok(&sub, ctx).await.expect("Unexpected invalid token");
+				let parsed = parse_tok(&sub).await.expect("Unexpected invalid token");
 				Ok((tail2, parsed))
 			}
 		}

orchid-std/src/macros/match_macros.rs

@@ -8,11 +8,11 @@ use orchid_api::ExprTicket;
 use orchid_api_derive::Coding;
 use orchid_base::error::{OrcRes, mk_errv};
 use orchid_base::format::fmt;
+use orchid_base::interner::is;
 use orchid_base::name::Sym;
 use orchid_base::sym;
 use orchid_extension::atom::{Atomic, TAtom};
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant, own};
-use orchid_extension::context::i;
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::{ExecHandle, exec};
 use orchid_extension::expr::{Expr, ExprHandle};
@@ -43,7 +43,7 @@ impl MatcherData {
 	pub fn keys(&self) -> impl Stream<Item = Sym> {
 		stream(async |mut h| {
 			for tk in &self.keys {
-				h.emit(Sym::from_api(*tk, &i()).await).await
+				h.emit(Sym::from_api(*tk).await).await
 			}
 		})
 	}
@@ -85,7 +85,7 @@ pub async fn gen_match_macro_lib() -> Vec<GenMember> {
 			},
 		),
 		fun(true, "matcher", async |names: HomoTpl<TAtom<SymAtom>>, matcher: Expr| MatcherAtom {
-			keys: join_all(names.0.iter().map(async |atm| Sym::from_api(atm.0, &i()).await)).await,
+			keys: join_all(names.0.iter().map(async |atm| Sym::from_api(atm.0).await)).await,
 			matcher,
 		}),
 		build_macro(None, ["match", "match_rule", "_row", "=>"])
@@ -93,7 +93,7 @@ pub async fn gen_match_macro_lib() -> Vec<GenMember> {
 				async |[value, rules]| {
 					exec(async move |mut h| {
 						let rule_lines = h
-							.exec::<TAtom<Tuple>>(call(sym_ref(sym!(macros::resolve; i())), [
+							.exec::<TAtom<Tuple>>(call(sym_ref(sym!(macros::resolve)), [
 								mactree!(macros::common::semi_list "push" rules.clone();).to_gen().await,
 							]))
 							.await?;
@@ -105,20 +105,20 @@ pub async fn gen_match_macro_lib() -> Vec<GenMember> {
 								))
 								.await?;
 							let Tpl((matcher, body)) = h
-								.exec(call(sym_ref(sym!(macros::resolve; i())), [
+								.exec(call(sym_ref(sym!(macros::resolve)), [
 									mactree!(pattern::_row "push" own(&line_mac).await ;).to_gen().await,
 								]))
 								.await?;
 							rule_atoms.push((matcher, body));
 						}
 						let base_case = lambda(0, [bot(mk_errv(
-							i().i("No branches match").await,
+							is("No branches match").await,
 							"None of the patterns matches this value",
 							[rules.pos()],
 						))]);
 						let match_expr = stream::iter(rule_atoms.into_iter().rev())
 							.fold(base_case, async |tail, (mat, body)| {
-								lambda(0, [call(sym_ref(sym!(pattern::match_one; i())), [
+								lambda(0, [call(sym_ref(sym!(pattern::match_one)), [
 									mat.to_gen().await,
 									arg(0),
 									body.to_gen().await,
@@ -144,14 +144,14 @@ pub async fn gen_match_macro_lib() -> Vec<GenMember> {
 				async |[pattern, mut value]| {
 					exec(async move |mut h| -> OrcRes<Tpl<(TAtom<MatcherAtom>, GExpr)>> {
 						let Ok(pat) = h
-							.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve; i())), [
+							.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [
 								mactree!(pattern::match_rule "push" pattern.clone();).to_gen().await,
 							]))
 							.await
 						else {
 							return Err(mk_errv(
-								i().i("Invalid pattern").await,
-								format!("Could not parse {} as a match pattern", fmt(&pattern, &i()).await),
+								is("Invalid pattern").await,
+								format!("Could not parse {} as a match pattern", fmt(&pattern).await),
 								[pattern.pos()],
 							));
 						};
@@ -169,18 +169,18 @@ pub async fn gen_match_macro_lib() -> Vec<GenMember> {
 			.rule(mactreev!(pattern::match_rule(pattern::ref "$" name)), [async |[name]| {
 				let MacTok::Name(name) = name.tok() else {
 					return Err(mk_errv(
-						i().i("pattern 'ref' requires a name to bind to").await,
+						is("pattern 'ref' requires a name to bind to").await,
 						format!(
 							"'ref' was interpreted as a binding matcher, \
 							but it was followed by {} instead of a name",
-							fmt(&name, &i()).await
+							fmt(&name).await
 						),
 						[name.pos()],
 					));
 				};
 				Ok(MatcherAtom {
 					keys: vec![name.clone()],
-					matcher: sym_ref(sym!(pattern::ref_body; i())).to_expr().await,
+					matcher: sym_ref(sym!(pattern::ref_body)).to_expr().await,
 				})
 			}])
 			.finish(),

orchid-std/src/macros/ph_lexer.rs

@@ -1,10 +1,10 @@
 use orchid_api_derive::Coding;
 use orchid_base::error::{OrcRes, mk_errv};
 use orchid_base::format::FmtUnit;
+use orchid_base::interner::{es, is};
 use orchid_base::parse::{name_char, name_start};
 use orchid_extension::atom::Atomic;
 use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
-use orchid_extension::context::i;
 use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
 use orchid_extension::tree::{GenTokTree, x_tok};
 
@@ -13,7 +13,7 @@ use crate::macros::mactree::{Ph, PhKind};
 #[derive(Clone, Coding)]
 pub struct PhAtom(orchid_api::TStr, PhKind);
 impl PhAtom {
-	pub async fn to_full(&self) -> Ph { Ph { kind: self.1, name: i().ex(self.0).await } }
+	pub async fn to_full(&self) -> Ph { Ph { kind: self.1, name: es(self.0).await } }
 }
 impl Atomic for PhAtom {
 	type Data = Self;
@@ -21,11 +21,11 @@ impl Atomic for PhAtom {
 }
 impl ThinAtom for PhAtom {
 	async fn print(&self) -> FmtUnit {
-		Ph { name: i().ex(self.0).await, kind: self.1 }.to_string().into()
+		Ph { name: es(self.0).await, kind: self.1 }.to_string().into()
 	}
 }
 
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub struct PhLexer;
 impl Lexer for PhLexer {
 	const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['$'..='$', '.'..='.'];
@@ -52,7 +52,7 @@ impl Lexer for PhLexer {
 							(prio_num, tail)
 						} else {
 							return Err(mk_errv(
-								i().i("Invalid priority, must be 0-255").await,
+								is("Invalid priority, must be 0-255").await,
 								format!("{prio} is not a valid placeholder priority"),
 								[ctx.pos_lt(prio.len(), tail)],
 							));
@@ -71,7 +71,7 @@ impl Lexer for PhLexer {
 				return Err(err_not_applicable().await);
 			}
 		};
-		let ph_atom = PhAtom(i().i::<String>(name).await.to_api(), phkind);
+		let ph_atom = PhAtom(is(name).await.to_api(), phkind);
 		Ok((tail, x_tok(ph_atom).await.at(ctx.pos_tt(line, tail))))
 	}
 }

orchid-std/src/macros/resolve.rs

@@ -6,12 +6,13 @@ use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use orchid_base::error::mk_errv;
 use orchid_base::format::fmt;
+use orchid_base::interner::is;
 use orchid_base::location::Pos;
-use orchid_base::name::Sym;
+use orchid_base::logging::log;
+use orchid_base::name::{NameLike, Sym, VPath};
 use orchid_base::tree::Paren;
 use orchid_extension::atom::TAtom;
 use orchid_extension::atom_owned::own;
-use orchid_extension::context::{ctx, i};
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::{ExecHandle, exec};
 use orchid_extension::gen_expr::{GExpr, arg, bot, call, lambda, sym_ref};
@@ -24,17 +25,19 @@ use crate::macros::mactree::MacTreeSeq;
 use crate::macros::rule::state::{MatchState, StateEntry};
 use crate::{MacTok, MacTree};
 
-pub async fn resolve(tpl: MacTree) -> GExpr {
+pub async fn resolve(val: MacTree) -> GExpr {
 	exec(async move |mut h| {
-		let ctx = ctx();
-		// if ctx.logger().is_active() {
-		writeln!(ctx.logger(), "Macro-resolving {}", fmt(&tpl, &i()).await);
-		// }
+		writeln!(log("debug"), "Macro-resolving {}", fmt(&val).await).await;
 		let root = refl();
 		let mut macros = HashMap::new();
-		for n in tpl.glossary() {
-			if let Ok(ReflMemKind::Const) = root.get_by_path(n).await.map(|m| m.kind()) {
-				let Ok(mac) = h.exec::<TAtom<Macro>>(sym_ref(n.clone())).await else { continue };
+		for n in val.glossary() {
+			let (foot, body) = n.split_last_seg();
+			let new_name = VPath::new(body.iter().cloned())
+				.name_with_suffix(is(&format!("__macro__{foot}")).await)
+				.to_sym()
+				.await;
+			if let Ok(ReflMemKind::Const) = root.get_by_path(&new_name).await.map(|m| m.kind()) {
+				let Ok(mac) = h.exec::<TAtom<Macro>>(sym_ref(new_name)).await else { continue };
 				let mac = own(&mac).await;
 				macros.entry(mac.0.canonical_name.clone()).or_insert(mac);
 			}
@@ -45,7 +48,7 @@ pub async fn resolve(tpl: MacTree) -> GExpr {
 		for (_, mac) in macros.iter() {
 			let mut record = FilteredMacroRecord { mac, rules: Vec::new() };
 			for (rule_i, rule) in mac.0.rules.iter().enumerate() {
-				if rule.pattern.glossary.is_subset(tpl.glossary()) {
+				if rule.pattern.glossary.is_subset(val.glossary()) {
 					record.rules.push(rule_i);
 				}
 			}
@@ -61,7 +64,15 @@ pub async fn resolve(tpl: MacTree) -> GExpr {
 			}
 		}
 		let mut rctx = ResolveCtx { h, exclusive, priod };
-		resolve_one(&mut rctx, Substack::Bottom, &tpl).await
+		let gex = resolve_one(&mut rctx, Substack::Bottom, &val).await;
+		writeln!(
+			log("debug"),
+			"Macro-resolution over {}\nreturned {}",
+			fmt(&val).await,
+			fmt(&gex).await
+		)
+		.await;
+		gex
 	})
 	.await
 }
@@ -98,7 +109,7 @@ async fn resolve_one(
 		MacTok::Lambda(arg, body) => {
 			let MacTok::Name(name) = &*arg.tok else {
 				return bot(mk_errv(
-					i().i("Syntax error after macros").await,
+					is("Syntax error after macros").await,
 					"This token ends up as a binding, consider replacing it with a name",
 					[arg.pos()],
 				));
@@ -109,8 +120,8 @@ async fn resolve_one(
 		},
 		MacTok::S(Paren::Round, body) => resolve_seq(ctx, arg_stk, body.clone(), value.pos()).await,
 		MacTok::S(..) => bot(mk_errv(
-			i().i("Leftover [] or {} not matched by macro").await,
-			format!("{} was not matched by any macro", fmt(value, &i()).await),
+			is("Leftover [] or {} not matched by macro").await,
+			format!("{} was not matched by any macro", fmt(value).await),
 			[value.pos()],
 		)),
 	}
@@ -138,7 +149,7 @@ async fn resolve_seq(
 ) -> GExpr {
 	if val.items.is_empty() {
 		return bot(mk_errv(
-			i().i("Empty sequence").await,
+			is("Empty sequence").await,
 			"() or (\\arg ) left after macro execution. \
 			This is usually caused by an incomplete call to a macro with bad error detection",
 			[fallback_pos],
@@ -212,7 +223,7 @@ async fn resolve_seq(
 				Err((lran, rran))
 			}
 		});
-		let mac_conflict_tk = i().i("Macro conflict").await;
+		let mac_conflict_tk = is("Macro conflict").await;
 		let error = conflict_sets
 			.filter(|r| 1 < r.len())
 			.map(|set| {
@@ -270,13 +281,12 @@ async fn resolve_seq(
 
 async fn mk_body_call(mac: &Macro, rule: &Rule, state: &MatchState<'_>, pos: Pos) -> GExpr {
 	let mut call_args = vec![];
-	for name in rule.placeholders.iter() {
+	for name in rule.ph_names.iter() {
 		call_args.push(match state.get(name).expect("Missing state entry for placeholder") {
 			StateEntry::Scalar(scal) => (**scal).clone().to_gen().await,
 			StateEntry::Vec(vec) =>
 				MacTok::S(Paren::Round, MacTreeSeq::new(vec.iter().cloned())).at(Pos::None).to_gen().await,
 		});
 	}
-	call(sym_ref(mac.0.module.suffix([rule.body_name.clone()], &i()).await), call_args)
-		.at(pos.clone())
+	call(sym_ref(mac.0.module.suffix([rule.body.clone()]).await), call_args).at(pos.clone())
 }

orchid-std/src/macros/rule/build.rs

@@ -2,17 +2,16 @@ use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
 use orchid_base::error::{OrcRes, mk_errv};
-use orchid_base::interner::Tok;
+use orchid_base::interner::{IStr, is};
 use orchid_base::join_ok;
 use orchid_base::side::Side;
-use orchid_extension::context::i;
 
 use super::shared::{AnyMatcher, ScalMatcher, VecMatcher};
 use super::vec_attrs::vec_attrs;
 use crate::macros::mactree::{Ph, PhKind};
 use crate::macros::{MacTok, MacTree};
 
-pub type MaxVecSplit<'a> = (&'a [MacTree], (Tok<String>, u8, bool), &'a [MacTree]);
+pub type MaxVecSplit<'a> = (&'a [MacTree], (IStr, u8, bool), &'a [MacTree]);
 
 /// Derive the details of the central vectorial and the two sides from a
 /// slice of Expr's
@@ -126,7 +125,7 @@ async fn mk_scalar(pattern: &MacTree) -> OrcRes<ScalMatcher> {
 		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(&body.items).boxed_local().await?)),
 		MacTok::Lambda(..) =>
 			return Err(mk_errv(
-				i().i("Lambda in matcher").await,
+				is("Lambda in matcher").await,
 				"Lambdas can't be matched for, only generated in templates",
 				[pattern.pos()],
 			)),
@@ -137,10 +136,11 @@ async fn mk_scalar(pattern: &MacTree) -> OrcRes<ScalMatcher> {
 
 #[cfg(test)]
 mod test {
+	use orchid_base::interner::local_interner::local_interner;
+	use orchid_base::interner::{is, with_interner};
 	use orchid_base::location::SrcRange;
 	use orchid_base::sym;
 	use orchid_base::tokens::Paren;
-	use orchid_extension::context::{i, mock_ctx, with_ctx};
 	use test_executors::spin_on;
 
 	use super::mk_any;
@@ -149,27 +149,27 @@ mod test {
 
 	#[test]
 	fn test_scan() {
-		spin_on(with_ctx(mock_ctx(), async {
-			let ex = |tok: MacTok| async { tok.at(SrcRange::mock(&i()).await.pos()) };
+		spin_on(with_interner(local_interner(), async {
+			let ex = |tok: MacTok| async { tok.at(SrcRange::mock().await.pos()) };
 			let pattern = vec![
 				ex(MacTok::Ph(Ph {
 					kind: PhKind::Vector { priority: 0, at_least_one: false },
-					name: i().i("::prefix").await,
+					name: is("::prefix").await,
 				}))
 				.await,
-				ex(MacTok::Name(sym!(prelude::do; i()))).await,
+				ex(MacTok::Name(sym!(prelude::do))).await,
 				ex(MacTok::S(
 					Paren::Round,
 					MacTreeSeq::new([
 						ex(MacTok::Ph(Ph {
 							kind: PhKind::Vector { priority: 0, at_least_one: false },
-							name: i().i("expr").await,
+							name: is("expr").await,
 						}))
 						.await,
-						ex(MacTok::Name(sym!(prelude::; ; i()))).await,
+						ex(MacTok::Name(sym!(prelude::;))).await,
 						ex(MacTok::Ph(Ph {
 							kind: PhKind::Vector { priority: 1, at_least_one: false },
-							name: i().i("rest").await,
+							name: is("rest").await,
 						}))
 						.await,
 					]),
@@ -177,7 +177,7 @@ mod test {
 				.await,
 				ex(MacTok::Ph(Ph {
 					kind: PhKind::Vector { priority: 0, at_least_one: false },
-					name: i().i("::suffix").await,
+					name: is("::suffix").await,
 				}))
 				.await,
 			];

orchid-std/src/macros/rule/matcher.rs

@@ -2,8 +2,8 @@ use std::fmt;
 use std::rc::Rc;
 
 use orchid_base::error::OrcRes;
+use orchid_base::interner::is;
 use orchid_base::name::Sym;
-use orchid_extension::context::i;
 
 use super::any_match::any_match;
 use super::build::mk_any;
@@ -24,12 +24,12 @@ impl Matcher {
 		let first = pattern.first().expect("Empty pattern is not allowed");
 		if vec_attrs(first).is_none() {
 			let pos = first.pos();
-			pattern.insert(0, MacTok::Ph(Ph { name: i().i("::before").await, kind }).at(pos));
+			pattern.insert(0, MacTok::Ph(Ph { name: is("::before").await, kind }).at(pos));
 		}
 		let last = pattern.last().expect("first returned Some above");
 		if vec_attrs(last).is_none() {
 			let pos = last.pos();
-			pattern.insert(0, MacTok::Ph(Ph { name: i().i("::after").await, kind }).at(pos));
+			pattern.insert(0, MacTok::Ph(Ph { name: is("::after").await, kind }).at(pos));
 		}
 		Ok(Matcher { inner: mk_any(&pattern).await? })
 	}
@@ -42,7 +42,7 @@ impl Matcher {
 	) -> Option<(&'a [MacTree], MatchState<'a>, &'a [MacTree])> {
 		let mut result = any_match(&self.inner, seq, &save_loc)?;
 		async fn remove_frame<'a>(result: &mut MatchState<'a>, key: &str) -> &'a [MacTree] {
-			match result.remove(i().i(key).await) {
+			match result.remove(is(key).await) {
 				Some(StateEntry::Scalar(_)) => panic!("{key} is defined in the constructor as a Vec"),
 				Some(StateEntry::Vec(v)) => v,
 				None => &[],

orchid-std/src/macros/rule/shared.rs

@@ -3,7 +3,7 @@
 use std::fmt;
 
 use itertools::Itertools;
-use orchid_base::interner::Tok;
+use orchid_base::interner::IStr;
 use orchid_base::name::Sym;
 use orchid_base::side::Side;
 use orchid_base::tokens::{PARENS, Paren};
@@ -11,12 +11,12 @@ use orchid_base::tokens::{PARENS, Paren};
 pub enum ScalMatcher {
 	Name(Sym),
 	S(Paren, Box<AnyMatcher>),
-	Placeh { key: Tok<String> },
+	Placeh { key: IStr },
 }
 
 pub enum VecMatcher {
 	Placeh {
-		key: Tok<String>,
+		key: IStr,
 		nonzero: bool,
 	},
 	Scan {
@@ -41,7 +41,7 @@ pub enum VecMatcher {
 		/// the length of matches on either side.
 		///
 		/// Vectorial keys that appear on either side, in priority order
-		key_order: Vec<Tok<String>>,
+		key_order: Vec<IStr>,
 	},
 }
 

orchid-std/src/macros/rule/state.rs

@@ -2,7 +2,7 @@
 use std::any::Any;
 
 use hashbrown::HashMap;
-use orchid_base::interner::Tok;
+use orchid_base::interner::IStr;
 use orchid_base::join::join_maps;
 use orchid_base::location::Pos;
 use orchid_base::match_mapping;
@@ -30,11 +30,11 @@ pub enum StateEntry<'a> {
 }
 #[derive(Clone, Debug)]
 pub struct MatchState<'a> {
-	placeholders: HashMap<Tok<String>, StateEntry<'a>>,
+	placeholders: HashMap<IStr, StateEntry<'a>>,
 	name_posv: HashMap<Sym, Vec<Pos>>,
 }
 impl<'a> MatchState<'a> {
-	pub fn from_ph(key: Tok<String>, entry: StateEntry<'a>) -> Self {
+	pub fn from_ph(key: IStr, entry: StateEntry<'a>) -> Self {
 		Self { placeholders: HashMap::from([(key, entry)]), name_posv: HashMap::new() }
 	}
 	pub fn combine(self, s: Self) -> Self {
@@ -45,7 +45,7 @@ impl<'a> MatchState<'a> {
 			}),
 		}
 	}
-	pub fn ph_len(&self, key: &Tok<String>) -> Option<usize> {
+	pub fn ph_len(&self, key: &IStr) -> Option<usize> {
 		match self.placeholders.get(key)? {
 			StateEntry::Vec(slc) => Some(slc.len()),
 			_ => None,
@@ -57,10 +57,8 @@ impl<'a> MatchState<'a> {
 	pub fn names(&self) -> impl Iterator<Item = (Sym, &[Pos])> {
 		self.name_posv.iter().map(|(sym, vec)| (sym.clone(), &vec[..]))
 	}
-	pub fn get(&self, key: &Tok<String>) -> Option<&StateEntry<'a>> { self.placeholders.get(key) }
-	pub fn remove(&mut self, name: Tok<String>) -> Option<StateEntry<'a>> {
-		self.placeholders.remove(&name)
-	}
+	pub fn get(&self, key: &IStr) -> Option<&StateEntry<'a>> { self.placeholders.get(key) }
+	pub fn remove(&mut self, name: IStr) -> Option<StateEntry<'a>> { self.placeholders.remove(&name) }
 	pub fn mk_owned(self) -> OwnedState {
 		OwnedState {
 			placeholders: (self.placeholders.into_iter())
@@ -88,10 +86,10 @@ pub enum OwnedEntry {
 	Scalar(MacTree),
 }
 pub struct OwnedState {
-	placeholders: HashMap<Tok<String>, OwnedEntry>,
+	placeholders: HashMap<IStr, OwnedEntry>,
 	name_posv: HashMap<Sym, Vec<Pos>>,
 }
 impl OwnedState {
-	pub fn get(&self, key: &Tok<String>) -> Option<&OwnedEntry> { self.placeholders.get(key) }
+	pub fn get(&self, key: &IStr) -> Option<&OwnedEntry> { self.placeholders.get(key) }
 	pub fn positions(&self, name: &Sym) -> &[Pos] { self.name_posv.get(name).map_or(&[], |v| &v[..]) }
 }

orchid-std/src/macros/rule/vec_attrs.rs

@@ -1,4 +1,4 @@
-use orchid_base::interner::Tok;
+use orchid_base::interner::IStr;
 
 use crate::macros::mactree::{Ph, PhKind};
 use crate::macros::{MacTok, MacTree};
@@ -6,7 +6,7 @@ use crate::macros::{MacTok, MacTree};
 /// Returns the name, priority and at_least_one of the expression if it is
 /// a vectorial placeholder
 #[must_use]
-pub fn vec_attrs(expr: &MacTree) -> Option<(Tok<String>, u8, bool)> {
+pub fn vec_attrs(expr: &MacTree) -> Option<(IStr, u8, bool)> {
 	match (*expr.tok).clone() {
 		MacTok::Ph(Ph { kind: PhKind::Vector { priority, at_least_one }, name }) =>
 			Some((name, priority, at_least_one)),

orchid-std/src/macros/std_macros.rs

@@ -3,7 +3,6 @@ use orchid_base::error::OrcRes;
 use orchid_base::sym;
 use orchid_extension::atom::TAtom;
 use orchid_extension::atom_owned::own;
-use orchid_extension::context::i;
 use orchid_extension::conv::ToExpr;
 use orchid_extension::coroutine_exec::exec;
 use orchid_extension::expr::Expr;
@@ -27,8 +26,8 @@ pub async fn gen_std_macro_lib() -> Vec<GenMember> {
 			fun(false, "is_none_body", async |val: OrcOpt<Expr>| {
 				if val.0.is_none() { OrcOpt(Some(Tpl(()))) } else { OrcOpt(None) }
 			}),
-			build_macro(None, ["of", "empty"])
-				.rule(mactreev!(pattern::match_rule ( std::option::of "...$" sub_pattern  0)), [
+			build_macro(None, ["some", "none"])
+				.rule(mactreev!(pattern::match_rule ( std::option::some "...$" sub_pattern  0)), [
 					|[sub]: [_; _]| {
 						exec(async move |mut h| {
 							let sub = h
@@ -39,7 +38,7 @@ pub async fn gen_std_macro_lib() -> Vec<GenMember> {
 							Ok(MatcherAtom {
 								keys: sub.keys().collect().await,
 								matcher: h
-									.register(call(sym_ref(sym!(std::option::is_some_body; i())), [sub
+									.register(call(sym_ref(sym!(std::option::is_some_body)), [sub
 										.to_gen()
 										.await]))
 									.await,
@@ -47,11 +46,11 @@ pub async fn gen_std_macro_lib() -> Vec<GenMember> {
 						})
 					},
 				])
-				.rule(mactreev!(pattern::match_rule(std::option::empty)), [|[]: [_; _]| {
+				.rule(mactreev!(pattern::match_rule(std::option::none)), [|[]: [_; _]| {
 					exec(async |mut h| {
 						Ok(MatcherAtom {
 							keys: vec![],
-							matcher: h.register(sym_ref(sym!(std::option::is_none_body; i()))).await,
+							matcher: h.register(sym_ref(sym!(std::option::is_none_body))).await,
 						})
 					})
 				}])
@@ -62,16 +61,16 @@ pub async fn gen_std_macro_lib() -> Vec<GenMember> {
 				.rule(mactreev!(std::tuple::t [ "...$" elements 0 ]), [|[elements]: [_; _]| {
 					exec(async move |mut h| {
 						let tup = h
-							.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve; i())), [
+							.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve)), [
 								mactree!((macros::common::comma_list "push" elements ;)).to_gen().await,
 							]))
 							.await?;
 						let val = stream::iter(&tup.0[..])
-							.fold(sym_ref(sym!(std::tuple::empty; i())), async |head, new| {
-								call(sym_ref(sym!(std::tuple::cat; i())), [
+							.fold(sym_ref(sym!(std::tuple::empty)), async |head, new| {
+								call(sym_ref(sym!(std::tuple::cat)), [
 									head,
-									call(sym_ref(sym!(std::tuple::one; i())), [call(
-										sym_ref(sym!(macros::resolve; i())),
+									call(sym_ref(sym!(std::tuple::one)), [call(
+										sym_ref(sym!(macros::resolve)),
 										[new.clone().to_gen().await],
 									)]),
 								])
@@ -102,7 +101,7 @@ pub async fn gen_std_macro_lib() -> Vec<GenMember> {
 fn parse_tpl(elements: MacTree, tail_matcher: Option<MacTree>) -> impl Future<Output = GExpr> {
 	exec(async move |mut h| -> OrcRes<MatcherAtom> {
 		let tup = h
-			.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve; i())), [
+			.exec::<HomoTpl<TAtom<MacTree>>>(call(sym_ref(sym!(macros::resolve)), [
 				mactree!((macros::common::comma_list "push" elements ;)).to_gen().await,
 			]))
 			.await?;
@@ -110,7 +109,7 @@ fn parse_tpl(elements: MacTree, tail_matcher: Option<MacTree>) -> impl Future<Ou
 		for mac_a in &tup.0[..] {
 			let mac = own(mac_a).await;
 			let sub = h
-				.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve; i())), [
+				.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [
 					mactree!(pattern::match_rule ("push" mac ;)).to_gen().await,
 				]))
 				.await?;
@@ -118,7 +117,7 @@ fn parse_tpl(elements: MacTree, tail_matcher: Option<MacTree>) -> impl Future<Ou
 		}
 		let tail_matcher = match tail_matcher {
 			Some(mac) => Some(
-				h.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve; i())), [
+				h.exec::<TAtom<MatcherAtom>>(call(sym_ref(sym!(macros::resolve)), [
 					mactree!(pattern::match_rule "push" mac ;).to_gen().await,
 				]))
 				.await?,
@@ -131,7 +130,7 @@ fn parse_tpl(elements: MacTree, tail_matcher: Option<MacTree>) -> impl Future<Ou
 				.chain(stream::iter(&tail_matcher).flat_map(|mat| mat.keys()))
 				.collect()
 				.await,
-			matcher: call(sym_ref(sym!(std::tuple::matcher_body; i())), [
+			matcher: call(sym_ref(sym!(std::tuple::matcher_body)), [
 				HomoTpl(subs).to_gen().await,
 				OrcOpt(tail_matcher).to_gen().await,
 			])
@@ -162,8 +161,8 @@ fn tuple_matcher_body(
 			None => (),
 			Some(tail_mat) => {
 				let tail_tpl = stream::iter(&value.0[children.0.len()..])
-					.fold(sym_ref(sym!(std::tuple::empty; i())), async |prefix, new| {
-						call(sym_ref(sym!(std::tuple::cat; i())), [prefix, new.clone().to_gen().await])
+					.fold(sym_ref(sym!(std::tuple::empty)), async |prefix, new| {
+						call(sym_ref(sym!(std::tuple::cat)), [prefix, new.clone().to_gen().await])
 					})
 					.await;
 				match tail_mat.run_matcher(&mut h, tail_tpl).await? {