base and extension fully compiles, host in good shape

Cargo.lock

@@ -26,6 +26,15 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "aho-corasick"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "allocator-api2"
 version = "0.2.21"
@@ -174,6 +183,43 @@ version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4288f83726785267c6f2ef073a3d83dc3f9b81464e9f99898240cced85fce35a"
 
+[[package]]
+name = "async-process"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb"
+dependencies = [
+ "async-channel 2.3.1",
+ "async-io",
+ "async-lock",
+ "async-signal",
+ "async-task",
+ "blocking",
+ "cfg-if",
+ "event-listener 5.4.0",
+ "futures-lite",
+ "rustix",
+ "tracing",
+]
+
+[[package]]
+name = "async-signal"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3"
+dependencies = [
+ "async-io",
+ "async-lock",
+ "atomic-waker",
+ "cfg-if",
+ "futures-core",
+ "futures-io",
+ "rustix",
+ "signal-hook-registry",
+ "slab",
+ "windows-sys",
+]
+
 [[package]]
 name = "async-std"
 version = "1.13.0"
@@ -200,6 +246,28 @@ dependencies = [
  "wasm-bindgen-futures",
 ]
 
+[[package]]
+name = "async-stream"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
+dependencies = [
+ "proc-macro2 1.0.92",
+ "quote 1.0.38",
+ "syn 2.0.95",
+]
+
 [[package]]
 name = "async-task"
 version = "4.7.1"
@@ -895,9 +963,11 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 name = "orchid-api"
 version = "0.1.0"
 dependencies = [
+ "async-std",
  "orchid-api-derive",
  "orchid-api-traits",
  "ordered-float",
+ "test_executors",
 ]
 
 [[package]]
@@ -916,6 +986,9 @@ dependencies = [
 name = "orchid-api-traits"
 version = "0.1.0"
 dependencies = [
+ "async-std",
+ "async-stream",
+ "futures",
  "itertools",
  "never",
  "ordered-float",
@@ -939,6 +1012,7 @@ dependencies = [
  "orchid-api-derive",
  "orchid-api-traits",
  "ordered-float",
+ "regex",
  "rust-embed",
  "rust_decimal",
  "some_executor 0.4.0",
@@ -978,7 +1052,9 @@ dependencies = [
 name = "orchid-host"
 version = "0.1.0"
 dependencies = [
+ "async-process",
  "async-std",
+ "async-stream",
  "derive_destructure",
  "futures",
  "hashbrown 0.15.2",
@@ -992,6 +1068,7 @@ dependencies = [
  "ordered-float",
  "paste",
  "substack",
+ "test_executors",
  "trait-set",
 ]
 
@@ -1196,6 +1273,35 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "regex"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
+
 [[package]]
 name = "rend"
 version = "0.4.2"
@@ -1361,6 +1467,15 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "simdutf8"
 version = "0.1.5"

SWAP.md

@@ -2,7 +2,11 @@
 
 convert host to async non-send
 
-convert extension's SysCtx to a typed context bag
+demonstrate operation with existing lex-hello example
+
+consider converting extension's SysCtx to a typed context bag
+
+align fn atom and macros on both sides with new design. No global state.
 
 ## alternate extension mechanism
 

orchid-api-derive/src/decode.rs

@@ -12,7 +12,9 @@ pub fn derive(input: TokenStream) -> TokenStream {
 	let decode = decode_body(&input.data);
 	let expanded = quote! {
 		impl #impl_generics orchid_api_traits::Decode for #name #ty_generics #where_clause {
-			fn decode<R: std::io::Read + ?Sized>(read: &mut R) -> Self { #decode }
+			async fn decode<R: async_std::io::Read + ?Sized>(mut read: std::pin::Pin<&mut R>) -> Self {
+				#decode
+			}
 		}
 	};
 	TokenStream::from(expanded)
@@ -22,11 +24,21 @@ fn decode_fields(fields: &syn::Fields) -> pm2::TokenStream {
 	match fields {
 		syn::Fields::Unit => quote! {},
 		syn::Fields::Named(_) => {
-			let names = fields.iter().map(|f| f.ident.as_ref().unwrap());
+			let exprs = fields.iter().map(|f| {
-			quote! { { #( #names: orchid_api_traits::Decode::decode(read), )* } }
+				let syn::Field { ty, ident, .. } = &f;
+				quote! {
+					#ident : < #ty as orchid_api_traits::Decode>::decode(read.as_mut()).await
+				}
+			});
+			quote! { { #( #exprs, )* } }
 		},
 		syn::Fields::Unnamed(_) => {
-			let exprs = fields.iter().map(|_| quote! { orchid_api_traits::Decode::decode(read), });
+			let exprs = fields.iter().map(|field| {
+				let ty = &field.ty;
+				quote! {
+					< #ty as orchid_api_traits::Decode>::decode(read.as_mut()).await,
+				}
+			});
 			quote! { ( #( #exprs )* ) }
 		},
 	}
@@ -46,7 +58,7 @@ fn decode_body(data: &syn::Data) -> proc_macro2::TokenStream {
 				quote! { #id => Self::#ident #fields, }
 			});
 			quote! {
-				match <u8 as orchid_api_traits::Decode>::decode(read) {
+				match <u8 as orchid_api_traits::Decode>::decode(read.as_mut()).await {
 					#(#opts)*
 					x => panic!("Unrecognized enum kind {x}")
 				}

orchid-api-derive/src/encode.rs

@@ -14,7 +14,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
 	let encode = encode_body(&input.data);
 	let expanded = quote! {
 		impl #e_impl_generics orchid_api_traits::Encode for #name #e_ty_generics #e_where_clause {
-			fn encode<W: std::io::Write + ?Sized>(&self, write: &mut W) { #encode }
+			async fn encode<W: async_std::io::Write + ?Sized>(&self, mut write: std::pin::Pin<&mut W>) { #encode }
 		}
 	};
 	TokenStream::from(expanded)
@@ -37,7 +37,7 @@ fn encode_body(data: &syn::Data) -> Option<pm2::TokenStream> {
 				let body = encode_items(&v.fields);
 				quote! {
 					Self::#ident #dest => {
-						(#i as u8).encode(write);
+						(#i as u8).encode(write.as_mut()).await;
 						#body
 					}
 				}
@@ -53,7 +53,7 @@ fn encode_body(data: &syn::Data) -> Option<pm2::TokenStream> {
 }
 
 fn encode_names<T: ToTokens>(names: impl Iterator<Item = T>) -> pm2::TokenStream {
-	quote! { #( #names .encode(write); )* }
+	quote! { #( #names .encode(write.as_mut()).await; )* }
 }
 
 fn encode_items(fields: &syn::Fields) -> Option<pm2::TokenStream> {

orchid-api-traits/Cargo.toml

@@ -6,6 +6,9 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-std = "1.13.0"
+async-stream = "0.3.6"
+futures = "0.3.31"
 itertools = "0.14.0"
 never = "0.1.0"
 ordered-float = "4.6.0"

orchid-api-traits/src/coding.rs

@@ -1,29 +1,38 @@
-use std::borrow::Cow;
 use std::collections::HashMap;
+use std::future::Future;
 use std::hash::Hash;
-use std::io::{Read, Write};
+use std::num::NonZero;
-use std::iter;
 use std::ops::{Range, RangeInclusive};
+use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
 
+use async_std::io::{Read, ReadExt, Write, WriteExt};
+use async_stream::stream;
+use futures::future::LocalBoxFuture;
+use futures::{FutureExt, StreamExt};
 use never::Never;
 use ordered_float::NotNan;
 
 use crate::encode_enum;
 
-pub trait Decode {
+pub trait Decode: 'static {
 	/// Decode an instance from the beginning of the buffer. Return the decoded
 	/// data and the remaining buffer.
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self;
+	fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> impl Future<Output = Self> + '_;
 }
 pub trait Encode {
 	/// Append an instance of the struct to the buffer
-	fn encode<W: Write + ?Sized>(&self, write: &mut W);
+	fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) -> impl Future<Output = ()>;
 }
 pub trait Coding: Encode + Decode + Clone {
-	fn get_decoder<T>(map: impl Fn(Self) -> T + 'static) -> impl Fn(&mut dyn Read) -> T {
+	fn get_decoder<T: 'static, F: Future<Output = T> + 'static>(
-		move |r| map(Self::decode(r))
+		map: impl Fn(Self) -> F + Clone + 'static,
+	) -> impl for<'a> Fn(Pin<&'a mut dyn Read>) -> LocalBoxFuture<'a, T> {
+		move |r| {
+			let map = map.clone();
+			async move { map(Self::decode(r).await).await }.boxed_local()
+		}
 	}
 }
 impl<T: Encode + Decode + Clone> Coding for T {}
@@ -31,15 +40,15 @@ impl<T: Encode + Decode + Clone> Coding for T {}
 macro_rules! num_impl {
 	($number:ty) => {
 		impl Decode for $number {
-			fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; (<$number>::BITS / 8) as usize];
-				read.read_exact(&mut bytes).unwrap();
+				read.read_exact(&mut bytes).await.unwrap();
 				<$number>::from_be_bytes(bytes)
 			}
 		}
 		impl Encode for $number {
-			fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-				write.write_all(&self.to_be_bytes()).expect("Could not write number")
+				write.write_all(&self.to_be_bytes()).await.expect("Could not write number")
 			}
 		}
 	};
@@ -57,41 +66,45 @@ num_impl!(i8);
 
 macro_rules! nonzero_impl {
 	($name:ty) => {
-		impl Decode for $name {
+		impl Decode for NonZero<$name> {
-			fn decode<R: Read + ?Sized>(read: &mut R) -> Self { Self::new(Decode::decode(read)).unwrap() }
+			async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+				Self::new(<$name as Decode>::decode(read).await).unwrap()
+			}
+		}
+		impl Encode for NonZero<$name> {
+			async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+				self.get().encode(write).await
 			}
-		impl Encode for $name {
-			fn encode<W: Write + ?Sized>(&self, write: &mut W) { self.get().encode(write) }
 		}
 	};
 }
 
-nonzero_impl!(std::num::NonZeroU8);
+nonzero_impl!(u8);
-nonzero_impl!(std::num::NonZeroU16);
+nonzero_impl!(u16);
-nonzero_impl!(std::num::NonZeroU32);
+nonzero_impl!(u32);
-nonzero_impl!(std::num::NonZeroU64);
+nonzero_impl!(u64);
-nonzero_impl!(std::num::NonZeroU128);
+nonzero_impl!(u128);
-nonzero_impl!(std::num::NonZeroI8);
+nonzero_impl!(i8);
-nonzero_impl!(std::num::NonZeroI16);
+nonzero_impl!(i16);
-nonzero_impl!(std::num::NonZeroI32);
+nonzero_impl!(i32);
-nonzero_impl!(std::num::NonZeroI64);
+nonzero_impl!(i64);
-nonzero_impl!(std::num::NonZeroI128);
+nonzero_impl!(i128);
 
 impl<T: Encode + ?Sized> Encode for &T {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
+	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) { (**self).encode(write).await }
 }
 macro_rules! float_impl {
 	($t:ty, $size:expr) => {
 		impl Decode for NotNan<$t> {
-			fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
 				let mut bytes = [0u8; $size];
-				read.read_exact(&mut bytes).unwrap();
+				read.read_exact(&mut bytes).await.unwrap();
 				NotNan::new(<$t>::from_be_bytes(bytes)).expect("Float was NaN")
 			}
 		}
 		impl Encode for NotNan<$t> {
-			fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-				write.write_all(&self.as_ref().to_be_bytes()).expect("Could not write number")
+				write.write_all(&self.as_ref().to_be_bytes()).await.expect("Could not write number")
 			}
 		}
 	};
@@ -101,98 +114,103 @@ float_impl!(f64, 8);
 float_impl!(f32, 4);
 
 impl Decode for String {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len = u64::decode(read).try_into().unwrap();
+		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
 		let mut data = vec![0u8; len];
-		read.read_exact(&mut data).unwrap();
+		read.read_exact(&mut data).await.unwrap();
 		std::str::from_utf8(&data).expect("String invalid UTF-8").to_owned()
 	}
 }
 impl Encode for String {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write);
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		write.write_all(self.as_bytes()).unwrap()
+		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl Encode for str {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write);
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		write.write_all(self.as_bytes()).unwrap()
+		write.write_all(self.as_bytes()).await.unwrap()
 	}
 }
 impl<T: Decode> Decode for Vec<T> {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len = u64::decode(read).try_into().unwrap();
+		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		iter::repeat_with(|| T::decode(read)).take(len).collect()
+		stream! { loop { yield T::decode(read.as_mut()).await } }.take(len).collect().await
 	}
 }
 impl<T: Encode> Encode for Vec<T> {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write);
+		self.as_slice().encode(write).await
-		self.iter().for_each(|t| t.encode(write));
 	}
 }
 impl<T: Encode> Encode for [T] {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write);
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		self.iter().for_each(|t| t.encode(write));
+		for t in self.iter() {
+			t.encode(write.as_mut()).await
+		}
 	}
 }
 impl<T: Decode> Decode for Option<T> {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		match u8::decode(read) {
+		match u8::decode(read.as_mut()).await {
 			0 => None,
-			1 => Some(T::decode(read)),
+			1 => Some(T::decode(read).await),
 			x => panic!("{x} is not a valid option value"),
 		}
 	}
 }
 impl<T: Encode> Encode for Option<T> {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-		let t = if let Some(t) = self { t } else { return 0u8.encode(write) };
+		let t = if let Some(t) = self { t } else { return 0u8.encode(write.as_mut()).await };
-		1u8.encode(write);
+		1u8.encode(write.as_mut()).await;
-		t.encode(write);
+		t.encode(write).await;
 	}
 }
 impl<T: Decode, E: Decode> Decode for Result<T, E> {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		match u8::decode(read) {
+		match u8::decode(read.as_mut()).await {
-			0 => Self::Ok(T::decode(read)),
+			0 => Self::Ok(T::decode(read).await),
-			1 => Self::Err(E::decode(read)),
+			1 => Self::Err(E::decode(read).await),
 			x => panic!("Invalid Result tag {x}"),
 		}
 	}
 }
 
 impl<T: Encode, E: Encode> Encode for Result<T, E> {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
 		match self {
-			Ok(t) => encode_enum(write, 0, |w| t.encode(w)),
+			Ok(t) => encode_enum(write, 0, |w| t.encode(w)).await,
-			Err(e) => encode_enum(write, 1, |w| e.encode(w)),
+			Err(e) => encode_enum(write, 1, |w| e.encode(w)).await,
 		}
 	}
 }
 impl<K: Decode + Eq + Hash, V: Decode> Decode for HashMap<K, V> {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		let len = u64::decode(read).try_into().unwrap();
+		let len = u64::decode(read.as_mut()).await.try_into().unwrap();
-		iter::repeat_with(|| <(K, V)>::decode(read)).take(len).collect()
+		stream! { loop { yield <(K, V)>::decode(read.as_mut()).await } }.take(len).collect().await
 	}
 }
 impl<K: Encode + Eq + Hash, V: Encode> Encode for HashMap<K, V> {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-		u64::try_from(self.len()).unwrap().encode(write);
+		u64::try_from(self.len()).unwrap().encode(write.as_mut()).await;
-		self.iter().for_each(|pair| pair.encode(write));
+		for pair in self.iter() {
+			pair.encode(write.as_mut()).await
+		}
 	}
 }
 macro_rules! tuple {
   (($($t:ident)*) ($($T:ident)*)) => {
     impl<$($T: Decode),*> Decode for ($($T,)*) {
-      fn decode<R: Read + ?Sized>(read: &mut R) -> Self { ($($T::decode(read),)*) }
+			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+				($($T::decode(read.as_mut()).await,)*)
+			}
     }
     impl<$($T: Encode),*> Encode for ($($T,)*) {
-      fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
         let ($($t,)*) = self;
-        $( $t.encode(write); )*
+        $( $t.encode(write.as_mut()).await; )*
       }
     }
   };
@@ -216,52 +234,59 @@ tuple!((t u v x y z a b c d e f g h i) (T U V X Y Z A B C D E F G H I));
 tuple!((t u v x y z a b c d e f g h i j) (T U V X Y Z A B C D E F G H I J)); // 16
 
 impl Decode for () {
-	fn decode<R: Read + ?Sized>(_: &mut R) -> Self {}
+	async fn decode<R: Read + ?Sized>(_: Pin<&mut R>) -> Self {}
 }
 impl Encode for () {
-	fn encode<W: Write + ?Sized>(&self, _: &mut W) {}
+	async fn encode<W: Write + ?Sized>(&self, _: Pin<&mut W>) {}
 }
 impl Decode for Never {
-	fn decode<R: Read + ?Sized>(_: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(_: Pin<&mut R>) -> Self {
 		unreachable!("A value of Never cannot exist so it can't have been serialized");
 	}
 }
 impl Encode for Never {
-	fn encode<W: Write + ?Sized>(&self, _: &mut W) { match *self {} }
+	async fn encode<W: Write + ?Sized>(&self, _: Pin<&mut W>) { match *self {} }
 }
 impl Decode for bool {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		let mut buf = [0];
-		read.read_exact(&mut buf).unwrap();
+		read.read_exact(&mut buf).await.unwrap();
 		buf[0] != 0
 	}
 }
 impl Encode for bool {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write.write_all(&[if *self { 0xff } else { 0 }]).unwrap()
+		write.write_all(&[if *self { 0xffu8 } else { 0u8 }]).await.unwrap()
 	}
 }
 impl<T: Decode, const N: usize> Decode for [T; N] {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		// TODO: figure out how to do this in safe rust on the stack
-		((0..N).map(|_| T::decode(read)).collect::<Vec<_>>().try_into())
+		let v =
-			.unwrap_or_else(|_| unreachable!("The length of this iterator is statically known"))
+			stream! { loop { yield T::decode(read.as_mut()).await } }.take(N).collect::<Vec<_>>().await;
+		v.try_into().unwrap_or_else(|_| unreachable!("The length of this stream is statically known"))
 	}
 }
 impl<T: Encode, const N: usize> Encode for [T; N] {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) { self.iter().for_each(|t| t.encode(write)) }
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
+		for t in self.iter() {
+			t.encode(write.as_mut()).await
+		}
+	}
 }
 
 macro_rules! two_end_range {
   ($this:ident, $name:tt, $op:tt, $start:expr, $end:expr) => {
     impl<T: Decode> Decode for $name<T> {
-      fn decode<R: Read + ?Sized>(read: &mut R) -> Self { T::decode(read) $op T::decode(read) }
+			async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
+				T::decode(read.as_mut()).await $op T::decode(read).await
+			}
     }
     impl<T: Encode> Encode for $name<T> {
-      fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+			async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
         let $this = self;
-        ($start).encode(write);
+        ($start).encode(write.as_mut()).await;
-        ($end).encode(write);
+        ($end).encode(write).await;
       }
     }
   }
@@ -273,10 +298,12 @@ two_end_range!(x, RangeInclusive, ..=, x.start(), x.end());
 macro_rules! smart_ptr {
 	($name:tt) => {
 		impl<T: Decode> Decode for $name<T> {
-			fn decode<R: Read + ?Sized>(read: &mut R) -> Self { $name::new(T::decode(read)) }
+			async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+				$name::new(T::decode(read).await)
+			}
 		}
 		impl<T: Encode> Encode for $name<T> {
-			fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
+			async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) { (**self).encode(write).await }
 		}
 	};
 }
@@ -285,18 +312,13 @@ smart_ptr!(Arc);
 smart_ptr!(Rc);
 smart_ptr!(Box);
 
-impl<T: ?Sized + ToOwned> Decode for Cow<'_, T>
-where T::Owned: Decode
-{
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self { Cow::Owned(T::Owned::decode(read)) }
-}
-impl<T: ?Sized + Encode + ToOwned> Encode for Cow<'_, T> {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
-}
-
 impl Decode for char {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self { char::from_u32(u32::decode(read)).unwrap() }
+	async fn decode<R: Read + ?Sized>(read: Pin<&mut R>) -> Self {
+		char::from_u32(u32::decode(read).await).unwrap()
+	}
 }
 impl Encode for char {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) { (*self as u32).encode(write) }
+	async fn encode<W: Write + ?Sized>(&self, write: Pin<&mut W>) {
+		(*self as u32).encode(write).await
+	}
 }

orchid-api-traits/src/helpers.rs

@@ -1,16 +1,22 @@
-use std::io::{Read, Write};
+use std::future::Future;
+use std::pin::Pin;
 
+use async_std::io::{Read, ReadExt, Write, WriteExt};
 use itertools::{Chunk, Itertools};
 
 use crate::Encode;
 
-pub fn encode_enum<W: Write + ?Sized>(write: &mut W, id: u8, f: impl FnOnce(&mut W)) {
+pub async fn encode_enum<'a, W: Write + ?Sized, F: Future<Output = ()>>(
-	id.encode(write);
+	mut write: Pin<&'a mut W>,
-	f(write)
+	id: u8,
+	f: impl FnOnce(Pin<&'a mut W>) -> F,
+) {
+	id.encode(write.as_mut()).await;
+	f(write).await
 }
 
-pub fn write_exact<W: Write + ?Sized>(write: &mut W, bytes: &'static [u8]) {
+pub async fn write_exact<W: Write + ?Sized>(mut write: Pin<&mut W>, bytes: &'static [u8]) {
-	write.write_all(bytes).expect("Failed to write exact bytes")
+	write.write_all(bytes).await.expect("Failed to write exact bytes")
 }
 
 pub fn print_bytes(b: &[u8]) -> String {
@@ -21,16 +27,16 @@ pub fn print_bytes(b: &[u8]) -> String {
 		.join("  ")
 }
 
-pub fn read_exact<R: Read + ?Sized>(read: &mut R, bytes: &'static [u8]) {
+pub async fn read_exact<R: Read + ?Sized>(mut read: Pin<&mut R>, bytes: &'static [u8]) {
 	let mut data = vec![0u8; bytes.len()];
-	read.read_exact(&mut data).expect("Failed to read bytes");
+	read.read_exact(&mut data).await.expect("Failed to read bytes");
 	if data != bytes {
 		panic!("Wrong bytes!\nExpected: {}\nFound: {}", print_bytes(bytes), print_bytes(&data));
 	}
 }
 
-pub fn enc_vec(enc: &impl Encode) -> Vec<u8> {
+pub async fn enc_vec(enc: &impl Encode) -> Vec<u8> {
 	let mut vec = Vec::new();
-	enc.encode(&mut vec);
+	enc.encode(Pin::new(&mut vec)).await;
 	vec
 }

orchid-api-traits/src/relations.rs

@@ -1,3 +1,5 @@
+use std::future::Future;
+
 use super::coding::Coding;
 use crate::helpers::enc_vec;
 
@@ -5,9 +7,12 @@ pub trait Request: Coding + Sized + Send + 'static {
 	type Response: Coding + Send + 'static;
 }
 
-pub fn respond<R: Request>(_: &R, rep: R::Response) -> Vec<u8> { enc_vec(&rep) }
+pub async fn respond<R: Request>(_: &R, rep: R::Response) -> Vec<u8> { enc_vec(&rep).await }
-pub fn respond_with<R: Request>(r: &R, f: impl FnOnce(&R) -> R::Response) -> Vec<u8> {
+pub async fn respond_with<R: Request, F: Future<Output = R::Response>>(
-	respond(r, f(r))
+	r: &R,
+	f: impl FnOnce(&R) -> F,
+) -> Vec<u8> {
+	respond(r, f(r).await).await
 }
 
 pub trait Channel: 'static {

orchid-api/Cargo.toml

@@ -9,3 +9,7 @@ edition = "2021"
 ordered-float = "4.6.0"
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
+async-std = "1.13.0"
+
+[dev-dependencies]
+test_executors = "0.3.2"

orchid-api/src/atom.rs

@@ -120,6 +120,13 @@ impl Request for AtomPrint {
 	type Response = String;
 }
 
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct ExtAtomPrint(pub Atom);
+impl Request for ExtAtomPrint {
+	type Response = String;
+}
+
 /// Requests that apply to an existing atom instance
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(HostExtReq)]

orchid-api/src/interner.rs

@@ -1,5 +1,4 @@
 use std::num::NonZeroU64;
-use std::sync::Arc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;

orchid-api/src/parser.rs

@@ -1,13 +1,20 @@
 use std::num::NonZeroU64;
 
-use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_derive::{Coding, Decode, Encode, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::{Comment, HostExtReq, OrcResult, SysId, TokenTree};
 
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
 pub struct ParsId(pub NonZeroU64);
 
+// impl orchid_api_traits::Decode for ParsId {
+// 	async fn decode<R: async_std::io::Read + ?Sized>(mut read:
+// std::pin::Pin<&mut R>) -> Self {
+// 		Self(orchid_api_traits::Decode::decode(read.as_mut()).await)
+// 	}
+// }
+
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct ParseLine {

orchid-api/src/proto.rs

@@ -22,8 +22,9 @@
 //! be preserved. Toolkits must ensure that the client code is able to observe
 //! the ordering of messages.
 
-use std::io::{Read, Write};
+use std::pin::Pin;
 
+use async_std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact};
 
@@ -34,15 +35,15 @@ pub struct HostHeader {
 	pub log_strategy: logging::LogStrategy,
 }
 impl Decode for HostHeader {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		read_exact(read, HOST_INTRO);
+		read_exact(read.as_mut(), HOST_INTRO).await;
-		Self { log_strategy: logging::LogStrategy::decode(read) }
+		Self { log_strategy: logging::LogStrategy::decode(read).await }
 	}
 }
 impl Encode for HostHeader {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write_exact(write, HOST_INTRO);
+		write_exact(write.as_mut(), HOST_INTRO).await;
-		self.log_strategy.encode(write)
+		self.log_strategy.encode(write).await
 	}
 }
 
@@ -52,16 +53,16 @@ pub struct ExtensionHeader {
 	pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
-	fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
+	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
-		read_exact(read, EXT_INTRO);
+		read_exact(read.as_mut(), EXT_INTRO).await;
-		Self { name: String::decode(read), systems: Vec::decode(read) }
+		Self { name: String::decode(read.as_mut()).await, systems: Vec::decode(read).await }
 	}
 }
 impl Encode for ExtensionHeader {
-	fn encode<W: Write + ?Sized>(&self, write: &mut W) {
+	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
-		write_exact(write, EXT_INTRO);
+		write_exact(write.as_mut(), EXT_INTRO).await;
-		self.name.encode(write);
+		self.name.encode(write.as_mut()).await;
-		self.systems.encode(write)
+		self.systems.encode(write).await
 	}
 }
 
@@ -78,6 +79,7 @@ pub enum ExtHostReq {
 	Ping(Ping),
 	IntReq(interner::IntReq),
 	Fwd(atom::Fwd),
+	ExtAtomPrint(atom::ExtAtomPrint),
 	SysFwd(system::SysFwd),
 	ExprReq(expr::ExprReq),
 	SubLex(lexer::SubLex),
@@ -150,20 +152,24 @@ impl MsgSet for HostMsgSet {
 mod tests {
 	use orchid_api_traits::enc_vec;
 	use ordered_float::NotNan;
+	use test_executors::spin_on;
 
 	use super::*;
 
 	#[test]
 	fn host_header_enc() {
+		spin_on(async {
 			let hh = HostHeader { log_strategy: logging::LogStrategy::File("SomeFile".to_string()) };
-		let mut enc = &enc_vec(&hh)[..];
+			let mut enc = &enc_vec(&hh).await[..];
 			eprintln!("Encoded to {enc:?}");
-		HostHeader::decode(&mut enc);
+			HostHeader::decode(Pin::new(&mut enc)).await;
 			assert_eq!(enc, []);
+		})
 	}
 
 	#[test]
 	fn ext_header_enc() {
+		spin_on(async {
 			let eh = ExtensionHeader {
 				name: "my_extension".to_string(),
 				systems: vec![system::SystemDecl {
@@ -173,9 +179,10 @@ mod tests {
 					priority: NotNan::new(1f64).unwrap(),
 				}],
 			};
-		let mut enc = &enc_vec(&eh)[..];
+			let mut enc = &enc_vec(&eh).await[..];
 			eprintln!("Encoded to {enc:?}");
-		ExtensionHeader::decode(&mut enc);
+			ExtensionHeader::decode(Pin::new(&mut enc)).await;
 			assert_eq!(enc, [])
+		})
 	}
 }

orchid-base/Cargo.toml

@@ -20,6 +20,7 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 ordered-float = "4.6.0"
+regex = "1.11.1"
 rust-embed = "8.5.0"
 rust_decimal = "1.36.0"
 some_executor = "0.4.0"

orchid-base/src/builtin.rs

@@ -12,9 +12,9 @@ use crate::api;
 /// There are no ordering guarantees about these
 pub trait ExtPort {
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()>;
-	fn recv<'a, 's: 'a>(
+	fn recv<'a>(
-		&'s self,
+		&'a self,
-		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'a, ()> + 'a>,
+		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
 	) -> LocalBoxFuture<'a, ()>;
 }
 
@@ -26,7 +26,7 @@ impl ExtInit {
 	pub async fn send(&self, msg: &[u8]) { self.port.send(msg).await }
 	pub async fn recv<'a, 's: 'a>(
 		&'s self,
-		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'a, ()> + 'a>,
+		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
 	) {
 		self.port.recv(Box::new(cb)).await
 	}

orchid-base/src/format.rs

@@ -0,0 +1,89 @@
+use std::convert::Infallible;
+use std::iter;
+use std::rc::Rc;
+use std::str::FromStr;
+
+use itertools::Itertools;
+use regex::Regex;
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct FmtUnit {
+	pub subs: Vec<FmtUnit>,
+	pub variants: Rc<Variants>,
+}
+impl FmtUnit {
+	pub fn new(variants: Rc<Variants>, subs: impl IntoIterator<Item = FmtUnit>) -> Self {
+		Self { subs: subs.into_iter().collect(), variants }
+	}
+}
+impl<T> From<T> for FmtUnit
+where Variants: From<T>
+{
+	fn from(value: T) -> Self { Self { subs: vec![], variants: Rc::new(Variants::from(value)) } }
+}
+impl FromStr for FmtUnit {
+	type Err = Infallible;
+	fn from_str(s: &str) -> Result<Self, Self::Err> {
+		Ok(Self { subs: vec![], variants: Rc::new(Variants::new([s])) })
+	}
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub enum FmtElement {
+	Sub(u8),
+	String(Rc<String>),
+	InlineSub(u8),
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct Variants(pub Vec<Vec<FmtElement>>);
+impl Variants {
+	pub fn new<'a>(variants: impl IntoIterator<Item = &'a str>) -> Self {
+		let re = Regex::new(r"(?<tpl>\{\d+?-?\})|(\{\{)|(\}\})").unwrap();
+		Self(Vec::from_iter(variants.into_iter().map(|s: &str| {
+			let matches = re.captures_iter(s);
+			let slots = matches.into_iter().filter_map(|m| m.name("tpl")).map(|tpl| {
+				let no_opencurly = tpl.as_str().strip_prefix("{").expect("required by regex");
+				let maybe_dash = no_opencurly.strip_suffix("}").expect("required by regex");
+				let (num, had_dash) =
+					maybe_dash.strip_suffix('-').map_or((maybe_dash, false), |s| (s, true));
+				let idx = num.parse::<u8>().expect("Decimal digits required by regex");
+				(tpl.range(), idx, had_dash)
+			});
+			(iter::once(None).chain(slots.into_iter().map(Some)).chain(None).tuple_windows())
+				.flat_map(|(l, r)| {
+					let string = match (l, &r) {
+						(None, Some((r, ..))) => &s[..r.start],
+						(Some((r1, ..)), Some((r2, ..))) => &s[r1.end..r2.start],
+						(Some((r, ..)), None) => &s[r.end..],
+						(None, None) => s,
+					};
+					let str_item = FmtElement::String(Rc::new(string.to_string()));
+					match r {
+						None => itertools::Either::Left([str_item]),
+						Some((_, idx, inline)) => itertools::Either::Right([str_item, match inline {
+							true => FmtElement::InlineSub(idx),
+							false => FmtElement::Sub(idx),
+						}]),
+					}
+					.into_iter()
+				})
+				.coalesce(|left, right| match (left, right) {
+					(FmtElement::String(left), FmtElement::String(right)) =>
+						Ok(FmtElement::String(Rc::new(left.to_string() + right.as_str()))),
+					tuple => Err(tuple),
+				})
+				.collect_vec()
+		})))
+	}
+}
+impl From<String> for Variants {
+	fn from(value: String) -> Self { Self(vec![vec![FmtElement::String(Rc::new(value))]]) }
+}
+impl From<Rc<String>> for Variants {
+	fn from(value: Rc<String>) -> Self { Self(vec![vec![FmtElement::String(value)]]) }
+}
+impl FromStr for Variants {
+	type Err = Infallible;
+	fn from_str(s: &str) -> Result<Self, Self::Err> { Ok(Self::new([s])) }
+}

orchid-base/src/interner.rs

@@ -247,7 +247,7 @@ impl Interner {
 		tok
 	}
 	/// Extern an identifier; query the data it represents if not known locally
-	async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
+	pub async fn ex<M: InternMarker>(&self, marker: M) -> Tok<M::Interned> {
 		if let Some(tok) = M::Interned::bimap(&mut *self.interners.lock().await).by_marker(marker) {
 			return tok;
 		}
@@ -284,6 +284,7 @@ pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) {
 #[cfg(test)]
 mod test {
 	use std::num::NonZero;
+	use std::pin::Pin;
 
 	use orchid_api_traits::{Decode, enc_vec};
 	use test_executors::spin_on;
@@ -300,9 +301,11 @@ mod test {
 
 	#[test]
 	fn test_coding() {
+		spin_on(async {
 			let coded = api::TStr(NonZero::new(3u64).unwrap());
-		let mut enc = &enc_vec(&coded)[..];
+			let mut enc = &enc_vec(&coded).await[..];
-		api::TStr::decode(&mut enc);
+			api::TStr::decode(Pin::new(&mut enc)).await;
 			assert_eq!(enc, [], "Did not consume all of {enc:?}")
+		})
 	}
 }

orchid-base/src/lib.rs

@@ -9,6 +9,7 @@ pub mod clone;
 pub mod combine;
 pub mod error;
 pub mod event;
+pub mod format;
 pub mod id_store;
 pub mod interner;
 pub mod join;

orchid-base/src/macros.rs

@@ -1,4 +1,5 @@
 use std::marker::PhantomData;
+use std::rc::Rc;
 use std::sync::Arc;
 
 use async_std::stream;
@@ -22,13 +23,14 @@ impl MacroSlot<'_> {
 
 trait_set! {
 	pub trait MacroAtomToApi<A> = for<'a> FnMut(&'a A) -> LocalBoxFuture<'a, api::MacroToken>;
-	pub trait MacroAtomFromApi<'a, A> = FnMut(&api::Atom) -> MTok<'a, A>;
+	pub trait MacroAtomFromApi<'a, A> =
+		for<'b> FnMut(&'b api::Atom) -> LocalBoxFuture<'b, MTok<'a, A>>;
 }
 
 #[derive(Clone, Debug)]
 pub struct MTree<'a, A> {
 	pub pos: Pos,
-	pub tok: Arc<MTok<'a, A>>,
+	pub tok: Rc<MTok<'a, A>>,
 }
 impl<'a, A> MTree<'a, A> {
 	pub(crate) async fn from_api(
@@ -38,7 +40,7 @@ impl<'a, A> MTree<'a, A> {
 	) -> Self {
 		Self {
 			pos: Pos::from_api(&api.location, i).await,
-			tok: Arc::new(MTok::from_api(&api.token, do_atom, i).await),
+			tok: Rc::new(MTok::from_api(&api.token, i, do_atom).await),
 		}
 	}
 	pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroTree {
@@ -66,17 +68,17 @@ pub enum MTok<'a, A> {
 impl<'a, A> MTok<'a, A> {
 	pub(crate) async fn from_api(
 		api: &api::MacroToken,
-		do_atom: &mut impl MacroAtomFromApi<'a, A>,
 		i: &Interner,
+		do_atom: &mut impl MacroAtomFromApi<'a, A>,
 	) -> Self {
 		match_mapping!(&api, api::MacroToken => MTok::<'a, A> {
-			Lambda(x => mtreev_from_api(x, do_atom, i).await, b => mtreev_from_api(b, do_atom, i).await),
+			Lambda(x => mtreev_from_api(x, i, do_atom).await, b => mtreev_from_api(b, i, do_atom).await),
 			Name(t => Sym::from_api(*t, i).await),
 			Slot(tk => MacroSlot(*tk, PhantomData)),
-			S(p.clone(), b => mtreev_from_api(b, do_atom, i).await),
+			S(p.clone(), b => mtreev_from_api(b, i, do_atom).await),
 			Ph(ph => Ph::from_api(ph, i).await),
 		} {
-			api::MacroToken::Atom(a) => do_atom(a)
+			api::MacroToken::Atom(a) => do_atom(a).await
 		})
 	}
 	pub(crate) async fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroToken {
@@ -93,13 +95,13 @@ impl<'a, A> MTok<'a, A> {
 			MTok::Atom(a) => do_atom(a).await,
 		})
 	}
-	pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Arc::new(self) } }
+	pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Rc::new(self) } }
 }
 
 pub async fn mtreev_from_api<'a, 'b, A>(
 	api: impl IntoIterator<Item = &'b api::MacroTree>,
-	do_atom: &mut impl MacroAtomFromApi<'a, A>,
 	i: &Interner,
+	do_atom: &mut impl MacroAtomFromApi<'a, A>,
 ) -> Vec<MTree<'a, A>> {
 	let do_atom_lk = Mutex::new(do_atom);
 	stream::from_iter(api)

orchid-base/src/msg.rs

@@ -6,7 +6,7 @@ use orchid_api_traits::{Decode, Encode};
 
 pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result<()> {
 	let mut len_buf = vec![];
-	u32::try_from(msg.len()).unwrap().encode(&mut len_buf);
+	u32::try_from(msg.len()).unwrap().encode(Pin::new(&mut len_buf)).await;
 	write.write_all(&len_buf).await?;
 	write.write_all(msg).await?;
 	write.flush().await
@@ -15,7 +15,7 @@ pub async fn send_msg(mut write: Pin<&mut impl Write>, msg: &[u8]) -> io::Result
 pub async fn recv_msg(mut read: Pin<&mut impl Read>) -> io::Result<Vec<u8>> {
 	let mut len_buf = [0u8; (u32::BITS / 8) as usize];
 	read.read_exact(&mut len_buf).await?;
-	let len = u32::decode(&mut &len_buf[..]);
+	let len = u32::decode(Pin::new(&mut &len_buf[..])).await;
 	let mut msg = vec![0u8; len as usize];
 	read.read_exact(&mut msg).await?;
 	Ok(msg)

orchid-base/src/parse.rs

@@ -33,6 +33,7 @@ impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
 	pub async fn i<T: Interned>(&self, arg: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
 		self.interner.i(arg).await
 	}
+	pub fn interner(&self) -> &'a Interner { self.interner }
 	pub fn split_at(self, pos: u32) -> (Self, Self) {
 		let Self { prev, cur, interner } = self;
 		let fst = Self { prev, cur: &cur[..pos as usize], interner };
@@ -177,7 +178,7 @@ pub async fn expect_tok<'a, 'b, A: AtomRepr, X: ExtraTok>(
 		Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
 		t => Err(mk_errv(
 			snip.i("Expected specific keyword").await,
-			format!("Expected {tok} but found {t}"),
+			format!("Expected {tok} but found {:?}", t.print().await),
 			[Pos::Range(head.range.clone()).into()],
 		)),
 	}
@@ -280,7 +281,7 @@ pub async fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
 				t => {
 					return Err(mk_errv(
 						tail.i("Unrecognized name end").await,
-						format!("Names cannot end with {t} tokens"),
+						format!("Names cannot end with {:?} tokens", t.print().await),
 						[Pos::Range(name.range.clone()).into()],
 					));
 				},

orchid-base/src/reqnot.rs

@@ -4,6 +4,7 @@ use std::future::Future;
 use std::marker::PhantomData;
 use std::mem;
 use std::ops::{BitAnd, Deref};
+use std::pin::Pin;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 
@@ -74,7 +75,7 @@ impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
 	pub async fn respond(&self, response: &impl Encode) -> Receipt<'a> {
 		assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded to {}", self.id);
 		let mut buf = (!self.id).to_be_bytes().to_vec();
-		response.encode(&mut buf);
+		response.encode(Pin::new(&mut buf)).await;
 		let mut send = clone_box(&*self.reqnot().0.lock().await.send);
 		(send)(&buf, self.parent.clone()).await;
 		Receipt(PhantomData)
@@ -126,18 +127,19 @@ impl<T: MsgSet> ReqNot<T> {
 		let mut g = self.0.lock().await;
 		let (id, payload) = get_id(message);
 		if id == 0 {
-			let mut notif = clone_box(&*g.notif);
+			let mut notif_cb = clone_box(&*g.notif);
 			mem::drop(g);
-			notif(<T::In as Channel>::Notif::decode(&mut &payload[..]), self.clone()).await
+			let notif_val = <T::In as Channel>::Notif::decode(Pin::new(&mut &payload[..])).await;
+			notif_cb(notif_val, self.clone()).await
 		} else if 0 < id.bitand(1 << 63) {
 			let sender = g.responses.remove(&!id).expect("Received response for invalid message");
 			sender.send(message.to_vec()).await.unwrap();
 		} else {
-			let message = <T::In as Channel>::Req::decode(&mut &payload[..]);
+			let message = <T::In as Channel>::Req::decode(Pin::new(&mut &payload[..])).await;
-			let mut req = clone_box(&*g.req);
+			let mut req_cb = clone_box(&*g.req);
 			mem::drop(g);
 			let rn = self.clone();
-			req(RequestHandle::new(rn, id), message).await;
+			req_cb(RequestHandle::new(rn, id), message).await;
 		}
 	}
 
@@ -145,7 +147,7 @@ impl<T: MsgSet> ReqNot<T> {
 		let mut send = clone_box(&*self.0.lock().await.send);
 		let mut buf = vec![0; 8];
 		let msg: <T::Out as Channel>::Notif = notif.into();
-		msg.encode(&mut buf);
+		msg.encode(Pin::new(&mut buf)).await;
 		send(&buf, self.clone()).await
 	}
 }
@@ -180,7 +182,7 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
 			let id = g.id;
 			g.id += 1;
 			let mut buf = id.to_be_bytes().to_vec();
-			req.encode(&mut buf);
+			req.encode(Pin::new(&mut buf)).await;
 			let (send, recv) = channel::bounded(1);
 			g.responses.insert(id, send);
 			let mut send = clone_box(&*g.send);
@@ -206,7 +208,7 @@ pub trait Requester: DynRequester {
 }
 impl<This: DynRequester + ?Sized> Requester for This {
 	async fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
-		R::Response::decode(&mut &self.raw_request(data.into()).await[..])
+		R::Response::decode(Pin::new(&mut &self.raw_request(data.into()).await[..])).await
 	}
 }
 

orchid-base/src/tree.rs

@@ -1,6 +1,6 @@
 use std::borrow::Borrow;
-use std::cell::RefCell;
 use std::fmt::{self, Debug, Display};
+use std::future::{Future, ready};
 use std::iter;
 use std::marker::PhantomData;
 use std::ops::Range;
@@ -9,7 +9,7 @@ use std::sync::Arc;
 pub use api::PhKind;
 use async_std::stream;
 use async_std::sync::Mutex;
-use futures::future::LocalBoxFuture;
+use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
 use never::Never;
@@ -47,15 +47,22 @@ pub fn recur<'a, A: AtomRepr, X: ExtraTok>(
 	})
 }
 
-pub trait AtomRepr: fmt::Display + Clone + fmt::Debug {
+pub trait AtomRepr: Clone {
 	type Ctx: ?Sized;
-	fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self;
+	fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> impl Future<Output = Self>;
-	fn to_api(&self) -> orchid_api::Atom;
+	fn to_api(&self) -> impl Future<Output = orchid_api::Atom> + '_;
+	fn print(&self) -> impl Future<Output = String> + '_;
 }
 impl AtomRepr for Never {
 	type Ctx = Never;
-	fn from_api(_: &api::Atom, _: Pos, _: &mut Self::Ctx) -> Self { panic!() }
+	#[allow(unreachable_code)]
-	fn to_api(&self) -> orchid_api::Atom { match *self {} }
+	fn from_api(_: &api::Atom, _: Pos, ctx: &mut Self::Ctx) -> impl Future<Output = Self> {
+		ready(match *ctx {})
+	}
+	#[allow(unreachable_code)]
+	fn to_api(&self) -> impl Future<Output = orchid_api::Atom> + '_ { ready(match *self {}) }
+	#[allow(unreachable_code)]
+	fn print(&self) -> impl Future<Output = String> + '_ { ready(match *self {}) }
 }
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
@@ -79,7 +86,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 	pub async fn from_api(tt: &api::TokenTree, ctx: &mut A::Ctx, i: &Interner) -> Self {
 		let tok = match_mapping!(&tt.token, api::Token => Token::<'b, A, X> {
 			BR, NS,
-			Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx)),
+			Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx).await),
 			Bottom(e => OrcErrv::from_api(e, i).await),
 			LambdaHead(arg => ttv_from_api(arg, ctx, i).await),
 			Name(n => Tok::from_api(*n, i).await),
@@ -94,7 +101,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 
 	pub async fn to_api(&self, do_extra: &mut impl RefDoExtra<X>) -> api::TokenTree {
 		let token = match_mapping!(&self.tok, Token => api::Token {
-			Atom(a.to_api()),
+			Atom(a.to_api().await),
 			BR,
 			NS,
 			Bottom(e.to_api()),
@@ -111,20 +118,20 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 		api::TokenTree { range: self.range.clone(), token }
 	}
 
-	pub fn into_api(
+	pub async fn into_api(
 		self,
 		do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
 	) -> api::TokenTree {
 		let token = match self.tok {
-			Token::Atom(a) => api::Token::Atom(a.to_api()),
+			Token::Atom(a) => api::Token::Atom(a.to_api().await),
 			Token::BR => api::Token::BR,
 			Token::NS => api::Token::NS,
 			Token::Bottom(e) => api::Token::Bottom(e.to_api()),
 			Token::Comment(c) => api::Token::Comment(c.clone()),
-			Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra)),
+			Token::LambdaHead(arg) => api::Token::LambdaHead(ttv_into_api(arg, do_extra).await),
 			Token::Name(n) => api::Token::Name(n.to_api()),
 			Token::Slot(tt) => api::Token::Slot(tt.ticket()),
-			Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra)),
+			Token::S(p, b) => api::Token::S(p, ttv_into_api(b, do_extra).await),
 			Token::Ph(Ph { kind, name }) =>
 				api::Token::Ph(api::Placeholder { name: name.to_api(), kind }),
 			Token::X(x) => return do_extra(x, self.range.clone()),
@@ -146,10 +153,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 		body.insert(0, Token::LambdaHead(arg).at(arg_range));
 		Token::S(Paren::Round, body).at(s_range)
 	}
-}
+	pub async fn print(&self) -> String { self.tok.print().await }
-
-impl<A: AtomRepr, X: ExtraTok> Display for TokTree<'_, A, X> {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
 }
 
 pub async fn ttv_from_api<A: AtomRepr, X: ExtraTok>(
@@ -161,7 +165,7 @@ pub async fn ttv_from_api<A: AtomRepr, X: ExtraTok>(
 	stream::from_iter(tokv.into_iter())
 		.then(|t| async {
 			let t = t;
-			TokTree::<A, X>::from_api(t.borrow(), *ctx_lk.lock().await, i).await
+			TokTree::<A, X>::from_api(t.borrow(), *ctx_lk.lock().await, i).boxed_local().await
 		})
 		.collect()
 		.await
@@ -178,11 +182,15 @@ pub async fn ttv_to_api<'a, A: AtomRepr, X: ExtraTok>(
 	output
 }
 
-pub fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>(
+pub async fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>(
 	tokv: impl IntoIterator<Item = TokTree<'a, A, X>>,
 	do_extra: &mut impl FnMut(X, Range<u32>) -> api::TokenTree,
 ) -> Vec<api::TokenTree> {
-	tokv.into_iter().map(|t| t.into_api(do_extra)).collect_vec()
+	let mut new_tokv = Vec::new();
+	for item in tokv {
+		new_tokv.push(item.into_api(do_extra).await)
+	}
+	new_tokv
 }
 
 /// This takes a position and not a range because it assigns the range to
@@ -237,50 +245,32 @@ impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
 			_ => None,
 		}
 	}
-}
+	pub async fn print(&self) -> String {
-impl<A: AtomRepr, X: ExtraTok> Display for Token<'_, A, X> {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		thread_local! {
-			static PAREN_LEVEL: RefCell<usize> = 0.into();
-		}
-		fn get_indent() -> usize { PAREN_LEVEL.with_borrow(|t| *t) }
-		fn with_indent<T>(f: impl FnOnce() -> T) -> T {
-			PAREN_LEVEL.with_borrow_mut(|t| *t += 1);
-			let r = f();
-			PAREN_LEVEL.with_borrow_mut(|t| *t -= 1);
-			r
-		}
 		match self {
-			Self::Atom(a) => f.write_str(&indent(&format!("{a} "), get_indent(), false)),
+			Self::Atom(a) => a.print().await,
-			Self::BR => write!(f, "\n{}", "  ".repeat(get_indent())),
+			Self::BR => "\n".to_string(),
-			Self::Bottom(err) if err.len() == 1 => write!(f, "Bottom({}) ", err.one().unwrap()),
+			Self::Bottom(err) if err.len() == 1 => format!("Bottom({}) ", err.one().unwrap()),
-			Self::Bottom(err) => {
+			Self::Bottom(err) => format!("Botttom(\n{}) ", indent(&err.to_string())),
-				write!(f, "Botttom(\n{}) ", indent(&err.to_string(), get_indent() + 1, true))
+			Self::Comment(c) => format!("--[{c}]-- "),
-			},
+			Self::LambdaHead(arg) => format!("\\ {} . ", indent(&ttv_fmt(arg).await)),
-			Self::Comment(c) => write!(f, "--[{c}]-- "),
+			Self::NS => ":: ".to_string(),
-			Self::LambdaHead(arg) => with_indent(|| write!(f, "\\ {} . ", ttv_fmt(arg))),
+			Self::Name(n) => format!("{n} "),
-			Self::NS => f.write_str(":: "),
+			Self::Slot(th) => format!("{th} "),
-			Self::Name(n) => write!(f, "{n} "),
-			Self::Slot(th) => write!(f, "{th} "),
 			Self::Ph(Ph { kind, name }) => match &kind {
-				PhKind::Scalar => write!(f, "${name}"),
+				PhKind::Scalar => format!("${name}"),
 				PhKind::Vector { at_least_one, priority } => {
-					if *at_least_one {
+					let prefix = if *at_least_one { "..." } else { ".." };
-						write!(f, ".")?
+					let suffix = if 0 < *priority { format!(":{priority}") } else { String::new() };
-					}
+					format!("{prefix}${name}{suffix}")
-					write!(f, "..${name}")?;
-					if 0 < *priority { write!(f, "{priority}") } else { Ok(()) }
 				},
 			},
 			Self::S(p, b) => {
 				let (lp, rp, _) = PARENS.iter().find(|(_, _, par)| par == p).unwrap();
-				write!(f, "{lp} ")?;
+				format!("{lp} {}{rp} ", indent(&ttv_fmt(b).await))
-				with_indent(|| f.write_str(&ttv_fmt(b)))?;
-				write!(f, "{rp} ")
 			},
-			Self::X(x) => write!(f, "{x} "),
+			Self::X(x) => format!("{x} "),
-			Self::Macro(None) => write!(f, "macro "),
+			Self::Macro(None) => "macro ".to_string(),
-			Self::Macro(Some(prio)) => write!(f, "macro({prio})"),
+			Self::Macro(Some(prio)) => format!("macro({prio})"),
 		}
 	}
 }
@@ -290,21 +280,13 @@ pub fn ttv_range(ttv: &[TokTree<'_, impl AtomRepr, impl ExtraTok>]) -> Range<u32
 	ttv.first().unwrap().range.start..ttv.last().unwrap().range.end
 }
 
-pub fn ttv_fmt<'a: 'b, 'b>(
+pub async fn ttv_fmt<'a: 'b, 'b>(
 	ttv: impl IntoIterator<Item = &'b TokTree<'a, impl AtomRepr + 'b, impl ExtraTok + 'b>>,
 ) -> String {
-	ttv.into_iter().join("")
+	join_all(ttv.into_iter().map(|tt| tt.print())).await.join("")
 }
 
-pub fn indent(s: &str, lvl: usize, first: bool) -> String {
+pub fn indent(s: &str) -> String { s.replace("\n", "\n  ") }
-	if first {
-		s.replace("\n", &("\n".to_string() + &"  ".repeat(lvl)))
-	} else if let Some((fst, rest)) = s.split_once('\n') {
-		fst.to_string() + "\n" + &indent(rest, lvl, true)
-	} else {
-		s.to_string()
-	}
-}
 
 #[derive(Clone, Debug)]
 pub struct Ph {

orchid-extension/src/atom.rs

@@ -1,12 +1,13 @@
 use std::any::{Any, TypeId, type_name};
 use std::fmt;
 use std::future::Future;
-use std::io::{Read, Write};
 use std::marker::PhantomData;
 use std::ops::Deref;
+use std::pin::Pin;
 use std::rc::Rc;
 
 use ahash::HashMap;
+use async_std::io::{Read, Write};
 use async_std::stream;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::LocalBoxFuture;
@@ -34,7 +35,7 @@ pub trait AtomCard: 'static + Sized {
 pub trait AtomicVariant {}
 pub trait Atomic: 'static + Sized {
 	type Variant: AtomicVariant;
-	type Data: Clone + Coding + Sized;
+	type Data: Clone + Coding + Sized + 'static;
 	fn reg_reqs() -> MethodSetBuilder<Self>;
 }
 impl<A: Atomic> AtomCard for A {
@@ -102,10 +103,10 @@ impl ForeignAtom<'static> {
 		let rep = (self.ctx.reqnot.request(api::Fwd(
 			self.atom.clone(),
 			Sym::parse(M::NAME, &self.ctx.i).await.unwrap().tok().to_api(),
-			enc_vec(&m),
+			enc_vec(&m).await,
 		)))
 		.await?;
-		Some(M::Response::decode(&mut &rep[..]))
+		Some(M::Response::decode(Pin::new(&mut &rep[..])).await)
 	}
 }
 impl fmt::Display for ForeignAtom<'_> {
@@ -118,10 +119,13 @@ impl fmt::Debug for ForeignAtom<'_> {
 }
 impl AtomRepr for ForeignAtom<'_> {
 	type Ctx = SysCtx;
-	fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self {
+	async fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self {
 		Self { atom: atom.clone(), _life: PhantomData, ctx: ctx.clone(), expr: None, pos }
 	}
-	fn to_api(&self) -> orchid_api::Atom { self.atom.clone() }
+	async fn to_api(&self) -> orchid_api::Atom { self.atom.clone() }
+	async fn print(&self) -> String {
+		self.ctx.reqnot.request(api::ExtAtomPrint(self.atom.clone())).await
+	}
 }
 
 pub struct NotTypAtom {
@@ -151,8 +155,8 @@ trait_set! {
 	trait AtomReqCb<A> = for<'a> Fn(
 		&'a A,
 		SysCtx,
-		&'a mut dyn Read,
+		Pin<&'a mut dyn Read>,
-		&'a mut dyn Write
+		Pin<&'a mut dyn Write>,
 	) -> LocalBoxFuture<'a, ()>
 }
 
@@ -167,8 +171,9 @@ impl<A: AtomCard> MethodSetBuilder<A> {
 		assert!(!M::NAME.is_empty(), "AtomMethod::NAME cannoot be empty");
 		self.handlers.push((
 			M::NAME,
-			Rc::new(move |a: &A, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write| {
+			Rc::new(move |a: &A, ctx: SysCtx, req: Pin<&mut dyn Read>, rep: Pin<&mut dyn Write>| {
-				async { Supports::<M>::handle(a, ctx, M::decode(req)).await.encode(rep) }.boxed_local()
+				async { Supports::<M>::handle(a, ctx, M::decode(req).await).await.encode(rep).await }
+					.boxed_local()
 			}),
 		));
 		self
@@ -197,8 +202,8 @@ impl<A: AtomCard> MethodSet<A> {
 		atom: &'a A,
 		ctx: SysCtx,
 		key: Sym,
-		req: &'a mut dyn Read,
+		req: Pin<&'a mut dyn Read>,
-		rep: &'a mut dyn Write,
+		rep: Pin<&'a mut dyn Write>,
 	) -> bool {
 		match self.handlers.get(&key) {
 			None => false,
@@ -228,14 +233,14 @@ impl<A: AtomicFeatures> TypAtom<'static, A> {
 				expr,
 				typ: Box::new(A::info()),
 			}),
-			Ok(atm) => match downcast_atom::<A>(atm) {
+			Ok(atm) => match downcast_atom::<A>(atm).await {
+				Ok(tatom) => Ok(tatom),
 				Err(fa) => Err(NotTypAtom {
 					pos: fa.pos.clone(),
 					ctx: fa.ctx.clone(),
 					expr: fa.ex(),
 					typ: Box::new(A::info()),
 				}),
-				Ok(tatom) => Ok(tatom),
 			},
 		}
 	}
@@ -243,15 +248,16 @@ impl<A: AtomicFeatures> TypAtom<'static, A> {
 impl<A: AtomicFeatures> TypAtom<'_, A> {
 	pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
 	where A: Supports<M> {
-		M::Response::decode(
+		M::Response::decode(Pin::new(
 			&mut &(self.data.ctx.reqnot.request(api::Fwd(
 				self.data.atom.clone(),
 				Sym::parse(M::NAME, &self.data.ctx.i).await.unwrap().tok().to_api(),
-				enc_vec(&req),
+				enc_vec(&req).await,
 			)))
 			.await
 			.unwrap()[..],
-		)
+		))
+		.await
 	}
 }
 impl<A: AtomicFeatures> Deref for TypAtom<'_, A> {
@@ -264,7 +270,7 @@ pub struct AtomCtx<'a>(pub &'a [u8], pub Option<api::AtomId>, pub SysCtx);
 pub trait AtomDynfo: 'static {
 	fn tid(&self) -> TypeId;
 	fn name(&self) -> &'static str;
-	fn decode(&self, ctx: AtomCtx<'_>) -> Box<dyn Any>;
+	fn decode<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>>;
 	fn call<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>;
 	fn call_ref<'a>(&'a self, ctx: AtomCtx<'a>, arg: api::ExprTicket) -> LocalBoxFuture<'a, GExpr>;
 	fn print<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, String>;
@@ -272,14 +278,14 @@ pub trait AtomDynfo: 'static {
 		&'a self,
 		ctx: AtomCtx<'a>,
 		key: Sym,
-		req: &'b mut dyn Read,
+		req: Pin<&'b mut dyn Read>,
-		rep: &'c mut dyn Write,
+		rep: Pin<&'c mut dyn Write>,
 	) -> LocalBoxFuture<'a, bool>;
 	fn command<'a>(&'a self, ctx: AtomCtx<'a>) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>>;
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
-		write: &'b mut dyn Write,
+		write: Pin<&'b mut dyn Write>,
 	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>>;
 	fn deserialize<'a>(
 		&'a self,

orchid-extension/src/atom_owned.rs

@@ -1,10 +1,11 @@
 use std::any::{Any, TypeId, type_name};
 use std::borrow::Cow;
 use std::future::Future;
-use std::io::{Read, Write};
+use std::pin::Pin;
 use std::rc::Rc;
 
 use async_once_cell::OnceCell;
+use async_std::io::{Read, Write};
 use futures::FutureExt;
 use futures::future::{LocalBoxFuture, ready};
 use itertools::Itertools;
@@ -31,8 +32,8 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
 		AtomFactory::new(move |ctx| async move {
 			let rec = ctx.obj_store.add(Box::new(self));
 			let (id, _) = get_info::<A>(ctx.cted.inst().card());
-			let mut data = enc_vec(&id);
+			let mut data = enc_vec(&id).await;
-			rec.encode(&mut data).await;
+			rec.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
 			api::Atom { drop: Some(api::AtomId(rec.id())), data, owner: ctx.id }
 		})
 	}
@@ -55,8 +56,11 @@ pub struct OwnedAtomDynfo<T: OwnedAtom> {
 impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	fn tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn name(&self) -> &'static str { type_name::<T>() }
-	fn decode(&self, AtomCtx(data, ..): AtomCtx) -> Box<dyn Any> {
+	fn decode<'a>(&'a self, AtomCtx(data, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
-		Box::new(<T as AtomCard>::Data::decode(&mut &data[..]))
+		async {
+			Box::new(<T as AtomCard>::Data::decode(Pin::new(&mut &data[..])).await) as Box<dyn Any>
+		}
+		.boxed_local()
 	}
 	fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
 		with_atom(id.unwrap(), &ctx, |a| a.remove()).dyn_call(ctx.clone(), arg)
@@ -82,8 +86,8 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx,
 		key: Sym,
-		req: &'b mut dyn Read,
+		req: Pin<&'b mut dyn Read>,
-		rep: &'c mut dyn Write,
+		rep: Pin<&'c mut dyn Write>,
 	) -> LocalBoxFuture<'a, bool> {
 		async move {
 			with_atom(id.unwrap(), &ctx, |a| {
@@ -110,11 +114,11 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
-		write: &'b mut dyn Write,
+		mut write: Pin<&'b mut dyn Write>,
 	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>> {
 		async move {
 			let id = id.unwrap();
-			id.encode(write);
+			id.encode(write.as_mut()).await;
 			with_atom(id, &ctx, |a| clone!(ctx; async move { a.dyn_serialize(ctx, write).await }))
 				.await
 				.map(|v| v.into_iter().map(|t| t.handle().tk).collect_vec())
@@ -138,20 +142,22 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 }
 
 pub trait DeserializeCtx: Sized {
-	fn read<T: Decode>(&mut self) -> T;
+	fn read<T: Decode>(&mut self) -> impl Future<Output = T>;
 	fn is_empty(&self) -> bool;
 	fn assert_empty(self) { assert!(self.is_empty(), "Bytes found after decoding") }
-	fn decode<T: Decode>(mut self) -> T {
+	fn decode<T: Decode>(mut self) -> impl Future<Output = T> {
-		let t = self.read();
+		async {
+			let t = self.read().await;
 			self.assert_empty();
 			t
 		}
+	}
 	fn sys(&self) -> SysCtx;
 }
 
 struct DeserCtxImpl<'a>(&'a [u8], &'a SysCtx);
 impl DeserializeCtx for DeserCtxImpl<'_> {
-	fn read<T: Decode>(&mut self) -> T { T::decode(&mut self.0) }
+	async fn read<T: Decode>(&mut self) -> T { T::decode(Pin::new(&mut self.0)).await }
 	fn is_empty(&self) -> bool { self.0.is_empty() }
 	fn sys(&self) -> SysCtx { self.1.clone() }
 }
@@ -228,7 +234,7 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	fn serialize(
 		&self,
 		ctx: SysCtx,
-		write: &mut (impl Write + ?Sized),
+		write: Pin<&mut (impl Write + ?Sized)>,
 	) -> impl Future<Output = Self::Refs> {
 		assert_serializable::<Self>();
 		async { panic!("Either implement serialize or set Refs to Never for {}", type_name::<Self>()) }
@@ -250,7 +256,7 @@ fn assert_serializable<T: OwnedAtom>() {
 pub trait DynOwnedAtom: 'static {
 	fn atom_tid(&self) -> TypeId;
 	fn as_any_ref(&self) -> &dyn Any;
-	fn encode<'a>(&'a self, buffer: &'a mut dyn Write) -> LocalBoxFuture<'a, ()>;
+	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()>;
 	fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr>;
 	fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket)
 	-> LocalBoxFuture<'static, GExpr>;
@@ -260,14 +266,14 @@ pub trait DynOwnedAtom: 'static {
 	fn dyn_serialize<'a>(
 		&'a self,
 		ctx: SysCtx,
-		sink: &'a mut dyn Write,
+		sink: Pin<&'a mut dyn Write>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>>;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn as_any_ref(&self) -> &dyn Any { self }
-	fn encode<'a>(&'a self, buffer: &'a mut dyn Write) -> LocalBoxFuture<'a, ()> {
+	fn encode<'a>(&'a self, buffer: Pin<&'a mut dyn Write>) -> LocalBoxFuture<'a, ()> {
-		async { self.val().await.as_ref().encode(buffer) }.boxed_local()
+		async { self.val().await.as_ref().encode(buffer).await }.boxed_local()
 	}
 	fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
 		self.call_ref(ExprHandle::from_args(ctx, arg)).boxed_local()
@@ -289,7 +295,7 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	fn dyn_serialize<'a>(
 		&'a self,
 		ctx: SysCtx,
-		sink: &'a mut dyn Write,
+		sink: Pin<&'a mut dyn Write>,
 	) -> LocalBoxFuture<'a, Option<Vec<Expr>>> {
 		match TypeId::of::<Never>() == TypeId::of::<<Self as OwnedAtom>::Refs>() {
 			true => ready(None).boxed_local(),

orchid-extension/src/atom_thin.rs

@@ -1,8 +1,9 @@
 use std::any::{Any, TypeId, type_name};
 use std::future::Future;
-use std::io::Write;
+use std::pin::Pin;
 
 use async_once_cell::OnceCell;
+use async_std::io::{Read, Write};
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
 use orchid_api_traits::{Coding, enc_vec};
@@ -24,8 +25,8 @@ impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant
 	fn _factory(self) -> AtomFactory {
 		AtomFactory::new(move |ctx| async move {
 			let (id, _) = get_info::<A>(ctx.cted.inst().card());
-			let mut buf = enc_vec(&id);
+			let mut buf = enc_vec(&id).await;
-			self.encode(&mut buf);
+			self.encode(Pin::new(&mut buf)).await;
 			api::Atom { drop: None, data: buf, owner: ctx.id }
 		})
 	}
@@ -39,53 +40,58 @@ pub struct ThinAtomDynfo<T: ThinAtom> {
 }
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 	fn print<'a>(&self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, String> {
-		async move { T::decode(&mut &buf[..]).print(ctx).await }.boxed_local()
+		async move { T::decode(Pin::new(&mut &buf[..])).await.print(ctx).await }.boxed_local()
 	}
 	fn tid(&self) -> TypeId { TypeId::of::<T>() }
 	fn name(&self) -> &'static str { type_name::<T>() }
-	fn decode(&self, AtomCtx(buf, ..): AtomCtx) -> Box<dyn Any> { Box::new(T::decode(&mut &buf[..])) }
+	fn decode<'a>(&'a self, AtomCtx(buf, ..): AtomCtx<'a>) -> LocalBoxFuture<'a, Box<dyn Any>> {
+		async { Box::new(T::decode(Pin::new(&mut &buf[..])).await) as Box<dyn Any> }.boxed_local()
+	}
 	fn call<'a>(
 		&'a self,
 		AtomCtx(buf, _, ctx): AtomCtx<'a>,
 		arg: api::ExprTicket,
 	) -> LocalBoxFuture<'a, GExpr> {
-		async move { T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg)).await }
+		Box::pin(async move {
-			.boxed_local()
+			T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await
+		})
 	}
 	fn call_ref<'a>(
 		&'a self,
 		AtomCtx(buf, _, ctx): AtomCtx<'a>,
 		arg: api::ExprTicket,
 	) -> LocalBoxFuture<'a, GExpr> {
-		async move { T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg)).await }
+		Box::pin(async move {
-			.boxed_local()
+			T::decode(Pin::new(&mut &buf[..])).await.call(ExprHandle::from_args(ctx, arg)).await
+		})
 	}
 	fn handle_req<'a, 'm1: 'a, 'm2: 'a>(
 		&'a self,
 		AtomCtx(buf, _, sys): AtomCtx<'a>,
 		key: Sym,
-		req: &'m1 mut dyn std::io::Read,
+		req: Pin<&'m1 mut dyn Read>,
-		rep: &'m2 mut dyn Write,
+		rep: Pin<&'m2 mut dyn Write>,
 	) -> LocalBoxFuture<'a, bool> {
-		async move {
+		Box::pin(async move {
 			let ms = self.ms.get_or_init(self.msbuild.pack(sys.clone())).await;
-			ms.dispatch(&T::decode(&mut &buf[..]), sys, key, req, rep).await
+			ms.dispatch(&T::decode(Pin::new(&mut &buf[..])).await, sys, key, req, rep).await
-		}
+		})
-		.boxed_local()
 	}
 	fn command<'a>(
 		&'a self,
 		AtomCtx(buf, _, ctx): AtomCtx<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>> {
-		async move { T::decode(&mut &buf[..]).command(ctx).await }.boxed_local()
+		async move { T::decode(Pin::new(&mut &buf[..])).await.command(ctx).await }.boxed_local()
 	}
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
 		ctx: AtomCtx<'a>,
-		write: &'b mut dyn Write,
+		write: Pin<&'b mut dyn Write>,
 	) -> LocalBoxFuture<'a, Option<Vec<api::ExprTicket>>> {
-		T::decode(&mut &ctx.0[..]).encode(write);
+		Box::pin(async {
-		async { Some(Vec::new()) }.boxed_local()
+			T::decode(Pin::new(&mut &ctx.0[..])).await.encode(write).await;
+			Some(Vec::new())
+		})
 	}
 	fn deserialize<'a>(
 		&'a self,
@@ -94,11 +100,11 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
 		refs: &'a [api::ExprTicket],
 	) -> LocalBoxFuture<'a, api::Atom> {
 		assert!(refs.is_empty(), "Refs found when deserializing thin atom");
-		async { T::decode(&mut &data[..])._factory().build(ctx).await }.boxed_local()
+		async { T::decode(Pin::new(&mut &data[..])).await._factory().build(ctx).await }.boxed_local()
 	}
 	fn drop<'a>(&'a self, AtomCtx(buf, _, ctx): AtomCtx<'a>) -> LocalBoxFuture<'a, ()> {
 		async move {
-			let string_self = T::decode(&mut &buf[..]).print(ctx.clone()).await;
+			let string_self = T::decode(Pin::new(&mut &buf[..])).await.print(ctx.clone()).await;
 			writeln!(ctx.logger, "Received drop signal for non-drop atom {string_self:?}");
 		}
 		.boxed_local()

orchid-extension/src/conv.rs

@@ -35,7 +35,7 @@ impl<A: AtomicFeatures> TryFromExpr for TypAtom<'_, A> {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
 		match expr.atom().await {
 			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), &ex.ctx().i).await.into()),
-			Ok(f) => match downcast_atom(f) {
+			Ok(f) => match downcast_atom(f).await {
 				Ok(a) => Ok(a),
 				Err(f) => Err(err_type(f.pos(), &f.ctx().i).await.into()),
 			},

orchid-extension/src/entrypoint.rs

@@ -3,6 +3,7 @@ use std::future::Future;
 use std::io::Write;
 use std::mem;
 use std::num::NonZero;
+use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -86,7 +87,7 @@ pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
 	let mut data = &atom.data[..];
 	let ctx = get_sys_ctx(atom.owner, reqnot).await;
 	let inst = ctx.cted.inst();
-	let id = api::AtomId::decode(&mut data);
+	let id = api::AtomId::decode(Pin::new(&mut data)).await;
 	let atom_record = atom_by_idx(inst.card(), id).expect("Atom ID reserved");
 	cb(atom_record, ctx, id, data).await
 }
@@ -114,23 +115,30 @@ impl ExtPort for ExtensionOwner {
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
 		self.rn.receive(msg).boxed_local()
 	}
-	fn recv<'a, 's: 'a>(
+	fn recv<'a>(
-		&'s self,
+		&'a self,
-		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'a, ()> + 'a>,
+		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
 	) -> LocalBoxFuture<'a, ()> {
-		async { cb(&self.out_recv.recv().await.unwrap()[..]).await }.boxed_local()
+		async {
+			let msg = self.out_recv.recv().await.unwrap();
+			cb(&msg[..]).await
+		}
+		.boxed_local()
 	}
 }
 
 async fn extension_main_logic(data: ExtensionData, spawner: Rc<dyn LocalSpawn>) {
-	let api::HostHeader { log_strategy } = api::HostHeader::decode(&mut std::io::stdin().lock());
+	let api::HostHeader { log_strategy } =
+		api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await;
 	let mut buf = Vec::new();
 	let decls = (data.systems.iter().enumerate())
 		.map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
 		.map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
 		.collect_vec();
 	let systems = Rc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
-	api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() }.encode(&mut buf);
+	api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() }
+		.encode(Pin::new(&mut buf))
+		.await;
 	std::io::stdout().write_all(&buf).unwrap();
 	std::io::stdout().flush().unwrap();
 	let exiting = Arc::new(AtomicBool::new(false));
@@ -333,8 +341,8 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc<dyn LocalSpawn>)
 									let actx = AtomCtx(buf, atom.drop, ctx.clone());
 									match &atom_req {
 										api::AtomReq::SerializeAtom(ser) => {
-											let mut buf = enc_vec(&id);
+											let mut buf = enc_vec(&id).await;
-											let refs_opt = nfo.serialize(actx, &mut buf).await;
+											let refs_opt = nfo.serialize(actx, Pin::<&mut Vec<_>>::new(&mut buf)).await;
 											hand.handle(ser, &refs_opt.map(|refs| (buf, refs))).await
 										},
 										api::AtomReq::AtomPrint(print @ api::AtomPrint(_)) =>
@@ -343,7 +351,14 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc<dyn LocalSpawn>)
 											let api::Fwded(_, key, payload) = &fwded;
 											let mut reply = Vec::new();
 											let key = Sym::from_api(*key, &i).await;
-											let some = nfo.handle_req(actx, key, &mut &payload[..], &mut reply).await;
+											let some = nfo
+												.handle_req(
+													actx,
+													key,
+													Pin::<&mut &[u8]>::new(&mut &payload[..]),
+													Pin::<&mut Vec<_>>::new(&mut reply),
+												)
+												.await;
 											hand.handle(fwded, &some.then_some(reply)).await
 										},
 										api::AtomReq::CallRef(call @ api::CallRef(_, arg)) => {
@@ -374,7 +389,7 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc<dyn LocalSpawn>)
 							let api::DeserAtom(sys, buf, refs) = &deser;
 							let mut read = &mut &buf[..];
 							let ctx = mk_ctx(*sys, hand.reqnot()).await;
-							let id = api::AtomId::decode(&mut read);
+							let id = api::AtomId::decode(Pin::new(&mut read)).await;
 							let inst = ctx.cted.inst();
 							let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
 							hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, refs).await).await
@@ -388,7 +403,7 @@ async fn extension_main_logic(data: ExtensionData, spawner: Rc<dyn LocalSpawn>)
 							for (k, v) in params {
 								ctx.args.insert(
 									Tok::from_api(k, &i).await,
-									mtreev_from_api(&v, &mut |_| panic!("No atom in macro prompt!"), &i).await,
+									mtreev_from_api(&v, &i, &mut |_| panic!("No atom in macro prompt!")).await,
 								);
 							}
 							let err_cascade = err_cascade(&i).await;

orchid-extension/src/expr.rs

@@ -21,22 +21,16 @@ pub struct ExprHandle {
 impl ExprHandle {
 	pub(crate) fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } }
 	pub fn get_ctx(&self) -> SysCtx { self.ctx.clone() }
+	pub async fn clone(&self) -> Self {
+		self.ctx.reqnot.notify(api::Acquire(self.ctx.id, self.tk)).await;
+		Self { ctx: self.ctx.clone(), tk: self.tk }
+	}
 }
 impl fmt::Debug for ExprHandle {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 		write!(f, "ExprHandle({})", self.tk.0)
 	}
 }
-impl Clone for ExprHandle {
-	fn clone(&self) -> Self {
-		let SysCtx { reqnot, spawner, .. } = self.ctx.clone();
-		let notif = api::Acquire(self.ctx.id, self.tk);
-		if let Err(e) = spawner.spawn_local(async move { reqnot.notify(notif).await }) {
-			panic!("Failed to schedule cloning notification, resource may not exist: {e}");
-		}
-		Self { ctx: self.ctx.clone(), tk: self.tk }
-	}
-}
 impl Drop for ExprHandle {
 	fn drop(&mut self) {
 		let notif = api::Release(self.ctx.id, self.tk);

orchid-extension/src/func_atom.rs

@@ -1,9 +1,10 @@
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::future::Future;
-use std::io;
+use std::pin::Pin;
 use std::rc::Rc;
 
+use async_std::io::Write;
 use async_std::sync::Mutex;
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
@@ -79,13 +80,13 @@ impl OwnedAtom for Fun {
 		}
 	}
 	async fn call(self, arg: ExprHandle) -> GExpr { self.call_ref(arg).await }
-	async fn serialize(&self, _: SysCtx, sink: &mut (impl io::Write + ?Sized)) -> Self::Refs {
+	async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs {
-		self.path.to_api().encode(sink);
+		self.path.to_api().encode(write).await;
 		self.args.clone()
 	}
 	async fn deserialize(ctx: impl DeserializeCtx, args: Self::Refs) -> Self {
 		let sys = ctx.sys();
-		let path = Sym::from_api(ctx.decode(), &sys.i).await;
+		let path = Sym::from_api(ctx.decode().await, &sys.i).await;
 		let (arity, fun) = FUNS.with(|f| f.clone()).lock().await.get(&path).unwrap().clone();
 		Self { args, arity, path, fun }
 	}

orchid-extension/src/macros.rs

@@ -48,7 +48,7 @@ impl<'a> RuleCtx<'a> {
 			return Err(err_cascade(&self.sys.i).await.into());
 		};
 		static ATOM_MSG: &str = "Returned atom from Rule recursion";
-		Ok(mtreev_from_api(&treev, &mut |_| panic!("{ATOM_MSG}"), &self.sys.i).await)
+		Ok(mtreev_from_api(&treev, &self.sys.i, &mut |_| panic!("{ATOM_MSG}")).await)
 	}
 	pub fn getv(&mut self, key: &Tok<String>) -> Vec<MTree<'a, Never>> {
 		self.args.remove(key).expect("Key not found")

orchid-extension/src/system.rs

@@ -2,10 +2,10 @@ use core::fmt;
 use std::any::TypeId;
 use std::future::Future;
 use std::num::NonZero;
+use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
 
-use futures::FutureExt;
 use futures::future::LocalBoxFuture;
 use futures::task::LocalSpawn;
 use hashbrown::HashMap;
@@ -68,8 +68,11 @@ pub fn atom_by_idx(
 	}
 }
 
-pub fn resolv_atom(sys: &(impl DynSystemCard + ?Sized), atom: &api::Atom) -> Box<dyn AtomDynfo> {
+pub async fn resolv_atom(
-	let tid = api::AtomId::decode(&mut &atom.data[..8]);
+	sys: &(impl DynSystemCard + ?Sized),
+	atom: &api::Atom,
+) -> Box<dyn AtomDynfo> {
+	let tid = api::AtomId::decode(Pin::new(&mut &atom.data[..8])).await;
 	atom_by_idx(sys, tid).expect("Value of nonexistent type found")
 }
 
@@ -102,21 +105,25 @@ impl<T: System> DynSystem for T {
 	fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
 	fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
 	fn dyn_request<'a>(&self, hand: ExtReq<'a>, req: Vec<u8>) -> LocalBoxFuture<'a, Receipt<'a>> {
-		Self::request(hand, <Self as SystemCard>::Req::decode(&mut &req[..])).boxed_local()
+		Box::pin(async move {
+			Self::request(hand, <Self as SystemCard>::Req::decode(Pin::new(&mut &req[..])).await).await
+		})
 	}
 	fn card(&self) -> &dyn DynSystemCard { self }
 }
 
-pub fn downcast_atom<A: AtomicFeatures>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom> {
+pub async fn downcast_atom<A>(foreign: ForeignAtom<'_>) -> Result<TypAtom<'_, A>, ForeignAtom<'_>>
+where A: AtomicFeatures {
 	let mut data = &foreign.atom.data[..];
 	let ctx = foreign.ctx.clone();
+	let value = api::AtomId::decode(Pin::new(&mut data)).await;
 	let info_ent = (ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
 		.map(|sys| get_info::<A>(sys.get_card()))
-		.filter(|(pos, _)| api::AtomId::decode(&mut data) == *pos);
+		.filter(|(pos, _)| value == *pos);
 	match info_ent {
 		None => Err(foreign),
 		Some((_, info)) => {
-			let val = info.decode(AtomCtx(data, foreign.atom.drop, ctx));
+			let val = info.decode(AtomCtx(data, foreign.atom.drop, ctx)).await;
 			let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
 			Ok(TypAtom { value, data: foreign })
 		},

orchid-host/Cargo.toml

@@ -6,7 +6,9 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-process = "2.3.0"
 async-std = "1.13.0"
+async-stream = "0.3.6"
 derive_destructure = "1.0.0"
 futures = "0.3.31"
 hashbrown = "0.15.2"
@@ -20,4 +22,5 @@ orchid-base = { version = "0.1.0", path = "../orchid-base" }
 ordered-float = "4.6.0"
 paste = "1.0.15"
 substack = "1.1.1"
+test_executors = "0.3.2"
 trait-set = "0.3.0"

orchid-host/src/atom.rs

@@ -0,0 +1,98 @@
+use std::fmt;
+use std::rc::{Rc, Weak};
+
+use derive_destructure::destructure;
+use orchid_base::location::Pos;
+use orchid_base::reqnot::Requester;
+use orchid_base::tree::AtomRepr;
+
+use crate::api;
+use crate::ctx::Ctx;
+use crate::expr::Expr;
+use crate::system::System;
+
+#[derive(destructure)]
+pub struct AtomData {
+	owner: System,
+	drop: Option<api::AtomId>,
+	data: Vec<u8>,
+}
+impl AtomData {
+	fn api(self) -> api::Atom {
+		let (owner, drop, data) = self.destructure();
+		api::Atom { data, drop, owner: owner.id() }
+	}
+	fn api_ref(&self) -> api::Atom {
+		api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
+	}
+}
+impl Drop for AtomData {
+	fn drop(&mut self) {
+		if let Some(id) = self.drop {
+			self.owner.drop_atom(id);
+		}
+	}
+}
+impl fmt::Debug for AtomData {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		f.debug_struct("AtomData")
+			.field("drop", &self.drop)
+			.field("data", &self.data)
+			.field("owner", &self.owner.id())
+			.finish()
+	}
+}
+
+#[derive(Clone, Debug)]
+pub struct AtomHand(Rc<AtomData>);
+impl AtomHand {
+	pub(crate) async fn new(api::Atom { data, drop, owner }: api::Atom, ctx: &Ctx) -> Self {
+		let create = || async {
+			let owner = ctx.system_inst(owner).await.expect("Dropped system created atom");
+			AtomHand(Rc::new(AtomData { data, owner, drop }))
+		};
+		if let Some(id) = drop {
+			let mut owned_g = ctx.owned_atoms.write().await;
+			if let Some(data) = owned_g.get(&id) {
+				if let Some(atom) = data.upgrade() {
+					return atom;
+				}
+			}
+			let new = create().await;
+			owned_g.insert(id, new.downgrade());
+			new
+		} else {
+			create().await
+		}
+	}
+	pub async fn call(self, arg: Expr) -> api::Expression {
+		let owner_sys = self.0.owner.clone();
+		let reqnot = owner_sys.reqnot();
+		owner_sys.ext().exprs().give_expr(arg.clone());
+		match Rc::try_unwrap(self.0) {
+			Ok(data) => reqnot.request(api::FinalCall(data.api(), arg.id())).await,
+			Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await,
+		}
+	}
+	pub async fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
+		self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await
+	}
+	pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
+	pub async fn to_string(&self) -> String {
+		self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())).await
+	}
+	pub fn downgrade(&self) -> WeakAtomHand { WeakAtomHand(Rc::downgrade(&self.0)) }
+}
+impl AtomRepr for AtomHand {
+	type Ctx = Ctx;
+	async fn from_api(atom: &orchid_api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
+		Self::new(atom.clone(), &ctx).await
+	}
+	async fn to_api(&self) -> orchid_api::Atom { self.api_ref() }
+	async fn print(&self) -> String { self.to_string().await }
+}
+
+pub struct WeakAtomHand(Weak<AtomData>);
+impl WeakAtomHand {
+	pub fn upgrade(&self) -> Option<AtomHand> { self.0.upgrade().map(AtomHand) }
+}

orchid-host/src/child.rs

@@ -1,44 +0,0 @@
-use std::sync::Mutex;
-use std::{fmt, io, mem, process};
-
-use orchid_base::msg::{recv_msg, send_msg};
-
-pub struct SharedChild {
-	child: process::Child,
-	stdin: Mutex<process::ChildStdin>,
-	stdout: Mutex<process::ChildStdout>,
-	debug: Option<(String, Mutex<Box<dyn fmt::Write>>)>,
-}
-impl SharedChild {
-	pub fn new(
-		command: &mut process::Command,
-		debug: Option<(&str, impl fmt::Write + 'static)>,
-	) -> io::Result<Self> {
-		let mut child =
-			command.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
-		let stdin = Mutex::new(child.stdin.take().expect("Piped stdin above"));
-		let stdout = Mutex::new(child.stdout.take().expect("Piped stdout above"));
-		let debug = debug.map(|(n, w)| (n.to_string(), Mutex::new(Box::new(w) as Box<dyn fmt::Write>)));
-		Ok(Self { child, stdin, stdout, debug })
-	}
-
-	pub fn send_msg(&self, msg: &[u8]) -> io::Result<()> {
-		if let Some((n, dbg)) = &self.debug {
-			let mut dbg = dbg.lock().unwrap();
-			writeln!(dbg, "To {n}: {msg:?}").unwrap();
-		}
-		send_msg(&mut *self.stdin.lock().unwrap(), msg)
-	}
-
-	pub fn recv_msg(&self) -> io::Result<Vec<u8>> {
-		let msg = recv_msg(&mut *self.stdout.lock().unwrap());
-		if let Some((n, dbg)) = &self.debug {
-			let mut dbg = dbg.lock().unwrap();
-			writeln!(dbg, "From {n}: {msg:?}").unwrap();
-		}
-		msg
-	}
-}
-impl Drop for SharedChild {
-	fn drop(&mut self) { mem::drop(self.child.kill()) }
-}

orchid-host/src/ctx.rs

@@ -0,0 +1,46 @@
+use std::cell::RefCell;
+use std::num::NonZeroU16;
+use std::rc::Rc;
+use std::{fmt, ops};
+
+use async_std::sync::RwLock;
+use futures::task::LocalSpawn;
+use hashbrown::HashMap;
+use orchid_api::SysId;
+use orchid_base::interner::Interner;
+
+use crate::api;
+use crate::atom::WeakAtomHand;
+use crate::system::{System, WeakSystem};
+
+pub struct CtxData {
+	pub i: Rc<Interner>,
+	pub spawn: Rc<dyn LocalSpawn>,
+	pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
+	pub system_id: RefCell<NonZeroU16>,
+	pub owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
+}
+#[derive(Clone)]
+pub struct Ctx(Rc<CtxData>);
+impl ops::Deref for Ctx {
+	type Target = CtxData;
+	fn deref(&self) -> &Self::Target { &*self.0 }
+}
+impl Ctx {
+	pub(crate) async fn system_inst(&self, id: api::SysId) -> Option<System> {
+		self.systems.read().await.get(&id).and_then(WeakSystem::upgrade)
+	}
+	pub(crate) fn next_sys_id(&self) -> api::SysId {
+		let mut g = self.system_id.borrow_mut();
+		*g = g.checked_add(1).unwrap_or(NonZeroU16::new(1).unwrap());
+		SysId(*g)
+	}
+}
+impl fmt::Debug for Ctx {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		f.debug_struct("Ctx")
+			.field("i", &self.i)
+			.field("system_id", &self.system_id)
+			.finish_non_exhaustive()
+	}
+}

orchid-host/src/expr.rs

@@ -1,10 +1,10 @@
 use std::collections::VecDeque;
 use std::num::NonZeroU64;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::rc::{Rc, Weak};
-use std::sync::{Arc, RwLock};
+use std::sync::atomic::AtomicBool;
 
-use hashbrown::HashMap;
+use async_std::sync::RwLock;
-use lazy_static::lazy_static;
+use futures::FutureExt;
 use orchid_base::error::OrcErrv;
 use orchid_base::location::Pos;
 use orchid_base::match_mapping;
@@ -12,69 +12,56 @@ use orchid_base::name::Sym;
 use orchid_base::tree::AtomRepr;
 
 use crate::api;
-use crate::extension::AtomHand;
+use crate::atom::AtomHand;
+use crate::extension::Extension;
 
-pub type ExprParseCtx = ();
+pub type ExprParseCtx = Extension;
+
+#[derive(Debug)]
+pub struct ExprData {
+	is_canonical: AtomicBool,
+	pos: Pos,
+	kind: RwLock<ExprKind>,
+}
 
 #[derive(Clone, Debug)]
-pub struct Expr {
+pub struct Expr(Rc<ExprData>);
-	is_canonical: Arc<AtomicBool>,
-	pos: Pos,
-	kind: Arc<RwLock<ExprKind>>,
-}
 impl Expr {
-	pub fn pos(&self) -> Pos { self.pos.clone() }
+	pub fn pos(&self) -> Pos { self.0.pos.clone() }
 	pub fn as_atom(&self) -> Option<AtomHand> { todo!() }
 	pub fn strong_count(&self) -> usize { todo!() }
 	pub fn id(&self) -> api::ExprTicket {
 		api::ExprTicket(
-			NonZeroU64::new(self.kind.as_ref() as *const RwLock<_> as usize as u64)
+			NonZeroU64::new(self.0.as_ref() as *const ExprData as usize as u64)
 				.expect("this is a ref, it cannot be null"),
 		)
 	}
-	pub fn canonicalize(&self) -> api::ExprTicket {
+	// pub fn canonicalize(&self) -> api::ExprTicket {
-		if !self.is_canonical.swap(true, Ordering::Relaxed) {
+	// 	if !self.is_canonical.swap(true, Ordering::Relaxed) {
-			KNOWN_EXPRS.write().unwrap().entry(self.id()).or_insert_with(|| self.clone());
+	// 		KNOWN_EXPRS.write().unwrap().entry(self.id()).or_insert_with(||
-		}
+	// self.clone()); 	}
-		self.id()
+	// 	self.id()
-	}
+	// }
-	pub fn resolve(tk: api::ExprTicket) -> Option<Self> {
+	// pub fn resolve(tk: api::ExprTicket) -> Option<Self> {
-		KNOWN_EXPRS.read().unwrap().get(&tk).cloned()
+	// 	KNOWN_EXPRS.read().unwrap().get(&tk).cloned()
-	}
+	// }
-	pub fn from_api(api: &api::Expression, ctx: &mut ExprParseCtx) -> Self {
+	pub async fn from_api(api: &api::Expression, ctx: &mut ExprParseCtx) -> Self {
 		if let api::ExpressionKind::Slot(tk) = &api.kind {
-			return Self::resolve(*tk).expect("Invalid slot");
+			return ctx.exprs().get_expr(*tk).expect("Invalid slot");
 		}
-		Self {
+		let pos = Pos::from_api(&api.location, &ctx.ctx().i).await;
-			kind: Arc::new(RwLock::new(ExprKind::from_api(&api.kind, ctx))),
+		let kind = RwLock::new(ExprKind::from_api(&api.kind, pos.clone(), ctx).boxed_local().await);
-			is_canonical: Arc::default(),
+		Self(Rc::new(ExprData { is_canonical: AtomicBool::new(false), pos, kind }))
-			pos: Pos::from_api(&api.location),
 	}
-	}
+	pub async fn to_api(&self) -> api::InspectedKind {
-	pub fn to_api(&self) -> api::InspectedKind {
 		use api::InspectedKind as K;
-		match &*self.kind.read().unwrap() {
+		match &*self.0.kind.read().await {
-			ExprKind::Atom(a) => K::Atom(a.to_api()),
+			ExprKind::Atom(a) => K::Atom(a.to_api().await),
 			ExprKind::Bottom(b) => K::Bottom(b.to_api()),
 			_ => K::Opaque,
 		}
 	}
 }
-impl Drop for Expr {
-	fn drop(&mut self) {
-		// If the only two references left are this and known, remove from known
-		if Arc::strong_count(&self.kind) == 2 && self.is_canonical.load(Ordering::Relaxed) {
-			// if known is poisoned, a leak is preferable to a panicking destructor
-			if let Ok(mut w) = KNOWN_EXPRS.write() {
-				w.remove(&self.id());
-			}
-		}
-	}
-}
-
-lazy_static! {
-	static ref KNOWN_EXPRS: RwLock<HashMap<api::ExprTicket, Expr>> = RwLock::default();
-}
 
 #[derive(Clone, Debug)]
 pub enum ExprKind {
@@ -87,16 +74,20 @@ pub enum ExprKind {
 	Const(Sym),
 }
 impl ExprKind {
-	pub fn from_api(api: &api::ExpressionKind, ctx: &mut ExprParseCtx) -> Self {
+	pub async fn from_api(api: &api::ExpressionKind, pos: Pos, ctx: &mut ExprParseCtx) -> Self {
 		match_mapping!(api, api::ExpressionKind => ExprKind {
-			Lambda(id => PathSet::from_api(*id, api), b => Expr::from_api(b, ctx)),
+			Lambda(id => PathSet::from_api(*id, api), b => Expr::from_api(b, ctx).await),
-			Bottom(b => OrcErrv::from_api(b)),
+			Bottom(b => OrcErrv::from_api(b, &ctx.ctx().i).await),
-			Call(f => Expr::from_api(f, ctx), x => Expr::from_api(x, ctx)),
+			Call(f => Expr::from_api(f, ctx).await, x => Expr::from_api(x, ctx).await),
-			Const(c => Sym::from_api(*c)),
+			Const(c => Sym::from_api(*c, &ctx.ctx().i).await),
-			Seq(a => Expr::from_api(a, ctx), b => Expr::from_api(b, ctx)),
+			Seq(a => Expr::from_api(a, ctx).await, b => Expr::from_api(b, ctx).await),
 		} {
 			api::ExpressionKind::Arg(_) => ExprKind::Arg,
-			api::ExpressionKind::NewAtom(a) => ExprKind::Atom(AtomHand::from_api(a.clone())),
+			api::ExpressionKind::NewAtom(a) => ExprKind::Atom(AtomHand::from_api(
+				a,
+				pos,
+				&mut ctx.ctx().clone()
+			).await),
 			api::ExpressionKind::Slot(_) => panic!("Handled in Expr"),
 		})
 	}
@@ -139,3 +130,8 @@ impl PathSet {
 		}
 	}
 }
+
+pub struct WeakExpr(Weak<ExprData>);
+impl WeakExpr {
+	pub fn upgrade(&self) -> Option<Expr> { self.0.upgrade().map(Expr) }
+}

orchid-host/src/expr_store.rs

@@ -0,0 +1,35 @@
+use std::cell::RefCell;
+use std::fmt;
+
+use hashbrown::HashMap;
+use hashbrown::hash_map::Entry;
+
+use crate::api;
+use crate::expr::Expr;
+
+#[derive(Default)]
+pub struct ExprStore(RefCell<HashMap<api::ExprTicket, (u32, Expr)>>);
+impl ExprStore {
+	pub fn give_expr(&self, expr: Expr) {
+		match self.0.borrow_mut().entry(expr.id()) {
+			Entry::Occupied(mut oe) => oe.get_mut().0 += 1,
+			Entry::Vacant(v) => {
+				v.insert((1, expr));
+			},
+		}
+	}
+	pub fn take_expr(&self, ticket: api::ExprTicket) {
+		(self.0.borrow_mut().entry(ticket))
+			.and_replace_entry_with(|_, (rc, rt)| (1 < rc).then_some((rc - 1, rt)));
+	}
+	pub fn get_expr(&self, ticket: api::ExprTicket) -> Option<Expr> {
+		self.0.borrow().get(&ticket).map(|(_, expr)| expr.clone())
+	}
+}
+impl fmt::Display for ExprStore {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		let r = self.0.borrow();
+		let rc: u32 = r.values().map(|v| v.0).sum();
+		write!(f, "Store holding {rc} refs to {} exprs", r.len())
+	}
+}

orchid-host/src/extension.rs

@@ -1,111 +1,35 @@
-use std::collections::VecDeque;
+use std::cell::RefCell;
-use std::num::NonZero;
+use std::future::Future;
-use std::ops::Deref;
+use std::io;
-use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering};
+use std::num::NonZeroU64;
-use std::sync::mpsc::{SyncSender, sync_channel};
+use std::rc::{Rc, Weak};
-use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
-use std::{fmt, io, thread};
 
+use async_std::channel::{self, Sender};
+use async_std::sync::Mutex;
 use derive_destructure::destructure;
+use futures::FutureExt;
+use futures::future::{join, join_all};
+use futures::task::LocalSpawnExt;
 use hashbrown::HashMap;
-use hashbrown::hash_map::Entry;
 use itertools::Itertools;
-use lazy_static::lazy_static;
+use orchid_api::HostMsgSet;
 use orchid_api_traits::Request;
-use orchid_base::builtin::{ExtFactory, ExtPort};
+use orchid_base::builtin::ExtInit;
-use orchid_base::char_filter::char_filter_match;
 use orchid_base::clone;
-use orchid_base::error::{OrcErrv, OrcRes};
+use orchid_base::interner::Tok;
-use orchid_base::interner::{Tok, intern};
-use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
 use orchid_base::macros::mtreev_from_api;
-use orchid_base::parse::Comment;
 use orchid_base::reqnot::{ReqNot, Requester as _};
-use orchid_base::tree::{AtomRepr, ttv_from_api};
+use orchid_base::tree::AtomRepr;
-use ordered_float::NotNan;
-use substack::{Stackframe, Substack};
 
 use crate::api;
-use crate::expr::Expr;
+use crate::atom::AtomHand;
+use crate::ctx::Ctx;
+use crate::expr_store::ExprStore;
 use crate::macros::{macro_recur, macro_treev_to_api};
-use crate::tree::{Member, ParsTokTree};
+use crate::system::SystemCtor;
 
-#[derive(Debug, destructure)]
+pub struct ReqPair<R: Request>(R, Sender<R::Response>);
-pub struct AtomData {
-	owner: System,
-	drop: Option<api::AtomId>,
-	data: Vec<u8>,
-}
-impl AtomData {
-	fn api(self) -> api::Atom {
-		let (owner, drop, data) = self.destructure();
-		api::Atom { data, drop, owner: owner.id() }
-	}
-	fn api_ref(&self) -> api::Atom {
-		api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
-	}
-}
-impl Drop for AtomData {
-	fn drop(&mut self) {
-		if let Some(id) = self.drop {
-			self.owner.reqnot().notify(api::AtomDrop(self.owner.id(), id))
-		}
-	}
-}
-
-#[derive(Clone, Debug)]
-pub struct AtomHand(Arc<AtomData>);
-impl AtomHand {
-	pub fn from_api(atom: api::Atom) -> Self {
-		fn create_new(api::Atom { data, drop, owner }: api::Atom) -> AtomHand {
-			let owner = System::resolve(owner).expect("Atom owned by non-existing system");
-			AtomHand(Arc::new(AtomData { data, drop, owner }))
-		}
-		if let Some(id) = atom.drop {
-			lazy_static! {
-				static ref OWNED_ATOMS: Mutex<HashMap<(api::SysId, api::AtomId), Weak<AtomData>>> =
-					Mutex::default();
-			}
-			let owner = atom.owner;
-			let mut owned_g = OWNED_ATOMS.lock().unwrap();
-			if let Some(data) = owned_g.get(&(owner, id)) {
-				if let Some(atom) = data.upgrade() {
-					return Self(atom);
-				}
-			}
-			let new = create_new(atom);
-			owned_g.insert((owner, id), Arc::downgrade(&new.0));
-			new
-		} else {
-			create_new(atom)
-		}
-	}
-	pub fn call(self, arg: Expr) -> api::Expression {
-		let owner_sys = self.0.owner.clone();
-		let reqnot = owner_sys.reqnot();
-		let ticket = owner_sys.give_expr(arg.canonicalize(), || arg);
-		match Arc::try_unwrap(self.0) {
-			Ok(data) => reqnot.request(api::FinalCall(data.api(), ticket)),
-			Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), ticket)),
-		}
-	}
-	pub fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
-		self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req))
-	}
-	pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
-	pub fn print(&self) -> String { self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())) }
-}
-impl AtomRepr for AtomHand {
-	type Ctx = ();
-	fn from_api(atom: &orchid_api::Atom, _: Pos, (): &mut Self::Ctx) -> Self {
-		Self::from_api(atom.clone())
-	}
-	fn to_api(&self) -> orchid_api::Atom { self.api_ref() }
-}
-impl fmt::Display for AtomHand {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.print()) }
-}
 
 /// Data held about an Extension. This is refcounted within [Extension]. It's
 /// important to only ever access parts of this struct through the [Arc] because
@@ -113,320 +37,194 @@ impl fmt::Display for AtomHand {
 /// upgrading fails.
 #[derive(destructure)]
 pub struct ExtensionData {
-	port: Mutex<Box<dyn ExtPort>>,
+	ctx: Ctx,
-	// child: Mutex<process::Child>,
+	init: ExtInit,
-	// child_stdin: Mutex<ChildStdin>,
 	reqnot: ReqNot<api::HostMsgSet>,
 	systems: Vec<SystemCtor>,
 	logger: Logger,
+	next_pars: RefCell<NonZeroU64>,
+	exprs: ExprStore,
+	lex_recur: Mutex<HashMap<api::ParsId, channel::Sender<ReqPair<api::SubLex>>>>,
 }
 impl Drop for ExtensionData {
 	fn drop(&mut self) { self.reqnot.notify(api::HostExtNotif::Exit); }
 }
 
-fn acq_expr(sys: api::SysId, extk: api::ExprTicket) {
-	(System::resolve(sys).expect("Expr acq'd by invalid system"))
-		.give_expr(extk, || Expr::resolve(extk).expect("Invalid expr acq'd"));
-}
-
-fn rel_expr(sys: api::SysId, extk: api::ExprTicket) {
-	let sys = System::resolve(sys).unwrap();
-	let mut exprs = sys.0.exprs.write().unwrap();
-	exprs.entry(extk).and_replace_entry_with(|_, (rc, rt)| {
-		(0 < rc.fetch_sub(1, Ordering::Relaxed)).then_some((rc, rt))
-	});
-}
-
 #[derive(Clone)]
-pub struct Extension(Arc<ExtensionData>);
+pub struct Extension(Rc<ExtensionData>);
 impl Extension {
-	pub fn new(fac: Box<dyn ExtFactory>, logger: Logger) -> io::Result<Self> {
+	pub fn new(init: ExtInit, logger: Logger, ctx: Ctx) -> io::Result<Self> {
-		Ok(Self(Arc::new_cyclic(|weak: &Weak<ExtensionData>| {
+		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
-			let (eh, port) = fac.run(Box::new(clone!(weak; move |msg| {
+			exprs: ExprStore::default(),
-				weak.upgrade().inspect(|xd| xd.reqnot.receive(msg));
+			ctx: ctx.clone(),
-			})));
+			systems: (init.systems.iter().cloned())
-			ExtensionData {
+				.map(|decl| SystemCtor { decl, ext: WeakExtension(weak.clone()) })
-				systems: (eh.systems.iter().cloned())
-					.map(|decl| SystemCtor { decl, ext: weak.clone() })
 				.collect(),
 			logger,
-				port: Mutex::new(port),
+			init,
+			next_pars: RefCell::new(NonZeroU64::new(1).unwrap()),
+			lex_recur: Mutex::default(),
 			reqnot: ReqNot::new(
-					clone!(weak; move |sfn, _| {
+				clone!(weak; move |sfn, _| clone!(weak; async move {
 					let data = weak.upgrade().unwrap();
 					data.logger.log_buf("Downsending", sfn);
-						data.port.lock().unwrap().send(sfn);
+					data.init.send(sfn).await
-					}),
+				}.boxed_local())),
-					clone!(weak; move |notif, _| match notif {
+				clone!(weak; move |notif, _| clone!(weak; async move {
-						api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => acq_expr(acq.0, acq.1),
+					let this = Extension(weak.upgrade().unwrap());
-						api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => rel_expr(rel.0, rel.1),
+					match notif {
+						api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
+							let target = this.0.exprs.get_expr(acq.1).expect("Invalid ticket");
+							this.0.exprs.give_expr(target)
+						}
+						api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
+							this.assert_own_sys(rel.0).await;
+							this.0.exprs.take_expr(rel.1)
+						}
 						api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
-							acq_expr(mov.inc, mov.expr);
+							this.assert_own_sys(mov.dec).await;
-							rel_expr(mov.dec, mov.expr);
+							let recp = this.ctx().system_inst(mov.inc).await.expect("invallid recipient sys id");
+							let expr = this.0.exprs.get_expr(mov.expr).expect("invalid ticket");
+							recp.ext().0.exprs.give_expr(expr);
+							this.0.exprs.take_expr(mov.expr);
 						},
-						api::ExtHostNotif::Log(api::Log(str)) => weak.upgrade().unwrap().logger.log(str),
+						api::ExtHostNotif::Log(api::Log(str)) => this.logger().log(str),
-					}),
+					}
-					|hand, req| match req {
+				}.boxed_local())),
-						api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()),
+				{
+					clone!(weak, ctx);
+					move |hand, req| {
+						clone!(weak, ctx);
+						async move {
+							let this = Self(weak.upgrade().unwrap());
+							let i = this.ctx().i.clone();
+							match req {
+								api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
 								api::ExtHostReq::IntReq(intreq) => match intreq {
-							api::IntReq::InternStr(s) => hand.handle(&s, &intern(&**s.0).to_api()),
+									api::IntReq::InternStr(s) => hand.handle(&s, &i.i(&*s.0).await.to_api()).await,
-							api::IntReq::InternStrv(v) => hand.handle(&v, &intern(&*v.0).to_api()),
+									api::IntReq::InternStrv(v) => {
-							api::IntReq::ExternStr(si) => hand.handle(&si, &Tok::<String>::from_api(si.0).arc()),
+										let tokens = join_all(v.0.iter().map(|m| i.ex(*m))).await;
-							api::IntReq::ExternStrv(vi) => hand.handle(
+										hand.handle(&v, &i.i(&tokens).await.to_api()).await
-								&vi,
+									},
-								&Arc::new(
+									api::IntReq::ExternStr(si) =>
-									Tok::<Vec<Tok<String>>>::from_api(vi.0).iter().map(|t| t.to_api()).collect_vec(),
+										hand.handle(&si, &Tok::<String>::from_api(si.0, &i).await.rc()).await,
-								),
+									api::IntReq::ExternStrv(vi) => {
-							),
+										let markerv = (i.ex(vi.0).await.iter()).map(|t| t.to_api()).collect_vec();
+										hand.handle(&vi, &markerv).await
+									},
 								},
 								api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
-							let sys = System::resolve(atom.owner).unwrap();
+									let sys = ctx.system_inst(atom.owner).await.expect("owner of live atom dropped");
-							hand.handle(fw, &sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())))
+									let reply =
+										sys.reqnot().request(api::Fwded(fw.0.clone(), *key, body.clone())).await;
+									hand.handle(fw, &reply).await
 								},
 								api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
-							let sys = System::resolve(id).unwrap();
+									let sys = ctx.system_inst(id).await.unwrap();
-							hand.handle(fw, &sys.request(body.clone()))
+									hand.handle(fw, &sys.request(body.clone()).await).await
 								},
 								api::ExtHostReq::SubLex(sl) => {
-							let (rep_in, rep_out) = sync_channel(0);
+									let (rep_in, rep_out) = channel::bounded(0);
-							let lex_g = LEX_RECUR.lock().unwrap();
+									let lex_g = this.0.lex_recur.lock().await;
 									let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
-							req_in.send(ReqPair(sl.clone(), rep_in)).unwrap();
+									req_in.send(ReqPair(sl.clone(), rep_in)).await.unwrap();
-							hand.handle(&sl, &rep_out.recv().unwrap())
+									hand.handle(&sl, &rep_out.recv().await.unwrap()).await
 								},
 								api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins @ api::Inspect { target })) => {
-							let expr = Expr::resolve(target).expect("Invalid ticket");
+									let expr = this.exprs().get_expr(target).expect("Invalid ticket");
-							hand.handle(&ins, &api::Inspected {
+									hand
+										.handle(&ins, &api::Inspected {
 											refcount: expr.strong_count() as u32,
 											location: expr.pos().to_api(),
-								kind: expr.to_api(),
+											kind: expr.to_api().await,
 										})
+										.await
 								},
-						api::ExtHostReq::RunMacros(ref rm @ api::RunMacros { ref run_id, ref query }) => hand
+								api::ExtHostReq::RunMacros(ref rm @ api::RunMacros { ref run_id, ref query }) => {
-							.handle(
+									let mtreev =
-								rm,
+										mtreev_from_api(query, &i, &mut |_| panic!("Atom in macro recur")).await;
-								&macro_recur(
+									match macro_recur(*run_id, mtreev).await {
-									*run_id,
+										Some(x) => hand.handle(rm, &Some(macro_treev_to_api(*run_id, x).await)).await,
-									mtreev_from_api(query, &mut |_| panic!("Recursion never contains atoms")),
+										None => hand.handle(rm, &None).await,
-								)
-								.map(|x| macro_treev_to_api(*run_id, x)),
-							),
-					},
-				),
 									}
+								},
+								api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) =>
+									hand.handle(eap, &AtomHand::new(atom.clone(), &ctx).await.print().await).await,
+							}
+						}
+						.boxed_local()
+					}
+				},
+			),
 		})))
 	}
-	pub fn systems(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
+	pub(crate) fn reqnot(&self) -> &ReqNot<HostMsgSet> { &self.0.reqnot }
-}
+	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
-
+	pub fn logger(&self) -> &Logger { &self.0.logger }
-pub struct SystemCtor {
+	pub fn system_ctors(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
-	decl: api::SystemDecl,
+	pub fn exprs(&self) -> &ExprStore { &self.0.exprs }
-	ext: Weak<ExtensionData>,
+	pub async fn is_own_sys(&self, id: api::SysId) -> bool {
-}
+		let sys = self.ctx().system_inst(id).await.expect("invalid sender sys id");
-impl SystemCtor {
+		Rc::ptr_eq(&self.0, &sys.ext().0)
-	pub fn name(&self) -> &str { &self.decl.name }
-	pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
-	pub fn depends(&self) -> impl ExactSizeIterator<Item = &str> {
-		self.decl.depends.iter().map(|s| &**s)
 	}
-	pub fn run<'a>(&self, depends: impl IntoIterator<Item = &'a System>) -> System {
+	pub async fn assert_own_sys(&self, id: api::SysId) {
-		let mut inst_g = SYSTEM_INSTS.write().unwrap();
+		assert!(self.is_own_sys(id).await, "Incoming message impersonates separate system");
-		let depends = depends.into_iter().map(|si| si.id()).collect_vec();
-		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
-		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
-		static NEXT_ID: AtomicU16 = AtomicU16::new(1);
-		let id =
-			api::SysId(NonZero::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped"));
-		let sys_inst = ext.reqnot.request(api::NewSystem { depends, id, system: self.decl.id });
-		let data = System(Arc::new(SystemInstData {
-			decl_id: self.decl.id,
-			ext: Extension(ext),
-			exprs: RwLock::default(),
-			lex_filter: sys_inst.lex_filter,
-			const_root: OnceLock::new(),
-			line_types: sys_inst.line_types.into_iter().map(Tok::from_api).collect(),
-			id,
-		}));
-		let root = (sys_inst.const_root.into_iter())
-			.map(|(k, v)| {
-				Member::from_api(
-					api::Member { name: k, kind: v },
-					Substack::Bottom.push(Tok::from_api(k)),
-					&data,
-				)
-			})
-			.collect_vec();
-		data.0.const_root.set(root).unwrap();
-		inst_g.insert(id, data.clone());
-		data
 	}
-}
+	pub fn next_pars(&self) -> NonZeroU64 {
-
+		let mut next_pars = self.0.next_pars.borrow_mut();
-lazy_static! {
+		*next_pars = next_pars.checked_add(1).unwrap_or(NonZeroU64::new(1).unwrap());
-	static ref SYSTEM_INSTS: RwLock<HashMap<api::SysId, System>> = RwLock::default();
+		*next_pars
-	static ref LEX_RECUR: Mutex<HashMap<api::ParsId, SyncSender<ReqPair<api::SubLex>>>> =
-		Mutex::default();
-}
-
-pub struct ReqPair<R: Request>(R, pub SyncSender<R::Response>);
-
-#[derive(destructure)]
-pub struct SystemInstData {
-	exprs: RwLock<HashMap<api::ExprTicket, (AtomicU32, Expr)>>,
-	ext: Extension,
-	decl_id: api::SysDeclId,
-	lex_filter: api::CharFilter,
-	id: api::SysId,
-	const_root: OnceLock<Vec<Member>>,
-	line_types: Vec<Tok<String>>,
-}
-impl Drop for SystemInstData {
-	fn drop(&mut self) {
-		self.ext.0.reqnot.notify(api::SystemDrop(self.id));
-		if let Ok(mut g) = SYSTEM_INSTS.write() {
-			g.remove(&self.id);
 	}
-	}
+	pub(crate) async fn lex_req<F: Future<Output = Option<api::SubLexed>>>(
-}
-#[derive(Clone)]
-pub struct System(Arc<SystemInstData>);
-impl System {
-	pub fn id(&self) -> api::SysId { self.id }
-	fn resolve(id: api::SysId) -> Option<System> { SYSTEM_INSTS.read().unwrap().get(&id).cloned() }
-	fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.ext.0.reqnot }
-	fn give_expr(&self, ticket: api::ExprTicket, get_expr: impl FnOnce() -> Expr) -> api::ExprTicket {
-		match self.0.exprs.write().unwrap().entry(ticket) {
-			Entry::Occupied(mut oe) => {
-				oe.get_mut().0.fetch_add(1, Ordering::Relaxed);
-			},
-			Entry::Vacant(v) => {
-				v.insert((AtomicU32::new(1), get_expr()));
-			},
-		}
-		ticket
-	}
-	pub fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
-		self.reqnot().request(api::GetMember(self.0.id, id))
-	}
-	pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
-	pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
-	/// Have this system lex a part of the source. It is assumed that
-	/// [Self::can_lex] was called and returned true.
-	pub fn lex(
 		&self,
 		source: Tok<String>,
 		pos: u32,
-		mut r: impl FnMut(u32) -> Option<api::SubLexed> + Send,
+		sys: api::SysId,
+		mut r: impl FnMut(u32) -> F,
 	) -> api::OrcResult<Option<api::LexedExpr>> {
 		// get unique lex ID
-		static LEX_ID: AtomicU64 = AtomicU64::new(1);
+		let id = api::ParsId(self.next_pars());
-		let id = api::ParsId(NonZero::new(LEX_ID.fetch_add(1, Ordering::Relaxed)).unwrap());
-		thread::scope(|s| {
 		// create and register channel
-			let (req_in, req_out) = sync_channel(0);
+		let (req_in, req_out) = channel::bounded(0);
-			LEX_RECUR.lock().unwrap().insert(id, req_in); // LEX_RECUR released
+		self.0.lex_recur.lock().await.insert(id, req_in); // lex_recur released
-			// spawn recursion handler which will exit when the sender is collected
+		let (ret, ()) = join(
-			s.spawn(move || {
+			async {
-				while let Ok(ReqPair(sublex, rep_in)) = req_out.recv() {
+				let res =
-					rep_in.send(r(sublex.pos)).unwrap()
+					(self.reqnot()).request(api::LexExpr { id, pos, sys, text: source.to_api() }).await;
-				}
+				// collect sender to unblock recursion handler branch before returning
-			});
+				self.0.lex_recur.lock().await.remove(&id);
-			// Pass control to extension
+				res
-			let ret =
-				self.reqnot().request(api::LexExpr { id, pos, sys: self.id(), text: source.to_api() });
-			// collect sender to unblock recursion handler thread before returning
-			LEX_RECUR.lock().unwrap().remove(&id);
-			ret.transpose()
-		}) // exit recursion handler thread
-	}
-	pub fn can_parse(&self, line_type: Tok<String>) -> bool { self.line_types.contains(&line_type) }
-	pub fn line_types(&self) -> impl Iterator<Item = Tok<String>> + '_ {
-		self.line_types.iter().cloned()
-	}
-	pub fn parse(
-		&self,
-		line: Vec<ParsTokTree>,
-		exported: bool,
-		comments: Vec<Comment>,
-	) -> OrcRes<Vec<ParsTokTree>> {
-		let line = line.iter().map(|t| t.to_api(&mut |n, _| match *n {})).collect_vec();
-		let comments = comments.iter().map(Comment::to_api).collect_vec();
-		let parsed =
-			(self.reqnot().request(api::ParseLine { exported, sys: self.id(), comments, line }))
-				.map_err(|e| OrcErrv::from_api(&e))?;
-		Ok(ttv_from_api(parsed, &mut ()))
-	}
-	pub fn request(&self, req: Vec<u8>) -> Vec<u8> {
-		self.reqnot().request(api::SysFwded(self.id(), req))
-	}
-}
-impl fmt::Debug for System {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let ctor = (self.0.ext.0.systems.iter().find(|c| c.decl.id == self.0.decl_id))
-			.expect("System instance with no associated constructor");
-		write!(f, "System({} @ {} #{})", ctor.decl.name, ctor.decl.priority, self.0.id.0)?;
-		match self.0.exprs.read() {
-			Err(_) => write!(f, "expressions unavailable"),
-			Ok(r) => {
-				let rc: u32 = r.values().map(|v| v.0.load(Ordering::Relaxed)).sum();
-				write!(f, "{rc} refs to {} exprs", r.len())
 			},
+			async {
+				while let Ok(ReqPair(sublex, rep_in)) = req_out.recv().await {
+					(rep_in.send(r(sublex.pos).await).await)
+						.expect("Response channel dropped while request pending")
 				}
+			},
+		)
+		.await;
+		ret.transpose()
 	}
-}
+	pub async fn recv_one(&self) {
-impl Deref for System {
+		let reqnot = self.0.reqnot.clone();
-	type Target = SystemInstData;
+		self
-	fn deref(&self) -> &Self::Target { self.0.as_ref() }
+			.0
+			.init
+			.recv(Box::new(move |msg| async move { reqnot.receive(msg).await }.boxed_local()))
+			.await;
+	}
+	pub fn system_drop(&self, id: api::SysId) {
+		let rc = self.clone();
+		(self.ctx().spawn.spawn_local(async move {
+			rc.reqnot().notify(api::SystemDrop(id)).await;
+			rc.ctx().systems.write().await.remove(&id);
+		}))
+		.expect("Failed to drop system!");
+	}
+	pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }
 }
 
-#[derive(Debug, Clone)]
+pub struct WeakExtension(Weak<ExtensionData>);
-pub enum SysResolvErr {
+impl WeakExtension {
-	Loop(Vec<String>),
+	pub fn upgrade(&self) -> Option<Extension> { self.0.upgrade().map(Extension) }
-	Missing(String),
-}
-
-pub fn init_systems(tgts: &[String], exts: &[Extension]) -> Result<Vec<System>, SysResolvErr> {
-	let mut to_load = HashMap::<&str, &SystemCtor>::new();
-	let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque<&str>>();
-	while let Some(target) = to_find.pop_front() {
-		if to_load.contains_key(target) {
-			continue;
-		}
-		let ctor = (exts.iter())
-			.flat_map(|e| e.systems().filter(|c| c.decl.name == target))
-			.max_by_key(|c| c.decl.priority)
-			.ok_or_else(|| SysResolvErr::Missing(target.to_string()))?;
-		to_load.insert(target, ctor);
-		to_find.extend(ctor.decl.depends.iter().map(|s| s.as_str()));
-	}
-	let mut to_load_ordered = Vec::new();
-	fn walk_deps<'a>(
-		graph: &mut HashMap<&str, &'a SystemCtor>,
-		list: &mut Vec<&'a SystemCtor>,
-		chain: Stackframe<&str>,
-	) -> Result<(), SysResolvErr> {
-		if let Some(ctor) = graph.remove(chain.item) {
-			// if the above is none, the system is already queued. Missing systems are
-			// detected above
-			for dep in ctor.decl.depends.iter() {
-				if Substack::Frame(chain).iter().any(|c| c == dep) {
-					let mut circle = vec![dep.to_string()];
-					circle.extend(Substack::Frame(chain).iter().map(|s| s.to_string()));
-					return Err(SysResolvErr::Loop(circle));
-				}
-				walk_deps(graph, list, Substack::Frame(chain).new_frame(dep))?
-			}
-			list.push(ctor);
-		}
-		Ok(())
-	}
-	for tgt in tgts {
-		walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?;
-	}
-	let mut systems = HashMap::<&str, System>::new();
-	for ctor in to_load_ordered.iter() {
-		let sys = ctor.run(ctor.depends().map(|n| &systems[n]));
-		systems.insert(ctor.name(), sys);
-	}
-	Ok(systems.into_values().collect_vec())
 }

orchid-host/src/lex.rs

@@ -1,18 +1,22 @@
 use std::num::NonZeroU64;
 use std::sync::Arc;
 
+use async_std::sync::Mutex;
+use futures::FutureExt;
 use hashbrown::HashMap;
 use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
-use orchid_base::interner::{Tok, intern};
+use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
+use orchid_base::match_mapping;
 use orchid_base::number::{num_to_err, parse_num};
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::PARENS;
-use orchid_base::tree::Ph;
+use orchid_base::tree::{AtomRepr, Ph};
-use orchid_base::{intern, match_mapping};
 
 use crate::api;
-use crate::extension::{AtomHand, System};
+use crate::atom::AtomHand;
+use crate::ctx::Ctx;
+use crate::system::System;
 use crate::tree::{ParsTok, ParsTokTree};
 
 pub struct LexCtx<'a> {
@@ -20,6 +24,7 @@ pub struct LexCtx<'a> {
 	pub source: &'a Tok<String>,
 	pub tail: &'a str,
 	pub sub_trees: &'a mut HashMap<api::TreeTicket, ParsTokTree>,
+	pub ctx: &'a Ctx,
 }
 impl<'a> LexCtx<'a> {
 	pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
@@ -29,6 +34,7 @@ impl<'a> LexCtx<'a> {
 			tail: &self.source[pos as usize..],
 			systems: self.systems,
 			sub_trees: &mut *self.sub_trees,
+			ctx: &self.ctx,
 		}
 	}
 	pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 }
@@ -70,7 +76,7 @@ impl<'a> LexCtx<'a> {
 	}
 }
 
-pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
+pub async fn lex_once<'a>(ctx: &mut LexCtx<'a>) -> OrcRes<ParsTokTree> {
 	let start = ctx.get_pos();
 	assert!(
 		!ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space),
@@ -82,11 +88,13 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 	} else if ctx.strip_prefix("::") {
 		ParsTok::NS
 	} else if ctx.strip_prefix("--[") {
-		let (cmt, tail) = ctx.tail.split_once("]--").ok_or_else(|| {
+		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
-			mk_errv(intern!(str: "Unterminated block comment"), "This block comment has no ending ]--", [
+			return Err(mk_errv(
-				Pos::Range(start..start + 3).into(),
+				ctx.ctx.i.i("Unterminated block comment").await,
-			])
+				"This block comment has no ending ]--",
-		})?;
+				[Pos::Range(start..start + 3).into()],
+			));
+		};
 		ctx.set_tail(tail);
 		ParsTok::Comment(Arc::new(cmt.to_string()))
 	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
@@ -99,12 +107,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 		while !ctx.strip_char('.') {
 			if ctx.tail.is_empty() {
 				return Err(mk_errv(
-					intern!(str: "Unclosed lambda"),
+					ctx.ctx.i.i("Unclosed lambda").await,
 					"Lambdae started with \\ should separate arguments from body with .",
 					[Pos::Range(start..start + 1).into()],
 				));
 			}
-			arg.push(lex_once(ctx)?);
+			arg.push(lex_once(ctx).boxed_local().await?);
 			ctx.trim_ws();
 		}
 		ParsTok::LambdaHead(arg)
@@ -114,12 +122,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 		while !ctx.strip_char(*rp) {
 			if ctx.tail.is_empty() {
 				return Err(mk_errv(
-					intern!(str: "unclosed paren"),
+					ctx.ctx.i.i("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
 					[Pos::Range(start..start + 1).into()],
 				));
 			}
-			body.push(lex_once(ctx)?);
+			body.push(lex_once(ctx).boxed_local().await?);
 			ctx.trim_ws();
 		}
 		ParsTok::S(*paren, body)
@@ -130,8 +138,10 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 		if ctx.strip_char('(') {
 			let pos = ctx.get_pos();
 			let numstr = ctx.get_start_matches(|x| x != ')').trim();
-			let num = parse_num(numstr).map_err(|e| num_to_err(e, pos))?;
+			match parse_num(numstr) {
-			ParsTok::Macro(Some(num.to_f64()))
+				Ok(num) => ParsTok::Macro(Some(num.to_f64())),
+				Err(e) => return Err(num_to_err(e, pos, &*ctx.ctx.i).await.into()),
+			}
 		} else {
 			ParsTok::Macro(None)
 		}
@@ -139,17 +149,26 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 		for sys in ctx.systems {
 			let mut errors = Vec::new();
 			if ctx.tail.starts_with(|c| sys.can_lex(c)) {
-				let lx =
+				let (source, pos) = (ctx.source.clone(), ctx.get_pos());
-					sys.lex(ctx.source.clone(), ctx.get_pos(), |pos| match lex_once(&mut ctx.push(pos)) {
+				let ctx_lck = &Mutex::new(&mut *ctx);
-						Ok(t) => Some(api::SubLexed { pos, ticket: ctx.add_subtree(t) }),
+				let errors_lck = &Mutex::new(&mut errors);
+				let lx = sys
+					.lex(source, pos, |pos| async move {
+						match lex_once(&mut ctx_lck.lock().await.push(pos)).boxed_local().await {
+							Ok(t) => Some(api::SubLexed { pos, ticket: ctx_lck.lock().await.add_subtree(t) }),
 							Err(e) => {
-							errors.push(e);
+								errors_lck.lock().await.push(e);
 								None
 							},
-					});
+						}
+					})
+					.await;
 				match lx {
-					Err(e) => return Err(errors.into_iter().fold(OrcErrv::from_api(&e), |a, b| a + b)),
+					Err(e) =>
-					Ok(Some(lexed)) => return Ok(tt_to_owned(&lexed.expr, &mut ctx.push(lexed.pos))),
+						return Err(
+							errors.into_iter().fold(OrcErrv::from_api(&e, &*ctx.ctx.i).await, |a, b| a + b),
+						),
+					Ok(Some(lexed)) => return Ok(tt_to_owned(&lexed.expr, &mut ctx.push(lexed.pos)).await),
 					Ok(None) => match errors.into_iter().reduce(|a, b| a + b) {
 						Some(errors) => return Err(errors),
 						None => continue,
@@ -158,12 +177,12 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 			}
 		}
 		if ctx.tail.starts_with(name_start) {
-			ParsTok::Name(intern(ctx.get_start_matches(name_char)))
+			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(name_char)).await)
 		} else if ctx.tail.starts_with(op_char) {
-			ParsTok::Name(intern(ctx.get_start_matches(op_char)))
+			ParsTok::Name(ctx.ctx.i.i(ctx.get_start_matches(op_char)).await)
 		} else {
 			return Err(mk_errv(
-				intern!(str: "Unrecognized character"),
+				ctx.ctx.i.i("Unrecognized character").await,
 				"The following syntax is meaningless.",
 				[Pos::Range(start..start + 1).into()],
 			));
@@ -172,16 +191,18 @@ pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
 	Ok(ParsTokTree { tok, range: start..ctx.get_pos() })
 }
 
-fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
+async fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
 	let tok = match_mapping!(&api.token, api::Token => ParsTok {
-		Atom(atom => AtomHand::from_api(atom.clone())),
+		Atom(atom =>
-		Bottom(err => OrcErrv::from_api(err)),
+			AtomHand::from_api(atom, Pos::Range(api.range.clone()), &mut ctx.ctx.clone()).await
-		LambdaHead(arg => ttv_to_owned(arg, ctx)),
+		),
-		Name(name => Tok::from_api(*name)),
+		Bottom(err => OrcErrv::from_api(err, &*ctx.ctx.i).await),
-		S(p.clone(), b.iter().map(|t| tt_to_owned(t, ctx)).collect()),
+		LambdaHead(arg => ttv_to_owned(arg, ctx).boxed_local().await),
+		Name(name => Tok::from_api(*name, &*ctx.ctx.i).await),
+		S(p.clone(), b => ttv_to_owned(b, ctx).boxed_local().await),
 		BR, NS,
 		Comment(c.clone()),
-		Ph(ph => Ph::from_api(ph)),
+		Ph(ph => Ph::from_api(ph, &*ctx.ctx.i).await),
 		Macro(*prio),
 	} {
 		api::Token::Slot(id) => return ctx.rm_subtree(*id),
@@ -189,20 +210,24 @@ fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
 	ParsTokTree { range: api.range.clone(), tok }
 }
 
-fn ttv_to_owned<'a>(
+async fn ttv_to_owned<'a>(
 	api: impl IntoIterator<Item = &'a api::TokenTree>,
 	ctx: &mut LexCtx<'_>,
 ) -> Vec<ParsTokTree> {
-	api.into_iter().map(|t| tt_to_owned(t, ctx)).collect()
+	let mut out = Vec::new();
+	for tt in api {
+		out.push(tt_to_owned(&tt, ctx).await)
+	}
+	out
 }
 
-pub fn lex(text: Tok<String>, systems: &[System]) -> OrcRes<Vec<ParsTokTree>> {
+pub async fn lex(text: Tok<String>, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
 	let mut sub_trees = HashMap::new();
-	let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems };
+	let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems, ctx };
 	let mut tokv = Vec::new();
 	ctx.trim(unrep_space);
 	while !ctx.tail.is_empty() {
-		tokv.push(lex_once(&mut ctx)?);
+		tokv.push(lex_once(&mut ctx).await?);
 		ctx.trim(unrep_space);
 	}
 	Ok(tokv)

orchid-host/src/lib.rs

@@ -1,11 +1,14 @@
 use orchid_api as api;
 
-pub mod child;
+pub mod atom;
+pub mod ctx;
 pub mod expr;
+pub mod expr_store;
 pub mod extension;
 pub mod lex;
 pub mod macros;
 pub mod parse;
 pub mod rule;
 pub mod subprocess;
+pub mod system;
 pub mod tree;

orchid-host/src/macros.rs

@@ -1,14 +1,16 @@
-use std::sync::{Arc, RwLock};
+use std::rc::Rc;
 
+use async_std::sync::RwLock;
+use futures::FutureExt;
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
-use lazy_static::lazy_static;
+use orchid_base::interner::Interner;
 use orchid_base::macros::{MTok, MTree, mtreev_from_api, mtreev_to_api};
 use orchid_base::name::Sym;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::extension::AtomHand;
+use crate::atom::AtomHand;
 use crate::rule::matcher::{NamedMatcher, PriodMatcher};
 use crate::rule::state::MatchState;
 use crate::tree::Code;
@@ -17,38 +19,41 @@ pub type MacTok = MTok<'static, AtomHand>;
 pub type MacTree = MTree<'static, AtomHand>;
 
 trait_set! {
-	trait MacroCB = Fn(Vec<MacTree>) -> Option<Vec<MacTree>> + Send + Sync;
+	trait MacroCB = Fn(Vec<MacTree>) -> Option<Vec<MacTree>>;
 }
 
-lazy_static! {
+thread_local! {
-	static ref RECURSION: RwLock<HashMap<api::ParsId, Box<dyn MacroCB>>> = RwLock::default();
+	static RECURSION: RwLock<HashMap<api::ParsId, Box<dyn MacroCB>>> = RwLock::default();
-	static ref MACRO_SLOTS: RwLock<HashMap<api::ParsId, HashMap<api::MacroTreeId, Arc<MacTok>>>> =
+	static MACRO_SLOTS: RwLock<HashMap<api::ParsId, HashMap<api::MacroTreeId, Rc<MacTok>>>> =
 		RwLock::default();
 }
 
-pub fn macro_recur(run_id: api::ParsId, input: Vec<MacTree>) -> Option<Vec<MacTree>> {
+pub async fn macro_recur(run_id: api::ParsId, input: Vec<MacTree>) -> Option<Vec<MacTree>> {
 	(RECURSION.read().unwrap()[&run_id])(input)
 }
 
-pub fn macro_treev_to_api(run_id: api::ParsId, mtree: Vec<MacTree>) -> Vec<api::MacroTree> {
+pub async fn macro_treev_to_api(run_id: api::ParsId, mtree: Vec<MacTree>) -> Vec<api::MacroTree> {
 	let mut g = MACRO_SLOTS.write().unwrap();
 	let run_cache = g.get_mut(&run_id).expect("Parser run not found");
 	mtreev_to_api(&mtree, &mut |a: &AtomHand| {
 		let id = api::MacroTreeId((run_cache.len() as u64 + 1).try_into().unwrap());
-		run_cache.insert(id, Arc::new(MacTok::Atom(a.clone())));
+		run_cache.insert(id, Rc::new(MacTok::Atom(a.clone())));
 		api::MacroToken::Slot(id)
 	})
 }
 
-pub fn macro_treev_from_api(api: Vec<api::MacroTree>) -> Vec<MacTree> {
+pub async fn macro_treev_from_api(api: Vec<api::MacroTree>, i: &Interner) -> Vec<MacTree> {
-	mtreev_from_api(&api, &mut |atom| MacTok::Atom(AtomHand::from_api(atom.clone())))
+	mtreev_from_api(&api, i, &mut |atom| {
+		async { MacTok::Atom(AtomHand::from_api(atom.clone())) }.boxed_local()
+	})
+	.await
 }
 
 pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option<Vec<MacTree>> {
 	let mut slots = (MACRO_SLOTS.write().unwrap()).remove(&run_id).expect("Run not found");
 	return work(&mut slots, tree);
 	fn work(
-		slots: &mut HashMap<api::MacroTreeId, Arc<MacTok>>,
+		slots: &mut HashMap<api::MacroTreeId, Rc<MacTok>>,
 		tree: &[MacTree],
 	) -> Option<Vec<MacTree>> {
 		let items = (tree.iter())
@@ -59,8 +64,8 @@ pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option<Vec<MacTree
 						MacTok::Ref(_) => panic!("Ref is an extension-local optimization"),
 						MacTok::Done(_) => panic!("Created and removed by matcher"),
 						MacTok::Slot(slot) => slots.get(&slot.id()).expect("Slot not found").clone(),
-						MacTok::S(paren, b) => Arc::new(MacTok::S(*paren, work(slots, b)?)),
+						MacTok::S(paren, b) => Rc::new(MacTok::S(*paren, work(slots, b)?)),
-						MacTok::Lambda(a, b) => Arc::new(match (work(slots, a), work(slots, b)) {
+						MacTok::Lambda(a, b) => Rc::new(match (work(slots, a), work(slots, b)) {
 							(None, None) => return None,
 							(Some(a), None) => MacTok::Lambda(a, b.clone()),
 							(None, Some(b)) => MacTok::Lambda(a.clone(), b),
@@ -92,7 +97,7 @@ pub struct MacroRepo {
 impl MacroRepo {
 	/// TODO: the recursion inside this function needs to be moved into Orchid.
 	/// See the markdown note
-	pub fn process_exprv(&self, target: &[MacTree]) -> Option<Vec<MacTree>> {
+	pub fn process_exprv(&self, target: &[MacTree], i: &Interner) -> Option<Vec<MacTree>> {
 		let mut workcp = target.to_vec();
 		let mut lexicon;
 
@@ -100,24 +105,25 @@ impl MacroRepo {
 			lexicon = HashSet::new();
 			target.iter().for_each(|tgt| fill_lexicon(tgt, &mut lexicon));
 
-			for (i, tree) in workcp.iter().enumerate() {
+			for (idx, tree) in workcp.iter().enumerate() {
 				let MacTok::Name(name) = &*tree.tok else { continue };
 				let matches = (self.named.get(name).into_iter().flatten())
 					.filter(|m| m.deps.is_subset(&lexicon))
 					.filter_map(|mac| {
-						mac.cases.iter().find_map(|cas| cas.0.apply(&workcp[i..], |_| false).map(|s| (cas, s)))
+						(mac.cases.iter())
+							.find_map(|cas| cas.0.apply(&workcp[idx..], i, |_| false).map(|s| (cas, s)))
 					})
 					.collect_vec();
 				assert!(
 					matches.len() < 2,
 					"Multiple conflicting matches on {:?}: {:?}",
-					&workcp[i..],
+					&workcp[idx..],
 					matches
 				);
 				let Some((case, (state, tail))) = matches.into_iter().next() else { continue };
 				let inj = (run_body(&case.1, state).into_iter())
-					.map(|MacTree { pos, tok }| MacTree { pos, tok: Arc::new(MacTok::Done(tok)) });
+					.map(|MacTree { pos, tok }| MacTree { pos, tok: Rc::new(MacTok::Done(tok)) });
-				workcp.splice(i..(workcp.len() - tail.len()), inj);
+				workcp.splice(idx..(workcp.len() - tail.len()), inj);
 				continue 'try_named;
 			}
 			break;
@@ -133,8 +139,9 @@ impl MacroRepo {
 
 		let results = (workcp.into_iter())
 			.map(|mt| match &*mt.tok {
-				MTok::S(p, body) => self.process_exprv(body).map(|body| MTok::S(*p, body).at(mt.pos)),
+				MTok::S(p, body) => self.process_exprv(body, i).map(|body| MTok::S(*p, body).at(mt.pos)),
-				MTok::Lambda(arg, body) => match (self.process_exprv(arg), self.process_exprv(body)) {
+				MTok::Lambda(arg, body) =>
+					match (self.process_exprv(arg, i), self.process_exprv(body, i)) {
 						(Some(arg), Some(body)) => Some(MTok::Lambda(arg, body).at(mt.pos)),
 						(Some(arg), None) => Some(MTok::Lambda(arg, body.to_vec()).at(mt.pos)),
 						(None, Some(body)) => Some(MTok::Lambda(arg.to_vec(), body).at(mt.pos)),
@@ -169,6 +176,6 @@ fn run_body(body: &Code, mut state: MatchState<'_>) -> Vec<MacTree> {
 	let inject: Vec<MacTree> = todo!("Call the interpreter with bindings");
 	inject
 		.into_iter()
-		.map(|MTree { pos, tok }| MTree { pos, tok: Arc::new(MTok::Done(tok)) })
+		.map(|MTree { pos, tok }| MTree { pos, tok: Rc::new(MTok::Done(tok)) })
 		.collect_vec()
 }

orchid-host/src/parse.rs

@@ -1,10 +1,10 @@
-use std::sync::Arc;
+use std::rc::Rc;
-use std::{iter, thread};
 
+use futures::FutureExt;
+use futures::future::join_all;
 use itertools::Itertools;
 use never::Never;
 use orchid_base::error::{OrcErrv, OrcRes, Reporter, mk_err, mk_errv};
-use orchid_base::intern;
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::macros::{MTok, MTree};
@@ -16,8 +16,9 @@ use orchid_base::parse::{
 use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
 
-use crate::extension::{AtomHand, System};
+use crate::atom::AtomHand;
 use crate::macros::MacTree;
+use crate::system::System;
 use crate::tree::{
 	Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, ParsTokTree, Rule, RuleKind,
 };
@@ -29,111 +30,104 @@ pub trait ParseCtx: Send + Sync {
 	fn reporter(&self) -> &impl Reporter;
 }
 
-pub fn parse_items(
+pub async fn parse_items(
 	ctx: &impl ParseCtx,
-	path: Substack<Tok<String>>,
+	path: Substack<'_, Tok<String>>,
-	items: ParsSnippet,
+	items: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	let lines = line_items(items);
+	let lines = line_items(items).await;
-	let mut ok = iter::from_fn(|| None).take(lines.len()).collect_vec();
+	let line_res =
-	thread::scope(|s| {
+		join_all(lines.into_iter().map(|p| parse_item(ctx, path.clone(), p.output, p.tail))).await;
-		let mut threads = Vec::new();
+	Ok(line_res.into_iter().flat_map(|l| l.ok().into_iter().flatten()).collect())
-		for (slot, Parsed { output: cmts, tail }) in ok.iter_mut().zip(lines.into_iter()) {
-			let path = &path;
-			threads.push(s.spawn(move || {
-				*slot = Some(parse_item(ctx, path.clone(), cmts, tail)?);
-				Ok::<(), OrcErrv>(())
-			}))
-		}
-		for t in threads {
-			t.join().unwrap().err().into_iter().flatten().for_each(|e| ctx.reporter().report(e))
-		}
-	});
-	Ok(ok.into_iter().flatten().flatten().collect_vec())
 }
 
-pub fn parse_item(
+pub async fn parse_item(
 	ctx: &impl ParseCtx,
-	path: Substack<Tok<String>>,
+	path: Substack<'_, Tok<String>>,
 	comments: Vec<Comment>,
-	item: ParsSnippet,
+	item: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
 	match item.pop_front() {
 		Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
-			n if *n == intern!(str: "export") => match try_pop_no_fluff(postdisc)? {
+			n if *n == item.i("export").await => match try_pop_no_fluff(postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
-					parse_exportable_item(ctx, path, comments, true, n.clone(), tail),
+					parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
 				Parsed { output: TokTree { tok: Token::NS, .. }, tail } => {
-					let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail)?;
+					let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail).await?;
 					let mut ok = Vec::new();
-					exports.into_iter().for_each(|(e, pos)| match (&e.path.as_slice(), e.name) {
+					for (e, pos) in exports {
+						match (&e.path.as_slice(), e.name) {
 							([], Some(n)) =>
 								ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }),
 							(_, Some(_)) => ctx.reporter().report(mk_err(
-							intern!(str: "Compound export"),
+								tail.i("Compound export").await,
 								"Cannot export compound names (names containing the :: separator)",
 								[pos.into()],
 							)),
 							(_, None) => ctx.reporter().report(mk_err(
-							intern!(str: "Wildcard export"),
+								tail.i("Wildcard export").await,
 								"Exports cannot contain the globstar *",
 								[pos.into()],
 							)),
-					});
+						}
-					expect_end(tail)?;
+					}
+					expect_end(tail).await?;
 					Ok(ok)
 				},
-				Parsed { output, .. } => Err(mk_errv(
+				Parsed { output, tail } => Err(mk_errv(
-					intern!(str: "Malformed export"),
+					tail.i("Malformed export").await,
 					"`export` can either prefix other lines or list names inside ::( ) or ::[ ]",
 					[Pos::Range(output.range.clone()).into()],
 				)),
 			},
-			n if *n == intern!(str: "import") => parse_import(ctx, postdisc).map(|v| {
+			n if *n == item.i("import").await => parse_import(ctx, postdisc).await.map(|v| {
 				Vec::from_iter(v.into_iter().map(|(t, pos)| Item {
 					comments: comments.clone(),
 					pos,
 					kind: ItemKind::Import(t),
 				}))
 			}),
-			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc),
+			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
 		},
 		Some(_) =>
-			Err(mk_errv(intern!(str: "Expected a line type"), "All lines must begin with a keyword", [
+			Err(mk_errv(item.i("Expected a line type").await, "All lines must begin with a keyword", [
 				Pos::Range(item.pos()).into(),
 			])),
 		None => unreachable!("These lines are filtered and aggregated in earlier stages"),
 	}
 }
 
-pub fn parse_import(ctx: &impl ParseCtx, tail: ParsSnippet) -> OrcRes<Vec<(Import, Pos)>> {
+pub async fn parse_import(
-	let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail)?;
+	ctx: &impl ParseCtx,
-	expect_end(tail)?;
+	tail: ParsSnippet<'_>,
+) -> OrcRes<Vec<(Import, Pos)>> {
+	let Parsed { output: imports, tail } = parse_multiname(ctx.reporter(), tail).await?;
+	expect_end(tail).await?;
 	Ok(imports)
 }
 
-pub fn parse_exportable_item(
+pub async fn parse_exportable_item(
 	ctx: &impl ParseCtx,
-	path: Substack<Tok<String>>,
+	path: Substack<'_, Tok<String>>,
 	comments: Vec<Comment>,
 	exported: bool,
 	discr: Tok<String>,
-	tail: ParsSnippet,
+	tail: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
-	let kind = if discr == intern!(str: "mod") {
+	let kind = if discr == tail.i("mod").await {
-		let (name, body) = parse_module(ctx, path, tail)?;
+		let (name, body) = parse_module(ctx, path, tail).await?;
 		ItemKind::Member(Member::new(name, MemberKind::Mod(body)))
-	} else if discr == intern!(str: "const") {
+	} else if discr == tail.i("const").await {
-		let (name, val) = parse_const(tail)?;
+		let (name, val) = parse_const(tail).await?;
-		let locator = CodeLocator::to_const(path.push(name.clone()).unreverse());
+		let locator = CodeLocator::to_const(tail.i(&path.push(name.clone()).unreverse()).await);
 		ItemKind::Member(Member::new(name, MemberKind::Const(Code::from_code(locator, val))))
 	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
-		let line = sys.parse(tail.to_vec(), exported, comments)?;
+		let line = sys.parse(tail.to_vec(), exported, comments).await?;
-		return parse_items(ctx, path, Snippet::new(tail.prev(), &line));
+		return parse_items(ctx, path, Snippet::new(tail.prev(), &line, tail.interner())).await;
 	} else {
 		let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
 		return Err(mk_errv(
-			intern!(str: "Unrecognized line type"),
+			tail.i("Unrecognized line type").await,
 			format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
 			[Pos::Range(tail.prev().range.clone()).into()],
 		));
@@ -141,82 +135,90 @@ pub fn parse_exportable_item(
 	Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }])
 }
 
-pub fn parse_module(
+pub async fn parse_module(
 	ctx: &impl ParseCtx,
-	path: Substack<Tok<String>>,
+	path: Substack<'_, Tok<String>>,
-	tail: ParsSnippet,
+	tail: ParsSnippet<'_>,
 ) -> OrcRes<(Tok<String>, Module)> {
-	let (name, tail) = match try_pop_no_fluff(tail)? {
+	let (name, tail) = match try_pop_no_fluff(tail).await? {
 		Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } => (n.clone(), tail),
 		Parsed { output, .. } => {
 			return Err(mk_errv(
-				intern!(str: "Missing module name"),
+				tail.i("Missing module name").await,
-				format!("A name was expected, {output} was found"),
+				format!("A name was expected, {} was found", output.print().await),
 				[Pos::Range(output.range.clone()).into()],
 			));
 		},
 	};
-	let Parsed { output, tail: surplus } = try_pop_no_fluff(tail)?;
+	let Parsed { output, tail: surplus } = try_pop_no_fluff(tail).await?;
-	expect_end(surplus)?;
+	expect_end(surplus).await?;
-	let body = output.as_s(Paren::Round).ok_or_else(|| {
+	let Some(body) = output.as_s(Paren::Round, tail.interner()) else {
-		mk_errv(
+		return Err(mk_errv(
-			intern!(str: "Expected module body"),
+			tail.i("Expected module body").await,
-			format!("A ( block ) was expected, {output} was found"),
+			format!("A ( block ) was expected, {} was found", output.print().await),
 			[Pos::Range(output.range.clone()).into()],
-		)
+		));
-	})?;
+	};
 	let path = path.push(name.clone());
-	Ok((name, Module::new(parse_items(ctx, path, body)?)))
+	Ok((name, Module::new(parse_items(ctx, path, body).await?)))
 }
 
-pub fn parse_const(tail: ParsSnippet) -> OrcRes<(Tok<String>, Vec<ParsTokTree>)> {
+pub async fn parse_const(tail: ParsSnippet<'_>) -> OrcRes<(Tok<String>, Vec<ParsTokTree>)> {
-	let Parsed { output, tail } = try_pop_no_fluff(tail)?;
+	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
-	let name = output.as_name().ok_or_else(|| {
+	let Some(name) = output.as_name() else {
-		mk_errv(
-			intern!(str: "Missing module name"),
-			format!("A name was expected, {output} was found"),
-			[Pos::Range(output.range.clone()).into()],
-		)
-	})?;
-	let Parsed { output, tail } = try_pop_no_fluff(tail)?;
-	if !output.is_kw(intern!(str: "=")) {
 		return Err(mk_errv(
-			intern!(str: "Missing walrus := separator"),
+			tail.i("Missing module name").await,
-			format!("Expected operator := , found {output}"),
+			format!("A name was expected, {} was found", output.print().await),
+			[Pos::Range(output.range.clone()).into()],
+		));
+	};
+	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
+	if !output.is_kw(tail.i("=").await) {
+		return Err(mk_errv(
+			tail.i("Missing walrus := separator").await,
+			format!("Expected operator := , found {}", output.print().await),
 			[Pos::Range(output.range.clone()).into()],
 		));
 	}
-	try_pop_no_fluff(tail)?;
+	try_pop_no_fluff(tail).await?;
 	Ok((name, tail.iter().flat_map(strip_fluff).collect_vec()))
 }
 
-pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
+pub async fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
 	let mut mtreev = Vec::new();
 	while let Some((ttree, tail)) = snip.pop_front() {
 		let (range, tok, tail) = match &ttree.tok {
-			Token::S(p, b) =>
+			Token::S(p, b) => (
-				(ttree.range.clone(), MTok::S(*p, parse_mtree(Snippet::new(ttree, b))?), tail),
+				ttree.range.clone(),
+				MTok::S(*p, parse_mtree(Snippet::new(ttree, b, snip.interner())).boxed_local().await?),
+				tail,
+			),
 			Token::Name(tok) => {
 				let mut segments = vec![tok.clone()];
 				let mut end = ttree.range.end;
 				while let Some((TokTree { tok: Token::NS, .. }, tail)) = snip.pop_front() {
-					let Parsed { output, tail } = try_pop_no_fluff(tail)?;
+					let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
-					segments.push(output.as_name().ok_or_else(|| {
+					let Some(seg) = output.as_name() else {
-						mk_errv(
+						return Err(mk_errv(
-							intern!(str: "Namespaced name interrupted"),
+							tail.i("Namespaced name interrupted").await,
 							"In expression context, :: must always be followed by a name.\n\
 											::() is permitted only in import and export items",
 							[Pos::Range(output.range.clone()).into()],
-						)
+						));
-					})?);
+					};
+					segments.push(seg);
 					snip = tail;
 					end = output.range.end;
 				}
-				(ttree.range.start..end, MTok::Name(Sym::new(segments).unwrap()), snip)
+				(
+					ttree.range.start..end,
+					MTok::Name(Sym::new(segments, snip.interner()).await.unwrap()),
+					snip,
+				)
 			},
 			Token::NS => {
 				return Err(mk_errv(
-					intern!(str: "Unexpected :: in macro pattern"),
+					tail.i("Unexpected :: in macro pattern").await,
 					":: can only follow a name outside export statements",
 					[Pos::Range(ttree.range.clone()).into()],
 				));
@@ -224,8 +226,11 @@ pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
 			Token::Ph(ph) => (ttree.range.clone(), MTok::Ph(ph.clone()), tail),
 			Token::Atom(_) | Token::Macro(_) => {
 				return Err(mk_errv(
-					intern!(str: "Unsupported token in macro patterns"),
+					tail.i("Unsupported token in macro patterns").await,
-					format!("Macro patterns can only contain names, braces, and lambda, not {ttree}."),
+					format!(
+						"Macro patterns can only contain names, braces, and lambda, not {}.",
+						ttree.print().await
+					),
 					[Pos::Range(ttree.range.clone()).into()],
 				));
 			},
@@ -233,50 +238,57 @@ pub fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
 			Token::Bottom(e) => return Err(e.clone()),
 			Token::LambdaHead(arg) => (
 				ttree.range.start..snip.pos().end,
-				MTok::Lambda(parse_mtree(Snippet::new(ttree, arg))?, parse_mtree(tail)?),
+				MTok::Lambda(
-				Snippet::new(ttree, &[]),
+					parse_mtree(Snippet::new(ttree, arg, snip.interner())).await?,
+					parse_mtree(tail).await?,
 				),
-			Token::Slot(_) | Token::X(_) => panic!("Did not expect {} in parsed token tree", &ttree.tok),
+				Snippet::new(ttree, &[], snip.interner()),
+			),
+			Token::Slot(_) | Token::X(_) =>
+				panic!("Did not expect {} in parsed token tree", &ttree.tok.print().await),
 		};
-		mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Arc::new(tok) });
+		mtreev.push(MTree { pos: Pos::Range(range.clone()), tok: Rc::new(tok) });
 		snip = tail;
 	}
 	Ok(mtreev)
 }
 
-pub fn parse_macro(
+pub async fn parse_macro(
-	tail: ParsSnippet,
+	tail: ParsSnippet<'_>,
 	macro_i: u16,
-	path: Substack<Tok<String>>,
+	path: Substack<'_, Tok<String>>,
 ) -> OrcRes<Vec<Rule>> {
-	let (surplus, prev, block) = match try_pop_no_fluff(tail)? {
+	let (surplus, prev, block) = match try_pop_no_fluff(tail).await? {
 		Parsed { tail, output: o @ TokTree { tok: Token::S(Paren::Round, b), .. } } => (tail, o, b),
 		Parsed { output, .. } => {
 			return Err(mk_errv(
-				intern!(str: "m"),
+				tail.i("m").await,
 				"Macro blocks must either start with a block or a ..$:number",
 				[Pos::Range(output.range.clone()).into()],
 			));
 		},
 	};
-	expect_end(surplus)?;
+	expect_end(surplus).await?;
 	let mut errors = Vec::new();
 	let mut rules = Vec::new();
-	for (i, item) in line_items(Snippet::new(prev, block)).into_iter().enumerate() {
+	for (i, item) in
-		let Parsed { tail, output } = try_pop_no_fluff(item.tail)?;
+		line_items(Snippet::new(&prev, block, tail.interner())).await.into_iter().enumerate()
-		if !output.is_kw(intern!(str: "rule")) {
+	{
+		let Parsed { tail, output } = try_pop_no_fluff(item.tail).await?;
+		if !output.is_kw(tail.i("rule").await) {
 			errors.extend(mk_errv(
-				intern!(str: "non-rule in macro"),
+				tail.i("non-rule in macro").await,
-				format!("Expected `rule`, got {output}"),
+				format!("Expected `rule`, got {}", output.print().await),
 				[Pos::Range(output.range.clone()).into()],
 			));
 			continue;
 		};
-		let (pat, body) = match tail.split_once(|t| t.is_kw(intern!(str: "=>"))) {
+		let arrow = tail.i("=>").await;
+		let (pat, body) = match tail.split_once(|t| t.is_kw(arrow.clone())) {
 			Some((a, b)) => (a, b),
 			None => {
 				errors.extend(mk_errv(
-					intern!(str: "no => in macro rule"),
+					tail.i("no => in macro rule").await,
 					"The pattern and body of a rule must be separated by a =>",
 					[Pos::Range(tail.pos()).into()],
 				));
@@ -286,9 +298,9 @@ pub fn parse_macro(
 		rules.push(Rule {
 			comments: item.output,
 			pos: Pos::Range(tail.pos()),
-			pattern: parse_mtree(pat)?,
+			pattern: parse_mtree(pat).await?,
 			kind: RuleKind::Native(Code::from_code(
-				CodeLocator::to_rule(path.unreverse(), macro_i, i as u16),
+				CodeLocator::to_rule(tail.i(&path.unreverse()).await, macro_i, i as u16),
 				body.to_vec(),
 			)),
 		})

orchid-host/src/rule/build.rs

@@ -107,43 +107,55 @@ fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
 
 #[cfg(test)]
 mod test {
-	use std::sync::Arc;
+	use std::rc::Rc;
 
 	use orchid_api::PhKind;
+	use orchid_base::interner::Interner;
 	use orchid_base::location::SourceRange;
+	use orchid_base::sym;
 	use orchid_base::tokens::Paren;
 	use orchid_base::tree::Ph;
-	use orchid_base::{intern, sym};
+	use test_executors::spin_on;
 
 	use super::mk_any;
 	use crate::macros::{MacTok, MacTree};
 
 	#[test]
 	fn test_scan() {
-		let ex = |tok: MacTok| MacTree { tok: Arc::new(tok), pos: SourceRange::mock().pos() };
+		spin_on(async {
+			let i = Interner::new_master();
+			let ex = |tok: MacTok| async {
+				MacTree { tok: Rc::new(tok), pos: SourceRange::mock(&i).await.pos() }
+			};
 			let pattern = vec![
 				ex(MacTok::Ph(Ph {
 					kind: PhKind::Vector { priority: 0, at_least_one: false },
-				name: intern!(str: "::prefix"),
+					name: i.i("::prefix").await,
-			})),
+				}))
-			ex(MacTok::Name(sym!(prelude::do))),
+				.await,
+				ex(MacTok::Name(sym!(prelude::do; i).await)).await,
 				ex(MacTok::S(Paren::Round, vec![
 					ex(MacTok::Ph(Ph {
 						kind: PhKind::Vector { priority: 0, at_least_one: false },
-					name: intern!(str: "expr"),
+						name: i.i("expr").await,
-				})),
+					}))
-				ex(MacTok::Name(sym!(prelude::;))),
+					.await,
+					ex(MacTok::Name(sym!(prelude::; ; i).await)).await,
 					ex(MacTok::Ph(Ph {
 						kind: PhKind::Vector { priority: 1, at_least_one: false },
-					name: intern!(str: "rest"),
+						name: i.i("rest").await,
-				})),
+					}))
-			])),
+					.await,
+				]))
+				.await,
 				ex(MacTok::Ph(Ph {
 					kind: PhKind::Vector { priority: 0, at_least_one: false },
-				name: intern!(str: "::suffix"),
+					name: i.i("::suffix").await,
-			})),
+				}))
+				.await,
 			];
 			let matcher = mk_any(&pattern);
 			println!("{matcher}");
+		})
 	}
 }

orchid-host/src/rule/matcher.rs

@@ -2,7 +2,7 @@ use std::fmt;
 
 use itertools::Itertools;
 use orchid_api::PhKind;
-use orchid_base::intern;
+use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::tree::Ph;
@@ -21,7 +21,7 @@ pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwra
 
 pub struct NamedMatcher(AnyMatcher);
 impl NamedMatcher {
-	pub fn new(pattern: &[MacTree]) -> Self {
+	pub async fn new(pattern: &[MacTree], i: &Interner) -> Self {
 		assert!(
 			matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))),
 			"Named matchers must begin with a name"
@@ -31,7 +31,7 @@ impl NamedMatcher {
 			true => Self(mk_any(pattern)),
 			false => {
 				let kind: PhKind = PhKind::Vector { priority: 0, at_least_one: false };
-				let suffix = [MacTok::Ph(Ph { name: intern!(str: "::after"), kind }).at(Pos::None)];
+				let suffix = [MacTok::Ph(Ph { name: i.i("::after").await, kind }).at(Pos::None)];
 				Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
 			},
 		}
@@ -39,18 +39,18 @@ impl NamedMatcher {
 	/// Also returns the tail, if any, which should be matched further
 	/// Note that due to how priod works below, the main usable information from
 	/// the tail is its length
-	pub fn apply<'a>(
+	pub async fn apply<'a>(
 		&self,
 		seq: &'a [MacTree],
+		i: &Interner,
 		save_loc: impl Fn(Sym) -> bool,
 	) -> Option<(MatchState<'a>, &'a [MacTree])> {
-		any_match(&self.0, seq, &save_loc).map(|mut state| {
+		let mut state = any_match(&self.0, seq, &save_loc)?;
-			match state.remove(intern!(str: "::after")) {
+		match state.remove(i.i("::after").await) {
 			Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"),
-				Some(StateEntry::Vec(v)) => (state, v),
+			Some(StateEntry::Vec(v)) => Some((state, v)),
-				None => (state, &[][..]),
+			None => Some((state, &[][..])),
 		}
-		})
 	}
 }
 impl fmt::Display for NamedMatcher {

orchid-host/src/rule/scal_match.rs

@@ -20,7 +20,7 @@ pub fn scal_match<'a>(
 		(ScalMatcher::Placeh { key }, _) =>
 			Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
 		(ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>
-			any_match(b_mat, &body[..], save_loc),
+			any_match(&b_mat, &body[..], save_loc),
 		(ScalMatcher::Lambda(arg_mat, b_mat), MacTok::Lambda(arg, body)) =>
 			Some(any_match(arg_mat, arg, save_loc)?.combine(any_match(b_mat, body, save_loc)?)),
 		_ => None,

orchid-host/src/subprocess.rs

@@ -1,79 +1,96 @@
-use std::io::{self, BufRead as _, Write};
+use std::cell::RefCell;
 use std::path::PathBuf;
-use std::sync::mpsc::sync_channel;
+use std::pin::Pin;
-use std::{process, thread};
+use std::rc::Rc;
+use std::thread;
 
+use async_process::{self, Child, ChildStdin, ChildStdout};
+use async_std::io::{self, BufReadExt, BufReader};
 use async_std::sync::Mutex;
+use futures::FutureExt;
+use futures::future::LocalBoxFuture;
+use futures::task::LocalSpawnExt;
 use orchid_api_traits::{Decode, Encode};
 use orchid_base::builtin::{ExtInit, ExtPort};
 use orchid_base::logging::Logger;
 use orchid_base::msg::{recv_msg, send_msg};
 
 use crate::api;
+use crate::ctx::Ctx;
 
-pub struct ExtensionCommand(pub process::Command, pub Logger);
+pub async fn ext_command(
-impl ExtFactory for ExtensionCommand {
+	cmd: std::process::Command,
-	fn run(self: Box<Self>, onmessage: OnMessage) -> ExtInit {
+	logger: Logger,
-		let Self(mut cmd, logger) = *self;
+	ctx: Ctx,
+) -> io::Result<ExtInit> {
 	let prog_pbuf = PathBuf::from(cmd.get_program());
 	let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy().to_string();
-		let mut child = cmd
+	let mut child = async_process::Command::from(cmd)
-			.stdin(process::Stdio::piped())
+		.stdin(async_process::Stdio::piped())
-			.stdout(process::Stdio::piped())
+		.stdout(async_process::Stdio::piped())
-			.stderr(process::Stdio::piped())
+		.stderr(async_process::Stdio::piped())
 		.spawn()?;
 	let mut stdin = child.stdin.take().unwrap();
-		api::HostHeader { log_strategy: logger.strat() }.encode(&mut stdin);
+	api::HostHeader { log_strategy: logger.strat() }.encode(Pin::new(&mut stdin));
-		stdin.flush()?;
 	let mut stdout = child.stdout.take().unwrap();
-		let header = api::ExtensionHeader::decode(&mut stdout);
+	let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await;
 	let child_stderr = child.stderr.take().unwrap();
-		let (set_onmessage, recv_onmessage) = sync_channel(0);
-		thread::Builder::new().name(format!("stdout-fwd:{prog}")).spawn(move || {
-			let mut onmessage: Box<dyn FnMut(&[u8]) + Send> = recv_onmessage.recv().unwrap();
-			drop(recv_onmessage);
-			loop {
-				match recv_msg(&mut stdout) {
-					Ok(msg) => onmessage(&msg[..]),
-					Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break,
-					Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
-				}
-			}
-		})?;
 	thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || {
-			let mut reader = io::BufReader::new(child_stderr);
+		async_std::task::block_on(async move {
+			let mut reader = BufReader::new(child_stderr);
 			loop {
 				let mut buf = String::new();
-				if 0 == reader.read_line(&mut buf).unwrap() {
+				if 0 == reader.read_line(&mut buf).await.unwrap() {
 					break;
 				}
 				logger.log(buf);
 			}
+		})
 	})?;
-		Ok(Subprocess { child: Mutex::new(child), stdin: Mutex::new(stdin), set_onmessage, header })
+	Ok(ExtInit {
-	}
+		header,
+		port: Box::new(Subprocess {
+			child: Rc::new(RefCell::new(child)),
+			stdin: Mutex::new(Box::pin(stdin)),
+			stdout: Mutex::new(Box::pin(stdout)),
+			ctx,
+		}),
+	})
 }
 
 pub struct Subprocess {
-	child: Mutex<process::Child>,
+	child: Rc<RefCell<Child>>,
-	stdin: Mutex<process::ChildStdin>,
+	stdin: Mutex<Pin<Box<ChildStdin>>>,
-	stdout: Mutex<process::ChildStdout>,
+	stdout: Mutex<Pin<Box<ChildStdout>>>,
-	header: api::ExtensionHeader,
+	ctx: Ctx,
-}
-impl Subprocess {
-	pub fn new(mut cmd: process::Command, logger: Logger) -> io::Result<Self> {}
 }
 impl Drop for Subprocess {
-	fn drop(&mut self) { self.child.lock().unwrap().wait().expect("Extension exited with error"); }
+	fn drop(&mut self) {
+		let child = self.child.clone();
+		(self.ctx.spawn.spawn_local(async move {
+			let status = child.borrow_mut().status().await.expect("Extension exited with error");
+			assert!(status.success(), "Extension exited with error {status}");
+		}))
+		.expect("Could not spawn process terminating future")
+	}
 }
 impl ExtPort for Subprocess {
-	fn send(&self, msg: &[u8]) {
+	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
 		if msg.starts_with(&[0, 0, 0, 0x1c]) {
 			panic!("Received unnecessary prefix");
 		}
-		send_msg(&mut *self.stdin.lock().unwrap(), msg).unwrap()
+		async { send_msg(Pin::new(&mut *self.stdin.lock().await), msg).await.unwrap() }.boxed_local()
 	}
-	fn recv<'a>(&self, cb: Box<dyn FnOnce(&[u8]) + Send + 'a>) -> futures::future::BoxFuture<()> {
+	fn recv<'a>(
-		async {}
+		&'a self,
+		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
+	) -> LocalBoxFuture<'a, ()> {
+		Box::pin(async {
+			match recv_msg(self.stdout.lock().await.as_mut()).await {
+				Ok(msg) => cb(&msg).await,
+				Err(e) if e.kind() == io::ErrorKind::BrokenPipe => (),
+				Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
+			}
+		})
 	}
 }

orchid-host/src/system.rs

@@ -0,0 +1,213 @@
+use std::collections::VecDeque;
+use std::fmt;
+use std::future::Future;
+use std::rc::{Rc, Weak};
+
+use async_stream::stream;
+use derive_destructure::destructure;
+use futures::StreamExt;
+use futures::future::join_all;
+use futures::task::LocalSpawnExt;
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchid_base::async_once_cell::OnceCell;
+use orchid_base::char_filter::char_filter_match;
+use orchid_base::clone;
+use orchid_base::error::{OrcErrv, OrcRes};
+use orchid_base::interner::Tok;
+use orchid_base::parse::Comment;
+use orchid_base::reqnot::{ReqNot, Requester};
+use orchid_base::tree::ttv_from_api;
+use ordered_float::NotNan;
+use substack::{Stackframe, Substack};
+
+use crate::api;
+use crate::ctx::Ctx;
+use crate::extension::{Extension, WeakExtension};
+use crate::tree::{Member, ParsTokTree};
+
+#[derive(destructure)]
+struct SystemInstData {
+	ctx: Ctx,
+	ext: Extension,
+	decl_id: api::SysDeclId,
+	lex_filter: api::CharFilter,
+	id: api::SysId,
+	const_root: OnceCell<Vec<Member>>,
+	line_types: Vec<Tok<String>>,
+}
+impl Drop for SystemInstData {
+	fn drop(&mut self) { self.ext.system_drop(self.id); }
+}
+impl fmt::Debug for SystemInstData {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		f.debug_struct("SystemInstData")
+			.field("decl_id", &self.decl_id)
+			.field("lex_filter", &self.lex_filter)
+			.field("id", &self.id)
+			.field("const_root", &self.const_root)
+			.field("line_types", &self.line_types)
+			.finish_non_exhaustive()
+	}
+}
+
+#[derive(Clone, Debug)]
+pub struct System(Rc<SystemInstData>);
+impl System {
+	pub fn id(&self) -> api::SysId { self.0.id }
+	pub fn ext(&self) -> &Extension { &self.0.ext }
+	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
+	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.ext.reqnot() }
+	pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
+		self.reqnot().request(api::GetMember(self.0.id, id)).await
+	}
+	pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
+	pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
+	/// Have this system lex a part of the source. It is assumed that
+	/// [Self::can_lex] was called and returned true.
+	pub async fn lex<F: Future<Output = Option<api::SubLexed>>>(
+		&self,
+		source: Tok<String>,
+		pos: u32,
+		r: impl FnMut(u32) -> F,
+	) -> api::OrcResult<Option<api::LexedExpr>> {
+		self.0.ext.lex_req(source, pos, self.id(), r).await
+	}
+	pub fn can_parse(&self, ltyp: Tok<String>) -> bool { self.0.line_types.contains(&ltyp) }
+	pub fn line_types(&self) -> impl Iterator<Item = &Tok<String>> + '_ { self.0.line_types.iter() }
+	pub async fn parse(
+		&self,
+		line: Vec<ParsTokTree>,
+		exported: bool,
+		comments: Vec<Comment>,
+	) -> OrcRes<Vec<ParsTokTree>> {
+		let line =
+			join_all(line.iter().map(|t| async { t.to_api(&mut |n, _| match *n {}).await })).await;
+		let comments = comments.iter().map(Comment::to_api).collect_vec();
+		match self.reqnot().request(api::ParseLine { exported, sys: self.id(), comments, line }).await {
+			Ok(parsed) => Ok(ttv_from_api(parsed, &mut self.ctx().clone(), &self.ctx().i).await),
+			Err(e) => Err(OrcErrv::from_api(&e, &self.ctx().i).await),
+		}
+	}
+	pub async fn request(&self, req: Vec<u8>) -> Vec<u8> {
+		self.reqnot().request(api::SysFwded(self.id(), req)).await
+	}
+	pub(crate) fn drop_atom(&self, drop: api::AtomId) {
+		let this = self.0.clone();
+		(self.0.ctx.spawn.spawn_local(async move {
+			this.ctx.owned_atoms.write().await.remove(&drop);
+		}))
+		.expect("Failed to drop atom");
+	}
+	pub async fn print(&self) -> String {
+		let ctor = (self.0.ext.system_ctors().find(|c| c.id() == self.0.decl_id))
+			.expect("System instance with no associated constructor");
+		format!("System({} @ {} #{})", ctor.name(), ctor.priority(), self.0.id.0)
+	}
+	pub fn downgrade(&self) -> WeakSystem { WeakSystem(Rc::downgrade(&self.0)) }
+}
+
+pub struct WeakSystem(Weak<SystemInstData>);
+impl WeakSystem {
+	pub fn upgrade(&self) -> Option<System> { self.0.upgrade().map(System) }
+}
+
+pub struct SystemCtor {
+	pub(crate) decl: api::SystemDecl,
+	pub(crate) ext: WeakExtension,
+}
+impl SystemCtor {
+	pub fn name(&self) -> &str { &self.decl.name }
+	pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
+	pub fn depends(&self) -> impl ExactSizeIterator<Item = &str> {
+		self.decl.depends.iter().map(|s| &**s)
+	}
+	pub fn id(&self) -> api::SysDeclId { self.decl.id }
+	pub async fn run<'a>(&self, depends: impl IntoIterator<Item = &'a System>) -> System {
+		let depends = depends.into_iter().map(|si| si.id()).collect_vec();
+		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
+		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
+		let id = ext.ctx().next_sys_id();
+		let sys_inst = ext.reqnot().request(api::NewSystem { depends, id, system: self.decl.id }).await;
+		let data = System(Rc::new(SystemInstData {
+			decl_id: self.decl.id,
+			ext: ext.clone(),
+			ctx: ext.ctx().clone(),
+			lex_filter: sys_inst.lex_filter,
+			const_root: OnceCell::new(),
+			line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
+				.await,
+			id,
+		}));
+		(data.0.const_root.get_or_init(
+			clone!(data, ext; stream! {
+				for (k, v) in sys_inst.const_root {
+					yield Member::from_api(
+						api::Member { name: k, kind: v },
+						&mut vec![Tok::from_api(k, &ext.ctx().i).await],
+						&data,
+					).await
+				}
+			})
+			.collect(),
+		))
+		.await;
+		ext.ctx().systems.write().await.insert(id, data.downgrade());
+		data
+	}
+}
+
+#[derive(Debug, Clone)]
+pub enum SysResolvErr {
+	Loop(Vec<String>),
+	Missing(String),
+}
+
+pub async fn init_systems(
+	tgts: &[String],
+	exts: &[Extension],
+) -> Result<Vec<System>, SysResolvErr> {
+	let mut to_load = HashMap::<&str, &SystemCtor>::new();
+	let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque<&str>>();
+	while let Some(target) = to_find.pop_front() {
+		if to_load.contains_key(target) {
+			continue;
+		}
+		let ctor = (exts.iter())
+			.flat_map(|e| e.system_ctors().filter(|c| c.name() == target))
+			.max_by_key(|c| c.priority())
+			.ok_or_else(|| SysResolvErr::Missing(target.to_string()))?;
+		to_load.insert(target, ctor);
+		to_find.extend(ctor.depends());
+	}
+	let mut to_load_ordered = Vec::new();
+	fn walk_deps<'a>(
+		graph: &mut HashMap<&str, &'a SystemCtor>,
+		list: &mut Vec<&'a SystemCtor>,
+		chain: Stackframe<&str>,
+	) -> Result<(), SysResolvErr> {
+		if let Some(ctor) = graph.remove(chain.item) {
+			// if the above is none, the system is already queued. Missing systems are
+			// detected above
+			for dep in ctor.depends() {
+				if Substack::Frame(chain).iter().any(|c| *c == dep) {
+					let mut circle = vec![dep.to_string()];
+					circle.extend(Substack::Frame(chain).iter().map(|s| s.to_string()));
+					return Err(SysResolvErr::Loop(circle));
+				}
+				walk_deps(graph, list, Substack::Frame(chain).new_frame(dep))?
+			}
+			list.push(ctor);
+		}
+		Ok(())
+	}
+	for tgt in tgts {
+		walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?;
+	}
+	let mut systems = HashMap::<&str, System>::new();
+	for ctor in to_load_ordered.iter() {
+		let sys = ctor.run(ctor.depends().map(|n| &systems[n])).await;
+		systems.insert(ctor.name(), sys);
+	}
+	Ok(systems.into_values().collect_vec())
+}

orchid-host/src/tree.rs

@@ -1,22 +1,25 @@
 use std::fmt::Debug;
 use std::sync::{Mutex, OnceLock};
 
+use async_stream::stream;
+use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
 use never::Never;
+use orchid_base::clone;
 use orchid_base::error::OrcRes;
-use orchid_base::interner::{Tok, intern};
+use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::macros::mtreev_from_api;
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Import};
-use orchid_base::tree::{TokTree, Token};
+use orchid_base::tree::{AtomRepr, TokTree, Token};
 use ordered_float::NotNan;
-use substack::{Substack, with_iter_stack};
 
 use crate::api;
+use crate::atom::AtomHand;
 use crate::expr::Expr;
-use crate::extension::{AtomHand, System};
 use crate::macros::{MacTok, MacTree};
+use crate::system::System;
 
 pub type ParsTokTree = TokTree<'static, AtomHand, Never>;
 pub type ParsTok = Token<'static, AtomHand, Never>;
@@ -37,54 +40,79 @@ pub enum ItemKind {
 }
 
 impl Item {
-	pub fn from_api(tree: api::Item, path: Substack<Tok<String>>, sys: &System) -> Self {
+	pub async fn from_api(tree: api::Item, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
 		let kind = match tree.kind {
-			api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys)),
+			api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys).await),
-			api::ItemKind::Import(i) =>
+			api::ItemKind::Import(name) => ItemKind::Import(Import {
-				ItemKind::Import(Import { path: Sym::from_api(i).iter().collect(), name: None }),
+				path: Sym::from_api(name, &sys.ctx().i).await.iter().collect(),
-			api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e)),
+				name: None,
-			api::ItemKind::Macro(api::MacroBlock { priority, rules }) => ItemKind::Macro(priority, {
-				Vec::from_iter(rules.into_iter().map(|api| Rule {
-					pos: Pos::from_api(&api.location),
-					pattern: mtreev_from_api(&api.pattern, &mut |a| {
-						MacTok::Atom(AtomHand::from_api(a.clone()))
-					}),
-					kind: RuleKind::Remote(sys.clone(), api.id),
-					comments: api.comments.iter().map(Comment::from_api).collect_vec(),
-				}))
 			}),
+			api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &sys.ctx().i).await),
+			api::ItemKind::Macro(macro_block) => {
+				let mut rules = Vec::new();
+				for rule in macro_block.rules {
+					let mut comments = Vec::new();
+					for comment in rule.comments {
+						comments.push(Comment::from_api(&comment, &sys.ctx().i).await);
+					}
+					let pos = Pos::from_api(&rule.location, &sys.ctx().i).await;
+					let pattern = mtreev_from_api(&rule.pattern, &sys.ctx().i, &mut {
+						clone!(pos, sys);
+						move |a| {
+							clone!(pos, sys);
+							Box::pin(async move {
+								MacTok::Atom(AtomHand::from_api(a, pos.clone(), &mut sys.ctx().clone()).await)
+							})
+						}
+					})
+					.await;
+					rules.push(Rule { pos, pattern, kind: RuleKind::Remote(sys.clone(), rule.id), comments });
+				}
+				ItemKind::Macro(macro_block.priority, rules)
+			},
 		};
-		let comments = tree.comments.iter().map(Comment::from_api).collect_vec();
+		let mut comments = Vec::new();
-		Self { pos: Pos::from_api(&tree.location), comments, kind }
+		for comment in tree.comments.iter() {
+			comments.push(Comment::from_api(comment, &sys.ctx().i).await)
+		}
+		Self { pos: Pos::from_api(&tree.location, &sys.ctx().i).await, comments, kind }
 	}
 }
 
-#[derive(Debug)]
 pub struct Member {
 	pub name: Tok<String>,
 	pub kind: OnceLock<MemberKind>,
 	pub lazy: Mutex<Option<LazyMemberHandle>>,
 }
 impl Member {
-	pub fn from_api(api: api::Member, path: Substack<Tok<String>>, sys: &System) -> Self {
+	pub async fn from_api(api: api::Member, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
-		let name = Tok::from_api(api.name);
+		path.push(Tok::from_api(api.name, &sys.ctx().i).await);
-		let full_path = path.push(name.clone());
 		let kind = match api.kind {
 			api::MemberKind::Lazy(id) => {
-				return LazyMemberHandle(id, sys.clone(), intern(&full_path.unreverse())).into_member(name);
+				let handle = LazyMemberHandle(id, sys.clone(), path.clone());
+				return handle.into_member(path.pop().unwrap());
 			},
 			api::MemberKind::Const(c) => MemberKind::Const(Code::from_expr(
-				CodeLocator::to_const(full_path.unreverse()),
+				CodeLocator::to_const(sys.ctx().i.i(&*path).await),
-				Expr::from_api(&c, &mut ()),
+				Expr::from_api(&c, &mut sys.ext().clone()).await,
 			)),
-			api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, full_path, sys)),
+			api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, path, sys).await),
 		};
+		let name = path.pop().unwrap();
 		Member { name, kind: OnceLock::from(kind), lazy: Mutex::default() }
 	}
 	pub fn new(name: Tok<String>, kind: MemberKind) -> Self {
 		Member { name, kind: OnceLock::from(kind), lazy: Mutex::default() }
 	}
 }
+impl Debug for Member {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Member")
+			.field("name", &self.name)
+			.field("kind", &self.kind)
+			.finish_non_exhaustive()
+	}
+}
 
 #[derive(Debug)]
 pub enum MemberKind {
@@ -109,30 +137,27 @@ impl Module {
 			.collect_vec();
 		Self { imports: vec![], exports, items }
 	}
-	pub fn from_api(m: api::Module, path: Substack<Tok<String>>, sys: &System) -> Self {
+	pub async fn from_api(m: api::Module, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
-		let mut output = Vec::new();
+		Self::new(
-		for item in m.items.into_iter() {
+			stream! { for item in m.items { yield Item::from_api(item, path, sys).boxed_local().await } }
-			let next = Item::from_api(item, path.clone(), sys);
+				.collect::<Vec<_>>()
-			output.push(next);
+				.await,
-		}
+		)
-		Self::new(output)
 	}
 }
 
-#[derive(Debug)]
+pub struct LazyMemberHandle(api::TreeId, System, Vec<Tok<String>>);
-pub struct LazyMemberHandle(api::TreeId, System, Tok<Vec<Tok<String>>>);
 impl LazyMemberHandle {
-	pub fn run(self) -> OrcRes<MemberKind> {
+	pub async fn run(self) -> OrcRes<MemberKind> {
-		match self.1.get_tree(self.0) {
+		match self.1.get_tree(self.0).await {
 			api::MemberKind::Const(c) => Ok(MemberKind::Const(Code {
-				bytecode: Expr::from_api(&c, &mut ()).into(),
+				bytecode: Expr::from_api(&c, &mut self.1.ext().clone()).await.into(),
-				locator: CodeLocator { steps: self.2, rule_loc: None },
+				locator: CodeLocator { steps: self.1.ctx().i.i(&self.2).await, rule_loc: None },
 				source: None,
 			})),
-			api::MemberKind::Module(m) => with_iter_stack(self.2.iter().cloned(), |path| {
+			api::MemberKind::Module(m) =>
-				Ok(MemberKind::Mod(Module::from_api(m, path, &self.1)))
+				Ok(MemberKind::Mod(Module::from_api(m, &mut { self.2 }, &self.1).await)),
-			}),
+			api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run().boxed_local().await,
-			api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run(),
 		}
 	}
 	pub fn into_member(self, name: Tok<String>) -> Member {
@@ -181,10 +206,8 @@ pub struct CodeLocator {
 	rule_loc: Option<(u16, u16)>,
 }
 impl CodeLocator {
-	pub fn to_const(path: impl IntoIterator<Item = Tok<String>>) -> Self {
+	pub fn to_const(steps: Tok<Vec<Tok<String>>>) -> Self { Self { steps, rule_loc: None } }
-		Self { steps: intern(&path.into_iter().collect_vec()), rule_loc: None }
+	pub fn to_rule(steps: Tok<Vec<Tok<String>>>, macro_i: u16, rule_i: u16) -> Self {
-	}
+		Self { steps, rule_loc: Some((macro_i, rule_i)) }
-	pub fn to_rule(path: impl IntoIterator<Item = Tok<String>>, macro_i: u16, rule_i: u16) -> Self {
-		Self { steps: intern(&path.into_iter().collect_vec()), rule_loc: Some((macro_i, rule_i)) }
 	}
 }