orchid-base moved to async

.gitattributes

@@ -1,2 +1,6 @@
 *.jpg filter=lfs diff=lfs merge=lfs -text
-*.pdf filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.rs text eol=lf 
+*.toml text eol=lf 
+*.orc text eol=lf 
+*.md text eol=lf 

orchid-base/src/builtin.rs

@@ -1,6 +1,6 @@
-use std::future::Future;
 use std::ops::Deref;
-use std::pin::Pin;
+
+use futures::future::LocalBoxFuture;
 
 use crate::api;
 
@@ -11,8 +11,8 @@ use crate::api;
 ///
 /// There are no ordering guarantees about these
 pub trait ExtPort {
-	fn send(&self, msg: &[u8]) -> Pin<Box<dyn Future<Output = ()>>>;
+	fn send(&self, msg: &[u8]) -> LocalBoxFuture<'_, ()>;
-	fn recv<'a>(&self, cb: Box<dyn FnOnce(&[u8]) + Send + 'a>) -> Pin<Box<dyn Future<Output = ()>>>;
+	fn recv<'s, 'a: 's>(&'s self, cb: Box<dyn FnOnce(&[u8]) + 'a>) -> LocalBoxFuture<'a, ()>;
 }
 
 pub struct ExtInit {

orchid-base/src/error.rs

@@ -2,6 +2,7 @@ use std::fmt;
 use std::ops::Add;
 use std::sync::Arc;
 
+use futures::future::join_all;
 use itertools::Itertools;
 
 use crate::api;
@@ -21,10 +22,10 @@ impl ErrPos {
 	pub fn new(msg: &str, position: Pos) -> Self {
 		Self { message: Some(Arc::new(msg.to_string())), position }
 	}
-	fn from_api(api: &api::ErrLocation) -> Self {
+	async fn from_api(api: &api::ErrLocation) -> Self {
 		Self {
 			message: Some(api.message.clone()).filter(|s| !s.is_empty()),
-			position: Pos::from_api(&api.location),
+			position: Pos::from_api(&api.location).await,
 		}
 	}
 	fn to_api(&self) -> api::ErrLocation {
@@ -52,11 +53,11 @@ impl OrcErr {
 			locations: self.positions.iter().map(ErrPos::to_api).collect(),
 		}
 	}
-	fn from_api(api: &api::OrcError) -> Self {
+	async fn from_api(api: &api::OrcError) -> Self {
 		Self {
-			description: Tok::from_api(api.description),
+			description: Tok::from_api(api.description).await,
 			message: api.message.clone(),
-			positions: api.locations.iter().map(ErrPos::from_api).collect(),
+			positions: join_all(api.locations.iter().map(ErrPos::from_api)).await,
 		}
 	}
 }
@@ -112,8 +113,8 @@ impl OrcErrv {
 		self.0.iter().flat_map(|e| e.positions.iter().cloned())
 	}
 	pub fn to_api(&self) -> Vec<api::OrcError> { self.0.iter().map(OrcErr::to_api).collect() }
-	pub fn from_api<'a>(api: impl IntoIterator<Item = &'a api::OrcError>) -> Self {
+	pub async fn from_api<'a>(api: impl IntoIterator<Item = &'a api::OrcError>) -> Self {
-		Self(api.into_iter().map(OrcErr::from_api).collect())
+		Self(join_all(api.into_iter().map(OrcErr::from_api)).await)
 	}
 }
 impl From<OrcErr> for OrcErrv {

orchid-base/src/interner.rs

@@ -1,5 +1,5 @@
 use std::borrow::Borrow;
-use std::future::IntoFuture;
+use std::future::Future;
 use std::hash::BuildHasher as _;
 use std::num::NonZeroU64;
 use std::ops::{Deref, DerefMut};
@@ -9,7 +9,7 @@ use std::{fmt, hash, mem};
 use async_once_cell::Lazy;
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools as _;
-use orchid_api_traits::{Decode, Encode, Request};
+use orchid_api_traits::Request;
 
 use crate::api;
 use crate::reqnot::{DynRequester, Requester};
@@ -61,19 +61,13 @@ impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
 		write!(f, "Token({} -> {:?})", self.to_api().get_id(), self.data.as_ref())
 	}
 }
-impl<T: Interned + Encode> Encode for Tok<T> {
-	fn encode<W: std::io::Write + ?Sized>(&self, write: &mut W) { self.data.encode(write) }
-}
-impl<T: Interned + Decode> Decode for Tok<T> {
-	fn decode<R: std::io::Read + ?Sized>(read: &mut R) -> Self { intern(&T::decode(read)) }
-}
 
 pub trait Interned: Eq + hash::Hash + Clone + fmt::Debug + Internable<Interned = Self> {
 	type Marker: InternMarker<Interned = Self> + Sized;
-	async fn intern(
+	fn intern(
 		self: Arc<Self>,
 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> Self::Marker;
+	) -> impl Future<Output = Self::Marker>;
 	fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
 }
 
@@ -84,10 +78,10 @@ pub trait Internable: fmt::Debug {
 
 pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash + Sized {
 	type Interned: Interned<Marker = Self>;
-	async fn resolve(
+	fn resolve(
 		self,
 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
-	) -> Tok<Self::Interned>;
+	) -> impl Future<Output = Tok<Self::Interned>>;
 	fn get_id(self) -> NonZeroU64;
 	fn from_id(id: NonZeroU64) -> Self;
 }
@@ -138,9 +132,9 @@ impl InternMarker for api::TStrv {
 		self,
 		req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
 	) -> Tok<Self::Interned> {
-		let data =
+		let rep = req.request(api::ExternStrv(self)).await;
-			Arc::new(req.request(api::ExternStrv(self)).await.iter().map(|m| deintern(*m)).collect_vec());
+		let data = futures::future::join_all(rep.iter().map(|m| deintern(*m))).await;
-		Tok::new(data, self)
+		Tok::new(Arc::new(data), self)
 	}
 	fn get_id(self) -> NonZeroU64 { self.0 }
 	fn from_id(id: NonZeroU64) -> Self { Self(id) }
@@ -314,21 +308,19 @@ pub fn sweep_replica() -> api::Retained {
 macro_rules! intern {
 	($ty:ty : $expr:expr) => {{
 		use std::future::Future;
+		use std::ops::Deref as _;
 		use std::pin::Pin;
 		type Interned = <$ty as $crate::interner::Internable>::Interned;
 		type Output = $crate::interner::Tok<Interned>;
 		type InternFuture = Pin<Box<dyn Future<Output = Output>>>;
 		thread_local! {
 			static VALUE:
-				Pin<std::rc::Rc<async_once_cell::Lazy<Output, InternFuture>>> =
+				Pin<std::rc::Rc<$crate::async_once_cell::Lazy<Output, InternFuture>>> =
-				std::rc::Rc::pin(async_once_cell::Lazy::new(Box::pin(async {
+				std::rc::Rc::pin($crate::async_once_cell::Lazy::new(Box::pin(async {
 					$crate::interner::intern::<Interned>($expr as &$ty).await
 				}) as InternFuture));
 		}
-		VALUE.with(|val| {
+		VALUE.with(|v| $crate::clone!(v; async move { v.as_ref().await.deref().clone() }))
-			let val: Pin<std::rc::Rc<async_once_cell::Lazy<Output, InternFuture>>> = val.clone();
-			async move { val.as_ref().await.deref().clone() }
-		})
 	}};
 }
 

orchid-base/src/lib.rs

@@ -1,3 +1,4 @@
+pub use async_once_cell;
 use orchid_api as api;
 
 pub mod box_cow;

orchid-base/src/location.rs

@@ -36,12 +36,12 @@ impl Pos {
 			other => format!("{other:?}"),
 		}
 	}
-	pub fn from_api(api: &api::Location) -> Self {
+	pub async fn from_api(api: &api::Location) -> Self {
 		match_mapping!(api, api::Location => Pos {
 			None, Inherit, SlotTarget,
 			Range(r.clone()),
-			Gen(cgi => CodeGenInfo::from_api(cgi)),
+			Gen(cgi => CodeGenInfo::from_api(cgi).await),
-			SourceRange(sr => SourceRange::from_api(sr))
+			SourceRange(sr => SourceRange::from_api(sr).await)
 		})
 	}
 	pub fn to_api(&self) -> api::Location {
@@ -67,7 +67,7 @@ impl SourceRange {
 	}
 	/// Create a dud [SourceRange] for testing. Its value is unspecified and
 	/// volatile.
-	pub fn mock() -> Self { Self { range: 0..1, path: sym!(test) } }
+	pub async fn mock() -> Self { Self { range: 0..1, path: sym!(test).await } }
 	/// Path the source text was loaded from
 	pub fn path(&self) -> Sym { self.path.clone() }
 	/// Byte range
@@ -92,8 +92,8 @@ impl SourceRange {
 		}
 	}
 	pub fn zw(path: Sym, pos: u32) -> Self { Self { path, range: pos..pos } }
-	fn from_api(api: &api::SourceRange) -> Self {
+	async fn from_api(api: &api::SourceRange) -> Self {
-		Self { path: Sym::from_api(api.path), range: api.range.clone() }
+		Self { path: Sym::from_api(api.path).await, range: api.range.clone() }
 	}
 	fn to_api(&self) -> api::SourceRange {
 		api::SourceRange { path: self.path.to_api(), range: self.range.clone() }
@@ -110,17 +110,22 @@ pub struct CodeGenInfo {
 }
 impl CodeGenInfo {
 	/// A codegen marker with no user message and parameters
-	pub fn no_details(generator: Sym) -> Self { Self { generator, details: intern!(str: "") } }
+	pub async fn new_short(generator: Sym) -> Self {
+		Self { generator, details: intern!(str: "").await }
+	}
 	/// A codegen marker with a user message or parameters
-	pub fn details(generator: Sym, details: impl AsRef<str>) -> Self {
+	pub async fn new_details(generator: Sym, details: impl AsRef<str>) -> Self {
-		Self { generator, details: intern(details.as_ref()) }
+		Self { generator, details: intern(details.as_ref()).await }
 	}
 	/// Syntactic location
 	pub fn pos(&self) -> Pos { Pos::Gen(self.clone()) }
-	fn from_api(api: &api::CodeGenInfo) -> Self {
+	pub async fn from_api(api: &api::CodeGenInfo) -> Self {
-		Self { generator: Sym::from_api(api.generator), details: Tok::from_api(api.details) }
+		Self {
+			generator: Sym::from_api(api.generator).await,
+			details: Tok::from_api(api.details).await,
+		}
 	}
-	fn to_api(&self) -> api::CodeGenInfo {
+	pub fn to_api(&self) -> api::CodeGenInfo {
 		api::CodeGenInfo { generator: self.generator.to_api(), details: self.details.to_api() }
 	}
 }

orchid-base/src/macros.rs

@@ -1,6 +1,9 @@
 use std::marker::PhantomData;
 use std::sync::Arc;
 
+use async_std::stream;
+use async_std::sync::Mutex;
+use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
 use never::Never;
 use trait_set::trait_set;
@@ -27,8 +30,14 @@ pub struct MTree<'a, A> {
 	pub tok: Arc<MTok<'a, A>>,
 }
 impl<'a, A> MTree<'a, A> {
-	pub(crate) fn from_api(api: &api::MacroTree, do_atom: &mut impl MacroAtomFromApi<'a, A>) -> Self {
+	pub(crate) async fn from_api(
-		Self { pos: Pos::from_api(&api.location), tok: Arc::new(MTok::from_api(&api.token, do_atom)) }
+		api: &api::MacroTree,
+		do_atom: &mut impl MacroAtomFromApi<'a, A>,
+	) -> Self {
+		Self {
+			pos: Pos::from_api(&api.location).await,
+			tok: Arc::new(MTok::from_api(&api.token, do_atom).await),
+		}
 	}
 	pub(crate) fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroTree {
 		api::MacroTree { location: self.pos.to_api(), token: self.tok.to_api(do_atom) }
@@ -50,16 +59,16 @@ pub enum MTok<'a, A> {
 	Done(Arc<MTok<'a, A>>),
 }
 impl<'a, A> MTok<'a, A> {
-	pub(crate) fn from_api(
+	pub(crate) async fn from_api(
 		api: &api::MacroToken,
 		do_atom: &mut impl MacroAtomFromApi<'a, A>,
 	) -> Self {
 		match_mapping!(&api, api::MacroToken => MTok::<'a, A> {
-			Lambda(x => mtreev_from_api(x, do_atom), b => mtreev_from_api(b, do_atom)),
+			Lambda(x => mtreev_from_api(x, do_atom).await, b => mtreev_from_api(b, do_atom).await),
-			Name(t => Sym::from_api(*t)),
+			Name(t => Sym::from_api(*t).await),
 			Slot(tk => MacroSlot(*tk, PhantomData)),
-			S(p.clone(), b => mtreev_from_api(b, do_atom)),
+			S(p.clone(), b => mtreev_from_api(b, do_atom).await),
-			Ph(ph => Ph::from_api(ph)),
+			Ph(ph => Ph::from_api(ph).await),
 		} {
 			api::MacroToken::Atom(a) => do_atom(a)
 		})
@@ -81,11 +90,15 @@ impl<'a, A> MTok<'a, A> {
 	pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Arc::new(self) } }
 }
 
-pub fn mtreev_from_api<'a, 'b, A>(
+pub async fn mtreev_from_api<'a, 'b, A>(
 	api: impl IntoIterator<Item = &'b api::MacroTree>,
 	do_atom: &mut impl MacroAtomFromApi<'a, A>,
 ) -> Vec<MTree<'a, A>> {
-	api.into_iter().map(|api| MTree::from_api(api, do_atom)).collect_vec()
+	let do_atom_lk = Mutex::new(do_atom);
+	stream::from_iter(api)
+		.then(|api| async { MTree::from_api(api, &mut *do_atom_lk.lock().await).boxed_local().await })
+		.collect()
+		.await
 }
 
 pub fn mtreev_to_api<'a: 'b, 'b, A: 'b>(

orchid-base/src/name.rs

@@ -8,6 +8,7 @@ use std::ops::{Bound, Deref, Index, RangeBounds};
 use std::path::Path;
 use std::{fmt, slice, vec};
 
+use futures::future::{OptionFuture, join_all};
 use itertools::Itertools;
 use trait_set::trait_set;
 
@@ -173,8 +174,8 @@ impl VPath {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Partition the string by `::` namespace separators
-	pub fn parse(s: &str) -> Self {
+	pub async fn parse(s: &str) -> Self {
-		Self(if s.is_empty() { vec![] } else { s.split("::").map(intern).collect() })
+		Self(if s.is_empty() { vec![] } else { join_all(s.split("::").map(intern)).await })
 	}
 	/// Walk over the segments
 	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
@@ -194,12 +195,14 @@ impl VPath {
 	}
 
 	/// Convert a fs path to a vpath
-	pub fn from_path(path: &Path) -> Option<(Self, bool)> {
+	pub async fn from_path(path: &Path, ext: &str) -> Option<(Self, bool)> {
-		let to_vpath =
+		async fn to_vpath(p: &Path) -> Option<VPath> {
-			|p: &Path| p.iter().map(|c| c.to_str().map(intern)).collect::<Option<_>>().map(VPath);
+			let tok_opt_v = join_all(p.iter().map(|c| OptionFuture::from(c.to_str().map(intern)))).await;
+			tok_opt_v.into_iter().collect::<Option<_>>().map(VPath)
+		}
 		match path.extension().map(|s| s.to_str()) {
-			Some(Some("orc")) => Some((to_vpath(&path.with_extension(""))?, true)),
+			Some(Some(s)) if s == ext => Some((to_vpath(&path.with_extension("")).await?, true)),
-			None => Some((to_vpath(path)?, false)),
+			None => Some((to_vpath(path).await?, false)),
 			Some(_) => None,
 		}
 	}
@@ -256,8 +259,8 @@ impl VName {
 		let data: Vec<_> = items.into_iter().collect();
 		if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
 	}
-	pub fn deintern(items: impl IntoIterator<Item = api::TStr>) -> Result<Self, EmptyNameError> {
+	pub async fn deintern(name: impl IntoIterator<Item = api::TStr>) -> Result<Self, EmptyNameError> {
-		Self::new(items.into_iter().map(Tok::from_api))
+		Self::new(join_all(name.into_iter().map(Tok::from_api)).await)
 	}
 	/// Unwrap the enclosed vector
 	pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
@@ -267,7 +270,7 @@ impl VName {
 	/// must never be empty.
 	pub fn vec_mut(&mut self) -> &mut Vec<Tok<String>> { &mut self.0 }
 	/// Intern the name and return a [Sym]
-	pub fn to_sym(&self) -> Sym { Sym(intern(&self.0[..])) }
+	pub async fn to_sym(&self) -> Sym { Sym(intern(&self.0[..]).await) }
 	/// If this name has only one segment, return it
 	pub fn as_root(&self) -> Option<Tok<String>> { self.0.iter().exactly_one().ok().cloned() }
 	/// Prepend the segments to this name
@@ -281,8 +284,8 @@ impl VName {
 		Self(self.0.into_iter().chain(items).collect())
 	}
 	/// Read a `::` separated namespaced name
-	pub fn parse(s: &str) -> Result<Self, EmptyNameError> { Self::new(VPath::parse(s)) }
+	pub async fn parse(s: &str) -> Result<Self, EmptyNameError> { Self::new(VPath::parse(s).await) }
-	pub fn literal(s: &'static str) -> Self { Self::parse(s).expect("empty literal !?") }
+	pub async fn literal(s: &'static str) -> Self { Self::parse(s).await.expect("empty literal !?") }
 	/// Obtain an iterator over the segments of the name
 	pub fn iter(&self) -> impl Iterator<Item = Tok<String>> + '_ { self.0.iter().cloned() }
 }
@@ -338,13 +341,13 @@ pub struct Sym(Tok<Vec<Tok<String>>>);
 impl Sym {
 	/// Assert that the sequence isn't empty, intern it and wrap it in a [Sym] to
 	/// represent this invariant
-	pub fn new(v: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
+	pub async fn new(v: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
 		let items = v.into_iter().collect_vec();
-		Self::from_tok(intern(&items[..]))
+		Self::from_tok(intern(&items[..]).await)
 	}
 	/// Read a `::` separated namespaced name.
-	pub fn parse(s: &str) -> Result<Self, EmptyNameError> {
+	pub async fn parse(s: &str) -> Result<Self, EmptyNameError> {
-		Ok(Sym(intern(&VName::parse(s)?.into_vec()[..])))
+		Ok(Sym(intern(&VName::parse(s).await?.into_vec()[..]).await))
 	}
 	/// Assert that a token isn't empty, and wrap it in a [Sym]
 	pub fn from_tok(t: Tok<Vec<Tok<String>>>) -> Result<Self, EmptyNameError> {
@@ -356,8 +359,8 @@ impl Sym {
 	pub fn id(&self) -> NonZeroU64 { self.0.to_api().get_id() }
 	/// Extern the sym for editing
 	pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
-	pub fn from_api(marker: api::TStrv) -> Sym {
+	pub async fn from_api(marker: api::TStrv) -> Sym {
-		Self::from_tok(Tok::from_api(marker)).expect("Empty sequence found for serialized Sym")
+		Self::from_tok(Tok::from_api(marker).await).expect("Empty sequence found for serialized Sym")
 	}
 	pub fn to_api(&self) -> api::TStrv { self.tok().to_api() }
 }
@@ -437,10 +440,13 @@ impl NameLike for VName {}
 #[macro_export]
 macro_rules! sym {
   ($seg1:tt $( :: $seg:tt)*) => { async {
-		$crate::name::Sym::from_tok($crate::intern!([$crate::interner::Tok<String>]: &[
+		$crate::name::Sym::from_tok(
-			$crate::intern!(str: stringify!($seg1)).await
+			$crate::intern!([$crate::interner::Tok<String>]: &[
-      $( , $crate::intern!(str: stringify!($seg)).await )*
+				$crate::intern!(str: stringify!($seg1)).await
-			])).unwrap()
+				$( , $crate::intern!(str: stringify!($seg)).await )*
+			])
+			.await
+		).unwrap()
 		}
   };
   (@NAME $seg:tt) => {}
@@ -451,12 +457,12 @@ macro_rules! sym {
 /// The components are interned much like in [sym].
 #[macro_export]
 macro_rules! vname {
-  ($seg1:tt $( :: $seg:tt)*) => {
+  ($seg1:tt $( :: $seg:tt)*) => { async {
     $crate::name::VName::new([
-      $crate::intern!(str: stringify!($seg1))
+      $crate::intern!(str: stringify!($seg1)).await
-      $( , $crate::intern!(str: stringify!($seg)) )*
+      $( , $crate::intern!(str: stringify!($seg)).await )*
     ]).unwrap()
-  };
+	} };
 }
 
 /// Create a [VPath] literal.
@@ -464,12 +470,12 @@ macro_rules! vname {
 /// The components are interned much like in [sym].
 #[macro_export]
 macro_rules! vpath {
-  ($seg1:tt $( :: $seg:tt)+) => {
+  ($seg1:tt $( :: $seg:tt)+) => { async {
     $crate::name::VPath(vec![
-      $crate::intern!(str: stringify!($seg1))
+      $crate::intern!(str: stringify!($seg1)).await
-      $( , $crate::intern!(str: stringify!($seg)) )+
+      $( , $crate::intern!(str: stringify!($seg)).await )+
-    ])
+		])
-  };
+	} };
   () => {
     $crate::name::VPath(vec![])
   }
@@ -479,15 +485,39 @@ macro_rules! vpath {
 ///
 /// The components are interned much like in [sym]
 #[macro_export]
-macro_rules! path_slice {
+macro_rules! with_path_slice {
-  ($seg1:tt $( :: $seg:tt)+) => {
+	(@UNIT $tt:tt) => { () };
-    $crate::name::PathSlice::new(&[
+  ($seg1:tt $( :: $seg:tt)* in $expr:expr) => { {
-      $crate::intern!(str: stringify!($seg1))
+		use std::future::Future;
-      $( , $crate::intern!(str: stringify!($seg)) )+
+		use std::ops::Deref as _;
-    ])
+		use std::pin::Pin;
-  };
+
-  () => {
+		const fn count_helper<const N: usize>(_: [(); N]) -> usize { N }
-    $crate::name::PathSlice::new(&[])
+
+		type Output = [Tok<String>; const {
+			count_helper([() $(, $crate::with_path_slice!(@UNIT $seg))*])
+		}];
+		type InternFuture = Pin<Box<dyn Future<Output = Output>>>;
+		thread_local! {
+			static VALUE: Pin<std::rc::Rc<async_once_cell::Lazy<Output, InternFuture>>> =
+				std::rc::Rc::pin(async_once_cell::Lazy::new(Box::pin(async {
+					[
+						$crate::intern!(str: stringify!($seg1)).await
+						$( , $crate::intern!(str: stringify!($seg)).await )+
+					]
+				})));
+		}
+		VALUE.with(|v| $crate::clone!(v; async move {
+			let expr = $expr;
+			let result = v.as_ref().await;
+			let ps: &PathSlice = $crate::name::PathSlice::new(&result.deref()[..]);
+			(expr)(ps).await
+		}))
+
+	} };
+  ($expr:expr) => {
+		let expr = $expr;
+    (expr)($crate::name::PathSlice::new(&[]))
   }
 }
 
@@ -495,33 +525,45 @@ macro_rules! path_slice {
 mod test {
 	use std::borrow::Borrow;
 
+	use test_executors::spin_on;
+
 	use super::{PathSlice, Sym, VName};
 	use crate::interner::{Tok, intern};
 	use crate::name::VPath;
 
 	#[test]
 	fn recur() {
-		let myname = vname!(foo::bar);
+		spin_on(async {
-		let _borrowed_slice: &[Tok<String>] = myname.borrow();
+			let myname = vname!(foo::bar).await;
-		let _borrowed_pathslice: &PathSlice = myname.borrow();
+			let _borrowed_slice: &[Tok<String>] = myname.borrow();
-		let _deref_pathslice: &PathSlice = &myname;
+			let _borrowed_pathslice: &PathSlice = myname.borrow();
-		let _as_slice_out: &[Tok<String>] = myname.as_slice();
+			let _deref_pathslice: &PathSlice = &myname;
+			let _as_slice_out: &[Tok<String>] = myname.as_slice();
+		})
 	}
 
 	#[test]
 	fn literals() {
-		assert_eq!(
+		spin_on(async {
-			sym!(foo::bar::baz),
+			assert_eq!(
-			Sym::new([intern("foo"), intern("bar"), intern("baz")]).unwrap()
+				sym!(foo::bar::baz).await,
-		);
+				Sym::new([intern("foo").await, intern("bar").await, intern("baz").await]).await.unwrap()
-		assert_eq!(
+			);
-			vname!(foo::bar::baz),
+			assert_eq!(
-			VName::new([intern("foo"), intern("bar"), intern("baz")]).unwrap()
+				vname!(foo::bar::baz).await,
-		);
+				VName::new([intern("foo").await, intern("bar").await, intern("baz").await]).unwrap()
-		assert_eq!(vpath!(foo::bar::baz), VPath::new([intern("foo"), intern("bar"), intern("baz")]));
+			);
-		assert_eq!(
+			assert_eq!(
-			path_slice!(foo::bar::baz),
+				vpath!(foo::bar::baz).await,
-			PathSlice::new(&[intern("foo"), intern("bar"), intern("baz")])
+				VPath::new([intern("foo").await, intern("bar").await, intern("baz").await])
-		);
+			);
+			with_path_slice!(foo::bar::baz in |val| async move {
+				assert_eq!(
+					val,
+					PathSlice::new(&[intern("foo").await, intern("bar").await, intern("baz").await])
+				);
+			})
+			.await
+		})
 	}
 }

orchid-base/src/number.rs

@@ -63,9 +63,9 @@ pub struct NumError {
 	pub kind: NumErrorKind,
 }
 
-pub fn num_to_err(NumError { kind, range }: NumError, offset: u32) -> OrcErr {
+pub async fn num_to_err(NumError { kind, range }: NumError, offset: u32) -> OrcErr {
 	mk_err(
-		intern!(str: "Failed to parse number"),
+		intern!(str: "Failed to parse number").await,
 		match kind {
 			NumErrorKind::NaN => "NaN emerged during parsing",
 			NumErrorKind::InvalidDigit => "non-digit character encountered",

orchid-base/src/parse.rs

@@ -1,6 +1,7 @@
 use std::iter;
 use std::ops::{Deref, Range};
 
+use futures::future::join_all;
 use itertools::Itertools;
 
 use crate::error::{OrcRes, Reporter, mk_err, mk_errv};
@@ -100,12 +101,12 @@ impl Comment {
 	pub fn to_api(&self) -> api::Comment {
 		api::Comment { location: self.pos.to_api(), text: self.text.to_api() }
 	}
-	pub fn from_api(api: &api::Comment) -> Self {
+	pub async fn from_api(api: &api::Comment) -> Self {
-		Self { pos: Pos::from_api(&api.location), text: Tok::from_api(api.text) }
+		Self { pos: Pos::from_api(&api.location).await, text: Tok::from_api(api.text).await }
 	}
 }
 
-pub fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
 	snip: Snippet<'a, 'b, A, X>,
 ) -> Vec<Parsed<'a, 'b, Vec<Comment>, A, X>> {
 	let mut items = Vec::new();
@@ -120,10 +121,14 @@ pub fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
 			None => comments.extend(line.cur),
 			Some(i) => {
 				let (cmts, tail) = line.split_at(i);
-				let comments = Vec::from_iter(comments.drain(..).chain(cmts.cur).map(|t| match &t.tok {
+				let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async {
-					Token::Comment(c) => Comment { text: intern(&**c), pos: Pos::Range(t.range.clone()) },
+					match &t.tok {
-					_ => unreachable!("All are comments checked above"),
+						Token::Comment(c) =>
-				}));
+							Comment { text: intern(&**c).await, pos: Pos::Range(t.range.clone()) },
+						_ => unreachable!("All are comments checked above"),
+					}
+				}))
+				.await;
 				items.push(Parsed { output: comments, tail });
 			},
 		}
@@ -131,22 +136,23 @@ pub fn line_items<'a, 'b, A: AtomRepr, X: ExtraTok>(
 	items
 }
 
-pub fn try_pop_no_fluff<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn try_pop_no_fluff<'a, 'b, A: AtomRepr, X: ExtraTok>(
 	snip: Snippet<'a, 'b, A, X>,
 ) -> ParseRes<'a, 'b, &'a TokTree<'b, A, X>, A, X> {
-	snip.skip_fluff().pop_front().map(|(output, tail)| Parsed { output, tail }).ok_or_else(|| {
+	match snip.skip_fluff().pop_front() {
-		mk_errv(
+		Some((output, tail)) => Ok(Parsed { output, tail }),
-			intern!(str: "Unexpected end"),
+		None =>
-			"Pattern ends abruptly",
+			Err(mk_errv(intern!(str: "Unexpected end").await, "Pattern ends abruptly", [Pos::Range(
-			[Pos::Range(snip.pos()).into()],
+				snip.pos(),
-		)
+			)
-	})
+			.into()])),
+	}
 }
 
-pub fn expect_end(snip: Snippet<'_, '_, impl AtomRepr, impl ExtraTok>) -> OrcRes<()> {
+pub async fn expect_end(snip: Snippet<'_, '_, impl AtomRepr, impl ExtraTok>) -> OrcRes<()> {
 	match snip.skip_fluff().get(0) {
 		Some(surplus) => Err(mk_errv(
-			intern!(str: "Extra code after end of line"),
+			intern!(str: "Extra code after end of line").await,
 			"Code found after the end of the line",
 			[Pos::Range(surplus.range.clone()).into()],
 		)),
@@ -154,15 +160,15 @@ pub fn expect_end(snip: Snippet<'_, '_, impl AtomRepr, impl ExtraTok>) -> OrcRes
 	}
 }
 
-pub fn expect_tok<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn expect_tok<'a, 'b, A: AtomRepr, X: ExtraTok>(
 	snip: Snippet<'a, 'b, A, X>,
 	tok: Tok<String>,
 ) -> ParseRes<'a, 'b, (), A, X> {
-	let Parsed { output: head, tail } = try_pop_no_fluff(snip)?;
+	let Parsed { output: head, tail } = try_pop_no_fluff(snip).await?;
 	match &head.tok {
 		Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
 		t => Err(mk_errv(
-			intern!(str: "Expected specific keyword"),
+			intern!(str: "Expected specific keyword").await,
 			format!("Expected {tok} but found {t}"),
 			[Pos::Range(head.range.clone()).into()],
 		)),
@@ -176,31 +182,34 @@ pub struct Parsed<'a, 'b, T, A: AtomRepr, X: ExtraTok> {
 
 pub type ParseRes<'a, 'b, T, A, X> = OrcRes<Parsed<'a, 'b, T, A, X>>;
 
-pub fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
+pub async fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
 	ctx: &impl Reporter,
 	tail: Snippet<'a, 'b, A, X>,
 ) -> ParseRes<'a, 'b, Vec<(Import, Pos)>, A, X> {
-	let ret = rec(ctx, tail);
+	let ret = rec(ctx, tail).await;
 	#[allow(clippy::type_complexity)] // it's an internal function
-	pub fn rec<'a, 'b, A: AtomRepr, X: ExtraTok>(
+	pub async fn rec<'a, 'b, A: AtomRepr, X: ExtraTok>(
 		ctx: &impl Reporter,
 		tail: Snippet<'a, 'b, A, X>,
 	) -> ParseRes<'a, 'b, Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>, A, X> {
-		let comma = intern!(str: ",");
+		let comma = intern!(str: ",").await;
-		let globstar = intern!(str: "*");
+		let globstar = intern!(str: "*").await;
-		let (name, tail) = tail.skip_fluff().pop_front().ok_or_else(|| {
+		let Some((name, tail)) = tail.skip_fluff().pop_front() else {
-			mk_err(intern!(str: "Expected name"), "Expected a name, a list of names, or a globstar.", [
+			return Err(mk_errv(
-				Pos::Range(tail.pos()).into(),
+				intern!(str: "Expected name").await,
-			])
+				"Expected a name, a list of names, or a globstar.",
-		})?;
+				[Pos::Range(tail.pos()).into()],
+			));
+		};
 		if let Some((Token::NS, tail)) = tail.skip_fluff().pop_front().map(|(tt, s)| (&tt.tok, s)) {
 			let n = match &name.tok {
 				Token::Name(n) if n.starts_with(name_start) => Ok(n),
-				_ => Err(mk_err(intern!(str: "Unexpected name prefix"), "Only names can precede ::", [
+				_ =>
-					Pos::Range(name.range.clone()).into(),
+					Err(mk_err(intern!(str: "Unexpected name prefix").await, "Only names can precede ::", [
-				])),
+						Pos::Range(name.range.clone()).into(),
+					])),
 			};
-			match (rec(ctx, tail), n) {
+			match (Box::pin(rec(ctx, tail)).await, n) {
 				(Err(ev), n) => Err(ev.extended(n.err())),
 				(Ok(Parsed { tail, .. }), Err(e)) => {
 					ctx.report(e);
@@ -218,7 +227,7 @@ pub fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
 						n if *n == globstar => None,
 						n if n.starts_with(op_char) => {
 							return Err(mk_errv(
-								intern!(str: "Unescaped operator in multiname"),
+								intern!(str: "Unescaped operator in multiname").await,
 								"Operators in multinames should be enclosed in []",
 								[Pos::Range(name.range.clone()).into()],
 							));
@@ -229,28 +238,30 @@ pub fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
 				},
 				Token::S(Paren::Square, b) => {
 					let mut ok = Vec::new();
-					b.iter().for_each(|tt| match &tt.tok {
+					for tt in b.iter() {
-						Token::Name(n) if n.starts_with(op_char) =>
+						match &tt.tok {
-							ok.push((vec![], Some(n.clone()), Pos::Range(tt.range.clone()))),
+							Token::Name(n) if n.starts_with(op_char) =>
-						Token::BR | Token::Comment(_) => (),
+								ok.push((vec![], Some(n.clone()), Pos::Range(tt.range.clone()))),
-						_ => ctx.report(mk_err(
+							Token::BR | Token::Comment(_) => (),
-							intern!(str: "Non-operator in escapement in multiname"),
+							_ => ctx.report(mk_err(
-							"In multinames, [] functions as a literal name list reserved for operators",
+								intern!(str: "Non-operator in escapement in multiname").await,
-							[Pos::Range(name.range.clone()).into()],
+								"In multinames, [] functions as a literal name list reserved for operators",
-						)),
+								[Pos::Range(name.range.clone()).into()],
-					});
+							)),
+						}
+					}
 					ok
 				},
 				Token::S(Paren::Round, b) => {
 					let mut ok = Vec::new();
 					let body = Snippet::new(name, b);
 					for csent in body.split(|n| matches!(n, Token::Name(n) if *n == comma)) {
-						match rec(ctx, csent) {
+						match Box::pin(rec(ctx, csent)).await {
 							Err(e) => ctx.report(e),
 							Ok(Parsed { output, tail }) => match tail.get(0) {
 								None => ok.extend(output),
 								Some(t) => ctx.report(mk_err(
-									intern!(str: "Unexpected token in multiname group"),
+									intern!(str: "Unexpected token in multiname group").await,
 									"Unexpected token. Likely missing a :: or , or wanted [] instead of ()",
 									[Pos::Range(t.range.clone()).into()],
 								)),
@@ -261,7 +272,7 @@ pub fn parse_multiname<'a, 'b, A: AtomRepr, X: ExtraTok>(
 				},
 				t => {
 					return Err(mk_errv(
-						intern!(str: "Unrecognized name end"),
+						intern!(str: "Unrecognized name end").await,
 						format!("Names cannot end with {t} tokens"),
 						[Pos::Range(name.range.clone()).into()],
 					));

orchid-base/src/reqnot.rs

@@ -2,32 +2,33 @@ use std::any::Any;
 use std::cell::RefCell;
 use std::future::Future;
 use std::marker::PhantomData;
+use std::mem;
 use std::ops::{BitAnd, Deref};
-use std::pin::Pin;
+use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::mpsc::{SyncSender, sync_channel};
-use std::sync::{Arc, Mutex};
-use std::{mem, thread};
 
-use async_std::channel::{self, Sender};
+use async_std::channel;
-use async_std::future;
+use async_std::sync::Mutex;
 use derive_destructure::destructure;
 use dyn_clone::{DynClone, clone_box};
+use futures::future::LocalBoxFuture;
 use hashbrown::HashMap;
 use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
 use trait_set::trait_set;
 
+use crate::clone;
+
 pub struct Receipt<'a>(PhantomData<&'a mut ()>);
 
 trait_set! {
 	pub trait SendFn<T: MsgSet> =
-		for<'a> FnMut(&'a [u8], ReqNot<T>) -> Pin<Box<dyn Future<Output = ()> + 'a>>
+		for<'a> FnMut(&'a [u8], ReqNot<T>) -> LocalBoxFuture<'a, ()>
 		+ DynClone + Send + 'static;
 	pub trait ReqFn<T: MsgSet> =
-		for<'a> FnMut(RequestHandle<'a, T>, <T::In as Channel>::Req) -> Pin<Box<dyn Future<Output = Receipt<'a>>>>
+		for<'a> FnMut(RequestHandle<'a, T>, <T::In as Channel>::Req) -> LocalBoxFuture<'a, Receipt<'a>>
 		+ DynClone + Send + Sync + 'static;
 	pub trait NotifFn<T: MsgSet> =
-		for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) -> Pin<Box<dyn Future<Output = ()>>>
+		FnMut(<T::In as Channel>::Notif, ReqNot<T>) -> LocalBoxFuture<'static, ()>
 		+ DynClone + Send + Sync + 'static;
 }
 
@@ -69,7 +70,7 @@ impl<'a, MS: MsgSet + 'static> RequestHandle<'a, MS> {
 		assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded to {}", self.id);
 		let mut buf = (!self.id).to_be_bytes().to_vec();
 		response.encode(&mut buf);
-		let mut send = clone_box(&*self.reqnot().0.lock().unwrap().send);
+		let mut send = clone_box(&*self.reqnot().0.lock().await.send);
 		(send)(&buf, self.parent.clone()).await;
 		Receipt(PhantomData)
 	}
@@ -117,7 +118,7 @@ impl<T: MsgSet> ReqNot<T> {
 
 	/// Can be called from a polling thread or dispatched in any other way
 	pub async fn receive(&self, message: &[u8]) {
-		let mut g = self.0.lock().unwrap();
+		let mut g = self.0.lock().await;
 		let (id, payload) = get_id(message);
 		if id == 0 {
 			let mut notif = clone_box(&*g.notif);
@@ -136,7 +137,7 @@ impl<T: MsgSet> ReqNot<T> {
 	}
 
 	pub async fn notify<N: Coding + Into<<T::Out as Channel>::Notif>>(&self, notif: N) {
-		let mut send = clone_box(&*self.0.lock().unwrap().send);
+		let mut send = clone_box(&*self.0.lock().await.send);
 		let mut buf = vec![0; 8];
 		let msg: <T::Out as Channel>::Notif = notif.into();
 		msg.encode(&mut buf);
@@ -147,40 +148,41 @@ impl<T: MsgSet> ReqNot<T> {
 pub trait DynRequester: Send + Sync {
 	type Transfer;
 	/// Encode and send a request, then receive the response buffer.
-	fn raw_request(&self, data: Self::Transfer) -> Pin<Box<dyn Future<Output = RawReply>>>;
+	fn raw_request(&self, data: Self::Transfer) -> LocalBoxFuture<'_, RawReply>;
 }
 
-pub struct MappedRequester<'a, T>(
+pub struct MappedRequester<'a, T: 'a>(
-	Box<dyn Fn(T) -> Pin<Box<dyn Future<Output = RawReply>>> + Send + Sync + 'a>,
+	Box<dyn Fn(T) -> LocalBoxFuture<'a, RawReply> + Send + Sync + 'a>,
 );
 impl<'a, T> MappedRequester<'a, T> {
 	fn new<U: DynRequester + 'a>(req: U) -> Self
 	where T: Into<U::Transfer> {
-		MappedRequester(Box::new(move |t| req.raw_request(t.into())))
+		let req_arc = Arc::new(req);
+		MappedRequester(Box::new(move |t| {
+			Box::pin(clone!(req_arc; async move { req_arc.raw_request(t.into()).await}))
+		}))
 	}
 }
 
 impl<T> DynRequester for MappedRequester<'_, T> {
 	type Transfer = T;
-	fn raw_request(&self, data: Self::Transfer) -> Pin<Box<dyn Future<Output = RawReply>>> {
+	fn raw_request(&self, data: Self::Transfer) -> LocalBoxFuture<'_, RawReply> { self.0(data) }
-		self.0(data)
-	}
 }
 
 impl<T: MsgSet> DynRequester for ReqNot<T> {
 	type Transfer = <T::Out as Channel>::Req;
-	fn raw_request(&self, req: Self::Transfer) -> Pin<Box<dyn Future<Output = RawReply>>> {
+	fn raw_request(&self, req: Self::Transfer) -> LocalBoxFuture<'_, RawReply> {
-		let mut g = self.0.lock().unwrap();
-		let id = g.id;
-		g.id += 1;
-		let mut buf = id.to_be_bytes().to_vec();
-		req.encode(&mut buf);
-		let (send, recv) = channel::bounded(1);
-		g.responses.insert(id, send);
-		let mut send = clone_box(&*g.send);
-		mem::drop(g);
-		let rn = self.clone();
 		Box::pin(async move {
+			let mut g = self.0.lock().await;
+			let id = g.id;
+			g.id += 1;
+			let mut buf = id.to_be_bytes().to_vec();
+			req.encode(&mut buf);
+			let (send, recv) = channel::bounded(1);
+			g.responses.insert(id, send);
+			let mut send = clone_box(&*g.send);
+			mem::drop(g);
+			let rn = self.clone();
 			send(&buf, rn).await;
 			RawReply(recv.recv().await.unwrap())
 		})
@@ -190,7 +192,10 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
 pub trait Requester: DynRequester {
 	#[must_use = "These types are subject to change with protocol versions. \
     If you don't want to use the return value, At a minimum, force the type."]
-	async fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response;
+	fn request<R: Request + Into<Self::Transfer>>(
+		&self,
+		data: R,
+	) -> impl Future<Output = R::Response>;
 	fn map<'a, U: Into<Self::Transfer>>(self) -> MappedRequester<'a, U>
 	where Self: Sized + 'a {
 		MappedRequester::new(self)
@@ -208,9 +213,10 @@ impl<T: MsgSet> Clone for ReqNot<T> {
 
 #[cfg(test)]
 mod test {
-	use std::sync::{Arc, Mutex};
+	use std::sync::Arc;
 
-	use async_std::future;
+	use async_std::sync::Mutex;
+	use futures::FutureExt;
 	use orchid_api_derive::Coding;
 	use orchid_api_traits::{Channel, Request};
 	use test_executors::spin_on;
@@ -239,49 +245,52 @@ mod test {
 
 	#[test]
 	fn notification() {
-		let received = Arc::new(Mutex::new(None));
+		spin_on(async {
-		let receiver = ReqNot::<TestMsgSet>::new(
+			let received = Arc::new(Mutex::new(None));
-			|_, _| panic!("Should not send anything"),
+			let receiver = ReqNot::<TestMsgSet>::new(
-			clone!(received; move |notif, _| {
+				|_, _| panic!("Should not send anything"),
-				*received.lock().unwrap() = Some(notif);
+				clone!(received; move |notif, _| clone!(received; async move {
-				Box::pin(future::ready(()))
+					*received.lock().await = Some(notif);
-			}),
+				}.boxed_local())),
-			|_, _| panic!("Not receiving a request"),
+				|_, _| panic!("Not receiving a request"),
-		);
+			);
-		let sender = ReqNot::<TestMsgSet>::new(
+			let sender = ReqNot::<TestMsgSet>::new(
-			clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move {
+				clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move {
-				receiver.receive(d).await
+					receiver.receive(d).await
-			}))),
+				}))),
-			|_, _| panic!("Should not receive notif"),
+				|_, _| panic!("Should not receive notif"),
-			|_, _| panic!("Should not receive request"),
+				|_, _| panic!("Should not receive request"),
-		);
+			);
-		sender.notify(3);
+			sender.notify(3).await;
-		assert_eq!(*received.lock().unwrap(), Some(3));
+			assert_eq!(*received.lock().await, Some(3));
-		sender.notify(4);
+			sender.notify(4).await;
-		assert_eq!(*received.lock().unwrap(), Some(4));
+			assert_eq!(*received.lock().await, Some(4));
+		})
 	}
 
 	#[test]
 	fn request() {
-		let receiver = Arc::new(Mutex::<Option<ReqNot<TestMsgSet>>>::new(None));
+		spin_on(async {
-		let sender = Arc::new(ReqNot::<TestMsgSet>::new(
+			let receiver = Arc::new(Mutex::<Option<ReqNot<TestMsgSet>>>::new(None));
-			clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move {
+			let sender = Arc::new(ReqNot::<TestMsgSet>::new(
-				receiver.lock().unwrap().as_ref().unwrap().receive(d).await
+				clone!(receiver; move |d, _| clone!(receiver; Box::pin(async move {
-			}))),
+					receiver.lock().await.as_ref().unwrap().receive(d).await
-			|_, _| panic!("Should not receive notif"),
+				}))),
-			|_, _| panic!("Should not receive request"),
+				|_, _| panic!("Should not receive notif"),
-		));
+				|_, _| panic!("Should not receive request"),
-		*receiver.lock().unwrap() = Some(ReqNot::new(
+			));
-			clone!(sender; move |d, _| clone!(sender; Box::pin(async move { sender.receive(d).await }))),
+			*receiver.lock().await = Some(ReqNot::new(
-			|_, _| panic!("Not receiving notifs"),
+				clone!(sender; move |d, _| clone!(sender; Box::pin(async move { sender.receive(d).await }))),
-			|hand, req| {
+				|_, _| panic!("Not receiving notifs"),
-				Box::pin(async move {
+				|hand, req| {
-					assert_eq!(req, TestReq(5));
+					Box::pin(async move {
-					hand.respond(&6u8).await
+						assert_eq!(req, TestReq(5));
-				})
+						hand.respond(&6u8).await
-			},
+					})
-		));
+				},
-		let response = spin_on(sender.request(TestReq(5)));
+			));
-		assert_eq!(response, 6);
+			let response = sender.request(TestReq(5)).await;
+			assert_eq!(response, 6);
+		})
 	}
 }

orchid-base/src/tree.rs

@@ -7,6 +7,9 @@ use std::ops::Range;
 use std::sync::Arc;
 
 pub use api::PhKind;
+use async_std::stream;
+use async_std::sync::Mutex;
+use futures::StreamExt;
 use itertools::Itertools;
 use never::Never;
 use ordered_float::NotNan;
@@ -70,17 +73,17 @@ pub struct TokTree<'a, A: AtomRepr, X: ExtraTok> {
 	pub range: Range<u32>,
 }
 impl<'a, A: AtomRepr, X: ExtraTok> TokTree<'a, A, X> {
-	pub fn from_api(tt: &api::TokenTree, ctx: &mut A::Ctx) -> Self {
+	pub async fn from_api(tt: &api::TokenTree, ctx: &mut A::Ctx) -> Self {
 		let tok = match_mapping!(&tt.token, api::Token => Token::<'a, A, X> {
 			BR, NS,
 			Atom(a => A::from_api(a, Pos::Range(tt.range.clone()), ctx)),
-			Bottom(e => OrcErrv::from_api(e)),
+			Bottom(e => OrcErrv::from_api(e).await),
-			LambdaHead(arg => ttv_from_api(arg, ctx)),
+			LambdaHead(arg => ttv_from_api(arg, ctx).await),
-			Name(n => Tok::from_api(*n)),
+			Name(n => Tok::from_api(*n).await),
-			S(*par, b => ttv_from_api(b, ctx)),
+			S(*par, b => ttv_from_api(b, ctx).await),
 			Comment(c.clone()),
 			Slot(id => TokHandle::new(*id)),
-			Ph(ph => Ph::from_api(ph)),
+			Ph(ph => Ph::from_api(ph).await),
 			Macro(*prio)
 		});
 		Self { range: tt.range.clone(), tok }
@@ -149,11 +152,18 @@ impl<A: AtomRepr, X: ExtraTok> Display for TokTree<'_, A, X> {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
 }
 
-pub fn ttv_from_api<A: AtomRepr, X: ExtraTok>(
+pub async fn ttv_from_api<A: AtomRepr, X: ExtraTok>(
 	tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>,
 	ctx: &mut A::Ctx,
 ) -> Vec<TokTree<'static, A, X>> {
-	tokv.into_iter().map(|t| TokTree::<A, X>::from_api(t.borrow(), ctx)).collect()
+	let ctx_lk = Mutex::new(ctx);
+	stream::from_iter(tokv.into_iter())
+		.then(|t| async {
+			let t = t;
+			TokTree::<A, X>::from_api(t.borrow(), *ctx_lk.lock().await).await
+		})
+		.collect()
+		.await
 }
 
 pub fn ttv_to_api<'a, A: AtomRepr, X: ExtraTok>(
@@ -297,8 +307,8 @@ pub struct Ph {
 	pub kind: PhKind,
 }
 impl Ph {
-	pub fn from_api(api: &api::Placeholder) -> Self {
+	pub async fn from_api(api: &api::Placeholder) -> Self {
-		Self { name: Tok::from_api(api.name), kind: api.kind }
+		Self { name: Tok::from_api(api.name).await, kind: api.kind }
 	}
 	pub fn to_api(&self) -> api::Placeholder {
 		api::Placeholder { name: self.name.to_api(), kind: self.kind }

orchid-extension/src/atom.rs

@@ -89,12 +89,13 @@ impl ForeignAtom<'static> {
 	pub(crate) fn new(handle: Arc<ExprHandle>, atom: api::Atom, pos: Pos) -> Self {
 		ForeignAtom { _life: PhantomData, atom, ctx: handle.ctx.clone(), expr: Some(handle), pos }
 	}
-	pub fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
+	pub async fn request<M: AtomMethod>(&self, m: M) -> Option<M::Response> {
-		let rep = self.ctx.reqnot.request(api::Fwd(
+		let rep = (self.ctx.reqnot.request(api::Fwd(
 			self.atom.clone(),
-			Sym::parse(M::NAME).unwrap().tok().to_api(),
+			Sym::parse(M::NAME).await.unwrap().tok().to_api(),
 			enc_vec(&m),
-		))?;
+		)))
+		.await?;
 		Some(M::Response::decode(&mut &rep[..]))
 	}
 }
@@ -116,9 +117,9 @@ impl AtomRepr for ForeignAtom<'_> {
 
 pub struct NotTypAtom(pub Pos, pub Expr, pub Box<dyn AtomDynfo>);
 impl NotTypAtom {
-	pub fn mk_err(&self) -> OrcErr {
+	pub async fn mk_err(&self) -> OrcErr {
 		mk_err(
-			intern!(str: "Not the expected type"),
+			intern!(str: "Not the expected type").await,
 			format!("This expression is not a {}", self.2.name()),
 			[self.0.clone().into()],
 		)
@@ -147,10 +148,10 @@ pub struct MethodSet<A: AtomCard> {
 impl<A: AtomCard> MethodSet<A> {
 	pub fn new() -> Self { Self { handlers: vec![] } }
 
-	pub fn handle<M: AtomMethod>(mut self) -> Self
+	pub async fn handle<M: AtomMethod>(mut self) -> Self
 	where A: Supports<M> {
 		self.handlers.push(AtomReqHandler {
-			key: Sym::parse(M::NAME).expect("AtomMethod::NAME cannoot be empty"),
+			key: Sym::parse(M::NAME).await.expect("AtomMethod::NAME cannoot be empty"),
 			cb: Box::new(move |a: &A, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write| {
 				Supports::<M>::handle(a, ctx, M::decode(req)).encode(rep);
 			}),
@@ -197,19 +198,16 @@ impl<A: AtomicFeatures> TypAtom<'static, A> {
 	}
 }
 impl<A: AtomicFeatures> TypAtom<'_, A> {
-	pub fn request<M: AtomMethod>(&self, req: M) -> M::Response
+	pub async fn request<M: AtomMethod>(&self, req: M) -> M::Response
 	where A: Supports<M> {
 		M::Response::decode(
-			&mut &self
+			&mut &(self.data.ctx.reqnot.request(api::Fwd(
-				.data
+				self.data.atom.clone(),
-				.ctx
+				Sym::parse(M::NAME).await.unwrap().tok().to_api(),
-				.reqnot
+				enc_vec(&req),
-				.request(api::Fwd(
+			)))
-					self.data.atom.clone(),
+			.await
-					Sym::parse(M::NAME).unwrap().tok().to_api(),
+			.unwrap()[..],
-					enc_vec(&req),
-				))
-				.unwrap()[..],
 		)
 	}
 }
@@ -255,10 +253,11 @@ impl fmt::Display for AtomFactory {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "AtomFactory") }
 }
 
-pub fn err_not_callable() -> OrcErr {
+pub async fn err_not_callable() -> OrcErr {
-	mk_err(intern!(str: "This atom is not callable"), "Attempted to apply value as function", [])
+	mk_err(intern!(str: "This atom is not callable").await, "Attempted to apply value as function", [
+	])
 }
 
-pub fn err_not_command() -> OrcErr {
+pub async fn err_not_command() -> OrcErr {
-	mk_err(intern!(str: "This atom is not a command"), "Settled on an inactionable value", [])
+	mk_err(intern!(str: "This atom is not a command").await, "Settled on an inactionable value", [])
 }

orchid-extension/src/conv.rs

@@ -20,12 +20,14 @@ impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
 	}
 }
 
-fn err_not_atom(pos: Pos) -> OrcErr {
+async fn err_not_atom(pos: Pos) -> OrcErr {
-	mk_err(intern!(str: "Expected an atom"), "This expression is not an atom", [pos.into()])
+	mk_err(intern!(str: "Expected an atom").await, "This expression is not an atom", [pos.into()])
 }
 
-fn err_type(pos: Pos) -> OrcErr {
+async fn err_type(pos: Pos) -> OrcErr {
-	mk_err(intern!(str: "Type error"), "The atom is a different type than expected", [pos.into()])
+	mk_err(intern!(str: "Type error").await, "The atom is a different type than expected", [
+		pos.into()
+	])
 }
 
 impl<A: AtomicFeatures> TryFromExpr for TypAtom<'_, A> {

orchid-extension/src/entrypoint.rs

@@ -29,7 +29,7 @@ use crate::system::{SysCtx, atom_by_idx};
 use crate::system_ctor::{CtedObj, DynSystemCtor};
 use crate::tree::{GenTok, GenTokTree, LazyMemberFactory, TIACtxImpl, do_extra};
 
-pub type ExtReq = RequestHandle<api::ExtMsgSet>;
+pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>;
 pub type ExtReqNot = ReqNot<api::ExtMsgSet>;
 
 pub struct ExtensionData {

orchid-extension/src/lexer.rs

@@ -10,18 +10,20 @@ use orchid_base::tree::TokHandle;
 use crate::api;
 use crate::tree::{GenTok, GenTokTree};
 
-pub fn err_cascade() -> OrcErr {
+pub async fn err_cascade() -> OrcErr {
 	mk_err(
-		intern!(str: "An error cascading from a recursive call"),
+		intern!(str: "An error cascading from a recursive call").await,
-		"This error should not surface. If you are seeing it, something is wrong",
+		"This error is a sentinel for the extension library.\
+		it should not be emitted by the extension.",
 		[Pos::None.into()],
 	)
 }
 
-pub fn err_not_applicable() -> OrcErr {
+pub async fn err_not_applicable() -> OrcErr {
 	mk_err(
-		intern!(str: "Pseudo-error to communicate that the current branch in a dispatch doesn't apply"),
+		intern!(str: "Pseudo-error to communicate that the current branch in a dispatch doesn't apply")
-		&*err_cascade().message,
+			.await,
+		&*err_cascade().await.message,
 		[Pos::None.into()],
 	)
 }