All sorts of test scaffolding works now

.cargo/config.toml

@@ -3,6 +3,7 @@ xtask = "run --quiet --package xtask --"
 orcx = "xtask orcx"
 
 [env]
+CARGO_WORKSPACE_DIR = { value = "", relative = true }
 ORCHID_EXTENSIONS = "target/debug/orchid-std"
 ORCHID_DEFAULT_SYSTEMS = "orchid::std"
 ORCHID_LOG_BUFFERS = "true"

Cargo.lock

@@ -388,6 +388,12 @@ dependencies = [
  "syn 2.0.95",
 ]
 
+[[package]]
+name = "bound"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "315d12cca12c2cd6c3c1de89aacabf340c628ace0f93bb56026d7a18acccb13b"
+
 [[package]]
 name = "bumpalo"
 version = "3.16.0"
@@ -532,6 +538,16 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "ctrlc"
+version = "3.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3"
+dependencies = [
+ "nix",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "darling"
 version = "0.20.10"
@@ -984,6 +1000,12 @@ version = "2.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
+[[package]]
+name = "memo-map"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.8.3"
@@ -1010,6 +1032,18 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91"
 
+[[package]]
+name = "nix"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -1111,6 +1145,7 @@ dependencies = [
  "itertools",
  "konst",
  "lazy_static",
+ "memo-map",
  "never",
  "once_cell",
  "orchid-api",
@@ -1128,9 +1163,11 @@ dependencies = [
 name = "orchid-host"
 version = "0.1.0"
 dependencies = [
+ "async-once-cell",
  "async-process",
  "async-std",
  "async-stream",
+ "bound",
  "derive_destructure",
  "futures",
  "hashbrown 0.15.2",
@@ -1171,9 +1208,11 @@ dependencies = [
 name = "orcx"
 version = "0.1.0"
 dependencies = [
+ "async-std",
  "async-stream",
  "camino",
  "clap",
+ "ctrlc",
  "futures",
  "itertools",
  "orchid-base",

orchid-api/src/logging.rs

@@ -2,7 +2,7 @@ use orchid_api_derive::{Coding, Hierarchy};
 
 use crate::ExtHostNotif;
 
-#[derive(Clone, Debug, Coding)]
+#[derive(Clone, Debug, Coding, PartialEq, Eq, Hash)]
 pub enum LogStrategy {
 	StdErr,
 	File(String),

orchid-api/src/proto.rs

@@ -33,17 +33,22 @@ use crate::{atom, expr, interner, lexer, logging, macros, parser, system, tree,
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader {
 	pub log_strategy: logging::LogStrategy,
+	pub msg_logs: logging::LogStrategy,
 }
 impl Decode for HostHeader {
 	async fn decode<R: Read + ?Sized>(mut read: Pin<&mut R>) -> Self {
 		read_exact(read.as_mut(), HOST_INTRO).await;
-		Self { log_strategy: logging::LogStrategy::decode(read).await }
+		Self {
+			log_strategy: logging::LogStrategy::decode(read.as_mut()).await,
+			msg_logs: logging::LogStrategy::decode(read.as_mut()).await,
+		}
 	}
 }
 impl Encode for HostHeader {
 	async fn encode<W: Write + ?Sized>(&self, mut write: Pin<&mut W>) {
 		write_exact(write.as_mut(), HOST_INTRO).await;
-		self.log_strategy.encode(write).await
+		self.log_strategy.encode(write.as_mut()).await;
+		self.msg_logs.encode(write.as_mut()).await
 	}
 }
 
@@ -159,7 +164,10 @@ mod tests {
 	#[test]
 	fn host_header_enc() {
 		spin_on(async {
-			let hh = HostHeader { log_strategy: logging::LogStrategy::File("SomeFile".to_string()) };
+			let hh = HostHeader {
+				log_strategy: logging::LogStrategy::File("SomeFile".to_string()),
+				msg_logs: logging::LogStrategy::File("SomeFile".to_string()),
+			};
 			let mut enc = &enc_vec(&hh).await[..];
 			eprintln!("Encoded to {enc:?}");
 			HostHeader::decode(Pin::new(&mut enc)).await;

orchid-api/src/tree.rs

@@ -32,6 +32,8 @@ pub enum Token {
 	LambdaHead(Vec<TokenTree>),
 	/// A name segment or an operator.
 	Name(TStr),
+	/// An absolute name
+	Reference(TStrv),
 	/// ::
 	NS,
 	/// Line break.

orchid-base/src/format.rs

@@ -140,7 +140,7 @@ impl Variants {
 					(Some((r, ..)), None) => &s[r.end..],
 					(None, None) => s,
 				};
-				let str_item = FmtElement::String(Rc::new(string.to_string()));
+				let str_item = FmtElement::String(Rc::new(string.replace("{{", "{").replace("}}", "}")));
 				match r {
 					None => itertools::Either::Left([str_item]),
 					Some((_, idx, bounded)) =>

orchid-base/src/logging.rs

@@ -21,7 +21,10 @@ impl Logger {
 	pub fn write_fmt(&self, fmt: Arguments) {
 		match &self.0 {
 			api::LogStrategy::Discard => (),
-			api::LogStrategy::StdErr => stderr().write_fmt(fmt).expect("Could not write to stderr!"),
+			api::LogStrategy::StdErr => {
+				stderr().write_fmt(fmt).expect("Could not write to stderr!");
+				stderr().flush().expect("Could not flush stderr")
+			},
 			api::LogStrategy::File(f) => {
 				let mut file = (File::options().write(true).create(true).truncate(true).open(f))
 					.expect("Could not open logfile");

orchid-base/src/name.rs

@@ -20,137 +20,6 @@ trait_set! {
 	pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
 }
 
-/// A borrowed name fragment which can be empty. See [VPath] for the owned
-/// variant.
-#[derive(Hash, PartialEq, Eq)]
-#[repr(transparent)]
-pub struct PathSlice([Tok<String>]);
-impl PathSlice {
-	/// Create a new [PathSlice]
-	pub fn new(slice: &[Tok<String>]) -> &PathSlice {
-		// SAFETY: This is ok because PathSlice is #[repr(transparent)]
-		unsafe { &*(slice as *const [Tok<String>] as *const PathSlice) }
-	}
-	/// Convert to an owned name fragment
-	pub fn to_vpath(&self) -> VPath { VPath(self.0.to_vec()) }
-	/// Iterate over the tokens
-	pub fn iter(&self) -> impl NameIter + '_ { self.into_iter() }
-	/// Iterate over the segments
-	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
-		Box::new(self.0.iter().map(|s| s.as_str()))
-	}
-	/// Find the longest shared prefix of this name and another sequence
-	pub fn coprefix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
-		&self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count() as u16]
-	}
-	/// Find the longest shared suffix of this name and another sequence
-	pub fn cosuffix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
-		&self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count() as u16]
-	}
-	/// Remove another
-	pub fn strip_prefix<'a>(&'a self, other: &PathSlice) -> Option<&'a PathSlice> {
-		let shared = self.coprefix(other).len();
-		(shared == other.len()).then_some(PathSlice::new(&self[shared..]))
-	}
-	/// Number of path segments
-	pub fn len(&self) -> u16 { self.0.len().try_into().expect("Too long name!") }
-	pub fn get<I: NameIndex>(&self, index: I) -> Option<&I::Output> { index.get(self) }
-	/// Whether there are any path segments. In other words, whether this is a
-	/// valid name
-	pub fn is_empty(&self) -> bool { self.len() == 0 }
-	/// Obtain a reference to the held slice. With all indexing traits shadowed,
-	/// this is better done explicitly
-	pub fn as_slice(&self) -> &[Tok<String>] { self }
-	/// Global empty path slice
-	pub fn empty() -> &'static Self { PathSlice::new(&[]) }
-}
-impl fmt::Debug for PathSlice {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
-}
-impl fmt::Display for PathSlice {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "{}", self.str_iter().join("::"))
-	}
-}
-impl Borrow<[Tok<String>]> for PathSlice {
-	fn borrow(&self) -> &[Tok<String>] { &self.0 }
-}
-impl<'a> IntoIterator for &'a PathSlice {
-	type IntoIter = Cloned<slice::Iter<'a, Tok<String>>>;
-	type Item = Tok<String>;
-	fn into_iter(self) -> Self::IntoIter { self.0.iter().cloned() }
-}
-
-pub trait NameIndex {
-	type Output: ?Sized;
-	fn get(self, name: &PathSlice) -> Option<&Self::Output>;
-}
-impl<T: NameIndex> Index<T> for PathSlice {
-	type Output = T::Output;
-	fn index(&self, index: T) -> &Self::Output { index.get(self).expect("Index out of bounds") }
-}
-
-mod idx_impls {
-	use std::ops;
-
-	use super::{NameIndex, PathSlice, conv_range};
-	use crate::interner::Tok;
-
-	impl NameIndex for u16 {
-		type Output = Tok<String>;
-		fn get(self, name: &PathSlice) -> Option<&Self::Output> { name.0.get(self as usize) }
-	}
-
-	impl NameIndex for ops::RangeFull {
-		type Output = PathSlice;
-		fn get(self, name: &PathSlice) -> Option<&Self::Output> { Some(name) }
-	}
-
-	macro_rules! impl_range_index_for_pathslice {
-		($range:ident) => {
-			impl ops::Index<ops::$range<u16>> for PathSlice {
-				type Output = Self;
-				fn index(&self, index: ops::$range<u16>) -> &Self::Output {
-					Self::new(&self.0[conv_range::<u16, usize>(index)])
-				}
-			}
-		};
-	}
-
-	impl_range_index_for_pathslice!(RangeFrom);
-	impl_range_index_for_pathslice!(RangeTo);
-	impl_range_index_for_pathslice!(Range);
-	impl_range_index_for_pathslice!(RangeInclusive);
-	impl_range_index_for_pathslice!(RangeToInclusive);
-}
-
-impl Deref for PathSlice {
-	type Target = [Tok<String>];
-
-	fn deref(&self) -> &Self::Target { &self.0 }
-}
-impl Borrow<PathSlice> for [Tok<String>] {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(self) }
-}
-impl<const N: usize> Borrow<PathSlice> for [Tok<String>; N] {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
-}
-impl Borrow<PathSlice> for Vec<Tok<String>> {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
-}
-pub fn conv_bound<T: Into<U> + Clone, U>(bound: Bound<&T>) -> Bound<U> {
-	match bound {
-		Bound::Included(i) => Bound::Included(i.clone().into()),
-		Bound::Excluded(i) => Bound::Excluded(i.clone().into()),
-		Bound::Unbounded => Bound::Unbounded,
-	}
-}
-pub fn conv_range<'a, T: Into<U> + Clone + 'a, U: 'a>(
-	range: impl RangeBounds<T>,
-) -> (Bound<U>, Bound<U>) {
-	(conv_bound(range.start_bound()), conv_bound(range.end_bound()))
-}
-
 /// A token path which may be empty. [VName] is the non-empty,
 /// [PathSlice] is the borrowed version
 #[derive(Clone, Default, Hash, PartialEq, Eq)]
@@ -227,22 +96,19 @@ impl IntoIterator for VPath {
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
 impl Borrow<[Tok<String>]> for VPath {
-	fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
+	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
-}
-impl Borrow<PathSlice> for VPath {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
 }
 impl Deref for VPath {
-	type Target = PathSlice;
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
 impl<T> Index<T> for VPath
-where PathSlice: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <PathSlice as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
-	fn index(&self, index: T) -> &Self::Output { &Borrow::<PathSlice>::borrow(self)[index] }
+	fn index(&self, index: T) -> &Self::Output { &Borrow::<[Tok<String>]>::borrow(self)[index] }
 }
 
 /// A mutable representation of a namespaced identifier of at least one segment.
@@ -311,20 +177,17 @@ impl IntoIterator for VName {
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
 impl<T> Index<T> for VName
-where PathSlice: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <PathSlice as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
 impl Borrow<[Tok<String>]> for VName {
 	fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
 }
-impl Borrow<PathSlice> for VName {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
-}
 impl Deref for VName {
-	type Target = PathSlice;
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -384,20 +247,17 @@ impl fmt::Display for Sym {
 	}
 }
 impl<T> Index<T> for Sym
-where PathSlice: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <PathSlice as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
 impl Borrow<[Tok<String>]> for Sym {
 	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
 }
-impl Borrow<PathSlice> for Sym {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
-}
 impl Deref for Sym {
-	type Target = PathSlice;
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -405,10 +265,10 @@ impl Deref for Sym {
 /// handled together in datastructures. The names can never be empty
 #[allow(clippy::len_without_is_empty)] // never empty
 pub trait NameLike:
-	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<PathSlice>
+	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[Tok<String>]>
 {
 	/// Convert into held slice
-	fn as_slice(&self) -> &[Tok<String>] { Borrow::<PathSlice>::borrow(self) }
+	fn as_slice(&self) -> &[Tok<String>] { Borrow::<[Tok<String>]>::borrow(self) }
 	/// Get iterator over tokens
 	fn iter(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
 	/// Get iterator over string segments
@@ -425,14 +285,14 @@ pub trait NameLike:
 		NonZeroUsize::try_from(self.iter().count()).expect("NameLike never empty")
 	}
 	/// Like slice's `split_first` except we know that it always returns Some
-	fn split_first(&self) -> (Tok<String>, &PathSlice) {
+	fn split_first(&self) -> (Tok<String>, &[Tok<String>]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
-		(foot.clone(), PathSlice::new(torso))
+		(foot.clone(), torso)
 	}
 	/// Like slice's `split_last` except we know that it always returns Some
-	fn split_last(&self) -> (Tok<String>, &PathSlice) {
+	fn split_last(&self) -> (Tok<String>, &[Tok<String>]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
-		(foot.clone(), PathSlice::new(torso))
+		(foot.clone(), torso)
 	}
 	/// Get the first element
 	fn first(&self) -> Tok<String> { self.split_first().0 }
@@ -498,7 +358,7 @@ mod test {
 
 	use test_executors::spin_on;
 
-	use super::{PathSlice, Sym, VName};
+	use super::{NameLike, Sym, VName};
 	use crate::interner::{Interner, Tok};
 	use crate::name::VPath;
 
@@ -508,8 +368,7 @@ mod test {
 			let i = Interner::new_master();
 			let myname = vname!(foo::bar; i).await;
 			let _borrowed_slice: &[Tok<String>] = myname.borrow();
-			let _borrowed_pathslice: &PathSlice = myname.borrow();
+			let _deref_pathslice: &[Tok<String>] = &myname;
-			let _deref_pathslice: &PathSlice = &myname;
 			let _as_slice_out: &[Tok<String>] = myname.as_slice();
 		})
 	}

orchid-base/src/parse.rs

@@ -10,7 +10,7 @@ use crate::error::{OrcRes, Reporter, mk_err, mk_errv};
 use crate::format::{Format, take_first_fmt};
 use crate::interner::{Interner, Tok};
 use crate::location::Pos;
-use crate::name::VPath;
+use crate::name::{VName, VPath};
 use crate::tree::{AtomRepr, ExtraTok, Paren, TokTree, Token};
 
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }

orchid-base/src/reqnot.rs

@@ -1,4 +1,4 @@
-use std::any::Any;
+use std::any::{Any, TypeId};
 use std::cell::RefCell;
 use std::future::Future;
 use std::marker::PhantomData;
@@ -17,8 +17,8 @@ use hashbrown::HashMap;
 use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
 use trait_set::trait_set;
 
-use crate::clone;
 use crate::logging::Logger;
+use crate::{api, clone};
 
 pub struct Receipt<'a>(PhantomData<&'a mut ()>);
 
@@ -204,7 +204,8 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
 			mem::drop(g);
 			let rn = self.clone();
 			send(&buf, rn).await;
-			RawReply(recv.recv().await.unwrap())
+			let items = recv.recv().await;
+			RawReply(items.unwrap())
 		})
 	}
 }
@@ -222,6 +223,7 @@ pub trait Requester: DynRequester {
 		MappedRequester::new(self, logger)
 	}
 }
+
 impl<This: DynRequester + ?Sized> Requester for This {
 	async fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
 		let req = format!("{data:?}");

orchid-base/src/tree.rs

@@ -1,7 +1,6 @@
 use std::borrow::Borrow;
 use std::fmt::{self, Debug, Display};
 use std::future::Future;
-use std::iter;
 use std::marker::PhantomData;
 use std::ops::Range;
 use std::rc::Rc;
@@ -20,7 +19,7 @@ use crate::error::OrcErrv;
 use crate::format::{FmtCtx, FmtUnit, Format, Variants};
 use crate::interner::{Interner, Tok};
 use crate::location::Pos;
-use crate::name::PathSlice;
+use crate::name::Sym;
 use crate::parse::Snippet;
 use crate::{api, match_mapping, tl_cache};
 
@@ -39,6 +38,7 @@ pub fn recur<'a, A: AtomRepr, X: ExtraTok>(
 		let tok = match tok {
 			tok @ (Token::Atom(_) | Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::NS) => tok,
 			tok @ (Token::Name(_) | Token::Slot(_) | Token::X(_) | Token::Ph(_) | Token::Macro(_)) => tok,
+			tok @ Token::Reference(_) => tok,
 			Token::LambdaHead(arg) =>
 				Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
 			Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
@@ -87,7 +87,8 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 			Comment(c.clone()),
 			Slot(id => TokHandle::new(*id)),
 			Ph(ph => Ph::from_api(ph, i).await),
-			Macro(*prio)
+			Macro(*prio),
+			Reference(tok => Sym::from_api(*tok, i).await)
 		});
 		Self { range: tt.range.clone(), tok }
 	}
@@ -105,6 +106,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 			S(*p, b => ttv_to_api(b, do_extra).boxed_local().await),
 			Ph(ph.to_api()),
 			Macro(*prio),
+			Reference(sym.to_api()),
 		} {
 			Token::X(x) => return do_extra(x, self.range.clone()).await
 		});
@@ -117,6 +119,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 	) -> api::TokenTree {
 		let token = match self.tok {
 			Token::Atom(a) => api::Token::Atom(a.to_api().await),
+			Token::Reference(sym) => api::Token::Reference(sym.to_api()),
 			Token::BR => api::Token::BR,
 			Token::NS => api::Token::NS,
 			Token::Bottom(e) => api::Token::Bottom(e.to_api()),
@@ -191,18 +194,6 @@ pub async fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>(
 	.await
 }
 
-/// This takes a position and not a range because it assigns the range to
-/// multiple leaf tokens, which is only valid if it's a zero-width range
-pub fn vname_tv<'a: 'b, 'b, A: AtomRepr + 'a, X: ExtraTok + 'a>(
-	name: &'b PathSlice,
-	pos: u32,
-) -> impl Iterator<Item = TokTree<'a, A, X>> + 'b {
-	let (head, tail) = name.split_first().expect("Empty vname");
-	iter::once(Token::Name(head.clone()))
-		.chain(tail.iter().flat_map(|t| [Token::NS, Token::Name(t.clone())]))
-		.map(move |t| t.at(pos..pos))
-}
-
 pub fn wrap_tokv<'a, A: AtomRepr, X: ExtraTok>(
 	items: impl IntoIterator<Item = TokTree<'a, A, X>>,
 ) -> TokTree<'a, A, X> {
@@ -219,19 +210,43 @@ pub fn wrap_tokv<'a, A: AtomRepr, X: ExtraTok>(
 
 pub use api::Paren;
 
+/// Lexer output variant
 #[derive(Clone, Debug)]
 pub enum Token<'a, A: AtomRepr, X: ExtraTok> {
+	/// Information about the code addressed to the human reader or dev tooling
+	/// It has no effect on the behaviour of the program unless it's explicitly
+	/// read via reflection
 	Comment(Arc<String>),
+	/// The part of a lambda between `\` and `.` enclosing the argument. The body
+	/// stretches to the end of the enclosing parens or the end of the const line
 	LambdaHead(Vec<TokTree<'a, A, X>>),
+	/// A binding, operator, or a segment of a namespaced::name
 	Name(Tok<String>),
+	/// The namespace separator ::
 	NS,
+	/// A line break
 	BR,
+	/// `()`, `[]`, or `{}`
 	S(Paren, Vec<TokTree<'a, A, X>>),
+	/// A fully formed reference to external code emitted by a lexer plugin
+	Reference(Sym),
+	/// A value emitted by a lexer plugin
 	Atom(A),
+	/// A grammar error emitted by a lexer plugin if it was possible to continue
+	/// reading. Parsers should treat it as an atom unless it prevents parsing,
+	/// in which case both this and a relevant error should be returned.
 	Bottom(OrcErrv),
+	/// An instruction from a plugin for the lexer to embed a subexpression
+	/// without retransmitting it. It should not appear anywhere outside lexer
+	/// plugin responses.
 	Slot(TokHandle<'a>),
+	/// Additional domain-specific token types
 	X(X),
+	/// A placeholder for metaprogramming, either $name, ..$name, ..$name:N,
+	/// ...$name, or ...$name:N
 	Ph(Ph),
+	/// `macro` or `macro(`X`)` where X is any valid floating point number
+	/// expression. `macro` is not a valid name in Orchid for this reason.
 	Macro(Option<NotNan<f64>>),
 }
 impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
@@ -258,6 +273,7 @@ impl<A: AtomRepr, X: ExtraTok> Format for Token<'_, A, X> {
 				]),
 			Self::NS => "::".to_string().into(),
 			Self::Name(n) => format!("{n}").into(),
+			Self::Reference(sym) => format!("{sym}").into(),
 			Self::Slot(th) => format!("{th}").into(),
 			Self::Ph(ph) => format!("{ph}").into(),
 			Self::S(p, b) => FmtUnit::new(

orchid-extension/Cargo.toml

@@ -16,6 +16,7 @@ hashbrown = "0.15.2"
 itertools = "0.14.0"
 konst = "0.3.16"
 lazy_static = "1.5.0"
+memo-map = "0.3.3"
 never = "0.1.0"
 once_cell = "1.20.2"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }

orchid-extension/src/atom.rs

@@ -2,6 +2,7 @@ use std::any::{Any, TypeId, type_name};
 use std::fmt;
 use std::future::Future;
 use std::marker::PhantomData;
+use std::num::NonZeroU32;
 use std::ops::Deref;
 use std::pin::Pin;
 use std::rc::Rc;
@@ -12,6 +13,7 @@ use async_std::stream;
 use dyn_clone::{DynClone, clone_box};
 use futures::future::LocalBoxFuture;
 use futures::{FutureExt, StreamExt};
+use orchid_api_derive::Coding;
 use orchid_api_traits::{Coding, Decode, Encode, Request, enc_vec};
 use orchid_base::clone;
 use orchid_base::error::{OrcErr, OrcRes, mk_err};
@@ -29,6 +31,9 @@ use crate::expr::{Expr, ExprData, ExprHandle, ExprKind};
 use crate::gen_expr::GExpr;
 use crate::system::{DynSystemCard, SysCtx, atom_info_for, downcast_atom};
 
+#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct AtomTypeId(pub NonZeroU32);
+
 pub trait AtomCard: 'static + Sized {
 	type Data: Clone + Coding + Sized;
 }
@@ -72,7 +77,7 @@ impl<A: Atomic + AtomicFeaturesImpl<A::Variant>> AtomicFeatures for A {
 
 pub fn get_info<A: AtomCard>(
 	sys: &(impl DynSystemCard + ?Sized),
-) -> (api::AtomId, Box<dyn AtomDynfo>) {
+) -> (AtomTypeId, Box<dyn AtomDynfo>) {
 	atom_info_for(sys, TypeId::of::<A>()).unwrap_or_else(|| {
 		panic!("Atom {} not associated with system {}", type_name::<A>(), sys.name())
 	})

orchid-extension/src/atom_owned.rs

@@ -1,20 +1,24 @@
 use std::any::{Any, TypeId, type_name};
 use std::borrow::Cow;
 use std::future::Future;
+use std::num::NonZero;
+use std::ops::Deref;
 use std::pin::Pin;
 use std::rc::Rc;
+use std::sync::atomic::AtomicU64;
 
 use async_once_cell::OnceCell;
 use async_std::io::{Read, Write};
+use async_std::sync::{RwLock, RwLockReadGuard};
 use futures::FutureExt;
 use futures::future::{LocalBoxFuture, ready};
 use itertools::Itertools;
+use memo_map::MemoMap;
 use never::Never;
+use orchid_api::AtomId;
 use orchid_api_traits::{Decode, Encode, enc_vec};
-use orchid_base::clone;
 use orchid_base::error::OrcRes;
 use orchid_base::format::FmtUnit;
-use orchid_base::id_store::{IdRecord, IdStore};
 use orchid_base::name::Sym;
 
 use crate::api;
@@ -31,23 +35,39 @@ impl AtomicVariant for OwnedVariant {}
 impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVariant> for A {
 	fn _factory(self) -> AtomFactory {
 		AtomFactory::new(move |ctx| async move {
-			let rec = ctx.obj_store.add(Box::new(self));
+			let serial = ctx.obj_store.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-			let (id, _) = get_info::<A>(ctx.cted.inst().card());
+			let atom_id = api::AtomId(NonZero::new(serial + 1).unwrap());
-			let mut data = enc_vec(&id).await;
+			let (typ_id, _) = get_info::<A>(ctx.cted.inst().card());
-			rec.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
+			let mut data = enc_vec(&typ_id).await;
-			api::Atom { drop: Some(api::AtomId(rec.id())), data, owner: ctx.id }
+			self.encode(Pin::<&mut Vec<u8>>::new(&mut data)).await;
+			ctx.obj_store.1.read().await.insert(atom_id, Box::new(self));
+			api::Atom { drop: Some(atom_id), data, owner: ctx.id }
 		})
 	}
 	fn _info() -> Self::_Info { OwnedAtomDynfo { msbuild: A::reg_reqs(), ms: OnceCell::new() } }
 	type _Info = OwnedAtomDynfo<A>;
 }
 
-fn with_atom<'a, U>(
+/// While an atom read guard is held, no atom can be removed.
+pub(crate) struct AtomReadGuard<'a> {
 	id: api::AtomId,
-	ctx: &'a SysCtx,
+	guard: RwLockReadGuard<'a, MemoMap<AtomId, Box<dyn DynOwnedAtom>>>,
-	f: impl FnOnce(IdRecord<'a, Box<dyn DynOwnedAtom>>) -> U,
+}
-) -> U {
+impl<'a> AtomReadGuard<'a> {
-	f(ctx.obj_store.get(id.0).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0)))
+	async fn new(id: api::AtomId, ctx: &'a SysCtx) -> Self {
+		let guard = ctx.obj_store.1.read().await;
+		assert!(guard.get(&id).is_some(), "Received invalid atom ID: {}", id.0);
+		Self { id, guard }
+	}
+}
+impl Deref for AtomReadGuard<'_> {
+	type Target = dyn DynOwnedAtom;
+	fn deref(&self) -> &Self::Target { &**self.guard.get(&self.id).unwrap() }
+}
+
+pub(crate) async fn take_atom(id: api::AtomId, ctx: &SysCtx) -> Box<dyn DynOwnedAtom> {
+	let mut g = ctx.obj_store.1.write().await;
+	g.remove(&id).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0))
 }
 
 pub struct OwnedAtomDynfo<T: OwnedAtom> {
@@ -64,23 +84,18 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 		.boxed_local()
 	}
 	fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> LocalBoxFuture<'_, GExpr> {
-		with_atom(id.unwrap(), &ctx, |a| a.remove()).dyn_call(ctx.clone(), arg)
+		async move { take_atom(id.unwrap(), &ctx).await.dyn_call(ctx.clone(), arg).await }.boxed_local()
 	}
 	fn call_ref<'a>(
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
 		arg: api::ExprTicket,
 	) -> LocalBoxFuture<'a, GExpr> {
-		async move {
+		async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_call_ref(ctx.clone(), arg).await }
-			with_atom(id.unwrap(), &ctx, |a| clone!(ctx; async move { a.dyn_call_ref(ctx, arg).await }))
-				.await
-		}
 			.boxed_local()
 	}
 	fn print(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> LocalBoxFuture<'_, FmtUnit> {
-		async move {
+		async move { AtomReadGuard::new(id.unwrap(), &ctx).await.dyn_print(ctx.clone()).await }
-			with_atom(id.unwrap(), &ctx, |a| clone!(ctx; async move { a.dyn_print(ctx).await })).await
-		}
 			.boxed_local()
 	}
 	fn handle_req<'a, 'b: 'a, 'c: 'a>(
@@ -91,13 +106,9 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 		rep: Pin<&'c mut dyn Write>,
 	) -> LocalBoxFuture<'a, bool> {
 		async move {
-			with_atom(id.unwrap(), &ctx, |a| {
+			let a = AtomReadGuard::new(id.unwrap(), &ctx).await;
-				clone!(ctx; async move {
 			let ms = self.ms.get_or_init(self.msbuild.pack(ctx.clone())).await;
-					ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), ctx, key, req, rep).await
+			ms.dispatch(a.as_any_ref().downcast_ref().unwrap(), ctx.clone(), key, req, rep).await
-				})
-			})
-			.await
 		}
 		.boxed_local()
 	}
@@ -105,12 +116,10 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 		&'a self,
 		AtomCtx(_, id, ctx): AtomCtx<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Option<GExpr>>> {
-		async move { with_atom(id.unwrap(), &ctx, |a| a.remove().dyn_command(ctx.clone())).await }
+		async move { take_atom(id.unwrap(), &ctx).await.dyn_command(ctx.clone()).await }.boxed_local()
-			.boxed_local()
 	}
 	fn drop(&self, AtomCtx(_, id, ctx): AtomCtx) -> LocalBoxFuture<'_, ()> {
-		async move { with_atom(id.unwrap(), &ctx, |a| a.remove().dyn_free(ctx.clone())).await }
+		async move { take_atom(id.unwrap(), &ctx).await.dyn_free(ctx.clone()).await }.boxed_local()
-			.boxed_local()
 	}
 	fn serialize<'a, 'b: 'a>(
 		&'a self,
@@ -120,9 +129,8 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
 		async move {
 			let id = id.unwrap();
 			id.encode(write.as_mut()).await;
-			with_atom(id, &ctx, |a| clone!(ctx; async move { a.dyn_serialize(ctx, write).await }))
+			let refs = AtomReadGuard::new(id, &ctx).await.dyn_serialize(ctx.clone(), write).await;
-				.await
+			refs.map(|v| v.into_iter().map(|t| t.handle().tk).collect_vec())
-				.map(|v| v.into_iter().map(|t| t.handle().tk).collect_vec())
 		}
 		.boxed_local()
 	}
@@ -305,4 +313,4 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	}
 }
 
-pub type ObjStore = Rc<IdStore<Box<dyn DynOwnedAtom>>>;
+pub type ObjStore = Rc<(AtomicU64, RwLock<MemoMap<api::AtomId, Box<dyn DynOwnedAtom>>>)>;

orchid-extension/src/conv.rs

@@ -35,7 +35,7 @@ impl<A: AtomicFeatures> TryFromExpr for TypAtom<'_, A> {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
 		match expr.atom().await {
 			Err(ex) => Err(err_not_atom(ex.data().await.pos.clone(), &ex.ctx().i).await.into()),
-			Ok(f) => match downcast_atom(f).await {
+			Ok(f) => match downcast_atom::<A>(f).await {
 				Ok(a) => Ok(a),
 				Err(f) => Err(err_type(f.pos(), &f.ctx().i).await.into()),
 			},

orchid-extension/src/entrypoint.rs

@@ -23,7 +23,7 @@ use orchid_base::clone;
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::logging::Logger;
 use orchid_base::macros::{mtreev_from_api, mtreev_to_api};
-use orchid_base::name::{PathSlice, Sym};
+use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Snippet};
 use orchid_base::reqnot::{ReqNot, RequestHandle, Requester};
 use orchid_base::tree::{ttv_from_api, ttv_to_api};
@@ -31,8 +31,8 @@ use substack::Substack;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::atom::{AtomCtx, AtomDynfo};
+use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId};
-use crate::atom_owned::ObjStore;
+use crate::atom_owned::{ObjStore, take_atom};
 use crate::fs::VirtFS;
 use crate::lexer::{LexContext, err_cascade, err_not_applicable};
 use crate::macros::{Rule, RuleCtx};
@@ -72,7 +72,7 @@ trait_set! {
 	pub trait WARCallback<'a, T> = FnOnce(
 		Box<dyn AtomDynfo>,
 		SysCtx,
-		api::AtomId,
+		AtomTypeId,
 		&'a [u8]
 	) -> LocalBoxFuture<'a, T>
 }
@@ -86,8 +86,8 @@ pub async fn with_atom_record<'a, F: Future<Output = SysCtx>, T>(
 	let mut data = &atom.data[..];
 	let ctx = get_sys_ctx(atom.owner, reqnot).await;
 	let inst = ctx.cted.inst();
-	let id = api::AtomId::decode(Pin::new(&mut data)).await;
+	let id = AtomTypeId::decode(Pin::new(&mut data)).await;
-	let atom_record = atom_by_idx(inst.card(), id).expect("Atom ID reserved");
+	let atom_record = atom_by_idx(inst.card(), id.clone()).expect("Atom ID reserved");
 	cb(atom_record, ctx, id, data).await
 }
 
@@ -127,7 +127,7 @@ impl ExtPort for ExtensionOwner {
 }
 
 pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
-	let api::HostHeader { log_strategy } =
+	let api::HostHeader { log_strategy, msg_logs } =
 		api::HostHeader::decode(Pin::new(&mut async_std::io::stdin())).await;
 	let mut buf = Vec::new();
 	let decls = (data.systems.iter().enumerate())
@@ -142,6 +142,7 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 	std::io::stdout().flush().unwrap();
 	let exiting = Arc::new(AtomicBool::new(false));
 	let logger = Logger::new(log_strategy);
+	let msg_logger = Logger::new(msg_logs);
 	let interner_cell = Rc::new(RefCell::new(None::<Rc<Interner>>));
 	let interner_weak = Rc::downgrade(&interner_cell);
 	let obj_store = ObjStore::default();
@@ -158,26 +159,29 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 			}.boxed_local())
 	});
 	let rn = ReqNot::<api::ExtMsgSet>::new(
-		logger.clone(),
+		msg_logger.clone(),
 		move |a, _| async move { send_parent_msg(a).await.unwrap() }.boxed_local(),
-		clone!(systems, exiting, mk_ctx, obj_store; move |n, reqnot| {
+		clone!(systems, exiting, mk_ctx; move |n, reqnot| {
-			clone!(systems, exiting, mk_ctx, obj_store; async move {
+			clone!(systems, exiting, mk_ctx; async move {
 				match n {
 					api::HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
 					api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
 						mem::drop(systems.lock().await.remove(&sys_id)),
-					api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) =>
+					api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) => {
-						obj_store.get(atom.0).unwrap().remove().dyn_free(mk_ctx(sys_id, reqnot).await).await,
+						let ctx = mk_ctx(sys_id, reqnot).await;
+						take_atom(atom, &ctx).await.dyn_free(ctx.clone()).await
+					}
 				}
 			}.boxed_local())
 		}),
 		{
-			clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls);
+			clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls, msg_logger);
 			move |hand, req| {
-				clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls);
+				clone!(systems, logger, mk_ctx, interner_weak, obj_store, spawner, decls, msg_logger);
 				async move {
 					let interner_cell = interner_weak.upgrade().expect("Interner dropped before request");
 					let i = interner_cell.borrow().clone().expect("Request arrived before interner set");
+					writeln!(msg_logger, "{} extension received request {req:?}", data.name);
 					match req {
 						api::HostExtReq::Ping(ping @ api::Ping) => hand.handle(&ping, &()).await,
 						api::HostExtReq::Sweep(sweep @ api::Sweep) =>
@@ -270,7 +274,7 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 							let ctx = mk_ctx(*sys_id, hand.reqnot()).await;
 							let systems_g = systems.lock().await;
 							let path = join_all(path.iter().map(|t| Tok::from_api(*t, &i))).await;
-							let vfs = systems_g[sys_id].vfses[vfs_id].load(PathSlice::new(&path), ctx).await;
+							let vfs = systems_g[sys_id].vfses[vfs_id].load(&path, ctx).await;
 							hand.handle(&vfs_read, &vfs).await
 						},
 						api::HostExtReq::LexExpr(lex @ api::LexExpr { sys, text, pos, id }) => {
@@ -386,7 +390,7 @@ pub async fn extension_main_logic(data: ExtensionData, spawner: Spawner) {
 							let api::DeserAtom(sys, buf, refs) = &deser;
 							let mut read = &mut &buf[..];
 							let ctx = mk_ctx(*sys, hand.reqnot()).await;
-							let id = api::AtomId::decode(Pin::new(&mut read)).await;
+							let id = AtomTypeId::decode(Pin::new(&mut read)).await;
 							let inst = ctx.cted.inst();
 							let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
 							hand.handle(&deser, &nfo.deserialize(ctx.clone(), read, refs).await).await

orchid-extension/src/expr.rs

@@ -3,7 +3,9 @@ use std::rc::Rc;
 
 use async_once_cell::OnceCell;
 use derive_destructure::destructure;
+use orchid_api::ExtAtomPrint;
 use orchid_base::error::OrcErrv;
+use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::location::Pos;
 use orchid_base::reqnot::Requester;
 
@@ -75,6 +77,16 @@ impl Expr {
 
 	pub fn gen(&self) -> GExpr { GExpr { pos: Pos::SlotTarget, kind: GExprKind::Slot(self.clone()) } }
 }
+impl Format for Expr {
+	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		match &self.data().await.kind {
+			ExprKind::Opaque => "OPAQUE".to_string().into(),
+			ExprKind::Bottom(b) => format!("Bottom({b})").into(),
+			ExprKind::Atom(a) =>
+				FmtUnit::from_api(&self.handle.ctx.reqnot.request(ExtAtomPrint(a.atom.clone())).await),
+		}
+	}
+}
 
 #[derive(Clone, Debug)]
 pub struct ExprData {

orchid-extension/src/fs.rs

@@ -3,8 +3,7 @@ use std::num::NonZero;
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
 use hashbrown::HashMap;
-use orchid_base::interner::Interner;
+use orchid_base::interner::{Interner, Tok};
-use orchid_base::name::PathSlice;
 
 use crate::api;
 use crate::system::SysCtx;
@@ -12,7 +11,7 @@ use crate::system::SysCtx;
 pub trait VirtFS: Send + Sync + 'static {
 	fn load<'a>(
 		&'a self,
-		path: &'a PathSlice,
+		path: &'a [Tok<String>],
 		ctx: SysCtx,
 	) -> LocalBoxFuture<'a, api::OrcResult<api::Loaded>>;
 }

orchid-extension/src/func_atom.rs

@@ -13,6 +13,7 @@ use never::Never;
 use orchid_api_traits::Encode;
 use orchid_base::clone;
 use orchid_base::error::OrcRes;
+use orchid_base::format::{FmtCtxImpl, Format, take_first};
 use orchid_base::name::Sym;
 use trait_set::trait_set;
 
@@ -61,6 +62,7 @@ impl Fun {
 		};
 		Self { args: vec![], arity: F::ARITY, path, fun }
 	}
+	pub fn arity(&self) -> u8 { self.arity }
 }
 impl Atomic for Fun {
 	type Data = ();
@@ -71,6 +73,7 @@ impl OwnedAtom for Fun {
 	type Refs = Vec<Expr>;
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
 	async fn call_ref(&self, arg: ExprHandle) -> GExpr {
+		std::io::Write::flush(&mut std::io::stderr()).unwrap();
 		let new_args = self.args.iter().cloned().chain([Expr::from_handle(Rc::new(arg))]).collect_vec();
 		if new_args.len() == self.arity.into() {
 			(self.fun)(new_args).await.to_expr()
@@ -90,6 +93,9 @@ impl OwnedAtom for Fun {
 		let (arity, fun) = FUNS.with(|f| f.clone()).lock().await.get(&path).unwrap().clone();
 		Self { args, arity, path, fun }
 	}
+	async fn print(&self, _: SysCtx) -> orchid_base::format::FmtUnit {
+		format!("{}:{}/{}", self.path, self.args.len(), self.arity).into()
+	}
 }
 
 /// An Atom representing a partially applied native lambda. These are not

orchid-extension/src/system.rs

@@ -1,5 +1,5 @@
 use core::fmt;
-use std::any::TypeId;
+use std::any::{TypeId, type_name};
 use std::future::Future;
 use std::num::NonZero;
 use std::pin::Pin;
@@ -15,7 +15,7 @@ use orchid_base::logging::Logger;
 use orchid_base::reqnot::{Receipt, ReqNot};
 
 use crate::api;
-use crate::atom::{AtomCtx, AtomDynfo, AtomicFeatures, ForeignAtom, TypAtom, get_info};
+use crate::atom::{AtomCtx, AtomDynfo, AtomTypeId, AtomicFeatures, ForeignAtom, TypAtom, get_info};
 use crate::atom_owned::ObjStore;
 use crate::entrypoint::ExtReq;
 use crate::fs::DeclFs;
@@ -49,21 +49,21 @@ fn general_atoms() -> impl Iterator<Item = Option<Box<dyn AtomDynfo>>> {
 pub fn atom_info_for(
 	sys: &(impl DynSystemCard + ?Sized),
 	tid: TypeId,
-) -> Option<(api::AtomId, Box<dyn AtomDynfo>)> {
+) -> Option<(AtomTypeId, Box<dyn AtomDynfo>)> {
-	(sys.atoms().enumerate().map(|(i, o)| (NonZero::new(i as u64 + 1).unwrap(), o)))
+	(sys.atoms().enumerate().map(|(i, o)| (NonZero::new(i as u32 + 1).unwrap(), o)))
-		.chain(general_atoms().enumerate().map(|(i, o)| (NonZero::new(!(i as u64)).unwrap(), o)))
+		.chain(general_atoms().enumerate().map(|(i, o)| (NonZero::new(!(i as u32)).unwrap(), o)))
-		.filter_map(|(i, o)| o.map(|a| (api::AtomId(i), a)))
+		.filter_map(|(i, o)| o.map(|a| (AtomTypeId(i), a)))
 		.find(|ent| ent.1.tid() == tid)
 }
 
 pub fn atom_by_idx(
 	sys: &(impl DynSystemCard + ?Sized),
-	tid: api::AtomId,
+	tid: AtomTypeId,
 ) -> Option<Box<dyn AtomDynfo>> {
-	if (u64::from(tid.0) >> (u64::BITS - 1)) & 1 == 1 {
+	if (u32::from(tid.0) >> (u32::BITS - 1)) & 1 == 1 {
-		general_atoms().nth(!u64::from(tid.0) as usize).unwrap()
+		general_atoms().nth(!u32::from(tid.0) as usize).unwrap()
 	} else {
-		sys.atoms().nth(u64::from(tid.0) as usize - 1).unwrap()
+		sys.atoms().nth(u32::from(tid.0) as usize - 1).unwrap()
 	}
 }
 
@@ -71,7 +71,7 @@ pub async fn resolv_atom(
 	sys: &(impl DynSystemCard + ?Sized),
 	atom: &api::Atom,
 ) -> Box<dyn AtomDynfo> {
-	let tid = api::AtomId::decode(Pin::new(&mut &atom.data[..8])).await;
+	let tid = AtomTypeId::decode(Pin::new(&mut &atom.data[..])).await;
 	atom_by_idx(sys, tid).expect("Value of nonexistent type found")
 }
 
@@ -115,18 +115,22 @@ pub async fn downcast_atom<A>(foreign: ForeignAtom<'_>) -> Result<TypAtom<'_, A>
 where A: AtomicFeatures {
 	let mut data = &foreign.atom.data[..];
 	let ctx = foreign.ctx.clone();
-	let value = api::AtomId::decode(Pin::new(&mut data)).await;
+	let value = AtomTypeId::decode(Pin::new(&mut data)).await;
-	let info_ent = (ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
+	let own_inst = ctx.cted.inst();
-		.map(|sys| get_info::<A>(sys.get_card()))
+	let owner = if ctx.id == foreign.atom.owner {
-		.filter(|(pos, _)| value == *pos);
+		own_inst.card()
-	match info_ent {
+	} else {
-		None => Err(foreign),
+		(ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
-		Some((_, info)) => {
+			.ok_or_else(|| foreign.clone())?
-			let val = info.decode(AtomCtx(data, foreign.atom.drop, ctx)).await;
+			.get_card()
+	};
+	let (typ_id, dynfo) = get_info::<A>(owner);
+	if value != typ_id {
+		return Err(foreign);
+	}
+	let val = dynfo.decode(AtomCtx(data, foreign.atom.drop, ctx)).await;
 	let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
 	Ok(TypAtom { value, data: foreign })
-		},
-	}
 }
 
 #[derive(Clone)]

orchid-extension/src/tree.rs

@@ -21,7 +21,7 @@ use crate::atom::{AtomFactory, ForeignAtom};
 use crate::conv::ToExpr;
 use crate::entrypoint::MemberRecord;
 use crate::func_atom::{ExprFunc, Fun};
-use crate::gen_expr::GExpr;
+use crate::gen_expr::{GExpr, arg, call, lambda, seq};
 use crate::macros::Rule;
 use crate::system::SysCtx;
 
@@ -92,8 +92,20 @@ pub fn root_mod(
 	(name.to_string(), kind)
 }
 pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenItem> {
-	let fac =
+	let fac = LazyMemberFactory::new(move |sym| async {
-		LazyMemberFactory::new(move |sym| async { MemKind::Const(Fun::new(sym, xf).await.to_expr()) });
+		return MemKind::Const(build_lambdas(Fun::new(sym, xf).await, 0));
+		fn build_lambdas(fun: Fun, i: u64) -> GExpr {
+			if i < fun.arity().into() {
+				return lambda(i, [build_lambdas(fun, i + 1)]);
+			}
+			let arity = fun.arity();
+			seq(
+				(0..arity)
+					.map(|i| arg(i as u64))
+					.chain([call([fun.to_expr()].into_iter().chain((0..arity).map(|i| arg(i as u64))))]),
+			)
+		}
+	});
 	with_export(GenMember { name: name.to_string(), kind: MemKind::Lazy(fac) }, exported)
 }
 pub fn macro_block(prio: Option<f64>, rules: impl IntoIterator<Item = Rule>) -> Vec<GenItem> {

orchid-host/Cargo.toml

@@ -6,9 +6,11 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-once-cell = "0.5.4"
 async-process = "2.3.0"
 async-std = "1.13.0"
 async-stream = "0.3.6"
+bound = "0.6.0"
 derive_destructure = "1.0.0"
 futures = "0.3.31"
 hashbrown = "0.15.2"

orchid-host/src/atom.rs

@@ -10,6 +10,7 @@ use orchid_base::tree::AtomRepr;
 use crate::api;
 use crate::ctx::Ctx;
 use crate::expr::Expr;
+use crate::extension::Extension;
 use crate::system::System;
 
 #[derive(destructure)]
@@ -75,6 +76,8 @@ impl AtomHand {
 			Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), arg.id())).await,
 		}
 	}
+	pub fn sys(&self) -> &System { &self.0.owner }
+	pub fn ext(&self) -> &Extension { self.sys().ext() }
 	pub async fn req(&self, key: api::TStrv, req: Vec<u8>) -> Option<Vec<u8>> {
 		self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), key, req)).await
 	}

orchid-host/src/ctx.rs

@@ -12,6 +12,7 @@ use orchid_base::interner::Interner;
 use crate::api;
 use crate::atom::WeakAtomHand;
 use crate::system::{System, WeakSystem};
+use crate::tree::Module;
 
 pub struct CtxData {
 	pub i: Rc<Interner>,
@@ -19,6 +20,7 @@ pub struct CtxData {
 	pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
 	pub system_id: RefCell<NonZeroU16>,
 	pub owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
+	pub root: RwLock<Module>,
 }
 #[derive(Clone)]
 pub struct Ctx(Rc<CtxData>);
@@ -34,6 +36,7 @@ impl Ctx {
 			systems: RwLock::default(),
 			system_id: RefCell::new(NonZero::new(1).unwrap()),
 			owned_atoms: RwLock::default(),
+			root: RwLock::new(Module::default()),
 		}))
 	}
 	pub(crate) async fn system_inst(&self, id: api::SysId) -> Option<System> {

orchid-host/src/dealias.rs

@@ -0,0 +1,240 @@
+use std::rc::Rc;
+
+use futures::FutureExt;
+use hashbrown::{HashMap, HashSet};
+use itertools::{Either, Itertools};
+use orchid_base::error::{OrcErr, Reporter, mk_err};
+use orchid_base::format::{FmtCtxImpl, Format, take_first};
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::Pos;
+use orchid_base::name::{NameLike, Sym, VName};
+
+use crate::macros::{MacTok, MacTree};
+use crate::tree::{ItemKind, MemberKind, Module, RuleKind, WalkErrorKind};
+
+/// Errors produced by absolute_path
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub enum AbsPathError {
+	/// `super` of root requested, for example, `app::main` referenced
+	/// `super::super::super::std`
+	TooManySupers,
+	/// root selected, for example, `app::main` referenced exactly `super::super`.
+	/// The empty path also triggers this.
+	RootPath,
+}
+impl AbsPathError {
+	pub async fn err_obj(self, i: &Interner, pos: Pos, path: &str) -> OrcErr {
+		let (descr, msg) = match self {
+			AbsPathError::RootPath => (
+				i.i("Path ends on root module").await,
+				format!(
+					"{path} is equal to the empty path. You cannot directly reference the root. \
+					Use one fewer 'super::' or add more segments to make it valid."
+				),
+			),
+			AbsPathError::TooManySupers => (
+				i.i("Too many 'super::' steps in path").await,
+				format!("{path} is leading outside the root."),
+			),
+		};
+		mk_err(descr, msg, [pos.into()])
+	}
+}
+
+/// Turn a relative (import) path into an absolute path.
+/// If the import path is empty, the return value is also empty.
+///
+/// # Errors
+///
+/// if the relative path contains as many or more `super` segments than the
+/// length of the absolute path.
+pub fn absolute_path(
+	mut cwd: &[Tok<String>],
+	mut rel: &[Tok<String>],
+) -> Result<VName, AbsPathError> {
+	let mut relative = false;
+	if rel.first().map(|t| t.as_str()) == Some("self") {
+		relative = true;
+		rel = rel.split_first().expect("checked above").1;
+	} else {
+		while rel.first().map(|t| t.as_str()) == Some("super") {
+			match cwd.split_last() {
+				Some((_, torso)) => cwd = torso,
+				None => return Err(AbsPathError::TooManySupers),
+			};
+			rel = rel.split_first().expect("checked above").1;
+			relative = true;
+		}
+	}
+	match relative {
+		true => VName::new(cwd.iter().chain(rel).cloned()),
+		false => VName::new(rel.to_vec()),
+	}
+	.map_err(|_| AbsPathError::RootPath)
+}
+
+pub async fn resolv_glob(
+	cwd: &[Tok<String>],
+	root: &Module,
+	abs_path: &[Tok<String>],
+	pos: Pos,
+	i: &Interner,
+	r: &impl Reporter,
+) -> Vec<Tok<String>> {
+	let coprefix_len = cwd.iter().zip(abs_path).take_while(|(a, b)| a == b).count();
+	let (co_prefix, diff_path) = abs_path.split_at(coprefix_len);
+	let co_parent = root.walk(false, co_prefix.iter().cloned()).await.expect("Invalid step in cwd");
+	let target_module = match co_parent.walk(true, diff_path.iter().cloned()).await {
+		Ok(t) => t,
+		Err(e) => {
+			let path = abs_path[..=coprefix_len + e.pos].iter().join("::");
+			let (tk, msg) = match e.kind {
+				WalkErrorKind::Constant =>
+					(i.i("Invalid import path").await, format!("{path} is a constant")),
+				WalkErrorKind::Missing => (i.i("Invalid import path").await, format!("{path} not found")),
+				WalkErrorKind::Private => (i.i("Import inaccessible").await, format!("{path} is private")),
+			};
+			r.report(mk_err(tk, msg, [pos.into()]));
+			return vec![];
+		},
+	};
+	target_module.exports.clone()
+}
+
+/// Read import statements and convert them into aliases, rasising any import
+/// errors in the process
+pub async fn imports_to_aliases(
+	module: &Module,
+	cwd: &mut Vec<Tok<String>>,
+	root: &Module,
+	alias_map: &mut HashMap<Sym, Sym>,
+	alias_rev_map: &mut HashMap<Sym, HashSet<Sym>>,
+	i: &Interner,
+	rep: &impl Reporter,
+) {
+	let mut import_locs = HashMap::<Sym, Vec<Pos>>::new();
+	for item in &module.items {
+		match &item.kind {
+			ItemKind::Import(imp) => match absolute_path(cwd, &imp.path) {
+				Err(e) => rep.report(e.err_obj(i, item.pos.clone(), &imp.path.iter().join("::")).await),
+				Ok(abs_path) => {
+					let names = match imp.name.as_ref() {
+						Some(n) => Either::Right([n.clone()].into_iter()),
+						None => Either::Left(
+							resolv_glob(cwd, root, &abs_path, item.pos.clone(), i, rep).await.into_iter(),
+						),
+					};
+					for name in names {
+						let mut tgt = abs_path.clone().suffix([name.clone()]).to_sym(i).await;
+						let src = Sym::new(cwd.iter().cloned().chain([name]), i).await.unwrap();
+						import_locs.entry(src.clone()).or_insert(vec![]).push(item.pos.clone());
+						if let Some(tgt2) = alias_map.get(&tgt) {
+							tgt = tgt2.clone();
+						}
+						if src == tgt {
+							rep.report(mk_err(
+								i.i("Circular references").await,
+								format!("{src} circularly refers to itself"),
+								[item.pos.clone().into()],
+							));
+							continue;
+						}
+						if let Some(fst_val) = alias_map.get(&src) {
+							let locations = (import_locs.get(&src))
+								.expect("The same name could only have appeared in the same module");
+							rep.report(mk_err(
+								i.i("Conflicting imports").await,
+								if fst_val == &src {
+									format!("{src} is imported multiple times")
+								} else {
+									format!("{} could refer to both {fst_val} and {src}", src.last())
+								},
+								locations.iter().map(|p| p.clone().into()).collect_vec(),
+							))
+						}
+						let mut srcv = vec![src.clone()];
+						if let Some(src_extra) = alias_rev_map.remove(&src) {
+							srcv.extend(src_extra);
+						}
+						for src in srcv {
+							alias_map.insert(src.clone(), tgt.clone());
+							alias_rev_map.entry(tgt.clone()).or_insert(HashSet::new()).insert(src);
+						}
+					}
+				},
+			},
+			ItemKind::Member(mem) => match mem.kind().await {
+				MemberKind::Const(_) => (),
+				MemberKind::Mod(m) => {
+					cwd.push(mem.name());
+					imports_to_aliases(m, cwd, root, alias_map, alias_rev_map, i, rep).boxed_local().await;
+					cwd.pop();
+				},
+			},
+			ItemKind::Export(_) | ItemKind::Macro(..) => (),
+		}
+	}
+}
+
+pub async fn dealias(module: &mut Module, alias_map: &HashMap<Sym, Sym>, i: &Interner) {
+	for item in &mut module.items {
+		match &mut item.kind {
+			ItemKind::Export(_) | ItemKind::Import(_) => (),
+			ItemKind::Member(mem) => match mem.kind_mut().await {
+				MemberKind::Const(c) => {
+					let Some(source) = c.source() else { continue };
+					let Some(new_source) = dealias_mactreev(source, alias_map, i).await else { continue };
+					c.set_source(new_source);
+				},
+				MemberKind::Mod(m) => dealias(m, alias_map, i).boxed_local().await,
+			},
+			ItemKind::Macro(_, rules) =>
+				for rule in rules.iter_mut() {
+					let RuleKind::Native(c) = &mut rule.kind else { continue };
+					let Some(source) = c.source() else { continue };
+					let Some(new_source) = dealias_mactreev(source, alias_map, i).await else { continue };
+					c.set_source(new_source);
+				},
+		}
+	}
+}
+
+async fn dealias_mactree(
+	mtree: &MacTree,
+	aliases: &HashMap<Sym, Sym>,
+	i: &Interner,
+) -> Option<MacTree> {
+	let new_tok = match &*mtree.tok {
+		MacTok::Atom(_) | MacTok::Ph(_) => return None,
+		tok @ (MacTok::Done(_) | MacTok::Ref(_) | MacTok::Slot(_)) => panic!(
+			"{} should not appear in retained pre-macro source",
+			take_first(&tok.print(&FmtCtxImpl { i }).await, true)
+		),
+		MacTok::Name(n) => MacTok::Name(aliases.get(n).unwrap_or(n).clone()),
+		MacTok::Lambda(arg, body) => {
+			match (dealias_mactreev(arg, aliases, i).await, dealias_mactreev(body, aliases, i).await) {
+				(None, None) => return None,
+				(Some(arg), None) => MacTok::Lambda(arg, body.clone()),
+				(None, Some(body)) => MacTok::Lambda(arg.clone(), body),
+				(Some(arg), Some(body)) => MacTok::Lambda(arg, body),
+			}
+		},
+		MacTok::S(p, b) => MacTok::S(*p, dealias_mactreev(b, aliases, i).await?),
+	};
+	Some(MacTree { pos: mtree.pos.clone(), tok: Rc::new(new_tok) })
+}
+
+async fn dealias_mactreev(
+	mtreev: &[MacTree],
+	aliases: &HashMap<Sym, Sym>,
+	i: &Interner,
+) -> Option<Vec<MacTree>> {
+	let mut results = Vec::with_capacity(mtreev.len());
+	let mut any_some = false;
+	for item in mtreev {
+		let out = dealias_mactree(item, aliases, i).boxed_local().await;
+		any_some |= out.is_some();
+		results.push(out.unwrap_or(item.clone()));
+	}
+	any_some.then_some(results)
+}

orchid-host/src/execute.rs

@@ -0,0 +1,223 @@
+use std::mem;
+
+use async_std::sync::RwLockWriteGuard;
+use bound::Bound;
+use futures::FutureExt;
+use orchid_base::error::{OrcErrv, mk_errv};
+use orchid_base::format::{FmtCtxImpl, Format, take_first};
+use orchid_base::location::Pos;
+use orchid_base::logging::Logger;
+use orchid_base::name::NameLike;
+
+use crate::ctx::Ctx;
+use crate::expr::{Expr, ExprKind, PathSet, Step};
+use crate::tree::{ItemKind, MemberKind};
+
+type ExprGuard = Bound<RwLockWriteGuard<'static, ExprKind>, Expr>;
+
+/// The stack operation associated with a transform
+enum StackOp {
+	Pop,
+	Nop,
+	Push(Expr),
+	Swap(Expr),
+	Unwind(OrcErrv),
+}
+
+pub enum ExecResult {
+	Value(Expr),
+	Gas(ExecCtx),
+	Err(OrcErrv),
+}
+
+pub struct ExecCtx {
+	ctx: Ctx,
+	gas: Option<u64>,
+	stack: Vec<ExprGuard>,
+	cur: ExprGuard,
+	cur_pos: Pos,
+	did_pop: bool,
+	logger: Logger,
+}
+impl ExecCtx {
+	pub async fn new(ctx: Ctx, logger: Logger, init: Expr) -> Self {
+		let cur_pos = init.pos();
+		let cur = Bound::async_new(init, |init| init.kind().write()).await;
+		Self { ctx, gas: None, stack: vec![], cur, cur_pos, did_pop: false, logger }
+	}
+	pub fn remaining_gas(&self) -> u64 { self.gas.expect("queried remaining_gas but no gas was set") }
+	pub fn set_gas(&mut self, gas: Option<u64>) { self.gas = gas }
+	pub fn idle(&self) -> bool { self.did_pop }
+	pub fn result(self) -> ExecResult {
+		if self.idle() {
+			match &*self.cur {
+				ExprKind::Bottom(errv) => ExecResult::Err(errv.clone()),
+				_ => ExecResult::Value(*self.cur.unbind()),
+			}
+		} else {
+			ExecResult::Gas(self)
+		}
+	}
+	pub fn use_gas(&mut self, amount: u64) -> bool {
+		if let Some(gas) = &mut self.gas {
+			*gas -= amount;
+		}
+		self.gas != Some(0)
+	}
+	pub async fn try_lock(&self, ex: &Expr) -> ExprGuard {
+		Bound::async_new(ex.clone(), |ex| ex.kind().write()).await
+	}
+	pub async fn unpack_ident(&self, ex: &Expr) -> Expr {
+		match ex.kind().try_write().as_deref_mut() {
+			Some(ExprKind::Identity(ex)) => {
+				let val = self.unpack_ident(ex).boxed_local().await;
+				*ex = val.clone();
+				val
+			},
+			Some(_) => ex.clone(),
+			None => panic!("Cycle encountered!"),
+		}
+	}
+	pub async fn execute(&mut self) {
+		while self.use_gas(1) {
+			let mut kind_swap = ExprKind::Missing;
+			mem::swap(&mut kind_swap, &mut self.cur);
+			let unit = kind_swap.print(&FmtCtxImpl { i: &self.ctx.i }).await;
+			writeln!(self.logger, "Exxecute lvl{} {}", self.stack.len(), take_first(&unit, true));
+			let (kind, op) = match kind_swap {
+				ExprKind::Identity(target) => {
+					let inner = self.unpack_ident(&target).await;
+					(ExprKind::Identity(inner.clone()), StackOp::Swap(inner))
+				},
+				ExprKind::Seq(a, b) if !self.did_pop => (ExprKind::Seq(a.clone(), b), StackOp::Push(a)),
+				ExprKind::Seq(_, b) => (ExprKind::Identity(b), StackOp::Nop),
+				ExprKind::Const(name) => {
+					let (cn, mp) = name.split_last();
+					let root_lock = self.ctx.root.read().await;
+					let module = root_lock.walk(true, mp.iter().cloned()).await.unwrap();
+					let member = (module.items.iter())
+						.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
+						.find(|m| m.name() == cn);
+					match member {
+						None => (
+							ExprKind::Bottom(mk_errv(
+								self.ctx.i.i("Constant does not exist").await,
+								format!("{name} does not refer to a constant"),
+								[self.cur_pos.clone().into()],
+							)),
+							StackOp::Pop,
+						),
+						Some(mem) => match mem.kind().await {
+							MemberKind::Mod(_) => (
+								ExprKind::Bottom(mk_errv(
+									self.ctx.i.i("module used as constant").await,
+									format!("{name} is a module"),
+									[self.cur_pos.clone().into()],
+								)),
+								StackOp::Pop,
+							),
+							MemberKind::Const(c) => {
+								let value = c.get_bytecode(&self.ctx).await;
+								(ExprKind::Identity(value.clone()), StackOp::Nop)
+							},
+						},
+					}
+				},
+				ExprKind::Arg => panic!("This should not appear outside function bodies"),
+				ek @ ExprKind::Atom(_) => (ek, StackOp::Pop),
+				ExprKind::Bottom(bot) => (ExprKind::Bottom(bot.clone()), StackOp::Unwind(bot)),
+				ExprKind::Call(f, x) if !self.did_pop => (ExprKind::Call(f.clone(), x), StackOp::Push(f)),
+				ExprKind::Call(f, x) => match f.try_into_owned_atom().await {
+					Ok(atom) => {
+						let mut ext = atom.sys().ext().clone();
+						let x_norm = self.unpack_ident(&x).await;
+						let val = Expr::from_api(&atom.call(x_norm).await, &mut ext).await;
+						(ExprKind::Identity(val.clone()), StackOp::Swap(val))
+					},
+					Err(f) => match &*f.kind().read().await {
+						ExprKind::Arg | ExprKind::Call(..) | ExprKind::Seq(..) | ExprKind::Const(_) =>
+							panic!("This should not appear outside function bodies"),
+						ExprKind::Missing => panic!("Should have been replaced"),
+						ExprKind::Atom(a) => {
+							let mut ext = a.sys().ext().clone();
+							let x_norm = self.unpack_ident(&x).await;
+							let val = Expr::from_api(&a.clone().call(x_norm).await, &mut ext).await;
+							(ExprKind::Identity(val.clone()), StackOp::Swap(val))
+						},
+						ExprKind::Bottom(exprv) => (ExprKind::Bottom(exprv.clone()), StackOp::Pop),
+						ExprKind::Lambda(None, body) =>
+							(ExprKind::Identity(body.clone()), StackOp::Swap(body.clone())),
+						ExprKind::Lambda(Some(path), body) => {
+							let output = substitute(body, &path.steps, path.next(), x).await;
+							(ExprKind::Identity(output.clone()), StackOp::Swap(output))
+						},
+						ExprKind::Identity(f) => (ExprKind::Call(f.clone(), x.clone()), StackOp::Nop),
+					},
+				},
+				l @ ExprKind::Lambda(..) => (l, StackOp::Pop),
+				ExprKind::Missing => panic!("Should have been replaced"),
+			};
+			self.did_pop = matches!(op, StackOp::Pop | StackOp::Unwind(_));
+			*self.cur = kind;
+			match op {
+				StackOp::Nop => (),
+				StackOp::Pop => match self.stack.pop() {
+					Some(top) => self.cur = top,
+					None => return,
+				},
+				StackOp::Push(sub) => {
+					self.cur_pos = sub.pos();
+					let mut new_guard = self.try_lock(&sub).await;
+					mem::swap(&mut self.cur, &mut new_guard);
+					self.stack.push(new_guard);
+				},
+				StackOp::Swap(new) => self.cur = self.try_lock(&new).await,
+				StackOp::Unwind(err) => {
+					for dependent in self.stack.iter_mut() {
+						**dependent = ExprKind::Bottom(err.clone());
+					}
+					*self.cur = ExprKind::Bottom(err.clone());
+					self.stack = vec![];
+					return;
+				},
+			}
+		}
+	}
+}
+
+async fn substitute(
+	src: &Expr,
+	path: &[Step],
+	next: Option<(&PathSet, &PathSet)>,
+	val: Expr,
+) -> Expr {
+	let exk = src.kind().try_read().expect("Cloned function body parts must never be written");
+	let kind = match (&*exk, path.split_first()) {
+		(ExprKind::Identity(x), _) => return substitute(x, path, next, val).boxed_local().await,
+		(ExprKind::Lambda(ps, b), _) =>
+			ExprKind::Lambda(ps.clone(), substitute(b, path, next, val).boxed_local().await),
+		(exk, None) => match (exk, next) {
+			(ExprKind::Arg, None) => return val.clone(),
+			(ExprKind::Call(f, x), Some((l, r))) => ExprKind::Call(
+				substitute(f, &l.steps, l.next(), val.clone()).boxed_local().await,
+				substitute(x, &r.steps, r.next(), val.clone()).boxed_local().await,
+			),
+			(ExprKind::Seq(a, b), Some((l, r))) => ExprKind::Seq(
+				substitute(a, &l.steps, l.next(), val.clone()).boxed_local().await,
+				substitute(b, &r.steps, r.next(), val.clone()).boxed_local().await,
+			),
+			(_, None) => panic!("Can only substitute Arg"),
+			(_, Some(_)) => panic!("Can only fork into Call and Seq"),
+		},
+		(ExprKind::Call(f, x), Some((Step::Left, tail))) =>
+			ExprKind::Call(substitute(f, tail, next, val).boxed_local().await, x.clone()),
+		(ExprKind::Call(f, x), Some((Step::Right, tail))) =>
+			ExprKind::Call(f.clone(), substitute(x, tail, next, val).boxed_local().await),
+		(ExprKind::Seq(f, x), Some((Step::Left, tail))) =>
+			ExprKind::Seq(substitute(f, tail, next, val).boxed_local().await, x.clone()),
+		(ExprKind::Seq(f, x), Some((Step::Right, tail))) =>
+			ExprKind::Seq(f.clone(), substitute(x, tail, next, val).boxed_local().await),
+		(ek, Some(_)) => panic!("Path leads into {ek:?}"),
+	};
+	kind.at(src.pos())
+}

orchid-host/src/expr.rs

@@ -1,23 +1,28 @@
+use std::cell::RefCell;
 use std::collections::VecDeque;
-use std::fmt;
 use std::num::NonZeroU64;
 use std::rc::{Rc, Weak};
+use std::{fmt, mem};
 
 use async_std::sync::RwLock;
 use futures::FutureExt;
 use hashbrown::HashSet;
 use itertools::Itertools;
-use orchid_api::ExprTicket;
+use orchid_base::error::{OrcErrv, mk_errv};
-use orchid_base::error::OrcErrv;
+use orchid_base::format::{FmtCtx, FmtCtxImpl, FmtUnit, Format, Variants, take_first};
-use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::location::Pos;
+use orchid_base::macros::mtreev_fmt;
 use orchid_base::name::Sym;
+use orchid_base::tokens::Paren;
 use orchid_base::tree::{AtomRepr, indent};
 use orchid_base::{match_mapping, tl_cache};
+use substack::Substack;
 
 use crate::api;
 use crate::atom::AtomHand;
+use crate::ctx::Ctx;
 use crate::extension::Extension;
+use crate::macros::{MacTok, MacTree};
 
 pub type ExprParseCtx = Extension;
 
@@ -31,8 +36,19 @@ pub struct ExprData {
 pub struct Expr(Rc<ExprData>);
 impl Expr {
 	pub fn pos(&self) -> Pos { self.0.pos.clone() }
-	pub fn as_atom(&self) -> Option<AtomHand> { todo!() }
+	pub async fn try_into_owned_atom(self) -> Result<AtomHand, Self> {
-	pub fn strong_count(&self) -> usize { todo!() }
+		match Rc::try_unwrap(self.0) {
+			Err(e) => Err(Self(e)),
+			Ok(data) => match data.kind.into_inner() {
+				ExprKind::Atom(a) => Ok(a),
+				inner => Err(Self(Rc::new(ExprData { kind: inner.into(), pos: data.pos }))),
+			},
+		}
+	}
+	pub async fn as_atom(&self) -> Option<AtomHand> {
+		if let ExprKind::Atom(a) = &*self.kind().read().await { Some(a.clone()) } else { None }
+	}
+	pub fn strong_count(&self) -> usize { Rc::strong_count(&self.0) }
 	pub fn id(&self) -> api::ExprTicket {
 		api::ExprTicket(
 			NonZeroU64::new(self.0.as_ref() as *const ExprData as usize as u64)
@@ -52,51 +68,27 @@ impl Expr {
 		match &*self.0.kind.read().await {
 			ExprKind::Atom(a) => K::Atom(a.to_api().await),
 			ExprKind::Bottom(b) => K::Bottom(b.to_api()),
+			ExprKind::Identity(ex) => ex.to_api().boxed_local().await,
 			_ => K::Opaque,
 		}
 	}
+	pub fn kind(&self) -> &RwLock<ExprKind> { &self.0.kind }
 }
 impl Format for Expr {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		return print_expr(self, c, &mut HashSet::new()).await;
+	}
+}
 async fn print_expr<'a>(
 	expr: &'a Expr,
 	c: &'a (impl FmtCtx + ?Sized + 'a),
-			visited: &mut HashSet<ExprTicket>,
+	visited: &mut HashSet<api::ExprTicket>,
 ) -> FmtUnit {
 	if visited.contains(&expr.id()) {
 		return "CYCLIC_EXPR".to_string().into();
 	}
 	visited.insert(expr.id());
-			match &*expr.0.kind.read().await {
+	print_exprkind(&*expr.kind().read().await, c, visited).boxed_local().await
-				ExprKind::Arg => "Arg".to_string().into(),
-				ExprKind::Atom(a) => a.print(c).await,
-				ExprKind::Bottom(e) if e.len() == 1 => format!("Bottom({e})").into(),
-				ExprKind::Bottom(e) => format!("Bottom(\n\t{}\n)", indent(&e.to_string())).into(),
-				ExprKind::Call(f, x) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
-					.unbounded("{0} {1l}")
-					.bounded("({0} {1b})")))
-				.units([
-					print_expr(f, c, visited).boxed_local().await,
-					print_expr(x, c, visited).boxed_local().await,
-				]),
-				ExprKind::Const(c) => format!("{c}").into(),
-				ExprKind::Lambda(None, body) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
-					.unbounded("\\.{0l}")
-					.bounded("(\\.{0b})")))
-				.units([print_expr(body, c, visited).boxed_local().await]),
-				ExprKind::Lambda(Some(path), body) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
-					.unbounded("\\{0b}. {1l}")
-					.bounded("(\\{0b}. {1b})")))
-				.units([format!("{path}").into(), print_expr(body, c, visited).boxed_local().await]),
-				ExprKind::Seq(l, r) =>
-					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("[{0b}]{1l}"))).units([
-						print_expr(l, c, visited).boxed_local().await,
-						print_expr(r, c, visited).boxed_local().await,
-					]),
-			}
-		}
-	}
 }
 
 #[derive(Clone, Debug)]
@@ -107,7 +99,12 @@ pub enum ExprKind {
 	Arg,
 	Lambda(Option<PathSet>, Expr),
 	Bottom(OrcErrv),
+	Identity(Expr),
 	Const(Sym),
+	/// Temporary expr kind assigned to a write guard to gain ownership of the
+	/// current value during normalization. While this is in place, the guard must
+	/// not be dropped.
+	Missing,
 }
 impl ExprKind {
 	pub async fn from_api(api: &api::ExpressionKind, pos: Pos, ctx: &mut ExprParseCtx) -> Self {
@@ -127,6 +124,48 @@ impl ExprKind {
 			api::ExpressionKind::Slot(_) => panic!("Handled in Expr"),
 		})
 	}
+	pub fn at(self, pos: Pos) -> Expr { Expr(Rc::new(ExprData { pos, kind: RwLock::new(self) })) }
+}
+impl Format for ExprKind {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		print_exprkind(self, c, &mut HashSet::new()).await
+	}
+}
+async fn print_exprkind<'a>(
+	ek: &ExprKind,
+	c: &'a (impl FmtCtx + ?Sized + 'a),
+	visited: &mut HashSet<api::ExprTicket>,
+) -> FmtUnit {
+	match &ek {
+		ExprKind::Arg => "Arg".to_string().into(),
+		ExprKind::Missing =>
+			panic!("This variant is swapped into write guards, so a read can never see it"),
+		ExprKind::Atom(a) => a.print(c).await,
+		ExprKind::Bottom(e) if e.len() == 1 => format!("Bottom({e})").into(),
+		ExprKind::Bottom(e) => format!("Bottom(\n\t{}\n)", indent(&e.to_string())).into(),
+		ExprKind::Call(f, x) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
+			.unbounded("{0} {1l}")
+			.bounded("({0} {1b})")))
+		.units([print_expr(f, c, visited).await, print_expr(x, c, visited).await]),
+		ExprKind::Identity(id) =>
+			tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{{{0}}}"))).units([print_expr(
+				id, c, visited,
+			)
+			.boxed_local()
+			.await]),
+		ExprKind::Const(c) => format!("{c}").into(),
+		ExprKind::Lambda(None, body) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
+			.unbounded("\\.{0l}")
+			.bounded("(\\.{0b})")))
+		.units([print_expr(body, c, visited).await]),
+		ExprKind::Lambda(Some(path), body) => tl_cache!(Rc<Variants>: Rc::new(Variants::default()
+			.unbounded("\\{0b}. {1l}")
+			.bounded("(\\{0b}. {1b})")))
+		.units([format!("{path}").into(), print_expr(body, c, visited).await]),
+		ExprKind::Seq(l, r) =>
+			tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("[{0b}]{1l}")))
+				.units([print_expr(l, c, visited).await, print_expr(r, c, visited).await]),
+	}
 }
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
@@ -138,34 +177,42 @@ pub enum Step {
 #[derive(Clone, Debug)]
 pub struct PathSet {
 	/// The single steps through [super::nort::Clause::Apply]
-	pub steps: VecDeque<Step>,
+	pub steps: Vec<Step>,
 	/// if Some, it splits at a [super::nort::Clause::Apply]. If None, it ends in
 	/// a [super::nort::Clause::LambdaArg]
 	pub next: Option<(Box<PathSet>, Box<PathSet>)>,
 }
 impl PathSet {
-	pub fn after(mut self, step: Step) -> Self {
+	pub fn next(&self) -> Option<(&PathSet, &PathSet)> {
-		self.steps.push_front(step);
+		self.next.as_ref().map(|(l, r)| (&**l, &**r))
-		self
 	}
 	pub fn from_api(id: u64, api: &api::ExpressionKind) -> Option<Self> {
 		use api::ExpressionKind as K;
+		struct Suffix(VecDeque<Step>, Option<(Box<PathSet>, Box<PathSet>)>);
+		fn seal(Suffix(steps, next): Suffix) -> PathSet { PathSet { steps: steps.into(), next } }
+		fn after(step: Step, mut suf: Suffix) -> Suffix {
+			suf.0.push_front(step);
+			suf
+		}
+		return from_api_inner(id, api).map(seal);
+		fn from_api_inner(id: u64, api: &api::ExpressionKind) -> Option<Suffix> {
 			match &api {
-			K::Arg(id2) => (id == *id2).then(|| Self { steps: VecDeque::new(), next: None }),
+				K::Arg(id2) => (id == *id2).then_some(Suffix(VecDeque::new(), None)),
 				K::Bottom(_) | K::Const(_) | K::NewAtom(_) | K::Slot(_) => None,
-			K::Lambda(_, b) => Self::from_api(id, &b.kind),
+				K::Lambda(_, b) => from_api_inner(id, &b.kind),
 				K::Call(l, r) | K::Seq(l, r) => {
-				match (Self::from_api(id, &l.kind), Self::from_api(id, &r.kind)) {
+					match (from_api_inner(id, &l.kind), from_api_inner(id, &r.kind)) {
 						(Some(a), Some(b)) =>
-						Some(Self { steps: VecDeque::new(), next: Some((Box::new(a), Box::new(b))) }),
+							Some(Suffix(VecDeque::new(), Some((Box::new(seal(a)), Box::new(seal(b)))))),
-					(Some(l), None) => Some(l.after(Step::Left)),
+						(Some(l), None) => Some(after(Step::Left, l)),
-					(None, Some(r)) => Some(r.after(Step::Right)),
+						(None, Some(r)) => Some(after(Step::Right, r)),
 						(None, None) => None,
 					}
 				},
 			}
 		}
 	}
+}
 impl fmt::Display for PathSet {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 		fn print_step(step: Step) -> &'static str { if step == Step::Left { "l" } else { "r" } }
@@ -177,12 +224,136 @@ impl fmt::Display for PathSet {
 				}
 				write!(f, "({left}|{right})")
 			},
-			None => write!(f, "{step_s}"),
+			None => write!(f, "{step_s}x"),
 		}
 	}
 }
 
+pub fn bot_expr(err: impl Into<OrcErrv>) -> Expr {
+	let errv: OrcErrv = err.into();
+	let pos = errv.pos_iter().next().map_or(Pos::None, |ep| ep.position.clone());
+	ExprKind::Bottom(errv).at(pos)
+}
+
 pub struct WeakExpr(Weak<ExprData>);
 impl WeakExpr {
 	pub fn upgrade(&self) -> Option<Expr> { self.0.upgrade().map(Expr) }
 }
+
+#[derive(Clone)]
+pub enum SrcToExprStep<'a> {
+	Left,
+	Right,
+	Lambda(Sym, &'a RefCell<Option<PathSet>>),
+}
+
+pub async fn mtreev_to_expr(
+	src: &[MacTree],
+	stack: Substack<'_, SrcToExprStep<'_>>,
+	ctx: &Ctx,
+) -> ExprKind {
+	let Some((x, f)) = src.split_last() else { panic!("Empty expression cannot be evaluated") };
+	let x_stack = if f.is_empty() { stack.clone() } else { stack.push(SrcToExprStep::Right) };
+	let x_kind = match &*x.tok {
+		MacTok::Atom(a) => ExprKind::Atom(a.clone()),
+		MacTok::Name(n) => 'name: {
+			let mut steps = VecDeque::new();
+			for step in x_stack.iter() {
+				match step {
+					SrcToExprStep::Left => steps.push_front(Step::Left),
+					SrcToExprStep::Right => steps.push_front(Step::Right),
+					SrcToExprStep::Lambda(name, _) if name != n => continue,
+					SrcToExprStep::Lambda(_, cell) => {
+						let mut ps = cell.borrow_mut();
+						match &mut *ps {
+							val @ None => *val = Some(PathSet { steps: steps.into(), next: None }),
+							Some(val) => {
+								let mut swap = PathSet { steps: Vec::new(), next: None };
+								mem::swap(&mut swap, val);
+								*val = merge(swap, &Vec::from(steps));
+								fn merge(ps: PathSet, steps: &[Step]) -> PathSet {
+									let diff_idx = ps.steps.iter().zip(steps).take_while(|(l, r)| l == r).count();
+									if diff_idx == ps.steps.len() {
+										if diff_idx == steps.len() {
+											match ps.next {
+												Some(_) => panic!("New path ends where old path forks"),
+												None => panic!("New path same as old path"),
+											}
+										}
+										let Some((left, right)) = ps.next else {
+											panic!("Old path ends where new path continues")
+										};
+										let next = match steps[diff_idx] {
+											Step::Left => Some((Box::new(merge(*left, &steps[diff_idx + 1..])), right)),
+											Step::Right => Some((left, Box::new(merge(*right, &steps[diff_idx + 1..])))),
+										};
+										PathSet { steps: ps.steps, next }
+									} else {
+										let shared_steps = ps.steps.iter().take(diff_idx).cloned().collect();
+										let main_steps = ps.steps.iter().skip(diff_idx + 1).cloned().collect();
+										let new_branch = steps[diff_idx + 1..].to_vec();
+										let main_side = PathSet { steps: main_steps, next: ps.next };
+										let new_side = PathSet { steps: new_branch, next: None };
+										let (left, right) = match steps[diff_idx] {
+											Step::Left => (new_side, main_side),
+											Step::Right => (main_side, new_side),
+										};
+										PathSet { steps: shared_steps, next: Some((Box::new(left), Box::new(right))) }
+									}
+								}
+							},
+						}
+						break 'name ExprKind::Arg;
+					},
+				}
+			}
+			ExprKind::Const(n.clone())
+		},
+		MacTok::Ph(_) | MacTok::Done(_) | MacTok::Ref(_) | MacTok::Slot(_) =>
+			ExprKind::Bottom(mk_errv(
+				ctx.i.i("placeholder in value").await,
+				"Placeholders cannot appear anywhere outside macro patterns",
+				[x.pos.clone().into()],
+			)),
+		MacTok::S(Paren::Round, b) if b.is_empty() =>
+			return ExprKind::Bottom(mk_errv(
+				ctx.i.i("Empty expression").await,
+				"Empty parens () are illegal",
+				[x.pos.clone().into()],
+			)),
+		MacTok::S(Paren::Round, b) => mtreev_to_expr(b, x_stack, ctx).boxed_local().await,
+		MacTok::S(..) => ExprKind::Bottom(mk_errv(
+			ctx.i.i("non-round parentheses after macros").await,
+			"[] or {} block was not consumed by macros; expressions may only contain ()",
+			[x.pos.clone().into()],
+		)),
+		MacTok::Lambda(_, b) if b.is_empty() =>
+			return ExprKind::Bottom(mk_errv(
+				ctx.i.i("Empty lambda").await,
+				"Lambdas must have a body",
+				[x.pos.clone().into()],
+			)),
+		MacTok::Lambda(arg, b) => 'lambda_converter: {
+			if let [MacTree { tok, .. }] = &**arg {
+				if let MacTok::Name(n) = &**tok {
+					let path = RefCell::new(None);
+					let b = mtreev_to_expr(b, x_stack.push(SrcToExprStep::Lambda(n.clone(), &path)), ctx)
+						.boxed_local()
+						.await;
+					break 'lambda_converter ExprKind::Lambda(path.into_inner(), b.at(x.pos.clone()));
+				}
+			}
+			let argstr = take_first(&mtreev_fmt(arg, &FmtCtxImpl { i: &ctx.i }).await, true);
+			ExprKind::Bottom(mk_errv(
+				ctx.i.i("Malformeed lambda").await,
+				format!("Lambda argument should be single name, found {argstr}"),
+				[x.pos.clone().into()],
+			))
+		},
+	};
+	if f.is_empty() {
+		return x_kind;
+	}
+	let f = mtreev_to_expr(f, stack.push(SrcToExprStep::Left), ctx).boxed_local().await;
+	ExprKind::Call(f.at(Pos::None), x_kind.at(x.pos.clone()))
+}

orchid-host/src/extension.rs

@@ -18,7 +18,7 @@ use orchid_base::clone;
 use orchid_base::format::{FmtCtxImpl, Format};
 use orchid_base::interner::Tok;
 use orchid_base::logging::Logger;
-use orchid_base::reqnot::{ReqNot, Requester as _};
+use orchid_base::reqnot::{DynRequester, ReqNot, Requester as _};
 
 use crate::api;
 use crate::atom::AtomHand;
@@ -54,7 +54,7 @@ impl Drop for ExtensionData {
 #[derive(Clone)]
 pub struct Extension(Rc<ExtensionData>);
 impl Extension {
-	pub fn new(init: ExtInit, logger: Logger, ctx: Ctx) -> io::Result<Self> {
+	pub fn new(init: ExtInit, logger: Logger, msg_logger: Logger, ctx: Ctx) -> io::Result<Self> {
 		Ok(Self(Rc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
 			exprs: ExprStore::default(),
 			ctx: ctx.clone(),
@@ -67,7 +67,7 @@ impl Extension {
 			lex_recur: Mutex::default(),
 			mac_recur: Mutex::default(),
 			reqnot: ReqNot::new(
-				logger,
+				msg_logger,
 				clone!(weak; move |sfn, _| clone!(weak; async move {
 					let data = weak.upgrade().unwrap();
 					data.init.send(sfn).await
@@ -100,6 +100,7 @@ impl Extension {
 						clone!(weak, ctx);
 						Box::pin(async move {
 							let this = Self(weak.upgrade().unwrap());
+							writeln!(this.reqnot().logger(), "Host received request {req:?}");
 							let i = this.ctx().i.clone();
 							match req {
 								api::ExtHostReq::Ping(ping) => hand.handle(&ping, &()).await,
@@ -213,7 +214,16 @@ impl Extension {
 	}
 	pub async fn recv_one(&self) {
 		let reqnot = self.0.reqnot.clone();
-		(self.0.init.recv(Box::new(move |msg| async move { reqnot.receive(msg).await }.boxed_local())))
+		let ctx = self.ctx().clone();
+		(self.0.init.recv(Box::new(move |msg| {
+			Box::pin(async move {
+				let msg = msg.to_vec();
+				let reqnot = reqnot.clone();
+				(ctx.spawn)(Box::pin(async move {
+					reqnot.receive(&msg).await;
+				}))
+			})
+		})))
 		.await;
 	}
 	pub fn system_drop(&self, id: api::SysId) {

orchid-host/src/lex.rs

@@ -8,6 +8,7 @@ use orchid_base::error::{OrcErrv, OrcRes, mk_errv};
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::match_mapping;
+use orchid_base::name::Sym;
 use orchid_base::number::{num_to_err, parse_num};
 use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::PARENS;
@@ -156,7 +157,7 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 					.lex(source, pos, |pos| async move {
 						let mut ctx_g = ctx_lck.lock().await;
 						match lex_once(&mut ctx_g.push(pos)).boxed_local().await {
-							Ok(t) => Some(api::SubLexed { pos, ticket: ctx_g.add_subtree(t) }),
+							Ok(t) => Some(api::SubLexed { pos: t.range.end, ticket: ctx_g.add_subtree(t) }),
 							Err(e) => {
 								errors_lck.lock().await.push(e);
 								None
@@ -203,6 +204,7 @@ async fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree
 		Bottom(err => OrcErrv::from_api(err, &ctx.ctx.i).await),
 		LambdaHead(arg => ttv_to_owned(arg, ctx).boxed_local().await),
 		Name(name => Tok::from_api(*name, &ctx.ctx.i).await),
+		Reference(tstrv => Sym::from_api(*tstrv, &ctx.ctx.i).await),
 		S(p.clone(), b => ttv_to_owned(b, ctx).boxed_local().await),
 		BR, NS,
 		Comment(c.clone()),

orchid-host/src/lib.rs

@@ -2,6 +2,8 @@ use orchid_api as api;
 
 pub mod atom;
 pub mod ctx;
+pub mod dealias;
+pub mod execute;
 pub mod expr;
 pub mod expr_store;
 pub mod extension;

orchid-host/src/parse.rs

@@ -10,8 +10,7 @@ use orchid_base::location::Pos;
 use orchid_base::macros::{MTok, MTree};
 use orchid_base::name::Sym;
 use orchid_base::parse::{
-	Comment, Import, Parsed, Snippet, expect_end, line_items, parse_multiname, strip_fluff,
+	Comment, Import, Parsed, Snippet, expect_end, line_items, parse_multiname, try_pop_no_fluff,
-	try_pop_no_fluff,
 };
 use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
@@ -19,9 +18,7 @@ use substack::Substack;
 use crate::atom::AtomHand;
 use crate::macros::MacTree;
 use crate::system::System;
-use crate::tree::{
+use crate::tree::{Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, Rule, RuleKind};
-	Code, CodeLocator, Item, ItemKind, Member, MemberKind, Module, ParsTokTree, Rule, RuleKind,
-};
 
 type ParsSnippet<'a> = Snippet<'a, 'static, AtomHand, Never>;
 
@@ -66,7 +63,7 @@ pub async fn parse_item(
 					let Parsed { output: exports, tail } = parse_multiname(ctx.reporter(), tail).await?;
 					let mut ok = Vec::new();
 					for (e, pos) in exports {
-						match (&e.path.as_slice(), e.name) {
+						match (&e.path[..], e.name) {
 							([], Some(n)) =>
 								ok.push(Item { comments: comments.clone(), pos, kind: ItemKind::Export(n) }),
 							(_, Some(_)) => ctx.reporter().report(mk_err(
@@ -129,7 +126,7 @@ pub async fn parse_exportable_item(
 		let (name, body) = parse_module(ctx, path, tail).await?;
 		ItemKind::Member(Member::new(name, MemberKind::Mod(body)))
 	} else if discr == tail.i().i("const").await {
-		let (name, val) = parse_const(tail).await?;
+		let (name, val) = parse_const(tail, path.clone()).await?;
 		let locator = CodeLocator::to_const(tail.i().i(&path.push(name.clone()).unreverse()).await);
 		ItemKind::Member(Member::new(name, MemberKind::Const(Code::from_code(locator, val))))
 	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
@@ -174,7 +171,10 @@ pub async fn parse_module(
 	Ok((name, Module::new(parse_items(ctx, path, body).await?)))
 }
 
-pub async fn parse_const(tail: ParsSnippet<'_>) -> OrcRes<(Tok<String>, Vec<ParsTokTree>)> {
+pub async fn parse_const(
+	tail: ParsSnippet<'_>,
+	path: Substack<'_, Tok<String>>,
+) -> OrcRes<(Tok<String>, Vec<MacTree>)> {
 	let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
 	let Some(name) = output.as_name() else {
 		return Err(mk_errv(
@@ -192,20 +192,25 @@ pub async fn parse_const(tail: ParsSnippet<'_>) -> OrcRes<(Tok<String>, Vec<Pars
 		));
 	}
 	try_pop_no_fluff(tail).await?;
-	Ok((name, tail.iter().flat_map(strip_fluff).collect_vec()))
+	Ok((name, parse_mtree(tail, path).await?))
 }
 
-pub async fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
+pub async fn parse_mtree(
+	mut snip: ParsSnippet<'_>,
+	path: Substack<'_, Tok<String>>,
+) -> OrcRes<Vec<MacTree>> {
 	let mut mtreev = Vec::new();
 	while let Some((ttree, tail)) = snip.pop_front() {
+		snip = tail;
 		let (range, tok, tail) = match &ttree.tok {
-			Token::S(p, b) => (
+			Token::S(p, b) => {
-				ttree.range.clone(),
+				let b = parse_mtree(Snippet::new(ttree, b, snip.i()), path.clone()).boxed_local().await?;
-				MTok::S(*p, parse_mtree(Snippet::new(ttree, b, snip.i())).boxed_local().await?),
+				(ttree.range.clone(), MTok::S(*p, b), tail)
-				tail,
+			},
-			),
+			Token::Reference(name) => (ttree.range.clone(), MTok::Name(name.clone()), tail),
 			Token::Name(tok) => {
-				let mut segments = vec![tok.clone()];
+				let mut segments = path.unreverse();
+				segments.push(tok.clone());
 				let mut end = ttree.range.end;
 				while let Some((TokTree { tok: Token::NS, .. }, tail)) = snip.pop_front() {
 					let Parsed { output, tail } = try_pop_no_fluff(tail).await?;
@@ -225,29 +230,27 @@ pub async fn parse_mtree(mut snip: ParsSnippet<'_>) -> OrcRes<Vec<MacTree>> {
 			},
 			Token::NS => {
 				return Err(mk_errv(
-					tail.i().i("Unexpected :: in macro pattern").await,
+					tail.i().i("Unexpected :: in expression").await,
-					":: can only follow a name outside export statements",
+					":: can only follow a name",
 					[Pos::Range(ttree.range.clone()).into()],
 				));
 			},
 			Token::Ph(ph) => (ttree.range.clone(), MTok::Ph(ph.clone()), tail),
-			Token::Atom(_) | Token::Macro(_) => {
+			Token::Macro(_) => {
 				return Err(mk_errv(
-					tail.i().i("Unsupported token in macro patterns").await,
+					tail.i().i("Invalid keyword in expression").await,
-					format!(
+					"Expressions cannot use `macro` as a name.",
-						"Macro patterns can only contain names, braces, and lambda, not {}.",
-						tail.fmt(ttree).await
-					),
 					[Pos::Range(ttree.range.clone()).into()],
 				));
 			},
+			Token::Atom(a) => (ttree.range.clone(), MTok::Atom(a.clone()), tail),
 			Token::BR | Token::Comment(_) => continue,
 			Token::Bottom(e) => return Err(e.clone()),
 			Token::LambdaHead(arg) => (
 				ttree.range.start..snip.pos().end,
 				MTok::Lambda(
-					parse_mtree(Snippet::new(ttree, arg, snip.i())).boxed_local().await?,
+					parse_mtree(Snippet::new(ttree, arg, snip.i()), path.clone()).boxed_local().await?,
-					parse_mtree(tail).boxed_local().await?,
+					parse_mtree(tail, path.clone()).boxed_local().await?,
 				),
 				Snippet::new(ttree, &[], snip.i()),
 			),
@@ -303,10 +306,10 @@ pub async fn parse_macro(
 		rules.push(Rule {
 			comments: item.output,
 			pos: Pos::Range(tail.pos()),
-			pattern: parse_mtree(pat).await?,
+			pattern: parse_mtree(pat, path.clone()).await?,
 			kind: RuleKind::Native(Code::from_code(
 				CodeLocator::to_rule(tail.i().i(&path.unreverse()).await, macro_i, i as u16),
-				body.to_vec(),
+				parse_mtree(body, path.clone()).await?,
 			)),
 		})
 	}

orchid-host/src/subprocess.rs

@@ -19,6 +19,7 @@ use crate::ctx::Ctx;
 pub async fn ext_command(
 	cmd: std::process::Command,
 	logger: Logger,
+	msg_logs: Logger,
 	ctx: Ctx,
 ) -> io::Result<ExtInit> {
 	let prog_pbuf = PathBuf::from(cmd.get_program());
@@ -29,7 +30,9 @@ pub async fn ext_command(
 		.stderr(async_process::Stdio::piped())
 		.spawn()?;
 	let mut stdin = child.stdin.take().unwrap();
-	api::HostHeader { log_strategy: logger.strat() }.encode(Pin::new(&mut stdin)).await;
+	api::HostHeader { log_strategy: logger.strat(), msg_logs: msg_logs.strat() }
+		.encode(Pin::new(&mut stdin))
+		.await;
 	let mut stdout = child.stdout.take().unwrap();
 	let header = api::ExtensionHeader::decode(Pin::new(&mut stdout)).await;
 	let child_stderr = child.stderr.take().unwrap();
@@ -41,7 +44,7 @@ pub async fn ext_command(
 				if 0 == reader.read_line(&mut buf).await.unwrap() {
 					break;
 				}
-				logger.log(buf);
+				logger.log(buf.strip_suffix('\n').expect("Readline implies this"));
 			}
 		})
 	})?;
@@ -73,9 +76,6 @@ impl Drop for Subprocess {
 }
 impl ExtPort for Subprocess {
 	fn send<'a>(&'a self, msg: &'a [u8]) -> LocalBoxFuture<'a, ()> {
-		if msg.starts_with(&[0, 0, 0, 0x1c]) {
-			panic!("Received unnecessary prefix");
-		}
 		async { send_msg(Pin::new(&mut *self.stdin.lock().await), msg).await.unwrap() }.boxed_local()
 	}
 	fn recv<'a>(
@@ -83,6 +83,7 @@ impl ExtPort for Subprocess {
 		cb: Box<dyn FnOnce(&[u8]) -> LocalBoxFuture<'_, ()> + 'a>,
 	) -> LocalBoxFuture<'a, ()> {
 		Box::pin(async {
+			std::io::Write::flush(&mut std::io::stderr()).unwrap();
 			match recv_msg(self.stdout.lock().await.as_mut()).await {
 				Ok(msg) => cb(&msg).await,
 				Err(e) if e.kind() == io::ErrorKind::BrokenPipe => (),

orchid-host/src/system.rs

@@ -1,7 +1,7 @@
 use std::collections::VecDeque;
-use std::fmt;
 use std::future::Future;
 use std::rc::{Rc, Weak};
+use std::{fmt, mem};
 
 use async_stream::stream;
 use derive_destructure::destructure;
@@ -9,12 +9,12 @@ use futures::StreamExt;
 use futures::future::join_all;
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::async_once_cell::OnceCell;
 use orchid_base::char_filter::char_filter_match;
 use orchid_base::clone;
 use orchid_base::error::{OrcErrv, OrcRes};
 use orchid_base::format::{FmtCtx, FmtUnit, Format};
 use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
 use orchid_base::parse::Comment;
 use orchid_base::reqnot::{ReqNot, Requester};
 use orchid_base::tree::ttv_from_api;
@@ -24,7 +24,7 @@ use substack::{Stackframe, Substack};
 use crate::api;
 use crate::ctx::Ctx;
 use crate::extension::{Extension, WeakExtension};
-use crate::tree::{Member, ParsTokTree};
+use crate::tree::{ItemKind, Member, Module, ParsTokTree};
 
 #[derive(destructure)]
 struct SystemInstData {
@@ -33,7 +33,6 @@ struct SystemInstData {
 	decl_id: api::SysDeclId,
 	lex_filter: api::CharFilter,
 	id: api::SysId,
-	const_root: OnceCell<Vec<Member>>,
 	line_types: Vec<Tok<String>>,
 }
 impl Drop for SystemInstData {
@@ -45,7 +44,6 @@ impl fmt::Debug for SystemInstData {
 			.field("decl_id", &self.decl_id)
 			.field("lex_filter", &self.lex_filter)
 			.field("id", &self.id)
-			.field("const_root", &self.const_root)
 			.field("line_types", &self.line_types)
 			.finish_non_exhaustive()
 	}
@@ -135,25 +133,26 @@ impl SystemCtor {
 			ext: ext.clone(),
 			ctx: ext.ctx().clone(),
 			lex_filter: sys_inst.lex_filter,
-			const_root: OnceCell::new(),
 			line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
 				.await,
 			id,
 		}));
-		(data.0.const_root.get_or_init(
+		let const_root = clone!(data, ext; stream! {
-			clone!(data, ext; stream! {
 			for (k, v) in sys_inst.const_root {
 				yield Member::from_api(
 					api::Member { name: k, kind: v },
 					&mut vec![Tok::from_api(k, &ext.ctx().i).await],
 					&data,
-					).await
+				).await;
 			}
 		})
-			.collect(),
+		.map(|mem| ItemKind::Member(mem).at(Pos::None))
-		))
+		.collect::<Vec<_>>()
 		.await;
 		ext.ctx().systems.write().await.insert(id, data.downgrade());
+		let mut swap = Module::default();
+		mem::swap(&mut swap, &mut *ext.ctx().root.write().await);
+		*ext.ctx().root.write().await = Module::new(swap.items.into_iter().chain(const_root));
 		data
 	}
 }

orchid-host/src/tree.rs

@@ -1,26 +1,28 @@
 use std::fmt::Debug;
 use std::rc::Rc;
-use std::sync::{Mutex, OnceLock};
 
+use async_once_cell::OnceCell;
+use async_std::sync::Mutex;
 use async_stream::stream;
 use futures::future::join_all;
 use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
 use never::Never;
-use orchid_base::error::OrcRes;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::macros::{mtreev_fmt, mtreev_from_api};
 use orchid_base::name::Sym;
 use orchid_base::parse::{Comment, Import};
-use orchid_base::tree::{AtomRepr, TokTree, Token, ttv_fmt};
+use orchid_base::tree::{AtomRepr, TokTree, Token};
 use orchid_base::{clone, tl_cache};
 use ordered_float::NotNan;
+use substack::Substack;
 
 use crate::api;
 use crate::atom::AtomHand;
-use crate::expr::Expr;
+use crate::ctx::Ctx;
+use crate::expr::{Expr, mtreev_to_expr};
 use crate::macros::{MacTok, MacTree};
 use crate::system::System;
 
@@ -41,13 +43,16 @@ pub enum ItemKind {
 	Import(Import),
 	Macro(Option<NotNan<f64>>, Vec<Rule>),
 }
+impl ItemKind {
+	pub fn at(self, pos: Pos) -> Item { Item { comments: vec![], pos, kind: self } }
+}
 
 impl Item {
 	pub async fn from_api(tree: api::Item, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
 		let kind = match tree.kind {
 			api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, path, sys).await),
 			api::ItemKind::Import(name) => ItemKind::Import(Import {
-				path: Sym::from_api(name, &sys.ctx().i).await.iter().collect(),
+				path: Sym::from_api(name, &sys.ctx().i).await.iter().cloned().collect(),
 				name: None,
 			}),
 			api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &sys.ctx().i).await),
@@ -108,11 +113,23 @@ impl Format for Item {
 }
 
 pub struct Member {
-	pub name: Tok<String>,
+	name: Tok<String>,
-	pub kind: OnceLock<MemberKind>,
+	kind: OnceCell<MemberKind>,
-	pub lazy: Mutex<Option<LazyMemberHandle>>,
+	lazy: Mutex<Option<LazyMemberHandle>>,
 }
 impl Member {
+	pub fn name(&self) -> Tok<String> { self.name.clone() }
+	pub async fn kind(&self) -> &MemberKind {
+		(self.kind.get_or_init(async {
+			let handle = self.lazy.lock().await.take().expect("Neither known nor lazy");
+			handle.run().await
+		}))
+		.await
+	}
+	pub async fn kind_mut(&mut self) -> &mut MemberKind {
+		self.kind().await;
+		self.kind.get_mut().expect("kind() already filled the cell")
+	}
 	pub async fn from_api(api: api::Member, path: &mut Vec<Tok<String>>, sys: &System) -> Self {
 		path.push(Tok::from_api(api.name, &sys.ctx().i).await);
 		let kind = match api.kind {
@@ -127,10 +144,10 @@ impl Member {
 			api::MemberKind::Module(m) => MemberKind::Mod(Module::from_api(m, path, sys).await),
 		};
 		let name = path.pop().unwrap();
-		Member { name, kind: OnceLock::from(kind), lazy: Mutex::default() }
+		Member { name, kind: OnceCell::from(kind), lazy: Mutex::default() }
 	}
 	pub fn new(name: Tok<String>, kind: MemberKind) -> Self {
-		Member { name, kind: OnceLock::from(kind), lazy: Mutex::default() }
+		Member { name, kind: OnceCell::from(kind), lazy: Mutex::default() }
 	}
 }
 impl Debug for Member {
@@ -148,7 +165,7 @@ pub enum MemberKind {
 	Mod(Module),
 }
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct Module {
 	pub imports: Vec<Sym>,
 	pub exports: Vec<Tok<String>>,
@@ -172,6 +189,40 @@ impl Module {
 				.await,
 		)
 	}
+	pub async fn walk(
+		&self,
+		allow_private: bool,
+		path: impl IntoIterator<Item = Tok<String>>,
+	) -> Result<&Module, WalkError> {
+		let mut cur = self;
+		for (pos, step) in path.into_iter().enumerate() {
+			let Some(member) = (cur.items.iter())
+				.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
+				.find(|m| m.name == step)
+			else {
+				return Err(WalkError { pos, kind: WalkErrorKind::Missing });
+			};
+			if !allow_private && !cur.exports.contains(&step) {
+				return Err(WalkError { pos, kind: WalkErrorKind::Private });
+			}
+			match member.kind().await {
+				MemberKind::Const(_) => return Err(WalkError { pos, kind: WalkErrorKind::Constant }),
+				MemberKind::Mod(m) => cur = m,
+			}
+		}
+		Ok(cur)
+	}
+}
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub enum WalkErrorKind {
+	Missing,
+	Private,
+	Constant,
+}
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct WalkError {
+	pub pos: usize,
+	pub kind: WalkErrorKind,
 }
 impl Format for Module {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -185,20 +236,20 @@ impl Format for Module {
 
 pub struct LazyMemberHandle(api::TreeId, System, Vec<Tok<String>>);
 impl LazyMemberHandle {
-	pub async fn run(self) -> OrcRes<MemberKind> {
+	pub async fn run(self) -> MemberKind {
 		match self.1.get_tree(self.0).await {
-			api::MemberKind::Const(c) => Ok(MemberKind::Const(Code {
+			api::MemberKind::Const(c) => MemberKind::Const(Code {
 				bytecode: Expr::from_api(&c, &mut self.1.ext().clone()).await.into(),
 				locator: CodeLocator { steps: self.1.ctx().i.i(&self.2).await, rule_loc: None },
 				source: None,
-			})),
+			}),
 			api::MemberKind::Module(m) =>
-				Ok(MemberKind::Mod(Module::from_api(m, &mut { self.2 }, &self.1).await)),
+				MemberKind::Mod(Module::from_api(m, &mut { self.2 }, &self.1).await),
 			api::MemberKind::Lazy(id) => Self(id, self.1, self.2).run().boxed_local().await,
 		}
 	}
 	pub fn into_member(self, name: Tok<String>) -> Member {
-		Member { name, kind: OnceLock::new(), lazy: Mutex::new(Some(self)) }
+		Member { name, kind: OnceCell::new(), lazy: Mutex::new(Some(self)) }
 	}
 }
 
@@ -237,15 +288,27 @@ pub enum RuleKind {
 #[derive(Debug)]
 pub struct Code {
 	locator: CodeLocator,
-	source: Option<Vec<ParsTokTree>>,
+	source: Option<Vec<MacTree>>,
-	bytecode: OnceLock<Expr>,
+	bytecode: OnceCell<Expr>,
 }
 impl Code {
 	pub fn from_expr(locator: CodeLocator, expr: Expr) -> Self {
 		Self { locator, source: None, bytecode: expr.into() }
 	}
-	pub fn from_code(locator: CodeLocator, code: Vec<ParsTokTree>) -> Self {
+	pub fn from_code(locator: CodeLocator, code: Vec<MacTree>) -> Self {
-		Self { locator, source: Some(code), bytecode: OnceLock::new() }
+		Self { locator, source: Some(code), bytecode: OnceCell::new() }
+	}
+	pub fn source(&self) -> Option<&Vec<MacTree>> { self.source.as_ref() }
+	pub fn set_source(&mut self, source: Vec<MacTree>) {
+		self.source = Some(source);
+		self.bytecode = OnceCell::new();
+	}
+	pub async fn get_bytecode(&self, ctx: &Ctx) -> &Expr {
+		(self.bytecode.get_or_init(async {
+			let src = self.source.as_ref().expect("no bytecode or source");
+			mtreev_to_expr(src, Substack::Bottom, ctx).await.at(Pos::None)
+		}))
+		.await
 	}
 }
 impl Format for Code {
@@ -254,7 +317,7 @@ impl Format for Code {
 			return bc.print(c).await;
 		}
 		if let Some(src) = &self.source {
-			return ttv_fmt(src, c).await;
+			return mtreev_fmt(src, c).await;
 		}
 		panic!("Code must be initialized with at least one state")
 	}

orchid-std/src/number/num_atom.rs

@@ -1,11 +1,13 @@
 use orchid_api_derive::Coding;
 use orchid_base::error::OrcRes;
+use orchid_base::format::FmtUnit;
 use orchid_extension::atom::{
 	AtomFactory, Atomic, AtomicFeatures, MethodSetBuilder, ToAtom, TypAtom,
 };
 use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
 use orchid_extension::conv::TryFromExpr;
 use orchid_extension::expr::Expr;
+use orchid_extension::system::SysCtx;
 use ordered_float::NotNan;
 
 #[derive(Clone, Debug, Coding)]
@@ -15,7 +17,9 @@ impl Atomic for Int {
 	type Data = Self;
 	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
-impl ThinAtom for Int {}
+impl ThinAtom for Int {
+	async fn print(&self, _: SysCtx) -> FmtUnit { self.0.to_string().into() }
+}
 impl TryFromExpr for Int {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
 		TypAtom::<Int>::try_from_expr(expr).await.map(|t| t.value)
@@ -29,7 +33,9 @@ impl Atomic for Float {
 	type Data = Self;
 	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
-impl ThinAtom for Float {}
+impl ThinAtom for Float {
+	async fn print(&self, _: SysCtx) -> FmtUnit { self.0.to_string().into() }
+}
 impl TryFromExpr for Float {
 	async fn try_from_expr(expr: Expr) -> OrcRes<Self> {
 		TypAtom::<Float>::try_from_expr(expr).await.map(|t| t.value)

orchid-std/src/std.rs

@@ -11,6 +11,7 @@ use orchid_extension::tree::{MemKind, comments, fun, module, root_mod};
 
 use crate::OrcString;
 use crate::number::num_atom::{Float, Int};
+use crate::number::num_lexer::NumLexer;
 use crate::string::str_atom::{IntStrAtom, StrAtom};
 use crate::string::str_lexer::StringLexer;
 
@@ -32,7 +33,7 @@ impl SystemCard for StdSystem {
 }
 impl System for StdSystem {
 	async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
-	fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer] }
+	fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer, &NumLexer] }
 	fn parsers() -> Vec<orchid_extension::parser::ParserObj> { vec![] }
 	fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
 	fn env() -> Vec<(String, MemKind)> {

orchid-std/src/string/str_atom.rs

@@ -49,6 +49,7 @@ impl OwnedAtom for StrAtom {
 	async fn serialize(&self, _: SysCtx, sink: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs {
 		self.deref().encode(sink).await
 	}
+	async fn print(&self, _: SysCtx) -> FmtUnit { format!("{:?}", &*self.0).into() }
 	async fn deserialize(mut ctx: impl DeserializeCtx, _: Self::Refs) -> Self {
 		Self::new(Rc::new(ctx.read::<String>().await))
 	}

orchid-std/src/string/str_lexer.rs

@@ -2,8 +2,8 @@ use itertools::Itertools;
 use orchid_base::error::{OrcErr, OrcRes, mk_err, mk_errv};
 use orchid_base::interner::Interner;
 use orchid_base::location::Pos;
-use orchid_base::tree::{vname_tv, wrap_tokv};
+use orchid_base::sym;
-use orchid_base::vname;
+use orchid_base::tree::wrap_tokv;
 use orchid_extension::atom::AtomicFeatures;
 use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
 use orchid_extension::tree::{GenTok, GenTokTree};
@@ -57,7 +57,7 @@ fn parse_string(str: &str) -> Result<String, StringError> {
 		}
 		let (mut pos, code) = iter.next().expect("lexer would have continued");
 		let next = match code {
-			c @ ('\\' | '"' | '$') => c,
+			c @ ('\\' | '"' | '\'' | '$') => c,
 			'b' => '\x08',
 			'f' => '\x0f',
 			'n' => '\n',
@@ -94,12 +94,14 @@ fn parse_string(str: &str) -> Result<String, StringError> {
 #[derive(Default)]
 pub struct StringLexer;
 impl Lexer for StringLexer {
-	const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"'];
+	const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"', '\''..='\''];
 	async fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> {
-		let Some(mut tail) = all.strip_prefix('"') else {
+		let Some((mut tail, delim)) = (all.strip_prefix('"').map(|t| (t, '"')))
+			.or_else(|| all.strip_prefix('\'').map(|t| (t, '\'')))
+		else {
 			return Err(err_not_applicable(ctx.i).await.into());
 		};
-		let mut ret = GenTok::X(IntStrAtom::from(ctx.i.i("").await).factory()).at(ctx.tok_ran(0, all));
+		let mut ret = None;
 		let mut cur = String::new();
 		let mut errors = vec![];
 		async fn str_to_gen<'a>(
@@ -116,19 +118,20 @@ impl Lexer for StringLexer {
 			GenTok::X(IntStrAtom::from(ctx.i.i(&*str_val).await).factory())
 				.at(ctx.tok_ran(str.len() as u32, tail)) as GenTokTree<'a>
 		}
-		let add_frag = |prev: GenTokTree<'a>, new: GenTokTree<'a>| async {
+		let add_frag = |prev: Option<GenTokTree<'a>>, new: GenTokTree<'a>| async {
-			wrap_tokv(
+			let Some(prev) = prev else { return new };
-				vname_tv(&vname!(std::string::concat; ctx.i).await, new.range.end).chain([prev, new]),
+			let concat_fn = GenTok::Reference(sym!(std::string::concat; ctx.i).await)
-			)
+				.at(prev.range.start..prev.range.start);
+			wrap_tokv([concat_fn, prev, new])
 		};
 		loop {
-			if let Some(rest) = tail.strip_prefix('"') {
+			if let Some(rest) = tail.strip_prefix(delim) {
 				return Ok((rest, add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await));
 			} else if let Some(rest) = tail.strip_prefix('$') {
-				ret = add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await;
+				ret = Some(add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await);
 				let (new_tail, tree) = ctx.recurse(rest).await?;
 				tail = new_tail;
-				ret = add_frag(ret, tree).await;
+				ret = Some(add_frag(ret, tree).await);
 			} else if tail.starts_with('\\') {
 				// parse_string will deal with it, we just have to skip the next char
 				tail = &tail[2..];

orcx/Cargo.toml

@@ -6,9 +6,11 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-std = "1.13.0"
 async-stream = "0.3.6"
 camino = "1.1.9"
 clap = { version = "4.5.24", features = ["derive", "env"] }
+ctrlc = "3.4.5"
 futures = "0.3.31"
 itertools = "0.14.0"
 orchid-base = { version = "0.1.0", path = "../orchid-base" }

orcx/src/main.rs

@@ -1,9 +1,10 @@
 use std::fs::File;
-use std::io::Read;
+use std::io::{Read, Write};
 use std::mem;
 use std::process::{Command, ExitCode};
 use std::rc::Rc;
 
+use async_std::io::stdin;
 use async_stream::try_stream;
 use camino::Utf8PathBuf;
 use clap::{Parser, Subcommand};
@@ -11,13 +12,17 @@ use futures::{Stream, TryStreamExt, io};
 use orchid_base::clone;
 use orchid_base::error::ReporterImpl;
 use orchid_base::format::{FmtCtxImpl, Format, take_first};
+use orchid_base::location::Pos;
 use orchid_base::logging::{LogStrategy, Logger};
+use orchid_base::macros::mtreev_fmt;
 use orchid_base::parse::Snippet;
 use orchid_base::tree::ttv_fmt;
 use orchid_host::ctx::Ctx;
+use orchid_host::execute::{ExecCtx, ExecResult};
+use orchid_host::expr::mtreev_to_expr;
 use orchid_host::extension::Extension;
 use orchid_host::lex::lex;
-use orchid_host::parse::{ParseCtxImpl, parse_items};
+use orchid_host::parse::{ParseCtxImpl, parse_items, parse_mtree};
 use orchid_host::subprocess::ext_command;
 use orchid_host::system::init_systems;
 use substack::Substack;
@@ -30,6 +35,8 @@ pub struct Args {
 	extension: Vec<Utf8PathBuf>,
 	#[arg(short, long, env = "ORCHID_DEFAULT_SYSTEMS", value_delimiter = ';')]
 	system: Vec<String>,
+	#[arg(short, long)]
+	verbose: bool,
 	#[command(subcommand)]
 	command: Commands,
 }
@@ -44,11 +51,17 @@ pub enum Commands {
 		#[arg(short, long)]
 		file: Utf8PathBuf,
 	},
+	Repl,
+	Execute {
+		#[arg()]
+		code: String,
+	},
 }
 
 fn get_all_extensions<'a>(
 	args: &'a Args,
 	logger: &'a Logger,
+	msg_logger: &'a Logger,
 	ctx: &'a Ctx,
 ) -> impl Stream<Item = io::Result<Extension>> + 'a {
 	try_stream! {
@@ -58,9 +71,9 @@ fn get_all_extensions<'a>(
 			} else {
 				ext_path.clone()
 			};
-			let init = ext_command(Command::new(exe.as_os_str()), logger.clone(), ctx.clone()).await
+			let init = ext_command(Command::new(exe.as_os_str()), logger.clone(), msg_logger.clone(), ctx.clone()).await
 				.unwrap();
-			let ext = Extension::new(init, logger.clone(), ctx.clone())?;
+			let ext = Extension::new(init, logger.clone(), msg_logger.clone(), ctx.clone())?;
 			spawn_local(clone!(ext; async move { loop { ext.recv_one().await }}));
 			yield ext
 		}
@@ -74,31 +87,35 @@ async fn main() -> io::Result<ExitCode> {
 		.run_until(async {
 			let args = Args::parse();
 			let ctx = &Ctx::new(Rc::new(|fut| mem::drop(spawn_local(fut))));
-			let logger = Logger::new(LogStrategy::Discard);
+			let i = &ctx.i;
-			let extensions =
+			let logger =
-				get_all_extensions(&args, &logger, ctx).try_collect::<Vec<Extension>>().await.unwrap();
+				Logger::new(if args.verbose { LogStrategy::StdErr } else { LogStrategy::Discard });
+			let extensions = get_all_extensions(&args, &logger, &Logger::new(LogStrategy::Discard), ctx)
+				.try_collect::<Vec<Extension>>()
+				.await
+				.unwrap();
 			match args.command {
 				Commands::Lex { file } => {
 					let systems = init_systems(&args.system, &extensions).await.unwrap();
 					let mut file = File::open(file.as_std_path()).unwrap();
 					let mut buf = String::new();
 					file.read_to_string(&mut buf).unwrap();
-					let lexemes = lex(ctx.i.i(&buf).await, &systems, ctx).await.unwrap();
+					let lexemes = lex(i.i(&buf).await, &systems, ctx).await.unwrap();
-					println!("{}", take_first(&ttv_fmt(&lexemes, &FmtCtxImpl { i: &ctx.i }).await, true))
+					println!("{}", take_first(&ttv_fmt(&lexemes, &FmtCtxImpl { i }).await, true))
 				},
 				Commands::Parse { file } => {
 					let systems = init_systems(&args.system, &extensions).await.unwrap();
 					let mut file = File::open(file.as_std_path()).unwrap();
 					let mut buf = String::new();
 					file.read_to_string(&mut buf).unwrap();
-					let lexemes = lex(ctx.i.i(&buf).await, &systems, ctx).await.unwrap();
+					let lexemes = lex(i.i(&buf).await, &systems, ctx).await.unwrap();
 					let Some(first) = lexemes.first() else {
 						println!("File empty!");
 						return;
 					};
 					let reporter = ReporterImpl::new();
 					let pctx = ParseCtxImpl { reporter: &reporter, systems: &systems };
-					let snip = Snippet::new(first, &lexemes, &ctx.i);
+					let snip = Snippet::new(first, &lexemes, i);
 					let ptree = parse_items(&pctx, Substack::Bottom, snip).await.unwrap();
 					if let Some(errv) = reporter.errv() {
 						eprintln!("{errv}");
@@ -111,7 +128,65 @@ async fn main() -> io::Result<ExitCode> {
 						return;
 					}
 					for item in ptree {
-						println!("{}", take_first(&item.print(&FmtCtxImpl { i: &ctx.i }).await, true))
+						println!("{}", take_first(&item.print(&FmtCtxImpl { i }).await, true))
+					}
+				},
+				Commands::Repl => loop {
+					let systems = init_systems(&args.system, &extensions).await.unwrap();
+					print!("\\.> ");
+					std::io::stdout().flush().unwrap();
+					let mut prompt = String::new();
+					stdin().read_line(&mut prompt).await.unwrap();
+					let lexemes = lex(i.i(prompt.trim()).await, &systems, ctx).await.unwrap();
+					if args.verbose {
+						println!("lexed: {}", take_first(&ttv_fmt(&lexemes, &FmtCtxImpl { i }).await, true));
+					}
+					let mtreev = parse_mtree(
+						Snippet::new(&lexemes[0], &lexemes, i),
+						Substack::Bottom.push(i.i("orcx").await).push(i.i("input").await),
+					)
+					.await
+					.unwrap();
+					if args.verbose {
+						let fmt = mtreev_fmt(&mtreev, &FmtCtxImpl { i }).await;
+						println!("parsed: {}", take_first(&fmt, true));
+					}
+					let expr = mtreev_to_expr(&mtreev, Substack::Bottom, ctx).await;
+					let mut xctx = ExecCtx::new(ctx.clone(), logger.clone(), expr.at(Pos::None)).await;
+					xctx.set_gas(Some(1000));
+					xctx.execute().await;
+					match xctx.result() {
+						ExecResult::Value(val) =>
+							println!("{}", take_first(&val.print(&FmtCtxImpl { i }).await, false)),
+						ExecResult::Err(e) => println!("error: {e}"),
+						ExecResult::Gas(_) => println!("Ran out of gas!"),
+					}
+				},
+				Commands::Execute { code } => {
+					let systems = init_systems(&args.system, &extensions).await.unwrap();
+					let lexemes = lex(i.i(code.trim()).await, &systems, ctx).await.unwrap();
+					if args.verbose {
+						println!("lexed: {}", take_first(&ttv_fmt(&lexemes, &FmtCtxImpl { i }).await, true));
+					}
+					let mtreev = parse_mtree(
+						Snippet::new(&lexemes[0], &lexemes, i),
+						Substack::Bottom.push(i.i("orcx").await).push(i.i("input").await),
+					)
+					.await
+					.unwrap();
+					if args.verbose {
+						let fmt = mtreev_fmt(&mtreev, &FmtCtxImpl { i }).await;
+						println!("parsed: {}", take_first(&fmt, true));
+					}
+					let expr = mtreev_to_expr(&mtreev, Substack::Bottom, ctx).await;
+					let mut xctx = ExecCtx::new(ctx.clone(), logger.clone(), expr.at(Pos::None)).await;
+					xctx.set_gas(Some(1000));
+					xctx.execute().await;
+					match xctx.result() {
+						ExecResult::Value(val) =>
+							println!("{}", take_first(&val.print(&FmtCtxImpl { i }).await, false)),
+						ExecResult::Err(e) => println!("error: {e}"),
+						ExecResult::Gas(_) => println!("Ran out of gas!"),
 					}
 				},
 			}

xtask/src/main.rs

@@ -1,9 +1,9 @@
 mod check_api_refs;
 mod orcx;
 
-use std::io;
 use std::process::ExitCode;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::{env, io};
 
 use check_api_refs::check_api_refs;
 use clap::{Parser, Subcommand};
@@ -29,6 +29,9 @@ pub enum Commands {
 pub static EXIT_OK: AtomicBool = AtomicBool::new(true);
 
 fn main() -> io::Result<ExitCode> {
+	if let Some(root) = env::var_os("CARGO_WORKSPACE_DIR") {
+		env::set_current_dir(root)?;
+	}
 	let args = Args::parse();
 	match &args.command {
 		Commands::CheckApiRefs => check_api_refs(&args)?,