it builds now

SWAP.md

@@ -1,21 +1,10 @@
-Decide whether we need patterns at runtime. Maybe macros aren't obligated to return MacTree so destructuring can be done in a safer and easier way?
+Consider placeholders as atoms produced by custom lexer. New prio syntax will be necessary to avoid clashing with potential future type syntax. Placeholders in this new format will be followed by an arbitrary tree, which patterns can read as a name and templates can read as interpolation
 
-Double-check type and templating logic in the note, it's a bit fishy.
+$name
+$1:name
+..$name
+..$1:name
+...$name
+...$1:name
 
-Consider whether all macros need to be loaded or the const references could be used to pre-filter for a given let line.
-
-## alternate extension mechanism
-
-The STD system will have a lot of traffic for trivial operations like algebra, stream IO will likely not be fast enough. A faster system is in order.
-
-Ideally, it should reuse `orchid-extension` for message routing and decoding.
-
-`orchid-host` accepts extensions as `impl ExtensionPort`
-
-## Preprocessor extension
-
-The macro system will not be privileged, it can take control from the interpreter via a custom top-level "let" line type.
-
-Must actually write macro system as recorded in note
-
-Check if any of this needs interpreter, if so, start with that
+Lexer needs to lex prio still so that we can read past it, but it already can do that. 

orchid-api/src/parser.rs

@@ -5,15 +5,12 @@ use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::{
-	Expression, ExtHostReq, HostExtReq, OrcResult, SourceRange, SysId, TStr, TStrv, TokenTree,
+	ExtHostReq, HostExtReq, OrcResult, ReflexiveTree, SourceRange, SysId, TStr, TStrv, TokenTree,
 };
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ParsId(pub NonZeroU64);
 
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-pub struct ParsedConstId(pub NonZeroU64);
-
 /// Parse a single source line. Return values can be modules, constants, or
 /// token sequences for re-parsing. These re-parsed token sequences can also
 /// represent raw language items such as modules, imports, and const. This is
@@ -59,18 +56,32 @@ pub struct ParsedMember {
 
 #[derive(Clone, Debug, Coding)]
 pub enum ParsedMemberKind {
-	Constant(ParsedConstId),
+	Constant(Vec<TokenTree>),
 	Module { lines: Vec<ParsedLine>, use_prelude: bool },
 }
 
-/// Obtain the value of a parsed constant. This is guaranteed to be called after
-/// the last [ParseLine] but before any [crate::AtomReq]. As such, in principle
-/// the macro engine could run here.
+/// Read context dependent syntax. This request is never sent before a
+/// [crate::GetMember] or [crate::ParseLine]. It may be called multiple times,
+/// and after its last call [ReflexiveDrop] will be sent.
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
-pub struct FetchParsedConst(pub SysId, pub ParsedConstId);
-impl Request for FetchParsedConst {
-	type Response = Expression;
+pub struct ReflexiveRead {
+	pub tree: ReflexiveTree,
+	pub path: TStrv,
+}
+impl Request for ReflexiveRead {
+	type Response = TokenTree;
+}
+
+/// Invalidates a [crate::ReflexiveTreeId] to reclaim the associated resources
+/// and indicate the end of the lowering phase. This is always sent for every
+/// [crate::ReflexiveTreeId] defined by any extension, and always before any
+/// [crate::AtomReq]
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct ReflexiveDrop(pub ReflexiveTree);
+impl Request for ReflexiveDrop {
+	type Response = ();
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -80,23 +91,23 @@ pub struct Comment {
 }
 
 /// Resolve relative names from the perspective of a constant. This can only be
-/// called during a [FetchParsedConst] call, but it can be called for a
-/// different [ParsedConstId] from the one in [FetchParsedConst].
+/// called for a given path after a [LazyRead] is called and before [LazyDrop]
+/// returns.
 ///
-/// Each name is either resolved to a valid name or a potential error error.
-/// The error is not raised by the interpreter itself, as names may have a
-/// primary meaning such as a local binding which can be overridden by specific
-/// true names such as those triggering macro keywords. It is not recommended to
-/// define syntax that can break by defining arbitrary constants, as line
-/// parsers can define new ones at will.
+/// The name is either resolved to a valid name or a potential error. The error
+/// is not raised by the interpreter itself, as names may have a primary meaning
+/// such as a local binding which can be overridden by specific true names such
+/// as those triggering macro keywords. It is not recommended to define syntax
+/// that can break by defining arbitrary constants, as line parsers can define
+/// new ones at will.
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
-pub struct ResolveNames {
+pub struct ResolveName {
 	pub sys: SysId,
-	pub constid: ParsedConstId,
-	pub names: Vec<TStrv>,
+	pub path: TStrv,
+	pub name: TStrv,
 }
 
-impl Request for ResolveNames {
-	type Response = Vec<OrcResult<TStrv>>;
+impl Request for ResolveName {
+	type Response = OrcResult<TStrv>;
 }

orchid-api/src/proto.rs

@@ -87,7 +87,7 @@ pub enum ExtHostReq {
 	ExprReq(expr::ExprReq),
 	SubLex(lexer::SubLex),
 	LsModule(tree::LsModule),
-	ResolveNames(parser::ResolveNames),
+	ResolveName(parser::ResolveName),
 }
 
 /// Notifications sent from the extension to the host
@@ -117,7 +117,8 @@ pub enum HostExtReq {
 	DeserAtom(atom::DeserAtom),
 	LexExpr(lexer::LexExpr),
 	ParseLine(parser::ParseLine),
-	FetchParsedConst(parser::FetchParsedConst),
+	ReflexiveRead(parser::ReflexiveRead),
+	ReflexiveDrop(parser::ReflexiveDrop),
 	GetMember(tree::GetMember),
 	SystemDrop(system::SystemDrop),
 	AtomDrop(atom::AtomDrop),

orchid-api/src/tree.rs

@@ -18,6 +18,18 @@ use crate::{ExprTicket, Expression, ExtHostReq, HostExtReq, OrcError, SysId, TSt
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct TreeTicket(pub NonZeroU64);
 
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct ReflexiveTreeId(pub NonZeroU64);
+
+/// These are constructed by extensions ad-hoc and referenced into a token tree.
+/// Since extensions never own, only ever borrow a token tree, they can borrow
+/// and return lazytree without any resource management concern.
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct ReflexiveTree {
+	pub sys: SysId,
+	pub id: ReflexiveTreeId,
+}
+
 #[derive(Clone, Debug, Coding)]
 pub struct TokenTree {
 	pub token: Token,
@@ -48,6 +60,13 @@ pub enum Token {
 	Bottom(Vec<OrcError>),
 	/// A comment
 	Comment(TStr),
+	/// Constructed by systems in place of syntax that has custom place-sensitive
+	/// behaviour, i.e. atoms that must resolve local names to determine their own
+	/// value. Parsers should transparently forward these if found in input. By
+	/// lowering time they should be in a constant, otherwise a parser error is
+	/// raised. These should not be able to evaluate to `name` or `(name ...)` as
+	/// that may generate confusing parser errors.
+	Reflexive(ReflexiveTree),
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]

orchid-base/src/error.rs

@@ -11,7 +11,7 @@ use futures::future::join_all;
 use itertools::Itertools;
 use task_local::task_local;
 
-use crate::{IStr, Pos, api, es, is};
+use crate::{IStr, Pos, SrcRange, api, es, is};
 
 /// A point of interest in resolving the error, such as the point where
 /// processing got stuck, a command that is likely to be incorrect
@@ -42,6 +42,9 @@ impl ErrPos {
 		}
 	}
 }
+impl From<SrcRange> for ErrPos {
+	fn from(value: SrcRange) -> Self { Self { position: value.pos(), message: None } }
+}
 impl From<Pos> for ErrPos {
 	fn from(origin: Pos) -> Self { Self { position: origin, message: None } }
 }

orchid-base/src/format.rs

@@ -316,27 +316,31 @@ pub fn take_first(unit: &FmtUnit, bounded: bool) -> String {
 }
 
 pub async fn take_first_fmt(v: &(impl Format + ?Sized)) -> String {
-	take_first(&v.print(&FmtCtxImpl { _foo: PhantomData }).await, false)
+	take_first(&v.print(&FmtCxImpl { _foo: PhantomData }).await, false)
 }
 
 /// [Default] this if you need one
 #[derive(Default)]
-pub struct FmtCtxImpl<'a> {
+pub struct FmtCxImpl<'a> {
 	_foo: PhantomData<&'a ()>,
 }
 
-/// Additional settings to the formatter. Implemented by [FmtCtxImpl]. Currently
+/// Additional settings to the formatter. Implemented by [FmtCxImpl]. Currently
 /// not in use
-pub trait FmtCtx {}
-impl FmtCtx for FmtCtxImpl<'_> {}
+pub trait FmtCx {
+	fn as_dyn(&self) -> &dyn FmtCx;
+}
+impl FmtCx for FmtCxImpl<'_> {
+	fn as_dyn(&self) -> &dyn FmtCx { self }
+}
 
 /// A value that can be formatted into a string with multiple possible forms
 pub trait Format {
 	#[must_use]
-	fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> + 'a;
+	fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> + 'a;
 }
 impl Format for Never {
-	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit { match *self {} }
+	async fn print<'a>(&'a self, _c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit { match *self {} }
 }
 
 /// Format with default strategy. Currently equal to [take_first_fmt]

orchid-base/src/location.rs

@@ -7,7 +7,7 @@ use std::ops::{Add, AddAssign, Range};
 use futures::future::join_all;
 use trait_set::trait_set;
 
-use crate::{ErrPos, IStr, IteratorPrint, Sym, api, es, is, match_mapping, sym};
+use crate::{IStr, IteratorPrint, Sym, api, es, is, match_mapping, sym};
 
 trait_set! {
 	pub trait GetSrc = FnMut(&Sym) -> IStr;
@@ -104,6 +104,9 @@ impl AddAssign for Pos {
 		*self = tmp + rhs;
 	}
 }
+impl From<SrcRange> for Pos {
+	fn from(value: SrcRange) -> Self { Pos::SrcRange(value) }
+}
 
 /// Exact source code location. Includes where the code was loaded from, what
 /// the original source code was, and a byte range.
@@ -148,6 +151,10 @@ impl SrcRange {
 	}
 	/// Zero-width position at a certain offset
 	pub fn zw(path: Sym, pos: u32) -> Self { Self { path, range: pos..pos } }
+	/// A zero-width position at the start of this position
+	pub fn prefix(&self) -> Self { Self::zw(self.path(), self.start()) }
+	/// A zero-width position at the end of this position
+	pub fn suffix(&self) -> Self { Self::zw(self.path(), self.end()) }
 	/// Deserialize from a message
 	pub async fn from_api(api: &api::SourceRange) -> Self {
 		Self { path: Sym::from_api(api.path).await, range: api.range.clone() }
@@ -166,9 +173,6 @@ impl SrcRange {
 		Self { path: self.path(), range: self.start().min(rhs.start())..self.end().max(rhs.end()) }
 	}
 }
-impl From<SrcRange> for ErrPos {
-	fn from(val: SrcRange) -> Self { val.pos().into() }
-}
 impl fmt::Display for SrcRange {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 		match self.range.len() {

orchid-base/src/name.rs

@@ -268,7 +268,7 @@ pub trait NameLike:
 	}
 	/// Like slice's `split_first` except we know that it always returns Some
 	fn split_first_seg(&self) -> (IStr, &[IStr]) {
-		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
+		let (foot, torso) = self.as_slice().split_first().expect("NameLike never empty");
 		(foot.clone(), torso)
 	}
 	/// Like slice's `split_last` except we know that it always returns Some

orchid-base/src/parse.rs

@@ -1,5 +1,5 @@
 use std::fmt::{self, Display};
-use std::iter;
+use std::iter::{self, FusedIterator};
 use std::ops::Deref;
 
 use futures::FutureExt;
@@ -7,8 +7,8 @@ use futures::future::join_all;
 use itertools::Itertools;
 
 use crate::{
-	ExprRepr, ExtraTok, FmtCtx, FmtUnit, Format, IStr, OrcErrv, OrcRes, Paren, SrcRange, Sym,
-	TokTree, Token, VName, VPath, api, es, fmt, is, mk_errv, report, ttv_fmt, ttv_range,
+	FmtCx, FmtUnit, Format, IStr, ManagedStrat, OrcErrv, OrcRes, Paren, SrcRange, Sym, TokTree,
+	Token, VName, VPath, api, es, fmt, is, mk_errv, report, ttv_fmt, ttv_range,
 };
 
 /// A character that can appear at the start of a name; `[a-zA-Z_]`
@@ -24,18 +24,19 @@ pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
 /// A cheaply copiable subsection of a document that holds onto context data and
 /// one token for error reporting on empty subsections.
 #[derive(Debug)]
-pub struct Snippet<'a, A: ExprRepr, X: ExtraTok> {
-	prev: &'a TokTree<A, X>,
-	cur: &'a [TokTree<A, X>],
+pub struct Snippet<'a, M: ManagedStrat> {
+	prev: &'a TokTree<M>,
+	cur: &'a [TokTree<M>],
 }
-impl<'a, A, X> Snippet<'a, A, X>
-where
-	A: ExprRepr,
-	X: ExtraTok,
-{
+impl<'a, M: ManagedStrat> Snippet<'a, M> {
+	pub fn iter<'b>(
+		&'b self,
+	) -> impl ExactSizeIterator<Item = &'a TokTree<M>> + DoubleEndedIterator + FusedIterator + 'b {
+		self.cur.iter()
+	}
 	/// Create a snippet from a fallback token for position tracking and a range
 	/// of tokens
-	pub fn new(prev: &'a TokTree<A, X>, cur: &'a [TokTree<A, X>]) -> Self { Self { prev, cur } }
+	pub fn new(prev: &'a TokTree<M>, cur: &'a [TokTree<M>]) -> Self { Self { prev, cur } }
 	/// Split snippet at index
 	pub fn split_at(self, pos: u32) -> (Self, Self) {
 		let Self { prev, cur } = self;
@@ -44,36 +45,44 @@ where
 		let snd = Self { prev: new_prev, cur: &self.cur[pos as usize..] };
 		(fst, snd)
 	}
+	/// Splits a sequence at the outermost (first) lambda
+	pub fn split_lambda(self) -> Option<(Self, &'a TokTree<M>, SrcRange, Self)> {
+		let (i, arg) =
+			self.cur.iter().enumerate().find_map(|(i, tt)| tt.as_lambda().map(|arg| (i, arg)))?;
+		let (prefix, lambda_ex) = self.split_at(i as u32);
+		let (_, body) = lambda_ex.split_first().expect("i is inside the slice");
+		Some((prefix, arg, lambda_ex.sr(), body))
+	}
 	/// Find the first index that matches a condition
-	pub fn find_idx(self, mut f: impl FnMut(&Token<A, X>) -> bool) -> Option<u32> {
+	pub fn find_idx(self, mut f: impl FnMut(&Token<M>) -> bool) -> Option<u32> {
 		self.cur.iter().position(|t| f(&t.tok)).map(|t| t as u32)
 	}
 	/// Get the n-th token
-	pub fn get(self, idx: u32) -> Option<&'a TokTree<A, X>> { self.cur.get(idx as usize) }
+	pub fn get(self, idx: u32) -> Option<&'a TokTree<M>> { self.cur.get(idx as usize) }
 	/// Count how many tokens long the current sequence is. Parenthesized
 	/// subsequences count as 1
 	pub fn len(self) -> u32 { self.cur.len() as u32 }
 	/// The fallback token that can be used for error reporting if this snippet is
 	/// unexpectedly empty
-	pub fn prev(self) -> &'a TokTree<A, X> { self.prev }
+	pub fn prev(self) -> &'a TokTree<M> { self.prev }
 	/// Create a position that describes all tokens in this snippet
 	pub fn sr(self) -> SrcRange { ttv_range(self.cur).unwrap_or_else(|| self.prev.sr.clone()) }
 	/// Split the first token
-	pub fn split_first(self) -> Option<(&'a TokTree<A, X>, Self)> {
+	pub fn split_first(self) -> Option<(&'a TokTree<M>, Self)> {
 		self.cur.first().map(|r| (r, self.split_at(1).1))
 	}
 	/// Split the last token
-	pub fn split_last(self) -> Option<(Self, &'a TokTree<A, X>)> {
+	pub fn split_last(self) -> Option<(Self, &'a TokTree<M>)> {
 		self.cur.last().map(|r| (self.split_at(self.len() - 1).0, r))
 	}
 	/// Split the snippet at the first token that matches the predicate
-	pub fn split_once(self, f: impl FnMut(&Token<A, X>) -> bool) -> Option<(Self, Self)> {
+	pub fn split_once(self, f: impl FnMut(&Token<M>) -> bool) -> Option<(Self, Self)> {
 		let idx = self.find_idx(f)?;
 		Some((self.split_at(idx).0, self.split_at(idx + 1).1))
 	}
 	/// Split the snippet at each occurrence of a delimiter matched by the
 	/// predicate
-	pub fn split(mut self, mut f: impl FnMut(&Token<A, X>) -> bool) -> impl Iterator<Item = Self> {
+	pub fn split(mut self, mut f: impl FnMut(&Token<M>) -> bool) -> impl Iterator<Item = Self> {
 		iter::from_fn(move || {
 			if self.is_empty() {
 				return None;
@@ -92,16 +101,16 @@ where
 		self.split_at(non_fluff_start.unwrap_or(self.len())).1
 	}
 }
-impl<A: ExprRepr, X: ExtraTok> Copy for Snippet<'_, A, X> {}
-impl<A: ExprRepr, X: ExtraTok> Clone for Snippet<'_, A, X> {
+impl<M: ManagedStrat> Copy for Snippet<'_, M> {}
+impl<M: ManagedStrat> Clone for Snippet<'_, M> {
 	fn clone(&self) -> Self { *self }
 }
-impl<A: ExprRepr, X: ExtraTok> Deref for Snippet<'_, A, X> {
-	type Target = [TokTree<A, X>];
+impl<M: ManagedStrat> Deref for Snippet<'_, M> {
+	type Target = [TokTree<M>];
 	fn deref(&self) -> &Self::Target { self.cur }
 }
-impl<A: ExprRepr, X: ExtraTok> Format for Snippet<'_, A, X> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+impl<M: ManagedStrat> Format for Snippet<'_, M> {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		ttv_fmt(&**self, c).await
 	}
 }
@@ -116,7 +125,7 @@ impl Comment {
 	pub async fn from_api(c: &api::Comment, src: Sym) -> Self {
 		Self { text: es(c.text).await, sr: SrcRange::new(c.range.clone(), &src) }
 	}
-	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>) -> Option<Self> {
+	pub async fn from_tk(tk: &TokTree<impl ManagedStrat>) -> Option<Self> {
 		match &tk.tok {
 			Token::Comment(text) => Some(Self { text: text.clone(), sr: tk.sr.clone() }),
 			_ => None,
@@ -140,9 +149,9 @@ impl fmt::Display for Comment {
 /// line separately
 ///
 /// This is the procedure for module parsing
-pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, A, X>,
-) -> Vec<Parsed<'a, Vec<Comment>, A, X>> {
+pub async fn line_items<'a, M: ManagedStrat>(
+	snip: Snippet<'a, M>,
+) -> Vec<Parsed<'a, Vec<Comment>, M>> {
 	let mut items = Vec::new();
 	let mut comments = Vec::new();
 	for mut line in snip.split(|t| matches!(t, Token::BR)) {
@@ -168,9 +177,9 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
 }
 
 /// Pop the next token that isn't a comment or line break
-pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, A, X>,
-) -> ParseRes<'a, &'a TokTree<A, X>, A, X> {
+pub async fn try_pop_no_fluff<'a, M: ManagedStrat>(
+	snip: Snippet<'a, M>,
+) -> ParseRes<'a, &'a TokTree<M>, M> {
 	match snip.skip_fluff().split_first() {
 		Some((output, tail)) => Ok(Parsed { output, tail }),
 		None =>
@@ -181,7 +190,7 @@ pub async fn try_pop_no_fluff<'a, A: ExprRepr, X: ExtraTok>(
 }
 
 /// Fail if the snippet isn't empty
-pub async fn expect_end(snip: Snippet<'_, impl ExprRepr, impl ExtraTok>) -> OrcRes<()> {
+pub async fn expect_end(snip: Snippet<'_, impl ManagedStrat>) -> OrcRes<()> {
 	match snip.skip_fluff().get(0) {
 		Some(surplus) => Err(mk_errv(
 			is("Extra code after end of line").await,
@@ -193,10 +202,10 @@ pub async fn expect_end(snip: Snippet<'_, impl ExprRepr, impl ExtraTok>) -> OrcR
 }
 
 /// Read a token and ensure that it matches the specified keyword
-pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
-	snip: Snippet<'a, A, X>,
+pub async fn expect_tok<'a, M: ManagedStrat>(
+	snip: Snippet<'a, M>,
 	tok: IStr,
-) -> ParseRes<'a, (), A, X> {
+) -> ParseRes<'a, (), M> {
 	let Parsed { output: head, tail } = try_pop_no_fluff(snip).await?;
 	match &head.tok {
 		Token::Name(n) if *n == tok => Ok(Parsed { output: (), tail }),
@@ -210,28 +219,28 @@ pub async fn expect_tok<'a, A: ExprRepr, X: ExtraTok>(
 
 /// Report an error related to a token that can conveniently use the token's
 /// text representation in the long message
-pub async fn token_errv<A: ExprRepr, X: ExtraTok>(
-	tok: &TokTree<A, X>,
+pub async fn token_errv<Strat: ManagedStrat, Msg: AsRef<str>>(
+	tok: &TokTree<Strat>,
 	description: &'static str,
-	message: impl FnOnce(&str) -> String,
+	message: impl FnOnce(&str) -> Msg,
 ) -> OrcErrv {
 	mk_errv(is(description).await, message(&fmt(tok).await), [tok.sr.pos()])
 }
 
 /// Success output of parsers
-pub struct Parsed<'a, T, H: ExprRepr, X: ExtraTok> {
+pub struct Parsed<'a, T, M: ManagedStrat> {
 	/// Information obtained from consumed tokens
 	pub output: T,
 	/// Input to next parser
-	pub tail: Snippet<'a, H, X>,
+	pub tail: Snippet<'a, M>,
 }
 
-pub type ParseRes<'a, T, H, X> = OrcRes<Parsed<'a, T, H, X>>;
+pub type ParseRes<'a, T, M> = OrcRes<Parsed<'a, T, M>>;
 
 /// Parse a `namespaced::name` or a `namespaced::(multi name)`
-pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
-	tail: Snippet<'a, A, X>,
-) -> ParseRes<'a, Vec<Import>, A, X> {
+pub async fn parse_multiname<'a, M: ManagedStrat>(
+	tail: Snippet<'a, M>,
+) -> ParseRes<'a, Vec<Import>, M> {
 	let Some((tt, tail)) = tail.skip_fluff().split_first() else {
 		return Err(mk_errv(
 			is("Expected token").await,
@@ -241,8 +250,8 @@ pub async fn parse_multiname<'a, A: ExprRepr, X: ExtraTok>(
 	};
 	let ret = rec(tt).await;
 	#[allow(clippy::type_complexity)] // it's an internal function
-	pub async fn rec<A: ExprRepr, X: ExtraTok>(
-		tt: &TokTree<A, X>,
+	pub async fn rec<M: ManagedStrat>(
+		tt: &TokTree<M>,
 	) -> OrcRes<Vec<(Vec<IStr>, Option<IStr>, SrcRange)>> {
 		let ttpos = tt.sr.pos();
 		match &tt.tok {
@@ -318,5 +327,5 @@ mod test {
 
 	use super::Snippet;
 
-	fn _covary_snip_a<'a>(x: Snippet<'static, Never, Never>) -> Snippet<'a, Never, Never> { x }
+	fn _covary_snip_a<'a>(x: Snippet<'static, Never>) -> Snippet<'a, Never> { x }
 }

orchid-base/src/tree.rs

@@ -8,11 +8,12 @@ use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
 use never::Never;
+use orchid_api::ReflexiveTree;
 use orchid_api_traits::Coding;
 use trait_set::trait_set;
 
 use crate::{
-	FmtCtx, FmtUnit, Format, IStr, OrcErrv, Pos, Snippet, SrcRange, Sym, VName, VPath, Variants, api,
+	FmtCx, FmtUnit, Format, IStr, OrcErrv, Pos, Snippet, SrcRange, Sym, VName, VPath, Variants, api,
 	es, match_mapping, tl_cache,
 };
 
@@ -25,64 +26,71 @@ pub type Paren = api::Paren;
 pub const PARENS: &[(char, char, Paren)] =
 	&[('(', ')', Paren::Round), ('[', ']', Paren::Square), ('{', '}', Paren::Curly)];
 
-/// Extension interface for embedded expressions and expression construction
-/// commands inside token trees
+/// Extension interface for complexly serialized tokens. If you set either cx
+/// type to [Never], a tree that mentions your token can never be converted in
+/// that direction even if the tree happens not to contain such a token.
 pub trait TokenVariant<ApiEquiv: Clone + Debug + Coding>: Format + Clone + fmt::Debug {
 	/// Additional arguments to the deserializer. If deserialization of a token
-	/// type is impossible, set this to a sentinel unit type that describes why.
-	/// If you set this to [Never], your token tree type can never be
-	/// deserialized.
-	type FromApiCtx<'a>;
+	/// type is impossible, set this to a sentinel unit type that describes why
+	/// and panic in the corresponding function.
+	type FromApiCx<'a>;
 	/// Additional arguments to the serializer. If serialization of a token type
 	/// is forbidden, set this to a sentinel unit type that describes how to avoid
 	/// it.
-	/// If you set this to [Never], your token tree type can never be serialized.
-	type ToApiCtx<'a>;
+	type ToApiCx<'a>;
 	/// Deserializer
 	#[must_use]
 	fn from_api(
 		api: ApiEquiv,
-		ctx: &mut Self::FromApiCtx<'_>,
+		ctx: &mut Self::FromApiCx<'_>,
 		pos: SrcRange,
 	) -> impl Future<Output = Self>;
 	/// Serializer
 	#[must_use]
-	fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> impl Future<Output = ApiEquiv>;
+	fn into_api(self, ctx: &mut Self::ToApiCx<'_>) -> impl Future<Output = ApiEquiv>;
 }
 impl<T: Clone + Debug + Coding> TokenVariant<T> for Never {
-	type FromApiCtx<'a> = ();
-	type ToApiCtx<'a> = ();
-	async fn from_api(_: T, _: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
+	type FromApiCx<'a> = ();
+	type ToApiCx<'a> = ();
+	async fn from_api(_: T, _: &mut Self::FromApiCx<'_>, _: SrcRange) -> Self {
 		panic!("Cannot deserialize Never")
 	}
-	async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> T { match self {} }
+	async fn into_api(self, _: &mut Self::ToApiCx<'_>) -> T { match self {} }
 }
 
-trait_set! {
+pub trait ManagedStrat {
 	/// [api::Token::Handle] variant
-	pub trait ExprRepr = TokenVariant<api::ExprTicket>;
+	type Expr: TokenVariant<api::ExprTicket>;
 	/// [api::Token::NewExpr] variant
-	pub trait ExtraTok = TokenVariant<api::Expression>;
+	type Extra: TokenVariant<api::Expression>;
+	/// [api::Token::Reflexive] variant
+	type Flex: TokenVariant<api::ReflexiveTree>;
+}
+
+impl ManagedStrat for Never {
+	type Expr = Never;
+	type Extra = Never;
+	type Flex = Never;
 }
 
 trait_set! {
 	/// Callback to callback to [recur].
-	pub trait RecurCB<'a, H: ExprRepr, X: ExtraTok> =
-		Fn(TokTree<H, X>) -> LocalBoxFuture<'a, TokTree<H, X>>;
+	pub trait RecurCB<'a, M: ManagedStrat> =
+		Fn(TokTree<M>) -> LocalBoxFuture<'a, TokTree<M>>;
 }
 
 /// An atom that can be passed through the API boundary as part of an
 /// expression. In particular, atoms created by extensions use this form.
 pub trait AtomRepr: Clone + Format {
-	type Ctx: ?Sized;
+	type Cx: ?Sized;
 	#[must_use]
-	fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> impl Future<Output = Self>;
+	fn from_api(api: &api::Atom, pos: Pos, ctx: &mut Self::Cx) -> impl Future<Output = Self>;
 	#[must_use]
 	fn to_api(&self) -> impl Future<Output = orchid_api::Atom> + '_;
 }
 impl AtomRepr for Never {
-	type Ctx = Never;
-	async fn from_api(_: &api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self { match *ctx {} }
+	type Cx = Never;
+	async fn from_api(_: &api::Atom, _: Pos, ctx: &mut Self::Cx) -> Self { match *ctx {} }
 	async fn to_api(&self) -> orchid_api::Atom { match *self {} }
 }
 
@@ -98,24 +106,40 @@ impl Display for TokHandle<'_> {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Handle({})", self.0.0) }
 }
 
+/// Individual contexts used for serializing the various custom leaf types
+pub struct TokToApiCx<'a, M: ManagedStrat> {
+	pub handle: &'a mut <M::Expr as TokenVariant<api::ExprTicket>>::ToApiCx<'a>,
+	pub extra: &'a mut <M::Extra as TokenVariant<api::Expression>>::ToApiCx<'a>,
+	pub lazy: &'a mut <M::Flex as TokenVariant<ReflexiveTree>>::ToApiCx<'a>,
+}
+
+/// Individual contexts used for deserializing the various custom leaf types
+pub struct TokFromApiCx<'a, M: ManagedStrat> {
+	/// Path to the source file the token is in
+	pub src: Sym,
+	pub handle: &'a mut <M::Expr as TokenVariant<api::ExprTicket>>::FromApiCx<'a>,
+	pub extra: &'a mut <M::Extra as TokenVariant<api::Expression>>::FromApiCx<'a>,
+	pub lazy: &'a mut <M::Flex as TokenVariant<ReflexiveTree>>::FromApiCx<'a>,
+}
+
 /// Lexer output
 #[derive(Clone, Debug)]
-pub struct TokTree<H: ExprRepr, X: ExtraTok> {
-	pub tok: Token<H, X>,
+pub struct TokTree<M: ManagedStrat> {
+	pub tok: Token<M>,
 	/// The protocol has a Range<u32> because these are always transmitted in the
 	/// context of a given snippet, but internal logic and error reporting is
 	/// easier if the in-memory representation also includes the snippet path.
 	pub sr: SrcRange,
 }
-impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
+impl<M: ManagedStrat> TokTree<M> {
 	/// Visit all tokens, modify them at will, and optionally recurse into them by
 	/// calling the callback passed to your callback
-	pub async fn recur(self, f: &impl AsyncFn(Self, &dyn RecurCB<'_, H, X>) -> Self) -> Self {
+	pub async fn recur(self, f: &impl AsyncFn(Self, &dyn RecurCB<'_, M>) -> Self) -> Self {
 		f(self, &|TokTree { sr: range, tok }| {
 			Box::pin(async {
 				let tok = match tok {
 					tok @ (Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::Name(_)) => tok,
-					tok @ (Token::Handle(_) | Token::NewExpr(_)) => tok,
+					tok @ (Token::Handle(_) | Token::NewExpr(_) | Token::Reflexive(_)) => tok,
 					Token::NS(n, b) => Token::NS(n, Box::new(b.recur(f).await)),
 					Token::LambdaHead(arg) => Token::LambdaHead(Box::new(arg.recur(f).await)),
 					Token::S(p, b) => Token::S(p, join_all(b.into_iter().map(|tt| tt.recur(f))).await),
@@ -126,43 +150,36 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 		.await
 	}
 
-	pub async fn from_api(
-		tt: api::TokenTree,
-		hctx: &mut H::FromApiCtx<'_>,
-		xctx: &mut X::FromApiCtx<'_>,
-		src: &Sym,
-	) -> Self {
-		let pos = SrcRange::new(tt.range, src);
-		let tok = match_mapping!(tt.token, api::Token => Token::<H, X> {
+	pub async fn from_api(tt: api::TokenTree, cx: &mut TokFromApiCx<'_, M>) -> Self {
+		let pos = SrcRange::new(tt.range, &cx.src);
+		let tok = match_mapping!(tt.token, api::Token => Token::<M> {
 			BR,
 			NS(n => es(n).await,
-				b => Box::new(Self::from_api(*b, hctx, xctx, src).boxed_local().await)),
+				b => Box::new(Self::from_api(*b, cx).boxed_local().await)),
 			Bottom(e => OrcErrv::from_api(e).await),
-			LambdaHead(arg => Box::new(Self::from_api(*arg, hctx, xctx, src).boxed_local().await)),
+			LambdaHead(arg => Box::new(Self::from_api(*arg, cx).boxed_local().await)),
 			Name(n => es(n).await),
-			S(par, b => ttv_from_api(b, hctx, xctx, src).await),
+			S(par, b => ttv_from_api(b, cx).await),
 			Comment(c => es(c).await),
-			NewExpr(expr => X::from_api(expr, xctx, pos.clone()).await),
-			Handle(tk => H::from_api(tk, hctx, pos.clone()).await)
+			NewExpr(expr => M::Extra::from_api(expr, cx.extra, pos.clone()).await),
+			Handle(tk => M::Expr::from_api(tk, cx.handle, pos.clone()).await),
+			Reflexive(l => M::Flex::from_api(l, cx.lazy, pos.clone()).await)
 		});
 		Self { sr: pos, tok }
 	}
 
-	pub async fn into_api(
-		self,
-		hctx: &mut H::ToApiCtx<'_>,
-		xctx: &mut X::ToApiCtx<'_>,
-	) -> api::TokenTree {
+	pub async fn into_api(self, cx: &mut TokToApiCx<'_, M>) -> api::TokenTree {
 		let token = match_mapping!(self.tok, Token => api::Token {
 			BR,
-			NS(n.to_api(), b => Box::new(b.into_api(hctx, xctx).boxed_local().await)),
+			NS(n.to_api(), b => Box::new(b.into_api(cx).boxed_local().await)),
 			Bottom(e.to_api()),
 			Comment(c.to_api()),
-			LambdaHead(arg => Box::new(arg.into_api(hctx, xctx).boxed_local().await)),
+			LambdaHead(arg => Box::new(arg.into_api(cx).boxed_local().await)),
 			Name(nn.to_api()),
-			S(p, b => ttv_into_api(b, hctx, xctx).boxed_local().await),
-			Handle(hand.into_api(hctx).await),
-			NewExpr(expr.into_api(xctx).await),
+			S(p, b => ttv_into_api(b, cx).boxed_local().await),
+			Handle(hand.into_api(cx.handle).await),
+			NewExpr(expr.into_api(cx.extra).await),
+			Reflexive(lazy.into_api(cx.lazy).await),
 		});
 		api::TokenTree { range: self.sr.range.clone(), token }
 	}
@@ -171,7 +188,7 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 	pub fn as_name(&self) -> Option<IStr> {
 		if let Token::Name(n) = &self.tok { Some(n.clone()) } else { None }
 	}
-	pub fn as_multiname(&self) -> Result<VName, &TokTree<H, X>> {
+	pub fn as_multiname(&self) -> Result<VName, &TokTree<M>> {
 		let mut segs = VPath::new([]);
 		let mut cur = self;
 		loop {
@@ -185,7 +202,7 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 			}
 		}
 	}
-	pub fn as_s(&self, par: Paren) -> Option<Snippet<'_, H, X>> {
+	pub fn as_s(&self, par: Paren) -> Option<Snippet<'_, M>> {
 		self.tok.as_s(par).map(|slc| Snippet::new(self, slc))
 	}
 	pub fn as_lambda(&self) -> Option<&Self> {
@@ -204,36 +221,33 @@ impl<H: ExprRepr, X: ExtraTok> TokTree<H, X> {
 	}
 	pub fn sr(&self) -> SrcRange { self.sr.clone() }
 }
-impl<H: ExprRepr, X: ExtraTok> Format for TokTree<H, X> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+impl<M: ManagedStrat> Format for TokTree<M> {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		self.tok.print(c).await
 	}
 }
 
 /// Receive a token sequence from API
-pub async fn ttv_from_api<H: ExprRepr, X: ExtraTok>(
+pub async fn ttv_from_api<M: ManagedStrat>(
 	tokv: impl IntoIterator<Item = api::TokenTree>,
-	hctx: &mut H::FromApiCtx<'_>,
-	xctx: &mut X::FromApiCtx<'_>,
-	src: &Sym,
-) -> Vec<TokTree<H, X>> {
-	stream(async |mut cx| {
+	cx: &mut TokFromApiCx<'_, M>,
+) -> Vec<TokTree<M>> {
+	stream(async |mut hand| {
 		for tok in tokv {
-			cx.emit(TokTree::<H, X>::from_api(tok, hctx, xctx, src).boxed_local().await).await
+			hand.emit(TokTree::<M>::from_api(tok, cx).boxed_local().await).await
 		}
 	})
 	.collect()
 	.await
 }
 /// Encode a token sequence for sending
-pub async fn ttv_into_api<H: ExprRepr, X: ExtraTok>(
-	tokv: impl IntoIterator<Item = TokTree<H, X>>,
-	hctx: &mut H::ToApiCtx<'_>,
-	xctx: &mut X::ToApiCtx<'_>,
+pub async fn ttv_into_api<M: ManagedStrat>(
+	tokv: impl IntoIterator<Item = TokTree<M>>,
+	cx: &mut TokToApiCx<'_, M>,
 ) -> Vec<api::TokenTree> {
-	stream(async |mut cx| {
+	stream(async |mut hand| {
 		for tok in tokv {
-			cx.emit(tok.into_api(hctx, xctx).await).await
+			hand.emit(tok.into_api(cx).await).await
 		}
 	})
 	.collect()
@@ -241,9 +255,7 @@ pub async fn ttv_into_api<H: ExprRepr, X: ExtraTok>(
 }
 
 /// Enclose the tokens in `()` if there is more than one
-pub fn wrap_tokv<H: ExprRepr, X: ExtraTok>(
-	items: impl IntoIterator<Item = TokTree<H, X>>,
-) -> TokTree<H, X> {
+pub fn wrap_tokv<M: ManagedStrat>(items: impl IntoIterator<Item = TokTree<M>>) -> TokTree<M> {
 	let items_v = items.into_iter().collect_vec();
 	match items_v.len() {
 		0 => panic!("A tokv with no elements is illegal"),
@@ -257,43 +269,44 @@ pub fn wrap_tokv<H: ExprRepr, X: ExtraTok>(
 
 /// Lexer output variant
 #[derive(Clone, Debug)]
-pub enum Token<H: ExprRepr, X: ExtraTok> {
+pub enum Token<M: ManagedStrat> {
 	/// Information about the code addressed to the human reader or dev tooling
 	/// It has no effect on the behaviour of the program unless it's explicitly
 	/// read via reflection
 	Comment(IStr),
 	/// The part of a lambda between `\` and `.` enclosing the argument. The body
 	/// stretches to the end of the enclosing parens or the end of the const line
-	LambdaHead(Box<TokTree<H, X>>),
+	LambdaHead(Box<TokTree<M>>),
 	/// A binding, operator, or a segment of a namespaced::name
 	Name(IStr),
 	/// A namespace prefix, like `my_ns::` followed by a token
-	NS(IStr, Box<TokTree<H, X>>),
+	NS(IStr, Box<TokTree<M>>),
 	/// A line break
 	BR,
 	/// `()`, `[]`, or `{}`
-	S(Paren, Vec<TokTree<H, X>>),
+	S(Paren, Vec<TokTree<M>>),
 	/// A newly instantiated expression
-	NewExpr(X),
+	NewExpr(M::Extra),
 	/// An existing expr from a nested lexer
-	Handle(H),
+	Handle(M::Expr),
 	/// A grammar error emitted by a lexer plugin if it was possible to continue
 	/// reading. Parsers should treat it as an atom unless it prevents parsing,
 	/// in which case both this and a relevant error should be returned.
 	Bottom(OrcErrv),
+	Reflexive(M::Flex),
 }
-impl<H: ExprRepr, X: ExtraTok> Token<H, X> {
-	pub fn at(self, sr: SrcRange) -> TokTree<H, X> { TokTree { sr, tok: self } }
+impl<M: ManagedStrat> Token<M> {
+	pub fn at(self, sr: SrcRange) -> TokTree<M> { TokTree { sr, tok: self } }
 	pub fn is_kw(&self, tk: IStr) -> bool { matches!(self, Token::Name(n) if *n == tk) }
-	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<H, X>]> {
+	pub fn as_s(&self, par: Paren) -> Option<&[TokTree<M>]> {
 		match self {
 			Self::S(p, b) if *p == par => Some(b),
 			_ => None,
 		}
 	}
 }
-impl<H: ExprRepr, X: ExtraTok> Format for Token<H, X> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+impl<M: ManagedStrat> Format for Token<M> {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		match self {
 			Self::BR => "\n".to_string().into(),
 			Self::Bottom(err) => match err.one() {
@@ -317,28 +330,29 @@ impl<H: ExprRepr, X: ExtraTok> Format for Token<H, X> {
 			),
 			Self::Handle(h) => h.print(c).await,
 			Self::NewExpr(ex) => ex.print(c).await,
+			Self::Reflexive(l) => l.print(c).await,
 		}
 	}
 }
 
 /// Find the location that best describes a sequence of tokens if the sequence
 /// isn't empty
-pub fn ttv_range<'a>(ttv: &[TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>]) -> Option<SrcRange> {
+pub fn ttv_range<'a>(ttv: &[TokTree<impl ManagedStrat + 'a>]) -> Option<SrcRange> {
 	let range = ttv.first()?.sr.range.start..ttv.last().unwrap().sr.range.end;
 	Some(SrcRange { path: ttv.first().unwrap().sr.path(), range })
 }
 
 /// Pretty-print a token sequence
 pub async fn ttv_fmt<'a: 'b, 'b>(
-	ttv: impl IntoIterator<Item = &'b TokTree<impl ExprRepr + 'a, impl ExtraTok + 'a>>,
-	c: &(impl FmtCtx + ?Sized),
+	ttv: impl IntoIterator<Item = &'b TokTree<impl ManagedStrat + 'a>>,
+	c: &(impl FmtCx + ?Sized),
 ) -> FmtUnit {
 	FmtUnit::sequence("", " ", "", true, join_all(ttv.into_iter().map(|t| t.print(c))).await)
 }
 
-pub struct FmtTTV<'a, H: ExprRepr, X: ExtraTok>(pub &'a [TokTree<H, X>]);
-impl<'b, H: ExprRepr, X: ExtraTok> Format for FmtTTV<'b, H, X> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+pub struct FmtTTV<'a, M: ManagedStrat>(pub &'a [TokTree<M>]);
+impl<'b, M: ManagedStrat> Format for FmtTTV<'b, M> {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		ttv_fmt(self.0, c).await
 	}
 }

orchid-extension/src/atom.rs

@@ -15,7 +15,7 @@ use futures::{AsyncWrite, FutureExt, StreamExt, stream};
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Coding, Decode, InHierarchy, Request, UnderRoot, enc_vec};
 use orchid_base::{
-	FmtCtx, FmtUnit, Format, IStr, OrcErrv, Pos, Receipt, ReqHandle, ReqReader, ReqReaderExt, Sym,
+	FmtCx, FmtUnit, Format, IStr, OrcErrv, Pos, Receipt, ReqHandle, ReqReader, ReqReaderExt, Sym,
 	fmt, is, mk_errv, mk_errv_floating, take_first,
 };
 use trait_set::trait_set;
@@ -137,7 +137,7 @@ impl fmt::Debug for ForeignAtom {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ForeignAtom({self})") }
 }
 impl Format for ForeignAtom {
-	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, _c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		FmtUnit::from_api(&request(api::ExtAtomPrint(self.atom.clone())).await)
 	}
 }
@@ -302,7 +302,7 @@ impl<A: AtomicFeatures> ToExpr for TAtom<A> {
 	async fn to_gen(self) -> GExpr { self.untyped.to_gen().await }
 }
 impl<A: AtomicFeatures> Format for TAtom<A> {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		self.untyped.print(c).await
 	}
 }
@@ -359,7 +359,7 @@ impl fmt::Display for AtomFactory {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self:?}") }
 }
 impl Format for AtomFactory {
-	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, _c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		self.to_string().into()
 	}
 }

orchid-extension/src/atom_owned.rs

@@ -17,7 +17,7 @@ use itertools::Itertools;
 use memo_map::MemoMap;
 use never::Never;
 use orchid_api_traits::{Decode, Encode, enc_vec};
-use orchid_base::{FmtCtx, FmtCtxImpl, FmtUnit, Format, Sym, log, take_first};
+use orchid_base::{FmtCx, FmtCxImpl, FmtUnit, Format, Sym, log, take_first};
 use task_local::task_local;
 
 use crate::gen_expr::{GExpr, bot};
@@ -94,7 +94,7 @@ impl<A: OwnedAtom> AtomOps for OwnedAtomOps<A> {
 				log("msg"),
 				"owned call {} {}",
 				take_first(&AtomReadGuard::new(id.unwrap()).await.dyn_print().await, false),
-				take_first(&arg.print(&FmtCtxImpl::default()).await, true),
+				take_first(&arg.print(&FmtCxImpl::default()).await, true),
 			)
 			.await;
 			take_atom(id.unwrap()).await.dyn_call(arg).await
@@ -106,7 +106,7 @@ impl<A: OwnedAtom> AtomOps for OwnedAtomOps<A> {
 				log("msg"),
 				"owned call_ref {} {}",
 				take_first(&AtomReadGuard::new(id.unwrap()).await.dyn_print().await, false),
-				take_first(&arg.print(&FmtCtxImpl::default()).await, true),
+				take_first(&arg.print(&FmtCxImpl::default()).await, true),
 			)
 			.await;
 			AtomReadGuard::new(id.unwrap()).await.dyn_call_ref(arg).await
@@ -261,7 +261,7 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Any + Clone + 'static {
 	/// Debug-print. This is the final fallback for Orchid's
 	/// `std::string::to_str`.
 	#[allow(unused_variables)]
-	fn print_atom<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> {
+	fn print_atom<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> impl Future<Output = FmtUnit> {
 		async { format!("OwnedAtom({})", type_name::<Self>()).into() }
 	}
 	/// Serialize this object. If the object is serializable you must override
@@ -316,7 +316,7 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
 	}
 	fn dyn_free(self: Box<Self>) -> LocalBoxFuture<'static, ()> { self.free().boxed_local() }
 	fn dyn_print(&self) -> LocalBoxFuture<'_, FmtUnit> {
-		async move { self.print_atom(&FmtCtxImpl::default()).await }.boxed_local()
+		async move { self.print_atom(&FmtCxImpl::default()).await }.boxed_local()
 	}
 	fn dyn_serialize<'a>(
 		&'a self,

orchid-extension/src/coroutine_exec.rs

@@ -8,7 +8,7 @@ use futures::lock::Mutex;
 use futures::stream::{self, LocalBoxStream};
 use futures::{FutureExt, SinkExt, StreamExt};
 use never::Never;
-use orchid_base::{FmtCtx, FmtUnit, OrcRes};
+use orchid_base::{FmtCx, FmtUnit, OrcRes};
 
 use crate::gen_expr::{GExpr, call, lam, new_atom, seq};
 use crate::{Atomic, Expr, OwnedAtom, OwnedVariant, ToExpr, TryFromExpr};
@@ -55,7 +55,7 @@ impl OwnedAtom for Replier {
 		std::mem::drop(self.reply);
 		self.builder.run().await
 	}
-	async fn print_atom<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print_atom<'a>(&'a self, _c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		format!("Replier<{}>", self.builder.0.name).into()
 	}
 }

orchid-extension/src/entrypoint.rs

@@ -16,23 +16,25 @@ use orchid_api_traits::{Decode, Encode, Request, UnderRoot, enc_vec};
 use orchid_async_utils::{Handle, JoinError, to_task};
 use orchid_base::{
 	Client, ClientExt, CommCx, Comment, IoComm, MsgReader, MsgReaderExt, ReqHandleExt, ReqReaderExt,
-	Snippet, Sym, TokenVariant, Witness, char_filter_match, char_filter_union, es, io_comm, is, log,
-	mk_char_filter, try_with_reporter, ttv_from_api, with_interner, with_logger, with_stash,
+	Snippet, Sym, TokFromApiCx, TokToApiCx, TokenVariant, Witness, char_filter_match,
+	char_filter_union, es, io_comm, is, log, mk_char_filter, try_with_reporter, ttv_from_api,
+	ttv_into_api, with_interner, with_logger, with_stash,
 };
 use substack::Substack;
 use task_local::task_local;
 use unsync_pipe::pipe;
 
 use crate::gen_expr::serialize;
+use crate::gen_tok::{GenTokTree, drop_flex_tree, read_flex_tree, with_flex_tree_context};
 use crate::interner::new_interner;
 use crate::logger::LoggerImpl;
 use crate::tree::{TreeIntoApiCtxImpl, get_lazy, with_lazy_member_store};
 use crate::trivial_req::TrivialReqCycle;
 use crate::{
 	AtomCtx, AtomTypeId, BorrowedExprStore, CtedObj, DynSystemCardExt, DynSystemCtor, Expr,
-	ExprHandle, ExtPort, LexContext, PTokTree, ParsCtx, SysCtx, SystemCtor, api, dyn_cted,
-	ekey_not_applicable, get_const, linev_into_api, resolve_atom_type, take_atom, with_funs_ctx,
-	with_obj_store, with_parsed_const_ctx, with_refl_roots, with_sys,
+	ExprHandle, ExtPort, LexContext, ParsCtx, SysCtx, SystemCtor, api, dyn_cted, ekey_not_applicable,
+	linev_into_api, resolve_atom_type, take_atom, with_funs_ctx, with_obj_store, with_refl_roots,
+	with_sys,
 };
 
 task_local::task_local! {
@@ -177,7 +179,7 @@ impl ExtensionBuilder {
 	/// managing the [ExtPort]
 	pub async fn run(mut self, mut ctx: ExtPort) {
 		self.add_context(with_funs_ctx);
-		self.add_context(with_parsed_const_ctx);
+		self.add_context(with_flex_tree_context);
 		self.add_context(with_obj_store);
 		self.add_context(with_lazy_member_store);
 		self.add_context(with_refl_roots);
@@ -308,10 +310,11 @@ impl ExtensionBuilder {
 											return handle.reply(&lex, Some(Err(e.to_api()))).await;
 										},
 										Ok((s, expr)) => {
-											let expr = join_all(
-												(expr.into_iter())
-													.map(|tok| async { tok.into_api(&mut (), &mut ()).await }),
-											)
+											let expr = ttv_into_api(expr, &mut TokToApiCx {
+												extra: &mut (),
+												handle: &mut (),
+												lazy: &mut (),
+											})
 											.await;
 											let pos = (text.len() - s.len()) as u32;
 											eprintln!("Lexation up to pos {pos} ({} - {})", text.len(), s.len());
@@ -334,7 +337,9 @@ impl ExtensionBuilder {
 								let comments =
 									join_all(comments.iter().map(|c| Comment::from_api(c, src.clone()))).await;
 								let expr_store = BorrowedExprStore::new();
-								let line: Vec<PTokTree> = ttv_from_api(line, &mut &expr_store, &mut (), &src).await;
+								let mut cx =
+									TokFromApiCx { handle: &mut &expr_store, extra: &mut (), lazy: &mut (), src };
+								let line: Vec<GenTokTree> = ttv_from_api(line, &mut cx).await;
 								let snip = Snippet::new(line.first().expect("Empty line"), &line);
 								let parser = parsers[idx as usize];
 								let module = Sym::from_api(module).await;
@@ -350,10 +355,17 @@ impl ExtensionBuilder {
 							})
 							.await
 						},
-						api::HostExtReq::FetchParsedConst(ref fpc @ api::FetchParsedConst(sys, id)) =>
-							with_sys_record(sys, async {
-								let cnst = get_const(id).await;
-								handle.reply(fpc, serialize(cnst).await).await
+						api::HostExtReq::ReflexiveRead(ref rr @ api::ReflexiveRead { tree, path }) =>
+							with_sys_record(tree.sys, async {
+								let tt = read_flex_tree(tree.id, Sym::from_api(path).await).await;
+								let mut cx = TokToApiCx { extra: &mut (), handle: &mut (), lazy: &mut () };
+								handle.reply(rr, tt.into_api(&mut cx).await).await
+							})
+							.await,
+						api::HostExtReq::ReflexiveDrop(ref rd @ api::ReflexiveDrop(tree)) =>
+							with_sys_record(tree.sys, async {
+								drop_flex_tree(tree.id);
+								handle.reply(rd, ()).await
 							})
 							.await,
 						api::HostExtReq::AtomReq(atom_req) => {

orchid-extension/src/expr.rs

@@ -8,7 +8,7 @@ use async_once_cell::OnceCell;
 use derive_destructure::destructure;
 use futures::future::join_all;
 use hashbrown::HashSet;
-use orchid_base::{FmtCtx, FmtUnit, Format, OrcErrv, Pos, stash};
+use orchid_base::{FmtCx, FmtUnit, Format, OrcErrv, Pos, stash};
 
 use crate::gen_expr::{GExpr, slot};
 use crate::{ForeignAtom, api, notify, request, sys_id};
@@ -165,7 +165,7 @@ impl Expr {
 	pub async fn serialize(self) -> api::ExprTicket { self.handle.serialize().await }
 }
 impl Format for Expr {
-	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, _c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		FmtUnit::from_api(&request(api::ExprPrint { target: self.handle.0 }).await)
 	}
 }

orchid-extension/src/func_atom.rs

@@ -11,7 +11,7 @@ use futures::{AsyncWrite, FutureExt};
 use itertools::Itertools;
 use never::Never;
 use orchid_api_traits::Encode;
-use orchid_base::{FmtCtx, FmtUnit, OrcRes, Pos, Sym, clone};
+use orchid_base::{FmtCx, FmtUnit, OrcRes, Pos, Sym, clone};
 use task_local::task_local;
 use trait_set::trait_set;
 
@@ -151,7 +151,7 @@ impl OwnedAtom for Fun {
 			.clone();
 		Self { args, path, record }
 	}
-	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print_atom<'a>(&'a self, _: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		format!("{}:{}/{}", self.path, self.args.len(), self.arity()).into()
 	}
 }

orchid-extension/src/gen_expr.rs

@@ -6,11 +6,12 @@ use std::rc::Rc;
 
 use futures::{FutureExt, Stream, StreamExt, stream};
 use orchid_base::{
-	FmtCtx, FmtUnit, Format, OrcErr, OrcErrv, Pos, Sym, Variants, match_mapping, tl_cache,
+	FmtCx, FmtUnit, Format, OrcErr, OrcErrv, Pos, Sym, Variants, match_mapping, tl_cache,
 };
 use substack::Substack;
 use task_local::task_local;
 
+use crate::gen_tok::GenTok;
 use crate::{AtomFactory, AtomicFeatures, Expr, ToExpr, ToExprFuture, api, request, sys_id};
 
 #[derive(Clone, Copy, Debug)]
@@ -62,9 +63,10 @@ impl GExpr {
 	pub async fn create(self) -> Expr {
 		Expr::deserialize(request(api::Create(sys_id(), serialize(self).await)).await).await
 	}
+	pub fn into_tok(self) -> GenTok { GenTok::NewExpr(self) }
 }
 impl Format for GExpr {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		self.kind.print(c).boxed_local().await
 	}
 }
@@ -136,7 +138,7 @@ impl GExprKind {
 	}
 }
 impl Format for GExprKind {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		match self {
 			GExprKind::Call(f, x) =>
 				tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("{0} ({1})")))

orchid-extension/src/gen_tok.rs

@@ -0,0 +1,147 @@
+use core::fmt;
+use std::any::type_name_of_val;
+use std::cell::RefCell;
+use std::num::NonZero;
+use std::rc::Rc;
+
+use futures::future::LocalBoxFuture;
+use orchid_base::{
+	FmtCx, FmtUnit, Format, IdStore, ManagedStrat, OrcErrv, OrcRes, Snippet, SrcRange, Sym, TokTree,
+	Token, TokenVariant,
+};
+use task_local::task_local;
+
+use crate::gen_expr::{GExpr, serialize};
+use crate::{BorrowedExprStore, Expr, ExprHandle, ToExpr, api, request, sys_id};
+
+pub trait BuildFlexTree {
+	fn build(&self, in_const: Sym) -> LocalBoxFuture<'_, GenTokTree>;
+	fn print_flextree(&self, c: &dyn FmtCx) -> LocalBoxFuture<'_, FmtUnit>;
+}
+impl<F: AsyncFn(Sym) -> GenTokTree> BuildFlexTree for F {
+	fn build(&self, in_const: Sym) -> LocalBoxFuture<'_, GenTokTree> { Box::pin((self)(in_const)) }
+	fn print_flextree(&self, _: &dyn FmtCx) -> LocalBoxFuture<'_, FmtUnit> {
+		Box::pin(async { format!("Lambda {}", type_name_of_val(self)).into() })
+	}
+}
+
+#[derive(Clone)]
+pub enum FlexTree {
+	Existing(api::ReflexiveTree),
+	New(Rc<dyn BuildFlexTree>),
+}
+impl Format for FlexTree {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
+		match self {
+			Self::Existing(api) => format!("{api:?}").into(),
+			Self::New(subtree) => subtree.print_flextree(c.as_dyn()).await,
+		}
+	}
+}
+impl fmt::Debug for FlexTree {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Self::Existing(api) => api.fmt(f),
+			Self::New(_) => f.write_str("LazyTree::New(..)"),
+		}
+	}
+}
+
+#[derive(Clone, Default)]
+struct FlexTreeStore(Rc<RefCell<IdStore<Rc<dyn BuildFlexTree>>>>);
+
+impl TokenVariant<api::ReflexiveTree> for FlexTree {
+	type FromApiCx<'a> = ();
+	async fn from_api(api: api::ReflexiveTree, (): &mut Self::FromApiCx<'_>, _: SrcRange) -> Self {
+		FlexTree::Existing(api)
+	}
+	type ToApiCx<'a> = ();
+	async fn into_api(self, (): &mut Self::ToApiCx<'_>) -> api::ReflexiveTree {
+		let store = get_flex_tree_store();
+		match self {
+			Self::Existing(api) => api,
+			Self::New(subtree) => {
+				let id =
+					api::ReflexiveTreeId(NonZero::new(store.0.borrow_mut().add(subtree) as u64 + 1).unwrap());
+				api::ReflexiveTree { sys: sys_id(), id }
+			},
+		}
+	}
+}
+
+#[derive(Copy, Clone, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct GenStrat;
+impl ManagedStrat for GenStrat {
+	type Expr = Expr;
+	type Extra = GExpr;
+	type Flex = FlexTree;
+}
+
+/// Tokens generated by lexers and parsers
+///
+/// See: [GenTok], [Token], [crate::Lexer], [crate::Parser]
+pub type GenTokTree = TokTree<GenStrat>;
+/// Tokens generated by lexers and parsers - without location data
+///
+/// See: [GenTokTree], [Token], [crate::Lexer], [crate::Parser]
+pub type GenTok = Token<GenStrat>;
+/// Wrapper [Snippet] around a slice of [PTokTree] with a fallback item for
+/// [orchid_base::Pos] metadata
+pub type GenSnippet<'a> = Snippet<'a, GenStrat>;
+
+impl TokenVariant<api::Expression> for GExpr {
+	type FromApiCx<'a> = ();
+	type ToApiCx<'a> = ();
+	async fn from_api(_: api::Expression, _: &mut Self::FromApiCx<'_>, _: SrcRange) -> Self {
+		panic!("Received new expression from host")
+	}
+	async fn into_api(self, _: &mut Self::ToApiCx<'_>) -> api::Expression { serialize(self).await }
+}
+
+impl TokenVariant<api::ExprTicket> for Expr {
+	type FromApiCx<'a> = &'a BorrowedExprStore;
+	async fn from_api(api: api::ExprTicket, exprs: &mut Self::FromApiCx<'_>, _: SrcRange) -> Self {
+		// SAFETY: receiving trees from sublexers implies borrowing
+		Expr::from_handle(ExprHandle::borrowed(api, exprs))
+	}
+	type ToApiCx<'a> = ();
+	async fn into_api(self, (): &mut Self::ToApiCx<'_>) -> api::ExprTicket { self.handle().ticket() }
+}
+
+task_local! {
+	static REFLEXIVE_TREES: FlexTreeStore;
+}
+
+fn get_flex_tree_store() -> FlexTreeStore { REFLEXIVE_TREES.get() }
+
+pub fn with_flex_tree_context<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
+	Box::pin(REFLEXIVE_TREES.scope(FlexTreeStore::default(), fut))
+}
+
+pub async fn read_flex_tree(id: api::ReflexiveTreeId, in_const: Sym) -> GenTokTree {
+	let builder = &REFLEXIVE_TREES.get().0.borrow()[id.0.get() as usize - 1].clone();
+	builder.build(in_const).await
+}
+
+pub fn drop_flex_tree(id: api::ReflexiveTreeId) {
+	REFLEXIVE_TREES.get().0.borrow_mut().remove(id.0.get() as usize - 1);
+}
+
+/// Embed a literal value in generated syntax.
+pub async fn x_tok(x: impl ToExpr) -> GenTok { GenTok::NewExpr(x.to_gen().await) }
+/// Embed a reference to a constant in generated syntax. The constant doesn't
+/// need to be visible per export rules, and if it doesn't exist, the expression
+/// will raise a runtime error
+pub async fn ref_tok(path: Sym) -> GenTok { GenTok::NewExpr(path.to_gen().await) }
+
+pub fn flex_tok<F: AsyncFn(Sym) -> GenTokTree + 'static>(f: F) -> GenTok {
+	GenTok::Reflexive(FlexTree::New(Rc::new(f) as Rc<dyn BuildFlexTree>))
+}
+
+pub async fn resolve_name(path: &Sym, local: &Sym) -> OrcRes<Sym> {
+	match request(api::ResolveName { sys: sys_id(), path: path.to_api(), name: local.to_api() }).await
+	{
+		Ok(sym) => Ok(Sym::from_api(sym).await),
+		Err(e) => Err(OrcErrv::from_api(e).await),
+	}
+}

orchid-extension/src/lexer.rs

@@ -5,10 +5,10 @@ use std::ops::RangeInclusive;
 
 use futures::FutureExt;
 use futures::future::LocalBoxFuture;
-use orchid_base::{IStr, OrcErrv, OrcRes, Pos, SrcRange, Sym, is, mk_errv};
+use orchid_base::{IStr, OrcErrv, OrcRes, Pos, SrcRange, Sym, TokFromApiCx, is, mk_errv};
 
-use crate::tree::GenTokTree;
-use crate::{BorrowedExprStore, PTokTree, api, request};
+use crate::gen_tok::GenTokTree;
+use crate::{BorrowedExprStore, api, request};
 
 pub(crate) async fn ekey_not_applicable() -> IStr {
 	is("Pseudo-error to communicate that the current branch in a dispatch doesn't apply").await
@@ -47,18 +47,19 @@ impl<'a> LexContext<'a> {
 	/// module
 	pub fn src(&self) -> &Sym { &self.src }
 	/// Lex an interpolated expression of some kind
-	///
-	/// This function returns [PTokTree] because it can never return
-	/// [orchid_base::Token::NewExpr]. You can use
-	/// [crate::parser::p_tree2gen] to convert this to [crate::tree::GenTokTree]
-	/// for embedding in the return value.
-	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, PTokTree)> {
+	pub async fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree)> {
 		let start = self.pos(tail);
 		let lx = match request(api::SubLex { pos: start, id: self.id }).await {
 			Err(orc_err) => return Err(OrcErrv::from_api(orc_err).await),
 			Ok(lx) => lx,
 		};
-		let tree = PTokTree::from_api(lx.tree, &mut { self.exprs }, &mut (), &self.src).await;
+		let tree = GenTokTree::from_api(lx.tree, &mut TokFromApiCx {
+			extra: &mut (),
+			handle: &mut { self.exprs },
+			lazy: &mut (),
+			src: self.src.clone(),
+		})
+		.await;
 		Ok((&self.text[lx.pos as usize..], tree))
 	}
 	/// Find the index of a cursor given the remaining, not-yet-consumed text

orchid-extension/src/lib.rs

@@ -26,6 +26,7 @@ pub use ext_port::*;
 mod func_atom;
 pub use func_atom::*;
 pub mod gen_expr;
+pub mod gen_tok;
 mod interner;
 mod lexer;
 pub use lexer::*;

orchid-extension/src/parser.rs

@@ -1,58 +1,11 @@
-use std::cell::RefCell;
 use std::marker::PhantomData;
-use std::num::NonZero;
 
-use async_fn_stream::stream;
+use futures::FutureExt;
 use futures::future::{LocalBoxFuture, join_all};
-use futures::{FutureExt, Stream, StreamExt};
-use hashbrown::HashMap;
-use itertools::Itertools;
-use never::Never;
-use orchid_base::{
-	Comment, IStr, OrcErrv, OrcRes, Snippet, SrcRange, Sym, TokTree, Token, match_mapping,
-	ttv_into_api,
-};
-use task_local::task_local;
+use orchid_base::{Comment, IStr, OrcRes, SrcRange, Sym, TokToApiCx, ttv_into_api};
 
-use crate::gen_expr::GExpr;
-use crate::tree::{GenTok, GenTokTree};
-use crate::{ExecHandle, Expr, ToExpr, TryFromExpr, api, exec, request, sys_id};
-
-/// [PTokTree] without [orchid_base::Pos] metadata
-pub type PTok = Token<Expr, Never>;
-/// [TokTree] received by [Parser], or returned from recursive [crate::Lexer]
-/// call. It is different from [GenTok] because it cannot be [Token::NewExpr].
-/// Convert with [p_tok2gen], [p_tree2gen], [p_v2gen]
-pub type PTokTree = TokTree<Expr, Never>;
-/// Wrapper [Snippet] around a slice of [PTokTree] with a fallback item for
-/// [orchid_base::Pos] metadata
-pub type PSnippet<'a> = Snippet<'a, Expr, Never>;
-
-/// Convert ingress syntax tokens into egress for interpolation into the output
-///
-/// See also [p_tree2gen], [p_v2gen]
-pub fn p_tok2gen(tok: PTok) -> GenTok {
-	match_mapping!(tok, PTok => GenTok {
-		Comment(s), Name(n), BR, Handle(ex), Bottom(err),
-		LambdaHead(arg => Box::new(p_tree2gen(*arg))),
-		NS(n, arg => Box::new(p_tree2gen(*arg))),
-		S(p, body () p_v2gen),
-	} {
-		PTok::NewExpr(never) => match never {}
-	})
-}
-/// Convert ingress syntax tokens into egress for interpolation into the output
-///
-/// See also [p_tok2gen], [p_v2gen]
-pub fn p_tree2gen(tree: PTokTree) -> GenTokTree {
-	TokTree { tok: p_tok2gen(tree.tok), sr: tree.sr }
-}
-/// Convert ingress syntax tokens into egress for interpolation into the output
-///
-/// See also [p_tok2gen], [p_tree2gen]
-pub fn p_v2gen(v: impl IntoIterator<Item = PTokTree>) -> Vec<GenTokTree> {
-	v.into_iter().map(p_tree2gen).collect_vec()
-}
+use crate::api;
+use crate::gen_tok::{GenSnippet, GenTokTree};
 
 /// A parser extension for the stage of parsing that converts source lines into
 /// a module tree. These are invoked by usercode intentionally through a custom
@@ -72,7 +25,7 @@ pub trait Parser: Send + Sync + Sized + Default + 'static {
 		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: PSnippet<'a>,
+		line: GenSnippet<'a>,
 	) -> impl Future<Output = OrcRes<Vec<ParsedLine>>> + 'a;
 }
 
@@ -86,7 +39,7 @@ pub trait DynParser: Send + Sync + 'static {
 		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: PSnippet<'a>,
+		line: GenSnippet<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>>;
 }
 
@@ -97,7 +50,7 @@ impl<T: Parser> DynParser for T {
 		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: PSnippet<'a>,
+		line: GenSnippet<'a>,
 	) -> LocalBoxFuture<'a, OrcRes<Vec<ParsedLine>>> {
 		Box::pin(async move { Self::parse(ctx, exported, comments, line).await })
 	}
@@ -118,18 +71,6 @@ impl<'a> ParsCtx<'a> {
 	pub fn module(&self) -> Sym { self.module.clone() }
 }
 
-type BoxConstCallback = Box<dyn for<'a> FnOnce(ConstCtx<'a>) -> LocalBoxFuture<'a, GExpr>>;
-
-task_local! {
-	static CONST_TBL: RefCell<HashMap<NonZero<u64>, BoxConstCallback>>;
-	static NEXT_CONST_ID: RefCell<NonZero<u64>>;
-}
-
-pub(crate) fn with_parsed_const_ctx<'a>(fut: LocalBoxFuture<'a, ()>) -> LocalBoxFuture<'a, ()> {
-	let id_cell = RefCell::new(NonZero::new(1).unwrap());
-	Box::pin(CONST_TBL.scope(RefCell::default(), NEXT_CONST_ID.scope(id_cell, fut)))
-}
-
 /// A logical subtree returned from a parser
 pub struct ParsedLine {
 	/// Source location associated with this item
@@ -143,16 +84,14 @@ impl ParsedLine {
 	/// Define a constant. The callback is only called later if the constant is
 	/// referenced, and it can call [crate::refl] to base its value on the module
 	/// tree or use its argument for context-specific queries
-	pub fn cnst<'a, R: ToExpr + 'static, F: for<'b> AsyncFnOnce(ConstCtx<'b>) -> R + 'static>(
+	pub fn cnst<'a>(
 		sr: &SrcRange,
 		comments: impl IntoIterator<Item = &'a Comment>,
 		exported: bool,
 		name: IStr,
-		f: F,
+		val: Vec<GenTokTree>,
 	) -> Self {
-		let cb: BoxConstCallback =
-			Box::new(|ctx| async move { f(ctx).await.to_gen().await }.boxed_local());
-		let kind = ParsedLineKind::Mem(ParsedMem { name, exported, kind: ParsedMemKind::Const(cb) });
+		let kind = ParsedLineKind::Mem(ParsedMem { name, exported, kind: ParsedMemKind::Const(val) });
 		let comments = comments.into_iter().cloned().collect();
 		ParsedLine { comments, sr: sr.clone(), kind }
 	}
@@ -184,10 +123,9 @@ impl ParsedLine {
 					name: mem.name.to_api(),
 					exported: mem.exported,
 					kind: match mem.kind {
-						ParsedMemKind::Const(cb) => {
-							let id = NEXT_CONST_ID.with(|c| c.replace_with(|id| id.checked_add(1).unwrap()));
-							CONST_TBL.with(|consts| consts.borrow_mut().insert(id, cb));
-							api::ParsedMemberKind::Constant(api::ParsedConstId(id))
+						ParsedMemKind::Const(tv) => {
+							let mut cx = TokToApiCx { extra: &mut (), handle: &mut (), lazy: &mut () };
+							api::ParsedMemberKind::Constant(ttv_into_api(tv, &mut cx).await)
 						},
 						ParsedMemKind::Mod { lines, use_prelude } => api::ParsedMemberKind::Module {
 							lines: linev_into_api(lines).boxed_local().await,
@@ -195,8 +133,10 @@ impl ParsedLine {
 						},
 					},
 				}),
-				ParsedLineKind::Rec(tv) =>
-					api::ParsedLineKind::Recursive(ttv_into_api(tv, &mut (), &mut ()).await),
+				ParsedLineKind::Rec(tv) => {
+					let mut cx = TokToApiCx { extra: &mut (), handle: &mut (), lazy: &mut () };
+					api::ParsedLineKind::Recursive(ttv_into_api(tv, &mut cx).await)
+				},
 			},
 		}
 	}
@@ -230,7 +170,7 @@ pub struct ParsedMem {
 /// Kind of [ParsedMem]
 pub enum ParsedMemKind {
 	/// A constant
-	Const(BoxConstCallback),
+	Const(Vec<GenTokTree>),
 	/// A module with additional descendants
 	Mod {
 		/// Logical lines inside the module
@@ -240,47 +180,3 @@ pub enum ParsedMemKind {
 		use_prelude: bool,
 	},
 }
-
-/// Enable a generated constant to query about its environment
-pub struct ConstCtx<'a> {
-	constid: api::ParsedConstId,
-	exec: ExecHandle<'a>,
-}
-impl<'a> ConstCtx<'a> {
-	/// Resolve a set of local names into the full names they would point to if
-	/// they were globally bound. Errors produced at this stage are soft, as the
-	/// names may still be found to be locally bound within the expression
-	pub fn names<'b>(
-		&'b self,
-		names: impl IntoIterator<Item = &'b Sym> + 'b,
-	) -> impl Stream<Item = OrcRes<Sym>> + 'b {
-		let names = names.into_iter().map(|n| n.to_api()).collect_vec();
-		stream(async |mut cx| {
-			if names.is_empty() {
-				return;
-			}
-			let resolve_names = api::ResolveNames { constid: self.constid, sys: sys_id(), names };
-			for name_opt in request(resolve_names).await {
-				cx.emit(match name_opt {
-					Err(e) => Err(OrcErrv::from_api(e).await),
-					Ok(name) => Ok(Sym::from_api(name).await),
-				})
-				.await
-			}
-		})
-	}
-	/// Static-length version of [Self::names]
-	pub async fn names_n<const N: usize>(&self, names: [&Sym; N]) -> [OrcRes<Sym>; N] {
-		self.names(names).collect::<Vec<_>>().await.try_into().expect("Lengths must match")
-	}
-	pub async fn exec<R: TryFromExpr>(&mut self, val: impl ToExpr) -> OrcRes<R> {
-		self.exec.exec(val).await
-	}
-	pub fn handle(&mut self) -> &mut ExecHandle<'a> { &mut self.exec }
-}
-
-pub(crate) async fn get_const(id: api::ParsedConstId) -> GExpr {
-	let cb = CONST_TBL
-		.with(|ent| ent.borrow_mut().remove(&id.0).expect("Bad ID or double read of parsed const"));
-	exec(async move |exec| cb(ConstCtx { constid: id, exec }).await).await
-}

orchid-extension/src/tree.rs

@@ -8,48 +8,13 @@ use futures::future::{LocalBoxFuture, join_all};
 use futures::{FutureExt, StreamExt};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::{IStr, SrcRange, Sym, TokTree, Token, TokenVariant, is};
+use orchid_base::{IStr, Sym, is};
 use substack::Substack;
 use task_local::task_local;
 use trait_set::trait_set;
 
 use crate::gen_expr::{GExpr, new_atom, serialize};
-use crate::{BorrowedExprStore, Expr, ExprFunc, ExprHandle, Fun, ToExpr, api};
-
-/// Tokens generated by lexers and parsers
-///
-/// See: [GenTok], [Token], [crate::Lexer], [crate::Parser]
-pub type GenTokTree = TokTree<Expr, GExpr>;
-/// Tokens generated by lexers and parsers - without location data
-///
-/// See: [GenTokTree], [Token], [crate::Lexer], [crate::Parser]
-pub type GenTok = Token<Expr, GExpr>;
-
-impl TokenVariant<api::Expression> for GExpr {
-	type FromApiCtx<'a> = ();
-	type ToApiCtx<'a> = ();
-	async fn from_api(_: api::Expression, _: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
-		panic!("Received new expression from host")
-	}
-	async fn into_api(self, _: &mut Self::ToApiCtx<'_>) -> api::Expression { serialize(self).await }
-}
-
-impl TokenVariant<api::ExprTicket> for Expr {
-	type FromApiCtx<'a> = &'a BorrowedExprStore;
-	async fn from_api(api: api::ExprTicket, exprs: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
-		// SAFETY: receiving trees from sublexers implies borrowing
-		Expr::from_handle(ExprHandle::borrowed(api, exprs))
-	}
-	type ToApiCtx<'a> = ();
-	async fn into_api(self, (): &mut Self::ToApiCtx<'_>) -> api::ExprTicket { self.handle().ticket() }
-}
-
-/// Embed a literal value in generated syntax.
-pub async fn x_tok(x: impl ToExpr) -> GenTok { GenTok::NewExpr(x.to_gen().await) }
-/// Embed a reference to a constant in generated syntax. The constant doesn't
-/// need to be visible per export rules, and if it doesn't exist, the expression
-/// will raise a runtime error
-pub async fn ref_tok(path: Sym) -> GenTok { GenTok::NewExpr(path.to_gen().await) }
+use crate::{ExprFunc, Fun, ToExpr, api};
 
 /// Create a new subtree that is evaluated as-needed, asynchronously, and can
 /// use its own path to determine its value

orchid-host/src/atom.rs

@@ -5,13 +5,13 @@ use async_once_cell::OnceCell;
 use derive_destructure::destructure;
 #[cfg(feature = "orchid-extension")]
 use orchid_api_traits::{Request, UnderRoot};
-use orchid_base::{AtomRepr, ClientExt, FmtCtx, FmtUnit, Format, Pos, take_first_fmt};
+use orchid_base::{AtomRepr, ClientExt, FmtCx, FmtUnit, Format, Pos, take_first_fmt};
 #[cfg(feature = "orchid-extension")]
 use orchid_extension::AtomMethod;
 
 use crate::api;
-use crate::ctx::Ctx;
-use crate::expr::{Expr, ExprFromApiCtx, PathSetBuilder};
+use crate::cx::Cx;
+use crate::expr::{Expr, ExprFromApiCx, PathSetBuilder};
 use crate::extension::Extension;
 use crate::system::System;
 
@@ -74,19 +74,16 @@ impl AtomHand {
 	#[must_use]
 	pub async fn call(self, arg: Expr) -> Expr {
 		let owner = self.0.owner.clone();
-		let ctx = owner.ctx();
+		let cx = owner.cx();
 		let client = owner.client();
-		ctx.exprs.give_expr(arg.clone());
+		cx.exprs.give_expr(arg.clone());
 		let ret = match Rc::try_unwrap(self.0) {
 			Ok(data) => client.request(api::FinalCall(data.api(), arg.id())).await.unwrap(),
 			Err(hand) => client.request(api::CallRef(hand.api_ref(), arg.id())).await.unwrap(),
 		};
-		let val = Expr::from_api(ret, PathSetBuilder::new(), ExprFromApiCtx {
-			sys: owner.id(),
-			ctx: ctx.clone(),
-		})
-		.await;
-		ctx.exprs.take_expr(arg.id());
+		let val =
+			Expr::from_api(ret, PathSetBuilder::new(), ExprFromApiCx { sys: owner.clone() }).await;
+		cx.exprs.take_expr(arg.id());
 		val
 	}
 	#[must_use]
@@ -104,7 +101,7 @@ impl AtomHand {
 	pub fn downgrade(&self) -> WeakAtomHand { WeakAtomHand(Rc::downgrade(&self.0)) }
 }
 impl Format for AtomHand {
-	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, _c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		(self.0.display.get_or_init(async {
 			FmtUnit::from_api(
 				&self.0.owner.client().request(api::AtomPrint(self.0.api_ref())).await.unwrap(),
@@ -115,10 +112,10 @@ impl Format for AtomHand {
 	}
 }
 impl AtomRepr for AtomHand {
-	type Ctx = Ctx;
-	async fn from_api(atom: &api::Atom, _: Pos, ctx: &mut Self::Ctx) -> Self {
+	type Cx = Cx;
+	async fn from_api(atom: &api::Atom, _: Pos, cx: &mut Self::Cx) -> Self {
 		let api::Atom { data, drop, owner } = atom.clone();
-		let sys = ctx.system_inst(owner).await.expect("Dropped system created atom");
+		let sys = cx.system_inst(owner).await.expect("Dropped system created atom");
 		if let Some(id) = drop {
 			sys.new_atom(data.0, id).await
 		} else {

orchid-host/src/cmd_system.rs

@@ -16,9 +16,9 @@ use orchid_async_utils::{LocalSet, LocalSetController, local_set};
 use orchid_base::{OrcErrv, Receipt, ReqHandle, Sym, fmt, is, log, mk_errv, sym};
 use orchid_extension::{self as ox, AtomicFeatures as _, get_arg};
 
-use crate::ctx::Ctx;
-use crate::execute::{ExecCtx, ExecResult};
-use crate::expr::{Expr, ExprFromApiCtx, PathSetBuilder};
+use crate::cx::Cx;
+use crate::execute::{ExecCx, ExecResult};
+use crate::expr::{Expr, ExprFromApiCx, PathSetBuilder};
 use crate::extension::Extension;
 use crate::inline::ext_inline;
 use crate::system::System;
@@ -35,7 +35,7 @@ enum Task {
 struct CommandQueueState {
 	new: VecDeque<Task>,
 	added: Rc<Event>,
-	ctx: Ctx,
+	cx: Cx,
 }
 
 /// Shared object serving as a communication point between the extension
@@ -43,11 +43,11 @@ struct CommandQueueState {
 #[derive(Clone)]
 struct CommandQueue(Rc<RefCell<CommandQueueState>>);
 impl CommandQueue {
-	fn new(ctx: Ctx, init: impl IntoIterator<Item = Expr>) -> Self {
+	fn new(cx: Cx, init: impl IntoIterator<Item = Expr>) -> Self {
 		Self(Rc::new(RefCell::new(CommandQueueState {
 			new: init.into_iter().map(Task::RunCommand).collect(),
 			added: Rc::default(),
-			ctx,
+			cx,
 		})))
 	}
 	pub fn push(&self, task: Task) {
@@ -112,7 +112,7 @@ enum TaskOutcome {
 pub struct CmdRunner {
 	queue: CommandQueue,
 	gas: Option<u64>,
-	interrupted: Option<ExecCtx>,
+	interrupted: Option<ExecCx>,
 	system: System,
 	recv: mpsc::Receiver<TaskOutcome>,
 	send: mpsc::Sender<TaskOutcome>,
@@ -120,10 +120,10 @@ pub struct CmdRunner {
 	local_set: LocalSet<'static, Never>,
 }
 impl CmdRunner {
-	pub async fn new(root: &mut Root, ctx: Ctx, init: impl IntoIterator<Item = Expr>) -> Self {
-		let queue = CommandQueue::new(ctx.clone(), init);
+	pub async fn new(root: &mut Root, cx: Cx, init: impl IntoIterator<Item = Expr>) -> Self {
+		let queue = CommandQueue::new(cx.clone(), init);
 		let ext_builder = ox::ExtensionBuilder::new("orchid::cmd").system(CmdSystemCtor(queue.clone()));
-		let extension = (Extension::new(ext_inline(ext_builder, ctx.clone()).await, ctx).await)
+		let extension = (Extension::new(ext_inline(ext_builder, cx.clone()).await, cx).await)
 			.expect("IO error on in-memory pipe");
 		let system_ctor = (extension.system_ctors().find(|ctor| ctor.decl.name == "orchid::cmd"))
 			.expect("Missing command system ctor");
@@ -163,22 +163,22 @@ impl CmdRunner {
 				Event::Request(Task::Exit) => return CmdEvent::Exit,
 				Event::Request(Task::Sleep(until, expr)) => {
 					let queue = self.queue.clone();
-					let ctx = queue.0.borrow_mut().ctx.clone();
+					let cx = queue.0.borrow_mut().cx.clone();
 					self.ctl.spawn(async move {
 						let delta = until - Utc::now();
 						match delta.to_std() {
 							Err(_) =>
 								writeln!(log("debug"), "Negative sleep found ({delta}), requeuing as instant").await,
-							Ok(delay) => ctx.sleep(delay).await,
+							Ok(delay) => cx.sleep(delay).await,
 						};
 						queue.push(Task::RunCommand(expr));
 						Ok(())
 					})
 				},
 				Event::Request(Task::RunCommand(expr)) => {
-					let mut xctx = ExecCtx::new(root.clone(), expr).await;
-					xctx.set_gas(self.gas);
-					let norm = match xctx.execute().await {
+					let mut xcx = ExecCx::new(root.clone(), expr).await;
+					xcx.set_gas(self.gas);
+					let norm = match xcx.execute().await {
 						ExecResult::Err(err, gas) => {
 							self.gas = gas;
 							return CmdEvent::Err(err);
@@ -187,23 +187,21 @@ impl CmdRunner {
 							self.gas = gas;
 							val
 						},
-						ExecResult::Gas(xctx) => {
-							self.interrupted = Some(xctx);
+						ExecResult::Gas(xcx) => {
+							self.interrupted = Some(xcx);
 							return CmdEvent::Gas;
 						},
 					};
 					let Some(atom) = norm.as_atom().await else {
 						return CmdEvent::NonCommand(norm);
 					};
-					let queue = self.queue.clone();
-					let ctx = queue.0.borrow_mut().ctx.clone();
 					let mut send = self.send.clone();
 					self.ctl.spawn(async move {
 						match atom.ipc(ox::std_reqs::StartCommand).await {
 							None => send.send(TaskOutcome::NonCommand(norm)).await.unwrap(),
 							Some(None) => send.send(TaskOutcome::None).await.unwrap(),
 							Some(Some(expr)) => {
-								let from_api_cx = ExprFromApiCtx { ctx, sys: atom.api_ref().owner };
+								let from_api_cx = ExprFromApiCx { sys: atom.sys().clone() };
 								let expr = Expr::from_api(expr, PathSetBuilder::new(), from_api_cx).await;
 								send.send(TaskOutcome::Next(expr)).await.unwrap();
 							},
@@ -257,7 +255,7 @@ impl ox::System for CmdSystemInst {
 					let side_xtk = side.clone().serialize().await;
 					let mut g = queue.0.borrow_mut();
 					let host_ex =
-						g.ctx.exprs.take_expr(side_xtk).expect("Host could not locate leaked expr by ID ");
+						g.cx.exprs.take_expr(side_xtk).expect("Host could not locate leaked expr by ID ");
 					g.new.push_back(Task::RunCommand(host_ex));
 					Some(cont.clone())
 				})
@@ -284,7 +282,7 @@ impl ox::System for CmdSystemInst {
 					let cont_xtk = cont.clone().serialize().await;
 					let mut g = queue.0.borrow_mut();
 					let host_ex =
-						g.ctx.exprs.take_expr(cont_xtk).expect("Host could not locate leaked expr by ID ");
+						g.cx.exprs.take_expr(cont_xtk).expect("Host could not locate leaked expr by ID ");
 					g.new.push_back(Task::Sleep(until, host_ex));
 					None::<Never>
 				})

orchid-host/src/cx.rs

@@ -25,7 +25,7 @@ pub trait JoinHandle {
 
 pub trait Spawner {
 	/// Spawn a future off-task, but in the same thread. If all objects associated
-	/// with the owning [Ctx] are dropped (eg. expressions, extensions, systems)
+	/// with the owning [cx] are dropped (eg. expressions, extensions, systems)
 	/// and no extensions create permanent ref loops, then all tasks will
 	/// eventually settle, therefore the implementor of this interface should not
 	/// exit while there are pending tasks to allow external communication
@@ -36,7 +36,7 @@ pub trait Spawner {
 	}
 }
 
-pub struct CtxData {
+pub struct CxData {
 	spawner: Rc<dyn Spawner>,
 	pub systems: RwLock<HashMap<api::SysId, WeakSystem>>,
 	pub system_id: RefCell<NonZeroU16>,
@@ -45,23 +45,23 @@ pub struct CtxData {
 	pub logger: LoggerImpl,
 }
 #[derive(Clone)]
-pub struct Ctx(Rc<CtxData>);
-impl ops::Deref for Ctx {
-	type Target = CtxData;
+pub struct Cx(Rc<CxData>);
+impl ops::Deref for Cx {
+	type Target = CxData;
 	fn deref(&self) -> &Self::Target { &self.0 }
 }
 #[derive(Clone)]
-pub struct WeakCtx(Weak<CtxData>);
-impl WeakCtx {
+pub struct Weakcx(Weak<CxData>);
+impl Weakcx {
 	#[must_use]
-	pub fn try_upgrade(&self) -> Option<Ctx> { Some(Ctx(self.0.upgrade()?)) }
+	pub fn try_upgrade(&self) -> Option<Cx> { Some(Cx(self.0.upgrade()?)) }
 	#[must_use]
-	pub fn upgrade(&self) -> Ctx { self.try_upgrade().expect("Ctx manually kept alive until exit") }
+	pub fn upgrade(&self) -> Cx { self.try_upgrade().expect("cx manually kept alive until exit") }
 }
-impl Ctx {
+impl Cx {
 	#[must_use]
 	pub fn new(spawner: impl Spawner + 'static, logger: LoggerImpl) -> Self {
-		Self(Rc::new(CtxData {
+		Self(Rc::new(CxData {
 			spawner: Rc::new(spawner),
 			systems: RwLock::default(),
 			system_id: RefCell::new(NonZero::new(1).unwrap()),
@@ -97,10 +97,10 @@ impl Ctx {
 		api::SysId(*g)
 	}
 	#[must_use]
-	pub fn downgrade(&self) -> WeakCtx { WeakCtx(Rc::downgrade(&self.0)) }
+	pub fn downgrade(&self) -> Weakcx { Weakcx(Rc::downgrade(&self.0)) }
 }
-impl fmt::Debug for Ctx {
+impl fmt::Debug for Cx {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		f.debug_struct("Ctx").field("system_id", &self.system_id).finish_non_exhaustive()
+		f.debug_struct("Cx").field("system_id", &self.system_id).finish_non_exhaustive()
 	}
 }

orchid-host/src/dealias.rs

@@ -61,13 +61,13 @@ pub async fn resolv_glob<Mod: Tree>(
 	root: &Mod,
 	abs_path: &[IStr],
 	pos: Pos,
-	ctx: &mut Mod::Ctx<'_>,
+	cx: &mut Mod::Cx<'_>,
 ) -> OrcRes<HashSet<IStr>> {
 	let coprefix_len = cwd.iter().zip(abs_path).take_while(|(a, b)| a == b).count();
 	let (co_prefix, diff_path) = abs_path.split_at(abs_path.len().min(coprefix_len + 1));
 	let fst_diff =
-		walk(root, false, co_prefix.iter().cloned(), ctx).await.expect("Invalid step in cwd");
-	let target_module = match walk(fst_diff, true, diff_path.iter().cloned(), ctx).await {
+		walk(root, false, co_prefix.iter().cloned(), cx).await.expect("Invalid step in cwd");
+	let target_module = match walk(fst_diff, true, diff_path.iter().cloned(), cx).await {
 		Ok(t) => t,
 		Err(e) => {
 			let path = abs_path[..=coprefix_len + e.pos].iter().join("::");
@@ -85,7 +85,7 @@ pub async fn resolv_glob<Mod: Tree>(
 pub type ChildResult<'a, T> = Result<&'a T, ChildErrorKind>;
 
 pub trait Tree {
-	type Ctx<'a>;
+	type Cx<'a>;
 	#[must_use]
 	fn children(&self, public_only: bool) -> HashSet<IStr>;
 	#[must_use]
@@ -93,7 +93,7 @@ pub trait Tree {
 		&self,
 		key: IStr,
 		public_only: bool,
-		ctx: &mut Self::Ctx<'_>,
+		cx: &mut Self::Cx<'_>,
 	) -> impl Future<Output = ChildResult<'_, Self>>;
 }
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
@@ -123,11 +123,11 @@ pub async fn walk<'a, T: Tree>(
 	root: &'a T,
 	public_only: bool,
 	path: impl IntoIterator<Item = IStr>,
-	ctx: &mut T::Ctx<'_>,
+	cx: &mut T::Cx<'_>,
 ) -> Result<&'a T, ChildError> {
 	let mut cur = root;
 	for (i, item) in path.into_iter().enumerate() {
-		match cur.child(item, public_only, ctx).await {
+		match cur.child(item, public_only, cx).await {
 			ChildResult::Ok(v) => cur = v,
 			ChildResult::Err(kind) => return Err(ChildError { pos: i, kind }),
 		}

orchid-host/src/dylib.rs

@@ -11,7 +11,7 @@ use orchid_base::{log, on_drop, vt_to_future};
 use unsync_pipe::pipe;
 
 use crate::api;
-use crate::ctx::Ctx;
+use crate::cx::Cx;
 use crate::extension::ExtPort;
 use crate::task_set::TaskSet;
 
@@ -28,12 +28,12 @@ fn load_dylib(path: &Path) -> Result<Arc<Library>, libloading::Error> {
 	}
 }
 
-pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Error> {
+pub async fn ext_dylib(path: &Path, cx: Cx) -> Result<ExtPort, libloading::Error> {
 	let (write_input, input) = pipe(1024);
 	let (output, read_output) = pipe(1024);
 	let (write_log, read_log) = pipe(1024);
 	let log_path = path.to_string_lossy().to_string();
-	let _ = ctx.spawn(Duration::ZERO, async move {
+	let _ = cx.spawn(Duration::ZERO, async move {
 		let mut lines = BufReader::new(read_log).lines();
 		while let Some(line) = lines.next().await {
 			match line {
@@ -50,7 +50,7 @@ pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Err
 	let entrypoint: Symbol<unsafe extern "C" fn(api::binary::ExtensionContext)> =
 		unsafe { library.get("orchid_extension_main") }?;
 
-	let data = Box::into_raw(Box::new(SpawnerState { ctx, tasks: tasks.clone() })) as *const ();
+	let data = Box::into_raw(Box::new(SpawnerState { cx, tasks: tasks.clone() })) as *const ();
 	let spawner = api::binary::SpawnerBin { data, drop, spawn };
 	let cx = api::binary::ExtensionContext { input, output, log: write_log, spawner };
 	unsafe { (entrypoint)(cx) };
@@ -62,7 +62,7 @@ pub async fn ext_dylib(path: &Path, ctx: Ctx) -> Result<ExtPort, libloading::Err
 }
 
 struct SpawnerState {
-	ctx: Ctx,
+	cx: Cx,
 	tasks: TaskSet,
 }
 extern "C" fn drop(data: *const ()) {
@@ -75,7 +75,7 @@ extern "C" fn spawn(data: *const (), delay: u64, vt: api::binary::FutureBin) {
 	let state = unsafe { (data as *mut SpawnerState).as_mut() }.unwrap();
 	state.tasks.with(|store| {
 		store.add_with(|id| {
-			state.ctx.spawn(Duration::from_millis(delay), async move {
+			state.cx.spawn(Duration::from_millis(delay), async move {
 				future.await;
 				// SAFETY: We know this is live because the drop handle that frees it also
 				// aborts the future

orchid-host/src/execute.rs

@@ -21,11 +21,11 @@ enum StackOp {
 
 pub enum ExecResult {
 	Value(Expr, Option<u64>),
-	Gas(ExecCtx),
+	Gas(ExecCx),
 	Err(OrcErrv, Option<u64>),
 }
 
-pub struct ExecCtx {
+pub struct ExecCx {
 	gas: Option<u64>,
 	stack: Vec<ExprGuard>,
 	cur: ExprGuard,
@@ -33,7 +33,7 @@ pub struct ExecCtx {
 	did_pop: bool,
 	root: Root,
 }
-impl ExecCtx {
+impl ExecCx {
 	#[must_use]
 	pub async fn new(root: Root, init: Expr) -> Self {
 		let cur_pos = init.pos();

orchid-host/src/expr.rs

@@ -8,15 +8,15 @@ use futures::FutureExt;
 use futures_locks::RwLock;
 use itertools::Itertools;
 use orchid_base::{
-	AtomRepr, FmtCtx, FmtUnit, Format, OrcErrv, Pos, SrcRange, Sym, TokenVariant, Variants, indent,
+	AtomRepr, FmtCx, FmtUnit, Format, OrcErrv, Pos, SrcRange, Sym, TokenVariant, Variants, indent,
 	tl_cache,
 };
 use substack::Substack;
 
 use crate::api;
 use crate::atom::AtomHand;
-use crate::ctx::Ctx;
 use crate::expr_store::ExprStore;
+use crate::system::System;
 
 #[derive(Debug)]
 pub struct ExprData {
@@ -25,9 +25,8 @@ pub struct ExprData {
 }
 
 #[derive(Clone, Debug)]
-pub struct ExprFromApiCtx {
-	pub sys: api::SysId,
-	pub ctx: Ctx,
+pub struct ExprFromApiCx {
+	pub sys: System,
 }
 
 #[derive(Clone, Debug)]
@@ -61,7 +60,7 @@ impl Expr {
 	pub async fn from_api(
 		api: api::Expression,
 		psb: PathSetBuilder<'_, u64>,
-		ctx: ExprFromApiCtx,
+		cx: ExprFromApiCx,
 	) -> Self {
 		let pos = Pos::from_api(&api.location).await;
 		let kind = match api.kind {
@@ -73,25 +72,26 @@ impl Expr {
 			api::ExpressionKind::Call(f, x) => {
 				let (lpsb, rpsb) = psb.split();
 				ExprKind::Call(
-					Expr::from_api(*f, lpsb, ctx.clone()).boxed_local().await,
-					Expr::from_api(*x, rpsb, ctx).boxed_local().await,
+					Expr::from_api(*f, lpsb, cx.clone()).boxed_local().await,
+					Expr::from_api(*x, rpsb, cx).boxed_local().await,
 				)
 			},
 			api::ExpressionKind::Const(name) => ExprKind::Const(Sym::from_api(name).await),
 			api::ExpressionKind::Lambda(x, body) => {
 				let lbuilder = psb.lambda(&x);
-				let body = Expr::from_api(*body, lbuilder.stack(), ctx).boxed_local().await;
+				let body = Expr::from_api(*body, lbuilder.stack(), cx).boxed_local().await;
 				ExprKind::Lambda(lbuilder.collect(), body)
 			},
 			api::ExpressionKind::NewAtom(a) => ExprKind::Atom(
-				AtomHand::from_api(&a.associate(ctx.sys), pos.clone(), &mut ctx.ctx.clone()).await,
+				AtomHand::from_api(&a.associate(cx.sys.id()), pos.clone(), &mut cx.sys.cx().clone()).await,
 			),
-			api::ExpressionKind::Slot(tk) => return ctx.ctx.exprs.take_expr(tk).expect("Invalid slot"),
+			api::ExpressionKind::Slot(tk) =>
+				return cx.sys.cx().exprs.take_expr(tk).expect("Invalid slot"),
 			api::ExpressionKind::Seq(a, b) => {
 				let (apsb, bpsb) = psb.split();
 				ExprKind::Seq(
-					Expr::from_api(*a, apsb, ctx.clone()).boxed_local().await,
-					Expr::from_api(*b, bpsb, ctx).boxed_local().await,
+					Expr::from_api(*a, apsb, cx.clone()).boxed_local().await,
+					Expr::from_api(*b, bpsb, cx).boxed_local().await,
 				)
 			},
 		};
@@ -111,13 +111,13 @@ impl Expr {
 	pub fn kind(&self) -> &RwLock<ExprKind> { &self.0.kind }
 }
 impl Format for Expr {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		return print_expr(self, c, Substack::Bottom, &[]).await;
 	}
 }
 pub async fn print_expr<'a>(
 	expr: &'a Expr,
-	c: &'a (impl FmtCtx + ?Sized + 'a),
+	c: &'a (impl FmtCx + ?Sized + 'a),
 	visited: Substack<'_, api::ExprTicket>,
 	id_only: &[api::ExprTicket],
 ) -> FmtUnit {
@@ -152,13 +152,13 @@ impl ExprKind {
 	pub fn at(self, pos: Pos) -> Expr { Expr(Rc::new(ExprData { pos, kind: RwLock::new(self) })) }
 }
 impl Format for ExprKind {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		print_exprkind(self, c, Substack::Bottom, &[]).await
 	}
 }
 async fn print_exprkind<'a>(
 	ek: &ExprKind,
-	c: &'a (impl FmtCtx + ?Sized + 'a),
+	c: &'a (impl FmtCx + ?Sized + 'a),
 	visited: Substack<'_, api::ExprTicket>,
 	id_only: &[api::ExprTicket],
 ) -> FmtUnit {
@@ -343,14 +343,14 @@ impl WeakExpr {
 }
 
 impl TokenVariant<api::ExprTicket> for Expr {
-	type FromApiCtx<'a> = ExprStore;
-	async fn from_api(api: api::ExprTicket, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
-		ctx.get_expr(api).expect("Invalid ticket")
+	type FromApiCx<'a> = ExprStore;
+	async fn from_api(api: api::ExprTicket, cx: &mut Self::FromApiCx<'_>, _: SrcRange) -> Self {
+		cx.get_expr(api).expect("Invalid ticket")
 	}
-	type ToApiCtx<'a> = ExprStore;
-	async fn into_api(self, ctx: &mut Self::ToApiCtx<'_>) -> api::ExprTicket {
+	type ToApiCx<'a> = ExprStore;
+	async fn into_api(self, cx: &mut Self::ToApiCx<'_>) -> api::ExprTicket {
 		let id = self.id();
-		ctx.give_expr(self);
+		cx.give_expr(self);
 		id
 	}
 }
@@ -360,12 +360,12 @@ impl TokenVariant<api::ExprTicket> for Expr {
 pub struct ExprWillPanic;
 
 impl TokenVariant<api::Expression> for Expr {
-	type FromApiCtx<'a> = ExprFromApiCtx;
-	async fn from_api(api: api::Expression, ctx: &mut Self::FromApiCtx<'_>, _: SrcRange) -> Self {
-		Self::from_api(api, PathSetBuilder::new(), ctx.clone()).await
+	type FromApiCx<'a> = &'a System;
+	async fn from_api(api: api::Expression, cx: &mut Self::FromApiCx<'_>, _: SrcRange) -> Self {
+		Self::from_api(api, PathSetBuilder::new(), ExprFromApiCx { sys: cx.clone() }).await
 	}
-	type ToApiCtx<'a> = ExprWillPanic;
-	async fn into_api(self, ExprWillPanic: &mut Self::ToApiCtx<'_>) -> api::Expression {
+	type ToApiCx<'a> = ExprWillPanic;
+	async fn into_api(self, ExprWillPanic: &mut Self::ToApiCx<'_>) -> api::Expression {
 		panic!("Cannot serialize expr!")
 	}
 }

orchid-host/src/extension.rs

@@ -7,7 +7,6 @@ use std::pin::Pin;
 use std::rc::{Rc, Weak};
 use std::time::Duration;
 
-use async_fn_stream::stream;
 use derive_destructure::destructure;
 use futures::channel::mpsc::{Sender, channel};
 use futures::future::{join, join_all};
@@ -17,15 +16,15 @@ use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use orchid_api_traits::{Decode, Encode, Request};
 use orchid_base::{
-	AtomRepr, Client, ClientExt, CommCx, FmtCtxImpl, Format, IStr, IStrv, IoComm, MsgReaderExt, Pos,
+	AtomRepr, Client, ClientExt, CommCx, FmtCxImpl, Format, IStr, IStrv, IoComm, MsgReaderExt, Pos,
 	ReqHandleExt, ReqReaderExt, Sym, Witness, es, ev, io_comm, is, iv, log, stash, with_stash,
 };
 
 use crate::api;
 use crate::atom::AtomHand;
-use crate::ctx::{Ctx, JoinHandle};
+use crate::cx::{Cx, JoinHandle};
 use crate::dealias::{ChildError, ChildErrorKind, walk};
-use crate::expr::{Expr, ExprFromApiCtx, PathSetBuilder};
+use crate::expr::{Expr, ExprFromApiCx, PathSetBuilder};
 use crate::system::SystemCtor;
 use crate::tree::MemKind;
 
@@ -50,7 +49,7 @@ pub struct ReqPair<R: Request>(R, Sender<R::Response>);
 #[derive(destructure)]
 pub struct ExtensionData {
 	name: String,
-	ctx: Ctx,
+	cx: Cx,
 	comm_cx: Option<CommCx>,
 	join_ext: Option<Box<dyn JoinHandle>>,
 	client: Rc<dyn Client>,
@@ -79,8 +78,8 @@ impl Drop for ExtensionData {
 #[derive(Clone)]
 pub struct Extension(Rc<ExtensionData>);
 impl Extension {
-	pub async fn new(mut init: ExtPort, ctx: Ctx) -> io::Result<Self> {
-		api::HostHeader { logger: ctx.logger.to_api() }.encode(init.input.as_mut()).await.unwrap();
+	pub async fn new(mut init: ExtPort, cx: Cx) -> io::Result<Self> {
+		api::HostHeader { logger: cx.logger.to_api() }.encode(init.input.as_mut()).await.unwrap();
 		init.input.flush().await.unwrap();
 
 		let header = api::ExtensionHeader::decode(init.output.as_mut()).await.unwrap();
@@ -90,8 +89,8 @@ impl Extension {
 			let IoComm { client, cx: comm_cx, srv } = io_comm(init.input, init.output);
 			let weak2 = weak;
 			let weak = weak.clone();
-			let ctx2 = ctx.clone();
-			let join_ext = ctx.clone().spawn(Duration::ZERO, async move {
+			let cx2 = cx.clone();
+			let join_ext = cx.clone().spawn(Duration::ZERO, async move {
 				srv
 					.listen(
 						async |reader| {
@@ -104,11 +103,11 @@ impl Extension {
 								}
 								match notif {
 									api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => {
-										let target = this.0.ctx.exprs.get_expr(acq.1).expect("Invalid ticket");
-										this.0.ctx.exprs.give_expr(target)
+										let target = this.0.cx.exprs.get_expr(acq.1).expect("Invalid ticket");
+										this.0.cx.exprs.give_expr(target)
 									},
 									api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => {
-										this.0.ctx.exprs.take_expr(rel.1);
+										this.0.cx.exprs.take_expr(rel.1);
 									},
 									api::ExtHostNotif::Log(api::Log { category, message }) =>
 										write!(log(&es(category).await), "{message}").await,
@@ -167,7 +166,7 @@ impl Extension {
 									},
 									api::ExtHostReq::Fwd(ref fw @ api::Fwd { ref target, ref method, ref body }) => {
 										let sys =
-											ctx.system_inst(target.owner).await.expect("owner of live atom dropped");
+											cx.system_inst(target.owner).await.expect("owner of live atom dropped");
 										let client = sys.client();
 										let reply = client
 											.request(api::Fwded(target.clone(), *method, body.clone()))
@@ -176,7 +175,7 @@ impl Extension {
 										handle.reply(fw, reply).await
 									},
 									api::ExtHostReq::SysFwd(ref fw @ api::SysFwd(id, ref body)) => {
-										let sys = ctx.system_inst(id).await.unwrap();
+										let sys = cx.system_inst(id).await.unwrap();
 										handle.reply(fw, sys.request(body.clone()).await).await
 									},
 									api::ExtHostReq::SubLex(sl) => {
@@ -192,7 +191,7 @@ impl Extension {
 									},
 									api::ExtHostReq::ExprReq(expr_req) => match expr_req {
 										api::ExprReq::Inspect(ins @ api::Inspect { target }) => {
-											let expr = ctx.exprs.get_expr(target).expect("Invalid ticket");
+											let expr = cx.exprs.get_expr(target).expect("Invalid ticket");
 											handle
 												.reply(&ins, api::Inspected {
 													refcount: expr.strong_count() as u32,
@@ -202,9 +201,9 @@ impl Extension {
 												.await
 										},
 										api::ExprReq::ExprPrint(prt @ api::ExprPrint { target }) => {
-											let msg = match ctx.exprs.get_expr(target) {
+											let msg = match cx.exprs.get_expr(target) {
 												None => "EXPIRED_TICKET".into(),
-												Some(expr) => expr.print(&FmtCtxImpl::default()).await,
+												Some(expr) => expr.print(&FmtCxImpl::default()).await,
 											}
 											.to_api();
 											handle.reply(&prt, msg).await
@@ -212,80 +211,73 @@ impl Extension {
 										api::ExprReq::Create(cre) => {
 											let req = Witness::of(&cre);
 											let api::Create(sys, expr) = cre;
-											let expr = Expr::from_api(expr, PathSetBuilder::new(), ExprFromApiCtx {
-												ctx: ctx.clone(),
-												sys,
+											let expr = Expr::from_api(expr, PathSetBuilder::new(), ExprFromApiCx {
+												sys: cx.system_inst(sys).await.unwrap(),
 											})
 											.await;
 											let expr_id = expr.id();
-											ctx.exprs.give_expr(expr);
+											cx.exprs.give_expr(expr);
 											handle.reply(req, expr_id).await
 										},
 									},
 									api::ExtHostReq::LsModule(ref ls @ api::LsModule(_sys, path)) => {
-										let reply: <api::LsModule as Request>::Response = 'reply: {
-											let path = ev(path).await;
-											let root = (ctx.root.read().await.upgrade())
-												.expect("LSModule called when root isn't in context");
-											let root_data = &*root.0.read().await;
-											let mut walk_ctx = (ctx.clone(), &root_data.consts);
-											let module =
-												match walk(&root_data.root, false, path.iter().cloned(), &mut walk_ctx)
-													.await
-												{
-													Ok(module) => module,
-													Err(ChildError { kind, .. }) => {
-														break 'reply Err(match kind {
-															ChildErrorKind::Private => panic!("Access checking was disabled"),
-															ChildErrorKind::Constant => api::LsModuleError::IsConstant,
-															ChildErrorKind::Missing => api::LsModuleError::InvalidPath,
-														});
-													},
-												};
-											let mut members = std::collections::HashMap::new();
-											for (k, v) in &module.members {
-												let kind = match v.kind(ctx.clone(), &root_data.consts).await {
-													MemKind::Const => api::MemberInfoKind::Constant,
-													MemKind::Module(_) => api::MemberInfoKind::Module,
-												};
-												members.insert(k.to_api(), api::MemberInfo { public: v.public, kind });
-											}
-											Ok(api::ModuleInfo { members })
-										};
+										let reply: <api::LsModule as Request>::Response =
+											'reply: {
+												let path = ev(path).await;
+												let root = (cx.root.read().await.upgrade())
+													.expect("LSModule called when root isn't in context");
+												let root_data = &*root.0.read().await;
+												let mut walk_cx = (cx.clone(), &root_data.consts);
+												let module =
+													match walk(&root_data.root, false, path.iter().cloned(), &mut walk_cx)
+														.await
+													{
+														Ok(module) => module,
+														Err(ChildError { kind, .. }) => {
+															break 'reply Err(match kind {
+																ChildErrorKind::Private => panic!("Access checking was disabled"),
+																ChildErrorKind::Constant => api::LsModuleError::IsConstant,
+																ChildErrorKind::Missing => api::LsModuleError::InvalidPath,
+															});
+														},
+													};
+												let mut members = std::collections::HashMap::new();
+												for (k, v) in &module.members {
+													let kind = match v.kind(cx.clone(), &root_data.consts).await {
+														MemKind::Const => api::MemberInfoKind::Constant,
+														MemKind::Module(_) => api::MemberInfoKind::Module,
+													};
+													members.insert(k.to_api(), api::MemberInfo { public: v.public, kind });
+												}
+												Ok(api::ModuleInfo { members })
+											};
 										handle.reply(ls, reply).await
 									},
-									api::ExtHostReq::ResolveNames(ref rn) => {
-										let api::ResolveNames { constid, names, sys } = rn;
-										let mut resolver = {
-											let systems = ctx.systems.read().await;
-											let weak_sys = systems.get(sys).expect("ResolveNames for invalid sys");
-											let sys = weak_sys.upgrade().expect("ResolveNames after sys drop");
-											sys.name_resolver(*constid).await
+									api::ExtHostReq::ResolveName(ref rn) => {
+										// TODO: decide if field is necessary
+										let api::ResolveName { name, path, sys: _ } = rn;
+										let root =
+											cx.root.read().await.upgrade().expect("ResolveName issued when root not set");
+										let result = root
+											.resolve_name(
+												cx.clone(),
+												Sym::from_api(*path).await,
+												Sym::from_api(*name).await,
+											)
+											.await;
+										let rep = match result {
+											Err(e) => {
+												(this.itable().borrow_mut().strings)
+													.extend(e.iter().map(|e| e.description.clone()));
+												Err(e.to_api())
+											},
+											Ok(name) => Ok(name.to_sym().await.to_api()),
 										};
-										let responses = stream(async |mut cx| {
-											for name in names {
-												cx.emit(match resolver(&ev(*name).await[..]).await {
-													Ok(abs) => {
-														let sym = abs.to_sym().await;
-														this.0.itable.borrow_mut().string_vecs.insert(sym.tok());
-														Ok(sym.to_api())
-													},
-													Err(e) => {
-														(this.0.itable.borrow_mut().strings)
-															.extend(e.iter().map(|e| e.description.clone()));
-														Err(e.to_api())
-													},
-												})
-												.await
-											}
-										})
-										.collect()
-										.await;
-										handle.reply(rn, responses).await
+										handle.reply(rn, rep).await
 									},
 									api::ExtHostReq::ExtAtomPrint(ref eap @ api::ExtAtomPrint(ref atom)) => {
-										let atom = AtomHand::from_api(atom, Pos::None, &mut ctx.clone()).await;
-										let unit = atom.print(&FmtCtxImpl::default()).await;
+										let atom = AtomHand::from_api(atom, Pos::None, &mut cx.clone()).await;
+										let unit = atom.print(&FmtCxImpl::default()).await;
 										handle.reply(eap, unit.to_api()).await
 									},
 								}
@@ -298,7 +290,7 @@ impl Extension {
 			});
 			ExtensionData {
 				name: header2.name.clone(),
-				ctx: ctx2,
+				cx: cx2,
 				comm_cx: Some(comm_cx),
 				systems: (header.systems.iter().cloned())
 					.map(|decl| SystemCtor { decl, ext: WeakExtension(weak2.clone()) })
@@ -316,7 +308,7 @@ impl Extension {
 	#[must_use]
 	pub fn client(&self) -> &dyn Client { &*self.0.client }
 	#[must_use]
-	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
+	pub fn cx(&self) -> &Cx { &self.0.cx }
 	pub fn system_ctors(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
 	#[must_use]
 	pub fn next_pars(&self) -> NonZeroU64 {
@@ -359,11 +351,11 @@ impl Extension {
 	}
 	pub fn system_drop(&self, id: api::SysId) {
 		let rc = self.clone();
-		let _ = self.ctx().spawn(
+		let _ = self.cx().spawn(
 			Duration::ZERO,
 			with_stash(async move {
 				rc.client().request(api::SystemDrop(id)).await.unwrap();
-				rc.ctx().systems.write().await.remove(&id);
+				rc.cx().systems.write().await.remove(&id);
 			}),
 		);
 	}

orchid-host/src/inline.rs

@@ -8,18 +8,18 @@ use orchid_base::{log, on_drop};
 use orchid_extension as ox;
 use unsync_pipe::pipe;
 
-use crate::ctx::Ctx;
+use crate::cx::Cx;
 use crate::extension::ExtPort;
 use crate::task_set::TaskSet;
 
-pub async fn ext_inline(builder: ox::ExtensionBuilder, ctx: Ctx) -> ExtPort {
+pub async fn ext_inline(builder: ox::ExtensionBuilder, cx: Cx) -> ExtPort {
 	let (in_stdin, out_stdin) = pipe(1024);
 	let (in_stdout, out_stdout) = pipe(1024);
 	let (in_stderr, out_stderr) = pipe(1024);
 
 	let name = builder.name;
 
-	std::mem::drop(ctx.spawn(Duration::ZERO, async move {
+	std::mem::drop(cx.spawn(Duration::ZERO, async move {
 		let mut lines = BufReader::new(out_stderr).lines();
 		while let Some(line) = lines.next().await {
 			match line {
@@ -35,7 +35,7 @@ pub async fn ext_inline(builder: ox::ExtensionBuilder, ctx: Ctx) -> ExtPort {
 	let task_set = TaskSet::default();
 	let task_set1 = task_set.clone();
 
-	std::mem::drop(ctx.clone().spawn(Duration::ZERO, async move {
+	std::mem::drop(cx.clone().spawn(Duration::ZERO, async move {
 		let task_set2 = task_set1.clone();
 		builder
 			.run(ox::ExtPort {
@@ -43,11 +43,11 @@ pub async fn ext_inline(builder: ox::ExtensionBuilder, ctx: Ctx) -> ExtPort {
 				output: Box::pin(in_stdout),
 				log: Box::pin(in_stderr),
 				spawn: Rc::new(move |delay, fut| {
-					let ctx1 = ctx.clone();
+					let cx1 = cx.clone();
 					let task_set2 = task_set1.clone();
 					task_set1.with(move |store| {
 						store.add_with(move |id| {
-							ctx1.spawn(delay, async move {
+							cx1.spawn(delay, async move {
 								fut.await;
 								task_set2.with(|store| store.remove(id));
 							})

orchid-host/src/lex.rs

@@ -5,38 +5,34 @@ use std::ops::Range;
 use futures::FutureExt;
 use futures::lock::Mutex;
 use orchid_base::{
-	IStr, OrcErrv, OrcRes, PARENS, SrcRange, Sym, clone, is, mk_errv, name_char, name_start, op_char,
-	report, unrep_space,
+	IStr, OrcErrv, OrcRes, PARENS, SrcRange, TokFromApiCx, clone, is, mk_errv, name_char, name_start,
+	op_char, report, unrep_space,
 };
 
 use crate::api;
-use crate::ctx::Ctx;
-use crate::expr::{Expr, ExprFromApiCtx};
+use crate::expr::Expr;
 use crate::expr_store::ExprStore;
 use crate::extension::InternedTable;
+use crate::parse::HostParseCx;
 use crate::parsed::{ParsTok, ParsTokTree, tt_to_api};
 use crate::system::System;
 
-pub struct LexCtx<'a> {
-	pub systems: &'a [System],
+pub struct LexCx<'a> {
+	pub pcx: &'a dyn HostParseCx,
 	pub source: &'a IStr,
-	pub path: &'a Sym,
 	pub tail: &'a str,
 	pub sub_trees: &'a mut Vec<Expr>,
-	pub ctx: &'a Ctx,
 	pub produced: &'a mut VecDeque<ParsTokTree>,
 }
-impl<'a> LexCtx<'a> {
+impl<'a> LexCx<'a> {
 	#[must_use]
-	pub fn sub<'b>(&'b mut self, pos: u32, produced: &'b mut VecDeque<ParsTokTree>) -> LexCtx<'b>
+	pub fn sub<'b>(&'b mut self, pos: u32, produced: &'b mut VecDeque<ParsTokTree>) -> LexCx<'b>
 	where 'a: 'b {
-		LexCtx {
+		LexCx {
+			pcx: self.pcx,
 			source: self.source,
-			path: self.path,
 			tail: &self.source[pos as usize..],
-			systems: self.systems,
 			sub_trees: &mut *self.sub_trees,
-			ctx: self.ctx,
 			produced,
 		}
 	}
@@ -63,17 +59,22 @@ impl<'a> LexCtx<'a> {
 		i: &RefCell<InternedTable>,
 		exprs: ExprStore,
 	) -> api::TokenTree {
-		tt_to_api(&mut { exprs }, i, subtree).await
+		tt_to_api(&mut { exprs }, &mut *self.pcx.flex_repo().lock().await, i, subtree).await
 	}
 	#[must_use]
 	pub async fn des_subtree(
 		&mut self,
 		tree: api::TokenTree,
-		sys: api::SysId,
+		sys: System,
 		exprs: ExprStore,
 	) -> ParsTokTree {
-		let mut cx = ExprFromApiCtx { ctx: self.ctx.clone(), sys };
-		ParsTokTree::from_api(tree, &mut { exprs }, &mut cx, self.path).await
+		ParsTokTree::from_api(tree, &mut TokFromApiCx {
+			handle: &mut exprs.clone(),
+			extra: &mut &sys,
+			lazy: &mut (sys.clone(), &mut *self.pcx.flex_repo().lock().await),
+			src: self.pcx.src(),
+		})
+		.await
 	}
 	#[must_use]
 	pub fn strip_char(&mut self, tgt: char) -> bool {
@@ -101,106 +102,106 @@ impl<'a> LexCtx<'a> {
 		Some(ret)
 	}
 	pub fn sr_to(&self, start: u32) -> SrcRange { self.sr(start..self.get_pos()) }
-	pub fn sr(&self, range: Range<u32>) -> SrcRange { SrcRange::new(range, self.path) }
+	pub fn sr(&self, range: Range<u32>) -> SrcRange { SrcRange::new(range, &self.pcx.src()) }
 }
 
-pub async fn lex_once(ctx: &mut LexCtx<'_>, exactly_one: bool) -> OrcRes<bool> {
-	ctx.trim(unrep_space);
-	if ctx.tail.is_empty() {
+pub async fn lex_once(cx: &mut LexCx<'_>, exactly_one: bool) -> OrcRes<bool> {
+	cx.trim(unrep_space);
+	if cx.tail.is_empty() {
 		return Ok(false);
 	}
-	let start = ctx.get_pos();
-	let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
+	let start = cx.get_pos();
+	let tok = if cx.strip_prefix("\r\n") || cx.strip_prefix("\r") || cx.strip_prefix("\n") {
 		ParsTok::BR
-	} else if let Some(tail) = (ctx.tail.starts_with(name_start).then_some(ctx.tail))
+	} else if let Some(tail) = (cx.tail.starts_with(name_start).then_some(cx.tail))
 		.and_then(|t| t.trim_start_matches(name_char).strip_prefix("::"))
 	{
-		let name = &ctx.tail[..ctx.tail.len() - tail.len() - "::".len()];
+		let name = &cx.tail[..cx.tail.len() - tail.len() - "::".len()];
 		let mut produced = VecDeque::new();
-		let mut sub_cx = ctx.sub(ctx.pos_from(tail), &mut produced);
+		let mut sub_cx = cx.sub(cx.pos_from(tail), &mut produced);
 		if !lex_once(&mut sub_cx, exactly_one).boxed_local().await? {
 			return Err(mk_errv(
 				is("Unexpected end of source text").await,
 				":: cannot be the last token",
-				[SrcRange::new(start..ctx.get_pos(), ctx.path)],
+				[SrcRange::new(start..cx.get_pos(), &cx.pcx.src())],
 			));
 		}
 		let pos = sub_cx.get_pos();
 		let body = produced.pop_front().expect("lex_once returned true");
-		ctx.set_pos(pos);
+		cx.set_pos(pos);
 		if exactly_one {
-			assert!(ctx.produced.is_empty(), "sub-lex of name:: respects exactly_one")
+			assert!(cx.produced.is_empty(), "sub-lex of name:: respects exactly_one")
 		} else {
-			ctx.produced.extend(produced.into_iter());
+			cx.produced.extend(produced);
 		}
 		ParsTok::NS(is(name).await, Box::new(body))
-	} else if ctx.strip_prefix("--[") {
-		let Some((cmt, tail)) = ctx.tail.split_once("]--") else {
+	} else if cx.strip_prefix("--[") {
+		let Some((cmt, tail)) = cx.tail.split_once("]--") else {
 			return Err(mk_errv(
 				is("Unterminated block comment").await,
 				"This block comment has no ending ]--",
-				[SrcRange::new(start..start + 3, ctx.path)],
+				[SrcRange::new(start..start + 3, &cx.pcx.src())],
 			));
 		};
-		ctx.set_tail(tail);
+		cx.set_tail(tail);
 		ParsTok::Comment(is(cmt).await)
-	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
+	} else if let Some(tail) = cx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
-		ctx.push_pos(end as u32);
+		cx.push_pos(end as u32);
 		ParsTok::Comment(is(&tail[2..end]).await)
-	} else if let Some(tail) = ctx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
-		let start = ctx.get_pos();
+	} else if let Some(tail) = cx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
+		let start = cx.get_pos();
 		let mut produced = VecDeque::new();
-		let mut sub_cx = ctx.sub(ctx.pos_from(tail), &mut produced);
+		let mut sub_cx = cx.sub(cx.pos_from(tail), &mut produced);
 		if !lex_once(&mut sub_cx, exactly_one).boxed_local().await? {
 			return Err(mk_errv(
 				is("Unexpected end of file").await,
 				"Expected a lambda argument and body",
-				[SrcRange::new(start..ctx.get_pos(), ctx.path)],
+				[SrcRange::new(start..cx.get_pos(), &cx.pcx.src())],
 			));
 		}
 		let pos = sub_cx.get_pos();
-		ctx.set_pos(pos);
+		cx.set_pos(pos);
 		let arg = produced.pop_front().expect("lex_once returned true");
 		if exactly_one {
 			assert!(produced.is_empty(), "sub-lexer for lambda arg respects exactly_one");
 		} else {
-			ctx.produced.extend(produced);
+			cx.produced.extend(produced);
 		}
 		ParsTok::LambdaHead(Box::new(arg))
-	} else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
+	} else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| cx.strip_char(*lp)) {
 		let mut body = VecDeque::new();
-		ctx.trim_ws();
-		while !ctx.strip_char(*rp) {
-			let mut sub_cx = ctx.sub(ctx.get_pos(), &mut body);
+		cx.trim_ws();
+		while !cx.strip_char(*rp) {
+			let mut sub_cx = cx.sub(cx.get_pos(), &mut body);
 			if !lex_once(&mut sub_cx, false).boxed_local().await? {
 				return Err(mk_errv(
 					is("unclosed paren").await,
 					format!("this {lp} has no matching {rp}"),
-					[SrcRange::new(start..start + 1, ctx.path)],
+					[SrcRange::new(start..start + 1, &cx.pcx.src())],
 				));
 			}
 			let pos = sub_cx.get_pos();
-			ctx.set_pos(pos);
+			cx.set_pos(pos);
 		}
 		ParsTok::S(*paren, body.into_iter().collect())
-	} else if let Some(res) = sys_lex(ctx).await {
+	} else if let Some(res) = sys_lex(cx).await {
 		let token = res?;
-		ctx.produced.extend(token);
+		cx.produced.extend(token);
 		return Ok(true);
-	} else if ctx.tail.starts_with(name_start) {
-		ParsTok::Name(is(ctx.get_start_matches(name_char)).await)
-	} else if ctx.tail.starts_with(op_char) {
-		let whole_tail = ctx.tail;
-		ctx.pop_char().expect("The above check would have failed");
-		let mut tail_after_op = ctx.tail;
+	} else if cx.tail.starts_with(name_start) {
+		ParsTok::Name(is(cx.get_start_matches(name_char)).await)
+	} else if cx.tail.starts_with(op_char) {
+		let whole_tail = cx.tail;
+		cx.pop_char().expect("The above check would have failed");
+		let mut tail_after_op = cx.tail;
 
 		let mut lookahead = Vec::new();
-		while !ctx.tail.is_empty() && ctx.tail.starts_with(op_char) {
-			match sys_lex(ctx).await {
+		while !cx.tail.is_empty() && cx.tail.starts_with(op_char) {
+			match sys_lex(cx).await {
 				None => {
-					ctx.pop_char();
-					tail_after_op = ctx.tail;
+					cx.pop_char();
+					tail_after_op = cx.tail;
 				},
 				Some(sys_res) => {
 					match sys_res {
@@ -213,49 +214,49 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>, exactly_one: bool) -> OrcRes<bool> {
 		}
 
 		let op_str = &whole_tail[0..whole_tail.len() - tail_after_op.len()];
-		ctx.produced.push_back(ParsTok::Name(is(op_str).await).at(ctx.sr_to(start)));
+		cx.produced.push_back(ParsTok::Name(is(op_str).await).at(cx.sr_to(start)));
 		if exactly_one {
-			ctx.pos_from(tail_after_op);
+			cx.pos_from(tail_after_op);
 		} else {
-			ctx.produced.extend(lookahead);
+			cx.produced.extend(lookahead);
 		}
 		return Ok(true);
 	} else {
 		return Err(mk_errv(
 			is("Unrecognized character").await,
 			"The following syntax is meaningless.",
-			[SrcRange::new(start..start + 1, ctx.path)],
+			[SrcRange::new(start..start + 1, &cx.pcx.src())],
 		));
 	};
-	ctx.produced.push_back(ParsTokTree { tok, sr: ctx.sr_to(start) });
+	cx.produced.push_back(ParsTokTree { tok, sr: cx.sr_to(start) });
 	Ok(true)
 }
 
 /// Parse one token via any of the systems, if we can
 ///
 /// This function never writes lookahead
-pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
-	for sys in ctx.systems {
-		if ctx.tail.starts_with(|c| sys.can_lex(c)) {
-			let (source, pos, path) = (ctx.source.clone(), ctx.get_pos(), ctx.path.clone());
-			let temp_store = ctx.ctx.exprs.derive();
-			let ctx_lck = &Mutex::new(&mut *ctx);
+pub async fn sys_lex(cx: &mut LexCx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
+	for sys in cx.pcx.systems() {
+		if cx.tail.starts_with(|c| sys.can_lex(c)) {
+			let (source, pos, path) = (cx.source.clone(), cx.get_pos(), cx.pcx.src());
+			let temp_store = cx.pcx.cx().exprs.derive();
+			let cx_lck = &Mutex::new(&mut *cx);
 			let temp_store_cb = temp_store.clone();
 			let lx = sys
 				.lex(source, path, pos, |pos| {
 					clone!(temp_store_cb);
 					async move {
-						let mut ctx_g = ctx_lck.lock().await;
+						let mut cx_g = cx_lck.lock().await;
 						let mut produced = VecDeque::new();
-						let mut sub_cx = ctx_g.sub(pos, &mut produced);
+						let mut sub_cx = cx_g.sub(pos, &mut produced);
 						let lex_res = lex_once(&mut sub_cx, true).boxed_local().await;
 						let pos1 = sub_cx.get_pos();
 						eprintln!("Setting sublex pos {pos1}");
-						ctx_g.set_pos(pos1);
+						cx_g.set_pos(pos1);
 						match lex_res {
 							Ok(false) => {
 								let errv = mk_errv(is("End of file").await, "Unexpected end of source text", [
-									ctx_g.sr_to(pos),
+									cx_g.sr_to(pos),
 								]);
 								Err(errv.to_api())
 							},
@@ -264,7 +265,7 @@ pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
 								assert!(produced.is_empty(), "sub-lexer in system respects exactly_one");
 								Ok(api::SubLexed {
 									pos: pos1,
-									tree: ctx_g.ser_subtree(tok, sys.ext().itable(), temp_store_cb.clone()).await,
+									tree: cx_g.ser_subtree(tok, sys.ext().itable(), temp_store_cb.clone()).await,
 								})
 							},
 							Err(e) => {
@@ -282,10 +283,10 @@ pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
 				},
 				Ok(Some(lexed)) => {
 					eprintln!("Lexer plugin ended at {}", lexed.pos);
-					ctx.set_pos(lexed.pos);
+					cx.set_pos(lexed.pos);
 					let mut stable_trees = Vec::new();
 					for tok in lexed.expr {
-						let tree = ctx.des_subtree(tok, sys.id(), temp_store.clone()).await;
+						let tree = cx.des_subtree(tok, sys.clone(), temp_store.clone()).await;
 						stable_trees.push(
 							tree
 								.recur(&async |tt, r| {
@@ -306,21 +307,19 @@ pub async fn sys_lex(ctx: &mut LexCtx<'_>) -> Option<OrcRes<Vec<ParsTokTree>>> {
 	None
 }
 
-pub async fn lex(text: IStr, path: Sym, systems: &[System], ctx: &Ctx) -> OrcRes<Vec<ParsTokTree>> {
+pub async fn lex(text: IStr, pcx: &dyn HostParseCx) -> OrcRes<Vec<ParsTokTree>> {
 	let mut sub_trees = Vec::new();
 	let mut produced = VecDeque::new();
-	let mut ctx = LexCtx {
+	let mut cx = LexCx {
+		pcx,
 		source: &text,
 		sub_trees: &mut sub_trees,
 		tail: &text[..],
-		systems,
-		path: &path,
-		ctx,
 		produced: &mut produced,
 	};
-	ctx.trim(unrep_space);
-	while lex_once(&mut ctx, false).await? {
-		ctx.trim(unrep_space);
+	cx.trim(unrep_space);
+	while lex_once(&mut cx, false).await? {
+		cx.trim(unrep_space);
 	}
 	Ok(produced.into())
 }

orchid-host/src/lib.rs

@@ -3,7 +3,7 @@ use orchid_api as api;
 pub mod atom;
 #[cfg(feature = "orchid-extension")]
 pub mod cmd_system;
-pub mod ctx;
+pub mod cx;
 pub mod dealias;
 #[cfg(feature = "tokio")]
 pub mod dylib;

orchid-host/src/parse.rs

@@ -1,48 +1,89 @@
+use std::rc::Rc;
+
 use futures::FutureExt;
+use futures_locks::Mutex;
 use itertools::Itertools;
 use orchid_base::{
-	Comment, IStr, Import, OrcRes, Paren, Parsed, Snippet, Sym, TokTree, Token, expect_end, fmt, is,
-	line_items, mk_errv, parse_multiname, report, try_pop_no_fluff,
+	Comment, IStr, Import, NameLike, OrcRes, Paren, Parsed, Snippet, SrcRange, Sym, TokTree, Token,
+	expect_end, fmt, is, line_items, mk_errv, parse_multiname, report, token_errv, try_pop_no_fluff,
 };
 use substack::Substack;
 
-use crate::ctx::Ctx;
-use crate::expr::Expr;
-use crate::parsed::{Item, ItemKind, ParsedMemKind, ParsedMember, ParsedModule};
+use crate::cx::Cx;
+use crate::expr::{Expr, ExprKind, PathSetBuilder, bot_expr};
+use crate::parsed::{
+	FlexTreeRepo, Item, ItemKind, ParsSnippet, ParsTok, ParsTokTree, ParsedMemKind, ParsedMember,
+	ParsedModule,
+};
 use crate::system::System;
 
-type ParsSnippet<'a> = Snippet<'a, Expr, Expr>;
-
-pub struct HostParseCtxImpl<'a> {
-	pub ctx: Ctx,
+#[derive(Clone)]
+struct HostParseCxImpl {
+	pub cx: Cx,
 	pub src: Sym,
-	pub systems: &'a [System],
+	pub systems: Vec<System>,
+	pub flex: Rc<Mutex<FlexTreeRepo>>,
 }
 
-impl HostParseCtx for HostParseCtxImpl<'_> {
-	fn ctx(&self) -> &Ctx { &self.ctx }
-	fn systems(&self) -> impl Iterator<Item = &System> { self.systems.iter() }
-	fn src_path(&self) -> Sym { self.src.clone() }
+impl HostParseCx for HostParseCxImpl {
+	fn cx(&self) -> &Cx { &self.cx }
+	fn systems(&self) -> &[System] { &self.systems[..] }
+	fn src(&self) -> Sym { self.src.clone() }
+	fn flex_repo(&self) -> &Mutex<FlexTreeRepo> { &self.flex }
+	fn with_src(&self, src: Sym) -> Self
+	where Self: Sized {
+		let mut this = self.clone();
+		this.src = src;
+		this
+	}
 }
 
-pub trait HostParseCtx {
+/// Generate a parsing context for use with [crate::lex::lex], functions in
+/// [crate::parse], and [crate::tree::Root::add_parsed]
+pub fn mk_parse_cx(
+	cx: &Cx,
+	src: Sym,
+	systems: impl IntoIterator<Item = System>,
+) -> impl HostParseCx {
+	HostParseCxImpl {
+		cx: cx.clone(),
+		src,
+		systems: systems.into_iter().collect_vec(),
+		flex: Rc::default(),
+	}
+}
+
+/// Values implementing this trait are produced with [mk_parse_cx]
+pub trait HostParseCx {
+	/// Environment to-be associated with the loaded source code
 	#[must_use]
-	fn ctx(&self) -> &Ctx;
+	fn cx(&self) -> &Cx;
+	/// Systems that can be referenced from the source file via eg. custom line
+	/// prefixes or lexers
 	#[must_use]
-	fn systems(&self) -> impl Iterator<Item = &System>;
+	fn systems(&self) -> &[System];
+	/// Path to the root module of the source file being processed, also the path
+	/// included in all position info
 	#[must_use]
-	fn src_path(&self) -> Sym;
+	fn src(&self) -> Sym;
+	/// Reflexive trees defined in the source code
+	#[must_use]
+	fn flex_repo(&self) -> &Mutex<FlexTreeRepo>;
+	/// Update root path to load different files
+	#[must_use]
+	fn with_src(&self, src: Sym) -> Self
+	where Self: Sized;
 }
 
 pub async fn parse_items(
-	ctx: &impl HostParseCtx,
+	cx: &impl HostParseCx,
 	path: Substack<'_, IStr>,
 	items: ParsSnippet<'_>,
 ) -> OrcRes<Vec<Item>> {
 	let lines = line_items(items).await;
 	let mut line_ok = Vec::new();
 	for Parsed { output: comments, tail } in lines {
-		match parse_item(ctx, path.clone(), comments, tail).boxed_local().await {
+		match parse_item(cx, path.clone(), comments, tail).boxed_local().await {
 			Err(e) => report(e),
 			Ok(l) => line_ok.extend(l),
 		}
@@ -51,7 +92,7 @@ pub async fn parse_items(
 }
 
 pub async fn parse_item(
-	ctx: &impl HostParseCtx,
+	cx: &impl HostParseCx,
 	path: Substack<'_, IStr>,
 	comments: Vec<Comment>,
 	item: ParsSnippet<'_>,
@@ -60,7 +101,7 @@ pub async fn parse_item(
 		Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
 			n if *n == is("export").await => match try_pop_no_fluff(postdisc).await? {
 				Parsed { output: TokTree { tok: Token::Name(n), .. }, tail } =>
-					parse_exportable_item(ctx, path, comments, true, n.clone(), tail).await,
+					parse_exportable_item(cx, path, comments, true, n.clone(), tail).await,
 				Parsed { output, tail: _ } => Err(mk_errv(
 					is("Malformed export").await,
 					"`export` can either prefix other lines or list names inside ( )",
@@ -75,7 +116,7 @@ pub async fn parse_item(
 					kind: ItemKind::Import(t),
 				})))
 			},
-			n => parse_exportable_item(ctx, path, comments, false, n.clone(), postdisc).await,
+			n => parse_exportable_item(cx, path, comments, false, n.clone(), postdisc).await,
 		},
 		Some(_) =>
 			Err(mk_errv(is("Expected a line type").await, "All lines must begin with a keyword", [
@@ -92,7 +133,7 @@ pub async fn parse_import<'a>(tail: ParsSnippet<'a>) -> OrcRes<Vec<Import>> {
 }
 
 pub async fn parse_exportable_item<'a>(
-	ctx: &impl HostParseCtx,
+	cx: &impl HostParseCx,
 	path: Substack<'_, IStr>,
 	comments: Vec<Comment>,
 	exported: bool,
@@ -100,17 +141,17 @@ pub async fn parse_exportable_item<'a>(
 	tail: ParsSnippet<'a>,
 ) -> OrcRes<Vec<Item>> {
 	let kind = if discr == is("mod").await {
-		let (name, body) = parse_module(ctx, path, tail).await?;
+		let (name, body) = parse_module(cx, path, tail).await?;
 		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemKind::Mod(body) })
-	} else if let Some(parser) = ctx.systems().find_map(|s| s.get_parser(discr.clone())) {
+	} else if let Some(parser) = cx.systems().iter().find_map(|s| s.get_parser(discr.clone())) {
 		return parser
-			.parse(ctx, path, tail.to_vec(), exported, comments, &mut async |stack, lines| {
+			.parse(cx, path, tail.to_vec(), exported, comments, &mut async |stack, lines| {
 				let source = Snippet::new(lines.first().unwrap(), &lines);
-				parse_items(ctx, stack, source).await
+				parse_items(cx, stack, source).await
 			})
 			.await;
 	} else {
-		let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
+		let ext_lines = cx.systems().iter().flat_map(System::line_types).join(", ");
 		return Err(mk_errv(
 			is("Unrecognized line type").await,
 			format!("Line types are: mod, {ext_lines}"),
@@ -121,7 +162,7 @@ pub async fn parse_exportable_item<'a>(
 }
 
 pub async fn parse_module<'a>(
-	ctx: &impl HostParseCtx,
+	cx: &impl HostParseCx,
 	path: Substack<'_, IStr>,
 	tail: ParsSnippet<'a>,
 ) -> OrcRes<(IStr, ParsedModule)> {
@@ -145,5 +186,113 @@ pub async fn parse_module<'a>(
 		));
 	};
 	let path = path.push(name.clone());
-	Ok((name, ParsedModule::new(true, parse_items(ctx, path, body).await?)))
+	Ok((name, ParsedModule::new(true, parse_items(cx, path, body).await?)))
+}
+
+pub async fn lower_v(
+	source: ParsSnippet<'_>,
+	builder: PathSetBuilder<'_, Sym>,
+	path: Sym,
+	cx: &impl HostParseCx,
+) -> Expr {
+	let Some((left, arg, sr, right)) = source.split_lambda() else {
+		return lower_call(source, builder, path, cx).boxed_local().await;
+	};
+	let Ok(argname) = arg.as_multiname() else {
+		return bot_expr(
+			token_errv(arg, "Expected name in lambda argument", |t| {
+				format!("Expected a plain name or namespaced::name, found {t}")
+			})
+			.await,
+		);
+	};
+	let argname = argname.to_sym().await;
+	if left.is_empty() {
+		lower_lambda(&argname, sr, right, builder, path, cx).await
+	} else {
+		let (head_builder, l_expr_builder) = builder.split();
+		ExprKind::Call(
+			lower_call(left, head_builder, path.clone(), cx).boxed_local().await,
+			lower_lambda(&argname, sr, right, l_expr_builder, path, cx).await,
+		)
+		.at(source.sr().pos())
+	}
+}
+
+pub async fn lower_lambda(
+	argname: &Sym,
+	sr: SrcRange,
+	body: ParsSnippet<'_>,
+	builder: PathSetBuilder<'_, Sym>,
+	path: Sym,
+	cx: &impl HostParseCx,
+) -> Expr {
+	let lambda_builder = builder.lambda(argname);
+	let body = lower_v(body, lambda_builder.stack(), path, cx).boxed_local().await;
+	ExprKind::Lambda(lambda_builder.collect(), body).at(sr.pos())
+}
+
+pub async fn lower_call(
+	source: ParsSnippet<'_>,
+	builder: PathSetBuilder<'_, Sym>,
+	path: Sym,
+	cx: &impl HostParseCx,
+) -> Expr {
+	let (f, x) = source.split_last().expect("cannot be called on empty");
+	if f.is_empty() {
+		return lower(x, builder, path, cx).await;
+	}
+	let (f_builder, x_builder) = builder.split();
+	let f = lower_call(f, f_builder, path.clone(), cx).boxed_local().await;
+	ExprKind::Call(f, lower(x, x_builder, path, cx).await).at(source.sr().pos())
+}
+
+pub async fn lower(
+	tt: &ParsTokTree,
+	builder: PathSetBuilder<'_, Sym>,
+	path: Sym,
+	cx: &impl HostParseCx,
+) -> Expr {
+	match &tt.tok {
+		ParsTok::BR | ParsTok::Comment(_) => panic!("Should have been stripped out before lowering"),
+		ParsTok::LambdaHead(_) => panic!("Should have been handled in lower_v"),
+		ParsTok::Bottom(err) => ExprKind::Bottom(err.clone()).at(tt.sr.pos()),
+		ParsTok::Handle(expr) | ParsTok::NewExpr(expr) => expr.clone(),
+		ParsTok::Reflexive(flex) => {
+			let flex_tt = flex.get(&mut *cx.flex_repo().lock().await, &path, &cx.src()).await;
+			lower(&flex_tt, builder, path, cx).boxed_local().await
+		},
+		ParsTok::NS(..) | ParsTok::Name(_) => {
+			let Ok(local_name) = tt.as_multiname() else {
+				return bot_expr(
+					token_errv(tt, "Namespace prefix not followed by name", |tok| {
+						format!("Inside expressions, :: can only be followed by a name. Found {tok}")
+					})
+					.await,
+				);
+			};
+			let local_sym = local_name.to_sym().await;
+			if builder.register_arg(&local_sym) {
+				return ExprKind::Arg.at(tt.sr.pos());
+			}
+			let (_, module) = path.split_last_seg();
+			ExprKind::Const(local_name.prefix(module.iter().cloned()).to_sym().await).at(tt.sr.pos())
+		},
+		ParsTok::S(Paren::Round, body) if body.is_empty() => bot_expr(
+			token_errv(tt, "() not a valid expression", |_| {
+				"All Orchid expressions must have a value, () is not a value".to_string()
+			})
+			.await,
+		),
+		ParsTok::S(Paren::Round, body) => lower_v(Snippet::new(tt, body), builder, path, cx).await,
+		ParsTok::S(..) => bot_expr(
+			token_errv(tt, "[] or {} not a valid expression", |tt| {
+				format!(
+					"[] and {{}} must be processed by custom lines and macros, \
+          {tt} is not a lambda expression"
+				)
+			})
+			.await,
+		),
+	}
 }

orchid-host/src/parsed.rs

@@ -4,11 +4,11 @@ use std::rc::Rc;
 
 use futures::FutureExt;
 use futures::future::{LocalBoxFuture, join_all};
-use hashbrown::HashSet;
+use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use orchid_base::{
-	Comment, FmtCtx, FmtUnit, Format, IStr, IStrv, Import, SrcRange, TokTree, Token, Variants,
-	tl_cache,
+	ClientExt, Comment, FmtCx, FmtUnit, Format, IStr, IStrv, Import, ManagedStrat, Snippet, SrcRange,
+	Sym, TokFromApiCx, TokToApiCx, TokTree, Token, TokenVariant, Variants, stash, tl_cache, ttv_fmt,
 };
 
 use crate::api;
@@ -18,8 +18,88 @@ use crate::expr_store::ExprStore;
 use crate::extension::InternedTable;
 use crate::system::System;
 
-pub type ParsTokTree = TokTree<Expr, Expr>;
-pub type ParsTok = Token<Expr, Expr>;
+struct FlexTreeHandle {
+	sys: System,
+	id: api::ReflexiveTreeId,
+}
+impl FlexTreeHandle {
+	pub fn to_api(&self) -> api::ReflexiveTree {
+		api::ReflexiveTree { sys: self.sys.id(), id: self.id }
+	}
+	pub async fn read(&self, path: &Sym) -> api::TokenTree {
+		(self.sys.client())
+			.request(api::ReflexiveRead { path: path.to_api(), tree: self.to_api() })
+			.await
+			.unwrap()
+	}
+}
+impl Drop for FlexTreeHandle {
+	fn drop(&mut self) {
+		let sys = self.sys.clone();
+		let api_tree = self.to_api();
+		stash(async move {
+			sys.client().request(api::ReflexiveDrop(api_tree)).await.unwrap();
+		})
+	}
+}
+#[derive(Clone)]
+pub struct FlexTree(Rc<FlexTreeHandle>);
+impl FlexTree {
+	pub async fn get(&self, flex: &mut FlexTreeRepo, path: &Sym, src: &Sym) -> ParsTokTree {
+		let tt = self.0.read(path).await;
+		ParsTokTree::from_api(tt, &mut TokFromApiCx {
+			extra: &mut &self.0.sys.clone(),
+			handle: &mut ExprStore::default(),
+			src: src.clone(),
+			lazy: &mut (self.0.sys.clone(), flex),
+		})
+		.await
+	}
+}
+impl Debug for FlexTree {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "{:?}/flextree/{:?}", self.0.id, &self.0.sys)
+	}
+}
+impl Format for FlexTree {
+	async fn print<'a>(&'a self, _c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
+		format!("{self:?}").into()
+	}
+}
+#[derive(Default)]
+pub struct FlexTreeRepo(HashMap<api::ReflexiveTree, FlexTree>);
+impl TokenVariant<api::ReflexiveTree> for FlexTree {
+	type FromApiCx<'a> = (System, &'a mut FlexTreeRepo);
+	async fn from_api(
+		api: api::ReflexiveTree,
+		(sys, repo): &mut Self::FromApiCx<'_>,
+		_: SrcRange,
+	) -> Self {
+		if let Some(tree) = repo.0.get(&api) {
+			tree.clone()
+		} else {
+			assert_eq!(sys.id(), api.sys, "Newly constructed flextree for wrong system");
+			FlexTree(Rc::new(FlexTreeHandle { sys: sys.clone(), id: api.id }))
+		}
+	}
+	type ToApiCx<'a> = FlexTreeRepo;
+	async fn into_api(self, ctx: &mut Self::ToApiCx<'_>) -> api::ReflexiveTree {
+		ctx.0.entry(self.0.to_api()).or_insert_with(|| self.clone());
+		self.0.to_api()
+	}
+}
+
+#[derive(Clone, Copy, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ParsStrat;
+impl ManagedStrat for ParsStrat {
+	type Expr = Expr;
+	type Extra = Expr;
+	type Flex = FlexTree;
+}
+
+pub type ParsTokTree = TokTree<ParsStrat>;
+pub type ParsTok = Token<ParsStrat>;
+pub type ParsSnippet<'a> = Snippet<'a, ParsStrat>;
 
 #[derive(Debug)]
 pub struct Item {
@@ -50,14 +130,14 @@ impl From<Import> for ItemKind {
 }
 
 impl Format for Item {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		let comment_text = self.comments.iter().join("\n");
 		let item_text = match &self.kind {
 			ItemKind::Import(i) => format!("import {i}").into(),
 			ItemKind::Member(mem) => match &mem.kind {
-				ParsedMemKind::Const(_, sys) =>
+				ParsedMemKind::Const(tt) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} via {1}")))
-						.units([mem.name.to_string().into(), sys.print(c).await]),
+						.units([mem.name.to_string().into(), ttv_fmt(tt, c).await]),
 				ParsedMemKind::Mod(module) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
 						.units([mem.name.to_string().into(), module.print(c).boxed_local().await]),
@@ -104,7 +184,7 @@ impl fmt::Debug for ParsedExpr {
 
 #[derive(Debug)]
 pub enum ParsedMemKind {
-	Const(api::ParsedConstId, System),
+	Const(Vec<ParsTokTree>),
 	Mod(ParsedModule),
 }
 impl From<ParsedModule> for ParsedMemKind {
@@ -144,12 +224,12 @@ impl ParsedModule {
 	}
 }
 impl Tree for ParsedModule {
-	type Ctx<'a> = ();
+	type Cx<'a> = ();
 	async fn child(
 		&self,
 		key: IStr,
 		public_only: bool,
-		(): &mut Self::Ctx<'_>,
+		(): &mut Self::Cx<'_>,
 	) -> ChildResult<'_, Self> {
 		if public_only && !self.exports.contains(&key) {
 			return ChildResult::Err(ChildErrorKind::Private);
@@ -180,7 +260,7 @@ impl Tree for ParsedModule {
 	}
 }
 impl Format for ParsedModule {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		let head_str = format!("export ::({})\n", self.exports.iter().join(", "));
 		Variants::default().sequence(self.items.len() + 1, "", "\n", "", false).units_own(
 			[head_str.into()].into_iter().chain(join_all(self.items.iter().map(|i| i.print(c))).await),
@@ -203,6 +283,7 @@ impl ConstPath {
 
 pub async fn tt_to_api(
 	exprs: &mut ExprStore,
+	flex: &mut FlexTreeRepo,
 	i: &RefCell<InternedTable>,
 	subtree: ParsTokTree,
 ) -> api::TokenTree {
@@ -217,5 +298,6 @@ pub async fn tt_to_api(
 			r(tt).await
 		})
 		.await;
-	without_new_expr.into_api(exprs, &mut ExprWillPanic).await
+	let mut cx = TokToApiCx { extra: &mut ExprWillPanic, handle: &mut exprs.clone(), lazy: flex };
+	without_new_expr.into_api(&mut cx).await
 }

orchid-host/src/subprocess.rs

@@ -5,10 +5,10 @@ use futures::{self, AsyncBufReadExt, StreamExt};
 use orchid_base::log;
 use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};
 
-use crate::ctx::Ctx;
+use crate::cx::Cx;
 use crate::extension::ExtPort;
 
-pub async fn ext_command(cmd: std::process::Command, ctx: Ctx) -> std::io::Result<ExtPort> {
+pub async fn ext_command(cmd: std::process::Command, cx: Cx) -> std::io::Result<ExtPort> {
 	let name = cmd.get_program().to_string_lossy().to_string();
 	let mut child = tokio::process::Command::from(cmd)
 		.stdin(std::process::Stdio::piped())
@@ -19,7 +19,7 @@ pub async fn ext_command(cmd: std::process::Command, ctx: Ctx) -> std::io::Resul
 	let stdin = child.stdin.take().unwrap();
 	let stdout = child.stdout.take().unwrap();
 	let child_stderr = child.stderr.take().unwrap();
-	std::mem::drop(ctx.spawn(Duration::ZERO, async move {
+	std::mem::drop(cx.spawn(Duration::ZERO, async move {
 		let _ = child;
 		let mut lines = BufReader::new(child_stderr.compat()).lines();
 		while let Some(line) = lines.next().await {

orchid-host/src/sys_parser.rs

@@ -1,15 +1,17 @@
-use futures::FutureExt;
+use async_fn_stream::stream;
 use futures::future::join_all;
+use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
-use orchid_base::{ClientExt, Comment, IStr, OrcErrv, OrcRes, SrcRange, Sym, es, ttv_from_api};
+use orchid_base::{
+	ClientExt, Comment, IStr, OrcErrv, OrcRes, SrcRange, Sym, TokFromApiCx, es, ttv_from_api,
+};
 use substack::Substack;
 
 use crate::api;
-use crate::expr::ExprFromApiCtx;
 use crate::expr_store::ExprStore;
-use crate::parse::HostParseCtx;
+use crate::parse::HostParseCx;
 use crate::parsed::{
-	Item, ItemKind, ParsTokTree, ParsedMemKind, ParsedMember, ParsedModule, tt_to_api,
+	FlexTreeRepo, Item, ItemKind, ParsTokTree, ParsedMemKind, ParsedMember, ParsedModule, tt_to_api,
 };
 use crate::system::System;
 
@@ -22,21 +24,32 @@ type ModPath<'a> = Substack<'a, IStr>;
 impl Parser {
 	pub async fn parse(
 		&self,
-		ctx: &impl HostParseCtx,
+		cx: &impl HostParseCx,
 		path: ModPath<'_>,
 		line: Vec<ParsTokTree>,
 		exported: bool,
 		comments: Vec<Comment>,
 		callback: &mut impl AsyncFnMut(ModPath<'_>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
 	) -> OrcRes<Vec<Item>> {
-		let mut temp_store = self.system.ctx().exprs.derive();
+		let mut temp_exprs = self.system.cx().exprs.derive();
 		let src_path = line.first().expect("cannot be empty").sr.path();
-		let line =
-			join_all((line.into_iter()).map(|t| async {
-				tt_to_api(&mut temp_store.clone(), self.system.ext().itable(), t).await
-			}))
-			.await;
-		let mod_path = ctx.src_path().suffix(path.unreverse()).await;
+		let line = stream(async |mut h| {
+			for t in line {
+				h.emit(
+					tt_to_api(
+						&mut temp_exprs.clone(),
+						&mut *cx.flex_repo().lock().await,
+						self.system.ext().itable(),
+						t,
+					)
+					.await,
+				)
+				.await
+			}
+		})
+		.collect::<Vec<_>>()
+		.await;
+		let mod_path = cx.src().suffix(path.unreverse()).await;
 		let comments = comments.iter().map(Comment::to_api).collect_vec();
 		let req = api::ParseLine {
 			idx: self.idx,
@@ -49,9 +62,9 @@ impl Parser {
 		};
 		match self.system.client().request(req).await.unwrap() {
 			Ok(parsed_v) =>
-				conv(parsed_v, path, callback, &mut ConvCtx {
-					mod_path: &mod_path,
-					ext_exprs: &mut temp_store,
+				conv(parsed_v, path, callback, &mut ConvCx {
+					ext_exprs: &mut temp_exprs,
+					ext_toks: &mut *cx.flex_repo().lock().await,
 					src_path: &src_path,
 					sys: &self.system,
 				})
@@ -61,17 +74,17 @@ impl Parser {
 	}
 }
 
-struct ConvCtx<'a> {
+struct ConvCx<'a> {
 	sys: &'a System,
-	mod_path: &'a Sym,
 	src_path: &'a Sym,
 	ext_exprs: &'a mut ExprStore,
+	ext_toks: &'a mut FlexTreeRepo,
 }
 async fn conv(
 	parsed_v: Vec<api::ParsedLine>,
 	module: Substack<'_, IStr>,
 	callback: &'_ mut impl AsyncFnMut(Substack<'_, IStr>, Vec<ParsTokTree>) -> OrcRes<Vec<Item>>,
-	ctx: &mut ConvCtx<'_>,
+	cx: &mut ConvCx<'_>,
 ) -> OrcRes<Vec<Item>> {
 	let mut items = Vec::new();
 	for parsed in parsed_v {
@@ -79,8 +92,13 @@ async fn conv(
 			api::ParsedLineKind::Member(api::ParsedMember { name, exported, kind }) =>
 				(name, exported, kind),
 			api::ParsedLineKind::Recursive(rec) => {
-				let mut cx = ExprFromApiCtx { ctx: ctx.sys.ctx().clone(), sys: ctx.sys.id() };
-				let tokens = ttv_from_api(rec, ctx.ext_exprs, &mut cx, ctx.src_path).await;
+				let tokens = ttv_from_api(rec, &mut TokFromApiCx {
+					handle: &mut cx.ext_exprs.clone(),
+					extra: &mut &cx.sys.clone(),
+					lazy: &mut (cx.sys.clone(), &mut *cx.ext_toks),
+					src: cx.src_path.clone(),
+				})
+				.await;
 				items.extend(callback(module.clone(), tokens).await?);
 				continue;
 			},
@@ -89,19 +107,22 @@ async fn conv(
 		let mem_path = module.push(name.clone());
 		let mkind = match kind {
 			api::ParsedMemberKind::Module { lines, use_prelude } => {
-				let items = conv(lines, mem_path, callback, ctx).boxed_local().await?;
+				let items = conv(lines, mem_path, callback, cx).boxed_local().await?;
 				ParsedMemKind::Mod(ParsedModule::new(use_prelude, items))
 			},
-			api::ParsedMemberKind::Constant(cid) => {
-				ctx.sys.0.const_paths.insert(cid, ctx.mod_path.suffix(mem_path.unreverse()).await);
-				ParsedMemKind::Const(cid, ctx.sys.clone())
-			},
+			api::ParsedMemberKind::Constant(tt) => ParsedMemKind::Const(
+				ttv_from_api(tt, &mut TokFromApiCx {
+					handle: &mut cx.ext_exprs.clone(),
+					extra: &mut &cx.sys.clone(),
+					lazy: &mut (cx.sys.clone(), &mut *cx.ext_toks),
+					src: cx.src_path.clone(),
+				})
+				.await,
+			),
 		};
 		items.push(Item {
-			comments: join_all(
-				parsed.comments.iter().map(|c| Comment::from_api(c, ctx.src_path.clone())),
-			)
-			.await,
+			comments: join_all(parsed.comments.iter().map(|c| Comment::from_api(c, cx.src_path.clone())))
+				.await,
 			sr: SrcRange::from_api(&parsed.source_range).await,
 			kind: ItemKind::Member(ParsedMember { name, exported, kind: mkind }),
 		})

orchid-host/src/system.rs

@@ -9,18 +9,13 @@ use futures::future::join_all;
 use futures_locks::RwLock;
 use hashbrown::HashMap;
 use itertools::Itertools;
-use memo_map::MemoMap;
-use orchid_base::{
-	Client, ClientExt, FmtCtx, FmtUnit, Format, IStr, IteratorPrint, NameLike, OrcRes, Sym, VName,
-	VPath, char_filter_match, es, is, mk_errv_floating,
-};
+use orchid_base::{Client, ClientExt, FmtCx, FmtUnit, Format, IStr, Sym, char_filter_match, es};
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
 use crate::api;
 use crate::atom::{AtomHand, WeakAtomHand};
-use crate::ctx::Ctx;
-use crate::dealias::walk;
+use crate::cx::Cx;
 use crate::extension::{Extension, WeakExtension};
 use crate::sys_parser::Parser;
 use crate::tree::Root;
@@ -28,7 +23,7 @@ use crate::tree::Root;
 #[derive(destructure)]
 pub(crate) struct SystemInstData {
 	deps: Vec<System>,
-	ctx: Ctx,
+	cx: Cx,
 	ext: Extension,
 	decl_id: api::SysDeclId,
 	lex_filter: api::CharFilter,
@@ -36,7 +31,6 @@ pub(crate) struct SystemInstData {
 	line_types: Vec<IStr>,
 	prelude: Vec<Sym>,
 	owned_atoms: RwLock<HashMap<api::AtomId, WeakAtomHand>>,
-	pub(crate) const_paths: MemoMap<api::ParsedConstId, Sym>,
 }
 impl Drop for SystemInstData {
 	fn drop(&mut self) { self.ext.system_drop(self.id); }
@@ -64,7 +58,7 @@ impl System {
 	#[must_use]
 	pub fn ext(&self) -> &Extension { &self.0.ext }
 	#[must_use]
-	pub fn ctx(&self) -> &Ctx { &self.0.ctx }
+	pub fn cx(&self) -> &Cx { &self.0.cx }
 	#[must_use]
 	pub fn deps(&self) -> &[System] { &self.0.deps }
 	#[must_use]
@@ -120,7 +114,7 @@ impl System {
 	}
 	pub(crate) fn drop_atom(&self, dropped_atom_id: api::AtomId) {
 		let this = self.0.clone();
-		let _ = self.0.ctx.spawn(Duration::ZERO, async move {
+		let _ = self.0.cx.spawn(Duration::ZERO, async move {
 			this.ext.client().request(api::AtomDrop(this.id, dropped_atom_id)).await.unwrap();
 			this.owned_atoms.write().await.remove(&dropped_atom_id);
 		});
@@ -129,52 +123,9 @@ impl System {
 	pub fn downgrade(&self) -> WeakSystem {
 		WeakSystem(Rc::downgrade(&self.0), self.0.decl_id, self.ext().downgrade())
 	}
-	/// Implementation of [api::ResolveNames]
-	pub(crate) async fn name_resolver(
-		&self,
-		orig: api::ParsedConstId,
-	) -> impl AsyncFnMut(&[IStr]) -> OrcRes<VName> + use<> {
-		let root = self.0.ctx.root.read().await.upgrade().expect("find_names when root not in context");
-		let orig = self.0.const_paths.get(&orig).expect("origin for find_names invalid").clone();
-		let ctx = self.0.ctx.clone();
-		async move |rel| {
-			let cwd = orig.split_last_seg().1;
-			let root_data = &mut *root.0.write().await;
-			let walk_ctx = &mut (ctx.clone(), &root_data.consts);
-			let cmod = (walk(&root_data.root, false, cwd.iter().cloned(), walk_ctx).await)
-				.expect("the parent module of a constant should exist");
-			let (selector, tail) = rel.split_first().expect("Names cannot be empty");
-			if cmod.members.get(selector).is_some() {
-				return Ok(VName::new(cwd.iter().chain(rel).cloned()).unwrap());
-			}
-			match cmod.imports.get(selector) {
-				Some(Ok(dest)) => return Ok(dest.target.to_vname().suffix(tail.iter().cloned())),
-				Some(Err(dests)) => {
-					return Err(mk_errv_floating(
-						is("Ambiguous name").await,
-						format!(
-							"{selector} could refer to {}",
-							dests.iter().map(|ri| &ri.target).display("or")
-						),
-					));
-				},
-				None => (),
-			}
-			if root_data.root.members.get(selector).is_some() {
-				return Ok(VName::new(rel.iter().cloned()).expect("split_first was called above"));
-			}
-			if tail.is_empty() {
-				return Ok(VPath::new(cwd.iter().cloned()).name_with_suffix(selector.clone()));
-			}
-			Err(mk_errv_floating(
-				is("Invalid name").await,
-				format!("{selector} doesn't refer to a module"),
-			))
-		}
-	}
 }
 impl Format for System {
-	async fn print<'a>(&'a self, _c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, _c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		let ctor = (self.0.ext.system_ctors().find(|c| c.id() == self.0.decl_id))
 			.expect("System instance with no associated constructor");
 		format!("System({} @ {} #{})", ctor.name(), ctor.priority(), self.0.id.0).into()
@@ -215,20 +166,19 @@ impl SystemCtor {
 		let depends = deps.iter().map(|si| si.id()).collect_vec();
 		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
 		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
-		let id = ext.ctx().next_sys_id();
+		let id = ext.cx().next_sys_id();
 		let sys_inst =
 			ext.client().request(api::NewSystem { depends, id, system: self.decl.id }).await.unwrap();
 		let data = System(Rc::new(SystemInstData {
 			deps,
 			decl_id: self.decl.id,
 			ext: ext.clone(),
-			ctx: ext.ctx().clone(),
+			cx: ext.cx().clone(),
 			lex_filter: sys_inst.lex_filter,
 			line_types: join_all(sys_inst.line_types.iter().map(|m| es(*m))).await,
 			id,
 			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok))).await,
 			owned_atoms: RwLock::new(HashMap::new()),
-			const_paths: MemoMap::new(),
 		}));
 		let api_module_root = api::Module {
 			members: (sys_inst.const_root.into_iter())
@@ -236,7 +186,7 @@ impl SystemCtor {
 				.collect_vec(),
 		};
 		let root = Root::from_api(api_module_root, &data).await;
-		ext.ctx().systems.write().await.insert(id, data.downgrade());
+		ext.cx().systems.write().await.insert(id, data.downgrade());
 		(root, data)
 	}
 }
@@ -289,7 +239,7 @@ pub async fn init_systems(
 		walk_deps(&mut to_load, &mut to_load_ordered, Substack::Bottom.new_frame(tgt))?;
 	}
 	let mut systems = HashMap::<&str, System>::new();
-	let mut root = Root::new(exts.first().unwrap().ctx().clone());
+	let mut root = Root::new(exts.first().unwrap().cx().clone());
 	for ctor in to_load_ordered.iter() {
 		let (sys_root, sys) = ctor.run(ctor.depends().map(|n| systems[n].clone()).collect()).await;
 		systems.insert(ctor.name(), sys);

orchid-host/src/task_set.rs

@@ -3,7 +3,7 @@ use std::rc::Rc;
 
 use orchid_base::IdStore;
 
-use crate::ctx::JoinHandle;
+use crate::cx::JoinHandle;
 
 #[derive(Clone)]
 pub struct TaskSet {

orchid-host/src/tree.rs

@@ -1,6 +1,7 @@
 //! This tree isn't Clone because lazy subtrees are guaranteed to only be loaded
 //! once
 use std::cell::RefCell;
+use std::collections::HashSet;
 use std::rc::{Rc, Weak};
 use std::slice;
 
@@ -13,35 +14,36 @@ use hashbrown::hash_map::Entry;
 use itertools::Itertools;
 use memo_map::MemoMap;
 use orchid_base::{
-	ClientExt, CodeGenInfo, IStr, IStrv, NameLike, OrcRes, Pos, Sym, VPath, clone, es, is, iv,
-	mk_errv, report,
+	CodeGenInfo, IStr, IStrv, IteratorPrint, NameLike, OrcRes, Pos, Snippet, Sym, VName, VPath,
+	clone, es, is, iv, mk_errv, mk_errv_floating, report,
 };
 
 use crate::api;
-use crate::ctx::Ctx;
+use crate::cx::Cx;
 use crate::dealias::{ChildErrorKind, Tree, absolute_path, resolv_glob, walk};
-use crate::expr::{Expr, ExprFromApiCtx, PathSetBuilder};
-use crate::parsed::{ItemKind, ParsedMemKind, ParsedModule};
+use crate::expr::{Expr, ExprFromApiCx, PathSetBuilder};
+use crate::parse::{HostParseCx, lower_v};
+use crate::parsed::{ItemKind, ParsTokTree, ParsedMemKind, ParsedModule};
 use crate::system::System;
 
 pub struct RootData {
 	pub root: Mod,
 	pub consts: MemoMap<Sym, Expr>,
-	pub ctx: Ctx,
+	pub cx: Cx,
 }
 #[derive(Clone)]
 pub struct Root(pub Rc<RwLock<RootData>>);
 impl Root {
 	#[must_use]
-	pub fn new(ctx: Ctx) -> Self {
-		Root(Rc::new(RwLock::new(RootData { root: Mod::default(), consts: MemoMap::default(), ctx })))
+	pub fn new(cx: Cx) -> Self {
+		Root(Rc::new(RwLock::new(RootData { root: Mod::default(), consts: MemoMap::default(), cx })))
 	}
 	#[must_use]
 	pub async fn from_api(api: api::Module, sys: &System) -> Self {
 		let consts = MemoMap::new();
-		let mut tfac = TreeFromApiCtx { consts: &consts, path: iv(&[][..]).await, sys };
+		let mut tfac = TreeFromApicx { consts: &consts, path: iv(&[][..]).await, sys };
 		let root = Mod::from_api(api, &mut tfac).await;
-		Root(Rc::new(RwLock::new(RootData { root, consts, ctx: sys.ctx().clone() })))
+		Root(Rc::new(RwLock::new(RootData { root, consts, cx: sys.cx().clone() })))
 	}
 	pub async fn merge(&self, new: &Root) -> Result<Self, MergeErr> {
 		let this = self.0.write().await;
@@ -50,25 +52,25 @@ impl Root {
 		for (k, v) in this.consts.iter().chain(that.consts.iter()) {
 			consts.insert(k.clone(), v.clone());
 		}
-		let root = this.root.merge(&that.root, this.ctx.clone(), &consts).await?;
-		Ok(Self(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() }))))
+		let root = this.root.merge(&that.root, this.cx.clone(), &consts).await?;
+		Ok(Self(Rc::new(RwLock::new(RootData { root, consts, cx: this.cx.clone() }))))
 	}
 	#[must_use]
-	pub async fn add_parsed(&self, parsed: &ParsedModule, pars_prefix: Sym) -> Self {
+	pub async fn add_parsed(&self, parsed: &ParsedModule, pcx: &impl HostParseCx) -> Self {
 		let mut ref_this = self.0.write().await;
 		let this = &mut *ref_this;
 		let mut deferred_consts = Vec::new();
 		let consts = this.consts.clone();
-		let mut tfpctx = FromParsedCtx {
+		let mut tfpcx = FromParsedCx {
 			pars_root: parsed,
 			deferred_consts: &mut deferred_consts,
 			consts: &consts,
-			pars_prefix: pars_prefix.clone(),
+			pars_prefix: pcx.src(),
 			root: &this.root,
-			ctx: &this.ctx,
+			cx: &this.cx,
 		};
-		let mut module = Mod::from_parsed(parsed, pars_prefix.clone(), &mut tfpctx).await;
-		for step in pars_prefix.iter().rev() {
+		let mut module = Mod::from_parsed(parsed, pcx.src(), &mut tfpcx).await;
+		for step in pcx.src().segs().rev() {
 			let kind = OnceCell::from(MemKind::Module(module));
 			let members = HashMap::from([(
 				step.clone(),
@@ -76,15 +78,14 @@ impl Root {
 			)]);
 			module = Mod { imports: HashMap::new(), members }
 		}
-		let root = (this.root.merge(&module, this.ctx.clone(), &consts).await)
+		let root = (this.root.merge(&module, this.cx.clone(), &consts).await)
 			.expect("Merge conflict between parsed and existing module");
-		let new = Root(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() })));
-		*this.ctx.root.write().await = new.downgrade();
-		for (path, sys_id, pc_id) in deferred_consts {
-			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
-			let api_expr = sys.client().request(api::FetchParsedConst(sys.id(), pc_id)).await.unwrap();
-			let cx = ExprFromApiCtx { sys: sys_id, ctx: this.ctx.clone() };
-			let expr = Expr::from_api(api_expr, PathSetBuilder::new(), cx).await;
+		let new = Root(Rc::new(RwLock::new(RootData { root, consts, cx: this.cx.clone() })));
+		*this.cx.root.write().await = new.downgrade();
+		for DeferredConstant { file, path, value } in deferred_consts {
+			let pcx = pcx.with_src(file);
+			let snip = Snippet::new(&value[0], &value[..]);
+			let expr = lower_v(snip, PathSetBuilder::new(), path.clone(), &pcx).await;
 			new.0.write().await.consts.insert(path, expr);
 		}
 		new
@@ -96,9 +97,9 @@ impl Root {
 			return Ok(val.clone());
 		}
 		// load the node, then check if this "walk" call added it to the map
-		let ctx = this.ctx.clone();
+		let cx = this.cx.clone();
 		let module =
-			walk(&this.root, false, name.iter().cloned(), &mut (ctx.clone(), &this.consts)).await;
+			walk(&this.root, false, name.iter().cloned(), &mut (cx.clone(), &this.consts)).await;
 		if let Some(val) = this.consts.get(&name) {
 			return Ok(val.clone());
 		}
@@ -121,6 +122,44 @@ impl Root {
 	}
 	#[must_use]
 	pub fn downgrade(&self) -> WeakRoot { WeakRoot(Rc::downgrade(&self.0)) }
+	pub async fn resolve_name(&self, cx: Cx, orig: Sym, rel: Sym) -> OrcRes<VName> {
+		let cwd = orig.split_last_seg().1;
+		let root_data = &mut *self.0.write().await;
+		let walk_cx = &mut (cx.clone(), &root_data.consts);
+		let cmod = (walk(&root_data.root, false, cwd.iter().cloned(), walk_cx).await)
+			.expect("the parent module of a constant should exist");
+		let (selector, tail) = rel.split_first_seg();
+		let mut options = HashSet::new();
+		if cmod.members.get(&selector).is_some() {
+			return Ok(VName::new(cwd.iter().chain(&rel[..]).cloned()).unwrap());
+		}
+		options.extend(cmod.members.keys().cloned());
+		match cmod.imports.get(&selector) {
+			Some(Ok(dest)) => return Ok(dest.target.to_vname().suffix(tail.iter().cloned())),
+			Some(Err(dests)) => {
+				return Err(mk_errv_floating(
+					is("Ambiguous name").await,
+					format!("{selector} could refer to {}", dests.iter().map(|ri| &ri.target).display("or")),
+				));
+			},
+			None => (),
+		}
+		options.extend(cmod.imports.keys().cloned());
+		if root_data.root.members.get(&selector).is_some() {
+			return Ok(VName::new(rel.iter().cloned()).expect("split_first was called above"));
+		}
+		options.extend(root_data.root.members.keys().cloned());
+		if tail.is_empty() {
+			return Ok(VPath::new(cwd.iter().cloned()).name_with_suffix(selector.clone()));
+		}
+		Err(mk_errv_floating(
+			is("Invalid name").await,
+			format!(
+				"{selector} doesn't refer to a module from {orig}, valid options are {}",
+				options.iter().join(", ")
+			),
+		))
+	}
 }
 
 #[derive(Clone)]
@@ -135,16 +174,16 @@ impl Default for WeakRoot {
 	fn default() -> Self { Self::new() }
 }
 
-pub struct TreeFromApiCtx<'a> {
+pub struct TreeFromApicx<'a> {
 	pub sys: &'a System,
 	pub consts: &'a MemoMap<Sym, Expr>,
 	pub path: IStrv,
 }
-impl<'a> TreeFromApiCtx<'a> {
+impl<'a> TreeFromApicx<'a> {
 	#[must_use]
-	pub async fn push<'c>(&'c self, name: IStr) -> TreeFromApiCtx<'c> {
+	pub async fn push<'c>(&'c self, name: IStr) -> TreeFromApicx<'c> {
 		let path = iv(&self.path.iter().cloned().chain([name]).collect_vec()).await;
-		TreeFromApiCtx { path, consts: self.consts, sys: self.sys }
+		TreeFromApicx { path, consts: self.consts, sys: self.sys }
 	}
 }
 
@@ -161,23 +200,23 @@ pub struct Mod {
 }
 impl Mod {
 	#[must_use]
-	pub async fn from_api(api: api::Module, ctx: &mut TreeFromApiCtx<'_>) -> Self {
+	pub async fn from_api(api: api::Module, cx: &mut TreeFromApicx<'_>) -> Self {
 		let mut members = HashMap::new();
 		for mem in api.members {
 			let mem_name = es(mem.name).await;
-			let vname = VPath::new(ctx.path.iter().cloned()).name_with_suffix(mem_name.clone());
+			let vname = VPath::new(cx.path.iter().cloned()).name_with_suffix(mem_name.clone());
 			let name = vname.to_sym().await;
 			let (lazy, kind) = match mem.kind {
 				api::MemberKind::Lazy(id) =>
-					(Some(LazyMemberHandle { id, sys: ctx.sys.id(), path: name.clone() }), None),
+					(Some(LazyMemberHandle { id, sys: cx.sys.id(), path: name.clone() }), None),
 				api::MemberKind::Const(val) => {
-					let cx = ExprFromApiCtx { ctx: ctx.sys.ctx().clone(), sys: ctx.sys.id() };
-					let expr = Expr::from_api(val, PathSetBuilder::new(), cx).await;
-					ctx.consts.insert(name.clone(), expr);
+					let xfacx = ExprFromApiCx { sys: cx.sys.clone() };
+					let expr = Expr::from_api(val, PathSetBuilder::new(), xfacx).await;
+					cx.consts.insert(name.clone(), expr);
 					(None, Some(MemKind::Const))
 				},
 				api::MemberKind::Module(m) => {
-					let m = Self::from_api(m, &mut ctx.push(mem_name.clone()).await).boxed_local().await;
+					let m = Self::from_api(m, &mut cx.push(mem_name.clone()).await).boxed_local().await;
 					(None, Some(MemKind::Module(m)))
 				},
 			};
@@ -193,7 +232,7 @@ impl Mod {
 		Self { members, imports: HashMap::new() }
 	}
 	#[must_use]
-	async fn from_parsed(parsed: &ParsedModule, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
+	async fn from_parsed(parsed: &ParsedModule, path: Sym, cx: &mut FromParsedCx<'_>) -> Self {
 		let imports_by_name = (parsed.get_imports().into_iter())
 			.filter_map(|i| Some((i.name.clone()?, i)))
 			.into_group_map();
@@ -203,15 +242,15 @@ impl Mod {
 			match absolute_path(&path, &import.path).await {
 				Err(e) => report(e.err_obj(pos, &import.path.to_string()).await),
 				Ok(abs_path) => {
-					let names_res = match abs_path.strip_prefix(&ctx.pars_prefix[..]) {
+					let names_res = match abs_path.strip_prefix(&cx.pars_prefix[..]) {
 						None => {
-							let mut tree_ctx = (ctx.ctx.clone(), ctx.consts);
-							resolv_glob(&path, ctx.root, &abs_path, pos, &mut tree_ctx).await
+							let mut tree_cx = (cx.cx.clone(), cx.consts);
+							resolv_glob(&path, cx.root, &abs_path, pos, &mut tree_cx).await
 						},
 						Some(sub_tgt) => {
-							let sub_path = (path.strip_prefix(&ctx.pars_prefix[..]))
+							let sub_path = (path.strip_prefix(&cx.pars_prefix[..]))
 								.expect("from_parsed called with path outside pars_prefix");
-							resolv_glob(sub_path, ctx.pars_root, sub_tgt, pos, &mut ()).await
+							resolv_glob(sub_path, cx.pars_root, sub_tgt, pos, &mut ()).await
 						},
 					};
 					let abs_path = abs_path.to_sym().await;
@@ -232,7 +271,7 @@ impl Mod {
 		}
 		let mut imports = HashMap::new();
 		if parsed.use_prelude {
-			let systems = ctx.ctx.systems.read().await;
+			let systems = cx.cx.systems.read().await;
 			for sys in systems.values().flat_map(|weak| weak.upgrade()) {
 				for prelude_item in sys.prelude() {
 					imports.insert(
@@ -301,7 +340,7 @@ impl Mod {
 			match &item.kind {
 				ItemKind::Member(mem) => {
 					let path = path.to_vname().suffix([mem.name.clone()]).to_sym().await;
-					let kind = OnceCell::from(MemKind::from_parsed(&mem.kind, path.clone(), ctx).await);
+					let kind = OnceCell::from(MemKind::from_parsed(&mem.kind, path.clone(), cx).await);
 					members.insert(
 						mem.name.clone(),
 						Rc::new(Member { kind, lazy: RefCell::default(), public: mem.exported }),
@@ -315,7 +354,7 @@ impl Mod {
 	pub async fn merge(
 		&self,
 		other: &Mod,
-		ctx: Ctx,
+		cx: Cx,
 		consts: &MemoMap<Sym, Expr>,
 	) -> Result<Mod, MergeErr> {
 		if !self.imports.is_empty() || !other.imports.is_empty() {
@@ -330,9 +369,9 @@ impl Mod {
 			if own.public != mem.public {
 				return Err(MergeErr { path: VPath::new([]), kind: MergeErrKind::Visibility });
 			}
-			match (own.kind(ctx.clone(), consts).await, mem.kind(ctx.clone(), consts).await) {
+			match (own.kind(cx.clone(), consts).await, mem.kind(cx.clone(), consts).await) {
 				(MemKind::Module(own_sub), MemKind::Module(sub)) => {
-					match own_sub.merge(sub, ctx.clone(), consts).boxed_local().await {
+					match own_sub.merge(sub, cx.clone(), consts).boxed_local().await {
 						Ok(module) => {
 							members.insert(
 								key.clone(),
@@ -373,22 +412,28 @@ pub enum MergeErrKind {
 	Const,
 }
 
-pub struct FromParsedCtx<'a> {
+struct DeferredConstant {
+	file: Sym,
+	path: Sym,
+	value: Vec<ParsTokTree>,
+}
+
+pub struct FromParsedCx<'a> {
 	pars_prefix: Sym,
 	pars_root: &'a ParsedModule,
 	root: &'a Mod,
-	ctx: &'a Ctx,
+	cx: &'a Cx,
 	consts: &'a MemoMap<Sym, Expr>,
-	deferred_consts: &'a mut Vec<(Sym, api::SysId, api::ParsedConstId)>,
+	deferred_consts: &'a mut Vec<DeferredConstant>,
 }
 
 impl Tree for Mod {
-	type Ctx<'a> = (Ctx, &'a MemoMap<Sym, Expr>);
+	type Cx<'a> = (Cx, &'a MemoMap<Sym, Expr>);
 	async fn child(
 		&self,
 		key: IStr,
 		public_only: bool,
-		(ctx, consts): &mut Self::Ctx<'_>,
+		(cx, consts): &mut Self::Cx<'_>,
 	) -> crate::dealias::ChildResult<'_, Self> {
 		let Some(member) = self.members.get(&key) else {
 			return Err(ChildErrorKind::Missing);
@@ -396,7 +441,7 @@ impl Tree for Mod {
 		if public_only && !member.public {
 			return Err(ChildErrorKind::Private);
 		}
-		match &member.kind(ctx.clone(), consts).await {
+		match &member.kind(cx.clone(), consts).await {
 			MemKind::Module(m) => Ok(m),
 			MemKind::Const => Err(ChildErrorKind::Constant),
 		}
@@ -413,10 +458,10 @@ pub struct Member {
 }
 impl Member {
 	#[must_use]
-	pub async fn kind<'a>(&'a self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> &'a MemKind {
+	pub async fn kind<'a>(&'a self, cx: Cx, consts: &MemoMap<Sym, Expr>) -> &'a MemKind {
 		(self.kind.get_or_init(async {
 			let handle = self.lazy.borrow_mut().take().expect("If kind is uninit, lazy must be Some");
-			handle.run(ctx, consts).await
+			handle.run(cx, consts).await
 		}))
 		.await
 	}
@@ -428,13 +473,15 @@ pub enum MemKind {
 }
 impl MemKind {
 	#[must_use]
-	async fn from_parsed(parsed: &ParsedMemKind, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
+	async fn from_parsed(parsed: &ParsedMemKind, path: Sym, cx: &mut FromParsedCx<'_>) -> Self {
 		match parsed {
-			ParsedMemKind::Const(id, sys) => {
-				ctx.deferred_consts.push((path, sys.id(), *id));
+			ParsedMemKind::Const(ptt) => {
+				let tt = (ptt.iter().next())
+					.unwrap_or_else(|| panic!("{path} is empty! this is an extension error"));
+				cx.deferred_consts.push(DeferredConstant { file: tt.sr.path(), path, value: ptt.clone() });
 				MemKind::Const
 			},
-			ParsedMemKind::Mod(m) => MemKind::Module(Mod::from_parsed(m, path, ctx).boxed_local().await),
+			ParsedMemKind::Mod(m) => MemKind::Module(Mod::from_parsed(m, path, cx).boxed_local().await),
 		}
 	}
 }
@@ -447,12 +494,12 @@ pub struct LazyMemberHandle {
 }
 impl LazyMemberHandle {
 	#[must_use]
-	pub async fn run(mut self, ctx: Ctx, consts: &MemoMap<Sym, Expr>) -> MemKind {
-		let sys = ctx.system_inst(self.sys).await.expect("Missing system for lazy member");
+	pub async fn run(mut self, cx: Cx, consts: &MemoMap<Sym, Expr>) -> MemKind {
+		let sys = cx.system_inst(self.sys).await.expect("Missing system for lazy member");
 		match sys.get_tree(self.id).await {
 			api::MemberKind::Const(c) => {
-				let ctx = ExprFromApiCtx { sys: sys.id(), ctx: ctx.clone() };
-				let expr = Expr::from_api(c, PathSetBuilder::new(), ctx).await;
+				let cx = ExprFromApiCx { sys };
+				let expr = Expr::from_api(c, PathSetBuilder::new(), cx).await;
 				let (.., path) = self.destructure();
 				consts.insert(path, expr);
 				MemKind::Const
@@ -460,12 +507,12 @@ impl LazyMemberHandle {
 			api::MemberKind::Module(m) => {
 				let (.., path) = self.destructure();
 				MemKind::Module(
-					Mod::from_api(m, &mut TreeFromApiCtx { sys: &sys, consts, path: path.tok() }).await,
+					Mod::from_api(m, &mut TreeFromApicx { sys: &sys, consts, path: path.tok() }).await,
 				)
 			},
 			api::MemberKind::Lazy(id) => {
 				self.id = id;
-				self.run(ctx, consts).boxed_local().await
+				self.run(cx, consts).boxed_local().await
 			},
 		}
 	}

orchid-std/src/macros/instantiate_tpl.rs

@@ -0,0 +1,63 @@
+use std::borrow::Cow;
+
+use never::Never;
+use orchid_extension::gen_expr::new_atom;
+use orchid_extension::{Atomic, Expr, OwnedAtom, OwnedVariant, TAtom, ToExpr, TryFromExpr, exec};
+
+use crate::macros::mactree::SlotFillStrategy;
+use crate::{MacTok, MacTree};
+
+/// Populate all the Slots in a MacTree with separate MacTrees given as
+/// arguments. It is expected that the template and subsequent arguments must
+/// all be dealiased
+#[derive(Clone)]
+pub struct InstantiateTplCall {
+	pub(crate) tpl: MacTree,
+	pub(crate) argv: Vec<Expr>,
+}
+impl Atomic for InstantiateTplCall {
+	type Variant = OwnedVariant;
+	type Data = ();
+}
+impl OwnedAtom for InstantiateTplCall {
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	type Refs = Never;
+	// Technically must be supported but shouldn't actually ever be called
+	async fn call_ref(&self, arg: Expr) -> impl ToExpr {
+		if !self.argv.is_empty() {
+			eprintln!(
+				"Copying partially applied instantiate_tpl call. This is an internal value.\
+				\nIt should be fully consumed within generated code."
+			);
+		}
+		self.clone().call(arg).await
+	}
+	async fn call(mut self, arg: Expr) -> impl ToExpr {
+		exec(async move |mut h| {
+			let expr = h.exec::<Expr>(arg.clone()).await.expect("Conversion to Expr shouldn't fail");
+			self.argv.push(expr);
+			if self.argv.len() < self.tpl.slotc {
+				return new_atom(self);
+			}
+			let mut args = self.argv.into_iter();
+			let ret = (self.tpl.map(&mut false, &mut async |mt| {
+				let strat = match mt.tok() {
+					MacTok::Slot(strat) => strat,
+					_ => return None,
+				};
+				let expr = args.next().expect("Not enough arguments to fill all slots");
+				match strat {
+					SlotFillStrategy::Value => Some(MacTok::Value(expr).at(mt.pos())),
+					SlotFillStrategy::Unwrap => match TAtom::<MacTree>::try_from_expr(expr).await {
+						Err(e) => Some(MacTok::Bottom(e).at(mt.pos())),
+						Ok(tree) => Some(tree.own().await),
+					},
+				}
+			}))
+			.await;
+			assert!(args.next().is_none(), "Too many arguments for all slots");
+			new_atom(ret)
+		})
+		.await
+	}
+}

orchid-std/src/macros/let_line.rs

@@ -1,20 +1,14 @@
-use std::pin::pin;
-
-use futures::{FutureExt, StreamExt, stream};
-use hashbrown::HashMap;
-use itertools::Itertools;
 use orchid_base::{
-	Comment, OrcRes, Paren, Parsed, Snippet, Sym, expect_tok, is, report, token_errv,
+	Comment, OrcErrv, OrcRes, Paren, Parsed, SrcRange, clone, expect_tok, is, sym, token_errv,
 	try_pop_no_fluff, with_reporter,
 };
-use orchid_extension::{
-	ConstCtx, PSnippet, PTok, PTokTree, ParsCtx, ParsedLine, Parser, TAtom, TryFromExpr,
-};
+use orchid_extension::gen_tok::{self, FlexTree, GenSnippet, GenTok, GenTokTree, flex_tok, x_tok};
+use orchid_extension::{ParsCtx, ParsedLine, Parser, gen_expr};
 
-use crate::macros::lower::lower;
-use crate::macros::mactree::{MacTok, MacTree, MacTreeSeq};
-use crate::macros::ph_lexer::PhAtom;
-use crate::macros::resolve::resolve;
+use crate::macros::mactree::{MacTok, PhKind, SlotFillStrategy};
+use crate::macros::mactree_dealias::dealias_mac_v;
+use crate::macros::mactree_parse::{MacParseStrat, parse_tokv};
+use crate::{MacTree, api};
 
 #[derive(Default)]
 pub struct LetLine;
@@ -24,7 +18,7 @@ impl Parser for LetLine {
 		_: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: PSnippet<'a>,
+		line: GenSnippet<'a>,
 	) -> OrcRes<Vec<ParsedLine>> {
 		let sr = line.sr();
 		let Parsed { output: name_tok, tail } = try_pop_no_fluff(line).await?;
@@ -35,91 +29,65 @@ impl Parser for LetLine {
 			return Err(err.await);
 		};
 		let Parsed { tail, .. } = expect_tok(tail, is("=").await).await?;
-		let aliased = parse_tokv(tail).await;
-		Ok(vec![ParsedLine::cnst(
-			&line.sr(),
-			&comments,
-			exported,
-			name.clone(),
-			async move |mut ctx| {
-				let macro_input =
-					MacTok::S(Paren::Round, with_reporter(dealias_mac_v(&aliased, &ctx)).await?).at(sr.pos());
-				let resolved = resolve(ctx.handle(), macro_input).await;
-				Ok(lower(&resolved, substack::Substack::Bottom).await)
-			},
-		)])
-	}
-}
-
-pub async fn dealias_mac_v(aliased: &MacTreeSeq, ctx: &ConstCtx<'_>) -> MacTreeSeq {
-	let keys = aliased.glossary().iter().cloned().collect_vec();
-	let mut names: HashMap<_, _> = HashMap::new();
-	let mut stream = pin!(ctx.names(&keys).zip(stream::iter(&keys)));
-	while let Some((canonical, local)) = stream.next().await {
-		match canonical {
-			Err(e) => report(e),
-			Ok(name) => {
-				names.insert(local.clone(), name);
-			},
-		}
-	}
-	aliased
-		.map(&mut false, &mut async |tree| match &*tree.tok {
-			MacTok::Name(n) => names.get(n).map(|new_n| MacTok::Name(new_n.clone()).at(tree.pos())),
-			_ => None,
-		})
-		.await
-}
-
-pub async fn parse_tokv(line: PSnippet<'_>) -> MacTreeSeq {
-	if let Some((idx, arg)) = line.iter().enumerate().find_map(|(i, x)| Some((i, x.as_lambda()?))) {
-		let (head, lambda) = line.split_at(idx as u32);
-		let (_, body) = lambda.split_first().unwrap();
-		let body = parse_tokv(body).boxed_local().await;
-		let mut all = parse_tokv_no_lambdas(&head).await;
-		match parse_tok(arg).await {
-			Some(arg) => all.push(MacTok::Lambda(arg, body).at(lambda.sr().pos())),
-			None => report(
-				token_errv(arg, "Lambda argument fluff", |arg| {
-					format!("Lambda arguments must be a valid token, found meaningless fragment {arg}")
-				})
-				.await,
-			),
-		};
-		MacTreeSeq::new(all)
-	} else {
-		MacTreeSeq::new(parse_tokv_no_lambdas(&line).await)
-	}
-}
-
-async fn parse_tokv_no_lambdas(line: &[PTokTree]) -> Vec<MacTree> {
-	stream::iter(line).filter_map(parse_tok).collect::<Vec<_>>().await
-}
-
-pub async fn parse_tok(tree: &PTokTree) -> Option<MacTree> {
-	let tok = match &tree.tok {
-		PTok::Bottom(errv) => MacTok::Bottom(errv.clone()),
-		PTok::BR | PTok::Comment(_) => return None,
-		PTok::Name(n) => MacTok::Name(Sym::new([n.clone()]).await.unwrap()),
-		PTok::NS(..) => match tree.as_multiname() {
-			Ok(mn) => MacTok::Name(mn.to_sym().await),
-			Err(nested) => {
-				report(
-					token_errv(tree, ":: can only be followed by a name in an expression", |tok| {
-						format!("Expected name, found {tok}")
+		// Parse a mactree that cannot contain placeholders and push all reflexive
+		// trees onto a vector to be returned separately and applied during execution
+		pub struct LetParsStrat(Vec<(api::ReflexiveTree, SrcRange)>);
+		impl<'a> MacParseStrat<'a> for LetParsStrat {
+			type Err = OrcErrv;
+			async fn with_placeholder(&mut self, ph: PhKind, next: &GenTokTree) -> OrcRes<MacTok> {
+				Err(
+					token_errv(next, "placeholder $ in let line", |next| {
+						format!("{ph}{next} can only be used in templates and patterns, not values")
 					})
 					.await,
-				);
-				return parse_tok(nested).boxed_local().await;
-			},
-		},
-		PTok::Handle(expr) => match TAtom::<PhAtom>::try_from_expr(expr.clone()).await {
-			Err(_) => MacTok::Value(expr.clone()),
-			Ok(ta) => MacTok::Ph(ta.value.to_full().await),
-		},
-		PTok::NewExpr(never) => match *never {},
-		PTok::LambdaHead(_) => panic!("Lambda-head handled in the sequence parser"),
-		PTok::S(p, body) => MacTok::S(*p, parse_tokv(Snippet::new(tree, body)).boxed_local().await),
-	};
-	Some(tok.at(tree.sr().pos()))
+				)
+			}
+			async fn with_reflexive(
+				&mut self,
+				flex: api::ReflexiveTree,
+				sr: SrcRange,
+			) -> OrcRes<MacTree> {
+				self.0.push((flex, sr.clone()));
+				Ok(MacTok::Slot(SlotFillStrategy::Value).at(sr))
+			}
+		}
+		let mut strat = LetParsStrat(Vec::new());
+		let aliased = match parse_tokv(tail, &mut strat).await {
+			Ok(aliased) => aliased,
+			Err(e) =>
+				return Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name.clone(), vec![
+					x_tok(gen_expr::bot(e)).await.at(line.sr()),
+				])]),
+		};
+		// Dealiased mactree with reflexive trees missing
+		let tpl = flex_tok(async move |path| {
+			clone!(aliased, sr);
+			match with_reporter(dealias_mac_v(&aliased, path)).await {
+				Err(e) => gen_tok::x_tok(gen_expr::bot(e)).await.at(sr),
+				Ok(tokv) => {
+					let mactree = MacTok::S(Paren::Round, tokv).at(sr.clone());
+					gen_tok::x_tok(gen_expr::new_atom(mactree)).await.at(sr)
+				},
+			}
+		})
+		.at(line.sr());
+		let body = if strat.0.is_empty() {
+			tpl
+		} else {
+			let mut call = vec![gen_tok::ref_tok(sym!(macros::instantiate_tpl)).await.at(tail.sr()), tpl];
+			for (flex, sr) in strat.0 {
+				call.push(GenTok::Reflexive(FlexTree::Existing(flex)).at(sr));
+			}
+			gen_tok::GenTok::S(Paren::Round, call).at(tail.sr())
+		};
+
+		Ok(vec![ParsedLine::cnst(&line.sr(), &comments, exported, name.clone(), vec![
+			gen_tok::ref_tok(sym!(macros::lower)).await.at(tail.sr().prefix()),
+			gen_tok::GenTok::S(Paren::Round, vec![
+				gen_tok::ref_tok(sym!(macros::resolve)).await.at(tail.sr().prefix()),
+				body,
+			])
+			.at(tail.sr()),
+		])])
+	}
 }

orchid-std/src/macros/lower.rs

@@ -15,7 +15,7 @@ pub async fn lower(mt: &MacTree, args: Substack<'_, Sym>) -> GExpr {
 	match mt.tok() {
 		MacTok::Resolved(inner) => lower(inner, args).boxed_local().await,
 		MacTok::Bottom(b) => bot(b.clone()),
-		MacTok::Slot => bot(mk_errv(
+		MacTok::Slot(_) => bot(mk_errv(
 			is("Lowering intermediary slotted mactree").await,
 			"Found a Slot during lowering, which should only exist in temporary templates",
 			[mt.pos()],

orchid-std/src/macros/macro_lib.rs

@@ -3,6 +3,7 @@ use orchid_extension::tree::{GenMember, fun, prefix};
 use orchid_extension::{TAtom, exec};
 use substack::Substack;
 
+use crate::macros::instantiate_tpl::InstantiateTplCall;
 use crate::macros::lower::lower;
 use crate::macros::mactree::MacTree;
 use crate::macros::resolve::resolve;
@@ -15,6 +16,14 @@ pub async fn gen_macro_lib() -> Vec<GenMember> {
 			exec(async move |mut h| new_atom(resolve(&mut h, tpl.own().await).await)).await
 		}),
 		fun(true, "lower", async |mt: TAtom<MacTree>| lower(&mt.own().await, Substack::Bottom).await),
+		fun(true, "instantiate_tpl", async |mt: TAtom<MacTree>| {
+			let tpl = mt.own().await;
+			if tpl.slotc == 0 {
+				new_atom(tpl)
+			} else {
+				new_atom(InstantiateTplCall { tpl, argv: vec![] })
+			}
+		}),
 		prefix("common", [
 			build_macro(None, ["..", "_", "="]).finish(),
 			build_macro(Some(30), ["+"])

orchid-std/src/macros/macro_line.rs

@@ -6,18 +6,20 @@ use async_once_cell::OnceCell;
 use futures::StreamExt;
 use itertools::Itertools;
 use orchid_base::{
-	Comment, OrcRes, Paren, Parsed, Pos, Snippet, Token, clone, expect_end, expect_tok, is,
-	line_items, mk_errv, report, token_errv, try_pop_no_fluff, with_reporter,
+	Comment, OrcErrv, OrcRes, Paren, Parsed, Snippet, SrcRange, Sym, Token, clone, expect_end,
+	expect_tok, is, line_items, mk_errv, report, token_errv, try_pop_no_fluff, with_reporter,
 };
 use orchid_extension::gen_expr::new_atom;
-use orchid_extension::{
-	PSnippet, ParsCtx, ParsedLine, ParsedLineKind, Parser, TAtom, ToExpr, TryFromExpr, p_v2gen,
-};
+use orchid_extension::gen_tok::{GenSnippet, GenTok, GenTokTree, flex_tok};
+use orchid_extension::{ParsCtx, ParsedLine, ParsedLineKind, Parser, TAtom, ToExpr, TryFromExpr};
 
-use crate::macros::let_line::{dealias_mac_v, parse_tokv};
+use super::mactree::PhKind;
 use crate::macros::macro_value::{Macro, MacroData, Rule};
+use crate::macros::mactree::Ph;
+use crate::macros::mactree_dealias::dealias_mac_v;
+use crate::macros::mactree_parse::{MacParseStrat, parse_tokv};
 use crate::macros::rule::matcher::Matcher;
-use crate::{Int, MacTok};
+use crate::{Int, MacTok, api};
 
 #[derive(Default)]
 pub struct MacroLine;
@@ -27,7 +29,7 @@ impl Parser for MacroLine {
 		ctx: ParsCtx<'a>,
 		exported: bool,
 		comments: Vec<Comment>,
-		line: PSnippet<'a>,
+		line: GenSnippet<'a>,
 	) -> OrcRes<Vec<ParsedLine>> {
 		if exported {
 			return Err(mk_errv(
@@ -108,21 +110,50 @@ impl Parser for MacroLine {
 				));
 				continue;
 			};
-			let pattern = parse_tokv(pattern).await;
-			let mut placeholders = Vec::new();
-			pattern
-				.map(&mut false, &mut async |tok| {
-					if let MacTok::Ph(ph) = tok.tok() {
-						placeholders.push((ph.clone(), tok.pos()))
-					}
-					None
-				})
-				.await;
-			let mut body_tt = VecDeque::from(p_v2gen(body.to_vec()));
-			for (ph, ph_pos) in placeholders.iter().rev() {
-				let Pos::SrcRange(ph_sr) = &ph_pos else {
-					panic!("Parsed from toktree always has srcrange")
-				};
+			struct Strat(Vec<(Ph, SrcRange)>);
+			impl<'a> MacParseStrat<'a> for Strat {
+				type Err = OrcErrv;
+				async fn with_reflexive(
+					&mut self,
+					_: api::ReflexiveTree,
+					sr: SrcRange,
+				) -> Result<crate::MacTree, Self::Err> {
+					Err(mk_errv(
+						is("Unsupported token in macro pattern").await,
+						"Patterns don't support atoms and other complex token types",
+						[sr],
+					))
+				}
+				async fn with_placeholder(
+					&mut self,
+					kind: PhKind,
+					next: &'a GenTokTree,
+				) -> Result<MacTok, Self::Err> {
+					let ph = match &next.tok {
+						GenTok::Name(name) => Ph { name: name.clone(), kind },
+						_ =>
+							return Err(
+								token_errv(next, "Expected placeholder name", |tok| {
+									format!("Expected a single name, found {tok}")
+								})
+								.await,
+							),
+					};
+					self.0.push((ph.clone(), next.sr.clone()));
+					Ok(MacTok::Ph(ph))
+				}
+			}
+			let mut strat = Strat(Vec::new());
+			let pattern = match parse_tokv(pattern, &mut strat).await {
+				Err(e) => {
+					report(e);
+					continue;
+				},
+				Ok(pattern) => pattern,
+			};
+			let placeholders = strat.0;
+			let mut body_tt = VecDeque::from(body.to_vec());
+			for (ph, ph_sr) in placeholders.iter().rev() {
 				let arg = Token::Name(ph.name.clone()).at(ph_sr.clone());
 				body_tt.push_front(Token::LambdaHead(Box::new(arg)).at(ph_sr.clone()));
 			}
@@ -142,34 +173,45 @@ impl Parser for MacroLine {
 		let mac_cell = Rc::new(OnceCell::new());
 		let rules = Rc::new(rules);
 		for (kw, sr) in &*keywords {
-			clone!(mac_cell, rules, module, macro_name, prio);
+			clone!(mac_cell, rules, module, macro_name, prio, kw);
 			let kw_key = is(&format!("__macro__{kw}")).await;
-			lines.push(ParsedLine::cnst(&sr.clone(), &comments, true, kw_key, async move |cctx| {
-				let mac_future = async {
-					let rules = with_reporter(
-						stream(async |mut h| {
-							for (body, ph_names, pattern_rel) in rules.iter() {
-								let pattern = dealias_mac_v(pattern_rel, &cctx).await;
-								let ph_names = ph_names.iter().map(|(ph, _)| ph.name.clone()).collect_vec();
-								match Matcher::new(pattern.clone()).await {
-									Ok(matcher) =>
-										h.emit(Rule { body: body.clone(), matcher, pattern, ph_names }).await,
-									Err(e) => report(e),
-								}
-							}
-						})
-						.collect::<Vec<_>>(),
-					)
-					.await?;
-					Ok(Macro(Rc::new(MacroData {
-						canonical_name: module.suffix([macro_name]).await,
-						module,
-						prio: prio.map(|i| i.0 as u64),
-						rules,
-					})))
-				};
-				mac_cell.get_or_init(mac_future).await.clone().map(new_atom).to_gen().await
-			}))
+			let sr1 = sr.clone();
+			lines.push(ParsedLine::cnst(&sr.clone(), &comments, true, kw_key, vec![
+				flex_tok(move |path: Sym| {
+					let sr = sr1.clone();
+					clone!(path, mac_cell, rules, module, macro_name, prio, kw);
+					async move {
+						let rules = &rules;
+						let mac_future = async {
+							let rules = with_reporter(
+								stream(async |mut h| {
+									for (body, ph_names, pattern_rel) in rules.iter() {
+										let pattern = dealias_mac_v(pattern_rel, path.clone()).await;
+										let ph_names = ph_names.iter().map(|(ph, _)| ph.name.clone()).collect_vec();
+										match Matcher::new(pattern.clone()).await {
+											Ok(matcher) =>
+												h.emit(Rule { body: body.clone(), matcher, pattern, ph_names }).await,
+											Err(e) => report(e),
+										}
+									}
+								})
+								.collect::<Vec<_>>(),
+							)
+							.await?;
+							Ok(Macro(Rc::new(MacroData {
+								canonical_name: module.suffix([macro_name]).await,
+								module,
+								prio: prio.map(|i| i.0 as u64),
+								rules,
+							})))
+						};
+						let mac_res = mac_cell.get_or_init(mac_future).await.clone();
+						let mac_or_bottom = mac_res.map(new_atom).to_gen().await;
+						mac_or_bottom.into_tok().at(sr.clone())
+					}
+				})
+				.at(sr.clone()),
+			]))
 		}
 		Ok(lines)
 	}

orchid-std/src/macros/macro_system.rs

@@ -5,11 +5,12 @@ use orchid_extension::{
 	AtomOps, AtomicFeatures, LexerObj, ParserObj, System, SystemCard, SystemCtor, SystemHandle,
 };
 
+use crate::macros::instantiate_tpl::InstantiateTplCall;
 use crate::macros::let_line::LetLine;
 use crate::macros::macro_lib::gen_macro_lib;
 use crate::macros::macro_line::MacroLine;
 use crate::macros::macro_value::Macro;
-use crate::macros::mactree_lexer::{InstantiateTplCall, MacTreeLexer};
+use crate::macros::mactree_lexer::MacTreeLexer;
 use crate::macros::match_macros::{MatcherAtom, gen_match_macro_lib};
 use crate::macros::ph_lexer::{PhAtom, PhLexer};
 use crate::macros::stdlib::gen_std_macro_lib;

orchid-std/src/macros/mactree.rs

@@ -8,7 +8,7 @@ use futures::{FutureExt, StreamExt};
 use hashbrown::HashSet;
 use orchid_api_derive::Coding;
 use orchid_base::{
-	FmtCtx, FmtUnit, Format, IStr, OrcErrv, Paren, Pos, Sym, Variants, indent, tl_cache,
+	FmtCx, FmtUnit, Format, IStr, OrcErrv, Paren, Pos, Sym, Variants, indent, tl_cache,
 };
 use orchid_extension::{Atomic, Expr, OwnedAtom, OwnedVariant};
 
@@ -42,6 +42,7 @@ pub struct MacTreeSeq {
 	pub items: Rc<Vec<MacTree>>,
 	pub top_glossary: Rc<HashSet<Sym>>,
 	pub glossary: Rc<HashSet<Sym>>,
+	pub slotc: usize,
 }
 impl MacTreeSeq {
 	pub fn new(i: impl IntoIterator<Item = MacTree>) -> Self {
@@ -55,7 +56,12 @@ impl MacTreeSeq {
 			}
 			items.push(item);
 		}
-		Self { items: Rc::new(items), top_glossary: Rc::new(top_glossary), glossary: Rc::new(glossary) }
+		Self {
+			slotc: items.iter().map(|i| i.slotc).sum(),
+			items: Rc::new(items),
+			top_glossary: Rc::new(top_glossary),
+			glossary: Rc::new(glossary),
+		}
 	}
 	pub async fn map<F: AsyncFnMut(MacTree) -> Option<MacTree>>(
 		&self,
@@ -95,6 +101,7 @@ impl MacTreeSeq {
 			(Err(left), Err(right)) => left.iter().chain(&right[..]).cloned().collect(),
 		};
 		Self {
+			slotc: items.iter().map(|i| i.slotc).sum(),
 			items: Rc::new(items),
 			top_glossary: union_rc_sets([self.top_glossary, other.top_glossary]),
 			glossary: union_rc_sets([self.glossary, other.glossary]),
@@ -102,7 +109,7 @@ impl MacTreeSeq {
 	}
 }
 impl Format for MacTreeSeq {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		mtreev_fmt(&self.items[..], c).await
 	}
 }
@@ -111,6 +118,7 @@ impl Format for MacTreeSeq {
 pub struct MacTree {
 	pub pos: Pos,
 	pub tok: Rc<MacTok>,
+	pub slotc: usize,
 	pub glossary: Rc<HashSet<Sym>>,
 }
 impl MacTree {
@@ -134,7 +142,7 @@ impl MacTree {
 				),
 				MacTok::Name(_) | MacTok::Value(_) => return self.clone(),
 				MacTok::Resolved(inner) => return inner.map(changed, map).boxed_local().await,
-				MacTok::Slot | MacTok::Ph(_) | MacTok::Bottom(_) => return self.clone(),
+				MacTok::Slot(_) | MacTok::Ph(_) | MacTok::Bottom(_) => return self.clone(),
 				MacTok::S(p, body) => MacTok::S(*p, body.map(changed, map).boxed_local().await),
 			},
 		};
@@ -149,23 +157,35 @@ impl OwnedAtom for MacTree {
 	type Refs = ();
 
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-	async fn print_atom<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print_atom<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("'{0}")))
 			.units([self.tok.print(c).await])
 	}
 }
 impl Format for MacTree {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		self.tok.print(c).await
 	}
 }
 
+/// Indicates how `macros::instantiate_tpl` should convert the corresponding
+/// argument into a [MacTok]
+#[derive(Debug, Clone)]
+pub enum SlotFillStrategy {
+	/// Only accepts an atom of [MacTree], inlines the tok directly in place of
+	/// the [MacTok::Slot]
+	Unwrap,
+	/// Accepts any [Expr], places it into a [MacTok::Value]
+	Value,
+}
+
 #[derive(Debug, Clone)]
 pub enum MacTok {
 	S(Paren, MacTreeSeq),
 	Name(Sym),
-	/// Only permitted in arguments to `instantiate_tpl`
-	Slot,
+	// TODO: does Slot need metadata regarding whether to attempt to unwrap the current argument?
+	/// Only permitted in arguments to `instantiate_tpl`.
+	Slot(SlotFillStrategy),
 	Value(Expr),
 	Lambda(MacTree, MacTreeSeq),
 	/// Only permitted in "pattern" values produced by macro blocks, which are
@@ -187,20 +207,37 @@ pub enum MacTok {
 impl MacTok {
 	pub fn build_glossary(&self) -> Rc<HashSet<Sym>> {
 		match self {
-			MacTok::Bottom(_) | MacTok::Ph(_) | MacTok::Slot | MacTok::Value(_) | MacTok::Resolved(_) =>
-				Rc::default(),
+			MacTok::Bottom(_)
+			| MacTok::Ph(_)
+			| MacTok::Slot(_)
+			| MacTok::Value(_)
+			| MacTok::Resolved(_) => Rc::default(),
 			MacTok::Name(sym) => Rc::new(HashSet::from([sym.clone()])),
 			MacTok::S(_, body) => union_rc_sets(body.items.iter().map(|mt| mt.glossary.clone())),
 			MacTok::Lambda(arg, body) =>
 				union_rc_sets(body.items.iter().chain([arg]).map(|mt| mt.glossary.clone())),
 		}
 	}
+	pub fn slotc(&self) -> usize {
+		match self {
+			MacTok::Bottom(_) | MacTok::Ph(_) | MacTok::Value(_) | MacTok::Name(_) => 0,
+			MacTok::Slot(_) => 1,
+			MacTok::Resolved(r) => r.slotc,
+			MacTok::Lambda(arg, body) => arg.slotc + body.slotc,
+			MacTok::S(_, body) => body.slotc,
+		}
+	}
 	pub fn at(self, pos: impl Into<Pos>) -> MacTree {
-		MacTree { pos: pos.into(), glossary: self.build_glossary(), tok: Rc::new(self) }
+		MacTree {
+			pos: pos.into(),
+			glossary: self.build_glossary(),
+			slotc: self.slotc(),
+			tok: Rc::new(self),
+		}
 	}
 }
 impl Format for MacTok {
-	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCx + ?Sized + 'a)) -> FmtUnit {
 		match self {
 			Self::Value(v) => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("$({0b})")))
 				.units([v.print(c).await]),
@@ -216,7 +253,8 @@ impl Format for MacTok {
 				Paren::Square => tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("[{0b}]"))),
 			}
 			.units([body.print(c).await]),
-			Self::Slot => "$SLOT".into(),
+			Self::Slot(SlotFillStrategy::Unwrap) => "$SLOT".into(),
+			Self::Slot(SlotFillStrategy::Value) => "$FLEX-SLOT".into(),
 			Self::Resolved(res) => res.print(c).boxed_local().await,
 			Self::Bottom(err) => match err.one() {
 				Some(err) => format!("Bottom({err}) ").into(),
@@ -228,7 +266,7 @@ impl Format for MacTok {
 
 pub async fn mtreev_fmt<'b>(
 	v: impl IntoIterator<Item = &'b MacTree>,
-	c: &(impl FmtCtx + ?Sized),
+	c: &(impl FmtCx + ?Sized),
 ) -> FmtUnit {
 	FmtUnit::sequence("", " ", "", true, join_all(v.into_iter().map(|t| t.print(c))).await)
 }
@@ -240,13 +278,7 @@ pub struct Ph {
 }
 impl Display for Ph {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		match self.kind {
-			PhKind::Scalar => write!(f, "${}", self.name),
-			PhKind::Vector { at_least_one: false, priority: 0 } => write!(f, "..${}", self.name),
-			PhKind::Vector { at_least_one: true, priority: 0 } => write!(f, "...${}", self.name),
-			PhKind::Vector { at_least_one: false, priority } => write!(f, "..${}:{priority}", self.name),
-			PhKind::Vector { at_least_one: true, priority } => write!(f, "...${}:{priority}", self.name),
-		}
+		write!(f, "{}{}", self.kind, self.name)
 	}
 }
 
@@ -256,6 +288,18 @@ pub enum PhKind {
 	Vector { at_least_one: bool, priority: u8 },
 }
 
+impl Display for PhKind {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self {
+			Self::Scalar => write!(f, "$"),
+			Self::Vector { at_least_one: false, priority: 0 } => write!(f, "..$"),
+			Self::Vector { at_least_one: true, priority: 0 } => write!(f, "...$"),
+			Self::Vector { at_least_one: false, priority } => write!(f, "..${priority}:"),
+			Self::Vector { at_least_one: true, priority } => write!(f, "...${priority}:"),
+		}
+	}
+}
+
 /// reverse "or". Inside, the flag is always false, but raising it will raise
 /// the outside flag too.
 async fn ro<T>(flag: &mut bool, cb: impl AsyncFnOnce(&mut bool) -> T) -> T {

orchid-std/src/macros/mactree_dealias.rs

@@ -0,0 +1,37 @@
+use hashbrown::HashMap;
+use orchid_base::{Sym, report};
+use orchid_extension::gen_tok::resolve_name;
+
+use crate::macros::mactree::MacTreeSeq;
+use crate::{MacTok, MacTree};
+
+pub async fn resolve_names(pos: Sym, keys: impl IntoIterator<Item = Sym>) -> HashMap<Sym, Sym> {
+	let mut names: HashMap<_, _> = HashMap::new();
+	for local in keys {
+		let canonical = resolve_name(&pos, &local).await;
+		match canonical {
+			Err(e) => report(e),
+			Ok(name) => {
+				names.insert(local.clone(), name);
+			},
+		}
+	}
+	names
+}
+
+fn dealias_mapping(tree: MacTree, names: &HashMap<Sym, Sym>) -> Option<MacTree> {
+	match &*tree.tok {
+		MacTok::Name(n) => names.get(n).map(|new_n| MacTok::Name(new_n.clone()).at(tree.pos())),
+		_ => None,
+	}
+}
+
+pub async fn dealias_mac(aliased: &MacTree, pos: Sym) -> MacTree {
+	let names = resolve_names(pos, aliased.glossary().iter().cloned()).await;
+	aliased.map(&mut false, &mut async |tree| dealias_mapping(tree, &names)).await
+}
+
+pub async fn dealias_mac_v(aliased: &MacTreeSeq, pos: Sym) -> MacTreeSeq {
+	let names = resolve_names(pos, aliased.glossary().iter().cloned()).await;
+	aliased.map(&mut false, &mut async |tree| dealias_mapping(tree, &names)).await
+}

orchid-std/src/macros/mactree_lexer.rs

@@ -1,21 +1,19 @@
 use std::borrow::Cow;
-use std::cell::RefCell;
 use std::ops::RangeInclusive;
 
-use futures::FutureExt;
-use itertools::chain;
+use itertools::{Itertools};
 use never::Never;
-use orchid_base::{OrcRes, PARENS, Paren, Pos, Sym, fmt, is, mk_errv};
+use orchid_base::{OrcRes, Paren, clone, token_errv};
 use orchid_extension::gen_expr::new_atom;
-use orchid_extension::tree::{GenTok, GenTokTree, x_tok};
-use orchid_extension::{
-	Atomic, Expr, LexContext, Lexer, OwnedAtom, OwnedVariant, TAtom, ToExpr, TryFromExpr,
-	err_not_applicable, exec, p_tree2gen,
-};
+use orchid_extension::gen_tok::{FlexTree, GenTok, GenTokTree, flex_tok, x_tok};
+use orchid_extension::{LexContext, Lexer, err_not_applicable};
 
+use super::mactree::PhKind;
 use crate::api;
-use crate::macros::let_line::parse_tok;
-use crate::macros::mactree::{MacTok, MacTree, MacTreeSeq};
+use crate::macros::instantiate_tpl::InstantiateTplCall;
+use crate::macros::mactree::{MacTok, MacTree, SlotFillStrategy};
+use crate::macros::mactree_dealias::dealias_mac;
+use crate::macros::mactree_parse::{MacParseStrat, ParseTokRes, parse_tok};
 
 #[derive(Debug, Default)]
 pub struct MacTreeLexer;
@@ -26,142 +24,75 @@ impl Lexer for MacTreeLexer {
 		let Some(tail2) = tail.strip_prefix('\'') else {
 			return Err(err_not_applicable().await);
 		};
-		let tail3 = tail2.trim_start();
-		let mut args = Vec::new();
-		let (tail4, mactree) = match mac_tree(tail3, &mut args, lctx).await {
-			Err(e) => return Ok((tail2, GenTok::Bottom(e).at(lctx.pos_lt(1, tail2)))),
-			Ok(res) => res,
-		};
-		let range = lctx.pos_tt(tail, tail4);
-		let tok = if args.is_empty() {
-			x_tok(new_atom(mactree)).await
-		} else {
-			let itpl_call = InstantiateTplCall { argc: args.len(), argv: vec![], tpl: mactree };
-			let call = chain!([x_tok(new_atom(itpl_call)).await.at(range.clone())], args);
-			GenTok::S(Paren::Round, call.collect())
-		};
-		return Ok((tail4, tok.at(range)));
-		async fn mac_tree<'a>(
-			tail: &'a str,
-			args: &mut Vec<GenTokTree>,
-			ctx: &'a LexContext<'a>,
-		) -> OrcRes<(&'a str, MacTree)> {
-			for (lp, rp, paren) in PARENS {
-				let Some(mut body_tail) = tail.strip_prefix(*lp) else { continue };
-				let mut items = Vec::new();
-				return loop {
-					let tail2 = body_tail.trim_start();
-					if let Some(tail3) = tail2.strip_prefix(*rp) {
-						let tok = MacTok::S(*paren, MacTreeSeq::new(items));
-						break Ok((tail3, tok.at(ctx.pos_tt(tail, tail3).pos())));
-					} else if tail2.is_empty() {
-						return Err(mk_errv(is("Unclosed block").await, format!("Expected closing {rp}"), [
-							ctx.pos_lt(1, tail),
-						]));
-					}
-					let (new_tail, new_item) = mac_tree(tail2, args, ctx).boxed_local().await?;
-					body_tail = new_tail;
-					items.push(new_item);
-				};
-			}
-			if let Some(tail2) = tail.strip_prefix("$") {
-				let (tail3, sub) = ctx.recurse(tail2).await?;
-				let sr = ctx.pos_tt(tail, tail3);
-				args.push(p_tree2gen(sub));
-				return Ok((tail3, MacTok::Slot.at(sr.pos())));
-			}
-			if let Some(tail2) = tail.strip_prefix("\\") {
-				let tail2 = tail2.trim_start();
-				let (mut tail3, param) = mac_tree(tail2, args, ctx).boxed_local().await?;
-				let mut body = Vec::new();
-				loop {
-					let tail4 = tail3.trim_start();
-					if tail4.is_empty() || tail4.starts_with(|c| ")]}".contains(c)) {
-						break;
-					};
-					let (tail5, body_tok) = mac_tree(tail4, args, ctx).boxed_local().await?;
-					body.push(body_tok);
-					tail3 = tail5;
+		let (tail3, recur) = lctx.recurse(tail2).await?;
+		#[derive(Clone)]
+		struct Strat<'a>(Vec<Cow<'a, GenTokTree>>);
+		impl<'a> MacParseStrat<'a> for Strat<'a> {
+			type Err = Never;
+			async fn with_placeholder(
+				&mut self,
+				ph: PhKind,
+				next: &'a GenTokTree,
+			) -> Result<MacTok, Self::Err> {
+				if !matches!(ph, PhKind::Scalar) {
+					return Ok(MacTok::Bottom(
+						token_errv(next, "Cannot spread into mactree", |tok| {
+							format!(
+								"{ph}{tok} is a spreading placeholder which is not supported by MacTree literals. \
+                Use $ for a non-spreading embed"
+							)
+						})
+						.await,
+					));
 				}
-				Ok((tail3, MacTok::Lambda(param, MacTreeSeq::new(body)).at(ctx.pos_tt(tail, tail3).pos())))
-			} else {
-				let (tail2, sub) = ctx.recurse(tail).await?;
-				let args = RefCell::new(&mut *args);
-				let parsed = (parse_tok(&sub).await.expect("Unexpected invalid token"))
-					.map(&mut false, &mut async |mt| {
-						if let MacTok::Name(n) = mt.tok() {
-							let Pos::SrcRange(sr) = mt.pos() else {
-								panic!("translated from TokTree must have SourceRange")
-							};
-							let arg = x_tok(new_atom(TemplateName(n.clone()))).await.at(sr);
-							args.borrow_mut().push(arg);
-							return Some(MacTok::Slot.at(mt.pos()));
-						}
-						None
-					})
-					.await;
-				Ok((tail2, parsed))
+				self.0.push(Cow::Borrowed(next));
+				Ok(MacTok::Slot(SlotFillStrategy::Unwrap))
+			}
+			async fn with_reflexive(
+				&mut self,
+				flex: api::ReflexiveTree,
+				sr: orchid_base::SrcRange,
+			) -> Result<MacTree, Self::Err> {
+				self.0.push(Cow::Owned(GenTok::Reflexive(FlexTree::Existing(flex)).at(sr)));
+				todo!("Review the todo in macro_line and replicate it here")
 			}
 		}
+		let mut strat = Strat(Vec::new());
+		let sr = lctx.pos_tt(tail, tail3);
+		let ret = match parse_tok(&recur, &mut strat).await.unwrap_or_else(|e| match e {}) {
+			ParseTokRes::Discard => GenTok::Bottom(
+				token_errv(&recur, "Untracked token literal", |tok| {
+					format!("{tok} is not tracked by the macro system so it cannot become a literal")
+				})
+				.await,
+			),
+			ParseTokRes::PhNeedsArg(_) => GenTok::Bottom(
+				token_errv(&recur, "Trivial interpolated token literal '$ found", |tok| {
+					format!("{tok} has no effect and should be removed")
+				})
+				.await,
+			),
+			ParseTokRes::Done(mactree) => {
+				clone!(sr);
+				let args = strat.0.into_iter().map(|cow| cow.into_owned()).collect_vec();
+				flex_tok(move |pos| {
+					clone!(mactree, args, sr);
+					let range = sr.clone();
+					async move {
+						let dealiased = dealias_mac(&mactree, pos).await;
+						let tok = if args.is_empty() {
+							x_tok(new_atom(dealiased)).await
+						} else {
+							let itpl_call = InstantiateTplCall { argv: vec![], tpl: dealiased };
+							let mut call = vec![x_tok(new_atom(itpl_call)).await.at(range.clone())];
+							call.extend(args);
+							GenTok::S(Paren::Round, call)
+						};
+						tok.at(range)
+					}
+				})
+			},
+		};
+		Ok((tail3, ret.at(sr)))
 	}
 }
-
-#[derive(Clone)]
-pub struct InstantiateTplCall {
-	pub(crate) tpl: MacTree,
-	pub(crate) argc: usize,
-	pub(crate) argv: Vec<MacTree>,
-}
-impl Atomic for InstantiateTplCall {
-	type Variant = OwnedVariant;
-	type Data = ();
-}
-impl OwnedAtom for InstantiateTplCall {
-	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-	type Refs = Never;
-	// Technically must be supported but shouldn't actually ever be called
-	async fn call_ref(&self, arg: Expr) -> impl ToExpr {
-		if !self.argv.is_empty() {
-			eprintln!(
-				"Copying partially applied instantiate_tpl call. This is an internal value.\
-				\nIt should be fully consumed within generated code."
-			);
-		}
-		self.clone().call(arg).await
-	}
-	async fn call(mut self, arg: Expr) -> impl ToExpr {
-		exec(async move |mut h| {
-			let expr = h.exec::<Expr>(arg.clone()).await.expect("Conversion to Expr shouldn't fail");
-			if let Ok(t) = TAtom::<MacTree>::try_from_expr(expr.clone()).await {
-				self.argv.push(t.own().await)
-			} else if let Ok(t) = TAtom::<TemplateName>::try_from_expr(expr.clone()).await {
-				self.argv.push(todo!())
-			} else {
-				panic!("Expected a macro param, found {}", fmt(&arg).await)
-			}
-			if self.argv.len() < self.argc {
-				return new_atom(self);
-			}
-			let mut args = self.argv.into_iter();
-			let ret = (self.tpl.map(&mut false, &mut async |mt| match mt.tok() {
-				MacTok::Slot => Some(args.next().expect("Not enough arguments to fill all slots")),
-				_ => None,
-			}))
-			.await;
-			assert!(args.next().is_none(), "Too many arguments for all slots");
-			new_atom(ret)
-		})
-		.await
-	}
-}
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-pub struct TemplateName(Sym);
-impl Atomic for TemplateName {
-	type Data = api::TStrv;
-	type Variant = OwnedVariant;
-}
-impl OwnedAtom for TemplateName {
-	type Refs = Never;
-	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.to_api()) }
-}

orchid-std/src/macros/mactree_parse.rs

@@ -0,0 +1,126 @@
+//! Read a macro-processable tree from a parser tree. After this, the tree
+//! contains names as written, and needs to be dealiased, then
+
+use async_fn_stream::try_stream;
+use futures::{FutureExt, TryStreamExt};
+use orchid_base::{Snippet, SrcRange, Sym, report, token_errv};
+use orchid_extension::gen_tok::{FlexTree, GenSnippet, GenTok, GenTokTree};
+use orchid_extension::{TAtom, TryFromExpr};
+
+use crate::macros::mactree::{MacTreeSeq, PhKind};
+use crate::macros::ph_lexer::PhAtom;
+use crate::{MacTok, MacTree, api};
+
+/// Trait to describe what to do with tokens that can't be directly translated
+/// into mactree
+pub trait MacParseStrat<'a> {
+	type Err;
+	async fn with_placeholder(
+		&mut self,
+		ph: PhKind,
+		next: &'a GenTokTree,
+	) -> Result<MacTok, Self::Err>;
+	async fn with_reflexive(
+		&mut self,
+		flex: api::ReflexiveTree,
+		sr: SrcRange,
+	) -> Result<MacTree, Self::Err>;
+}
+
+pub async fn parse_tokv<'a, S: MacParseStrat<'a> + 'a>(
+	line: GenSnippet<'a>,
+	strat: &mut S,
+) -> Result<MacTreeSeq, S::Err> {
+	let Some((head, arg, sr, body)) = line.split_lambda() else {
+		return Ok(MacTreeSeq::new(parse_tokv_no_lambdas(line, strat).await?));
+	};
+	let body = parse_tokv(body, strat).boxed_local().await?;
+	let mut all = parse_tokv_no_lambdas(head, strat).await?;
+	match parse_tok(arg, strat).await? {
+		ParseTokRes::Done(arg) => all.push(MacTok::Lambda(arg, body).at(sr)),
+		ParseTokRes::Discard | ParseTokRes::PhNeedsArg(_) => report(
+			token_errv(arg, "Lambda argument not valid macro token", |arg| {
+				format!("Lambda arguments must be a valid token, found meaningless fragment {arg}")
+			})
+			.await,
+		),
+	};
+	Ok(MacTreeSeq::new(all))
+}
+
+async fn report_trailing_placeholder(tok: &GenTokTree) {
+	report(
+		token_errv(
+			tok,
+			"Placeholder with no arg",
+			|_| "Placeholders are always followed by a name or a token",
+		)
+		.await,
+	)
+}
+
+async fn parse_tokv_no_lambdas<'a, S: MacParseStrat<'a> + 'a>(
+	line: GenSnippet<'a>,
+	strat: &mut S,
+) -> Result<Vec<MacTree>, S::Err> {
+	try_stream(async move |mut h| {
+		let mut line_toks = line.iter();
+		while let Some(tok) = line_toks.next() {
+			match parse_tok(tok, strat).await? {
+				ParseTokRes::Done(out) => h.emit(out).await,
+				ParseTokRes::Discard => (),
+				ParseTokRes::PhNeedsArg(arg) => {
+					let Some(next) = line_toks.next() else {
+						report_trailing_placeholder(tok).await;
+						continue;
+					};
+					h.emit(strat.with_placeholder(arg, next).await?.at(tok.sr())).await
+				},
+			}
+		}
+		Ok(h)
+	})
+	.try_collect::<Vec<_>>()
+	.await
+}
+
+pub enum ParseTokRes {
+	Discard,
+	Done(MacTree),
+	PhNeedsArg(PhKind),
+}
+
+pub async fn parse_tok<'a, S: MacParseStrat<'a> + 'a>(
+	tree: &'a GenTokTree,
+	strat: &mut S,
+) -> Result<ParseTokRes, S::Err> {
+	let tok = match &tree.tok {
+		GenTok::Bottom(errv) => MacTok::Bottom(errv.clone()),
+		GenTok::BR | GenTok::Comment(_) => return Ok(ParseTokRes::Discard),
+		GenTok::Name(n) => MacTok::Name(Sym::new([n.clone()]).await.unwrap()),
+		GenTok::NS(..) => match tree.as_multiname() {
+			Ok(mn) => MacTok::Name(mn.to_sym().await),
+			Err(nested) => {
+				report(
+					token_errv(tree, ":: can only be followed by a name in an expression", |tok| {
+						format!("Expected name, found {tok}")
+					})
+					.await,
+				);
+				return parse_tok(nested, strat).boxed_local().await;
+			},
+		},
+		GenTok::Reflexive(FlexTree::Existing(flex)) =>
+			return Ok(ParseTokRes::Done(strat.with_reflexive(*flex, tree.sr()).await?)),
+		GenTok::Handle(expr) => match TAtom::<PhAtom>::try_from_expr(expr.clone()).await {
+			Err(_) => MacTok::Value(expr.clone()),
+			Ok(ta) => return Ok(ParseTokRes::PhNeedsArg(ta.value.get())),
+		},
+		GenTok::NewExpr(_) | GenTok::Reflexive(FlexTree::New(_)) =>
+			panic!("parse_tok can only process clean input, newly constructed atom found"),
+		GenTok::LambdaHead(_) => panic!("Lambda-head handled in the sequence parser"),
+		GenTok::S(p, body) =>
+			MacTok::S(*p, parse_tokv(Snippet::new(tree, body), strat).boxed_local().await?),
+	};
+	Ok(ParseTokRes::Done(tok.at(tree.sr().pos())))
+}

orchid-std/src/macros/mod.rs

@@ -1,3 +1,4 @@
+mod instantiate_tpl;
 mod let_line;
 pub mod lower;
 mod macro_lib;
@@ -5,7 +6,9 @@ mod macro_line;
 pub mod macro_system;
 mod macro_value;
 pub mod mactree;
+pub mod mactree_dealias;
 mod mactree_lexer;
+pub mod mactree_parse;
 pub mod match_macros;
 mod ph_lexer;
 mod resolve;

orchid-std/src/macros/ph_lexer.rs

@@ -1,24 +1,22 @@
 use orchid_api_derive::Coding;
-use orchid_base::{FmtUnit, OrcRes, es, is, mk_errv, name_char, name_start};
+use orchid_base::{FmtUnit, OrcRes, is, mk_errv};
 use orchid_extension::gen_expr::new_atom;
-use orchid_extension::tree::{GenTokTree, x_tok};
+use orchid_extension::gen_tok::{GenTokTree, x_tok};
 use orchid_extension::{Atomic, LexContext, Lexer, ThinAtom, ThinVariant, err_not_applicable};
 
-use crate::macros::mactree::{Ph, PhKind};
+use crate::macros::mactree::PhKind;
 
 #[derive(Clone, Coding)]
-pub struct PhAtom(orchid_api::TStr, PhKind);
+pub struct PhAtom(PhKind);
 impl PhAtom {
-	pub async fn to_full(&self) -> Ph { Ph { kind: self.1, name: es(self.0).await } }
+	pub fn get(&self) -> PhKind { self.0 }
 }
 impl Atomic for PhAtom {
 	type Data = Self;
 	type Variant = ThinVariant;
 }
 impl ThinAtom for PhAtom {
-	async fn print(&self) -> FmtUnit {
-		Ph { name: es(self.0).await, kind: self.1 }.to_string().into()
-	}
+	async fn print(&self) -> FmtUnit { self.0.to_string().into() }
 }
 
 #[derive(Debug, Default)]
@@ -26,48 +24,37 @@ pub struct PhLexer;
 impl Lexer for PhLexer {
 	const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['$'..='$', '.'..='.'];
 	async fn lex<'a>(line: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree)> {
-		let (tail, name, phkind) = if let Some(tail) = line.strip_prefix("$")
-			&& tail.starts_with(name_start)
+		let (tail, phkind) = if let Some(tail) = line.strip_prefix("$")
+			&& tail.starts_with(|c: char| !c.is_ascii_whitespace())
 		{
-			let name = tail.split_once(|c| !name_char(c)).map_or("", |(h, _)| h);
-			let tail = tail.split_at(name.len()).1;
-			(tail, name, PhKind::Scalar)
+			(tail, PhKind::Scalar)
 		} else {
-			async fn name_and_prio<'a>(
-				tail: &'a str,
-				ctx: &'a LexContext<'a>,
-			) -> OrcRes<(&'a str, u8, &'a str)> {
-				let name = tail.split_once(|c| !name_char(c)).map_or("", |(h, _)| h);
-				let tail = tail.split_at(name.len()).1;
-				let (prio, tail) = match tail.strip_prefix(":") {
-					None => (0, tail),
-					Some(tail) => {
-						let prio = tail.split_once(|c: char| c.is_ascii_digit()).map_or("", |(h, _)| h);
-						let tail = tail.split_at(prio.len()).1;
-						if let Ok(prio_num) = prio.parse::<u8>() {
-							(prio_num, tail)
-						} else {
-							return Err(mk_errv(
-								is("Invalid priority, must be 0-255").await,
-								format!("{prio} is not a valid placeholder priority"),
-								[ctx.pos_lt(prio.len(), tail)],
-							));
-						}
-					},
-				};
-				Ok((name, prio, tail))
-			}
-			if let Some(tail) = line.strip_prefix("..$") {
-				let (name, priority, tail) = name_and_prio(tail, ctx).await?;
-				(tail, name, PhKind::Vector { at_least_one: false, priority })
+			let (tail, at_least_one) = if let Some(tail) = line.strip_prefix("..$") {
+				(tail, false)
 			} else if let Some(tail) = line.strip_prefix("...$") {
-				let (name, priority, tail) = name_and_prio(tail, ctx).await?;
-				(tail, name, PhKind::Vector { at_least_one: true, priority })
+				(tail, true)
 			} else {
 				return Err(err_not_applicable().await);
+			};
+			let colon_idx = tail.find(|c: char| !c.is_ascii_digit()).unwrap_or(tail.len());
+			let (prio_s, mut tail) = tail.split_at(colon_idx);
+			if !prio_s.is_empty() {
+				if let Some(new_tail) = tail.strip_prefix(":")
+					&& new_tail.starts_with(|c: char| !c.is_ascii_whitespace())
+				{
+					tail = new_tail
+				} else {
+					return Err(err_not_applicable().await);
+				};
+			}
+			match prio_s.parse::<u8>() {
+				Ok(priority) => (tail, PhKind::Vector { at_least_one, priority }),
+				Err(e) =>
+					return Err(mk_errv(is("Invalid placeholder priority").await, e.to_string(), [
+						ctx.pos_tt(line, tail)
+					])),
 			}
 		};
-		let ph_atom = PhAtom(is(name).await.to_api(), phkind);
-		Ok((tail, x_tok(new_atom(ph_atom)).await.at(ctx.pos_tt(line, tail))))
+		Ok((tail, x_tok(new_atom(PhAtom(phkind))).await.at(ctx.pos_tt(line, tail))))
 	}
 }

orchid-std/src/macros/resolve.rs

@@ -75,7 +75,7 @@ struct ResolveCtx<'a, 'b> {
 
 async fn resolve_one(ctx: &mut ResolveCtx<'_, '_>, value: &MacTree) -> Option<MacTree> {
 	match value.tok() {
-		MacTok::Ph(_) | MacTok::Slot => panic!("Forbidden element in value mactree"),
+		MacTok::Ph(_) | MacTok::Slot(_) => panic!("Forbidden element in value mactree"),
 		MacTok::Bottom(_) | MacTok::Value(_) | MacTok::Name(_) | MacTok::Resolved(_) => None,
 		MacTok::Lambda(arg, body) => {
 			let new_arg = resolve_one(ctx, arg).boxed_local().await;

orchid-std/src/macros/rule/build.rs

@@ -128,7 +128,7 @@ async fn mk_scalar(pattern: &MacTree) -> OrcRes<ScalMatcher> {
 			));
 		},
 		MacTok::Resolved(_) => panic!("Can only appear in macro output, not in matcher"),
-		MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
+		MacTok::Value(_) | MacTok::Slot(_) => panic!("Only used for templating"),
 		MacTok::Bottom(errv) => return Err(errv.clone()),
 	})
 }

orchid-std/src/std/number/num_lexer.rs

@@ -1,7 +1,7 @@
 use std::ops::RangeInclusive;
 
 use orchid_base::{OrcRes, num_to_errv, parse_num};
-use orchid_extension::tree::{GenTokTree, x_tok};
+use orchid_extension::gen_tok::{GenTokTree, x_tok};
 use orchid_extension::{LexContext, Lexer};
 
 use super::num_atom::Num;

orchid-std/src/std/ops/subscript_lexer.rs

@@ -1,6 +1,6 @@
 use orchid_base::{OrcRes, is, name_char, name_start};
 use orchid_extension::gen_expr::new_atom;
-use orchid_extension::tree::GenTok;
+use orchid_extension::gen_tok::GenTok;
 use orchid_extension::{LexContext, LexedData, Lexer, err_not_applicable};
 
 use crate::std::string::str_atom::IntStrAtom;

orchid-std/src/std/protocol/parse_impls.rs

@@ -3,13 +3,14 @@ use orchid_base::{
 	IStr, Import, OrcRes, Paren, Parsed, Snippet, Sym, Token, expect_tok, is, line_items, mk_errv,
 	parse_multiname, token_errv,
 };
-use orchid_extension::{PTokTree, ParsCtx, ParsedLine, ParsedLineKind, p_tree2gen, p_v2gen};
+use orchid_extension::gen_tok::GenTokTree;
+use orchid_extension::{ParsCtx, ParsedLine, ParsedLineKind};
 
 pub async fn parse_impls(
 	_: &ParsCtx<'_>,
 	lines: &mut Vec<ParsedLine>,
 	impls: &mut Vec<(Sym, IStr)>,
-	body_tt: &PTokTree,
+	body_tt: &GenTokTree,
 ) -> OrcRes<()> {
 	let body = match &body_tt.tok {
 		Token::S(Paren::Round, body) => line_items(Snippet::new(body_tt, body)).await,
@@ -58,7 +59,7 @@ pub async fn parse_impls(
 					[Token::Name(is("let").await).at(line.sr())],
 					[Token::Name(cnst_name.clone()).at(name_sr)],
 					[Token::Name(is("=").await).at(line.sr())],
-					tail.iter().cloned().map(p_tree2gen),
+					tail.iter().cloned(),
 				])),
 			});
 			impls.push((name, cnst_name));
@@ -66,7 +67,7 @@ pub async fn parse_impls(
 			lines.push(ParsedLine {
 				sr: line.sr(),
 				comments,
-				kind: ParsedLineKind::Rec(p_v2gen(line.to_vec())),
+				kind: ParsedLineKind::Rec(line.to_vec()),
 			});
 		}
 	}