Macro processing factored into Orchid functions

2025-01-08 01:34:40 +00:00
parent e780969c6c
commit 7cdfe7e3c4
36 changed files with 631 additions and 289 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4

 [[package]]
 name = "ahash"
@@ -335,6 +335,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

+[[package]]
+name = "foldhash"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"
+
 [[package]]
 name = "funty"
 version = "2.0.0"
@@ -376,9 +382,16 @@ name = "hashbrown"
 version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+
+[[package]]
+name = "hashbrown"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
 dependencies = [
- "ahash 0.8.11",
 "allocator-api2",
+ "equivalent",
+ "foldhash",
 ]

 [[package]]
@@ -418,6 +431,15 @@ dependencies = [
 "either",
 ]

+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -504,7 +526,7 @@ name = "orchid-api-derive"
 version = "0.1.0"
 dependencies = [
 "darling",
- "itertools",
+ "itertools 0.13.0",
 "orchid-api-traits",
 "proc-macro2 1.0.78",
 "quote 1.0.35",
@@ -515,7 +537,7 @@ dependencies = [
 name = "orchid-api-traits"
 version = "0.1.0"
 dependencies = [
- "itertools",
+ "itertools 0.13.0",
 "never",
 "ordered-float",
 ]
@@ -526,8 +548,8 @@ version = "0.1.0"
 dependencies = [
 "derive_destructure",
 "dyn-clone",
- "hashbrown 0.14.5",
- "itertools",
+ "hashbrown 0.15.2",
+ "itertools 0.14.0",
 "lazy_static",
 "never",
 "num-traits",
@@ -548,8 +570,8 @@ dependencies = [
 "ahash 0.8.11",
 "derive_destructure",
 "dyn-clone",
- "hashbrown 0.14.5",
- "itertools",
+ "hashbrown 0.15.2",
+ "itertools 0.14.0",
 "konst",
 "lazy_static",
 "never",
@@ -569,8 +591,8 @@ name = "orchid-host"
 version = "0.1.0"
 dependencies = [
 "derive_destructure",
- "hashbrown 0.14.5",
- "itertools",
+ "hashbrown 0.15.2",
+ "itertools 0.14.0",
 "lazy_static",
 "never",
 "num-traits",
@@ -587,7 +609,7 @@ dependencies = [
 name = "orchid-std"
 version = "0.1.0"
 dependencies = [
- "itertools",
+ "itertools 0.13.0",
 "never",
 "once_cell",
 "orchid-api",
@@ -604,7 +626,7 @@ version = "0.1.0"
 dependencies = [
 "camino",
 "clap",
- "itertools",
+ "itertools 0.13.0",
 "orchid-base",
 "orchid-host",
 ]
--- a/SWAP.md
+++ b/SWAP.md
@@ -0,0 +1,12 @@
+### Swapfile
+A loose collection of ideas to quickly resume work
+
+---
+
+Must figure out how preprocessor can both be a System and referenced in the interpreter
+
+Must actually write macro system as recorded in note
+
+At this point swappable preprocessors aren't a target because interaction with module system sounds complicated
+
+Check if any of this needs interpreter, if so, start with that
--- a/notes/new_macro_model.md
+++ b/notes/new_macro_model.md
@@ -1,9 +1,11 @@
-# Code sample for the new macro system
+We make a distinction between positional and prioritized macros.
+
+# Named macro

 ```
 macro (
 	rule match ...$expr { ...$body } => \recurse. '(
-		fn::pass (...$expr) \match::value. ...$(
+		fn::pass (...$expr) \match::value. ..$(
 			fn::pass (quote::split body ';) \cases.
 			fn::pass (list::map cases \x. (
 				fn::pass (quote::split_once x '=>) \pair.
@@ -21,25 +23,79 @@ macro (
 		)
 	)
 )
+```

--[
-	Macros are run from the top down.
-	For every token
-		1. If it's a name token, test all macros starting with that name
-		2. If none match and this is the first token in a list, test all macros starting with vectorials
-	Test all in a set of macros
-		1. Take the first rule that matches in each block
-		2. If there are multiple matches across blocks, raise an ambiguity error
-		3. If the single match is in the recursion stack, raise a recursion error
-		4. Add the matching rule to the recursion stack, then execute the body.
-]--
+Named macro patterns must start with a name token. They are always evaluated first. If they don't end with a vectorial placeholder, macro evaluation continues after them so that they can become first arguments to infix operators.

--[
-	1. Macro patterns are held in the host, they don't contain atoms, and atoms are never considered equal, so the matcher doesn't have to call an extension.
-	2. The body of macros may be defined in Rust. If it isn't, the entire interpreter will recurse on the macro to calculate the output.
-]--
+# Prioritized macro

--[
-	1. if the rule body uses the same macro, fail with the rule
-	2. if the rule explicitly recursively invokes the same macro, fail with the first match
-]--
+```
+macro 3 (
+	...$lhs + ...$rhs:1 => \recurse. '(add (..$(recurse lhs)) (..$(recurse rhs)))
+)
+```
+
+Prioritised macro patterns must start and end with a vectorial placeholder. They represent infix operators.
+
+# Algorithm
+
+Macros are checked from the outermost block inwards.
+
+1. For every name token, test all named macros starting with that name
+   1. Take the first rule that matches in each block
+   2. If there are multiple matches across blocks, raise an ambiguity error
+   3. If the tail is implicit, recurse on it
+2. Test all prioritized macros
+   1. Take the first rule that matches in the highest prioritized block
+
+Test all in a set of macros
+1. Take the first rule that matches in each block
+2. If there are multiple matches across blocks, raise an ambiguity error
+3. If the single match is in the recursion stack, raise a recursion error
+4. Add the matching rule to the recursion stack, then execute the body.
+
+# Considerations
+
+Maxims for the location of macros
+
+1. Macro patterns are held in the host, they don't contain atoms, and atoms are never considered equal, so the matcher doesn't have to call an extension.
+2. The body of macros may be defined in Rust. If it isn't, the entire interpreter will recurse on the macro to calculate the output.
+
+On recursion, the following errors can be detected
+
+1. if the rule body uses the same macro, fail with the rule
+2. if the rule explicitly recursively invokes the same macro, fail with the first match
+
+# Elements of the extension
+
+Recursion has to happen through the interpreter itself, so the macro system is defined in terms of atoms just like an extension
+
+- atom `MacTree` depicts a single token. `MacTree(tpl)` is also a `MacTree` but it can contain the independently unrepresentable templated slot node
+
+- lexer `'` followed by any single token always generates `MacTree`. If it contains placeholders which are tokens prefixed with `$` or `..$`, it generates a call to `instantiate_tpl` with a prepared `MacTree(tpl)` as the first argument and the placeholder values after. `MacTree(tpl)` only exists as an internal subresult routed directly to `instantiate_tpl`.
+
+- line parser `macro` parses a macro with the existing logic
+- atom `MacRecurState` holds the recursion state
+- function `resolve_recur` finds all matches on a MacTree
+  - type: `MacRecurState -> MacTree -> MacTree`
+  - use all macros to find all matches in the tree
+  - for each match
+    - check for recursion violations
+    - wrap the body in iife-s corresponding to the named values in the match state
+    - emit a recursive call to process and run the body, and pass the same recursive call as argument for the macro to use
+		```
+		(\recur. lower (recur $body) recur)
+			(resolve_recur $mac_recur_state)
+		```
+  - emit a single call to `instantiate_tpl` which receives all of these
+- function `instantiate_tpl` inserts `MacTree` values into a `MacTree(tpl)`
+  - type: `MacTree(tpl) [-> MacTree] -> MacTree`  
+		_this function deduces the number of arguments from the first argument. This combines poorly with autocurry, but it's an easy way to avoid representing standalone tree lists_
+  - walks the tree to find max template slot number, reads and type checks as many template values
+  - returns the populated tree
+- function `resolve` is the main entry point of the code
+  - type: `MacTree -> MacTree`
+  - invokes `resolve_recur` with an empty `MacRecurState`
+- function `lower` is the main exit point of the code
+  - type: `MacTree -> any`
+  - Lowers `MacTree` into the equivalent `Expr`.
--- a/orchid-api-traits/src/coding.rs
+++ b/orchid-api-traits/src/coding.rs
@@ -77,7 +77,7 @@ nonzero_impl!(std::num::NonZeroI32);
 nonzero_impl!(std::num::NonZeroI64);
 nonzero_impl!(std::num::NonZeroI128);

-impl<'a, T: Encode + ?Sized> Encode for &'a T {
+impl<T: Encode + ?Sized> Encode for &T {
  fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
 }
 macro_rules! float_impl {
@@ -285,12 +285,12 @@ smart_ptr!(Arc);
 smart_ptr!(Rc);
 smart_ptr!(Box);

-impl<'a, T: ?Sized + ToOwned> Decode for Cow<'a, T>
+impl<T: ?Sized + ToOwned> Decode for Cow<'_, T>
 where T::Owned: Decode
 {
  fn decode<R: Read + ?Sized>(read: &mut R) -> Self { Cow::Owned(T::Owned::decode(read)) }
 }
-impl<'a, T: ?Sized + Encode + ToOwned> Encode for Cow<'a, T> {
+impl<T: ?Sized + Encode + ToOwned> Encode for Cow<'_, T> {
  fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
 }

--- a/orchid-api/src/atom.rs
+++ b/orchid-api/src/atom.rs
@@ -67,7 +67,7 @@ impl Request for FinalCall {
 #[extends(AtomReq, HostExtReq)]
 pub struct SerializeAtom(pub Atom);
 impl Request for SerializeAtom {
-  type Response = (Vec<u8>, Vec<ExprTicket>);
+  type Response = Option<(Vec<u8>, Vec<ExprTicket>)>;
 }

 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
--- a/orchid-base/Cargo.toml
+++ b/orchid-base/Cargo.toml
@@ -8,8 +8,8 @@ edition = "2021"
 [dependencies]
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.17"
-hashbrown = "0.14.3"
-itertools = "0.13.0"
+hashbrown = "0.15.2"
+itertools = "0.14.0"
 lazy_static = "1.4.0"
 never = "0.1.0"
 num-traits = "0.2.19"
--- a/orchid-base/src/box_cow.rs
+++ b/orchid-base/src/box_cow.rs
@@ -6,10 +6,10 @@ pub enum ArcCow<'a, T: ?Sized + ToOwned> {
  Borrowed(&'a T),
  Owned(Arc<T::Owned>),
 }
-impl<'a, T: ?Sized + ToOwned> ArcCow<'a, T> {
+impl<T: ?Sized + ToOwned> ArcCow<'_, T> {
  pub fn owned(value: T::Owned) -> Self { Self::Owned(Arc::new(value)) }
 }
-impl<'a, T: ?Sized + ToOwned> Clone for ArcCow<'a, T> {
+impl<T: ?Sized + ToOwned> Clone for ArcCow<'_, T> {
  fn clone(&self) -> Self {
    match self {
      Self::Borrowed(r) => Self::Borrowed(r),
@@ -18,7 +18,7 @@ impl<'a, T: ?Sized + ToOwned> Clone for ArcCow<'a, T> {
  }
 }

-impl<'a, T: ?Sized + ToOwned> Deref for ArcCow<'a, T> {
+impl<T: ?Sized + ToOwned> Deref for ArcCow<'_, T> {
  type Target = T;
  fn deref(&self) -> &Self::Target {
    match self {
--- a/orchid-base/src/id_store.rs
+++ b/orchid-base/src/id_store.rs
@@ -33,17 +33,17 @@ impl<T> Default for IdStore<T> {
 }

 pub struct IdRecord<'a, T>(NonZeroU64, MutexGuard<'a, HashMap<NonZeroU64, T>>);
-impl<'a, T> IdRecord<'a, T> {
+impl<T> IdRecord<'_, T> {
  pub fn id(&self) -> NonZeroU64 { self.0 }
  pub fn remove(mut self) -> T { self.1.remove(&self.0).unwrap() }
 }
-impl<'a, T> Deref for IdRecord<'a, T> {
+impl<T> Deref for IdRecord<'_, T> {
  type Target = T;
  fn deref(&self) -> &Self::Target {
    self.1.get(&self.0).expect("Existence checked on construction")
  }
 }
-impl<'a, T> DerefMut for IdRecord<'a, T> {
+impl<T> DerefMut for IdRecord<'_, T> {
  fn deref_mut(&mut self) -> &mut Self::Target {
    self.1.get_mut(&self.0).expect("Existence checked on construction")
  }
--- a/orchid-base/src/macros.rs
+++ b/orchid-base/src/macros.rs
@@ -1,15 +1,18 @@
+use std::marker::PhantomData;
+use std::sync::Arc;
+
 use itertools::Itertools;
 use never::Never;
 use trait_set::trait_set;

-use crate::{match_mapping, name::Sym, tree::{Paren, Ph}};
-use std::{marker::PhantomData, sync::Arc};
-
-use crate::{api, location::Pos};
+use crate::location::Pos;
+use crate::name::Sym;
+use crate::tree::{Paren, Ph};
+use crate::{api, match_mapping};

 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct MacroSlot<'a>(api::MacroTreeId, PhantomData<&'a ()>);
-impl<'a> MacroSlot<'a> {
+impl MacroSlot<'_> {
  pub fn id(self) -> api::MacroTreeId { self.0 }
 }

@@ -21,7 +24,7 @@ trait_set! {
 #[derive(Clone, Debug)]
 pub struct MTree<'a, A> {
  pub pos: Pos,
-  pub tok: Arc<MTok<'a, A>>
+  pub tok: Arc<MTok<'a, A>>,
 }
 impl<'a, A> MTree<'a, A> {
  pub(crate) fn from_api(api: &api::MacroTree, do_atom: &mut impl MacroAtomFromApi<'a, A>) -> Self {
@@ -40,12 +43,16 @@ pub enum MTok<'a, A> {
  Lambda(Vec<MTree<'a, A>>, Vec<MTree<'a, A>>),
  Ph(Ph),
  Atom(A),
-  Ref(Box<MTok<'a, Never>>),
+  /// Used in extensions to directly return input
+  Ref(Arc<MTok<'a, Never>>),
+  /// Used in the matcher to skip previous macro output which can only go in
+  /// vectorial placeholders
+  Done(Arc<MTok<'a, A>>),
 }
 impl<'a, A> MTok<'a, A> {
  pub(crate) fn from_api(
    api: &api::MacroToken,
-    do_atom: &mut impl MacroAtomFromApi<'a, A>
+    do_atom: &mut impl MacroAtomFromApi<'a, A>,
  ) -> Self {
    match_mapping!(&api, api::MacroToken => MTok::<'a, A> {
      Lambda(x => mtreev_from_api(x, do_atom), b => mtreev_from_api(b, do_atom)),
@@ -58,6 +65,7 @@ impl<'a, A> MTok<'a, A> {
    })
  }
  pub(crate) fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroToken {
+    fn sink(n: &Never) -> api::MacroToken { match *n {} }
    match_mapping!(&self, MTok => api::MacroToken {
      Lambda(x => mtreev_to_api(x, do_atom), b => mtreev_to_api(b, do_atom)),
      Name(t.tok().to_api()),
@@ -65,22 +73,24 @@ impl<'a, A> MTok<'a, A> {
      S(p.clone(), b => mtreev_to_api(b, do_atom)),
      Slot(tk.0.clone()),
    } {
-      MTok::Ref(r) => r.to_api(&mut |e| match *e {}),
+      MTok::Ref(r) => r.to_api(&mut sink),
+      MTok::Done(t) => t.to_api(do_atom),
      MTok::Atom(a) => do_atom(a),
    })
  }
+  pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Arc::new(self) } }
 }

 pub fn mtreev_from_api<'a, 'b, A>(
  api: impl IntoIterator<Item = &'b api::MacroTree>,
-  do_atom: &mut impl MacroAtomFromApi<'a, A>
+  do_atom: &mut impl MacroAtomFromApi<'a, A>,
 ) -> Vec<MTree<'a, A>> {
  api.into_iter().map(|api| MTree::from_api(api, do_atom)).collect_vec()
 }

 pub fn mtreev_to_api<'a: 'b, 'b, A: 'b>(
  v: impl IntoIterator<Item = &'b MTree<'a, A>>,
-  do_atom: &mut impl MacroAtomToApi<A>
+  do_atom: &mut impl MacroAtomToApi<A>,
 ) -> Vec<api::MacroTree> {
  v.into_iter().map(|t| t.to_api(do_atom)).collect_vec()
 }
--- a/orchid-base/src/match_mapping.rs
+++ b/orchid-base/src/match_mapping.rs
@@ -75,6 +75,9 @@ macro_rules! match_mapping {
  (@PAT_MUNCH $ctx:tt ($($names:ident)*) $name:ident => $value:expr , $($tail:tt)*) => {
    match_mapping!(@PAT_MUNCH $ctx ($($names)* $name) $($tail)*)
  };
+  (@PAT_MUNCH $ctx:tt ($($names:ident)*) $name:ident () $value:expr , $($tail:tt)*) => {
+    match_mapping!(@PAT_MUNCH $ctx ($($names)* $name) $($tail)*)
+  };
  (@PAT_MUNCH $ctx:tt ($($names:ident)*) $name:ident . $($tail:tt)*) => { 
    match_mapping!(@PAT_DOT_MUNCH $ctx ($($names)* $name) $($tail)*)
  };
@@ -97,6 +100,9 @@ macro_rules! match_mapping {
  (@VAL_MUNCH $ctx:tt ($($prefix:tt)*) $name:ident => $value:expr , $($tail:tt)*) => {
    match_mapping!(@VAL_MUNCH $ctx ($($prefix)* ($name ($value)) ) $($tail)*)
  };
+  (@VAL_MUNCH $ctx:tt ($($prefix:tt)*) $name:ident () $value:expr , $($tail:tt)*) => {
+    match_mapping!(@VAL_MUNCH $ctx ($($prefix)* ($name ($value($name))) ) $($tail)*)
+  };
  (@VAL_MUNCH $ctx:tt $fields:tt $name:ident . $member:tt $($tail:tt)*) => {
    match_mapping!(@VAL_DOT_MUNCH $ctx $fields $name ($name . $member ) $($tail)*)
  };
--- a/orchid-base/src/parse.rs
+++ b/orchid-base/src/parse.rs
@@ -67,11 +67,11 @@ impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
    self.split_at(non_fluff_start.unwrap_or(self.len())).1
  }
 }
-impl<'a, 'b, A: AtomRepr, X: ExtraTok> Copy for Snippet<'a, 'b, A, X> {}
-impl<'a, 'b, A: AtomRepr, X: ExtraTok> Clone for Snippet<'a, 'b, A, X> {
+impl<A: AtomRepr, X: ExtraTok> Copy for Snippet<'_, '_, A, X> {}
+impl<A: AtomRepr, X: ExtraTok> Clone for Snippet<'_, '_, A, X> {
  fn clone(&self) -> Self { *self }
 }
-impl<'a, 'b, A: AtomRepr, X: ExtraTok> Deref for Snippet<'a, 'b, A, X> {
+impl<'b, A: AtomRepr, X: ExtraTok> Deref for Snippet<'_, 'b, A, X> {
  type Target = [TokTree<'b, A, X>];
  fn deref(&self) -> &Self::Target { self.cur }
 }
--- a/orchid-base/src/reqnot.rs
+++ b/orchid-base/src/reqnot.rs
@@ -105,16 +105,16 @@ impl<T: MsgSet> ReqNot<T> {
  }

  /// Can be called from a polling thread or dispatched in any other way
-  pub fn receive(&self, message: Vec<u8>) {
+  pub fn receive(&self, message: &[u8]) {
    let mut g = self.0.lock().unwrap();
-    let (id, payload) = get_id(&message[..]);
+    let (id, payload) = get_id(message);
    if id == 0 {
      let mut notif = clone_box(&*g.notif);
      mem::drop(g);
      notif(<T::In as Channel>::Notif::decode(&mut &payload[..]), self.clone())
    } else if 0 < id.bitand(1 << 63) {
      let sender = g.responses.remove(&!id).expect("Received response for invalid message");
-      sender.send(message).unwrap();
+      sender.send(message.to_vec()).unwrap();
    } else {
      let message = <T::In as Channel>::Req::decode(&mut &payload[..]);
      let mut req = clone_box(&*g.req);
@@ -150,7 +150,7 @@ impl<'a, T> MappedRequester<'a, T> {
  }
 }

-impl<'a, T> DynRequester for MappedRequester<'a, T> {
+impl<T> DynRequester for MappedRequester<'_, T> {
  type Transfer = T;
  fn raw_request(&self, data: Self::Transfer) -> RawReply { self.0(data) }
 }
@@ -181,7 +181,7 @@ pub trait Requester: DynRequester {
    MappedRequester::new(self)
  }
 }
-impl<'a, This: DynRequester + ?Sized + 'a> Requester for This {
+impl<This: DynRequester + ?Sized> Requester for This {
  fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
    R::Response::decode(&mut &self.raw_request(data.into())[..])
  }
@@ -229,7 +229,7 @@ mod test {
      |_, _| panic!("Not receiving a request"),
    );
    let sender = ReqNot::<TestMsgSet>::new(
-      clone!(receiver; move |d, _| receiver.receive(d.to_vec())),
+      clone!(receiver; move |d, _| receiver.receive(d)),
      |_, _| panic!("Should not receive notif"),
      |_, _| panic!("Should not receive request"),
    );
@@ -245,7 +245,7 @@ mod test {
    let sender = Arc::new(ReqNot::<TestMsgSet>::new(
      {
        let receiver = receiver.clone();
-        move |d, _| receiver.lock().unwrap().as_ref().unwrap().receive(d.to_vec())
+        move |d, _| receiver.lock().unwrap().as_ref().unwrap().receive(d)
      },
      |_, _| panic!("Should not receive notif"),
      |_, _| panic!("Should not receive request"),
@@ -253,7 +253,7 @@ mod test {
    *receiver.lock().unwrap() = Some(ReqNot::new(
      {
        let sender = sender.clone();
-        move |d, _| sender.receive(d.to_vec())
+        move |d, _| sender.receive(d)
      },
      |_, _| panic!("Not receiving notifs"),
      |hand, req| {
--- a/orchid-base/src/tree.rs
+++ b/orchid-base/src/tree.rs
@@ -58,10 +58,10 @@ pub struct TokHandle<'a>(api::TreeTicket, PhantomData<&'a ()>);
 impl TokHandle<'static> {
  pub fn new(tt: api::TreeTicket) -> Self { TokHandle(tt, PhantomData) }
 }
-impl<'a> TokHandle<'a> {
+impl TokHandle<'_> {
  pub fn ticket(self) -> api::TreeTicket { self.0 }
 }
-impl<'a> Display for TokHandle<'a> {
+impl Display for TokHandle<'_> {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Handle({})", self.0.0) }
 }

@@ -146,7 +146,7 @@ impl<'a, A: AtomRepr, X: ExtraTok> TokTree<'a, A, X> {
  }
 }

-impl<'a, A: AtomRepr, X: ExtraTok> Display for TokTree<'a, A, X> {
+impl<A: AtomRepr, X: ExtraTok> Display for TokTree<'_, A, X> {
  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
 }

@@ -226,7 +226,7 @@ impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
    }
  }
 }
-impl<'a, A: AtomRepr, X: ExtraTok> Display for Token<'a, A, X> {
+impl<A: AtomRepr, X: ExtraTok> Display for Token<'_, A, X> {
  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    thread_local! {
      static PAREN_LEVEL: RefCell<usize> = 0.into();
--- a/orchid-extension/Cargo.toml
+++ b/orchid-extension/Cargo.toml
@@ -9,8 +9,8 @@ edition = "2021"
 ahash = "0.8.11"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.17"
-hashbrown = "0.14.5"
-itertools = "0.13.0"
+hashbrown = "0.15.2"
+itertools = "0.14.0"
 konst = "0.3.9"
 lazy_static = "1.5.0"
 never = "0.1.0"
--- a/orchid-extension/src/atom.rs
+++ b/orchid-extension/src/atom.rs
@@ -77,7 +77,7 @@ pub struct ForeignAtom<'a> {
  pub atom: api::Atom,
  pub pos: Pos,
 }
-impl<'a> ForeignAtom<'a> {
+impl ForeignAtom<'_> {
  pub fn oex_opt(self) -> Option<Expr> {
    let (handle, pos) = (self.expr.as_ref()?.clone(), self.pos.clone());
    let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { _life: PhantomData, ..self }) };
@@ -98,15 +98,15 @@ impl ForeignAtom<'static> {
    Some(M::Response::decode(&mut &rep[..]))
  }
 }
-impl<'a> fmt::Display for ForeignAtom<'a> {
+impl fmt::Display for ForeignAtom<'_> {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
    write!(f, "{}::{:?}", if self.expr.is_some() { "Clause" } else { "Tok" }, self.atom)
  }
 }
-impl<'a> fmt::Debug for ForeignAtom<'a> {
+impl fmt::Debug for ForeignAtom<'_> {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ForeignAtom({self})") }
 }
-impl<'a> AtomRepr for ForeignAtom<'a> {
+impl AtomRepr for ForeignAtom<'_> {
  type Ctx = SysCtx;
  fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self {
    Self { atom: atom.clone(), _life: PhantomData, ctx: ctx.clone(), expr: None, pos }
@@ -197,7 +197,7 @@ impl<A: AtomicFeatures> TypAtom<'static, A> {
    }
  }
 }
-impl<'a, A: AtomicFeatures> TypAtom<'a, A> {
+impl<A: AtomicFeatures> TypAtom<'_, A> {
  pub fn request<M: AtomMethod>(&self, req: M) -> M::Response where A: Supports<M> {
    M::Response::decode(
      &mut &self.data.ctx.reqnot.request(api::Fwd(
@@ -208,7 +208,7 @@ impl<'a, A: AtomicFeatures> TypAtom<'a, A> {
    )
  }
 }
-impl<'a, A: AtomicFeatures> Deref for TypAtom<'a, A> {
+impl<A: AtomicFeatures> Deref for TypAtom<'_, A> {
  type Target = A::Data;
  fn deref(&self) -> &Self::Target { &self.value }
 }
@@ -224,7 +224,7 @@ pub trait AtomDynfo: Send + Sync + 'static {
  fn print(&self, ctx: AtomCtx<'_>) -> String;
  fn handle_req(&self, ctx: AtomCtx<'_>, key: Sym, req: &mut dyn Read, rep: &mut dyn Write) -> bool;
  fn command(&self, ctx: AtomCtx<'_>) -> OrcRes<Option<Expr>>;
-  fn serialize(&self, ctx: AtomCtx<'_>, write: &mut dyn Write) -> Vec<api::ExprTicket>;
+  fn serialize(&self, ctx: AtomCtx<'_>, write: &mut dyn Write) -> Option<Vec<api::ExprTicket>>;
  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> api::Atom;
  fn drop(&self, ctx: AtomCtx<'_>);
 }
--- a/orchid-extension/src/atom_owned.rs
+++ b/orchid-extension/src/atom_owned.rs
@@ -1,19 +1,21 @@
-use std::any::{type_name, Any, TypeId};
+use std::any::{Any, TypeId, type_name};
 use std::borrow::Cow;
 use std::io::{Read, Write};
 use std::sync::Arc;

 use itertools::Itertools;
-use orchid_api_traits::{enc_vec, Decode, Encode};
+use never::Never;
+use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::id_store::{IdRecord, IdStore};
 use orchid_base::name::Sym;

 use crate::api;
 use crate::atom::{
-  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, MethodSet, Atomic, AtomicFeaturesImpl, AtomicVariant,
+  AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
+  err_not_callable, err_not_command, get_info,
 };
-use crate::expr::{bot, Expr, ExprHandle};
+use crate::expr::{Expr, ExprHandle, bot};
 use crate::system::SysCtx;

 pub struct OwnedVariant;
@@ -28,9 +30,7 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
      api::Atom { drop: Some(api::AtomId(rec.id())), data, owner: ctx.id }
    })
  }
-  fn _info() -> Self::_Info {
-    OwnedAtomDynfo(A::reg_reqs())
-  }
+  fn _info() -> Self::_Info { OwnedAtomDynfo(A::reg_reqs()) }
  type _Info = OwnedAtomDynfo<A>;
 }

@@ -54,7 +54,13 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
  fn call_ref(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> Expr {
    with_atom(id.unwrap(), |a| a.dyn_call_ref(ctx, arg))
  }
-  fn handle_req(&self, AtomCtx(_, id, ctx): AtomCtx, key: Sym, req: &mut dyn Read, rep: &mut dyn Write) -> bool {
+  fn handle_req(
+    &self,
+    AtomCtx(_, id, ctx): AtomCtx,
+    key: Sym,
+    req: &mut dyn Read,
+    rep: &mut dyn Write,
+  ) -> bool {
    with_atom(id.unwrap(), |a| {
      self.0.dispatch(a.as_any_ref().downcast_ref().unwrap(), ctx, key, req, rep)
    })
@@ -69,13 +75,11 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
    &self,
    AtomCtx(_, id, ctx): AtomCtx<'_>,
    write: &mut dyn Write,
-  ) -> Vec<api::ExprTicket> {
+  ) -> Option<Vec<api::ExprTicket>> {
    let id = id.unwrap();
    id.encode(write);
    with_atom(id, |a| a.dyn_serialize(ctx, write))
-      .into_iter()
-      .map(|t| t.handle.unwrap().tk)
-      .collect_vec()
+      .map(|v| v.into_iter().map(|t| t.handle.unwrap().tk).collect_vec())
  }
  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> orchid_api::Atom {
    let refs = refs.iter().map(|tk| Expr::new(Arc::new(ExprHandle::from_args(ctx.clone(), *tk))));
@@ -97,7 +101,7 @@ pub trait DeserializeCtx: Sized {
 }

 struct DeserCtxImpl<'a>(&'a [u8], &'a SysCtx);
-impl<'a> DeserializeCtx for DeserCtxImpl<'a> {
+impl DeserializeCtx for DeserCtxImpl<'_> {
  fn read<T: Decode>(&mut self) -> T { T::decode(&mut self.0) }
  fn is_empty(&self) -> bool { self.0.is_empty() }
  fn sys(&self) -> SysCtx { self.1.clone() }
@@ -108,6 +112,13 @@ pub trait RefSet {
  fn to_vec(self) -> Vec<Expr>;
 }

+static E_NON_SER: &str = "Never is a stand-in refset for non-serializable atoms";
+
+impl RefSet for Never {
+  fn from_iter<I>(_: I) -> Self { panic!("{E_NON_SER}") }
+  fn to_vec(self) -> Vec<Expr> { panic!("{E_NON_SER}") }
+}
+
 impl RefSet for () {
  fn to_vec(self) -> Vec<Expr> { Vec::new() }
  fn from_iter<I: Iterator<Item = Expr> + ExactSizeIterator>(refs: I) -> Self {
@@ -130,6 +141,16 @@ impl<const N: usize> RefSet for [Expr; N] {

 /// Atoms that have a [Drop]
 pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone + 'static {
+  /// If serializable, the collection that best stores subexpression references
+  /// for this atom.
+  /// 
+  /// - `()` for no subexppressions,
+  /// - `[Expr; N]` for a static number of subexpressions
+  /// - `Vec<Expr>` for a variable number of subexpressions
+  /// - `Never` if not serializable
+  ///
+  /// If this isn't `Never`, you must override the default, panicking
+  /// `serialize` and `deserialize` implementation
  type Refs: RefSet;
  fn val(&self) -> Cow<'_, Self::Data>;
  #[allow(unused_variables)]
@@ -147,8 +168,21 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone
  #[allow(unused_variables)]
  fn print(&self, ctx: SysCtx) -> String { format!("OwnedAtom({})", type_name::<Self>()) }
  #[allow(unused_variables)]
-  fn serialize(&self, ctx: SysCtx, write: &mut (impl Write + ?Sized)) -> Self::Refs;
-  fn deserialize(ctx: impl DeserializeCtx, refs: Self::Refs) -> Self;
+  fn serialize(&self, ctx: SysCtx, write: &mut (impl Write + ?Sized)) -> Self::Refs {
+    assert!(
+      TypeId::of::<Self::Refs>() != TypeId::of::<Never>(),
+      "The extension scaffold is broken, this function should never be called on Never Refs"
+    );
+    panic!("Either implement serialize or set Refs to Never for {}", type_name::<Self>())
+  }
+  #[allow(unused_variables)]
+  fn deserialize(ctx: impl DeserializeCtx, refs: Self::Refs) -> Self {
+    assert!(
+      TypeId::of::<Self::Refs>() != TypeId::of::<Never>(),
+      "The extension scaffold is broken, this function should never be called on Never Refs"
+    );
+    panic!("Either implement deserialize or set Refs to Never for {}", type_name::<Self>())
+  }
 }
 pub trait DynOwnedAtom: Send + Sync + 'static {
  fn atom_tid(&self) -> TypeId;
@@ -159,7 +193,7 @@ pub trait DynOwnedAtom: Send + Sync + 'static {
  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<Expr>>;
  fn dyn_free(self: Box<Self>, ctx: SysCtx);
  fn dyn_print(&self, ctx: SysCtx) -> String;
-  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<Expr>;
+  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Option<Vec<Expr>>;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
  fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
@@ -174,8 +208,9 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<Expr>> { self.command(ctx) }
  fn dyn_free(self: Box<Self>, ctx: SysCtx) { self.free(ctx) }
  fn dyn_print(&self, ctx: SysCtx) -> String { self.print(ctx) }
-  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<Expr> {
-    self.serialize(ctx, sink).to_vec()
+  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Option<Vec<Expr>> {
+    (TypeId::of::<Never>() != TypeId::of::<<Self as OwnedAtom>::Refs>())
+      .then(|| self.serialize(ctx, sink).to_vec())
  }
 }

--- a/orchid-extension/src/atom_thin.rs
+++ b/orchid-extension/src/atom_thin.rs
@@ -1,15 +1,16 @@
-use std::any::{type_name, Any, TypeId};
+use std::any::{Any, TypeId, type_name};
 use std::io::Write;

-use orchid_api_traits::{enc_vec, Coding};
+use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::name::Sym;

 use crate::api;
 use crate::atom::{
-  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, MethodSet, Atomic, AtomicFeaturesImpl, AtomicVariant
+  AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
+  err_not_callable, err_not_command, get_info,
 };
-use crate::expr::{bot, Expr, ExprHandle};
+use crate::expr::{Expr, ExprHandle, bot};
 use crate::system::SysCtx;

 pub struct ThinVariant;
@@ -23,9 +24,7 @@ impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant
      api::Atom { drop: None, data: buf, owner: ctx.id }
    })
  }
-  fn _info() -> Self::_Info {
-    ThinAtomDynfo(Self::reg_reqs())
-  }
+  fn _info() -> Self::_Info { ThinAtomDynfo(Self::reg_reqs()) }
  type _Info = ThinAtomDynfo<Self>;
 }

@@ -55,9 +54,9 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
  fn command(&self, AtomCtx(buf, _, ctx): AtomCtx<'_>) -> OrcRes<Option<Expr>> {
    T::decode(&mut &buf[..]).command(ctx)
  }
-  fn serialize(&self, actx: AtomCtx<'_>, write: &mut dyn Write) -> Vec<api::ExprTicket> {
+  fn serialize(&self, actx: AtomCtx<'_>, write: &mut dyn Write) -> Option<Vec<api::ExprTicket>> {
    T::decode(&mut &actx.0[..]).encode(write);
-    Vec::new()
+    Some(Vec::new())
  }
  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> api::Atom {
    assert!(refs.is_empty(), "Refs found when deserializing thin atom");
--- a/orchid-extension/src/conv.rs
+++ b/orchid-extension/src/conv.rs
@@ -28,7 +28,7 @@ fn err_type(pos: Pos) -> OrcErr {
  mk_err(intern!(str: "Type error"), "The atom is a different type than expected", [pos.into()])
 }

-impl<'a, A: AtomicFeatures> TryFromExpr for TypAtom<'a, A> {
+impl<A: AtomicFeatures> TryFromExpr for TypAtom<'_, A> {
  fn try_from_expr(expr: Expr) -> OrcRes<Self> {
    (expr.foreign_atom())
      .map_err(|ex| err_not_atom(ex.pos.clone()).into())
--- a/orchid-extension/src/entrypoint.rs
+++ b/orchid-extension/src/entrypoint.rs
@@ -232,8 +232,8 @@ fn extension_main_logic(data: ExtensionData) {
          match &atom_req {
            api::AtomReq::SerializeAtom(ser) => {
              let mut buf = enc_vec(&id);
-              let refs = nfo.serialize(actx, &mut buf);
-              hand.handle(ser, &(buf, refs))
+              let refs_opt = nfo.serialize(actx, &mut buf);
+              hand.handle(ser, &refs_opt.map(|refs| (buf, refs)))
            }
            api::AtomReq::AtomPrint(print@api::AtomPrint(_)) =>
              hand.handle(print, &nfo.print(actx)),
@@ -299,6 +299,6 @@ fn extension_main_logic(data: ExtensionData) {
  init_replica(rn.clone().map());
  while !exiting.load(Ordering::Relaxed) {
    let rcvd = recv_parent_msg().unwrap();
-    rn.receive(rcvd)
+    rn.receive(&rcvd)
  }
 }
--- a/orchid-extension/src/expr.rs
+++ b/orchid-extension/src/expr.rs
@@ -1,8 +1,9 @@
-use std::fmt;
 use std::ops::Deref;
 use std::sync::{Arc, OnceLock};
+use std::{backtrace, fmt};

 use derive_destructure::destructure;
+use orchid_base::clone;
 use orchid_base::error::{OrcErr, OrcErrv};
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
@@ -10,6 +11,8 @@ use orchid_base::reqnot::Requester;

 use crate::api;
 use crate::atom::{AtomFactory, ForeignAtom, ToAtom};
+use crate::conv::{ToExpr, TryFromExpr};
+use crate::func_atom::Lambda;
 use crate::system::SysCtx;

 #[derive(destructure)]
@@ -147,7 +150,7 @@ pub fn seq(ops: impl IntoIterator<Item = Expr>) -> Expr {
  recur(ops.into_iter()).expect("Empty list provided to seq!")
 }

-pub fn arg(n: u64) -> ExprKind { ExprKind::Arg(n) }
+pub fn arg(n: u64) -> Expr { inherit(ExprKind::Arg(n)) }

 pub fn lambda(n: u64, b: impl IntoIterator<Item = Expr>) -> Expr {
  inherit(ExprKind::Lambda(n, Box::new(call(b))))
@@ -162,3 +165,10 @@ pub fn call(v: impl IntoIterator<Item = Expr>) -> Expr {
 pub fn bot(ev: impl IntoIterator<Item = OrcErr>) -> Expr {
  inherit(ExprKind::Bottom(OrcErrv::new(ev).unwrap()))
 }
+
+pub fn with<I: TryFromExpr, O: ToExpr>(
+  expr: Expr,
+  cont: impl Fn(I) -> O + Clone + Send + Sync + 'static,
+) -> Expr {
+  call([lambda(0, [seq([arg(0), call([Lambda::new(cont).to_expr(), arg(0)])])]), expr])
+}
--- a/orchid-extension/src/func_atom.rs
+++ b/orchid-extension/src/func_atom.rs
@@ -5,13 +5,14 @@ use std::sync::{Arc, Mutex};

 use itertools::Itertools;
 use lazy_static::lazy_static;
+use never::Never;
 use orchid_api_traits::Encode;
 use orchid_base::error::OrcRes;
 use orchid_base::interner::Tok;
 use orchid_base::name::Sym;
 use trait_set::trait_set;

-use crate::atom::{MethodSet, Atomic};
+use crate::atom::{Atomic, MethodSet};
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use crate::conv::ToExpr;
 use crate::expr::{Expr, ExprHandle};
@@ -30,6 +31,11 @@ lazy_static! {
  static ref FUNS: Mutex<HashMap<Sym, (u8, Arc<dyn FunCB>)>> = Mutex::default();
 }

+/// An Atom representing a partially applied named native function. These
+/// partial calls are serialized into the name of the native function and the
+/// argument list.
+/// 
+/// See [Lambda] for the non-serializable variant
 #[derive(Clone)]
 pub(crate) struct Fun {
  path: Sym,
@@ -79,6 +85,40 @@ impl OwnedAtom for Fun {
  }
 }

+/// An Atom representing a partially applied native lambda. These are not serializable.
+/// 
+/// See [Fun] for the serializable variant
+#[derive(Clone)]
+pub struct Lambda {
+  args: Vec<Expr>,
+  arity: u8,
+  fun: Arc<dyn FunCB>,
+}
+impl Lambda {
+  pub fn new<I, O, F: ExprFunc<I, O>>(f: F) -> Self {
+    let fun = Arc::new(move |v| f.apply(v));
+    Self { args: vec![], arity: F::ARITY, fun }
+  }
+}
+impl Atomic for Lambda {
+  type Data = ();
+  type Variant = OwnedVariant;
+  fn reg_reqs() -> MethodSet<Self> { MethodSet::new() }
+}
+impl OwnedAtom for Lambda {
+  type Refs = Never;
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+  fn call_ref(&self, arg: ExprHandle) -> Expr {
+    let new_args = self.args.iter().cloned().chain([Expr::new(Arc::new(arg))]).collect_vec();
+    if new_args.len() == self.arity.into() {
+      (self.fun)(new_args).to_expr()
+    } else {
+      Self { args: new_args, arity: self.arity, fun: self.fun.clone() }.to_expr()
+    }
+  }
+  fn call(self, arg: ExprHandle) -> Expr { self.call_ref(arg) }
+}
+
 mod expr_func_derives {
  use orchid_base::error::OrcRes;

--- a/orchid-extension/src/lib.rs
+++ b/orchid-extension/src/lib.rs
@@ -5,7 +5,6 @@ pub mod atom_owned;
 pub mod atom_thin;
 pub mod conv;
 pub mod entrypoint;
-// pub mod error;
 pub mod expr;
 pub mod fs;
 pub mod func_atom;
--- a/orchid-extension/src/system.rs
+++ b/orchid-extension/src/system.rs
@@ -13,6 +13,7 @@ use crate::api;
 use crate::atom::{get_info, AtomCtx, AtomDynfo, AtomicFeatures, ForeignAtom, TypAtom};
 use crate::entrypoint::ExtReq;
 use crate::fs::DeclFs;
+use crate::func_atom::Fun;
 // use crate::fun::Fun;
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
@@ -37,7 +38,7 @@ pub trait DynSystemCard: Send + Sync + 'static {
 /// The indices of these are bitwise negated, such that the MSB of an atom index
 /// marks whether it belongs to this package (0) or the importer (1)
 fn general_atoms() -> impl Iterator<Item = Option<Box<dyn AtomDynfo>>> {
-  [/*Some(Fun::INFO)*/].into_iter()
+  [Some(Fun::dynfo())].into_iter()
 }

 pub fn atom_info_for(
--- a/orchid-extension/src/tree.rs
+++ b/orchid-extension/src/tree.rs
@@ -1,10 +1,10 @@
 use std::num::NonZero;
 use std::ops::Range;

-use dyn_clone::{clone_box, DynClone};
+use dyn_clone::{DynClone, clone_box};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::interner::{intern, Tok};
+use orchid_base::interner::{Tok, intern};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::parse::Comment;
@@ -15,7 +15,7 @@ use trait_set::trait_set;

 use crate::api;
 use crate::atom::{AtomFactory, ForeignAtom};
-use crate::conv::ToExpr;
+use crate::conv::{ToExpr, TryFromExpr};
 use crate::entrypoint::MemberRecord;
 use crate::expr::Expr;
 use crate::func_atom::{ExprFunc, Fun};
@@ -48,8 +48,8 @@ impl GenItem {
      GenItemKind::Import(cn) => api::ItemKind::Import(cn.tok().to_api()),
      GenItemKind::Macro(prio, rules) => api::ItemKind::Macro(api::MacroBlock {
        priority: prio,
-        rules: rules.into_iter().map(|r| r.to_api() ).collect_vec(),
-      })
+        rules: rules.into_iter().map(|r| r.to_api()).collect_vec(),
+      }),
    };
    let comments = self.comments.into_iter().map(|c| c.to_api()).collect_vec();
    api::Item { location: self.pos.to_api(), comments, kind }
--- a/orchid-host/Cargo.toml
+++ b/orchid-host/Cargo.toml
@@ -7,8 +7,8 @@ edition = "2021"

 [dependencies]
 derive_destructure = "1.0.0"
-hashbrown = "0.14.5"
-itertools = "0.13.0"
+hashbrown = "0.15.2"
+itertools = "0.14.0"
 lazy_static = "1.4.0"
 never = "0.1.0"
 num-traits = "0.2.19"
--- a/orchid-host/src/extension.rs
+++ b/orchid-host/src/extension.rs
@@ -2,26 +2,26 @@ use std::collections::VecDeque;
 use std::num::NonZero;
 use std::ops::Deref;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering};
-use std::sync::mpsc::{sync_channel, SyncSender};
+use std::sync::mpsc::{SyncSender, sync_channel};
 use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
 use std::{fmt, io, thread};

 use derive_destructure::destructure;
-use hashbrown::hash_map::Entry;
 use hashbrown::HashMap;
+use hashbrown::hash_map::Entry;
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use orchid_api_traits::Request;
 use orchid_base::char_filter::char_filter_match;
+use orchid_base::clone;
 use orchid_base::error::{OrcErrv, OrcRes};
-use orchid_base::interner::{intern, Tok};
+use orchid_base::interner::{Tok, intern};
 use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
 use orchid_base::macros::mtreev_from_api;
 use orchid_base::parse::Comment;
 use orchid_base::reqnot::{ReqNot, Requester as _};
-use orchid_base::tree::{ttv_from_api, AtomRepr};
-use orchid_base::clone;
+use orchid_base::tree::{AtomRepr, ttv_from_api};
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};

@@ -106,6 +106,8 @@ impl fmt::Display for AtomHand {
  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.print()) }
 }

+pub type OnMessage = Box<dyn FnMut(&[u8]) + Send>;
+
 /// The 3 primary contact points with an extension are
 /// - send a message
 /// - wait for a message to arrive
@@ -113,8 +115,8 @@ impl fmt::Display for AtomHand {
 ///
 /// There are no ordering guarantees about these
 pub trait ExtensionPort: Send + Sync {
+  fn set_onmessage(&self, callback: OnMessage);
  fn send(&self, msg: &[u8]);
-  fn receive(&self) -> Option<Vec<u8>>;
  fn header(&self) -> &api::ExtensionHeader;
 }

@@ -180,12 +182,12 @@ impl Extension {
            api::IntReq::InternStr(s) => hand.handle(&s, &intern(&**s.0).to_api()),
            api::IntReq::InternStrv(v) => hand.handle(&v, &intern(&*v.0).to_api()),
            api::IntReq::ExternStr(si) => hand.handle(&si, &Tok::<String>::from_api(si.0).arc()),
-            api::IntReq::ExternStrv(vi) => hand.handle(&vi, &Arc::new(
-              Tok::<Vec<Tok<String>>>::from_api(vi.0)
-                .iter()
-                .map(|t| t.to_api())
-                .collect_vec()
-            )),
+            api::IntReq::ExternStrv(vi) => hand.handle(
+              &vi,
+              &Arc::new(
+                Tok::<Vec<Tok<String>>>::from_api(vi.0).iter().map(|t| t.to_api()).collect_vec(),
+              ),
+            ),
          },
          api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
            let sys = System::resolve(atom.owner).unwrap();
@@ -210,30 +212,24 @@ impl Extension {
              kind: expr.to_api(),
            })
          },
-          api::ExtHostReq::RunMacros(ref rm @ api::RunMacros{ ref run_id, ref query }) => {
-            hand.handle(rm,
-              &macro_recur(*run_id,
-                mtreev_from_api(query, &mut |_| panic!("Recursion never contains atoms"))
+          api::ExtHostReq::RunMacros(ref rm @ api::RunMacros { ref run_id, ref query }) => hand
+            .handle(
+              rm,
+              &macro_recur(
+                *run_id,
+                mtreev_from_api(query, &mut |_| panic!("Recursion never contains atoms")),
              )
-              .map(|x| macro_treev_to_api(*run_id, x))
-            )
-          }
+              .map(|x| macro_treev_to_api(*run_id, x)),
+            ),
        },
      ),
    });
    let weak = Arc::downgrade(&ret);
-    thread::Builder::new()
-      .name(format!("host-end:{}", eh.name))
-      .spawn::<_, Option<()>>(move || {
-        loop {
-          // thread will exit if either the peer exits or the extension object is dropped.
-          // It holds a strong reference to the port so the port's destructor will not be
-          // called until the
-          let msg = port.receive()?;
-          weak.upgrade()?.reqnot.receive(msg);
-        }
-      })
-      .unwrap();
+    port.set_onmessage(Box::new(move |msg| {
+      if let Some(xd) = weak.upgrade() {
+        xd.reqnot.receive(msg)
+      }
+    }));
    Ok(Self(ret))
  }
  pub fn systems(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
@@ -268,11 +264,13 @@ impl SystemCtor {
      id,
    }));
    let root = (sys_inst.const_root.into_iter())
-      .map(|(k, v)| Member::from_api(
-        api::Member { name: k, kind: v },
-        Substack::Bottom.push(Tok::from_api(k)),
-        &data
-      ))
+      .map(|(k, v)| {
+        Member::from_api(
+          api::Member { name: k, kind: v },
+          Substack::Bottom.push(Tok::from_api(k)),
+          &data,
+        )
+      })
      .collect_vec();
    data.0.const_root.set(root).unwrap();
    inst_g.insert(id, data.clone());
--- a/orchid-host/src/macros.rs
+++ b/orchid-host/src/macros.rs
@@ -1,26 +1,29 @@
-use crate::{api, rule::shared::Matcher, tree::Code};
 use std::sync::{Arc, RwLock};

 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use lazy_static::lazy_static;
-use orchid_base::{macros::{mtreev_from_api, mtreev_to_api, MTok, MTree}, name::Sym};
-use ordered_float::NotNan;
+use orchid_base::macros::{MTok, MTree, mtreev_from_api, mtreev_to_api};
+use orchid_base::name::Sym;
 use trait_set::trait_set;
+
+use crate::api;
 use crate::extension::AtomHand;
+use crate::rule::matcher::{NamedMatcher, PriodMatcher};
+use crate::rule::state::{MatchState, OwnedState};
+use crate::tree::Code;

 pub type MacTok = MTok<'static, AtomHand>;
 pub type MacTree = MTree<'static, AtomHand>;

-trait_set!{
+trait_set! {
  trait MacroCB = Fn(Vec<MacTree>) -> Option<Vec<MacTree>> + Send + Sync;
 }

-lazy_static!{
+lazy_static! {
  static ref RECURSION: RwLock<HashMap<api::ParsId, Box<dyn MacroCB>>> = RwLock::default();
-  static ref MACRO_SLOTS: RwLock<HashMap<api::ParsId,
-    HashMap<api::MacroTreeId, Arc<MacTok>>
-  >> = RwLock::default();
+  static ref MACRO_SLOTS: RwLock<HashMap<api::ParsId, HashMap<api::MacroTreeId, Arc<MacTok>>>> =
+    RwLock::default();
 }

 pub fn macro_recur(run_id: api::ParsId, input: Vec<MacTree>) -> Option<Vec<MacTree>> {
@@ -42,29 +45,31 @@ pub fn macro_treev_from_api(api: Vec<api::MacroTree>) -> Vec<MacTree> {
 }

 pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option<Vec<MacTree>> {
-  let mut slots = (MACRO_SLOTS.write().unwrap())
-    .remove(&run_id).expect("Run not found");
+  let mut slots = (MACRO_SLOTS.write().unwrap()).remove(&run_id).expect("Run not found");
  return work(&mut slots, tree);
  fn work(
    slots: &mut HashMap<api::MacroTreeId, Arc<MacTok>>,
-    tree: &[MacTree]
+    tree: &[MacTree],
  ) -> Option<Vec<MacTree>> {
    let items = (tree.iter())
-      .map(|t| Some(MacTree {
-        tok: match &*t.tok {
-          MacTok::Atom(_) | MacTok::Name(_) | MacTok::Ph(_) => return None,
-          MacTok::Ref(_) => panic!("Ref is an extension-local optimization"),
-          MacTok::Slot(slot) => slots.get(&slot.id()).expect("Slot not found").clone(),
-          MacTok::S(paren, b) => Arc::new(MacTok::S(*paren, work(slots, b)?)),
-          MacTok::Lambda(a, b) => Arc::new(match (work(slots, a), work(slots, b)) {
-            (None, None) => return None,
-            (Some(a), None) => MacTok::Lambda(a, b.clone()),
-            (None, Some(b)) => MacTok::Lambda(a.clone(), b),
-            (Some(a), Some(b)) => MacTok::Lambda(a, b),
-          }),
-        },
-        pos: t.pos.clone()
-      }))
+      .map(|t| {
+        Some(MacTree {
+          tok: match &*t.tok {
+            MacTok::Atom(_) | MacTok::Name(_) | MacTok::Ph(_) => return None,
+            MacTok::Ref(_) => panic!("Ref is an extension-local optimization"),
+            MacTok::Done(_) => panic!("Created and removed by matcher"),
+            MacTok::Slot(slot) => slots.get(&slot.id()).expect("Slot not found").clone(),
+            MacTok::S(paren, b) => Arc::new(MacTok::S(*paren, work(slots, b)?)),
+            MacTok::Lambda(a, b) => Arc::new(match (work(slots, a), work(slots, b)) {
+              (None, None) => return None,
+              (Some(a), None) => MacTok::Lambda(a, b.clone()),
+              (None, Some(b)) => MacTok::Lambda(a.clone(), b),
+              (Some(a), Some(b)) => MacTok::Lambda(a, b),
+            }),
+          },
+          pos: t.pos.clone(),
+        })
+      })
      .collect_vec();
    let any_changed = items.iter().any(Option::is_some);
    any_changed.then(|| {
@@ -75,11 +80,95 @@ pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option<Vec<MacTree
  }
 }

-pub struct MacroRepo{
-  no_prio: Vec<(HashSet<Sym>, Matcher, Code)>,
-  prio: Vec<(HashSet<Sym>, NotNan<f64>, Matcher, Code)>,
+pub struct Macro<Matcher> {
+  deps: HashSet<Sym>,
+  cases: Vec<(Matcher, Code)>,
 }

-pub fn match_on_exprv<'a>(target: &'a [MacTree], pattern: &[MacTree]) -> Option<MatchhState<'a>> {
-  
+pub struct MacroRepo {
+  named: HashMap<Sym, Vec<Macro<NamedMatcher>>>,
+  prio: Vec<Macro<PriodMatcher>>,
+}
+impl MacroRepo {
+  /// TODO: the recursion inside this function needs to be moved into Orchid.
+  /// See the markdown note
+  pub fn process_exprv(&self, target: &[MacTree]) -> Option<Vec<MacTree>> {
+    let mut workcp = target.to_vec();
+    let mut lexicon;
+
+    'try_named: loop {
+      lexicon = HashSet::new();
+      target.iter().for_each(|tgt| fill_lexicon(tgt, &mut lexicon));
+
+      for (i, tree) in workcp.iter().enumerate() {
+        let MacTok::Name(name) = &*tree.tok else { continue };
+        let matches = (self.named.get(name).into_iter().flatten())
+          .filter(|m| m.deps.is_subset(&lexicon))
+          .filter_map(|mac| {
+            mac.cases.iter().find_map(|cas| cas.0.apply(&workcp[i..], |_| false).map(|s| (cas, s)))
+          })
+          .collect_vec();
+        assert!(
+          matches.len() < 2,
+          "Multiple conflicting matches on {:?}: {:?}",
+          &workcp[i..],
+          matches
+        );
+        let Some((case, (state, tail))) = matches.into_iter().next() else { continue };
+        let inj = (run_body(&case.1, state).into_iter())
+          .map(|MacTree { pos, tok }| MacTree { pos, tok: Arc::new(MacTok::Done(tok)) });
+        workcp.splice(i..(workcp.len() - tail.len()), inj);
+        continue 'try_named;
+      }
+      break;
+    }
+
+    if let Some(((_, body), state)) = (self.prio.iter())
+      .filter(|mac| mac.deps.is_subset(&lexicon))
+      .flat_map(|mac| &mac.cases)
+      .find_map(|case| case.0.apply(&workcp, |_| false).map(|state| (case, state)))
+    {
+      return Some(run_body(body, state));
+    }
+
+    let results = (workcp.into_iter())
+      .map(|mt| match &*mt.tok {
+        MTok::S(p, body) => self.process_exprv(body).map(|body| MTok::S(*p, body).at(mt.pos)),
+        MTok::Lambda(arg, body) => match (self.process_exprv(arg), self.process_exprv(body)) {
+          (Some(arg), Some(body)) => Some(MTok::Lambda(arg, body).at(mt.pos)),
+          (Some(arg), None) => Some(MTok::Lambda(arg, body.to_vec()).at(mt.pos)),
+          (None, Some(body)) => Some(MTok::Lambda(arg.to_vec(), body).at(mt.pos)),
+          (None, None) => None,
+        },
+        _ => None,
+      })
+      .collect_vec();
+    results.iter().any(Option::is_some).then(|| {
+      (results.into_iter().zip(target))
+        .map(|(opt, fb)| opt.unwrap_or_else(|| fb.clone()))
+        .collect_vec()
+    })
+  }
+}
+
+fn fill_lexicon(tgt: &MacTree, lexicon: &mut HashSet<Sym>) {
+  match &*tgt.tok {
+    MTok::Name(n) => {
+      lexicon.insert(n.clone());
+    },
+    MTok::Lambda(arg, body) => {
+      arg.iter().for_each(|t| fill_lexicon(t, lexicon));
+      body.iter().for_each(|t| fill_lexicon(t, lexicon))
+    },
+    MTok::S(_, body) => body.iter().for_each(|t| fill_lexicon(t, lexicon)),
+    _ => (),
+  }
+}
+
+fn run_body(body: &Code, mut state: MatchState<'_>) -> Vec<MacTree> {
+  let inject: Vec<MacTree> = todo!("Call the interpreter with bindings");
+  inject
+    .into_iter()
+    .map(|MTree { pos, tok }| MTree { pos, tok: Arc::new(MTok::Done(tok)) })
+    .collect_vec()
 }
--- a/orchid-host/src/rule/build.rs
+++ b/orchid-host/src/rule/build.rs
@@ -46,7 +46,7 @@ fn mk_scalv(pattern: &[MacTree]) -> Vec<ScalMatcher> { pattern.iter().map(mk_sca

 /// Pattern MUST start and end with a vectorial placeholder
 #[must_use]
-fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
+pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
  debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
  debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
  debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
@@ -91,7 +91,7 @@ fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
 #[must_use]
 fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
  match &*pattern.tok {
-    MacTok::Atom(_) => panic!("Atoms aren't supported in matchers"),
+    MacTok::Atom(_) | MacTok::Done(_) => panic!("Atoms and Done aren't supported in matchers"),
    MacTok::Name(n) => ScalMatcher::Name(n.clone()),
    MacTok::Ph(Ph { name, kind }) => match kind {
      PhKind::Vector { .. } => {
--- a/orchid-host/src/rule/matcher.rs
+++ b/orchid-host/src/rule/matcher.rs
@@ -1,21 +1,85 @@
-//! Abstract definition of a rule matcher, so that the implementation can
-//! eventually be swapped out for a different one.
-
-use std::rc::Rc;
+use std::fmt;

+use itertools::Itertools;
+use orchid_api::PhKind;
+use orchid_base::intern;
+use orchid_base::location::Pos;
 use orchid_base::name::Sym;
+use orchid_base::tree::Ph;

-use super::state::State;
-use crate::macros::MacTree;
+use super::any_match::any_match;
+use super::build::mk_any;
+use super::shared::{AnyMatcher, VecMatcher};
+use super::state::{MatchState, StateEntry};
+use super::vec_attrs::vec_attrs;
+use super::vec_match::vec_match;
+use crate::macros::{MacTok, MacTree};
+use crate::rule::build::mk_vec;

-/// Cacheable optimized structures for matching patterns on slices. This is
-/// injected to allow experimentation in the matcher implementation.
-pub trait Matcher {
-  /// Build matcher for a pattern
-  #[must_use]
-  fn new(pattern: Rc<Vec<MacTree>>) -> Self;
-  /// Apply matcher to a token sequence
-  #[must_use]
-  fn apply<'a>(&self, source: &'a [MacTree], save_loc: &impl Fn(Sym) -> bool)
-  -> Option<State<'a>>;
+pub fn first_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.first().unwrap()).is_some() }
+pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwrap()).is_some() }
+
+pub struct NamedMatcher(AnyMatcher);
+impl NamedMatcher {
+  pub fn new(pattern: &[MacTree]) -> Self {
+    assert!(
+      matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))),
+      "Named matchers must begin with a name"
+    );
+
+    match last_is_vec(pattern) {
+      true => Self(mk_any(pattern)),
+      false => {
+        let kind: PhKind = PhKind::Vector { priority: 0, at_least_one: false };
+        let suffix = [MacTok::Ph(Ph { name: intern!(str: "::after"), kind }).at(Pos::None)];
+        Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
+      },
+    }
+  }
+  /// Also returns the tail, if any, which should be matched further
+  /// Note that due to how priod works below, the main usable information from the tail is
+  /// its length
+  pub fn apply<'a>(
+    &self,
+    seq: &'a [MacTree],
+    save_loc: impl Fn(Sym) -> bool,
+  ) -> Option<(MatchState<'a>, &'a [MacTree])> {
+    any_match(&self.0, seq, &save_loc).map(|mut state| match state.remove(intern!(str: "::after")) {
+      Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"),
+      Some(StateEntry::Vec(v)) => (state, v),
+      None => (state, &[][..]),
+    })
+  }
+}
+impl fmt::Display for NamedMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}
+impl fmt::Debug for NamedMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "NamedMatcher({self})") }
+}
+
+pub struct PriodMatcher(VecMatcher);
+impl PriodMatcher {
+  pub fn new(pattern: &[MacTree]) -> Self {
+    assert!(
+      pattern.first().and_then(vec_attrs).is_some()
+      && pattern.last().and_then(vec_attrs).is_some(),
+      "Prioritized matchers must start and end with a vectorial",
+    );
+    Self(mk_vec(pattern))
+  }
+  /// tokens before the offset always match the prefix
+  pub fn apply<'a>(
+    &self,
+    seq: &'a [MacTree],
+    save_loc: impl Fn(Sym) -> bool,
+  ) -> Option<MatchState<'a>> {
+    vec_match(&self.0, seq, &save_loc)
+  }
+}
+impl fmt::Display for PriodMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}
+impl fmt::Debug for PriodMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "PriodMatcher({self})") }
 }
--- a/orchid-host/src/rule/mod.rs
+++ b/orchid-host/src/rule/mod.rs
@@ -21,4 +21,5 @@ pub mod shared;
 mod vec_match;
 pub mod state;
 mod vec_attrs;
+pub mod matcher;
 // pub mod matcher;
--- a/orchid-host/src/rule/scal_match.rs
+++ b/orchid-host/src/rule/scal_match.rs
@@ -2,7 +2,8 @@ use orchid_base::name::Sym;

 use super::any_match::any_match;
 use super::shared::ScalMatcher;
-use crate::{macros::{MacTok, MacTree}, rule::state::{MatchState, StateEntry}};
+use crate::macros::{MacTok, MacTree};
+use crate::rule::state::{MatchState, StateEntry};

 #[must_use]
 pub fn scal_match<'a>(
@@ -15,6 +16,7 @@ pub fn scal_match<'a>(
      true => MatchState::from_name(n1.clone(), expr.pos.clone()),
      false => MatchState::default(),
    }),
+    (ScalMatcher::Placeh { .. }, MacTok::Done(_)) => None,
    (ScalMatcher::Placeh { key }, _) =>
      Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
    (ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>
--- a/orchid-host/src/rule/shared.rs
+++ b/orchid-host/src/rule/shared.rs
@@ -4,14 +4,9 @@ use std::fmt;

 use itertools::Itertools;
 use orchid_base::interner::Tok;
-
-use super::any_match::any_match;
-use super::build::mk_any;
 use orchid_base::name::Sym;
-use crate::macros::MacTree;
-use crate::rule::state::MatchState;
 use orchid_base::side::Side;
-use orchid_base::tokens::{Paren, PARENS};
+use orchid_base::tokens::{PARENS, Paren};

 pub enum ScalMatcher {
  Name(Sym),
@@ -104,18 +99,3 @@ impl fmt::Display for AnyMatcher {
    }
  }
 }
-
-// ################ External ################
-
-/// A priority-order tree of the vectorial placeholders with scalars as leaves.
-pub struct Matcher(AnyMatcher);
-impl Matcher {
-  pub fn new(pattern: &[MacTree]) -> Self { Self(mk_any(pattern)) }
-  pub fn apply<'a>(&self, seq: &'a [MacTree], save_loc: &impl Fn(Sym) -> bool) -> Option<MatchState<'a>> {
-    any_match(&self.0, seq, save_loc)
-  }
-}
-
-impl fmt::Display for Matcher {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
-}
--- a/orchid-host/src/rule/state.rs
+++ b/orchid-host/src/rule/state.rs
@@ -1,20 +1,36 @@
-use std::sync::Arc;
+#![allow(unused)]
+use std::any::Any;

 use hashbrown::HashMap;
-use orchid_api::PhKind;
-use orchid_base::tree::Ph;
-use orchid_base::{interner::Tok, join::join_maps};
+use orchid_base::interner::Tok;
+use orchid_base::join::join_maps;
 use orchid_base::location::Pos;
-
-use crate::macros::{MacTok, MacTree};
+use orchid_base::match_mapping;
 use orchid_base::name::Sym;

+use crate::macros::MacTree;
+
+enum StackAction {
+  Return(Box<dyn Any>),
+  Call {
+    target: Box<dyn FnOnce(Box<dyn Any>) -> StackAction>,
+    param: Box<dyn Any>,
+    tail: Box<dyn FnOnce(Box<dyn Any>) -> StackAction>
+  }
+}
+
+struct Trampoline {
+  stack: Vec<Box<dyn FnOnce(Box<dyn Any>) -> StackAction>>
+}
+
+
+
 #[derive(Clone, Copy, Debug)]
 pub enum StateEntry<'a> {
  Vec(&'a [MacTree]),
  Scalar(&'a MacTree),
 }
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct MatchState<'a> {
  placeholders: HashMap<Tok<String>, StateEntry<'a>>,
  name_posv: HashMap<Sym, Vec<Pos>>,
@@ -26,9 +42,7 @@ impl<'a> MatchState<'a> {
  pub fn combine(self, s: Self) -> Self {
    Self {
      placeholders: self.placeholders.into_iter().chain(s.placeholders).collect(),
-      name_posv: join_maps(self.name_posv, s.name_posv, |_, l, r| {
-        l.into_iter().chain(r).collect()
-      }),
+      name_posv: join_maps(self.name_posv, s.name_posv, |_, l, r| l.into_iter().chain(r).collect()),
    }
  }
  pub fn ph_len(&self, key: &Tok<String>) -> Option<usize> {
@@ -40,45 +54,40 @@ impl<'a> MatchState<'a> {
  pub fn from_name(name: Sym, location: Pos) -> Self {
    Self { name_posv: HashMap::from([(name, vec![location])]), placeholders: HashMap::new() }
  }
+  pub fn remove(&mut self, name: Tok<String>) -> Option<StateEntry<'a>> {
+    self.placeholders.remove(&name)
+  }
+  pub fn mk_owned(self) -> OwnedState {
+    OwnedState {
+      placeholders: (self.placeholders.into_iter())
+        .map(|(k, v)| {
+          (
+            k.clone(),
+            match_mapping!(v, StateEntry => OwnedEntry {
+              Scalar(tree.clone()),
+              Vec(v.to_vec()),
+            }),
+          )
+        })
+        .collect(),
+      name_posv: self.name_posv,
+    }
+  }
 }
 impl Default for MatchState<'static> {
  fn default() -> Self { Self { name_posv: HashMap::new(), placeholders: HashMap::new() } }
 }

-#[must_use]
-pub fn apply_exprv(template: &[MacTree], state: &MatchState) -> Vec<MacTree> {
-  template.iter().map(|e| apply_expr(e, state)).flat_map(Vec::into_iter).collect()
+#[derive(Clone, Debug)]
+pub enum OwnedEntry {
+  Vec(Vec<MacTree>),
+  Scalar(MacTree),
 }
-
-#[must_use]
-pub fn apply_expr(template: &MacTree, state: &MatchState) -> Vec<MacTree> {
-  let MacTree { pos, tok } = template;
-  match &**tok {
-    MacTok::Name(n) => match state.name_posv.get(n) {
-      None => vec![template.clone()],
-      Some(locs) => vec![MacTree { tok: tok.clone(), pos: locs[0].clone() }],
-    },
-    MacTok::Atom(_) => vec![template.clone()],
-    MacTok::S(c, body) => vec![MacTree {
-      pos: pos.clone(), tok: Arc::new(MacTok::S(*c, apply_exprv(body.as_slice(), state))),
-    }],
-    MacTok::Ph(Ph { name, kind }) => {
-      let Some(value) = state.placeholders.get(name) else {
-        panic!("Placeholder does not have a value in state")
-      };
-      match (kind, value) {
-        (PhKind::Scalar, StateEntry::Scalar(item)) => vec![(*item).clone()],
-        (PhKind::Vector { .. }, StateEntry::Vec(chunk)) => chunk.to_vec(),
-        _ => panic!("Type mismatch between template and state"),
-      }
-    },
-    MacTok::Lambda(arg, body) => vec![MacTree {
-      pos: pos.clone(),
-      tok: Arc::new(MacTok::Lambda(
-        apply_exprv(arg, state),
-        apply_exprv(&body[..], state),
-      )),
-    }],
-    MacTok::Slot(_) | MacTok::Ref(_) => panic!("Extension-only variants"),
-  }
+pub struct OwnedState {
+  placeholders: HashMap<Tok<String>, OwnedEntry>,
+  name_posv: HashMap<Sym, Vec<Pos>>,
+}
+impl OwnedState {
+  pub fn get(&self, key: &Tok<String>) -> Option<&OwnedEntry> { self.placeholders.get(key) }
+  pub fn positions(&self, name: &Sym) -> &[Pos] { self.name_posv.get(name).map_or(&[], |v| &v[..]) }
 }
--- a/orchid-host/src/rule/vec_match.rs
+++ b/orchid-host/src/rule/vec_match.rs
@@ -18,7 +18,7 @@ pub fn vec_match<'a>(
      if *nonzero && seq.is_empty() {
        return None;
      }
-      return Some(MatchState::from_ph(key.clone(), StateEntry::Vec(seq)));
+      Some(MatchState::from_ph(key.clone(), StateEntry::Vec(seq)))
    },
    VecMatcher::Scan { left, sep, right, direction } => {
      if seq.len() < sep.len() {
--- a/orchid-host/src/subprocess.rs
+++ b/orchid-host/src/subprocess.rs
@@ -1,5 +1,6 @@
 use std::io::{self, BufRead as _, Write};
 use std::path::PathBuf;
+use std::sync::mpsc::{sync_channel, SyncSender};
 use std::sync::Mutex;
 use std::{process, thread};

@@ -8,12 +9,12 @@ use orchid_base::logging::Logger;
 use orchid_base::msg::{recv_msg, send_msg};

 use crate::api;
-use crate::extension::ExtensionPort;
+use crate::extension::{ExtensionPort, OnMessage};

 pub struct Subprocess {
  child: Mutex<process::Child>,
  stdin: Mutex<process::ChildStdin>,
-  stdout: Mutex<process::ChildStdout>,
+  set_onmessage: SyncSender<OnMessage>,
  header: api::ExtensionHeader,
 }
 impl Subprocess {
@@ -31,6 +32,18 @@ impl Subprocess {
    let mut stdout = child.stdout.take().unwrap();
    let header = api::ExtensionHeader::decode(&mut stdout);
    let child_stderr = child.stderr.take().unwrap();
+    let (set_onmessage, recv_onmessage) = sync_channel(0);
+    thread::Builder::new().name(format!("stdout-fwd:{prog}")).spawn(move || {
+      let mut onmessage: Box<dyn FnMut(&[u8]) + Send> = recv_onmessage.recv().unwrap();
+      drop(recv_onmessage);
+      loop {
+        match recv_msg(&mut stdout) {
+          Ok(msg) => onmessage(&msg[..]),
+          Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break,
+          Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
+        }
+      }
+    })?;
    thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || {
      let mut reader = io::BufReader::new(child_stderr);
      loop {
@@ -44,7 +57,7 @@ impl Subprocess {
    Ok(Self {
      child: Mutex::new(child),
      stdin: Mutex::new(stdin),
-      stdout: Mutex::new(stdout),
+      set_onmessage,
      header,
    })
  }
@@ -53,6 +66,9 @@ impl Drop for Subprocess {
  fn drop(&mut self) { self.child.lock().unwrap().wait().expect("Extension exited with error"); }
 }
 impl ExtensionPort for Subprocess {
+  fn set_onmessage(&self, callback: OnMessage) {
+    self.set_onmessage.send(callback).unwrap();
+  }
  fn header(&self) -> &orchid_api::ExtensionHeader { &self.header }
  fn send(&self, msg: &[u8]) {
    if msg.starts_with(&[0, 0, 0, 0x1c]) {
@@ -60,11 +76,4 @@ impl ExtensionPort for Subprocess {
    }
    send_msg(&mut *self.stdin.lock().unwrap(), msg).unwrap()
  }
-  fn receive(&self) -> Option<Vec<u8>> {
-    match recv_msg(&mut *self.stdout.lock().unwrap()) {
-      Ok(msg) => Some(msg),
-      Err(e) if e.kind() == io::ErrorKind::BrokenPipe => None,
-      Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
-    }
-  }
 }
--- a/orchid-std/src/string/str_atom.rs
+++ b/orchid-std/src/string/str_atom.rs
@@ -77,7 +77,7 @@ pub enum OrcString<'a> {
  Val(TypAtom<'a, StrAtom>),
  Int(TypAtom<'a, IntStrAtom>),
 }
-impl<'a> OrcString<'a> {
+impl OrcString<'_> {
  pub fn get_string(&self) -> Arc<String> {
    match &self {
      Self::Int(tok) => Tok::from_api(tok.value).arc(),