Macro processing factored into Orchid functions

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "ahash"
@@ -335,6 +335,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"
+
 [[package]]
 name = "funty"
 version = "2.0.0"
@@ -376,9 +382,16 @@ name = "hashbrown"
 version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+
+[[package]]
+name = "hashbrown"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
 dependencies = [
- "ahash 0.8.11",
  "allocator-api2",
+ "equivalent",
+ "foldhash",
 ]
 
 [[package]]
@@ -418,6 +431,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -504,7 +526,7 @@ name = "orchid-api-derive"
 version = "0.1.0"
 dependencies = [
  "darling",
- "itertools",
+ "itertools 0.13.0",
  "orchid-api-traits",
  "proc-macro2 1.0.78",
  "quote 1.0.35",
@@ -515,7 +537,7 @@ dependencies = [
 name = "orchid-api-traits"
 version = "0.1.0"
 dependencies = [
- "itertools",
+ "itertools 0.13.0",
  "never",
  "ordered-float",
 ]
@@ -526,8 +548,8 @@ version = "0.1.0"
 dependencies = [
  "derive_destructure",
  "dyn-clone",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.2",
- "itertools",
+ "itertools 0.14.0",
  "lazy_static",
  "never",
  "num-traits",
@@ -548,8 +570,8 @@ dependencies = [
  "ahash 0.8.11",
  "derive_destructure",
  "dyn-clone",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.2",
- "itertools",
+ "itertools 0.14.0",
  "konst",
  "lazy_static",
  "never",
@@ -569,8 +591,8 @@ name = "orchid-host"
 version = "0.1.0"
 dependencies = [
  "derive_destructure",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.2",
- "itertools",
+ "itertools 0.14.0",
  "lazy_static",
  "never",
  "num-traits",
@@ -587,7 +609,7 @@ dependencies = [
 name = "orchid-std"
 version = "0.1.0"
 dependencies = [
- "itertools",
+ "itertools 0.13.0",
  "never",
  "once_cell",
  "orchid-api",
@@ -604,7 +626,7 @@ version = "0.1.0"
 dependencies = [
  "camino",
  "clap",
- "itertools",
+ "itertools 0.13.0",
  "orchid-base",
  "orchid-host",
 ]

SWAP.md

@@ -0,0 +1,12 @@
+### Swapfile
+A loose collection of ideas to quickly resume work
+
+---
+
+Must figure out how preprocessor can both be a System and referenced in the interpreter
+
+Must actually write macro system as recorded in note
+
+At this point swappable preprocessors aren't a target because interaction with module system sounds complicated
+
+Check if any of this needs interpreter, if so, start with that

notes/new_macro_model.md

@@ -1,9 +1,11 @@
-# Code sample for the new macro system
+We make a distinction between positional and prioritized macros.
+
+# Named macro
 
 ```
 macro (
 	rule match ...$expr { ...$body } => \recurse. '(
-		fn::pass (...$expr) \match::value. ...$(
+		fn::pass (...$expr) \match::value. ..$(
 			fn::pass (quote::split body ';) \cases.
 			fn::pass (list::map cases \x. (
 				fn::pass (quote::split_once x '=>) \pair.
@@ -21,25 +23,79 @@ macro (
 		)
 	)
 )
+```
 
---[
+Named macro patterns must start with a name token. They are always evaluated first. If they don't end with a vectorial placeholder, macro evaluation continues after them so that they can become first arguments to infix operators.
-	Macros are run from the top down.
+
-	For every token
+# Prioritized macro
-		1. If it's a name token, test all macros starting with that name
+
-		2. If none match and this is the first token in a list, test all macros starting with vectorials
+```
-	Test all in a set of macros
+macro 3 (
+	...$lhs + ...$rhs:1 => \recurse. '(add (..$(recurse lhs)) (..$(recurse rhs)))
+)
+```
+
+Prioritised macro patterns must start and end with a vectorial placeholder. They represent infix operators.
+
+# Algorithm
+
+Macros are checked from the outermost block inwards.
+
+1. For every name token, test all named macros starting with that name
    1. Take the first rule that matches in each block
    2. If there are multiple matches across blocks, raise an ambiguity error
-		3. If the single match is in the recursion stack, raise a recursion error
+   3. If the tail is implicit, recurse on it
-		4. Add the matching rule to the recursion stack, then execute the body.
+2. Test all prioritized macros
-]--
+   1. Take the first rule that matches in the highest prioritized block
 
---[
+Test all in a set of macros
-	1. Macro patterns are held in the host, they don't contain atoms, and atoms are never considered equal, so the matcher doesn't have to call an extension.
+1. Take the first rule that matches in each block
-	2. The body of macros may be defined in Rust. If it isn't, the entire interpreter will recurse on the macro to calculate the output.
+2. If there are multiple matches across blocks, raise an ambiguity error
-]--
+3. If the single match is in the recursion stack, raise a recursion error
+4. Add the matching rule to the recursion stack, then execute the body.
 
---[
+# Considerations
-	1. if the rule body uses the same macro, fail with the rule
+
-	2. if the rule explicitly recursively invokes the same macro, fail with the first match
+Maxims for the location of macros
-]--
+
+1. Macro patterns are held in the host, they don't contain atoms, and atoms are never considered equal, so the matcher doesn't have to call an extension.
+2. The body of macros may be defined in Rust. If it isn't, the entire interpreter will recurse on the macro to calculate the output.
+
+On recursion, the following errors can be detected
+
+1. if the rule body uses the same macro, fail with the rule
+2. if the rule explicitly recursively invokes the same macro, fail with the first match
+
+# Elements of the extension
+
+Recursion has to happen through the interpreter itself, so the macro system is defined in terms of atoms just like an extension
+
+- atom `MacTree` depicts a single token. `MacTree(tpl)` is also a `MacTree` but it can contain the independently unrepresentable templated slot node
+
+- lexer `'` followed by any single token always generates `MacTree`. If it contains placeholders which are tokens prefixed with `$` or `..$`, it generates a call to `instantiate_tpl` with a prepared `MacTree(tpl)` as the first argument and the placeholder values after. `MacTree(tpl)` only exists as an internal subresult routed directly to `instantiate_tpl`.
+
+- line parser `macro` parses a macro with the existing logic
+- atom `MacRecurState` holds the recursion state
+- function `resolve_recur` finds all matches on a MacTree
+  - type: `MacRecurState -> MacTree -> MacTree`
+  - use all macros to find all matches in the tree
+  - for each match
+    - check for recursion violations
+    - wrap the body in iife-s corresponding to the named values in the match state
+    - emit a recursive call to process and run the body, and pass the same recursive call as argument for the macro to use
+		```
+		(\recur. lower (recur $body) recur)
+			(resolve_recur $mac_recur_state)
+		```
+  - emit a single call to `instantiate_tpl` which receives all of these
+- function `instantiate_tpl` inserts `MacTree` values into a `MacTree(tpl)`
+  - type: `MacTree(tpl) [-> MacTree] -> MacTree`  
+		_this function deduces the number of arguments from the first argument. This combines poorly with autocurry, but it's an easy way to avoid representing standalone tree lists_
+  - walks the tree to find max template slot number, reads and type checks as many template values
+  - returns the populated tree
+- function `resolve` is the main entry point of the code
+  - type: `MacTree -> MacTree`
+  - invokes `resolve_recur` with an empty `MacRecurState`
+- function `lower` is the main exit point of the code
+  - type: `MacTree -> any`
+  - Lowers `MacTree` into the equivalent `Expr`.

orchid-api-traits/src/coding.rs

@@ -77,7 +77,7 @@ nonzero_impl!(std::num::NonZeroI32);
 nonzero_impl!(std::num::NonZeroI64);
 nonzero_impl!(std::num::NonZeroI128);
 
-impl<'a, T: Encode + ?Sized> Encode for &'a T {
+impl<T: Encode + ?Sized> Encode for &T {
   fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
 }
 macro_rules! float_impl {
@@ -285,12 +285,12 @@ smart_ptr!(Arc);
 smart_ptr!(Rc);
 smart_ptr!(Box);
 
-impl<'a, T: ?Sized + ToOwned> Decode for Cow<'a, T>
+impl<T: ?Sized + ToOwned> Decode for Cow<'_, T>
 where T::Owned: Decode
 {
   fn decode<R: Read + ?Sized>(read: &mut R) -> Self { Cow::Owned(T::Owned::decode(read)) }
 }
-impl<'a, T: ?Sized + Encode + ToOwned> Encode for Cow<'a, T> {
+impl<T: ?Sized + Encode + ToOwned> Encode for Cow<'_, T> {
   fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
 }
 

orchid-api/src/atom.rs

@@ -67,7 +67,7 @@ impl Request for FinalCall {
 #[extends(AtomReq, HostExtReq)]
 pub struct SerializeAtom(pub Atom);
 impl Request for SerializeAtom {
-  type Response = (Vec<u8>, Vec<ExprTicket>);
+  type Response = Option<(Vec<u8>, Vec<ExprTicket>)>;
 }
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]

orchid-base/Cargo.toml

@@ -8,8 +8,8 @@ edition = "2021"
 [dependencies]
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.17"
-hashbrown = "0.14.3"
+hashbrown = "0.15.2"
-itertools = "0.13.0"
+itertools = "0.14.0"
 lazy_static = "1.4.0"
 never = "0.1.0"
 num-traits = "0.2.19"

orchid-base/src/box_cow.rs

@@ -6,10 +6,10 @@ pub enum ArcCow<'a, T: ?Sized + ToOwned> {
   Borrowed(&'a T),
   Owned(Arc<T::Owned>),
 }
-impl<'a, T: ?Sized + ToOwned> ArcCow<'a, T> {
+impl<T: ?Sized + ToOwned> ArcCow<'_, T> {
   pub fn owned(value: T::Owned) -> Self { Self::Owned(Arc::new(value)) }
 }
-impl<'a, T: ?Sized + ToOwned> Clone for ArcCow<'a, T> {
+impl<T: ?Sized + ToOwned> Clone for ArcCow<'_, T> {
   fn clone(&self) -> Self {
     match self {
       Self::Borrowed(r) => Self::Borrowed(r),
@@ -18,7 +18,7 @@ impl<'a, T: ?Sized + ToOwned> Clone for ArcCow<'a, T> {
   }
 }
 
-impl<'a, T: ?Sized + ToOwned> Deref for ArcCow<'a, T> {
+impl<T: ?Sized + ToOwned> Deref for ArcCow<'_, T> {
   type Target = T;
   fn deref(&self) -> &Self::Target {
     match self {

orchid-base/src/id_store.rs

@@ -33,17 +33,17 @@ impl<T> Default for IdStore<T> {
 }
 
 pub struct IdRecord<'a, T>(NonZeroU64, MutexGuard<'a, HashMap<NonZeroU64, T>>);
-impl<'a, T> IdRecord<'a, T> {
+impl<T> IdRecord<'_, T> {
   pub fn id(&self) -> NonZeroU64 { self.0 }
   pub fn remove(mut self) -> T { self.1.remove(&self.0).unwrap() }
 }
-impl<'a, T> Deref for IdRecord<'a, T> {
+impl<T> Deref for IdRecord<'_, T> {
   type Target = T;
   fn deref(&self) -> &Self::Target {
     self.1.get(&self.0).expect("Existence checked on construction")
   }
 }
-impl<'a, T> DerefMut for IdRecord<'a, T> {
+impl<T> DerefMut for IdRecord<'_, T> {
   fn deref_mut(&mut self) -> &mut Self::Target {
     self.1.get_mut(&self.0).expect("Existence checked on construction")
   }

orchid-base/src/macros.rs

@@ -1,15 +1,18 @@
+use std::marker::PhantomData;
+use std::sync::Arc;
+
 use itertools::Itertools;
 use never::Never;
 use trait_set::trait_set;
 
-use crate::{match_mapping, name::Sym, tree::{Paren, Ph}};
+use crate::location::Pos;
-use std::{marker::PhantomData, sync::Arc};
+use crate::name::Sym;
-
+use crate::tree::{Paren, Ph};
-use crate::{api, location::Pos};
+use crate::{api, match_mapping};
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct MacroSlot<'a>(api::MacroTreeId, PhantomData<&'a ()>);
-impl<'a> MacroSlot<'a> {
+impl MacroSlot<'_> {
   pub fn id(self) -> api::MacroTreeId { self.0 }
 }
 
@@ -21,7 +24,7 @@ trait_set! {
 #[derive(Clone, Debug)]
 pub struct MTree<'a, A> {
   pub pos: Pos,
-  pub tok: Arc<MTok<'a, A>>
+  pub tok: Arc<MTok<'a, A>>,
 }
 impl<'a, A> MTree<'a, A> {
   pub(crate) fn from_api(api: &api::MacroTree, do_atom: &mut impl MacroAtomFromApi<'a, A>) -> Self {
@@ -40,12 +43,16 @@ pub enum MTok<'a, A> {
   Lambda(Vec<MTree<'a, A>>, Vec<MTree<'a, A>>),
   Ph(Ph),
   Atom(A),
-  Ref(Box<MTok<'a, Never>>),
+  /// Used in extensions to directly return input
+  Ref(Arc<MTok<'a, Never>>),
+  /// Used in the matcher to skip previous macro output which can only go in
+  /// vectorial placeholders
+  Done(Arc<MTok<'a, A>>),
 }
 impl<'a, A> MTok<'a, A> {
   pub(crate) fn from_api(
     api: &api::MacroToken,
-    do_atom: &mut impl MacroAtomFromApi<'a, A>
+    do_atom: &mut impl MacroAtomFromApi<'a, A>,
   ) -> Self {
     match_mapping!(&api, api::MacroToken => MTok::<'a, A> {
       Lambda(x => mtreev_from_api(x, do_atom), b => mtreev_from_api(b, do_atom)),
@@ -58,6 +65,7 @@ impl<'a, A> MTok<'a, A> {
     })
   }
   pub(crate) fn to_api(&self, do_atom: &mut impl MacroAtomToApi<A>) -> api::MacroToken {
+    fn sink(n: &Never) -> api::MacroToken { match *n {} }
     match_mapping!(&self, MTok => api::MacroToken {
       Lambda(x => mtreev_to_api(x, do_atom), b => mtreev_to_api(b, do_atom)),
       Name(t.tok().to_api()),
@@ -65,22 +73,24 @@ impl<'a, A> MTok<'a, A> {
       S(p.clone(), b => mtreev_to_api(b, do_atom)),
       Slot(tk.0.clone()),
     } {
-      MTok::Ref(r) => r.to_api(&mut |e| match *e {}),
+      MTok::Ref(r) => r.to_api(&mut sink),
+      MTok::Done(t) => t.to_api(do_atom),
       MTok::Atom(a) => do_atom(a),
     })
   }
+  pub fn at(self, pos: Pos) -> MTree<'a, A> { MTree { pos, tok: Arc::new(self) } }
 }
 
 pub fn mtreev_from_api<'a, 'b, A>(
   api: impl IntoIterator<Item = &'b api::MacroTree>,
-  do_atom: &mut impl MacroAtomFromApi<'a, A>
+  do_atom: &mut impl MacroAtomFromApi<'a, A>,
 ) -> Vec<MTree<'a, A>> {
   api.into_iter().map(|api| MTree::from_api(api, do_atom)).collect_vec()
 }
 
 pub fn mtreev_to_api<'a: 'b, 'b, A: 'b>(
   v: impl IntoIterator<Item = &'b MTree<'a, A>>,
-  do_atom: &mut impl MacroAtomToApi<A>
+  do_atom: &mut impl MacroAtomToApi<A>,
 ) -> Vec<api::MacroTree> {
   v.into_iter().map(|t| t.to_api(do_atom)).collect_vec()
 }

orchid-base/src/match_mapping.rs

@@ -75,6 +75,9 @@ macro_rules! match_mapping {
   (@PAT_MUNCH $ctx:tt ($($names:ident)*) $name:ident => $value:expr , $($tail:tt)*) => {
     match_mapping!(@PAT_MUNCH $ctx ($($names)* $name) $($tail)*)
   };
+  (@PAT_MUNCH $ctx:tt ($($names:ident)*) $name:ident () $value:expr , $($tail:tt)*) => {
+    match_mapping!(@PAT_MUNCH $ctx ($($names)* $name) $($tail)*)
+  };
   (@PAT_MUNCH $ctx:tt ($($names:ident)*) $name:ident . $($tail:tt)*) => { 
     match_mapping!(@PAT_DOT_MUNCH $ctx ($($names)* $name) $($tail)*)
   };
@@ -97,6 +100,9 @@ macro_rules! match_mapping {
   (@VAL_MUNCH $ctx:tt ($($prefix:tt)*) $name:ident => $value:expr , $($tail:tt)*) => {
     match_mapping!(@VAL_MUNCH $ctx ($($prefix)* ($name ($value)) ) $($tail)*)
   };
+  (@VAL_MUNCH $ctx:tt ($($prefix:tt)*) $name:ident () $value:expr , $($tail:tt)*) => {
+    match_mapping!(@VAL_MUNCH $ctx ($($prefix)* ($name ($value($name))) ) $($tail)*)
+  };
   (@VAL_MUNCH $ctx:tt $fields:tt $name:ident . $member:tt $($tail:tt)*) => {
     match_mapping!(@VAL_DOT_MUNCH $ctx $fields $name ($name . $member ) $($tail)*)
   };

orchid-base/src/parse.rs

@@ -67,11 +67,11 @@ impl<'a, 'b, A: AtomRepr, X: ExtraTok> Snippet<'a, 'b, A, X> {
     self.split_at(non_fluff_start.unwrap_or(self.len())).1
   }
 }
-impl<'a, 'b, A: AtomRepr, X: ExtraTok> Copy for Snippet<'a, 'b, A, X> {}
+impl<A: AtomRepr, X: ExtraTok> Copy for Snippet<'_, '_, A, X> {}
-impl<'a, 'b, A: AtomRepr, X: ExtraTok> Clone for Snippet<'a, 'b, A, X> {
+impl<A: AtomRepr, X: ExtraTok> Clone for Snippet<'_, '_, A, X> {
   fn clone(&self) -> Self { *self }
 }
-impl<'a, 'b, A: AtomRepr, X: ExtraTok> Deref for Snippet<'a, 'b, A, X> {
+impl<'b, A: AtomRepr, X: ExtraTok> Deref for Snippet<'_, 'b, A, X> {
   type Target = [TokTree<'b, A, X>];
   fn deref(&self) -> &Self::Target { self.cur }
 }

orchid-base/src/reqnot.rs

@@ -105,16 +105,16 @@ impl<T: MsgSet> ReqNot<T> {
   }
 
   /// Can be called from a polling thread or dispatched in any other way
-  pub fn receive(&self, message: Vec<u8>) {
+  pub fn receive(&self, message: &[u8]) {
     let mut g = self.0.lock().unwrap();
-    let (id, payload) = get_id(&message[..]);
+    let (id, payload) = get_id(message);
     if id == 0 {
       let mut notif = clone_box(&*g.notif);
       mem::drop(g);
       notif(<T::In as Channel>::Notif::decode(&mut &payload[..]), self.clone())
     } else if 0 < id.bitand(1 << 63) {
       let sender = g.responses.remove(&!id).expect("Received response for invalid message");
-      sender.send(message).unwrap();
+      sender.send(message.to_vec()).unwrap();
     } else {
       let message = <T::In as Channel>::Req::decode(&mut &payload[..]);
       let mut req = clone_box(&*g.req);
@@ -150,7 +150,7 @@ impl<'a, T> MappedRequester<'a, T> {
   }
 }
 
-impl<'a, T> DynRequester for MappedRequester<'a, T> {
+impl<T> DynRequester for MappedRequester<'_, T> {
   type Transfer = T;
   fn raw_request(&self, data: Self::Transfer) -> RawReply { self.0(data) }
 }
@@ -181,7 +181,7 @@ pub trait Requester: DynRequester {
     MappedRequester::new(self)
   }
 }
-impl<'a, This: DynRequester + ?Sized + 'a> Requester for This {
+impl<This: DynRequester + ?Sized> Requester for This {
   fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
     R::Response::decode(&mut &self.raw_request(data.into())[..])
   }
@@ -229,7 +229,7 @@ mod test {
       |_, _| panic!("Not receiving a request"),
     );
     let sender = ReqNot::<TestMsgSet>::new(
-      clone!(receiver; move |d, _| receiver.receive(d.to_vec())),
+      clone!(receiver; move |d, _| receiver.receive(d)),
       |_, _| panic!("Should not receive notif"),
       |_, _| panic!("Should not receive request"),
     );
@@ -245,7 +245,7 @@ mod test {
     let sender = Arc::new(ReqNot::<TestMsgSet>::new(
       {
         let receiver = receiver.clone();
-        move |d, _| receiver.lock().unwrap().as_ref().unwrap().receive(d.to_vec())
+        move |d, _| receiver.lock().unwrap().as_ref().unwrap().receive(d)
       },
       |_, _| panic!("Should not receive notif"),
       |_, _| panic!("Should not receive request"),
@@ -253,7 +253,7 @@ mod test {
     *receiver.lock().unwrap() = Some(ReqNot::new(
       {
         let sender = sender.clone();
-        move |d, _| sender.receive(d.to_vec())
+        move |d, _| sender.receive(d)
       },
       |_, _| panic!("Not receiving notifs"),
       |hand, req| {

orchid-base/src/tree.rs

@@ -58,10 +58,10 @@ pub struct TokHandle<'a>(api::TreeTicket, PhantomData<&'a ()>);
 impl TokHandle<'static> {
   pub fn new(tt: api::TreeTicket) -> Self { TokHandle(tt, PhantomData) }
 }
-impl<'a> TokHandle<'a> {
+impl TokHandle<'_> {
   pub fn ticket(self) -> api::TreeTicket { self.0 }
 }
-impl<'a> Display for TokHandle<'a> {
+impl Display for TokHandle<'_> {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Handle({})", self.0.0) }
 }
 
@@ -146,7 +146,7 @@ impl<'a, A: AtomRepr, X: ExtraTok> TokTree<'a, A, X> {
   }
 }
 
-impl<'a, A: AtomRepr, X: ExtraTok> Display for TokTree<'a, A, X> {
+impl<A: AtomRepr, X: ExtraTok> Display for TokTree<'_, A, X> {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
 }
 
@@ -226,7 +226,7 @@ impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
     }
   }
 }
-impl<'a, A: AtomRepr, X: ExtraTok> Display for Token<'a, A, X> {
+impl<A: AtomRepr, X: ExtraTok> Display for Token<'_, A, X> {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     thread_local! {
       static PAREN_LEVEL: RefCell<usize> = 0.into();

orchid-extension/Cargo.toml

@@ -9,8 +9,8 @@ edition = "2021"
 ahash = "0.8.11"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.17"
-hashbrown = "0.14.5"
+hashbrown = "0.15.2"
-itertools = "0.13.0"
+itertools = "0.14.0"
 konst = "0.3.9"
 lazy_static = "1.5.0"
 never = "0.1.0"

orchid-extension/src/atom.rs

@@ -77,7 +77,7 @@ pub struct ForeignAtom<'a> {
   pub atom: api::Atom,
   pub pos: Pos,
 }
-impl<'a> ForeignAtom<'a> {
+impl ForeignAtom<'_> {
   pub fn oex_opt(self) -> Option<Expr> {
     let (handle, pos) = (self.expr.as_ref()?.clone(), self.pos.clone());
     let data = ExprData { pos, kind: ExprKind::Atom(ForeignAtom { _life: PhantomData, ..self }) };
@@ -98,15 +98,15 @@ impl ForeignAtom<'static> {
     Some(M::Response::decode(&mut &rep[..]))
   }
 }
-impl<'a> fmt::Display for ForeignAtom<'a> {
+impl fmt::Display for ForeignAtom<'_> {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     write!(f, "{}::{:?}", if self.expr.is_some() { "Clause" } else { "Tok" }, self.atom)
   }
 }
-impl<'a> fmt::Debug for ForeignAtom<'a> {
+impl fmt::Debug for ForeignAtom<'_> {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ForeignAtom({self})") }
 }
-impl<'a> AtomRepr for ForeignAtom<'a> {
+impl AtomRepr for ForeignAtom<'_> {
   type Ctx = SysCtx;
   fn from_api(atom: &api::Atom, pos: Pos, ctx: &mut Self::Ctx) -> Self {
     Self { atom: atom.clone(), _life: PhantomData, ctx: ctx.clone(), expr: None, pos }
@@ -197,7 +197,7 @@ impl<A: AtomicFeatures> TypAtom<'static, A> {
     }
   }
 }
-impl<'a, A: AtomicFeatures> TypAtom<'a, A> {
+impl<A: AtomicFeatures> TypAtom<'_, A> {
   pub fn request<M: AtomMethod>(&self, req: M) -> M::Response where A: Supports<M> {
     M::Response::decode(
       &mut &self.data.ctx.reqnot.request(api::Fwd(
@@ -208,7 +208,7 @@ impl<'a, A: AtomicFeatures> TypAtom<'a, A> {
     )
   }
 }
-impl<'a, A: AtomicFeatures> Deref for TypAtom<'a, A> {
+impl<A: AtomicFeatures> Deref for TypAtom<'_, A> {
   type Target = A::Data;
   fn deref(&self) -> &Self::Target { &self.value }
 }
@@ -224,7 +224,7 @@ pub trait AtomDynfo: Send + Sync + 'static {
   fn print(&self, ctx: AtomCtx<'_>) -> String;
   fn handle_req(&self, ctx: AtomCtx<'_>, key: Sym, req: &mut dyn Read, rep: &mut dyn Write) -> bool;
   fn command(&self, ctx: AtomCtx<'_>) -> OrcRes<Option<Expr>>;
-  fn serialize(&self, ctx: AtomCtx<'_>, write: &mut dyn Write) -> Vec<api::ExprTicket>;
+  fn serialize(&self, ctx: AtomCtx<'_>, write: &mut dyn Write) -> Option<Vec<api::ExprTicket>>;
   fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> api::Atom;
   fn drop(&self, ctx: AtomCtx<'_>);
 }

orchid-extension/src/atom_owned.rs

@@ -1,19 +1,21 @@
-use std::any::{type_name, Any, TypeId};
+use std::any::{Any, TypeId, type_name};
 use std::borrow::Cow;
 use std::io::{Read, Write};
 use std::sync::Arc;
 
 use itertools::Itertools;
-use orchid_api_traits::{enc_vec, Decode, Encode};
+use never::Never;
+use orchid_api_traits::{Decode, Encode, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::id_store::{IdRecord, IdStore};
 use orchid_base::name::Sym;
 
 use crate::api;
 use crate::atom::{
-  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, MethodSet, Atomic, AtomicFeaturesImpl, AtomicVariant,
+  AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
+  err_not_callable, err_not_command, get_info,
 };
-use crate::expr::{bot, Expr, ExprHandle};
+use crate::expr::{Expr, ExprHandle, bot};
 use crate::system::SysCtx;
 
 pub struct OwnedVariant;
@@ -28,9 +30,7 @@ impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVari
       api::Atom { drop: Some(api::AtomId(rec.id())), data, owner: ctx.id }
     })
   }
-  fn _info() -> Self::_Info {
+  fn _info() -> Self::_Info { OwnedAtomDynfo(A::reg_reqs()) }
-    OwnedAtomDynfo(A::reg_reqs())
-  }
   type _Info = OwnedAtomDynfo<A>;
 }
 
@@ -54,7 +54,13 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
   fn call_ref(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> Expr {
     with_atom(id.unwrap(), |a| a.dyn_call_ref(ctx, arg))
   }
-  fn handle_req(&self, AtomCtx(_, id, ctx): AtomCtx, key: Sym, req: &mut dyn Read, rep: &mut dyn Write) -> bool {
+  fn handle_req(
+    &self,
+    AtomCtx(_, id, ctx): AtomCtx,
+    key: Sym,
+    req: &mut dyn Read,
+    rep: &mut dyn Write,
+  ) -> bool {
     with_atom(id.unwrap(), |a| {
       self.0.dispatch(a.as_any_ref().downcast_ref().unwrap(), ctx, key, req, rep)
     })
@@ -69,13 +75,11 @@ impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
     &self,
     AtomCtx(_, id, ctx): AtomCtx<'_>,
     write: &mut dyn Write,
-  ) -> Vec<api::ExprTicket> {
+  ) -> Option<Vec<api::ExprTicket>> {
     let id = id.unwrap();
     id.encode(write);
     with_atom(id, |a| a.dyn_serialize(ctx, write))
-      .into_iter()
+      .map(|v| v.into_iter().map(|t| t.handle.unwrap().tk).collect_vec())
-      .map(|t| t.handle.unwrap().tk)
-      .collect_vec()
   }
   fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> orchid_api::Atom {
     let refs = refs.iter().map(|tk| Expr::new(Arc::new(ExprHandle::from_args(ctx.clone(), *tk))));
@@ -97,7 +101,7 @@ pub trait DeserializeCtx: Sized {
 }
 
 struct DeserCtxImpl<'a>(&'a [u8], &'a SysCtx);
-impl<'a> DeserializeCtx for DeserCtxImpl<'a> {
+impl DeserializeCtx for DeserCtxImpl<'_> {
   fn read<T: Decode>(&mut self) -> T { T::decode(&mut self.0) }
   fn is_empty(&self) -> bool { self.0.is_empty() }
   fn sys(&self) -> SysCtx { self.1.clone() }
@@ -108,6 +112,13 @@ pub trait RefSet {
   fn to_vec(self) -> Vec<Expr>;
 }
 
+static E_NON_SER: &str = "Never is a stand-in refset for non-serializable atoms";
+
+impl RefSet for Never {
+  fn from_iter<I>(_: I) -> Self { panic!("{E_NON_SER}") }
+  fn to_vec(self) -> Vec<Expr> { panic!("{E_NON_SER}") }
+}
+
 impl RefSet for () {
   fn to_vec(self) -> Vec<Expr> { Vec::new() }
   fn from_iter<I: Iterator<Item = Expr> + ExactSizeIterator>(refs: I) -> Self {
@@ -130,6 +141,16 @@ impl<const N: usize> RefSet for [Expr; N] {
 
 /// Atoms that have a [Drop]
 pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone + 'static {
+  /// If serializable, the collection that best stores subexpression references
+  /// for this atom.
+  /// 
+  /// - `()` for no subexppressions,
+  /// - `[Expr; N]` for a static number of subexpressions
+  /// - `Vec<Expr>` for a variable number of subexpressions
+  /// - `Never` if not serializable
+  ///
+  /// If this isn't `Never`, you must override the default, panicking
+  /// `serialize` and `deserialize` implementation
   type Refs: RefSet;
   fn val(&self) -> Cow<'_, Self::Data>;
   #[allow(unused_variables)]
@@ -147,8 +168,21 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone
   #[allow(unused_variables)]
   fn print(&self, ctx: SysCtx) -> String { format!("OwnedAtom({})", type_name::<Self>()) }
   #[allow(unused_variables)]
-  fn serialize(&self, ctx: SysCtx, write: &mut (impl Write + ?Sized)) -> Self::Refs;
+  fn serialize(&self, ctx: SysCtx, write: &mut (impl Write + ?Sized)) -> Self::Refs {
-  fn deserialize(ctx: impl DeserializeCtx, refs: Self::Refs) -> Self;
+    assert!(
+      TypeId::of::<Self::Refs>() != TypeId::of::<Never>(),
+      "The extension scaffold is broken, this function should never be called on Never Refs"
+    );
+    panic!("Either implement serialize or set Refs to Never for {}", type_name::<Self>())
+  }
+  #[allow(unused_variables)]
+  fn deserialize(ctx: impl DeserializeCtx, refs: Self::Refs) -> Self {
+    assert!(
+      TypeId::of::<Self::Refs>() != TypeId::of::<Never>(),
+      "The extension scaffold is broken, this function should never be called on Never Refs"
+    );
+    panic!("Either implement deserialize or set Refs to Never for {}", type_name::<Self>())
+  }
 }
 pub trait DynOwnedAtom: Send + Sync + 'static {
   fn atom_tid(&self) -> TypeId;
@@ -159,7 +193,7 @@ pub trait DynOwnedAtom: Send + Sync + 'static {
   fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<Expr>>;
   fn dyn_free(self: Box<Self>, ctx: SysCtx);
   fn dyn_print(&self, ctx: SysCtx) -> String;
-  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<Expr>;
+  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Option<Vec<Expr>>;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
   fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
@@ -174,8 +208,9 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
   fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<Expr>> { self.command(ctx) }
   fn dyn_free(self: Box<Self>, ctx: SysCtx) { self.free(ctx) }
   fn dyn_print(&self, ctx: SysCtx) -> String { self.print(ctx) }
-  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<Expr> {
+  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Option<Vec<Expr>> {
-    self.serialize(ctx, sink).to_vec()
+    (TypeId::of::<Never>() != TypeId::of::<<Self as OwnedAtom>::Refs>())
+      .then(|| self.serialize(ctx, sink).to_vec())
   }
 }
 

orchid-extension/src/atom_thin.rs

@@ -1,15 +1,16 @@
-use std::any::{type_name, Any, TypeId};
+use std::any::{Any, TypeId, type_name};
 use std::io::Write;
 
-use orchid_api_traits::{enc_vec, Coding};
+use orchid_api_traits::{Coding, enc_vec};
 use orchid_base::error::OrcRes;
 use orchid_base::name::Sym;
 
 use crate::api;
 use crate::atom::{
-  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, MethodSet, Atomic, AtomicFeaturesImpl, AtomicVariant
+  AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, MethodSet,
+  err_not_callable, err_not_command, get_info,
 };
-use crate::expr::{bot, Expr, ExprHandle};
+use crate::expr::{Expr, ExprHandle, bot};
 use crate::system::SysCtx;
 
 pub struct ThinVariant;
@@ -23,9 +24,7 @@ impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant
       api::Atom { drop: None, data: buf, owner: ctx.id }
     })
   }
-  fn _info() -> Self::_Info {
+  fn _info() -> Self::_Info { ThinAtomDynfo(Self::reg_reqs()) }
-    ThinAtomDynfo(Self::reg_reqs())
-  }
   type _Info = ThinAtomDynfo<Self>;
 }
 
@@ -55,9 +54,9 @@ impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
   fn command(&self, AtomCtx(buf, _, ctx): AtomCtx<'_>) -> OrcRes<Option<Expr>> {
     T::decode(&mut &buf[..]).command(ctx)
   }
-  fn serialize(&self, actx: AtomCtx<'_>, write: &mut dyn Write) -> Vec<api::ExprTicket> {
+  fn serialize(&self, actx: AtomCtx<'_>, write: &mut dyn Write) -> Option<Vec<api::ExprTicket>> {
     T::decode(&mut &actx.0[..]).encode(write);
-    Vec::new()
+    Some(Vec::new())
   }
   fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> api::Atom {
     assert!(refs.is_empty(), "Refs found when deserializing thin atom");

orchid-extension/src/conv.rs

@@ -28,7 +28,7 @@ fn err_type(pos: Pos) -> OrcErr {
   mk_err(intern!(str: "Type error"), "The atom is a different type than expected", [pos.into()])
 }
 
-impl<'a, A: AtomicFeatures> TryFromExpr for TypAtom<'a, A> {
+impl<A: AtomicFeatures> TryFromExpr for TypAtom<'_, A> {
   fn try_from_expr(expr: Expr) -> OrcRes<Self> {
     (expr.foreign_atom())
       .map_err(|ex| err_not_atom(ex.pos.clone()).into())

orchid-extension/src/entrypoint.rs

@@ -232,8 +232,8 @@ fn extension_main_logic(data: ExtensionData) {
           match &atom_req {
             api::AtomReq::SerializeAtom(ser) => {
               let mut buf = enc_vec(&id);
-              let refs = nfo.serialize(actx, &mut buf);
+              let refs_opt = nfo.serialize(actx, &mut buf);
-              hand.handle(ser, &(buf, refs))
+              hand.handle(ser, &refs_opt.map(|refs| (buf, refs)))
             }
             api::AtomReq::AtomPrint(print@api::AtomPrint(_)) =>
               hand.handle(print, &nfo.print(actx)),
@@ -299,6 +299,6 @@ fn extension_main_logic(data: ExtensionData) {
   init_replica(rn.clone().map());
   while !exiting.load(Ordering::Relaxed) {
     let rcvd = recv_parent_msg().unwrap();
-    rn.receive(rcvd)
+    rn.receive(&rcvd)
   }
 }

orchid-extension/src/expr.rs

@@ -1,8 +1,9 @@
-use std::fmt;
 use std::ops::Deref;
 use std::sync::{Arc, OnceLock};
+use std::{backtrace, fmt};
 
 use derive_destructure::destructure;
+use orchid_base::clone;
 use orchid_base::error::{OrcErr, OrcErrv};
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
@@ -10,6 +11,8 @@ use orchid_base::reqnot::Requester;
 
 use crate::api;
 use crate::atom::{AtomFactory, ForeignAtom, ToAtom};
+use crate::conv::{ToExpr, TryFromExpr};
+use crate::func_atom::Lambda;
 use crate::system::SysCtx;
 
 #[derive(destructure)]
@@ -147,7 +150,7 @@ pub fn seq(ops: impl IntoIterator<Item = Expr>) -> Expr {
   recur(ops.into_iter()).expect("Empty list provided to seq!")
 }
 
-pub fn arg(n: u64) -> ExprKind { ExprKind::Arg(n) }
+pub fn arg(n: u64) -> Expr { inherit(ExprKind::Arg(n)) }
 
 pub fn lambda(n: u64, b: impl IntoIterator<Item = Expr>) -> Expr {
   inherit(ExprKind::Lambda(n, Box::new(call(b))))
@@ -162,3 +165,10 @@ pub fn call(v: impl IntoIterator<Item = Expr>) -> Expr {
 pub fn bot(ev: impl IntoIterator<Item = OrcErr>) -> Expr {
   inherit(ExprKind::Bottom(OrcErrv::new(ev).unwrap()))
 }
+
+pub fn with<I: TryFromExpr, O: ToExpr>(
+  expr: Expr,
+  cont: impl Fn(I) -> O + Clone + Send + Sync + 'static,
+) -> Expr {
+  call([lambda(0, [seq([arg(0), call([Lambda::new(cont).to_expr(), arg(0)])])]), expr])
+}

orchid-extension/src/func_atom.rs

@@ -5,13 +5,14 @@ use std::sync::{Arc, Mutex};
 
 use itertools::Itertools;
 use lazy_static::lazy_static;
+use never::Never;
 use orchid_api_traits::Encode;
 use orchid_base::error::OrcRes;
 use orchid_base::interner::Tok;
 use orchid_base::name::Sym;
 use trait_set::trait_set;
 
-use crate::atom::{MethodSet, Atomic};
+use crate::atom::{Atomic, MethodSet};
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use crate::conv::ToExpr;
 use crate::expr::{Expr, ExprHandle};
@@ -30,6 +31,11 @@ lazy_static! {
   static ref FUNS: Mutex<HashMap<Sym, (u8, Arc<dyn FunCB>)>> = Mutex::default();
 }
 
+/// An Atom representing a partially applied named native function. These
+/// partial calls are serialized into the name of the native function and the
+/// argument list.
+/// 
+/// See [Lambda] for the non-serializable variant
 #[derive(Clone)]
 pub(crate) struct Fun {
   path: Sym,
@@ -79,6 +85,40 @@ impl OwnedAtom for Fun {
   }
 }
 
+/// An Atom representing a partially applied native lambda. These are not serializable.
+/// 
+/// See [Fun] for the serializable variant
+#[derive(Clone)]
+pub struct Lambda {
+  args: Vec<Expr>,
+  arity: u8,
+  fun: Arc<dyn FunCB>,
+}
+impl Lambda {
+  pub fn new<I, O, F: ExprFunc<I, O>>(f: F) -> Self {
+    let fun = Arc::new(move |v| f.apply(v));
+    Self { args: vec![], arity: F::ARITY, fun }
+  }
+}
+impl Atomic for Lambda {
+  type Data = ();
+  type Variant = OwnedVariant;
+  fn reg_reqs() -> MethodSet<Self> { MethodSet::new() }
+}
+impl OwnedAtom for Lambda {
+  type Refs = Never;
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+  fn call_ref(&self, arg: ExprHandle) -> Expr {
+    let new_args = self.args.iter().cloned().chain([Expr::new(Arc::new(arg))]).collect_vec();
+    if new_args.len() == self.arity.into() {
+      (self.fun)(new_args).to_expr()
+    } else {
+      Self { args: new_args, arity: self.arity, fun: self.fun.clone() }.to_expr()
+    }
+  }
+  fn call(self, arg: ExprHandle) -> Expr { self.call_ref(arg) }
+}
+
 mod expr_func_derives {
   use orchid_base::error::OrcRes;
 

orchid-extension/src/lib.rs

@@ -5,7 +5,6 @@ pub mod atom_owned;
 pub mod atom_thin;
 pub mod conv;
 pub mod entrypoint;
-// pub mod error;
 pub mod expr;
 pub mod fs;
 pub mod func_atom;

orchid-extension/src/system.rs

@@ -13,6 +13,7 @@ use crate::api;
 use crate::atom::{get_info, AtomCtx, AtomDynfo, AtomicFeatures, ForeignAtom, TypAtom};
 use crate::entrypoint::ExtReq;
 use crate::fs::DeclFs;
+use crate::func_atom::Fun;
 // use crate::fun::Fun;
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
@@ -37,7 +38,7 @@ pub trait DynSystemCard: Send + Sync + 'static {
 /// The indices of these are bitwise negated, such that the MSB of an atom index
 /// marks whether it belongs to this package (0) or the importer (1)
 fn general_atoms() -> impl Iterator<Item = Option<Box<dyn AtomDynfo>>> {
-  [/*Some(Fun::INFO)*/].into_iter()
+  [Some(Fun::dynfo())].into_iter()
 }
 
 pub fn atom_info_for(

orchid-extension/src/tree.rs

@@ -1,10 +1,10 @@
 use std::num::NonZero;
 use std::ops::Range;
 
-use dyn_clone::{clone_box, DynClone};
+use dyn_clone::{DynClone, clone_box};
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_base::interner::{intern, Tok};
+use orchid_base::interner::{Tok, intern};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::parse::Comment;
@@ -15,7 +15,7 @@ use trait_set::trait_set;
 
 use crate::api;
 use crate::atom::{AtomFactory, ForeignAtom};
-use crate::conv::ToExpr;
+use crate::conv::{ToExpr, TryFromExpr};
 use crate::entrypoint::MemberRecord;
 use crate::expr::Expr;
 use crate::func_atom::{ExprFunc, Fun};
@@ -48,8 +48,8 @@ impl GenItem {
       GenItemKind::Import(cn) => api::ItemKind::Import(cn.tok().to_api()),
       GenItemKind::Macro(prio, rules) => api::ItemKind::Macro(api::MacroBlock {
         priority: prio,
-        rules: rules.into_iter().map(|r| r.to_api() ).collect_vec(),
+        rules: rules.into_iter().map(|r| r.to_api()).collect_vec(),
-      })
+      }),
     };
     let comments = self.comments.into_iter().map(|c| c.to_api()).collect_vec();
     api::Item { location: self.pos.to_api(), comments, kind }

orchid-host/Cargo.toml

@@ -7,8 +7,8 @@ edition = "2021"
 
 [dependencies]
 derive_destructure = "1.0.0"
-hashbrown = "0.14.5"
+hashbrown = "0.15.2"
-itertools = "0.13.0"
+itertools = "0.14.0"
 lazy_static = "1.4.0"
 never = "0.1.0"
 num-traits = "0.2.19"

orchid-host/src/extension.rs

@@ -2,26 +2,26 @@ use std::collections::VecDeque;
 use std::num::NonZero;
 use std::ops::Deref;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering};
-use std::sync::mpsc::{sync_channel, SyncSender};
+use std::sync::mpsc::{SyncSender, sync_channel};
 use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
 use std::{fmt, io, thread};
 
 use derive_destructure::destructure;
-use hashbrown::hash_map::Entry;
 use hashbrown::HashMap;
+use hashbrown::hash_map::Entry;
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use orchid_api_traits::Request;
 use orchid_base::char_filter::char_filter_match;
+use orchid_base::clone;
 use orchid_base::error::{OrcErrv, OrcRes};
-use orchid_base::interner::{intern, Tok};
+use orchid_base::interner::{Tok, intern};
 use orchid_base::location::Pos;
 use orchid_base::logging::Logger;
 use orchid_base::macros::mtreev_from_api;
 use orchid_base::parse::Comment;
 use orchid_base::reqnot::{ReqNot, Requester as _};
-use orchid_base::tree::{ttv_from_api, AtomRepr};
+use orchid_base::tree::{AtomRepr, ttv_from_api};
-use orchid_base::clone;
 use ordered_float::NotNan;
 use substack::{Stackframe, Substack};
 
@@ -106,6 +106,8 @@ impl fmt::Display for AtomHand {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.print()) }
 }
 
+pub type OnMessage = Box<dyn FnMut(&[u8]) + Send>;
+
 /// The 3 primary contact points with an extension are
 /// - send a message
 /// - wait for a message to arrive
@@ -113,8 +115,8 @@ impl fmt::Display for AtomHand {
 ///
 /// There are no ordering guarantees about these
 pub trait ExtensionPort: Send + Sync {
+  fn set_onmessage(&self, callback: OnMessage);
   fn send(&self, msg: &[u8]);
-  fn receive(&self) -> Option<Vec<u8>>;
   fn header(&self) -> &api::ExtensionHeader;
 }
 
@@ -180,12 +182,12 @@ impl Extension {
             api::IntReq::InternStr(s) => hand.handle(&s, &intern(&**s.0).to_api()),
             api::IntReq::InternStrv(v) => hand.handle(&v, &intern(&*v.0).to_api()),
             api::IntReq::ExternStr(si) => hand.handle(&si, &Tok::<String>::from_api(si.0).arc()),
-            api::IntReq::ExternStrv(vi) => hand.handle(&vi, &Arc::new(
+            api::IntReq::ExternStrv(vi) => hand.handle(
-              Tok::<Vec<Tok<String>>>::from_api(vi.0)
+              &vi,
-                .iter()
+              &Arc::new(
-                .map(|t| t.to_api())
+                Tok::<Vec<Tok<String>>>::from_api(vi.0).iter().map(|t| t.to_api()).collect_vec(),
-                .collect_vec()
+              ),
-            )),
+            ),
           },
           api::ExtHostReq::Fwd(ref fw @ api::Fwd(ref atom, ref key, ref body)) => {
             let sys = System::resolve(atom.owner).unwrap();
@@ -210,30 +212,24 @@ impl Extension {
               kind: expr.to_api(),
             })
           },
-          api::ExtHostReq::RunMacros(ref rm @ api::RunMacros{ ref run_id, ref query }) => {
+          api::ExtHostReq::RunMacros(ref rm @ api::RunMacros { ref run_id, ref query }) => hand
-            hand.handle(rm,
+            .handle(
-              &macro_recur(*run_id,
+              rm,
-                mtreev_from_api(query, &mut |_| panic!("Recursion never contains atoms"))
+              &macro_recur(
+                *run_id,
+                mtreev_from_api(query, &mut |_| panic!("Recursion never contains atoms")),
               )
-              .map(|x| macro_treev_to_api(*run_id, x))
+              .map(|x| macro_treev_to_api(*run_id, x)),
-            )
+            ),
-          }
         },
       ),
     });
     let weak = Arc::downgrade(&ret);
-    thread::Builder::new()
+    port.set_onmessage(Box::new(move |msg| {
-      .name(format!("host-end:{}", eh.name))
+      if let Some(xd) = weak.upgrade() {
-      .spawn::<_, Option<()>>(move || {
+        xd.reqnot.receive(msg)
-        loop {
-          // thread will exit if either the peer exits or the extension object is dropped.
-          // It holds a strong reference to the port so the port's destructor will not be
-          // called until the
-          let msg = port.receive()?;
-          weak.upgrade()?.reqnot.receive(msg);
       }
-      })
+    }));
-      .unwrap();
     Ok(Self(ret))
   }
   pub fn systems(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
@@ -268,11 +264,13 @@ impl SystemCtor {
       id,
     }));
     let root = (sys_inst.const_root.into_iter())
-      .map(|(k, v)| Member::from_api(
+      .map(|(k, v)| {
+        Member::from_api(
           api::Member { name: k, kind: v },
           Substack::Bottom.push(Tok::from_api(k)),
-        &data
+          &data,
-      ))
+        )
+      })
       .collect_vec();
     data.0.const_root.set(root).unwrap();
     inst_g.insert(id, data.clone());

orchid-host/src/macros.rs

@@ -1,26 +1,29 @@
-use crate::{api, rule::shared::Matcher, tree::Code};
 use std::sync::{Arc, RwLock};
 
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
 use lazy_static::lazy_static;
-use orchid_base::{macros::{mtreev_from_api, mtreev_to_api, MTok, MTree}, name::Sym};
+use orchid_base::macros::{MTok, MTree, mtreev_from_api, mtreev_to_api};
-use ordered_float::NotNan;
+use orchid_base::name::Sym;
 use trait_set::trait_set;
+
+use crate::api;
 use crate::extension::AtomHand;
+use crate::rule::matcher::{NamedMatcher, PriodMatcher};
+use crate::rule::state::{MatchState, OwnedState};
+use crate::tree::Code;
 
 pub type MacTok = MTok<'static, AtomHand>;
 pub type MacTree = MTree<'static, AtomHand>;
 
-trait_set!{
+trait_set! {
   trait MacroCB = Fn(Vec<MacTree>) -> Option<Vec<MacTree>> + Send + Sync;
 }
 
-lazy_static!{
+lazy_static! {
   static ref RECURSION: RwLock<HashMap<api::ParsId, Box<dyn MacroCB>>> = RwLock::default();
-  static ref MACRO_SLOTS: RwLock<HashMap<api::ParsId,
+  static ref MACRO_SLOTS: RwLock<HashMap<api::ParsId, HashMap<api::MacroTreeId, Arc<MacTok>>>> =
-    HashMap<api::MacroTreeId, Arc<MacTok>>
+    RwLock::default();
-  >> = RwLock::default();
 }
 
 pub fn macro_recur(run_id: api::ParsId, input: Vec<MacTree>) -> Option<Vec<MacTree>> {
@@ -42,18 +45,19 @@ pub fn macro_treev_from_api(api: Vec<api::MacroTree>) -> Vec<MacTree> {
 }
 
 pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option<Vec<MacTree>> {
-  let mut slots = (MACRO_SLOTS.write().unwrap())
+  let mut slots = (MACRO_SLOTS.write().unwrap()).remove(&run_id).expect("Run not found");
-    .remove(&run_id).expect("Run not found");
   return work(&mut slots, tree);
   fn work(
     slots: &mut HashMap<api::MacroTreeId, Arc<MacTok>>,
-    tree: &[MacTree]
+    tree: &[MacTree],
   ) -> Option<Vec<MacTree>> {
     let items = (tree.iter())
-      .map(|t| Some(MacTree {
+      .map(|t| {
+        Some(MacTree {
           tok: match &*t.tok {
             MacTok::Atom(_) | MacTok::Name(_) | MacTok::Ph(_) => return None,
             MacTok::Ref(_) => panic!("Ref is an extension-local optimization"),
+            MacTok::Done(_) => panic!("Created and removed by matcher"),
             MacTok::Slot(slot) => slots.get(&slot.id()).expect("Slot not found").clone(),
             MacTok::S(paren, b) => Arc::new(MacTok::S(*paren, work(slots, b)?)),
             MacTok::Lambda(a, b) => Arc::new(match (work(slots, a), work(slots, b)) {
@@ -63,8 +67,9 @@ pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option<Vec<MacTree
               (Some(a), Some(b)) => MacTok::Lambda(a, b),
             }),
           },
-        pos: t.pos.clone()
+          pos: t.pos.clone(),
-      }))
+        })
+      })
       .collect_vec();
     let any_changed = items.iter().any(Option::is_some);
     any_changed.then(|| {
@@ -75,11 +80,95 @@ pub fn deslot_macro(run_id: api::ParsId, tree: &[MacTree]) -> Option<Vec<MacTree
   }
 }
 
-pub struct MacroRepo{
+pub struct Macro<Matcher> {
-  no_prio: Vec<(HashSet<Sym>, Matcher, Code)>,
+  deps: HashSet<Sym>,
-  prio: Vec<(HashSet<Sym>, NotNan<f64>, Matcher, Code)>,
+  cases: Vec<(Matcher, Code)>,
 }
 
-pub fn match_on_exprv<'a>(target: &'a [MacTree], pattern: &[MacTree]) -> Option<MatchhState<'a>> {
+pub struct MacroRepo {
-  
+  named: HashMap<Sym, Vec<Macro<NamedMatcher>>>,
+  prio: Vec<Macro<PriodMatcher>>,
+}
+impl MacroRepo {
+  /// TODO: the recursion inside this function needs to be moved into Orchid.
+  /// See the markdown note
+  pub fn process_exprv(&self, target: &[MacTree]) -> Option<Vec<MacTree>> {
+    let mut workcp = target.to_vec();
+    let mut lexicon;
+
+    'try_named: loop {
+      lexicon = HashSet::new();
+      target.iter().for_each(|tgt| fill_lexicon(tgt, &mut lexicon));
+
+      for (i, tree) in workcp.iter().enumerate() {
+        let MacTok::Name(name) = &*tree.tok else { continue };
+        let matches = (self.named.get(name).into_iter().flatten())
+          .filter(|m| m.deps.is_subset(&lexicon))
+          .filter_map(|mac| {
+            mac.cases.iter().find_map(|cas| cas.0.apply(&workcp[i..], |_| false).map(|s| (cas, s)))
+          })
+          .collect_vec();
+        assert!(
+          matches.len() < 2,
+          "Multiple conflicting matches on {:?}: {:?}",
+          &workcp[i..],
+          matches
+        );
+        let Some((case, (state, tail))) = matches.into_iter().next() else { continue };
+        let inj = (run_body(&case.1, state).into_iter())
+          .map(|MacTree { pos, tok }| MacTree { pos, tok: Arc::new(MacTok::Done(tok)) });
+        workcp.splice(i..(workcp.len() - tail.len()), inj);
+        continue 'try_named;
+      }
+      break;
+    }
+
+    if let Some(((_, body), state)) = (self.prio.iter())
+      .filter(|mac| mac.deps.is_subset(&lexicon))
+      .flat_map(|mac| &mac.cases)
+      .find_map(|case| case.0.apply(&workcp, |_| false).map(|state| (case, state)))
+    {
+      return Some(run_body(body, state));
+    }
+
+    let results = (workcp.into_iter())
+      .map(|mt| match &*mt.tok {
+        MTok::S(p, body) => self.process_exprv(body).map(|body| MTok::S(*p, body).at(mt.pos)),
+        MTok::Lambda(arg, body) => match (self.process_exprv(arg), self.process_exprv(body)) {
+          (Some(arg), Some(body)) => Some(MTok::Lambda(arg, body).at(mt.pos)),
+          (Some(arg), None) => Some(MTok::Lambda(arg, body.to_vec()).at(mt.pos)),
+          (None, Some(body)) => Some(MTok::Lambda(arg.to_vec(), body).at(mt.pos)),
+          (None, None) => None,
+        },
+        _ => None,
+      })
+      .collect_vec();
+    results.iter().any(Option::is_some).then(|| {
+      (results.into_iter().zip(target))
+        .map(|(opt, fb)| opt.unwrap_or_else(|| fb.clone()))
+        .collect_vec()
+    })
+  }
+}
+
+fn fill_lexicon(tgt: &MacTree, lexicon: &mut HashSet<Sym>) {
+  match &*tgt.tok {
+    MTok::Name(n) => {
+      lexicon.insert(n.clone());
+    },
+    MTok::Lambda(arg, body) => {
+      arg.iter().for_each(|t| fill_lexicon(t, lexicon));
+      body.iter().for_each(|t| fill_lexicon(t, lexicon))
+    },
+    MTok::S(_, body) => body.iter().for_each(|t| fill_lexicon(t, lexicon)),
+    _ => (),
+  }
+}
+
+fn run_body(body: &Code, mut state: MatchState<'_>) -> Vec<MacTree> {
+  let inject: Vec<MacTree> = todo!("Call the interpreter with bindings");
+  inject
+    .into_iter()
+    .map(|MTree { pos, tok }| MTree { pos, tok: Arc::new(MTok::Done(tok)) })
+    .collect_vec()
 }

orchid-host/src/rule/build.rs

@@ -46,7 +46,7 @@ fn mk_scalv(pattern: &[MacTree]) -> Vec<ScalMatcher> { pattern.iter().map(mk_sca
 
 /// Pattern MUST start and end with a vectorial placeholder
 #[must_use]
-fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
+pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
   debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
   debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
   debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
@@ -91,7 +91,7 @@ fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
 #[must_use]
 fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
   match &*pattern.tok {
-    MacTok::Atom(_) => panic!("Atoms aren't supported in matchers"),
+    MacTok::Atom(_) | MacTok::Done(_) => panic!("Atoms and Done aren't supported in matchers"),
     MacTok::Name(n) => ScalMatcher::Name(n.clone()),
     MacTok::Ph(Ph { name, kind }) => match kind {
       PhKind::Vector { .. } => {

orchid-host/src/rule/matcher.rs

@@ -1,21 +1,85 @@
-//! Abstract definition of a rule matcher, so that the implementation can
+use std::fmt;
-//! eventually be swapped out for a different one.
-
-use std::rc::Rc;
 
+use itertools::Itertools;
+use orchid_api::PhKind;
+use orchid_base::intern;
+use orchid_base::location::Pos;
 use orchid_base::name::Sym;
+use orchid_base::tree::Ph;
 
-use super::state::State;
+use super::any_match::any_match;
-use crate::macros::MacTree;
+use super::build::mk_any;
+use super::shared::{AnyMatcher, VecMatcher};
+use super::state::{MatchState, StateEntry};
+use super::vec_attrs::vec_attrs;
+use super::vec_match::vec_match;
+use crate::macros::{MacTok, MacTree};
+use crate::rule::build::mk_vec;
 
-/// Cacheable optimized structures for matching patterns on slices. This is
+pub fn first_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.first().unwrap()).is_some() }
-/// injected to allow experimentation in the matcher implementation.
+pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwrap()).is_some() }
-pub trait Matcher {
+
-  /// Build matcher for a pattern
+pub struct NamedMatcher(AnyMatcher);
-  #[must_use]
+impl NamedMatcher {
-  fn new(pattern: Rc<Vec<MacTree>>) -> Self;
+  pub fn new(pattern: &[MacTree]) -> Self {
-  /// Apply matcher to a token sequence
+    assert!(
-  #[must_use]
+      matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))),
-  fn apply<'a>(&self, source: &'a [MacTree], save_loc: &impl Fn(Sym) -> bool)
+      "Named matchers must begin with a name"
-  -> Option<State<'a>>;
+    );
+
+    match last_is_vec(pattern) {
+      true => Self(mk_any(pattern)),
+      false => {
+        let kind: PhKind = PhKind::Vector { priority: 0, at_least_one: false };
+        let suffix = [MacTok::Ph(Ph { name: intern!(str: "::after"), kind }).at(Pos::None)];
+        Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
+      },
+    }
+  }
+  /// Also returns the tail, if any, which should be matched further
+  /// Note that due to how priod works below, the main usable information from the tail is
+  /// its length
+  pub fn apply<'a>(
+    &self,
+    seq: &'a [MacTree],
+    save_loc: impl Fn(Sym) -> bool,
+  ) -> Option<(MatchState<'a>, &'a [MacTree])> {
+    any_match(&self.0, seq, &save_loc).map(|mut state| match state.remove(intern!(str: "::after")) {
+      Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"),
+      Some(StateEntry::Vec(v)) => (state, v),
+      None => (state, &[][..]),
+    })
+  }
+}
+impl fmt::Display for NamedMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}
+impl fmt::Debug for NamedMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "NamedMatcher({self})") }
+}
+
+pub struct PriodMatcher(VecMatcher);
+impl PriodMatcher {
+  pub fn new(pattern: &[MacTree]) -> Self {
+    assert!(
+      pattern.first().and_then(vec_attrs).is_some()
+      && pattern.last().and_then(vec_attrs).is_some(),
+      "Prioritized matchers must start and end with a vectorial",
+    );
+    Self(mk_vec(pattern))
+  }
+  /// tokens before the offset always match the prefix
+  pub fn apply<'a>(
+    &self,
+    seq: &'a [MacTree],
+    save_loc: impl Fn(Sym) -> bool,
+  ) -> Option<MatchState<'a>> {
+    vec_match(&self.0, seq, &save_loc)
+  }
+}
+impl fmt::Display for PriodMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}
+impl fmt::Debug for PriodMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "PriodMatcher({self})") }
 }

orchid-host/src/rule/mod.rs

@@ -21,4 +21,5 @@ pub mod shared;
 mod vec_match;
 pub mod state;
 mod vec_attrs;
+pub mod matcher;
 // pub mod matcher;

orchid-host/src/rule/scal_match.rs

@@ -2,7 +2,8 @@ use orchid_base::name::Sym;
 
 use super::any_match::any_match;
 use super::shared::ScalMatcher;
-use crate::{macros::{MacTok, MacTree}, rule::state::{MatchState, StateEntry}};
+use crate::macros::{MacTok, MacTree};
+use crate::rule::state::{MatchState, StateEntry};
 
 #[must_use]
 pub fn scal_match<'a>(
@@ -15,6 +16,7 @@ pub fn scal_match<'a>(
       true => MatchState::from_name(n1.clone(), expr.pos.clone()),
       false => MatchState::default(),
     }),
+    (ScalMatcher::Placeh { .. }, MacTok::Done(_)) => None,
     (ScalMatcher::Placeh { key }, _) =>
       Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
     (ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>

orchid-host/src/rule/shared.rs

@@ -4,14 +4,9 @@ use std::fmt;
 
 use itertools::Itertools;
 use orchid_base::interner::Tok;
-
-use super::any_match::any_match;
-use super::build::mk_any;
 use orchid_base::name::Sym;
-use crate::macros::MacTree;
-use crate::rule::state::MatchState;
 use orchid_base::side::Side;
-use orchid_base::tokens::{Paren, PARENS};
+use orchid_base::tokens::{PARENS, Paren};
 
 pub enum ScalMatcher {
   Name(Sym),
@@ -104,18 +99,3 @@ impl fmt::Display for AnyMatcher {
     }
   }
 }
-
-// ################ External ################
-
-/// A priority-order tree of the vectorial placeholders with scalars as leaves.
-pub struct Matcher(AnyMatcher);
-impl Matcher {
-  pub fn new(pattern: &[MacTree]) -> Self { Self(mk_any(pattern)) }
-  pub fn apply<'a>(&self, seq: &'a [MacTree], save_loc: &impl Fn(Sym) -> bool) -> Option<MatchState<'a>> {
-    any_match(&self.0, seq, save_loc)
-  }
-}
-
-impl fmt::Display for Matcher {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
-}

orchid-host/src/rule/state.rs

@@ -1,20 +1,36 @@
-use std::sync::Arc;
+#![allow(unused)]
+use std::any::Any;
 
 use hashbrown::HashMap;
-use orchid_api::PhKind;
+use orchid_base::interner::Tok;
-use orchid_base::tree::Ph;
+use orchid_base::join::join_maps;
-use orchid_base::{interner::Tok, join::join_maps};
 use orchid_base::location::Pos;
-
+use orchid_base::match_mapping;
-use crate::macros::{MacTok, MacTree};
 use orchid_base::name::Sym;
 
+use crate::macros::MacTree;
+
+enum StackAction {
+  Return(Box<dyn Any>),
+  Call {
+    target: Box<dyn FnOnce(Box<dyn Any>) -> StackAction>,
+    param: Box<dyn Any>,
+    tail: Box<dyn FnOnce(Box<dyn Any>) -> StackAction>
+  }
+}
+
+struct Trampoline {
+  stack: Vec<Box<dyn FnOnce(Box<dyn Any>) -> StackAction>>
+}
+
+
+
 #[derive(Clone, Copy, Debug)]
 pub enum StateEntry<'a> {
   Vec(&'a [MacTree]),
   Scalar(&'a MacTree),
 }
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct MatchState<'a> {
   placeholders: HashMap<Tok<String>, StateEntry<'a>>,
   name_posv: HashMap<Sym, Vec<Pos>>,
@@ -26,9 +42,7 @@ impl<'a> MatchState<'a> {
   pub fn combine(self, s: Self) -> Self {
     Self {
       placeholders: self.placeholders.into_iter().chain(s.placeholders).collect(),
-      name_posv: join_maps(self.name_posv, s.name_posv, |_, l, r| {
+      name_posv: join_maps(self.name_posv, s.name_posv, |_, l, r| l.into_iter().chain(r).collect()),
-        l.into_iter().chain(r).collect()
-      }),
     }
   }
   pub fn ph_len(&self, key: &Tok<String>) -> Option<usize> {
@@ -40,45 +54,40 @@ impl<'a> MatchState<'a> {
   pub fn from_name(name: Sym, location: Pos) -> Self {
     Self { name_posv: HashMap::from([(name, vec![location])]), placeholders: HashMap::new() }
   }
+  pub fn remove(&mut self, name: Tok<String>) -> Option<StateEntry<'a>> {
+    self.placeholders.remove(&name)
+  }
+  pub fn mk_owned(self) -> OwnedState {
+    OwnedState {
+      placeholders: (self.placeholders.into_iter())
+        .map(|(k, v)| {
+          (
+            k.clone(),
+            match_mapping!(v, StateEntry => OwnedEntry {
+              Scalar(tree.clone()),
+              Vec(v.to_vec()),
+            }),
+          )
+        })
+        .collect(),
+      name_posv: self.name_posv,
+    }
+  }
 }
 impl Default for MatchState<'static> {
   fn default() -> Self { Self { name_posv: HashMap::new(), placeholders: HashMap::new() } }
 }
 
-#[must_use]
+#[derive(Clone, Debug)]
-pub fn apply_exprv(template: &[MacTree], state: &MatchState) -> Vec<MacTree> {
+pub enum OwnedEntry {
-  template.iter().map(|e| apply_expr(e, state)).flat_map(Vec::into_iter).collect()
+  Vec(Vec<MacTree>),
+  Scalar(MacTree),
 }
-
+pub struct OwnedState {
-#[must_use]
+  placeholders: HashMap<Tok<String>, OwnedEntry>,
-pub fn apply_expr(template: &MacTree, state: &MatchState) -> Vec<MacTree> {
+  name_posv: HashMap<Sym, Vec<Pos>>,
-  let MacTree { pos, tok } = template;
+}
-  match &**tok {
+impl OwnedState {
-    MacTok::Name(n) => match state.name_posv.get(n) {
+  pub fn get(&self, key: &Tok<String>) -> Option<&OwnedEntry> { self.placeholders.get(key) }
-      None => vec![template.clone()],
+  pub fn positions(&self, name: &Sym) -> &[Pos] { self.name_posv.get(name).map_or(&[], |v| &v[..]) }
-      Some(locs) => vec![MacTree { tok: tok.clone(), pos: locs[0].clone() }],
-    },
-    MacTok::Atom(_) => vec![template.clone()],
-    MacTok::S(c, body) => vec![MacTree {
-      pos: pos.clone(), tok: Arc::new(MacTok::S(*c, apply_exprv(body.as_slice(), state))),
-    }],
-    MacTok::Ph(Ph { name, kind }) => {
-      let Some(value) = state.placeholders.get(name) else {
-        panic!("Placeholder does not have a value in state")
-      };
-      match (kind, value) {
-        (PhKind::Scalar, StateEntry::Scalar(item)) => vec![(*item).clone()],
-        (PhKind::Vector { .. }, StateEntry::Vec(chunk)) => chunk.to_vec(),
-        _ => panic!("Type mismatch between template and state"),
-      }
-    },
-    MacTok::Lambda(arg, body) => vec![MacTree {
-      pos: pos.clone(),
-      tok: Arc::new(MacTok::Lambda(
-        apply_exprv(arg, state),
-        apply_exprv(&body[..], state),
-      )),
-    }],
-    MacTok::Slot(_) | MacTok::Ref(_) => panic!("Extension-only variants"),
-  }
 }

orchid-host/src/rule/vec_match.rs

@@ -18,7 +18,7 @@ pub fn vec_match<'a>(
       if *nonzero && seq.is_empty() {
         return None;
       }
-      return Some(MatchState::from_ph(key.clone(), StateEntry::Vec(seq)));
+      Some(MatchState::from_ph(key.clone(), StateEntry::Vec(seq)))
     },
     VecMatcher::Scan { left, sep, right, direction } => {
       if seq.len() < sep.len() {

orchid-host/src/subprocess.rs

@@ -1,5 +1,6 @@
 use std::io::{self, BufRead as _, Write};
 use std::path::PathBuf;
+use std::sync::mpsc::{sync_channel, SyncSender};
 use std::sync::Mutex;
 use std::{process, thread};
 
@@ -8,12 +9,12 @@ use orchid_base::logging::Logger;
 use orchid_base::msg::{recv_msg, send_msg};
 
 use crate::api;
-use crate::extension::ExtensionPort;
+use crate::extension::{ExtensionPort, OnMessage};
 
 pub struct Subprocess {
   child: Mutex<process::Child>,
   stdin: Mutex<process::ChildStdin>,
-  stdout: Mutex<process::ChildStdout>,
+  set_onmessage: SyncSender<OnMessage>,
   header: api::ExtensionHeader,
 }
 impl Subprocess {
@@ -31,6 +32,18 @@ impl Subprocess {
     let mut stdout = child.stdout.take().unwrap();
     let header = api::ExtensionHeader::decode(&mut stdout);
     let child_stderr = child.stderr.take().unwrap();
+    let (set_onmessage, recv_onmessage) = sync_channel(0);
+    thread::Builder::new().name(format!("stdout-fwd:{prog}")).spawn(move || {
+      let mut onmessage: Box<dyn FnMut(&[u8]) + Send> = recv_onmessage.recv().unwrap();
+      drop(recv_onmessage);
+      loop {
+        match recv_msg(&mut stdout) {
+          Ok(msg) => onmessage(&msg[..]),
+          Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break,
+          Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
+        }
+      }
+    })?;
     thread::Builder::new().name(format!("stderr-fwd:{prog}")).spawn(move || {
       let mut reader = io::BufReader::new(child_stderr);
       loop {
@@ -44,7 +57,7 @@ impl Subprocess {
     Ok(Self {
       child: Mutex::new(child),
       stdin: Mutex::new(stdin),
-      stdout: Mutex::new(stdout),
+      set_onmessage,
       header,
     })
   }
@@ -53,6 +66,9 @@ impl Drop for Subprocess {
   fn drop(&mut self) { self.child.lock().unwrap().wait().expect("Extension exited with error"); }
 }
 impl ExtensionPort for Subprocess {
+  fn set_onmessage(&self, callback: OnMessage) {
+    self.set_onmessage.send(callback).unwrap();
+  }
   fn header(&self) -> &orchid_api::ExtensionHeader { &self.header }
   fn send(&self, msg: &[u8]) {
     if msg.starts_with(&[0, 0, 0, 0x1c]) {
@@ -60,11 +76,4 @@ impl ExtensionPort for Subprocess {
     }
     send_msg(&mut *self.stdin.lock().unwrap(), msg).unwrap()
   }
-  fn receive(&self) -> Option<Vec<u8>> {
-    match recv_msg(&mut *self.stdout.lock().unwrap()) {
-      Ok(msg) => Some(msg),
-      Err(e) if e.kind() == io::ErrorKind::BrokenPipe => None,
-      Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
-    }
-  }
 }

orchid-std/src/string/str_atom.rs

@@ -77,7 +77,7 @@ pub enum OrcString<'a> {
   Val(TypAtom<'a, StrAtom>),
   Int(TypAtom<'a, IntStrAtom>),
 }
-impl<'a> OrcString<'a> {
+impl OrcString<'_> {
   pub fn get_string(&self) -> Arc<String> {
     match &self {
       Self::Int(tok) => Tok::from_api(tok.value).arc(),