Macro processing factored into Orchid functions

orchid-host/src/rule/build.rs

@@ -46,7 +46,7 @@ fn mk_scalv(pattern: &[MacTree]) -> Vec<ScalMatcher> { pattern.iter().map(mk_sca
 
 /// Pattern MUST start and end with a vectorial placeholder
 #[must_use]
-fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
+pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
   debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
   debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
   debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
@@ -91,7 +91,7 @@ fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
 #[must_use]
 fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
   match &*pattern.tok {
-    MacTok::Atom(_) => panic!("Atoms aren't supported in matchers"),
+    MacTok::Atom(_) | MacTok::Done(_) => panic!("Atoms and Done aren't supported in matchers"),
     MacTok::Name(n) => ScalMatcher::Name(n.clone()),
     MacTok::Ph(Ph { name, kind }) => match kind {
       PhKind::Vector { .. } => {

orchid-host/src/rule/matcher.rs

@@ -1,21 +1,85 @@
-//! Abstract definition of a rule matcher, so that the implementation can
-//! eventually be swapped out for a different one.
-
-use std::rc::Rc;
+use std::fmt;
 
+use itertools::Itertools;
+use orchid_api::PhKind;
+use orchid_base::intern;
+use orchid_base::location::Pos;
 use orchid_base::name::Sym;
+use orchid_base::tree::Ph;
 
-use super::state::State;
-use crate::macros::MacTree;
+use super::any_match::any_match;
+use super::build::mk_any;
+use super::shared::{AnyMatcher, VecMatcher};
+use super::state::{MatchState, StateEntry};
+use super::vec_attrs::vec_attrs;
+use super::vec_match::vec_match;
+use crate::macros::{MacTok, MacTree};
+use crate::rule::build::mk_vec;
 
-/// Cacheable optimized structures for matching patterns on slices. This is
-/// injected to allow experimentation in the matcher implementation.
-pub trait Matcher {
-  /// Build matcher for a pattern
-  #[must_use]
-  fn new(pattern: Rc<Vec<MacTree>>) -> Self;
-  /// Apply matcher to a token sequence
-  #[must_use]
-  fn apply<'a>(&self, source: &'a [MacTree], save_loc: &impl Fn(Sym) -> bool)
-  -> Option<State<'a>>;
+pub fn first_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.first().unwrap()).is_some() }
+pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwrap()).is_some() }
+
+pub struct NamedMatcher(AnyMatcher);
+impl NamedMatcher {
+  pub fn new(pattern: &[MacTree]) -> Self {
+    assert!(
+      matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))),
+      "Named matchers must begin with a name"
+    );
+
+    match last_is_vec(pattern) {
+      true => Self(mk_any(pattern)),
+      false => {
+        let kind: PhKind = PhKind::Vector { priority: 0, at_least_one: false };
+        let suffix = [MacTok::Ph(Ph { name: intern!(str: "::after"), kind }).at(Pos::None)];
+        Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
+      },
+    }
+  }
+  /// Also returns the tail, if any, which should be matched further
+  /// Note that due to how priod works below, the main usable information from the tail is
+  /// its length
+  pub fn apply<'a>(
+    &self,
+    seq: &'a [MacTree],
+    save_loc: impl Fn(Sym) -> bool,
+  ) -> Option<(MatchState<'a>, &'a [MacTree])> {
+    any_match(&self.0, seq, &save_loc).map(|mut state| match state.remove(intern!(str: "::after")) {
+      Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"),
+      Some(StateEntry::Vec(v)) => (state, v),
+      None => (state, &[][..]),
+    })
+  }
+}
+impl fmt::Display for NamedMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}
+impl fmt::Debug for NamedMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "NamedMatcher({self})") }
+}
+
+pub struct PriodMatcher(VecMatcher);
+impl PriodMatcher {
+  pub fn new(pattern: &[MacTree]) -> Self {
+    assert!(
+      pattern.first().and_then(vec_attrs).is_some()
+      && pattern.last().and_then(vec_attrs).is_some(),
+      "Prioritized matchers must start and end with a vectorial",
+    );
+    Self(mk_vec(pattern))
+  }
+  /// tokens before the offset always match the prefix
+  pub fn apply<'a>(
+    &self,
+    seq: &'a [MacTree],
+    save_loc: impl Fn(Sym) -> bool,
+  ) -> Option<MatchState<'a>> {
+    vec_match(&self.0, seq, &save_loc)
+  }
+}
+impl fmt::Display for PriodMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}
+impl fmt::Debug for PriodMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "PriodMatcher({self})") }
 }

orchid-host/src/rule/mod.rs

@@ -21,4 +21,5 @@ pub mod shared;
 mod vec_match;
 pub mod state;
 mod vec_attrs;
+pub mod matcher;
 // pub mod matcher;

orchid-host/src/rule/scal_match.rs

@@ -2,7 +2,8 @@ use orchid_base::name::Sym;
 
 use super::any_match::any_match;
 use super::shared::ScalMatcher;
-use crate::{macros::{MacTok, MacTree}, rule::state::{MatchState, StateEntry}};
+use crate::macros::{MacTok, MacTree};
+use crate::rule::state::{MatchState, StateEntry};
 
 #[must_use]
 pub fn scal_match<'a>(
@@ -15,6 +16,7 @@ pub fn scal_match<'a>(
       true => MatchState::from_name(n1.clone(), expr.pos.clone()),
       false => MatchState::default(),
     }),
+    (ScalMatcher::Placeh { .. }, MacTok::Done(_)) => None,
     (ScalMatcher::Placeh { key }, _) =>
       Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
     (ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>

orchid-host/src/rule/shared.rs

@@ -4,14 +4,9 @@ use std::fmt;
 
 use itertools::Itertools;
 use orchid_base::interner::Tok;
-
-use super::any_match::any_match;
-use super::build::mk_any;
 use orchid_base::name::Sym;
-use crate::macros::MacTree;
-use crate::rule::state::MatchState;
 use orchid_base::side::Side;
-use orchid_base::tokens::{Paren, PARENS};
+use orchid_base::tokens::{PARENS, Paren};
 
 pub enum ScalMatcher {
   Name(Sym),
@@ -104,18 +99,3 @@ impl fmt::Display for AnyMatcher {
     }
   }
 }
-
-// ################ External ################
-
-/// A priority-order tree of the vectorial placeholders with scalars as leaves.
-pub struct Matcher(AnyMatcher);
-impl Matcher {
-  pub fn new(pattern: &[MacTree]) -> Self { Self(mk_any(pattern)) }
-  pub fn apply<'a>(&self, seq: &'a [MacTree], save_loc: &impl Fn(Sym) -> bool) -> Option<MatchState<'a>> {
-    any_match(&self.0, seq, save_loc)
-  }
-}
-
-impl fmt::Display for Matcher {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
-}

orchid-host/src/rule/state.rs

@@ -1,20 +1,36 @@
-use std::sync::Arc;
+#![allow(unused)]
+use std::any::Any;
 
 use hashbrown::HashMap;
-use orchid_api::PhKind;
-use orchid_base::tree::Ph;
-use orchid_base::{interner::Tok, join::join_maps};
+use orchid_base::interner::Tok;
+use orchid_base::join::join_maps;
 use orchid_base::location::Pos;
-
-use crate::macros::{MacTok, MacTree};
+use orchid_base::match_mapping;
 use orchid_base::name::Sym;
 
+use crate::macros::MacTree;
+
+enum StackAction {
+  Return(Box<dyn Any>),
+  Call {
+    target: Box<dyn FnOnce(Box<dyn Any>) -> StackAction>,
+    param: Box<dyn Any>,
+    tail: Box<dyn FnOnce(Box<dyn Any>) -> StackAction>
+  }
+}
+
+struct Trampoline {
+  stack: Vec<Box<dyn FnOnce(Box<dyn Any>) -> StackAction>>
+}
+
+
+
 #[derive(Clone, Copy, Debug)]
 pub enum StateEntry<'a> {
   Vec(&'a [MacTree]),
   Scalar(&'a MacTree),
 }
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct MatchState<'a> {
   placeholders: HashMap<Tok<String>, StateEntry<'a>>,
   name_posv: HashMap<Sym, Vec<Pos>>,
@@ -26,9 +42,7 @@ impl<'a> MatchState<'a> {
   pub fn combine(self, s: Self) -> Self {
     Self {
       placeholders: self.placeholders.into_iter().chain(s.placeholders).collect(),
-      name_posv: join_maps(self.name_posv, s.name_posv, |_, l, r| {
-        l.into_iter().chain(r).collect()
-      }),
+      name_posv: join_maps(self.name_posv, s.name_posv, |_, l, r| l.into_iter().chain(r).collect()),
     }
   }
   pub fn ph_len(&self, key: &Tok<String>) -> Option<usize> {
@@ -40,45 +54,40 @@ impl<'a> MatchState<'a> {
   pub fn from_name(name: Sym, location: Pos) -> Self {
     Self { name_posv: HashMap::from([(name, vec![location])]), placeholders: HashMap::new() }
   }
+  pub fn remove(&mut self, name: Tok<String>) -> Option<StateEntry<'a>> {
+    self.placeholders.remove(&name)
+  }
+  pub fn mk_owned(self) -> OwnedState {
+    OwnedState {
+      placeholders: (self.placeholders.into_iter())
+        .map(|(k, v)| {
+          (
+            k.clone(),
+            match_mapping!(v, StateEntry => OwnedEntry {
+              Scalar(tree.clone()),
+              Vec(v.to_vec()),
+            }),
+          )
+        })
+        .collect(),
+      name_posv: self.name_posv,
+    }
+  }
 }
 impl Default for MatchState<'static> {
   fn default() -> Self { Self { name_posv: HashMap::new(), placeholders: HashMap::new() } }
 }
 
-#[must_use]
-pub fn apply_exprv(template: &[MacTree], state: &MatchState) -> Vec<MacTree> {
-  template.iter().map(|e| apply_expr(e, state)).flat_map(Vec::into_iter).collect()
+#[derive(Clone, Debug)]
+pub enum OwnedEntry {
+  Vec(Vec<MacTree>),
+  Scalar(MacTree),
 }
-
-#[must_use]
-pub fn apply_expr(template: &MacTree, state: &MatchState) -> Vec<MacTree> {
-  let MacTree { pos, tok } = template;
-  match &**tok {
-    MacTok::Name(n) => match state.name_posv.get(n) {
-      None => vec![template.clone()],
-      Some(locs) => vec![MacTree { tok: tok.clone(), pos: locs[0].clone() }],
-    },
-    MacTok::Atom(_) => vec![template.clone()],
-    MacTok::S(c, body) => vec![MacTree {
-      pos: pos.clone(), tok: Arc::new(MacTok::S(*c, apply_exprv(body.as_slice(), state))),
-    }],
-    MacTok::Ph(Ph { name, kind }) => {
-      let Some(value) = state.placeholders.get(name) else {
-        panic!("Placeholder does not have a value in state")
-      };
-      match (kind, value) {
-        (PhKind::Scalar, StateEntry::Scalar(item)) => vec![(*item).clone()],
-        (PhKind::Vector { .. }, StateEntry::Vec(chunk)) => chunk.to_vec(),
-        _ => panic!("Type mismatch between template and state"),
-      }
-    },
-    MacTok::Lambda(arg, body) => vec![MacTree {
-      pos: pos.clone(),
-      tok: Arc::new(MacTok::Lambda(
-        apply_exprv(arg, state),
-        apply_exprv(&body[..], state),
-      )),
-    }],
-    MacTok::Slot(_) | MacTok::Ref(_) => panic!("Extension-only variants"),
-  }
+pub struct OwnedState {
+  placeholders: HashMap<Tok<String>, OwnedEntry>,
+  name_posv: HashMap<Sym, Vec<Pos>>,
+}
+impl OwnedState {
+  pub fn get(&self, key: &Tok<String>) -> Option<&OwnedEntry> { self.placeholders.get(key) }
+  pub fn positions(&self, name: &Sym) -> &[Pos] { self.name_posv.get(name).map_or(&[], |v| &v[..]) }
 }

orchid-host/src/rule/vec_match.rs

@@ -18,7 +18,7 @@ pub fn vec_match<'a>(
       if *nonzero && seq.is_empty() {
         return None;
       }
-      return Some(MatchState::from_ph(key.clone(), StateEntry::Vec(seq)));
+      Some(MatchState::from_ph(key.clone(), StateEntry::Vec(seq)))
     },
     VecMatcher::Scan { left, sep, right, direction } => {
       if seq.len() < sep.len() {