in midst of refactor

orchidlang/src/rule/matcher_vectree/any_match.rs

@@ -0,0 +1,29 @@
+use super::scal_match::scalv_match;
+use super::shared::AnyMatcher;
+use super::vec_match::vec_match;
+use crate::name::Sym;
+use crate::rule::matcher::RuleExpr;
+use crate::rule::state::State;
+
+#[must_use]
+pub fn any_match<'a>(
+  matcher: &AnyMatcher,
+  seq: &'a [RuleExpr],
+  save_loc: &impl Fn(Sym) -> bool,
+) -> Option<State<'a>> {
+  match matcher {
+    AnyMatcher::Scalar(scalv) => scalv_match(scalv, seq, save_loc),
+    AnyMatcher::Vec { left, mid, right } => {
+      if seq.len() < left.len() + right.len() {
+        return None;
+      };
+      let left_split = left.len();
+      let right_split = seq.len() - right.len();
+      Some(
+        scalv_match(left, &seq[..left_split], save_loc)?
+          .combine(scalv_match(right, &seq[right_split..], save_loc)?)
+          .combine(vec_match(mid, &seq[left_split..right_split], save_loc)?),
+      )
+    },
+  }
+}

orchidlang/src/rule/matcher_vectree/build.rs

@@ -0,0 +1,149 @@
+use intern_all::Tok;
+use itertools::Itertools;
+
+use super::shared::{AnyMatcher, ScalMatcher, VecMatcher};
+use crate::parse::parsed::{Clause, PHClass, Placeholder};
+use crate::rule::matcher::RuleExpr;
+use crate::rule::vec_attrs::vec_attrs;
+use crate::utils::side::Side;
+
+pub type MaxVecSplit<'a> = (&'a [RuleExpr], (Tok<String>, usize, bool), &'a [RuleExpr]);
+
+/// Derive the details of the central vectorial and the two sides from a
+/// slice of Expr's
+#[must_use]
+fn split_at_max_vec(pattern: &[RuleExpr]) -> Option<MaxVecSplit> {
+  let rngidx = pattern
+    .iter()
+    .position_max_by_key(|expr| vec_attrs(expr).map(|attrs| attrs.1 as i64).unwrap_or(-1))?;
+  let (left, not_left) = pattern.split_at(rngidx);
+  let (placeh, right) =
+    not_left.split_first().expect("The index of the greatest element must be less than the length");
+  vec_attrs(placeh).map(|attrs| (left, attrs, right))
+}
+
+#[must_use]
+fn scal_cnt<'a>(iter: impl Iterator<Item = &'a RuleExpr>) -> usize {
+  iter.take_while(|expr| vec_attrs(expr).is_none()).count()
+}
+
+#[must_use]
+pub fn mk_any(pattern: &[RuleExpr]) -> AnyMatcher {
+  let left_split = scal_cnt(pattern.iter());
+  if pattern.len() <= left_split {
+    return AnyMatcher::Scalar(mk_scalv(pattern));
+  }
+  let (left, not_left) = pattern.split_at(left_split);
+  let right_split = not_left.len() - scal_cnt(pattern.iter().rev());
+  let (mid, right) = not_left.split_at(right_split);
+  AnyMatcher::Vec { left: mk_scalv(left), mid: mk_vec(mid), right: mk_scalv(right) }
+}
+
+/// Pattern MUST NOT contain vectorial placeholders
+#[must_use]
+fn mk_scalv(pattern: &[RuleExpr]) -> Vec<ScalMatcher> { pattern.iter().map(mk_scalar).collect() }
+
+/// Pattern MUST start and end with a vectorial placeholder
+#[must_use]
+fn mk_vec(pattern: &[RuleExpr]) -> VecMatcher {
+  debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
+  debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
+  debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
+  let (left, (key, _, nonzero), right) = split_at_max_vec(pattern)
+    .expect("pattern must have vectorial placeholders at least at either end");
+  let r_sep_size = scal_cnt(right.iter());
+  let (r_sep, r_side) = right.split_at(r_sep_size);
+  let l_sep_size = scal_cnt(left.iter().rev());
+  let (l_side, l_sep) = left.split_at(left.len() - l_sep_size);
+  let main = VecMatcher::Placeh { key: key.clone(), nonzero };
+  match (left, right) {
+    (&[], &[]) => VecMatcher::Placeh { key, nonzero },
+    (&[], _) => VecMatcher::Scan {
+      direction: Side::Left,
+      left: Box::new(main),
+      sep: mk_scalv(r_sep),
+      right: Box::new(mk_vec(r_side)),
+    },
+    (_, &[]) => VecMatcher::Scan {
+      direction: Side::Right,
+      left: Box::new(mk_vec(l_side)),
+      sep: mk_scalv(l_sep),
+      right: Box::new(main),
+    },
+    (..) => {
+      let mut key_order =
+        l_side.iter().chain(r_side.iter()).filter_map(vec_attrs).collect::<Vec<_>>();
+      key_order.sort_by_key(|(_, prio, _)| -(*prio as i64));
+      VecMatcher::Middle {
+        left: Box::new(mk_vec(l_side)),
+        left_sep: mk_scalv(l_sep),
+        mid: Box::new(main),
+        right_sep: mk_scalv(r_sep),
+        right: Box::new(mk_vec(r_side)),
+        key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
+      }
+    },
+  }
+}
+
+/// Pattern MUST NOT be a vectorial placeholder
+#[must_use]
+fn mk_scalar(pattern: &RuleExpr) -> ScalMatcher {
+  match &pattern.value {
+    Clause::Atom(a) => ScalMatcher::Atom(a.clone()),
+    Clause::Name(n) => ScalMatcher::Name(n.clone()),
+    Clause::Placeh(Placeholder { name, class }) => match class {
+      PHClass::Vec { .. } => {
+        panic!("Scalar matcher cannot be built from vector pattern")
+      },
+      PHClass::Scalar | PHClass::Name =>
+        ScalMatcher::Placeh { key: name.clone(), name_only: class == &PHClass::Name },
+    },
+    Clause::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
+    Clause::Lambda(arg, body) => ScalMatcher::Lambda(Box::new(mk_any(arg)), Box::new(mk_any(body))),
+  }
+}
+
+#[cfg(test)]
+mod test {
+  use std::rc::Rc;
+
+  use intern_all::i;
+
+  use super::mk_any;
+  use crate::location::SourceRange;
+  use crate::parse::parsed::{Clause, PHClass, PType, Placeholder};
+  use crate::sym;
+
+  #[test]
+  fn test_scan() {
+    let ex = |c: Clause| c.into_expr(SourceRange::mock());
+    let pattern = vec![
+      ex(Clause::Placeh(Placeholder {
+        class: PHClass::Vec { nonzero: false, prio: 0 },
+        name: i!(str: "::prefix"),
+      })),
+      ex(Clause::Name(sym!(prelude::do))),
+      ex(Clause::S(
+        PType::Par,
+        Rc::new(vec![
+          ex(Clause::Placeh(Placeholder {
+            class: PHClass::Vec { nonzero: false, prio: 0 },
+            name: i!(str: "expr"),
+          })),
+          ex(Clause::Name(sym!(prelude::;))),
+          ex(Clause::Placeh(Placeholder {
+            class: PHClass::Vec { nonzero: false, prio: 1 },
+            name: i!(str: "rest"),
+          })),
+        ]),
+      )),
+      ex(Clause::Placeh(Placeholder {
+        class: PHClass::Vec { nonzero: false, prio: 0 },
+        name: i!(str: "::suffix"),
+      })),
+    ];
+    let matcher = mk_any(&pattern);
+    println!("{matcher}");
+  }
+}

orchidlang/src/rule/matcher_vectree/mod.rs

@@ -0,0 +1,21 @@
+//! Optimized form of macro pattern that can be quickly tested against the AST.
+//!
+//! # Construction
+//!
+//! convert pattern into hierarchy of plain, scan, middle
+//! - plain: accept any sequence or any non-empty sequence
+//! - scan: a single scalar pattern moves LTR or RTL, submatchers on either
+//! side
+//! - middle: two scalar patterns walk over all permutations of matches
+//! while getting progressively closer to each other
+//!
+//! # Application
+//!
+//! walk over the current matcher's valid options and poll the submatchers
+//! for each of them
+
+mod any_match;
+mod build;
+mod scal_match;
+pub mod shared;
+mod vec_match;

orchidlang/src/rule/matcher_vectree/scal_match.rs

@@ -0,0 +1,47 @@
+use super::any_match::any_match;
+use super::shared::ScalMatcher;
+use crate::name::Sym;
+use crate::parse::parsed::Clause;
+use crate::rule::matcher::RuleExpr;
+use crate::rule::state::{State, StateEntry};
+
+#[must_use]
+pub fn scal_match<'a>(
+  matcher: &ScalMatcher,
+  expr: &'a RuleExpr,
+  save_loc: &impl Fn(Sym) -> bool,
+) -> Option<State<'a>> {
+  match (matcher, &expr.value) {
+    (ScalMatcher::Atom(a1), Clause::Atom(a2)) if a1.run().0.parser_eq(&*a2.run().0) =>
+      Some(State::default()),
+    (ScalMatcher::Name(n1), Clause::Name(n2)) if n1 == n2 => Some(match save_loc(n1.clone()) {
+      true => State::from_name(n1.clone(), expr.range.clone()),
+      false => State::default(),
+    }),
+    (ScalMatcher::Placeh { key, name_only: true }, Clause::Name(n)) =>
+      Some(State::from_ph(key.clone(), StateEntry::Name(n, &expr.range))),
+    (ScalMatcher::Placeh { key, name_only: false }, _) =>
+      Some(State::from_ph(key.clone(), StateEntry::Scalar(expr))),
+    (ScalMatcher::S(c1, b_mat), Clause::S(c2, body)) if c1 == c2 =>
+      any_match(b_mat, &body[..], save_loc),
+    (ScalMatcher::Lambda(arg_mat, b_mat), Clause::Lambda(arg, body)) =>
+      Some(any_match(arg_mat, arg, save_loc)?.combine(any_match(b_mat, body, save_loc)?)),
+    _ => None,
+  }
+}
+
+#[must_use]
+pub fn scalv_match<'a>(
+  matchers: &[ScalMatcher],
+  seq: &'a [RuleExpr],
+  save_loc: &impl Fn(Sym) -> bool,
+) -> Option<State<'a>> {
+  if seq.len() != matchers.len() {
+    return None;
+  }
+  let mut state = State::default();
+  for (matcher, expr) in matchers.iter().zip(seq.iter()) {
+    state = state.combine(scal_match(matcher, expr, save_loc)?);
+  }
+  Some(state)
+}

orchidlang/src/rule/matcher_vectree/shared.rs

@@ -0,0 +1,141 @@
+//! Datastructures for cached pattern
+
+use std::fmt;
+use std::rc::Rc;
+
+use intern_all::Tok;
+use itertools::Itertools;
+
+use super::any_match::any_match;
+use super::build::mk_any;
+use crate::foreign::atom::AtomGenerator;
+use crate::name::Sym;
+use crate::parse::parsed::PType;
+use crate::rule::matcher::{Matcher, RuleExpr};
+use crate::rule::state::State;
+use crate::utils::side::Side;
+
+pub(super) enum ScalMatcher {
+  Atom(AtomGenerator),
+  Name(Sym),
+  S(PType, Box<AnyMatcher>),
+  Lambda(Box<AnyMatcher>, Box<AnyMatcher>),
+  Placeh { key: Tok<String>, name_only: bool },
+}
+
+pub(super) enum VecMatcher {
+  Placeh {
+    key: Tok<String>,
+    nonzero: bool,
+  },
+  Scan {
+    left: Box<VecMatcher>,
+    sep: Vec<ScalMatcher>,
+    right: Box<VecMatcher>,
+    /// The separator traverses the sequence towards this side
+    direction: Side,
+  },
+  Middle {
+    /// Matches the left outer region
+    left: Box<VecMatcher>,
+    /// Matches the left separator
+    left_sep: Vec<ScalMatcher>,
+    /// Matches the middle - can only ever be a plain placeholder
+    mid: Box<VecMatcher>,
+    /// Matches the right separator
+    right_sep: Vec<ScalMatcher>,
+    /// Matches the right outer region
+    right: Box<VecMatcher>,
+    /// Order of significance for sorting equally good projects based on
+    /// the length of matches on either side.
+    ///
+    /// Vectorial keys that appear on either side, in priority order
+    key_order: Vec<Tok<String>>,
+  },
+}
+
+pub(super) enum AnyMatcher {
+  Scalar(Vec<ScalMatcher>),
+  Vec { left: Vec<ScalMatcher>, mid: VecMatcher, right: Vec<ScalMatcher> },
+}
+impl Matcher for AnyMatcher {
+  fn new(pattern: Rc<Vec<RuleExpr>>) -> Self { mk_any(&pattern) }
+
+  fn apply<'a>(
+    &self,
+    source: &'a [RuleExpr],
+    save_loc: &impl Fn(Sym) -> bool,
+  ) -> Option<State<'a>> {
+    any_match(self, source, save_loc)
+  }
+}
+
+// ################ Display ################
+
+impl fmt::Display for ScalMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Atom(a) => write!(f, "{a:?}"),
+      Self::Placeh { key, name_only } => match name_only {
+        false => write!(f, "${key}"),
+        true => write!(f, "$_{key}"),
+      },
+      Self::Name(n) => write!(f, "{n}"),
+      Self::S(t, body) => write!(f, "{}{body}{}", t.l(), t.r()),
+      Self::Lambda(arg, body) => write!(f, "\\{arg}.{body}"),
+    }
+  }
+}
+
+impl fmt::Display for VecMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Placeh { key, nonzero: true } => write!(f, "...${key}"),
+      Self::Placeh { key, nonzero: false } => write!(f, "..${key}"),
+      Self::Scan { left, sep, right, direction } => {
+        let arrow = if direction == &Side::Left { "<==" } else { "==>" };
+        write!(f, "Scan{{{left} {arrow} {} {arrow} {right}}}", sep.iter().join(" "))
+      },
+      Self::Middle { left, left_sep, mid, right_sep, right, .. } => {
+        let left_sep_s = left_sep.iter().join(" ");
+        let right_sep_s = right_sep.iter().join(" ");
+        write!(f, "Middle{{{left}|{left_sep_s}|{mid}|{right_sep_s}|{right}}}")
+      },
+    }
+  }
+}
+
+impl fmt::Display for AnyMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Scalar(s) => {
+        write!(f, "({})", s.iter().join(" "))
+      },
+      Self::Vec { left, mid, right } => {
+        let lefts = left.iter().join(" ");
+        let rights = right.iter().join(" ");
+        write!(f, "[{lefts}|{mid}|{rights}]")
+      },
+    }
+  }
+}
+
+// ################ External ################
+
+/// A [Matcher] implementation that builds a priority-order tree of the
+/// vectorial placeholders and handles the scalars on leaves.
+pub struct VectreeMatcher(AnyMatcher);
+impl Matcher for VectreeMatcher {
+  fn new(pattern: Rc<Vec<RuleExpr>>) -> Self { Self(AnyMatcher::new(pattern)) }
+
+  fn apply<'a>(
+    &self,
+    source: &'a [RuleExpr],
+    save_loc: &impl Fn(Sym) -> bool,
+  ) -> Option<State<'a>> {
+    self.0.apply(source, save_loc)
+  }
+}
+impl fmt::Display for VectreeMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}

orchidlang/src/rule/matcher_vectree/vec_match.rs

@@ -0,0 +1,95 @@
+use std::cmp::Ordering;
+
+use itertools::Itertools;
+
+use super::scal_match::scalv_match;
+use super::shared::VecMatcher;
+use crate::name::Sym;
+use crate::rule::matcher::RuleExpr;
+use crate::rule::state::{State, StateEntry};
+
+#[must_use]
+pub fn vec_match<'a>(
+  matcher: &VecMatcher,
+  seq: &'a [RuleExpr],
+  save_loc: &impl Fn(Sym) -> bool,
+) -> Option<State<'a>> {
+  match matcher {
+    VecMatcher::Placeh { key, nonzero } => {
+      if *nonzero && seq.is_empty() {
+        return None;
+      }
+      return Some(State::from_ph(key.clone(), StateEntry::Vec(seq)));
+    },
+    VecMatcher::Scan { left, sep, right, direction } => {
+      if seq.len() < sep.len() {
+        return None;
+      }
+      for lpos in direction.walk(0..=seq.len() - sep.len()) {
+        let rpos = lpos + sep.len();
+        let state = vec_match(left, &seq[..lpos], save_loc)
+          .and_then(|s| Some(s.combine(scalv_match(sep, &seq[lpos..rpos], save_loc)?)))
+          .and_then(|s| Some(s.combine(vec_match(right, &seq[rpos..], save_loc)?)));
+        if let Some(s) = state {
+          return Some(s);
+        }
+      }
+      None
+    },
+    // XXX predict heap space usage and allocation count
+    VecMatcher::Middle { left, left_sep, mid, right_sep, right, key_order } => {
+      if seq.len() < left_sep.len() + right_sep.len() {
+        return None;
+      }
+      // Valid locations for the left separator
+      let lposv = seq[..seq.len() - right_sep.len()]
+        .windows(left_sep.len())
+        .enumerate()
+        .filter_map(|(i, window)| scalv_match(left_sep, window, save_loc).map(|s| (i, s)))
+        .collect::<Vec<_>>();
+      // Valid locations for the right separator
+      let rposv = seq[left_sep.len()..]
+        .windows(right_sep.len())
+        .enumerate()
+        .filter_map(|(i, window)| scalv_match(right_sep, window, save_loc).map(|s| (i, s)))
+        .collect::<Vec<_>>();
+      // Valid combinations of locations for the separators
+      let mut pos_pairs = lposv
+        .into_iter()
+        .cartesian_product(rposv)
+        .filter(|((lpos, _), (rpos, _))| lpos + left_sep.len() <= *rpos)
+        .map(|((lpos, lstate), (rpos, rstate))| (lpos, rpos, lstate.combine(rstate)))
+        .collect::<Vec<_>>();
+      // In descending order of size
+      pos_pairs.sort_by_key(|(l, r, _)| -((r - l) as i64));
+      let eql_clusters = pos_pairs.into_iter().group_by(|(al, ar, _)| ar - al);
+      for (_gap_size, cluster) in eql_clusters.into_iter() {
+        let best_candidate = cluster
+          .into_iter()
+          .filter_map(|(lpos, rpos, state)| {
+            Some(
+              state
+                .combine(vec_match(left, &seq[..lpos], save_loc)?)
+                .combine(vec_match(mid, &seq[lpos + left_sep.len()..rpos], save_loc)?)
+                .combine(vec_match(right, &seq[rpos + right_sep.len()..], save_loc)?),
+            )
+          })
+          .max_by(|a, b| {
+            for key in key_order {
+              let alen = a.ph_len(key).expect("key_order references scalar or missing");
+              let blen = b.ph_len(key).expect("key_order references scalar or missing");
+              match alen.cmp(&blen) {
+                Ordering::Equal => (),
+                any => return any,
+              }
+            }
+            Ordering::Equal
+          });
+        if let Some(state) = best_candidate {
+          return Some(state);
+        }
+      }
+      None
+    },
+  }
+}