Gitbutler >:(

I don't understand this piece of software at all

orchid-host/src/rule/any_match.rs

@@ -0,0 +1,29 @@
+use super::scal_match::scalv_match;
+use super::shared::AnyMatcher;
+use super::vec_match::vec_match;
+use orchid_base::name::Sym;
+use crate::macros::MacTree;
+use crate::rule::state::MatchState;
+
+#[must_use]
+pub fn any_match<'a>(
+  matcher: &AnyMatcher,
+  seq: &'a [MacTree],
+  save_loc: &impl Fn(Sym) -> bool,
+) -> Option<MatchState<'a>> {
+  match matcher {
+    AnyMatcher::Scalar(scalv) => scalv_match(scalv, seq, save_loc),
+    AnyMatcher::Vec { left, mid, right } => {
+      if seq.len() < left.len() + right.len() {
+        return None;
+      };
+      let left_split = left.len();
+      let right_split = seq.len() - right.len();
+      Some(
+        scalv_match(left, &seq[..left_split], save_loc)?
+          .combine(scalv_match(right, &seq[right_split..], save_loc)?)
+          .combine(vec_match(mid, &seq[left_split..right_split], save_loc)?),
+      )
+    },
+  }
+}

orchid-host/src/rule/build.rs

@@ -0,0 +1,151 @@
+use orchid_api::PhKind;
+use orchid_base::interner::Tok;
+use itertools::Itertools;
+use orchid_base::side::Side;
+use orchid_base::tree::Ph;
+
+use super::shared::{AnyMatcher, ScalMatcher, VecMatcher};
+use crate::macros::{MacTok, MacTree};
+use crate::rule::vec_attrs::vec_attrs;
+
+pub type MaxVecSplit<'a> = (&'a [MacTree], (Tok<String>, u8, bool), &'a [MacTree]);
+
+/// Derive the details of the central vectorial and the two sides from a
+/// slice of Expr's
+#[must_use]
+fn split_at_max_vec(pattern: &[MacTree]) -> Option<MaxVecSplit> {
+  let rngidx = pattern
+    .iter()
+    .position_max_by_key(|expr| vec_attrs(expr).map(|attrs| attrs.1 as i64).unwrap_or(-1))?;
+  let (left, not_left) = pattern.split_at(rngidx);
+  let (placeh, right) =
+    not_left.split_first().expect("The index of the greatest element must be less than the length");
+  vec_attrs(placeh).map(|attrs| (left, attrs, right))
+}
+
+#[must_use]
+fn scal_cnt<'a>(iter: impl Iterator<Item = &'a MacTree>) -> usize {
+  iter.take_while(|expr| vec_attrs(expr).is_none()).count()
+}
+
+#[must_use]
+pub fn mk_any(pattern: &[MacTree]) -> AnyMatcher {
+  let left_split = scal_cnt(pattern.iter());
+  if pattern.len() <= left_split {
+    return AnyMatcher::Scalar(mk_scalv(pattern));
+  }
+  let (left, not_left) = pattern.split_at(left_split);
+  let right_split = not_left.len() - scal_cnt(pattern.iter().rev());
+  let (mid, right) = not_left.split_at(right_split);
+  AnyMatcher::Vec { left: mk_scalv(left), mid: mk_vec(mid), right: mk_scalv(right) }
+}
+
+/// Pattern MUST NOT contain vectorial placeholders
+#[must_use]
+fn mk_scalv(pattern: &[MacTree]) -> Vec<ScalMatcher> { pattern.iter().map(mk_scalar).collect() }
+
+/// Pattern MUST start and end with a vectorial placeholder
+#[must_use]
+fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
+  debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
+  debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
+  debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
+  let (left, (key, _, nonzero), right) = split_at_max_vec(pattern)
+    .expect("pattern must have vectorial placeholders at least at either end");
+  let r_sep_size = scal_cnt(right.iter());
+  let (r_sep, r_side) = right.split_at(r_sep_size);
+  let l_sep_size = scal_cnt(left.iter().rev());
+  let (l_side, l_sep) = left.split_at(left.len() - l_sep_size);
+  let main = VecMatcher::Placeh { key: key.clone(), nonzero };
+  match (left, right) {
+    (&[], &[]) => VecMatcher::Placeh { key, nonzero },
+    (&[], _) => VecMatcher::Scan {
+      direction: Side::Left,
+      left: Box::new(main),
+      sep: mk_scalv(r_sep),
+      right: Box::new(mk_vec(r_side)),
+    },
+    (_, &[]) => VecMatcher::Scan {
+      direction: Side::Right,
+      left: Box::new(mk_vec(l_side)),
+      sep: mk_scalv(l_sep),
+      right: Box::new(main),
+    },
+    (..) => {
+      let mut key_order =
+        l_side.iter().chain(r_side.iter()).filter_map(vec_attrs).collect::<Vec<_>>();
+      key_order.sort_by_key(|(_, prio, _)| -(*prio as i64));
+      VecMatcher::Middle {
+        left: Box::new(mk_vec(l_side)),
+        left_sep: mk_scalv(l_sep),
+        mid: Box::new(main),
+        right_sep: mk_scalv(r_sep),
+        right: Box::new(mk_vec(r_side)),
+        key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
+      }
+    },
+  }
+}
+
+/// Pattern MUST NOT be a vectorial placeholder
+#[must_use]
+fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
+  match &*pattern.tok {
+    MacTok::Atom(_) => panic!("Atoms aren't supported in matchers"),
+    MacTok::Name(n) => ScalMatcher::Name(n.clone()),
+    MacTok::Ph(Ph { name, kind }) => match kind {
+      PhKind::Vector { .. } => {
+        panic!("Scalar matcher cannot be built from vector pattern")
+      },
+      PhKind::Scalar =>
+        ScalMatcher::Placeh { key: name.clone() },
+    },
+    MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
+    MacTok::Lambda(arg, body) => ScalMatcher::Lambda(Box::new(mk_any(arg)), Box::new(mk_any(body))),
+    MacTok::Ref(_) | MacTok::Slot(_) => panic!("Extension-only variants"),
+  }
+}
+
+#[cfg(test)]
+mod test {
+  use std::sync::Arc;
+
+  use orchid_api::PhKind;
+use orchid_base::{intern, location::SourceRange, sym, tree::Ph, tokens::Paren};
+
+use crate::macros::{MacTok, MacTree};
+
+use super::mk_any;
+
+  #[test]
+  fn test_scan() {
+    let ex = |tok: MacTok| MacTree{ tok: Arc::new(tok), pos: SourceRange::mock().pos() };
+    let pattern = vec![
+      ex(MacTok::Ph(Ph {
+        kind: PhKind::Vector { priority: 0, at_least_one: false },
+        name: intern!(str: "::prefix"),
+      })),
+      ex(MacTok::Name(sym!(prelude::do))),
+      ex(MacTok::S(
+        Paren::Round,
+        vec![
+          ex(MacTok::Ph(Ph {
+            kind: PhKind::Vector { priority: 0, at_least_one: false },
+            name: intern!(str: "expr"),
+          })),
+          ex(MacTok::Name(sym!(prelude::;))),
+          ex(MacTok::Ph(Ph {
+            kind: PhKind::Vector { priority: 1, at_least_one: false },
+            name: intern!(str: "rest"),
+          })),
+        ],
+      )),
+      ex(MacTok::Ph(Ph {
+        kind: PhKind::Vector { priority: 0, at_least_one: false },
+        name: intern!(str: "::suffix"),
+      })),
+    ];
+    let matcher = mk_any(&pattern);
+    println!("{matcher}");
+  }
+}

orchid-host/src/rule/matcher.rs

@@ -0,0 +1,21 @@
+//! Abstract definition of a rule matcher, so that the implementation can
+//! eventually be swapped out for a different one.
+
+use std::rc::Rc;
+
+use orchid_base::name::Sym;
+
+use super::state::State;
+use crate::macros::MacTree;
+
+/// Cacheable optimized structures for matching patterns on slices. This is
+/// injected to allow experimentation in the matcher implementation.
+pub trait Matcher {
+  /// Build matcher for a pattern
+  #[must_use]
+  fn new(pattern: Rc<Vec<MacTree>>) -> Self;
+  /// Apply matcher to a token sequence
+  #[must_use]
+  fn apply<'a>(&self, source: &'a [MacTree], save_loc: &impl Fn(Sym) -> bool)
+  -> Option<State<'a>>;
+}

orchid-host/src/rule/mod.rs

@@ -0,0 +1,24 @@
+//! Optimized form of macro pattern that can be quickly tested against the AST.
+//!
+//! # Construction
+//!
+//! convert pattern into hierarchy of plain, scan, middle
+//! - plain: accept any sequence or any non-empty sequence
+//! - scan: a single scalar pattern moves LTR or RTL, submatchers on either
+//!   side
+//! - middle: two scalar patterns walk over all permutations of matches
+//!   while getting progressively closer to each other
+//!
+//! # Application
+//!
+//! walk over the current matcher's valid options and poll the submatchers
+//! for each of them
+
+mod any_match;
+mod build;
+mod scal_match;
+pub mod shared;
+mod vec_match;
+pub mod state;
+mod vec_attrs;
+// pub mod matcher;

orchid-host/src/rule/scal_match.rs

@@ -0,0 +1,42 @@
+use orchid_base::name::Sym;
+
+use super::any_match::any_match;
+use super::shared::ScalMatcher;
+use crate::{macros::{MacTok, MacTree}, rule::state::{MatchState, StateEntry}};
+
+#[must_use]
+pub fn scal_match<'a>(
+  matcher: &ScalMatcher,
+  expr: &'a MacTree,
+  save_loc: &impl Fn(Sym) -> bool,
+) -> Option<MatchState<'a>> {
+  match (matcher, &*expr.tok) {
+    (ScalMatcher::Name(n1), MacTok::Name(n2)) if n1 == n2 => Some(match save_loc(n1.clone()) {
+      true => MatchState::from_name(n1.clone(), expr.pos.clone()),
+      false => MatchState::default(),
+    }),
+    (ScalMatcher::Placeh { key }, _) =>
+      Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
+    (ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>
+      any_match(b_mat, &body[..], save_loc),
+    (ScalMatcher::Lambda(arg_mat, b_mat), MacTok::Lambda(arg, body)) =>
+      Some(any_match(arg_mat, arg, save_loc)?.combine(any_match(b_mat, body, save_loc)?)),
+    _ => None,
+  }
+}
+
+#[must_use]
+pub fn scalv_match<'a>(
+  matchers: &[ScalMatcher],
+  seq: &'a [MacTree],
+  save_loc: &impl Fn(Sym) -> bool,
+) -> Option<MatchState<'a>> {
+  if seq.len() != matchers.len() {
+    return None;
+  }
+  let mut state = MatchState::default();
+  for (matcher, expr) in matchers.iter().zip(seq.iter()) {
+    state = state.combine(scal_match(matcher, expr, save_loc)?);
+  }
+  Some(state)
+}

orchid-host/src/rule/shared.rs

@@ -0,0 +1,121 @@
+//! Datastructures for cached pattern
+
+use std::fmt;
+
+use itertools::Itertools;
+use orchid_base::interner::Tok;
+
+use super::any_match::any_match;
+use super::build::mk_any;
+use orchid_base::name::Sym;
+use crate::macros::MacTree;
+use crate::rule::state::MatchState;
+use orchid_base::side::Side;
+use orchid_base::tokens::{Paren, PARENS};
+
+pub enum ScalMatcher {
+  Name(Sym),
+  S(Paren, Box<AnyMatcher>),
+  Lambda(Box<AnyMatcher>, Box<AnyMatcher>),
+  Placeh { key: Tok<String> },
+}
+
+pub enum VecMatcher {
+  Placeh {
+    key: Tok<String>,
+    nonzero: bool,
+  },
+  Scan {
+    left: Box<VecMatcher>,
+    sep: Vec<ScalMatcher>,
+    right: Box<VecMatcher>,
+    /// The separator traverses the sequence towards this side
+    direction: Side,
+  },
+  Middle {
+    /// Matches the left outer region
+    left: Box<VecMatcher>,
+    /// Matches the left separator
+    left_sep: Vec<ScalMatcher>,
+    /// Matches the middle - can only ever be a plain placeholder
+    mid: Box<VecMatcher>,
+    /// Matches the right separator
+    right_sep: Vec<ScalMatcher>,
+    /// Matches the right outer region
+    right: Box<VecMatcher>,
+    /// Order of significance for sorting equally good projects based on
+    /// the length of matches on either side.
+    ///
+    /// Vectorial keys that appear on either side, in priority order
+    key_order: Vec<Tok<String>>,
+  },
+}
+
+pub enum AnyMatcher {
+  Scalar(Vec<ScalMatcher>),
+  Vec { left: Vec<ScalMatcher>, mid: VecMatcher, right: Vec<ScalMatcher> },
+}
+
+// ################ Display ################
+
+impl fmt::Display for ScalMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Placeh { key } => write!(f, "${key}"),
+      Self::Name(n) => write!(f, "{n}"),
+      Self::S(t, body) => {
+        let (l, r, _) = PARENS.iter().find(|r| r.2 == *t).unwrap();
+        write!(f, "{l}{body}{r}")
+      },
+      Self::Lambda(arg, body) => write!(f, "\\{arg}.{body}"),
+    }
+  }
+}
+
+impl fmt::Display for VecMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Placeh { key, nonzero: true } => write!(f, "...${key}"),
+      Self::Placeh { key, nonzero: false } => write!(f, "..${key}"),
+      Self::Scan { left, sep, right, direction } => {
+        let arrow = if direction == &Side::Left { "<==" } else { "==>" };
+        write!(f, "Scan{{{left} {arrow} {} {arrow} {right}}}", sep.iter().join(" "))
+      },
+      Self::Middle { left, left_sep, mid, right_sep, right, .. } => {
+        let left_sep_s = left_sep.iter().join(" ");
+        let right_sep_s = right_sep.iter().join(" ");
+        write!(f, "Middle{{{left}|{left_sep_s}|{mid}|{right_sep_s}|{right}}}")
+      },
+    }
+  }
+}
+
+impl fmt::Display for AnyMatcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Scalar(s) => {
+        write!(f, "({})", s.iter().join(" "))
+      },
+      Self::Vec { left, mid, right } => {
+        let lefts = left.iter().join(" ");
+        let rights = right.iter().join(" ");
+        write!(f, "[{lefts}|{mid}|{rights}]")
+      },
+    }
+  }
+}
+
+// ################ External ################
+
+/// A priority-order tree of the vectorial placeholders with scalars as leaves.
+pub struct Matcher(AnyMatcher);
+impl Matcher {
+  pub fn new(pattern: &[MacTree]) -> Self { Self(mk_any(pattern)) }
+  pub fn apply<'a>(&self, seq: &'a [MacTree], save_loc: &impl Fn(Sym) -> bool) -> Option<MatchState<'a>> {
+    any_match(&self.0, seq, save_loc)
+  }
+}
+
+impl fmt::Display for Matcher {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}

orchid-host/src/rule/state.rs

@@ -0,0 +1,84 @@
+use std::sync::Arc;
+
+use hashbrown::HashMap;
+use orchid_api::PhKind;
+use orchid_base::tree::Ph;
+use orchid_base::{interner::Tok, join::join_maps};
+use orchid_base::location::Pos;
+
+use crate::macros::{MacTok, MacTree};
+use orchid_base::name::Sym;
+
+#[derive(Clone, Copy, Debug)]
+pub enum StateEntry<'a> {
+  Vec(&'a [MacTree]),
+  Scalar(&'a MacTree),
+}
+#[derive(Clone)]
+pub struct MatchState<'a> {
+  placeholders: HashMap<Tok<String>, StateEntry<'a>>,
+  name_posv: HashMap<Sym, Vec<Pos>>,
+}
+impl<'a> MatchState<'a> {
+  pub fn from_ph(key: Tok<String>, entry: StateEntry<'a>) -> Self {
+    Self { placeholders: HashMap::from([(key, entry)]), name_posv: HashMap::new() }
+  }
+  pub fn combine(self, s: Self) -> Self {
+    Self {
+      placeholders: self.placeholders.into_iter().chain(s.placeholders).collect(),
+      name_posv: join_maps(self.name_posv, s.name_posv, |_, l, r| {
+        l.into_iter().chain(r).collect()
+      }),
+    }
+  }
+  pub fn ph_len(&self, key: &Tok<String>) -> Option<usize> {
+    match self.placeholders.get(key)? {
+      StateEntry::Vec(slc) => Some(slc.len()),
+      _ => None,
+    }
+  }
+  pub fn from_name(name: Sym, location: Pos) -> Self {
+    Self { name_posv: HashMap::from([(name, vec![location])]), placeholders: HashMap::new() }
+  }
+}
+impl Default for MatchState<'static> {
+  fn default() -> Self { Self { name_posv: HashMap::new(), placeholders: HashMap::new() } }
+}
+
+#[must_use]
+pub fn apply_exprv(template: &[MacTree], state: &MatchState) -> Vec<MacTree> {
+  template.iter().map(|e| apply_expr(e, state)).flat_map(Vec::into_iter).collect()
+}
+
+#[must_use]
+pub fn apply_expr(template: &MacTree, state: &MatchState) -> Vec<MacTree> {
+  let MacTree { pos, tok } = template;
+  match &**tok {
+    MacTok::Name(n) => match state.name_posv.get(n) {
+      None => vec![template.clone()],
+      Some(locs) => vec![MacTree { tok: tok.clone(), pos: locs[0].clone() }],
+    },
+    MacTok::Atom(_) => vec![template.clone()],
+    MacTok::S(c, body) => vec![MacTree {
+      pos: pos.clone(), tok: Arc::new(MacTok::S(*c, apply_exprv(body.as_slice(), state))),
+    }],
+    MacTok::Ph(Ph { name, kind }) => {
+      let Some(value) = state.placeholders.get(name) else {
+        panic!("Placeholder does not have a value in state")
+      };
+      match (kind, value) {
+        (PhKind::Scalar, StateEntry::Scalar(item)) => vec![(*item).clone()],
+        (PhKind::Vector { .. }, StateEntry::Vec(chunk)) => chunk.to_vec(),
+        _ => panic!("Type mismatch between template and state"),
+      }
+    },
+    MacTok::Lambda(arg, body) => vec![MacTree {
+      pos: pos.clone(),
+      tok: Arc::new(MacTok::Lambda(
+        apply_exprv(arg, state),
+        apply_exprv(&body[..], state),
+      )),
+    }],
+    MacTok::Slot(_) | MacTok::Ref(_) => panic!("Extension-only variants"),
+  }
+}

orchid-host/src/rule/vec_attrs.rs

@@ -0,0 +1,16 @@
+use orchid_api::PhKind;
+use orchid_base::interner::Tok;
+use orchid_base::tree::Ph;
+
+use crate::macros::{MacTok, MacTree};
+
+/// Returns the name, priority and at_least_one of the expression if it is
+/// a vectorial placeholder
+#[must_use]
+pub fn vec_attrs(expr: &MacTree) -> Option<(Tok<String>, u8, bool)> {
+  match (*expr.tok).clone() {
+    MacTok::Ph(Ph { kind: PhKind::Vector { priority, at_least_one }, name }) =>
+      Some((name, priority, at_least_one)),
+    _ => None,
+  }
+}

orchid-host/src/rule/vec_match.rs

@@ -0,0 +1,94 @@
+use std::cmp::Ordering;
+
+use itertools::Itertools;
+
+use super::scal_match::scalv_match;
+use super::shared::VecMatcher;
+use orchid_base::name::Sym;
+use crate::{macros::MacTree, rule::state::{MatchState, StateEntry}};
+
+#[must_use]
+pub fn vec_match<'a>(
+  matcher: &VecMatcher,
+  seq: &'a [MacTree],
+  save_loc: &impl Fn(Sym) -> bool,
+) -> Option<MatchState<'a>> {
+  match matcher {
+    VecMatcher::Placeh { key, nonzero } => {
+      if *nonzero && seq.is_empty() {
+        return None;
+      }
+      return Some(MatchState::from_ph(key.clone(), StateEntry::Vec(seq)));
+    },
+    VecMatcher::Scan { left, sep, right, direction } => {
+      if seq.len() < sep.len() {
+        return None;
+      }
+      for lpos in direction.walk(0..=seq.len() - sep.len()) {
+        let rpos = lpos + sep.len();
+        let state = vec_match(left, &seq[..lpos], save_loc)
+          .and_then(|s| Some(s.combine(scalv_match(sep, &seq[lpos..rpos], save_loc)?)))
+          .and_then(|s| Some(s.combine(vec_match(right, &seq[rpos..], save_loc)?)));
+        if let Some(s) = state {
+          return Some(s);
+        }
+      }
+      None
+    },
+    // XXX predict heap space usage and allocation count
+    VecMatcher::Middle { left, left_sep, mid, right_sep, right, key_order } => {
+      if seq.len() < left_sep.len() + right_sep.len() {
+        return None;
+      }
+      // Valid locations for the left separator
+      let lposv = seq[..seq.len() - right_sep.len()]
+        .windows(left_sep.len())
+        .enumerate()
+        .filter_map(|(i, window)| scalv_match(left_sep, window, save_loc).map(|s| (i, s)))
+        .collect::<Vec<_>>();
+      // Valid locations for the right separator
+      let rposv = seq[left_sep.len()..]
+        .windows(right_sep.len())
+        .enumerate()
+        .filter_map(|(i, window)| scalv_match(right_sep, window, save_loc).map(|s| (i, s)))
+        .collect::<Vec<_>>();
+      // Valid combinations of locations for the separators
+      let mut pos_pairs = lposv
+        .into_iter()
+        .cartesian_product(rposv)
+        .filter(|((lpos, _), (rpos, _))| lpos + left_sep.len() <= *rpos)
+        .map(|((lpos, lstate), (rpos, rstate))| (lpos, rpos, lstate.combine(rstate)))
+        .collect::<Vec<_>>();
+      // In descending order of size
+      pos_pairs.sort_by_key(|(l, r, _)| -((r - l) as i64));
+      let eql_clusters = pos_pairs.into_iter().chunk_by(|(al, ar, _)| ar - al);
+      for (_gap_size, cluster) in eql_clusters.into_iter() {
+        let best_candidate = cluster
+          .into_iter()
+          .filter_map(|(lpos, rpos, state)| {
+            Some(
+              state
+                .combine(vec_match(left, &seq[..lpos], save_loc)?)
+                .combine(vec_match(mid, &seq[lpos + left_sep.len()..rpos], save_loc)?)
+                .combine(vec_match(right, &seq[rpos + right_sep.len()..], save_loc)?),
+            )
+          })
+          .max_by(|a, b| {
+            for key in key_order {
+              let alen = a.ph_len(key).expect("key_order references scalar or missing");
+              let blen = b.ph_len(key).expect("key_order references scalar or missing");
+              match alen.cmp(&blen) {
+                Ordering::Equal => (),
+                any => return any,
+              }
+            }
+            Ordering::Equal
+          });
+        if let Some(state) = best_candidate {
+          return Some(state);
+        }
+      }
+      None
+    },
+  }
+}