commit before easter break

orchid-std/src/lib.rs

@@ -1,7 +1,6 @@
-mod number;
+mod macros;
 mod std;
-mod string;
 
-pub use std::StdSystem;
-
-pub use string::str_atom::OrcString;
+pub use std::number::num_atom::{Float, HomoArray, Int, Num};
+pub use std::std_system::StdSystem;
+pub use std::string::str_atom::OrcString;

orchid-std/src/macros/macro_system.rs

@@ -0,0 +1,32 @@
+use never::Never;
+use orchid_base::reqnot::Receipt;
+use orchid_extension::atom::AtomDynfo;
+use orchid_extension::entrypoint::ExtReq;
+use orchid_extension::fs::DeclFs;
+use orchid_extension::lexer::LexerObj;
+use orchid_extension::parser::ParserObj;
+use orchid_extension::system::{System, SystemCard};
+use orchid_extension::system_ctor::SystemCtor;
+use orchid_extension::tree::GenItem;
+
+#[derive(Default)]
+pub struct MacroSystem;
+impl SystemCtor for MacroSystem {
+	type Deps = ();
+	type Instance = Self;
+	const NAME: &'static str = "macros";
+	const VERSION: f64 = 0.00_01;
+	fn inst() -> Option<Self::Instance> { Some(Self) }
+}
+impl SystemCard for MacroSystem {
+	type Ctor = Self;
+	type Req = Never;
+	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> { [] }
+}
+impl System for MacroSystem {
+	async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
+	fn vfs() -> orchid_extension::fs::DeclFs { DeclFs::Mod(&[]) }
+	fn lexers() -> Vec<LexerObj> { vec![] }
+	fn parsers() -> Vec<ParserObj> { vec![] }
+	fn env() -> Vec<GenItem> { vec![] }
+}

orchid-std/src/macros/mactree.rs

@@ -0,0 +1,75 @@
+use std::borrow::Cow;
+use std::rc::Rc;
+
+use futures::future::join_all;
+use orchid_api::Paren;
+use orchid_base::error::OrcErrv;
+use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::tl_cache;
+use orchid_base::tree::Ph;
+use orchid_extension::atom::{Atomic, MethodSetBuilder};
+use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+use orchid_extension::expr::Expr;
+
+#[derive(Debug, Clone)]
+pub struct MacTree {
+	pub pos: Pos,
+	pub tok: Rc<MacTok>,
+}
+impl MacTree {}
+impl Atomic for MacTree {
+	type Data = ();
+	type Variant = OwnedVariant;
+}
+impl OwnedAtom for MacTree {
+	type Refs = ();
+
+	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		self.tok.print(c).await
+	}
+}
+
+#[derive(Debug, Clone)]
+pub enum MacTok {
+	S(Paren, Vec<MacTree>),
+	Name(Sym),
+	/// Only permitted in arguments to `instantiate_tpl`
+	Slot,
+	Value(Expr),
+	Lambda(Vec<MacTree>, Vec<MacTree>),
+	Ph(Ph),
+}
+impl Format for MacTok {
+	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		match self {
+			Self::Value(v) => v.print(c).await,
+			Self::Lambda(arg, b) => FmtUnit::new(
+				tl_cache!(Rc<Variants>: Rc::new(Variants::default()
+					.unbounded("\\{0b}.{1l}")
+					.bounded("(\\{0b}.{1b})"))),
+				[mtreev_fmt(arg, c).await, mtreev_fmt(b, c).await],
+			),
+			Self::Name(n) => format!("{n}").into(),
+			Self::Ph(ph) => format!("{ph}").into(),
+			Self::S(p, body) => FmtUnit::new(
+				match *p {
+					Paren::Round => Rc::new(Variants::default().bounded("({0b})")),
+					Paren::Curly => Rc::new(Variants::default().bounded("{{0b}}")),
+					Paren::Square => Rc::new(Variants::default().bounded("[{0b}]")),
+				},
+				[mtreev_fmt(body, c).await],
+			),
+			Self::Slot => format!("SLOT").into(),
+		}
+	}
+}
+
+pub async fn mtreev_fmt<'b>(
+	v: impl IntoIterator<Item = &'b MacTree>,
+	c: &(impl FmtCtx + ?Sized),
+) -> FmtUnit {
+	FmtUnit::sequence(" ", None, join_all(v.into_iter().map(|t| t.print(c))).await)
+}

orchid-std/src/macros/mactree_lexer.rs

@@ -0,0 +1,64 @@
+use std::ops::RangeInclusive;
+use std::rc::Rc;
+
+use futures::FutureExt;
+use orchid_base::error::{OrcRes, mk_errv};
+use orchid_base::location::Pos;
+use orchid_base::parse::name_start;
+use orchid_base::tokens::PARENS;
+use orchid_extension::atom::AtomicFeatures;
+use orchid_extension::gen_expr::atom;
+use orchid_extension::lexer::{LexContext, Lexer, err_not_applicable};
+use orchid_extension::tree::{GenTok, GenTokTree};
+
+use crate::macros::mactree::{MacTok, MacTree};
+
+#[derive(Default)]
+pub struct MacTreeLexer;
+impl Lexer for MacTreeLexer {
+	const CHAR_FILTER: &'static [RangeInclusive<char>] = &['\''..='\''];
+	async fn lex<'a>(tail: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> {
+		let Some(tail2) = tail.strip_prefix('\'') else {
+			return Err(err_not_applicable(ctx.i).await.into());
+		};
+		let tail3 = tail2.trim_start();
+		return match mac_tree(tail3, ctx).await {
+			Ok((tail4, mactree)) =>
+				Ok((tail4, GenTok::X(mactree.factory()).at(ctx.pos(tail)..ctx.pos(tail4)))),
+			Err(e) => Ok((tail2, GenTok::Bottom(e).at(ctx.tok_ran(1, tail2)))),
+		};
+		async fn mac_tree<'a>(tail: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, MacTree)> {
+			for (lp, rp, paren) in PARENS {
+				let Some(mut body_tail) = tail.strip_prefix(*lp) else { continue };
+				let mut items = Vec::new();
+				return loop {
+					let tail2 = body_tail.trim();
+					if let Some(tail3) = tail2.strip_prefix(*rp) {
+						break Ok((tail3, MacTree {
+							pos: Pos::Range(ctx.pos(tail)..ctx.pos(tail3)),
+							tok: Rc::new(MacTok::S(*paren, items)),
+						}));
+					} else if tail2.is_empty() {
+						return Err(mk_errv(
+							ctx.i.i("Unclosed block").await,
+							format!("Expected closing {rp}"),
+							[Pos::Range(ctx.tok_ran(1, tail)).into()],
+						));
+					}
+					let (new_tail, new_item) = mac_tree(tail2, ctx).boxed_local().await?;
+					body_tail = new_tail;
+					items.push(new_item);
+				};
+			}
+			const INTERPOL: &[&str] = &["$", "..$"];
+			for pref in INTERPOL {
+				let Some(code) = tail.strip_prefix(pref) else { continue };
+			}
+			return Err(mk_errv(
+				ctx.i.i("Expected token after '").await,
+				format!("Expected a token after ', found {tail:?}"),
+				[Pos::Range(ctx.tok_ran(1, tail)).into()],
+			));
+		}
+	}
+}

orchid-std/src/macros/mod.rs

@@ -0,0 +1,6 @@
+mod macro_system;
+pub mod mactree;
+mod mactree_lexer;
+mod rule;
+
+use mactree::{MacTok, MacTree};

orchid-std/src/macros/rule/any_match.rs

@@ -0,0 +1,30 @@
+use orchid_base::name::Sym;
+
+use super::scal_match::scalv_match;
+use super::shared::AnyMatcher;
+use super::state::MatchState;
+use super::vec_match::vec_match;
+use crate::macros::MacTree;
+
+#[must_use]
+pub fn any_match<'a>(
+	matcher: &AnyMatcher,
+	seq: &'a [MacTree],
+	save_loc: &impl Fn(Sym) -> bool,
+) -> Option<MatchState<'a>> {
+	match matcher {
+		AnyMatcher::Scalar(scalv) => scalv_match(scalv, seq, save_loc),
+		AnyMatcher::Vec { left, mid, right } => {
+			if seq.len() < left.len() + right.len() {
+				return None;
+			};
+			let left_split = left.len();
+			let right_split = seq.len() - right.len();
+			Some(
+				scalv_match(left, &seq[..left_split], save_loc)?
+					.combine(scalv_match(right, &seq[right_split..], save_loc)?)
+					.combine(vec_match(mid, &seq[left_split..right_split], save_loc)?),
+			)
+		},
+	}
+}

orchid-std/src/macros/rule/build.rs

@@ -0,0 +1,160 @@
+use itertools::Itertools;
+use orchid_api::PhKind;
+use orchid_base::interner::Tok;
+use orchid_base::side::Side;
+use orchid_base::tree::Ph;
+
+use super::shared::{AnyMatcher, ScalMatcher, VecMatcher};
+use super::vec_attrs::vec_attrs;
+use crate::macros::{MacTok, MacTree};
+
+pub type MaxVecSplit<'a> = (&'a [MacTree], (Tok<String>, u8, bool), &'a [MacTree]);
+
+/// Derive the details of the central vectorial and the two sides from a
+/// slice of Expr's
+#[must_use]
+fn split_at_max_vec(pattern: &[MacTree]) -> Option<MaxVecSplit> {
+	let rngidx = pattern
+		.iter()
+		.position_max_by_key(|expr| vec_attrs(expr).map(|attrs| attrs.1 as i64).unwrap_or(-1))?;
+	let (left, not_left) = pattern.split_at(rngidx);
+	let (placeh, right) =
+		not_left.split_first().expect("The index of the greatest element must be less than the length");
+	vec_attrs(placeh).map(|attrs| (left, attrs, right))
+}
+
+#[must_use]
+fn scal_cnt<'a>(iter: impl Iterator<Item = &'a MacTree>) -> usize {
+	iter.take_while(|expr| vec_attrs(expr).is_none()).count()
+}
+
+#[must_use]
+pub fn mk_any(pattern: &[MacTree]) -> AnyMatcher {
+	let left_split = scal_cnt(pattern.iter());
+	if pattern.len() <= left_split {
+		return AnyMatcher::Scalar(mk_scalv(pattern));
+	}
+	let (left, not_left) = pattern.split_at(left_split);
+	let right_split = not_left.len() - scal_cnt(pattern.iter().rev());
+	let (mid, right) = not_left.split_at(right_split);
+	AnyMatcher::Vec { left: mk_scalv(left), mid: mk_vec(mid), right: mk_scalv(right) }
+}
+
+/// Pattern MUST NOT contain vectorial placeholders
+#[must_use]
+fn mk_scalv(pattern: &[MacTree]) -> Vec<ScalMatcher> { pattern.iter().map(mk_scalar).collect() }
+
+/// Pattern MUST start and end with a vectorial placeholder
+#[must_use]
+pub fn mk_vec(pattern: &[MacTree]) -> VecMatcher {
+	debug_assert!(!pattern.is_empty(), "pattern cannot be empty");
+	debug_assert!(pattern.first().map(vec_attrs).is_some(), "pattern must start with a vectorial");
+	debug_assert!(pattern.last().map(vec_attrs).is_some(), "pattern must end with a vectorial");
+	let (left, (key, _, nonzero), right) = split_at_max_vec(pattern)
+		.expect("pattern must have vectorial placeholders at least at either end");
+	let r_sep_size = scal_cnt(right.iter());
+	let (r_sep, r_side) = right.split_at(r_sep_size);
+	let l_sep_size = scal_cnt(left.iter().rev());
+	let (l_side, l_sep) = left.split_at(left.len() - l_sep_size);
+	let main = VecMatcher::Placeh { key: key.clone(), nonzero };
+	match (left, right) {
+		(&[], &[]) => VecMatcher::Placeh { key, nonzero },
+		(&[], _) => VecMatcher::Scan {
+			direction: Side::Left,
+			left: Box::new(main),
+			sep: mk_scalv(r_sep),
+			right: Box::new(mk_vec(r_side)),
+		},
+		(_, &[]) => VecMatcher::Scan {
+			direction: Side::Right,
+			left: Box::new(mk_vec(l_side)),
+			sep: mk_scalv(l_sep),
+			right: Box::new(main),
+		},
+		(..) => {
+			let mut key_order =
+				l_side.iter().chain(r_side.iter()).filter_map(vec_attrs).collect::<Vec<_>>();
+			key_order.sort_by_key(|(_, prio, _)| -(*prio as i64));
+			VecMatcher::Middle {
+				left: Box::new(mk_vec(l_side)),
+				left_sep: mk_scalv(l_sep),
+				mid: Box::new(main),
+				right_sep: mk_scalv(r_sep),
+				right: Box::new(mk_vec(r_side)),
+				key_order: key_order.into_iter().map(|(n, ..)| n).collect(),
+			}
+		},
+	}
+}
+
+/// Pattern MUST NOT be a vectorial placeholder
+#[must_use]
+fn mk_scalar(pattern: &MacTree) -> ScalMatcher {
+	match &*pattern.tok {
+		MacTok::Name(n) => ScalMatcher::Name(n.clone()),
+		MacTok::Ph(Ph { name, kind }) => match kind {
+			PhKind::Vector { .. } => {
+				panic!("Scalar matcher cannot be built from vector pattern")
+			},
+			PhKind::Scalar => ScalMatcher::Placeh { key: name.clone() },
+		},
+		MacTok::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
+		MacTok::Lambda(arg, body) => ScalMatcher::Lambda(Box::new(mk_any(arg)), Box::new(mk_any(body))),
+		MacTok::Value(_) | MacTok::Slot => panic!("Only used for templating"),
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use std::rc::Rc;
+
+	use orchid_api::PhKind;
+	use orchid_base::interner::Interner;
+	use orchid_base::location::SourceRange;
+	use orchid_base::sym;
+	use orchid_base::tokens::Paren;
+	use orchid_base::tree::Ph;
+	use test_executors::spin_on;
+
+	use super::mk_any;
+	use crate::macros::{MacTok, MacTree};
+
+	#[test]
+	fn test_scan() {
+		spin_on(async {
+			let i = Interner::new_master();
+			let ex = |tok: MacTok| async {
+				MacTree { tok: Rc::new(tok), pos: SourceRange::mock(&i).await.pos() }
+			};
+			let pattern = vec![
+				ex(MacTok::Ph(Ph {
+					kind: PhKind::Vector { priority: 0, at_least_one: false },
+					name: i.i("::prefix").await,
+				}))
+				.await,
+				ex(MacTok::Name(sym!(prelude::do; i).await)).await,
+				ex(MacTok::S(Paren::Round, vec![
+					ex(MacTok::Ph(Ph {
+						kind: PhKind::Vector { priority: 0, at_least_one: false },
+						name: i.i("expr").await,
+					}))
+					.await,
+					ex(MacTok::Name(sym!(prelude::; ; i).await)).await,
+					ex(MacTok::Ph(Ph {
+						kind: PhKind::Vector { priority: 1, at_least_one: false },
+						name: i.i("rest").await,
+					}))
+					.await,
+				]))
+				.await,
+				ex(MacTok::Ph(Ph {
+					kind: PhKind::Vector { priority: 0, at_least_one: false },
+					name: i.i("::suffix").await,
+				}))
+				.await,
+			];
+			let matcher = mk_any(&pattern);
+			println!("{matcher}");
+		})
+	}
+}

orchid-std/src/macros/rule/matcher.rs

@@ -0,0 +1,87 @@
+use std::fmt;
+use std::rc::Rc;
+
+use itertools::Itertools;
+use orchid_api::PhKind;
+use orchid_base::interner::Interner;
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
+use orchid_base::tree::Ph;
+
+use super::any_match::any_match;
+use super::build::{mk_any, mk_vec};
+use super::shared::{AnyMatcher, VecMatcher};
+use super::state::{MatchState, StateEntry};
+use super::vec_attrs::vec_attrs;
+use super::vec_match::vec_match;
+use crate::macros::{MacTok, MacTree};
+
+pub fn first_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.first().unwrap()).is_some() }
+pub fn last_is_vec(pattern: &[MacTree]) -> bool { vec_attrs(pattern.last().unwrap()).is_some() }
+
+pub struct NamedMatcher(AnyMatcher);
+impl NamedMatcher {
+	pub async fn new(pattern: &[MacTree], i: &Interner) -> Self {
+		assert!(
+			matches!(pattern.first().map(|tree| &*tree.tok), Some(MacTok::Name(_))),
+			"Named matchers must begin with a name"
+		);
+
+		match last_is_vec(pattern) {
+			true => Self(mk_any(pattern)),
+			false => {
+				let kind = PhKind::Vector { priority: 0, at_least_one: false };
+				let tok = MacTok::Ph(Ph { name: i.i("::after").await, kind });
+				let suffix = [MacTree { pos: Pos::None, tok: Rc::new(tok) }];
+				Self(mk_any(&pattern.iter().chain(&suffix).cloned().collect_vec()))
+			},
+		}
+	}
+	/// Also returns the tail, if any, which should be matched further
+	/// Note that due to how priod works below, the main usable information from
+	/// the tail is its length
+	pub async fn apply<'a>(
+		&self,
+		seq: &'a [MacTree],
+		i: &Interner,
+		save_loc: impl Fn(Sym) -> bool,
+	) -> Option<(MatchState<'a>, &'a [MacTree])> {
+		let mut state = any_match(&self.0, seq, &save_loc)?;
+		match state.remove(i.i("::after").await) {
+			Some(StateEntry::Scalar(_)) => panic!("::after can never be a scalar entry!"),
+			Some(StateEntry::Vec(v)) => Some((state, v)),
+			None => Some((state, &[][..])),
+		}
+	}
+}
+impl fmt::Display for NamedMatcher {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}
+impl fmt::Debug for NamedMatcher {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "NamedMatcher({self})") }
+}
+
+pub struct PriodMatcher(VecMatcher);
+impl PriodMatcher {
+	pub fn new(pattern: &[MacTree]) -> Self {
+		assert!(
+			pattern.first().and_then(vec_attrs).is_some() && pattern.last().and_then(vec_attrs).is_some(),
+			"Prioritized matchers must start and end with a vectorial",
+		);
+		Self(mk_vec(pattern))
+	}
+	/// tokens before the offset always match the prefix
+	pub fn apply<'a>(
+		&self,
+		seq: &'a [MacTree],
+		save_loc: impl Fn(Sym) -> bool,
+	) -> Option<MatchState<'a>> {
+		vec_match(&self.0, seq, &save_loc)
+	}
+}
+impl fmt::Display for PriodMatcher {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) }
+}
+impl fmt::Debug for PriodMatcher {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "PriodMatcher({self})") }
+}

orchid-std/src/macros/rule/mod.rs

@@ -0,0 +1,24 @@
+//! Optimized form of macro pattern that can be quickly tested against the AST.
+//!
+//! # Construction
+//!
+//! convert pattern into hierarchy of plain, scan, middle
+//! - plain: accept any sequence or any non-empty sequence
+//! - scan: a single scalar pattern moves LTR or RTL, submatchers on either side
+//! - middle: two scalar patterns walk over all permutations of matches while
+//!   getting progressively closer to each other
+//!
+//! # Application
+//!
+//! walk over the current matcher's valid options and poll the submatchers
+//! for each of them
+
+mod any_match;
+mod build;
+pub mod matcher;
+mod scal_match;
+pub mod shared;
+pub mod state;
+mod vec_attrs;
+mod vec_match;
+// pub mod matcher;

orchid-std/src/macros/rule/scal_match.rs

@@ -0,0 +1,43 @@
+use orchid_base::name::Sym;
+
+use super::any_match::any_match;
+use super::shared::ScalMatcher;
+use super::state::{MatchState, StateEntry};
+use crate::macros::{MacTok, MacTree};
+
+#[must_use]
+pub fn scal_match<'a>(
+	matcher: &ScalMatcher,
+	expr: &'a MacTree,
+	save_loc: &impl Fn(Sym) -> bool,
+) -> Option<MatchState<'a>> {
+	match (matcher, &*expr.tok) {
+		(ScalMatcher::Name(n1), MacTok::Name(n2)) if n1 == n2 => Some(match save_loc(n1.clone()) {
+			true => MatchState::from_name(n1.clone(), expr.pos.clone()),
+			false => MatchState::default(),
+		}),
+		(ScalMatcher::Placeh { key }, _) =>
+			Some(MatchState::from_ph(key.clone(), StateEntry::Scalar(expr))),
+		(ScalMatcher::S(c1, b_mat), MacTok::S(c2, body)) if c1 == c2 =>
+			any_match(b_mat, &body[..], save_loc),
+		(ScalMatcher::Lambda(arg_mat, b_mat), MacTok::Lambda(arg, body)) =>
+			Some(any_match(arg_mat, arg, save_loc)?.combine(any_match(b_mat, body, save_loc)?)),
+		_ => None,
+	}
+}
+
+#[must_use]
+pub fn scalv_match<'a>(
+	matchers: &[ScalMatcher],
+	seq: &'a [MacTree],
+	save_loc: &impl Fn(Sym) -> bool,
+) -> Option<MatchState<'a>> {
+	if seq.len() != matchers.len() {
+		return None;
+	}
+	let mut state = MatchState::default();
+	for (matcher, expr) in matchers.iter().zip(seq.iter()) {
+		state = state.combine(scal_match(matcher, expr, save_loc)?);
+	}
+	Some(state)
+}

orchid-std/src/macros/rule/shared.rs

@@ -0,0 +1,101 @@
+//! Datastructures for cached pattern
+
+use std::fmt;
+
+use itertools::Itertools;
+use orchid_base::interner::Tok;
+use orchid_base::name::Sym;
+use orchid_base::side::Side;
+use orchid_base::tokens::{PARENS, Paren};
+
+pub enum ScalMatcher {
+	Name(Sym),
+	S(Paren, Box<AnyMatcher>),
+	Lambda(Box<AnyMatcher>, Box<AnyMatcher>),
+	Placeh { key: Tok<String> },
+}
+
+pub enum VecMatcher {
+	Placeh {
+		key: Tok<String>,
+		nonzero: bool,
+	},
+	Scan {
+		left: Box<VecMatcher>,
+		sep: Vec<ScalMatcher>,
+		right: Box<VecMatcher>,
+		/// The separator traverses the sequence towards this side
+		direction: Side,
+	},
+	Middle {
+		/// Matches the left outer region
+		left: Box<VecMatcher>,
+		/// Matches the left separator
+		left_sep: Vec<ScalMatcher>,
+		/// Matches the middle - can only ever be a plain placeholder
+		mid: Box<VecMatcher>,
+		/// Matches the right separator
+		right_sep: Vec<ScalMatcher>,
+		/// Matches the right outer region
+		right: Box<VecMatcher>,
+		/// Order of significance for sorting equally good projects based on
+		/// the length of matches on either side.
+		///
+		/// Vectorial keys that appear on either side, in priority order
+		key_order: Vec<Tok<String>>,
+	},
+}
+
+pub enum AnyMatcher {
+	Scalar(Vec<ScalMatcher>),
+	Vec { left: Vec<ScalMatcher>, mid: VecMatcher, right: Vec<ScalMatcher> },
+}
+
+// ################ Display ################
+
+impl fmt::Display for ScalMatcher {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Self::Placeh { key } => write!(f, "${key}"),
+			Self::Name(n) => write!(f, "{n}"),
+			Self::S(t, body) => {
+				let (l, r, _) = PARENS.iter().find(|r| r.2 == *t).unwrap();
+				write!(f, "{l}{body}{r}")
+			},
+			Self::Lambda(arg, body) => write!(f, "\\{arg}.{body}"),
+		}
+	}
+}
+
+impl fmt::Display for VecMatcher {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Self::Placeh { key, nonzero: true } => write!(f, "...${key}"),
+			Self::Placeh { key, nonzero: false } => write!(f, "..${key}"),
+			Self::Scan { left, sep, right, direction } => {
+				let arrow = if direction == &Side::Left { "<==" } else { "==>" };
+				write!(f, "Scan{{{left} {arrow} {} {arrow} {right}}}", sep.iter().join(" "))
+			},
+			Self::Middle { left, left_sep, mid, right_sep, right, .. } => {
+				let left_sep_s = left_sep.iter().join(" ");
+				let right_sep_s = right_sep.iter().join(" ");
+				write!(f, "Middle{{{left}|{left_sep_s}|{mid}|{right_sep_s}|{right}}}")
+			},
+		}
+	}
+}
+
+impl fmt::Display for AnyMatcher {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			Self::Scalar(s) => {
+				write!(f, "({})", s.iter().join(" "))
+			},
+			Self::Vec { left, mid, right } => {
+				let lefts = left.iter().join(" ");
+				let rights = right.iter().join(" ");
+				write!(f, "[{lefts}|{mid}|{rights}]")
+			},
+		}
+	}
+}

orchid-std/src/macros/rule/state.rs

@@ -0,0 +1,93 @@
+#![allow(unused)]
+use std::any::Any;
+
+use hashbrown::HashMap;
+use orchid_base::interner::Tok;
+use orchid_base::join::join_maps;
+use orchid_base::location::Pos;
+use orchid_base::match_mapping;
+use orchid_base::name::Sym;
+
+use crate::macros::MacTree;
+
+enum StackAction {
+	Return(Box<dyn Any>),
+	Call {
+		target: Box<dyn FnOnce(Box<dyn Any>) -> StackAction>,
+		param: Box<dyn Any>,
+		tail: Box<dyn FnOnce(Box<dyn Any>) -> StackAction>,
+	},
+}
+
+struct Trampoline {
+	stack: Vec<Box<dyn FnOnce(Box<dyn Any>) -> StackAction>>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum StateEntry<'a> {
+	Vec(&'a [MacTree]),
+	Scalar(&'a MacTree),
+}
+#[derive(Clone, Debug)]
+pub struct MatchState<'a> {
+	placeholders: HashMap<Tok<String>, StateEntry<'a>>,
+	name_posv: HashMap<Sym, Vec<Pos>>,
+}
+impl<'a> MatchState<'a> {
+	pub fn from_ph(key: Tok<String>, entry: StateEntry<'a>) -> Self {
+		Self { placeholders: HashMap::from([(key, entry)]), name_posv: HashMap::new() }
+	}
+	pub fn combine(self, s: Self) -> Self {
+		Self {
+			placeholders: self.placeholders.into_iter().chain(s.placeholders).collect(),
+			name_posv: join_maps::<_, Vec<Pos>>(self.name_posv, s.name_posv, |_, l, r| {
+				l.into_iter().chain(r).collect()
+			}),
+		}
+	}
+	pub fn ph_len(&self, key: &Tok<String>) -> Option<usize> {
+		match self.placeholders.get(key)? {
+			StateEntry::Vec(slc) => Some(slc.len()),
+			_ => None,
+		}
+	}
+	pub fn from_name(name: Sym, location: Pos) -> Self {
+		Self { name_posv: HashMap::from([(name, vec![location])]), placeholders: HashMap::new() }
+	}
+	pub fn remove(&mut self, name: Tok<String>) -> Option<StateEntry<'a>> {
+		self.placeholders.remove(&name)
+	}
+	pub fn mk_owned(self) -> OwnedState {
+		OwnedState {
+			placeholders: (self.placeholders.into_iter())
+				.map(|(k, v)| {
+					(
+						k.clone(),
+						match_mapping!(v, StateEntry => OwnedEntry {
+							Scalar(tree.clone()),
+							Vec(v.to_vec()),
+						}),
+					)
+				})
+				.collect(),
+			name_posv: self.name_posv,
+		}
+	}
+}
+impl Default for MatchState<'static> {
+	fn default() -> Self { Self { name_posv: HashMap::new(), placeholders: HashMap::new() } }
+}
+
+#[derive(Clone, Debug)]
+pub enum OwnedEntry {
+	Vec(Vec<MacTree>),
+	Scalar(MacTree),
+}
+pub struct OwnedState {
+	placeholders: HashMap<Tok<String>, OwnedEntry>,
+	name_posv: HashMap<Sym, Vec<Pos>>,
+}
+impl OwnedState {
+	pub fn get(&self, key: &Tok<String>) -> Option<&OwnedEntry> { self.placeholders.get(key) }
+	pub fn positions(&self, name: &Sym) -> &[Pos] { self.name_posv.get(name).map_or(&[], |v| &v[..]) }
+}

orchid-std/src/macros/rule/vec_attrs.rs

@@ -0,0 +1,16 @@
+use orchid_api::PhKind;
+use orchid_base::interner::Tok;
+use orchid_base::tree::Ph;
+
+use crate::macros::{MacTok, MacTree};
+
+/// Returns the name, priority and at_least_one of the expression if it is
+/// a vectorial placeholder
+#[must_use]
+pub fn vec_attrs(expr: &MacTree) -> Option<(Tok<String>, u8, bool)> {
+	match (*expr.tok).clone() {
+		MacTok::Ph(Ph { kind: PhKind::Vector { priority, at_least_one }, name }) =>
+			Some((name, priority, at_least_one)),
+		_ => None,
+	}
+}

orchid-std/src/macros/rule/vec_match.rs

@@ -0,0 +1,95 @@
+use std::cmp::Ordering;
+
+use itertools::Itertools;
+use orchid_base::name::Sym;
+
+use super::scal_match::scalv_match;
+use super::shared::VecMatcher;
+use super::state::{MatchState, StateEntry};
+use crate::macros::MacTree;
+
+#[must_use]
+pub fn vec_match<'a>(
+	matcher: &VecMatcher,
+	seq: &'a [MacTree],
+	save_loc: &impl Fn(Sym) -> bool,
+) -> Option<MatchState<'a>> {
+	match matcher {
+		VecMatcher::Placeh { key, nonzero } => {
+			if *nonzero && seq.is_empty() {
+				return None;
+			}
+			Some(MatchState::from_ph(key.clone(), StateEntry::Vec(seq)))
+		},
+		VecMatcher::Scan { left, sep, right, direction } => {
+			if seq.len() < sep.len() {
+				return None;
+			}
+			for lpos in direction.walk(0..=seq.len() - sep.len()) {
+				let rpos = lpos + sep.len();
+				let state = vec_match(left, &seq[..lpos], save_loc)
+					.and_then(|s| Some(s.combine(scalv_match(sep, &seq[lpos..rpos], save_loc)?)))
+					.and_then(|s| Some(s.combine(vec_match(right, &seq[rpos..], save_loc)?)));
+				if let Some(s) = state {
+					return Some(s);
+				}
+			}
+			None
+		},
+		// XXX predict heap space usage and allocation count
+		VecMatcher::Middle { left, left_sep, mid, right_sep, right, key_order } => {
+			if seq.len() < left_sep.len() + right_sep.len() {
+				return None;
+			}
+			// Valid locations for the left separator
+			let lposv = seq[..seq.len() - right_sep.len()]
+				.windows(left_sep.len())
+				.enumerate()
+				.filter_map(|(i, window)| scalv_match(left_sep, window, save_loc).map(|s| (i, s)))
+				.collect::<Vec<_>>();
+			// Valid locations for the right separator
+			let rposv = seq[left_sep.len()..]
+				.windows(right_sep.len())
+				.enumerate()
+				.filter_map(|(i, window)| scalv_match(right_sep, window, save_loc).map(|s| (i, s)))
+				.collect::<Vec<_>>();
+			// Valid combinations of locations for the separators
+			let mut pos_pairs = lposv
+				.into_iter()
+				.cartesian_product(rposv)
+				.filter(|((lpos, _), (rpos, _))| lpos + left_sep.len() <= *rpos)
+				.map(|((lpos, lstate), (rpos, rstate))| (lpos, rpos, lstate.combine(rstate)))
+				.collect::<Vec<_>>();
+			// In descending order of size
+			pos_pairs.sort_by_key(|(l, r, _)| -((r - l) as i64));
+			let eql_clusters = pos_pairs.into_iter().chunk_by(|(al, ar, _)| ar - al);
+			for (_gap_size, cluster) in eql_clusters.into_iter() {
+				let best_candidate = cluster
+					.into_iter()
+					.filter_map(|(lpos, rpos, state)| {
+						Some(
+							state
+								.combine(vec_match(left, &seq[..lpos], save_loc)?)
+								.combine(vec_match(mid, &seq[lpos + left_sep.len()..rpos], save_loc)?)
+								.combine(vec_match(right, &seq[rpos + right_sep.len()..], save_loc)?),
+						)
+					})
+					.max_by(|a, b| {
+						for key in key_order {
+							let alen = a.ph_len(key).expect("key_order references scalar or missing");
+							let blen = b.ph_len(key).expect("key_order references scalar or missing");
+							match alen.cmp(&blen) {
+								Ordering::Equal => (),
+								any => return any,
+							}
+						}
+						Ordering::Equal
+					});
+				if let Some(state) = best_candidate {
+					return Some(state);
+				}
+			}
+			None
+		},
+	}
+}

orchid-std/src/std.rs

@@ -1,77 +0,0 @@
-use std::rc::Rc;
-
-use never::Never;
-use orchid_base::number::Numeric;
-use orchid_base::reqnot::Receipt;
-use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
-use orchid_extension::entrypoint::ExtReq;
-use orchid_extension::fs::DeclFs;
-use orchid_extension::system::{System, SystemCard};
-use orchid_extension::system_ctor::SystemCtor;
-use orchid_extension::tree::{MemKind, comments, fun, module, root_mod};
-use ordered_float::NotNan;
-
-use crate::OrcString;
-use crate::number::num_atom::{Float, HomoArray, Int, Num};
-use crate::number::num_lexer::NumLexer;
-use crate::string::str_atom::{IntStrAtom, StrAtom};
-use crate::string::str_lexer::StringLexer;
-
-#[derive(Default)]
-pub struct StdSystem;
-impl SystemCtor for StdSystem {
-	type Deps = ();
-	type Instance = Self;
-	const NAME: &'static str = "orchid::std";
-	const VERSION: f64 = 0.00_01;
-	fn inst() -> Option<Self::Instance> { Some(StdSystem) }
-}
-impl SystemCard for StdSystem {
-	type Ctor = Self;
-	type Req = Never;
-	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> {
-		[Some(Int::dynfo()), Some(Float::dynfo()), Some(StrAtom::dynfo()), Some(IntStrAtom::dynfo())]
-	}
-}
-impl System for StdSystem {
-	async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
-	fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer, &NumLexer] }
-	fn parsers() -> Vec<orchid_extension::parser::ParserObj> { vec![] }
-	fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
-	fn env() -> Vec<(String, MemKind)> {
-		vec![root_mod("std", [], [
-			module(true, "string", [], [comments(
-				["Concatenate two strings"],
-				fun(true, "concat", |left: OrcString<'static>, right: OrcString<'static>| async move {
-					StrAtom::new(Rc::new(left.get_string().await.to_string() + &right.get_string().await))
-				}),
-			)]),
-			module(true, "number", [], [
-				fun(true, "add", |a: Num, b: Num| async move {
-					Num(match HomoArray::new([a.0, b.0]) {
-						HomoArray::Int([a, b]) => Numeric::Int(a + b),
-						HomoArray::Float([a, b]) => Numeric::Float(a + b),
-					})
-				}),
-				fun(true, "neg", |a: Num| async move {
-					Num(match a.0 {
-						Numeric::Int(i) => Numeric::Int(-i),
-						Numeric::Float(f) => Numeric::Float(-f),
-					})
-				}),
-				fun(true, "mul", |a: Num, b: Num| async move {
-					Num(match HomoArray::new([a.0, b.0]) {
-						HomoArray::Int([a, b]) => Numeric::Int(a * b),
-						HomoArray::Float([a, b]) => Numeric::Float(a * b),
-					})
-				}),
-				fun(true, "idiv", |a: Int, b: Int| async move { Int(a.0 / b.0) }),
-				fun(true, "imod", |a: Int, b: Int| async move { Int(a.0 % b.0) }),
-				fun(true, "fdiv", |a: Float, b: Float| async move { Float(a.0 / b.0) }),
-				fun(true, "fmod", |a: Float, b: Float| async move {
-					Float(a.0 - NotNan::new((a.0 / b.0).trunc()).unwrap() * b.0)
-				}),
-			]),
-		])]
-	}
-}

orchid-std/src/std/mod.rs

@@ -0,0 +1,4 @@
+pub mod number;
+pub mod string;
+
+pub mod std_system;

orchid-std/src/std/number/mod.rs

@@ -1,2 +1,3 @@
 pub mod num_atom;
 pub mod num_lexer;
+pub mod num_lib;

orchid-std/src/std/number/num_atom.rs

@@ -17,7 +17,6 @@ pub struct Int(pub i64);
 impl Atomic for Int {
 	type Variant = ThinVariant;
 	type Data = Self;
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl ThinAtom for Int {
 	async fn print(&self, _: SysCtx) -> FmtUnit { self.0.to_string().into() }
@@ -33,7 +32,6 @@ pub struct Float(pub NotNan<f64>);
 impl Atomic for Float {
 	type Variant = ThinVariant;
 	type Data = Self;
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl ThinAtom for Float {
 	async fn print(&self, _: SysCtx) -> FmtUnit { self.0.to_string().into() }

orchid-std/src/std/number/num_lib.rs

@@ -0,0 +1,34 @@
+use orchid_base::number::Numeric;
+use orchid_extension::tree::{GenItem, fun, prefix};
+use ordered_float::NotNan;
+
+use super::num_atom::{Float, HomoArray, Int, Num};
+
+pub fn gen_num_lib() -> Vec<GenItem> {
+	prefix("std::number", [
+		fun(true, "add", |a: Num, b: Num| async move {
+			Num(match HomoArray::new([a.0, b.0]) {
+				HomoArray::Int([a, b]) => Numeric::Int(a + b),
+				HomoArray::Float([a, b]) => Numeric::Float(a + b),
+			})
+		}),
+		fun(true, "neg", |a: Num| async move {
+			Num(match a.0 {
+				Numeric::Int(i) => Numeric::Int(-i),
+				Numeric::Float(f) => Numeric::Float(-f),
+			})
+		}),
+		fun(true, "mul", |a: Num, b: Num| async move {
+			Num(match HomoArray::new([a.0, b.0]) {
+				HomoArray::Int([a, b]) => Numeric::Int(a * b),
+				HomoArray::Float([a, b]) => Numeric::Float(a * b),
+			})
+		}),
+		fun(true, "idiv", |a: Int, b: Int| async move { Int(a.0 / b.0) }),
+		fun(true, "imod", |a: Int, b: Int| async move { Int(a.0 % b.0) }),
+		fun(true, "fdiv", |a: Float, b: Float| async move { Float(a.0 / b.0) }),
+		fun(true, "fmod", |a: Float, b: Float| async move {
+			Float(a.0 - NotNan::new((a.0 / b.0).trunc()).unwrap() * b.0)
+		}),
+	])
+}

orchid-std/src/std/std_system.rs

@@ -0,0 +1,41 @@
+use never::Never;
+use orchid_base::reqnot::Receipt;
+use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
+use orchid_extension::entrypoint::ExtReq;
+use orchid_extension::fs::DeclFs;
+use orchid_extension::lexer::LexerObj;
+use orchid_extension::parser::ParserObj;
+use orchid_extension::system::{System, SystemCard};
+use orchid_extension::system_ctor::SystemCtor;
+use orchid_extension::tree::{GenItem, merge_trivial};
+
+use super::number::num_lib::gen_num_lib;
+use super::string::str_atom::{IntStrAtom, StrAtom};
+use super::string::str_lib::gen_str_lib;
+use crate::std::number::num_lexer::NumLexer;
+use crate::std::string::str_lexer::StringLexer;
+use crate::{Float, Int};
+
+#[derive(Default)]
+pub struct StdSystem;
+impl SystemCtor for StdSystem {
+	type Deps = ();
+	type Instance = Self;
+	const NAME: &'static str = "orchid::std";
+	const VERSION: f64 = 0.00_01;
+	fn inst() -> Option<Self::Instance> { Some(Self) }
+}
+impl SystemCard for StdSystem {
+	type Ctor = Self;
+	type Req = Never;
+	fn atoms() -> impl IntoIterator<Item = Option<Box<dyn AtomDynfo>>> {
+		[Some(Int::dynfo()), Some(Float::dynfo()), Some(StrAtom::dynfo()), Some(IntStrAtom::dynfo())]
+	}
+}
+impl System for StdSystem {
+	async fn request(_: ExtReq<'_>, req: Self::Req) -> Receipt<'_> { match req {} }
+	fn lexers() -> Vec<LexerObj> { vec![&StringLexer, &NumLexer] }
+	fn parsers() -> Vec<ParserObj> { vec![] }
+	fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
+	fn env() -> Vec<GenItem> { merge_trivial([gen_num_lib(), gen_str_lib()]) }
+}

orchid-std/src/std/string/mod.rs

@@ -1,2 +1,3 @@
 pub mod str_atom;
 pub mod str_lexer;
+pub mod str_lib;

orchid-std/src/std/string/str_atom.rs

@@ -7,7 +7,7 @@ use async_std::io::Write;
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Encode, Request};
 use orchid_base::error::{OrcRes, mk_errv};
-use orchid_base::format::FmtUnit;
+use orchid_base::format::{FmtCtx, FmtUnit};
 use orchid_base::interner::Tok;
 use orchid_extension::atom::{AtomMethod, Atomic, MethodSetBuilder, Supports, TypAtom};
 use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
@@ -49,7 +49,9 @@ impl OwnedAtom for StrAtom {
 	async fn serialize(&self, _: SysCtx, sink: Pin<&mut (impl Write + ?Sized)>) -> Self::Refs {
 		self.deref().encode(sink).await
 	}
-	async fn print(&self, _: SysCtx) -> FmtUnit { format!("{:?}", &*self.0).into() }
+	async fn print<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		format!("{:?}", &*self.0).into()
+	}
 	async fn deserialize(mut ctx: impl DeserializeCtx, _: Self::Refs) -> Self {
 		Self::new(Rc::new(ctx.read::<String>().await))
 	}
@@ -60,7 +62,6 @@ pub struct IntStrAtom(Tok<String>);
 impl Atomic for IntStrAtom {
 	type Variant = OwnedVariant;
 	type Data = orchid_api::TStr;
-	fn reg_reqs() -> MethodSetBuilder<Self> { MethodSetBuilder::new() }
 }
 impl From<Tok<String>> for IntStrAtom {
 	fn from(value: Tok<String>) -> Self { Self(value) }
@@ -68,7 +69,9 @@ impl From<Tok<String>> for IntStrAtom {
 impl OwnedAtom for IntStrAtom {
 	type Refs = ();
 	async fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.to_api()) }
-	async fn print(&self, _ctx: SysCtx) -> FmtUnit { format!("{:?}i", *self.0).into() }
+	async fn print<'a>(&'a self, _: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
+		format!("{:?}i", *self.0).into()
+	}
 	async fn serialize(&self, _: SysCtx, write: Pin<&mut (impl Write + ?Sized)>) {
 		self.0.encode(write).await
 	}

orchid-std/src/std/string/str_lexer.rs

@@ -94,11 +94,9 @@ fn parse_string(str: &str) -> Result<String, StringError> {
 #[derive(Default)]
 pub struct StringLexer;
 impl Lexer for StringLexer {
-	const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"', '\''..='\''];
+	const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"', '`'..='`'];
 	async fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> {
-		let Some((mut tail, delim)) = (all.strip_prefix('"').map(|t| (t, '"')))
-			.or_else(|| all.strip_prefix('\'').map(|t| (t, '\'')))
-		else {
+		let Some(mut tail) = all.strip_prefix('"') else {
 			return Err(err_not_applicable(ctx.i).await.into());
 		};
 		let mut ret = None;
@@ -125,7 +123,7 @@ impl Lexer for StringLexer {
 			wrap_tokv([concat_fn, prev, new])
 		};
 		loop {
-			if let Some(rest) = tail.strip_prefix(delim) {
+			if let Some(rest) = tail.strip_prefix('"') {
 				return Ok((rest, add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await));
 			} else if let Some(rest) = tail.strip_prefix('$') {
 				ret = Some(add_frag(ret, str_to_gen(&mut cur, tail, &mut errors, ctx).await).await);

orchid-std/src/std/string/str_lib.rs

@@ -0,0 +1,15 @@
+use std::rc::Rc;
+
+use orchid_extension::tree::{GenItem, comments, fun, prefix};
+
+use super::str_atom::StrAtom;
+use crate::OrcString;
+
+pub fn gen_str_lib() -> Vec<GenItem> {
+	prefix("std::string", [comments(
+		["Concatenate two strings"],
+		fun(true, "concat", |left: OrcString<'static>, right: OrcString<'static>| async move {
+			StrAtom::new(Rc::new(left.get_string().await.to_string() + &right.get_string().await))
+		}),
+	)])
+}