All sorts of test scaffolding works now

orchid-base/src/format.rs

@@ -140,7 +140,7 @@ impl Variants {
 					(Some((r, ..)), None) => &s[r.end..],
 					(None, None) => s,
 				};
-				let str_item = FmtElement::String(Rc::new(string.to_string()));
+				let str_item = FmtElement::String(Rc::new(string.replace("{{", "{").replace("}}", "}")));
 				match r {
 					None => itertools::Either::Left([str_item]),
 					Some((_, idx, bounded)) =>

orchid-base/src/logging.rs

@@ -21,7 +21,10 @@ impl Logger {
 	pub fn write_fmt(&self, fmt: Arguments) {
 		match &self.0 {
 			api::LogStrategy::Discard => (),
-			api::LogStrategy::StdErr => stderr().write_fmt(fmt).expect("Could not write to stderr!"),
+			api::LogStrategy::StdErr => {
+				stderr().write_fmt(fmt).expect("Could not write to stderr!");
+				stderr().flush().expect("Could not flush stderr")
+			},
 			api::LogStrategy::File(f) => {
 				let mut file = (File::options().write(true).create(true).truncate(true).open(f))
 					.expect("Could not open logfile");

orchid-base/src/name.rs

@@ -20,137 +20,6 @@ trait_set! {
 	pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
 }
 
-/// A borrowed name fragment which can be empty. See [VPath] for the owned
-/// variant.
-#[derive(Hash, PartialEq, Eq)]
-#[repr(transparent)]
-pub struct PathSlice([Tok<String>]);
-impl PathSlice {
-	/// Create a new [PathSlice]
-	pub fn new(slice: &[Tok<String>]) -> &PathSlice {
-		// SAFETY: This is ok because PathSlice is #[repr(transparent)]
-		unsafe { &*(slice as *const [Tok<String>] as *const PathSlice) }
-	}
-	/// Convert to an owned name fragment
-	pub fn to_vpath(&self) -> VPath { VPath(self.0.to_vec()) }
-	/// Iterate over the tokens
-	pub fn iter(&self) -> impl NameIter + '_ { self.into_iter() }
-	/// Iterate over the segments
-	pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
-		Box::new(self.0.iter().map(|s| s.as_str()))
-	}
-	/// Find the longest shared prefix of this name and another sequence
-	pub fn coprefix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
-		&self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count() as u16]
-	}
-	/// Find the longest shared suffix of this name and another sequence
-	pub fn cosuffix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
-		&self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count() as u16]
-	}
-	/// Remove another
-	pub fn strip_prefix<'a>(&'a self, other: &PathSlice) -> Option<&'a PathSlice> {
-		let shared = self.coprefix(other).len();
-		(shared == other.len()).then_some(PathSlice::new(&self[shared..]))
-	}
-	/// Number of path segments
-	pub fn len(&self) -> u16 { self.0.len().try_into().expect("Too long name!") }
-	pub fn get<I: NameIndex>(&self, index: I) -> Option<&I::Output> { index.get(self) }
-	/// Whether there are any path segments. In other words, whether this is a
-	/// valid name
-	pub fn is_empty(&self) -> bool { self.len() == 0 }
-	/// Obtain a reference to the held slice. With all indexing traits shadowed,
-	/// this is better done explicitly
-	pub fn as_slice(&self) -> &[Tok<String>] { self }
-	/// Global empty path slice
-	pub fn empty() -> &'static Self { PathSlice::new(&[]) }
-}
-impl fmt::Debug for PathSlice {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
-}
-impl fmt::Display for PathSlice {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "{}", self.str_iter().join("::"))
-	}
-}
-impl Borrow<[Tok<String>]> for PathSlice {
-	fn borrow(&self) -> &[Tok<String>] { &self.0 }
-}
-impl<'a> IntoIterator for &'a PathSlice {
-	type IntoIter = Cloned<slice::Iter<'a, Tok<String>>>;
-	type Item = Tok<String>;
-	fn into_iter(self) -> Self::IntoIter { self.0.iter().cloned() }
-}
-
-pub trait NameIndex {
-	type Output: ?Sized;
-	fn get(self, name: &PathSlice) -> Option<&Self::Output>;
-}
-impl<T: NameIndex> Index<T> for PathSlice {
-	type Output = T::Output;
-	fn index(&self, index: T) -> &Self::Output { index.get(self).expect("Index out of bounds") }
-}
-
-mod idx_impls {
-	use std::ops;
-
-	use super::{NameIndex, PathSlice, conv_range};
-	use crate::interner::Tok;
-
-	impl NameIndex for u16 {
-		type Output = Tok<String>;
-		fn get(self, name: &PathSlice) -> Option<&Self::Output> { name.0.get(self as usize) }
-	}
-
-	impl NameIndex for ops::RangeFull {
-		type Output = PathSlice;
-		fn get(self, name: &PathSlice) -> Option<&Self::Output> { Some(name) }
-	}
-
-	macro_rules! impl_range_index_for_pathslice {
-		($range:ident) => {
-			impl ops::Index<ops::$range<u16>> for PathSlice {
-				type Output = Self;
-				fn index(&self, index: ops::$range<u16>) -> &Self::Output {
-					Self::new(&self.0[conv_range::<u16, usize>(index)])
-				}
-			}
-		};
-	}
-
-	impl_range_index_for_pathslice!(RangeFrom);
-	impl_range_index_for_pathslice!(RangeTo);
-	impl_range_index_for_pathslice!(Range);
-	impl_range_index_for_pathslice!(RangeInclusive);
-	impl_range_index_for_pathslice!(RangeToInclusive);
-}
-
-impl Deref for PathSlice {
-	type Target = [Tok<String>];
-
-	fn deref(&self) -> &Self::Target { &self.0 }
-}
-impl Borrow<PathSlice> for [Tok<String>] {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(self) }
-}
-impl<const N: usize> Borrow<PathSlice> for [Tok<String>; N] {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
-}
-impl Borrow<PathSlice> for Vec<Tok<String>> {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
-}
-pub fn conv_bound<T: Into<U> + Clone, U>(bound: Bound<&T>) -> Bound<U> {
-	match bound {
-		Bound::Included(i) => Bound::Included(i.clone().into()),
-		Bound::Excluded(i) => Bound::Excluded(i.clone().into()),
-		Bound::Unbounded => Bound::Unbounded,
-	}
-}
-pub fn conv_range<'a, T: Into<U> + Clone + 'a, U: 'a>(
-	range: impl RangeBounds<T>,
-) -> (Bound<U>, Bound<U>) {
-	(conv_bound(range.start_bound()), conv_bound(range.end_bound()))
-}
-
 /// A token path which may be empty. [VName] is the non-empty,
 /// [PathSlice] is the borrowed version
 #[derive(Clone, Default, Hash, PartialEq, Eq)]
@@ -227,22 +96,19 @@ impl IntoIterator for VPath {
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
 impl Borrow<[Tok<String>]> for VPath {
-	fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
-}
-impl Borrow<PathSlice> for VPath {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
+	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
 }
 impl Deref for VPath {
-	type Target = PathSlice;
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
 impl<T> Index<T> for VPath
-where PathSlice: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <PathSlice as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
-	fn index(&self, index: T) -> &Self::Output { &Borrow::<PathSlice>::borrow(self)[index] }
+	fn index(&self, index: T) -> &Self::Output { &Borrow::<[Tok<String>]>::borrow(self)[index] }
 }
 
 /// A mutable representation of a namespaced identifier of at least one segment.
@@ -311,20 +177,17 @@ impl IntoIterator for VName {
 	fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
 impl<T> Index<T> for VName
-where PathSlice: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <PathSlice as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
 impl Borrow<[Tok<String>]> for VName {
 	fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
 }
-impl Borrow<PathSlice> for VName {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
-}
 impl Deref for VName {
-	type Target = PathSlice;
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -384,20 +247,17 @@ impl fmt::Display for Sym {
 	}
 }
 impl<T> Index<T> for Sym
-where PathSlice: Index<T>
+where [Tok<String>]: Index<T>
 {
-	type Output = <PathSlice as Index<T>>::Output;
+	type Output = <[Tok<String>] as Index<T>>::Output;
 
 	fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
 impl Borrow<[Tok<String>]> for Sym {
 	fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
 }
-impl Borrow<PathSlice> for Sym {
-	fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
-}
 impl Deref for Sym {
-	type Target = PathSlice;
+	type Target = [Tok<String>];
 	fn deref(&self) -> &Self::Target { self.borrow() }
 }
 
@@ -405,10 +265,10 @@ impl Deref for Sym {
 /// handled together in datastructures. The names can never be empty
 #[allow(clippy::len_without_is_empty)] // never empty
 pub trait NameLike:
-	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<PathSlice>
+	'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<[Tok<String>]>
 {
 	/// Convert into held slice
-	fn as_slice(&self) -> &[Tok<String>] { Borrow::<PathSlice>::borrow(self) }
+	fn as_slice(&self) -> &[Tok<String>] { Borrow::<[Tok<String>]>::borrow(self) }
 	/// Get iterator over tokens
 	fn iter(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
 	/// Get iterator over string segments
@@ -425,14 +285,14 @@ pub trait NameLike:
 		NonZeroUsize::try_from(self.iter().count()).expect("NameLike never empty")
 	}
 	/// Like slice's `split_first` except we know that it always returns Some
-	fn split_first(&self) -> (Tok<String>, &PathSlice) {
+	fn split_first(&self) -> (Tok<String>, &[Tok<String>]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
-		(foot.clone(), PathSlice::new(torso))
+		(foot.clone(), torso)
 	}
 	/// Like slice's `split_last` except we know that it always returns Some
-	fn split_last(&self) -> (Tok<String>, &PathSlice) {
+	fn split_last(&self) -> (Tok<String>, &[Tok<String>]) {
 		let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
-		(foot.clone(), PathSlice::new(torso))
+		(foot.clone(), torso)
 	}
 	/// Get the first element
 	fn first(&self) -> Tok<String> { self.split_first().0 }
@@ -498,7 +358,7 @@ mod test {
 
 	use test_executors::spin_on;
 
-	use super::{PathSlice, Sym, VName};
+	use super::{NameLike, Sym, VName};
 	use crate::interner::{Interner, Tok};
 	use crate::name::VPath;
 
@@ -508,8 +368,7 @@ mod test {
 			let i = Interner::new_master();
 			let myname = vname!(foo::bar; i).await;
 			let _borrowed_slice: &[Tok<String>] = myname.borrow();
-			let _borrowed_pathslice: &PathSlice = myname.borrow();
-			let _deref_pathslice: &PathSlice = &myname;
+			let _deref_pathslice: &[Tok<String>] = &myname;
 			let _as_slice_out: &[Tok<String>] = myname.as_slice();
 		})
 	}

orchid-base/src/parse.rs

@@ -10,7 +10,7 @@ use crate::error::{OrcRes, Reporter, mk_err, mk_errv};
 use crate::format::{Format, take_first_fmt};
 use crate::interner::{Interner, Tok};
 use crate::location::Pos;
-use crate::name::VPath;
+use crate::name::{VName, VPath};
 use crate::tree::{AtomRepr, ExtraTok, Paren, TokTree, Token};
 
 pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }

orchid-base/src/reqnot.rs

@@ -1,4 +1,4 @@
-use std::any::Any;
+use std::any::{Any, TypeId};
 use std::cell::RefCell;
 use std::future::Future;
 use std::marker::PhantomData;
@@ -17,8 +17,8 @@ use hashbrown::HashMap;
 use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
 use trait_set::trait_set;
 
-use crate::clone;
 use crate::logging::Logger;
+use crate::{api, clone};
 
 pub struct Receipt<'a>(PhantomData<&'a mut ()>);
 
@@ -204,7 +204,8 @@ impl<T: MsgSet> DynRequester for ReqNot<T> {
 			mem::drop(g);
 			let rn = self.clone();
 			send(&buf, rn).await;
-			RawReply(recv.recv().await.unwrap())
+			let items = recv.recv().await;
+			RawReply(items.unwrap())
 		})
 	}
 }
@@ -222,6 +223,7 @@ pub trait Requester: DynRequester {
 		MappedRequester::new(self, logger)
 	}
 }
+
 impl<This: DynRequester + ?Sized> Requester for This {
 	async fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
 		let req = format!("{data:?}");

orchid-base/src/tree.rs

@@ -1,7 +1,6 @@
 use std::borrow::Borrow;
 use std::fmt::{self, Debug, Display};
 use std::future::Future;
-use std::iter;
 use std::marker::PhantomData;
 use std::ops::Range;
 use std::rc::Rc;
@@ -20,7 +19,7 @@ use crate::error::OrcErrv;
 use crate::format::{FmtCtx, FmtUnit, Format, Variants};
 use crate::interner::{Interner, Tok};
 use crate::location::Pos;
-use crate::name::PathSlice;
+use crate::name::Sym;
 use crate::parse::Snippet;
 use crate::{api, match_mapping, tl_cache};
 
@@ -39,6 +38,7 @@ pub fn recur<'a, A: AtomRepr, X: ExtraTok>(
 		let tok = match tok {
 			tok @ (Token::Atom(_) | Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::NS) => tok,
 			tok @ (Token::Name(_) | Token::Slot(_) | Token::X(_) | Token::Ph(_) | Token::Macro(_)) => tok,
+			tok @ Token::Reference(_) => tok,
 			Token::LambdaHead(arg) =>
 				Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
 			Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
@@ -87,7 +87,8 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 			Comment(c.clone()),
 			Slot(id => TokHandle::new(*id)),
 			Ph(ph => Ph::from_api(ph, i).await),
-			Macro(*prio)
+			Macro(*prio),
+			Reference(tok => Sym::from_api(*tok, i).await)
 		});
 		Self { range: tt.range.clone(), tok }
 	}
@@ -105,6 +106,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 			S(*p, b => ttv_to_api(b, do_extra).boxed_local().await),
 			Ph(ph.to_api()),
 			Macro(*prio),
+			Reference(sym.to_api()),
 		} {
 			Token::X(x) => return do_extra(x, self.range.clone()).await
 		});
@@ -117,6 +119,7 @@ impl<'b, A: AtomRepr, X: ExtraTok> TokTree<'b, A, X> {
 	) -> api::TokenTree {
 		let token = match self.tok {
 			Token::Atom(a) => api::Token::Atom(a.to_api().await),
+			Token::Reference(sym) => api::Token::Reference(sym.to_api()),
 			Token::BR => api::Token::BR,
 			Token::NS => api::Token::NS,
 			Token::Bottom(e) => api::Token::Bottom(e.to_api()),
@@ -191,18 +194,6 @@ pub async fn ttv_into_api<'a, A: AtomRepr, X: ExtraTok>(
 	.await
 }
 
-/// This takes a position and not a range because it assigns the range to
-/// multiple leaf tokens, which is only valid if it's a zero-width range
-pub fn vname_tv<'a: 'b, 'b, A: AtomRepr + 'a, X: ExtraTok + 'a>(
-	name: &'b PathSlice,
-	pos: u32,
-) -> impl Iterator<Item = TokTree<'a, A, X>> + 'b {
-	let (head, tail) = name.split_first().expect("Empty vname");
-	iter::once(Token::Name(head.clone()))
-		.chain(tail.iter().flat_map(|t| [Token::NS, Token::Name(t.clone())]))
-		.map(move |t| t.at(pos..pos))
-}
-
 pub fn wrap_tokv<'a, A: AtomRepr, X: ExtraTok>(
 	items: impl IntoIterator<Item = TokTree<'a, A, X>>,
 ) -> TokTree<'a, A, X> {
@@ -219,19 +210,43 @@ pub fn wrap_tokv<'a, A: AtomRepr, X: ExtraTok>(
 
 pub use api::Paren;
 
+/// Lexer output variant
 #[derive(Clone, Debug)]
 pub enum Token<'a, A: AtomRepr, X: ExtraTok> {
+	/// Information about the code addressed to the human reader or dev tooling
+	/// It has no effect on the behaviour of the program unless it's explicitly
+	/// read via reflection
 	Comment(Arc<String>),
+	/// The part of a lambda between `\` and `.` enclosing the argument. The body
+	/// stretches to the end of the enclosing parens or the end of the const line
 	LambdaHead(Vec<TokTree<'a, A, X>>),
+	/// A binding, operator, or a segment of a namespaced::name
 	Name(Tok<String>),
+	/// The namespace separator ::
 	NS,
+	/// A line break
 	BR,
+	/// `()`, `[]`, or `{}`
 	S(Paren, Vec<TokTree<'a, A, X>>),
+	/// A fully formed reference to external code emitted by a lexer plugin
+	Reference(Sym),
+	/// A value emitted by a lexer plugin
 	Atom(A),
+	/// A grammar error emitted by a lexer plugin if it was possible to continue
+	/// reading. Parsers should treat it as an atom unless it prevents parsing,
+	/// in which case both this and a relevant error should be returned.
 	Bottom(OrcErrv),
+	/// An instruction from a plugin for the lexer to embed a subexpression
+	/// without retransmitting it. It should not appear anywhere outside lexer
+	/// plugin responses.
 	Slot(TokHandle<'a>),
+	/// Additional domain-specific token types
 	X(X),
+	/// A placeholder for metaprogramming, either $name, ..$name, ..$name:N,
+	/// ...$name, or ...$name:N
 	Ph(Ph),
+	/// `macro` or `macro(`X`)` where X is any valid floating point number
+	/// expression. `macro` is not a valid name in Orchid for this reason.
 	Macro(Option<NotNan<f64>>),
 }
 impl<'a, A: AtomRepr, X: ExtraTok> Token<'a, A, X> {
@@ -258,6 +273,7 @@ impl<A: AtomRepr, X: ExtraTok> Format for Token<'_, A, X> {
 				]),
 			Self::NS => "::".to_string().into(),
 			Self::Name(n) => format!("{n}").into(),
+			Self::Reference(sym) => format!("{sym}").into(),
 			Self::Slot(th) => format!("{th}").into(),
 			Self::Ph(ph) => format!("{ph}").into(),
 			Self::S(p, b) => FmtUnit::new(