in midst of refactor

orchid-base/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "orchid-base"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+derive_destructure = "1.0.0"
+dyn-clone = "1.0.17"
+hashbrown = "0.14.3"
+itertools = "0.12.1"
+lazy_static = "1.4.0"
+never = "0.1.0"
+orchid-api = { version = "0.1.0", path = "../orchid-api" }
+orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
+orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
+ordered-float = "4.2.0"
+rust-embed = "8.3.0"
+substack = "1.1.0"
+trait-set = "0.3.0"

orchid-base/src/boxed_iter.rs

@@ -0,0 +1,23 @@
+/// Utility functions to get rid of tedious explicit casts to
+/// BoxedIter
+use std::iter;
+
+/// A trait object of [Iterator] to be assigned to variables that may be
+/// initialized form multiple iterators of incompatible types
+pub type BoxedIter<'a, T> = Box<dyn Iterator<Item = T> + 'a>;
+/// creates a [BoxedIter] of a single element
+pub fn box_once<'a, T: 'a>(t: T) -> BoxedIter<'a, T> { Box::new(iter::once(t)) }
+/// creates an empty [BoxedIter]
+pub fn box_empty<'a, T: 'a>() -> BoxedIter<'a, T> { Box::new(iter::empty()) }
+
+/// Chain various iterators into a [BoxedIter]
+macro_rules! box_chain {
+  ($curr:expr) => {
+    Box::new($curr) as BoxedIter<_>
+  };
+  ($curr:expr, $($rest:expr),*) => {
+    Box::new($curr$(.chain($rest))*) as $crate::utils::boxed_iter::BoxedIter<_>
+  };
+}
+
+pub(crate) use box_chain;

orchid-base/src/child.rs

@@ -0,0 +1,44 @@
+use std::io::{self, Read, Write};
+use std::sync::Mutex;
+use std::{mem, process};
+
+pub struct SharedChild {
+  child: process::Child,
+  stdin: Mutex<process::ChildStdin>,
+  stdout: Mutex<process::ChildStdout>,
+}
+impl SharedChild {
+  pub fn new(cmd: &mut process::Command) -> io::Result<Self> {
+    let mut child = cmd.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
+    let stdin = Mutex::new(child.stdin.take().expect("Piped stdin above"));
+    let stdout = Mutex::new(child.stdout.take().expect("Piped stdout above"));
+    Ok(Self { stdin, stdout, child })
+  }
+
+  pub fn send_msg(&self, msg: &[u8]) -> io::Result<()> {
+    send_msg(&mut *self.stdin.lock().unwrap(), msg)
+  }
+
+  pub fn recv_msg(&self) -> io::Result<Vec<u8>> { recv_msg(&mut *self.stdout.lock().unwrap()) }
+}
+impl Drop for SharedChild {
+  fn drop(&mut self) { mem::drop(self.child.kill()) }
+}
+
+pub fn send_msg(write: &mut impl Write, msg: &[u8]) -> io::Result<()> {
+  write.write_all(&(u32::try_from(msg.len()).unwrap()).to_be_bytes())?;
+  write.write_all(msg)?;
+  write.flush()
+}
+
+pub fn recv_msg(read: &mut impl Read) -> io::Result<Vec<u8>> {
+  let mut len = [0u8; 4];
+  read.read_exact(&mut len)?;
+  let len = u32::from_be_bytes(len);
+  let mut msg = vec![0u8; len as usize];
+  read.read_exact(&mut msg)?;
+  Ok(msg)
+}
+
+pub fn send_parent_msg(msg: &[u8]) -> io::Result<()> { send_msg(&mut io::stdout().lock(), msg) }
+pub fn recv_parent_msg() -> io::Result<Vec<u8>> { recv_msg(&mut io::stdin().lock()) }

orchid-base/src/clone.rs

@@ -0,0 +1,9 @@
+#[macro_export]
+macro_rules! clone {
+  ($($n:ident),+; $body:expr) => (
+    {
+      $( let $n = $n.clone(); )+
+      $body
+    }
+  );
+}

orchid-base/src/combine.rs

@@ -0,0 +1,24 @@
+//! The concept of a fallible merger
+
+use never::Never;
+
+/// Fallible, type-preserving variant of [std::ops::Add] implemented by a
+/// variety of types for different purposes. Very broadly, if the operation
+/// succeeds, the result should represent _both_ inputs.
+pub trait Combine: Sized {
+  /// Information about the failure
+  type Error;
+
+  /// Merge two values into a value that represents both, if this is possible.
+  fn combine(self, other: Self) -> Result<Self, Self::Error>;
+}
+
+impl Combine for Never {
+  type Error = Never;
+  fn combine(self, _: Self) -> Result<Self, Self::Error> { match self {} }
+}
+
+impl Combine for () {
+  type Error = Never;
+  fn combine(self, (): Self) -> Result<Self, Self::Error> { Ok(()) }
+}

orchid-base/src/event.rs

@@ -0,0 +1,63 @@
+//! Multiple-listener-single-delivery event system.
+
+use std::mem;
+use std::sync::mpsc::{self, sync_channel};
+use std::sync::Mutex;
+
+struct Reply<T, U> {
+  resub: bool,
+  outcome: Result<U, T>,
+}
+
+struct Listener<T, E> {
+  sink: mpsc::SyncSender<T>,
+  source: mpsc::Receiver<Reply<T, E>>,
+}
+
+pub struct Event<T, U> {
+  listeners: Mutex<Vec<Listener<T, U>>>,
+}
+impl<T, U> Event<T, U> {
+  pub const fn new() -> Self { Self { listeners: Mutex::new(Vec::new()) } }
+
+  pub fn dispatch(&self, mut ev: T) -> Option<U> {
+    let mut listeners = self.listeners.lock().unwrap();
+    let mut alt_list = Vec::with_capacity(listeners.len());
+    mem::swap(&mut *listeners, &mut alt_list);
+    let mut items = alt_list.into_iter();
+    while let Some(l) = items.next() {
+      l.sink.send(ev).unwrap();
+      let Reply { resub, outcome } = l.source.recv().unwrap();
+      if resub {
+        listeners.push(l);
+      }
+      match outcome {
+        Ok(res) => {
+          listeners.extend(items);
+          return Some(res);
+        },
+        Err(next) => {
+          ev = next;
+        },
+      }
+    }
+    None
+  }
+
+  pub fn get_one<V>(&self, mut filter: impl FnMut(&T) -> bool, f: impl FnOnce(T) -> (U, V)) -> V {
+    let mut listeners = self.listeners.lock().unwrap();
+    let (sink, request) = sync_channel(0);
+    let (response, source) = sync_channel(0);
+    listeners.push(Listener { sink, source });
+    mem::drop(listeners);
+    loop {
+      let t = request.recv().unwrap();
+      if filter(&t) {
+        let (u, v) = f(t);
+        response.send(Reply { resub: false, outcome: Ok(u) }).unwrap();
+        return v;
+      }
+      response.send(Reply { resub: true, outcome: Err(t) }).unwrap();
+    }
+  }
+}

orchid-base/src/gen/impls.rs

@@ -0,0 +1,67 @@
+use std::any::TypeId;
+
+use itertools::Itertools;
+use orchid_api::expr::{Clause, Expr};
+use orchid_api::location::Location;
+
+use super::traits::{GenClause, Generable};
+use crate::expr::RtExpr;
+use crate::host::AtomHand;
+use crate::intern::{deintern, intern};
+
+fn safely_reinterpret<In: 'static, Out: 'static>(x: In) -> Result<Out, In> {
+  if TypeId::of::<In>() != TypeId::of::<Out>() {
+    return Err(x);
+  }
+  let bx = Box::new(x);
+  // SAFETY: type sameness asserted above, from_raw and into_raw pair up
+  Ok(*unsafe { Box::from_raw(Box::into_raw(bx) as *mut Out) })
+}
+
+/// impls of the gen traits for external types
+
+impl GenClause for Expr {
+  fn generate<T: super::traits::Generable>(&self, ctx: T::Ctx<'_>, pop: &impl Fn() -> T) -> T {
+    match &self.clause {
+      Clause::Arg(arg) => T::arg(ctx, deintern(*arg).as_str()),
+      Clause::Atom(atom) => T::atom(ctx, AtomHand::from_api(atom.clone())),
+      Clause::Call(f, x) => T::apply(ctx, |c| f.generate(c, pop), |c| x.generate(c, pop)),
+      Clause::Lambda(arg, b) => T::lambda(ctx, deintern(*arg).as_str(), |ctx| b.generate(ctx, pop)),
+      Clause::Seq(n1, n2) => T::seq(ctx, |c| n1.generate(c, pop), |c| n2.generate(c, pop)),
+      Clause::Const(int) => T::constant(ctx, deintern(*int).iter().map(|t| t.as_str())),
+      Clause::Slot(expr) => {
+        let rte = RtExpr::resolve(*expr).expect("expired ticket");
+        safely_reinterpret(rte).expect("ticket slots make no sense for anything other than rte")
+      },
+    }
+  }
+}
+
+fn to_expr(clause: Clause) -> Expr { Expr { clause, location: Location::None } }
+
+impl Generable for Expr {
+  type Ctx<'a> = ();
+
+  fn arg(_ctx: Self::Ctx<'_>, name: &str) -> Self { to_expr(Clause::Arg(intern(name).marker())) }
+  fn apply(
+    ctx: Self::Ctx<'_>,
+    f: impl FnOnce(Self::Ctx<'_>) -> Self,
+    x: impl FnOnce(Self::Ctx<'_>) -> Self,
+  ) -> Self {
+    to_expr(Clause::Call(Box::new(f(ctx)), Box::new(x(ctx))))
+  }
+  fn atom(_ctx: Self::Ctx<'_>, a: crate::host::AtomHand) -> Self { to_expr(Clause::Atom(a.api_ref())) }
+  fn constant<'a>(_ctx: Self::Ctx<'_>, name: impl IntoIterator<Item = &'a str>) -> Self {
+    to_expr(Clause::Const(intern(&name.into_iter().map(intern).collect_vec()[..]).marker()))
+  }
+  fn lambda(ctx: Self::Ctx<'_>, name: &str, body: impl FnOnce(Self::Ctx<'_>) -> Self) -> Self {
+    to_expr(Clause::Lambda(intern(name).marker(), Box::new(body(ctx))))
+  }
+  fn seq(
+    ctx: Self::Ctx<'_>,
+    a: impl FnOnce(Self::Ctx<'_>) -> Self,
+    b: impl FnOnce(Self::Ctx<'_>) -> Self,
+  ) -> Self {
+    to_expr(Clause::Seq(Box::new(a(ctx)), Box::new(b(ctx))))
+  }
+}

orchid-base/src/gen/mod.rs

@@ -0,0 +1,8 @@
+//! Abstractions and primitives for defining Orchid code in compile-time Rust
+//! constants. This is used both to generate glue code such as function call
+//! expressions at runtime and to define completely static intrinsics and
+//! constants accessible to usercode.
+pub mod tpl;
+pub mod traits;
+pub mod tree;
+pub mod impls;

orchid-base/src/gen/tpl.rs

@@ -0,0 +1,61 @@
+//! Various elemental components to build expression trees that all implement
+//! [GenClause].
+
+use super::traits::{GenClause, Generable};
+use crate::host::AtomHand;
+
+/// A trivial atom
+#[derive(Clone, Debug)]
+pub struct SysAtom(pub AtomHand);
+impl GenClause for SysAtom {
+  fn generate<T: Generable>(&self, ctx: T::Ctx<'_>, _: &impl Fn() -> T) -> T {
+    T::atom(ctx, self.0.clone())
+  }
+}
+
+/// Const, Reference a constant from the execution environment. Unlike Orchid
+/// syntax, this doesn't include lambda arguments. For that, use [P]
+#[derive(Debug, Clone)]
+pub struct C(pub &'static str);
+impl GenClause for C {
+  fn generate<T: Generable>(&self, ctx: T::Ctx<'_>, _: &impl Fn() -> T) -> T {
+    T::constant(ctx, self.0.split("::"))
+  }
+}
+
+/// Apply a function to a value provided by [L]
+#[derive(Debug, Clone)]
+pub struct A<F: GenClause, X: GenClause>(pub F, pub X);
+impl<F: GenClause, X: GenClause> GenClause for A<F, X> {
+  fn generate<T: Generable>(&self, ctx: T::Ctx<'_>, p: &impl Fn() -> T) -> T {
+    T::apply(ctx, |gen| self.0.generate(gen, p), |gen| self.1.generate(gen, p))
+  }
+}
+
+/// Apply a function to two arguments
+pub fn a2(f: impl GenClause, x: impl GenClause, y: impl GenClause) -> impl GenClause {
+  A(A(f, x), y)
+}
+
+/// Lambda expression. The argument can be referenced with [P]
+#[derive(Debug, Clone)]
+pub struct L<B: GenClause>(pub &'static str, pub B);
+impl<B: GenClause> GenClause for L<B> {
+  fn generate<T: Generable>(&self, ctx: T::Ctx<'_>, p: &impl Fn() -> T) -> T {
+    T::lambda(ctx, self.0, |gen| self.1.generate(gen, p))
+  }
+}
+
+/// Parameter to a lambda expression
+#[derive(Debug, Clone)]
+pub struct P(pub &'static str);
+impl GenClause for P {
+  fn generate<T: Generable>(&self, ctx: T::Ctx<'_>, _: &impl Fn() -> T) -> T { T::arg(ctx, self.0) }
+}
+
+/// Slot for an Orchid value to be specified during execution
+#[derive(Debug, Clone)]
+pub struct Slot;
+impl GenClause for Slot {
+  fn generate<T: Generable>(&self, _: T::Ctx<'_>, pop: &impl Fn() -> T) -> T { pop() }
+}

orchid-base/src/gen/traits.rs

@@ -0,0 +1,78 @@
+//! Abstractions used to generate Orchid expressions
+
+use std::backtrace::Backtrace;
+use std::cell::RefCell;
+use std::collections::VecDeque;
+use std::fmt;
+
+use crate::host::AtomHand;
+
+/// Representations of the Orchid expression tree that can describe basic
+/// language elements.
+pub trait Generable: Sized + 'static {
+  /// Context information defined by parents. Generators just forward this.
+  type Ctx<'a>: Sized;
+  /// Wrap external data.
+  fn atom(ctx: Self::Ctx<'_>, a: AtomHand) -> Self;
+  /// Generate a reference to a constant
+  fn constant<'a>(ctx: Self::Ctx<'_>, name: impl IntoIterator<Item = &'a str>) -> Self;
+  /// Generate a function call given the function and its argument
+  fn apply(
+    ctx: Self::Ctx<'_>,
+    f: impl FnOnce(Self::Ctx<'_>) -> Self,
+    x: impl FnOnce(Self::Ctx<'_>) -> Self,
+  ) -> Self;
+  /// Generate a dependency between the top clause and the bottom clause
+  fn seq(
+    ctx: Self::Ctx<'_>,
+    a: impl FnOnce(Self::Ctx<'_>) -> Self,
+    b: impl FnOnce(Self::Ctx<'_>) -> Self,
+  ) -> Self;
+  /// Generate a function. The argument name is only valid within the same
+  /// [Generable].
+  fn lambda(ctx: Self::Ctx<'_>, name: &str, body: impl FnOnce(Self::Ctx<'_>) -> Self) -> Self;
+
+  /// Generate a reference to a function argument. The argument name is only
+  /// valid within the same [Generable].
+  fn arg(ctx: Self::Ctx<'_>, name: &str) -> Self;
+}
+
+/// Expression templates which can be instantiated in multiple representations
+/// of Orchid. Expressions can be built from the elements defined in
+/// [super::tpl].
+///
+/// Do not depend on this trait, use [Gen] instead. Conversely, implement this
+/// instead of [Gen].
+pub trait GenClause: fmt::Debug + Sized {
+  /// Enact the template at runtime to build a given type.
+  /// `pop` pops from the runtime template parameter list passed to the
+  /// generator.
+  ///
+  /// Do not call this, it's the backing operation of [Gen#template]
+  fn generate<T: Generable>(&self, ctx: T::Ctx<'_>, pop: &impl Fn() -> T) -> T;
+}
+
+/// Expression generators
+///
+/// Do not implement this trait, it's the frontend for [GenClause]. Conversely,
+/// do not consume [GenClause].
+pub trait Gen<T: Generable, U>: fmt::Debug {
+  /// Create an instance of this template with some parameters
+  fn template(&self, ctx: T::Ctx<'_>, params: U) -> T;
+}
+
+impl<T: Generable, I: IntoIterator<Item = T>, G: GenClause> Gen<T, I> for G {
+  fn template(&self, ctx: T::Ctx<'_>, params: I) -> T {
+    let values = RefCell::new(params.into_iter().collect::<VecDeque<_>>());
+    let t =
+      self.generate(ctx, &|| values.borrow_mut().pop_front().expect("Not enough values provided"));
+    let leftover = values.borrow().len();
+    assert_eq!(
+      leftover,
+      0,
+      "Too many values provided ({leftover} left) {}",
+      Backtrace::force_capture()
+    );
+    t
+  }
+}

orchid-base/src/gen/tree.rs

@@ -0,0 +1,79 @@
+//! Components to build in-memory module trees that in Orchid. These modules
+//! can only contain constants and other modules.
+
+use std::fmt;
+
+use dyn_clone::{clone_box, DynClone};
+use orchid_api::expr::Expr;
+use trait_set::trait_set;
+
+use super::tpl;
+use super::traits::{Gen, GenClause};
+use crate::combine::Combine;
+use crate::host::AtomHand;
+use crate::tree::{ModEntry, ModMember, TreeConflict};
+
+trait_set! {
+  trait TreeLeaf = Gen<Expr, [Expr; 0]> + DynClone + Send;
+}
+
+/// A leaf in the [ConstTree]
+pub struct GenConst(Box<dyn TreeLeaf>);
+impl GenConst {
+  fn c(data: impl GenClause + Send + Clone + 'static) -> Self { Self(Box::new(data)) }
+}
+impl fmt::Debug for GenConst {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self.0) }
+}
+impl Clone for GenConst {
+  fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
+}
+
+/// Error condition when constant trees that define the the same constant are
+/// merged. Produced during system loading if multiple modules define the
+/// same constant
+#[derive(Debug, Clone)]
+pub struct ConflictingConsts;
+
+impl Combine for GenConst {
+  type Error = ConflictingConsts;
+  fn combine(self, _: Self) -> Result<Self, Self::Error> { Err(ConflictingConsts) }
+}
+
+/// A lightweight module tree that can be built declaratively by hand to
+/// describe libraries of external functions in Rust. It implements [Combine]
+/// for merging libraries.
+pub type ConstTree = ModEntry<GenConst, (), ()>;
+
+/// Describe a constant
+#[must_use]
+pub fn leaf(value: impl GenClause + Clone + Send + 'static) -> ConstTree {
+  ModEntry::wrap(ModMember::Item(GenConst::c(value)))
+}
+
+/// Describe a constant which appears in [ConstTree::tree].
+///
+/// The unarray tricks rustfmt into keeping this call as a single line even if
+/// it chooses to break the argument into a block.
+pub fn ent<K: AsRef<str>>(
+  key: K,
+  [g]: [impl GenClause + Clone + Send + 'static; 1],
+) -> (K, ConstTree) {
+  (key, leaf(g))
+}
+
+/// Describe an [Atomic]
+#[must_use]
+pub fn atom_leaf(atom: AtomHand) -> ConstTree { leaf(tpl::SysAtom(atom)) }
+
+/// Describe an [Atomic] which appears as an entry in a [ConstTree::tree]
+///
+/// The unarray is used to trick rustfmt into breaking the atom into a block
+/// without breaking this call into a block
+#[must_use]
+pub fn atom_ent<K: AsRef<str>>(key: K, [atom]: [AtomHand; 1]) -> (K, ConstTree) {
+  (key, atom_leaf(atom))
+}
+
+/// Errors produced duriung the merger of constant trees
+pub type ConstCombineErr = TreeConflict<GenConst, (), ()>;

orchid-base/src/handler.rs

@@ -0,0 +1,116 @@
+//! Impure functions that can be triggered by Orchid code when a command
+//! evaluates to an atom representing a command
+
+use std::any::{Any, TypeId};
+use std::cell::RefCell;
+
+use hashbrown::HashMap;
+use trait_set::trait_set;
+
+use super::nort::Expr;
+use crate::foreign::atom::Atomic;
+use crate::foreign::error::RTResult;
+use crate::foreign::to_clause::ToClause;
+use crate::location::CodeLocation;
+
+trait_set! {
+  trait Handler = for<'a> Fn(&'a dyn Any, CodeLocation) -> Expr;
+}
+
+enum HTEntry<'a> {
+  Handler(Box<dyn Handler + 'a>),
+  Forward(&'a (dyn Handler + 'a)),
+}
+impl<'a> AsRef<dyn Handler + 'a> for HTEntry<'a> {
+  fn as_ref(&self) -> &(dyn Handler + 'a) {
+    match self {
+      HTEntry::Handler(h) => &**h,
+      HTEntry::Forward(h) => *h,
+    }
+  }
+}
+
+/// A table of impure command handlers exposed to Orchid
+#[derive(Default)]
+pub struct HandlerTable<'a> {
+  handlers: HashMap<TypeId, HTEntry<'a>>,
+}
+impl<'a> HandlerTable<'a> {
+  /// Create a new [HandlerTable]
+  #[must_use]
+  pub fn new() -> Self { Self { handlers: HashMap::new() } }
+
+  /// Add a handler function to interpret a command and select the continuation.
+  /// See [HandlerTable#with] for a declarative option.
+  pub fn register<T: 'static, R: ToClause>(&mut self, f: impl for<'b> FnMut(&'b T) -> R + 'a) {
+    let cell = RefCell::new(f);
+    let cb = move |a: &dyn Any, loc: CodeLocation| {
+      cell.borrow_mut()(a.downcast_ref().expect("found by TypeId")).to_expr(loc)
+    };
+    let prev = self.handlers.insert(TypeId::of::<T>(), HTEntry::Handler(Box::new(cb)));
+    assert!(prev.is_none(), "A handler for this type is already registered");
+  }
+
+  /// Add a handler function to interpret a command and select the continuation.
+  /// See [HandlerTable#register] for a procedural option.
+  pub fn with<T: 'static>(mut self, f: impl FnMut(&T) -> RTResult<Expr> + 'a) -> Self {
+    self.register(f);
+    self
+  }
+
+  /// Find and execute the corresponding handler for this type
+  pub fn dispatch(&self, arg: &dyn Atomic, loc: CodeLocation) -> Option<Expr> {
+    (self.handlers.get(&arg.as_any_ref().type_id())).map(|ent| ent.as_ref()(arg.as_any_ref(), loc))
+  }
+
+  /// Combine two non-overlapping handler sets
+  #[must_use]
+  pub fn combine(mut self, other: Self) -> Self {
+    for (key, value) in other.handlers {
+      let prev = self.handlers.insert(key, value);
+      assert!(prev.is_none(), "Duplicate handlers")
+    }
+    self
+  }
+
+  /// Add entries that forward requests to a borrowed non-overlapping handler
+  /// set
+  pub fn link<'b: 'a>(mut self, other: &'b HandlerTable<'b>) -> Self {
+    for (key, value) in other.handlers.iter() {
+      let prev = self.handlers.insert(*key, HTEntry::Forward(value.as_ref()));
+      assert!(prev.is_none(), "Duplicate handlers")
+    }
+    self
+  }
+}
+
+#[cfg(test)]
+#[allow(unconditional_recursion)]
+#[allow(clippy::ptr_arg)]
+mod test {
+  use std::marker::PhantomData;
+
+  use super::HandlerTable;
+
+  /// Ensure that the method I use to verify covariance actually passes with
+  /// covariant and fails with invariant
+  ///
+  /// The failing case:
+  /// ```
+  /// struct Cov2<'a>(PhantomData<&'a mut &'a ()>);
+  /// fn fail<'a>(_c: &Cov2<'a>, _s: &'a String) { fail(_c, &String::new()) }
+  /// ```
+  #[allow(unused)]
+  fn covariant_control() {
+    struct Cov<'a>(PhantomData<&'a ()>);
+    fn pass<'a>(_c: &Cov<'a>, _s: &'a String) { pass(_c, &String::new()) }
+  }
+
+  /// The &mut ensures that 'a in the two functions must be disjoint, and that
+  /// ht must outlive both. For this to compile, Rust has to cast ht to the
+  /// shorter lifetimes, ensuring covariance
+  #[allow(unused)]
+  fn assert_covariant() {
+    fn pass<'a>(_ht: HandlerTable<'a>, _s: &'a String) { pass(_ht, &String::new()) }
+  }
+}

orchid-base/src/intern.rs

@@ -0,0 +1,303 @@
+use std::borrow::Borrow;
+use std::hash::BuildHasher as _;
+use std::num::NonZeroU64;
+use std::ops::{Deref, DerefMut};
+use std::sync::{atomic, Arc, Mutex, MutexGuard};
+use std::{fmt, hash};
+
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools as _;
+use orchid_api::intern::{
+  ExternStr, ExternStrv, IntReq, InternStr, InternStrv, Retained, TStr, TStrv,
+};
+use orchid_api_traits::Request;
+
+use crate::reqnot::{DynRequester, Requester};
+
+#[derive(Clone)]
+pub struct Token<T: ?Sized + Interned> {
+  data: Arc<T>,
+  marker: T::Marker,
+}
+impl<T: Interned + ?Sized> Token<T> {
+  pub fn marker(&self) -> T::Marker { self.marker }
+  pub fn arc(&self) -> Arc<T> { self.data.clone() }
+}
+impl<T: Interned + ?Sized> Deref for Token<T> {
+  type Target = T;
+
+  fn deref(&self) -> &Self::Target { self.data.as_ref() }
+}
+impl<T: Interned + ?Sized> Ord for Token<T> {
+  fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.marker().cmp(&other.marker()) }
+}
+impl<T: Interned + ?Sized> PartialOrd for Token<T> {
+  fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
+}
+impl<T: Interned + ?Sized> Eq for Token<T> {}
+impl<T: Interned + ?Sized> PartialEq for Token<T> {
+  fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
+}
+impl<T: Interned + ?Sized> hash::Hash for Token<T> {
+  fn hash<H: hash::Hasher>(&self, state: &mut H) { self.marker().hash(state) }
+}
+impl<T: Interned + ?Sized + fmt::Display> fmt::Display for Token<T> {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    write!(f, "{}", &*self.data)
+  }
+}
+impl<T: Interned + ?Sized + fmt::Debug> fmt::Debug for Token<T> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Token({} -> {:?})", self.marker().get_id(), self.data.as_ref())
+  }
+}
+
+pub trait Interned: Eq + hash::Hash + Clone {
+  type Marker: InternMarker<Interned = Self>;
+  fn intern(self: Arc<Self>, req: &(impl DynRequester<Transfer = IntReq> + ?Sized))
+  -> Self::Marker;
+  fn bimap(interner: &mut Interner) -> &mut Bimap<Self>;
+}
+
+pub trait Internable {
+  type Interned: Interned;
+  fn get_owned(&self) -> Arc<Self::Interned>;
+}
+
+pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash {
+  type Interned: Interned<Marker = Self>;
+  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Token<Self::Interned>;
+  fn get_id(self) -> NonZeroU64;
+  fn from_id(id: NonZeroU64) -> Self;
+}
+
+impl Interned for String {
+  type Marker = TStr;
+  fn intern(
+    self: Arc<Self>,
+    req: &(impl DynRequester<Transfer = IntReq> + ?Sized),
+  ) -> Self::Marker {
+    req.request(InternStr(self))
+  }
+  fn bimap(interner: &mut Interner) -> &mut Bimap<Self> { &mut interner.strings }
+}
+
+impl InternMarker for TStr {
+  type Interned = String;
+  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Token<Self::Interned> {
+    Token { marker: self, data: req.request(ExternStr(self)) }
+  }
+  fn get_id(self) -> NonZeroU64 { self.0 }
+  fn from_id(id: NonZeroU64) -> Self { Self(id) }
+}
+
+impl Internable for str {
+  type Interned = String;
+  fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_string()) }
+}
+
+impl Internable for String {
+  type Interned = String;
+  fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_string()) }
+}
+
+impl Interned for Vec<Token<String>> {
+  type Marker = TStrv;
+  fn intern(
+    self: Arc<Self>,
+    req: &(impl DynRequester<Transfer = IntReq> + ?Sized),
+  ) -> Self::Marker {
+    req.request(InternStrv(Arc::new(self.iter().map(|t| t.marker()).collect())))
+  }
+  fn bimap(interner: &mut Interner) -> &mut Bimap<Self> { &mut interner.vecs }
+}
+
+impl InternMarker for TStrv {
+  type Interned = Vec<Token<String>>;
+  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Token<Self::Interned> {
+    let data = Arc::new(req.request(ExternStrv(self)).iter().map(|m| deintern(*m)).collect_vec());
+    Token { marker: self, data }
+  }
+  fn get_id(self) -> NonZeroU64 { self.0 }
+  fn from_id(id: NonZeroU64) -> Self { Self(id) }
+}
+
+impl Internable for [Token<String>] {
+  type Interned = Vec<Token<String>>;
+  fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_vec()) }
+}
+
+impl Internable for Vec<Token<String>> {
+  type Interned = Vec<Token<String>>;
+  fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_vec()) }
+}
+
+impl Internable for Vec<TStr> {
+  type Interned = Vec<Token<String>>;
+  fn get_owned(&self) -> Arc<Self::Interned> {
+    Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
+  }
+}
+
+impl Internable for [TStr] {
+  type Interned = Vec<Token<String>>;
+  fn get_owned(&self) -> Arc<Self::Interned> {
+    Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
+  }
+}
+
+/// The number of references held to any token by the interner.
+const BASE_RC: usize = 3;
+
+#[test]
+fn base_rc_correct() {
+  let tok = Token { marker: TStr(1.try_into().unwrap()), data: Arc::new("foo".to_string()) };
+  let mut bimap = Bimap::default();
+  bimap.insert(tok.clone());
+  assert_eq!(Arc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the current instance");
+}
+
+pub struct Bimap<T: Interned + ?Sized> {
+  intern: HashMap<Arc<T>, Token<T>>,
+  by_id: HashMap<T::Marker, Token<T>>,
+}
+impl<T: Interned + ?Sized> Bimap<T> {
+  pub fn insert(&mut self, token: Token<T>) {
+    self.intern.insert(token.data.clone(), token.clone());
+    self.by_id.insert(token.marker(), token);
+  }
+
+  pub fn by_marker(&self, marker: T::Marker) -> Option<Token<T>> {
+    self.by_id.get(&marker).cloned()
+  }
+
+  pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Token<T>>
+  where T: Borrow<Q> {
+    (self.intern.raw_entry())
+      .from_hash(self.intern.hasher().hash_one(q), |k| k.as_ref().borrow() == q)
+      .map(|p| p.1.clone())
+  }
+
+  pub fn sweep_replica(&mut self) -> Vec<T::Marker> {
+    (self.intern)
+      .extract_if(|k, _| Arc::strong_count(k) == BASE_RC)
+      .map(|(_, v)| {
+        self.by_id.remove(&v.marker());
+        v.marker()
+      })
+      .collect()
+  }
+
+  pub fn sweep_master(&mut self, retained: HashSet<T::Marker>) {
+    self.intern.retain(|k, v| BASE_RC < Arc::strong_count(k) || retained.contains(&v.marker()))
+  }
+}
+
+impl<T: Interned + ?Sized> Default for Bimap<T> {
+  fn default() -> Self { Self { by_id: HashMap::new(), intern: HashMap::new() } }
+}
+
+pub trait UpComm {
+  fn up<R: Request>(&self, req: R) -> R::Response;
+}
+
+#[derive(Default)]
+pub struct Interner {
+  strings: Bimap<String>,
+  vecs: Bimap<Vec<Token<String>>>,
+  master: Option<Box<dyn DynRequester<Transfer = IntReq>>>,
+}
+
+static ID: atomic::AtomicU64 = atomic::AtomicU64::new(1);
+static INTERNER: Mutex<Option<Interner>> = Mutex::new(None);
+
+pub fn interner() -> impl DerefMut<Target = Interner> {
+  struct G(MutexGuard<'static, Option<Interner>>);
+  impl Deref for G {
+    type Target = Interner;
+    fn deref(&self) -> &Self::Target { self.0.as_ref().expect("Guard pre-initialized") }
+  }
+  impl DerefMut for G {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+      self.0.as_mut().expect("Guard pre-iniitialized")
+    }
+  }
+  let mut g = INTERNER.lock().unwrap();
+  g.get_or_insert_with(Interner::default);
+  G(g)
+}
+
+pub fn init_replica(req: impl DynRequester<Transfer = IntReq> + 'static) {
+  let mut g = INTERNER.lock().unwrap();
+  assert!(g.is_none(), "Attempted to initialize replica interner after first use");
+  *g = Some(Interner {
+    strings: Bimap::default(),
+    vecs: Bimap::default(),
+    master: Some(Box::new(req)),
+  })
+}
+
+pub fn intern<T: Interned>(t: &(impl Internable<Interned = T> + ?Sized)) -> Token<T> {
+  let mut g = interner();
+  let data = t.get_owned();
+  let marker = (g.master.as_mut()).map_or_else(
+    || T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
+    |c| data.clone().intern(&**c),
+  );
+  let tok = Token { marker, data };
+  T::bimap(&mut g).insert(tok.clone());
+  tok
+}
+
+pub fn deintern<M: InternMarker>(marker: M) -> Token<M::Interned> {
+  let mut g = interner();
+  if let Some(tok) = M::Interned::bimap(&mut g).by_marker(marker) {
+    return tok;
+  }
+  let master = g.master.as_mut().expect("ID not in local interner and this is master");
+  let token = marker.resolve(&**master);
+  M::Interned::bimap(&mut g).insert(token.clone());
+  token
+}
+
+pub fn merge_retained(into: &mut Retained, from: &Retained) {
+  into.strings = into.strings.iter().chain(&from.strings).copied().unique().collect();
+  into.vecs = into.vecs.iter().chain(&from.vecs).copied().unique().collect();
+}
+
+pub fn sweep_replica() -> Retained {
+  let mut g = interner();
+  assert!(g.master.is_some(), "Not a replica");
+  Retained { strings: g.strings.sweep_replica(), vecs: g.vecs.sweep_replica() }
+}
+
+pub fn sweep_master(retained: Retained) {
+  let mut g = interner();
+  assert!(g.master.is_none(), "Not master");
+  g.strings.sweep_master(retained.strings.into_iter().collect());
+  g.vecs.sweep_master(retained.vecs.into_iter().collect());
+}
+
+/// Create a thread-local token instance and copy it. This ensures that the
+/// interner will only be called the first time the expresion is executed,
+/// and subsequent calls will just copy the token. Accepts a single static
+/// expression (i.e. a literal).
+#[macro_export]
+macro_rules! intern {
+  ($ty:ty : $expr:expr) => {{
+    thread_local! {
+      static VALUE: $crate::intern::Token<<$ty as $crate::intern::Internable>::Interned>
+        = $crate::intern::intern($expr as &$ty);
+    }
+    VALUE.with(|v| v.clone())
+  }};
+}
+
+#[allow(unused)]
+fn test_i() {
+  let _: Token<String> = intern!(str: "foo");
+  let _: Token<Vec<Token<String>>> = intern!([Token<String>]: &[
+    intern!(str: "bar"),
+    intern!(str: "baz")
+  ]);
+}

orchid-base/src/join.rs

@@ -0,0 +1,34 @@
+//! Join hashmaps with a callback for merging or failing on conflicting keys.
+
+use std::hash::Hash;
+
+use hashbrown::HashMap;
+use never::Never;
+
+/// Combine two hashmaps via an infallible value merger. See also
+/// [try_join_maps]
+pub fn join_maps<K: Eq + Hash, V>(
+  left: HashMap<K, V>,
+  right: HashMap<K, V>,
+  mut merge: impl FnMut(&K, V, V) -> V,
+) -> HashMap<K, V> {
+  try_join_maps(left, right, |k, l, r| Ok(merge(k, l, r))).unwrap_or_else(|e: Never| match e {})
+}
+
+/// Combine two hashmaps via a fallible value merger. See also [join_maps]
+pub fn try_join_maps<K: Eq + Hash, V, E>(
+  left: HashMap<K, V>,
+  mut right: HashMap<K, V>,
+  mut merge: impl FnMut(&K, V, V) -> Result<V, E>,
+) -> Result<HashMap<K, V>, E> {
+  let mut mixed = HashMap::with_capacity(left.len() + right.len());
+  for (key, lval) in left {
+    let val = match right.remove(&key) {
+      None => lval,
+      Some(rval) => merge(&key, lval, rval)?,
+    };
+    mixed.insert(key, val);
+  }
+  mixed.extend(right);
+  Ok(mixed)
+}

orchid-base/src/lib.rs

@@ -0,0 +1,17 @@
+pub mod boxed_iter;
+pub mod child;
+pub mod clone;
+pub mod combine;
+pub mod event;
+pub mod expr;
+pub mod gen;
+pub mod intern;
+pub mod location;
+pub mod name;
+pub mod proj_error;
+pub mod reqnot;
+pub mod tree;
+pub mod virt_fs;
+pub mod join;
+pub mod sequence;
+pub mod api_utils;

orchid-base/src/location.rs

@@ -0,0 +1,188 @@
+//! Structures that show where code or semantic elements came from
+
+use std::fmt;
+use std::hash::Hash;
+use std::ops::Range;
+use std::sync::Arc;
+
+use itertools::Itertools;
+
+use crate::name::Sym;
+use crate::sym;
+
+/// A full source code unit, such as a source file
+#[derive(Clone, Eq)]
+pub struct SourceCode {
+  pub(crate) path: Sym,
+  pub(crate) text: Arc<String>,
+}
+impl SourceCode {
+  /// Create a new source file description
+  pub fn new(path: Sym, source: Arc<String>) -> Self { Self { path, text: source } }
+  /// Location the source code was loaded from in the virtual tree
+  pub fn path(&self) -> Sym { self.path.clone() }
+  /// Raw source code string
+  pub fn text(&self) -> Arc<String> { self.text.clone() }
+}
+impl PartialEq for SourceCode {
+  fn eq(&self, other: &Self) -> bool { self.path == other.path }
+}
+impl Hash for SourceCode {
+  fn hash<H: std::hash::Hasher>(&self, state: &mut H) { self.path.hash(state) }
+}
+impl fmt::Debug for SourceCode {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeInfo({self})") }
+}
+impl fmt::Display for SourceCode {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}.orc", self.path.str_iter().join("/"))
+  }
+}
+impl AsRef<str> for SourceCode {
+  fn as_ref(&self) -> &str { &self.text }
+}
+
+/// Exact source code location. Includes where the code was loaded from, what
+/// the original source code was, and a byte range.
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct SourceRange {
+  pub(crate) code: SourceCode,
+  pub(crate) range: Range<usize>,
+}
+impl SourceRange {
+  /// Create a new instance.
+  pub fn new(range: Range<usize>, code: SourceCode) -> Self { Self { code, range } }
+  /// Create a dud [SourceRange] for testing. Its value is unspecified and
+  /// volatile.
+  pub fn mock() -> Self { Self::new(0..1, SourceCode::new(sym!(test), Arc::new(String::new()))) }
+  /// Source code
+  pub fn code(&self) -> SourceCode { self.code.clone() }
+  /// Source text
+  pub fn text(&self) -> Arc<String> { self.code.text.clone() }
+  /// Path the source text was loaded from
+  pub fn path(&self) -> Sym { self.code.path.clone() }
+  /// Byte range
+  pub fn range(&self) -> Range<usize> { self.range.clone() }
+  /// 0-based index of first byte
+  pub fn start(&self) -> usize { self.range.start }
+  /// 0-based index of last byte + 1
+  pub fn end(&self) -> usize { self.range.end }
+  /// Syntactic location
+  pub fn origin(&self) -> CodeOrigin { CodeOrigin::Source(self.clone()) }
+  /// Transform the numeric byte range
+  pub fn map_range(&self, map: impl FnOnce(Range<usize>) -> Range<usize>) -> Self {
+    Self::new(map(self.range()), self.code())
+  }
+}
+impl fmt::Debug for SourceRange {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeRange({self})") }
+}
+impl fmt::Display for SourceRange {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    let Self { code, range } = self;
+    let (sl, sc) = pos2lc(code.text.as_str(), range.start);
+    let (el, ec) = pos2lc(code.text.as_str(), range.end);
+    write!(f, "{code} {sl}:{sc}")?;
+    if el == sl {
+      if sc + 1 == ec { Ok(()) } else { write!(f, "..{ec}") }
+    } else {
+      write!(f, "..{el}:{ec}")
+    }
+  }
+}
+
+/// Information about a code generator attached to the generated code
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct CodeGenInfo {
+  /// formatted like a Rust namespace
+  pub generator: Sym,
+  /// Unformatted user message with relevant circumstances and parameters
+  pub details: Arc<String>,
+}
+impl CodeGenInfo {
+  /// A codegen marker with no user message and parameters
+  pub fn no_details(generator: Sym) -> Self { Self { generator, details: Arc::new(String::new()) } }
+  /// A codegen marker with a user message or parameters
+  pub fn details(generator: Sym, details: impl AsRef<str>) -> Self {
+    Self { generator, details: Arc::new(details.as_ref().to_string()) }
+  }
+}
+impl fmt::Debug for CodeGenInfo {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeGenInfo({self})") }
+}
+impl fmt::Display for CodeGenInfo {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "generated by {}", self.generator)?;
+    if !self.details.is_empty() { write!(f, ", details: {}", self.details) } else { write!(f, ".") }
+  }
+}
+
+/// identifies a sequence of characters that contributed to the enclosing
+/// construct or the reason the code was generated for generated code.
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub enum CodeOrigin {
+  /// Character sequence
+  Source(SourceRange),
+  /// Generated construct
+  Gen(CodeGenInfo),
+}
+
+impl fmt::Display for CodeOrigin {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Gen(info) => write!(f, "{info}"),
+      Self::Source(cr) => write!(f, "{cr}"),
+    }
+  }
+}
+
+impl fmt::Debug for CodeOrigin {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeOrigin({self})") }
+}
+
+/// Location data associated with any code fragment. Identifies where the code
+/// came from and where it resides in the tree.
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct CodeLocation {
+  pub(crate) origin: CodeOrigin,
+  pub(crate) module: Sym,
+}
+impl CodeLocation {
+  pub(crate) fn new_src(range: SourceRange, module: Sym) -> Self {
+    Self { origin: CodeOrigin::Source(range), module }
+  }
+  /// Create a location for generated code. The generator string must not be
+  /// empty. For code, the generator string must contain at least one `::`
+  pub fn new_gen(gen: CodeGenInfo) -> Self {
+    Self { module: gen.generator.clone(), origin: CodeOrigin::Gen(gen) }
+  }
+
+  /// Get the syntactic location
+  pub fn origin(&self) -> CodeOrigin { self.origin.clone() }
+  /// Get the name of the containing module
+  pub fn module(&self) -> Sym { self.module.clone() }
+}
+
+impl fmt::Display for CodeLocation {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    if self.module[..].is_empty() {
+      write!(f, "global {}", self.origin)
+    } else {
+      write!(f, "{} in {}", self.origin, self.module)
+    }
+  }
+}
+
+impl fmt::Debug for CodeLocation {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeLocation({self})") }
+}
+
+#[must_use]
+fn pos2lc(s: &str, i: usize) -> (usize, usize) {
+  s.chars().take(i).fold(
+    (1, 1),
+    |(line, col), char| {
+      if char == '\n' { (line + 1, 1) } else { (line, col + 1) }
+    },
+  )
+}

orchid-base/src/name.rs

@@ -0,0 +1,488 @@
+//! Various datatypes that all represent namespaced names.
+
+use std::borrow::Borrow;
+use std::hash::Hash;
+use std::iter::Cloned;
+use std::num::{NonZeroU64, NonZeroUsize};
+use std::ops::{Deref, Index};
+use std::path::Path;
+use std::{fmt, slice, vec};
+
+use itertools::Itertools;
+use trait_set::trait_set;
+
+use crate::intern::{intern, InternMarker, Token};
+
+trait_set! {
+  /// Traits that all name iterators should implement
+  pub trait NameIter = Iterator<Item = Token<String>> + DoubleEndedIterator + ExactSizeIterator;
+}
+
+/// A borrowed name fragment which can be empty. See [VPath] for the owned
+/// variant.
+#[derive(Hash, PartialEq, Eq)]
+#[repr(transparent)]
+pub struct PathSlice([Token<String>]);
+impl PathSlice {
+  /// Create a new [PathSlice]
+  pub fn new(slice: &[Token<String>]) -> &PathSlice {
+    // SAFETY: This is ok because PathSlice is #[repr(transparent)]
+    unsafe { &*(slice as *const [Token<String>] as *const PathSlice) }
+  }
+  /// Convert to an owned name fragment
+  pub fn to_vpath(&self) -> VPath { VPath(self.0.to_vec()) }
+  /// Iterate over the tokens
+  pub fn iter(&self) -> impl NameIter + '_ { self.into_iter() }
+  /// Iterate over the segments
+  pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
+    Box::new(self.0.iter().map(|s| s.as_str()))
+  }
+  /// Find the longest shared prefix of this name and another sequence
+  pub fn coprefix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
+    &self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count()]
+  }
+  /// Find the longest shared suffix of this name and another sequence
+  pub fn cosuffix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
+    &self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count()]
+  }
+  /// Remove another
+  pub fn strip_prefix<'a>(&'a self, other: &PathSlice) -> Option<&'a PathSlice> {
+    let shared = self.coprefix(other).len();
+    (shared == other.len()).then_some(PathSlice::new(&self[shared..]))
+  }
+  /// Number of path segments
+  pub fn len(&self) -> usize { self.0.len() }
+  /// Whether there are any path segments. In other words, whether this is a
+  /// valid name
+  pub fn is_empty(&self) -> bool { self.len() == 0 }
+  /// Obtain a reference to the held slice. With all indexing traits shadowed,
+  /// this is better done explicitly
+  pub fn as_slice(&self) -> &[Token<String>] { self }
+  /// Global empty path slice
+  pub fn empty() -> &'static Self { PathSlice::new(&[]) }
+}
+impl fmt::Debug for PathSlice {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
+}
+impl fmt::Display for PathSlice {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.str_iter().join("::"))
+  }
+}
+impl Borrow<[Token<String>]> for PathSlice {
+  fn borrow(&self) -> &[Token<String>] { &self.0 }
+}
+impl<'a> IntoIterator for &'a PathSlice {
+  type IntoIter = Cloned<slice::Iter<'a, Token<String>>>;
+  type Item = Token<String>;
+  fn into_iter(self) -> Self::IntoIter { self.0.iter().cloned() }
+}
+
+mod idx_impls {
+  use std::ops;
+
+  use super::PathSlice;
+  use crate::intern::Token;
+
+  impl ops::Index<usize> for PathSlice {
+    type Output = Token<String>;
+    fn index(&self, index: usize) -> &Self::Output { &self.0[index] }
+  }
+  macro_rules! impl_range_index_for_pathslice {
+    ($range:ty) => {
+      impl ops::Index<$range> for PathSlice {
+        type Output = Self;
+        fn index(&self, index: $range) -> &Self::Output { Self::new(&self.0[index]) }
+      }
+    };
+  }
+
+  impl_range_index_for_pathslice!(ops::RangeFull);
+  impl_range_index_for_pathslice!(ops::RangeFrom<usize>);
+  impl_range_index_for_pathslice!(ops::RangeTo<usize>);
+  impl_range_index_for_pathslice!(ops::Range<usize>);
+  impl_range_index_for_pathslice!(ops::RangeInclusive<usize>);
+  impl_range_index_for_pathslice!(ops::RangeToInclusive<usize>);
+}
+
+impl Deref for PathSlice {
+  type Target = [Token<String>];
+
+  fn deref(&self) -> &Self::Target { &self.0 }
+}
+impl Borrow<PathSlice> for [Token<String>] {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(self) }
+}
+impl<const N: usize> Borrow<PathSlice> for [Token<String>; N] {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
+}
+impl Borrow<PathSlice> for Vec<Token<String>> {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
+}
+
+/// A token path which may be empty. [VName] is the non-empty,
+/// [PathSlice] is the borrowed version
+#[derive(Clone, Default, Hash, PartialEq, Eq)]
+pub struct VPath(pub Vec<Token<String>>);
+impl VPath {
+  /// Collect segments into a vector
+  pub fn new(items: impl IntoIterator<Item = Token<String>>) -> Self {
+    Self(items.into_iter().collect())
+  }
+  /// Number of path segments
+  pub fn len(&self) -> usize { self.0.len() }
+  /// Whether there are any path segments. In other words, whether this is a
+  /// valid name
+  pub fn is_empty(&self) -> bool { self.len() == 0 }
+  /// Prepend some tokens to the path
+  pub fn prefix(self, items: impl IntoIterator<Item = Token<String>>) -> Self {
+    Self(items.into_iter().chain(self.0).collect())
+  }
+  /// Append some tokens to the path
+  pub fn suffix(self, items: impl IntoIterator<Item = Token<String>>) -> Self {
+    Self(self.0.into_iter().chain(items).collect())
+  }
+  /// Partition the string by `::` namespace separators
+  pub fn parse(s: &str) -> Self {
+    Self(if s.is_empty() { vec![] } else { s.split("::").map(intern).collect() })
+  }
+  /// Walk over the segments
+  pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
+    Box::new(self.0.iter().map(|s| s.as_str()))
+  }
+  /// Try to convert into non-empty version
+  pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
+  /// Add a token to the path. Since now we know that it can't be empty, turn it
+  /// into a name.
+  pub fn name_with_prefix(self, name: Token<String>) -> VName {
+    VName(self.into_iter().chain([name]).collect())
+  }
+  /// Add a token to the beginning of the. Since now we know that it can't be
+  /// empty, turn it into a name.
+  pub fn name_with_suffix(self, name: Token<String>) -> VName {
+    VName([name].into_iter().chain(self).collect())
+  }
+
+  /// Convert a fs path to a vpath
+  pub fn from_path(path: &Path) -> Option<(Self, bool)> {
+    let to_vpath =
+      |p: &Path| p.iter().map(|c| c.to_str().map(intern)).collect::<Option<_>>().map(VPath);
+    match path.extension().map(|s| s.to_str()) {
+      Some(Some("orc")) => Some((to_vpath(&path.with_extension(""))?, true)),
+      None => Some((to_vpath(path)?, false)),
+      Some(_) => None,
+    }
+  }
+}
+impl fmt::Debug for VPath {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
+}
+impl fmt::Display for VPath {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.str_iter().join("::"))
+  }
+}
+impl FromIterator<Token<String>> for VPath {
+  fn from_iter<T: IntoIterator<Item = Token<String>>>(iter: T) -> Self {
+    Self(iter.into_iter().collect())
+  }
+}
+impl IntoIterator for VPath {
+  type Item = Token<String>;
+  type IntoIter = vec::IntoIter<Self::Item>;
+  fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
+}
+impl Borrow<[Token<String>]> for VPath {
+  fn borrow(&self) -> &[Token<String>] { self.0.borrow() }
+}
+impl Borrow<PathSlice> for VPath {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
+}
+impl Deref for VPath {
+  type Target = PathSlice;
+  fn deref(&self) -> &Self::Target { self.borrow() }
+}
+
+impl<T> Index<T> for VPath
+where PathSlice: Index<T>
+{
+  type Output = <PathSlice as Index<T>>::Output;
+
+  fn index(&self, index: T) -> &Self::Output { &Borrow::<PathSlice>::borrow(self)[index] }
+}
+
+/// A mutable representation of a namespaced identifier of at least one segment.
+///
+/// These names may be relative or otherwise partially processed.
+///
+/// See also [Sym] for the immutable representation, and [VPath] for possibly
+/// empty values
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub struct VName(Vec<Token<String>>);
+impl VName {
+  /// Assert that the sequence isn't empty and wrap it in [VName] to represent
+  /// this invariant
+  pub fn new(items: impl IntoIterator<Item = Token<String>>) -> Result<Self, EmptyNameError> {
+    let data: Vec<_> = items.into_iter().collect();
+    if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
+  }
+  /// Unwrap the enclosed vector
+  pub fn into_vec(self) -> Vec<Token<String>> { self.0 }
+  /// Get a reference to the enclosed vector
+  pub fn vec(&self) -> &Vec<Token<String>> { &self.0 }
+  /// Mutable access to the underlying vector. To ensure correct results, this
+  /// must never be empty.
+  pub fn vec_mut(&mut self) -> &mut Vec<Token<String>> { &mut self.0 }
+  /// Intern the name and return a [Sym]
+  pub fn to_sym(&self) -> Sym { Sym(intern(&self.0[..])) }
+  /// If this name has only one segment, return it
+  pub fn as_root(&self) -> Option<Token<String>> { self.0.iter().exactly_one().ok().cloned() }
+  /// Prepend the segments to this name
+  #[must_use = "This is a pure function"]
+  pub fn prefix(self, items: impl IntoIterator<Item = Token<String>>) -> Self {
+    Self(items.into_iter().chain(self.0).collect())
+  }
+  /// Append the segments to this name
+  #[must_use = "This is a pure function"]
+  pub fn suffix(self, items: impl IntoIterator<Item = Token<String>>) -> Self {
+    Self(self.0.into_iter().chain(items).collect())
+  }
+  /// Read a `::` separated namespaced name
+  pub fn parse(s: &str) -> Result<Self, EmptyNameError> { Self::new(VPath::parse(s)) }
+  /// Obtain an iterator over the segments of the name
+  pub fn iter(&self) -> impl Iterator<Item = Token<String>> + '_ { self.0.iter().cloned() }
+}
+impl fmt::Debug for VName {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
+}
+impl fmt::Display for VName {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.str_iter().join("::"))
+  }
+}
+impl IntoIterator for VName {
+  type Item = Token<String>;
+  type IntoIter = vec::IntoIter<Self::Item>;
+  fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
+}
+impl<T> Index<T> for VName
+where PathSlice: Index<T>
+{
+  type Output = <PathSlice as Index<T>>::Output;
+
+  fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
+}
+impl Borrow<[Token<String>]> for VName {
+  fn borrow(&self) -> &[Token<String>] { self.0.borrow() }
+}
+impl Borrow<PathSlice> for VName {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
+}
+impl Deref for VName {
+  type Target = PathSlice;
+  fn deref(&self) -> &Self::Target { self.borrow() }
+}
+
+/// Error produced when a non-empty name [VName] or [Sym] is constructed with an
+/// empty sequence
+#[derive(Debug, Copy, Clone, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct EmptyNameError;
+impl TryFrom<&[Token<String>]> for VName {
+  type Error = EmptyNameError;
+  fn try_from(value: &[Token<String>]) -> Result<Self, Self::Error> {
+    Self::new(value.iter().cloned())
+  }
+}
+
+/// An interned representation of a namespaced identifier.
+///
+/// These names are always absolute.
+///
+/// See also [VName]
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub struct Sym(Token<Vec<Token<String>>>);
+impl Sym {
+  /// Assert that the sequence isn't empty, intern it and wrap it in a [Sym] to
+  /// represent this invariant
+  pub fn new(v: impl IntoIterator<Item = Token<String>>) -> Result<Self, EmptyNameError> {
+    let items = v.into_iter().collect_vec();
+    Self::from_tok(intern(&items[..]))
+  }
+  /// Read a `::` separated namespaced name.
+  pub fn parse(s: &str) -> Result<Self, EmptyNameError> {
+    Ok(Sym(intern(&VName::parse(s)?.into_vec()[..])))
+  }
+  /// Assert that a token isn't empty, and wrap it in a [Sym]
+  pub fn from_tok(t: Token<Vec<Token<String>>>) -> Result<Self, EmptyNameError> {
+    if t.is_empty() { Err(EmptyNameError) } else { Ok(Self(t)) }
+  }
+  /// Grab the interner token
+  pub fn tok(&self) -> Token<Vec<Token<String>>> { self.0.clone() }
+  /// Get a number unique to this name suitable for arbitrary ordering.
+  pub fn id(&self) -> NonZeroU64 { self.0.marker().get_id() }
+  /// Extern the sym for editing
+  pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
+}
+impl fmt::Debug for Sym {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Sym({self})") }
+}
+impl fmt::Display for Sym {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.str_iter().join("::"))
+  }
+}
+impl<T> Index<T> for Sym
+where PathSlice: Index<T>
+{
+  type Output = <PathSlice as Index<T>>::Output;
+
+  fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
+}
+impl Borrow<[Token<String>]> for Sym {
+  fn borrow(&self) -> &[Token<String>] { &self.0[..] }
+}
+impl Borrow<PathSlice> for Sym {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
+}
+impl Deref for Sym {
+  type Target = PathSlice;
+  fn deref(&self) -> &Self::Target { self.borrow() }
+}
+
+/// An abstraction over tokenized vs non-tokenized names so that they can be
+/// handled together in datastructures. The names can never be empty
+#[allow(clippy::len_without_is_empty)] // never empty
+pub trait NameLike:
+  'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<PathSlice>
+{
+  /// Convert into held slice
+  fn as_slice(&self) -> &[Token<String>] { Borrow::<PathSlice>::borrow(self) }
+  /// Get iterator over tokens
+  fn iter(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
+  /// Get iterator over string segments
+  fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ {
+    self.as_slice().iter().map(|t| t.as_str())
+  }
+  /// Fully resolve the name for printing
+  #[must_use]
+  fn to_strv(&self) -> Vec<String> { self.iter().map(|s| s.to_string()).collect() }
+  /// Format the name as an approximate filename
+  fn as_src_path(&self) -> String { format!("{}.orc", self.iter().join("/")) }
+  /// Return the number of segments in the name
+  fn len(&self) -> NonZeroUsize {
+    NonZeroUsize::try_from(self.iter().count()).expect("NameLike never empty")
+  }
+  /// Like slice's `split_first` except we know that it always returns Some
+  fn split_first(&self) -> (Token<String>, &PathSlice) {
+    let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
+    (foot.clone(), PathSlice::new(torso))
+  }
+  /// Like slice's `split_last` except we know that it always returns Some
+  fn split_last(&self) -> (Token<String>, &PathSlice) {
+    let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
+    (foot.clone(), PathSlice::new(torso))
+  }
+  /// Get the first element
+  fn first(&self) -> Token<String> { self.split_first().0 }
+  /// Get the last element
+  fn last(&self) -> Token<String> { self.split_last().0 }
+}
+
+impl NameLike for Sym {}
+impl NameLike for VName {}
+
+/// Create a [Sym] literal.
+///
+/// Both the name and its components will be cached in a thread-local static so
+/// that subsequent executions of the expression only incur an Arc-clone for
+/// cloning the token.
+#[macro_export]
+macro_rules! sym {
+  ($seg1:tt $( :: $seg:tt)*) => {
+    $crate::name::Sym::from_tok($crate::intern!([$crate::intern::Token<String>]: &[
+      $crate::intern!(str: stringify!($seg1))
+      $( , $crate::intern!(str: stringify!($seg)) )*
+    ])).unwrap()
+  };
+  (@NAME $seg:tt) => {}
+}
+
+/// Create a [VName] literal.
+///
+/// The components are interned much like in [sym].
+#[macro_export]
+macro_rules! vname {
+  ($seg1:tt $( :: $seg:tt)*) => {
+    $crate::name::VName::new([
+      $crate::intern!(str: stringify!($seg1))
+      $( , $crate::intern!(str: stringify!($seg)) )*
+    ]).unwrap()
+  };
+}
+
+/// Create a [VPath] literal.
+///
+/// The components are interned much like in [sym].
+#[macro_export]
+macro_rules! vpath {
+  ($seg1:tt $( :: $seg:tt)+) => {
+    $crate::name::VPath(vec![
+      $crate::intern!(str: stringify!($seg1))
+      $( , $crate::intern!(str: stringify!($seg)) )+
+    ])
+  };
+  () => {
+    $crate::name::VPath(vec![])
+  }
+}
+
+/// Create a &[PathSlice] literal.
+///
+/// The components are interned much like in [sym]
+#[macro_export]
+macro_rules! path_slice {
+  ($seg1:tt $( :: $seg:tt)+) => {
+    $crate::name::PathSlice::new(&[
+      $crate::intern!(str: stringify!($seg1))
+      $( , $crate::intern!(str: stringify!($seg)) )+
+    ])
+  };
+  () => {
+    $crate::name::PathSlice::new(&[])
+  }
+}
+
+#[cfg(test)]
+mod test {
+  use std::borrow::Borrow;
+
+  use super::{PathSlice, Sym, VName};
+  use crate::intern::{intern, Token};
+  use crate::name::VPath;
+
+  #[test]
+  fn recur() {
+    let myname = vname!(foo::bar);
+    let _borrowed_slice: &[Token<String>] = myname.borrow();
+    let _borrowed_pathslice: &PathSlice = myname.borrow();
+    let _deref_pathslice: &PathSlice = &myname;
+    let _as_slice_out: &[Token<String>] = myname.as_slice();
+  }
+
+  #[test]
+  fn literals() {
+    assert_eq!(
+      sym!(foo::bar::baz),
+      Sym::new([intern("foo"), intern("bar"), intern("baz")]).unwrap()
+    );
+    assert_eq!(
+      vname!(foo::bar::baz),
+      VName::new([intern("foo"), intern("bar"), intern("baz")]).unwrap()
+    );
+    assert_eq!(vpath!(foo::bar::baz), VPath::new([intern("foo"), intern("bar"), intern("baz")]));
+    assert_eq!(
+      path_slice!(foo::bar::baz),
+      PathSlice::new(&[intern("foo"), intern("bar"), intern("baz")])
+    );
+  }
+}

orchid-base/src/nort.rs

@@ -0,0 +1,283 @@
+//! The NORT (Normal Order Referencing Tree) is the interpreter's runtime
+//! representation of Orchid programs.
+//!
+//! It uses a locator tree to find bound variables in lambda functions, which
+//! necessitates a normal reduction order because modifying the body by reducing
+//! expressions would invalidate any locators in enclosing lambdas.
+//!
+//! Clauses are held in a mutable `Arc<Mutex<_>>`, so that after substitution
+//! the instances of the argument remain linked and a reduction step applied to
+//! any instance transforms all of them.
+//!
+//! To improve locality and make the tree less deep and locators shorter,
+//! function calls store multiple arguments in a deque.
+
+use std::collections::VecDeque;
+use std::fmt;
+use std::ops::DerefMut;
+use std::sync::{Arc, Mutex, MutexGuard, TryLockError};
+
+use itertools::Itertools;
+
+use super::path_set::PathSet;
+use crate::foreign::atom::Atom;
+#[allow(unused)] // for doc
+use crate::foreign::atom::Atomic;
+use crate::foreign::error::{RTErrorObj, RTResult};
+use crate::foreign::try_from_expr::TryFromExpr;
+use crate::location::CodeLocation;
+use crate::name::Sym;
+#[allow(unused)] // for doc
+use crate::parse::parsed;
+use crate::utils::ddispatch::request;
+
+/// Kinda like [AsMut] except it supports a guard
+pub(crate) trait AsDerefMut<T> {
+  fn as_deref_mut(&mut self) -> impl DerefMut<Target = T> + '_;
+}
+
+/// An expression with metadata
+#[derive(Clone)]
+pub struct Expr {
+  /// The actual value
+  pub clause: ClauseInst,
+  /// Information about the code that produced this value
+  pub location: CodeLocation,
+}
+impl Expr {
+  /// Constructor
+  pub fn new(clause: ClauseInst, location: CodeLocation) -> Self { Self { clause, location } }
+  /// Obtain the location of the expression
+  pub fn location(&self) -> CodeLocation { self.location.clone() }
+
+  /// Convert into any type that implements [TryFromExpr]. Calls to this
+  /// function are generated wherever a conversion is elided in an extern
+  /// function.
+  pub fn downcast<T: TryFromExpr>(self) -> RTResult<T> {
+    let Expr { mut clause, location } = self;
+    loop {
+      let cls_deref = clause.cls_mut();
+      match &*cls_deref {
+        Clause::Identity(alt) => {
+          let temp = alt.clone();
+          drop(cls_deref);
+          clause = temp;
+        },
+        _ => {
+          drop(cls_deref);
+          return T::from_expr(Expr { clause, location });
+        },
+      };
+    }
+  }
+
+  /// Visit all expressions in the tree. The search can be exited early by
+  /// returning [Some]
+  ///
+  /// See also [parsed::Expr::search_all]
+  pub fn search_all<T>(&self, predicate: &mut impl FnMut(&Self) -> Option<T>) -> Option<T> {
+    if let Some(t) = predicate(self) {
+      return Some(t);
+    }
+    self.clause.inspect(|c| match c {
+      Clause::Identity(_alt) => unreachable!("Handled by inspect"),
+      Clause::Apply { f, x } =>
+        (f.search_all(predicate)).or_else(|| x.iter().find_map(|x| x.search_all(predicate))),
+      Clause::Lambda { body, .. } => body.search_all(predicate),
+      Clause::Constant(_) | Clause::LambdaArg | Clause::Atom(_) | Clause::Bottom(_) => None,
+    })
+  }
+
+  /// Clone the refcounted [ClauseInst] out of the expression
+  #[must_use]
+  pub fn clsi(&self) -> ClauseInst { self.clause.clone() }
+
+  /// Read-Write access to the [Clause]
+  ///
+  /// # Panics
+  ///
+  /// if the clause is already borrowed
+  pub fn cls_mut(&self) -> MutexGuard<'_, Clause> { self.clause.cls_mut() }
+}
+
+impl fmt::Debug for Expr {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{:?}@{}", self.clause, self.location)
+  }
+}
+
+impl fmt::Display for Expr {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.clause) }
+}
+
+impl AsDerefMut<Clause> for Expr {
+  fn as_deref_mut(&mut self) -> impl DerefMut<Target = Clause> + '_ { self.clause.cls_mut() }
+}
+
+/// A wrapper around expressions to handle their multiple occurences in
+/// the tree together
+#[derive(Clone)]
+pub struct ClauseInst(pub Arc<Mutex<Clause>>);
+impl ClauseInst {
+  /// Wrap a [Clause] in a shared container so that normalization steps are
+  /// applied to all references
+  #[must_use]
+  pub fn new(cls: Clause) -> Self { Self(Arc::new(Mutex::new(cls))) }
+
+  /// Take the [Clause] out of this container if it's the last reference to it,
+  /// or return self.
+  pub fn try_unwrap(self) -> Result<Clause, ClauseInst> {
+    Arc::try_unwrap(self.0).map(|c| c.into_inner().unwrap()).map_err(Self)
+  }
+
+  /// Read-Write access to the shared clause instance
+  ///
+  /// if the clause is already borrowed, this will block until it is released.
+  pub fn cls_mut(&self) -> MutexGuard<'_, Clause> { self.0.lock().unwrap() }
+
+  /// Call a predicate on the clause, returning whatever the
+  /// predicate returns. This is a convenience function for reaching
+  /// through the [Mutex]. The clause will never be [Clause::Identity].
+  #[must_use]
+  pub fn inspect<T>(&self, predicate: impl FnOnce(&Clause) -> T) -> T {
+    match &*self.cls_mut() {
+      Clause::Identity(sub) => sub.inspect(predicate),
+      x => predicate(x),
+    }
+  }
+
+  /// If this expression is an [Atomic], request an object of the given type.
+  /// If it's not an atomic, fail the request automatically.
+  #[must_use = "your request might not have succeeded"]
+  pub fn request<T: 'static>(&self) -> Option<T> {
+    match &*self.cls_mut() {
+      Clause::Atom(a) => request(&*a.0),
+      Clause::Identity(alt) => alt.request(),
+      _ => None,
+    }
+  }
+
+  /// Associate a location with this clause
+  pub fn into_expr(self, location: CodeLocation) -> Expr {
+    Expr { clause: self.clone(), location: location.clone() }
+  }
+  /// Check ahead-of-time if this clause contains an atom. Calls
+  /// [ClauseInst#cls] for read access.
+  ///
+  /// Since atoms cannot become normalizable, if this is true and previous
+  /// normalization failed, the atom is known to be in normal form.
+  pub fn is_atom(&self) -> bool { matches!(&*self.cls_mut(), Clause::Atom(_)) }
+
+  /// Tries to unwrap the [Arc]. If that fails, clones it field by field.
+  /// If it's a [Clause::Atom] which cannot be cloned, wraps it in a
+  /// [Clause::Identity].
+  ///
+  /// Implementation of [crate::foreign::to_clause::ToClause::to_clause]. The
+  /// trait is more general so it requires a location which this one doesn't.
+  pub fn into_cls(self) -> Clause {
+    self.try_unwrap().unwrap_or_else(|clsi| match &*clsi.cls_mut() {
+      Clause::Apply { f, x } => Clause::Apply { f: f.clone(), x: x.clone() },
+      Clause::Atom(_) => Clause::Identity(clsi.clone()),
+      Clause::Bottom(e) => Clause::Bottom(e.clone()),
+      Clause::Constant(c) => Clause::Constant(c.clone()),
+      Clause::Identity(sub) => Clause::Identity(sub.clone()),
+      Clause::Lambda { args, body } => Clause::Lambda { args: args.clone(), body: body.clone() },
+      Clause::LambdaArg => Clause::LambdaArg,
+    })
+  }
+
+  /// Decides if this clause is the exact same instance as another. Most useful
+  /// to detect potential deadlocks.
+  pub fn is_same(&self, other: &Self) -> bool { Arc::ptr_eq(&self.0, &other.0) }
+}
+
+impl fmt::Debug for ClauseInst {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self.0.try_lock() {
+      Ok(expr) => write!(f, "{expr:?}"),
+      Err(TryLockError::Poisoned(_)) => write!(f, "<poisoned>"),
+      Err(TryLockError::WouldBlock) => write!(f, "<locked>"),
+    }
+  }
+}
+
+impl fmt::Display for ClauseInst {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self.0.try_lock() {
+      Ok(expr) => write!(f, "{expr}"),
+      Err(TryLockError::Poisoned(_)) => write!(f, "<poisoned>"),
+      Err(TryLockError::WouldBlock) => write!(f, "<locked>"),
+    }
+  }
+}
+
+impl AsDerefMut<Clause> for ClauseInst {
+  fn as_deref_mut(&mut self) -> impl DerefMut<Target = Clause> + '_ { self.cls_mut() }
+}
+
+/// Distinct types of expressions recognized by the interpreter
+#[derive(Debug)]
+pub enum Clause {
+  /// An expression that causes an error
+  Bottom(RTErrorObj),
+  /// Indicates that this [ClauseInst] has the same value as the other
+  /// [ClauseInst]. This has two benefits;
+  ///
+  /// - [Clause] and therefore [Atomic] doesn't have to be [Clone] which saves
+  ///   many synchronization primitives and reference counters in usercode
+  /// - it enforces on the type level that all copies are normalized together,
+  ///   so accidental inefficiency in the interpreter is rarer.
+  ///
+  /// That being said, it's still arbitrary many indirections, so when possible
+  /// APIs should be usable with a [ClauseInst] directly.
+  Identity(ClauseInst),
+  /// An opaque non-callable value, eg. a file handle
+  Atom(Atom),
+  /// A function application
+  Apply {
+    /// Function to be applied
+    f: Expr,
+    /// Argument to be substituted in the function
+    x: VecDeque<Expr>,
+  },
+  /// A name to be looked up in the interpreter's symbol table
+  Constant(Sym),
+  /// A function
+  Lambda {
+    /// A collection of (zero or more) paths to placeholders belonging to this
+    /// function
+    args: Option<PathSet>,
+    /// The tree produced by this function, with placeholders where the
+    /// argument will go
+    body: Expr,
+  },
+  /// A placeholder within a function that will be replaced upon application
+  LambdaArg,
+}
+impl Clause {
+  /// Wrap a clause in a refcounted lock
+  pub fn into_inst(self) -> ClauseInst { ClauseInst::new(self) }
+  /// Wrap a clause in an expression.
+  pub fn into_expr(self, location: CodeLocation) -> Expr { self.into_inst().into_expr(location) }
+}
+
+impl fmt::Display for Clause {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Clause::Atom(a) => write!(f, "{a:?}"),
+      Clause::Bottom(err) => write!(f, "bottom({err})"),
+      Clause::LambdaArg => write!(f, "arg"),
+      Clause::Apply { f: fun, x } => write!(f, "({fun} {})", x.iter().join(" ")),
+      Clause::Lambda { args, body } => match args {
+        Some(path) => write!(f, "[\\{path}.{body}]"),
+        None => write!(f, "[\\_.{body}]"),
+      },
+      Clause::Constant(t) => write!(f, "{t}"),
+      Clause::Identity(other) => write!(f, "{{{other}}}"),
+    }
+  }
+}
+
+impl AsDerefMut<Clause> for Clause {
+  fn as_deref_mut(&mut self) -> impl DerefMut<Target = Clause> + '_ { self }
+}

orchid-base/src/old-atom.rs

@@ -0,0 +1,261 @@
+//! Adaptor trait to embed Rust values in Orchid expressions
+
+use std::any::Any;
+use std::fmt;
+use std::sync::{Arc, Mutex};
+
+use never::Never;
+
+use super::error::{RTError, RTResult};
+use crate::interpreter::context::{RunEnv, RunParams};
+use crate::interpreter::nort;
+use crate::location::{CodeLocation, SourceRange};
+use crate::name::NameLike;
+use crate::parse::lexer::Lexeme;
+use crate::parse::parsed;
+use crate::utils::ddispatch::{request, Request, Responder};
+
+/// Information returned by [Atomic::run].
+pub enum AtomicReturn {
+  /// No work was done. If the atom takes an argument, it can be provided now
+  Inert(Atom),
+  /// Work was done, returns new clause and consumed gas. 1 gas is already
+  /// consumed by the virtual call, so nonzero values indicate expensive
+  /// operations.
+  Change(usize, nort::Clause),
+}
+impl AtomicReturn {
+  /// Report indicating that the value is inert. The result here is always [Ok],
+  /// it's meant to match the return type of [Atomic::run]
+  #[allow(clippy::unnecessary_wraps)]
+  pub fn inert<T: Atomic, E>(this: T) -> Result<Self, E> { Ok(Self::Inert(Atom::new(this))) }
+}
+
+/// Returned by [Atomic::run]
+pub type AtomicResult = RTResult<AtomicReturn>;
+
+/// General error produced when a non-function [Atom] is applied to something as
+/// a function.
+#[derive(Clone)]
+pub struct NotAFunction(pub nort::Expr);
+impl RTError for NotAFunction {}
+impl fmt::Display for NotAFunction {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{:?} is not a function", self.0)
+  }
+}
+
+/// Information about a function call presented to an external function
+pub struct CallData<'a, 'b> {
+  /// Location of the function expression
+  pub location: CodeLocation,
+  /// The argument the function was called on. Functions are curried
+  pub arg: nort::Expr,
+  /// Globally available information such as the list of all constants
+  pub env: &'a RunEnv<'b>,
+  /// Resource limits and other details specific to this interpreter run
+  pub params: &'a mut RunParams,
+}
+
+/// Information about a normalization run presented to an atom
+pub struct RunData<'a, 'b> {
+  /// Location of the atom
+  pub location: CodeLocation,
+  /// Globally available information such as the list of all constants
+  pub env: &'a RunEnv<'b>,
+  /// Resource limits and other details specific to this interpreter run
+  pub params: &'a mut RunParams,
+}
+
+/// Functionality the interpreter needs to handle a value
+///
+/// # Lifecycle methods
+///
+/// Atomics expose the methods [Atomic::redirect], [Atomic::run],
+/// [Atomic::apply] and [Atomic::apply_mut] to interact with the interpreter.
+/// The interpreter first tries to call `redirect` to find a subexpression to
+/// normalize. If it returns `None` or the subexpression is inert, `run` is
+/// called. `run` takes ownership of the value and returns a new one.
+///
+/// If `run` indicated in its return value that the result is inert and the atom
+/// is in the position of a function, `apply` or `apply_mut` is called depending
+/// upon whether the atom is referenced elsewhere. `apply` falls back to
+/// `apply_mut` so implementing it is considered an optimization to avoid
+/// excessive copying.
+///
+/// Atoms don't generally have to be copyable because clauses are refcounted in
+/// the interpreter, but Orchid code is always free to duplicate the references
+/// and apply them as functions to multiple different arguments so atoms that
+/// represent functions have to support application by-ref without consuming the
+/// function itself.
+pub trait Atomic: Any + fmt::Debug + Responder + Send
+where Self: 'static
+{
+  /// Casts this value to [Any] so that its original value can be salvaged
+  /// during introspection by other external code.
+  ///
+  /// This function should be implemented in exactly one way:
+  ///
+  /// ```ignore
+  /// fn as_any(self: Box<Self>) -> Box<dyn Any> { self }
+  /// ```
+  #[must_use]
+  fn as_any(self: Box<Self>) -> Box<dyn Any>;
+  /// See [Atomic::as_any], exactly the same but for references
+  #[must_use]
+  fn as_any_ref(&self) -> &dyn Any;
+  /// Print the atom's type name. Should only ever be implemented as
+  ///
+  /// ```ignore
+  /// fn type_name(&self) -> &'static str { std::any::type_name::<Self>() }
+  /// ```
+  fn type_name(&self) -> &'static str;
+
+  /// Returns a reference to a possible expression held inside the atom which
+  /// can be reduced. For an overview of the lifecycle see [Atomic]
+  fn redirect(&mut self) -> Option<&mut nort::Expr>;
+
+  /// Attempt to normalize this value. If it wraps a value, this should report
+  /// inert. If it wraps a computation, it should execute one logical step of
+  /// the computation and return a structure representing the next.
+  ///
+  /// For an overview of the lifecycle see [Atomic]
+  fn run(self: Box<Self>, run: RunData) -> AtomicResult;
+
+  /// Combine the function with an argument to produce a new clause. Falls back
+  /// to [Atomic::apply_mut] by default.
+  ///
+  /// For an overview of the lifecycle see [Atomic]
+  fn apply(mut self: Box<Self>, call: CallData) -> RTResult<nort::Clause> { self.apply_mut(call) }
+
+  /// Combine the function with an argument to produce a new clause
+  ///
+  /// For an overview of the lifecycle see [Atomic]
+  fn apply_mut(&mut self, call: CallData) -> RTResult<nort::Clause>;
+
+  /// Must return true for atoms parsed from identical source.
+  /// If the atom cannot be parsed from source, it can safely be ignored
+  #[allow(unused_variables)]
+  fn parser_eq(&self, other: &dyn Atomic) -> bool { false }
+
+  /// Wrap the atom in a clause to be placed in an [AtomicResult].
+  #[must_use]
+  fn atom_cls(self) -> nort::Clause
+  where Self: Sized {
+    nort::Clause::Atom(Atom(Box::new(self)))
+  }
+
+  /// Shorthand for `self.atom_cls().to_inst()`
+  fn atom_clsi(self) -> nort::ClauseInst
+  where Self: Sized {
+    self.atom_cls().into_inst()
+  }
+
+  /// Wrap the atom in a new expression instance to be placed in a tree
+  #[must_use]
+  fn atom_expr(self, location: CodeLocation) -> nort::Expr
+  where Self: Sized {
+    self.atom_clsi().into_expr(location)
+  }
+
+  /// Wrap the atom in a clause to be placed in a
+  /// [crate::parse::parsed::SourceLine].
+  #[must_use]
+  fn ast_cls(self) -> parsed::Clause
+  where Self: Sized + Clone {
+    parsed::Clause::Atom(AtomGenerator::cloner(self))
+  }
+
+  /// Wrap the atom in an expression to be placed in a
+  /// [crate::parse::parsed::SourceLine].
+  #[must_use]
+  fn ast_exp<N: NameLike>(self, range: SourceRange) -> parsed::Expr
+  where Self: Sized + Clone {
+    self.ast_cls().into_expr(range)
+  }
+
+  /// Wrap this atomic value in a lexeme. This means that the atom will
+  /// participate in macro reproject, so it must implement [Atomic::parser_eq].
+  fn lexeme(self) -> Lexeme
+  where Self: Sized + Clone {
+    Lexeme::Atom(AtomGenerator::cloner(self))
+  }
+}
+
+/// A struct for generating any number of [Atom]s. Since atoms aren't Clone,
+/// this represents the ability to create any number of instances of an atom
+#[derive(Clone)]
+pub struct AtomGenerator(Arc<dyn Fn() -> Atom + Send + Sync>);
+impl AtomGenerator {
+  /// Use a factory function to create any number of atoms
+  pub fn new(f: impl Fn() -> Atom + Send + Sync + 'static) -> Self { Self(Arc::new(f)) }
+  /// Clone a representative atom when called
+  pub fn cloner(atom: impl Atomic + Clone) -> Self {
+    let lock = Mutex::new(atom);
+    Self::new(move || Atom::new(lock.lock().unwrap().clone()))
+  }
+  /// Generate an atom
+  pub fn run(&self) -> Atom { self.0() }
+}
+impl fmt::Debug for AtomGenerator {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self.run()) }
+}
+impl PartialEq for AtomGenerator {
+  fn eq(&self, other: &Self) -> bool { self.run().0.parser_eq(&*other.run().0) }
+}
+
+/// Represents a black box unit of data with its own normalization steps.
+/// Typically Rust functions integrated with [super::fn_bridge::xfn] will
+/// produce and consume [Atom]s to represent both raw data, pending
+/// computational tasks, and curried partial calls awaiting their next argument.
+pub struct Atom(pub Box<dyn Atomic>);
+impl Atom {
+  /// Wrap an [Atomic] in a type-erased box
+  #[must_use]
+  pub fn new<T: 'static + Atomic>(data: T) -> Self { Self(Box::new(data) as Box<dyn Atomic>) }
+  /// Get the contained data
+  #[must_use]
+  pub fn data(&self) -> &dyn Atomic { self.0.as_ref() as &dyn Atomic }
+  /// Test the type of the contained data without downcasting
+  #[must_use]
+  pub fn is<T: Atomic>(&self) -> bool { self.data().as_any_ref().is::<T>() }
+  /// Downcast contained data, panic if it isn't the specified type
+  #[must_use]
+  pub fn downcast<T: Atomic>(self) -> T {
+    *self.0.as_any().downcast().expect("Type mismatch on Atom::cast")
+  }
+  /// Normalize the contained data
+  pub fn run(self, run: RunData<'_, '_>) -> AtomicResult { self.0.run(run) }
+  /// Request a delegate from the encapsulated data
+  pub fn request<T: 'static>(&self) -> Option<T> { request(self.0.as_ref()) }
+  /// Downcast the atom to a concrete atomic type, or return the original atom
+  /// if it is not the specified type
+  pub fn try_downcast<T: Atomic>(self) -> Result<T, Self> {
+    match self.0.as_any_ref().is::<T>() {
+      true => Ok(*self.0.as_any().downcast().expect("checked just above")),
+      false => Err(self),
+    }
+  }
+  /// Downcast an atom by reference
+  pub fn downcast_ref<T: Atomic>(&self) -> Option<&T> { self.0.as_any_ref().downcast_ref() }
+  /// Combine the function with an argument to produce a new clause
+  pub fn apply(self, call: CallData) -> RTResult<nort::Clause> { self.0.apply(call) }
+  /// Combine the function with an argument to produce a new clause
+  pub fn apply_mut(&mut self, call: CallData) -> RTResult<nort::Clause> { self.0.apply_mut(call) }
+}
+
+impl fmt::Debug for Atom {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self.data()) }
+}
+
+impl Responder for Never {
+  fn respond(&self, _request: Request) { match *self {} }
+}
+impl Atomic for Never {
+  fn as_any(self: Box<Self>) -> Box<dyn Any> { match *self {} }
+  fn as_any_ref(&self) -> &dyn Any { match *self {} }
+  fn type_name(&self) -> &'static str { match *self {} }
+  fn redirect(&mut self) -> Option<&mut nort::Expr> { match *self {} }
+  fn run(self: Box<Self>, _: RunData) -> AtomicResult { match *self {} }
+  fn apply_mut(&mut self, _: CallData) -> RTResult<nort::Clause> { match *self {} }
+}

orchid-base/src/proj_error.rs

@@ -0,0 +1,285 @@
+//! Abstractions for handling various code-related errors under a common trait
+//! object.
+
+use std::any::Any;
+use std::cell::RefCell;
+use std::sync::Arc;
+use std::{fmt, process};
+
+use dyn_clone::{clone_box, DynClone};
+use itertools::Itertools;
+
+use crate::location::CodeOrigin;
+use crate::boxed_iter::{box_once, BoxedIter};
+#[allow(unused)] // for doc
+use crate::virt_fs::CodeNotFound;
+
+/// A point of interest in resolving the error, such as the point where
+/// processing got stuck, a command that is likely to be incorrect
+#[derive(Clone)]
+pub struct ErrorPosition {
+  /// The suspected origin
+  pub origin: CodeOrigin,
+  /// Any information about the role of this origin
+  pub message: Option<String>,
+}
+impl From<CodeOrigin> for ErrorPosition {
+  fn from(origin: CodeOrigin) -> Self { Self { origin, message: None } }
+}
+
+/// Errors addressed to the developer which are to be resolved with
+/// code changes
+pub trait ProjectError: Sized + Send + Sync + 'static {
+  /// A general description of this type of error
+  const DESCRIPTION: &'static str;
+  /// A formatted message that includes specific parameters
+  #[must_use]
+  fn message(&self) -> String { self.description().to_string() }
+  /// Code positions relevant to this error. If you don't implement this, you
+  /// must implement [ProjectError::one_position]
+  #[must_use]
+  fn positions(&self) -> impl IntoIterator<Item = ErrorPosition> + '_ {
+    box_once(ErrorPosition { origin: self.one_position(), message: None })
+  }
+  /// Short way to provide a single origin. If you don't implement this, you
+  /// must implement [ProjectError::positions]
+  #[must_use]
+  fn one_position(&self) -> CodeOrigin { unimplemented!() }
+  /// Convert the error into an `Arc<dyn DynProjectError>` to be able to
+  /// handle various errors together
+  #[must_use]
+  fn pack(self) -> ProjectErrorObj { Arc::new(self) }
+}
+
+/// Object-safe version of [ProjectError]. Implement that instead of this.
+pub trait DynProjectError: Send + Sync {
+  /// Access type information about this error
+  #[must_use]
+  fn as_any_ref(&self) -> &dyn Any;
+  /// Pack the error into a trait object, or leave it as-is if it's already a
+  /// trait object
+  #[must_use]
+  fn into_packed(self: Arc<Self>) -> ProjectErrorObj;
+  /// A general description of this type of error
+  #[must_use]
+  fn description(&self) -> &str;
+  /// A formatted message that includes specific parameters
+  #[must_use]
+  fn message(&self) -> String { self.description().to_string() }
+  /// Code positions relevant to this error.
+  #[must_use]
+  fn positions(&self) -> BoxedIter<'_, ErrorPosition>;
+}
+
+impl<T> DynProjectError for T
+where T: ProjectError
+{
+  fn as_any_ref(&self) -> &dyn Any { self }
+  fn into_packed(self: Arc<Self>) -> ProjectErrorObj { self }
+  fn description(&self) -> &str { T::DESCRIPTION }
+  fn message(&self) -> String { ProjectError::message(self) }
+  fn positions(&self) -> BoxedIter<ErrorPosition> {
+    Box::new(ProjectError::positions(self).into_iter())
+  }
+}
+
+impl DynProjectError for ProjectErrorObj {
+  fn as_any_ref(&self) -> &dyn Any { (**self).as_any_ref() }
+  fn description(&self) -> &str { (**self).description() }
+  fn into_packed(self: Arc<Self>) -> ProjectErrorObj { (*self).clone() }
+  fn message(&self) -> String { (**self).message() }
+  fn positions(&self) -> BoxedIter<'_, ErrorPosition> { (**self).positions() }
+}
+
+impl fmt::Display for dyn DynProjectError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    let description = self.description();
+    let message = self.message();
+    let positions = self.positions().collect::<Vec<_>>();
+    writeln!(f, "Project error: {description}\n{message}")?;
+    if positions.is_empty() {
+      writeln!(f, "No origins specified")?;
+    } else {
+      for ErrorPosition { origin, message } in positions {
+        match message {
+          None => writeln!(f, "@{origin}"),
+          Some(msg) => writeln!(f, "@{origin}: {msg}"),
+        }?
+      }
+    }
+    Ok(())
+  }
+}
+
+impl fmt::Debug for dyn DynProjectError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") }
+}
+
+/// Type-erased [ProjectError] implementor through the [DynProjectError]
+/// object-trait
+pub type ProjectErrorObj = Arc<dyn DynProjectError>;
+/// Alias for a result with an error of [ProjectErrorObj]. This is the type of
+/// result most commonly returned by pre-runtime operations.
+pub type ProjectResult<T> = Result<T, ProjectErrorObj>;
+
+/// A trait for error types that are only missing an origin. Do not depend on
+/// this trait, refer to [DynErrorSansOrigin] instead.
+pub trait ErrorSansOrigin: Clone + Sized + Send + Sync + 'static {
+  /// General description of the error condition
+  const DESCRIPTION: &'static str;
+  /// Specific description of the error including code fragments or concrete
+  /// data if possible
+  fn message(&self) -> String { Self::DESCRIPTION.to_string() }
+  /// Convert the error to a type-erased structure for handling on shared
+  /// channels
+  fn pack(self) -> ErrorSansOriginObj { Box::new(self) }
+  /// A shortcut to streamline switching code between [ErrorSansOriginObj] and
+  /// concrete types
+  fn bundle(self, origin: &CodeOrigin) -> ProjectErrorObj { self.pack().bundle(origin) }
+}
+
+/// Object-safe equivalent to [ErrorSansOrigin]. Implement that one instead of
+/// this. Typically found as [ErrorSansOriginObj]
+pub trait DynErrorSansOrigin: Any + Send + Sync + DynClone {
+  /// Allow to downcast the base object to distinguish between various errors.
+  /// The main intended purpose is to trigger a fallback when [CodeNotFound] is
+  /// encountered, but the possibilities are not limited to that.
+  fn as_any_ref(&self) -> &dyn Any;
+  /// Regularize the type
+  fn into_packed(self: Box<Self>) -> ErrorSansOriginObj;
+  /// Generic description of the error condition
+  fn description(&self) -> &str;
+  /// Specific description of this particular error
+  fn message(&self) -> String;
+  /// Add an origin
+  fn bundle(self: Box<Self>, origin: &CodeOrigin) -> ProjectErrorObj;
+}
+
+/// Type-erased [ErrorSansOrigin] implementor through the object-trait
+/// [DynErrorSansOrigin]. This can be turned into a [ProjectErrorObj] with
+/// [ErrorSansOriginObj::bundle].
+pub type ErrorSansOriginObj = Box<dyn DynErrorSansOrigin>;
+/// A generic project result without origin
+pub type ResultSansOrigin<T> = Result<T, ErrorSansOriginObj>;
+
+impl<T: ErrorSansOrigin + 'static> DynErrorSansOrigin for T {
+  fn description(&self) -> &str { Self::DESCRIPTION }
+  fn message(&self) -> String { (*self).message() }
+  fn as_any_ref(&self) -> &dyn Any { self }
+  fn into_packed(self: Box<Self>) -> ErrorSansOriginObj { (*self).pack() }
+  fn bundle(self: Box<Self>, origin: &CodeOrigin) -> ProjectErrorObj {
+    Arc::new(OriginBundle(origin.clone(), *self))
+  }
+}
+impl Clone for ErrorSansOriginObj {
+  fn clone(&self) -> Self { clone_box(&**self) }
+}
+impl DynErrorSansOrigin for ErrorSansOriginObj {
+  fn description(&self) -> &str { (**self).description() }
+  fn message(&self) -> String { (**self).message() }
+  fn as_any_ref(&self) -> &dyn Any { (**self).as_any_ref() }
+  fn into_packed(self: Box<Self>) -> ErrorSansOriginObj { *self }
+  fn bundle(self: Box<Self>, origin: &CodeOrigin) -> ProjectErrorObj { (*self).bundle(origin) }
+}
+impl fmt::Display for ErrorSansOriginObj {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    writeln!(f, "{}\nOrigin missing from error", self.message())
+  }
+}
+impl fmt::Debug for ErrorSansOriginObj {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") }
+}
+
+struct OriginBundle<T: ErrorSansOrigin>(CodeOrigin, T);
+impl<T: ErrorSansOrigin> DynProjectError for OriginBundle<T> {
+  fn as_any_ref(&self) -> &dyn Any { self.1.as_any_ref() }
+  fn into_packed(self: Arc<Self>) -> ProjectErrorObj { self }
+  fn description(&self) -> &str { self.1.description() }
+  fn message(&self) -> String { self.1.message() }
+  fn positions(&self) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { origin: self.0.clone(), message: None })
+  }
+}
+
+/// A collection for tracking fatal errors without halting. Participating
+/// functions return [ProjectResult] even if they only ever construct [Ok]. When
+/// they call other participating functions, instead of directly forwarding
+/// errors with `?` they should prefer constructing a fallback value with
+/// [Reporter::fallback]. If any error is added to a [Reporter] in a function,
+/// the return value is valid but its meaning need not be related in any way to
+/// the inputs.
+///
+/// Returning [Err] from a function that accepts `&mut Reporter` indicates not
+/// that there was a fatal error but that it wasn't possible to construct a
+/// fallback, so if it can, the caller should construct one.
+pub struct Reporter(RefCell<Vec<ProjectErrorObj>>);
+impl Reporter {
+  /// Create a new error reporter
+  pub fn new() -> Self { Self(RefCell::new(Vec::new())) }
+  /// Returns true if any errors were regorded. If this ever returns true, it
+  /// will always return true in the future.
+  pub fn failing(&self) -> bool { !self.0.borrow().is_empty() }
+  /// Report a fatal error
+  pub fn report(&self, error: ProjectErrorObj) { self.0.borrow_mut().push(error) }
+  /// Catch a fatal error, report it, and substitute the value
+  pub fn fallback<T>(&self, res: ProjectResult<T>, cb: impl FnOnce(ProjectErrorObj) -> T) -> T {
+    res.inspect_err(|e| self.report(e.clone())).unwrap_or_else(cb)
+  }
+  /// Panic if there were errors
+  pub fn assert(&self) { self.unwrap(Ok(())) }
+  /// Exit with code -1 if there were errors
+  pub fn assert_exit(&self) { self.unwrap_exit(Ok(())) }
+  /// Panic with descriptive messages if there were errors. If there were no
+  /// errors, unwrap the result
+  pub fn unwrap<T>(&self, res: ProjectResult<T>) -> T {
+    if self.failing() {
+      panic!("Errors were encountered: \n{}", self.0.borrow().iter().join("\n"));
+    }
+    res.unwrap()
+  }
+  /// Print errors and exit if any occurred.  If there were no errors, unwrap
+  /// the result
+  pub fn unwrap_exit<T>(&self, res: ProjectResult<T>) -> T {
+    if self.failing() {
+      eprintln!("Errors were encountered: \n{}", self.0.borrow().iter().join("\n"));
+      process::exit(-1)
+    }
+    res.unwrap_or_else(|e| {
+      eprintln!("{e}");
+      process::exit(-1)
+    })
+  }
+  /// Take the errors out of the reporter
+  #[must_use]
+  pub fn into_errors(self) -> Option<Vec<ProjectErrorObj>> {
+    let v = self.0.into_inner();
+    if v.is_empty() { None } else { Some(v) }
+  }
+  /// Raise an error if the reporter contains any errors
+  pub fn bind(self) -> ProjectResult<()> {
+    match self.into_errors() {
+      None => Ok(()),
+      Some(v) if v.len() == 1 => Err(v.into_iter().exactly_one().unwrap()),
+      Some(v) => Err(MultiError(v).pack()),
+    }
+  }
+}
+
+impl Default for Reporter {
+  fn default() -> Self { Self::new() }
+}
+
+struct MultiError(Vec<ProjectErrorObj>);
+impl ProjectError for MultiError {
+  const DESCRIPTION: &'static str = "Multiple errors occurred";
+  fn message(&self) -> String { format!("{} errors occurred", self.0.len()) }
+  fn positions(&self) -> impl IntoIterator<Item = ErrorPosition> + '_ {
+    self.0.iter().flat_map(|e| {
+      e.positions().map(|pos| {
+        let emsg = e.message();
+        let msg = if let Some(pmsg) = pos.message { format!("{emsg}: {pmsg}") } else { emsg };
+        ErrorPosition { origin: pos.origin, message: Some(msg) }
+      })
+    })
+  }
+}

orchid-base/src/reqnot.rs

@@ -0,0 +1,222 @@
+use std::mem;
+use std::ops::{BitAnd, Deref};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{sync_channel, SyncSender};
+use std::sync::{Arc, Mutex};
+
+use dyn_clone::{clone_box, DynClone};
+use hashbrown::HashMap;
+use orchid_api_traits::{Coding, Decode, Encode, MsgSet, Request};
+use trait_set::trait_set;
+
+trait_set! {
+  pub trait SendFn<T: MsgSet> = for<'a> FnMut(&'a [u8], ReqNot<T>) + DynClone + Send + 'static;
+  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + Send + 'static;
+  pub trait NotifFn<T: MsgSet> = for<'a> FnMut(T::InNot, ReqNot<T>) + Send + Sync + 'static;
+}
+
+fn get_id(message: &[u8]) -> (u64, &[u8]) {
+  (u64::from_be_bytes(message[..8].to_vec().try_into().unwrap()), &message[8..])
+}
+
+pub struct RequestHandle<T: MsgSet> {
+  id: u64,
+  message: T::InReq,
+  send: Box<dyn SendFn<T>>,
+  parent: ReqNot<T>,
+  fulfilled: AtomicBool,
+}
+impl<MS: MsgSet> RequestHandle<MS> {
+  pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
+  pub fn req(&self) -> &MS::InReq { &self.message }
+  fn respond(&self, response: &impl Encode) {
+    assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded");
+    let mut buf = (!self.id).to_be_bytes().to_vec();
+    response.encode(&mut buf);
+    clone_box(&*self.send)(&buf, self.parent.clone());
+  }
+  pub fn handle<T: Request>(&self, _: &T, rep: &T::Response) { self.respond(rep) }
+}
+impl<MS: MsgSet> Drop for RequestHandle<MS> {
+  fn drop(&mut self) {
+    debug_assert!(self.fulfilled.load(Ordering::Relaxed), "Request dropped without response")
+  }
+}
+
+pub fn respond_with<R: Request>(r: &R, f: impl FnOnce(&R) -> R::Response) -> Vec<u8> {
+  r.respond(f(r))
+}
+
+pub struct ReqNotData<T: MsgSet> {
+  id: u64,
+  send: Box<dyn SendFn<T>>,
+  notif: Box<dyn NotifFn<T>>,
+  req: Box<dyn ReqFn<T>>,
+  responses: HashMap<u64, SyncSender<Vec<u8>>>,
+}
+
+pub struct RawReply(Vec<u8>);
+impl Deref for RawReply {
+  type Target = [u8];
+  fn deref(&self) -> &Self::Target { get_id(&self.0[..]).1 }
+}
+
+pub struct ReqNot<T: MsgSet>(Arc<Mutex<ReqNotData<T>>>);
+impl<T: MsgSet> ReqNot<T> {
+  pub fn new(send: impl SendFn<T>, notif: impl NotifFn<T>, req: impl ReqFn<T>) -> Self {
+    Self(Arc::new(Mutex::new(ReqNotData {
+      id: 1,
+      send: Box::new(send),
+      notif: Box::new(notif),
+      req: Box::new(req),
+      responses: HashMap::new(),
+    })))
+  }
+
+  /// Can be called from a polling thread or dispatched in any other way
+  pub fn receive(&self, message: Vec<u8>) {
+    let mut g = self.0.lock().unwrap();
+    let (id, payload) = get_id(&message[..]);
+    if id == 0 {
+      (g.notif)(T::InNot::decode(&mut &payload[..]), self.clone())
+    } else if 0 < id.bitand(1 << 63) {
+      let sender = g.responses.remove(&!id).expect("Received response for invalid message");
+      sender.send(message).unwrap();
+    } else {
+      let send = clone_box(&*g.send);
+      let message = T::InReq::decode(&mut &payload[..]);
+      (g.req)(RequestHandle { id, message, send, fulfilled: false.into(), parent: self.clone() })
+    }
+  }
+
+  pub fn notify<N: Coding + Into<T::OutNot>>(&self, notif: N) {
+    let mut send = clone_box(&*self.0.lock().unwrap().send);
+    let mut buf = vec![0; 8];
+    let msg: T::OutNot = notif.into();
+    msg.encode(&mut buf);
+    send(&buf, self.clone())
+  }
+}
+
+pub struct MappedRequester<'a, T>(Box<dyn Fn(T) -> RawReply + Send + Sync + 'a>);
+impl<'a, T> MappedRequester<'a, T> {
+  fn new<U: DynRequester + 'a>(req: U) -> Self
+  where T: Into<U::Transfer> {
+    MappedRequester(Box::new(move |t| req.raw_request(t.into())))
+  }
+}
+
+impl<'a, T> DynRequester for MappedRequester<'a, T> {
+  type Transfer = T;
+  fn raw_request(&self, data: Self::Transfer) -> RawReply { self.0(data) }
+}
+
+impl<T: MsgSet> DynRequester for ReqNot<T> {
+  type Transfer = T::OutReq;
+  fn raw_request(&self, req: Self::Transfer) -> RawReply {
+    let mut g = self.0.lock().unwrap();
+    let id = g.id;
+    g.id += 1;
+    let mut buf = id.to_be_bytes().to_vec();
+    req.encode(&mut buf);
+    let (send, recv) = sync_channel(1);
+    g.responses.insert(id, send);
+    let mut send = clone_box(&*g.send);
+    mem::drop(g);
+    send(&buf, self.clone());
+    RawReply(recv.recv().unwrap())
+  }
+}
+
+pub trait DynRequester: Send + Sync {
+  type Transfer;
+  fn raw_request(&self, data: Self::Transfer) -> RawReply;
+}
+pub trait Requester: DynRequester {
+  #[must_use = "These types are subject to change with protocol versions. \
+    If you don't want to use the return value, At a minimum, force the type."]
+  fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response;
+  fn map<'a, U: Into<Self::Transfer>>(self) -> MappedRequester<'a, U>
+  where Self: Sized + 'a {
+    MappedRequester::new(self)
+  }
+}
+impl<'a, This: DynRequester + ?Sized + 'a> Requester for This {
+  fn request<R: Request + Into<Self::Transfer>>(&self, data: R) -> R::Response {
+    R::Response::decode(&mut &self.raw_request(data.into())[..])
+  }
+}
+
+impl<T: MsgSet> Clone for ReqNot<T> {
+  fn clone(&self) -> Self { Self(self.0.clone()) }
+}
+
+#[cfg(test)]
+mod test {
+  use std::sync::{Arc, Mutex};
+
+  use orchid_api_derive::Coding;
+  use orchid_api_traits::Request;
+
+  use super::{MsgSet, ReqNot};
+  use crate::{clone, reqnot::Requester as _};
+
+  #[derive(Coding, Debug, PartialEq)]
+  pub struct TestReq(u8);
+  impl Request for TestReq {
+    type Response = u8;
+  }
+
+  pub struct TestMsgSet;
+  impl MsgSet for TestMsgSet {
+    type InNot = u8;
+    type InReq = TestReq;
+    type OutNot = u8;
+    type OutReq = TestReq;
+  }
+
+  #[test]
+  fn notification() {
+    let received = Arc::new(Mutex::new(None));
+    let receiver = ReqNot::<TestMsgSet>::new(
+      |_, _| panic!("Should not send anything"),
+      clone!(received; move |notif, _| *received.lock().unwrap() = Some(notif)),
+      |_| panic!("Not receiving a request"),
+    );
+    let sender = ReqNot::<TestMsgSet>::new(
+      clone!(receiver; move |d, _| receiver.receive(d.to_vec())),
+      |_, _| panic!("Should not receive notif"),
+      |_| panic!("Should not receive request"),
+    );
+    sender.notify(3);
+    assert_eq!(*received.lock().unwrap(), Some(3));
+    sender.notify(4);
+    assert_eq!(*received.lock().unwrap(), Some(4));
+  }
+
+  #[test]
+  fn request() {
+    let receiver = Arc::new(Mutex::<Option<ReqNot<TestMsgSet>>>::new(None));
+    let sender = Arc::new(ReqNot::<TestMsgSet>::new(
+      {
+        let receiver = receiver.clone();
+        move |d, _| receiver.lock().unwrap().as_ref().unwrap().receive(d.to_vec())
+      },
+      |_, _| panic!("Should not receive notif"),
+      |_| panic!("Should not receive request"),
+    ));
+    *receiver.lock().unwrap() = Some(ReqNot::new(
+      {
+        let sender = sender.clone();
+        move |d, _| sender.receive(d.to_vec())
+      },
+      |_, _| panic!("Not receiving notifs"),
+      |req| {
+        assert_eq!(req.req(), &TestReq(5));
+        req.respond(&6u8);
+      },
+    ));
+    let response = sender.request(TestReq(5));
+    assert_eq!(response, 6);
+  }
+}

orchid-base/src/rt_error.rs

@@ -0,0 +1,67 @@
+//! Errors produced by the interpreter
+
+use std::error::Error;
+use std::fmt;
+use std::sync::Arc;
+
+use dyn_clone::DynClone;
+
+use crate::error::ProjectErrorObj;
+use crate::location::CodeLocation;
+
+/// Errors produced by external code when runtime-enforced assertions are
+/// violated.
+pub trait RTError: fmt::Display + Send + Sync + DynClone {
+  /// Convert into trait object
+  #[must_use]
+  fn pack(self) -> RTErrorObj
+  where Self: 'static + Sized {
+    Arc::new(self)
+  }
+}
+
+impl fmt::Debug for dyn RTError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "ExternError({self})") }
+}
+
+impl Error for dyn RTError {}
+
+impl RTError for ProjectErrorObj {}
+
+/// An error produced by Rust code called form Orchid. The error is type-erased.
+pub type RTErrorObj = Arc<dyn RTError>;
+
+/// A result produced by Rust code called from Orchid.
+pub type RTResult<T> = Result<T, RTErrorObj>;
+
+/// Some expectation (usually about the argument types of a function) did not
+/// hold.
+#[derive(Clone)]
+pub struct AssertionError {
+  location: CodeLocation,
+  message: &'static str,
+  details: String,
+}
+
+impl AssertionError {
+  /// Construct, upcast and wrap in a Result that never succeeds for easy
+  /// short-circuiting
+  pub fn fail<T>(location: CodeLocation, message: &'static str, details: String) -> RTResult<T> {
+    Err(Self::ext(location, message, details))
+  }
+
+  /// Construct and upcast to [RTErrorObj]
+  pub fn ext(location: CodeLocation, message: &'static str, details: String) -> RTErrorObj {
+    Self { location, message, details }.pack()
+  }
+}
+
+impl fmt::Display for AssertionError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Error: expected {}", self.message)?;
+    write!(f, " at {}", self.location)?;
+    write!(f, " details: {}", self.details)
+  }
+}
+
+impl RTError for AssertionError {}

orchid-base/src/run_ctx.rs

@@ -0,0 +1,95 @@
+//! Addiitional information passed to the interpreter
+
+use std::cell::RefCell;
+use std::fmt;
+
+use hashbrown::HashMap;
+
+use super::handler::HandlerTable;
+use super::nort::{Clause, Expr};
+use crate::foreign::error::{RTError, RTErrorObj, RTResult};
+use crate::location::CodeLocation;
+use crate::name::Sym;
+
+/// Data that must not change except in well-defined ways while any data
+/// associated with this process persists
+pub struct RunEnv<'a> {
+  /// Mutable callbacks the code can invoke with continuation passing
+  pub handlers: HandlerTable<'a>,
+  /// Constants referenced in the code in [super::nort::Clause::Constant] nodes
+  pub symbols: RefCell<HashMap<Sym, RTResult<Expr>>>,
+  /// Callback to invoke when a symbol is not found
+  pub symbol_cb: Box<dyn Fn(Sym, CodeLocation) -> RTResult<Expr> + 'a>,
+}
+
+impl<'a> RunEnv<'a> {
+  /// Create a new context. The return values of the symbol callback are cached
+  pub fn new(
+    handlers: HandlerTable<'a>,
+    symbol_cb: impl Fn(Sym, CodeLocation) -> RTResult<Expr> + 'a,
+  ) -> Self {
+    Self { handlers, symbols: RefCell::new(HashMap::new()), symbol_cb: Box::new(symbol_cb) }
+  }
+
+  /// Produce an error indicating that a symbol was missing
+  pub fn sym_not_found(sym: Sym, location: CodeLocation) -> RTErrorObj {
+    MissingSymbol { location, sym }.pack()
+  }
+
+  /// Load a symbol from cache or invoke the callback
+  pub fn load(&self, sym: Sym, location: CodeLocation) -> RTResult<Expr> {
+    let mut guard = self.symbols.borrow_mut();
+    let (_, r) = (guard.raw_entry_mut().from_key(&sym))
+      .or_insert_with(|| (sym.clone(), (self.symbol_cb)(sym, location)));
+    r.clone()
+  }
+
+  /// Attempt to resolve the command with the command handler table
+  pub fn dispatch(&self, expr: &Clause, location: CodeLocation) -> Option<Expr> {
+    match expr {
+      Clause::Atom(at) => self.handlers.dispatch(&*at.0, location),
+      _ => None,
+    }
+  }
+}
+
+/// Limits and other context that is subject to change
+pub struct RunParams {
+  /// Number of reduction steps permitted before the program is preempted
+  pub gas: Option<usize>,
+  /// Maximum recursion depth. Orchid uses a soft stack so this can be very
+  /// large, but it must not be
+  pub stack: usize,
+}
+impl RunParams {
+  /// Consume some gas if it is being counted
+  pub fn use_gas(&mut self, amount: usize) {
+    if let Some(g) = self.gas.as_mut() {
+      *g = g.saturating_sub(amount)
+    }
+  }
+  /// Gas is being counted and there is none left
+  pub fn no_gas(&self) -> bool { self.gas == Some(0) }
+  /// Add gas to make execution longer, or to resume execution in a preempted
+  /// expression
+  pub fn add_gas(&mut self, amount: usize) {
+    if let Some(g) = self.gas.as_mut() {
+      *g = g.saturating_add(amount)
+    }
+  }
+}
+
+/// The interpreter's sole output excluding error conditions is an expression
+pub type Halt = Expr;
+
+#[derive(Clone)]
+pub(crate) struct MissingSymbol {
+  pub sym: Sym,
+  pub location: CodeLocation,
+}
+impl RTError for MissingSymbol {}
+impl fmt::Display for MissingSymbol {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}, called at {} is not loaded", self.sym, self.location)
+  }
+}

orchid-base/src/sequence.rs

@@ -0,0 +1,27 @@
+//! An alternative to `Iterable` in many languages, a [Fn] that returns an
+//! iterator.
+
+use std::rc::Rc;
+
+use trait_set::trait_set;
+
+use super::boxed_iter::BoxedIter;
+
+trait_set! {
+  trait Payload<'a, T> = Fn() -> BoxedIter<'a, T> + 'a;
+}
+
+/// Dynamic iterator building callback. Given how many trait objects this
+/// involves, it may actually be slower than C#.
+pub struct Sequence<'a, T: 'a>(Rc<dyn Payload<'a, T>>);
+impl<'a, T: 'a> Sequence<'a, T> {
+  /// Construct from a concrete function returning a concrete iterator
+  pub fn new<I: IntoIterator<Item = T> + 'a>(f: impl Fn() -> I + 'a) -> Self {
+    Self(Rc::new(move || Box::new(f().into_iter())))
+  }
+  /// Get an iterator from the function
+  pub fn iter(&self) -> impl Iterator<Item = T> + '_ { (self.0)() }
+}
+impl<'a, T: 'a> Clone for Sequence<'a, T> {
+  fn clone(&self) -> Self { Self(self.0.clone()) }
+}

orchid-base/src/system.rs

@@ -0,0 +1,70 @@
+//! Unified extension struct instances of which are catalogued by
+//! [super::loader::Loader]. Language extensions must implement [IntoSystem].
+
+use crate::error::{ErrorPosition, ProjectError};
+use crate::gen::tree::ConstTree;
+use crate::interpreter::handler::HandlerTable;
+use crate::name::VName;
+use crate::parse::lex_plugin::LexerPlugin;
+use crate::parse::parse_plugin::ParseLinePlugin;
+use crate::pipeline::load_project::Prelude;
+use crate::virt_fs::DeclTree;
+
+/// A description of every point where an external library can hook into Orchid.
+/// Intuitively, this can be thought of as a plugin
+pub struct System<'a> {
+  /// An identifier for the system used eg. in error reporting.
+  pub name: &'a str,
+  /// External functions and other constant values defined in AST form
+  pub constants: ConstTree,
+  /// Orchid libraries defined by this system
+  pub code: DeclTree,
+  /// Prelude lines to be added to the head of files to expose the
+  /// functionality of this system. A glob import from the first path is
+  /// added to every file outside the prefix specified by the second path
+  pub prelude: Vec<Prelude>,
+  /// Handlers for actions defined in this system
+  pub handlers: HandlerTable<'a>,
+  /// Custom lexer for the source code representation atomic data.
+  /// These take priority over builtin lexers so the syntax they
+  /// match should be unambiguous
+  pub lexer_plugins: Vec<Box<dyn LexerPlugin + 'a>>,
+  /// Parser that processes custom line types into their representation in the
+  /// module tree
+  pub line_parsers: Vec<Box<dyn ParseLinePlugin>>,
+}
+impl<'a> System<'a> {
+  /// Intern the name of the system so that it can be used as an Orchid
+  /// namespace
+  #[must_use]
+  pub fn vname(&self) -> VName {
+    VName::parse(self.name).expect("Systems must have a non-empty name")
+  }
+}
+
+/// An error raised when a system fails to load a path. This usually means that
+/// another system the current one depends on did not get loaded
+#[derive(Debug)]
+struct MissingSystemCode {
+  path: VName,
+  system: Vec<String>,
+  referrer: VName,
+}
+impl ProjectError for MissingSystemCode {
+  const DESCRIPTION: &'static str = "A system tried to import a path that doesn't exist";
+  fn message(&self) -> String {
+    format!(
+      "Path {} imported by {} is not defined by {} or any system before it",
+      self.path,
+      self.referrer,
+      self.system.join("::")
+    )
+  }
+  fn positions(&self) -> impl IntoIterator<Item = ErrorPosition> { [] }
+}
+
+/// Trait for objects that can be converted into a [System].
+pub trait IntoSystem<'a> {
+  /// Convert this object into a system using an interner
+  fn into_system(self) -> System<'a>;
+}

orchid-base/src/tree.rs

@@ -0,0 +1,630 @@
+//! Generic module tree structure
+//!
+//! Used by various stages of the pipeline with different parameters
+use std::fmt;
+
+use hashbrown::HashMap;
+use itertools::Itertools as _;
+use never::Never;
+use substack::Substack;
+use trait_set::trait_set;
+
+use crate::boxed_iter::BoxedIter;
+use crate::combine::Combine;
+use crate::intern::{intern, Token};
+use crate::join::try_join_maps;
+use crate::location::CodeOrigin;
+use crate::name::{VName, VPath};
+use crate::proj_error::{ProjectError, ProjectErrorObj};
+use crate::sequence::Sequence;
+
+/// An umbrella trait for operations you can carry out on any part of the tree
+/// structure
+pub trait TreeTransforms: Sized {
+  /// Data held at the leaves of the tree
+  type Item;
+  /// Data associated with modules
+  type XMod;
+  /// Data associated with entries inside modules
+  type XEnt;
+  /// Recursive type to enable [TreeTransforms::map_data] to transform the whole
+  /// tree
+  type SelfType<T, U, V>: TreeTransforms<Item = T, XMod = U, XEnt = V>;
+
+  /// Implementation for [TreeTransforms::map_data]
+  fn map_data_rec<T, U, V>(
+    self,
+    item: &mut impl FnMut(Substack<Token<String>>, Self::Item) -> T,
+    module: &mut impl FnMut(Substack<Token<String>>, Self::XMod) -> U,
+    entry: &mut impl FnMut(Substack<Token<String>>, Self::XEnt) -> V,
+    path: Substack<Token<String>>,
+  ) -> Self::SelfType<T, U, V>;
+
+  /// Transform all the data in the tree without changing its structure
+  fn map_data<T, U, V>(
+    self,
+    mut item: impl FnMut(Substack<Token<String>>, Self::Item) -> T,
+    mut module: impl FnMut(Substack<Token<String>>, Self::XMod) -> U,
+    mut entry: impl FnMut(Substack<Token<String>>, Self::XEnt) -> V,
+  ) -> Self::SelfType<T, U, V> {
+    self.map_data_rec(&mut item, &mut module, &mut entry, Substack::Bottom)
+  }
+
+  /// Visit all elements in the tree. This is like [TreeTransforms::search] but
+  /// without the early exit
+  ///
+  /// * init - can be used for reduce, otherwise pass `()`
+  /// * callback - a callback applied on every module.
+  ///   * [`Substack<Token<String>>`] - the walked path
+  ///   * [Module] - the current module
+  ///   * `T` - data for reduce.
+  fn search_all<'a, T>(
+    &'a self,
+    init: T,
+    mut callback: impl FnMut(
+      Substack<Token<String>>,
+      ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
+      T,
+    ) -> T,
+  ) -> T {
+    let res =
+      self.search(init, |stack, member, state| Ok::<T, Never>(callback(stack, member, state)));
+    res.unwrap_or_else(|e| match e {})
+  }
+
+  /// Visit elements in the tree depth first with the provided function
+  ///
+  /// * init - can be used for reduce, otherwise pass `()`
+  /// * callback - a callback applied on every module. Can return [Err] to
+  ///   short-circuit the walk
+  ///   * [`Substack<Token<String>>`] - the walked path
+  ///   * [Module] - the current module
+  ///   * `T` - data for reduce.
+  fn search<'a, T, E>(
+    &'a self,
+    init: T,
+    mut callback: impl FnMut(
+      Substack<Token<String>>,
+      ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
+      T,
+    ) -> Result<T, E>,
+  ) -> Result<T, E> {
+    self.search_rec(init, Substack::Bottom, &mut callback)
+  }
+
+  /// Internal version of [TreeTransforms::search_all]
+  fn search_rec<'a, T, E>(
+    &'a self,
+    state: T,
+    stack: Substack<Token<String>>,
+    callback: &mut impl FnMut(
+      Substack<Token<String>>,
+      ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
+      T,
+    ) -> Result<T, E>,
+  ) -> Result<T, E>;
+}
+
+/// The member in a [ModEntry] which is associated with a name in a [Module]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ModMember<Item, XMod, XEnt> {
+  /// Arbitrary data
+  Item(Item),
+  /// A child module
+  Sub(Module<Item, XMod, XEnt>),
+}
+
+impl<Item, XMod, XEnt> TreeTransforms for ModMember<Item, XMod, XEnt> {
+  type Item = Item;
+  type XEnt = XEnt;
+  type XMod = XMod;
+  type SelfType<T, U, V> = ModMember<T, U, V>;
+
+  fn map_data_rec<T, U, V>(
+    self,
+    item: &mut impl FnMut(Substack<Token<String>>, Item) -> T,
+    module: &mut impl FnMut(Substack<Token<String>>, XMod) -> U,
+    entry: &mut impl FnMut(Substack<Token<String>>, XEnt) -> V,
+    path: Substack<Token<String>>,
+  ) -> Self::SelfType<T, U, V> {
+    match self {
+      Self::Item(it) => ModMember::Item(item(path, it)),
+      Self::Sub(sub) => ModMember::Sub(sub.map_data_rec(item, module, entry, path)),
+    }
+  }
+
+  fn search_rec<'a, T, E>(
+    &'a self,
+    state: T,
+    stack: Substack<Token<String>>,
+    callback: &mut impl FnMut(
+      Substack<Token<String>>,
+      ModMemberRef<'a, Item, XMod, XEnt>,
+      T,
+    ) -> Result<T, E>,
+  ) -> Result<T, E> {
+    match self {
+      Self::Item(it) => callback(stack, ModMemberRef::Item(it), state),
+      Self::Sub(m) => m.search_rec(state, stack, callback),
+    }
+  }
+}
+
+/// Reasons why merging trees might fail
+pub enum ConflictKind<Item: Combine, XMod: Combine, XEnt: Combine> {
+  /// Error during the merging of items
+  Item(Item::Error),
+  /// Error during the merging of module metadata
+  Module(XMod::Error),
+  /// Error during the merging of entry metadata
+  XEnt(XEnt::Error),
+  /// An item appeared in one tree where the other contained a submodule
+  ItemModule,
+}
+
+macro_rules! impl_for_conflict {
+  ($target:ty, ($($deps:tt)*), $for:ty, $body:tt) => {
+    impl<Item: Combine, XMod: Combine, XEnt: Combine> $target
+    for $for
+    where
+      Item::Error: $($deps)*,
+      XMod::Error: $($deps)*,
+      XEnt::Error: $($deps)*,
+    $body
+  };
+}
+
+impl_for_conflict!(Clone, (Clone), ConflictKind<Item, XMod, XEnt>, {
+  fn clone(&self) -> Self {
+    match self {
+      ConflictKind::Item(it_e) => ConflictKind::Item(it_e.clone()),
+      ConflictKind::Module(mod_e) => ConflictKind::Module(mod_e.clone()),
+      ConflictKind::XEnt(ent_e) => ConflictKind::XEnt(ent_e.clone()),
+      ConflictKind::ItemModule => ConflictKind::ItemModule,
+    }
+  }
+});
+
+impl_for_conflict!(fmt::Debug, (fmt::Debug), ConflictKind<Item, XMod, XEnt>, {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      ConflictKind::Item(it_e) =>
+        f.debug_tuple("TreeCombineErr::Item").field(it_e).finish(),
+      ConflictKind::Module(mod_e) =>
+        f.debug_tuple("TreeCombineErr::Module").field(mod_e).finish(),
+      ConflictKind::XEnt(ent_e) =>
+        f.debug_tuple("TreeCombineErr::XEnt").field(ent_e).finish(),
+      ConflictKind::ItemModule => write!(f, "TreeCombineErr::Item2Module"),
+    }
+  }
+});
+
+/// Error produced when two trees cannot be merged
+pub struct TreeConflict<Item: Combine, XMod: Combine, XEnt: Combine> {
+  /// Which subtree caused the failure
+  pub path: VPath,
+  /// What type of failure occurred
+  pub kind: ConflictKind<Item, XMod, XEnt>,
+}
+impl<Item: Combine, XMod: Combine, XEnt: Combine> TreeConflict<Item, XMod, XEnt> {
+  fn new(kind: ConflictKind<Item, XMod, XEnt>) -> Self { Self { path: VPath::new([]), kind } }
+
+  fn push(self, seg: Token<String>) -> Self {
+    Self { path: self.path.prefix([seg]), kind: self.kind }
+  }
+}
+
+impl_for_conflict!(Clone, (Clone), TreeConflict<Item, XMod, XEnt>, {
+  fn clone(&self) -> Self {
+    Self { path: self.path.clone(), kind: self.kind.clone() }
+  }
+});
+
+impl_for_conflict!(fmt::Debug, (fmt::Debug), TreeConflict<Item, XMod, XEnt>, {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    f.debug_struct("TreeConflict")
+      .field("path", &self.path)
+      .field("kind", &self.kind)
+      .finish()
+  }
+});
+
+impl<Item: Combine, XMod: Combine, XEnt: Combine> Combine for ModMember<Item, XMod, XEnt> {
+  type Error = TreeConflict<Item, XMod, XEnt>;
+
+  fn combine(self, other: Self) -> Result<Self, Self::Error> {
+    match (self, other) {
+      (Self::Item(i1), Self::Item(i2)) => match i1.combine(i2) {
+        Ok(i) => Ok(Self::Item(i)),
+        Err(e) => Err(TreeConflict::new(ConflictKind::Item(e))),
+      },
+      (Self::Sub(m1), Self::Sub(m2)) => m1.combine(m2).map(Self::Sub),
+      (..) => Err(TreeConflict::new(ConflictKind::ItemModule)),
+    }
+  }
+}
+
+/// Data about a name in a [Module]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ModEntry<Item, XMod, XEnt> {
+  /// The submodule or item
+  pub member: ModMember<Item, XMod, XEnt>,
+  /// Additional fields
+  pub x: XEnt,
+}
+impl<Item: Combine, XMod: Combine, XEnt: Combine> Combine for ModEntry<Item, XMod, XEnt> {
+  type Error = TreeConflict<Item, XMod, XEnt>;
+  fn combine(self, other: Self) -> Result<Self, Self::Error> {
+    match self.x.combine(other.x) {
+      Err(e) => Err(TreeConflict::new(ConflictKind::XEnt(e))),
+      Ok(x) => Ok(Self { x, member: self.member.combine(other.member)? }),
+    }
+  }
+}
+impl<Item, XMod, XEnt> ModEntry<Item, XMod, XEnt> {
+  /// Returns the item in this entry if it contains one.
+  #[must_use]
+  pub fn item(&self) -> Option<&Item> {
+    match &self.member {
+      ModMember::Item(it) => Some(it),
+      ModMember::Sub(_) => None,
+    }
+  }
+}
+
+impl<Item, XMod, XEnt> TreeTransforms for ModEntry<Item, XMod, XEnt> {
+  type Item = Item;
+  type XEnt = XEnt;
+  type XMod = XMod;
+  type SelfType<T, U, V> = ModEntry<T, U, V>;
+
+  fn map_data_rec<T, U, V>(
+    self,
+    item: &mut impl FnMut(Substack<Token<String>>, Item) -> T,
+    module: &mut impl FnMut(Substack<Token<String>>, XMod) -> U,
+    entry: &mut impl FnMut(Substack<Token<String>>, XEnt) -> V,
+    path: Substack<Token<String>>,
+  ) -> Self::SelfType<T, U, V> {
+    ModEntry {
+      member: self.member.map_data_rec(item, module, entry, path.clone()),
+      x: entry(path, self.x),
+    }
+  }
+
+  fn search_rec<'a, T, E>(
+    &'a self,
+    state: T,
+    stack: Substack<Token<String>>,
+    callback: &mut impl FnMut(
+      Substack<Token<String>>,
+      ModMemberRef<'a, Item, XMod, XEnt>,
+      T,
+    ) -> Result<T, E>,
+  ) -> Result<T, E> {
+    self.member.search_rec(state, stack, callback)
+  }
+}
+impl<Item, XMod, XEnt: Default> ModEntry<Item, XMod, XEnt> {
+  /// Wrap a member directly with trivial metadata
+  pub fn wrap(member: ModMember<Item, XMod, XEnt>) -> Self { Self { member, x: XEnt::default() } }
+  /// Wrap an item directly with trivial metadata
+  pub fn leaf(item: Item) -> Self { Self::wrap(ModMember::Item(item)) }
+}
+impl<Item, XMod: Default, XEnt: Default> ModEntry<Item, XMod, XEnt> {
+  /// Create an empty submodule
+  pub fn empty() -> Self { Self::wrap(ModMember::Sub(Module::wrap([]))) }
+
+  /// Create a module
+  #[must_use]
+  pub fn tree<K: AsRef<str>>(arr: impl IntoIterator<Item = (K, Self)>) -> Self {
+    Self::wrap(ModMember::Sub(Module::wrap(arr.into_iter().map(|(k, v)| (intern(k.as_ref()), v)))))
+  }
+
+  /// Create a record in the list passed to [ModEntry#tree] which describes a
+  /// submodule. This mostly exists to deal with strange rustfmt block
+  /// breaking behaviour
+  pub fn tree_ent<K: AsRef<str>>(key: K, arr: impl IntoIterator<Item = (K, Self)>) -> (K, Self) {
+    (key, Self::tree(arr))
+  }
+
+  /// Namespace the tree with the list of names
+  ///
+  /// The unarray is used to trick rustfmt into breaking the sub-item
+  /// into a block without breaking anything else.
+  #[must_use]
+  pub fn ns(name: impl AsRef<str>, [mut end]: [Self; 1]) -> Self {
+    let elements = name.as_ref().split("::").collect::<Vec<_>>();
+    for name in elements.into_iter().rev() {
+      end = Self::tree([(name, end)]);
+    }
+    end
+  }
+
+  fn not_mod_panic<T>() -> T { panic!("Expected module but found leaf") }
+
+  /// Return the wrapped module. Panic if the entry wraps an item
+  pub fn unwrap_mod(self) -> Module<Item, XMod, XEnt> {
+    if let ModMember::Sub(m) = self.member { m } else { Self::not_mod_panic() }
+  }
+
+  /// Return the wrapped module. Panic if the entry wraps an item
+  pub fn unwrap_mod_ref(&self) -> &Module<Item, XMod, XEnt> {
+    if let ModMember::Sub(m) = &self.member { m } else { Self::not_mod_panic() }
+  }
+}
+
+/// A module, containing imports,
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Module<Item, XMod, XEnt> {
+  /// Submodules and items by name
+  pub entries: HashMap<Token<String>, ModEntry<Item, XMod, XEnt>>,
+  /// Additional fields
+  pub x: XMod,
+}
+
+trait_set! {
+  /// A filter applied to a module tree
+  pub trait Filter<'a, Item, XMod, XEnt> =
+    for<'b> Fn(&'b ModEntry<Item, XMod, XEnt>) -> bool + Clone + 'a
+}
+
+/// A line in a [Module]
+pub type Record<Item, XMod, XEnt> = (Token<String>, ModEntry<Item, XMod, XEnt>);
+
+impl<Item, XMod, XEnt> Module<Item, XMod, XEnt> {
+  /// Returns child names for which the value matches a filter
+  #[must_use]
+  pub fn keys<'a>(
+    &'a self,
+    filter: impl for<'b> Fn(&'b ModEntry<Item, XMod, XEnt>) -> bool + 'a,
+  ) -> BoxedIter<Token<String>> {
+    Box::new((self.entries.iter()).filter(move |(_, v)| filter(v)).map(|(k, _)| k.clone()))
+  }
+
+  /// Return the module at the end of the given path
+  pub fn walk_ref<'a: 'b, 'b>(
+    &'a self,
+    prefix: &'b [Token<String>],
+    path: &'b [Token<String>],
+    filter: impl Filter<'b, Item, XMod, XEnt>,
+  ) -> Result<&'a Self, WalkError<'b>> {
+    let mut module = self;
+    for (pos, step) in path.iter().enumerate() {
+      let kind = match module.entries.get(step) {
+        None => ErrKind::Missing,
+        Some(ent) if !filter(ent) => ErrKind::Filtered,
+        Some(ModEntry { member: ModMember::Item(_), .. }) => ErrKind::NotModule,
+        Some(ModEntry { member: ModMember::Sub(next), .. }) => {
+          module = next;
+          continue;
+        },
+      };
+      let options = Sequence::new(move || module.keys(filter.clone()));
+      return Err(WalkError { kind, prefix, path, pos, options });
+    }
+    Ok(module)
+  }
+
+  /// Return the member at the end of the given path
+  ///
+  /// # Panics
+  ///
+  /// if path is empty, since the reference cannot be forwarded that way
+  pub fn walk1_ref<'a: 'b, 'b>(
+    &'a self,
+    prefix: &'b [Token<String>],
+    path: &'b [Token<String>],
+    filter: impl Filter<'b, Item, XMod, XEnt>,
+  ) -> Result<(&'a ModEntry<Item, XMod, XEnt>, &'a Self), WalkError<'b>> {
+    let (last, parent) = path.split_last().expect("Path cannot be empty");
+    let pos = path.len() - 1;
+    let module = self.walk_ref(prefix, parent, filter.clone())?;
+    let err_kind = match &module.entries.get(last) {
+      Some(entry) if filter(entry) => return Ok((entry, module)),
+      Some(_) => ErrKind::Filtered,
+      None => ErrKind::Missing,
+    };
+    let options = Sequence::new(move || module.keys(filter.clone()));
+    Err(WalkError { kind: err_kind, options, prefix, path, pos })
+  }
+
+  /// Walk from one node to another in a tree, asserting that the origin can see
+  /// the target.
+  ///
+  /// # Panics
+  ///
+  /// If the target is the root node
+  pub fn inner_walk<'a: 'b, 'b>(
+    &'a self,
+    origin: &[Token<String>],
+    target: &'b [Token<String>],
+    is_exported: impl for<'c> Fn(&'c ModEntry<Item, XMod, XEnt>) -> bool + Clone + 'b,
+  ) -> Result<(&'a ModEntry<Item, XMod, XEnt>, &'a Self), WalkError<'b>> {
+    let ignore_vis_len = 1 + origin.iter().zip(target).take_while(|(a, b)| a == b).count();
+    if target.len() <= ignore_vis_len {
+      return self.walk1_ref(&[], target, |_| true);
+    }
+    let (ignore_vis_path, hidden_path) = target.split_at(ignore_vis_len);
+    let first_divergence = self.walk_ref(&[], ignore_vis_path, |_| true)?;
+    first_divergence.walk1_ref(ignore_vis_path, hidden_path, is_exported)
+  }
+
+  /// Wrap entry table in a module with trivial metadata
+  pub fn wrap(entries: impl IntoIterator<Item = Record<Item, XMod, XEnt>>) -> Self
+  where XMod: Default {
+    Self { entries: entries.into_iter().collect(), x: XMod::default() }
+  }
+}
+
+impl<Item, XMod, XEnt> TreeTransforms for Module<Item, XMod, XEnt> {
+  type Item = Item;
+  type XEnt = XEnt;
+  type XMod = XMod;
+  type SelfType<T, U, V> = Module<T, U, V>;
+
+  fn map_data_rec<T, U, V>(
+    self,
+    item: &mut impl FnMut(Substack<Token<String>>, Item) -> T,
+    module: &mut impl FnMut(Substack<Token<String>>, XMod) -> U,
+    entry: &mut impl FnMut(Substack<Token<String>>, XEnt) -> V,
+    path: Substack<Token<String>>,
+  ) -> Self::SelfType<T, U, V> {
+    Module {
+      x: module(path.clone(), self.x),
+      entries: (self.entries.into_iter())
+        .map(|(k, e)| (k.clone(), e.map_data_rec(item, module, entry, path.push(k))))
+        .collect(),
+    }
+  }
+
+  fn search_rec<'a, T, E>(
+    &'a self,
+    mut state: T,
+    stack: Substack<Token<String>>,
+    callback: &mut impl FnMut(
+      Substack<Token<String>>,
+      ModMemberRef<'a, Item, XMod, XEnt>,
+      T,
+    ) -> Result<T, E>,
+  ) -> Result<T, E> {
+    state = callback(stack.clone(), ModMemberRef::Mod(self), state)?;
+    for (key, value) in &self.entries {
+      state = value.search_rec(state, stack.push(key.clone()), callback)?;
+    }
+    Ok(state)
+  }
+}
+
+impl<Item: Combine, XMod: Combine, XEnt: Combine> Combine for Module<Item, XMod, XEnt> {
+  type Error = TreeConflict<Item, XMod, XEnt>;
+  fn combine(self, Self { entries, x }: Self) -> Result<Self, Self::Error> {
+    let entries =
+      try_join_maps(self.entries, entries, |k, l, r| l.combine(r).map_err(|e| e.push(k.clone())))?;
+    let x = (self.x.combine(x)).map_err(|e| TreeConflict::new(ConflictKind::Module(e)))?;
+    Ok(Self { x, entries })
+  }
+}
+
+impl<Item: fmt::Display, TExt: fmt::Display, XEnt: fmt::Display> fmt::Display
+  for Module<Item, TExt, XEnt>
+{
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "module {{")?;
+    for (name, ModEntry { member, x: extra }) in &self.entries {
+      match member {
+        ModMember::Sub(module) => write!(f, "\n{name} {extra} = {module}"),
+        ModMember::Item(item) => write!(f, "\n{name} {extra} = {item}"),
+      }?;
+    }
+    write!(f, "\n\n{}\n}}", &self.x)
+  }
+}
+
+/// A non-owning version of [ModMember]. Either an item-ref or a module-ref.
+pub enum ModMemberRef<'a, Item, XMod, XEnt> {
+  /// Leaf
+  Item(&'a Item),
+  /// Node
+  Mod(&'a Module<Item, XMod, XEnt>),
+}
+
+/// Possible causes why the path could not be walked
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum ErrKind {
+  /// `require_exported` was set to `true` and a module wasn't exported
+  Filtered,
+  /// A module was not found
+  Missing,
+  /// The path leads into a leaf node
+  NotModule,
+}
+
+#[derive(Clone)]
+/// All details about a failed tree-walk
+pub struct WalkError<'a> {
+  /// Failure mode
+  kind: ErrKind,
+  /// Path to the module where the walk started
+  prefix: &'a [Token<String>],
+  /// Planned walk path
+  path: &'a [Token<String>],
+  /// Index into walked path where the error occurred
+  pos: usize,
+  /// Alternatives to the failed steps
+  options: Sequence<'a, Token<String>>,
+}
+impl<'a> WalkError<'a> {
+  /// Total length of the path represented by this error
+  #[must_use]
+  pub fn depth(&self) -> usize { self.prefix.len() + self.pos + 1 }
+
+  /// Attach a location to the error and convert into trait object for reporting
+  #[must_use]
+  pub fn at(self, origin: &CodeOrigin) -> ProjectErrorObj {
+    let details = WalkErrorDetails {
+      origin: origin.clone(),
+      path: VName::new((self.prefix.iter()).chain(self.path.iter().take(self.pos + 1)).cloned())
+        .expect("empty paths don't cause an error"),
+      options: self.options.iter().collect(),
+    };
+    match self.kind {
+      ErrKind::Filtered => FilteredError(details).pack(),
+      ErrKind::Missing => MissingError(details).pack(),
+      ErrKind::NotModule => NotModuleError(details).pack(),
+    }
+  }
+  /// Construct an error for the very last item in a slice. This is often done
+  /// outside [super::tree] so it gets a function rather than exposing the
+  /// fields of [WalkError]
+  pub fn last(
+    path: &'a [Token<String>],
+    kind: ErrKind,
+    options: Sequence<'a, Token<String>>,
+  ) -> Self {
+    WalkError { kind, path, options, pos: path.len() - 1, prefix: &[] }
+  }
+}
+impl<'a> fmt::Debug for WalkError<'a> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    f.debug_struct("WalkError")
+      .field("kind", &self.kind)
+      .field("prefix", &self.prefix)
+      .field("path", &self.path)
+      .field("pos", &self.pos)
+      .finish_non_exhaustive()
+  }
+}
+
+struct WalkErrorDetails {
+  path: VName,
+  options: Vec<Token<String>>,
+  origin: CodeOrigin,
+}
+impl WalkErrorDetails {
+  fn print_options(&self) -> String { format!("options are {}", self.options.iter().join(", ")) }
+}
+
+struct FilteredError(WalkErrorDetails);
+impl ProjectError for FilteredError {
+  const DESCRIPTION: &'static str = "The path leads into a private module";
+  fn one_position(&self) -> CodeOrigin { self.0.origin.clone() }
+  fn message(&self) -> String { format!("{} is private, {}", self.0.path, self.0.print_options()) }
+}
+
+struct MissingError(WalkErrorDetails);
+impl ProjectError for MissingError {
+  const DESCRIPTION: &'static str = "Nonexistent path";
+  fn one_position(&self) -> CodeOrigin { self.0.origin.clone() }
+  fn message(&self) -> String {
+    format!("{} does not exist, {}", self.0.path, self.0.print_options())
+  }
+}
+
+struct NotModuleError(WalkErrorDetails);
+impl ProjectError for NotModuleError {
+  const DESCRIPTION: &'static str = "The path leads into a leaf";
+  fn one_position(&self) -> CodeOrigin { self.0.origin.clone() }
+  fn message(&self) -> String {
+    format!("{} is not a module, {}", self.0.path, self.0.print_options())
+  }
+}

orchid-base/src/try_from_expr.rs

@@ -0,0 +1,30 @@
+//! Conversions from Orchid expressions to Rust values. Many APIs and
+//! [super::fn_bridge] in particular use this to automatically convert values on
+//! the boundary. Failures cause an interpreter exit
+
+use super::error::RTResult;
+use crate::interpreter::nort::{ClauseInst, Expr};
+use crate::location::CodeLocation;
+
+/// Types automatically convertible from an [Expr]. Most notably, this is how
+/// foreign functions request automatic argument downcasting.
+pub trait TryFromExpr: Sized {
+  /// Match and clone the value out of an [Expr]
+  fn from_expr(expr: Expr) -> RTResult<Self>;
+}
+
+impl TryFromExpr for Expr {
+  fn from_expr(expr: Expr) -> RTResult<Self> { Ok(expr) }
+}
+
+impl TryFromExpr for ClauseInst {
+  fn from_expr(expr: Expr) -> RTResult<Self> { Ok(expr.clsi()) }
+}
+
+/// Request a value of a particular type and also return its location for
+/// further error reporting
+#[derive(Debug, Clone)]
+pub struct WithLoc<T>(pub CodeLocation, pub T);
+impl<T: TryFromExpr> TryFromExpr for WithLoc<T> {
+  fn from_expr(expr: Expr) -> RTResult<Self> { Ok(Self(expr.location(), T::from_expr(expr)?)) }
+}

orchid-base/src/virt_fs/common.rs

@@ -0,0 +1,95 @@
+use std::rc::Rc;
+use std::sync::Arc;
+
+use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
+use crate::intern::Token;
+use crate::name::{PathSlice, VPath};
+
+/// Represents the result of loading code from a string-tree form such
+/// as the file system. Cheap to clone.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum Loaded {
+  /// Conceptually equivalent to a sourcefile
+  Code(Arc<String>),
+  /// Conceptually equivalent to the list of *.orc files in a folder, without
+  /// the extension
+  Collection(Arc<Vec<Token<String>>>),
+}
+impl Loaded {
+  /// Is the loaded item source code (not a collection)?
+  pub fn is_code(&self) -> bool { matches!(self, Loaded::Code(_)) }
+  /// Collect the elements in a collection rreport
+  pub fn collection(items: impl IntoIterator<Item = Token<String>>) -> Self {
+    Self::Collection(Arc::new(items.into_iter().collect()))
+  }
+}
+
+/// Returned by any source loading callback
+pub type FSResult = Result<Loaded, ErrorSansOriginObj>;
+
+/// Type that indicates the type of an entry without reading the contents
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+pub enum FSKind {
+  /// Invalid path or read error
+  None,
+  /// Source code
+  Code,
+  /// Internal tree node
+  Collection,
+}
+
+/// Distinguished error for missing code
+#[derive(Clone, PartialEq, Eq)]
+pub struct CodeNotFound(pub VPath);
+impl CodeNotFound {
+  /// Instantiate error
+  pub fn new(path: VPath) -> Self { Self(path) }
+}
+impl ErrorSansOrigin for CodeNotFound {
+  const DESCRIPTION: &'static str = "No source code for path";
+  fn message(&self) -> String { format!("{} not found", self.0) }
+}
+
+/// A simplified view of a file system for the purposes of source code loading.
+/// This includes the real FS and source code, but also various in-memory
+/// formats and other sources for libraries and dependencies.
+pub trait VirtFS {
+  /// Implementation of [VirtFS::read]
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult;
+  /// Discover information about a path without reading it.
+  ///
+  /// Implement this if your vfs backend can do expensive operations
+  fn kind(&self, path: &PathSlice) -> FSKind {
+    match self.read(path) {
+      Err(_) => FSKind::None,
+      Ok(Loaded::Code(_)) => FSKind::Code,
+      Ok(Loaded::Collection(_)) => FSKind::Collection,
+    }
+  }
+  /// Convert a path into a human-readable string that is meaningful in the
+  /// target context.
+  fn display(&self, path: &[Token<String>]) -> Option<String>;
+  /// Convert the FS handler into a type-erased version of itself for packing in
+  /// a tree.
+  fn rc(self) -> Rc<dyn VirtFS>
+  where Self: Sized + 'static {
+    Rc::new(self)
+  }
+  /// Read a path, returning either a text file, a directory listing or an
+  /// error. Wrapper for [VirtFS::get]
+  fn read(&self, path: &PathSlice) -> FSResult { self.get(path, path) }
+}
+
+impl VirtFS for &dyn VirtFS {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    (*self).get(path, full_path)
+  }
+  fn display(&self, path: &[Token<String>]) -> Option<String> { (*self).display(path) }
+}
+
+impl<T: VirtFS + ?Sized> VirtFS for Rc<T> {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    (**self).get(path, full_path)
+  }
+  fn display(&self, path: &[Token<String>]) -> Option<String> { (**self).display(path) }
+}

orchid-base/src/virt_fs/decl.rs

@@ -0,0 +1,73 @@
+use std::rc::Rc;
+use std::sync::Arc;
+
+use super::common::CodeNotFound;
+use super::{FSResult, Loaded, VirtFS};
+use crate::intern::Token;
+use crate::proj_error::ErrorSansOrigin;
+use crate::name::PathSlice;
+use crate::tree::{ModEntry, ModMember};
+use crate::combine::Combine;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ConflictingTrees;
+
+impl Combine for Rc<dyn VirtFS> {
+  type Error = ConflictingTrees;
+  fn combine(self, _: Self) -> Result<Self, Self::Error> { Err(ConflictingTrees) }
+}
+
+impl Combine for Arc<dyn VirtFS> {
+  type Error = ConflictingTrees;
+  fn combine(self, _: Self) -> Result<Self, Self::Error> { Err(ConflictingTrees) }
+}
+
+impl<'a> Combine for &'a dyn VirtFS {
+  type Error = ConflictingTrees;
+  fn combine(self, _: Self) -> Result<Self, Self::Error> { Err(ConflictingTrees) }
+}
+
+/// A declarative in-memory tree with [VirtFS] objects for leaves. Paths are
+/// followed to a leaf and the leftover handled by it.
+pub type DeclTree = ModEntry<Rc<dyn VirtFS>, (), ()>;
+
+impl VirtFS for DeclTree {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    match &self.member {
+      ModMember::Item(it) => it.get(path, full_path),
+      ModMember::Sub(module) => match path.split_first() {
+        None => Ok(Loaded::collection(module.keys(|_| true))),
+        Some((head, tail)) => (module.entries.get(head))
+          .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
+          .and_then(|ent| ent.get(tail, full_path)),
+      },
+    }
+  }
+
+  fn display(&self, path: &[Token<String>]) -> Option<String> {
+    let (head, tail) = path.split_first()?;
+    match &self.member {
+      ModMember::Item(it) => it.display(path),
+      ModMember::Sub(module) => module.entries.get(head)?.display(tail),
+    }
+  }
+}
+
+impl VirtFS for String {
+  fn display(&self, _: &[Token<String>]) -> Option<String> { None }
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    (path.is_empty().then(|| Loaded::Code(Arc::new(self.as_str().to_string()))))
+      .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
+  }
+}
+
+impl<'a> VirtFS for &'a str {
+  fn display(&self, _: &[Token<String>]) -> Option<String> { None }
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    (path.is_empty().then(|| Loaded::Code(Arc::new(self.to_string()))))
+      .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
+  }
+}
+
+/// Insert a file by cleartext contents in the [DeclTree].
+pub fn decl_file(s: &str) -> DeclTree { DeclTree::leaf(Rc::new(s.to_string())) }

orchid-base/src/virt_fs/dir.rs

@@ -0,0 +1,121 @@
+use std::cell::RefCell;
+use std::fs::File;
+use std::io;
+use std::io::{ErrorKind, Read};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex};
+
+use hashbrown::HashMap;
+
+use super::common::CodeNotFound;
+use super::{FSResult, Loaded, VirtFS};
+use crate::intern::{intern, Token};
+use crate::proj_error::{ErrorSansOrigin, ErrorSansOriginObj};
+use crate::name::PathSlice;
+
+#[derive(Clone)]
+struct OpenError {
+  file: Arc<Mutex<io::Error>>,
+  dir: Arc<Mutex<io::Error>>,
+}
+impl OpenError {
+  pub fn wrap(file: io::Error, dir: io::Error) -> ErrorSansOriginObj {
+    Self { dir: Arc::new(Mutex::new(dir)), file: Arc::new(Mutex::new(file)) }.pack()
+  }
+}
+impl ErrorSansOrigin for OpenError {
+  const DESCRIPTION: &'static str = "A file system error occurred";
+  fn message(&self) -> String {
+    let Self { dir, file } = self;
+    format!(
+      "File system errors other than not found occurred\n\
+      as a file: {}\nas a directory: {}",
+      file.lock().unwrap(),
+      dir.lock().unwrap()
+    )
+  }
+}
+
+#[derive(Clone)]
+struct IOError(Arc<Mutex<io::Error>>);
+impl IOError {
+  pub fn wrap(inner: io::Error) -> ErrorSansOriginObj { Self(Arc::new(Mutex::new(inner))).pack() }
+}
+impl ErrorSansOrigin for IOError {
+  const DESCRIPTION: &'static str = "an I/O error occured";
+  fn message(&self) -> String { format!("File read error: {}", self.0.lock().unwrap()) }
+}
+
+#[derive(Clone)]
+struct NotUtf8(PathBuf);
+impl NotUtf8 {
+  pub fn wrap(path: &Path) -> ErrorSansOriginObj { Self(path.to_owned()).pack() }
+}
+impl ErrorSansOrigin for NotUtf8 {
+  const DESCRIPTION: &'static str = "Source files must be UTF-8";
+  fn message(&self) -> String {
+    format!("{} is a source file but contains invalid UTF-8", self.0.display())
+  }
+}
+
+/// A real file system directory linked into the virtual FS
+pub struct DirNode {
+  cached: RefCell<HashMap<PathBuf, FSResult>>,
+  root: PathBuf,
+  suffix: &'static str,
+}
+impl DirNode {
+  /// Reference a real file system directory in the virtual FS
+  pub fn new(root: PathBuf, suffix: &'static str) -> Self {
+    assert!(suffix.starts_with('.'), "Extension must begin with .");
+    Self { cached: RefCell::default(), root, suffix }
+  }
+
+  fn ext(&self) -> &str { self.suffix.strip_prefix('.').expect("Checked in constructor") }
+
+  fn load_file(&self, fpath: &Path, orig_path: &PathSlice) -> FSResult {
+    match fpath.read_dir() {
+      Err(dir_e) => {
+        let fpath = fpath.with_extension(self.ext());
+        let mut file =
+          File::open(&fpath).map_err(|file_e| match (dir_e.kind(), file_e.kind()) {
+            (ErrorKind::NotFound, ErrorKind::NotFound) =>
+              CodeNotFound::new(orig_path.to_vpath()).pack(),
+            _ => OpenError::wrap(file_e, dir_e),
+          })?;
+        let mut buf = Vec::new();
+        file.read_to_end(&mut buf).map_err(IOError::wrap)?;
+        let text = String::from_utf8(buf).map_err(|_| NotUtf8::wrap(&fpath))?;
+        Ok(Loaded::Code(Arc::new(text)))
+      },
+      Ok(dir) => Ok(Loaded::collection(dir.filter_map(|ent_r| {
+        let ent = ent_r.ok()?;
+        let name = ent.file_name().into_string().ok()?;
+        match ent.metadata().ok()?.is_dir() {
+          false => Some(intern(name.strip_suffix(self.suffix)?)),
+          true => Some(intern(&name)),
+        }
+      }))),
+    }
+  }
+
+  fn mk_pathbuf(&self, path: &[Token<String>]) -> PathBuf {
+    let mut fpath = self.root.clone();
+    path.iter().for_each(|seg| fpath.push(seg.as_str()));
+    fpath
+  }
+}
+impl VirtFS for DirNode {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    let fpath = self.mk_pathbuf(path);
+    let mut binding = self.cached.borrow_mut();
+    let (_, res) = (binding.raw_entry_mut().from_key(&fpath))
+      .or_insert_with(|| (fpath.clone(), self.load_file(&fpath, full_path)));
+    res.clone()
+  }
+
+  fn display(&self, path: &[Token<String>]) -> Option<String> {
+    let pathbuf = self.mk_pathbuf(path).with_extension(self.ext());
+    Some(pathbuf.to_string_lossy().to_string())
+  }
+}

orchid-base/src/virt_fs/embed.rs

@@ -0,0 +1,74 @@
+use std::sync::Arc;
+
+use itertools::Itertools as _;
+use rust_embed::RustEmbed;
+
+use super::common::CodeNotFound;
+use super::{FSResult, Loaded, VirtFS};
+use crate::intern::{intern, Token};
+use crate::proj_error::ErrorSansOrigin;
+use crate::location::CodeGenInfo;
+use crate::name::PathSlice;
+use crate::tree::{ModEntry, ModMember, Module};
+
+/// An in-memory FS tree for libraries managed internally by the interpreter
+pub struct EmbeddedFS {
+  tree: Module<Arc<String>, (), ()>,
+  suffix: &'static str,
+  gen: CodeGenInfo,
+}
+impl EmbeddedFS {
+  /// Expose a directory embedded in a binary wiht [RustEmbed] to the
+  /// interpreter
+  pub fn new<T: RustEmbed>(suffix: &'static str, gen: CodeGenInfo) -> Self {
+    let mut tree = Module::wrap([]);
+    for path in T::iter() {
+      let data_buf = T::get(&path).expect("path from iterator").data.to_vec();
+      let data = String::from_utf8(data_buf).expect("embed must be utf8");
+      let mut cur_node = &mut tree;
+      let path_no_suffix = path.strip_suffix(suffix).expect("embed filtered for suffix");
+      let mut segments = path_no_suffix.split('/').map(intern);
+      let mut cur_seg = segments.next().expect("Embed is a directory");
+      for next_seg in segments {
+        if !cur_node.entries.contains_key(&cur_seg) {
+          let ent = ModEntry::wrap(ModMember::Sub(Module::wrap([])));
+          cur_node.entries.insert(cur_seg.clone(), ent);
+        }
+        let ent = cur_node.entries.get_mut(&cur_seg).expect("just constructed");
+        match &mut ent.member {
+          ModMember::Sub(submod) => cur_node = submod,
+          _ => panic!("Aliased file and folder"),
+        };
+        cur_seg = next_seg;
+      }
+      let data_ent = ModEntry::wrap(ModMember::Item(Arc::new(data)));
+      let prev = cur_node.entries.insert(cur_seg, data_ent);
+      debug_assert!(prev.is_none(), "file name unique");
+    }
+    // if gen.generator == "std" {
+    //   panic!(
+    //     "{:?}",
+    //     tree.map_data(&|_, s| (), &|_, x| x, &|_, x| x, Substack::Bottom)
+    //   );
+    // };
+    Self { gen, suffix, tree }
+  }
+}
+
+impl VirtFS for EmbeddedFS {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+    if path.is_empty() {
+      return Ok(Loaded::collection(self.tree.keys(|_| true)));
+    }
+    let entry = (self.tree.walk1_ref(&[], path, |_| true))
+      .map_err(|_| CodeNotFound::new(full_path.to_vpath()).pack())?;
+    Ok(match &entry.0.member {
+      ModMember::Item(text) => Loaded::Code(text.clone()),
+      ModMember::Sub(sub) => Loaded::collection(sub.keys(|_| true)),
+    })
+  }
+  fn display(&self, path: &[Token<String>]) -> Option<String> {
+    let Self { gen, suffix, .. } = self;
+    Some(format!("{}{suffix} in {gen}", path.iter().join("/")))
+  }
+}

orchid-base/src/virt_fs/mod.rs

@@ -0,0 +1,18 @@
+//! Abstractions and primitives to help define the namespace tree used by
+//! Orchid.
+//!
+//! Although this may make it seem like the namespace tree is very flexible,
+//! libraries are generally permitted and expected to hardcode their own
+//! location in the tree, so it's up to the embedder to ensure that the flexible
+//! structure retains the assumed location of all code.
+mod common;
+mod decl;
+mod dir;
+mod embed;
+mod prefix;
+
+pub use common::{CodeNotFound, FSKind, FSResult, Loaded, VirtFS};
+pub use decl::{decl_file, DeclTree};
+pub use dir::DirNode;
+pub use embed::EmbeddedFS;
+pub use prefix::PrefixFS;

orchid-base/src/virt_fs/prefix.rs

@@ -0,0 +1,38 @@
+use itertools::Itertools;
+
+use super::common::CodeNotFound;
+use super::VirtFS;
+use crate::intern::Token;
+use crate::name::{PathSlice, VPath};
+use crate::proj_error::ErrorSansOrigin;
+
+/// Modify the prefix of a nested file tree
+pub struct PrefixFS<'a> {
+  remove: VPath,
+  add: VPath,
+  wrapped: Box<dyn VirtFS + 'a>,
+}
+impl<'a> PrefixFS<'a> {
+  /// Modify the prefix of a file tree
+  pub fn new(wrapped: impl VirtFS + 'a, remove: impl AsRef<str>, add: impl AsRef<str>) -> Self {
+    Self {
+      wrapped: Box::new(wrapped),
+      remove: VPath::parse(remove.as_ref()),
+      add: VPath::parse(add.as_ref()),
+    }
+  }
+  fn proc_path(&self, path: &[Token<String>]) -> Option<Vec<Token<String>>> {
+    let path = path.strip_prefix(self.remove.as_slice())?;
+    Some(self.add.0.iter().chain(path).cloned().collect_vec())
+  }
+}
+impl<'a> VirtFS for PrefixFS<'a> {
+  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> super::FSResult {
+    let path =
+      self.proc_path(path).ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())?;
+    self.wrapped.get(&path, full_path)
+  }
+  fn display(&self, path: &[Token<String>]) -> Option<String> {
+    self.wrapped.display(&self.proc_path(path)?)
+  }
+}