Public API and docs

Cargo.lock

@@ -140,7 +140,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.13",
 ]
 
 [[package]]
@@ -298,8 +298,8 @@ dependencies = [
  "hashbrown 0.13.2",
  "itertools",
  "ordered-float",
- "smallvec",
  "thiserror",
+ "trait-set",
 ]
 
 [[package]]
@@ -352,12 +352,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "smallvec"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
-
 [[package]]
 name = "stacker"
 version = "0.1.15"
@@ -377,6 +371,17 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "syn"
 version = "2.0.13"
@@ -405,7 +410,18 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.13",
+]
+
+[[package]]
+name = "trait-set"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b79e2e9c9ab44c6d7c20d5976961b47e8f49ac199154daa514b77cd1ab536625"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
 ]
 
 [[package]]

Cargo.toml

@@ -6,6 +6,15 @@ authors = [
   "Lawrence Bethlenfalvy <lbfalvy@protonmail.com>"
 ]
 
+[lib]
+name = "orchid"
+path = "src/lib.rs"
+
+[[bin]]
+name = "orcx"
+path = "src/bin/main.rs"
+doc = false
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -14,6 +23,6 @@ chumsky = "0.9.2"
 hashbrown = "0.13.2"
 ordered-float = "3.0"
 itertools = "0.10"
-smallvec = { version = "1.10.0", features = ['const_generics'] }
 dyn-clone = "1.0.11"
 clap = { version = "4.2.4", features = ["derive"] }
+trait-set = "0.3.0"

src/bin/main.rs

@@ -1,20 +1,45 @@
-use std::path::Path;
+use std::borrow::Borrow;
+use std::fs::File;
+use std::path::{Path, PathBuf};
 use std::rc::Rc;
 
+use clap::Parser;
 use hashbrown::HashMap;
 use itertools::Itertools;
-
+use orchid::interner::{InternedDisplay, Interner, Sym};
-use crate::external::handle;
+use orchid::pipeline::file_loader::{mk_cache, Loaded};
-use crate::interner::{InternedDisplay, Interner, Sym};
+use orchid::pipeline::{
-use crate::interpreter::Return;
-use crate::pipeline::file_loader::{mk_cache, Loaded};
-use crate::pipeline::{
   collect_consts, collect_rules, from_const_tree, parse_layer, ProjectTree,
 };
-use crate::representations::sourcefile::{FileEntry, Import};
+use orchid::rule::Repo;
-use crate::representations::{ast_to_postmacro, postmacro_to_interpreted};
+use orchid::sourcefile::{FileEntry, Import};
-use crate::rule::Repo;
+use orchid::{ast_to_interpreted, interpreter, stl};
-use crate::{external, interpreter, xloop};
+
+/// Orchid interpreter
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+  /// Folder containing main.orc
+  #[arg(short, long, default_value = ".")]
+  pub project: String,
+}
+impl Args {
+  pub fn chk_proj(&self) -> Result<(), String> {
+    let mut path = PathBuf::from(&self.project);
+    path.push(PathBuf::from("main.orc"));
+    if File::open(&path).is_ok() {
+      Ok(())
+    } else {
+      Err(format!("{} not found", path.display()))
+    }
+  }
+}
+
+fn main() {
+  let args = Args::parse();
+  args.chk_proj().unwrap_or_else(|e| panic!("{e}"));
+  run_dir(PathBuf::try_from(args.project).unwrap().borrow());
+}
 
 static PRELUDE_TXT: &str = r#"
 import std::(
@@ -63,7 +88,7 @@ fn entrypoint(i: &Interner) -> Sym {
 
 fn load_environment(i: &Interner) -> ProjectTree {
   let env = from_const_tree(
-    HashMap::from([(i.i("std"), external::std::std(i))]),
+    HashMap::from([(i.i("std"), stl::mk_stl(i))]),
     &[i.i("std")],
     i,
   );
@@ -90,7 +115,6 @@ fn load_dir(i: &Interner, dir: &Path) -> ProjectTree {
     .expect("Failed to load source code")
 }
 
-#[allow(unused)]
 pub fn run_dir(dir: &Path) {
   let i = Interner::new();
   let project = load_dir(&i, dir);
@@ -113,7 +137,11 @@ pub fn run_dir(dir: &Path) {
     let displayname = i.extern_vec(*name).join("::");
     let macro_timeout = 100;
     println!("Executing macros in {displayname}...",);
-    let unmatched = xloop!(let mut idx = 0; idx < macro_timeout; idx += 1; {
+    let mut idx = 0;
+    let unmatched = loop {
+      if idx == macro_timeout {
+        panic!("Macro execution in {displayname} didn't halt")
+      }
       match repo.step(&tree) {
         None => break tree,
         Some(phase) => {
@@ -121,10 +149,10 @@ pub fn run_dir(dir: &Path) {
           tree = phase;
         },
       }
-    }; panic!("Macro execution in {displayname} didn't halt"));
+      idx += 1;
-    let pmtree = ast_to_postmacro::expr(&unmatched)
+    };
+    let runtree = ast_to_interpreted(&unmatched)
       .unwrap_or_else(|e| panic!("Postmacro conversion error: {e}"));
-    let runtree = postmacro_to_interpreted::expr(&pmtree);
     exec_table.insert(*name, runtree);
   }
   println!("macro execution complete");
@@ -139,9 +167,9 @@ pub fn run_dir(dir: &Path) {
         .join(", ")
     )
   });
-  let io_handler = handle;
+  let io_handler = orchid::stl::handleIO;
   let ret = interpreter::run_handler(entrypoint.clone(), io_handler, ctx);
-  let Return { gas, state, inert } =
+  let interpreter::Return { gas, state, inert } =
     ret.unwrap_or_else(|e| panic!("Runtime error: {}", e));
   if inert {
     println!("Settled at {}", state.expr().clause.bundle(&i));

src/cli.rs

@@ -1,22 +0,0 @@
-use std::fmt::Display;
-use std::io::{stdin, stdout, BufRead, Write};
-
-pub fn prompt<T: Display, E: Display>(
-  prompt: &str,
-  default: T,
-  mut try_cast: impl FnMut(String) -> Result<T, E>,
-) -> T {
-  loop {
-    print!("{prompt} ({default}): ");
-    stdout().lock().flush().unwrap();
-    let mut input = String::with_capacity(100);
-    stdin().lock().read_line(&mut input).unwrap();
-    if input.is_empty() {
-      return default;
-    }
-    match try_cast(input) {
-      Ok(t) => return t,
-      Err(e) => println!("Error: {e}"),
-    }
-  }
-}

src/foreign.rs

@@ -1,3 +1,7 @@
+//! Interaction with foreign code
+//!
+//! Structures and traits used in the exposure of external functions and values
+//! to Orchid code
 use std::any::Any;
 use std::error::Error;
 use std::fmt::{Debug, Display};
@@ -11,6 +15,8 @@ pub use crate::representations::interpreted::Clause;
 use crate::representations::interpreted::ExprInst;
 use crate::representations::Primitive;
 
+/// Information returned by [Atomic::run]. This mirrors
+/// [crate::interpreter::Return] but with a clause instead of an Expr.
 pub struct AtomicReturn {
   pub clause: Clause,
   pub gas: Option<usize>,
@@ -23,12 +29,14 @@ impl AtomicReturn {
   }
 }
 
-// Aliases for concise macros
+/// A type-erased error in external code
 pub type RcError = Rc<dyn ExternError>;
+/// Returned by [Atomic::run]
 pub type AtomicResult = Result<AtomicReturn, RuntimeError>;
+/// Returned by [ExternFn::apply]
 pub type XfnResult = Result<Clause, RcError>;
-pub type RcExpr = ExprInst;
 
+/// Errors produced by external code
 pub trait ExternError: Display {
   fn into_extern(self) -> Rc<dyn ExternError>
   where
@@ -50,11 +58,15 @@ impl Error for dyn ExternError {}
 /// the executor. Since Orchid lacks basic numerical operations,
 /// these are also external functions.
 pub trait ExternFn: DynClone {
+  /// Display name of the function
   fn name(&self) -> &str;
+  /// Combine the function with an argument to produce a new clause
   fn apply(&self, arg: ExprInst, ctx: Context) -> XfnResult;
-  fn hash(&self, state: &mut dyn std::hash::Hasher) {
+  /// Hash the name to get a somewhat unique hash.
-    state.write_str(self.name())
+  fn hash(&self, mut state: &mut dyn std::hash::Hasher) {
+    self.name().hash(&mut state)
   }
+  /// Wrap this function in a clause to be placed in an [AtomicResult].
   fn to_xfn_cls(self) -> Clause
   where
     Self: Sized + 'static,
@@ -80,12 +92,20 @@ impl Debug for dyn ExternFn {
   }
 }
 
+/// Functionality the interpreter needs to handle a value
 pub trait Atomic: Any + Debug + DynClone
 where
   Self: 'static,
 {
+  /// Casts this value to [Any] so that its original value can be salvaged
+  /// during introspection by other external code. There is no other way to
+  /// interact with values of unknown types at the moment.
   fn as_any(&self) -> &dyn Any;
+  /// Attempt to normalize this value. If it wraps a value, this should report
+  /// inert. If it wraps a computation, it should execute one logical step of
+  /// the computation and return a structure representing the ntext.
   fn run(&self, ctx: Context) -> AtomicResult;
+  /// Wrap the atom in a clause to be placed in an [AtomicResult].
   fn to_atom_cls(self) -> Clause
   where
     Self: Sized,
@@ -96,12 +116,11 @@ where
 
 /// Represents a black box unit of code with its own normalization steps.
 /// Typically [ExternFn] will produce an [Atom] when applied to a [Clause],
-/// this [Atom] will then forward `run_*` calls to the argument until it
+/// this [Atom] will then forward `run` calls to the argument until it becomes
-/// yields [InternalError::NonReducible] at which point the [Atom] will
+/// inert at which point the [Atom] will validate and process the argument,
-/// validate and process the argument, returning a different [Atom]
+/// returning a different [Atom] intended for processing by external code, a new
-/// intended for processing by external code, a new [ExternFn] to capture
+/// [ExternFn] to capture an additional argument, or an Orchid expression
-/// an additional argument, or an Orchid expression
+/// to pass control back to the interpreter.btop
-/// to pass control back to the interpreter.
 pub struct Atom(pub Box<dyn Atomic>);
 impl Atom {
   pub fn new<T: 'static + Atomic>(data: T) -> Self {
@@ -110,8 +129,8 @@ impl Atom {
   pub fn data(&self) -> &dyn Atomic {
     self.0.as_ref() as &dyn Atomic
   }
-  pub fn try_cast<T: Atomic>(&self) -> Result<&T, ()> {
+  pub fn try_cast<T: Atomic>(&self) -> Option<&T> {
-    self.data().as_any().downcast_ref().ok_or(())
+    self.data().as_any().downcast_ref()
   }
   pub fn is<T: 'static>(&self) -> bool {
     self.data().as_any().is::<T>()

src/foreign_macros/atomic_defaults.rs

@@ -2,10 +2,10 @@
 use crate::foreign::Atomic;
 
 /// A macro that generates the straightforward, syntactically invariant part of
-/// implementing [Atomic]. Implemented fns are [Atomic::as_any],
+/// implementing [Atomic].
-/// [Atomic::definitely_eq] and [Atomic::hash].
 ///
-/// It depends on [Eq] and [Hash]
+/// Currently implements
+/// - [Atomic::as_any]
 #[macro_export]
 macro_rules! atomic_defaults {
   () => {

src/foreign_macros/atomic_impl.rs

@@ -14,10 +14,10 @@ use crate::representations::Primitive;
 /// A macro that generates implementations of [Atomic] to simplify the
 /// development of external bindings for Orchid.
 ///
-/// The macro depends on implementations of [AsRef<Clause>] and [From<(&Self,
+/// The macro depends on implementations of [`AsRef<Clause>`] and
-/// Clause)>] for extracting the clause to be processed and then reconstructing
+/// [`From<(&Self, Clause)>`] for extracting the clause to be processed and then
-/// the [Atomic]. Naturally, supertraits of [Atomic] are also dependencies.
+/// reconstructing the [Atomic]. Naturally, supertraits of [Atomic] are also
-/// These are [Any], [Debug] and [DynClone].
+/// dependencies. These are [Any], [Debug] and [DynClone].
 ///
 /// The simplest form just requires the typename to be specified. This
 /// additionally depends on an implementation of [ExternFn] because after the
@@ -37,8 +37,9 @@ use crate::representations::Primitive;
 /// ```
 /// // excerpt from the exact implementation of Multiply
 /// atomic_impl!(Multiply0, |Self(a, cls): &Self| {
-///   let b: Numeric = cls.clone().try_into().map_err(AssertionError::into_extern)?;
+///   let b: Numeric =
-///   Ok(*a * b).into())
+///     cls.clone().try_into().map_err(AssertionError::into_extern)?;
+///   Ok(*a * b).into()
 /// })
 /// ```
 #[macro_export]
@@ -58,14 +59,18 @@ macro_rules! atomic_impl {
         ctx: $crate::interpreter::Context,
       ) -> $crate::foreign::AtomicResult {
         // extract the expression
-        let expr =
+        let expr = <Self as AsRef<
-          <Self as AsRef<$crate::foreign::RcExpr>>::as_ref(self).clone();
+          $crate::representations::interpreted::ExprInst,
+        >>::as_ref(self)
+        .clone();
         // run the expression
         let ret = $crate::interpreter::run(expr, ctx.clone())?;
         let $crate::interpreter::Return { gas, state, inert } = ret;
         // rebuild the atomic
-        let next_self =
+        let next_self = <Self as From<(
-          <Self as From<(&Self, $crate::foreign::RcExpr)>>::from((self, state));
+          &Self,
+          $crate::representations::interpreted::ExprInst,
+        )>>::from((self, state));
         // branch off or wrap up
         let clause = if inert {
           let closure = $next_phase;

src/foreign_macros/atomic_redirect.rs

@@ -1,7 +1,7 @@
 #[allow(unused)]
-use super::atomic_impl;
+use crate::atomic_impl;
 
-/// Implement the traits required by [atomic_impl] to redirect run_* functions
+/// Implement the traits required by [atomic_impl] to redirect run calls
 /// to a field with a particular name.
 #[macro_export]
 macro_rules! atomic_redirect {
@@ -18,14 +18,18 @@ macro_rules! atomic_redirect {
     }
   };
   ($typ:ident, $field:ident) => {
-    impl AsRef<$crate::foreign::RcExpr> for $typ {
+    impl AsRef<$crate::representations::interpreted::ExprInst> for $typ {
-      fn as_ref(&self) -> &$crate::foreign::RcExpr {
+      fn as_ref(&self) -> &$crate::representations::interpreted::ExprInst {
         &self.$field
       }
     }
-    impl From<(&Self, $crate::foreign::RcExpr)> for $typ {
+    impl From<(&Self, $crate::representations::interpreted::ExprInst)>
+      for $typ
+    {
       #[allow(clippy::needless_update)]
-      fn from((old, $field): (&Self, $crate::foreign::RcExpr)) -> Self {
+      fn from(
+        (old, $field): (&Self, $crate::representations::interpreted::ExprInst),
+      ) -> Self {
         Self { $field, ..old.clone() }
       }
     }

src/foreign_macros/externfn_impl.rs

@@ -28,7 +28,7 @@ macro_rules! externfn_impl {
       }
       fn apply(
         &self,
-        arg: $crate::foreign::RcExpr,
+        arg: $crate::representations::interpreted::ExprInst,
         _ctx: $crate::interpreter::Context,
       ) -> $crate::foreign::XfnResult {
         let closure = $next_atomic;

src/interner/display.rs

@@ -20,13 +20,7 @@ pub trait InternedDisplay {
 
   /// Converts the value to a string to be displayed
   fn to_string_i(&self, i: &Interner) -> String {
-    // Copied from <https://doc.rust-lang.org/src/alloc/string.rs.html#2526>
+    self.bundle(i).to_string()
-    let mut buf = String::new();
-    let mut formatter = Formatter::new(&mut buf);
-    // Bypass format_args!() to avoid write_str with zero-length strs
-    Self::fmt_i(self, &mut formatter, i)
-      .expect("a Display implementation returned an error unexpectedly");
-    buf
   }
 
   fn bundle<'a>(&'a self, interner: &'a Interner) -> DisplayBundle<'a, Self> {
@@ -47,12 +41,14 @@ where
   }
 }
 
+/// A reference to an [InternedDisplay] type and an [Interner] tied together
+/// to implement [Display]
 pub struct DisplayBundle<'a, T: InternedDisplay + ?Sized> {
   interner: &'a Interner,
   data: &'a T,
 }
 
-impl<'a, T: InternedDisplay> Display for DisplayBundle<'a, T> {
+impl<'a, T: InternedDisplay + ?Sized> Display for DisplayBundle<'a, T> {
   fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
     self.data.fmt_i(f, self.interner)
   }

src/interner/mod.rs

@@ -1,3 +1,7 @@
+//! A type-agnostic interner
+//!
+//! Can be used to deduplicate various structures for fast equality comparisons.
+//! The parser uses it to intern strings.
 mod display;
 mod monotype;
 mod multitype;

src/interner/monotype.rs

@@ -8,7 +8,8 @@ use hashbrown::HashMap;
 use super::token::Tok;
 
 /// An interner for any type that implements [Borrow]. This is inspired by
-/// Lasso but much simpler, in part because not much can be known about the type.
+/// Lasso but much simpler, in part because not much can be known about the
+/// type.
 pub struct TypedInterner<T: 'static + Eq + Hash + Clone> {
   tokens: RefCell<HashMap<&'static T, Tok<T>>>,
   values: RefCell<Vec<(&'static T, bool)>>,
@@ -45,7 +46,7 @@ impl<T: Eq + Hash + Clone> TypedInterner<T> {
     *kv.1
   }
 
-  /// Resolve a token, obtaining an object
+  /// Resolve a token, obtaining a reference to the held object.
   /// It is illegal to use a token obtained from one interner with
   /// another.
   pub fn r(&self, t: Tok<T>) -> &T {
@@ -74,6 +75,12 @@ impl<T: Eq + Hash + Clone> TypedInterner<T> {
   }
 }
 
+impl<T: Eq + Hash + Clone> Default for TypedInterner<T> {
+  fn default() -> Self {
+    Self::new()
+  }
+}
+
 impl<T: Eq + Hash + Clone> Drop for TypedInterner<T> {
   fn drop(&mut self) {
     // make sure all values leaked by us are dropped

src/interner/multitype.rs

@@ -16,10 +16,12 @@ pub struct Interner {
   interners: RefCell<HashMap<TypeId, Rc<dyn Any>>>,
 }
 impl Interner {
+  /// Create a new interner
   pub fn new() -> Self {
     Self { interners: RefCell::new(HashMap::new()) }
   }
 
+  /// Intern something
   pub fn i<Q: ?Sized + Eq + Hash + ToOwned>(&self, q: &Q) -> Tok<Q::Owned>
   where
     Q::Owned: 'static + Eq + Hash + Clone + Borrow<Q>,
@@ -29,6 +31,7 @@ impl Interner {
     interner.i(q)
   }
 
+  /// Resolve a token to a reference
   pub fn r<T: 'static + Eq + Hash + Clone>(&self, t: Tok<T>) -> &T {
     let mut interners = self.interners.borrow_mut();
     let interner = get_interner(&mut interners);
@@ -36,7 +39,7 @@ impl Interner {
     unsafe { (interner.r(t) as *const T).as_ref().unwrap() }
   }
 
-  /// Fully resolve
+  /// Fully resolve an interned list of interned things
   /// TODO: make this generic over containers
   pub fn extern_vec<T: 'static + Eq + Hash + Clone>(
     &self,
@@ -49,6 +52,7 @@ impl Interner {
     v.iter().map(|t| t_int.r(*t)).cloned().collect()
   }
 
+  /// Fully resolve a list of interned things.
   pub fn extern_all<T: 'static + Eq + Hash + Clone>(
     &self,
     s: &[Tok<T>],

src/interpreter/apply.rs

@@ -53,12 +53,16 @@ fn map_at<E>(
     .map(|p| p.0)
 }
 
+/// TODO replace when `!` gets stabilized
+#[derive(Debug)]
+enum Never {}
+
 /// Replace the [Clause::LambdaArg] placeholders at the ends of the [PathSet]
 /// with the value in the body. Note that a path may point to multiple
 /// placeholders.
 fn substitute(paths: &PathSet, value: Clause, body: ExprInst) -> ExprInst {
   let PathSet { steps, next } = paths;
-  map_at(steps, body, &mut |checkpoint| -> Result<Clause, !> {
+  map_at(steps, body, &mut |checkpoint| -> Result<Clause, Never> {
     match (checkpoint, next) {
       (Clause::Lambda { .. }, _) => unreachable!("Handled by map_at"),
       (Clause::Apply { f, x }, Some((left, right))) => Ok(Clause::Apply {
@@ -72,7 +76,7 @@ fn substitute(paths: &PathSet, value: Clause, body: ExprInst) -> ExprInst {
         panic!("Substitution path leads into something other than Apply"),
     }
   })
-  .into_ok()
+  .unwrap()
 }
 
 /// Apply a function-like expression to a parameter.

src/interpreter/context.rs

@@ -8,8 +8,8 @@ use crate::representations::interpreted::ExprInst;
 pub struct Context<'a> {
   /// Table used to resolve constants
   pub symbols: &'a HashMap<Sym, ExprInst>,
-  /// The interner used for strings internally, so external functions can deduce
+  /// The interner used for strings internally, so external functions can
-  /// referenced constant names on the fly
+  /// deduce referenced constant names on the fly
   pub interner: &'a Interner,
   /// The number of reduction steps the interpreter can take before returning
   pub gas: Option<usize>,

src/interpreter/error.rs

@@ -7,7 +7,9 @@ use crate::representations::interpreted::ExprInst;
 /// Problems in the process of execution
 #[derive(Clone, Debug)]
 pub enum RuntimeError {
+  /// A Rust function encountered an error
   Extern(Rc<dyn ExternError>),
+  /// Primitive applied as function
   NonFunctionApplication(ExprInst),
 }
 

src/interpreter/mod.rs

@@ -1,3 +1,4 @@
+//! functions to interact with Orchid code
 mod apply;
 mod context;
 mod error;
@@ -5,4 +6,4 @@ mod run;
 
 pub use context::{Context, Return};
 pub use error::RuntimeError;
-pub use run::{run, run_handler, Handler, HandlerParm, HandlerRes};
+pub use run::{run, run_handler, Handler, HandlerErr, HandlerParm, HandlerRes};

src/interpreter/run.rs

@@ -78,23 +78,24 @@ impl From<HandlerParm> for HandlerErr {
   }
 }
 
-/// Various possible outcomes of a [Handler] execution.
+/// Various possible outcomes of a [Handler] execution. Ok returns control to
+/// the interpreter. The meaning of Err is decided by the value in it.
 pub type HandlerRes = Result<ExprInst, HandlerErr>;
 
 /// A trait for things that may be able to handle commands returned by Orchid
-/// code. This trait is implemented for [FnMut(HandlerParm) -> HandlerRes] and
+/// code. This trait is implemented for `FnMut(HandlerParm) -> HandlerRes` and
-/// [(Handler, Handler)], users are not supposed to implement it themselves.
+/// `(Handler, Handler)`, users are not supposed to implement it themselves.
 ///
 /// A handler receives an arbitrary inert [Atomic] and uses [Atomic::as_any]
-/// then [std::any::Any::downcast_ref] to obtain a known type. If this fails, it
+/// then downcast_ref of [std::any::Any] to obtain a known type. If this fails,
-/// returns the box in [HandlerErr::NA] which will be passed to the next
+/// it returns the box in [HandlerErr::NA] which will be passed to the next
 /// handler.
 pub trait Handler {
   /// Attempt to resolve a command with this handler.
   fn resolve(&mut self, data: HandlerParm) -> HandlerRes;
 
   /// If this handler isn't applicable, try the other one.
-  fn or<T: Handler>(self, t: T) -> impl Handler
+  fn or<T: Handler>(self, t: T) -> (Self, T)
   where
     Self: Sized,
   {

src/lib.rs

@@ -0,0 +1,16 @@
+pub mod foreign;
+mod foreign_macros;
+pub mod interner;
+pub mod interpreter;
+mod parse;
+pub mod pipeline;
+mod representations;
+pub mod rule;
+pub mod stl;
+mod utils;
+
+pub use representations::ast_to_interpreted::ast_to_interpreted;
+pub use representations::{
+  ast, interpreted, sourcefile, tree, Literal, Location, PathSet, Primitive,
+};
+pub use utils::{Side, Substack};

src/main.rs

@@ -1,57 +0,0 @@
-#![feature(generators, generator_trait)]
-#![feature(never_type)]
-#![feature(unwrap_infallible)]
-#![feature(arc_unwrap_or_clone)]
-#![feature(hasher_prefixfree_extras)]
-#![feature(closure_lifetime_binder)]
-#![feature(generic_arg_infer)]
-#![feature(array_chunks)]
-#![feature(fmt_internals)]
-#![feature(map_try_insert)]
-#![feature(slice_group_by)]
-#![feature(trait_alias)]
-#![feature(return_position_impl_trait_in_trait)]
-
-mod cli;
-mod external;
-pub(crate) mod foreign;
-mod foreign_macros;
-mod interner;
-mod interpreter;
-mod parse;
-mod pipeline;
-mod representations;
-mod rule;
-mod run_dir;
-mod utils;
-use std::fs::File;
-use std::path::PathBuf;
-
-use clap::Parser;
-use cli::prompt;
-pub use representations::ast;
-use run_dir::run_dir;
-
-/// Orchid interpreter
-#[derive(Parser, Debug)]
-#[command(author, version, about, long_about = None)]
-struct Args {
-  /// Folder containing main.orc
-  #[arg(short, long)]
-  pub project: Option<String>,
-}
-
-fn main() {
-  let args = Args::parse();
-  let path = args.project.unwrap_or_else(|| {
-    prompt("Enter a project root", ".".to_string(), |p| {
-      let mut path: PathBuf = p.trim().into();
-      path.push("main.orc");
-      match File::open(&path) {
-        Ok(_) => Ok(p),
-        Err(e) => Err(format!("{}: {e}", path.display())),
-      }
-    })
-  });
-  run_dir(&PathBuf::try_from(path).unwrap());
-}

src/parse/decls.rs

@@ -3,10 +3,13 @@ use std::hash::Hash;
 use chumsky::prelude::Simple;
 use chumsky::recursive::Recursive;
 use chumsky::{BoxedParser, Parser};
+use trait_set::trait_set;
 
-/// Wrapper around [Parser] with [Simple] error to avoid repeating the input
+trait_set! {
-pub trait SimpleParser<I: Eq + Hash + Clone, O> =
+  /// Wrapper around [Parser] with [Simple] error to avoid repeating the input
-  Parser<I, O, Error = Simple<I>>;
+  pub trait SimpleParser<I: Eq + Hash + Clone, O> =
+    Parser<I, O, Error = Simple<I>>;
+}
 /// Boxed version of [SimpleParser]
 pub type BoxedSimpleParser<'a, I, O> = BoxedParser<'a, I, O, Simple<I>>;
 /// [Recursive] specialization of [SimpleParser] to parameterize calls to

src/parse/enum_filter.rs

@@ -8,7 +8,6 @@
 /// // Foo::Bar(T) into Quz::Bar(T)
 /// // Foo::Baz(U) into Quz::Baz(U)
 /// ```
-#[macro_export]
 macro_rules! enum_filter {
   ($p:path | $m:tt) => {
     {
@@ -48,3 +47,5 @@ macro_rules! enum_filter {
     enum_filter!($p | {concat!("Expected ", stringify!($p))})
   };
 }
+
+pub(crate) use enum_filter;

src/parse/expression.rs

@@ -6,8 +6,8 @@ use chumsky::{self, Parser};
 
 use super::context::Context;
 use super::decls::SimpleParser;
+use super::enum_filter::enum_filter;
 use super::lexer::{filter_map_lex, Entry, Lexeme};
-use crate::enum_filter;
 use crate::interner::Sym;
 use crate::representations::ast::{Clause, Expr};
 use crate::representations::location::Location;

src/parse/import.rs

@@ -4,18 +4,17 @@ use itertools::Itertools;
 
 use super::context::Context;
 use super::decls::{SimpleParser, SimpleRecursive};
+use super::enum_filter::enum_filter;
 use super::lexer::{filter_map_lex, Lexeme};
 use super::Entry;
 use crate::interner::Tok;
 use crate::representations::sourcefile::Import;
 use crate::utils::iter::{
-  box_flatten, box_once, into_boxed_iter, BoxedIterIter,
+  box_chain, box_flatten, box_once, into_boxed_iter, BoxedIterIter,
 };
-use crate::{box_chain, enum_filter};
 
-/// initialize a BoxedIter<BoxedIter<String>> with a single element.
+/// initialize an iterator of iterator with a single element.
 fn init_table(name: Tok<String>) -> BoxedIterIter<'static, Tok<String>> {
-  // I'm not at all confident that this is a good approach.
   box_once(box_once(name))
 }
 
@@ -24,7 +23,7 @@ fn init_table(name: Tok<String>) -> BoxedIterIter<'static, Tok<String>> {
 /// semi and the delimiters are plain parentheses. Namespaces should
 /// preferably contain crossplatform filename-legal characters but the
 /// symbols are explicitly allowed to go wild.
-/// There's a blacklist in [name]
+/// There's a blacklist in [crate::parse::name::NOT_NAME_CHAR]
 pub fn import_parser<'a>(
   ctx: impl Context + 'a,
 ) -> impl SimpleParser<Entry, Vec<Import>> + 'a {

src/parse/name.rs

@@ -24,7 +24,7 @@ fn op_parser<'a>(
 /// Characters that cannot be parsed as part of an operator
 ///
 /// The initial operator list overrides this.
-static NOT_NAME_CHAR: &[char] = &[
+pub static NOT_NAME_CHAR: &[char] = &[
   ':', // used for namespacing and type annotations
   '\\', '@', // parametric expression starters
   '"', '\'', // parsed as primitives and therefore would never match

src/parse/sourcefile.rs

@@ -7,12 +7,12 @@ use itertools::Itertools;
 
 use super::context::Context;
 use super::decls::{SimpleParser, SimpleRecursive};
+use super::enum_filter::enum_filter;
 use super::expression::xpr_parser;
 use super::import::import_parser;
 use super::lexer::{filter_map_lex, Lexeme};
 use super::Entry;
 use crate::ast::{Clause, Constant, Expr, Rule};
-use crate::enum_filter;
 use crate::representations::location::Location;
 use crate::representations::sourcefile::{FileEntry, Member, Namespace};
 
@@ -24,10 +24,10 @@ fn rule_parser<'a>(
     .at_least(1)
     .then(filter_map_lex(enum_filter!(Lexeme::Rule)))
     .then(xpr_parser(ctx).repeated().at_least(1))
-    .map(|((s, (prio, _)), t)| Rule {
+    .map(|((p, (prio, _)), t)| Rule {
-      source: Rc::new(s),
+      pattern: Rc::new(p),
       prio,
-      target: Rc::new(t),
+      template: Rc::new(t),
     })
     .labelled("Rule")
 }

src/pipeline/error/mod.rs

@@ -1,3 +1,4 @@
+//! Various errors the pipeline can produce
 mod module_not_found;
 mod not_exported;
 mod parse_error_with_path;

src/pipeline/error/not_exported.rs

@@ -4,6 +4,7 @@ use super::{ErrorPosition, ProjectError};
 use crate::representations::location::Location;
 use crate::utils::BoxedIter;
 
+/// An import refers to a symbol which exists but is not exported.
 #[derive(Debug)]
 pub struct NotExported {
   pub file: Vec<String>,

src/pipeline/error/project_error.rs

@@ -29,7 +29,7 @@ pub trait ProjectError: Debug {
   }
   /// Code positions relevant to this error
   fn positions(&self) -> BoxedIter<ErrorPosition>;
-  /// Convert the error into an [Rc<dyn ProjectError>] to be able to
+  /// Convert the error into an `Rc<dyn ProjectError>` to be able to
   /// handle various errors together
   fn rc(self) -> Rc<dyn ProjectError>
   where

src/pipeline/error/visibility_mismatch.rs

@@ -5,6 +5,7 @@ use crate::representations::location::Location;
 use crate::utils::iter::box_once;
 use crate::utils::BoxedIter;
 
+/// Multiple occurences of the same namespace with different visibility
 #[derive(Debug)]
 pub struct VisibilityMismatch {
   pub namespace: Vec<String>,

src/pipeline/file_loader.rs

@@ -1,3 +1,4 @@
+//! File system implementation of the source loader callback
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use std::{fs, io};
@@ -9,6 +10,7 @@ use crate::pipeline::error::{
 use crate::utils::iter::box_once;
 use crate::utils::{BoxedIter, Cache};
 
+/// All the data available about a failed source load call
 #[derive(Debug)]
 pub struct FileLoadingError {
   file: io::Error,
@@ -40,10 +42,9 @@ impl Loaded {
   }
 }
 
+/// Returned by any source loading callback
 pub type IOResult = Result<Loaded, Rc<dyn ProjectError>>;
 
-pub type FileCache<'a> = Cache<'a, Sym, IOResult>;
-
 /// Load a file from a path expressed in Rust strings, but relative to
 /// a root expressed as an OS Path.
 pub fn load_file(root: &Path, path: &[impl AsRef<str>]) -> IOResult {
@@ -81,7 +82,7 @@ pub fn load_file(root: &Path, path: &[impl AsRef<str>]) -> IOResult {
 }
 
 /// Generates a cached file loader for a directory
-pub fn mk_cache(root: PathBuf, i: &Interner) -> FileCache {
+pub fn mk_cache(root: PathBuf, i: &Interner) -> Cache<Sym, IOResult> {
   Cache::new(move |token: Sym, _this| -> IOResult {
     let path = i.r(token).iter().map(|t| i.r(*t).as_str()).collect::<Vec<_>>();
     load_file(&root, &path)

src/pipeline/import_resolution/apply_aliases.rs

@@ -90,17 +90,17 @@ fn apply_aliases_rec(
     .rules
     .iter()
     .map(|rule| {
-      let Rule { source, prio, target } = rule;
+      let Rule { pattern, prio, template } = rule;
       Rule {
         prio: *prio,
-        source: Rc::new(
+        pattern: Rc::new(
-          source
+          pattern
             .iter()
             .map(|expr| process_expr(expr, alias_map, injected_as, i))
             .collect::<Vec<_>>(),
         ),
-        target: Rc::new(
+        template: Rc::new(
-          target
+          template
             .iter()
             .map(|expr| process_expr(expr, alias_map, injected_as, i))
             .collect::<Vec<_>>(),

src/pipeline/import_resolution/decls.rs

@@ -1,3 +1,7 @@
+use trait_set::trait_set;
+
 use crate::interner::{Sym, Tok};
 
-pub trait InjectedAsFn = Fn(&[Tok<String>]) -> Option<Sym>;
+trait_set! {
+  pub trait InjectedAsFn = Fn(&[Tok<String>]) -> Option<Sym>;
+}

src/pipeline/mod.rs

@@ -1,3 +1,4 @@
+//! Loading Orchid modules from source
 pub mod error;
 pub mod file_loader;
 mod import_abs_path;

src/pipeline/project_tree/build_tree.rs

@@ -1,6 +1,7 @@
 use std::rc::Rc;
 
 use hashbrown::HashMap;
+use itertools::Itertools;
 
 use super::collect_ops::InjectedOperatorsFn;
 use super::parse_file::parse_file;
@@ -87,7 +88,7 @@ fn source_to_module(
           Member::Namespace(ns) => box_once(mk_ent(ns.name)),
           Member::Rule(rule) => {
             let mut names = Vec::new();
-            for e in rule.source.iter() {
+            for e in rule.pattern.iter() {
               e.visit_names(Substack::Bottom, &mut |n| {
                 if let Some([name]) = i.r(n).strip_prefix(&path_v[..]) {
                   names.push((*name, n))
@@ -177,7 +178,7 @@ fn source_to_module(
 
 fn files_to_module(
   path: Substack<Tok<String>>,
-  files: &[ParsedSource],
+  files: Vec<ParsedSource>,
   i: &Interner,
 ) -> Rc<Module<Expr, ProjectExt>> {
   let lvl = path.len();
@@ -192,11 +193,13 @@ fn files_to_module(
     );
   }
   let items = files
-    .group_by(|a, b| a.path[lvl] == b.path[lvl])
+    .into_iter()
-    .map(|files| {
+    .group_by(|f| f.path[lvl])
-      let namespace = files[0].path[lvl];
+    .into_iter()
+    .map(|(namespace, files)| {
       let subpath = path.push(namespace);
-      let module = files_to_module(subpath, files, i);
+      let files_v = files.collect::<Vec<_>>();
+      let module = files_to_module(subpath, files_v, i);
       let member = ModMember::Sub(module);
       (namespace, ModEntry { exported: true, member })
     })
@@ -206,11 +209,6 @@ fn files_to_module(
     .copied()
     .map(|name| (name, i.i(&pushed(&path_v, name))))
     .collect();
-  // println!(
-  //   "Constructing module {} with items ({})",
-  //   i.extern_all(&path_v[..]).join("::"),
-  //   exports.keys().map(|t| i.r(*t)).join(", ")
-  // );
   Rc::new(Module {
     items,
     imports: vec![],
@@ -250,5 +248,5 @@ pub fn build_tree(
       path: path.clone(),
     })
     .collect::<Vec<_>>();
-  Ok(ProjectTree(files_to_module(Substack::Bottom, &files, i)))
+  Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)))
 }

src/pipeline/project_tree/collect_ops/exported_ops.rs

@@ -3,6 +3,7 @@ use std::rc::Rc;
 
 use hashbrown::HashSet;
 use itertools::Itertools;
+use trait_set::trait_set;
 
 use crate::interner::{Interner, Sym, Tok};
 use crate::pipeline::error::{ModuleNotFound, ProjectError};
@@ -14,7 +15,9 @@ use crate::utils::Cache;
 pub type OpsResult = Result<Rc<HashSet<Tok<String>>>, Rc<dyn ProjectError>>;
 pub type ExportedOpsCache<'a> = Cache<'a, Sym, OpsResult>;
 
-pub trait InjectedOperatorsFn = Fn(Sym) -> Option<Rc<HashSet<Tok<String>>>>;
+trait_set! {
+  pub trait InjectedOperatorsFn = Fn(Sym) -> Option<Rc<HashSet<Tok<String>>>>;
+}
 
 fn coprefix<T: Eq>(
   l: impl Iterator<Item = T>,

src/pipeline/project_tree/const_tree.rs

@@ -12,23 +12,30 @@ use crate::representations::tree::{ModEntry, ModMember, Module};
 use crate::representations::Primitive;
 use crate::utils::{pushed, Substack};
 
+/// A lightweight module tree that can be built declaratively by hand to
+/// describe libraries of external functions in Rust. It implements [Add] for
+/// added convenience
 pub enum ConstTree {
   Const(Expr),
   Tree(HashMap<Tok<String>, ConstTree>),
 }
 impl ConstTree {
+  /// Describe a [Primitive]
   pub fn primitive(primitive: Primitive) -> Self {
     Self::Const(Expr {
       location: Location::Unknown,
       value: Clause::P(primitive),
     })
   }
+  /// Describe an [ExternFn]
   pub fn xfn(xfn: impl ExternFn + 'static) -> Self {
     Self::primitive(Primitive::ExternFn(Box::new(xfn)))
   }
+  /// Describe an [Atomic]
   pub fn atom(atom: impl Atomic + 'static) -> Self {
     Self::primitive(Primitive::Atom(Atom(Box::new(atom))))
   }
+  /// Describe a module
   pub fn tree(arr: impl IntoIterator<Item = (Tok<String>, Self)>) -> Self {
     Self::Tree(arr.into_iter().collect())
   }
@@ -89,6 +96,8 @@ fn from_const_tree_rec(
   }
 }
 
+/// Convert a map of [ConstTree] into a [ProjectTree] that can be used with the
+/// layered parsing system
 pub fn from_const_tree(
   consts: HashMap<Tok<String>, ConstTree>,
   file: &[Tok<String>],

src/pipeline/project_tree/prefix.rs

@@ -35,11 +35,11 @@ fn member_rec(
     }),
     Member::Rule(rule) => Member::Rule(Rule {
       prio: rule.prio,
-      source: Rc::new(
+      pattern: Rc::new(
-        rule.source.iter().map(|e| e.prefix(prefix, i, &except)).collect(),
+        rule.pattern.iter().map(|e| e.prefix(prefix, i, &except)).collect(),
       ),
-      target: Rc::new(
+      template: Rc::new(
-        rule.target.iter().map(|e| e.prefix(prefix, i, &except)).collect(),
+        rule.template.iter().map(|e| e.prefix(prefix, i, &except)).collect(),
       ),
     }),
   }

src/pipeline/project_tree/tree.rs

@@ -8,11 +8,13 @@ use crate::interner::{Interner, Sym, Tok};
 use crate::representations::tree::{ModMember, Module};
 use crate::utils::Substack;
 
+/// Additional data about a loaded module beyond the list of constants and
+/// submodules
 #[derive(Clone, Debug, Default)]
 pub struct ProjectExt {
   /// Pairs each foreign token to the module it was imported from
   pub imports_from: HashMap<Tok<String>, Sym>,
-  /// Pairs each exported token to its original full name.
+  /// Pairs each exported token to its original full name
   pub exports: HashMap<Tok<String>, Sym>,
   /// All rules defined in this module, exported or not
   pub rules: Vec<Rule>,
@@ -35,7 +37,10 @@ impl Add for ProjectExt {
   }
 }
 
+/// A node in the tree describing the project
 pub type ProjectModule = Module<Expr, ProjectExt>;
+
+/// Module corresponding to the root of a project
 pub struct ProjectTree(pub Rc<ProjectModule>);
 
 fn collect_rules_rec(bag: &mut Vec<Rule>, module: &ProjectModule) {
@@ -47,6 +52,8 @@ fn collect_rules_rec(bag: &mut Vec<Rule>, module: &ProjectModule) {
   }
 }
 
+/// Collect the complete list of rules to be used by the rule repository from
+/// the [ProjectTree]
 pub fn collect_rules(project: &ProjectTree) -> Vec<Rule> {
   let mut rules = Vec::new();
   collect_rules_rec(&mut rules, project.0.as_ref());
@@ -72,6 +79,7 @@ fn collect_consts_rec(
   }
 }
 
+/// Extract the symbol table from a [ProjectTree]
 pub fn collect_consts(
   project: &ProjectTree,
   i: &Interner,

src/representations/ast.rs

@@ -1,3 +1,10 @@
+//! Datastructures representing syntax as written
+//!
+//! These structures are produced by the parser, processed by the macro
+//! executor, and then converted to other usable formats. This module is public
+//! in order to allow users to define injected libraries programmatically,
+//! although going through the parser is usually preferable.
+
 use std::hash::Hash;
 use std::rc::Rc;
 
@@ -9,7 +16,7 @@ use super::primitive::Primitive;
 use crate::interner::{InternedDisplay, Interner, Sym, Tok};
 use crate::utils::Substack;
 
-/// An S-expression with a type
+/// A [Clause] with associated metadata
 #[derive(Clone, Debug, PartialEq)]
 pub struct Expr {
   pub value: Clause,
@@ -17,10 +24,12 @@ pub struct Expr {
 }
 
 impl Expr {
+  /// Obtain the contained clause
   pub fn into_clause(self) -> Clause {
     self.value
   }
 
+  /// Call the function on every name in this expression
   pub fn visit_names(&self, binds: Substack<Sym>, cb: &mut impl FnMut(Sym)) {
     let Expr { value, .. } = self;
     value.visit_names(binds, cb);
@@ -61,12 +70,21 @@ impl InternedDisplay for Expr {
   }
 }
 
+/// Various types of placeholders
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum PHClass {
-  Vec { nonzero: bool, prio: u64 },
+  /// Matches multiple tokens, lambdas or parenthesized groups
+  Vec {
+    /// If true, must match at least one clause
+    nonzero: bool,
+    /// Greediness in the allocation of tokens
+    prio: u64,
+  },
+  /// Matches exactly one token, lambda or parenthesized group
   Scalar,
 }
 
+/// Properties of a placeholder that matches unknown tokens in macros
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub struct Placeholder {
   pub name: Tok<String>,
@@ -95,6 +113,7 @@ impl InternedDisplay for Placeholder {
 /// An S-expression as read from a source file
 #[derive(Debug, PartialEq, Clone)]
 pub enum Clause {
+  /// A primitive
   P(Primitive),
   /// A c-style name or an operator, eg. `+`, `i`, `foo::bar`
   Name(Sym),
@@ -257,7 +276,7 @@ fn fmt_expr_seq<'a>(
   Ok(())
 }
 
-pub fn fmt_name(
+pub(crate) fn fmt_name(
   name: Sym,
   f: &mut std::fmt::Formatter,
   i: &Interner,
@@ -302,15 +321,17 @@ impl InternedDisplay for Clause {
 /// A substitution rule as read from the source
 #[derive(Debug, Clone, PartialEq)]
 pub struct Rule {
-  pub source: Rc<Vec<Expr>>,
+  pub pattern: Rc<Vec<Expr>>,
   pub prio: NotNan<f64>,
-  pub target: Rc<Vec<Expr>>,
+  pub template: Rc<Vec<Expr>>,
 }
 
 impl Rule {
+  /// Return a list of all names that don't contain a namespace separator `::`.
+  /// These are exported when the rule is exported
   pub fn collect_single_names(&self, i: &Interner) -> Vec<Tok<String>> {
     let mut names = Vec::new();
-    for e in self.source.iter() {
+    for e in self.pattern.iter() {
       e.visit_names(Substack::Bottom, &mut |tok| {
         let ns_name = i.r(tok);
         let (name, excess) =
@@ -324,6 +345,7 @@ impl Rule {
     names
   }
 
+  /// Namespace all tokens in the rule
   pub fn prefix(
     &self,
     prefix: Sym,
@@ -332,11 +354,11 @@ impl Rule {
   ) -> Self {
     Self {
       prio: self.prio,
-      source: Rc::new(
+      pattern: Rc::new(
-        self.source.iter().map(|e| e.prefix(prefix, i, except)).collect(),
+        self.pattern.iter().map(|e| e.prefix(prefix, i, except)).collect(),
       ),
-      target: Rc::new(
+      template: Rc::new(
-        self.target.iter().map(|e| e.prefix(prefix, i, except)).collect(),
+        self.template.iter().map(|e| e.prefix(prefix, i, except)).collect(),
       ),
     }
   }
@@ -348,12 +370,12 @@ impl InternedDisplay for Rule {
     f: &mut std::fmt::Formatter<'_>,
     i: &Interner,
   ) -> std::fmt::Result {
-    for e in self.source.iter() {
+    for e in self.pattern.iter() {
       e.fmt_i(f, i)?;
       write!(f, " ")?;
     }
     write!(f, "={}=>", self.prio)?;
-    for e in self.target.iter() {
+    for e in self.template.iter() {
       write!(f, " ")?;
       e.fmt_i(f, i)?;
     }

src/representations/ast_to_interpreted.rs

@@ -0,0 +1,13 @@
+use super::{ast, ast_to_postmacro, interpreted, postmacro_to_interpreted};
+
+#[allow(unused)]
+pub type AstError = ast_to_postmacro::Error;
+
+/// Attempt to convert the AST processed by macros into an executable format
+#[allow(unused)]
+pub fn ast_to_interpreted(
+  ast: &ast::Expr,
+) -> Result<interpreted::ExprInst, AstError> {
+  let pmtree = ast_to_postmacro::expr(ast)?;
+  Ok(postmacro_to_interpreted::expr(&pmtree))
+}

src/representations/ast_to_postmacro.rs

@@ -63,7 +63,7 @@ impl<'a> Context<'a> {
   }
 }
 
-/// Recursive state of [exprv]
+/// Process an expression sequence
 fn exprv_rec<'a>(
   v: &'a [ast::Expr],
   ctx: Context<'a>,
@@ -78,7 +78,7 @@ fn exprv_rec<'a>(
   Ok(postmacro::Expr { value, location: Location::Unknown })
 }
 
-/// Recursive state of [expr]
+/// Process an expression
 fn expr_rec<'a>(
   ast::Expr { value, location }: &'a ast::Expr,
   ctx: Context<'a>,
@@ -95,11 +95,7 @@ fn expr_rec<'a>(
   }
 }
 
-// (\t:(@T. Pair T T). t \left.\right. left) @number -- this will fail
+/// Process a clause
-// (@T. \t:Pair T T. t \left.\right. left) @number -- this is the correct
-// phrasing
-
-/// Recursive state of [clause]
 fn clause_rec<'a>(
   cls: &'a ast::Clause,
   ctx: Context<'a>,

src/representations/interpreted.rs

@@ -1,3 +1,7 @@
+//! The interpreter's changing internal representation of the code at runtime
+//!
+//! This code may be generated to minimize the number of states external
+//! functions have to define
 use std::cell::RefCell;
 use std::fmt::Debug;
 use std::ops::{Deref, DerefMut};
@@ -12,6 +16,7 @@ use crate::utils::sym2string;
 
 // TODO: implement Debug, Eq and Hash with cycle detection
 
+/// An expression with metadata
 pub struct Expr {
   pub clause: Clause,
   pub location: Location,
@@ -43,8 +48,12 @@ impl InternedDisplay for Expr {
   }
 }
 
+/// [ExprInst::with_literal] produces this marker unit to indicate that the
+/// expression is not a literal
+pub struct NotALiteral;
+
 /// A wrapper around expressions to handle their multiple occurences in
-/// the tree
+/// the tree together
 #[derive(Clone)]
 pub struct ExprInst(pub Rc<RefCell<Expr>>);
 impl ExprInst {
@@ -88,15 +97,17 @@ impl ExprInst {
     predicate(&self.expr().clause)
   }
 
+  /// Call the predicate on the value inside this expression if it is a
+  /// primitive
   pub fn with_literal<T>(
     &self,
     predicate: impl FnOnce(&Literal) -> T,
-  ) -> Result<T, ()> {
+  ) -> Result<T, NotALiteral> {
     let expr = self.expr();
     if let Clause::P(Primitive::Literal(l)) = &expr.clause {
       Ok(predicate(l))
     } else {
-      Err(())
+      Err(NotALiteral)
     }
   }
 }
@@ -123,12 +134,18 @@ impl InternedDisplay for ExprInst {
   }
 }
 
+/// Distinct types of expressions recognized by the interpreter
 #[derive(Debug, Clone)]
 pub enum Clause {
+  /// An unintrospectable unit
   P(Primitive),
+  /// A function application
   Apply { f: ExprInst, x: ExprInst },
+  /// A name to be looked up in the interpreter's symbol table
   Constant(Sym),
+  /// A function
   Lambda { args: Option<PathSet>, body: ExprInst },
+  /// A placeholder within a function that will be replaced upon application
   LambdaArg,
 }
 impl Clause {

src/representations/literal.rs

@@ -2,8 +2,8 @@ use std::fmt::Debug;
 
 use ordered_float::NotNan;
 
-/// An exact value, read from the AST and unmodified in shape until
+/// Exact values read from the AST which have a shared meaning recognized by all
-/// compilation
+/// external functions
 #[derive(Clone, PartialEq, Eq, Hash)]
 pub enum Literal {
   Num(NotNan<f64>),

src/representations/location.rs

@@ -4,6 +4,8 @@ use std::rc::Rc;
 
 use itertools::Itertools;
 
+/// A location in a file, identifies a sequence of suspect characters for any
+/// error. Meaningful within the context of a project.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum Location {
   Unknown,

src/representations/mod.rs

@@ -1,14 +1,17 @@
 pub mod ast;
+pub mod ast_to_interpreted;
 pub mod ast_to_postmacro;
 pub mod interpreted;
 pub mod literal;
 pub mod location;
 pub mod path_set;
 pub mod postmacro;
+pub mod postmacro_to_interpreted;
 pub mod primitive;
 pub mod sourcefile;
 pub mod tree;
+
+pub use literal::Literal;
+pub use location::Location;
 pub use path_set::PathSet;
 pub use primitive::Primitive;
-pub mod postmacro_to_interpreted;
-pub use literal::Literal;

src/representations/path_set.rs

@@ -4,7 +4,8 @@ use std::rc::Rc;
 
 use crate::utils::Side;
 
-/// A set of paths into a Lambda expression
+/// A branching path selecting some placeholders (but at least one) in a Lambda
+/// expression
 #[derive(Clone, PartialEq, Eq, Hash)]
 pub struct PathSet {
   /// The definite steps
@@ -40,7 +41,8 @@ impl Add<Side> for PathSet {
   type Output = Self;
   fn add(self, rhs: Side) -> Self::Output {
     let PathSet { steps, next } = self;
-    let mut new_steps = Rc::unwrap_or_clone(steps);
+    let mut new_steps =
+      Rc::try_unwrap(steps).unwrap_or_else(|rc| rc.as_ref().clone());
     new_steps.insert(0, rhs);
     Self { steps: Rc::new(new_steps), next }
   }

src/representations/primitive.rs

@@ -3,12 +3,13 @@ use std::fmt::Debug;
 use super::Literal;
 use crate::foreign::{Atom, ExternFn};
 
+/// A value the interpreter can't inspect
 pub enum Primitive {
   /// A literal value, eg. `1`, `"hello"`
   Literal(Literal),
-  /// An opaque function, eg. an effectful function employing CPS.
+  /// An opaque function, eg. an effectful function employing CPS
   ExternFn(Box<dyn ExternFn>),
-  /// An opaque non-callable value, eg. a file handle.
+  /// An opaque non-callable value, eg. a file handle
   Atom(Atom),
 }
 

src/representations/sourcefile.rs

@@ -1,10 +1,12 @@
+//! Building blocks of a source file
 use itertools::{Either, Itertools};
 
 use crate::ast::{Constant, Rule};
 use crate::interner::{Interner, Sym, Tok};
-use crate::unwrap_or;
+use crate::utils::{unwrap_or, BoxedIter};
-use crate::utils::BoxedIter;
 
+/// An import pointing at another module, either specifying the symbol to be
+/// imported or importing all available symbols with a globstar (*)
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct Import {
   pub path: Sym,
@@ -112,6 +114,11 @@ pub fn normalize_namespaces(
   Ok(rest)
 }
 
+/// Produced by [absolute_path] if there are more `super` segments in the
+/// import than the length of the current absolute path
+#[derive(Debug, Clone)]
+pub struct TooManySupers;
+
 /// Turn a relative (import) path into an absolute path.
 /// If the import path is empty, the return value is also empty.
 ///
@@ -123,12 +130,12 @@ pub fn absolute_path(
   abs_location: &[Tok<String>],
   rel_path: &[Tok<String>],
   i: &Interner,
-) -> Result<Vec<Tok<String>>, ()> {
+) -> Result<Vec<Tok<String>>, TooManySupers> {
   let (head, tail) = unwrap_or!(rel_path.split_first();
     return Ok(vec![])
   );
   if *head == i.i("super") {
-    let (_, new_abs) = abs_location.split_last().ok_or(())?;
+    let (_, new_abs) = abs_location.split_last().ok_or(TooManySupers)?;
     if tail.is_empty() {
       Ok(new_abs.to_vec())
     } else {

src/representations/tree.rs

@@ -1,3 +1,6 @@
+//! Generic module tree structure
+//!
+//! Used by various stages of the pipeline with different parameters
 use std::ops::Add;
 use std::rc::Rc;
 
@@ -27,20 +30,29 @@ pub struct Module<TItem: Clone, TExt: Clone> {
   pub extra: TExt,
 }
 
+/// Possible causes why the path could not be walked
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum WalkErrorKind {
+  /// `require_exported` was set to `true` and a module wasn't exported
   Private,
+  /// A module was not found
   Missing,
 }
 
+/// Error produced by [Module::walk]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct WalkError {
+  /// The 0-based index of the offending segment
   pub pos: usize,
+  /// The cause of the error
   pub kind: WalkErrorKind,
 }
 
+/// The path taken to reach a given module
 pub type ModPath<'a> = Substack<'a, Tok<String>>;
+
 impl<TItem: Clone, TExt: Clone> Module<TItem, TExt> {
+  /// Return the module at the end of the given path.
   pub fn walk(
     self: &Rc<Self>,
     path: &[Tok<String>],
@@ -78,6 +90,8 @@ impl<TItem: Clone, TExt: Clone> Module<TItem, TExt> {
     Ok(())
   }
 
+  /// Call the provided function on every import in the tree. Can be
+  /// short-circuited by returning Err
   pub fn visit_all_imports<E>(
     &self,
     callback: &mut impl FnMut(ModPath, &Self, &Import) -> Result<(), E>,

src/rule/matcher.rs

@@ -3,7 +3,11 @@ use std::rc::Rc;
 use super::state::State;
 use crate::ast::Expr;
 
+/// Cacheable optimized structures for matching patterns on slices. This is
+/// injected to allow experimentation in the matcher implementation.
 pub trait Matcher {
+  /// Build matcher for a pattern
   fn new(pattern: Rc<Vec<Expr>>) -> Self;
+  /// Apply matcher to a token sequence
   fn apply<'a>(&self, source: &'a [Expr]) -> Option<State<'a>>;
 }

src/rule/matcher_vectree/build.rs

@@ -25,9 +25,7 @@ fn scal_cnt<'a>(iter: impl Iterator<Item = &'a Expr>) -> usize {
   iter.take_while(|expr| vec_attrs(expr).is_none()).count()
 }
 
-/// Recursively convert this pattern into a matcher that can be
+pub fn mk_any(pattern: &[Expr]) -> AnyMatcher {
-/// efficiently applied to slices.
-pub fn mk_matcher(pattern: &[Expr]) -> AnyMatcher {
   let left_split = scal_cnt(pattern.iter());
   if pattern.len() <= left_split {
     return AnyMatcher::Scalar(mk_scalv(pattern));
@@ -113,9 +111,9 @@ fn mk_scalar(pattern: &Expr) -> ScalMatcher {
       );
       ScalMatcher::Placeh(*name)
     },
-    Clause::S(c, body) => ScalMatcher::S(*c, Box::new(mk_matcher(body))),
+    Clause::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
     Clause::Lambda(arg, body) =>
-      ScalMatcher::Lambda(Box::new(mk_scalar(arg)), Box::new(mk_matcher(body))),
+      ScalMatcher::Lambda(Box::new(mk_scalar(arg)), Box::new(mk_any(body))),
   }
 }
 
@@ -123,7 +121,7 @@ fn mk_scalar(pattern: &Expr) -> ScalMatcher {
 mod test {
   use std::rc::Rc;
 
-  use super::mk_matcher;
+  use super::mk_any;
   use crate::ast::{Clause, PHClass, Placeholder};
   use crate::interner::{InternedDisplay, Interner};
 
@@ -160,7 +158,7 @@ mod test {
       })
       .into_expr(),
     ];
-    let matcher = mk_matcher(&pattern);
+    let matcher = mk_any(&pattern);
     println!("{}", matcher.bundle(&i));
   }
 }

src/rule/matcher_vectree/mod.rs

@@ -16,5 +16,5 @@ mod scal_match;
 mod shared;
 mod vec_match;
 
-pub use build::mk_matcher;
+// pub use build::mk_matcher;
-pub use shared::AnyMatcher;
+pub use shared::VectreeMatcher;

src/rule/matcher_vectree/shared.rs

@@ -2,14 +2,13 @@ use std::fmt::Write;
 use std::rc::Rc;
 
 use super::any_match::any_match;
-use super::build::mk_matcher;
+use super::build::mk_any;
 use crate::ast::Expr;
 use crate::interner::{InternedDisplay, Interner, Sym, Tok};
 use crate::representations::Primitive;
 use crate::rule::matcher::Matcher;
 use crate::rule::state::State;
-use crate::unwrap_or;
+use crate::utils::{sym2string, unwrap_or, Side};
-use crate::utils::{sym2string, Side};
 
 pub enum ScalMatcher {
   P(Primitive),
@@ -56,7 +55,7 @@ pub enum AnyMatcher {
 }
 impl Matcher for AnyMatcher {
   fn new(pattern: Rc<Vec<Expr>>) -> Self {
-    mk_matcher(&pattern)
+    mk_any(&pattern)
   }
 
   fn apply<'a>(&self, source: &'a [Expr]) -> Option<State<'a>> {
@@ -177,3 +176,27 @@ impl InternedDisplay for AnyMatcher {
     }
   }
 }
+
+// ################ External ################
+
+/// A [Matcher] implementation that builds a priority-order tree of the
+/// vectorial placeholders and handles the scalars on leaves.
+pub struct VectreeMatcher(AnyMatcher);
+impl Matcher for VectreeMatcher {
+  fn new(pattern: Rc<Vec<Expr>>) -> Self {
+    Self(AnyMatcher::new(pattern))
+  }
+
+  fn apply<'a>(&self, source: &'a [Expr]) -> Option<State<'a>> {
+    self.0.apply(source)
+  }
+}
+impl InternedDisplay for VectreeMatcher {
+  fn fmt_i(
+    &self,
+    f: &mut std::fmt::Formatter<'_>,
+    i: &Interner,
+  ) -> std::fmt::Result {
+    self.0.fmt_i(f, i)
+  }
+}

src/rule/matcher_vectree/vec_match.rs

@@ -6,7 +6,7 @@ use super::scal_match::scalv_match;
 use super::shared::VecMatcher;
 use crate::ast::Expr;
 use crate::rule::state::{State, StateEntry};
-use crate::unwrap_or;
+use crate::utils::unwrap_or;
 
 pub fn vec_match<'a>(
   matcher: &VecMatcher,

src/rule/mod.rs

@@ -1,5 +1,6 @@
+//! Substitution rule processing
 mod matcher;
-mod matcher_second;
+mod matcher_vectree;
 mod prepare_rule;
 mod repository;
 mod rule_error;
@@ -7,6 +8,7 @@ mod state;
 mod update_first_seq;
 mod vec_attrs;
 
-pub use matcher_second::AnyMatcher;
+pub use matcher::Matcher;
+pub use matcher_vectree::VectreeMatcher;
 pub use repository::{Repo, Repository};
 pub use rule_error::RuleError;

src/rule/prepare_rule.rs

@@ -22,24 +22,24 @@ fn pad(mut rule: Rule, i: &Interner) -> Rule {
     location: Location::Unknown,
     value: Clause::Placeh(Placeholder { name: i.i("::suffix"), class }),
   }];
-  let rule_head = rule.source.first().expect("Src can never be empty!");
+  let rule_head = rule.pattern.first().expect("Src can never be empty!");
   let prefix_explicit = vec_attrs(rule_head).is_some();
-  let rule_tail = rule.source.last().expect("Unreachable branch!");
+  let rule_tail = rule.pattern.last().expect("Unreachable branch!");
   let suffix_explicit = vec_attrs(rule_tail).is_some();
   let prefix_v = if prefix_explicit { empty } else { prefix };
   let suffix_v = if suffix_explicit { empty } else { suffix };
-  rule.source = Rc::new(
+  rule.pattern = Rc::new(
     prefix_v
       .iter()
-      .chain(rule.source.iter())
+      .chain(rule.pattern.iter())
       .chain(suffix_v.iter())
       .cloned()
       .collect(),
   );
-  rule.target = Rc::new(
+  rule.template = Rc::new(
     prefix_v
       .iter()
-      .chain(rule.target.iter())
+      .chain(rule.template.iter())
       .chain(suffix_v.iter())
       .cloned()
       .collect(),
@@ -118,8 +118,8 @@ fn check_rec_exprv(
 pub fn prepare_rule(rule: Rule, i: &Interner) -> Result<Rule, RuleError> {
   // Dimension check
   let mut types = HashMap::new();
-  check_rec_exprv(&rule.source, &mut types, false)?;
+  check_rec_exprv(&rule.pattern, &mut types, false)?;
-  check_rec_exprv(&rule.target, &mut types, true)?;
+  check_rec_exprv(&rule.template, &mut types, true)?;
   // Padding
   Ok(pad(rule, i))
 }

src/rule/repository.rs

@@ -8,7 +8,7 @@ use ordered_float::NotNan;
 use super::matcher::Matcher;
 use super::prepare_rule::prepare_rule;
 use super::state::apply_exprv;
-use super::{update_first_seq, AnyMatcher, RuleError};
+use super::{update_first_seq, RuleError, VectreeMatcher};
 use crate::ast::{Expr, Rule};
 use crate::interner::{InternedDisplay, Interner, Sym};
 use crate::utils::Substack;
@@ -16,7 +16,7 @@ use crate::utils::Substack;
 #[derive(Debug)]
 pub struct CachedRule<M: Matcher> {
   matcher: M,
-  source: Rc<Vec<Expr>>,
+  pattern: Rc<Vec<Expr>>,
   template: Rc<Vec<Expr>>,
 }
 
@@ -26,7 +26,7 @@ impl<M: InternedDisplay + Matcher> InternedDisplay for CachedRule<M> {
     f: &mut std::fmt::Formatter<'_>,
     i: &Interner,
   ) -> std::fmt::Result {
-    for item in self.source.iter() {
+    for item in self.pattern.iter() {
       item.fmt_i(f, i)?;
       f.write_char(' ')?;
     }
@@ -35,8 +35,13 @@ impl<M: InternedDisplay + Matcher> InternedDisplay for CachedRule<M> {
   }
 }
 
-/// Manages a priority queue of substitution rules and allows to apply
+/// Substitution rule scheduler
-/// them
+///
+/// Manages a priority queue of rules and offers functions to apply them. The
+/// rules are stored in an optimized structure but the repository is generic
+/// over the implementation of this optimized form.
+///
+/// If you don't know what to put in the generic parameter, use [Repo]
 pub struct Repository<M: Matcher> {
   cache: Vec<(CachedRule<M>, HashSet<Sym>, NotNan<f64>)>,
 }
@@ -52,14 +57,17 @@ impl<M: Matcher> Repository<M> {
         let prio = r.prio;
         let rule = prepare_rule(r.clone(), i).map_err(|e| (r, e))?;
         let mut glossary = HashSet::new();
-        for e in rule.source.iter() {
+        for e in rule.pattern.iter() {
           e.visit_names(Substack::Bottom, &mut |op| {
             glossary.insert(op);
           })
         }
-        let matcher = M::new(rule.source.clone());
+        let matcher = M::new(rule.pattern.clone());
-        let prep =
+        let prep = CachedRule {
-          CachedRule { matcher, source: rule.source, template: rule.target };
+          matcher,
+          pattern: rule.pattern,
+          template: rule.template,
+        };
         Ok((prep, glossary, prio))
       })
       .collect::<Result<Vec<_>, _>>()?;
@@ -163,4 +171,5 @@ impl<M: InternedDisplay + Matcher> InternedDisplay for Repository<M> {
   }
 }
 
-pub type Repo = Repository<AnyMatcher>;
+/// Repository with the default matcher implementation
+pub type Repo = Repository<VectreeMatcher>;

src/rule/rule_error.rs

@@ -2,13 +2,16 @@ use std::fmt;
 
 use crate::interner::{InternedDisplay, Interner, Tok};
 
+/// Various reasons why a substitution rule may be invalid
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum RuleError {
+  /// A key is present in the template but not the pattern
   Missing(Tok<String>),
+  /// A key uses a different arity in the template and in the pattern
   TypeMismatch(Tok<String>),
-  /// Multiple occurences of a placeholder in a pattern are no longer
+  /// Multiple occurences of a placeholder in a pattern
-  /// supported.
   Multiple(Tok<String>),
+  /// Two vectorial placeholders are next to each other
   VecNeighbors(Tok<String>, Tok<String>),
 }
 

src/rule/state.rs

@@ -5,7 +5,7 @@ use itertools::Itertools;
 
 use crate::ast::{Clause, Expr, PHClass, Placeholder};
 use crate::interner::Tok;
-use crate::unwrap_or;
+use crate::utils::unwrap_or;
 
 #[derive(Clone, Copy, Debug, PartialEq)]
 pub enum StateEntry<'a> {

src/stl/assertion_error.rs

@@ -13,10 +13,10 @@ pub struct AssertionError {
 }
 
 impl AssertionError {
-  pub fn fail(
+  pub fn fail<T>(
     value: ExprInst,
     assertion: &'static str,
-  ) -> Result<!, Rc<dyn ExternError>> {
+  ) -> Result<T, Rc<dyn ExternError>> {
     return Err(Self { value, assertion }.into_extern());
   }
 

src/stl/bool/equals.rs

@@ -1,8 +1,8 @@
 use std::fmt::Debug;
 
 use super::super::assertion_error::AssertionError;
+use super::super::litconv::with_lit;
 use super::boolean::Boolean;
-use crate::external::litconv::with_lit;
 use crate::representations::interpreted::ExprInst;
 use crate::representations::Literal;
 use crate::{atomic_impl, atomic_redirect, externfn_impl};

src/stl/bool/ifthenelse.rs

@@ -1,8 +1,8 @@
 use std::fmt::Debug;
 use std::rc::Rc;
 
+use super::super::assertion_error::AssertionError;
 use super::Boolean;
-use crate::external::assertion_error::AssertionError;
 use crate::representations::interpreted::{Clause, ExprInst};
 use crate::representations::PathSet;
 use crate::{atomic_impl, atomic_redirect, externfn_impl};

src/stl/conv/parse_float.rs

@@ -3,7 +3,7 @@ use std::fmt::Debug;
 use chumsky::Parser;
 
 use super::super::assertion_error::AssertionError;
-use crate::external::litconv::with_lit;
+use super::super::litconv::with_lit;
 use crate::parse::float_parser;
 use crate::representations::interpreted::ExprInst;
 use crate::representations::Literal;

src/stl/conv/parse_uint.rs

@@ -2,8 +2,8 @@ use std::fmt::Debug;
 
 use chumsky::Parser;
 
-use crate::external::assertion_error::AssertionError;
+use super::super::assertion_error::AssertionError;
-use crate::external::litconv::with_lit;
+use super::super::litconv::with_lit;
 use crate::parse::int_parser;
 use crate::representations::interpreted::ExprInst;
 use crate::representations::Literal;

src/stl/conv/to_string.rs

@@ -1,6 +1,6 @@
 use std::fmt::Debug;
 
-use crate::external::litconv::with_lit;
+use super::super::litconv::with_lit;
 use crate::representations::interpreted::ExprInst;
 use crate::representations::Literal;
 use crate::{atomic_impl, atomic_redirect, externfn_impl};

src/stl/cpsio/io.rs

@@ -1,10 +1,11 @@
 use std::io::{self, Write};
 
-use crate::external::runtime_error::RuntimeError;
+use super::super::runtime_error::RuntimeError;
+use crate::atomic_inert;
 use crate::interpreter::{HandlerParm, HandlerRes};
 use crate::representations::interpreted::{Clause, ExprInst};
 use crate::representations::{Literal, Primitive};
-use crate::{atomic_inert, unwrap_or};
+use crate::utils::unwrap_or;
 
 /// An IO command to be handled by the host application.
 #[derive(Clone, Debug)]

src/stl/cpsio/panic.rs

@@ -1,6 +1,6 @@
 use std::fmt::Display;
 
-use crate::external::litconv::with_str;
+use super::super::litconv::with_str;
 use crate::foreign::ExternError;
 use crate::representations::interpreted::ExprInst;
 use crate::{atomic_impl, atomic_redirect, externfn_impl};

src/stl/cpsio/print.rs

@@ -1,7 +1,7 @@
 use std::fmt::Debug;
 
+use super::super::litconv::with_str;
 use super::io::IO;
-use crate::external::litconv::with_str;
 use crate::foreign::{Atomic, AtomicResult, AtomicReturn};
 use crate::interpreter::Context;
 use crate::representations::interpreted::ExprInst;

src/stl/litconv.rs

@@ -1,6 +1,6 @@
 use std::rc::Rc;
 
-use crate::external::assertion_error::AssertionError;
+use super::assertion_error::AssertionError;
 use crate::foreign::ExternError;
 use crate::representations::interpreted::ExprInst;
 use crate::representations::Literal;
@@ -12,7 +12,7 @@ pub fn with_lit<T>(
   predicate: impl FnOnce(&Literal) -> Result<T, Rc<dyn ExternError>>,
 ) -> Result<T, Rc<dyn ExternError>> {
   x.with_literal(predicate)
-    .map_err(|()| AssertionError::ext(x.clone(), "a literal value"))
+    .map_err(|_| AssertionError::ext(x.clone(), "a literal value"))
     .and_then(|r| r)
 }
 

src/stl/mk_stl.rs

@@ -6,6 +6,6 @@ use super::str::str;
 use crate::interner::Interner;
 use crate::pipeline::ConstTree;
 
-pub fn std(i: &Interner) -> ConstTree {
+pub fn mk_stl(i: &Interner) -> ConstTree {
   cpsio(i) + conv(i) + bool(i) + str(i) + num(i)
 }

src/stl/mod.rs

@@ -3,9 +3,10 @@ mod bool;
 mod conv;
 mod cpsio;
 mod litconv;
+mod mk_stl;
 mod num;
 mod runtime_error;
-pub mod std;
 mod str;
 
-pub use cpsio::{handle, IO};
+pub use cpsio::{handle as handleIO, IO};
+pub use mk_stl::mk_stl;

src/stl/num/numeric.rs

@@ -3,8 +3,8 @@ use std::rc::Rc;
 
 use ordered_float::NotNan;
 
-use crate::external::assertion_error::AssertionError;
+use super::super::assertion_error::AssertionError;
-use crate::external::litconv::with_lit;
+use super::super::litconv::with_lit;
 use crate::foreign::ExternError;
 use crate::representations::interpreted::{Clause, ExprInst};
 use crate::representations::{Literal, Primitive};

src/stl/runtime_error.rs

@@ -11,10 +11,10 @@ pub struct RuntimeError {
 }
 
 impl RuntimeError {
-  pub fn fail(
+  pub fn fail<T>(
     message: String,
     operation: &'static str,
-  ) -> Result<!, Rc<dyn ExternError>> {
+  ) -> Result<T, Rc<dyn ExternError>> {
     return Err(Self { message, operation }.into_extern());
   }
 

src/stl/str/char_at.rs

@@ -1,7 +1,7 @@
 use std::fmt::Debug;
 
-use crate::external::litconv::{with_str, with_uint};
+use super::super::litconv::{with_str, with_uint};
-use crate::external::runtime_error::RuntimeError;
+use super::super::runtime_error::RuntimeError;
 use crate::representations::interpreted::{Clause, ExprInst};
 use crate::representations::{Literal, Primitive};
 use crate::{atomic_impl, atomic_redirect, externfn_impl};

src/stl/str/concatenate.rs

@@ -1,6 +1,6 @@
 use std::fmt::Debug;
 
-use crate::external::litconv::with_str;
+use super::super::litconv::with_str;
 use crate::representations::interpreted::{Clause, ExprInst};
 use crate::representations::{Literal, Primitive};
 use crate::{atomic_impl, atomic_redirect, externfn_impl};

src/utils/cache.rs

@@ -2,10 +2,12 @@ use std::cell::RefCell;
 use std::hash::Hash;
 
 use hashbrown::HashMap;
+use trait_set::trait_set;
 
 // TODO: make this a crate
-pub trait Callback<'a, I, O: 'static> = Fn(I, &Cache<'a, I, O>) -> O;
+trait_set! {
-
+  pub trait Callback<'a, I, O: 'static> = Fn(I, &Cache<'a, I, O>) -> O;
+}
 pub type CbBox<'a, I, O> = Box<dyn Callback<'a, I, O> + 'a>;
 
 /// Cache the return values of an effectless closure in a hashmap

src/utils/iter.rs

@@ -17,7 +17,6 @@ pub fn box_empty<'a, T: 'a>() -> BoxedIter<'a, T> {
 }
 
 /// Chain various iterators into a [BoxedIter]
-#[macro_export]
 macro_rules! box_chain {
   ($curr:expr) => {
     Box::new($curr) as BoxedIter<_>
@@ -27,6 +26,8 @@ macro_rules! box_chain {
   };
 }
 
+pub(crate) use box_chain;
+
 /// Flatten an iterator of iterators into a boxed iterator of the inner
 /// nested values
 pub fn box_flatten<
@@ -40,7 +41,7 @@ pub fn box_flatten<
   Box::new(i.flatten())
 }
 
-/// Convert an iterator into a Box<dyn Iterator>
+/// Convert an iterator into a `Box<dyn Iterator>`
 pub fn into_boxed_iter<'a, T: 'a + IntoIterator>(
   t: T,
 ) -> BoxedIter<'a, <T as IntoIterator>::Item> {

src/utils/mod.rs

@@ -6,7 +6,6 @@ mod side;
 mod string_from_charset;
 mod substack;
 mod unwrap_or;
-mod xloop;
 
 pub use cache::Cache;
 pub use print_nname::sym2string;
@@ -14,6 +13,7 @@ pub use pushed::pushed;
 pub use replace_first::replace_first;
 pub use side::Side;
 pub use substack::{Stackframe, Substack, SubstackIterator};
+pub(crate) use unwrap_or::unwrap_or;
 pub mod iter;
 pub use iter::BoxedIter;
 pub use string_from_charset::string_from_charset;

src/utils/side.rs

@@ -66,8 +66,8 @@ impl Side {
       Side::Right => (opposite, this),
     }
   }
-  /// Produces an increasing sequence on Right, and a decreasing sequence
+  /// Walk a double ended iterator (assumed to be left-to-right) in this
-  /// on Left
+  /// direction
   pub fn walk<'a, I: DoubleEndedIterator + 'a>(
     &self,
     iter: I,

src/utils/substack.rs

@@ -56,6 +56,9 @@ impl<'a, T> Substack<'a, T> {
       Self::Bottom => 0,
     }
   }
+  pub fn is_empty(&self) -> bool {
+    self.len() == 0
+  }
 }
 
 impl<'a, T: Debug> Debug for Substack<'a, T> {

src/utils/unwrap_or.rs

@@ -1,8 +1,9 @@
 /// A macro version of [Option::unwrap_or_else] which supports flow
 /// control statements such as `return` and `break` in the "else" branch.
-#[macro_export]
 macro_rules! unwrap_or {
   ($m:expr; $fail:expr) => {{
     if let Some(res) = ($m) { res } else { $fail }
   }};
 }
+
+pub(crate) use unwrap_or;

src/utils/xloop.rs

@@ -1,95 +0,0 @@
-/// Imitates a regular for loop with an exit clause using Rust's `loop`
-/// keyword. This macro brings the break value to all existing Rust loops,
-/// by allowing you to specify an exit expression in case the loop was
-/// broken by the condition and not an explicit `break`.
-///
-/// Since the exit expression can also be a block, this also allows you to
-/// execute other code when the condition fails. This can also be used to
-/// re-enter the loop with an explicit `continue` statement.
-///
-/// The macro also adds support for classic for loops familiar to everyone
-/// since C, except with the addition of an exit statement these too can
-/// be turned into expressions.
-///
-/// ```
-/// xloop!(for i in 0..10; {
-///   connection.try_connect()
-///   if connection.ready() {
-///     break Some(connection)
-///   }
-/// }; None)
-/// ```
-///
-/// While loop with reentry. This is a very convoluted example but
-/// displays the idea quite clearly.
-///
-/// ```
-/// xloop!(while socket.is_open(); {
-///   let (data, is_end) = socket.read();
-///   all_data.append(data)
-///   if is_end { break Ok(all_data) }
-/// }; {
-///   if let Ok(new_sock) = open_socket(socket.position()) {
-///     new_sock.set_position(socket.position());
-///     socket = new_sock;
-///     continue
-///   } else {
-///     Err(DownloadError::ConnectionLost)
-///   }
-/// })
-/// ```
-///
-/// CUDA algorythm for O(log n) summation using a C loop
-///
-/// ```
-/// xloop!(let mut leap = 1; own_id*2 + leap < batch_size; leap *= 2; {
-///   batch[own_id*2] += batch[own_id*2 + leap]
-/// })
-/// ```
-///
-/// The above loop isn't used as an expression, but an exit expression -
-/// or block - can be added to these as well just like the others. In all
-/// cases the exit expression is optional, its default value is `()`.
-///
-/// TODO: find a valid use case for While let for a demo
-/// TODO: break out into crate
-#[macro_export]
-macro_rules! xloop {
-  (for $p:pat in $it:expr; $body:stmt) => {
-    xloop!(for $p in $it; $body; ())
-  };
-  (for $p:pat in $iit:expr; $body:stmt; $exit:stmt) => {
-    {
-      let mut i = $iit.into_iter();
-      xloop!(let Some($p) = i.next(); $body; $exit)
-    }
-  };
-  (let $p:pat = $e:expr; $body:stmt) => {
-    xloop!(let $p = $e; $body; ())
-  };
-  (let $p:pat = $e:expr; $body:stmt; $exit:stmt) => {
-    {
-      loop {
-        if let $p = $e { $body }
-        else { break { $exit } }
-      }
-    }
-  };
-  (while $cond:expr; $body:stmt) => {
-    xloop!($cond; $body; ())
-  };
-  (while $cond:expr; $body:stmt; $exit:stmt) => {
-    {
-      loop {
-        if $cond { $body }
-        else { break { $exit } }
-      }
-    }
-  };
-  ($init:stmt; $cond:expr; $step:stmt; $body:stmt) => {
-    xloop!(for ( $init; $cond; $step ) $body; ())
-  };
-  ($init:stmt; $cond:expr; $step:stmt; $body:stmt; $exit:stmt) => {
-    { $init; xloop!(while $cond; { $body; $step }; $exit) }
-  };
-}