in midst of refactor

orchidlang/src/bin/cli/mod.rs

@@ -0,0 +1,2 @@
+mod prompt;
+pub use prompt::cmd_prompt;

orchidlang/src/bin/cli/prompt.rs

@@ -0,0 +1,11 @@
+use std::io::{self, Error, Write};
+
+pub fn cmd_prompt(prompt: &str) -> Result<(String, Vec<String>), Error> {
+  print!("{}", prompt);
+  io::stdout().flush()?;
+  let mut cmdln = String::new();
+  io::stdin().read_line(&mut cmdln)?;
+  let mut segments = cmdln.split(' ');
+  let cmd = if let Some(cmd) = segments.next() { cmd } else { "" };
+  Ok((cmd.to_string(), segments.map(str::to_string).collect()))
+}

orchidlang/src/bin/features/macro_debug.rs

@@ -0,0 +1,74 @@
+use itertools::Itertools;
+use orchidlang::error::Reporter;
+use orchidlang::facade::macro_runner::MacroRunner;
+use orchidlang::libs::std::exit_status::OrcExitStatus;
+use orchidlang::location::{CodeGenInfo, CodeLocation};
+use orchidlang::name::Sym;
+use orchidlang::pipeline::project::{ItemKind, ProjItem, ProjectTree};
+use orchidlang::sym;
+
+use crate::cli::cmd_prompt;
+
+/// A little utility to step through the reproject of a macro set
+pub fn main(tree: ProjectTree, symbol: Sym) -> OrcExitStatus {
+  print!("Macro debugger starting on {symbol}");
+  let location = CodeLocation::new_gen(CodeGenInfo::no_details(sym!(orcx::macro_runner)));
+  let expr_ent = match tree.0.walk1_ref(&[], &symbol[..], |_| true) {
+    Ok((e, _)) => e.clone(),
+    Err(e) => {
+      eprintln!("{}", e.at(&location.origin()));
+      return OrcExitStatus::Failure;
+    },
+  };
+  let mut expr = match expr_ent.item() {
+    Some(ProjItem { kind: ItemKind::Const(c) }) => c.clone(),
+    _ => {
+      eprintln!("macro-debug argument must be a constant");
+      return OrcExitStatus::Failure;
+    },
+  };
+  let reporter = Reporter::new();
+  let macro_runner = MacroRunner::new(&tree, None, &reporter);
+  reporter.assert_exit();
+  println!("\nInitial state: {expr}");
+  // print_for_debug(&code);
+  let mut steps = macro_runner.step(expr.clone()).enumerate();
+  loop {
+    let (cmd, _) = cmd_prompt("\ncmd> ").unwrap();
+    match cmd.trim() {
+      "" | "n" | "next" => match steps.next() {
+        None => print!("Halted"),
+        Some((idx, c)) => {
+          expr = c;
+          print!("Step {idx}: {expr}");
+        },
+      },
+      "p" | "print" => {
+        let glossary = expr.value.collect_names();
+        let gl_str = glossary.iter().join(", ");
+        print!("code: {expr}\nglossary: {gl_str}")
+      },
+      "d" | "dump" => print!("Rules: {}", macro_runner.repo),
+      "q" | "quit" => return OrcExitStatus::Success,
+      "complete" => {
+        match steps.last() {
+          Some((idx, c)) => print!("Step {idx}: {c}"),
+          None => print!("Already halted"),
+        }
+        return OrcExitStatus::Success;
+      },
+      "h" | "help" => print!(
+        "Available commands:
+        \t<blank>, n, next\t\ttake a step
+        \tp, print\t\tprint the current state
+        \td, dump\t\tprint the rule table
+        \tq, quit\t\texit
+        \th, help\t\tprint this text"
+      ),
+      _ => {
+        print!("unrecognized command \"{}\", try \"help\"", cmd);
+        continue;
+      },
+    }
+  }
+}

orchidlang/src/bin/features/mod.rs

@@ -0,0 +1,4 @@
+pub mod macro_debug;
+pub mod print_project;
+pub mod shared;
+pub mod tests;

orchidlang/src/bin/features/print_project.rs

@@ -0,0 +1,55 @@
+use itertools::Itertools;
+use orchidlang::pipeline::project::{ItemKind, ProjItem, ProjectMod};
+use orchidlang::tree::{ModEntry, ModMember};
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+pub struct ProjPrintOpts {
+  pub width: u16,
+  pub hide_locations: bool,
+}
+
+fn indent(amount: u16) -> String { "  ".repeat(amount.into()) }
+
+pub fn print_proj_mod(module: &ProjectMod, lvl: u16, opts: ProjPrintOpts) -> String {
+  let mut acc = String::new();
+  let tab = indent(lvl);
+  for (key, ModEntry { member, x }) in &module.entries {
+    let mut line_acc = String::new();
+    for c in &x.comments {
+      line_acc += &format!("{tab}, --[|{}|]--\n", c);
+    }
+    if x.exported {
+      line_acc += &format!("{tab}export ");
+    } else {
+      line_acc += &tab
+    }
+    match member {
+      ModMember::Sub(module) => {
+        line_acc += &format!("module {key} {{\n");
+        line_acc += &print_proj_mod(module, lvl + 1, opts);
+        line_acc += &format!("{tab}}}");
+      },
+      ModMember::Item(ProjItem { kind: ItemKind::None }) => {
+        line_acc += &format!("keyword {key}");
+      },
+      ModMember::Item(ProjItem { kind: ItemKind::Alias(tgt) }) => {
+        line_acc += &format!("alias {key} => {tgt}");
+      },
+      ModMember::Item(ProjItem { kind: ItemKind::Const(val) }) => {
+        line_acc += &format!("const {key} := {val}");
+      },
+    }
+    if !x.locations.is_empty() && !opts.hide_locations {
+      let locs = x.locations.iter().map(|l| l.to_string()).join(", ");
+      let line_len = line_acc.split('\n').last().unwrap().len();
+      match usize::from(opts.width).checked_sub(locs.len() + line_len + 4) {
+        Some(padding) => line_acc += &" ".repeat(padding),
+        None => line_acc += &format!("\n{tab}  @ "),
+      }
+      line_acc += &locs;
+    }
+    line_acc += "\n";
+    acc += &line_acc
+  }
+  acc
+}

orchidlang/src/bin/features/shared.rs

@@ -0,0 +1,64 @@
+use std::io::BufReader;
+use std::thread;
+
+use orchidlang::facade::loader::Loader;
+use orchidlang::libs::asynch::system::AsynchSystem;
+use orchidlang::libs::directfs::DirectFS;
+use orchidlang::libs::io::{IOService, Sink, Source, Stream};
+use orchidlang::libs::scheduler::system::SeqScheduler;
+use orchidlang::libs::std::std_system::StdConfig;
+
+pub fn stdin_source() -> Source { BufReader::new(Box::new(std::io::stdin())) }
+pub fn stdout_sink() -> Sink { Box::new(std::io::stdout()) }
+pub fn stderr_sink() -> Sink { Box::new(std::io::stderr()) }
+
+pub fn with_std_env<T>(cb: impl for<'a> FnOnce(Loader<'a>) -> T) -> T {
+  with_env(stdin_source(), stdout_sink(), stderr_sink(), cb)
+}
+
+pub fn with_env<T>(
+  stdin: Source,
+  stdout: Sink,
+  stderr: Sink,
+  cb: impl for<'a> FnOnce(Loader<'a>) -> T,
+) -> T {
+  let mut asynch = AsynchSystem::new();
+  let scheduler = SeqScheduler::new(&mut asynch);
+  let std_streams = [
+    ("stdin", Stream::Source(stdin)),
+    ("stdout", Stream::Sink(stdout)),
+    ("stderr", Stream::Sink(stderr)),
+  ];
+  let env = Loader::new()
+    .add_system(StdConfig { impure: true })
+    .add_system(asynch)
+    .add_system(scheduler.clone())
+    .add_system(IOService::new(scheduler.clone(), std_streams))
+    .add_system(DirectFS::new(scheduler));
+  cb(env)
+}
+
+pub fn worker_cnt() -> usize { thread::available_parallelism().map(usize::from).unwrap_or(1) }
+
+macro_rules! unwrap_exit {
+  ($param:expr) => {
+    match $param {
+      Ok(v) => v,
+      Err(e) => {
+        eprintln!("{e}");
+        return ExitCode::FAILURE;
+      },
+    }
+  };
+  ($param:expr; $error:expr) => {
+    match $param {
+      Ok(v) => v,
+      Err(e) => {
+        eprintln!("{e}");
+        return $error;
+      },
+    }
+  };
+}
+
+pub(crate) use unwrap_exit;

orchidlang/src/bin/features/tests.rs

@@ -0,0 +1,111 @@
+use std::fmt;
+use std::io::BufReader;
+use std::path::Path;
+
+use hashbrown::HashMap;
+use itertools::Itertools;
+use orchidlang::error::{ProjectError, ProjectResult, Reporter};
+use orchidlang::facade::loader::Loader;
+use orchidlang::facade::macro_runner::MacroRunner;
+use orchidlang::facade::merge_trees::NortConst;
+use orchidlang::facade::process::Process;
+use orchidlang::foreign::error::{RTError, RTErrorObj, RTResult};
+use orchidlang::foreign::inert::Inert;
+use orchidlang::interpreter::error::RunError;
+use orchidlang::interpreter::nort;
+use orchidlang::libs::io::{Sink, Source};
+use orchidlang::libs::std::exit_status::OrcExitStatus;
+use orchidlang::name::Sym;
+use rayon::iter::ParallelIterator;
+use rayon::slice::ParallelSlice;
+
+use super::shared::{with_env, worker_cnt};
+
+pub fn mock_source() -> Source { BufReader::new(Box::new(&[][..])) }
+pub fn mock_sink() -> Sink { Box::<Vec<u8>>::default() }
+pub fn with_mock_env<T>(cb: impl for<'a> FnOnce(Loader<'a>) -> T) -> T {
+  with_env(mock_source(), mock_sink(), mock_sink(), cb)
+}
+
+#[derive(Clone)]
+pub struct TestDidNotHalt(Sym);
+impl RTError for TestDidNotHalt {}
+impl fmt::Display for TestDidNotHalt {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Test {} did not halt", self.0)
+  }
+}
+
+#[derive(Clone)]
+pub struct TestDidNotSucceed(Sym, nort::Expr);
+impl RTError for TestDidNotSucceed {}
+impl fmt::Display for TestDidNotSucceed {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Test {} settled on {}", self.0, self.1)
+  }
+}
+
+pub fn run_test(proc: &mut Process, name: Sym, data: NortConst) -> RTResult<()> {
+  let res = proc.run(data.value, Some(10_000)).map_err(|e| match e {
+    RunError::Extern(e) => e,
+    RunError::Interrupted(_) => TestDidNotHalt(name.clone()).pack(),
+  })?;
+  match res.clone().downcast()? {
+    Inert(OrcExitStatus::Success) => Ok(()),
+    _ => Err(TestDidNotSucceed(name, res).pack()),
+  }
+}
+pub fn run_tests(
+  dir: &Path,
+  macro_limit: usize,
+  threads: Option<usize>,
+  tests: &[(Sym, NortConst)],
+) -> ProjectResult<()> {
+  with_mock_env(|env| {
+    let reporter = Reporter::new();
+    env.proc_dir(dir.to_owned(), true, Some(macro_limit), &reporter);
+    reporter.bind()
+  })?;
+  let threads = threads.unwrap_or_else(worker_cnt);
+  rayon::ThreadPoolBuilder::new().num_threads(threads).build_global().unwrap();
+  let batch_size = tests.len().div_ceil(threads);
+  let errors = (tests.par_chunks(batch_size))
+    .map(|tests| {
+      with_mock_env(|env| {
+        let reporter = Reporter::new();
+        let mut proc = env.proc_dir(dir.to_owned(), true, Some(macro_limit), &reporter);
+        reporter.assert(); // checked above
+        (tests.iter())
+          .filter_map(|(test, constant)| {
+            Some((test.clone(), run_test(&mut proc, test.clone(), constant.clone()).err()?))
+          })
+          .collect_vec()
+      })
+    })
+    .collect::<Vec<_>>()
+    .into_iter()
+    .flatten()
+    .collect::<HashMap<_, _>>();
+  if errors.is_empty() { Ok(()) } else { Err(TestsFailed(errors).pack()) }
+}
+
+pub struct TestsFailed(HashMap<Sym, RTErrorObj>);
+impl ProjectError for TestsFailed {
+  const DESCRIPTION: &'static str = "Various tests failed";
+  fn message(&self) -> String {
+    ([format!("{} tests failed. Errors:", self.0.len())].into_iter())
+      .chain(self.0.iter().map(|(k, e)| format!("In {k}, {e}")))
+      .join("\n")
+  }
+}
+
+pub fn get_tree_tests(dir: &Path, reporter: &Reporter) -> ProjectResult<Vec<(Sym, NortConst)>> {
+  with_mock_env(|env| {
+    let tree = env.load_dir(dir.to_owned(), reporter);
+    let tree = MacroRunner::new(&tree, Some(10_000), reporter).run_macros(tree, reporter);
+    (tree.all_consts().into_iter())
+      .filter(|(_, rep)| rep.comments.iter().any(|s| s.trim() == "test"))
+      .map(|(k, v)| Ok((k.clone(), NortConst::convert_from(v, reporter))))
+      .collect::<ProjectResult<Vec<_>>>()
+  })
+}

orchidlang/src/bin/orcx.rs

@@ -0,0 +1,283 @@
+mod cli;
+mod features;
+
+use std::fs::File;
+use std::io::{stdin, stdout, Write};
+use std::path::PathBuf;
+use std::process::ExitCode;
+
+use clap::{Parser, Subcommand};
+use hashbrown::HashSet;
+use itertools::Itertools;
+use never::Never;
+use orchidlang::error::Reporter;
+use orchidlang::facade::macro_runner::MacroRunner;
+use orchidlang::facade::merge_trees::{merge_trees, NortConst};
+use orchidlang::facade::process::Process;
+use orchidlang::foreign::inert::Inert;
+use orchidlang::gen::tpl;
+use orchidlang::gen::traits::Gen;
+use orchidlang::interpreter::gen_nort::nort_gen;
+use orchidlang::interpreter::nort::{self};
+use orchidlang::libs::std::exit_status::OrcExitStatus;
+use orchidlang::libs::std::string::OrcString;
+use orchidlang::location::{CodeGenInfo, CodeLocation, SourceRange};
+use orchidlang::name::Sym;
+use orchidlang::parse::context::FlatLocContext;
+use orchidlang::parse::lexer::{lex, Lexeme};
+use orchidlang::sym;
+use orchidlang::tree::{ModMemberRef, TreeTransforms};
+use orchidlang::virt_fs::{decl_file, DeclTree};
+
+use crate::features::macro_debug;
+use crate::features::print_project::{print_proj_mod, ProjPrintOpts};
+use crate::features::shared::{stderr_sink, stdout_sink, unwrap_exit, with_env, with_std_env};
+use crate::features::tests::{get_tree_tests, mock_source, run_test, run_tests, with_mock_env};
+
+#[derive(Subcommand, Debug)]
+enum Command {
+  /// Run unit tests, any constant annotated --[[ test ]]--
+  Test {
+    /// Specify an exact test to run
+    #[arg(long)]
+    only: Option<String>,
+    #[arg(long, short)]
+    threads: Option<usize>,
+    #[arg(long)]
+    system: Option<String>,
+  },
+  #[command(arg_required_else_help = true)]
+  MacroDebug {
+    #[arg(long, short)]
+    symbol: String,
+  },
+  ListMacros,
+  ProjectTree {
+    #[arg(long, default_value_t = false)]
+    hide_locations: bool,
+    #[arg(long)]
+    width: Option<u16>,
+  },
+  Repl,
+}
+/// Orchid interpreter
+#[derive(Parser, Debug)]
+#[command(name = "Orchid Executor")]
+#[command(author = "Lawrence Bethlenfalvy <lbfalvy@protonmail.com>")]
+#[command(long_about = Some("Execute Orchid projects from the file system"))]
+struct Args {
+  /// Folder containing main.orc or the manually specified entry module
+  #[arg(short, long, default_value = ".")]
+  pub dir: String,
+  /// Alternative entrypoint for the interpreter
+  #[arg(short, long)]
+  pub main: Option<String>,
+  /// Maximum number of steps taken by the macro executor
+  #[arg(long, default_value_t = 10_000)]
+  pub macro_limit: usize,
+
+  #[command(subcommand)]
+  pub command: Option<Command>,
+}
+impl Args {
+  /// Validate the project directory and the
+  pub fn chk_dir_main(&self) -> Result<(), String> {
+    let dir_path = PathBuf::from(&self.dir);
+    if !dir_path.is_dir() {
+      return Err(format!("{} is not a directory", dir_path.display()));
+    }
+    let segs = match &self.main {
+      Some(s) => s.split("::").collect::<Vec<_>>(),
+      None => match File::open("./main.orc") {
+        Ok(_) => return Ok(()),
+        Err(e) => return Err(format!("Cannot open './main.orc'\n{e}")),
+      },
+    };
+    if segs.len() < 2 {
+      return Err("Entry point too short".to_string());
+    };
+    let (_, pathsegs) = segs.split_last().unwrap();
+    let mut possible_files = pathsegs.iter().scan(dir_path, |path, seg| {
+      path.push(seg);
+      Some(path.with_extension("orc"))
+    });
+    if possible_files.all(|p| File::open(p).is_err()) {
+      let out_path = pathsegs.join("::");
+      let pbuf = PathBuf::from(&self.dir);
+      return Err(format!("{out_path} not found in {}", pbuf.display()));
+    }
+    Ok(())
+  }
+
+  pub fn chk_proj(&self) -> Result<(), String> { self.chk_dir_main() }
+}
+
+pub fn main() -> ExitCode {
+  let args = Args::parse();
+  unwrap_exit!(args.chk_proj());
+  let dir = PathBuf::from(args.dir);
+  let main_s = args.main.as_ref().map_or("tree::main::main", |s| s);
+  let main = Sym::parse(main_s).expect("--main cannot be empty");
+  let location = CodeLocation::new_gen(CodeGenInfo::no_details(sym!(orcx::entrypoint)));
+  let reporter = Reporter::new();
+
+  // subcommands
+  #[allow(clippy::blocks_in_conditions)]
+  match args.command {
+    Some(Command::ListMacros) => with_mock_env(|env| {
+      let tree = env.load_main(dir, [main], &reporter);
+      let mr = MacroRunner::new(&tree, None, &reporter);
+      println!("Parsed rules: {}", mr.repo);
+      ExitCode::SUCCESS
+    }),
+    Some(Command::ProjectTree { hide_locations, width }) => {
+      let tree = with_mock_env(|env| env.load_main(dir, [main], &reporter));
+      let w = width.or_else(|| termsize::get().map(|s| s.cols)).unwrap_or(74);
+      let print_opts = ProjPrintOpts { width: w, hide_locations };
+      println!("Project tree: {}", print_proj_mod(&tree.0, 0, print_opts));
+      ExitCode::SUCCESS
+    },
+    Some(Command::MacroDebug { symbol }) => with_mock_env(|env| {
+      let tree = env.load_main(dir, [main], &reporter);
+      let symbol = Sym::parse(&symbol).expect("macro-debug needs an argument");
+      macro_debug::main(tree, symbol).code()
+    }),
+    Some(Command::Test { only: Some(_), threads: Some(_), .. }) => {
+      eprintln!(
+        "Each test case runs in a single thread.
+        --only and --threads cannot both be specified"
+      );
+      ExitCode::FAILURE
+    },
+    Some(Command::Test { only: Some(_), system: Some(_), .. }) => {
+      eprintln!(
+        "Conflicting test filters applied. --only runs a single test by
+        symbol name, while --system runs all tests in a system"
+      );
+      ExitCode::FAILURE
+    },
+    Some(Command::Test { only: None, threads, system: None }) => {
+      let tree_tests = reporter.unwrap_exit(get_tree_tests(&dir, &reporter));
+      unwrap_exit!(run_tests(&dir, args.macro_limit, threads, &tree_tests));
+      ExitCode::SUCCESS
+    },
+    Some(Command::Test { only: Some(symbol), threads: None, system: None }) => {
+      let symbol = Sym::parse(&symbol).expect("Test needs an argument");
+      with_env(mock_source(), stdout_sink(), stderr_sink(), |env| {
+        // iife in lieu of try blocks
+        let tree = env.load_main(dir.clone(), [symbol.clone()], &reporter);
+        let mr = MacroRunner::new(&tree, Some(args.macro_limit), &reporter);
+        let consts = mr.run_macros(tree, &reporter).all_consts();
+        let test = consts.get(&symbol).expect("Test not found");
+        let nc = NortConst::convert_from(test.clone(), &reporter);
+        let mut proc = Process::new(merge_trees(consts, env.systems(), &reporter), env.handlers());
+        unwrap_exit!(run_test(&mut proc, symbol.clone(), nc.clone()));
+        ExitCode::SUCCESS
+      })
+    },
+    Some(Command::Test { only: None, threads, system: Some(system) }) => {
+      let subtrees = unwrap_exit!(with_mock_env(|env| {
+        match env.systems().find(|s| s.name == system) {
+          None => Err(format!("System {system} not found")),
+          Some(sys) => {
+            let mut paths = HashSet::new();
+            sys.code.search_all((), |path, node, ()| {
+              if matches!(node, ModMemberRef::Item(_)) {
+                let name = Sym::new(path.unreverse()).expect("Empty path means global file");
+                paths.insert(name);
+              }
+            });
+            Ok(paths)
+          },
+        }
+      }));
+      let in_subtrees = |sym: Sym| subtrees.iter().any(|sub| sym[..].starts_with(&sub[..]));
+      let tests = with_mock_env(|env| {
+        let tree = env.load_main(dir.clone(), [main.clone()], &reporter);
+        let mr = MacroRunner::new(&tree, Some(args.macro_limit), &reporter);
+        let src_consts = mr.run_macros(tree, &reporter).all_consts();
+        let consts = merge_trees(src_consts, env.systems(), &reporter);
+        (consts.into_iter())
+          .filter(|(k, v)| in_subtrees(k.clone()) && v.comments.iter().any(|c| c.trim() == "test"))
+          .collect_vec()
+      });
+      eprintln!("Running {} tests", tests.len());
+      unwrap_exit!(run_tests(&dir, args.macro_limit, threads, &tests));
+      eprintln!("All tests pass");
+      ExitCode::SUCCESS
+    },
+    None => with_std_env(|env| {
+      let proc = env.proc_main(dir, [main.clone()], true, Some(args.macro_limit), &reporter);
+      reporter.assert_exit();
+      let ret = unwrap_exit!(proc.run(nort::Clause::Constant(main).into_expr(location), None));
+      drop(proc);
+      match ret.clone().downcast() {
+        Ok(Inert(OrcExitStatus::Success)) => ExitCode::SUCCESS,
+        Ok(Inert(OrcExitStatus::Failure)) => ExitCode::FAILURE,
+        Err(_) => {
+          println!("{}", ret.clause);
+          ExitCode::SUCCESS
+        },
+      }
+    }),
+    Some(Command::Repl) => with_std_env(|env| {
+      let sctx = env.project_ctx(&reporter);
+      loop {
+        let reporter = Reporter::new();
+        print!("orc");
+        let mut src = String::new();
+        let mut paren_tally = 0;
+        loop {
+          print!("> ");
+          stdout().flush().unwrap();
+          let mut buf = String::new();
+          stdin().read_line(&mut buf).unwrap();
+          src += &buf;
+          let range = SourceRange::mock();
+          let spctx = sctx.parsing(range.code());
+          let pctx = FlatLocContext::new(&spctx, &range);
+          let res =
+            lex(Vec::new(), &buf, &pctx, |_| Ok::<_, Never>(false)).unwrap_or_else(|e| match e {});
+          res.tokens.iter().for_each(|e| match &e.lexeme {
+            Lexeme::LP(_) => paren_tally += 1,
+            Lexeme::RP(_) => paren_tally -= 1,
+            _ => (),
+          });
+          if 0 == paren_tally {
+            break;
+          }
+        }
+        let tree = env.load_project_main(
+          [sym!(tree::main::__repl_input__)],
+          DeclTree::ns("tree::main", [decl_file(&format!("const __repl_input__ := {src}"))]),
+          &reporter,
+        );
+        let mr = MacroRunner::new(&tree, Some(args.macro_limit), &reporter);
+        let proj_consts = mr.run_macros(tree, &reporter).all_consts();
+        let consts = merge_trees(proj_consts, env.systems(), &reporter);
+        let ctx = nort_gen(location.clone());
+        let to_string_tpl = tpl::A(tpl::C("std::string::convert"), tpl::Slot);
+        if let Err(err) = reporter.bind() {
+          eprintln!("{err}");
+          continue;
+        }
+        let proc = Process::new(consts, env.handlers());
+        let prompt = tpl::C("tree::main::__repl_input__").template(ctx.clone(), []);
+        let out = match proc.run(prompt, Some(1000)) {
+          Ok(out) => out,
+          Err(e) => {
+            eprintln!("{e}");
+            continue;
+          },
+        };
+        if let Ok(out) = proc.run(to_string_tpl.template(ctx, [out.clone()]), Some(1000)) {
+          if let Ok(s) = out.clone().downcast::<Inert<OrcString>>() {
+            println!("{}", s.0.as_str());
+            continue;
+          }
+        }
+        println!("{out}")
+      }
+    }),
+  }
+}

orchidlang/src/facade/loader.rs

@@ -0,0 +1,203 @@
+//! The main structure of the façade, collects systems and exposes various
+//! operations over the whole set.
+
+use std::borrow::Borrow;
+use std::path::PathBuf;
+
+use intern_all::i;
+
+use super::macro_runner::MacroRunner;
+use super::merge_trees::merge_trees;
+use super::process::Process;
+use super::system::{IntoSystem, System};
+use super::unbound_ref::validate_refs;
+use crate::error::Reporter;
+use crate::gen::tree::ConstTree;
+use crate::interpreter::context::RunEnv;
+use crate::interpreter::handler::HandlerTable;
+use crate::location::{CodeGenInfo, CodeOrigin};
+use crate::name::{PathSlice, Sym, VPath};
+use crate::pipeline::load_project::{load_project, ProjectContext};
+use crate::pipeline::project::ProjectTree;
+use crate::sym;
+use crate::utils::combine::Combine;
+use crate::utils::sequence::Sequence;
+use crate::virt_fs::{DeclTree, DirNode, Loaded, VirtFS};
+
+/// A compiled environment ready to load user code. It stores the list of
+/// systems and combines with usercode to produce a [Process]
+pub struct Loader<'a> {
+  systems: Vec<System<'a>>,
+}
+impl<'a> Loader<'a> {
+  /// Initialize a new environment
+  #[must_use]
+  pub fn new() -> Self { Self { systems: Vec::new() } }
+
+  /// Retrieve the list of systems
+  pub fn systems(&self) -> impl Iterator<Item = &System<'a>> { self.systems.iter() }
+
+  /// Register a new system in the environment
+  #[must_use]
+  pub fn add_system<'b: 'a>(mut self, is: impl IntoSystem<'b> + 'b) -> Self {
+    self.systems.push(Box::new(is).into_system());
+    self
+  }
+
+  /// Extract the systems from the environment
+  pub fn into_systems(self) -> Vec<System<'a>> { self.systems }
+
+  /// Initialize an environment with a prepared list of systems
+  pub fn from_systems(sys: impl IntoIterator<Item = System<'a>>) -> Self {
+    Self { systems: sys.into_iter().collect() }
+  }
+
+  /// Combine the `constants` fields of all systems
+  pub fn constants(&self) -> ConstTree {
+    (self.systems())
+      .try_fold(ConstTree::tree::<&str>([]), |acc, sys| acc.combine(sys.constants.clone()))
+      .expect("Conflicting const trees")
+  }
+
+  /// Extract the command handlers from the systems, consuming the loader in the
+  /// process. This has to consume the systems because handler tables aren't
+  /// Copy. It also establishes the practice that environments live on the
+  /// stack.
+  pub fn handlers(&self) -> HandlerTable<'_> {
+    (self.systems.iter()).fold(HandlerTable::new(), |t, sys| t.link(&sys.handlers))
+  }
+
+  /// Compile the environment from the set of systems and return it directly.
+  /// See [#load_dir]
+  pub fn project_ctx<'b>(&self, reporter: &'b Reporter) -> ProjectContext<'_, 'b> {
+    ProjectContext {
+      lexer_plugins: Sequence::new(|| {
+        self.systems().flat_map(|sys| &sys.lexer_plugins).map(|b| &**b)
+      }),
+      line_parsers: Sequence::new(|| {
+        self.systems().flat_map(|sys| &sys.line_parsers).map(|b| &**b)
+      }),
+      preludes: Sequence::new(|| self.systems().flat_map(|sys| &sys.prelude)),
+      reporter,
+    }
+  }
+
+  /// Combine source code from all systems with the specified directory into a
+  /// common [VirtFS]
+  pub fn make_dir_fs(&self, dir: PathBuf) -> DeclTree {
+    let dir_node = DirNode::new(dir, ".orc").rc();
+    DeclTree::tree([("tree", DeclTree::leaf(dir_node))])
+  }
+
+  /// All system trees merged into one
+  pub fn system_fs(&self) -> DeclTree {
+    (self.systems().try_fold(DeclTree::empty(), |acc, sub| acc.combine(sub.code.clone())))
+      .expect("Conflicting system trees")
+  }
+
+  /// A wrapper around [load_project] that only takes the arguments that aren't
+  /// fully specified by systems
+  pub fn load_project_main(
+    &self,
+    entrypoints: impl IntoIterator<Item = Sym>,
+    root: DeclTree,
+    reporter: &Reporter,
+  ) -> ProjectTree {
+    let tgt_loc = CodeOrigin::Gen(CodeGenInfo::no_details(sym!(facade::entrypoint)));
+    let constants = self.constants().unwrap_mod();
+    let targets = entrypoints.into_iter().map(|s| (s, tgt_loc.clone()));
+    let root = self.system_fs().combine(root).expect("System trees conflict with root");
+    load_project(&self.project_ctx(reporter), targets, &constants, &root)
+  }
+
+  /// A wrapper around [load_project] that only takes the arguments that aren't
+  /// fully specified by systems
+  pub fn load_project(&self, root: DeclTree, reporter: &Reporter) -> ProjectTree {
+    let mut orc_files: Vec<VPath> = Vec::new();
+    find_all_orc_files([].borrow(), &mut orc_files, &root);
+    let entrypoints = (orc_files.into_iter()).map(|p| p.name_with_suffix(i!(str: "tree")).to_sym());
+    let tgt_loc = CodeOrigin::Gen(CodeGenInfo::no_details(sym!(facade::entrypoint)));
+    let constants = self.constants().unwrap_mod();
+    let targets = entrypoints.into_iter().map(|s| (s, tgt_loc.clone()));
+    let root = self.system_fs().combine(root).expect("System trees conflict with root");
+    load_project(&self.project_ctx(reporter), targets, &constants, &root)
+  }
+
+  /// Load a directory from the local file system as an Orchid project.
+  /// File loading proceeds along import statements and ignores all files
+  /// not reachable from the specified file.
+  pub fn load_main(
+    &self,
+    dir: PathBuf,
+    targets: impl IntoIterator<Item = Sym>,
+    reporter: &Reporter,
+  ) -> ProjectTree {
+    self.load_project_main(targets, self.make_dir_fs(dir), reporter)
+  }
+
+  /// Load every orchid file in a directory
+  pub fn load_dir(&self, dir: PathBuf, reporter: &Reporter) -> ProjectTree {
+    self.load_project(self.make_dir_fs(dir), reporter)
+  }
+
+  /// Build a process by calling other utilities in [crate::facade]. A sort of
+  /// facade over the facade. If you need a custom file system, consider
+  /// combining this with [Loader::load_project]. For usage with
+  /// [Loader::load_main] and [Loader::load_dir] we offer the shorthands
+  /// [Loader::proc_main] and [Loader::proc_dir].
+  pub fn proc(
+    &'a self,
+    tree: ProjectTree,
+    check_refs: bool,
+    macro_limit: Option<usize>,
+    reporter: &Reporter,
+  ) -> Process<'a> {
+    let mr = MacroRunner::new(&tree, macro_limit, reporter);
+    let pm_tree = mr.run_macros(tree, reporter);
+    let consts = merge_trees(pm_tree.all_consts(), self.systems(), reporter);
+    if check_refs {
+      validate_refs(consts.keys().cloned().collect(), reporter, &mut |sym, location| {
+        (consts.get(&sym).map(|nc| nc.value.clone()))
+          .ok_or_else(|| RunEnv::sym_not_found(sym, location))
+      });
+    }
+    Process::new(consts, self.handlers())
+  }
+
+  /// Load a project and process everything
+  pub fn proc_dir(
+    &'a self,
+    dir: PathBuf,
+    check_refs: bool,
+    macro_limit: Option<usize>,
+    reporter: &Reporter,
+  ) -> Process<'a> {
+    self.proc(self.load_dir(dir.to_owned(), reporter), check_refs, macro_limit, reporter)
+  }
+
+  /// Load a project and process everything to load specific symbols
+  pub fn proc_main(
+    &'a self,
+    dir: PathBuf,
+    targets: impl IntoIterator<Item = Sym>,
+    check_refs: bool,
+    macro_limit: Option<usize>,
+    reporter: &Reporter,
+  ) -> Process<'a> {
+    self.proc(self.load_main(dir.to_owned(), targets, reporter), check_refs, macro_limit, reporter)
+  }
+}
+
+impl<'a> Default for Loader<'a> {
+  fn default() -> Self { Self::new() }
+}
+
+fn find_all_orc_files(path: &PathSlice, paths: &mut Vec<VPath>, vfs: &impl VirtFS) {
+  match vfs.read(path) {
+    Err(_) => (),
+    Ok(Loaded::Code(_)) => paths.push(path.to_vpath()),
+    Ok(Loaded::Collection(items)) => items
+      .iter()
+      .for_each(|suffix| find_all_orc_files(&path.to_vpath().suffix([suffix.clone()]), paths, vfs)),
+  }
+}

orchidlang/src/facade/macro_runner.rs

@@ -0,0 +1,102 @@
+//! Encapsulates the macro runner's scaffolding. Relies on a [ProjectTree]
+//! loaded by the [super::loader::Loader]
+
+use std::iter;
+
+use crate::error::{ErrorPosition, ProjectError, ProjectErrorObj, ProjectResult, Reporter};
+use crate::location::CodeOrigin;
+use crate::parse::parsed;
+use crate::pipeline::project::{ItemKind, ProjItem, ProjectTree};
+use crate::rule::repository::Repo;
+use crate::tree::TreeTransforms;
+
+/// Encapsulates the macro repository and the constant list, and allows querying
+/// for macro execution results
+pub struct MacroRunner {
+  /// Optimized catalog of substitution rules
+  pub repo: Repo,
+  /// Runtime code containing macro invocations
+  pub timeout: Option<usize>,
+}
+impl MacroRunner {
+  /// Initialize a macro runner
+  pub fn new(tree: &ProjectTree, timeout: Option<usize>, reporter: &Reporter) -> Self {
+    let rules = tree.all_rules();
+    let repo = Repo::new(rules, reporter);
+    Self { repo, timeout }
+  }
+
+  /// Process the macros in an expression.
+  pub fn process_expr(&self, expr: parsed::Expr) -> ProjectResult<parsed::Expr> {
+    match self.timeout {
+      None => Ok((self.repo.pass(&expr)).unwrap_or_else(|| expr.clone())),
+      Some(limit) => {
+        let (o, leftover_gas) = self.repo.long_step(&expr, limit + 1);
+        if 0 < leftover_gas {
+          return Ok(o);
+        }
+        Err(MacroTimeout { location: expr.range.origin(), limit }.pack())
+      },
+    }
+  }
+
+  /// Run all macros in the project.
+  pub fn run_macros(&self, tree: ProjectTree, reporter: &Reporter) -> ProjectTree {
+    ProjectTree(tree.0.map_data(
+      |_, item| match &item.kind {
+        ItemKind::Const(c) => match self.process_expr(c.clone()) {
+          Ok(expr) => ProjItem { kind: ItemKind::Const(expr) },
+          Err(e) => {
+            reporter.report(e);
+            item
+          },
+        },
+        _ => item,
+      },
+      |_, x| x,
+      |_, x| x,
+    ))
+  }
+
+  /// Obtain an iterator that steps through the preprocessing of a constant
+  /// for debugging macros
+  pub fn step(&self, mut expr: parsed::Expr) -> impl Iterator<Item = parsed::Expr> + '_ {
+    iter::from_fn(move || {
+      expr = self.repo.step(&expr)?;
+      Some(expr.clone())
+    })
+  }
+}
+
+/// Error raised when a macro runs too long
+#[derive(Debug)]
+pub struct MacroTimeout {
+  location: CodeOrigin,
+  limit: usize,
+}
+impl ProjectError for MacroTimeout {
+  const DESCRIPTION: &'static str = "Macro execution has not halted";
+
+  fn message(&self) -> String {
+    let Self { limit, .. } = self;
+    format!("Macro processing took more than {limit} steps")
+  }
+
+  fn one_position(&self) -> CodeOrigin { self.location.clone() }
+}
+
+struct MacroErrors(Vec<ProjectErrorObj>);
+impl ProjectError for MacroErrors {
+  const DESCRIPTION: &'static str = "Errors occurred during macro execution";
+  fn positions(&self) -> impl IntoIterator<Item = ErrorPosition> + '_ {
+    self.0.iter().enumerate().flat_map(|(i, e)| {
+      e.positions().map(move |ep| ErrorPosition {
+        origin: ep.origin,
+        message: Some(match ep.message {
+          Some(msg) => format!("Error #{}: {}; {msg}", i + 1, e.message()),
+          None => format!("Error #{}: {}", i + 1, e.message()),
+        }),
+      })
+    })
+  }
+}

orchidlang/src/facade/merge_trees.rs

@@ -0,0 +1,75 @@
+//! Combine constants from [super::macro_runner::MacroRunner::run_macros] with
+//! systems from [super::loader::Loader::systems]
+
+use std::sync::Arc;
+
+use hashbrown::HashMap;
+
+use super::system::System;
+use crate::error::Reporter;
+use crate::foreign::inert::Inert;
+use crate::foreign::to_clause::ToClause;
+use crate::intermediate::ast_to_ir::ast_to_ir;
+use crate::intermediate::ir_to_nort::ir_to_nort;
+use crate::interpreter::nort;
+use crate::location::{CodeGenInfo, CodeLocation};
+use crate::name::{NameLike, Sym};
+use crate::pipeline::project::ConstReport;
+use crate::sym;
+use crate::tree::{ModMemberRef, TreeTransforms};
+use crate::utils::unwrap_or::unwrap_or;
+
+/// Equivalent of [crate::pipeline::project::ConstReport] for the interpreter's
+/// representation, [crate::interpreter::nort].
+#[derive(Clone)]
+pub struct NortConst {
+  /// Comments associated with the constant which may affect its interpretation
+  pub comments: Vec<Arc<String>>,
+  /// Location of the definition, if known
+  pub location: CodeLocation,
+  /// Value assigned to the constant
+  pub value: nort::Expr,
+}
+impl NortConst {
+  /// Convert into NORT constant from AST constant
+  pub fn convert_from(value: ConstReport, reporter: &Reporter) -> NortConst {
+    let module = Sym::new(value.name.split_last().1[..].iter())
+      .expect("Constant names from source are at least 2 long");
+    let location = CodeLocation::new_src(value.range.clone(), value.name);
+    let nort = match ast_to_ir(value.value, value.range, module.clone()) {
+      Ok(ir) => ir_to_nort(&ir),
+      Err(e) => {
+        reporter.report(e);
+        Inert(0).to_expr(location.clone())
+      },
+    };
+    Self { value: nort, location, comments: value.comments }
+  }
+}
+
+/// Combine a list of symbols loaded from source and the constant trees from
+/// each system.
+pub fn merge_trees<'a: 'b, 'b>(
+  source: impl IntoIterator<Item = (Sym, ConstReport)>,
+  systems: impl IntoIterator<Item = &'b System<'a>> + 'b,
+  reporter: &Reporter,
+) -> HashMap<Sym, NortConst> {
+  let mut out = HashMap::new();
+  for (name, rep) in source.into_iter() {
+    out.insert(name.clone(), NortConst::convert_from(rep, reporter));
+  }
+  for system in systems {
+    let const_module = system.constants.unwrap_mod_ref();
+    const_module.search_all((), |stack, node, ()| {
+      let c = unwrap_or!(node => ModMemberRef::Item; return);
+      let location = CodeLocation::new_gen(CodeGenInfo::details(
+        sym!(facade::merge_tree),
+        format!("system.name={}", system.name),
+      ));
+      let value = c.clone().gen_nort(stack.clone(), location.clone());
+      let crep = NortConst { value, comments: vec![], location };
+      out.insert(Sym::new(stack.unreverse()).expect("root item is forbidden"), crep);
+    });
+  }
+  out
+}

orchidlang/src/facade/mod.rs

@@ -0,0 +1,9 @@
+//! A simplified set of commands each grouping a large subset of the operations
+//! exposed by Orchid to make writing embeddings faster in the typical case.
+
+pub mod loader;
+pub mod macro_runner;
+pub mod merge_trees;
+pub mod process;
+pub mod system;
+pub mod unbound_ref;

orchidlang/src/facade/process.rs

@@ -0,0 +1,39 @@
+//! Run Orchid commands in the context of the loaded environment. Either
+//! returned by [super::loader::Loader::proc], or constructed manually from the
+//! return value of [super::merge_trees::merge_trees] and
+//! [super::loader::Loader::handlers].
+
+use hashbrown::HashMap;
+
+use super::merge_trees::NortConst;
+use crate::interpreter::context::{Halt, RunEnv, RunParams};
+use crate::interpreter::error::RunError;
+use crate::interpreter::handler::HandlerTable;
+use crate::interpreter::nort::Expr;
+use crate::interpreter::run::run;
+use crate::name::Sym;
+
+/// This struct ties the state of systems to loaded code, and allows to call
+/// Orchid-defined functions
+pub struct Process<'a>(RunEnv<'a>);
+impl<'a> Process<'a> {
+  /// Build a process from the return value of [crate::facade::merge_trees] and
+  pub fn new(
+    consts: impl IntoIterator<Item = (Sym, NortConst)>,
+    handlers: HandlerTable<'a>,
+  ) -> Self {
+    let symbols: HashMap<_, _> = consts.into_iter().map(|(k, v)| (k, v.value)).collect();
+    Self(RunEnv::new(handlers, move |sym, location| {
+      symbols.get(&sym).cloned().ok_or_else(|| RunEnv::sym_not_found(sym, location))
+    }))
+  }
+
+  /// Execute the given command in this process. If gas is specified, at most as
+  /// many steps will be executed and then the partial result returned.
+  ///
+  /// This is useful to catch infinite loops or ensure that a tenant program
+  /// yields
+  pub fn run(&self, prompt: Expr, gas: Option<usize>) -> Result<Halt, RunError<'_>> {
+    run(prompt, &self.0, &mut RunParams { stack: 1000, gas })
+  }
+}

orchidlang/src/facade/unbound_ref.rs

@@ -0,0 +1,94 @@
+//! Referencing a constant that doesn't exist is a runtime error in Orchid, even
+//! though this common error condition is usually caused by faulty macro
+//! execution. This module constains functions to detect and raise these errors
+//! eagerly.
+
+use std::fmt;
+
+use hashbrown::HashSet;
+use trait_set::trait_set;
+
+use crate::error::{ProjectError, Reporter};
+use crate::interpreter::nort::{Clause, Expr};
+use crate::location::{CodeGenInfo, CodeLocation};
+use crate::name::Sym;
+use crate::sym;
+
+/// Start with a symbol
+pub fn unbound_refs_sym<E: SubError>(
+  symbol: Sym,
+  location: CodeLocation,
+  visited: &mut HashSet<Sym>,
+  load: &mut impl FnMut(Sym, CodeLocation) -> Result<Expr, E>,
+  reporter: &Reporter,
+) {
+  if visited.insert(symbol.clone()) {
+    match load(symbol.clone(), location.clone()) {
+      Err(error) => reporter.report(MissingSymbol { symbol, location, error }.pack()),
+      Ok(expr) => unbound_refs_expr(expr, visited, load, reporter),
+    }
+  }
+}
+
+/// Find all unbound constant names in a snippet. This is mostly useful to
+/// detect macro errors.
+pub fn unbound_refs_expr<E: SubError>(
+  expr: Expr,
+  visited: &mut HashSet<Sym>,
+  load: &mut impl FnMut(Sym, CodeLocation) -> Result<Expr, E>,
+  reporter: &Reporter,
+) {
+  expr.search_all(&mut |s: &Expr| {
+    if let Clause::Constant(symbol) = &*s.cls_mut() {
+      unbound_refs_sym(symbol.clone(), s.location(), visited, load, reporter)
+    }
+    None::<()>
+  });
+}
+
+/// Assert that the code contains no invalid references that reference missing
+/// symbols. [Clause::Constant]s can be created procedurally, so this isn't a
+/// total guarantee, more of a convenience.
+pub fn validate_refs<E: SubError>(
+  all_syms: HashSet<Sym>,
+  reporter: &Reporter,
+  load: &mut impl FnMut(Sym, CodeLocation) -> Result<Expr, E>,
+) -> HashSet<Sym> {
+  let mut visited = HashSet::new();
+  for sym in all_syms {
+    let location = CodeLocation::new_gen(CodeGenInfo::no_details(sym!(orchidlang::validate_refs)));
+    unbound_refs_sym(sym, location, &mut visited, load, reporter);
+  }
+  visited
+}
+
+trait_set! {
+  /// Any error the reference walker can package into a [MissingSymbol]
+  pub trait SubError = fmt::Display + Clone + Send + Sync + 'static;
+}
+
+/// Information about a reproject failure
+#[derive(Clone)]
+pub struct MissingSymbol<E: SubError> {
+  /// The error returned by the loader function. This is usually a ismple "not
+  /// found", but with unusual setups it might provide some useful info.
+  pub error: E,
+  /// Location of the first reference to the missing symbol.
+  pub location: CodeLocation,
+  /// The symbol in question
+  pub symbol: Sym,
+}
+impl<E: SubError> ProjectError for MissingSymbol<E> {
+  const DESCRIPTION: &'static str = "A name not referring to a known symbol was found in the source after \
+     macro execution. This can either mean that a symbol name was mistyped, or \
+     that macro execution didn't correctly halt.";
+  fn message(&self) -> String { format!("{}: {}", self.symbol, self.error) }
+  fn one_position(&self) -> crate::location::CodeOrigin { self.location.origin() }
+}
+
+// struct MissingSymbols {
+//   errors: Vec<ErrorPosition>,
+// }
+// impl ProjectError for MissingSymbols {
+//   fn positions(&self) -> impl IntoIterator<Item = ErrorPosition> {
+// self.errors.iter().cloned() } }

orchidlang/src/foreign/cps_box.rs

@@ -0,0 +1,97 @@
+//! Automated wrappers to make working with CPS commands easier.
+
+use std::fmt;
+
+use trait_set::trait_set;
+
+use super::atom::{Atomic, AtomicResult, AtomicReturn, CallData, NotAFunction, RunData};
+use super::error::{RTError, RTResult};
+use crate::interpreter::nort::{Clause, Expr};
+use crate::location::CodeLocation;
+use crate::utils::ddispatch::{Request, Responder};
+use crate::utils::pure_seq::pushed_ref;
+
+trait_set! {
+  /// A "well behaved" type that can be used as payload in a CPS box
+  pub trait CPSPayload = Clone + fmt::Debug + Send + 'static;
+  /// A function to handle a CPS box with a specific payload
+  pub trait CPSHandler<T: CPSPayload> = FnMut(&T, &Expr) -> RTResult<Expr>;
+}
+
+/// An Orchid Atom value encapsulating a payload and continuation points
+#[derive(Debug, Clone)]
+pub struct CPSBox<T: CPSPayload> {
+  /// Number of arguments not provided yet
+  pub argc: usize,
+  /// Details about the command
+  pub payload: T,
+  /// Possible continuations, in the order they were provided
+  pub continuations: Vec<Expr>,
+}
+impl<T: CPSPayload> CPSBox<T> {
+  /// Create a new command prepared to receive exacly N continuations
+  #[must_use]
+  pub fn new(argc: usize, payload: T) -> Self {
+    debug_assert!(argc > 0, "Null-ary CPS functions are invalid");
+    Self { argc, continuations: Vec::new(), payload }
+  }
+  /// Unpack the wrapped command and the continuation
+  #[must_use]
+  pub fn unpack1(&self) -> (&T, Expr) {
+    match &self.continuations[..] {
+      [cont] => (&self.payload, cont.clone()),
+      _ => panic!("size mismatch"),
+    }
+  }
+  /// Unpack the wrapped command and 2 continuations (usually an async and a
+  /// sync)
+  #[must_use]
+  pub fn unpack2(&self) -> (&T, Expr, Expr) {
+    match &self.continuations[..] {
+      [c1, c2] => (&self.payload, c1.clone(), c2.clone()),
+      _ => panic!("size mismatch"),
+    }
+  }
+  /// Unpack the wrapped command and 3 continuations (usually an async success,
+  /// an async fail and a sync)
+  #[must_use]
+  pub fn unpack3(&self) -> (&T, Expr, Expr, Expr) {
+    match &self.continuations[..] {
+      [c1, c2, c3] => (&self.payload, c1.clone(), c2.clone(), c3.clone()),
+      _ => panic!("size mismatch"),
+    }
+  }
+
+  fn assert_applicable(&self, err_loc: &CodeLocation) -> RTResult<()> {
+    match self.argc {
+      0 => Err(NotAFunction(self.clone().atom_expr(err_loc.clone())).pack()),
+      _ => Ok(()),
+    }
+  }
+}
+impl<T: CPSPayload> Responder for CPSBox<T> {
+  fn respond(&self, _request: Request) {}
+}
+impl<T: CPSPayload> Atomic for CPSBox<T> {
+  fn as_any(self: Box<Self>) -> Box<dyn std::any::Any> { self }
+  fn as_any_ref(&self) -> &dyn std::any::Any { self }
+  fn type_name(&self) -> &'static str { std::any::type_name::<Self>() }
+  fn parser_eq(&self, _: &dyn Atomic) -> bool { false }
+  fn redirect(&mut self) -> Option<&mut Expr> { None }
+  fn run(self: Box<Self>, _: RunData) -> AtomicResult { AtomicReturn::inert(*self) }
+  fn apply(mut self: Box<Self>, call: CallData) -> RTResult<Clause> {
+    self.assert_applicable(&call.location)?;
+    self.argc -= 1;
+    self.continuations.push(call.arg);
+    Ok(self.atom_cls())
+  }
+  fn apply_mut(&mut self, call: CallData) -> RTResult<Clause> {
+    self.assert_applicable(&call.location)?;
+    let new = Self {
+      argc: self.argc - 1,
+      continuations: pushed_ref(&self.continuations, call.arg),
+      payload: self.payload.clone(),
+    };
+    Ok(new.atom_cls())
+  }
+}

orchidlang/src/foreign/fn_bridge.rs

@@ -0,0 +1,212 @@
+//! Insert Rust functions in Orchid code almost seamlessly
+
+use std::any::{Any, TypeId};
+use std::fmt;
+use std::marker::PhantomData;
+
+use intern_all::{i, Tok};
+
+use super::atom::{Atomic, AtomicResult, AtomicReturn, CallData, RunData};
+use super::error::RTResult;
+use super::to_clause::ToClause;
+use super::try_from_expr::TryFromExpr;
+use crate::interpreter::nort::{Clause, Expr};
+use crate::utils::ddispatch::Responder;
+
+/// Return a unary lambda wrapped in this struct to take an additional argument
+/// in a function passed to Orchid through [super::fn_bridge::xfn].
+///
+/// Container for a unary [FnOnce] that uniquely states the argument and return
+/// type. Rust functions are never overloaded, but inexplicably the [Fn] traits
+/// take the argument tuple as a generic parameter which means that it cannot
+/// be a unique dispatch target.
+///
+/// If the function takes an instance of [Thunk], it will contain the expression
+/// the function was applied to without any specific normalization. If it takes
+/// any other type, the argument will be fully normalized and cast using the
+/// type's [TryFromExpr] impl.
+pub struct Param<T, U, F> {
+  data: F,
+  name: Tok<String>,
+  _t: PhantomData<T>,
+  _u: PhantomData<U>,
+}
+unsafe impl<T, U, F: Send> Send for Param<T, U, F> {}
+impl<T, U, F> Param<T, U, F> {
+  /// Wrap a new function in a parametric struct
+  pub fn new(name: Tok<String>, f: F) -> Self
+  where F: FnOnce(T) -> U {
+    Self { name, data: f, _t: PhantomData, _u: PhantomData }
+  }
+  /// Take out the function
+  pub fn get(self) -> F { self.data }
+}
+impl<T, U, F: Clone> Clone for Param<T, U, F> {
+  fn clone(&self) -> Self {
+    Self { name: self.name.clone(), data: self.data.clone(), _t: PhantomData, _u: PhantomData }
+  }
+}
+impl<T, U, F> fmt::Display for Param<T, U, F> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.name) }
+}
+impl<T, U, F> fmt::Debug for Param<T, U, F> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    f.debug_tuple("Param").field(&*self.name).finish()
+  }
+}
+
+/// A marker struct that gets assigned an expression without normalizing it.
+/// This behaviour cannot be replicated in usercode, it's implemented with an
+/// explicit runtime [TypeId] check invoked by [Param].
+#[derive(Debug, Clone)]
+pub struct Thunk(pub Expr);
+impl TryFromExpr for Thunk {
+  fn from_expr(expr: Expr) -> RTResult<Self> { Ok(Thunk(expr)) }
+}
+
+struct FnMiddleStage<T, U, F> {
+  arg: Expr,
+  f: Param<T, U, F>,
+}
+
+impl<T, U, F: Clone> Clone for FnMiddleStage<T, U, F> {
+  fn clone(&self) -> Self { Self { arg: self.arg.clone(), f: self.f.clone() } }
+}
+impl<T, U, F> fmt::Debug for FnMiddleStage<T, U, F> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "FnMiddleStage({} {})", self.f, self.arg)
+  }
+}
+impl<T, U, F> Responder for FnMiddleStage<T, U, F> {}
+impl<
+  T: 'static + TryFromExpr,
+  U: 'static + ToClause,
+  F: 'static + Clone + FnOnce(T) -> U + Any + Send,
+> Atomic for FnMiddleStage<T, U, F>
+{
+  fn as_any(self: Box<Self>) -> Box<dyn std::any::Any> { self }
+  fn as_any_ref(&self) -> &dyn std::any::Any { self }
+  fn type_name(&self) -> &'static str { std::any::type_name::<Self>() }
+  fn redirect(&mut self) -> Option<&mut Expr> {
+    // this should be ctfe'd
+    (TypeId::of::<T>() != TypeId::of::<Thunk>()).then_some(&mut self.arg)
+  }
+  fn run(self: Box<Self>, r: RunData) -> AtomicResult {
+    let Self { arg, f: Param { data: f, .. } } = *self;
+    Ok(AtomicReturn::Change(0, f(arg.downcast()?).to_clause(r.location)))
+  }
+  fn apply_mut(&mut self, _: CallData) -> RTResult<Clause> { panic!("Atom should have decayed") }
+}
+
+impl<T, U, F> Responder for Param<T, U, F> {}
+
+impl<
+  T: 'static + TryFromExpr + Clone,
+  U: 'static + ToClause,
+  F: 'static + Clone + Send + FnOnce(T) -> U,
+> Atomic for Param<T, U, F>
+{
+  fn as_any(self: Box<Self>) -> Box<dyn std::any::Any> { self }
+  fn as_any_ref(&self) -> &dyn std::any::Any { self }
+  fn type_name(&self) -> &'static str { std::any::type_name::<Self>() }
+  fn redirect(&mut self) -> Option<&mut Expr> { None }
+  fn run(self: Box<Self>, _: RunData) -> AtomicResult { AtomicReturn::inert(*self) }
+  fn apply_mut(&mut self, call: CallData) -> RTResult<Clause> {
+    Ok(FnMiddleStage { arg: call.arg, f: self.clone() }.atom_cls())
+  }
+  fn apply(self: Box<Self>, call: CallData) -> RTResult<Clause> {
+    Ok(FnMiddleStage { arg: call.arg, f: *self }.atom_cls())
+  }
+}
+
+/// Convert a Rust function to Orchid. If you can, register your Rust functions
+/// statically with functions in [crate::gen::tree].
+pub fn xfn<const N: usize, Argv, Ret>(
+  name: &str,
+  x: impl Xfn<N, Argv, Ret>,
+) -> impl Atomic + Clone {
+  x.to_atomic(i(name))
+}
+
+/// Trait for functions that can be directly passed to Orchid. Constraints in a
+/// nutshell:
+///
+/// - the function must live as long as ['static]
+/// - All arguments must implement [TryFromExpr]
+/// - all but the last argument must implement [Clone] and [Send]
+/// - the return type must implement [ToClause]
+///
+/// Take [Thunk] to consume the argument as-is, without normalization.
+pub trait Xfn<const N: usize, Argv, Ret>: Clone + Send + 'static {
+  /// Convert Rust type to Orchid function, given a name for logging
+  fn to_atomic(self, name: Tok<String>) -> impl Atomic + Clone;
+}
+
+/// Conversion functions from [Fn] traits into [Atomic]. Since Rust's type
+/// system allows overloaded [Fn] implementations, we must specify the arity and
+/// argument types for this process. Arities are only defined up to 9, but the
+/// function can always return another call to `xfn_`N`ary` to consume more
+/// arguments.
+pub mod xfn_impls {
+  use intern_all::{i, Tok};
+
+  use super::super::atom::Atomic;
+  use super::super::try_from_expr::TryFromExpr;
+  #[allow(unused)] // for doc
+  use super::Thunk;
+  use super::{Param, ToClause, Xfn};
+
+  macro_rules! xfn_variant {
+    (
+      $number:expr,
+      ($($t:ident)*)
+      ($($alt:expr)*)
+    ) => {
+      paste::paste!{
+        impl<
+          $( $t : TryFromExpr + Clone + Send + 'static, )*
+          TLast: TryFromExpr + Clone + 'static,
+          TReturn: ToClause + Send + 'static,
+          TFunction: FnOnce( $( $t , )* TLast )
+            -> TReturn + Clone + Send + 'static
+        > Xfn<$number, ($($t,)* TLast,), TReturn> for TFunction {
+          fn to_atomic(self, name: Tok<String>) -> impl Atomic + Clone {
+            #[allow(unused_variables)]
+            let argc = 0;
+            let stage_n = name.clone();
+            xfn_variant!(@BODY_LOOP self name stage_n argc
+              ( $( ( $t [< $t:lower >] ) )* )
+              ( $( [< $t:lower >] )* )
+            )
+          }
+        }
+      }
+    };
+    (@BODY_LOOP $function:ident $name:ident $stage_n:ident $argc:ident (
+      ( $Next:ident $next:ident )
+      $( ( $T:ident $t:ident ) )*
+    ) $full:tt) => {{
+      Param::new($stage_n, move |$next : $Next| {
+        let $argc = $argc + 1;
+        let $stage_n = i(&format!("{}/{}", $name, $argc));
+        xfn_variant!(@BODY_LOOP $function $name $stage_n $argc ( $( ( $T $t ) )* ) $full)
+      })
+    }};
+    (@BODY_LOOP $function:ident $name:ident $stage_n:ident $argc:ident (
+
+    ) ( $( $t:ident )* )) => {{
+      Param::new($stage_n, |last: TLast| $function ( $( $t , )* last ))
+    }};
+  }
+
+  xfn_variant!(1, () (2 3 4 5 6 7 8 9 10 11 12 13 14 15 16));
+  xfn_variant!(2, (A) (1 3 4 5 6 7 8 9 10 11 12 13 14 15 16));
+  xfn_variant!(3, (A B) (1 2 4 5 6 7 8 9 10 11 12 13 14 15 16));
+  xfn_variant!(4, (A B C) (1 2 3 5 6 7 8 9 10 11 12 13 14 15 16));
+  xfn_variant!(5, (A B C D) (1 2 3 4 6 7 8 9 10 11 12 13 14 15 16));
+  xfn_variant!(6, (A B C D E) (1 2 3 4 5 7 8 9 10 11 12 13 14 15 16));
+  xfn_variant!(7, (A B C D E F) (1 2 3 4 5 6 8 9 10 11 12 13 14 15 16));
+  xfn_variant!(8, (A B C D E F G) (1 2 3 4 5 6 7 9 10 11 12 13 14 15 16));
+  xfn_variant!(9, (A B C D E F G H) (1 2 3 4 5 6 7 8 10 11 12 13 14 15 16));
+  // at higher arities rust-analyzer fails to load the project
+}

orchidlang/src/foreign/inert.rs

@@ -0,0 +1,129 @@
+//! An [Atomic] that wraps inert data.
+
+use std::any::Any;
+use std::fmt;
+use std::ops::{Deref, DerefMut};
+
+use ordered_float::NotNan;
+
+use super::atom::{Atom, Atomic, AtomicResult, AtomicReturn, CallData, NotAFunction, RunData};
+use super::error::{RTError, RTResult};
+use super::try_from_expr::TryFromExpr;
+use crate::foreign::error::AssertionError;
+use crate::interpreter::nort::{Clause, Expr};
+use crate::libs::std::number::Numeric;
+use crate::libs::std::string::OrcString;
+use crate::utils::ddispatch::{Request, Responder};
+
+/// A proxy trait that implements [Atomic] for blobs of data in Rust code that
+/// cannot be processed and always report inert. Since these are expected to be
+/// parameters of Rust functions it also automatically implements [TryFromExpr]
+/// so that `Inert<MyType>` arguments can be parsed directly.
+pub trait InertPayload: fmt::Debug + Clone + Send + 'static {
+  /// Typename to be shown in the error when a conversion from [Expr] fails
+  ///
+  /// This will default to `type_name::<Self>()` when it becomes stable
+  const TYPE_STR: &'static str;
+  /// Proxies to [Responder] so that you don't have to implmeent it manually if
+  /// you need it, but behaves exactly as the default implementation.
+  #[allow(unused_mut, unused_variables)] // definition should show likely usage
+  fn respond(&self, mut request: Request) {}
+  /// Equality comparison used by the pattern matcher. Since the pattern matcher
+  /// only works with parsed code, you only need to implement this if your type
+  /// is directly parseable.
+  ///
+  /// If your type implements [PartialEq], this can simply be implemented as
+  /// ```ignore
+  /// fn strict_eq(&self, other: &Self) -> bool { self == other }
+  /// ```
+  #[allow(unused_variables)]
+  fn strict_eq(&self, other: &Self) -> bool { false }
+}
+
+/// An atom that stores a value and rejects all interpreter interactions. It is
+/// used to reference foreign data in Orchid.
+#[derive(Debug, Clone)]
+pub struct Inert<T: InertPayload>(pub T);
+impl<T: InertPayload> Inert<T> {
+  /// Wrap the argument in a type-erased [Atom] for embedding in Orchid
+  /// structures.
+  pub fn atom(t: T) -> Atom { Atom::new(Inert(t)) }
+}
+
+impl<T: InertPayload> Deref for Inert<T> {
+  type Target = T;
+  fn deref(&self) -> &Self::Target { &self.0 }
+}
+
+impl<T: InertPayload> DerefMut for Inert<T> {
+  fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }
+}
+
+impl<T: InertPayload> Responder for Inert<T> {
+  fn respond(&self, mut request: Request) {
+    if request.can_serve::<T>() { request.serve(self.0.clone()) } else { self.0.respond(request) }
+  }
+}
+impl<T: InertPayload> Atomic for Inert<T> {
+  fn as_any(self: Box<Self>) -> Box<dyn Any> { self }
+  fn as_any_ref(&self) -> &dyn Any { self }
+  fn type_name(&self) -> &'static str { std::any::type_name::<Self>() }
+
+  fn redirect(&mut self) -> Option<&mut Expr> { None }
+  fn run(self: Box<Self>, _: RunData) -> AtomicResult { AtomicReturn::inert(*self) }
+  fn apply_mut(&mut self, call: CallData) -> RTResult<Clause> {
+    Err(NotAFunction(self.clone().atom_expr(call.location)).pack())
+  }
+  fn parser_eq(&self, other: &dyn Atomic) -> bool {
+    other.as_any_ref().downcast_ref::<Self>().map_or(false, |other| self.0.strict_eq(&other.0))
+  }
+}
+
+impl<T: InertPayload> TryFromExpr for Inert<T> {
+  fn from_expr(expr: Expr) -> RTResult<Self> {
+    let Expr { clause, location } = expr;
+    match clause.try_unwrap() {
+      Ok(Clause::Atom(at)) => at
+        .try_downcast::<Self>()
+        .map_err(|a| AssertionError::ext(location, T::TYPE_STR, format!("{a:?}"))),
+      Err(inst) => match &*inst.cls_mut() {
+        Clause::Atom(at) => at
+          .downcast_ref::<Self>()
+          .cloned()
+          .ok_or_else(|| AssertionError::ext(location, T::TYPE_STR, format!("{inst}"))),
+        cls => AssertionError::fail(location, "atom", format!("{cls}")),
+      },
+      Ok(cls) => AssertionError::fail(location, "atom", format!("{cls}")),
+    }
+  }
+}
+
+impl<T: InertPayload + fmt::Display> fmt::Display for Inert<T> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) }
+}
+
+impl InertPayload for bool {
+  const TYPE_STR: &'static str = "bool";
+  fn strict_eq(&self, other: &Self) -> bool { self == other }
+  fn respond(&self, mut request: Request) {
+    request.serve_with(|| OrcString::from(self.to_string()))
+  }
+}
+
+impl InertPayload for usize {
+  const TYPE_STR: &'static str = "usize";
+  fn strict_eq(&self, other: &Self) -> bool { self == other }
+  fn respond(&self, mut request: Request) {
+    request.serve(Numeric::Uint(*self));
+    request.serve_with(|| OrcString::from(self.to_string()))
+  }
+}
+
+impl InertPayload for NotNan<f64> {
+  const TYPE_STR: &'static str = "NotNan<f64>";
+  fn strict_eq(&self, other: &Self) -> bool { self == other }
+  fn respond(&self, mut request: Request) {
+    request.serve(Numeric::Float(*self));
+    request.serve_with(|| OrcString::from(self.to_string()))
+  }
+}

orchidlang/src/foreign/mod.rs

@@ -0,0 +1,12 @@
+//! Interaction with foreign code
+//!
+//! Structures and traits used in the exposure of external functions and values
+//! to Orchid code
+pub mod atom;
+pub mod cps_box;
+pub mod error;
+pub mod fn_bridge;
+pub mod inert;
+pub mod process;
+pub mod to_clause;
+pub mod try_from_expr;

orchidlang/src/foreign/process.rs

@@ -0,0 +1,40 @@
+//! An [Atomic] implementor that runs a callback and turns into the return
+//! value. Useful to generate expressions that depend on values in the
+//! interpreter's context, and to defer the generation of expensive
+//! subexpressions
+use std::fmt;
+
+use super::atom::{Atomic, AtomicResult, AtomicReturn, CallData, RunData};
+use super::error::RTResult;
+use super::to_clause::ToClause;
+use crate::interpreter::nort::{Clause, Expr};
+use crate::utils::ddispatch::Responder;
+
+/// An atom that immediately decays to the result of the function when
+/// normalized. Can be used to build infinite recursive datastructures from
+/// Rust.
+#[derive(Clone)]
+pub struct Unstable<F>(F);
+impl<F: FnOnce(RunData) -> R + Send + 'static, R: ToClause> Unstable<F> {
+  /// Wrap a function in an Unstable
+  pub const fn new(f: F) -> Self { Self(f) }
+}
+impl<F> Responder for Unstable<F> {}
+impl<F> fmt::Debug for Unstable<F> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    f.debug_struct("Unstable").finish_non_exhaustive()
+  }
+}
+impl<F: FnOnce(RunData) -> R + Send + 'static, R: ToClause> Atomic for Unstable<F> {
+  fn as_any(self: Box<Self>) -> Box<dyn std::any::Any> { self }
+  fn as_any_ref(&self) -> &dyn std::any::Any { self }
+  fn type_name(&self) -> &'static str { std::any::type_name::<Self>() }
+
+  fn apply_mut(&mut self, _: CallData) -> RTResult<Clause> { panic!("This atom decays instantly") }
+  fn run(self: Box<Self>, run: RunData) -> AtomicResult {
+    let loc = run.location.clone();
+    let clause = self.0(run).to_clause(loc);
+    Ok(AtomicReturn::Change(0, clause))
+  }
+  fn redirect(&mut self) -> Option<&mut Expr> { None }
+}

orchidlang/src/foreign/to_clause.rs

@@ -0,0 +1,202 @@
+//! Conversions from Rust values to Orchid expressions. Many APIs and
+//! [super::fn_bridge] in particular use this to automatically convert values on
+//! the boundary. The opposite conversion is [super::try_from_expr::TryFromExpr]
+
+use super::atom::{Atomic, RunData};
+use super::process::Unstable;
+use crate::gen::tpl;
+use crate::gen::traits::Gen;
+use crate::interpreter::gen_nort::nort_gen;
+use crate::interpreter::nort::{Clause, ClauseInst, Expr};
+use crate::location::CodeLocation;
+use crate::utils::clonable_iter::Clonable;
+
+/// A trait for things that are infallibly convertible to [ClauseInst]. These
+/// types can be returned by callbacks passed to [super::fn_bridge::xfn].
+pub trait ToClause: Sized {
+  /// Convert this value to a [Clause]. If your value can only be directly
+  /// converted to a [ClauseInst], you can call `ClauseInst::to_clause` to
+  /// unwrap it if possible or fall back to [Clause::Identity].
+  fn to_clause(self, location: CodeLocation) -> Clause;
+
+  /// Convert the type to a [Clause].
+  fn to_clsi(self, location: CodeLocation) -> ClauseInst {
+    ClauseInst::new(self.to_clause(location))
+  }
+
+  /// Convert to an expression via [ToClause].
+  fn to_expr(self, location: CodeLocation) -> Expr {
+    Expr { clause: self.to_clsi(location.clone()), location }
+  }
+}
+
+impl<T: Atomic + Clone> ToClause for T {
+  fn to_clause(self, _: CodeLocation) -> Clause { self.atom_cls() }
+}
+impl ToClause for Clause {
+  fn to_clause(self, _: CodeLocation) -> Clause { self }
+}
+impl ToClause for ClauseInst {
+  fn to_clause(self, _: CodeLocation) -> Clause { self.into_cls() }
+  fn to_clsi(self, _: CodeLocation) -> ClauseInst { self }
+}
+impl ToClause for Expr {
+  fn to_clause(self, location: CodeLocation) -> Clause { self.clause.to_clause(location) }
+  fn to_clsi(self, _: CodeLocation) -> ClauseInst { self.clause }
+  fn to_expr(self, _: CodeLocation) -> Expr { self }
+}
+
+struct ListGen<I>(Clonable<I>)
+where
+  I: Iterator + Send,
+  I::Item: ToClause + Send;
+impl<I> Clone for ListGen<I>
+where
+  I: Iterator + Send,
+  I::Item: ToClause + Send,
+{
+  fn clone(&self) -> Self { Self(self.0.clone()) }
+}
+impl<I> ToClause for ListGen<I>
+where
+  I: Iterator + Send + 'static,
+  I::Item: ToClause + Clone + Send,
+{
+  fn to_clause(mut self, location: CodeLocation) -> Clause {
+    let ctx = nort_gen(location.clone());
+    match self.0.next() {
+      None => tpl::C("std::list::end").template(ctx, []),
+      Some(val) => {
+        let atom = Unstable::new(|run| self.to_clause(run.location));
+        tpl::a2(tpl::C("std::list::cons"), tpl::Slot, tpl::V(atom))
+          .template(ctx, [val.to_clause(location)])
+      },
+    }
+  }
+}
+
+/// Convert an iterator into a lazy-evaluated Orchid list.
+pub fn list<I>(items: I) -> impl ToClause
+where
+  I: IntoIterator + Clone + Send + Sync + 'static,
+  I::IntoIter: Send,
+  I::Item: ToClause + Clone + Send,
+{
+  Unstable::new(move |RunData { location, .. }| {
+    ListGen(Clonable::new(items.clone().into_iter().map(move |t| t.to_clsi(location.clone()))))
+  })
+}
+
+mod implementations {
+  use std::any::Any;
+  use std::fmt;
+  use std::sync::Arc;
+
+  use super::{list, ToClause};
+  use crate::foreign::atom::{Atom, Atomic, AtomicResult, CallData, RunData};
+  use crate::foreign::error::{AssertionError, RTErrorObj, RTResult};
+  use crate::foreign::inert::Inert;
+  use crate::foreign::try_from_expr::TryFromExpr;
+  use crate::gen::tpl;
+  use crate::gen::traits::Gen;
+  use crate::interpreter::gen_nort::nort_gen;
+  use crate::interpreter::nort::{Clause, Expr};
+  use crate::libs::std::tuple::Tuple;
+  use crate::location::CodeLocation;
+  use crate::utils::ddispatch::Responder;
+
+  impl<T: ToClause> ToClause for Option<T> {
+    fn to_clause(self, location: CodeLocation) -> Clause {
+      let ctx = nort_gen(location.clone());
+      match self {
+        None => tpl::C("std::option::none").template(ctx, []),
+        Some(t) =>
+          tpl::A(tpl::C("std::option::some"), tpl::Slot).template(ctx, [t.to_clause(location)]),
+      }
+    }
+  }
+
+  impl<T: ToClause, U: ToClause> ToClause for Result<T, U> {
+    fn to_clause(self, location: CodeLocation) -> Clause {
+      let ctx = nort_gen(location.clone());
+      match self {
+        Ok(t) =>
+          tpl::A(tpl::C("std::result::ok"), tpl::Slot).template(ctx, [t.to_clause(location)]),
+        Err(e) =>
+          tpl::A(tpl::C("std::result::err"), tpl::Slot).template(ctx, [e.to_clause(location)]),
+      }
+    }
+  }
+
+  struct PendingError(RTErrorObj);
+  impl Responder for PendingError {}
+  impl fmt::Debug for PendingError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+      write!(f, "PendingError({})", self.0)
+    }
+  }
+  impl Atomic for PendingError {
+    fn as_any(self: Box<Self>) -> Box<dyn Any> { self }
+    fn as_any_ref(&self) -> &dyn Any { self }
+    fn type_name(&self) -> &'static str { std::any::type_name::<Self>() }
+
+    fn redirect(&mut self) -> Option<&mut Expr> { None }
+    fn run(self: Box<Self>, _: RunData) -> AtomicResult { Err(self.0) }
+    fn apply_mut(&mut self, _: CallData) -> RTResult<Clause> {
+      panic!("This atom decays instantly")
+    }
+  }
+
+  impl<T: ToClause> ToClause for RTResult<T> {
+    fn to_clause(self, location: CodeLocation) -> Clause {
+      match self {
+        Err(e) => PendingError(e).atom_cls(),
+        Ok(t) => t.to_clause(location),
+      }
+    }
+  }
+
+  impl<T: ToClause + Clone + Send + Sync + 'static> ToClause for Vec<T> {
+    fn to_clause(self, location: CodeLocation) -> Clause { list(self).to_clause(location) }
+  }
+
+  impl ToClause for Atom {
+    fn to_clause(self, _: CodeLocation) -> Clause { Clause::Atom(self) }
+  }
+
+  macro_rules! gen_tuple_impl {
+    ( ($($T:ident)*) ($($t:ident)*)) => {
+      impl<$($T: ToClause),*> ToClause for ($($T,)*) {
+        fn to_clause(self, location: CodeLocation) -> Clause {
+          let ($($t,)*) = self;
+          Inert(Tuple(Arc::new(vec![
+            $($t.to_expr(location.clone()),)*
+          ]))).atom_cls()
+        }
+      }
+
+      impl<$($T: TryFromExpr),*> TryFromExpr for ($($T,)*) {
+        fn from_expr(ex: Expr) -> RTResult<Self> {
+          let Inert(Tuple(slice)) = ex.clone().downcast()?;
+          match &slice[..] {
+            [$($t),*] => Ok(($($t.clone().downcast()?,)*)),
+            _ => AssertionError::fail(ex.location(), "Tuple length mismatch", format!("{ex}"))
+          }
+        }
+      }
+    };
+  }
+
+  gen_tuple_impl!((A)(a));
+  gen_tuple_impl!((A B) (a b));
+  gen_tuple_impl!((A B C) (a b c));
+  gen_tuple_impl!((A B C D) (a b c d));
+  gen_tuple_impl!((A B C D E) (a b c d e));
+  gen_tuple_impl!((A B C D E F) (a b c d e f));
+  gen_tuple_impl!((A B C D E F G) (a b c d e f g));
+  gen_tuple_impl!((A B C D E F G H) (a b c d e f g h));
+  gen_tuple_impl!((A B C D E F G H I) (a b c d e f g h i));
+  gen_tuple_impl!((A B C D E F G H I J) (a b c d e f g h i j));
+  gen_tuple_impl!((A B C D E F G H I J K) (a b c d e f g h i j k));
+  gen_tuple_impl!((A B C D E F G H I J K L) (a b c d e f g h i j k l));
+}

orchidlang/src/intermediate/ast_to_ir.rs

@@ -0,0 +1,140 @@
+//! Convert the preprocessed AST into IR
+
+use std::collections::VecDeque;
+use std::rc::Rc;
+
+use substack::Substack;
+
+use super::ir;
+use crate::error::{ProjectError, ProjectResult};
+use crate::location::{CodeOrigin, SourceRange};
+use crate::name::Sym;
+use crate::parse::parsed;
+use crate::utils::unwrap_or::unwrap_or;
+
+trait IRErrorKind: Clone + Send + Sync + 'static {
+  const DESCR: &'static str;
+}
+
+#[derive(Clone)]
+struct IRError<T: IRErrorKind> {
+  at: SourceRange,
+  sym: SourceRange,
+  _kind: T,
+}
+impl<T: IRErrorKind> IRError<T> {
+  fn new(at: SourceRange, sym: SourceRange, _kind: T) -> Self { Self { at, sym, _kind } }
+}
+impl<T: IRErrorKind> ProjectError for IRError<T> {
+  const DESCRIPTION: &'static str = T::DESCR;
+  fn message(&self) -> String { format!("In {}, {}", self.sym, T::DESCR) }
+  fn one_position(&self) -> CodeOrigin { CodeOrigin::Source(self.at.clone()) }
+}
+
+#[derive(Clone)]
+struct EmptyS;
+impl IRErrorKind for EmptyS {
+  const DESCR: &'static str = "`()` as a clause is meaningless in lambda calculus";
+}
+
+#[derive(Clone)]
+struct BadGroup;
+impl IRErrorKind for BadGroup {
+  const DESCR: &'static str = "Only `(...)` may be used after macros. \
+  `[...]` and `{...}` left in the code are signs of incomplete macro execution";
+}
+
+#[derive(Clone)]
+struct InvalidArg;
+impl IRErrorKind for InvalidArg {
+  const DESCR: &'static str = "Argument names can only be Name nodes";
+}
+
+#[derive(Clone)]
+struct PhLeak;
+impl IRErrorKind for PhLeak {
+  const DESCR: &'static str = "Placeholders shouldn't even appear \
+    in the code during macro execution, this is likely a compiler bug";
+}
+
+/// Try to convert an expression from AST format to typed lambda
+pub fn ast_to_ir(expr: parsed::Expr, symbol: SourceRange, module: Sym) -> ProjectResult<ir::Expr> {
+  expr_rec(expr, Context::new(symbol, module))
+}
+
+#[derive(Clone)]
+struct Context<'a> {
+  names: Substack<'a, Sym>,
+  range: SourceRange,
+  module: Sym,
+}
+
+impl<'a> Context<'a> {
+  #[must_use]
+  fn w_name<'b>(&'b self, name: Sym) -> Context<'b>
+  where 'a: 'b {
+    Context { names: self.names.push(name), range: self.range.clone(), module: self.module.clone() }
+  }
+}
+impl Context<'static> {
+  #[must_use]
+  fn new(symbol: SourceRange, module: Sym) -> Self {
+    Self { names: Substack::Bottom, range: symbol, module }
+  }
+}
+
+/// Process an expression sequence
+fn exprv_rec(
+  mut v: VecDeque<parsed::Expr>,
+  ctx: Context<'_>,
+  location: SourceRange,
+) -> ProjectResult<ir::Expr> {
+  let last = unwrap_or! {v.pop_back(); {
+    return Err(IRError::new(location, ctx.range, EmptyS).pack());
+  }};
+  let v_end = match v.back() {
+    None => return expr_rec(last, ctx),
+    Some(penultimate) => penultimate.range.end(),
+  };
+  let f = exprv_rec(v, ctx.clone(), location.map_range(|r| r.start..v_end))?;
+  let x = expr_rec(last, ctx.clone())?;
+  let value = ir::Clause::Apply(Rc::new(f), Rc::new(x));
+  Ok(ir::Expr::new(value, location, ctx.module))
+}
+
+/// Process an expression
+fn expr_rec(parsed::Expr { value, range }: parsed::Expr, ctx: Context) -> ProjectResult<ir::Expr> {
+  match value {
+    parsed::Clause::S(parsed::PType::Par, body) => {
+      return exprv_rec(body.to_vec().into(), ctx, range);
+    },
+    parsed::Clause::S(..) => return Err(IRError::new(range, ctx.range, BadGroup).pack()),
+    _ => (),
+  }
+  let value = match value {
+    parsed::Clause::Atom(a) => ir::Clause::Atom(a.clone()),
+    parsed::Clause::Lambda(arg, b) => {
+      let name = match &arg[..] {
+        [parsed::Expr { value: parsed::Clause::Name(name), .. }] => name,
+        [parsed::Expr { value: parsed::Clause::Placeh { .. }, .. }] =>
+          return Err(IRError::new(range.clone(), ctx.range, PhLeak).pack()),
+        _ => return Err(IRError::new(range.clone(), ctx.range, InvalidArg).pack()),
+      };
+      let body_ctx = ctx.w_name(name.clone());
+      let body = exprv_rec(b.to_vec().into(), body_ctx, range.clone())?;
+      ir::Clause::Lambda(Rc::new(body))
+    },
+    parsed::Clause::Name(name) => {
+      let lvl_opt = (ctx.names.iter()).enumerate().find(|(_, n)| **n == name).map(|(lvl, _)| lvl);
+      match lvl_opt {
+        Some(lvl) => ir::Clause::LambdaArg(lvl),
+        None => ir::Clause::Constant(name.clone()),
+      }
+    },
+    parsed::Clause::S(parsed::PType::Par, entries) =>
+      exprv_rec(entries.to_vec().into(), ctx.clone(), range.clone())?.value,
+    parsed::Clause::S(..) => return Err(IRError::new(range, ctx.range, BadGroup).pack()),
+    parsed::Clause::Placeh { .. } => return Err(IRError::new(range, ctx.range, PhLeak).pack()),
+  };
+  Ok(ir::Expr::new(value, range, ctx.module))
+}

orchidlang/src/intermediate/ir.rs

@@ -0,0 +1,114 @@
+//! IR is an abstract representation of Orchid expressions that's impractical
+//! for all purposes except converting to and from other representations. Future
+//! innovations in the processing and execution of code will likely operate on
+//! this representation.
+
+use std::fmt;
+use std::rc::Rc;
+
+use crate::foreign::atom::AtomGenerator;
+use crate::location::{CodeLocation, SourceRange};
+use crate::name::Sym;
+use crate::utils::string_from_charset::string_from_charset;
+
+/// Indicates whether either side needs to be wrapped. Syntax whose end is
+/// ambiguous on that side must use parentheses, or forward the flag
+#[derive(PartialEq, Eq, Clone, Copy)]
+struct Wrap(bool, bool);
+
+/// Code element with associated metadata
+#[derive(Clone)]
+pub struct Expr {
+  /// Code element
+  pub value: Clause,
+  /// Location metadata
+  pub location: CodeLocation,
+}
+
+impl Expr {
+  /// Create an IR expression
+  pub fn new(value: Clause, location: SourceRange, module: Sym) -> Self {
+    Self { value, location: CodeLocation::new_src(location, module) }
+  }
+
+  fn deep_fmt(&self, f: &mut fmt::Formatter<'_>, depth: usize, tr: Wrap) -> fmt::Result {
+    let Expr { value, .. } = self;
+    value.deep_fmt(f, depth, tr)?;
+    Ok(())
+  }
+}
+
+impl fmt::Debug for Expr {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    self.deep_fmt(f, 0, Wrap(false, false))
+  }
+}
+
+/// Semantic code element
+#[derive(Clone)]
+pub enum Clause {
+  /// Function call expression
+  Apply(Rc<Expr>, Rc<Expr>),
+  /// Function expression
+  Lambda(Rc<Expr>),
+  /// Reference to an external constant
+  Constant(Sym),
+  /// Reference to a function argument
+  LambdaArg(usize),
+  /// An opaque non-callable value, eg. a file handle
+  Atom(AtomGenerator),
+}
+
+const ARGNAME_CHARSET: &str = "abcdefghijklmnopqrstuvwxyz";
+
+fn parametric_fmt(
+  f: &mut fmt::Formatter<'_>,
+  depth: usize,
+  prefix: &str,
+  body: &Expr,
+  wrap_right: bool,
+) -> fmt::Result {
+  if wrap_right {
+    write!(f, "(")?;
+  }
+  f.write_str(prefix)?;
+  f.write_str(&string_from_charset(depth as u64, ARGNAME_CHARSET))?;
+  f.write_str(".")?;
+  body.deep_fmt(f, depth + 1, Wrap(false, false))?;
+  if wrap_right {
+    write!(f, ")")?;
+  }
+  Ok(())
+}
+
+impl Clause {
+  fn deep_fmt(&self, f: &mut fmt::Formatter<'_>, depth: usize, Wrap(wl, wr): Wrap) -> fmt::Result {
+    match self {
+      Self::Atom(a) => write!(f, "{a:?}"),
+      Self::Lambda(body) => parametric_fmt(f, depth, "\\", body, wr),
+      Self::LambdaArg(skip) => {
+        let lambda_depth = (depth - skip - 1).try_into().unwrap();
+        f.write_str(&string_from_charset(lambda_depth, ARGNAME_CHARSET))
+      },
+      Self::Apply(func, x) => {
+        if wl {
+          write!(f, "(")?;
+        }
+        func.deep_fmt(f, depth, Wrap(false, true))?;
+        write!(f, " ")?;
+        x.deep_fmt(f, depth, Wrap(true, wr && !wl))?;
+        if wl {
+          write!(f, ")")?;
+        }
+        Ok(())
+      },
+      Self::Constant(token) => write!(f, "{token}"),
+    }
+  }
+}
+
+impl fmt::Debug for Clause {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    self.deep_fmt(f, 0, Wrap(false, false))
+  }
+}

orchidlang/src/intermediate/ir_to_nort.rs

@@ -0,0 +1,37 @@
+//! Convert IR to the interpreter's NORT representation
+
+use super::ir;
+use crate::interpreter::nort;
+use crate::interpreter::nort_builder::NortBuilder;
+
+fn expr(expr: &ir::Expr, ctx: NortBuilder<(), usize>) -> nort::Expr {
+  clause(&expr.value, ctx).into_expr(expr.location.clone())
+}
+
+fn clause(cls: &ir::Clause, ctx: NortBuilder<(), usize>) -> nort::Clause {
+  match cls {
+    ir::Clause::Constant(name) => nort::Clause::Constant(name.clone()),
+    ir::Clause::Atom(a) => nort::Clause::Atom(a.run()),
+    ir::Clause::LambdaArg(n) => {
+      ctx.arg_logic(n);
+      nort::Clause::LambdaArg
+    },
+    ir::Clause::Apply(f, x) => ctx.apply_logic(|c| expr(f, c), |c| expr(x, c)),
+    ir::Clause::Lambda(body) => ctx.lambda_logic(&(), |c| expr(body, c)),
+  }
+}
+
+/// Convert an expression.
+pub fn ir_to_nort(expr: &ir::Expr) -> nort::Expr {
+  let c = NortBuilder::new(&|count| {
+    let mut count: usize = *count;
+    Box::new(move |()| match count {
+      0 => true,
+      _ => {
+        count -= 1;
+        false
+      },
+    })
+  });
+  nort::ClauseInst::new(clause(&expr.value, c)).into_expr(expr.location.clone())
+}

orchidlang/src/intermediate/mod.rs

@@ -0,0 +1,7 @@
+//! Intermediate representation. Currently just an indirection between
+//! [super::parse::parsed] and [super::interpreter::nort], in the future
+//! hopefully a common point for alternate encodings, optimizations and targets.
+
+pub mod ast_to_ir;
+pub mod ir;
+pub mod ir_to_nort;

orchidlang/src/interpreter/apply.rs

@@ -0,0 +1,109 @@
+use never::Never;
+
+use super::context::{RunEnv, RunParams};
+use super::nort::{Clause, ClauseInst, Expr};
+use super::path_set::{PathSet, Step};
+use crate::foreign::atom::CallData;
+use crate::foreign::error::RTResult;
+
+/// Process the clause at the end of the provided path. Note that paths always
+/// point to at least one target. Note also that this is not cached as a
+/// normalization step in the intermediate expressions.
+fn map_at<E>(
+  mut path: impl Iterator<Item = Step>,
+  source: &Clause,
+  mapper: &mut impl FnMut(&Clause) -> Result<Clause, E>,
+) -> Result<Clause, E> {
+  // Pass through some unambiguous wrapper clauses
+  match source {
+    Clause::Identity(alt) => return map_at(path, &alt.cls_mut(), mapper),
+    Clause::Lambda { args, body: Expr { location: b_loc, clause } } =>
+      return Ok(Clause::Lambda {
+        args: args.clone(),
+        body: Expr {
+          clause: map_at(path, &clause.cls_mut(), mapper)?.into_inst(),
+          location: b_loc.clone(),
+        },
+      }),
+    _ => (),
+  }
+  Ok(match (source, path.next()) {
+    (Clause::Lambda { .. } | Clause::Identity(_), _) => unreachable!("Handled above"),
+    // If the path ends and this isn't a lambda, process it
+    (val, None) => mapper(val)?,
+    // If it's an Apply, execute the next step in the path
+    (Clause::Apply { f, x }, Some(head)) => {
+      let proc = |x: &Expr| Ok(map_at(path, &x.cls_mut(), mapper)?.into_expr(x.location()));
+      match head {
+        None => Clause::Apply { f: proc(f)?, x: x.clone() },
+        Some(n) => {
+          let i = x.len() - n - 1;
+          let mut argv = x.clone();
+          argv[i] = proc(&x[i])?;
+          Clause::Apply { f: f.clone(), x: argv }
+        },
+      }
+    },
+    (_, Some(_)) => panic!("Path leads into node that isn't Apply or Lambda"),
+  })
+}
+
+/// Replace the [Clause::LambdaArg] placeholders at the ends of the [PathSet]
+/// with the value in the body. Note that a path may point to multiple
+/// placeholders.
+#[must_use]
+pub fn substitute(
+  paths: &PathSet,
+  value: ClauseInst,
+  body: &Clause,
+  on_sub: &mut impl FnMut(),
+) -> Clause {
+  let PathSet { steps, next } = paths;
+  map_at(steps.iter().cloned(), body, &mut |chkpt| -> Result<Clause, Never> {
+    match (chkpt, next) {
+      (Clause::Lambda { .. } | Clause::Identity(_), _) => {
+        unreachable!("Handled by map_at")
+      },
+      (Clause::Apply { f, x }, Some(conts)) => {
+        let mut argv = x.clone();
+        let f = match conts.get(&None) {
+          None => f.clone(),
+          Some(sp) => substitute(sp, value.clone(), &f.cls_mut(), on_sub).into_expr(f.location()),
+        };
+        for (i, old) in argv.iter_mut().rev().enumerate() {
+          if let Some(sp) = conts.get(&Some(i)) {
+            let tmp = substitute(sp, value.clone(), &old.cls_mut(), on_sub);
+            *old = tmp.into_expr(old.location());
+          }
+        }
+        Ok(Clause::Apply { f, x: argv })
+      },
+      (Clause::LambdaArg, None) => {
+        on_sub();
+        Ok(Clause::Identity(value.clone()))
+      },
+      (_, None) => panic!("Argument path must point to LambdaArg"),
+      (_, Some(_)) => panic!("Argument path can only fork at Apply"),
+    }
+  })
+  .unwrap_or_else(|e| match e {})
+}
+
+pub(super) fn apply_as_atom(
+  f: Expr,
+  arg: Expr,
+  env: &RunEnv,
+  params: &mut RunParams,
+) -> RTResult<Clause> {
+  let call = CallData { location: f.location(), arg, env, params };
+  match f.clause.try_unwrap() {
+    Ok(clause) => match clause {
+      Clause::Atom(atom) => Ok(atom.apply(call)?),
+      _ => panic!("Not an atom"),
+    },
+    Err(clsi) => match &mut *clsi.cls_mut() {
+      Clause::Atom(atom) => Ok(atom.apply_mut(call)?),
+      _ => panic!("Not an atom"),
+    },
+  }
+}

orchidlang/src/interpreter/error.rs

@@ -0,0 +1,33 @@
+//! Error produced by the interpreter.
+
+use std::fmt;
+
+use super::run::State;
+use crate::foreign::error::{RTError, RTErrorObj};
+
+/// Error produced by the interpreter. This could be because the code is faulty,
+/// but equally because gas was being counted and it ran out.
+#[derive(Debug)]
+pub enum RunError<'a> {
+  /// A Rust function encountered an error
+  Extern(RTErrorObj),
+  /// Ran out of gas
+  Interrupted(State<'a>),
+}
+
+impl<'a, T: RTError + 'static> From<T> for RunError<'a> {
+  fn from(value: T) -> Self { Self::Extern(value.pack()) }
+}
+
+impl<'a> From<RTErrorObj> for RunError<'a> {
+  fn from(value: RTErrorObj) -> Self { Self::Extern(value) }
+}
+
+impl<'a> fmt::Display for RunError<'a> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Interrupted(i) => write!(f, "Ran out of gas:\n{i}"),
+      Self::Extern(e) => write!(f, "Program fault: {e}"),
+    }
+  }
+}

orchidlang/src/interpreter/gen_nort.rs

@@ -0,0 +1,94 @@
+//! Implementations of [Generable] for [super::nort]
+
+use intern_all::i;
+
+use super::nort_builder::NortBuilder;
+use crate::foreign::atom::Atom;
+use crate::foreign::to_clause::ToClause;
+use crate::gen::traits::Generable;
+use crate::interpreter::nort::{Clause, ClauseInst, Expr};
+use crate::location::CodeLocation;
+use crate::name::Sym;
+
+/// Context data for instantiating templated expressions as [super::nort].
+/// Instances of this type are created via [nort_gen]
+pub type NortGenCtx<'a> = (CodeLocation, NortBuilder<'a, str, str>);
+
+/// Create [NortGenCtx] instances to generate interpreted expressions
+pub fn nort_gen<'a>(location: CodeLocation) -> NortGenCtx<'a> {
+  (location, NortBuilder::new(&|l| Box::new(move |r| l == r)))
+}
+
+impl Generable for Expr {
+  type Ctx<'a> = NortGenCtx<'a>;
+  fn apply(
+    ctx: Self::Ctx<'_>,
+    f_cb: impl FnOnce(Self::Ctx<'_>) -> Self,
+    x_cb: impl FnOnce(Self::Ctx<'_>) -> Self,
+  ) -> Self {
+    (ctx.1.apply_logic(|c| f_cb((ctx.0.clone(), c)), |c| x_cb((ctx.0.clone(), c))))
+      .into_expr(ctx.0.clone())
+  }
+  fn arg(ctx: Self::Ctx<'_>, name: &str) -> Self {
+    Clause::arg(ctx.clone(), name).into_expr(ctx.0.clone())
+  }
+  fn atom(ctx: Self::Ctx<'_>, a: Atom) -> Self {
+    Clause::atom(ctx.clone(), a).into_expr(ctx.0.clone())
+  }
+  fn constant<'a>(ctx: Self::Ctx<'_>, name: impl IntoIterator<Item = &'a str>) -> Self {
+    Clause::constant(ctx.clone(), name).into_expr(ctx.0.clone())
+  }
+  fn lambda(ctx: Self::Ctx<'_>, name: &str, body: impl FnOnce(Self::Ctx<'_>) -> Self) -> Self {
+    (ctx.1.lambda_logic(name, |c| body((ctx.0.clone(), c)))).into_expr(ctx.0.clone())
+  }
+}
+
+impl Generable for ClauseInst {
+  type Ctx<'a> = NortGenCtx<'a>;
+  fn arg(ctx: Self::Ctx<'_>, name: &str) -> Self { Clause::arg(ctx, name).into_inst() }
+  fn atom(ctx: Self::Ctx<'_>, a: Atom) -> Self { Clause::atom(ctx, a).into_inst() }
+  fn constant<'a>(ctx: Self::Ctx<'_>, name: impl IntoIterator<Item = &'a str>) -> Self {
+    Clause::constant(ctx, name).into_inst()
+  }
+  fn lambda(ctx: Self::Ctx<'_>, name: &str, body: impl FnOnce(Self::Ctx<'_>) -> Self) -> Self {
+    (ctx.1.lambda_logic(name, |c| body((ctx.0.clone(), c)).into_expr(ctx.0.clone())))
+      .to_clsi(ctx.0.clone())
+  }
+  fn apply(
+    ctx: Self::Ctx<'_>,
+    f: impl FnOnce(Self::Ctx<'_>) -> Self,
+    x: impl FnOnce(Self::Ctx<'_>) -> Self,
+  ) -> Self {
+    (ctx.1.apply_logic(
+      |c| f((ctx.0.clone(), c)).into_expr(ctx.0.clone()),
+      |c| x((ctx.0.clone(), c)).into_expr(ctx.0.clone()),
+    ))
+    .to_clsi(ctx.0.clone())
+  }
+}
+
+impl Generable for Clause {
+  type Ctx<'a> = NortGenCtx<'a>;
+  fn atom(_: Self::Ctx<'_>, a: Atom) -> Self { Clause::Atom(a) }
+  fn constant<'a>(_: Self::Ctx<'_>, name: impl IntoIterator<Item = &'a str>) -> Self {
+    let sym = Sym::new(name.into_iter().map(i)).expect("Empty constant");
+    Clause::Constant(sym)
+  }
+  fn apply(
+    ctx: Self::Ctx<'_>,
+    f: impl FnOnce(Self::Ctx<'_>) -> Self,
+    x: impl FnOnce(Self::Ctx<'_>) -> Self,
+  ) -> Self {
+    ctx.1.apply_logic(
+      |c| f((ctx.0.clone(), c)).into_expr(ctx.0.clone()),
+      |c| x((ctx.0.clone(), c)).into_expr(ctx.0.clone()),
+    )
+  }
+  fn arg(ctx: Self::Ctx<'_>, name: &str) -> Self {
+    ctx.1.arg_logic(name);
+    Clause::LambdaArg
+  }
+  fn lambda(ctx: Self::Ctx<'_>, name: &str, body: impl FnOnce(Self::Ctx<'_>) -> Self) -> Self {
+    ctx.1.lambda_logic(name, |c| body((ctx.0.clone(), c)).into_expr(ctx.0.clone()))
+  }
+}

orchidlang/src/interpreter/mod.rs

@@ -0,0 +1,10 @@
+//! functions to execute Orchid code
+mod apply;
+pub mod context;
+pub mod error;
+pub mod gen_nort;
+pub mod handler;
+pub mod nort;
+pub mod nort_builder;
+pub(crate) mod path_set;
+pub mod run;

orchidlang/src/interpreter/nort_builder.rs

@@ -0,0 +1,148 @@
+//! Helper for generating the interpreter's internal representation
+
+use std::cell::RefCell;
+use std::mem;
+
+use substack::Substack;
+
+use super::nort::{AsDerefMut, Clause, Expr};
+use super::path_set::PathSet;
+use crate::utils::pure_seq::pushed;
+
+enum IntGenData<'a, T: ?Sized> {
+  Lambda(&'a T, &'a RefCell<Option<PathSet>>),
+  /// Counts left steps within a chain of [Clause::Apply] for collapsing.
+  Apply(&'a RefCell<usize>),
+  /// Replaces [IntGenData::Apply] when stepping left into non-apply to record
+  /// a [None] [super::path_set::Step].
+  AppF,
+  /// Replaces [IntGenData::Apply] when stepping right to freeze the value.
+  AppArg(usize),
+}
+
+impl<'a, T: ?Sized> Copy for IntGenData<'a, T> {}
+impl<'a, T: ?Sized> Clone for IntGenData<'a, T> {
+  fn clone(&self) -> Self { *self }
+}
+
+struct ArgCollector(RefCell<Option<PathSet>>);
+impl ArgCollector {
+  pub fn new() -> Self { Self(RefCell::new(None)) }
+  pub fn into_path(self) -> Option<PathSet> { self.0.into_inner() }
+}
+
+/// Strategy used to find the lambda corresponding to a given argument in the
+/// stack. The function is called on the data associated with the argument, then
+/// the callback it returns is called on every lambda ancestor's associated
+/// data from closest to outermost ancestor. The first lambda where this
+/// callback returns true is considered to own the argument.
+pub type LambdaPicker<'a, T, U> = &'a dyn for<'b> Fn(&'b U) -> Box<dyn FnMut(&T) -> bool + 'b>;
+
+/// Bundle of information passed down through recursive fnuctions to instantiate
+/// runtime [Expr], [super::nort::ClauseInst] or [Clause].
+///
+/// The context used by [crate::gen::traits::Gen] to convert templates is which
+/// includes this type is constructed with [super::gen_nort::nort_gen].
+pub struct NortBuilder<'a, T: ?Sized, U: ?Sized> {
+  stack: Substack<'a, IntGenData<'a, T>>,
+  lambda_picker: LambdaPicker<'a, T, U>,
+}
+impl<'a, T: ?Sized, U: ?Sized> NortBuilder<'a, T, U> {
+  /// Create a new recursive [super::nort] builder from a location that will be
+  pub fn new(lambda_picker: LambdaPicker<'a, T, U>) -> Self {
+    Self { stack: Substack::Bottom, lambda_picker }
+  }
+  /// [Substack::pop] and clone the location
+  fn pop<'b>(&'b self, count: usize) -> NortBuilder<'b, T, U>
+  where 'a: 'b {
+    let mut new = *self;
+    new.stack = *self.stack.pop(count);
+    new
+  }
+  /// [Substack::push] and clone the location
+  fn push<'b>(&'b self, data: IntGenData<'a, T>) -> NortBuilder<'b, T, U>
+  where 'a: 'b {
+    let mut new = *self;
+    new.stack = self.stack.push(data);
+    new
+  }
+  fn non_app_step<V>(self, f: impl FnOnce(NortBuilder<T, U>) -> V) -> V {
+    if let Some(IntGenData::Apply(_)) = self.stack.value() {
+      f(self.pop(1).push(IntGenData::AppF))
+    } else {
+      f(self)
+    }
+  }
+
+  /// Climb back through the stack and find a lambda associated with this
+  /// argument, then record the path taken from the lambda to this argument in
+  /// the lambda's mutable cell.
+  pub fn arg_logic(self, name: &'a U) {
+    let mut lambda_chk = (self.lambda_picker)(name);
+    self.non_app_step(|ctx| {
+      let res = ctx.stack.iter().try_fold(vec![], |path, item| match item {
+        IntGenData::Apply(_) => panic!("This is removed after handling"),
+        IntGenData::Lambda(n, rc) => match lambda_chk(n) {
+          false => Ok(path),
+          true => Err((path, *rc)),
+        },
+        IntGenData::AppArg(n) => Ok(pushed(path, Some(*n))),
+        IntGenData::AppF => Ok(pushed(path, None)),
+      });
+      let (mut path, slot) = res.expect_err("Argument not wrapped in matching lambda");
+      path.reverse();
+      match &mut *slot.borrow_mut() {
+        slot @ None => *slot = Some(PathSet::end(path)),
+        Some(slot) => take_mut::take(slot, |p| p.overlay(PathSet::end(path))),
+      }
+    })
+  }
+
+  /// Push a stackframe corresponding to a lambda expression, build the body,
+  /// then record the path set collected by [NortBuilder::arg_logic] calls
+  /// within the body.
+  pub fn lambda_logic(self, name: &T, body: impl FnOnce(NortBuilder<T, U>) -> Expr) -> Clause {
+    let coll = ArgCollector::new();
+    let frame = IntGenData::Lambda(name, &coll.0);
+    let body = self.non_app_step(|ctx| body(ctx.push(frame)));
+    let args = coll.into_path();
+    Clause::Lambda { args, body }
+  }
+
+  /// Logic for collapsing Apply clauses. Different steps of the logic
+  /// communicate via mutable variables on the stack
+  pub fn apply_logic(
+    self,
+    f: impl FnOnce(NortBuilder<T, U>) -> Expr,
+    x: impl FnOnce(NortBuilder<T, U>) -> Expr,
+  ) -> Clause {
+    let mut fun: Expr;
+    let arg: Expr;
+    if let Some(IntGenData::Apply(rc)) = self.stack.value() {
+      // argument side commits backidx
+      arg = x(self.pop(1).push(IntGenData::AppArg(*rc.borrow())));
+      // function side increments backidx
+      *rc.borrow_mut() += 1;
+      fun = f(self);
+    } else {
+      // function side starts from backidx 1
+      fun = f(self.push(IntGenData::Apply(&RefCell::new(1))));
+      // argument side commits 0
+      arg = x(self.push(IntGenData::AppArg(0)));
+    };
+    let mut cls_lk = fun.as_deref_mut();
+    if let Clause::Apply { x, f: _ } = &mut *cls_lk {
+      x.push_back(arg);
+      mem::drop(cls_lk);
+      fun.clause.into_cls()
+    } else {
+      mem::drop(cls_lk);
+      Clause::Apply { f: fun, x: [arg].into() }
+    }
+  }
+}
+
+impl<'a, T: ?Sized, U: ?Sized> Copy for NortBuilder<'a, T, U> {}
+impl<'a, T: ?Sized, U: ?Sized> Clone for NortBuilder<'a, T, U> {
+  fn clone(&self) -> Self { *self }
+}

orchidlang/src/interpreter/path_set.rs

@@ -0,0 +1,160 @@
+use std::collections::VecDeque;
+use std::fmt;
+
+use hashbrown::HashMap;
+use itertools::Itertools;
+
+use crate::utils::join::join_maps;
+
+/// A step into a [super::nort::Clause::Apply]. If [None], it steps to the
+/// function. If [Some(n)], it steps to the `n`th _last_ argument.
+pub type Step = Option<usize>;
+fn print_step(step: Step) -> String {
+  if let Some(n) = step { format!("{n}") } else { "f".to_string() }
+}
+
+/// A branching path selecting some placeholders (but at least one) in a Lambda
+/// expression
+#[derive(Clone)]
+pub struct PathSet {
+  /// The single steps through [super::nort::Clause::Apply]
+  pub steps: VecDeque<Step>,
+  /// if Some, it splits at a [super::nort::Clause::Apply]. If None, it ends in
+  /// a [super::nort::Clause::LambdaArg]
+  pub next: Option<HashMap<Step, PathSet>>,
+}
+
+impl PathSet {
+  /// Create a path set for more than one target
+  pub fn branch(
+    steps: impl IntoIterator<Item = Step>,
+    conts: impl IntoIterator<Item = (Step, Self)>,
+  ) -> Self {
+    let conts = conts.into_iter().collect::<HashMap<_, _>>();
+    assert!(1 < conts.len(), "Branching pathsets need multiple continuations");
+    Self { steps: steps.into_iter().collect(), next: Some(conts) }
+  }
+
+  /// Create a path set for one target
+  pub fn end(steps: impl IntoIterator<Item = Step>) -> Self {
+    Self { steps: steps.into_iter().collect(), next: None }
+  }
+
+  /// Create a path set that points to a slot that is a direct
+  /// child of the given lambda with no applications. In essence, this means
+  /// that this argument will be picked as the value of the expression after an
+  /// arbitrary amount of subsequent discarded parameters.
+  pub fn pick() -> Self { Self { steps: VecDeque::new(), next: None } }
+
+  /// Merge two paths into one path that points to all targets of both. Only
+  /// works if both paths select leaf nodes of the same partial tree.
+  ///
+  /// # Panics
+  ///
+  /// if either path selects a node the other path dissects
+  pub fn overlay(self, other: Self) -> Self {
+    let (mut short, mut long) = match self.steps.len() < other.steps.len() {
+      true => (self, other),
+      false => (other, self),
+    };
+    let short_len = short.steps.len();
+    let long_len = long.steps.len();
+    let match_len = (short.steps.iter()).zip(long.steps.iter()).take_while(|(a, b)| a == b).count();
+    // fact: match_len <= short_len <= long_len
+    if short_len == match_len && match_len == long_len {
+      // implies match_len == short_len == long_len
+      match (short.next, long.next) {
+        (None, None) => Self::end(short.steps.iter().cloned()),
+        (Some(_), None) | (None, Some(_)) => {
+          panic!("One of these paths is faulty")
+        },
+        (Some(s), Some(l)) =>
+          Self::branch(short.steps.iter().cloned(), join_maps(s, l, |_, l, r| l.overlay(r))),
+      }
+    } else if short_len == match_len {
+      // implies match_len == short_len < long_len
+      // long.steps[0..match_len] is in steps
+      // long.steps[match_len] becomes the choice of branch below
+      // long.steps[match_len + 1..] is in tail
+      let mut conts = short.next.expect("One path ends inside the other");
+      let tail_steps = long.steps.split_off(match_len + 1);
+      let tail = match long.next {
+        Some(n) => Self::branch(tail_steps, n),
+        None => Self::end(tail_steps),
+      };
+      let branch = long.steps[match_len];
+      let prev_c = conts.remove(&branch);
+      let new_c = if let Some(x) = prev_c { x.overlay(tail) } else { tail };
+      conts.insert(branch, new_c);
+      Self::branch(short.steps, conts)
+    } else {
+      // implies match_len < short_len <= long_len
+      // steps[0..match_len] is in shared
+      // steps[match_len] become the branches below
+      // steps[match_len + 1..] is in new_long and new_short
+      let new_short_steps = short.steps.split_off(match_len + 1);
+      let short_last = short.steps.pop_back().expect("split at n + 1");
+      let new_short = Self { next: short.next.clone(), steps: new_short_steps };
+      let new_long_steps = long.steps.split_off(match_len + 1);
+      let new_long = Self { next: long.next.clone(), steps: new_long_steps };
+      Self::branch(short.steps, [(short_last, new_short), (long.steps[match_len], new_long)])
+    }
+  }
+
+  /// Prepend a step to a path. If it had previously started at a node that is
+  /// at the specified step within an Apply clause, it now starts at the Apply.
+  ///
+  /// This is only valid if the new Apply is **separate** from the previous
+  /// root.
+  pub fn prepend(&mut self, step: Step) { self.steps.push_front(step); }
+}
+
+impl fmt::Display for PathSet {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    let step_s = self.steps.iter().copied().map(print_step).join(">");
+    match &self.next {
+      Some(conts) => {
+        let opts = (conts.iter())
+          .sorted_unstable_by_key(|(k, _)| k.map_or(0, |n| n + 1))
+          .map(|(h, t)| format!("{}>{t}", print_step(*h)))
+          .join("|");
+        if !step_s.is_empty() {
+          write!(f, "{step_s}>")?;
+        }
+        write!(f, "({opts})")
+      },
+      None => write!(f, "{step_s}"),
+    }
+  }
+}
+
+impl fmt::Debug for PathSet {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "PathSet({self})") }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn test_combine() {
+    let ps1 = PathSet { next: None, steps: VecDeque::from([Some(2), None]) };
+    let ps2 = PathSet { next: None, steps: VecDeque::from([Some(3), Some(1)]) };
+    let sum = ps1.clone().overlay(ps2.clone());
+    assert_eq!(format!("{sum}"), "(2>f|3>1)");
+  }
+
+  fn extend_scaffold() -> PathSet {
+    PathSet::branch([None, Some(1), None], [
+      (None, PathSet::end([None, Some(1)])),
+      (Some(1), PathSet::end([None, Some(2)])),
+    ])
+  }
+
+  #[test]
+  fn test_extend_noclone() {
+    let mut ps = extend_scaffold();
+    ps.prepend(Some(0));
+    assert_eq!(format!("{ps}"), "0>f>1>f>(f>f>1|1>f>2)");
+  }
+}

orchidlang/src/interpreter/run.rs

@@ -0,0 +1,249 @@
+//! Executes Orchid code
+
+use std::ops::{Deref, DerefMut};
+use std::sync::{MutexGuard, TryLockError};
+use std::{fmt, mem};
+
+use bound::Bound;
+use itertools::Itertools;
+
+use super::context::{Halt, RunEnv, RunParams};
+use super::error::RunError;
+use super::nort::{Clause, Expr};
+use crate::foreign::atom::{AtomicReturn, RunData};
+use crate::foreign::error::{RTError, RTErrorObj};
+use crate::interpreter::apply::{apply_as_atom, substitute};
+use crate::location::CodeLocation;
+use crate::utils::take_with_output::take_with_output;
+
+#[derive(Debug)]
+struct Stackframe {
+  expr: Expr,
+  cls: Bound<MutexGuard<'static, Clause>, Expr>,
+}
+impl Stackframe {
+  pub fn new(expr: Expr) -> Option<Self> {
+    match Bound::try_new(expr.clone(), |e| e.clause.0.try_lock()) {
+      Ok(cls) => Some(Stackframe { cls, expr }),
+      Err(bound_e) if matches!(bound_e.wrapped(), TryLockError::WouldBlock) => None,
+      Err(bound_e) => panic!("{:?}", bound_e.wrapped()),
+    }
+  }
+  pub fn wait_new(expr: Expr) -> Self {
+    Self { cls: Bound::new(expr.clone(), |e| e.clause.0.lock().unwrap()), expr }
+  }
+  pub fn record_cycle(&mut self) -> RTErrorObj {
+    let err = CyclicalExpression(self.expr.clone()).pack();
+    *self.cls = Clause::Bottom(err.clone());
+    err
+  }
+}
+impl Deref for Stackframe {
+  type Target = Clause;
+  fn deref(&self) -> &Self::Target { &self.cls }
+}
+impl DerefMut for Stackframe {
+  fn deref_mut(&mut self) -> &mut Self::Target { &mut self.cls }
+}
+impl fmt::Display for Stackframe {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}\n    at {}", *self.cls, self.expr.location)
+  }
+}
+
+/// Interpreter state when processing was interrupted
+pub struct State<'a> {
+  stack: Vec<Stackframe>,
+  popped: Option<Expr>,
+  env: &'a RunEnv<'a>,
+}
+impl<'a> State<'a> {
+  /// Create a new trivial state with a specified stack size and a single
+  /// element on the stack
+  fn new(base: Expr, env: &'a RunEnv<'a>) -> Self {
+    let stack = vec![Stackframe::new(base).expect("Initial state should not be locked")];
+    State { stack, popped: None, env }
+  }
+
+  /// Try to push an expression on the stack, raise appropriate errors if the
+  /// expression is already on the stack (and thus references itself), or if the
+  /// stack now exceeds the pre-defined height
+  fn push_expr(&'_ mut self, expr: Expr, params: &RunParams) -> Result<(), RunError<'a>> {
+    let sf = match Stackframe::new(expr.clone()) {
+      Some(sf) => sf,
+      None => match self.stack.iter_mut().rev().find(|sf| sf.expr.clause.is_same(&expr.clause)) {
+        None => Stackframe::wait_new(expr),
+        Some(sf) => return Err(RunError::Extern(sf.record_cycle())),
+      },
+    };
+    self.stack.push(sf);
+    if params.stack < self.stack.len() {
+      let so = StackOverflow(self.stack.iter().map(|sf| sf.expr.clone()).collect());
+      return Err(RunError::Extern(so.pack()));
+    }
+    Ok(())
+  }
+
+  /// Process this state until it either completes, runs out of gas as specified
+  /// in the context, or produces an error.
+  pub fn run(mut self, params: &mut RunParams) -> Result<Halt, RunError<'a>> {
+    loop {
+      if params.no_gas() {
+        return Err(RunError::Interrupted(self));
+      }
+      params.use_gas(1);
+      let top = self.stack.last_mut().expect("Stack never empty");
+      let location = top.expr.location();
+      let op = take_with_output(&mut *top.cls, |c| {
+        match step(c, self.popped, location, self.env, params) {
+          Err(e) => (Clause::Bottom(e.clone()), Err(RunError::Extern(e))),
+          Ok((cls, cmd)) => (cls, Ok(cmd)),
+        }
+      })?;
+      self.popped = None;
+      match op {
+        StackOp::Nop => continue,
+        StackOp::Push(ex) => self.push_expr(ex, params)?,
+        StackOp::Swap(ex) => {
+          self.stack.pop().expect("Stack never empty");
+          self.push_expr(ex, params)?
+        },
+        StackOp::Pop => {
+          let ret = self.stack.pop().expect("last_mut called above");
+          if self.stack.is_empty() {
+            if let Some(alt) = self.env.dispatch(&ret.cls, ret.expr.location()) {
+              self.push_expr(alt, params)?;
+              params.use_gas(1);
+              continue;
+            }
+            return Ok(ret.expr);
+          } else {
+            self.popped = Some(ret.expr);
+          }
+        },
+      }
+    }
+  }
+}
+impl<'a> fmt::Display for State<'a> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.stack.iter().rev().join("\n"))
+  }
+}
+impl<'a> fmt::Debug for State<'a> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "State({self})") }
+}
+
+/// Process an expression with specific resource limits
+pub fn run<'a>(
+  base: Expr,
+  env: &'a RunEnv<'a>,
+  params: &mut RunParams,
+) -> Result<Halt, RunError<'a>> {
+  State::new(base, env).run(params)
+}
+
+enum StackOp {
+  Pop,
+  Nop,
+  Swap(Expr),
+  Push(Expr),
+}
+
+fn step(
+  top: Clause,
+  popped: Option<Expr>,
+  location: CodeLocation,
+  env: &RunEnv,
+  params: &mut RunParams,
+) -> Result<(Clause, StackOp), RTErrorObj> {
+  match top {
+    Clause::Bottom(err) => Err(err),
+    Clause::LambdaArg => Ok((Clause::Bottom(UnboundArg(location).pack()), StackOp::Nop)),
+    l @ Clause::Lambda { .. } => Ok((l, StackOp::Pop)),
+    Clause::Identity(other) =>
+      Ok((Clause::Identity(other.clone()), StackOp::Swap(other.into_expr(location)))),
+    Clause::Constant(name) => {
+      let expr = env.load(name, location)?;
+      Ok((Clause::Identity(expr.clsi()), StackOp::Swap(expr.clone())))
+    },
+    Clause::Atom(mut at) => {
+      if let Some(delegate) = at.0.redirect() {
+        match popped {
+          Some(popped) => *delegate = popped,
+          None => {
+            let tmp = delegate.clone();
+            return Ok((Clause::Atom(at), StackOp::Push(tmp)));
+          },
+        }
+      }
+      match at.run(RunData { params, env, location })? {
+        AtomicReturn::Inert(at) => Ok((Clause::Atom(at), StackOp::Pop)),
+        AtomicReturn::Change(gas, c) => {
+          params.use_gas(gas);
+          Ok((c, StackOp::Nop))
+        },
+      }
+    },
+    Clause::Apply { mut f, mut x } => {
+      if x.is_empty() {
+        return Ok((Clause::Identity(f.clsi()), StackOp::Swap(f)));
+      }
+      f = match popped {
+        None => return Ok((Clause::Apply { f: f.clone(), x }, StackOp::Push(f))),
+        Some(ex) => ex,
+      };
+      let val = x.pop_front().expect("Empty args handled above");
+      let f_mut = f.clause.cls_mut();
+      let mut cls = match &*f_mut {
+        Clause::Lambda { args, body } => match args {
+          None => Clause::Identity(body.clsi()),
+          Some(args) => substitute(args, val.clsi(), &body.cls_mut(), &mut || params.use_gas(1)),
+        },
+        Clause::Atom(_) => {
+          mem::drop(f_mut);
+          apply_as_atom(f, val, env, params)?
+        },
+        c => unreachable!("Run should never settle on {c}"),
+      };
+      if !x.is_empty() {
+        cls = Clause::Apply { f: cls.into_expr(location), x };
+      }
+      Ok((cls, StackOp::Nop))
+    },
+  }
+}
+
+#[derive(Clone)]
+pub(crate) struct StackOverflow(Vec<Expr>);
+impl RTError for StackOverflow {}
+impl fmt::Display for StackOverflow {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    writeln!(f, "Stack depth exceeded {}:", self.0.len() - 1)?; // 1 for failed call, 1 for current
+    for item in self.0.iter().rev() {
+      match Stackframe::new(item.clone()) {
+        Some(sf) => writeln!(f, "{sf}")?,
+        None => writeln!(f, "Locked frame at {}", item.location)?,
+      }
+    }
+    Ok(())
+  }
+}
+
+#[derive(Clone)]
+pub(crate) struct UnboundArg(CodeLocation);
+impl RTError for UnboundArg {}
+impl fmt::Display for UnboundArg {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Unbound argument found at {}. This is likely a codegen error", self.0)
+  }
+}
+
+#[derive(Clone)]
+pub(crate) struct CyclicalExpression(Expr);
+impl RTError for CyclicalExpression {}
+impl fmt::Display for CyclicalExpression {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "The expression {} contains itself", self.0)
+  }
+}

orchidlang/src/lib.rs

@@ -0,0 +1,22 @@
+#![warn(missing_docs)]
+#![warn(unit_bindings)]
+#![warn(clippy::unnecessary_wraps)]
+#![doc(html_logo_url = "https://raw.githubusercontent.com/lbfalvy/orchid/master/icon.svg")]
+#![doc(html_favicon_url = "https://raw.githubusercontent.com/lbfalvy/orchid/master/icon.svg")]
+//! Orchid is a lazy, pure scripting language to be embedded in Rust
+//! applications. Check out the repo for examples and other links.
+pub mod error;
+pub mod facade;
+pub mod foreign;
+pub mod gen;
+pub mod intermediate;
+pub mod interpreter;
+pub mod libs;
+pub mod location;
+pub mod name;
+pub mod parse;
+pub mod pipeline;
+pub mod rule;
+pub mod tree;
+pub mod utils;
+pub mod virt_fs;

orchidlang/src/libs/asynch/async.orc

@@ -0,0 +1,7 @@
+import std::panic
+
+export const block_on := \action. \cont. (
+  action cont
+    (\e.panic "unwrapped asynch call")
+    \c.yield
+)

orchidlang/src/libs/asynch/delete_cell.rs

@@ -0,0 +1,13 @@
+use std::sync::{Arc, Mutex};
+
+pub struct DeleteCell<T>(pub Arc<Mutex<Option<T>>>);
+impl<T> DeleteCell<T> {
+  pub fn new(t: T) -> Self { Self(Arc::new(Mutex::new(Some(t)))) }
+  pub fn take(&self) -> Option<T> { self.0.lock().unwrap().take() }
+}
+impl<T: Clone> DeleteCell<T> {
+  pub fn clone_out(&self) -> Option<T> { self.0.lock().unwrap().clone() }
+}
+impl<T> Clone for DeleteCell<T> {
+  fn clone(&self) -> Self { Self(self.0.clone()) }
+}

orchidlang/src/libs/asynch/mod.rs

@@ -0,0 +1,9 @@
+//! An event queue other systems can use to trigger events on the main
+//! interpreter thread. These events are handled when the Orchid code returns
+//! `system::async::yield`, and may cause additional Orchid code to be executed
+//! beyond being general Rust functions.
+//! It also exposes timers.
+
+mod delete_cell;
+pub mod poller;
+pub mod system;

orchidlang/src/libs/asynch/poller.rs

@@ -0,0 +1,151 @@
+//! Abstract implementation of the poller
+
+use std::collections::BinaryHeap;
+use std::mem;
+use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender};
+use std::thread::sleep;
+use std::time::{Duration, Instant};
+
+use super::delete_cell::DeleteCell;
+
+enum TimerKind<TOnce, TRec> {
+  Once(DeleteCell<TOnce>),
+  Recurring { period: Duration, data_cell: DeleteCell<TRec> },
+}
+impl<TOnce, TRec> Clone for TimerKind<TOnce, TRec> {
+  fn clone(&self) -> Self {
+    match self {
+      Self::Once(c) => Self::Once(c.clone()),
+      Self::Recurring { period, data_cell: data } =>
+        Self::Recurring { period: *period, data_cell: data.clone() },
+    }
+  }
+}
+
+/// Indicates a bit of code which is to be executed at a
+/// specific point in time
+///
+/// In order to work with Rust's builtin [BinaryHeap] which is a max heap, the
+/// [Ord] implemenetation of this struct is reversed; it can be intuitively
+/// thought of as ordering by urgency.
+struct Timer<TOnce, TRec> {
+  expires: Instant,
+  kind: TimerKind<TOnce, TRec>,
+}
+impl<TOnce, TRec> Clone for Timer<TOnce, TRec> {
+  fn clone(&self) -> Self { Self { expires: self.expires, kind: self.kind.clone() } }
+}
+impl<TOnce, TRec> Eq for Timer<TOnce, TRec> {}
+impl<TOnce, TRec> PartialEq for Timer<TOnce, TRec> {
+  fn eq(&self, other: &Self) -> bool { self.expires.eq(&other.expires) }
+}
+impl<TOnce, TRec> PartialOrd for Timer<TOnce, TRec> {
+  fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(other.cmp(self)) }
+}
+impl<TOnce, TRec> Ord for Timer<TOnce, TRec> {
+  fn cmp(&self, other: &Self) -> std::cmp::Ordering { other.expires.cmp(&self.expires) }
+}
+
+/// Representation of a scheduled timer
+#[derive(Clone)]
+pub struct TimerHandle<T>(DeleteCell<T>);
+impl<T> TimerHandle<T> {
+  /// Cancel the timer
+  pub fn cancel(self) { mem::drop(self.0.take()) }
+}
+
+/// The abstract event poller implementation used by the standard asynch
+/// subsystem.
+pub struct Poller<TEv, TOnce, TRec: Clone> {
+  timers: BinaryHeap<Timer<TOnce, TRec>>,
+  receiver: Receiver<TEv>,
+}
+
+impl<TEv, TOnce, TRec: Clone> Poller<TEv, TOnce, TRec> {
+  /// Create an event poller and a [Sender] that can produce events on it.
+  pub fn new() -> (Sender<TEv>, Self) {
+    let (sender, receiver) = channel();
+    let this = Self { receiver, timers: BinaryHeap::new() };
+    (sender, this)
+  }
+
+  /// Set a single-fire timer
+  pub fn set_timeout(&mut self, duration: Duration, data: TOnce) -> TimerHandle<TOnce> {
+    let data_cell = DeleteCell::new(data);
+    self
+      .timers
+      .push(Timer { kind: TimerKind::Once(data_cell.clone()), expires: Instant::now() + duration });
+    TimerHandle(data_cell)
+  }
+
+  /// Set a recurring timer
+  pub fn set_interval(&mut self, period: Duration, data: TRec) -> TimerHandle<TRec> {
+    let data_cell = DeleteCell::new(data);
+    self.timers.push(Timer {
+      expires: Instant::now() + period,
+      kind: TimerKind::Recurring { period, data_cell: data_cell.clone() },
+    });
+    TimerHandle(data_cell)
+  }
+
+  /// Process a timer popped from the timers heap of this event loop.
+  /// This function returns [None] if the timer had been cancelled. **This
+  /// behaviour is different from [EventLoop::run] which is returns None if
+  /// the event loop is empty, even though the types are compatible.**
+  fn process_next_timer(
+    &mut self,
+    Timer { expires, kind }: Timer<TOnce, TRec>,
+  ) -> Option<PollEvent<TEv, TOnce, TRec>> {
+    Some(match kind {
+      TimerKind::Once(data) => PollEvent::Once(data.take()?),
+      TimerKind::Recurring { period, data_cell } => {
+        let data = data_cell.clone_out()?;
+        self.timers.push(Timer {
+          expires: expires + period,
+          kind: TimerKind::Recurring { period, data_cell },
+        });
+        PollEvent::Recurring(data)
+      },
+    })
+  }
+
+  /// Block until a message is received or the first timer expires
+  pub fn run(&mut self) -> Option<PollEvent<TEv, TOnce, TRec>> {
+    loop {
+      if let Some(expires) = self.timers.peek().map(|t| t.expires) {
+        return match self.receiver.recv_timeout(expires - Instant::now()) {
+          Ok(t) => Some(PollEvent::Event(t)),
+          Err(e) => {
+            if e == RecvTimeoutError::Disconnected {
+              // The receiver is now inert, but the timer must finish
+              sleep(expires - Instant::now());
+            }
+            // pop and process the timer we've been waiting on
+            let timer = self.timers.pop().expect("checked before wait");
+            let result = self.process_next_timer(timer);
+            // if the timer had been cancelled, repeat
+            if result.is_none() {
+              continue;
+            }
+            result
+          },
+        };
+      } else {
+        return match self.receiver.recv() {
+          Ok(t) => Some(PollEvent::Event(t)),
+          Err(RecvError) => None,
+        };
+      }
+    }
+  }
+}
+
+/// Events produced by [Poller].
+pub enum PollEvent<TEv, TOnce, TRec> {
+  /// An event was sent to the [Sender] associated with the [Poller].
+  Event(TEv),
+  /// A single-fire timer expired
+  Once(TOnce),
+  /// A recurring event fired
+  Recurring(TRec),
+}

orchidlang/src/libs/asynch/system.rs

@@ -0,0 +1,210 @@
+//! Object to pass to [crate::facade::loader::Loader::add_system] to enable the
+//! I/O subsystem. Also many other systems depend on it, these take a mut ref to
+//! register themselves.
+
+use std::any::{type_name, Any, TypeId};
+use std::cell::RefCell;
+use std::collections::VecDeque;
+use std::fmt;
+use std::rc::Rc;
+use std::sync::mpsc::Sender;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use hashbrown::HashMap;
+use ordered_float::NotNan;
+use rust_embed::RustEmbed;
+
+use super::poller::{PollEvent, Poller, TimerHandle};
+use crate::facade::system::{IntoSystem, System};
+use crate::foreign::atom::Atomic;
+use crate::foreign::cps_box::CPSBox;
+use crate::foreign::error::RTError;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::gen::tpl;
+use crate::gen::traits::Gen;
+use crate::gen::tree::{atom_ent, xfn_ent, ConstTree};
+use crate::interpreter::gen_nort::nort_gen;
+use crate::interpreter::handler::HandlerTable;
+use crate::interpreter::nort::Expr;
+use crate::libs::std::number::Numeric;
+use crate::location::{CodeGenInfo, CodeLocation};
+use crate::sym;
+use crate::utils::unwrap_or::unwrap_or;
+use crate::virt_fs::{DeclTree, EmbeddedFS, PrefixFS, VirtFS};
+
+#[derive(Debug, Clone)]
+struct Timer {
+  recurring: bool,
+  delay: NotNan<f64>,
+}
+
+fn set_timer(rec: Inert<bool>, delay: Numeric) -> CPSBox<Timer> {
+  CPSBox::new(2, Timer { recurring: rec.0, delay: delay.as_float() })
+}
+
+#[derive(Clone)]
+struct CancelTimer(Arc<Mutex<dyn Fn() + Send>>);
+impl CancelTimer {
+  pub fn new<T: Send + Clone + 'static>(canceller: TimerHandle<T>) -> Self {
+    Self(Arc::new(Mutex::new(move || canceller.clone().cancel())))
+  }
+  pub fn cancel(&self) { self.0.lock().unwrap()() }
+}
+impl fmt::Debug for CancelTimer {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    f.debug_struct("CancelTimer").finish_non_exhaustive()
+  }
+}
+
+#[derive(Clone, Debug)]
+struct Yield;
+impl InertPayload for Yield {
+  const TYPE_STR: &'static str = "asynch::yield";
+}
+
+/// Error indicating a yield command when all event producers and timers had
+/// exited
+#[derive(Clone)]
+pub struct InfiniteBlock;
+impl RTError for InfiniteBlock {}
+impl fmt::Display for InfiniteBlock {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    static MSG: &str = "User code yielded, but there are no timers or event \
+                        producers to wake it up in the future";
+    write!(f, "{}", MSG)
+  }
+}
+
+/// A thread-safe handle that can be used to send events of any type
+#[derive(Clone)]
+pub struct MessagePort(Sender<Box<dyn Any + Send>>);
+impl MessagePort {
+  /// Send an event. Any type is accepted, handlers are dispatched by type ID
+  pub fn send<T: Send + 'static>(&mut self, message: T) { let _ = self.0.send(Box::new(message)); }
+}
+
+fn gen() -> CodeGenInfo { CodeGenInfo::no_details(sym!(asynch)) }
+
+#[derive(RustEmbed)]
+#[folder = "src/libs/asynch"]
+#[include = "*.orc"]
+struct AsynchEmbed;
+
+fn code() -> DeclTree {
+  DeclTree::ns("system::async", [DeclTree::leaf(
+    PrefixFS::new(EmbeddedFS::new::<AsynchEmbed>(".orc", gen()), "", "async").rc(),
+  )])
+}
+
+type AnyHandler<'a> = Box<dyn FnMut(Box<dyn Any>) -> Vec<Expr> + 'a>;
+
+/// Datastructures the asynch system will eventually be constructed from.
+pub struct AsynchSystem<'a> {
+  poller: Poller<Box<dyn Any + Send>, Expr, Expr>,
+  sender: Sender<Box<dyn Any + Send>>,
+  handlers: HashMap<TypeId, AnyHandler<'a>>,
+}
+
+impl<'a> AsynchSystem<'a> {
+  /// Create a new async event loop that allows registering handlers and taking
+  /// references to the port before it's converted into a [System]
+  #[must_use]
+  pub fn new() -> Self {
+    let (sender, poller) = Poller::new();
+    Self { poller, sender, handlers: HashMap::new() }
+  }
+
+  /// Register a callback to be called on the owning thread when an object of
+  /// the given type is found on the queue. Each type should signify a single
+  /// command so each type should have exactly one handler.
+  ///
+  /// # Panics
+  ///
+  /// if the given type is already handled.
+  pub fn register<T: 'static>(&mut self, mut f: impl FnMut(Box<T>) -> Vec<Expr> + 'a) {
+    let cb = move |a: Box<dyn Any>| f(a.downcast().expect("keyed by TypeId"));
+    let prev = self.handlers.insert(TypeId::of::<T>(), Box::new(cb));
+    assert!(prev.is_none(), "Duplicate handlers for async event {}", type_name::<T>())
+  }
+
+  /// Obtain a message port for sending messages to the main thread. If an
+  /// object is passed to the MessagePort that does not have a handler, the
+  /// main thread panics.
+  #[must_use]
+  pub fn get_port(&self) -> MessagePort { MessagePort(self.sender.clone()) }
+}
+
+impl<'a> Default for AsynchSystem<'a> {
+  fn default() -> Self { Self::new() }
+}
+
+impl<'a> IntoSystem<'a> for AsynchSystem<'a> {
+  fn into_system(self) -> System<'a> {
+    let Self { mut handlers, poller, .. } = self;
+    let mut handler_table = HandlerTable::new();
+    let polly = Rc::new(RefCell::new(poller));
+    handler_table.register({
+      let polly = polly.clone();
+      move |t: &CPSBox<Timer>| {
+        let mut polly = polly.borrow_mut();
+        let (Timer { delay, recurring }, action, cont) = t.unpack2();
+        let duration = Duration::from_secs_f64(**delay);
+        let cancel_timer = match *recurring {
+          true => CancelTimer::new(polly.set_interval(duration, action)),
+          false => CancelTimer::new(polly.set_timeout(duration, action)),
+        };
+        let tpl = tpl::A(tpl::Slot, tpl::V(CPSBox::new(1, cancel_timer)));
+        tpl.template(nort_gen(cont.location()), [cont])
+      }
+    });
+    handler_table.register(move |t: &CPSBox<CancelTimer>| {
+      let (command, cont) = t.unpack1();
+      command.cancel();
+      cont
+    });
+    handler_table.register({
+      let polly = polly.clone();
+      let mut microtasks = VecDeque::new();
+      move |_: &Inert<Yield>| {
+        if let Some(expr) = microtasks.pop_front() {
+          return Ok(expr);
+        }
+        let mut polly = polly.borrow_mut();
+        loop {
+          let next = unwrap_or!(polly.run();
+            return Err(InfiniteBlock.pack())
+          );
+          match next {
+            PollEvent::Once(expr) => return Ok(expr),
+            PollEvent::Recurring(expr) => return Ok(expr),
+            PollEvent::Event(ev) => {
+              let handler = (handlers.get_mut(&ev.as_ref().type_id()))
+                .unwrap_or_else(|| panic!("Unhandled messgae type: {:?}", (*ev).type_id()));
+              let events = handler(ev);
+              // we got new microtasks
+              if !events.is_empty() {
+                microtasks = VecDeque::from(events);
+                // trampoline
+                let loc = CodeLocation::new_gen(CodeGenInfo::no_details(sym!(system::asynch)));
+                return Ok(Inert(Yield).atom_expr(loc));
+              }
+            },
+          }
+        }
+      }
+    });
+    System {
+      name: "system::asynch",
+      lexer_plugins: vec![],
+      line_parsers: vec![],
+      constants: ConstTree::ns("system::async", [ConstTree::tree([
+        xfn_ent("set_timer", [set_timer]),
+        atom_ent("yield", [Inert(Yield)]),
+      ])]),
+      code: code(),
+      prelude: Vec::new(),
+      handlers: handler_table,
+    }
+  }
+}

orchidlang/src/libs/directfs/commands.rs

@@ -0,0 +1,191 @@
+use std::ffi::OsString;
+use std::fs::File;
+use std::path::{Path, PathBuf};
+
+use super::osstring::os_string_lib;
+use crate::facade::system::{IntoSystem, System};
+use crate::foreign::atom::Atomic;
+use crate::foreign::cps_box::CPSBox;
+use crate::foreign::error::RTResult;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::foreign::process::Unstable;
+use crate::foreign::to_clause::ToClause;
+use crate::gen::tpl;
+use crate::gen::traits::Gen;
+use crate::gen::tree::{atom_ent, xfn_ent, ConstTree};
+use crate::interpreter::gen_nort::nort_gen;
+use crate::interpreter::handler::HandlerTable;
+use crate::interpreter::nort::{Clause, Expr};
+use crate::libs::io::instances::io_error_handler;
+use crate::libs::io::{Sink, Source};
+use crate::libs::scheduler::system::{SeqScheduler, SharedHandle};
+use crate::libs::std::runtime_error::RuntimeError;
+use crate::utils::combine::Combine;
+use crate::virt_fs::DeclTree;
+
+#[derive(Debug, Clone)]
+struct ReadFileCmd(OsString);
+impl InertPayload for ReadFileCmd {
+  const TYPE_STR: &'static str = "readfile command";
+}
+
+#[derive(Debug, Clone)]
+struct ReadDirCmd(OsString);
+impl InertPayload for ReadDirCmd {
+  const TYPE_STR: &'static str = "readdir command";
+}
+
+#[derive(Debug, Clone)]
+struct WriteFile {
+  name: OsString,
+  append: bool,
+}
+impl InertPayload for WriteFile {
+  const TYPE_STR: &'static str = "writefile command";
+}
+
+#[must_use]
+fn read_file(sched: &SeqScheduler, cmd: &CPSBox<ReadFileCmd>) -> Expr {
+  let (ReadFileCmd(name), succ, fail, cont) = cmd.unpack3();
+  let name = name.clone();
+  let cancel = sched.run_orphan(
+    move |_| File::open(name),
+    |file, _| match file {
+      Err(e) => vec![io_error_handler(e, fail)],
+      Ok(f) => {
+        let source_handle = SharedHandle::wrap(Source::new(Box::new(f)));
+        let tpl = tpl::A(tpl::Slot, tpl::V(Inert(source_handle)));
+        vec![tpl.template(nort_gen(succ.location()), [succ])]
+      },
+    },
+  );
+  let tpl = tpl::A(tpl::Slot, tpl::V(CPSBox::new(1, cancel)));
+  tpl.template(nort_gen(cont.location()), [cont])
+}
+
+#[must_use]
+fn read_dir(sched: &SeqScheduler, cmd: &CPSBox<ReadDirCmd>) -> Expr {
+  let (ReadDirCmd(name), succ, fail, cont) = cmd.unpack3();
+  let name = name.clone();
+  let cancel = sched.run_orphan(
+    move |_| {
+      Path::new(&name)
+        .read_dir()?
+        .map(|r| r.and_then(|e| Ok((e.file_name(), e.file_type()?.is_dir()))))
+        .collect()
+    },
+    |items: std::io::Result<Vec<(OsString, bool)>>, _| match items {
+      Err(e) => vec![io_error_handler(e, fail)],
+      Ok(os_namev) => {
+        let converted = (os_namev.into_iter())
+          .map(|(n, d)| {
+            Ok((Inert(n).atom_expr(succ.location()), Inert(d).atom_expr(succ.location())))
+          })
+          .collect::<Result<Vec<_>, Clause>>();
+        match converted {
+          Err(e) => {
+            let e = e.into_expr(fail.location());
+            let tpl = tpl::A(tpl::Slot, tpl::Slot);
+            vec![tpl.template(nort_gen(fail.location()), [fail, e])]
+          },
+          Ok(names) => {
+            let names = names.to_expr(succ.location());
+            let tpl = tpl::A(tpl::Slot, tpl::Slot);
+            vec![tpl.template(nort_gen(succ.location()), [succ, names])]
+          },
+        }
+      },
+    },
+  );
+  let tpl = tpl::A(tpl::Slot, tpl::V(CPSBox::new(1, cancel)));
+  tpl.template(nort_gen(cont.location()), [cont])
+}
+
+#[must_use]
+fn write_file(sched: &SeqScheduler, cmd: &CPSBox<WriteFile>) -> Expr {
+  let (cmd, succ, fail, cont) = cmd.unpack3();
+  let cmd = cmd.clone();
+  let cancel = sched.run_orphan(
+    move |_| File::options().write(true).append(cmd.append).open(&cmd.name),
+    |file, _| match file {
+      Err(e) => vec![io_error_handler(e, fail)],
+      Ok(f) => {
+        let sink_handle = SharedHandle::wrap(Box::new(f) as Sink);
+        let tpl = tpl::A(tpl::Slot, tpl::V(Inert(sink_handle)));
+        vec![tpl.template(nort_gen(succ.location()), [succ])]
+      },
+    },
+  );
+  let tpl = tpl::A(tpl::Slot, tpl::V(CPSBox::new(1, cancel)));
+  tpl.template(nort_gen(cont.location()), [cont])
+}
+
+fn open_file_read_cmd(name: OsString) -> CPSBox<ReadFileCmd> { CPSBox::new(3, ReadFileCmd(name)) }
+
+fn read_dir_cmd(name: OsString) -> CPSBox<ReadDirCmd> { CPSBox::new(3, ReadDirCmd(name)) }
+
+fn open_file_write_cmd(name: OsString) -> CPSBox<WriteFile> {
+  CPSBox::new(3, WriteFile { name, append: false })
+}
+
+fn open_file_append_cmd(name: OsString) -> CPSBox<WriteFile> {
+  CPSBox::new(3, WriteFile { name, append: true })
+}
+
+fn join_paths(root: OsString, sub: OsString) -> OsString {
+  let mut path = PathBuf::from(root);
+  path.push(sub);
+  path.into_os_string()
+}
+
+fn pop_path(path: Inert<OsString>) -> Option<(Inert<OsString>, Inert<OsString>)> {
+  let mut path = PathBuf::from(path.0);
+  let sub = path.file_name()?.to_owned();
+  assert!(path.pop(), "file_name above returned Some");
+  Some((Inert(path.into_os_string()), Inert(sub)))
+}
+
+/// A rudimentary system to read and write files.
+#[derive(Clone)]
+pub struct DirectFS {
+  scheduler: SeqScheduler,
+}
+impl DirectFS {
+  /// Create a new instance of the system.
+  pub fn new(scheduler: SeqScheduler) -> Self { Self { scheduler } }
+}
+
+impl IntoSystem<'static> for DirectFS {
+  fn into_system(self) -> System<'static> {
+    let mut handlers = HandlerTable::new();
+    let sched = self.scheduler.clone();
+    handlers.register(move |cmd| read_file(&sched, cmd));
+    let sched = self.scheduler.clone();
+    handlers.register(move |cmd| read_dir(&sched, cmd));
+    let sched = self.scheduler;
+    handlers.register(move |cmd| write_file(&sched, cmd));
+    System {
+      name: "system::directfs",
+      code: DeclTree::empty(),
+      prelude: Vec::new(),
+      lexer_plugins: vec![],
+      line_parsers: vec![],
+      constants: ConstTree::ns("system::fs", [ConstTree::tree([
+        xfn_ent("read_file", [open_file_read_cmd]),
+        xfn_ent("read_dir", [read_dir_cmd]),
+        xfn_ent("write_file", [open_file_write_cmd]),
+        xfn_ent("append_file", [open_file_append_cmd]),
+        xfn_ent("join_paths", [join_paths]),
+        xfn_ent("pop_path", [pop_path]),
+        atom_ent("cwd", [Unstable::new(|_| -> RTResult<_> {
+          let path =
+            std::env::current_dir().map_err(|e| RuntimeError::ext(e.to_string(), "reading CWD"))?;
+          Ok(Inert(path.into_os_string()))
+        })]),
+      ])])
+      .combine(os_string_lib())
+      .expect("os_string library and directfs conflict"),
+      handlers,
+    }
+  }
+}

orchidlang/src/libs/directfs/mod.rs

@@ -0,0 +1,9 @@
+//! A rudimentary system exposing methods for Orchid to interact with the file
+//! system. All paths are strings.
+//!
+//! The system depends on [crate::libs::scheduler] for scheduling blocking I/O
+//! on a separate thread.
+mod commands;
+mod osstring;
+
+pub use commands::DirectFS;

orchidlang/src/libs/directfs/osstring.rs

@@ -0,0 +1,42 @@
+use std::ffi::OsString;
+
+use crate::foreign::atom::Atomic;
+use crate::foreign::error::RTResult;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::foreign::to_clause::ToClause;
+use crate::foreign::try_from_expr::TryFromExpr;
+use crate::gen::tree::{xfn_ent, ConstTree};
+use crate::interpreter::nort::{Clause, Expr};
+use crate::libs::std::string::OrcString;
+use crate::location::CodeLocation;
+
+impl InertPayload for OsString {
+  const TYPE_STR: &'static str = "OsString";
+  fn respond(&self, mut request: crate::utils::ddispatch::Request) {
+    request.serve_with(|| OrcString::from(self.to_string_lossy().to_string()))
+  }
+}
+impl TryFromExpr for OsString {
+  fn from_expr(exi: Expr) -> RTResult<Self> { Ok(Inert::from_expr(exi)?.0) }
+}
+impl ToClause for OsString {
+  fn to_clause(self, _: CodeLocation) -> Clause { Inert(self).atom_cls() }
+}
+
+pub fn os_to_string(os: Inert<OsString>) -> Result<Inert<OrcString>, Inert<OsString>> {
+  os.0.into_string().map(|s| Inert(s.into())).map_err(Inert)
+}
+
+pub fn string_to_os(str: Inert<OrcString>) -> Inert<OsString> { Inert(str.0.get_string().into()) }
+
+pub fn os_print(os: Inert<OsString>) -> Inert<OrcString> {
+  Inert(os.0.to_string_lossy().to_string().into())
+}
+
+pub fn os_string_lib() -> ConstTree {
+  ConstTree::ns("system::fs", [ConstTree::tree([
+    xfn_ent("os_to_string", [os_to_string]),
+    xfn_ent("string_to_os", [string_to_os]),
+    xfn_ent("os_print", [os_print]),
+  ])])
+}

orchidlang/src/libs/io/bindings.rs

@@ -0,0 +1,71 @@
+use super::flow::IOCmdHandlePack;
+use super::instances::{BRead, ReadCmd, SRead, WriteCmd};
+use super::service::{Sink, Source};
+use crate::foreign::cps_box::CPSBox;
+use crate::foreign::error::RTResult;
+use crate::foreign::inert::Inert;
+use crate::gen::tree::{xfn_ent, ConstTree};
+use crate::libs::scheduler::system::SharedHandle;
+use crate::libs::std::binary::Binary;
+use crate::libs::std::runtime_error::RuntimeError;
+use crate::libs::std::string::OrcString;
+use crate::utils::combine::Combine;
+
+pub type WriteHandle = Inert<SharedHandle<Sink>>;
+pub type ReadHandle = Inert<SharedHandle<Source>>;
+
+type ReadCmdPack = CPSBox<IOCmdHandlePack<ReadCmd>>;
+type WriteCmdPack = CPSBox<IOCmdHandlePack<WriteCmd>>;
+
+pub fn read_string(Inert(handle): ReadHandle) -> ReadCmdPack {
+  let cmd = ReadCmd::RStr(SRead::All);
+  CPSBox::new(3, IOCmdHandlePack { handle, cmd })
+}
+pub fn read_line(Inert(handle): ReadHandle) -> ReadCmdPack {
+  let cmd = ReadCmd::RStr(SRead::Line);
+  CPSBox::new(3, IOCmdHandlePack { handle, cmd })
+}
+pub fn read_bin(Inert(handle): ReadHandle) -> ReadCmdPack {
+  let cmd = ReadCmd::RBytes(BRead::All);
+  CPSBox::new(3, IOCmdHandlePack { handle, cmd })
+}
+pub fn read_bytes(Inert(handle): ReadHandle, n: Inert<usize>) -> ReadCmdPack {
+  let cmd = ReadCmd::RBytes(BRead::N(n.0));
+  CPSBox::new(3, IOCmdHandlePack { cmd, handle })
+}
+pub fn read_until(
+  Inert(handle): ReadHandle,
+  Inert(pattern): Inert<usize>,
+) -> RTResult<ReadCmdPack> {
+  let pattern = pattern.try_into().map_err(|_| {
+    let msg = format!("{pattern} doesn't fit into a byte");
+    RuntimeError::ext(msg, "converting number to byte")
+  })?;
+  let cmd = ReadCmd::RBytes(BRead::Until(pattern));
+  Ok(CPSBox::new(3, IOCmdHandlePack { handle, cmd }))
+}
+pub fn write_str(Inert(handle): WriteHandle, string: Inert<OrcString>) -> WriteCmdPack {
+  let cmd = WriteCmd::WStr(string.0.get_string());
+  CPSBox::new(3, IOCmdHandlePack { handle, cmd })
+}
+pub fn write_bin(Inert(handle): WriteHandle, bytes: Inert<Binary>) -> WriteCmdPack {
+  CPSBox::new(3, IOCmdHandlePack { handle, cmd: WriteCmd::WBytes(bytes.0) })
+}
+pub fn flush(Inert(handle): WriteHandle) -> WriteCmdPack {
+  CPSBox::new(3, IOCmdHandlePack { handle, cmd: WriteCmd::Flush })
+}
+
+pub fn io_bindings<'a>(std_streams: impl IntoIterator<Item = (&'a str, ConstTree)>) -> ConstTree {
+  ConstTree::ns("system::io", [ConstTree::tree([
+    xfn_ent("read_string", [read_string]),
+    xfn_ent("read_line", [read_line]),
+    xfn_ent("read_bin", [read_bin]),
+    xfn_ent("read_n_bytes", [read_bytes]),
+    xfn_ent("read_until", [read_until]),
+    xfn_ent("write_str", [write_str]),
+    xfn_ent("write_bin", [write_bin]),
+    xfn_ent("flush", [flush]),
+  ])
+  .combine(ConstTree::tree(std_streams))
+  .expect("std_stream name clashing with io functions")])
+}

orchidlang/src/libs/io/flow.rs

@@ -0,0 +1,41 @@
+use std::fmt;
+
+use crate::foreign::error::RTError;
+use crate::libs::scheduler::cancel_flag::CancelFlag;
+
+pub trait IOHandler<T> {
+  type Product;
+
+  fn handle(self, result: T) -> Self::Product;
+  fn early_cancel(self) -> Self::Product;
+}
+
+pub trait IOResult: Send {
+  type Handler;
+  type HandlerProduct;
+
+  fn handle(self, handler: Self::Handler) -> Self::HandlerProduct;
+}
+
+pub trait IOCmd: Send {
+  type Stream: Send;
+  type Result: Send;
+  type Handle;
+
+  fn execute(self, stream: &mut Self::Stream, cancel: CancelFlag) -> Self::Result;
+}
+
+#[derive(Debug, Clone)]
+pub struct IOCmdHandlePack<Cmd: IOCmd> {
+  pub cmd: Cmd,
+  pub handle: Cmd::Handle,
+}
+
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+pub struct NoActiveStream(usize);
+impl RTError for NoActiveStream {}
+impl fmt::Display for NoActiveStream {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "The stream {} had already been closed", self.0)
+  }
+}

orchidlang/src/libs/io/instances.rs

@@ -0,0 +1,131 @@
+use std::io::{self, BufRead, Read, Write};
+use std::sync::Arc;
+
+use super::flow::IOCmd;
+use super::service::{Sink, Source};
+use crate::foreign::inert::Inert;
+use crate::gen::tpl;
+use crate::gen::traits::Gen;
+use crate::interpreter::gen_nort::nort_gen;
+use crate::interpreter::nort::Expr;
+use crate::libs::scheduler::cancel_flag::CancelFlag;
+use crate::libs::scheduler::system::SharedHandle;
+use crate::libs::std::binary::Binary;
+use crate::libs::std::string::OrcString;
+use crate::location::{CodeGenInfo, CodeLocation};
+use crate::sym;
+
+/// String reading command
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub(super) enum SRead {
+  All,
+  Line,
+}
+
+/// Binary reading command
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub(super) enum BRead {
+  All,
+  N(usize),
+  Until(u8),
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub(super) enum ReadCmd {
+  RBytes(BRead),
+  RStr(SRead),
+}
+
+impl IOCmd for ReadCmd {
+  type Stream = Source;
+  type Result = ReadResult;
+  type Handle = SharedHandle<Source>;
+
+  // This is a buggy rule, check manually
+  #[allow(clippy::read_zero_byte_vec)]
+  fn execute(self, stream: &mut Self::Stream, _cancel: CancelFlag) -> Self::Result {
+    match self {
+      Self::RBytes(bread) => {
+        let mut buf = Vec::new();
+        let result = match &bread {
+          BRead::All => stream.read_to_end(&mut buf).map(|_| ()),
+          BRead::Until(b) => stream.read_until(*b, &mut buf).map(|_| ()),
+          BRead::N(n) => {
+            buf.resize(*n, 0);
+            stream.read_exact(&mut buf)
+          },
+        };
+        ReadResult::RBin(bread, result.map(|_| buf))
+      },
+      Self::RStr(sread) => {
+        let mut buf = String::new();
+        let sresult = match &sread {
+          SRead::All => stream.read_to_string(&mut buf).map(|_| ()),
+          SRead::Line => stream.read_line(&mut buf).map(|_| {
+            if buf.ends_with('\n') {
+              buf.pop();
+            }
+          }),
+        };
+        ReadResult::RStr(sread, sresult.map(|()| buf))
+      },
+    }
+  }
+}
+
+/// Reading command (string or binary)
+pub(super) enum ReadResult {
+  RStr(SRead, io::Result<String>),
+  RBin(BRead, io::Result<Vec<u8>>),
+}
+impl ReadResult {
+  pub fn dispatch(self, succ: Expr, fail: Expr) -> Vec<Expr> {
+    vec![match self {
+      ReadResult::RBin(_, Err(e)) | ReadResult::RStr(_, Err(e)) => io_error_handler(e, fail),
+      ReadResult::RBin(_, Ok(bytes)) => tpl::A(tpl::Slot, tpl::V(Inert(Binary(Arc::new(bytes)))))
+        .template(nort_gen(succ.location()), [succ]),
+      ReadResult::RStr(_, Ok(text)) => tpl::A(tpl::Slot, tpl::V(Inert(OrcString::from(text))))
+        .template(nort_gen(succ.location()), [succ]),
+    }]
+  }
+}
+
+/// Function to convert [io::Error] to Orchid data
+pub(crate) fn io_error_handler(_e: io::Error, handler: Expr) -> Expr {
+  let ctx = nort_gen(CodeLocation::new_gen(CodeGenInfo::no_details(sym!(system::io::io_error))));
+  tpl::A(tpl::Slot, tpl::V(Inert(0usize))).template(ctx, [handler])
+}
+
+/// Writing command (string or binary)
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub(super) enum WriteCmd {
+  WBytes(Binary),
+  WStr(String),
+  Flush,
+}
+
+impl IOCmd for WriteCmd {
+  type Stream = Sink;
+  type Handle = SharedHandle<Sink>;
+  type Result = WriteResult;
+
+  fn execute(self, stream: &mut Self::Stream, _cancel: CancelFlag) -> Self::Result {
+    let result = match &self {
+      Self::Flush => stream.flush(),
+      Self::WStr(str) => write!(stream, "{}", str).map(|_| ()),
+      Self::WBytes(bytes) => stream.write_all(bytes.0.as_ref()).map(|_| ()),
+    };
+    WriteResult { result, cmd: self }
+  }
+}
+
+pub(super) struct WriteResult {
+  #[allow(unused)]
+  pub cmd: WriteCmd,
+  pub result: io::Result<()>,
+}
+impl WriteResult {
+  pub fn dispatch(self, succ: Expr, fail: Expr) -> Vec<Expr> {
+    vec![self.result.map_or_else(|e| io_error_handler(e, fail), |()| succ)]
+  }
+}

orchidlang/src/libs/io/io.orc

@@ -0,0 +1,35 @@
+import std::panic
+import system::io
+import system::async::yield
+
+export const print := \text. \ok. (
+  io::write_str io::stdout text
+    (io::flush io::stdout
+      ok
+      (\e. panic "println threw on flush")
+      \_. yield
+    )
+    (\e. panic "print threw on write")
+    \_. yield
+)
+
+export const println := \line. \ok. (
+  print (line ++ "\n") ok
+)
+
+export const readln := \ok. (
+  io::read_line io::stdin
+    ok
+    (\e. panic "readln threw")
+    \_. yield
+)
+
+export const prompt := \line. \ok. (
+  print line (readln ok)
+)
+
+export module prelude (
+  import super::*
+
+  export ::(print, println, readln, prompt)
+)

orchidlang/src/libs/io/mod.rs

@@ -0,0 +1,38 @@
+//! System that allows Orchid to interact with trait objects of Rust's `Writer`
+//! and with `BufReader`s of `Reader` trait objects.
+//!
+//! You can pass standard streams during initialization, the stllib expects
+//! `stdin`, `stdout` and `stderr`. This system depends on
+//! [crate::libs::scheduler] to run blocking I/O operations off-thread, which in
+//! turn depends on [crate::libs::asynch] to process results on the main thread,
+//! and [crate::libs::std] for `std::panic`.
+//!
+//! ```
+//! use std::io::BufReader;
+//!
+//! use orchidlang::facade::loader::Loader;
+//! use orchidlang::libs::asynch::system::AsynchSystem;
+//! use orchidlang::libs::io::{IOService, Stream};
+//! use orchidlang::libs::scheduler::system::SeqScheduler;
+//! use orchidlang::libs::std::std_system::StdConfig;
+//!
+//! let mut asynch = AsynchSystem::new();
+//! let scheduler = SeqScheduler::new(&mut asynch);
+//! let std_streams = [
+//!   ("stdin", Stream::Source(BufReader::new(Box::new(std::io::stdin())))),
+//!   ("stdout", Stream::Sink(Box::new(std::io::stdout()))),
+//!   ("stderr", Stream::Sink(Box::new(std::io::stderr()))),
+//! ];
+//! let env = Loader::new()
+//!   .add_system(StdConfig { impure: false })
+//!   .add_system(asynch)
+//!   .add_system(scheduler.clone())
+//!   .add_system(IOService::new(scheduler.clone(), std_streams));
+//! ```
+
+mod bindings;
+mod flow;
+pub(super) mod instances;
+mod service;
+
+pub use service::{IOService, Sink, Source, Stream};

orchidlang/src/libs/io/service.rs

@@ -0,0 +1,133 @@
+//! Object to pass to [crate::facade::loader::Loader::add_system] to enable the
+//! I/O subsystem
+
+use std::io::{BufReader, Read, Write};
+
+use rust_embed::RustEmbed;
+use trait_set::trait_set;
+
+use super::bindings::io_bindings;
+use super::flow::{IOCmd, IOCmdHandlePack};
+use super::instances::{ReadCmd, WriteCmd};
+use crate::facade::system::{IntoSystem, System};
+use crate::foreign::cps_box::CPSBox;
+use crate::foreign::inert::Inert;
+use crate::gen::tpl;
+use crate::gen::traits::Gen;
+use crate::gen::tree::leaf;
+use crate::interpreter::gen_nort::nort_gen;
+use crate::interpreter::handler::HandlerTable;
+use crate::libs::scheduler::system::{SeqScheduler, SharedHandle};
+use crate::location::CodeGenInfo;
+use crate::pipeline::load_project::Prelude;
+use crate::virt_fs::{DeclTree, EmbeddedFS, PrefixFS, VirtFS};
+use crate::{sym, vname};
+
+/// Any type that we can read controlled amounts of data from
+pub type Source = BufReader<Box<dyn Read + Send>>;
+/// Any type that we can write data to
+pub type Sink = Box<dyn Write + Send>;
+
+/// A shared type for sinks and sources
+pub enum Stream {
+  /// A Source, aka. a BufReader
+  Source(Source),
+  /// A Sink, aka. a Writer
+  Sink(Sink),
+}
+
+trait_set! {
+  /// The table of default streams to be overlain on the I/O module, typicially
+  /// stdin, stdout, stderr.
+  pub(super) trait StreamTable<'a> = IntoIterator<Item = (&'a str, Stream)>
+}
+
+fn gen() -> CodeGenInfo { CodeGenInfo::no_details(sym!(system::io)) }
+
+#[derive(RustEmbed)]
+#[folder = "src/libs/io"]
+#[include = "*.orc"]
+struct IOEmbed;
+
+fn code() -> DeclTree {
+  DeclTree::ns("system::io", [DeclTree::leaf(
+    PrefixFS::new(EmbeddedFS::new::<IOEmbed>(".orc", gen()), "", "io").rc(),
+  )])
+}
+
+/// A streaming I/O service for interacting with Rust's [std::io::Write] and
+/// [std::io::Read] traits.
+pub struct IOService<'a, ST: IntoIterator<Item = (&'a str, Stream)>> {
+  scheduler: SeqScheduler,
+  global_streams: ST,
+}
+impl<'a, ST: IntoIterator<Item = (&'a str, Stream)>> IOService<'a, ST> {
+  /// Construct a new instance of the service
+  pub fn new(scheduler: SeqScheduler, global_streams: ST) -> Self {
+    Self { scheduler, global_streams }
+  }
+}
+
+impl<'a, ST: IntoIterator<Item = (&'a str, Stream)>> IntoSystem<'static> for IOService<'a, ST> {
+  fn into_system(self) -> System<'static> {
+    let scheduler = self.scheduler.clone();
+    let mut handlers = HandlerTable::new();
+    handlers.register(move |cps: &CPSBox<IOCmdHandlePack<ReadCmd>>| {
+      let (IOCmdHandlePack { cmd, handle }, succ, fail, cont) = cps.unpack3();
+      let (cmd, fail1) = (*cmd, fail.clone());
+      let result = scheduler.schedule(
+        handle.clone(),
+        move |mut stream, cancel| {
+          let ret = cmd.execute(&mut stream, cancel);
+          (stream, ret)
+        },
+        move |stream, res, _cancel| (stream, res.dispatch(succ, fail1)),
+        |stream| (stream, Vec::new()),
+      );
+      match result {
+        Ok(cancel) => tpl::A(tpl::Slot, tpl::V(CPSBox::new(1, cancel)))
+          .template(nort_gen(cont.location()), [cont]),
+        Err(e) => tpl::A(tpl::Slot, tpl::V(Inert(e))).template(nort_gen(fail.location()), [fail]),
+      }
+    });
+    let scheduler = self.scheduler.clone();
+    handlers.register(move |cps: &CPSBox<IOCmdHandlePack<WriteCmd>>| {
+      let (IOCmdHandlePack { cmd, handle }, succ, fail, cont) = cps.unpack3();
+      let (succ1, fail1, cmd) = (succ, fail.clone(), cmd.clone());
+      let result = scheduler.schedule(
+        handle.clone(),
+        move |mut stream, cancel| {
+          let ret = cmd.execute(&mut stream, cancel);
+          (stream, ret)
+        },
+        move |stream, res, _cancel| (stream, res.dispatch(succ1, fail1)),
+        |stream| (stream, Vec::new()),
+      );
+      match result {
+        Ok(cancel) => tpl::A(tpl::Slot, tpl::V(CPSBox::new(1, cancel)))
+          .template(nort_gen(cont.location()), [cont]),
+        Err(e) => tpl::A(tpl::Slot, tpl::V(Inert(e))).template(nort_gen(fail.location()), [fail]),
+      }
+    });
+    let streams = self.global_streams.into_iter().map(|(n, stream)| {
+      let handle = match stream {
+        Stream::Sink(sink) => leaf(tpl::V(Inert(SharedHandle::wrap(sink)))),
+        Stream::Source(source) => leaf(tpl::V(Inert(SharedHandle::wrap(source)))),
+      };
+      (n, handle)
+    });
+    System {
+      handlers,
+      name: "system::io",
+      constants: io_bindings(streams),
+      code: code(),
+      prelude: vec![Prelude {
+        target: vname!(system::io::prelude),
+        exclude: vname!(system::io),
+        owner: gen(),
+      }],
+      lexer_plugins: vec![],
+      line_parsers: vec![],
+    }
+  }
+}

orchidlang/src/libs/mod.rs

@@ -0,0 +1,7 @@
+//! Constants exposed to usercode by the interpreter
+pub mod asynch;
+pub mod directfs;
+pub mod io;
+pub mod parse_custom_line;
+pub mod scheduler;
+pub mod std;

orchidlang/src/libs/parse_custom_line.rs

@@ -0,0 +1,47 @@
+//! A helper for defining custom lines. See [custom_line]
+use intern_all::Tok;
+
+use crate::error::ProjectResult;
+use crate::location::SourceRange;
+use crate::parse::errors::ParseErrorKind;
+use crate::parse::frag::Frag;
+use crate::parse::lexer::Lexeme;
+use crate::parse::parse_plugin::ParsePluginReq;
+
+/// An exported line with a name for which the line parser denies exports
+pub struct Unexportable(Lexeme);
+impl ParseErrorKind for Unexportable {
+  const DESCRIPTION: &'static str = "this line type cannot be exported";
+  fn message(&self) -> String { format!("{} cannot be exported", &self.0) }
+}
+
+/// Parse a line identified by the specified leading keyword. Although not
+/// required, plugins are encouraged to prefix their lines with a globally
+/// unique keyword which makes or breaks their parsing, to avoid accidental
+/// failure to recognize
+pub fn custom_line<'a>(
+  tail: Frag<'a>,
+  keyword: Tok<String>,
+  exportable: bool,
+  req: &dyn ParsePluginReq,
+) -> Option<ProjectResult<(bool, Frag<'a>, SourceRange)>> {
+  let line_loc = req.frag_loc(tail);
+  let (fst, tail) = req.pop(tail).ok()?;
+  let fst_name = req.expect_name(fst).ok()?;
+  let (exported, n_ent, tail) = if fst_name == keyword {
+    (false, fst, tail.trim())
+  } else if fst_name.as_str() == "export" {
+    let (snd, tail) = req.pop(tail).ok()?;
+    req.expect(Lexeme::Name(keyword), snd).ok()?;
+    (true, snd, tail.trim())
+  } else {
+    return None;
+  };
+  Some(match exported && !exportable {
+    true => {
+      let err = Unexportable(n_ent.lexeme.clone());
+      Err(err.pack(req.range_loc(n_ent.range.clone())))
+    },
+    false => Ok((exported, tail, line_loc)),
+  })
+}

orchidlang/src/libs/scheduler/busy.rs

@@ -0,0 +1,122 @@
+use std::any::Any;
+use std::collections::VecDeque;
+
+use super::cancel_flag::CancelFlag;
+use crate::interpreter::nort::Expr;
+
+pub type SyncResult<T> = (T, Box<dyn Any + Send>);
+/// Output from handlers contains the resource being processed and any Orchid
+/// handlers executed as a result of the operation
+pub type HandlerRes<T> = (T, Vec<Expr>);
+pub type SyncOperation<T> = Box<dyn FnOnce(T, CancelFlag) -> SyncResult<T> + Send>;
+pub type SyncOpResultHandler<T> =
+  Box<dyn FnOnce(T, Box<dyn Any + Send>, CancelFlag) -> (T, Vec<Expr>) + Send>;
+
+struct SyncQueueItem<T> {
+  cancelled: CancelFlag,
+  operation: SyncOperation<T>,
+  handler: SyncOpResultHandler<T>,
+  early_cancel: Box<dyn FnOnce(T) -> (T, Vec<Expr>) + Send>,
+}
+
+pub enum NextItemReportKind<T> {
+  Free(T),
+  Next { instance: T, cancelled: CancelFlag, operation: SyncOperation<T>, rest: BusyState<T> },
+  Taken,
+}
+
+pub struct NextItemReport<T> {
+  pub kind: NextItemReportKind<T>,
+  pub events: Vec<Expr>,
+}
+
+pub(super) struct BusyState<T> {
+  handler: SyncOpResultHandler<T>,
+  queue: VecDeque<SyncQueueItem<T>>,
+  seal: Option<Box<dyn FnOnce(T) -> Vec<Expr> + Send>>,
+}
+impl<T> BusyState<T> {
+  pub fn new<U: 'static + Send>(
+    handler: impl FnOnce(T, U, CancelFlag) -> HandlerRes<T> + Send + 'static,
+  ) -> Self {
+    BusyState {
+      handler: Box::new(|t, payload, cancel| {
+        let u = *payload.downcast().expect("mismatched initial handler and operation");
+        handler(t, u, cancel)
+      }),
+      queue: VecDeque::new(),
+      seal: None,
+    }
+  }
+
+  /// Add a new operation to the queue. Returns Some if the operation was
+  /// successfully enqueued and None if the queue is already sealed.
+  pub fn enqueue<U: 'static + Send>(
+    &mut self,
+    operation: impl FnOnce(T, CancelFlag) -> (T, U) + Send + 'static,
+    handler: impl FnOnce(T, U, CancelFlag) -> HandlerRes<T> + Send + 'static,
+    early_cancel: impl FnOnce(T) -> HandlerRes<T> + Send + 'static,
+  ) -> Option<CancelFlag> {
+    if self.seal.is_some() {
+      return None;
+    }
+    let cancelled = CancelFlag::new();
+    self.queue.push_back(SyncQueueItem {
+      cancelled: cancelled.clone(),
+      early_cancel: Box::new(early_cancel),
+      operation: Box::new(|t, c| {
+        let (t, r) = operation(t, c);
+        (t, Box::new(r))
+      }),
+      handler: Box::new(|t, u, c| {
+        let u: Box<U> = u.downcast().expect("mismatched handler and operation");
+        handler(t, *u, c)
+      }),
+    });
+    Some(cancelled)
+  }
+
+  pub fn seal(&mut self, recipient: impl FnOnce(T) -> Vec<Expr> + Send + 'static) {
+    assert!(self.seal.is_none(), "Already sealed");
+    self.seal = Some(Box::new(recipient))
+  }
+
+  pub fn is_sealed(&self) -> bool { self.seal.is_some() }
+
+  pub fn rotate(
+    mut self,
+    instance: T,
+    result: Box<dyn Any + Send>,
+    cancelled: CancelFlag,
+  ) -> NextItemReport<T> {
+    let (mut instance, mut events) = (self.handler)(instance, result, cancelled);
+    let next_item = loop {
+      if let Some(candidate) = self.queue.pop_front() {
+        if candidate.cancelled.is_cancelled() {
+          let ret = (candidate.early_cancel)(instance);
+          instance = ret.0;
+          events.extend(ret.1);
+        } else {
+          break candidate;
+        }
+      } else if let Some(seal) = self.seal.take() {
+        seal(instance);
+        let kind = NextItemReportKind::Taken;
+        return NextItemReport { events, kind };
+      } else {
+        let kind = NextItemReportKind::Free(instance);
+        return NextItemReport { events, kind };
+      }
+    };
+    self.handler = next_item.handler;
+    NextItemReport {
+      events,
+      kind: NextItemReportKind::Next {
+        instance,
+        cancelled: next_item.cancelled,
+        operation: next_item.operation,
+        rest: self,
+      },
+    }
+  }
+}

orchidlang/src/libs/scheduler/cancel_flag.rs

@@ -0,0 +1,23 @@
+//! Flag for cancelling scheduled operations
+
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+/// A single-fire thread-safe boolean flag with relaxed ordering
+#[derive(Debug, Clone)]
+pub struct CancelFlag(Arc<AtomicBool>);
+
+impl CancelFlag {
+  /// Create a new canceller
+  pub fn new() -> Self { CancelFlag(Arc::new(AtomicBool::new(false))) }
+
+  /// Check whether the operation has been cancelled
+  pub fn is_cancelled(&self) -> bool { self.0.load(Ordering::Relaxed) }
+
+  /// Cancel the operation
+  pub fn cancel(&self) { self.0.store(true, Ordering::Relaxed) }
+}
+
+impl Default for CancelFlag {
+  fn default() -> Self { Self::new() }
+}

orchidlang/src/libs/scheduler/id_map.rs

@@ -0,0 +1,49 @@
+use hashbrown::HashMap;
+
+/// A set that automatically assigns a unique ID to every entry.
+///
+/// # How unique?
+///
+/// If you inserted a new entry every nanosecond, it would take more than
+/// 550_000 years to run out of indices. Realistically Orchid might insert a new
+/// entry every 10ms, so these 64-bit indices will probably outlast humanity.
+#[derive(Clone, Debug)]
+pub struct IdMap<T> {
+  next_id: u64,
+  data: HashMap<u64, T>,
+}
+impl<T> IdMap<T> {
+  /// Create a new empty set
+  pub fn new() -> Self { Self { next_id: 0, data: HashMap::new() } }
+
+  /// Insert an element with a new ID and return the ID
+  pub fn insert(&mut self, t: T) -> u64 {
+    let id = self.next_id;
+    self.next_id += 1;
+    (self.data.try_insert(id, t)).unwrap_or_else(|_| panic!("IdMap keys should be unique"));
+    id
+  }
+
+  /// Remove the element with the given ID from the set. The ID will not be
+  /// reused.
+  pub fn remove(&mut self, id: u64) -> Option<T> { self.data.remove(&id) }
+}
+
+impl<T> Default for IdMap<T> {
+  fn default() -> Self { Self::new() }
+}
+
+#[cfg(test)]
+mod test {
+  use super::IdMap;
+
+  #[test]
+  fn basic_test() {
+    let mut map = IdMap::new();
+    let a = map.insert(1);
+    let b = map.insert(2);
+    assert_eq!(map.remove(a), Some(1));
+    assert_eq!(map.remove(a), None);
+    assert_eq!(map.data.get(&b), Some(&2));
+  }
+}

orchidlang/src/libs/scheduler/mod.rs

@@ -0,0 +1,8 @@
+//! A generic utility to sequence long blocking mutations that require a mutable
+//! reference to a shared resource.
+
+mod busy;
+pub mod cancel_flag;
+mod id_map;
+pub mod system;
+pub mod thread_pool;

orchidlang/src/libs/scheduler/system.rs

@@ -0,0 +1,344 @@
+//! Object to pass to [crate::facade::loader::Loader::add_system] to enable the
+//! scheduling subsystem. Other systems also take clones as dependencies.
+//!
+//! ```
+//! use orchidlang::facade::loader::Loader;
+//! use orchidlang::libs::asynch::system::AsynchSystem;
+//! use orchidlang::libs::scheduler::system::SeqScheduler;
+//! use orchidlang::libs::std::std_system::StdConfig;
+//!
+//! let mut asynch = AsynchSystem::new();
+//! let scheduler = SeqScheduler::new(&mut asynch);
+//! let env = Loader::new()
+//!   .add_system(StdConfig { impure: false })
+//!   .add_system(asynch)
+//!   .add_system(scheduler.clone());
+//! ```
+
+use std::any::{type_name, Any};
+use std::cell::RefCell;
+use std::fmt;
+use std::rc::Rc;
+use std::sync::{Arc, Mutex};
+
+use trait_set::trait_set;
+
+use super::busy::{BusyState, HandlerRes, NextItemReportKind, SyncOperation};
+use super::cancel_flag::CancelFlag;
+use super::id_map::IdMap;
+use super::thread_pool::ThreadPool;
+use crate::facade::system::{IntoSystem, System};
+use crate::foreign::cps_box::CPSBox;
+use crate::foreign::error::{AssertionError, RTResult};
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::gen::tree::{xfn_ent, ConstTree};
+use crate::interpreter::handler::HandlerTable;
+use crate::interpreter::nort::Expr;
+use crate::libs::asynch::system::{AsynchSystem, MessagePort};
+use crate::utils::ddispatch::Request;
+use crate::utils::take_with_output::take_with_output;
+use crate::utils::unwrap_or::unwrap_or;
+use crate::virt_fs::DeclTree;
+
+pub(super) enum SharedResource<T> {
+  Free(T),
+  Busy(BusyState<T>),
+  Taken,
+}
+
+/// Possible states of a shared resource
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub enum SharedState {
+  /// The resource is ready to be used or taken
+  Free,
+  /// The resource is currently in use but operations can be asynchronously
+  /// scheduled on it
+  Busy,
+  /// The resource is currently in use and a consuming seal has already been
+  /// scheduled, therefore further operations cannot access it and it will
+  /// transition to [SharedState::Taken] as soon as the currently pending
+  /// operations finish or are cancelled.
+  Sealed,
+  /// The resource has been removed from this location.
+  Taken,
+}
+
+/// A shared handle for a resource of type `T` that can be used with a
+/// [SeqScheduler] to execute mutating operations one by one in worker threads.
+pub struct SharedHandle<T>(pub(super) Arc<Mutex<SharedResource<T>>>);
+
+impl<T> SharedHandle<T> {
+  /// Wrap a value to be accessible to a [SeqScheduler].
+  pub fn wrap(t: T) -> Self { Self(Arc::new(Mutex::new(SharedResource::Free(t)))) }
+
+  /// Check the state of the handle
+  pub fn state(&self) -> SharedState {
+    match &*self.0.lock().unwrap() {
+      SharedResource::Busy(b) if b.is_sealed() => SharedState::Sealed,
+      SharedResource::Busy(_) => SharedState::Busy,
+      SharedResource::Free(_) => SharedState::Free,
+      SharedResource::Taken => SharedState::Taken,
+    }
+  }
+
+  /// Remove the value from the handle if it's free. To interact with a handle
+  /// you probably want to use a [SeqScheduler], but sometimes this makes
+  /// sense as eg. an optimization. You can return the value after processing
+  /// via [SharedHandle::untake].
+  pub fn take(&self) -> Option<T> {
+    take_with_output(&mut *self.0.lock().unwrap(), |state| match state {
+      SharedResource::Free(t) => (SharedResource::Taken, Some(t)),
+      _ => (state, None),
+    })
+  }
+
+  /// Return the value to a handle that doesn't have one. The intended use case
+  /// is to return values synchronously after they have been removed with
+  /// [SharedHandle::take].
+  pub fn untake(&self, value: T) -> Result<(), T> {
+    take_with_output(&mut *self.0.lock().unwrap(), |state| match state {
+      SharedResource::Taken => (SharedResource::Free(value), Ok(())),
+      _ => (state, Err(value)),
+    })
+  }
+}
+impl<T> Clone for SharedHandle<T> {
+  fn clone(&self) -> Self { Self(self.0.clone()) }
+}
+impl<T> fmt::Debug for SharedHandle<T> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    f.debug_struct("SharedHandle")
+      .field("state", &self.state())
+      .field("type", &type_name::<T>())
+      .finish()
+  }
+}
+impl<T: Send + 'static> InertPayload for SharedHandle<T> {
+  const TYPE_STR: &'static str = "a SharedHandle";
+  fn respond(&self, mut request: Request) {
+    request.serve_with(|| {
+      let this = self.clone();
+      TakeCmd(Arc::new(move |sch| {
+        let _ = sch.seal(this.clone(), |_| Vec::new());
+      }))
+    })
+  }
+}
+
+#[derive(Clone)]
+struct TakeCmd(pub Arc<dyn Fn(SeqScheduler) + Send + Sync>);
+impl fmt::Debug for TakeCmd {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "A command to drop a shared resource")
+  }
+}
+
+/// Error produced when an operation is scheduled or a seal placed on a resource
+/// which is either already sealed or taken.
+#[derive(Debug, Clone)]
+pub struct SealedOrTaken;
+impl InertPayload for SealedOrTaken {
+  const TYPE_STR: &'static str = "SealedOrTaken";
+}
+
+fn take_and_drop(x: Expr) -> RTResult<CPSBox<TakeCmd>> {
+  match x.clause.request() {
+    Some(t) => Ok(CPSBox::<TakeCmd>::new(1, t)),
+    None => AssertionError::fail(x.location(), "SharedHandle", format!("{x}")),
+  }
+}
+
+fn is_taken_e(x: Expr) -> Inert<bool> { Inert(x.downcast::<Inert<SealedOrTaken>>().is_ok()) }
+
+trait_set! {
+  /// The part of processing a blocking I/O task that cannot be done on a remote
+  /// thread, eg. because it accesses other systems or Orchid code.
+  trait NonSendFn = FnOnce(Box<dyn Any + Send>, SeqScheduler) -> Vec<Expr>;
+}
+
+struct SyncReply {
+  opid: u64,
+  data: Box<dyn Any + Send>,
+}
+
+struct CheshireCat {
+  pool: ThreadPool<Box<dyn FnOnce() + Send>>,
+  pending: RefCell<IdMap<Box<dyn NonSendFn>>>,
+  port: MessagePort,
+}
+
+/// A task scheduler that executes long blocking operations that have mutable
+/// access to a shared one by one on a worker thread. The resources are
+/// held in [SharedHandle]s
+#[derive(Clone)]
+pub struct SeqScheduler(Rc<CheshireCat>);
+impl SeqScheduler {
+  /// Creates a new [SeqScheduler]. The new object is also kept alive by a
+  /// callback in the provided [AsynchSystem]. There should be at most one
+  pub fn new(asynch: &mut AsynchSystem) -> Self {
+    let this = Self(Rc::new(CheshireCat {
+      pending: RefCell::new(IdMap::new()),
+      pool: ThreadPool::new(),
+      port: asynch.get_port(),
+    }));
+    let this1 = this.clone();
+    // referenced by asynch, references this
+    asynch.register(move |res: Box<SyncReply>| {
+      let callback = this1.0.pending.borrow_mut().remove(res.opid).expect(
+        "Received reply for task we didn't start. This likely means that \
+         there are multiple SequencingContexts attached to the same \
+         AsynchSystem.",
+      );
+      callback(res.data, this1.clone())
+    });
+    this
+  }
+
+  /// Submit an action to be executed on a worker thread which can own the data
+  /// in the handle.
+  ///
+  /// * handle - data to be transformed
+  /// * operation - long blocking mutation to execute off-thread.
+  /// * handler - process the results, talk to other systems, generate and run
+  ///   Orchid code.
+  /// * early_cancel - clean up in case the task got cancelled before it was
+  ///   scheduled. This is an optimization so that threads aren't spawned if a
+  ///   large batch of tasks is scheduled and then cancelled.
+  pub fn schedule<T: Send + 'static, U: Send + 'static>(
+    &self,
+    handle: SharedHandle<T>,
+    operation: impl FnOnce(T, CancelFlag) -> (T, U) + Send + 'static,
+    handler: impl FnOnce(T, U, CancelFlag) -> HandlerRes<T> + Send + 'static,
+    early_cancel: impl FnOnce(T) -> HandlerRes<T> + Send + 'static,
+  ) -> Result<CancelFlag, SealedOrTaken> {
+    take_with_output(&mut *handle.0.lock().unwrap(), {
+      let handle = handle.clone();
+      |state| {
+        match state {
+          SharedResource::Taken => (SharedResource::Taken, Err(SealedOrTaken)),
+          SharedResource::Busy(mut b) => match b.enqueue(operation, handler, early_cancel) {
+            Some(cancelled) => (SharedResource::Busy(b), Ok(cancelled)),
+            None => (SharedResource::Busy(b), Err(SealedOrTaken)),
+          },
+          SharedResource::Free(t) => {
+            let cancelled = CancelFlag::new();
+            drop(early_cancel); // cannot possibly be useful
+            let op_erased: SyncOperation<T> = Box::new(|t, c| {
+              let (t, u) = operation(t, c);
+              (t, Box::new(u))
+            });
+            self.submit(t, handle, cancelled.clone(), op_erased);
+            (SharedResource::Busy(BusyState::new(handler)), Ok(cancelled))
+          },
+        }
+      }
+    })
+  }
+
+  /// Run an operation asynchronously and then process its result in thread,
+  /// without queuing on any particular data.
+  pub fn run_orphan<T: Send + 'static>(
+    &self,
+    operation: impl FnOnce(CancelFlag) -> T + Send + 'static,
+    handler: impl FnOnce(T, CancelFlag) -> Vec<Expr> + 'static,
+  ) -> CancelFlag {
+    let cancelled = CancelFlag::new();
+    let canc1 = cancelled.clone();
+    let opid = self.0.pending.borrow_mut().insert(Box::new(|data: Box<dyn Any + Send>, _| {
+      handler(*data.downcast().expect("This is associated by ID"), canc1)
+    }));
+    let canc1 = cancelled.clone();
+    let mut port = self.0.port.clone();
+    self.0.pool.submit(Box::new(move || {
+      port.send(SyncReply { opid, data: Box::new(operation(canc1)) });
+    }));
+    cancelled
+  }
+
+  /// Schedule a function that will consume the value. After this the handle is
+  /// considered sealed and all [SeqScheduler::schedule] calls will fail.
+  pub fn seal<T>(
+    &self,
+    handle: SharedHandle<T>,
+    seal: impl FnOnce(T) -> Vec<Expr> + Sync + Send + 'static,
+  ) -> Result<Vec<Expr>, SealedOrTaken> {
+    take_with_output(&mut *handle.0.lock().unwrap(), |state| match state {
+      SharedResource::Busy(mut b) if !b.is_sealed() => {
+        b.seal(seal);
+        (SharedResource::Busy(b), Ok(Vec::new()))
+      },
+      SharedResource::Busy(_) => (state, Err(SealedOrTaken)),
+      SharedResource::Taken => (SharedResource::Taken, Err(SealedOrTaken)),
+      SharedResource::Free(t) => (SharedResource::Taken, Ok(seal(t))),
+    })
+  }
+
+  /// Asynchronously recursive function to schedule a new task for execution and
+  /// act upon its completion. The self-reference is passed into the callback
+  /// from the callback passed to the [AsynchSystem] so that if the task is
+  /// never resolved but the [AsynchSystem] through which the resolving event
+  /// would arrive is dropped this [SeqScheduler] is also dropped.
+  fn submit<T: Send + 'static>(
+    &self,
+    t: T,
+    handle: SharedHandle<T>,
+    cancelled: CancelFlag,
+    operation: SyncOperation<T>,
+  ) {
+    // referenced by self until run, references handle
+    let opid = self.0.pending.borrow_mut().insert(Box::new({
+      let cancelled = cancelled.clone();
+      move |data: Box<dyn Any + Send>, this: SeqScheduler| {
+        let (t, u): (T, Box<dyn Any + Send>) = *data.downcast().expect("This is associated by ID");
+        let handle2 = handle.clone();
+        take_with_output(&mut *handle.0.lock().unwrap(), |state| {
+          let busy = unwrap_or! { state => SharedResource::Busy;
+            panic!("Handle with outstanding invocation must be busy")
+          };
+          let report = busy.rotate(t, u, cancelled);
+          match report.kind {
+            NextItemReportKind::Free(t) => (SharedResource::Free(t), report.events),
+            NextItemReportKind::Taken => (SharedResource::Taken, report.events),
+            NextItemReportKind::Next { instance, cancelled, operation, rest } => {
+              this.submit(instance, handle2, cancelled, operation);
+              (SharedResource::Busy(rest), report.events)
+            },
+          }
+        })
+      }
+    }));
+    let mut port = self.0.port.clone();
+    // referenced by thread until run, references port
+    self.0.pool.submit(Box::new(move || {
+      port.send(SyncReply { opid, data: Box::new(operation(t, cancelled)) })
+    }))
+  }
+}
+
+impl IntoSystem<'static> for SeqScheduler {
+  fn into_system(self) -> System<'static> {
+    let mut handlers = HandlerTable::new();
+    handlers.register(|cmd: &CPSBox<CancelFlag>| {
+      let (canceller, cont) = cmd.unpack1();
+      canceller.cancel();
+      cont
+    });
+    handlers.register(move |cmd: &CPSBox<TakeCmd>| {
+      let (TakeCmd(cb), cont) = cmd.unpack1();
+      cb(self.clone());
+      cont
+    });
+    System {
+      name: "system::scheduler",
+      prelude: Vec::new(),
+      code: DeclTree::empty(),
+      handlers,
+      lexer_plugins: vec![],
+      line_parsers: vec![],
+      constants: ConstTree::ns("system::scheduler", [ConstTree::tree([
+        xfn_ent("is_taken_e", [is_taken_e]),
+        xfn_ent("take_and_drop", [take_and_drop]),
+      ])]),
+    }
+  }
+}

orchidlang/src/libs/scheduler/thread_pool.rs

@@ -0,0 +1,172 @@
+//! A thread pool for executing tasks in parallel, spawning threads as workload
+//! increases and terminating them as tasks finish. This is not terribly
+//! efficient, its main design goal is to parallelize blocking I/O calls.
+//!
+//! This is the abstract implementation of the scheduler.
+
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{sync_channel, SyncSender};
+use std::sync::{Arc, Mutex};
+use std::thread::spawn;
+
+/// A trait for a task dispatched on a [ThreadPool]. The task owns all relevant
+/// data, is safe to pass between threads and is executed only once.
+pub trait Task: Send + 'static {
+  /// Execute the task. At a minimum, this involves signaling some other thread,
+  /// otherwise the task has no effect.
+  fn run(self);
+}
+
+impl<F: FnOnce() + Send + 'static> Task for F {
+  fn run(self) { self() }
+}
+
+/// An async unit of work that produces some result, see [Task]. This can be
+/// wrapped in a generic reporter to create a task.
+pub trait Query: Send + 'static {
+  /// The value produced by the query
+  type Result: Send + 'static;
+
+  /// Execute the query, producing some value which can then be sent to another
+  /// thread
+  fn run(self) -> Self::Result;
+
+  /// Associate the query with a reporter expressed in a plain function.
+  /// Note that because every lambda has a distinct type and every thread pool
+  /// runs exactly one type of task, this can appear only once in the code for
+  /// a given thread pool. It is practical in a narrow set of cases, most of the
+  /// time however you are better off defining an explicit reporter.
+  fn then<F: FnOnce(Self::Result) + Send + 'static>(self, callback: F) -> QueryTask<Self, F>
+  where Self: Sized {
+    QueryTask { query: self, callback }
+  }
+}
+impl<F: FnOnce() -> R + Send + 'static, R: Send + 'static> Query for F {
+  type Result = R;
+
+  fn run(self) -> Self::Result { self() }
+}
+
+/// A reporter that calls a statically known function with the result of a
+/// query. Constructed with [Query::then]
+pub struct QueryTask<Q: Query, F: FnOnce(Q::Result) + Send + 'static> {
+  query: Q,
+  callback: F,
+}
+impl<Q: Query, F: FnOnce(Q::Result) + Send + 'static> Task for QueryTask<Q, F> {
+  fn run(self) { (self.callback)(self.query.run()) }
+}
+
+enum Message<T: Task> {
+  Stop,
+  Task(T),
+}
+
+struct ThreadPoolData<T: Task> {
+  rdv_point: Mutex<Option<SyncSender<Message<T>>>>,
+  stopping: AtomicBool,
+}
+
+/// A thread pool to execute blocking I/O operations in parallel.
+/// This thread pool is pretty inefficient for CPU-bound operations because it
+/// spawns an unbounded number of concurrent threads and destroys them eagerly.
+/// It is assumed that the tasks at hand are substnatially but not incomparably
+/// more expensive than spawning a new thread.
+///
+/// If multiple threads finish their tasks, one waiting thread is kept, the
+/// rest exit. If all threads are busy, new threads are spawned when tasks
+/// arrive. To get rid of the last waiting thread, drop the thread pool.
+///
+/// ```
+/// use orchidlang::libs::scheduler::thread_pool::{Task, ThreadPool};
+///
+/// struct MyTask(&'static str);
+/// impl Task for MyTask {
+///   fn run(self) { println!("{}", self.0) }
+/// }
+///
+/// let pool = ThreadPool::new();
+///
+/// // spawns first thread
+/// pool.submit(MyTask("foo"));
+/// // probably spawns second thread
+/// pool.submit(MyTask("bar"));
+/// // either spawns third thread or reuses first
+/// pool.submit(MyTask("baz"));
+/// ```
+pub struct ThreadPool<T: Task> {
+  data: Arc<ThreadPoolData<T>>,
+}
+impl<T: Task> ThreadPool<T> {
+  /// Create a new thread pool. This just initializes the threadsafe
+  /// datastructures used to synchronize tasks and doesn't spawn any threads.
+  /// The first submission spawns the first thread.
+  pub fn new() -> Self {
+    Self {
+      data: Arc::new(ThreadPoolData {
+        rdv_point: Mutex::new(None),
+        stopping: AtomicBool::new(false),
+      }),
+    }
+  }
+
+  /// Submit a task to the thread pool. This tries to send the task to the
+  /// waiting thread, or spawn a new one. If a thread is done with its task
+  /// and finds that it another thread is already waiting, it exits.
+  pub fn submit(&self, task: T) {
+    let mut standby = self.data.rdv_point.lock().unwrap();
+    if let Some(port) = standby.take() {
+      (port.try_send(Message::Task(task))).expect(
+        "This channel cannot be disconnected unless the receiver crashes
+        between registering the sender and blocking for receive, and it cannot
+        be full because it's taken before insertion",
+      );
+    } else {
+      drop(standby);
+      let data = self.data.clone();
+      // worker thread created if all current ones are busy
+      spawn(move || {
+        let mut cur_task = task;
+        loop {
+          // Handle the task
+          cur_task.run();
+          // Apply for a new task if no other thread is doing so already
+          let mut standby_spot = data.rdv_point.lock().unwrap();
+          if standby_spot.is_some() {
+            return; // exit if we would be the second in line
+          }
+          let (sender, receiver) = sync_channel(1);
+          *standby_spot = Some(sender);
+          drop(standby_spot);
+          if data.stopping.load(Ordering::SeqCst) {
+            return; // exit if the pool was dropped before we applied
+          }
+          // Wait for the next event on the pool
+          let msg = (receiver.recv()).expect("We are holding a reference");
+          match msg {
+            // repeat with next task
+            Message::Task(task) => cur_task = task,
+            // exit if the pool is dropped
+            Message::Stop => return,
+          }
+        }
+      });
+    }
+  }
+}
+
+impl<T: Task> Default for ThreadPool<T> {
+  fn default() -> Self { Self::new() }
+}
+
+impl<T: Task> Drop for ThreadPool<T> {
+  // Ensure all threads exit properly
+  fn drop(&mut self) {
+    self.data.stopping.store(true, Ordering::SeqCst);
+    let mut rdv_point = self.data.rdv_point.lock().unwrap();
+    if let Some(pending) = rdv_point.take() {
+      // the worker has read the value of `stopping`
+      let _ = pending.send(Message::Stop);
+    }
+  }
+}

orchidlang/src/libs/std/arithmetic_error.rs

@@ -0,0 +1,31 @@
+//! Error produced by numeric opperations
+
+use std::fmt;
+
+use crate::foreign::error::RTError;
+
+/// Various errors produced by arithmetic operations
+#[derive(Clone)]
+pub enum ArithmeticError {
+  /// Integer overflow
+  Overflow,
+  /// Float overflow
+  Infinity,
+  /// Division or modulo by zero
+  DivByZero,
+  /// Other, unexpected operation produced NaN
+  NaN,
+}
+
+impl fmt::Display for ArithmeticError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::NaN => write!(f, "Operation resulted in NaN"),
+      Self::Overflow => write!(f, "Integer overflow"),
+      Self::Infinity => write!(f, "Operation resulted in Infinity"),
+      Self::DivByZero => write!(f, "A division by zero was attempted"),
+    }
+  }
+}
+
+impl RTError for ArithmeticError {}

orchidlang/src/libs/std/binary.rs

@@ -0,0 +1,137 @@
+//! `std::binary` Operations on binary buffers.
+
+use std::fmt;
+use std::ops::Deref;
+use std::sync::Arc;
+
+use itertools::Itertools;
+
+use super::runtime_error::RuntimeError;
+use crate::foreign::atom::Atomic;
+use crate::foreign::error::RTResult;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::gen::tree::{atom_ent, xfn_ent, ConstTree};
+use crate::interpreter::nort::Clause;
+use crate::utils::iter_find::iter_find;
+use crate::utils::unwrap_or::unwrap_or;
+
+const INT_BYTES: usize = usize::BITS as usize / 8;
+
+/// A block of binary data
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub struct Binary(pub Arc<Vec<u8>>);
+impl InertPayload for Binary {
+  const TYPE_STR: &'static str = "a binary blob";
+}
+
+impl Deref for Binary {
+  type Target = Vec<u8>;
+  fn deref(&self) -> &Self::Target { &self.0 }
+}
+
+impl fmt::Debug for Binary {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    let mut iter = self.0.iter().copied();
+    f.write_str("Binary")?;
+    for mut chunk in iter.by_ref().take(32).chunks(4).into_iter() {
+      let a = chunk.next().expect("Chunks cannot be empty");
+      let b = unwrap_or!(chunk.next(); return write!(f, "{a:02x}"));
+      let c = unwrap_or!(chunk.next(); return write!(f, "{a:02x}{b:02x}"));
+      let d = unwrap_or!(chunk.next(); return write!(f, "{a:02x}{b:02x}{c:02x}"));
+      write!(f, "{a:02x}{b:02x}{c:02x}{d:02x}")?
+    }
+    if iter.next().is_some() { write!(f, "...") } else { Ok(()) }
+  }
+}
+
+/// Append two binary data blocks
+pub fn concatenate(a: Inert<Binary>, b: Inert<Binary>) -> Inert<Binary> {
+  let data = (*a).iter().chain(b.0.0.iter()).copied().collect();
+  Inert(Binary(Arc::new(data)))
+}
+
+/// Extract a subsection of the binary data
+pub fn slice(s: Inert<Binary>, i: Inert<usize>, len: Inert<usize>) -> RTResult<Inert<Binary>> {
+  if i.0 + len.0 < s.0.0.len() {
+    RuntimeError::fail("Byte index out of bounds".to_string(), "indexing binary")?
+  }
+  Ok(Inert(Binary(Arc::new(s.0.0[i.0..i.0 + len.0].to_vec()))))
+}
+
+/// Return the index where the first argument first contains the second, if any
+pub fn find(haystack: Inert<Binary>, needle: Inert<Binary>) -> Option<Clause> {
+  let found = iter_find(haystack.0.0.iter(), needle.0.0.iter());
+  found.map(|i| Inert(i).atom_cls())
+}
+
+/// Split binary data block into two smaller blocks
+pub fn split(bin: Inert<Binary>, i: Inert<usize>) -> RTResult<(Inert<Binary>, Inert<Binary>)> {
+  if bin.0.0.len() < i.0 {
+    RuntimeError::fail("Byte index out of bounds".to_string(), "splitting binary")?
+  }
+  let (asl, bsl) = bin.0.0.split_at(i.0);
+  Ok((Inert(Binary(Arc::new(asl.to_vec()))), Inert(Binary(Arc::new(bsl.to_vec())))))
+}
+
+/// Read a number from a binary blob
+pub fn get_num(
+  buf: Inert<Binary>,
+  loc: Inert<usize>,
+  size: Inert<usize>,
+  is_le: Inert<bool>,
+) -> RTResult<Inert<usize>> {
+  if buf.0.0.len() < (loc.0 + size.0) {
+    RuntimeError::fail("section out of range".to_string(), "reading number from binary data")?
+  }
+  if INT_BYTES < size.0 {
+    RuntimeError::fail(
+      "more than std::bin::int_bytes bytes provided".to_string(),
+      "reading number from binary data",
+    )?
+  }
+  let mut data = [0u8; INT_BYTES];
+  let section = &buf.0.0[loc.0..(loc.0 + size.0)];
+  let num = if is_le.0 {
+    data[0..size.0].copy_from_slice(section);
+    usize::from_le_bytes(data)
+  } else {
+    data[INT_BYTES - size.0..].copy_from_slice(section);
+    usize::from_be_bytes(data)
+  };
+  Ok(Inert(num))
+}
+
+/// Convert a number into a blob
+pub fn from_num(
+  size: Inert<usize>,
+  is_le: Inert<bool>,
+  data: Inert<usize>,
+) -> RTResult<Inert<Binary>> {
+  if INT_BYTES < size.0 {
+    RuntimeError::fail(
+      "more than std::bin::int_bytes bytes requested".to_string(),
+      "converting number to binary",
+    )?
+  }
+  let bytes = match is_le.0 {
+    true => data.0.to_le_bytes()[0..size.0].to_vec(),
+    false => data.0.to_be_bytes()[8 - size.0..].to_vec(),
+  };
+  Ok(Inert(Binary(Arc::new(bytes))))
+}
+
+/// Detect the number of bytes in the blob
+pub fn size(b: Inert<Binary>) -> Inert<usize> { Inert(b.0.len()) }
+
+pub(super) fn bin_lib() -> ConstTree {
+  ConstTree::ns("std::binary", [ConstTree::tree([
+    xfn_ent("concat", [concatenate]),
+    xfn_ent("slice", [slice]),
+    xfn_ent("find", [find]),
+    xfn_ent("split", [split]),
+    xfn_ent("get_num", [get_num]),
+    xfn_ent("from_num", [from_num]),
+    xfn_ent("size", [size]),
+    atom_ent("int_bytes", [Inert(INT_BYTES)]),
+  ])])
+}

orchidlang/src/libs/std/bool.orc

@@ -0,0 +1,47 @@
+import std::(pmatch, inspect)
+import std::known::(=)
+
+export ::(!=, ==)
+
+export const not := \bool. if bool then false else true
+macro ...$a != ...$b =0x3p36=> (not (...$a == ...$b))
+macro ...$a == ...$b =0x3p36=> (equals (...$a) (...$b))
+export macro ...$a and ...$b =0x4p36=> (ifthenelse (...$a) (...$b) false)
+export macro ...$a or ...$b =0x4p36=> (ifthenelse (...$a) true (...$b))
+export macro if ...$cond then ...$true else ...$false:1 =0x1p84=> (
+  ifthenelse (...$cond) (...$true) (...$false)
+)
+
+(
+  macro pmatch::request (= ...$other)
+  =0x1p230=> pmatch::response (
+    if pmatch::value == (...$other)
+    then pmatch::pass
+    else pmatch::fail
+  )
+  ( pmatch::no_binds )
+)
+
+(
+  macro pmatch::request (!= ...$other)
+  =0x1p230=> pmatch::response (
+    if pmatch::value != (...$other)
+    then pmatch::pass
+    else pmatch::fail
+  )
+  ( pmatch::no_binds )
+)
+
+(
+  macro pmatch::request (true)
+  =0x1p230=> pmatch::response
+    (if pmatch::value then pmatch::pass else pmatch::fail)
+    ( pmatch::no_binds )
+)
+
+(
+  macro pmatch::request (false)
+  =0x1p230=> pmatch::response
+    (if pmatch::value then pmatch::fail else pmatch::pass)
+    ( pmatch::no_binds )
+)

orchidlang/src/libs/std/bool.rs

@@ -0,0 +1,48 @@
+use super::number::Numeric;
+use super::string::OrcString;
+use crate::foreign::error::{AssertionError, RTResult};
+use crate::foreign::inert::Inert;
+use crate::foreign::try_from_expr::WithLoc;
+use crate::gen::tpl;
+use crate::gen::traits::{Gen, GenClause};
+use crate::gen::tree::{atom_ent, xfn_ent, ConstTree};
+use crate::interpreter::gen_nort::nort_gen;
+use crate::interpreter::nort::Expr;
+
+const fn left() -> impl GenClause { tpl::L("l", tpl::L("_", tpl::P("l"))) }
+const fn right() -> impl GenClause { tpl::L("_", tpl::L("r", tpl::P("r"))) }
+
+/// Takes a boolean and two branches, runs the first if the bool is true, the
+/// second if it's false.
+// Even though it's a ternary function, IfThenElse is implemented as an unary
+// foreign function, as the rest of the logic can be defined in Orchid.
+pub fn if_then_else(WithLoc(loc, b): WithLoc<Inert<bool>>) -> Expr {
+  let ctx = nort_gen(loc);
+  if b.0 { left().template(ctx, []) } else { right().template(ctx, []) }
+}
+
+/// Compares the inner values if
+///
+/// - both are string,
+/// - both are bool,
+/// - both are either uint or num
+pub fn equals(WithLoc(loc, a): WithLoc<Expr>, b: Expr) -> RTResult<Inert<bool>> {
+  Ok(Inert(if let Ok(l) = a.clone().downcast::<Inert<OrcString>>() {
+    b.downcast::<Inert<OrcString>>().is_ok_and(|r| *l == *r)
+  } else if let Ok(l) = a.clone().downcast::<Inert<bool>>() {
+    b.downcast::<Inert<bool>>().is_ok_and(|r| *l == *r)
+  } else if let Some(l) = a.clause.request::<Numeric>() {
+    b.clause.request::<Numeric>().is_some_and(|r| l.as_float() == r.as_float())
+  } else {
+    AssertionError::fail(loc, "string, bool or numeric", format!("{a}"))?
+  }))
+}
+
+pub fn bool_lib() -> ConstTree {
+  ConstTree::ns("std::bool", [ConstTree::tree([
+    xfn_ent("ifthenelse", [if_then_else]),
+    xfn_ent("equals", [equals]),
+    atom_ent("true", [Inert(true)]),
+    atom_ent("false", [Inert(false)]),
+  ])])
+}

orchidlang/src/libs/std/conv.rs

@@ -0,0 +1,45 @@
+use ordered_float::NotNan;
+
+use super::number::Numeric;
+use super::string::OrcString;
+use crate::foreign::error::{AssertionError, RTResult};
+use crate::foreign::inert::Inert;
+use crate::foreign::try_from_expr::WithLoc;
+use crate::gen::tpl;
+use crate::gen::tree::{leaf, xfn_ent, ConstTree};
+use crate::interpreter::nort::ClauseInst;
+use crate::parse::numeric::parse_num;
+
+fn to_numeric(WithLoc(loc, a): WithLoc<ClauseInst>) -> RTResult<Numeric> {
+  if let Some(n) = a.request::<Numeric>() {
+    return Ok(n);
+  }
+  if let Some(s) = a.request::<OrcString>() {
+    return parse_num(s.as_str())
+      .map_err(|e| AssertionError::ext(loc, "number syntax", format!("{e:?}")));
+  }
+  AssertionError::fail(loc, "string or number", format!("{a}"))
+}
+
+/// parse a number. Accepts the same syntax Orchid does.
+pub fn to_float(a: WithLoc<ClauseInst>) -> RTResult<Inert<NotNan<f64>>> {
+  to_numeric(a).map(|n| Inert(n.as_float()))
+}
+
+/// Parse an unsigned integer. Accepts the same formats Orchid does. If the
+/// input is a number, floors it.
+pub fn to_uint(a: WithLoc<ClauseInst>) -> RTResult<Inert<usize>> {
+  to_numeric(a).map(|n| match n {
+    Numeric::Float(f) => Inert(f.floor() as usize),
+    Numeric::Uint(i) => Inert(i),
+  })
+}
+
+pub fn conv_lib() -> ConstTree {
+  ConstTree::ns("std", [ConstTree::tree([ConstTree::tree_ent("conv", [
+    xfn_ent("to_float", [to_float]),
+    xfn_ent("to_uint", [to_uint]),
+    // conversion logic moved to the string library
+    ("to_string", leaf(tpl::C("std::string::convert"))),
+  ])])])
+}

orchidlang/src/libs/std/cross_pipeline.rs

@@ -0,0 +1,65 @@
+use std::iter;
+use std::rc::Rc;
+
+use itertools::Itertools;
+
+use crate::foreign::atom::Atomic;
+use crate::foreign::fn_bridge::xfn;
+use crate::foreign::process::Unstable;
+use crate::foreign::to_clause::ToClause;
+use crate::foreign::try_from_expr::{TryFromExpr, WithLoc};
+use crate::location::SourceRange;
+use crate::parse::parsed::{self, PType};
+use crate::utils::pure_seq::pushed;
+
+pub trait DeferredRuntimeCallback<T, R: ToClause>:
+  FnOnce(Vec<T>) -> R + Clone + Send + 'static
+{
+}
+impl<T, R: ToClause, F: FnOnce(Vec<T>) -> R + Clone + Send + 'static> DeferredRuntimeCallback<T, R>
+  for F
+{
+}
+
+/// Lazy-recursive function that takes the next value from the interpreter
+/// and acts upon it
+///
+/// # Panics
+///
+/// If the list of remaining keys is empty
+fn table_receiver_rec<T: TryFromExpr + Clone + Send + 'static, R: ToClause + 'static>(
+  results: Vec<T>,
+  items: usize,
+  callback: impl DeferredRuntimeCallback<T, R>,
+) -> impl Atomic {
+  xfn("__table_receiver__", move |WithLoc(loc, t): WithLoc<T>| {
+    let results: Vec<T> = pushed(results, t);
+    match items == results.len() {
+      true => callback(results).to_clause(loc),
+      false => table_receiver_rec(results, items, callback).atom_cls(),
+    }
+  })
+}
+
+fn table_receiver<T: TryFromExpr + Clone + Send + 'static, R: ToClause + 'static>(
+  items: usize,
+  callback: impl DeferredRuntimeCallback<T, R>,
+) -> parsed::Clause {
+  if items == 0 {
+    Unstable::new(move |_| callback(Vec::new())).ast_cls()
+  } else {
+    Unstable::new(move |_| table_receiver_rec(Vec::new(), items, callback).atom_cls()).ast_cls()
+  }
+}
+
+/// Defers the execution of the callback to runtime, allowing it to depend on
+/// the result of Otchid expressions.
+pub fn defer_to_runtime<T: TryFromExpr + Clone + Send + 'static, R: ToClause + 'static>(
+  range: SourceRange,
+  exprs: impl Iterator<Item = Vec<parsed::Expr>>,
+  callback: impl DeferredRuntimeCallback<T, R>,
+) -> parsed::Clause {
+  let argv = exprs.into_iter().map(|v| parsed::Clause::S(PType::Par, Rc::new(v))).collect_vec();
+  let items = iter::once(table_receiver(argv.len(), callback)).chain(argv);
+  parsed::Clause::s('(', items, range)
+}

orchidlang/src/libs/std/exit_status.rs

@@ -0,0 +1,43 @@
+//! `std::exit_status` Exit status of a program or effectful subprogram.
+//!
+//! There is no support for custom codes, and the success/failure state can be
+//! inspected.
+
+use std::process::ExitCode;
+
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::gen::tree::{atom_ent, xfn_ent, ConstTree};
+
+/// An Orchid equivalent to Rust's binary exit status model
+///
+/// The "namespaced" name is to easily separate in autocomplete lists from both
+/// [std::process::ExitStatus] and [std::process::ExitCode]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum OrcExitStatus {
+  /// unix exit code 0
+  Success,
+  /// unix exit code 1
+  Failure,
+}
+impl OrcExitStatus {
+  /// Convert to Rust-land [ExitCode]
+  pub fn code(self) -> ExitCode {
+    match self {
+      Self::Success => ExitCode::SUCCESS,
+      Self::Failure => ExitCode::FAILURE,
+    }
+  }
+}
+
+impl InertPayload for OrcExitStatus {
+  const TYPE_STR: &'static str = "ExitStatus";
+}
+
+pub(super) fn exit_status_lib() -> ConstTree {
+  let is_success = |es: Inert<OrcExitStatus>| Inert(es.0 == OrcExitStatus::Success);
+  ConstTree::ns("std::exit_status", [ConstTree::tree([
+    atom_ent("success", [Inert(OrcExitStatus::Success)]),
+    atom_ent("failure", [Inert(OrcExitStatus::Failure)]),
+    xfn_ent("is_success", [is_success]),
+  ])])
+}

orchidlang/src/libs/std/fn.orc

@@ -0,0 +1,43 @@
+import super::known::*
+import super::pmatch
+import super::pmatch::(match, =>)
+import super::macro
+
+--[ Do nothing. Especially useful as a passive cps operation ]--
+export const identity := \x.x
+--[
+  Apply the function to the given value. Can be used to assign a
+  concrete value in a cps assignment statement.
+]--
+export const pass := \val. \cont. cont val
+--[
+  Apply the function to the given pair of values. Mainly useful to assign
+  a concrete pair of values in a cps multi-assignment statement
+]--
+export const pass2 := \a. \b. \cont. cont a b
+--[
+  A function that returns the given value for any input. Also useful as a
+  "break" statement in a "do" block.
+]--
+export const return := \a. \b.a
+
+export macro ...$prefix $ ...$suffix:1 =0x1p38=> ...$prefix (...$suffix)
+export macro ...$prefix |> $fn ..$suffix:1 =0x2p32=> $fn (...$prefix) ..$suffix
+
+( macro (..$argv) => ...$body
+  =0x3p127=> lambda_walker macro::comma_list (..$argv) (...$body)
+)
+( macro $_arg => ...$body
+  =0x2p127=> \$_arg. ...$body
+)
+( macro lambda_walker ( macro::list_item ($_argname) $tail ) $body
+  =0x2p254=> \$_argname. lambda_walker $tail $body
+)
+( macro lambda_walker ( macro::list_item (...$head) $tail ) $body
+  =0x1p254=> \arg. match arg {
+    ...$head => lambda_walker $tail $body;
+  }
+)
+( macro lambda_walker macro::list_end $body
+  =0x1p254=> $body
+)

orchidlang/src/libs/std/inspect.rs

@@ -0,0 +1,16 @@
+use crate::foreign::fn_bridge::Thunk;
+use crate::gen::tree::{xfn_ent, ConstTree};
+use crate::interpreter::nort::Expr;
+
+pub fn inspect_lib() -> ConstTree {
+  ConstTree::ns("std", [ConstTree::tree([
+    xfn_ent("inspect", [|x: Thunk| {
+      eprintln!("{}", x.0);
+      x.0
+    }]),
+    xfn_ent("tee", [|x: Expr| {
+      eprintln!("{x}");
+      x
+    }]),
+  ])])
+}

orchidlang/src/libs/std/known.orc

@@ -0,0 +1 @@
+export ::[, _ ; . =]

orchidlang/src/libs/std/list.orc

@@ -0,0 +1,139 @@
+import super::(option, tuple, tuple::t, panic, pmatch, pmatch::=>, macro, tee)
+import super::(fn::*, procedural::*)
+import super::(loop::*, bool::*, known::*, number::*)
+
+as_type ()
+
+export const cons := \hd. \tl. wrap (option::some t[hd, unwrap tl])
+export const end := wrap option::none
+export const pop := \list. \default. \f. (
+  pmatch::match (unwrap list) {
+    option::none => default;
+    option::some t[hd, tl] => f hd (wrap tl);
+  }
+)
+
+-- Operators
+
+--[ Fold each element into an accumulator using an `acc -> el -> acc`. #eager ]--
+export const fold := \list. \acc. \f. (
+  loop_over (list, acc) {
+    cps head, list = pop list acc;
+    let acc = f acc head;
+  }
+)
+
+--[ Fold each element into an accumulator in reverse order. #eager-notail ]--
+export const rfold := \list. \acc. \f. (
+  recursive r (list)
+    pop list acc \head. \tail.
+      f (r tail) head
+)
+
+--[ Reverse a list. #eager ]--
+export const reverse := \list. fold list end \tl. \hd. cons hd tl
+
+--[ Fold each element into a shared element with an `el -> el -> el`. #eager-notail ]--
+export const reduce := \list. \f. do{
+  cps head, list = pop list option::none;
+  option::some $ fold list head f
+}
+
+--[
+  Return a new list that contains only the elements from the input list
+  for which the function returns true. #lazy
+]--
+export const filter := \list. \f. (
+  pop list end \head. \tail.
+    if (f head)
+    then cons head (filter tail f)
+    else filter tail f
+)
+
+--[ Transform each element of the list with an `el -> any`. #lazy ]--
+export const map := \list. \f. (
+  recursive r (list)
+    pop list end \head. \tail.
+      cons (f head) (r tail)
+)
+
+--[ Skip `n` elements from the list and return the tail. #lazy ]--
+export const skip := \foo. \n. (
+  loop_over (foo, n) {
+    cps _head, foo = if n <= 0
+      then return foo
+      else pop foo end;
+    let n = n - 1;
+  }
+)
+
+--[ Return `n` elements from the list and discard the rest. #lazy ]--
+export const take := \list. \n. (
+  recursive r (list, n)
+    if n == 0
+    then end
+    else pop list end \head. \tail.
+      cons head $ r tail $ n - 1
+)
+
+--[ Return the `n`th element from the list. #eager ]--
+export const get := \list. \n. (
+  loop_over (list, n) {
+    cps head, list = pop list option::none;
+    cps if n == 0
+      then return (option::some head)
+      else identity;
+    let n = n - 1;
+  }
+)
+
+--[ Map every element to a pair of the index and the original element. #lazy ]--
+export const enumerate := \list. (
+  recursive r (list, n = 0) 
+    pop list end \head. \tail.
+      cons t[n, head] $ r tail $ n + 1
+)
+
+export const count := \list. fold list 0 \a. \n. a + 1
+
+--[
+  Turn a list of CPS commands into a sequence. This is achieved by calling every
+  element on the return value of the next element with the tail passed to it.
+  The continuation is passed to the very last argument. #lazy
+]--
+export const chain := \list. \cont. loop_over (list) {
+  cps head, list = pop list cont;
+  cps head;
+}
+
+macro new[..$items] =0x2p84=> mk_list macro::comma_list (..$items)
+
+macro mk_list ( macro::list_item $item $tail ) =0x1p254=> (cons $item mk_list $tail)
+macro mk_list macro::list_end =0x1p254=> end
+
+export ::(new)
+
+( macro pmatch::request (cons $head $tail)
+  =0x1p230=> await_subpatterns
+    (pmatch::request ($head))
+    (pmatch::request ($tail))
+)
+( macro await_subpatterns
+    (pmatch::response $h_expr ( $h_binds ))
+    (pmatch::response $t_expr ( $t_binds ))
+  =0x1p230=> pmatch::response (
+    pop
+      pmatch::value
+      pmatch::fail
+      \head. \tail. (
+        (\pmatch::pass. (\pmatch::value. $h_expr) head)
+        (pmatch::take_binds $h_binds (
+          (\pmatch::pass. (\pmatch::value. $t_expr) tail)
+          (pmatch::take_binds $t_binds (
+            pmatch::give_binds (pmatch::chain_binds $h_binds $t_binds) pmatch::pass
+          ))
+        ))
+      )
+  )
+  ( (pmatch::chain_binds $h_binds $t_binds) )
+)

orchidlang/src/libs/std/loop.orc

@@ -0,0 +1,75 @@
+import super::procedural::*
+import super::bool::*
+import super::fn::(return, identity)
+import super::known::*
+
+--[
+  Bare fixpoint combinator. Due to its many pitfalls, usercode is
+  recommended to use one of the wrappers such as [recursive] or
+  [loop_over] instead.
+]--
+export const Y := \f.(\x.f (x x))(\x.f (x x))
+
+--[
+  A syntax construct that encapsulates the Y combinator and encourages
+  single tail recursion. It's possible to use this for multiple or
+  non-tail recursion by using cps statements, but it's more ergonomic
+  than [Y] and more flexible than [std::list::fold].
+
+  To break out of the loop, use [std::fn::return] in a cps statement
+]--
+export macro loop_over (..$binds) {
+  ...$body
+} =0x5p129=> Y (\r. 
+  def_binds parse_binds (..$binds) do{
+    ...$body;
+    r apply_binds parse_binds (..$binds)
+  }
+) init_binds parse_binds (..$binds)
+
+-- parse_binds builds a conslist
+macro parse_binds (...$item, ...$tail:1) =0x2p250=> (
+  parse_bind (...$item)
+  parse_binds (...$tail)
+)
+macro parse_binds (...$item) =0x1p250=> (
+  parse_bind (...$item)
+  ()
+)
+
+-- while loop
+export macro statement (
+  while ..$condition (..$binds) {
+    ...$body
+  }
+) $next =0x5p129=> loop_over (..$binds) {
+  cps if (..$condition) then identity else return $next;
+  ...$body;
+}
+
+-- parse_bind converts items to pairs
+macro parse_bind ($name) =0x1p250=> ($name bind_no_value)
+macro parse_bind ($name = ...$value) =0x1p250=> ($name (...$value))
+
+-- def_binds creates name bindings for everything
+macro def_binds ( ($name $value) $tail ) ...$body =0x1p250=> (
+  \$name. def_binds $tail ...$body
+)
+macro def_binds () ...$body =0x1p250=> ...$body
+
+-- init_binds passes the value for initializers
+macro init_binds ( ($name bind_no_value) $tail ) =0x2p250=> $name init_binds $tail
+macro init_binds ( ($name $value) $tail ) =0x1p250=> $value init_binds $tail
+-- avoid empty templates by assuming that there is a previous token
+macro $fn init_binds () =0x1p250=> $fn
+
+-- apply_binds passes the name for initializers
+macro apply_binds ( ($name $value) $tail ) =0x1p250=> $name apply_binds $tail
+macro $fn apply_binds () =0x1p250=> $fn
+
+--[
+  Alias for the Y-combinator to avoid some universal pitfalls
+]--
+export macro recursive $name (..$binds) ...$body =0x5p129=> Y (\$name.
+  def_binds parse_binds (..$binds) ...$body
+) init_binds parse_binds (..$binds)

orchidlang/src/libs/std/macro.orc

@@ -0,0 +1,68 @@
+import std::number::add
+import std::known::*
+
+-- convert a comma-separated list into a linked list, with support for trailing commas
+export ::comma_list
+( macro comma_list ( ...$head, ...$tail:1 )
+  =0x2p254=> ( await_comma_list ( ...$head ) comma_list ( ...$tail ) )
+)
+( macro comma_list (...$only)
+  =0x1p254=> ( list_item (...$only) list_end )
+)
+( macro ( await_comma_list $head $tail )
+  =0x2p254=> ( list_item $head $tail )
+)
+( macro comma_list ()
+  =0x1p254=> list_end
+)
+( macro comma_list (...$data,)
+  =0x3p254=> comma_list (...$data)
+)
+
+-- convert a comma-separated list into a linked list, with support for trailing commas
+export ::semi_list
+( macro semi_list ( ...$head; ...$tail:1 )
+  =0x2p254=> ( await_semi_list ( ...$head ) semi_list ( ...$tail ) )
+)
+( macro semi_list (...$only)
+  =0x1p254=> ( list_item (...$only) list_end )
+)
+( macro ( await_semi_list $head $tail )
+  =0x2p254=> ( list_item $head $tail )
+)
+( macro semi_list ()
+  =0x1p254=> list_end
+)
+( macro semi_list (...$data;)
+  =0x3p254=> semi_list (...$data)
+)
+
+-- calculate the length of a linked list
+export ::length
+( macro length ( list_item $discard $tail )
+  =0x1p254=> await_length ( length $tail )
+)
+( macro await_length ( $len )
+  =0x1p254=> (add 1 $len)
+)
+macro length list_end =0x1p254=> (0)
+
+
+export ::error
+( macro ( ..$prefix error $details ..$suffix )
+  =0x2p255=> error $details
+)
+( macro [ ..$prefix error $details ..$suffix ]
+  =0x2p255=> error $details
+)
+( macro { ..$prefix error $details ..$suffix }
+  =0x2p255=> error $details
+)
+( macro error $details
+  =0x1p255=> 
+)
+
+export ::leftover_error
+( macro leftover_error $details
+  =0x1p255=> error ( "Token fails to parse" $details )
+)

orchidlang/src/libs/std/map.orc

@@ -0,0 +1,94 @@
+import super::(bool::*, fn::*, known::*, loop::*, procedural::*, string::*)
+import super::(panic, pmatch, macro, option, list, string, tuple, conv, pmatch::[=>])
+
+as_type (
+  impl string::conversion := \map. "map[" ++ (
+    unwrap map
+      |> list::map (
+        (tuple::t[k, v]) => conv::to_string k ++ " = " ++ conv::to_string v
+      )
+      |> list::reduce (\l. \r. l ++ ", " ++ r)
+      |> option::fallback ""
+  ) ++ "]"
+)
+
+--[ Constructors ]--
+
+const empty := wrap list::end
+const add := \m. \k. \v. wrap (
+  list::cons
+    tuple::t[k, v]
+    (unwrap m)
+)
+
+--[ List constructor ]--
+
+export ::new
+macro new[..$items] =0x2p84=> mk_map macro::comma_list (..$items)
+
+macro mk_map macro::list_end =0x1p254=> empty
+( macro mk_map ( macro::list_item ( ...$key = ...$value:1 ) $tail )
+  =0x1p254=> ( set mk_map $tail (...$key) (...$value) )
+)
+
+--[ Queries ]--
+
+-- return the last occurrence of a key if exists
+export const get := \m. \key. (
+  loop_over (m=unwrap m) {
+    cps record, m = list::pop m option::none;
+    cps if tuple::pick record 0 == key
+      then return $ option::some $ tuple::pick record 1
+      else identity;
+  }
+)
+
+--[ Commands ]--
+
+-- remove one occurrence of a key
+export const del := \m. \k. wrap (
+  recursive r (m=unwrap m)
+    list::pop m list::end \head. \tail.
+      if tuple::pick head 0 == k then tail
+      else list::cons head $ r tail
+)
+
+-- replace at most one occurrence of a key
+export const set := \m. \k. \v. m |> del k |> add k v
+
+export ::having
+( macro pmatch::request (having [..$items])
+  =0x1p230=> having_pattern (
+    pattern_walker
+      macro::comma_list ( ..$items )
+  )
+)
+( macro having_pattern ( tail_result $expr ( $binds ) )
+  =0x1p254=> pmatch::response $expr ( $binds )
+)
+( macro pattern_walker macro::list_end
+  =0x1p254=> tail_result pmatch::pass ( pmatch::no_binds )
+)
+( macro pattern_walker ( macro::list_item ( ...$key = ...$value:1 ) $tail )
+  =0x1p254=> await_pattern ( ...$key )
+    ( pmatch::request (...$value) )
+    ( pattern_walker $tail )
+)
+( macro await_pattern $key
+    ( pmatch::response $expr ( $binds ) )
+    ( tail_result $t_expr ( $t_binds ) )
+  =0x1p254=> tail_result (
+    option::handle (get pmatch::value $key)
+      pmatch::fail
+      \value. (\pmatch::pass. (\pmatch::value. $expr) value) (
+        pmatch::take_binds $binds (
+          (\pmatch::pass. $t_expr) (
+            pmatch::take_binds $t_binds (
+              pmatch::give_binds (pmatch::chain_binds $binds $t_binds) pmatch::pass
+            )
+          )
+        )
+      )
+  )
+  ( (pmatch::chain_binds $binds $t_binds) )
+)

orchidlang/src/libs/std/mod.rs

@@ -0,0 +1,18 @@
+//! Basic types and their functions, frequently used tools with no environmental
+//! dependencies.
+pub mod arithmetic_error;
+pub mod binary;
+mod bool;
+mod conv;
+mod cross_pipeline;
+pub mod exit_status;
+mod inspect;
+pub mod number;
+mod panic;
+pub mod protocol;
+pub mod reflect;
+pub mod runtime_error;
+mod state;
+pub mod std_system;
+pub mod string;
+pub mod tuple;

orchidlang/src/libs/std/number.orc

@@ -0,0 +1,16 @@
+import super::bool::*
+
+export ::(+, -, [*], %, /, <, >, <=, >=)
+
+const less_than_or_equal := \a. \b. a < b or a == b
+
+macro ...$a + ...$b =0x2p36=> (add (...$a) (...$b))
+macro ...$a:1 - ...$b =0x2p36=> (subtract (...$a) (...$b))
+macro ...$a * ...$b =0x1p36=> (multiply (...$a) (...$b))
+macro ...$a:1 % ...$b =0x1p36=> (remainder (...$a) (...$b))
+macro ...$a:1 / ...$b =0x1p36=> (divide (...$a) (...$b))
+macro ...$a:1 < ...$b =0x3p36=> (less_than (...$a) (...$b))
+macro ...$a:1 > ...$b =0x3p36=> ((...$b) < (...$a))
+macro ...$a:1 <= ...$b =0x3p36=> (less_than_or_equal (...$a) (...$b))
+macro ...$a:1 >= ...$b =0x3p36=> ((...$b) <= (...$a))
+

orchidlang/src/libs/std/number.rs

@@ -0,0 +1,141 @@
+//! `std::number` Numeric operations.
+
+use ordered_float::NotNan;
+
+use super::arithmetic_error::ArithmeticError;
+use crate::foreign::atom::Atomic;
+use crate::foreign::error::{AssertionError, RTError, RTResult};
+use crate::foreign::inert::Inert;
+use crate::foreign::to_clause::ToClause;
+use crate::foreign::try_from_expr::TryFromExpr;
+use crate::gen::tree::{xfn_ent, ConstTree};
+use crate::interpreter::nort::{Clause, Expr};
+use crate::location::CodeLocation;
+
+/// A number, either floating point or unsigned int, visible to Orchid.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum Numeric {
+  /// A nonnegative integer such as a size, index or count
+  Uint(usize),
+  /// A float other than NaN. Orchid has no silent errors
+  Float(NotNan<f64>),
+}
+
+impl Numeric {
+  /// Return the enclosed float, or cast the enclosed int to a float
+  pub fn as_f64(&self) -> f64 {
+    match self {
+      Numeric::Float(n) => **n,
+      Numeric::Uint(i) => *i as f64,
+    }
+  }
+
+  /// Returns the enclosed [NotNan], or casts and wraps the enclosed int
+  pub fn as_float(&self) -> NotNan<f64> {
+    match self {
+      Numeric::Float(n) => *n,
+      Numeric::Uint(i) => NotNan::new(*i as f64).expect("ints cannot cast to NaN"),
+    }
+  }
+
+  /// Wrap a f64 in a Numeric
+  pub fn new(value: f64) -> RTResult<Self> {
+    match value.is_finite() {
+      false => Err(ArithmeticError::Infinity.pack()),
+      true => match NotNan::new(value) {
+        Ok(f) => Ok(Self::Float(f)),
+        Err(_) => Err(ArithmeticError::NaN.pack()),
+      },
+    }
+  }
+}
+impl TryFromExpr for Numeric {
+  fn from_expr(exi: Expr) -> RTResult<Self> {
+    (exi.clause.request())
+      .ok_or_else(|| AssertionError::ext(exi.location(), "a numeric value", format!("{exi}")))
+  }
+}
+
+impl ToClause for Numeric {
+  fn to_clause(self, _: CodeLocation) -> Clause {
+    match self {
+      Numeric::Uint(i) => Inert(i).atom_cls(),
+      Numeric::Float(n) => Inert(n).atom_cls(),
+    }
+  }
+}
+
+/// Add two numbers. If they're both uint, the output is uint. If either is
+/// number, the output is number.
+pub fn add(a: Numeric, b: Numeric) -> RTResult<Numeric> {
+  match (a, b) {
+    (Numeric::Uint(a), Numeric::Uint(b)) =>
+      a.checked_add(b).map(Numeric::Uint).ok_or_else(|| ArithmeticError::Overflow.pack()),
+    (Numeric::Float(a), Numeric::Float(b)) => Numeric::new(*(a + b)),
+    (Numeric::Float(a), Numeric::Uint(b)) | (Numeric::Uint(b), Numeric::Float(a)) =>
+      Numeric::new(*a + b as f64),
+  }
+}
+
+/// Subtract a number from another. Always returns Number.
+pub fn subtract(a: Numeric, b: Numeric) -> RTResult<Numeric> {
+  match (a, b) {
+    (Numeric::Uint(a), Numeric::Uint(b)) => Numeric::new(a as f64 - b as f64),
+    (Numeric::Float(a), Numeric::Float(b)) => Numeric::new(*(a - b)),
+    (Numeric::Float(a), Numeric::Uint(b)) => Numeric::new(*a - b as f64),
+    (Numeric::Uint(a), Numeric::Float(b)) => Numeric::new(a as f64 - *b),
+  }
+}
+
+/// Multiply two numbers. If they're both uint, the output is uint. If either
+/// is number, the output is number.
+pub fn multiply(a: Numeric, b: Numeric) -> RTResult<Numeric> {
+  match (a, b) {
+    (Numeric::Uint(a), Numeric::Uint(b)) =>
+      a.checked_mul(b).map(Numeric::Uint).ok_or_else(|| ArithmeticError::Overflow.pack()),
+    (Numeric::Float(a), Numeric::Float(b)) => Numeric::new(*(a * b)),
+    (Numeric::Uint(a), Numeric::Float(b)) | (Numeric::Float(b), Numeric::Uint(a)) =>
+      Numeric::new(a as f64 * *b),
+  }
+}
+
+/// Divide a number by another. Always returns Number.
+pub fn divide(a: Numeric, b: Numeric) -> RTResult<Numeric> {
+  let a: f64 = a.as_f64();
+  let b: f64 = b.as_f64();
+  if b == 0.0 {
+    return Err(ArithmeticError::DivByZero.pack());
+  }
+  Numeric::new(a / b)
+}
+
+/// Take the remainder of two numbers.  If they're both uint, the output is
+/// uint. If either is number, the output is number.
+pub fn remainder(a: Numeric, b: Numeric) -> RTResult<Numeric> {
+  match (a, b) {
+    (Numeric::Uint(a), Numeric::Uint(b)) =>
+      a.checked_rem(b).map(Numeric::Uint).ok_or_else(|| ArithmeticError::DivByZero.pack()),
+    (Numeric::Float(a), Numeric::Float(b)) => Numeric::new(*(a % b)),
+    (Numeric::Uint(a), Numeric::Float(b)) => Numeric::new(a as f64 % *b),
+    (Numeric::Float(a), Numeric::Uint(b)) => Numeric::new(*a % b as f64),
+  }
+}
+
+/// Tries to use integer comparison, casts to float otherwise
+pub fn less_than(a: Numeric, b: Numeric) -> Inert<bool> {
+  match (a, b) {
+    (Numeric::Uint(a), Numeric::Uint(b)) => Inert(a < b),
+    (a, b) => Inert(a.as_f64() < b.as_f64()),
+  }
+}
+
+pub(super) fn num_lib() -> ConstTree {
+  ConstTree::ns("std::number", [ConstTree::tree([
+    xfn_ent("add", [add]),
+    xfn_ent("subtract", [subtract]),
+    xfn_ent("multiply", [multiply]),
+    xfn_ent("divide", [divide]),
+    xfn_ent("remainder", [remainder]),
+    xfn_ent("less_than", [less_than]),
+  ])])
+}

orchidlang/src/libs/std/option.orc

@@ -0,0 +1,42 @@
+import std::(panic, pmatch, string, conv)
+import std::(fn::*, string::*)
+
+as_type (
+  impl string::conversion := \opt. (
+    handle opt "none" \x. "some(" ++ conv::to_string x ++ ")"
+  )
+)
+
+export const some := \v. wrap \d. \f. f v
+export const none := wrap \d. \f. d
+
+export const handle := \t. \d. \f. (unwrap t) d f
+
+export const map := \option. \f. handle option none \x. some $ f x
+export const fallback := \option. \fallback. handle option fallback \data. data
+export const flatten := \option. handle option none \opt. wrap $ unwrap opt -- assert type
+export const flatmap := \option. \f. handle option none \opt. wrap $ unwrap $ f opt -- assert return
+export const assume := \option. handle option (panic "value expected") \x.x
+
+(
+  macro pmatch::request ( none )
+  =0x1p230=> pmatch::response (
+    handle pmatch::value
+      pmatch::pass
+      \_. pmatch::fail
+  ) ( pmatch::no_binds )
+)
+
+(
+  macro pmatch::request ( some ...$value )
+  =0x1p230=> await_some_subpattern ( pmatch::request (...$value) )
+)
+
+(
+  macro await_some_subpattern ( pmatch::response $expr ( $binds ) )
+  =0x1p254=> pmatch::response (
+    handle pmatch::value
+      pmatch::fail
+      \pmatch::value. $expr
+  ) ( $binds )
+)

orchidlang/src/libs/std/panic.rs

@@ -0,0 +1,30 @@
+use std::fmt;
+use std::sync::Arc;
+
+use never::Never;
+
+use super::string::OrcString;
+use crate::foreign::error::{RTError, RTResult};
+use crate::foreign::inert::Inert;
+use crate::gen::tree::{xfn_leaf, ConstTree};
+
+/// An unrecoverable error in Orchid land. Because Orchid is lazy, this only
+/// invalidates expressions that reference the one that generated it.
+#[derive(Clone)]
+pub struct OrchidPanic(Arc<String>);
+
+impl fmt::Display for OrchidPanic {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Orchid code panicked: {}", self.0)
+  }
+}
+
+impl RTError for OrchidPanic {}
+
+/// Takes a message, returns an [ExternError] unconditionally.
+pub fn orc_panic(msg: Inert<OrcString>) -> RTResult<Never> {
+  // any return value would work, but Clause is the simplest
+  Err(OrchidPanic(Arc::new(msg.0.get_string())).pack())
+}
+
+pub fn panic_lib() -> ConstTree { ConstTree::ns("std::panic", [xfn_leaf(orc_panic)]) }

orchidlang/src/libs/std/pmatch.orc

@@ -0,0 +1,105 @@
+import std::known::(_, ;)
+import std::procedural
+import std::bool
+import std::macro
+import std::panic
+
+--[
+  The protocol:
+
+  Request contains the pattern
+  Response contains an expression and the list of names 
+]--
+
+export ::(match, value, pass, fail, request, response, =>)
+
+(
+  macro ..$prefix:1 match ...$argument:0 { ..$body } ..$suffix:1
+  =0x1p130=> ..$prefix (
+    (\value. match_walker macro::semi_list ( ..$body ) )
+    ( ...$argument )
+  ) ..$suffix
+)
+
+macro match_walker macro::list_end =0x1p254=> panic "no arms match"
+( macro match_walker ( macro::list_item (...$pattern => ...$handler:1) $tail )
+  =0x1p254=> match_await ( request (...$pattern) ) (...$handler) ( match_walker $tail )
+)
+( macro match_await ( response $expr ( $binds ) ) $handler $tail
+  =0x1p254=> (\fail. (\pass. $expr) (take_binds $binds $handler)) $tail
+)
+
+macro request (( ..$pattern )) =0x1p254=> request ( ..$pattern )
+
+-- bindings list
+
+export ::(no_binds, add_bind, chain_binds, give_binds, take_binds)
+
+macro ( add_bind $_new no_binds ) =0x1p254=> ( binds_list $_new no_binds )
+( macro ( add_bind $_new (binds_list ...$tail) )
+  =0x1p254=> ( binds_list $_new (binds_list ...$tail) )
+)
+macro ( give_binds no_binds $cont ) =0x1p254=> $cont
+( macro ( give_binds ( binds_list $_name $tail ) $cont )
+  =0x1p254=> (( give_binds $tail $cont ) $_name )
+)
+macro ( take_binds no_binds $cont ) =0x1p254=> $cont
+( macro ( take_binds ( binds_list $_name $tail ) $cont )
+  =0x1p254=> ( take_binds $tail \$_name. $cont )
+)
+macro ( chain_binds no_binds $second ) =0x1p254=> $second
+( macro ( chain_binds ( binds_list $_head $tail ) $second )
+  =0x1p254=> ( add_bind $_head ( chain_binds $tail $second ))
+)
+
+--[ primitive pattern ( _ ) ]--
+
+(
+  macro request ( _ )
+  =0x1p230=> response pass ( no_binds )
+)
+
+--[ primitive name pattern ]--
+
+(
+  macro request ( $_name )
+  =0x1p226=> response ( pass value ) ( ( add_bind $_name no_binds ) )
+)
+
+--[ primitive pattern ( and ) ]--
+
+( macro request ( ...$lhs bool::and ...$rhs )
+  =0x3p230=> await_and_subpatterns ( request (...$lhs ) ) ( request ( ...$rhs ) )
+)
+
+( macro await_and_subpatterns ( response $lh_expr ( $lh_binds ) ) ( response $rh_expr ( $rh_binds ) )
+  =0x1p254=> response (
+      (\pass. $lh_expr) (take_binds $lh_binds (
+        (\pass. $rh_expr) (take_binds $rh_binds (
+          give_binds (chain_binds $lh_binds $rh_binds) pass
+        ))
+      ))
+    )
+    ( (chain_binds $lh_binds $rh_binds) ) 
+)
+
+--[ primitive pattern ( or ) ]--
+
+(
+  macro request ( ...$lhs bool::or ...$rhs )
+  =0x3p230=> await_or_subpatterns
+    ( request ( ...$lhs ) )
+    ( request ( ...$rhs ) )
+)
+
+( -- for this to work, lh and rh must produce the same bindings
+  macro await_or_subpatterns ( response $lh_expr ( $lh_binds) ) ( response $rh_expr ( $rh_binds ) )
+  =0x1p254=> response (
+    (\fail. $lh_expr) -- lh works with pass directly because its bindings are reported up
+    ($rh_expr (take_binds $rh_binds -- rh runs if lh cancels
+      (give_binds $lh_binds pass) -- translate rh binds to lh binds
+    ))
+  )
+  ( $lh_binds ) -- report lh bindings
+)
+

orchidlang/src/libs/std/prelude.orc

@@ -0,0 +1,25 @@
+import std::number::*
+export ::[+ - * / % < > <= >=]
+import std::string::*
+export ::[++]
+import std::bool::*
+export ::([== !=], if, then, else, true, false, and, or, not)
+import std::fn::*
+export ::([$ |>], identity, pass, pass2, return)
+import std::procedural::*
+export ::(do, let, cps)
+import std::tuple::t
+export ::(t)
+import std::pmatch::(match, [=>])
+export ::(match, [=>])
+import std::loop::*
+export ::(loop_over, recursive, while)
+
+import std::known::*
+export ::[, _ ; . =]
+
+import std::(tuple, list, map, option, exit_status)
+export ::(tuple, list, map, option, exit_status)
+
+import std
+export ::(std)

orchidlang/src/libs/std/procedural.orc

@@ -0,0 +1,36 @@
+import super::pmatch::=>
+import super::known::*
+
+export ::(do, statement, [;])
+
+-- remove duplicate ;-s
+macro do {
+  ...$statement ; ; ...$rest:1
+} =0x3p130=> do { 
+  ...$statement ; ...$rest
+}
+-- modular operation block that returns a value
+macro do {
+  ...$statement ; ...$rest:1
+} =0x2p130=> statement (...$statement) (do { ...$rest })
+macro do { ...$return } =0x1p130=> (...$return)
+
+export ::let
+
+macro statement (let $_name = ...$value) (...$next) =0x2p230=> (
+  ( \$_name. ...$next) (...$value)
+)
+macro statement (let ...$pattern = ...$value:1) (...$next) =0x1p230=> (
+  ( (...$pattern) => (...$next) ) (...$value)
+)
+
+export ::cps
+
+-- modular operation block that returns a CPS function
+macro do cps { ...$body } =0x1p130=> \cont. do { ...$body ; cont }
+macro statement (cps ...$names = ...$operation:1) (...$next) =0x2p230=> (
+  (...$operation) ( (...$names) => ...$next )
+)
+macro statement (cps ...$operation) (...$next) =0x1p230=> (
+  (...$operation) (...$next)
+)

orchidlang/src/libs/std/protocol.orc

@@ -0,0 +1,8 @@
+import std::(map, option, fn::*)
+
+export const vcall := \proto. \key. \val. (
+  resolve proto val
+    |> map::get key
+    |> option::assume
+    $ break val
+)

orchidlang/src/libs/std/protocol.rs

@@ -0,0 +1,411 @@
+//! Polymorphism through Elixir-style protocols associated with type-tagged
+//! values. A type-tag seals the value, and can only be unwrapped explicitly by
+//! providing the correct type. This ensures that code that uses this explicit
+//! polymorphism never uses the implicit polymorphism of dynamic typing.
+//!
+//! Atoms can also participate in this controlled form of polymorphism by
+//! offering a [Tag] in their [crate::utils::ddispatch::Responder] callback.
+//!
+//! Protocols and types are modules with magic elements that distinguish them
+//! from regular modules.
+
+use std::sync::Arc;
+use std::{fmt, iter};
+
+use const_format::formatcp;
+use hashbrown::HashMap;
+use intern_all::{i, Tok};
+use itertools::Itertools;
+
+use super::cross_pipeline::defer_to_runtime;
+use super::reflect::refer_seq;
+use super::runtime_error::RuntimeError;
+use crate::error::ProjectResult;
+use crate::foreign::atom::Atomic;
+use crate::foreign::error::RTResult;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::foreign::process::Unstable;
+use crate::gen::tpl;
+use crate::gen::traits::GenClause;
+use crate::gen::tree::{atom_ent, leaf, xfn_ent, ConstTree};
+use crate::interpreter::nort;
+use crate::interpreter::nort::ClauseInst;
+use crate::libs::parse_custom_line::custom_line;
+use crate::location::SourceRange;
+use crate::name::{Sym, VName};
+use crate::parse::frag::Frag;
+use crate::parse::lexer::Lexeme;
+use crate::parse::parse_plugin::{ParseLinePlugin, ParsePluginReq};
+use crate::parse::parsed::{
+  self, Constant, Member, MemberKind, ModuleBlock, PType, SourceLine, SourceLineKind,
+};
+use crate::utils::ddispatch::Request;
+
+// TODO: write an example that thoroughly tests this module. Test rust-interop
+// with Tuple
+
+/// Information available both for protocols and for tagged values
+#[derive(Clone)]
+pub struct TypeData {
+  /// The full path of the module designated to represent this type or protocol.
+  /// Also used to key impl tables in the counterpart (tag or protocol)
+  pub id: Sym,
+  /// Maps IDs of the counterpart (tag or protocol) to implementations
+  pub impls: Arc<HashMap<Sym, nort::Expr>>,
+}
+impl TypeData {
+  /// Create a new type data record from a known name and impls
+  pub fn new(id: Sym, impls: impl IntoIterator<Item = (Sym, nort::Expr)>) -> Self {
+    Self { id, impls: Arc::new(impls.into_iter().collect()) }
+  }
+}
+
+fn mk_mod<'a>(
+  rest: impl IntoIterator<Item = (&'a str, ConstTree)>,
+  impls: HashMap<Sym, nort::Expr>,
+  profile: ImplsProfile<impl WrapImpl>,
+) -> ConstTree {
+  ConstTree::tree(rest.into_iter().chain([
+    (profile.own_id, leaf(tpl::A(tpl::C("std::reflect::modname"), tpl::V(Inert(1))))),
+    atom_ent(TYPE_KEY, [use_wrap(profile.wrap, impls)]),
+  ]))
+}
+
+fn to_mod<'a>(
+  rest: impl IntoIterator<Item = (&'a str, ConstTree)>,
+  data: TypeData,
+  profile: ImplsProfile<impl WrapImpl>,
+) -> ConstTree {
+  let id = data.id.clone();
+  ConstTree::tree(rest.into_iter().chain([
+    atom_ent(profile.own_id, [Unstable::new(move |r| {
+      assert!(r.location.module == id, "Pre-initilaized type lib mounted on wrong prefix");
+      Inert(r.location.module)
+    })]),
+    atom_ent(TYPE_KEY, [profile.wrap.wrap(data)]),
+  ]))
+}
+
+/// Key for type data. The value is either [Inert<Protocol>] or [Inert<Tag>]
+const TYPE_KEY: &str = "__type_data__";
+
+/// A shared behaviour that may implement itself for types, and may be
+/// implemented by types.
+#[derive(Clone)]
+pub struct Protocol(pub TypeData);
+impl Protocol {
+  /// Name of the member the ID must be assigned to for a module to be
+  /// recognized as a protocol.
+  pub const ID_KEY: &'static str = "__protocol_id__";
+  const fn profile() -> ImplsProfile<impl WrapImpl> {
+    ImplsProfile {
+      wrap: |t| Inert(Protocol(t)),
+      own_id: Protocol::ID_KEY,
+      other_id: Tag::ID_KEY,
+      prelude: formatcp!(
+        "import std
+        const {} := std::reflect::modname 1
+        const resolve := std::protocol::resolve {TYPE_KEY}
+        const vcall := std::protocol::vcall {TYPE_KEY}",
+        Protocol::ID_KEY
+      ),
+    }
+  }
+
+  /// Create a new protocol with a pre-determined name
+  pub fn new(id: Sym, impls: impl IntoIterator<Item = (Sym, nort::Expr)>) -> Self {
+    Self(TypeData::new(id, impls))
+  }
+
+  /// Attach a pre-existing protocol to the tree. Consider [Protocol::tree].
+  pub fn to_tree<'a>(&self, rest: impl IntoIterator<Item = (&'a str, ConstTree)>) -> ConstTree {
+    to_mod(rest, self.0.clone(), Self::profile())
+  }
+
+  /// Create a new protocol definition
+  pub fn tree<'a>(
+    impls: impl IntoIterator<Item = (Sym, nort::Expr)>,
+    rest: impl IntoIterator<Item = (&'a str, ConstTree)>,
+  ) -> ConstTree {
+    mk_mod(rest, impls.into_iter().collect(), Self::profile())
+  }
+}
+impl fmt::Debug for Protocol {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Protocol({})", self.0.id) }
+}
+impl InertPayload for Protocol {
+  const TYPE_STR: &'static str = "Protocol";
+}
+
+/// A type marker that can be attached to values to form a [Tagged]
+#[derive(Clone)]
+pub struct Tag(pub TypeData);
+impl Tag {
+  const ID_KEY: &'static str = "__type_id__";
+  const fn profile() -> ImplsProfile<impl WrapImpl> {
+    ImplsProfile {
+      wrap: |t| Inert(Tag(t)),
+      own_id: Tag::ID_KEY,
+      other_id: Protocol::ID_KEY,
+      prelude: formatcp!(
+        "import std
+        const {} := std::reflect::modname 1
+        const unwrap := std::protocol::unwrap {TYPE_KEY}
+        const wrap := std::protocol::wrap {TYPE_KEY}",
+        Tag::ID_KEY
+      ),
+    }
+  }
+
+  /// Create a new type-tag with a pre-determined name
+  pub fn new(id: Sym, impls: impl IntoIterator<Item = (Sym, nort::Expr)>) -> Self {
+    Self(TypeData::new(id, impls))
+  }
+
+  /// Attach a pre-existing type-tag to the tree. Consider [Tag::tree]
+  pub fn to_tree<'a>(&self, rest: impl IntoIterator<Item = (&'a str, ConstTree)>) -> ConstTree {
+    to_mod(rest, self.0.clone(), Self::profile())
+  }
+
+  /// Create a new tag
+  pub fn tree<'a>(
+    impls: impl IntoIterator<Item = (Sym, nort::Expr)>,
+    rest: impl IntoIterator<Item = (&'a str, ConstTree)>,
+  ) -> ConstTree {
+    mk_mod(rest, impls.into_iter().collect(), Self::profile())
+  }
+}
+impl fmt::Debug for Tag {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Tag({})", self.0.id) }
+}
+impl InertPayload for Tag {
+  const TYPE_STR: &'static str = "Tag";
+  fn strict_eq(&self, other: &Self) -> bool { self.0.id == other.0.id }
+}
+
+/// A value with a type [Tag]
+#[derive(Clone)]
+pub struct Tagged {
+  /// Type information
+  pub tag: Tag,
+  /// Value
+  pub value: nort::Expr,
+}
+impl fmt::Debug for Tagged {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Tagged({:?} {})", self.tag, self.value)
+  }
+}
+impl InertPayload for Tagged {
+  const TYPE_STR: &'static str = "Tagged";
+  fn respond(&self, mut request: Request) { request.serve_with(|| self.tag.clone()) }
+}
+
+fn parse_impl(
+  tail: Frag,
+  req: &dyn ParsePluginReq,
+) -> Option<ProjectResult<(VName, parsed::Expr)>> {
+  custom_line(tail, i!(str: "impl"), false, req).map(|res| {
+    let (_, tail, _) = res?;
+    let (name, tail) = req.parse_nsname(tail)?;
+    let (walrus, tail) = req.pop(tail.trim())?;
+    req.expect(Lexeme::Walrus, walrus)?;
+    let (body, empty) = req.parse_exprv(tail, None)?;
+    req.expect_empty(empty)?;
+    let value = req.vec_to_single(tail.fallback, body)?;
+    Ok((name, value))
+  })
+}
+
+struct Impl {
+  target: Sym,
+  value: parsed::Expr,
+}
+
+fn extract_impls(
+  tail: Frag,
+  req: &dyn ParsePluginReq,
+  range: SourceRange,
+  typeid_name: Tok<String>,
+) -> ProjectResult<(Vec<SourceLine>, Vec<Impl>)> {
+  let mut lines = Vec::new();
+  let mut impls = Vec::new(); // name1, value1, name2, value2, etc...
+  for line in req.split_lines(tail) {
+    match parse_impl(line, req) {
+      Some(result) => {
+        let (name, value) = result?;
+        let target = Sym::new(name.suffix([typeid_name.clone()])).unwrap();
+        impls.push(Impl { target, value });
+      },
+      None => lines.extend((req.parse_line(line)?.into_iter()).map(|k| k.wrap(range.clone()))),
+    }
+  }
+  Ok((lines, impls))
+}
+
+trait WrapImpl: Clone + Copy + Send + Sync + 'static {
+  type R: Atomic + Clone + 'static;
+  fn wrap(&self, data: TypeData) -> Self::R;
+}
+impl<R: Atomic + Clone + 'static, F: Fn(TypeData) -> R + Clone + Copy + Send + Sync + 'static>
+  WrapImpl for F
+{
+  type R = R;
+  fn wrap(&self, data: TypeData) -> Self::R { self(data) }
+}
+fn use_wrap(wrap: impl WrapImpl, impls: HashMap<Sym, nort::Expr>) -> impl Atomic + Clone + 'static {
+  Unstable::new(move |r| wrap.wrap(TypeData::new(r.location.module, impls)))
+}
+
+#[derive(Debug, Clone)]
+struct ImplsProfile<W: WrapImpl> {
+  wrap: W,
+  own_id: &'static str,
+  other_id: &'static str,
+  prelude: &'static str,
+}
+
+fn parse_body_with_impls(
+  body: Frag,
+  req: &dyn ParsePluginReq,
+  range: SourceRange,
+  profile: ImplsProfile<impl WrapImpl>,
+) -> ProjectResult<Vec<SourceLine>> {
+  let ImplsProfile { other_id, prelude, wrap, .. } = profile.clone();
+  let (mut lines, impls) = extract_impls(body, req, range.clone(), i(other_id))?;
+  let line_loc = range.clone();
+  let type_data = defer_to_runtime(
+    range.clone(),
+    impls.into_iter().flat_map(move |Impl { target, value }| {
+      [vec![parsed::Clause::Name(target).into_expr(line_loc.clone())], vec![value]]
+    }),
+    move |pairs: Vec<nort::Expr>| -> RTResult<_> {
+      debug_assert_eq!(pairs.len() % 2, 0, "key-value pairs");
+      let mut impls = HashMap::with_capacity(pairs.len() / 2);
+      for (name, value) in pairs.into_iter().tuples() {
+        impls.insert(name.downcast::<Inert<Sym>>()?.0, value);
+      }
+      Ok(use_wrap(wrap, impls))
+    },
+  );
+  let type_data_line = Constant { name: i(TYPE_KEY), value: type_data.into_expr(range.clone()) };
+  lines.extend(req.parse_entries(prelude, range.clone()));
+  lines.push(MemberKind::Constant(type_data_line).into_line(true, range));
+  Ok(lines)
+}
+
+#[derive(Clone)]
+struct ProtocolParser;
+impl ParseLinePlugin for ProtocolParser {
+  fn parse(&self, req: &dyn ParsePluginReq) -> Option<ProjectResult<Vec<SourceLineKind>>> {
+    custom_line(req.frag(), i!(str: "protocol"), true, req).map(|res| {
+      let (exported, tail, line_loc) = res?;
+      let (name, tail) = req.pop(tail)?;
+      let name = req.expect_name(name)?;
+      let tail = req.expect_block(tail, PType::Par)?;
+      let body = parse_body_with_impls(tail, req, line_loc, Protocol::profile())?;
+      let kind = MemberKind::Module(ModuleBlock { name, body });
+      Ok(vec![SourceLineKind::Member(Member { exported, kind })])
+    })
+  }
+}
+
+#[derive(Clone)]
+struct TypeParser;
+impl ParseLinePlugin for TypeParser {
+  fn parse(&self, req: &dyn ParsePluginReq) -> Option<ProjectResult<Vec<SourceLineKind>>> {
+    custom_line(req.frag(), i!(str: "type"), true, req).map(|res| {
+      let (exported, tail, line_loc) = res?;
+      let (name, tail) = req.pop(tail)?;
+      let name = req.expect_name(name)?;
+      let tail = req.expect_block(tail, PType::Par)?;
+      let body = parse_body_with_impls(tail, req, line_loc, Tag::profile())?;
+      let kind = MemberKind::Module(ModuleBlock { name, body });
+      Ok(vec![SourceLineKind::Member(Member { exported, kind })])
+    })
+  }
+}
+
+#[derive(Clone)]
+struct AsProtocolParser;
+impl ParseLinePlugin for AsProtocolParser {
+  fn parse(&self, req: &dyn ParsePluginReq) -> Option<ProjectResult<Vec<SourceLineKind>>> {
+    custom_line(req.frag(), i!(str: "as_protocol"), false, req).map(|res| {
+      let (_, tail, line_loc) = res?;
+      let body = req.expect_block(tail, PType::Par)?;
+      parse_body_with_impls(body, req, line_loc, Protocol::profile())
+        .map(|v| v.into_iter().map(|e| e.kind).collect())
+    })
+  }
+}
+
+#[derive(Clone)]
+struct AsTypeParser;
+impl ParseLinePlugin for AsTypeParser {
+  fn parse(&self, req: &dyn ParsePluginReq) -> Option<ProjectResult<Vec<SourceLineKind>>> {
+    custom_line(req.frag(), i!(str: "as_type"), false, req).map(|res| {
+      let (_, tail, line_loc) = res?;
+      let body = req.expect_block(tail, PType::Par)?;
+      parse_body_with_impls(body, req, line_loc, Tag::profile())
+        .map(|v| v.into_iter().map(|e| e.kind).collect())
+    })
+  }
+}
+
+/// Collection of all the parser plugins defined here
+pub fn parsers() -> Vec<Box<dyn ParseLinePlugin>> {
+  vec![
+    Box::new(ProtocolParser),
+    Box::new(TypeParser),
+    Box::new(AsTypeParser),
+    Box::new(AsProtocolParser),
+  ]
+}
+
+/// Check and remove the type tag from a value
+pub fn unwrap(tag: Inert<Tag>, tagged: Inert<Tagged>) -> RTResult<nort::Expr> {
+  if tagged.tag.strict_eq(&tag) {
+    return Ok(tagged.value.clone());
+  }
+  let msg = format!("expected {:?} but got {:?}", tag, tagged.tag);
+  RuntimeError::fail(msg, "unwrapping type-tagged value")
+}
+
+/// Attach a type tag to a value
+pub fn wrap(tag: Inert<Tag>, value: nort::Expr) -> Inert<Tagged> {
+  Inert(Tagged { tag: tag.0, value })
+}
+
+/// Find the implementation of a protocol for a given value
+pub fn resolve(protocol: Inert<Protocol>, value: ClauseInst) -> RTResult<nort::Expr> {
+  let tag = value.request::<Tag>().ok_or_else(|| {
+    let msg = format!("{value} is not type-tagged");
+    RuntimeError::ext(msg, "resolving protocol impl")
+  })?;
+  if let Some(implem) = protocol.0.0.impls.get(&tag.0.id) {
+    Ok(implem.clone())
+  } else if let Some(implem) = tag.0.impls.get(&protocol.0.0.id) {
+    Ok(implem.clone())
+  } else {
+    let message = format!("{tag:?} doesn't implement {protocol:?}");
+    RuntimeError::fail(message, "dispatching protocol")
+  }
+}
+
+/// Generate a call to [resolve] bound to the given protocol
+pub const fn gen_resolv(name: &'static str) -> impl GenClause {
+  tpl::A(
+    tpl::C("std::protocol::resolve"),
+    tpl::V(Unstable::new(move |_| refer_seq(name.split("::").chain(iter::once(TYPE_KEY))))),
+  )
+}
+
+/// All the functions exposed by the std::protocol library
+pub fn protocol_lib() -> ConstTree {
+  ConstTree::ns("std::protocol", [ConstTree::tree([
+    xfn_ent("unwrap", [unwrap]),
+    xfn_ent("wrap", [wrap]),
+    xfn_ent("resolve", [resolve]),
+    xfn_ent("break", [|t: Inert<Tagged>| t.0.value]),
+  ])])
+}

orchidlang/src/libs/std/reflect.rs

@@ -0,0 +1,78 @@
+//! `std::reflect` Abstraction-breaking operations for dynamically constructing
+//! [Clause::Constant] references.
+
+use std::hash::Hash;
+use std::sync::atomic::{self, AtomicUsize};
+use std::{cmp, fmt};
+
+use intern_all::i;
+
+use super::runtime_error::RuntimeError;
+use super::string::OrcString;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::foreign::try_from_expr::WithLoc;
+use crate::gen::tree::{xfn_ent, ConstTree};
+use crate::interpreter::nort::{self, Clause};
+use crate::name::Sym;
+
+impl InertPayload for Sym {
+  const TYPE_STR: &'static str = "SymbolName";
+  fn strict_eq(&self, o: &Self) -> bool { self == o }
+}
+
+/// Generate a constant reference at runtime. Referencing a nonexistent constant
+/// is a runtime error.
+pub fn refer_seq(name: impl IntoIterator<Item = &'static str>) -> Clause {
+  Clause::Constant(Sym::new(name.into_iter().map(i)).expect("Empty name"))
+}
+
+/// Generate a constant reference at runtime. Referencing a nonexistent constant
+/// is a runtime error.
+pub fn refer(name: &'static str) -> Clause { refer_seq(name.split("::")) }
+
+static COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+/// A struct that equals its own copies and only its own copies
+#[derive(Clone)]
+pub struct RefEqual(usize);
+impl RefEqual {
+  /// Create a new [RefEqual] which is initially completely unique
+  #[allow(clippy::new_without_default)] // new has semantic meaning
+  pub fn new() -> Self { Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) }
+  /// Return the unique identifier of this [RefEqual] and its copies
+  pub fn id(&self) -> usize { self.0 }
+}
+impl fmt::Debug for RefEqual {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    f.debug_tuple("RefEqual").field(&self.id()).finish()
+  }
+}
+impl InertPayload for RefEqual {
+  const TYPE_STR: &'static str = "RefEqual";
+  fn strict_eq(&self, other: &Self) -> bool { self == other }
+}
+impl Eq for RefEqual {}
+impl PartialEq for RefEqual {
+  fn eq(&self, other: &Self) -> bool { self.id() == other.id() }
+}
+impl Ord for RefEqual {
+  fn cmp(&self, other: &Self) -> cmp::Ordering { self.id().cmp(&other.id()) }
+}
+impl PartialOrd for RefEqual {
+  fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> { Some(self.cmp(other)) }
+}
+impl Hash for RefEqual {
+  fn hash<H: std::hash::Hasher>(&self, state: &mut H) { self.id().hash(state) }
+}
+
+pub(super) fn reflect_lib() -> ConstTree {
+  ConstTree::ns("std::reflect", [ConstTree::tree([
+    xfn_ent("ref_equal", [|l: Inert<RefEqual>, r: Inert<RefEqual>| Inert(l.0.id() == r.0.id())]),
+    xfn_ent("modname", [|WithLoc(loc, _): WithLoc<nort::Expr>| Inert(loc.module)]),
+    xfn_ent("symbol", [|s: Inert<OrcString>| {
+      Sym::parse(s.0.as_str())
+        .map(Inert)
+        .map_err(|_| RuntimeError::ext("empty string".to_string(), "converting string to Symbol"))
+    }]),
+  ])])
+}

orchidlang/src/libs/std/result.orc

@@ -0,0 +1,12 @@
+import std::panic
+
+as_type ()
+
+export const ok := \v. wrap \fe. \fv. fv v
+export const err := \e. wrap \fe. \fv. fe e
+
+export const map := \result. \fv. unwrap result err fv
+export const map_err := \result. \fe. unwrap result fe ok
+export const flatten := \result. unwrap result err \res. wrap (unwrap res)
+export const and_then := \result. \f. unwrap result err \v. f v
+export const assume := \result. unwrap result (\e. panic "value expected") \v.v

orchidlang/src/libs/std/runtime_error.rs

@@ -0,0 +1,34 @@
+//! Errors thrown by the standard library in lieu of in-language error handling
+//! for runtime errors such as missing files.
+
+use std::fmt;
+
+use crate::foreign::error::{RTError, RTErrorObj, RTResult};
+
+/// Some external event prevented the operation from succeeding
+#[derive(Clone)]
+pub struct RuntimeError {
+  message: String,
+  operation: &'static str,
+}
+
+impl RuntimeError {
+  /// Construct, upcast and wrap in a Result that never succeeds for easy
+  /// short-circuiting
+  pub fn fail<T>(message: String, operation: &'static str) -> RTResult<T> {
+    Err(Self { message, operation }.pack())
+  }
+
+  /// Construct and upcast to [RTErrorObj]
+  pub fn ext(message: String, operation: &'static str) -> RTErrorObj {
+    Self { message, operation }.pack()
+  }
+}
+
+impl fmt::Display for RuntimeError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Error while {}: {}", self.operation, self.message)
+  }
+}
+
+impl RTError for RuntimeError {}

orchidlang/src/libs/std/state.rs

@@ -0,0 +1,80 @@
+use std::sync::{Arc, Mutex};
+
+use crate::foreign::fn_bridge::Thunk;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::gen::tpl;
+use crate::gen::traits::Gen;
+use crate::gen::tree::{xfn_ent, ConstTree};
+use crate::interpreter::gen_nort::nort_gen;
+use crate::interpreter::handler::HandlerTable;
+use crate::interpreter::nort::Expr;
+
+#[derive(Debug, Clone)]
+pub struct State(Arc<Mutex<Expr>>);
+impl InertPayload for State {
+  const TYPE_STR: &'static str = "State";
+}
+
+#[derive(Debug, Clone)]
+struct NewStateCmd(Expr, Expr);
+impl InertPayload for NewStateCmd {
+  const TYPE_STR: &'static str = "NewStateCmd";
+  fn strict_eq(&self, _: &Self) -> bool { true }
+}
+#[derive(Debug, Clone)]
+struct SetStateCmd(State, Expr, Expr);
+impl InertPayload for SetStateCmd {
+  const TYPE_STR: &'static str = "SetStateCmd";
+}
+
+#[derive(Debug, Clone)]
+struct GetStateCmd(State, Expr);
+impl InertPayload for GetStateCmd {
+  const TYPE_STR: &'static str = "GetStateCmd";
+}
+
+fn new_state(default: Thunk, cont: Thunk) -> Inert<NewStateCmd> {
+  Inert(NewStateCmd(default.0, cont.0))
+}
+
+fn get_state(s: Inert<State>, cont: Thunk) -> Inert<GetStateCmd> { Inert(GetStateCmd(s.0, cont.0)) }
+
+fn set_state(s: Inert<State>, value: Thunk, cont: Thunk) -> Inert<SetStateCmd> {
+  Inert(SetStateCmd(s.0, value.0, cont.0))
+}
+
+fn new_state_handler(cmd: &Inert<NewStateCmd>) -> Expr {
+  let Inert(NewStateCmd(default, handler)) = cmd;
+  let state = State(Arc::new(Mutex::new(default.clone())));
+  let tpl = tpl::A(tpl::Slot, tpl::V(Inert(state)));
+  tpl.template(nort_gen(handler.location()), [handler.clone()])
+}
+
+fn set_state_handler(cmd: &Inert<SetStateCmd>) -> Expr {
+  let Inert(SetStateCmd(state, value, handler)) = cmd;
+  *state.0.lock().unwrap() = value.clone();
+  handler.clone()
+}
+
+fn get_state_handler(cmd: &Inert<GetStateCmd>) -> Expr {
+  let Inert(GetStateCmd(state, handler)) = cmd;
+  let val = state.0.lock().unwrap().clone();
+  let tpl = tpl::A(tpl::Slot, tpl::Slot);
+  tpl.template(nort_gen(handler.location()), [handler.clone(), val])
+}
+
+pub fn state_handlers() -> HandlerTable<'static> {
+  let mut handlers = HandlerTable::new();
+  handlers.register(new_state_handler);
+  handlers.register(get_state_handler);
+  handlers.register(set_state_handler);
+  handlers
+}
+
+pub fn state_lib() -> ConstTree {
+  ConstTree::ns("std::state", [ConstTree::tree([
+    xfn_ent("new_state", [new_state]),
+    xfn_ent("get_state", [get_state]),
+    xfn_ent("set_state", [set_state]),
+  ])])
+}

orchidlang/src/libs/std/std_system.rs

@@ -0,0 +1,78 @@
+//! Add the standard library's constants and mcacros to an Orchid environment
+
+#![allow(non_upper_case_globals)]
+
+use rust_embed::RustEmbed;
+
+use super::binary::bin_lib;
+use super::bool::bool_lib;
+use super::conv::conv_lib;
+use super::exit_status::exit_status_lib;
+use super::inspect::inspect_lib;
+use super::number::num_lib;
+use super::panic::panic_lib;
+use super::protocol::{parsers, protocol_lib};
+use super::reflect::reflect_lib;
+use super::state::{state_handlers, state_lib};
+use super::string::{str_lib, StringLexer};
+use super::tuple::tuple_lib;
+use crate::facade::system::{IntoSystem, System};
+use crate::gen::tree::{ConstCombineErr, ConstTree};
+use crate::location::CodeGenInfo;
+use crate::pipeline::load_project::Prelude;
+use crate::tree::ModEntry;
+use crate::utils::combine::Combine;
+use crate::virt_fs::{EmbeddedFS, VirtFS};
+use crate::{sym, vname};
+
+#[derive(RustEmbed)]
+#[folder = "src/libs/std"]
+#[include = "*.orc"]
+struct StdEmbed;
+
+/// Feature flags for the STL.
+#[derive(Default)]
+pub struct StdConfig {
+  /// Whether impure functions (such as io::debug) are allowed. An embedder
+  /// would typically disable this flag
+  pub impure: bool,
+}
+impl StdConfig {
+  fn stdlib(&self) -> Result<ConstTree, ConstCombineErr> {
+    let pure_tree = tuple_lib()
+      .combine(bin_lib())?
+      .combine(bool_lib())?
+      .combine(conv_lib())?
+      .combine(exit_status_lib())?
+      .combine(num_lib())?
+      .combine(panic_lib())?
+      .combine(protocol_lib())?
+      .combine(reflect_lib())?
+      .combine(state_lib())?
+      .combine(str_lib())?;
+    if !self.impure {
+      return Ok(pure_tree);
+    }
+    pure_tree.combine(inspect_lib())
+  }
+}
+
+impl IntoSystem<'static> for StdConfig {
+  fn into_system(self) -> System<'static> {
+    System {
+      name: "stdlib",
+      constants: self.stdlib().expect("stdlib tree is malformed"),
+      code: ModEntry::ns("std", [ModEntry::leaf(
+        EmbeddedFS::new::<StdEmbed>(".orc", CodeGenInfo::no_details(sym!(std::fs))).rc(),
+      )]),
+      prelude: vec![Prelude {
+        target: vname!(std::prelude),
+        exclude: vname!(std),
+        owner: CodeGenInfo::no_details(sym!(std::prelude)),
+      }],
+      handlers: state_handlers(),
+      lexer_plugins: vec![Box::new(StringLexer)],
+      line_parsers: parsers(),
+    }
+  }
+}

orchidlang/src/libs/std/string.orc

@@ -0,0 +1,10 @@
+import super::(procedural::*, bool::*, panic, inspect, known::*)
+
+export macro ...$a ++ ...$b =0x4p36=> (concat (...$a) (...$b))
+
+export const char_at := \s. \i. do{
+  let slc = slice s i 1;
+  if len slc == 1
+  then slc
+  else panic "Character index out of bounds"
+}

orchidlang/src/libs/std/string.rs

@@ -0,0 +1,401 @@
+//! `std::string` String processing
+
+use std::fmt;
+use std::fmt::Write as _;
+use std::hash::Hash;
+use std::ops::Deref;
+use std::sync::Arc;
+
+use intern_all::{i, Tok};
+use itertools::Itertools;
+use unicode_segmentation::UnicodeSegmentation;
+
+use super::protocol::{gen_resolv, Protocol};
+use super::runtime_error::RuntimeError;
+use crate::error::{ProjectErrorObj, ProjectResult};
+use crate::foreign::atom::{AtomGenerator, Atomic};
+use crate::foreign::error::RTResult;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::foreign::to_clause::ToClause;
+use crate::foreign::try_from_expr::{TryFromExpr, WithLoc};
+use crate::gen::tpl;
+use crate::gen::traits::Gen;
+use crate::gen::tree::{xfn_ent, ConstTree};
+use crate::interpreter::gen_nort::nort_gen;
+use crate::interpreter::nort::{Clause, Expr};
+use crate::location::CodeLocation;
+use crate::parse::context::ParseCtx;
+use crate::parse::errors::ParseErrorKind;
+use crate::parse::lex_plugin::{LexPluginRecur, LexPluginReq, LexerPlugin};
+use crate::parse::lexer::{Entry, LexRes, Lexeme};
+use crate::parse::parsed::PType;
+use crate::utils::iter_find::iter_find;
+
+/// An Orchid string which may or may not be interned
+#[derive(Clone, Eq)]
+pub enum OrcString {
+  /// An interned string. Equality-conpared by reference.
+  Interned(Tok<String>),
+  /// An uninterned bare string. Equality-compared by character
+  Runtime(Arc<String>),
+}
+
+impl fmt::Debug for OrcString {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Runtime(s) => write!(f, "r\"{s}\""),
+      Self::Interned(t) => write!(f, "i\"{t}\""),
+    }
+  }
+}
+
+impl OrcString {
+  /// Intern the contained string
+  pub fn intern(&mut self) {
+    if let Self::Runtime(t) = self {
+      *self = Self::Interned(i(t.as_str()))
+    }
+  }
+  /// Clone out the plain Rust [String]
+  #[must_use]
+  pub fn get_string(self) -> String {
+    match self {
+      Self::Interned(s) => s.as_str().to_owned(),
+      Self::Runtime(rc) => Arc::unwrap_or_clone(rc),
+    }
+  }
+}
+
+impl Deref for OrcString {
+  type Target = String;
+
+  fn deref(&self) -> &Self::Target {
+    match self {
+      Self::Interned(t) => t,
+      Self::Runtime(r) => r,
+    }
+  }
+}
+
+impl Hash for OrcString {
+  fn hash<H: std::hash::Hasher>(&self, state: &mut H) { self.as_str().hash(state) }
+}
+
+impl From<String> for OrcString {
+  fn from(value: String) -> Self { Self::Runtime(Arc::new(value)) }
+}
+
+impl From<&str> for OrcString {
+  fn from(value: &str) -> Self { Self::from(value.to_string()) }
+}
+
+impl From<Tok<String>> for OrcString {
+  fn from(value: Tok<String>) -> Self { Self::Interned(value) }
+}
+
+impl PartialEq for OrcString {
+  fn eq(&self, other: &Self) -> bool {
+    match (self, other) {
+      (Self::Interned(t1), Self::Interned(t2)) => t1 == t2,
+      _ => **self == **other,
+    }
+  }
+}
+
+impl InertPayload for OrcString {
+  const TYPE_STR: &'static str = "OrcString";
+  fn strict_eq(&self, other: &Self) -> bool { self == other }
+}
+
+impl ToClause for String {
+  fn to_clause(self, _: CodeLocation) -> Clause { Inert(OrcString::from(self)).atom_cls() }
+}
+
+impl TryFromExpr for String {
+  fn from_expr(exi: Expr) -> RTResult<Self> {
+    Ok(exi.downcast::<Inert<OrcString>>()?.0.get_string())
+  }
+}
+
+pub(super) fn str_lib() -> ConstTree {
+  ConstTree::ns("std::string", [ConstTree::tree([
+    // String conversion protocol implementable by external types
+    ("conversion", Protocol::tree([], [])),
+    xfn_ent("slice", [|s: Inert<OrcString>, i: Inert<usize>, len: Inert<usize>| {
+      let graphs = s.0.as_str().graphemes(true);
+      if i.0 == 0 {
+        return Ok(graphs.take(len.0).collect::<String>());
+      }
+      let mut prefix = graphs.skip(i.0 - 1);
+      if prefix.next().is_none() {
+        return Err(RuntimeError::ext(
+          "Character index out of bounds".to_string(),
+          "indexing string",
+        ));
+      }
+      let mut count = 0;
+      let ret = (prefix.take(len.0))
+        .map(|x| {
+          count += 1;
+          x
+        })
+        .collect::<String>();
+      if count == len.0 {
+        Ok(ret)
+      } else {
+        RuntimeError::fail("Character index out of bounds".to_string(), "indexing string")
+      }
+    }]),
+    xfn_ent("concat", [|a: String, b: Inert<OrcString>| a + b.0.as_str()]),
+    xfn_ent("find", [|haystack: Inert<OrcString>, needle: Inert<OrcString>| {
+      let haystack_graphs = haystack.0.as_str().graphemes(true);
+      iter_find(haystack_graphs, needle.0.as_str().graphemes(true)).map(Inert)
+    }]),
+    xfn_ent("split", [|s: String, i: Inert<usize>| -> (String, String) {
+      let mut graphs = s.as_str().graphemes(true);
+      (graphs.by_ref().take(i.0).collect(), graphs.collect())
+    }]),
+    xfn_ent("len", [|s: Inert<OrcString>| Inert(s.0.graphemes(true).count())]),
+    xfn_ent("size", [|s: Inert<OrcString>| Inert(s.0.as_bytes().len())]),
+    xfn_ent("intern", [|s: Inert<OrcString>| {
+      Inert(match s.0 {
+        OrcString::Runtime(s) => OrcString::Interned(i(&*s)),
+        x => x,
+      })
+    }]),
+    xfn_ent("convert", [|WithLoc(loc, a): WithLoc<Expr>| match a.clone().downcast() {
+      Ok(str) => Inert::<OrcString>::atom_expr(str, loc),
+      Err(_) => match a.clause.request::<OrcString>() {
+        Some(str) => Inert(str).atom_expr(loc),
+        None => tpl::a2(gen_resolv("std::string::conversion"), tpl::Slot, tpl::Slot)
+          .template(nort_gen(loc), [a.clone(), a]),
+      },
+    }]),
+  ])])
+}
+
+/// Reasons why [parse_string] might fail. See [StringError]
+enum StringErrorKind {
+  /// A unicode escape sequence wasn't followed by 4 hex digits
+  NotHex,
+  /// A unicode escape sequence contained an unassigned code point
+  BadCodePoint,
+  /// An unrecognized escape sequence was found
+  BadEscSeq,
+}
+
+/// Error produced by [parse_string]
+struct StringError {
+  /// Character where the error occured
+  pos: usize,
+  /// Reason for the error
+  kind: StringErrorKind,
+}
+
+impl StringError {
+  /// Convert into project error for reporting
+  pub fn into_proj(self, ctx: &dyn ParseCtx, pos: usize) -> ProjectErrorObj {
+    let start = pos + self.pos;
+    let location = ctx.range_loc(&(start..start + 1));
+    match self.kind {
+      StringErrorKind::NotHex => NotHex.pack(location),
+      StringErrorKind::BadCodePoint => BadCodePoint.pack(location),
+      StringErrorKind::BadEscSeq => BadEscapeSequence.pack(location),
+    }
+  }
+}
+
+/// Process escape sequences in a string literal
+fn parse_string(str: &str) -> Result<String, StringError> {
+  let mut target = String::new();
+  let mut iter = str.char_indices();
+  while let Some((_, c)) = iter.next() {
+    if c != '\\' {
+      target.push(c);
+      continue;
+    }
+    let (mut pos, code) = iter.next().expect("lexer would have continued");
+    let next = match code {
+      c @ ('\\' | '/' | '"') => c,
+      'b' => '\x08',
+      'f' => '\x0f',
+      'n' => '\n',
+      'r' => '\r',
+      't' => '\t',
+      '\n' => 'skipws: loop {
+        match iter.next() {
+          None => return Ok(target),
+          Some((_, c)) =>
+            if !c.is_whitespace() {
+              break 'skipws c;
+            },
+        }
+      },
+      'u' => {
+        let acc = ((0..4).rev())
+          .map(|radical| {
+            let (j, c) = (iter.next()).ok_or(StringError { pos, kind: StringErrorKind::NotHex })?;
+            pos = j;
+            let b = u32::from_str_radix(&String::from(c), 16)
+              .map_err(|_| StringError { pos, kind: StringErrorKind::NotHex })?;
+            Ok(16u32.pow(radical) + b)
+          })
+          .fold_ok(0, u32::wrapping_add)?;
+        char::from_u32(acc).ok_or(StringError { pos, kind: StringErrorKind::BadCodePoint })?
+      },
+      _ => return Err(StringError { pos, kind: StringErrorKind::BadEscSeq }),
+    };
+    target.push(next);
+  }
+  Ok(target)
+}
+
+/// [LexerPlugin] for a string literal that supports interpolateion.
+#[derive(Clone)]
+pub struct StringLexer;
+impl LexerPlugin for StringLexer {
+  fn lex<'a>(&self, req: &'_ dyn LexPluginReq<'a>) -> Option<ProjectResult<LexRes<'a>>> {
+    req.tail().strip_prefix('\"').map(|mut txt| {
+      let ctx = req.ctx();
+      let mut parts = vec![Entry::new(ctx.range(0, txt), Lexeme::LP(PType::Par))];
+      let mut str = String::new();
+      let commit_str = |str: &mut String, tail: &str, parts: &mut Vec<Entry>| -> ProjectResult<_> {
+        let str_val = parse_string(str).unwrap_or_else(|e| {
+          ctx.reporter().report(e.into_proj(ctx, ctx.pos(txt)));
+          String::new()
+        });
+        let ag = AtomGenerator::cloner(Inert(OrcString::from(i(&str_val))));
+        parts.push(Entry::new(ctx.range(str.len(), tail), Lexeme::Atom(ag)));
+        *str = String::new();
+        Ok(())
+      };
+      loop {
+        if let Some(rest) = txt.strip_prefix('"') {
+          commit_str(&mut str, txt, &mut parts)?;
+          parts.push(Entry::new(ctx.range(0, rest), Lexeme::RP(PType::Par)));
+          if parts.len() == 3 {
+            return Ok(LexRes { tail: rest, tokens: vec![parts[1].clone()] });
+          }
+          return Ok(LexRes { tail: rest, tokens: parts });
+        }
+        if let Some(rest) = txt.strip_prefix("${") {
+          let mut depth = 0;
+          commit_str(&mut str, rest, &mut parts)?;
+          parts.extend(req.insert("++ std::string::convert (", ctx.source_range(0, rest)));
+          let res = req.recurse(LexPluginRecur {
+            tail: rest,
+            exit: &mut |c| {
+              match c.chars().next() {
+                None => return Err(UnclosedInterpolation.pack(ctx.source_range(2, rest))),
+                Some('{') => depth += 1,
+                Some('}') if depth == 0 => return Ok(true),
+                Some('}') => depth -= 1,
+                _ => (),
+              }
+              Ok(false)
+            },
+          })?;
+          txt = &res.tail[1..]; // account for final }
+          parts.extend(res.tokens);
+          parts.extend(req.insert(") ++", ctx.source_range(0, txt)));
+        } else {
+          let mut chars = txt.chars();
+          match chars.next() {
+            None => return Err(NoStringEnd.pack(ctx.source_range(req.tail().len(), ""))),
+            Some('\\') => match chars.next() {
+              None => write!(str, "\\").expect("writing \\ into string"),
+              Some(next) => write!(str, "\\{next}").expect("writing \\ and char into string"),
+            },
+            Some(c) => write!(str, "{c}").expect("writing char into string"),
+          }
+          txt = chars.as_str();
+        }
+      }
+    })
+  }
+}
+
+/// An interpolated string section started with ${ wasn't closed with a balanced
+/// }
+pub struct UnclosedInterpolation;
+impl ParseErrorKind for UnclosedInterpolation {
+  const DESCRIPTION: &'static str = "A ${ block within a $-string wasn't closed";
+}
+
+/// String literal never ends
+pub(super) struct NoStringEnd;
+impl ParseErrorKind for NoStringEnd {
+  const DESCRIPTION: &'static str = "A string literal was not closed with `\"`";
+}
+
+/// A unicode escape sequence contains something other than a hex digit
+pub(super) struct NotHex;
+impl ParseErrorKind for NotHex {
+  const DESCRIPTION: &'static str = "Expected a hex digit";
+}
+
+/// A unicode escape sequence contains a number that isn't a unicode code point.
+pub(super) struct BadCodePoint;
+impl ParseErrorKind for BadCodePoint {
+  const DESCRIPTION: &'static str = "\\uXXXX escape sequence does not describe valid code point";
+}
+
+/// An unrecognized escape sequence occurred in a string.
+pub(super) struct BadEscapeSequence;
+impl ParseErrorKind for BadEscapeSequence {
+  const DESCRIPTION: &'static str = "Unrecognized escape sequence";
+}
+#[cfg(test)]
+mod test {
+  use intern_all::i;
+
+  use super::StringLexer;
+  use crate::foreign::atom::Atomic;
+  use crate::foreign::inert::Inert;
+  use crate::libs::std::string::OrcString;
+  use crate::parse::context::MockContext;
+  use crate::parse::lex_plugin::{LexPlugReqImpl, LexerPlugin};
+  use crate::parse::lexer::Lexeme;
+  use crate::parse::parsed::PType;
+
+  #[test]
+  fn plain_string() {
+    let source = r#""Hello world!" - says the programmer"#;
+    let ctx = MockContext::new();
+    let req = LexPlugReqImpl { ctx: &ctx, tail: source };
+    let res = (StringLexer.lex(&req))
+      .expect("the snippet starts with a quote")
+      .expect("it contains a valid string");
+    let expected = [Inert(OrcString::from("Hello world!")).lexeme()];
+    assert_eq!(res.tokens, expected);
+    assert_eq!(res.tail, " - says the programmer");
+    assert!(!ctx.0.failing(), "No errors were generated")
+  }
+
+  #[test]
+  #[rustfmt::skip]
+  fn template_string() {
+    let source = r#""I <${1 + 2} parsers" - this dev"#;
+    let ctx = MockContext::new();
+    let req = LexPlugReqImpl { ctx: &ctx, tail: source };
+    let res = (StringLexer.lex(&req))
+      .expect("the snippet starts with a quote")
+      .expect("it contains a valid string");
+    use Lexeme::{Name, LP, NS, RP};
+    let expected = [
+      LP(PType::Par),
+      Inert(OrcString::from("I <")).lexeme(),
+      Name(i!(str: "++")),
+      // std::string::convert
+      Name(i!(str: "std")), NS, Name(i!(str: "string")), NS, Name(i!(str: "convert")),
+      // (1 + 1)
+      LP(PType::Par), Inert(1).lexeme(), Name(i!(str: "+")), Inert(2).lexeme(), RP(PType::Par),
+      Name(i!(str: "++")),
+      Inert(OrcString::from(" parsers")).lexeme(),
+      RP(PType::Par),
+    ];
+    assert_eq!(res.tokens, expected);
+    assert_eq!(res.tail, " - this dev");
+    assert!(!ctx.0.failing(), "No errors were generated");
+  }
+}

orchidlang/src/libs/std/tuple.orc

@@ -0,0 +1,76 @@
+import super::(known::*, bool::*, number::*, string::*, fn::*)
+import super::loop::recursive
+import super::(pmatch, macro, panic, conv, list, option)
+
+-- referenced in the impl table in Rust
+const to_string_impl := \t. "tuple[" ++ (
+  to_list t
+    |> list::map conv::to_string
+    |> list::reduce (\l. \r. l ++ ", " ++ r)
+    |> option::fallback ""
+) ++ "]"
+
+export const to_list := \t. (
+  recursive r (n=length t, l=list::end)
+    if n == 0 then l
+    else r (n - 1) (list::cons (pick t $ conv::to_uint $ n - 1) l)
+)
+
+macro gen_tuple $tup macro::list_end =0x1p254=> $tup
+macro gen_tuple $tup ( macro::list_item $item $tail ) =0x1p254=> (gen_tuple (push $tup $item) $tail)
+export macro new ( $list ) =0x1p84=> (gen_tuple empty $list)
+
+macro t[..$items] =0x2p84=> ( new ( macro::comma_list (..$items) ) )
+
+export ::(t, size)
+
+--[
+  request l -> tuple_pattern pattern_walker l
+  pattern_walker end -> pattern_result
+  pattern_walker h ++ t -> pattern_await ( request h ) ( pattern_walker t )
+  pattern_await response pattern_result -> pattern_result
+  tuple_pattern pattern_result -> response
+]--
+
+( macro pmatch::request ( t[ ..$items ] )
+  =0x1p230=> tuple_pattern
+    ( macro::length macro::comma_list ( ..$items ) )
+    (
+      pattern_walker
+        macro::comma_list ( ..$items ) -- leftover items
+    )
+)
+( macro tuple_pattern $length ( pattern_result $expr ( $binds ) )
+  =0x1p254=> pmatch::response (
+    if length pmatch::value == $length
+      then ((\tuple_idx. $expr ) 0)
+      else pmatch::fail
+  ) ( $binds )
+)
+( macro pattern_walker macro::list_end
+  =0x1p254=> pattern_result pmatch::pass ( pmatch::no_binds )
+)
+( macro pattern_walker ( macro::list_item $next $tail )
+  =0x1p254=> pattern_await
+    ( pmatch::request $next )
+    ( pattern_walker $tail )
+)
+( macro pattern_await
+    ( pmatch::response $expr ( $binds ) )
+    ( pattern_result $tail_expr ( $tail_binds ) )
+  =0x1p254=>
+    pattern_result
+      (
+        (\pmatch::pass. (\pmatch::value. $expr) (pick pmatch::value tuple_idx)) (
+          pmatch::take_binds $binds (
+            (\pmatch::pass. (\tuple_idx. $tail_expr) (tuple_idx + 1))
+            ( pmatch::take_binds $tail_binds (
+              pmatch::give_binds
+                (pmatch::chain_binds $binds $tail_binds)
+                pmatch::pass
+            ))
+          )
+        )
+      )
+      ( ( pmatch::chain_binds $binds $tail_binds ) )
+)

orchidlang/src/libs/std/tuple.rs

@@ -0,0 +1,64 @@
+//! `std::tuple` A vector-based sequence for storing short sequences.
+
+use std::fmt;
+use std::sync::Arc;
+
+use once_cell::sync::Lazy;
+
+use super::protocol::Tag;
+use super::reflect::refer;
+use crate::foreign::error::{AssertionError, RTResult};
+use crate::foreign::fn_bridge::Thunk;
+use crate::foreign::inert::{Inert, InertPayload};
+use crate::foreign::try_from_expr::WithLoc;
+use crate::gen::tree::{atom_ent, xfn_ent, ConstTree};
+use crate::interpreter::nort::Expr;
+use crate::location::{CodeGenInfo, CodeLocation};
+use crate::sym;
+use crate::utils::ddispatch::Request;
+use crate::utils::pure_seq::pushed;
+
+static TUPLE_TAG: Lazy<Tag> = Lazy::new(|| {
+  let location = CodeLocation::new_gen(CodeGenInfo::no_details(sym!(std::tuple)));
+  Tag::new(sym!(std::tuple), [(
+    sym!(std::string::conversion),
+    refer("std::tuple::to_string_impl").into_expr(location),
+  )])
+});
+
+/// A short contiquous random access sequence of Orchid values.
+#[derive(Clone)]
+pub struct Tuple(pub Arc<Vec<Expr>>);
+impl InertPayload for Tuple {
+  const TYPE_STR: &'static str = "tuple";
+  fn respond(&self, mut request: Request) { request.serve_with(|| TUPLE_TAG.clone()) }
+}
+impl fmt::Debug for Tuple {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "Tuple")?;
+    f.debug_list().entries(self.0.iter().map(|e| &e.clause)).finish()
+  }
+}
+
+fn length(tuple: Inert<Tuple>) -> Inert<usize> { Inert(tuple.0.0.len()) }
+
+fn pick(WithLoc(loc, tuple): WithLoc<Inert<Tuple>>, idx: Inert<usize>) -> RTResult<Expr> {
+  (tuple.0.0.get(idx.0).cloned()).ok_or_else(|| {
+    let msg = format!("{} <= {idx}", tuple.0.0.len());
+    AssertionError::ext(loc, "Tuple index out of bounds", msg)
+  })
+}
+
+fn push(Inert(tuple): Inert<Tuple>, item: Thunk) -> Inert<Tuple> {
+  let items = Arc::unwrap_or_clone(tuple.0);
+  Inert(Tuple(Arc::new(pushed(items, item.0))))
+}
+
+pub(super) fn tuple_lib() -> ConstTree {
+  ConstTree::ns("std::tuple", [TUPLE_TAG.to_tree([
+    atom_ent("empty", [Inert(Tuple(Arc::new(Vec::new())))]),
+    xfn_ent("length", [length]),
+    xfn_ent("pick", [pick]),
+    xfn_ent("push", [push]),
+  ])])
+}

orchidlang/src/name.rs

@@ -0,0 +1,476 @@
+//! Various datatypes that all represent namespaced names.
+
+use std::borrow::Borrow;
+use std::hash::Hash;
+use std::iter::Cloned;
+use std::num::NonZeroUsize;
+use std::ops::{Deref, Index};
+use std::path::Path;
+use std::{fmt, slice, vec};
+
+use intern_all::{i, Tok};
+use itertools::Itertools;
+use trait_set::trait_set;
+
+trait_set! {
+  /// Traits that all name iterators should implement
+  pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
+}
+
+/// A borrowed name fragment which can be empty. See [VPath] for the owned
+/// variant.
+#[derive(Hash, PartialEq, Eq)]
+#[repr(transparent)]
+pub struct PathSlice([Tok<String>]);
+impl PathSlice {
+  /// Create a new [PathSlice]
+  pub fn new(slice: &[Tok<String>]) -> &PathSlice {
+    // SAFETY: This is ok because PathSlice is #[repr(transparent)]
+    unsafe { &*(slice as *const [Tok<String>] as *const PathSlice) }
+  }
+  /// Convert to an owned name fragment
+  pub fn to_vpath(&self) -> VPath { VPath(self.0.to_vec()) }
+  /// Iterate over the tokens
+  pub fn iter(&self) -> impl NameIter + '_ { self.into_iter() }
+  /// Iterate over the segments
+  pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
+    Box::new(self.0.iter().map(|s| s.as_str()))
+  }
+  /// Find the longest shared prefix of this name and another sequence
+  pub fn coprefix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
+    &self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count()]
+  }
+  /// Find the longest shared suffix of this name and another sequence
+  pub fn cosuffix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
+    &self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count()]
+  }
+  /// Remove another
+  pub fn strip_prefix<'a>(&'a self, other: &PathSlice) -> Option<&'a PathSlice> {
+    let shared = self.coprefix(other).len();
+    (shared == other.len()).then_some(PathSlice::new(&self[shared..]))
+  }
+  /// Number of path segments
+  pub fn len(&self) -> usize { self.0.len() }
+  /// Whether there are any path segments. In other words, whether this is a
+  /// valid name
+  pub fn is_empty(&self) -> bool { self.len() == 0 }
+  /// Obtain a reference to the held slice. With all indexing traits shadowed,
+  /// this is better done explicitly
+  pub fn as_slice(&self) -> &[Tok<String>] { self }
+  /// Global empty path slice
+  pub fn empty() -> &'static Self { PathSlice::new(&[]) }
+}
+impl fmt::Debug for PathSlice {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
+}
+impl fmt::Display for PathSlice {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.str_iter().join("::"))
+  }
+}
+impl Borrow<[Tok<String>]> for PathSlice {
+  fn borrow(&self) -> &[Tok<String>] { &self.0 }
+}
+impl<'a> IntoIterator for &'a PathSlice {
+  type IntoIter = Cloned<slice::Iter<'a, Tok<String>>>;
+  type Item = Tok<String>;
+  fn into_iter(self) -> Self::IntoIter { self.0.iter().cloned() }
+}
+
+mod idx_impls {
+  use std::ops;
+
+  use intern_all::Tok;
+
+  use super::PathSlice;
+
+  impl ops::Index<usize> for PathSlice {
+    type Output = Tok<String>;
+    fn index(&self, index: usize) -> &Self::Output { &self.0[index] }
+  }
+  macro_rules! impl_range_index_for_pathslice {
+    ($range:ty) => {
+      impl ops::Index<$range> for PathSlice {
+        type Output = Self;
+        fn index(&self, index: $range) -> &Self::Output { Self::new(&self.0[index]) }
+      }
+    };
+  }
+
+  impl_range_index_for_pathslice!(ops::RangeFull);
+  impl_range_index_for_pathslice!(ops::RangeFrom<usize>);
+  impl_range_index_for_pathslice!(ops::RangeTo<usize>);
+  impl_range_index_for_pathslice!(ops::Range<usize>);
+  impl_range_index_for_pathslice!(ops::RangeInclusive<usize>);
+  impl_range_index_for_pathslice!(ops::RangeToInclusive<usize>);
+}
+
+impl Deref for PathSlice {
+  type Target = [Tok<String>];
+
+  fn deref(&self) -> &Self::Target { &self.0 }
+}
+impl Borrow<PathSlice> for [Tok<String>] {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(self) }
+}
+impl<const N: usize> Borrow<PathSlice> for [Tok<String>; N] {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
+}
+impl Borrow<PathSlice> for Vec<Tok<String>> {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
+}
+
+/// A token path which may be empty. [VName] is the non-empty,
+/// [PathSlice] is the borrowed version
+#[derive(Clone, Default, Hash, PartialEq, Eq)]
+pub struct VPath(pub Vec<Tok<String>>);
+impl VPath {
+  /// Collect segments into a vector
+  pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Self {
+    Self(items.into_iter().collect())
+  }
+  /// Number of path segments
+  pub fn len(&self) -> usize { self.0.len() }
+  /// Whether there are any path segments. In other words, whether this is a
+  /// valid name
+  pub fn is_empty(&self) -> bool { self.len() == 0 }
+  /// Prepend some tokens to the path
+  pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+    Self(items.into_iter().chain(self.0).collect())
+  }
+  /// Append some tokens to the path
+  pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+    Self(self.0.into_iter().chain(items).collect())
+  }
+  /// Partition the string by `::` namespace separators
+  pub fn parse(s: &str) -> Self {
+    Self(if s.is_empty() { vec![] } else { s.split("::").map(i).collect() })
+  }
+  /// Walk over the segments
+  pub fn str_iter(&self) -> impl Iterator<Item = &'_ str> {
+    Box::new(self.0.iter().map(|s| s.as_str()))
+  }
+  /// Try to convert into non-empty version
+  pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
+  /// Add a token to the path. Since now we know that it can't be empty, turn it
+  /// into a name.
+  pub fn name_with_prefix(self, name: Tok<String>) -> VName {
+    VName(self.into_iter().chain([name]).collect())
+  }
+  /// Add a token to the beginning of the. Since now we know that it can't be
+  /// empty, turn it into a name.
+  pub fn name_with_suffix(self, name: Tok<String>) -> VName {
+    VName([name].into_iter().chain(self).collect())
+  }
+
+  /// Convert a fs path to a vpath
+  pub fn from_path(path: &Path) -> Option<(Self, bool)> {
+    let to_vpath = |p: &Path| p.iter().map(|c| c.to_str().map(i)).collect::<Option<_>>().map(VPath);
+    match path.extension().map(|s| s.to_str()) {
+      Some(Some("orc")) => Some((to_vpath(&path.with_extension(""))?, true)),
+      None => Some((to_vpath(path)?, false)),
+      Some(_) => None,
+    }
+  }
+}
+impl fmt::Debug for VPath {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
+}
+impl fmt::Display for VPath {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.str_iter().join("::"))
+  }
+}
+impl FromIterator<Tok<String>> for VPath {
+  fn from_iter<T: IntoIterator<Item = Tok<String>>>(iter: T) -> Self {
+    Self(iter.into_iter().collect())
+  }
+}
+impl IntoIterator for VPath {
+  type Item = Tok<String>;
+  type IntoIter = vec::IntoIter<Self::Item>;
+  fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
+}
+impl Borrow<[Tok<String>]> for VPath {
+  fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
+}
+impl Borrow<PathSlice> for VPath {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
+}
+impl Deref for VPath {
+  type Target = PathSlice;
+  fn deref(&self) -> &Self::Target { self.borrow() }
+}
+
+impl<T> Index<T> for VPath
+where PathSlice: Index<T>
+{
+  type Output = <PathSlice as Index<T>>::Output;
+
+  fn index(&self, index: T) -> &Self::Output { &Borrow::<PathSlice>::borrow(self)[index] }
+}
+
+/// A mutable representation of a namespaced identifier of at least one segment.
+///
+/// These names may be relative or otherwise partially processed.
+///
+/// See also [Sym] for the immutable representation, and [VPath] for possibly
+/// empty values
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub struct VName(Vec<Tok<String>>);
+impl VName {
+  /// Assert that the sequence isn't empty and wrap it in [VName] to represent
+  /// this invariant
+  pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
+    let data: Vec<_> = items.into_iter().collect();
+    if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
+  }
+  /// Unwrap the enclosed vector
+  pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
+  /// Get a reference to the enclosed vector
+  pub fn vec(&self) -> &Vec<Tok<String>> { &self.0 }
+  /// Mutable access to the underlying vector. To ensure correct results, this
+  /// must never be empty.
+  pub fn vec_mut(&mut self) -> &mut Vec<Tok<String>> { &mut self.0 }
+  /// Intern the name and return a [Sym]
+  pub fn to_sym(&self) -> Sym { Sym(i(&self.0)) }
+  /// If this name has only one segment, return it
+  pub fn as_root(&self) -> Option<Tok<String>> { self.0.iter().exactly_one().ok().cloned() }
+  /// Prepend the segments to this name
+  #[must_use = "This is a pure function"]
+  pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+    Self(items.into_iter().chain(self.0).collect())
+  }
+  /// Append the segments to this name
+  #[must_use = "This is a pure function"]
+  pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
+    Self(self.0.into_iter().chain(items).collect())
+  }
+  /// Read a `::` separated namespaced name
+  pub fn parse(s: &str) -> Result<Self, EmptyNameError> { Self::new(VPath::parse(s)) }
+  /// Obtain an iterator over the segments of the name
+  pub fn iter(&self) -> impl Iterator<Item = Tok<String>> + '_ { self.0.iter().cloned() }
+}
+impl fmt::Debug for VName {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
+}
+impl fmt::Display for VName {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.str_iter().join("::"))
+  }
+}
+impl IntoIterator for VName {
+  type Item = Tok<String>;
+  type IntoIter = vec::IntoIter<Self::Item>;
+  fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
+}
+impl<T> Index<T> for VName
+where PathSlice: Index<T>
+{
+  type Output = <PathSlice as Index<T>>::Output;
+
+  fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
+}
+impl Borrow<[Tok<String>]> for VName {
+  fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
+}
+impl Borrow<PathSlice> for VName {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
+}
+impl Deref for VName {
+  type Target = PathSlice;
+  fn deref(&self) -> &Self::Target { self.borrow() }
+}
+
+/// Error produced when a non-empty name [VName] or [Sym] is constructed with an
+/// empty sequence
+#[derive(Debug, Copy, Clone, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct EmptyNameError;
+impl TryFrom<&[Tok<String>]> for VName {
+  type Error = EmptyNameError;
+  fn try_from(value: &[Tok<String>]) -> Result<Self, Self::Error> {
+    Self::new(value.iter().cloned())
+  }
+}
+
+/// An interned representation of a namespaced identifier.
+///
+/// These names are always absolute.
+///
+/// See also [VName]
+#[derive(Clone, Hash, PartialEq, Eq)]
+pub struct Sym(Tok<Vec<Tok<String>>>);
+impl Sym {
+  /// Assert that the sequence isn't empty, intern it and wrap it in a [Sym] to
+  /// represent this invariant
+  pub fn new(v: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
+    let items = v.into_iter().collect::<Vec<_>>();
+    Self::from_tok(i(&items))
+  }
+  /// Read a `::` separated namespaced name.
+  pub fn parse(s: &str) -> Result<Self, EmptyNameError> { Ok(Sym(i(&VName::parse(s)?.into_vec()))) }
+  /// Assert that a token isn't empty, and wrap it in a [Sym]
+  pub fn from_tok(t: Tok<Vec<Tok<String>>>) -> Result<Self, EmptyNameError> {
+    if t.is_empty() { Err(EmptyNameError) } else { Ok(Self(t)) }
+  }
+  /// Grab the interner token
+  pub fn tok(&self) -> Tok<Vec<Tok<String>>> { self.0.clone() }
+  /// Get a number unique to this name suitable for arbitrary ordering.
+  pub fn id(&self) -> NonZeroUsize { self.0.id() }
+  /// Extern the sym for editing
+  pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
+}
+impl fmt::Debug for Sym {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Sym({self})") }
+}
+impl fmt::Display for Sym {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}", self.str_iter().join("::"))
+  }
+}
+impl<T> Index<T> for Sym
+where PathSlice: Index<T>
+{
+  type Output = <PathSlice as Index<T>>::Output;
+
+  fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
+}
+impl Borrow<[Tok<String>]> for Sym {
+  fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
+}
+impl Borrow<PathSlice> for Sym {
+  fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
+}
+impl Deref for Sym {
+  type Target = PathSlice;
+  fn deref(&self) -> &Self::Target { self.borrow() }
+}
+
+/// An abstraction over tokenized vs non-tokenized names so that they can be
+/// handled together in datastructures. The names can never be empty
+#[allow(clippy::len_without_is_empty)] // never empty
+pub trait NameLike:
+  'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<PathSlice>
+{
+  /// Convert into held slice
+  fn as_slice(&self) -> &[Tok<String>] { Borrow::<PathSlice>::borrow(self) }
+  /// Get iterator over tokens
+  fn iter(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
+  /// Get iterator over string segments
+  fn str_iter(&self) -> impl Iterator<Item = &'_ str> + '_ {
+    self.as_slice().iter().map(|t| t.as_str())
+  }
+  /// Fully resolve the name for printing
+  #[must_use]
+  fn to_strv(&self) -> Vec<String> { self.iter().map(|s| s.to_string()).collect() }
+  /// Format the name as an approximate filename
+  fn as_src_path(&self) -> String { format!("{}.orc", self.iter().join("/")) }
+  /// Return the number of segments in the name
+  fn len(&self) -> NonZeroUsize {
+    NonZeroUsize::try_from(self.iter().count()).expect("NameLike never empty")
+  }
+  /// Like slice's `split_first` except we know that it always returns Some
+  fn split_first(&self) -> (Tok<String>, &PathSlice) {
+    let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
+    (foot.clone(), PathSlice::new(torso))
+  }
+  /// Like slice's `split_last` except we know that it always returns Some
+  fn split_last(&self) -> (Tok<String>, &PathSlice) {
+    let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
+    (foot.clone(), PathSlice::new(torso))
+  }
+  /// Get the first element
+  fn first(&self) -> Tok<String> { self.split_first().0 }
+  /// Get the last element
+  fn last(&self) -> Tok<String> { self.split_last().0 }
+}
+
+impl NameLike for Sym {}
+impl NameLike for VName {}
+
+/// Create a [Sym] literal.
+///
+/// Both the name and its components will be cached in a thread-local static so
+/// that subsequent executions of the expression only incur an Arc-clone for
+/// cloning the token.
+#[macro_export]
+macro_rules! sym {
+  ($seg1:tt $( :: $seg:tt)*) => {
+    $crate::name::Sym::from_tok(intern_all::i!([intern_all::Tok<String>]: &[
+      intern_all::i!(str: stringify!($seg1))
+      $( , intern_all::i!(str: stringify!($seg)) )*
+    ][..])).unwrap()
+  };
+}
+
+/// Create a [VName] literal.
+///
+/// The components are interned much like in [sym].
+#[macro_export]
+macro_rules! vname {
+  ($seg1:tt $( :: $seg:tt)*) => {
+    $crate::name::VName::new([
+      intern_all::i!(str: stringify!($seg1))
+      $( , intern_all::i!(str: stringify!($seg)) )*
+    ]).unwrap()
+  };
+}
+
+/// Create a [VPath] literal.
+///
+/// The components are interned much like in [sym].
+#[macro_export]
+macro_rules! vpath {
+  ($seg1:tt $( :: $seg:tt)+) => {
+    $crate::name::VPath(vec![
+      intern_all::i!(str: stringify!($seg1))
+      $( , intern_all::i!(str: stringify!($seg)) )+
+    ])
+  };
+  () => {
+    $crate::name::VPath(vec![])
+  }
+}
+
+/// Create a &[PathSlice] literal.
+///
+/// The components are interned much like in [sym]
+#[macro_export]
+macro_rules! path_slice {
+  ($seg1:tt $( :: $seg:tt)+) => {
+    $crate::name::PathSlice::new(&[
+      intern_all::i!(str: stringify!($seg1))
+      $( , intern_all::i!(str: stringify!($seg)) )+
+    ])
+  };
+  () => {
+    $crate::name::PathSlice::new(&[])
+  }
+}
+
+#[cfg(test)]
+mod test {
+  use std::borrow::Borrow;
+
+  use intern_all::{i, Tok};
+
+  use super::{PathSlice, Sym, VName};
+  use crate::name::VPath;
+
+  #[test]
+  fn recur() {
+    let myname = vname!(foo::bar);
+    let _borrowed_slice: &[Tok<String>] = myname.borrow();
+    let _borrowed_pathslice: &PathSlice = myname.borrow();
+    let _deref_pathslice: &PathSlice = &myname;
+    let _as_slice_out: &[Tok<String>] = myname.as_slice();
+  }
+
+  #[test]
+  fn literals() {
+    assert_eq!(sym!(foo::bar::baz), Sym::new([i("foo"), i("bar"), i("baz")]).unwrap());
+    assert_eq!(vname!(foo::bar::baz), VName::new([i("foo"), i("bar"), i("baz")]).unwrap());
+    assert_eq!(vpath!(foo::bar::baz), VPath::new([i("foo"), i("bar"), i("baz")]));
+    assert_eq!(path_slice!(foo::bar::baz), PathSlice::new(&[i("foo"), i("bar"), i("baz")]));
+  }
+}

orchidlang/src/parse/context.rs

@@ -0,0 +1,163 @@
+//! Definition and implementations of the parsing context, which is used
+
+use std::ops::Range;
+use std::sync::Arc;
+
+use super::lex_plugin::LexerPlugin;
+use super::parse_plugin::ParseLinePlugin;
+use crate::error::Reporter;
+use crate::location::{SourceCode, SourceRange};
+use crate::utils::boxed_iter::{box_empty, BoxedIter};
+use crate::utils::sequence::Sequence;
+
+/// Trait enclosing all context features
+///
+/// The main implementation is [ParseCtxImpl]
+pub trait ParseCtx {
+  /// Get an object describing the file this source code comes from
+  #[must_use]
+  fn code_info(&self) -> SourceCode;
+  /// Get the list of all lexer plugins
+  #[must_use]
+  fn lexers(&self) -> BoxedIter<'_, &dyn LexerPlugin>;
+  /// Get the list of all parser plugins
+  #[must_use]
+  fn line_parsers(&self) -> BoxedIter<'_, &dyn ParseLinePlugin>;
+  /// Error reporter
+  #[must_use]
+  fn reporter(&self) -> &Reporter;
+  /// Find our position in the text given the text we've yet to parse
+  #[must_use]
+  fn pos(&self, tail: &str) -> usize {
+    let tail_len = tail.len();
+    let source_len = self.source().len();
+    (self.source().len().checked_sub(tail.len())).unwrap_or_else(|| {
+      panic!("tail.len()={tail_len} greater than self.source().len()={source_len}; tail={tail:?}")
+    })
+  }
+  /// Generate a location given the length of a token and the unparsed text
+  /// after it. See also [ParseCtx::range_loc] if the maths gets complex.
+  #[must_use]
+  fn range(&self, len: usize, tl: &str) -> Range<usize> {
+    match self.pos(tl).checked_sub(len) {
+      Some(start) => start..self.pos(tl),
+      None => {
+        panic!("len={len} greater than tail.len()={}; tail={tl:?}", tl.len())
+      },
+    }
+  }
+  /// Create a contextful location for error reporting
+  #[must_use]
+  fn source_range(&self, len: usize, tl: &str) -> SourceRange {
+    self.range_loc(&self.range(len, tl))
+  }
+  /// Create a contentful location from a range directly.
+  #[must_use]
+  fn range_loc(&self, range: &Range<usize>) -> SourceRange {
+    SourceRange { code: self.code_info(), range: range.clone() }
+  }
+  /// Get a reference to the full source text. This should not be used for
+  /// position math.
+  #[must_use]
+  fn source(&self) -> Arc<String> { self.code_info().text.clone() }
+}
+
+impl<'a, C: ParseCtx + 'a + ?Sized> ParseCtx for &'a C {
+  fn reporter(&self) -> &Reporter { (*self).reporter() }
+  fn lexers(&self) -> BoxedIter<'_, &dyn LexerPlugin> { (*self).lexers() }
+  fn line_parsers(&self) -> BoxedIter<'_, &dyn ParseLinePlugin> { (*self).line_parsers() }
+  fn pos(&self, tail: &str) -> usize { (*self).pos(tail) }
+  fn code_info(&self) -> SourceCode { (*self).code_info() }
+  fn source(&self) -> Arc<String> { (*self).source() }
+  fn range(&self, l: usize, t: &str) -> Range<usize> { (*self).range(l, t) }
+}
+
+/// Struct implementing context
+#[derive(Clone)]
+pub struct ParseCtxImpl<'a, 'b> {
+  /// File to be parsed; where it belongs in the tree and its text
+  pub code: SourceCode,
+  /// Error aggregator
+  pub reporter: &'b Reporter,
+  /// Lexer plugins for parsing custom literals
+  pub lexers: Sequence<'a, &'a (dyn LexerPlugin + 'a)>,
+  /// Parser plugins for parsing custom line structures
+  pub line_parsers: Sequence<'a, &'a dyn ParseLinePlugin>,
+}
+impl<'a, 'b> ParseCtx for ParseCtxImpl<'a, 'b> {
+  fn reporter(&self) -> &Reporter { self.reporter }
+  // Rust doesn't realize that this lifetime is covariant
+  #[allow(clippy::map_identity)]
+  fn lexers(&self) -> BoxedIter<'_, &dyn LexerPlugin> { Box::new(self.lexers.iter().map(|r| r)) }
+  #[allow(clippy::map_identity)]
+  fn line_parsers(&self) -> BoxedIter<'_, &dyn ParseLinePlugin> {
+    Box::new(self.line_parsers.iter().map(|r| r))
+  }
+  fn code_info(&self) -> SourceCode { self.code.clone() }
+}
+
+/// Context instance for testing. Implicitly provides a reporter and panics if
+/// any errors are reported
+pub struct MockContext(pub Reporter);
+impl MockContext {
+  /// Create a new mock
+  pub fn new() -> Self { Self(Reporter::new()) }
+}
+impl Default for MockContext {
+  fn default() -> Self { Self::new() }
+}
+impl ParseCtx for MockContext {
+  fn reporter(&self) -> &Reporter { &self.0 }
+  fn pos(&self, tail: &str) -> usize { usize::MAX / 2 - tail.len() }
+  // these are expendable
+  fn code_info(&self) -> SourceCode { SourceRange::mock().code() }
+  fn lexers(&self) -> BoxedIter<'_, &dyn LexerPlugin> { box_empty() }
+  fn line_parsers(&self) -> BoxedIter<'_, &dyn ParseLinePlugin> { box_empty() }
+}
+impl Drop for MockContext {
+  fn drop(&mut self) { self.0.assert() }
+}
+
+/// Context that assigns the same location to every subset of the source code.
+/// Its main use case is to process source code that was dynamically generated
+/// in response to some user code. See also [ReporterContext]
+pub struct FlatLocContext<'a, C: ParseCtx + ?Sized> {
+  sub: &'a C,
+  range: &'a SourceRange,
+}
+impl<'a, C: ParseCtx + ?Sized> FlatLocContext<'a, C> {
+  /// Create a new context that will use the same provided range for every
+  /// parsed token
+  pub fn new(sub: &'a C, range: &'a SourceRange) -> Self { Self { sub, range } }
+}
+impl<'a, C: ParseCtx + ?Sized> ParseCtx for FlatLocContext<'a, C> {
+  fn reporter(&self) -> &Reporter { self.sub.reporter() }
+  fn pos(&self, _: &str) -> usize { 0 }
+  fn lexers(&self) -> BoxedIter<'_, &dyn LexerPlugin> { self.sub.lexers() }
+  fn line_parsers(&self) -> BoxedIter<'_, &dyn ParseLinePlugin> { self.sub.line_parsers() }
+  fn code_info(&self) -> SourceCode { self.range.code() }
+  fn range(&self, _: usize, _: &str) -> Range<usize> { self.range.range() }
+}
+
+/// Context that forwards everything to a wrapped context except for error
+/// reporting. See also [FlatLocContext]
+pub struct ReporterContext<'a, C: ParseCtx + ?Sized> {
+  sub: &'a C,
+  reporter: &'a Reporter,
+}
+impl<'a, C: ParseCtx + ?Sized> ReporterContext<'a, C> {
+  /// Create a new context that will collect errors separately and forward
+  /// everything else to an enclosed context
+  pub fn new(sub: &'a C, reporter: &'a Reporter) -> Self { Self { sub, reporter } }
+}
+impl<'a, C: ParseCtx + ?Sized> ParseCtx for ReporterContext<'a, C> {
+  fn reporter(&self) -> &Reporter { self.reporter }
+  fn pos(&self, tail: &str) -> usize { self.sub.pos(tail) }
+  fn lexers(&self) -> BoxedIter<'_, &dyn LexerPlugin> { self.sub.lexers() }
+  fn line_parsers(&self) -> BoxedIter<'_, &dyn ParseLinePlugin> { self.sub.line_parsers() }
+  fn code_info(&self) -> SourceCode { self.sub.code_info() }
+  fn range(&self, len: usize, tl: &str) -> Range<usize> { self.sub.range(len, tl) }
+  fn range_loc(&self, range: &Range<usize>) -> SourceRange { self.sub.range_loc(range) }
+  fn source(&self) -> Arc<String> { self.sub.source() }
+  fn source_range(&self, len: usize, tl: &str) -> SourceRange { self.sub.source_range(len, tl) }
+}

orchidlang/src/parse/errors.rs

@@ -0,0 +1,215 @@
+//! Errors produced by the parser. Plugins are encouraged to reuse these where
+//! applicable.
+
+use intern_all::Tok;
+use itertools::Itertools;
+
+use super::context::ParseCtx;
+use super::frag::Frag;
+use super::lexer::{Entry, Lexeme};
+use crate::error::{ProjectError, ProjectErrorObj, ProjectResult};
+use crate::location::{CodeOrigin, SourceRange};
+use crate::parse::parsed::PType;
+
+/// Parse error information without a location. Location data is added by the
+/// parser.
+pub trait ParseErrorKind: Sized + Send + Sync + 'static {
+  /// A general description of the error condition
+  const DESCRIPTION: &'static str;
+  /// A specific description of the error with concrete text sections
+  fn message(&self) -> String { Self::DESCRIPTION.to_string() }
+  /// Convert this error to a type-erased [ProjectError] to be handled together
+  /// with other Orchid errors.
+  fn pack(self, range: SourceRange) -> ProjectErrorObj { ParseError { kind: self, range }.pack() }
+}
+
+struct ParseError<T> {
+  pub range: SourceRange,
+  pub kind: T,
+}
+impl<T: ParseErrorKind> ProjectError for ParseError<T> {
+  const DESCRIPTION: &'static str = T::DESCRIPTION;
+  fn one_position(&self) -> CodeOrigin { self.range.origin() }
+  fn message(&self) -> String { self.kind.message() }
+}
+
+/// A line does not begin with an identifying keyword. Raised on the first token
+pub(super) struct LineNeedsPrefix(pub Lexeme);
+impl ParseErrorKind for LineNeedsPrefix {
+  const DESCRIPTION: &'static str = "This linetype requires a prefix";
+  fn message(&self) -> String { format!("{} cannot appear at the beginning of a line", self.0) }
+}
+
+/// The line ends abruptly. Raised on the last token
+pub(super) struct UnexpectedEOL(pub Lexeme);
+impl ParseErrorKind for UnexpectedEOL {
+  const DESCRIPTION: &'static str = "The line ended abruptly";
+  fn message(&self) -> String {
+    "In Orchid, all line breaks outside parentheses start a new declaration".to_string()
+  }
+}
+
+/// The line should have ended. Raised on last valid or first excess token
+pub(super) struct ExpectedEOL;
+impl ParseErrorKind for ExpectedEOL {
+  const DESCRIPTION: &'static str = "Expected the end of the line";
+}
+
+/// A name was expected.
+pub(super) struct ExpectedName(pub Lexeme);
+impl ParseErrorKind for ExpectedName {
+  const DESCRIPTION: &'static str = "A name was expected";
+  fn message(&self) -> String { format!("Expected a name, found {}", self.0) }
+}
+
+/// Unwrap a name or operator.
+pub(super) fn expect_name(
+  Entry { lexeme, range }: &Entry,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<Tok<String>> {
+  match lexeme {
+    Lexeme::Name(n) => Ok(n.clone()),
+    lex => Err(ExpectedName(lex.clone()).pack(ctx.range_loc(range))),
+  }
+}
+
+/// A specific lexeme was expected
+pub(super) struct Expected {
+  /// The lexemes that would have been acceptable
+  pub expected: Vec<Lexeme>,
+  /// Whether a name would also have been acceptable (multiname)
+  pub or_name: bool,
+  /// What was actually found
+  pub found: Lexeme,
+}
+impl ParseErrorKind for Expected {
+  const DESCRIPTION: &'static str = "A concrete token was expected";
+  fn message(&self) -> String {
+    let list = match &self.expected[..] {
+      &[] => return "Unsatisfiable expectation".to_string(),
+      [only] => only.to_string(),
+      [a, b] => format!("either {a} or {b}"),
+      [variants @ .., last] => {
+        format!("any of {} or {last}", variants.iter().join(", "))
+      },
+    };
+    let or_name = if self.or_name { " or a name" } else { "" };
+    format!("Expected {list}{or_name} but found {}", self.found)
+  }
+}
+/// Assert that the entry contains exactly the specified lexeme
+pub(super) fn expect(l: Lexeme, e: &Entry, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<()> {
+  if e.lexeme.strict_eq(&l) {
+    return Ok(());
+  }
+  let found = e.lexeme.clone();
+  let kind = Expected { expected: vec![l], or_name: false, found };
+  Err(kind.pack(ctx.range_loc(&e.range)))
+}
+
+/// A token reserved for future use was found in the code
+pub(super) struct ReservedToken(pub Lexeme);
+impl ParseErrorKind for ReservedToken {
+  const DESCRIPTION: &'static str = "Syntax reserved for future use";
+  fn message(&self) -> String { format!("{} is a reserved token", self.0) }
+}
+
+/// A token was found where it doesn't belong
+pub(super) struct BadTokenInRegion {
+  /// What was found
+  pub lexeme: Lexeme,
+  /// Human-readable name of the region where it should not appear
+  pub region: &'static str,
+}
+impl ParseErrorKind for BadTokenInRegion {
+  const DESCRIPTION: &'static str = "An unexpected token was found";
+  fn message(&self) -> String { format!("{} cannot appear in {}", self.lexeme, self.region) }
+}
+
+/// Some construct was searched but not found.
+pub(super) struct NotFound(pub &'static str);
+impl ParseErrorKind for NotFound {
+  const DESCRIPTION: &'static str = "A specific lexeme was expected";
+  fn message(&self) -> String { format!("{} was expected", self.0) }
+}
+
+/// :: found on its own somewhere other than a general export
+pub(super) struct LeadingNS;
+impl ParseErrorKind for LeadingNS {
+  const DESCRIPTION: &'static str = ":: can only follow a name token";
+}
+
+/// Parens don't pair up
+pub(super) struct MisalignedParen(pub Lexeme);
+impl ParseErrorKind for MisalignedParen {
+  const DESCRIPTION: &'static str = "(), [] and {} must always pair up";
+  fn message(&self) -> String { format!("This {} has no pair", self.0) }
+}
+
+/// Export line contains a complex name
+pub(super) struct NamespacedExport;
+impl ParseErrorKind for NamespacedExport {
+  const DESCRIPTION: &'static str = "Only local names may be exported";
+}
+
+/// Export line contains *
+pub(super) struct GlobExport;
+impl ParseErrorKind for GlobExport {
+  const DESCRIPTION: &'static str = "Globstars are not allowed in exports";
+}
+
+/// Comment never ends
+pub(super) struct NoCommentEnd;
+impl ParseErrorKind for NoCommentEnd {
+  const DESCRIPTION: &'static str = "a comment was not closed with `]--`";
+}
+
+/// A placeholder's priority is a floating point number
+pub(super) struct FloatPlacehPrio;
+impl ParseErrorKind for FloatPlacehPrio {
+  const DESCRIPTION: &'static str =
+    "a placeholder priority has a decimal point or a negative exponent";
+}
+
+/// A number literal decodes to NaN
+pub(super) struct NaNLiteral;
+impl ParseErrorKind for NaNLiteral {
+  const DESCRIPTION: &'static str = "float literal decoded to NaN";
+}
+
+/// A sequence of digits in a number literal overflows [usize].
+pub(super) struct LiteralOverflow;
+impl ParseErrorKind for LiteralOverflow {
+  const DESCRIPTION: &'static str = "number literal described number greater than usize::MAX";
+}
+
+/// A digit was expected but something else was found
+pub(super) struct ExpectedDigit;
+impl ParseErrorKind for ExpectedDigit {
+  const DESCRIPTION: &'static str = "expected a digit";
+}
+
+/// Expected a parenthesized block at the end of the line
+pub(super) struct ExpectedBlock;
+impl ParseErrorKind for ExpectedBlock {
+  const DESCRIPTION: &'static str = "Expected a parenthesized block";
+}
+/// Remove two parentheses from the ends of the cursor
+pub(super) fn expect_block<'a>(
+  tail: Frag<'a>,
+  typ: PType,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<Frag<'a>> {
+  let (lp, tail) = tail.trim().pop(ctx)?;
+  expect(Lexeme::LP(typ), lp, ctx)?;
+  let (rp, tail) = tail.pop_back(ctx)?;
+  expect(Lexeme::RP(typ), rp, ctx)?;
+  Ok(tail.trim())
+}
+
+/// A namespaced name was expected but a glob pattern or a branching multiname
+/// was found.
+pub(super) struct ExpectedSingleName;
+impl ParseErrorKind for ExpectedSingleName {
+  const DESCRIPTION: &'static str = "expected a single name, no wildcards, no branches";
+}

orchidlang/src/parse/facade.rs

@@ -0,0 +1,42 @@
+//! Entrypoints to the parser that combine lexing and parsing
+
+use never::Never;
+
+use super::context::{FlatLocContext, ParseCtx, ReporterContext};
+use super::frag::Frag;
+use super::lexer::lex;
+use super::sourcefile::parse_module_body;
+use crate::error::Reporter;
+use crate::location::SourceRange;
+use crate::parse::parsed::SourceLine;
+use crate::parse::sourcefile::{parse_line, split_lines};
+
+/// Parse a file
+pub fn parse_file(ctx: &impl ParseCtx) -> Vec<SourceLine> {
+  let tokens = lex(vec![], ctx.source().as_str(), ctx, |_| Ok::<_, Never>(false))
+    .unwrap_or_else(|e| match e {})
+    .tokens;
+  if tokens.is_empty() { Vec::new() } else { parse_module_body(Frag::from_slice(&tokens), ctx) }
+}
+
+/// Parse a statically defined line sequence
+///
+/// # Panics
+///
+/// On any parse error, which is why it only accepts a string literal
+pub fn parse_entries(
+  ctx: &dyn ParseCtx,
+  text: &'static str,
+  range: SourceRange,
+) -> Vec<SourceLine> {
+  let reporter = Reporter::new();
+  let flctx = FlatLocContext::new(ctx, &range);
+  let ctx = ReporterContext::new(&flctx, &reporter);
+  let res = lex(vec![], text, &ctx, |_| Ok::<_, Never>(false)).unwrap_or_else(|e| match e {});
+  let out = split_lines(Frag::from_slice(&res.tokens), &ctx)
+    .flat_map(|tokens| parse_line(tokens, &ctx).expect("pre-specified source"))
+    .map(|kind| kind.wrap(range.clone()))
+    .collect();
+  reporter.assert();
+  out
+}

orchidlang/src/parse/frag.rs

@@ -0,0 +1,133 @@
+//! The [Frag] is the main input datastructure of parsers. Beyond the slice of
+//! tokens, it contains a fallback value that can be used for error reporting if
+//! the fragment is empty.
+
+use std::ops::Range;
+
+use super::context::ParseCtx;
+use super::errors::{ExpectedEOL, NotFound, ParseErrorKind, UnexpectedEOL};
+use super::lexer::{Entry, Lexeme};
+use crate::error::ProjectResult;
+
+/// Represents a slice which may or may not contain items, and a fallback entry
+/// used for error reporting whenever the errant fragment is empty.
+#[must_use = "fragment of code should not be discarded implicitly"]
+#[derive(Clone, Copy)]
+pub struct Frag<'a> {
+  /// Entry to place in errors if the fragment contains no tokens
+  pub fallback: &'a Entry,
+  /// Tokens to parse
+  pub data: &'a [Entry],
+}
+impl<'a> Frag<'a> {
+  /// Create a new fragment
+  pub fn new(fallback: &'a Entry, data: &'a [Entry]) -> Self { Self { fallback, data } }
+
+  /// Remove comments and line breaks from both ends of the text
+  pub fn trim(self) -> Self {
+    let Self { data, fallback } = self;
+    let front = data.iter().take_while(|e| e.is_filler()).count();
+    let (_, right) = data.split_at(front);
+    let back = right.iter().rev().take_while(|e| e.is_filler()).count();
+    let (data, _) = right.split_at(right.len() - back);
+    Self { fallback, data }
+  }
+
+  /// Discard the first entry
+  pub fn step(self, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<Self> {
+    let Self { data, fallback: Entry { lexeme, range } } = self;
+    match data.split_first() {
+      Some((fallback, data)) => Ok(Frag { data, fallback }),
+      None => Err(UnexpectedEOL(lexeme.clone()).pack(ctx.range_loc(range))),
+    }
+  }
+
+  /// Get the first entry
+  pub fn pop(self, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<(&'a Entry, Self)> {
+    Ok((self.get(0, ctx)?, self.step(ctx)?))
+  }
+
+  /// Retrieve an index from a slice or raise an error if it isn't found.
+  pub fn get(self, idx: usize, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<&'a Entry> {
+    self.data.get(idx).ok_or_else(|| {
+      let entry = self.data.last().unwrap_or(self.fallback).clone();
+      UnexpectedEOL(entry.lexeme).pack(ctx.range_loc(&entry.range))
+    })
+  }
+
+  /// Area covered by this fragment
+  #[must_use]
+  pub fn range(self) -> Range<usize> {
+    self.data.first().map_or_else(
+      || self.fallback.range.clone(),
+      |f| f.range.start..self.data.last().unwrap().range.end,
+    )
+  }
+
+  /// Find a given token, split the fragment there and read some value from the
+  /// separator. See also [Frag::find]
+  pub fn find_map<T>(
+    self,
+    msg: &'static str,
+    ctx: &(impl ParseCtx + ?Sized),
+    mut f: impl FnMut(&'a Lexeme) -> Option<T>,
+  ) -> ProjectResult<(Self, T, Self)> {
+    let Self { data, fallback } = self;
+    let (dot_idx, output) = skip_parenthesized(data.iter())
+      .find_map(|(i, e)| f(&e.lexeme).map(|t| (i, t)))
+      .ok_or_else(|| NotFound(msg).pack(ctx.range_loc(&self.range())))?;
+    let (left, not_left) = data.split_at(dot_idx);
+    let (middle_ent, right) = not_left.split_first().unwrap();
+    Ok((Self::new(fallback, left), output, Self::new(middle_ent, right)))
+  }
+
+  /// Split the fragment at a token and return just the two sides.
+  /// See also [Frag::find_map].
+  pub fn find(
+    self,
+    descr: &'static str,
+    ctx: &(impl ParseCtx + ?Sized),
+    mut f: impl FnMut(&Lexeme) -> bool,
+  ) -> ProjectResult<(Self, Self)> {
+    let (l, _, r) = self.find_map(descr, ctx, |l| Some(l).filter(|l| f(l)))?;
+    Ok((l, r))
+  }
+
+  /// Remove the last item from the fragment
+  pub fn pop_back(self, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<(&'a Entry, Self)> {
+    let Self { data, fallback } = self;
+    let (last, data) = (data.split_last())
+      .ok_or_else(|| UnexpectedEOL(fallback.lexeme.clone()).pack(ctx.range_loc(&fallback.range)))?;
+    Ok((last, Self { fallback, data }))
+  }
+
+  /// # Panics
+  ///
+  /// If the slice is empty
+  pub fn from_slice(data: &'a [Entry]) -> Self {
+    let fallback = (data.first()).expect("Empty slice cannot be converted into a parseable");
+    Self { data, fallback }
+  }
+
+  /// Assert that the fragment is empty.
+  pub fn expect_empty(self, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<()> {
+    match self.data.first() {
+      Some(x) => Err(ExpectedEOL.pack(ctx.range_loc(&x.range))),
+      None => Ok(()),
+    }
+  }
+}
+
+fn skip_parenthesized<'a>(
+  it: impl Iterator<Item = &'a Entry>,
+) -> impl Iterator<Item = (usize, &'a Entry)> {
+  let mut paren_lvl = 1;
+  it.enumerate().filter(move |(_, e)| {
+    match e.lexeme {
+      Lexeme::LP(_) => paren_lvl += 1,
+      Lexeme::RP(_) => paren_lvl -= 1,
+      _ => (),
+    }
+    paren_lvl <= 1
+  })
+}

orchidlang/src/parse/lex_plugin.rs

@@ -0,0 +1,65 @@
+//! Abstractions for dynamic extensions to the lexer to parse custom literals
+
+use dyn_clone::DynClone;
+use never::Never;
+
+use super::context::{FlatLocContext, ParseCtx};
+use super::lexer::{lex, Entry, LexRes};
+use crate::error::ProjectResult;
+use crate::location::SourceRange;
+
+/// Data passed to the recursive sub-lexer
+pub struct LexPluginRecur<'a, 'b> {
+  /// Text to tokenize
+  pub tail: &'a str,
+  /// Callback that will be called between lexemes on the leftover text.
+  /// When it returns true, the lexer exits and leaves the remaining text for
+  /// you.
+  pub exit: &'b mut dyn for<'c> FnMut(&'c str) -> ProjectResult<bool>,
+}
+
+/// Data and actions available to a lexer plugin
+pub trait LexPluginReq<'a> {
+  /// Text to tokenize
+  fn tail(&self) -> &'a str;
+  /// [ParseCtx] instance for calculating locations and such
+  fn ctx(&self) -> &dyn ParseCtx;
+  /// Start a child lexer that calls back between lexemes and exits on your
+  /// command. You can combine this with custom atoms to create holes for
+  /// expressions in your literals like the template strings of most languages
+  /// other than Rust.
+  fn recurse(&self, req: LexPluginRecur<'a, '_>) -> ProjectResult<LexRes<'a>>;
+  /// Lex an inserted piece of text, especially when translating custom syntax
+  /// into multiple lexemes.
+  ///
+  /// # Panics
+  ///
+  /// If tokenization fails
+  fn insert(&self, data: &str, range: SourceRange) -> Vec<Entry>;
+}
+
+/// External plugin that parses a literal into recognized Orchid lexemes, most
+/// likely atoms.
+pub trait LexerPlugin: Send + Sync + DynClone {
+  /// Run the lexer
+  fn lex<'a>(&self, req: &'_ dyn LexPluginReq<'a>) -> Option<ProjectResult<LexRes<'a>>>;
+}
+
+/// Implementation of [LexPluginReq]
+pub struct LexPlugReqImpl<'a, 'b, TCtx: ParseCtx> {
+  /// Text to be lexed
+  pub tail: &'a str,
+  /// Context data
+  pub ctx: &'b TCtx,
+}
+impl<'a, 'b, TCtx: ParseCtx> LexPluginReq<'a> for LexPlugReqImpl<'a, 'b, TCtx> {
+  fn tail(&self) -> &'a str { self.tail }
+  fn ctx(&self) -> &dyn ParseCtx { self.ctx }
+  fn recurse(&self, req: LexPluginRecur<'a, '_>) -> ProjectResult<LexRes<'a>> {
+    lex(Vec::new(), req.tail, self.ctx, |s| (req.exit)(s))
+  }
+  fn insert(&self, data: &str, range: SourceRange) -> Vec<Entry> {
+    let ctx = FlatLocContext::new(self.ctx as &dyn ParseCtx, &range);
+    lex(Vec::new(), data, &ctx, |_| Ok::<_, Never>(false)).unwrap_or_else(|e| match e {}).tokens
+  }
+}

orchidlang/src/parse/lexer.rs

@@ -0,0 +1,318 @@
+//! Convert source text into a sequence of tokens. Newlines and comments are
+//! included, but spacing is converted into numerical ranges on the elements.
+//!
+//! Literals lose their syntax form here and are handled in an abstract
+//! representation hence
+
+use std::fmt;
+use std::ops::Range;
+use std::sync::Arc;
+
+use intern_all::{i, Tok};
+use itertools::Itertools;
+use ordered_float::NotNan;
+
+use super::context::ParseCtx;
+use super::errors::{FloatPlacehPrio, NoCommentEnd};
+use super::lex_plugin::LexerPlugin;
+use super::numeric::{numstart, parse_num, print_nat16};
+use crate::foreign::atom::AtomGenerator;
+use crate::libs::std::number::Numeric;
+use crate::parse::errors::ParseErrorKind;
+use crate::parse::lex_plugin::LexPlugReqImpl;
+use crate::parse::numeric::{numchar, NumericLexer};
+use crate::parse::parsed::{PHClass, PType, Placeholder};
+
+/// A lexeme and the location where it was found
+#[derive(Clone, Debug)]
+pub struct Entry {
+  /// the lexeme
+  pub lexeme: Lexeme,
+  /// the range in bytes
+  pub range: Range<usize>,
+}
+impl Entry {
+  /// Checks if the lexeme is a comment or line break
+  #[must_use]
+  pub fn is_filler(&self) -> bool { matches!(self.lexeme, Lexeme::Comment(_) | Lexeme::BR) }
+
+  /// Create a new entry
+  #[must_use]
+  pub fn new(range: Range<usize>, lexeme: Lexeme) -> Self { Self { lexeme, range } }
+}
+
+impl fmt::Display for Entry {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.lexeme.fmt(f) }
+}
+
+impl PartialEq<Lexeme> for Entry {
+  fn eq(&self, other: &Lexeme) -> bool { self.lexeme == *other }
+}
+
+/// A unit of syntax
+#[derive(Clone, Debug, PartialEq)]
+pub enum Lexeme {
+  /// Atoms parsed by plugins
+  Atom(AtomGenerator),
+  /// Keyword or name
+  Name(Tok<String>),
+  /// Macro operator `=`number`=>`
+  Arrow(NotNan<f64>),
+  /// `:=`
+  Walrus,
+  /// Line break
+  BR,
+  /// `::`
+  NS,
+  /// Left paren `([{`
+  LP(PType),
+  /// Right paren `)]}`
+  RP(PType),
+  /// `\`
+  BS,
+  /// `@``
+  At,
+  /// `:`
+  Type,
+  /// comment
+  Comment(Arc<String>),
+  /// placeholder in a macro.
+  Placeh(Placeholder),
+}
+
+impl fmt::Display for Lexeme {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Atom(a) => write!(f, "{a:?}"),
+      Self::Name(token) => write!(f, "{}", **token),
+      Self::Walrus => write!(f, ":="),
+      Self::Arrow(prio) => write!(f, "={}=>", print_nat16(*prio)),
+      Self::NS => write!(f, "::"),
+      Self::LP(t) => write!(f, "{}", t.l()),
+      Self::RP(t) => write!(f, "{}", t.r()),
+      Self::BR => writeln!(f),
+      Self::BS => write!(f, "\\"),
+      Self::At => write!(f, "@"),
+      Self::Type => write!(f, ":"),
+      Self::Comment(text) => write!(f, "--[{}]--", text),
+      Self::Placeh(ph) => write!(f, "{ph}"),
+    }
+  }
+}
+
+impl Lexeme {
+  /// Compare lexemes for equality. It's `strict` because for atoms it uses the
+  /// strict equality comparison
+  pub fn strict_eq(&self, other: &Self) -> bool {
+    match (self, other) {
+      (Self::Arrow(f1), Self::Arrow(f2)) => f1 == f2,
+      (Self::At, Self::At) | (Self::BR, Self::BR) => true,
+      (Self::BS, Self::BS) => true,
+      (Self::NS, Self::NS) | (Self::Type, Self::Type) => true,
+      (Self::Walrus, Self::Walrus) => true,
+      (Self::Atom(a1), Self::Atom(a2)) => a1.run().0.parser_eq(&*a2.run().0),
+      (Self::Comment(c1), Self::Comment(c2)) => c1 == c2,
+      (Self::LP(p1), Self::LP(p2)) | (Self::RP(p1), Self::RP(p2)) => p1 == p2,
+      (Self::Name(n1), Self::Name(n2)) => n1 == n2,
+      (Self::Placeh(ph1), Self::Placeh(ph2)) => ph1 == ph2,
+      (..) => false,
+    }
+  }
+}
+
+/// Data returned from the lexer
+pub struct LexRes<'a> {
+  /// Leftover text. If the bail callback never returned true, this is empty
+  pub tail: &'a str,
+  /// Lexemes extracted from the text
+  pub tokens: Vec<Entry>,
+}
+
+/// Neatly format source code
+#[allow(unused)]
+pub fn format(lexed: &[Entry]) -> String { lexed.iter().join(" ") }
+
+/// Character filter that can appear in a keyword or name
+pub fn namechar(c: char) -> bool { c.is_alphanumeric() | (c == '_') }
+/// Character filter that can start a name
+pub fn namestart(c: char) -> bool { c.is_alphabetic() | (c == '_') }
+/// Character filter that can appear in operators.
+pub fn opchar(c: char) -> bool {
+  !namestart(c) && !numstart(c) && !c.is_whitespace() && !"()[]{},'\"\\".contains(c)
+}
+
+/// Split off all characters from the beginning that match a filter
+pub fn split_filter(s: &str, mut pred: impl FnMut(char) -> bool) -> (&str, &str) {
+  s.find(|c| !pred(c)).map_or((s, ""), |i| s.split_at(i))
+}
+
+fn lit_table() -> impl IntoIterator<Item = (&'static str, Lexeme)> {
+  [
+    ("\\", Lexeme::BS),
+    ("@", Lexeme::At),
+    ("(", Lexeme::LP(PType::Par)),
+    ("[", Lexeme::LP(PType::Sqr)),
+    ("{", Lexeme::LP(PType::Curl)),
+    (")", Lexeme::RP(PType::Par)),
+    ("]", Lexeme::RP(PType::Sqr)),
+    ("}", Lexeme::RP(PType::Curl)),
+    ("\n", Lexeme::BR),
+    (":=", Lexeme::Walrus),
+    ("::", Lexeme::NS),
+    (":", Lexeme::Type),
+  ]
+}
+
+static BUILTIN_ATOMS: &[&dyn LexerPlugin] = &[&NumericLexer];
+
+/// Convert source code to a flat list of tokens. The bail callback will be
+/// called between lexemes. When it returns true, the remaining text is
+/// returned without processing.
+pub fn lex<'a, E>(
+  mut tokens: Vec<Entry>,
+  mut data: &'a str,
+  ctx: &'_ impl ParseCtx,
+  mut bail: impl FnMut(&str) -> Result<bool, E>,
+) -> Result<LexRes<'a>, E> {
+  let mut prev_len = data.len() + 1;
+  'tail: loop {
+    if prev_len == data.len() {
+      panic!("got stuck at {data:?}, parsed {:?}", tokens.last().unwrap());
+    }
+    prev_len = data.len();
+    data = data.trim_start_matches(|c: char| c.is_whitespace() && c != '\n');
+    if bail(data)? {
+      return Ok(LexRes { tokens, tail: data });
+    }
+    let mut chars = data.chars();
+    let head = match chars.next() {
+      None => return Ok(LexRes { tokens, tail: data }),
+      Some(h) => h,
+    };
+    for lexer in ctx.lexers().chain(BUILTIN_ATOMS.iter().copied()) {
+      let req = LexPlugReqImpl { tail: data, ctx };
+      if let Some(res) = lexer.lex(&req) {
+        let LexRes { tail, tokens: mut new_tokens } =
+          ctx.reporter().fallback(res, |_| LexRes { tail: "", tokens: vec![] });
+        // fallback: no tokens left, no additional tokens parsed
+        if tail.len() == data.len() {
+          panic!("lexer plugin consumed 0 characters")
+        }
+        tokens.append(&mut new_tokens);
+        data = tail;
+        continue 'tail;
+      }
+    }
+    for (prefix, lexeme) in lit_table() {
+      if let Some(tail) = data.strip_prefix(prefix) {
+        tokens.push(Entry::new(ctx.range(prefix.len(), tail), lexeme.clone()));
+        data = tail;
+        continue 'tail;
+      }
+    }
+
+    if let Some(tail) = data.strip_prefix(',') {
+      tokens.push(Entry::new(ctx.range(1, tail), Lexeme::Name(i!(str: ","))));
+      data = tail;
+      continue 'tail;
+    }
+    if let Some(tail) = data.strip_prefix("--[") {
+      let (note, tail) = tail.split_once("]--").unwrap_or_else(|| {
+        ctx.reporter().report(NoCommentEnd.pack(ctx.source_range(tail.len(), "")));
+        (tail, "") // fallback: the rest of the file is in the comment
+      });
+      let lexeme = Lexeme::Comment(Arc::new(note.to_string()));
+      tokens.push(Entry::new(ctx.range(note.len() + 3, tail), lexeme));
+      data = tail;
+      continue 'tail;
+    }
+    if let Some(tail) = data.strip_prefix("--") {
+      let (note, tail) = split_filter(tail, |c| c != '\n');
+      let lexeme = Lexeme::Comment(Arc::new(note.to_string()));
+      tokens.push(Entry::new(ctx.range(note.len(), tail), lexeme));
+      data = tail;
+      continue 'tail;
+    }
+    // Parse a rule arrow
+    if let Some(tail) = data.strip_prefix('=') {
+      if tail.chars().next().map_or(false, numstart) {
+        let (num, post_num) = split_filter(tail, numchar);
+        if let Some(tail) = post_num.strip_prefix("=>") {
+          let prio = parse_num(num).unwrap_or_else(|e| {
+            ctx.reporter().report(e.into_proj(num.len(), post_num, ctx));
+            Numeric::Uint(0)
+          });
+          let lexeme = Lexeme::Arrow(prio.as_float());
+          tokens.push(Entry::new(ctx.range(num.len() + 3, tail), lexeme));
+          data = tail;
+          continue 'tail;
+        }
+      }
+    }
+    // Parse scalar placeholder $_name or $name
+    if let Some(tail) = data.strip_prefix('$') {
+      let (nameonly, tail) = tail.strip_prefix('_').map_or((false, tail), |t| (true, t));
+      let (name, tail) = split_filter(tail, namechar);
+      if !name.is_empty() {
+        let class = if nameonly { PHClass::Name } else { PHClass::Scalar };
+        let lexeme = Lexeme::Placeh(Placeholder { name: i(name), class });
+        tokens.push(Entry::new(ctx.range(name.len() + 1, tail), lexeme));
+        data = tail;
+        continue 'tail;
+      }
+    }
+    // Parse vectorial placeholder. `..` or `...`, then `$name`, then an optional
+    // `:n` where n is a number.
+    if let Some(tail) = data.strip_prefix("..") {
+      let (nonzero, tail) = tail.strip_prefix('.').map_or((false, tail), |t| (true, t));
+      if let Some(tail) = tail.strip_prefix('$') {
+        let (name, tail) = split_filter(tail, namechar);
+        if !name.is_empty() {
+          let (prio, priolen, tail) = tail
+            .strip_prefix(':')
+            .map(|tail| split_filter(tail, numchar))
+            .filter(|(num, _)| !num.is_empty())
+            .map(|(num_str, tail)| {
+              let p = ctx.reporter().fallback(
+                parse_num(num_str).map_err(|e| e.into_proj(num_str.len(), tail, ctx)).and_then(
+                  |num| match num {
+                    Numeric::Uint(usize) => Ok(usize),
+                    Numeric::Float(_) =>
+                      Err(FloatPlacehPrio.pack(ctx.source_range(num_str.len(), tail))),
+                  },
+                ),
+                |_| 0,
+              );
+              (p, num_str.len() + 1, tail)
+            })
+            .unwrap_or((0, 0, tail));
+          let byte_len = if nonzero { 4 } else { 3 } + priolen + name.len();
+          let class = PHClass::Vec { nonzero, prio };
+          let lexeme = Lexeme::Placeh(Placeholder { name: i(name), class });
+          tokens.push(Entry::new(ctx.range(byte_len, tail), lexeme));
+          data = tail;
+          continue 'tail;
+        }
+      }
+    }
+    if namestart(head) {
+      let (name, tail) = split_filter(data, namechar);
+      if !name.is_empty() {
+        let lexeme = Lexeme::Name(i(name));
+        tokens.push(Entry::new(ctx.range(name.len(), tail), lexeme));
+        data = tail;
+        continue 'tail;
+      }
+    }
+    if opchar(head) {
+      let (name, tail) = split_filter(data, opchar);
+      if !name.is_empty() {
+        let lexeme = Lexeme::Name(i(name));
+        tokens.push(Entry::new(ctx.range(name.len(), tail), lexeme));
+        data = tail;
+        continue 'tail;
+      }
+    }
+    unreachable!(r#"opchar is pretty much defined as "not namechar" "#)
+  }
+}

orchidlang/src/parse/mod.rs

@@ -0,0 +1,12 @@
+//! Parser, and abstractions for interacting with it from language extensions
+pub mod context;
+pub mod errors;
+pub mod facade;
+pub mod frag;
+pub mod lex_plugin;
+pub mod lexer;
+pub mod multiname;
+pub mod numeric;
+pub mod parse_plugin;
+pub mod parsed;
+mod sourcefile;

orchidlang/src/parse/multiname.rs

@@ -0,0 +1,146 @@
+//! Parse the tree-like name sets used to represent imports
+
+use std::collections::VecDeque;
+use std::ops::Range;
+
+use intern_all::{i, Tok};
+
+use super::context::ParseCtx;
+use super::errors::{Expected, ParseErrorKind};
+use super::frag::Frag;
+use super::lexer::{Entry, Lexeme};
+use crate::error::ProjectResult;
+use crate::location::SourceRange;
+use crate::name::VPath;
+use crate::parse::parsed::{Import, PType};
+use crate::utils::boxed_iter::{box_chain, box_once, BoxedIter};
+
+struct Subresult {
+  glob: bool,
+  deque: VecDeque<Tok<String>>,
+  range: Range<usize>,
+}
+impl Subresult {
+  #[must_use]
+  fn new_glob(range: &Range<usize>) -> Self {
+    Self { glob: true, deque: VecDeque::new(), range: range.clone() }
+  }
+
+  #[must_use]
+  fn new_named(name: Tok<String>, range: &Range<usize>) -> Self {
+    Self { glob: false, deque: VecDeque::from([name]), range: range.clone() }
+  }
+
+  #[must_use]
+  fn push_front(mut self, name: Tok<String>) -> Self {
+    self.deque.push_front(name);
+    self
+  }
+
+  #[must_use]
+  fn finalize(self, ctx: &(impl ParseCtx + ?Sized)) -> Import {
+    let Self { mut deque, glob, range } = self;
+    debug_assert!(glob || !deque.is_empty(), "The constructors forbid this");
+    let name = if glob { None } else { deque.pop_back() };
+    let range = ctx.range_loc(&range);
+    Import { name, range, path: VPath(deque.into()) }
+  }
+}
+
+fn parse_multiname_branch<'a>(
+  cursor: Frag<'a>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<(BoxedIter<'a, Subresult>, Frag<'a>)> {
+  let comma = i!(str: ",");
+  let (subnames, cursor) = parse_multiname_rec(cursor, ctx)?;
+  let (Entry { lexeme, range }, cursor) = cursor.trim().pop(ctx)?;
+  match &lexeme {
+    Lexeme::RP(PType::Par) => Ok((subnames, cursor)),
+    Lexeme::Name(n) if n == &comma => {
+      let (tail, cont) = parse_multiname_branch(cursor, ctx)?;
+      Ok((box_chain!(subnames, tail), cont))
+    },
+    _ => {
+      let expected = vec![Lexeme::Name(comma), Lexeme::RP(PType::Par)];
+      let err = Expected { expected, or_name: false, found: lexeme.clone() };
+      Err(err.pack(SourceRange { range: range.clone(), code: ctx.code_info() }))
+    },
+  }
+}
+
+fn parse_multiname_rec<'a>(
+  cursor: Frag<'a>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<(BoxedIter<'a, Subresult>, Frag<'a>)> {
+  let (head, mut cursor) = cursor.trim().pop(ctx)?;
+  match &head.lexeme {
+    Lexeme::LP(PType::Par) => parse_multiname_branch(cursor, ctx),
+    Lexeme::LP(PType::Sqr) => {
+      let mut names = Vec::new();
+      loop {
+        let (Entry { lexeme, range }, tail) = cursor.trim().pop(ctx)?;
+        cursor = tail;
+        match lexeme {
+          Lexeme::Name(n) => names.push((n.clone(), range)),
+          Lexeme::RP(PType::Sqr) => break,
+          _ => {
+            let err = Expected {
+              expected: vec![Lexeme::RP(PType::Sqr)],
+              or_name: true,
+              found: head.lexeme.clone(),
+            };
+            return Err(err.pack(ctx.range_loc(range)));
+          },
+        }
+      }
+      Ok((
+        Box::new(
+          names.into_iter().map(|(name, location)| Subresult::new_named(name.clone(), location)),
+        ),
+        cursor,
+      ))
+    },
+    Lexeme::Name(n) if *n == i!(str: "*") =>
+      Ok((box_once(Subresult::new_glob(&head.range)), cursor)),
+    Lexeme::Name(n) if ![i!(str: ","), i!(str: "*")].contains(n) => {
+      let cursor = cursor.trim();
+      if cursor.get(0, ctx).map_or(false, |e| e.lexeme.strict_eq(&Lexeme::NS)) {
+        let cursor = cursor.step(ctx)?;
+        let (out, cursor) = parse_multiname_rec(cursor, ctx)?;
+        let out = Box::new(out.map(|sr| sr.push_front(n.clone())));
+        Ok((out, cursor))
+      } else {
+        Ok((box_once(Subresult::new_named(n.clone(), &head.range)), cursor))
+      }
+    },
+    _ => {
+      let expected = vec![Lexeme::LP(PType::Par)];
+      let err = Expected { expected, or_name: true, found: head.lexeme.clone() };
+      Err(err.pack(ctx.range_loc(&head.range)))
+    },
+  }
+}
+
+/// Parse a tree that describes several names. The tree can be
+///
+/// - name (except `,` or `*`)
+/// - name (except `,` or `*`) `::` tree
+/// - `(` tree `,` tree ... `)`
+/// - `*` (wildcard)
+/// - `[` name name ... `]` (including `,` or `*`).
+///
+/// Examples of valid syntax:
+///
+/// ```txt
+/// foo
+/// foo::bar::baz
+/// foo::bar::(baz, quz::quux, fimble::*)
+/// foo::bar::[baz quz * +]
+/// ```
+pub fn parse_multiname<'a>(
+  cursor: Frag<'a>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<(Vec<Import>, Frag<'a>)> {
+  let (output, cont) = parse_multiname_rec(cursor, ctx)?;
+  Ok((output.map(|sr| sr.finalize(ctx)).collect(), cont))
+}

orchidlang/src/parse/numeric.rs

@@ -0,0 +1,179 @@
+//! Parse a float or integer. These functions are also used for the macro
+//! priority numbers
+
+use std::num::IntErrorKind;
+use std::ops::Range;
+
+use ordered_float::NotNan;
+
+use super::context::ParseCtx;
+use super::errors::{ExpectedDigit, LiteralOverflow, NaNLiteral, ParseErrorKind};
+use super::lex_plugin::LexPluginReq;
+#[allow(unused)] // for doc
+use super::lex_plugin::LexerPlugin;
+use super::lexer::{split_filter, Entry, LexRes, Lexeme};
+use crate::error::{ProjectErrorObj, ProjectResult};
+use crate::foreign::atom::AtomGenerator;
+use crate::foreign::inert::Inert;
+use crate::libs::std::number::Numeric;
+
+/// Rasons why [parse_num] might fail. See [NumError].
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum NumErrorKind {
+  /// The literal describes [f64::NAN]
+  NaN,
+  /// Some integer appearing in the literal overflows [usize]
+  Overflow,
+  /// A character that isn't a digit in the given base was found
+  InvalidDigit,
+}
+impl NumErrorKind {
+  fn from_int(kind: &IntErrorKind) -> Self {
+    match kind {
+      IntErrorKind::InvalidDigit => Self::InvalidDigit,
+      IntErrorKind::NegOverflow | IntErrorKind::PosOverflow => Self::Overflow,
+      _ => panic!("Impossible error condition"),
+    }
+  }
+}
+
+/// Error produced by [parse_num]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct NumError {
+  /// Location
+  pub range: Range<usize>,
+  /// Reason
+  pub kind: NumErrorKind,
+}
+
+impl NumError {
+  /// Convert into [ProjectErrorObj]
+  pub fn into_proj(
+    self,
+    len: usize,
+    tail: &str,
+    ctx: &(impl ParseCtx + ?Sized),
+  ) -> ProjectErrorObj {
+    let start = ctx.source().len() - tail.len() - len + self.range.start;
+    let location = ctx.range_loc(&(start..start + self.range.len()));
+    match self.kind {
+      NumErrorKind::NaN => NaNLiteral.pack(location),
+      NumErrorKind::InvalidDigit => ExpectedDigit.pack(location),
+      NumErrorKind::Overflow => LiteralOverflow.pack(location),
+    }
+  }
+}
+
+/// Parse a numbre literal out of text
+pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
+  let overflow_err = NumError { range: 0..string.len(), kind: NumErrorKind::Overflow };
+  let (radix, noprefix, pos) = (string.strip_prefix("0x").map(|s| (16u8, s, 2)))
+    .or_else(|| string.strip_prefix("0b").map(|s| (2u8, s, 2)))
+    .or_else(|| string.strip_prefix("0o").map(|s| (8u8, s, 2)))
+    .unwrap_or((10u8, string, 0));
+  // identity
+  let (base, exponent) = match noprefix.split_once('p') {
+    Some((b, e)) => {
+      let (s, d, len) = e.strip_prefix('-').map_or((1, e, 0), |ue| (-1, ue, 1));
+      (b, s * int_parse(d, 10, pos + b.len() + 1 + len)? as i32)
+    },
+    None => (noprefix, 0),
+  };
+  match base.split_once('.') {
+    None => {
+      let base_usize = int_parse(base, radix, pos)?;
+      if let Ok(pos_exp) = u32::try_from(exponent) {
+        if let Some(radical) = usize::from(radix).checked_pow(pos_exp) {
+          let number = base_usize.checked_mul(radical).ok_or(overflow_err)?;
+          return Ok(Numeric::Uint(number));
+        }
+      }
+      let f = (base_usize as f64) * (radix as f64).powi(exponent);
+      let err = NumError { range: 0..string.len(), kind: NumErrorKind::NaN };
+      Ok(Numeric::Float(NotNan::new(f).map_err(|_| err)?))
+    },
+    Some((whole, part)) => {
+      let whole_n = int_parse(whole, radix, pos)? as f64;
+      let part_n = int_parse(part, radix, pos + whole.len() + 1)? as f64;
+      let real_val = whole_n + (part_n / (radix as f64).powi(part.len() as i32));
+      let f = real_val * (radix as f64).powi(exponent);
+      Ok(Numeric::Float(NotNan::new(f).expect("None of the inputs are NaN")))
+    },
+  }
+}
+
+fn int_parse(s: &str, radix: u8, start: usize) -> Result<usize, NumError> {
+  let s = s.chars().filter(|c| *c != '_').collect::<String>();
+  let range = start..(start + s.len());
+  usize::from_str_radix(&s, radix as u32)
+    .map_err(|e| NumError { range, kind: NumErrorKind::from_int(e.kind()) })
+}
+
+/// Filter for characters that can appear in numbers
+pub fn numchar(c: char) -> bool { c.is_alphanumeric() | "._-".contains(c) }
+/// Filter for characters that can start numbers
+pub fn numstart(c: char) -> bool { c.is_ascii_digit() }
+
+/// Print a number as a base-16 floating point literal
+#[must_use]
+pub fn print_nat16(num: NotNan<f64>) -> String {
+  if *num == 0.0 {
+    return "0x0".to_string();
+  } else if num.is_infinite() {
+    return match num.is_sign_positive() {
+      true => "Infinity".to_string(),
+      false => "-Infinity".to_string(),
+    };
+  } else if num.is_nan() {
+    return "NaN".to_string();
+  }
+  let exp = num.log(16.0).floor();
+  let man = *num / 16_f64.powf(exp);
+  format!("0x{man}p{exp:.0}")
+}
+
+/// [LexerPlugin] for a number literal
+#[derive(Clone)]
+pub struct NumericLexer;
+impl LexerPlugin for NumericLexer {
+  fn lex<'b>(&self, req: &'_ dyn LexPluginReq<'b>) -> Option<ProjectResult<LexRes<'b>>> {
+    req.tail().chars().next().filter(|c| numstart(*c)).map(|_| {
+      let (num_str, tail) = split_filter(req.tail(), numchar);
+      let ag = match parse_num(num_str) {
+        Ok(Numeric::Float(f)) => AtomGenerator::cloner(Inert(f)),
+        Ok(Numeric::Uint(i)) => AtomGenerator::cloner(Inert(i)),
+        Err(e) => return Err(e.into_proj(num_str.len(), tail, req.ctx())),
+      };
+      let range = req.ctx().range(num_str.len(), tail);
+      let entry = Entry { lexeme: Lexeme::Atom(ag), range };
+      Ok(LexRes { tail, tokens: vec![entry] })
+    })
+  }
+}
+
+#[cfg(test)]
+mod test {
+  use crate::libs::std::number::Numeric;
+  use crate::parse::numeric::parse_num;
+
+  #[test]
+  fn just_ints() {
+    let test = |s, n| assert_eq!(parse_num(s), Ok(Numeric::Uint(n)));
+    test("12345", 12345);
+    test("0xcafebabe", 0xcafebabe);
+    test("0o751", 0o751);
+    test("0b111000111", 0b111000111);
+  }
+
+  #[test]
+  fn decimals() {
+    let test = |s, n| assert_eq!(parse_num(s).map(|n| n.as_f64()), Ok(n));
+    test("3.1417", 3.1417);
+    test("3.1417", 3_f64 + 1417_f64 / 10000_f64);
+    test("0xf.cafe", 0xf as f64 + 0xcafe as f64 / 0x10000 as f64);
+    test("34p3", 34000f64);
+    test("0x2p3", (0x2 * 0x1000) as f64);
+    test("1.5p3", 1500f64);
+    test("0x2.5p3", (0x25 * 0x100) as f64);
+  }
+}

orchidlang/src/parse/parse_plugin.rs

@@ -0,0 +1,142 @@
+//! Abstractions for dynamic extensions to the parser that act across entries.
+//! Macros are the primary syntax extension  mechanism, but they only operate
+//! within a constant and can't interfere with name reproject.
+
+use std::ops::Range;
+
+use dyn_clone::DynClone;
+use intern_all::Tok;
+
+use super::context::ParseCtx;
+use super::errors::{expect, expect_block, expect_name};
+use super::facade::parse_entries;
+use super::frag::Frag;
+use super::lexer::{Entry, Lexeme};
+use super::parsed::{Constant, Expr, ModuleBlock, PType, Rule, SourceLine, SourceLineKind};
+use super::sourcefile::{
+  exprv_to_single, parse_const, parse_exprv, parse_line, parse_module, parse_module_body,
+  parse_nsname, parse_rule, split_lines,
+};
+use crate::error::{ProjectErrorObj, ProjectResult};
+use crate::location::SourceRange;
+use crate::name::VName;
+use crate::utils::boxed_iter::BoxedIter;
+
+/// Information and actions exposed to [ParseLinePlugin]. A plugin should never
+/// import and call the parser directly because it might be executed in a
+/// different version of the parser.
+pub trait ParsePluginReq<'t> {
+  // ################ Frag and ParseCtx ################
+
+  /// The token sequence this parser must parse
+  fn frag(&self) -> Frag;
+  /// Get the location of a fragment
+  fn frag_loc(&self, f: Frag) -> SourceRange;
+  /// Convert a numeric byte range into a location
+  fn range_loc(&self, r: Range<usize>) -> SourceRange;
+  /// Remove the first token of the fragment
+  fn pop<'a>(&self, f: Frag<'a>) -> ProjectResult<(&'a Entry, Frag<'a>)>;
+  /// Remove the last element of the fragment
+  fn pop_back<'a>(&self, f: Frag<'a>) -> ProjectResult<(&'a Entry, Frag<'a>)>;
+
+  // ################ Parser states ################
+
+  /// Split up the lines in a fragment. The fragment must outlive the iterator
+  /// and the request itself must outlive both
+  fn split_lines<'a: 'b, 'b>(&'b self, f: Frag<'a>) -> BoxedIter<'b, Frag<'a>>
+  where 't: 'b + 'a;
+  /// Parse a sequence of source lines separated by line breaks
+  fn parse_module_body(&self, frag: Frag) -> ProjectResult<Vec<SourceLine>>;
+  /// Parse a single source line. This returns a vector because plugins can
+  /// convert a single line into multiple entries
+  fn parse_line(&self, frag: Frag) -> ProjectResult<Vec<SourceLineKind>>;
+  /// Parse a macro rule `<exprv> =prio=> <exprv>`
+  fn parse_rule(&self, frag: Frag) -> ProjectResult<Rule>;
+  /// Parse a constant declaration `<name> := <exprv>`
+  fn parse_const(&self, frag: Frag) -> ProjectResult<Constant>;
+  /// Parse a namespaced name `name::name`
+  fn parse_nsname<'a>(&self, f: Frag<'a>) -> ProjectResult<(VName, Frag<'a>)>;
+  /// Parse a module declaration. `<name> ( <module_body> )`
+  fn parse_module(&self, frag: Frag) -> ProjectResult<ModuleBlock>;
+  /// Parse a sequence of expressions. In principle, it never makes sense to
+  /// parse a single expression because it could always be a macro invocation.
+  fn parse_exprv<'a>(&self, f: Frag<'a>, p: Option<PType>) -> ProjectResult<(Vec<Expr>, Frag<'a>)>;
+  /// Parse a prepared string of code
+  fn parse_entries(&self, t: &'static str, r: SourceRange) -> Vec<SourceLine>;
+  /// Convert a sequence of expressions to a single one by parenthesization if
+  /// necessary
+  fn vec_to_single(&self, fallback: &Entry, v: Vec<Expr>) -> ProjectResult<Expr>;
+
+  // ################ Assertions ################
+
+  /// Unwrap a single name token or raise an error
+  fn expect_name(&self, entry: &Entry) -> ProjectResult<Tok<String>>;
+  /// Assert that the entry contains exactly the specified lexeme
+  fn expect(&self, l: Lexeme, e: &Entry) -> ProjectResult<()>;
+  /// Remove two parentheses from the ends of the cursor
+  fn expect_block<'a>(&self, f: Frag<'a>, p: PType) -> ProjectResult<Frag<'a>>;
+  /// Ensure that the fragment is empty
+  fn expect_empty(&self, f: Frag) -> ProjectResult<()>;
+  /// Report a fatal error while also producing output to be consumed by later
+  /// stages for improved error reporting
+  fn report_err(&self, e: ProjectErrorObj);
+}
+
+/// External plugin that parses an unrecognized source line into lines of
+/// recognized types
+pub trait ParseLinePlugin: Sync + Send + DynClone {
+  /// Attempt to parse a line. Returns [None] if the line isn't recognized,
+  /// [Some][Err] if it's recognized but incorrect.
+  fn parse(&self, req: &dyn ParsePluginReq) -> Option<ProjectResult<Vec<SourceLineKind>>>;
+}
+
+/// Implementation of [ParsePluginReq] exposing sub-parsers and data to the
+/// plugin via dynamic dispatch
+pub struct ParsePlugReqImpl<'a, TCtx: ParseCtx + ?Sized> {
+  /// Fragment of text to be parsed by the plugin
+  pub frag: Frag<'a>,
+  /// Context for recursive commands and to expose to the plugin
+  pub ctx: &'a TCtx,
+}
+impl<'ty, TCtx: ParseCtx + ?Sized> ParsePluginReq<'ty> for ParsePlugReqImpl<'ty, TCtx> {
+  fn frag(&self) -> Frag { self.frag }
+  fn frag_loc(&self, f: Frag) -> SourceRange { self.range_loc(f.range()) }
+  fn range_loc(&self, r: Range<usize>) -> SourceRange { self.ctx.range_loc(&r) }
+  fn pop<'a>(&self, f: Frag<'a>) -> ProjectResult<(&'a Entry, Frag<'a>)> { f.pop(self.ctx) }
+  fn pop_back<'a>(&self, f: Frag<'a>) -> ProjectResult<(&'a Entry, Frag<'a>)> {
+    f.pop_back(self.ctx)
+  }
+  fn split_lines<'a: 'b, 'b>(&'b self, f: Frag<'a>) -> BoxedIter<'b, Frag<'a>>
+  where
+    'ty: 'b,
+    'ty: 'a,
+  {
+    Box::new(split_lines(f, self.ctx))
+  }
+  fn parse_module_body(&self, f: Frag) -> ProjectResult<Vec<SourceLine>> {
+    Ok(parse_module_body(f, self.ctx))
+  }
+  fn parse_line(&self, f: Frag) -> ProjectResult<Vec<SourceLineKind>> { parse_line(f, self.ctx) }
+  fn parse_rule(&self, f: Frag) -> ProjectResult<Rule> { parse_rule(f, self.ctx) }
+  fn parse_const(&self, f: Frag) -> ProjectResult<Constant> { parse_const(f, self.ctx) }
+  fn parse_nsname<'a>(&self, f: Frag<'a>) -> ProjectResult<(VName, Frag<'a>)> {
+    parse_nsname(f, self.ctx)
+  }
+  fn parse_module(&self, f: Frag) -> ProjectResult<ModuleBlock> { parse_module(f, self.ctx) }
+  fn parse_exprv<'a>(&self, f: Frag<'a>, p: Option<PType>) -> ProjectResult<(Vec<Expr>, Frag<'a>)> {
+    parse_exprv(f, p, self.ctx)
+  }
+  fn parse_entries(&self, s: &'static str, r: SourceRange) -> Vec<SourceLine> {
+    parse_entries(&self.ctx, s, r)
+  }
+  fn vec_to_single(&self, fb: &Entry, v: Vec<Expr>) -> ProjectResult<Expr> {
+    exprv_to_single(fb, v, self.ctx)
+  }
+  fn expect_name(&self, e: &Entry) -> ProjectResult<Tok<String>> { expect_name(e, self.ctx) }
+  fn expect(&self, l: Lexeme, e: &Entry) -> ProjectResult<()> { expect(l, e, self.ctx) }
+  fn expect_block<'a>(&self, f: Frag<'a>, t: PType) -> ProjectResult<Frag<'a>> {
+    expect_block(f, t, self.ctx)
+  }
+  fn expect_empty(&self, f: Frag) -> ProjectResult<()> { f.expect_empty(self.ctx) }
+  fn report_err(&self, e: ProjectErrorObj) { self.ctx.reporter().report(e) }
+}

orchidlang/src/parse/parsed.rs

@@ -0,0 +1,507 @@
+//! Datastructures representing the units of macro execution
+//!
+//! These structures are produced by the pipeline, processed by the macro
+//! executor, and then converted to other usable formats.
+
+use std::fmt;
+use std::hash::Hash;
+use std::rc::Rc;
+
+use hashbrown::HashSet;
+use intern_all::Tok;
+use itertools::Itertools;
+use ordered_float::NotNan;
+
+use crate::foreign::atom::AtomGenerator;
+#[allow(unused)] // for doc
+use crate::interpreter::nort;
+use crate::location::SourceRange;
+use crate::name::{Sym, VName, VPath};
+use crate::parse::numeric::print_nat16;
+
+/// A [Clause] with associated metadata
+#[derive(Clone, Debug)]
+pub struct Expr {
+  /// The actual value
+  pub value: Clause,
+  /// Information about the code that produced this value
+  pub range: SourceRange,
+}
+
+impl Expr {
+  /// Process all names with the given mapper.
+  /// Return a new object if anything was processed
+  #[must_use]
+  pub fn map_names(&self, pred: &mut impl FnMut(Sym) -> Option<Sym>) -> Option<Self> {
+    (self.value.map_names(pred)).map(|value| Self { value, range: self.range.clone() })
+  }
+
+  /// Visit all expressions in the tree. The search can be exited early by
+  /// returning [Some]
+  ///
+  /// See also [crate::interpreter::nort::Expr::search_all]
+  pub fn search_all<T>(&self, f: &mut impl FnMut(&Self) -> Option<T>) -> Option<T> {
+    f(self).or_else(|| self.value.search_all(f))
+  }
+}
+
+/// Visit all expression sequences including this sequence itself.
+pub fn search_all_slcs<T>(this: &[Expr], f: &mut impl FnMut(&[Expr]) -> Option<T>) -> Option<T> {
+  f(this).or_else(|| this.iter().find_map(|expr| expr.value.search_all_slcs(f)))
+}
+
+impl Expr {
+  /// Add the specified prefix to every Name
+  #[must_use]
+  pub fn prefix(&self, prefix: &[Tok<String>], except: &impl Fn(Tok<String>) -> bool) -> Self {
+    Self { value: self.value.prefix(prefix, except), range: self.range.clone() }
+  }
+}
+
+impl fmt::Display for Expr {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.value.fmt(f) }
+}
+
+/// Various types of placeholders
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum PHClass {
+  /// Matches multiple tokens, lambdas or parenthesized groups
+  Vec {
+    /// If true, must match at least one clause
+    nonzero: bool,
+    /// Greediness in the allocation of tokens
+    prio: usize,
+  },
+  /// Matches exactly one token, lambda or parenthesized group
+  Scalar,
+  /// Matches exactly one name
+  Name,
+}
+
+/// Properties of a placeholder that matches unknown tokens in macros
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct Placeholder {
+  /// Identifier to pair placeholders in the pattern and template
+  pub name: Tok<String>,
+  /// The nature of the token set matched by this placeholder
+  pub class: PHClass,
+}
+
+impl fmt::Display for Placeholder {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    let name = &self.name;
+    match self.class {
+      PHClass::Scalar => write!(f, "${name}"),
+      PHClass::Name => write!(f, "$_{name}"),
+      PHClass::Vec { nonzero, prio } => {
+        if nonzero { write!(f, "...") } else { write!(f, "..") }?;
+        write!(f, "${name}:{prio}")
+      },
+    }
+  }
+}
+
+/// Different types of brackets supported by Orchid
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
+pub enum PType {
+  /// ()
+  Par,
+  /// []
+  Sqr,
+  /// {}
+  Curl,
+}
+impl PType {
+  /// Left paren character for this paren type
+  pub fn l(self) -> char {
+    match self {
+      PType::Curl => '{',
+      PType::Par => '(',
+      PType::Sqr => '[',
+    }
+  }
+
+  /// Right paren character for this paren type
+  pub fn r(self) -> char {
+    match self {
+      PType::Curl => '}',
+      PType::Par => ')',
+      PType::Sqr => ']',
+    }
+  }
+}
+
+/// An S-expression as read from a source file
+#[derive(Debug, Clone)]
+pub enum Clause {
+  /// An opaque non-callable value, eg. a file handle
+  Atom(AtomGenerator),
+  /// A c-style name or an operator, eg. `+`, `i`, `foo::bar`
+  Name(Sym),
+  /// A parenthesized expression
+  /// eg. `(print out "hello")`, `[1, 2, 3]`, `{Some(t) => t}`
+  S(PType, Rc<Vec<Expr>>),
+  /// A function expression, eg. `\x. x + 1`
+  Lambda(Rc<Vec<Expr>>, Rc<Vec<Expr>>),
+  /// A placeholder for macros, eg. `$name`, `...$body`, `...$lhs:1`
+  Placeh(Placeholder),
+}
+
+impl Clause {
+  /// Extract the expressions from an auto, lambda or S
+  #[must_use]
+  pub fn body(&self) -> Option<Rc<Vec<Expr>>> {
+    match self {
+      Self::Lambda(_, body) | Self::S(_, body) => Some(body.clone()),
+      _ => None,
+    }
+  }
+
+  /// Convert with identical meaning
+  #[must_use]
+  pub fn into_expr(self, range: SourceRange) -> Expr {
+    if let Self::S(PType::Par, body) = &self {
+      if let [wrapped] = &body[..] {
+        return wrapped.clone();
+      }
+    }
+    Expr { value: self, range }
+  }
+
+  /// Convert with identical meaning
+  #[must_use]
+  pub fn from_exprs(exprs: &[Expr]) -> Option<Self> {
+    match exprs {
+      [] => None,
+      [only] => Some(only.value.clone()),
+      _ => Some(Self::S(PType::Par, Rc::new(exprs.to_vec()))),
+    }
+  }
+
+  /// Convert with identical meaning
+  #[must_use]
+  pub fn from_exprv(exprv: &Rc<Vec<Expr>>) -> Option<Clause> {
+    if exprv.len() < 2 { Self::from_exprs(exprv) } else { Some(Self::S(PType::Par, exprv.clone())) }
+  }
+
+  /// Collect all names that appear in this expression.
+  /// NOTICE: this isn't the total set of unbound names, it's mostly useful to
+  /// make weak statements for optimization.
+  #[must_use]
+  pub fn collect_names(&self) -> HashSet<Sym> {
+    if let Self::Name(n) = self {
+      return HashSet::from([n.clone()]);
+    }
+    let mut glossary = HashSet::new();
+    let result = self.search_all(&mut |e| {
+      if let Clause::Name(n) = &e.value {
+        glossary.insert(n.clone());
+      }
+      None::<()>
+    });
+    assert!(result.is_none(), "Callback never returns Some");
+    glossary
+  }
+
+  /// Process all names with the given mapper.
+  /// Return a new object if anything was processed
+  #[must_use]
+  pub fn map_names(&self, pred: &mut impl FnMut(Sym) -> Option<Sym>) -> Option<Self> {
+    match self {
+      Clause::Atom(_) | Clause::Placeh(_) => None,
+      Clause::Name(name) => pred(name.clone()).map(Clause::Name),
+      Clause::S(c, body) => {
+        let mut any_some = false;
+        let new_body = body
+          .iter()
+          .map(|e| {
+            let val = e.map_names(pred);
+            any_some |= val.is_some();
+            val.unwrap_or_else(|| e.clone())
+          })
+          .collect();
+        if any_some { Some(Clause::S(*c, Rc::new(new_body))) } else { None }
+      },
+      Clause::Lambda(arg, body) => {
+        let mut any_some = false;
+        let new_arg = (arg.iter())
+          .map(|e| {
+            let val = e.map_names(pred);
+            any_some |= val.is_some();
+            val.unwrap_or_else(|| e.clone())
+          })
+          .collect();
+        let new_body = (body.iter())
+          .map(|e| {
+            let val = e.map_names(pred);
+            any_some |= val.is_some();
+            val.unwrap_or_else(|| e.clone())
+          })
+          .collect();
+        if any_some { Some(Clause::Lambda(Rc::new(new_arg), Rc::new(new_body))) } else { None }
+      },
+    }
+  }
+
+  /// Pair of [Expr::search_all]
+  pub fn search_all<T>(&self, f: &mut impl FnMut(&Expr) -> Option<T>) -> Option<T> {
+    match self {
+      Clause::Lambda(arg, body) =>
+        arg.iter().chain(body.iter()).find_map(|expr| expr.search_all(f)),
+      Clause::Name(_) | Clause::Atom(_) | Clause::Placeh(_) => None,
+      Clause::S(_, body) => body.iter().find_map(|expr| expr.search_all(f)),
+    }
+  }
+
+  /// Visit all expression sequences. Most useful when looking for some pattern
+  pub fn search_all_slcs<T>(&self, f: &mut impl FnMut(&[Expr]) -> Option<T>) -> Option<T> {
+    match self {
+      Clause::Lambda(arg, body) => search_all_slcs(arg, f).or_else(|| search_all_slcs(body, f)),
+      Clause::Name(_) | Clause::Atom(_) | Clause::Placeh(_) => None,
+      Clause::S(_, body) => search_all_slcs(body, f),
+    }
+  }
+
+  /// Generate a parenthesized expression sequence
+  pub fn s(delimiter: char, body: impl IntoIterator<Item = Self>, range: SourceRange) -> Self {
+    let ptype = match delimiter {
+      '(' => PType::Par,
+      '[' => PType::Sqr,
+      '{' => PType::Curl,
+      _ => panic!("not an opening paren"),
+    };
+    let body = body.into_iter().map(|it| it.into_expr(range.clone())).collect();
+    Self::S(ptype, Rc::new(body))
+  }
+}
+
+impl Clause {
+  /// Add the specified prefix to every Name
+  #[must_use]
+  pub fn prefix(&self, prefix: &[Tok<String>], except: &impl Fn(Tok<String>) -> bool) -> Self {
+    self
+      .map_names(&mut |name| match except(name[0].clone()) {
+        true => None,
+        false => {
+          let prefixed = prefix.iter().cloned().chain(name.iter()).collect::<Vec<_>>();
+          Some(Sym::from_tok(name.tok().interner().i(&prefixed)).unwrap())
+        },
+      })
+      .unwrap_or_else(|| self.clone())
+  }
+}
+
+impl fmt::Display for Clause {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Atom(a) => write!(f, "{a:?}"),
+      Self::Name(name) => write!(f, "{}", name),
+      Self::S(t, items) => {
+        let body = items.iter().join(" ");
+        write!(f, "{}{body}{}", t.l(), t.r())
+      },
+      Self::Lambda(arg, body) => {
+        let args = arg.iter().join(" ");
+        let bodys = body.iter().join(" ");
+        write!(f, "\\{args}.{bodys}")
+      },
+      Self::Placeh(ph) => ph.fmt(f),
+    }
+  }
+}
+
+/// A substitution rule as loaded from source
+#[derive(Debug, Clone)]
+pub struct Rule {
+  /// Expressions on the left side of the arrow
+  pub pattern: Vec<Expr>,
+  /// Priority number written inside the arrow
+  pub prio: NotNan<f64>,
+  /// Expressions on the right side of the arrow
+  pub template: Vec<Expr>,
+}
+
+impl fmt::Display for Rule {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(
+      f,
+      "rule {} ={}=> {}",
+      self.pattern.iter().join(" "),
+      print_nat16(self.prio),
+      self.template.iter().join(" ")
+    )
+  }
+}
+
+/// A named constant
+#[derive(Debug, Clone)]
+pub struct Constant {
+  /// Used to reference the constant
+  pub name: Tok<String>,
+  /// The constant value inserted where the name is found
+  pub value: Expr,
+}
+
+impl fmt::Display for Constant {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "const {} := {}", *self.name, self.value)
+  }
+}
+
+/// An import pointing at another module, either specifying the symbol to be
+/// imported or importing all available symbols with a globstar (*)
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Import {
+  /// Import path, a sequence of module names. Can either start with
+  ///
+  /// - `self` to reference the current module
+  /// - any number of `super` to reference the parent module of the implied
+  ///   `self`
+  /// - a root name
+  pub path: VPath,
+  /// If name is None, this is a wildcard import
+  pub name: Option<Tok<String>>,
+  /// Location of the final name segment, which uniquely identifies this name
+  pub range: SourceRange,
+}
+impl Import {
+  /// Constructor
+  pub fn new(
+    path: impl IntoIterator<Item = Tok<String>>,
+    name: Option<Tok<String>>,
+    range: SourceRange,
+  ) -> Self {
+    let path = VPath(path.into_iter().collect());
+    assert!(name.is_some() || !path.0.is_empty(), "import * not allowed");
+    Self { range, name, path }
+  }
+
+  /// Get the preload target space for this import - the prefix below
+  /// which all files should be included in the compilation
+  ///
+  /// Returns the path if this is a glob import, or the path plus the
+  /// name if this is a specific import
+  #[must_use]
+  pub fn nonglob_path(&self) -> VName {
+    VName::new(self.path.0.iter().chain(&self.name).cloned())
+      .expect("Everything import (`import *`) not allowed")
+  }
+}
+
+impl fmt::Display for Import {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match &self.name {
+      None => write!(f, "{}::*", self.path),
+      Some(n) => write!(f, "{}::{}", self.path, n),
+    }
+  }
+}
+
+/// A namespace block
+#[derive(Debug, Clone)]
+pub struct ModuleBlock {
+  /// Name prefixed to all names in the block
+  pub name: Tok<String>,
+  /// Prefixed entries
+  pub body: Vec<SourceLine>,
+}
+
+impl fmt::Display for ModuleBlock {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    let bodys = self.body.iter().map(|e| e.to_string()).join("\n");
+    write!(f, "module {} {{\n{}\n}}", self.name, bodys)
+  }
+}
+
+/// see [Member]
+#[derive(Debug, Clone)]
+pub enum MemberKind {
+  /// A substitution rule. Rules apply even when they're not in scope, if the
+  /// absolute names are present eg. because they're produced by other rules
+  Rule(Rule),
+  /// A constant (or function) associated with a name
+  Constant(Constant),
+  /// A prefixed set of other entries
+  Module(ModuleBlock),
+}
+impl MemberKind {
+  /// Convert to [SourceLine]
+  pub fn into_line(self, exported: bool, range: SourceRange) -> SourceLine {
+    SourceLineKind::Member(Member { exported, kind: self }).wrap(range)
+  }
+}
+
+impl fmt::Display for MemberKind {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Constant(c) => c.fmt(f),
+      Self::Module(m) => m.fmt(f),
+      Self::Rule(r) => r.fmt(f),
+    }
+  }
+}
+
+/// Things that may be prefixed with an export
+/// see [MemberKind]
+#[derive(Debug, Clone)]
+pub struct Member {
+  /// Various members
+  pub kind: MemberKind,
+  /// Whether this member is exported or not
+  pub exported: bool,
+}
+
+impl fmt::Display for Member {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self { exported: true, kind } => write!(f, "export {kind}"),
+      Self { exported: false, kind } => write!(f, "{kind}"),
+    }
+  }
+}
+
+/// See [SourceLine]
+#[derive(Debug, Clone)]
+pub enum SourceLineKind {
+  /// Imports one or all names in a module
+  Import(Vec<Import>),
+  /// Comments are kept here in case dev tooling wants to parse documentation
+  Comment(String),
+  /// An element with visibility information
+  Member(Member),
+  /// A list of tokens exported explicitly. This can also create new exported
+  /// tokens that the local module doesn't actually define a role for
+  Export(Vec<(Tok<String>, SourceRange)>),
+}
+impl SourceLineKind {
+  /// Wrap with no location
+  pub fn wrap(self, range: SourceRange) -> SourceLine { SourceLine { kind: self, range } }
+}
+
+impl fmt::Display for SourceLineKind {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::Comment(s) => write!(f, "--[{s}]--"),
+      Self::Export(s) => {
+        write!(f, "export ::({})", s.iter().map(|t| &**t.0).join(", "))
+      },
+      Self::Member(member) => write!(f, "{member}"),
+      Self::Import(i) => {
+        write!(f, "import ({})", i.iter().map(|i| i.to_string()).join(", "))
+      },
+    }
+  }
+}
+
+/// Anything the parser might encounter in a file. See [SourceLineKind]
+#[derive(Debug, Clone)]
+pub struct SourceLine {
+  /// What we encountered
+  pub kind: SourceLineKind,
+  /// Where we encountered it.
+  pub range: SourceRange,
+}
+
+impl fmt::Display for SourceLine {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.kind.fmt(f) }
+}

orchidlang/src/parse/sourcefile.rs

@@ -0,0 +1,313 @@
+//! Internal states of the parser.
+
+use std::iter;
+use std::rc::Rc;
+
+use intern_all::i;
+use itertools::Itertools;
+
+use super::context::ParseCtx;
+use super::errors::{
+  expect, expect_block, expect_name, BadTokenInRegion, ExpectedSingleName, GlobExport, LeadingNS,
+  MisalignedParen, NamespacedExport, ParseErrorKind, ReservedToken, UnexpectedEOL,
+};
+use super::frag::Frag;
+use super::lexer::{Entry, Lexeme};
+use super::multiname::parse_multiname;
+use super::parse_plugin::ParsePlugReqImpl;
+use crate::error::ProjectResult;
+use crate::name::VName;
+use crate::parse::parsed::{
+  Clause, Constant, Expr, Import, Member, MemberKind, ModuleBlock, PType, Rule, SourceLine,
+  SourceLineKind,
+};
+use crate::sym;
+
+/// Split the fragment at each line break outside parentheses
+pub fn split_lines<'a>(
+  module: Frag<'a>,
+  ctx: &'a (impl ParseCtx + ?Sized),
+) -> impl Iterator<Item = Frag<'a>> {
+  let mut source = module.data.iter().enumerate();
+  let mut fallback = module.fallback;
+  let mut last_slice = 0;
+  let mut finished = false;
+  iter::from_fn(move || {
+    let mut paren_count = 0;
+    for (i, Entry { lexeme, .. }) in source.by_ref() {
+      match lexeme {
+        Lexeme::LP(_) => paren_count += 1,
+        Lexeme::RP(_) => paren_count -= 1,
+        Lexeme::BR if paren_count == 0 => {
+          let begin = last_slice;
+          last_slice = i + 1;
+          let cur_prev = fallback;
+          fallback = &module.data[i];
+          return Some(Frag::new(cur_prev, &module.data[begin..i]));
+        },
+        _ => (),
+      }
+    }
+    // Include last line even without trailing newline
+    if !finished {
+      finished = true;
+      return Some(Frag::new(fallback, &module.data[last_slice..]));
+    }
+    None
+  })
+  .map(Frag::trim)
+  .map(|s| {
+    match s.pop(ctx).and_then(|(f, i)| i.pop_back(ctx).map(|(l, i)| (&f.lexeme, i, &l.lexeme))) {
+      Ok((Lexeme::LP(PType::Par), inner, Lexeme::RP(PType::Par))) => inner.trim(),
+      _ => s,
+    }
+  })
+  .filter(|l| !l.data.is_empty())
+}
+
+/// Parse linebreak-separated entries
+pub fn parse_module_body(cursor: Frag<'_>, ctx: &(impl ParseCtx + ?Sized)) -> Vec<SourceLine> {
+  let mut lines = Vec::new();
+  for l in split_lines(cursor, ctx) {
+    let kinds = ctx.reporter().fallback(parse_line(l, ctx), |_| vec![]);
+    let r = ctx.range_loc(&l.range());
+    lines.extend(kinds.into_iter().map(|kind| SourceLine { range: r.clone(), kind }));
+  }
+  lines
+}
+
+/// Parse a single, possibly exported entry
+pub fn parse_line(
+  cursor: Frag<'_>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<Vec<SourceLineKind>> {
+  let req = ParsePlugReqImpl { ctx, frag: cursor };
+  for line_parser in ctx.line_parsers() {
+    if let Some(result) = line_parser.parse(&req) {
+      return result;
+    }
+  }
+  let head = cursor.get(0, ctx)?;
+  match &head.lexeme {
+    Lexeme::Comment(cmt) => cmt.strip_prefix('|').and_then(|c| c.strip_suffix('|')).map_or_else(
+      || parse_line(cursor.step(ctx)?, ctx),
+      |cmt| Ok(vec![SourceLineKind::Comment(cmt.to_string())]),
+    ),
+    Lexeme::BR => parse_line(cursor.step(ctx)?, ctx),
+    Lexeme::Name(n) if **n == "export" =>
+      parse_export_line(cursor.step(ctx)?, ctx).map(|k| vec![k]),
+    Lexeme::Name(n) if ["const", "macro", "module"].contains(&n.as_str()) => {
+      let member = Member { exported: false, kind: parse_member(cursor, ctx)? };
+      Ok(vec![SourceLineKind::Member(member)])
+    },
+    Lexeme::Name(n) if **n == "import" => {
+      let (imports, cont) = parse_multiname(cursor.step(ctx)?, ctx)?;
+      cont.expect_empty(ctx)?;
+      Ok(vec![SourceLineKind::Import(imports)])
+    },
+    lexeme => {
+      let lexeme = lexeme.clone();
+      Err(BadTokenInRegion { lexeme, region: "start of line" }.pack(ctx.range_loc(&head.range)))
+    },
+  }
+}
+
+fn parse_export_line(
+  cursor: Frag<'_>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<SourceLineKind> {
+  let cursor = cursor.trim();
+  let head = cursor.get(0, ctx)?;
+  match &head.lexeme {
+    Lexeme::NS => {
+      let (names, cont) = parse_multiname(cursor.step(ctx)?, ctx)?;
+      cont.expect_empty(ctx)?;
+      let names = (names.into_iter())
+        .map(|Import { name, path, range }| match name {
+          Some(n) if path.is_empty() => Ok((n, range)),
+          Some(_) => Err(NamespacedExport.pack(range)),
+          None => Err(GlobExport.pack(range)),
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+      Ok(SourceLineKind::Export(names))
+    },
+    Lexeme::Name(n) if ["const", "macro", "module"].contains(&n.as_str()) =>
+      Ok(SourceLineKind::Member(Member { kind: parse_member(cursor, ctx)?, exported: true })),
+    lexeme => {
+      let lexeme = lexeme.clone();
+      let err = BadTokenInRegion { lexeme, region: "exported line" };
+      Err(err.pack(ctx.range_loc(&head.range)))
+    },
+  }
+}
+
+fn parse_member(cursor: Frag<'_>, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<MemberKind> {
+  let (typemark, cursor) = cursor.trim().pop(ctx)?;
+  match &typemark.lexeme {
+    Lexeme::Name(n) if **n == "const" => {
+      let constant = parse_const(cursor, ctx)?;
+      Ok(MemberKind::Constant(constant))
+    },
+    Lexeme::Name(n) if **n == "macro" => {
+      let rule = parse_rule(cursor, ctx)?;
+      Ok(MemberKind::Rule(rule))
+    },
+    Lexeme::Name(n) if **n == "module" => {
+      let module = parse_module(cursor, ctx)?;
+      Ok(MemberKind::Module(module))
+    },
+    lexeme => {
+      let lexeme = lexeme.clone();
+      let err = BadTokenInRegion { lexeme, region: "member type" };
+      Err(err.pack(ctx.range_loc(&typemark.range)))
+    },
+  }
+}
+
+/// Parse a macro rule
+pub fn parse_rule(cursor: Frag<'_>, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<Rule> {
+  let (pattern, prio, template) = cursor.find_map("arrow", ctx, |a| match a {
+    Lexeme::Arrow(p) => Some(*p),
+    _ => None,
+  })?;
+  let (pattern, _) = parse_exprv(pattern, None, ctx)?;
+  let (template, _) = parse_exprv(template, None, ctx)?;
+  Ok(Rule { pattern, prio, template })
+}
+
+/// Parse a constant declaration
+pub fn parse_const(cursor: Frag<'_>, ctx: &(impl ParseCtx + ?Sized)) -> ProjectResult<Constant> {
+  let (name_ent, cursor) = cursor.trim().pop(ctx)?;
+  let name = expect_name(name_ent, ctx)?;
+  let (walrus_ent, cursor) = cursor.trim().pop(ctx)?;
+  expect(Lexeme::Walrus, walrus_ent, ctx)?;
+  let value = ctx.reporter().fallback(
+    parse_exprv(cursor, None, ctx).and_then(|(body, _)| exprv_to_single(walrus_ent, body, ctx)),
+    |_| Clause::Name(sym!(__syntax_error__)).into_expr(ctx.range_loc(&cursor.range())),
+  );
+  Ok(Constant { name, value })
+}
+
+/// Parse a namespaced name. TODO: use this for modules
+pub fn parse_nsname<'a>(
+  cursor: Frag<'a>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<(VName, Frag<'a>)> {
+  let (name, tail) = parse_multiname(cursor, ctx)?;
+  match name.into_iter().exactly_one() {
+    Ok(Import { name: Some(name), path, .. }) =>
+      Ok((VName::new([name]).unwrap().prefix(path), tail)),
+    Err(_) | Ok(Import { name: None, .. }) => {
+      let range = cursor.data[0].range.start..tail.data[0].range.end;
+      Err(ExpectedSingleName.pack(ctx.range_loc(&range)))
+    },
+  }
+}
+
+/// Parse a submodule declaration
+pub fn parse_module(
+  cursor: Frag<'_>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<ModuleBlock> {
+  let (name_ent, cursor) = cursor.trim().pop(ctx)?;
+  let name = expect_name(name_ent, ctx)?;
+  let body = expect_block(cursor, PType::Par, ctx)?;
+  Ok(ModuleBlock { name, body: parse_module_body(body, ctx) })
+}
+
+/// Parse a sequence of expressions
+pub fn parse_exprv<'a>(
+  mut cursor: Frag<'a>,
+  paren: Option<PType>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<(Vec<Expr>, Frag<'a>)> {
+  let mut output = Vec::new();
+  cursor = cursor.trim();
+  while let Ok(current) = cursor.get(0, ctx) {
+    match &current.lexeme {
+      Lexeme::BR | Lexeme::Comment(_) => unreachable!("Fillers skipped"),
+      Lexeme::At | Lexeme::Type => {
+        let err = ReservedToken(current.lexeme.clone());
+        return Err(err.pack(ctx.range_loc(&current.range)));
+      },
+      Lexeme::Atom(a) => {
+        let value = Clause::Atom(a.clone());
+        output.push(Expr { value, range: ctx.range_loc(&current.range) });
+        cursor = cursor.step(ctx)?;
+      },
+      Lexeme::Placeh(ph) => {
+        output
+          .push(Expr { value: Clause::Placeh(ph.clone()), range: ctx.range_loc(&current.range) });
+        cursor = cursor.step(ctx)?;
+      },
+      Lexeme::Name(n) => {
+        let mut range = ctx.range_loc(&current.range);
+        let mut fullname = VName::new([n.clone()]).unwrap();
+        while cursor.get(1, ctx).is_ok_and(|e| e.lexeme.strict_eq(&Lexeme::NS)) {
+          let next_seg = cursor.get(2, ctx)?;
+          range.range.end = next_seg.range.end;
+          fullname = fullname.suffix([expect_name(next_seg, ctx)?]);
+          cursor = cursor.step(ctx)?.step(ctx)?;
+        }
+        let clause = Clause::Name(fullname.to_sym());
+        output.push(Expr { value: clause, range });
+        cursor = cursor.step(ctx)?;
+      },
+      Lexeme::NS => return Err(LeadingNS.pack(ctx.range_loc(&current.range))),
+      Lexeme::RP(c) => match paren {
+        Some(exp_c) if exp_c == *c => return Ok((output, cursor.step(ctx)?)),
+        _ => {
+          let err = MisalignedParen(current.lexeme.clone());
+          return Err(err.pack(ctx.range_loc(&current.range)));
+        },
+      },
+      Lexeme::LP(c) => {
+        let (result, leftover) = parse_exprv(cursor.step(ctx)?, Some(*c), ctx)?;
+        let range = current.range.start..leftover.fallback.range.end;
+        let value = Clause::S(*c, Rc::new(result));
+        output.push(Expr { value, range: ctx.range_loc(&range) });
+        cursor = leftover;
+      },
+      Lexeme::BS => {
+        let dot = i!(str: ".");
+        let (arg, body) =
+          (cursor.step(ctx))?.find("A '.'", ctx, |l| l.strict_eq(&Lexeme::Name(dot.clone())))?;
+        let (arg, _) = parse_exprv(arg, None, ctx)?;
+        let (body, leftover) = parse_exprv(body, paren, ctx)?;
+        output.push(Expr {
+          range: ctx.range_loc(&cursor.range()),
+          value: Clause::Lambda(Rc::new(arg), Rc::new(body)),
+        });
+        return Ok((output, leftover));
+      },
+      lexeme => {
+        let lexeme = lexeme.clone();
+        let err = BadTokenInRegion { lexeme, region: "expression" };
+        return Err(err.pack(ctx.range_loc(&current.range)));
+      },
+    }
+    cursor = cursor.trim();
+  }
+  Ok((output, Frag::new(cursor.fallback, &[])))
+}
+
+/// Wrap an expression list in parentheses if necessary
+pub fn exprv_to_single(
+  fallback: &Entry,
+  v: Vec<Expr>,
+  ctx: &(impl ParseCtx + ?Sized),
+) -> ProjectResult<Expr> {
+  match v.len() {
+    0 => {
+      let err = UnexpectedEOL(fallback.lexeme.clone());
+      Err(err.pack(ctx.range_loc(&fallback.range)))
+    },
+    1 => Ok(v.into_iter().exactly_one().unwrap()),
+    _ => {
+      let f_range = &v.first().unwrap().range;
+      let l_range = &v.last().unwrap().range;
+      let range = f_range.map_range(|r| r.start..l_range.end());
+      Ok(Expr { range, value: Clause::S(PType::Par, Rc::new(v)) })
+    },
+  }
+}