Most files suffered major changes

- Less ambiguous syntax
- Better parser (Chumsky only does tokenization now)
- Tidy(|ier) error handling
- Facade for simplified embedding
- External code grouped in (fairly) self-contained Systems
- Dynamic action dispatch
- Many STL additions

Cargo.lock

@@ -33,6 +33,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
+
 [[package]]
 name = "anstream"
 version = "0.3.2"
@@ -183,6 +189,21 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
+[[package]]
+name = "concurrent-queue"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "convert_case"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.7"
@@ -192,6 +213,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -202,6 +232,19 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "derive_more"
+version = "0.99.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
+dependencies = [
+ "convert_case",
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -306,11 +349,12 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.13.2"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
 dependencies = [
  "ahash 0.8.3",
+ "allocator-api2",
 ]
 
 [[package]]
@@ -350,9 +394,9 @@ dependencies = [
 
 [[package]]
 name = "itertools"
-version = "0.10.5"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
 dependencies = [
  "either",
 ]
@@ -405,12 +449,14 @@ version = "0.2.2"
 dependencies = [
  "chumsky",
  "clap",
+ "derive_more",
  "duplicate",
  "dyn-clone",
- "hashbrown 0.13.2",
+ "hashbrown 0.14.0",
  "itertools",
  "ordered-float",
  "paste",
+ "polling",
  "rust-embed",
  "take_mut",
  "thiserror",
@@ -433,6 +479,28 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
 
+[[package]]
+name = "pin-project-lite"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57"
+
+[[package]]
+name = "polling"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce"
+dependencies = [
+ "autocfg",
+ "bitflags",
+ "cfg-if",
+ "concurrent-queue",
+ "libc",
+ "log",
+ "pin-project-lite",
+ "windows-sys",
+]
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -536,6 +604,15 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "rustc_version"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "0.37.19"
@@ -559,6 +636,12 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
+
 [[package]]
 name = "serde"
 version = "1.0.160"

Cargo.toml

@@ -16,7 +16,7 @@ path = "src/lib.rs"
 
 [[bin]]
 name = "orcx"
-path = "src/bin/main.rs"
+path = "src/bin/orcx.rs"
 doc = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -24,9 +24,9 @@ doc = false
 [dependencies]
 thiserror = "1.0"
 chumsky = "0.9"
-hashbrown = "0.13"
+hashbrown = "0.14"
 ordered-float = "3.7"
-itertools = "0.10"
+itertools = "0.11"
 dyn-clone = "1.0"
 clap = { version = "4.3", features = ["derive"] }
 trait-set = "0.3"
@@ -35,3 +35,5 @@ rust-embed = { version = "6.6", features = ["include-exclude"] }
 duplicate = "1.0.0"
 take_mut = "0.2.2"
 unicode-segmentation = "1.10.1"
+polling = "2.8.0"
+derive_more = "0.99.17"

ROADMAP.md

@@ -1,5 +1,49 @@
-# IO
+This document is a wishlist, its items aren't ordered in any way other than inline notes about dependency relations
 
-All IO is event-based via callbacks.
+# Language
-Driven streams such as stdin expose single-fire events for the results of functions such as "read until terminator" or "read N bytes".
+
-Network IO exposes repeated events such as "connect", "message", etc.
+## Operator declarations
+A dedicated (exportable) line type for declaring operators. Still just names, only you can write them next to other things without whitespace
+
+- ops may not contain c-ident-safe characters
+- clusters of operator characters are broken up with a greedy algorithm
+
+## Typeclasses
+Elixir-style protocols probably, only with n-ary dispatch which I saw in SICP-js
+
+# Rules
+
+## Placeholder constraints
+Simultaneously match a pattern to a subexpression and give it a name to copy it over
+
+- Copy unique 1->1 names over by default to preserve their location info
+
+# STL
+
+## Command short-circuiting
+Functions for each command type which destructure it and pass it to an Orchid callback
+
+## Runtime error handling
+result? multipath cps utils? Not sure yet.
+
+## Pattern matching
+This was the main trick in Orchid, still want to do it, still need to polish the language first
+
+## Macro error handling
+Error tokens with rules to lift them out. Kinda depends on preservation of location info in rules to be really useful
+
+# Systems
+
+## Async
+Join allows to run code when a tuple of pending events all resolve on the event poller
+
+## New: FS
+Exposes tree operations to Orchid
+Uses existing IO to open and read files
+Uses the event bus to read directories in batches without blocking other Orchid code
+
+## New: Network
+Event-driven I/O with single-fire events and resubscription to relay backpressure to the OS. Initially TCP
+
+## New: Marshall
+Serialization of Orchid data, including code, given customizable sets of serializable foreign items. Alternatively, code reflection so that all this can go in the STL

examples/calculator/main.orc

@@ -1,14 +1,15 @@
-import std::(proc::*, to_float, to_string, io::(readline, print))
+import std::(proc::*, to_float, to_string, panic, str::char_at)
 
-export main := do{
+export const main := do{
   cps print "left operand: ";
-  cps data = readline;
+  cps data = readln;
   let a = to_float data;
   cps print "operator: ";
-  cps op = readline;
+  cps op = readln;
-  cps print ("you selected \"" ++ op ++ "\"\n");
+  let op = char_at op 0;
+  cps println ("you selected \"" ++ op ++ "\"");
   cps print "right operand: ";
-  cps data = readline;
+  cps data = readln;
   let b = to_float data;
   let result = (
     if op == "+" then a + b
@@ -17,6 +18,6 @@ export main := do{
     else if op == "/" then a / b
     else (panic "Unsupported operation")
   );
-  cps print ("Result: " ++ to_string result ++ "\n");
+  cps println ("Result: " ++ to_string result);
   0
 }

examples/hello-world/main.orc

@@ -1,3 +1,5 @@
-import std::io::print
+const main := (
-
+  println "Hello, world!"
-main := print "Hello, world!\n" "goodbye"
+    "success"
+)
+-- main := "Hello, World!\n"

examples/list-processing/main.orc

@@ -1,6 +1,6 @@
-import std::(proc::*, io::print, to_string)
+import std::(proc::*, to_string)
 
-export main := do{
+export const main := do{
   let foo = list::new[1, 2, 3, 4, 5, 6];
   let bar = list::map foo n => n * 2;
   let sum = bar
@@ -8,6 +8,6 @@ export main := do{
     |> list::take 3
     |> list::reduce (\a.\b. a + b)
     |> option::unwrap;
-  cps print $ to_string sum ++ "\n";
+  cps println $ to_string sum;
   0
 }

examples/maps/main.orc

@@ -1,6 +1,6 @@
-import std::(proc::*, io::print, to_string)
+import std::(proc::*, to_string)
 
-export main := do{
+export const main := do{
   let foo = map::new[
     "foo" = 1,
     "bar" = 2,
@@ -9,6 +9,6 @@ export main := do{
   ];
   let num = map::get foo "bar"
     |> option::unwrap;
-  cps print (to_string num ++ "\n");
+  cps println $ to_string num;
   0
 }

orchid.code-workspace

@@ -36,7 +36,6 @@
 	},
 	"extensions": {
 		"recommendations": [
-			"bungcip.better-toml",
 			"maptz.regionfolder",
 			"serayuzgur.crates",
 			"tamasfe.even-better-toml",

src/bin/main.rs

@@ -1,183 +0,0 @@
-mod cli;
-
-use std::fs::File;
-use std::path::{Path, PathBuf};
-use std::{iter, process};
-
-use clap::Parser;
-use hashbrown::HashMap;
-use itertools::Itertools;
-use orchidlang::interner::InternedDisplay;
-use orchidlang::{
-  ast, ast_to_interpreted, collect_consts, collect_rules, interpreter,
-  pipeline, rule, stl, vname_to_sym_tree, Interner, ProjectTree, Stok, Sym,
-  VName,
-};
-
-use crate::cli::cmd_prompt;
-
-/// Orchid interpreter
-#[derive(Parser, Debug)]
-#[command(author, version, about, long_about = None)]
-struct Args {
-  /// Folder containing main.orc or the manually specified entry module
-  #[arg(short, long, default_value = ".")]
-  pub dir: String,
-  /// Entrypoint for the interpreter
-  #[arg(short, long, default_value = "main::main")]
-  pub main: String,
-  /// Maximum number of steps taken by the macro executor
-  #[arg(long, default_value_t = 10_000)]
-  pub macro_limit: usize,
-  /// Print the parsed ruleset and exit
-  #[arg(long)]
-  pub dump_repo: bool,
-  /// Step through the macro execution process in the specified symbol
-  #[arg(long, default_value = "")]
-  pub macro_debug: String,
-}
-impl Args {
-  /// Validate the project directory and the
-  pub fn chk_dir_main(&self) -> Result<(), String> {
-    let dir_path = PathBuf::from(&self.dir);
-    if !dir_path.is_dir() {
-      return Err(format!("{} is not a directory", dir_path.display()));
-    }
-    let segs = self.main.split("::").collect::<Vec<_>>();
-    if segs.len() < 2 {
-      return Err("Entry point too short".to_string());
-    }
-    let (pathsegs, _) = segs.split_at(segs.len() - 1);
-    let mut possible_files = pathsegs.iter().scan(dir_path, |path, seg| {
-      path.push(seg);
-      Some(path.with_extension("orc"))
-    });
-    if possible_files.all(|p| File::open(p).is_err()) {
-      return Err(format!(
-        "{} not found in {}",
-        pathsegs.join("::"),
-        PathBuf::from(&self.dir).display()
-      ));
-    }
-    Ok(())
-  }
-
-  pub fn chk_proj(&self) -> Result<(), String> {
-    self.chk_dir_main()
-  }
-}
-
-/// Load and parse all source related to the symbol `target` or all symbols
-/// in the namespace `target` in the context of the STL. All sourcefiles must
-/// reside within `dir`.
-fn load_dir(dir: &Path, target: &[Stok], i: &Interner) -> ProjectTree<VName> {
-  let file_cache = pipeline::file_loader::mk_dir_cache(dir.to_path_buf(), i);
-  let library = stl::mk_stl(i, stl::StlOptions::default());
-  pipeline::parse_layer(
-    iter::once(target),
-    &|path| file_cache.find(path),
-    &library,
-    &stl::mk_prelude(i),
-    i,
-  )
-  .expect("Failed to load source code")
-}
-
-pub fn to_vname(data: &str, i: &Interner) -> VName {
-  data.split("::").map(|s| i.i(s)).collect::<Vec<_>>()
-}
-
-/// A little utility to step through the resolution of a macro set
-pub fn macro_debug(repo: rule::Repo, mut code: ast::Expr<Sym>, i: &Interner) {
-  let mut idx = 0;
-  println!("Macro debugger working on {}", code.bundle(i));
-  loop {
-    let (cmd, _) = cmd_prompt("cmd> ").unwrap();
-    match cmd.trim() {
-      "" | "n" | "next" =>
-        if let Some(c) = repo.step(&code) {
-          idx += 1;
-          code = c;
-          println!("Step {idx}: {}", code.bundle(i));
-        },
-      "p" | "print" => println!("Step {idx}: {}", code.bundle(i)),
-      "d" | "dump" => println!("Rules: {}", repo.bundle(i)),
-      "q" | "quit" => return,
-      "h" | "help" => println!(
-        "Available commands:
-        \t<blank>, n, next\t\ttake a step
-        \tp, print\t\tprint the current state
-        \tq, quit\t\texit
-        \th, help\t\tprint this text"
-      ),
-      _ => {
-        println!("unrecognized command \"{}\", try \"help\"", cmd);
-        continue;
-      },
-    }
-  }
-}
-
-pub fn main() {
-  let args = Args::parse();
-  args.chk_proj().unwrap_or_else(|e| panic!("{e}"));
-  let dir = PathBuf::try_from(args.dir).unwrap();
-  let i = Interner::new();
-  let main = to_vname(&args.main, &i);
-  let project = vname_to_sym_tree(load_dir(&dir, &main, &i), &i);
-  let rules = collect_rules(&project);
-  let consts = collect_consts(&project, &i);
-  let repo = rule::Repo::new(rules, &i).unwrap_or_else(|(rule, error)| {
-    panic!(
-      "Rule error: {}
-      Offending rule: {}",
-      error.bundle(&i),
-      rule.bundle(&i)
-    )
-  });
-  if args.dump_repo {
-    println!("Parsed rules: {}", repo.bundle(&i));
-    return;
-  } else if !args.macro_debug.is_empty() {
-    let name = i.i(&to_vname(&args.macro_debug, &i));
-    let code = consts
-      .get(&name)
-      .unwrap_or_else(|| panic!("Constant {} not found", args.macro_debug));
-    return macro_debug(repo, code.clone(), &i);
-  }
-  let mut exec_table = HashMap::new();
-  for (name, source) in consts.iter() {
-    let displayname = i.extern_vec(*name).join("::");
-    let (unmatched, steps_left) = repo.long_step(source, args.macro_limit + 1);
-    assert!(steps_left > 0, "Macro execution in {displayname} did not halt");
-    let runtree = ast_to_interpreted(&unmatched).unwrap_or_else(|e| {
-      panic!("Postmacro conversion error in {displayname}: {e}")
-    });
-    exec_table.insert(*name, runtree);
-  }
-  let ctx =
-    interpreter::Context { symbols: &exec_table, interner: &i, gas: None };
-  let entrypoint = exec_table.get(&i.i(&main)).unwrap_or_else(|| {
-    let main = args.main;
-    let symbols =
-      exec_table.keys().map(|t| i.extern_vec(*t).join("::")).join(", ");
-    panic!(
-      "Entrypoint not found!
-      Entrypoint was {main}
-      known keys are {symbols}"
-    )
-  });
-  let io_handler = orchidlang::stl::handleIO;
-  let ret = interpreter::run_handler(entrypoint.clone(), io_handler, ctx);
-  let interpreter::Return { gas, state, inert } =
-    ret.unwrap_or_else(|e| panic!("Runtime error: {}", e));
-  if inert {
-    println!("Settled at {}", state.expr().clause.bundle(&i));
-    if let Some(g) = gas {
-      println!("Remaining gas: {g}")
-    }
-  } else if gas == Some(0) {
-    eprintln!("Ran out of gas!");
-    process::exit(-1);
-  }
-}

src/bin/orcx.rs

@@ -0,0 +1,172 @@
+mod cli;
+
+use std::fs::File;
+use std::io::BufReader;
+use std::path::PathBuf;
+use std::process;
+
+use clap::Parser;
+use itertools::Itertools;
+use orchidlang::facade::{Environment, PreMacro};
+use orchidlang::interner::InternedDisplay;
+use orchidlang::systems::stl::StlConfig;
+use orchidlang::systems::{io_system, AsynchConfig, IOStream};
+use orchidlang::{ast, interpreted, interpreter, Interner, Sym, VName};
+
+use crate::cli::cmd_prompt;
+
+/// Orchid interpreter
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+  /// Folder containing main.orc or the manually specified entry module
+  #[arg(short, long, default_value = ".")]
+  pub dir: String,
+  /// Entrypoint for the interpreter
+  #[arg(short, long, default_value = "main::main")]
+  pub main: String,
+  /// Maximum number of steps taken by the macro executor
+  #[arg(long, default_value_t = 10_000)]
+  pub macro_limit: usize,
+  /// Print the parsed ruleset and exit
+  #[arg(long)]
+  pub dump_repo: bool,
+  /// Step through the macro execution process in the specified symbol
+  #[arg(long, default_value = "")]
+  pub macro_debug: String,
+}
+impl Args {
+  /// Validate the project directory and the
+  pub fn chk_dir_main(&self) -> Result<(), String> {
+    let dir_path = PathBuf::from(&self.dir);
+    if !dir_path.is_dir() {
+      return Err(format!("{} is not a directory", dir_path.display()));
+    }
+    let segs = self.main.split("::").collect::<Vec<_>>();
+    if segs.len() < 2 {
+      return Err("Entry point too short".to_string());
+    }
+    let (pathsegs, _) = segs.split_at(segs.len() - 1);
+    let mut possible_files = pathsegs.iter().scan(dir_path, |path, seg| {
+      path.push(seg);
+      Some(path.with_extension("orc"))
+    });
+    if possible_files.all(|p| File::open(p).is_err()) {
+      return Err(format!(
+        "{} not found in {}",
+        pathsegs.join("::"),
+        PathBuf::from(&self.dir).display()
+      ));
+    }
+    Ok(())
+  }
+
+  pub fn chk_proj(&self) -> Result<(), String> {
+    self.chk_dir_main()
+  }
+}
+
+pub fn to_vname(data: &str, i: &Interner) -> VName {
+  data.split("::").map(|s| i.i(s)).collect::<Vec<_>>()
+}
+
+fn print_for_debug(e: &ast::Expr<Sym>, i: &Interner) {
+  print!(
+    "code: {}\nglossary: {}",
+    e.bundle(i),
+    (e.value.collect_names().into_iter())
+      .map(|t| i.extern_vec(t).join("::"))
+      .join(", ")
+  )
+}
+
+/// A little utility to step through the resolution of a macro set
+pub fn macro_debug(premacro: PreMacro, sym: Sym, i: &Interner) {
+  let (mut code, location) = (premacro.consts.get(&sym))
+    .unwrap_or_else(|| {
+      panic!(
+        "Symbol {} not found\nvalid symbols: \n\t{}\n",
+        i.extern_vec(sym).join("::"),
+        (premacro.consts.keys())
+          .map(|t| i.extern_vec(*t).join("::"))
+          .join("\n\t")
+      )
+    })
+    .clone();
+  println!(
+    "Debugging macros in {} defined at {}.
+    Initial state: ",
+    i.extern_vec(sym).join("::"),
+    location
+  );
+  print_for_debug(&code, i);
+  let mut steps = premacro.step(sym).enumerate();
+  loop {
+    let (cmd, _) = cmd_prompt("\ncmd> ").unwrap();
+    match cmd.trim() {
+      "" | "n" | "next" =>
+        if let Some((idx, c)) = steps.next() {
+          code = c;
+          print!("Step {idx}: ");
+          print_for_debug(&code, i);
+        } else {
+          print!("Halted")
+        },
+      "p" | "print" => print_for_debug(&code, i),
+      "d" | "dump" => print!("Rules: {}", premacro.repo.bundle(i)),
+      "q" | "quit" => return,
+      "h" | "help" => print!(
+        "Available commands:
+        \t<blank>, n, next\t\ttake a step
+        \tp, print\t\tprint the current state
+        \tq, quit\t\texit
+        \th, help\t\tprint this text"
+      ),
+      _ => {
+        print!("unrecognized command \"{}\", try \"help\"", cmd);
+        continue;
+      },
+    }
+  }
+}
+
+pub fn main() {
+  let args = Args::parse();
+  args.chk_proj().unwrap_or_else(|e| panic!("{e}"));
+  let dir = PathBuf::try_from(args.dir).unwrap();
+  let i = Interner::new();
+  let main = to_vname(&args.main, &i);
+  let mut asynch = AsynchConfig::new();
+  let io = io_system(&mut asynch, None, None, [
+    ("stdin", IOStream::Source(BufReader::new(Box::new(std::io::stdin())))),
+    ("stdout", IOStream::Sink(Box::new(std::io::stdout()))),
+    ("stderr", IOStream::Sink(Box::new(std::io::stderr()))),
+  ]);
+  let env = Environment::new(&i)
+    .add_system(StlConfig { impure: true })
+    .add_system(asynch)
+    .add_system(io);
+  let premacro = i.unwrap(env.load_dir(&dir, &main));
+  if args.dump_repo {
+    println!("Parsed rules: {}", premacro.repo.bundle(&i));
+    return;
+  }
+  if !args.macro_debug.is_empty() {
+    let sym = i.i(&to_vname(&args.macro_debug, &i));
+    return macro_debug(premacro, sym, &i);
+  }
+  let mut proc = i.unwrap(premacro.build_process(Some(args.macro_limit)));
+  let main = interpreted::Clause::Constant(i.i(&main)).wrap();
+  let ret = i.unwrap(proc.run(main, None));
+  let interpreter::Return { gas, state, inert } = ret;
+  drop(proc);
+  if inert {
+    println!("Settled at {}", state.expr().clause.bundle(&i));
+    if let Some(g) = gas {
+      println!("Remaining gas: {g}")
+    }
+  } else if gas == Some(0) {
+    eprintln!("Ran out of gas!");
+    process::exit(-1);
+  }
+}

src/error/import_all.rs

@@ -4,27 +4,31 @@ use super::{ErrorPosition, ProjectError};
 use crate::representations::location::Location;
 use crate::utils::iter::box_once;
 use crate::utils::BoxedIter;
+use crate::{Interner, VName};
 
 /// Error produced for the statement `import *`
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct ImportAll {
   /// The file containing the offending import
-  pub offender_file: Vec<String>,
+  pub offender_file: Rc<Vec<String>>,
   /// The module containing the offending import
-  pub offender_mod: Vec<String>,
+  pub offender_mod: Rc<VName>,
 }
 impl ProjectError for ImportAll {
   fn description(&self) -> &str {
     "a top-level glob import was used"
   }
-  fn message(&self) -> String {
+  fn message(&self, i: &Interner) -> String {
-    format!("{} imports *", self.offender_mod.join("::"))
+    format!("{} imports *", i.extern_all(&self.offender_mod).join("::"))
   }
 
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self, i: &Interner) -> BoxedIter<ErrorPosition> {
     box_once(ErrorPosition {
-      location: Location::File(Rc::new(self.offender_file.clone())),
+      location: Location::File(self.offender_file.clone()),
-      message: Some(format!("{} imports *", self.offender_mod.join("::"))),
+      message: Some(format!(
+        "{} imports *",
+        i.extern_all(&self.offender_mod).join("::")
+      )),
     })
   }
 }

src/error/mod.rs

@@ -1,18 +1,20 @@
 //! Various errors the pipeline can produce
 mod import_all;
+mod no_targets;
 mod not_exported;
 mod not_found;
-mod parse_error_with_path;
+mod parse_error_with_tokens;
 mod project_error;
 mod too_many_supers;
 mod unexpected_directory;
 mod visibility_mismatch;
 
 pub use import_all::ImportAll;
+pub use no_targets::NoTargets;
 pub use not_exported::NotExported;
 pub use not_found::NotFound;
-pub use parse_error_with_path::ParseErrorWithPath;
+pub use parse_error_with_tokens::ParseErrorWithTokens;
-pub use project_error::{ErrorPosition, ProjectError};
+pub use project_error::{ErrorPosition, ProjectError, ProjectResult};
 pub use too_many_supers::TooManySupers;
 pub use unexpected_directory::UnexpectedDirectory;
 pub use visibility_mismatch::VisibilityMismatch;

src/error/no_targets.rs

@@ -0,0 +1,23 @@
+use super::{ErrorPosition, ProjectError};
+#[allow(unused)] // for doc
+use crate::parse_layer;
+use crate::utils::iter::box_empty;
+use crate::utils::BoxedIter;
+use crate::Interner;
+
+/// Error produced when [parse_layer] is called without targets. This function
+/// produces an error instead of returning a straightforward empty tree because
+/// the edge case of no targets is often an error and should generally be
+/// handled explicitly
+#[derive(Debug)]
+pub struct NoTargets;
+
+impl ProjectError for NoTargets {
+  fn description(&self) -> &str {
+    "No targets were specified for layer parsing"
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_empty()
+  }
+}

src/error/not_exported.rs

@@ -3,35 +3,39 @@ use std::rc::Rc;
 use super::{ErrorPosition, ProjectError};
 use crate::representations::location::Location;
 use crate::utils::BoxedIter;
+use crate::{Interner, VName};
 
 /// An import refers to a symbol which exists but is not exported.
 #[derive(Debug)]
 pub struct NotExported {
   /// The containing file - files are always exported
-  pub file: Vec<String>,
+  pub file: VName,
   /// The path leading to the unexported module
-  pub subpath: Vec<String>,
+  pub subpath: VName,
   /// The offending file
-  pub referrer_file: Vec<String>,
+  pub referrer_file: VName,
   /// The module containing the offending import
-  pub referrer_subpath: Vec<String>,
+  pub referrer_subpath: VName,
 }
 impl ProjectError for NotExported {
   fn description(&self) -> &str {
     "An import refers to a symbol that exists but isn't exported"
   }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self, i: &Interner) -> BoxedIter<ErrorPosition> {
     Box::new(
       [
         ErrorPosition {
-          location: Location::File(Rc::new(self.file.clone())),
+          location: Location::File(Rc::new(i.extern_all(&self.file))),
-          message: Some(format!("{} isn't exported", self.subpath.join("::"))),
+          message: Some(format!(
+            "{} isn't exported",
+            i.extern_all(&self.subpath).join("::")
+          )),
         },
         ErrorPosition {
-          location: Location::File(Rc::new(self.referrer_file.clone())),
+          location: Location::File(Rc::new(i.extern_all(&self.referrer_file))),
           message: Some(format!(
             "{} cannot see this symbol",
-            self.referrer_subpath.join("::")
+            i.extern_all(&self.referrer_subpath).join("::")
           )),
         },
       ]

src/error/not_found.rs

@@ -5,15 +5,17 @@ use crate::tree::Module;
 use crate::tree::WalkError;
 use crate::utils::iter::box_once;
 use crate::utils::BoxedIter;
-use crate::{Interner, NameLike, Tok};
+use crate::{Interner, NameLike, Tok, VName};
 
 /// Error produced when an import refers to a nonexistent module
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct NotFound {
-  /// The file containing the invalid import
+  /// The module that imported the invalid path
-  pub file: Vec<String>,
+  pub source: Option<VName>,
+  /// The file not containing the expected path
+  pub file: VName,
   /// The invalid import path
-  pub subpath: Vec<String>,
+  pub subpath: VName,
 }
 impl NotFound {
   /// Produce this error from the parameters of [Module]`::walk_ref` and a
@@ -27,23 +29,27 @@ impl NotFound {
   /// Basically, if `e` was not produced by the `walk*` methods called on
   /// `path`.
   pub fn from_walk_error(
+    source: &[Tok<String>],
     prefix: &[Tok<String>],
     path: &[Tok<String>],
     orig: &ProjectModule<impl NameLike>,
     e: WalkError,
-    i: &Interner,
   ) -> Self {
     let last_mod =
       orig.walk_ref(&path[..e.pos], false).expect("error occured on next step");
-    let mut whole_path =
+    let mut whole_path = prefix.iter().chain(path.iter()).copied();
-      prefix.iter().chain(path.iter()).map(|t| i.r(*t)).cloned();
     if let Some(file) = &last_mod.extra.file {
       Self {
+        source: Some(source.to_vec()),
         file: whole_path.by_ref().take(file.len()).collect(),
         subpath: whole_path.collect(),
       }
     } else {
-      Self { file: whole_path.collect(), subpath: Vec::new() }
+      Self {
+        source: Some(source.to_vec()),
+        file: whole_path.collect(),
+        subpath: Vec::new(),
+      }
     }
   }
 }
@@ -51,14 +57,14 @@ impl ProjectError for NotFound {
   fn description(&self) -> &str {
     "an import refers to a nonexistent module"
   }
-  fn message(&self) -> String {
+  fn message(&self, i: &Interner) -> String {
     format!(
       "module {} in {} was not found",
-      self.subpath.join("::"),
+      i.extern_all(&self.subpath).join("::"),
-      self.file.join("/"),
+      i.extern_all(&self.file).join("/"),
     )
   }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self, i: &Interner) -> BoxedIter<ErrorPosition> {
-    box_once(ErrorPosition::just_file(self.file.clone()))
+    box_once(ErrorPosition::just_file(i.extern_all(&self.file)))
   }
 }

src/error/parse_error_with_tokens.rs

@@ -0,0 +1,34 @@
+use std::rc::Rc;
+
+use itertools::Itertools;
+
+use super::{ErrorPosition, ProjectError};
+use crate::interner::InternedDisplay;
+use crate::parse::Entry;
+use crate::utils::BoxedIter;
+use crate::Interner;
+
+/// Produced by stages that parse text when it fails.
+pub struct ParseErrorWithTokens {
+  /// The complete source of the faulty file
+  pub full_source: String,
+  /// Tokens, if the error did not occur during tokenization
+  pub tokens: Vec<Entry>,
+  /// The parse error produced by Chumsky
+  pub error: Rc<dyn ProjectError>,
+}
+impl ProjectError for ParseErrorWithTokens {
+  fn description(&self) -> &str {
+    self.error.description()
+  }
+  fn message(&self, i: &Interner) -> String {
+    format!(
+      "Failed to parse code: {}\nTokenized source for context:\n{}",
+      self.error.message(i),
+      self.tokens.iter().map(|t| t.to_string_i(i)).join(" "),
+    )
+  }
+  fn positions(&self, i: &Interner) -> BoxedIter<ErrorPosition> {
+    self.error.positions(i)
+  }
+}

src/error/project_error.rs

@@ -1,8 +1,9 @@
-use std::fmt::{Debug, Display};
 use std::rc::Rc;
 
+use crate::interner::InternedDisplay;
 use crate::representations::location::Location;
 use crate::utils::BoxedIter;
+use crate::Interner;
 
 /// A point of interest in resolving the error, such as the point where
 /// processing got stuck, a command that is likely to be incorrect
@@ -22,15 +23,15 @@ impl ErrorPosition {
 
 /// Errors addressed to the developer which are to be resolved with
 /// code changes
-pub trait ProjectError: Debug {
+pub trait ProjectError {
   /// A general description of this type of error
   fn description(&self) -> &str;
   /// A formatted message that includes specific parameters
-  fn message(&self) -> String {
+  fn message(&self, _i: &Interner) -> String {
-    String::new()
+    self.description().to_string()
   }
   /// Code positions relevant to this error
-  fn positions(&self) -> BoxedIter<ErrorPosition>;
+  fn positions(&self, i: &Interner) -> BoxedIter<ErrorPosition>;
   /// Convert the error into an `Rc<dyn ProjectError>` to be able to
   /// handle various errors together
   fn rc(self) -> Rc<dyn ProjectError>
@@ -41,14 +42,18 @@ pub trait ProjectError: Debug {
   }
 }
 
-impl Display for dyn ProjectError {
+impl InternedDisplay for dyn ProjectError {
-  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+  fn fmt_i(
+    &self,
+    f: &mut std::fmt::Formatter<'_>,
+    i: &Interner,
+  ) -> std::fmt::Result {
     let description = self.description();
-    let message = self.message();
+    let message = self.message(i);
-    let positions = self.positions();
+    let positions = self.positions(i);
-    write!(f, "Problem with the project: {description}; {message}")?;
+    writeln!(f, "Project error: {description}\n{message}")?;
     for ErrorPosition { location, message } in positions {
-      write!(
+      writeln!(
         f,
         "@{location}: {}",
         message.unwrap_or("location of interest".to_string())
@@ -57,3 +62,7 @@ impl Display for dyn ProjectError {
     Ok(())
   }
 }
+
+/// Alias for a result with an error of [Rc] of [ProjectError] trait object.
+/// This is the type of result most commonly returned by pre-run operations.
+pub type ProjectResult<T> = Result<T, Rc<dyn ProjectError>>;

src/error/too_many_supers.rs

@@ -4,38 +4,39 @@ use super::{ErrorPosition, ProjectError};
 use crate::representations::location::Location;
 use crate::utils::iter::box_once;
 use crate::utils::BoxedIter;
+use crate::{Interner, VName};
 
 /// Error produced when an import path starts with more `super` segments
 /// than the current module's absolute path
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct TooManySupers {
   /// The offending import path
-  pub path: Vec<String>,
+  pub path: VName,
   /// The file containing the offending import
-  pub offender_file: Vec<String>,
+  pub offender_file: VName,
   /// The module containing the offending import
-  pub offender_mod: Vec<String>,
+  pub offender_mod: VName,
 }
 impl ProjectError for TooManySupers {
   fn description(&self) -> &str {
     "an import path starts with more `super` segments than the current \
      module's absolute path"
   }
-  fn message(&self) -> String {
+  fn message(&self, i: &Interner) -> String {
     format!(
       "path {} in {} contains too many `super` steps.",
-      self.path.join("::"),
+      i.extern_all(&self.path).join("::"),
-      self.offender_mod.join("::")
+      i.extern_all(&self.offender_mod).join("::")
     )
   }
 
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self, i: &Interner) -> BoxedIter<ErrorPosition> {
     box_once(ErrorPosition {
-      location: Location::File(Rc::new(self.offender_file.clone())),
+      location: Location::File(Rc::new(i.extern_all(&self.offender_file))),
       message: Some(format!(
         "path {} in {} contains too many `super` steps.",
-        self.path.join("::"),
+        i.extern_all(&self.path).join("::"),
-        self.offender_mod.join("::")
+        i.extern_all(&self.offender_mod).join("::")
       )),
     })
   }

src/error/unexpected_directory.rs

@@ -1,26 +1,27 @@
 use super::{ErrorPosition, ProjectError};
 use crate::utils::iter::box_once;
 use crate::utils::BoxedIter;
+use crate::{Interner, VName};
 
 /// Produced when a stage that deals specifically with code encounters
 /// a path that refers to a directory
 #[derive(Debug)]
 pub struct UnexpectedDirectory {
   /// Path to the offending collection
-  pub path: Vec<String>,
+  pub path: VName,
 }
 impl ProjectError for UnexpectedDirectory {
   fn description(&self) -> &str {
     "A stage that deals specifically with code encountered a path that refers \
      to a directory"
   }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self, i: &Interner) -> BoxedIter<ErrorPosition> {
-    box_once(ErrorPosition::just_file(self.path.clone()))
+    box_once(ErrorPosition::just_file(i.extern_all(&self.path)))
   }
-  fn message(&self) -> String {
+  fn message(&self, i: &Interner) -> String {
     format!(
       "{} was expected to be a file but a directory was found",
-      self.path.join("/")
+      i.extern_all(&self.path).join("/")
     )
   }
 }

src/error/visibility_mismatch.rs

@@ -4,12 +4,13 @@ use super::project_error::{ErrorPosition, ProjectError};
 use crate::representations::location::Location;
 use crate::utils::iter::box_once;
 use crate::utils::BoxedIter;
+use crate::{Interner, VName};
 
 /// Multiple occurences of the same namespace with different visibility
 #[derive(Debug)]
 pub struct VisibilityMismatch {
   /// The namespace with ambiguous visibility
-  pub namespace: Vec<String>,
+  pub namespace: VName,
   /// The file containing the namespace
   pub file: Rc<Vec<String>>,
 }
@@ -17,12 +18,12 @@ impl ProjectError for VisibilityMismatch {
   fn description(&self) -> &str {
     "Some occurences of a namespace are exported but others are not"
   }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self, i: &Interner) -> BoxedIter<ErrorPosition> {
     box_once(ErrorPosition {
       location: Location::File(self.file.clone()),
       message: Some(format!(
         "{} is opened multiple times with different visibilities",
-        self.namespace.join("::")
+        i.extern_all(&self.namespace).join("::")
       )),
     })
   }

src/facade/environment.rs

@@ -0,0 +1,91 @@
+use std::iter;
+use std::path::Path;
+
+use hashbrown::HashMap;
+
+use super::system::{IntoSystem, System};
+use super::PreMacro;
+use crate::error::ProjectResult;
+use crate::pipeline::file_loader;
+use crate::sourcefile::FileEntry;
+use crate::{
+  from_const_tree, parse_layer, vname_to_sym_tree, Interner, ProjectTree, Stok,
+  VName,
+};
+
+/// A compiled environment ready to load user code. It stores the list of
+/// systems and combines with usercode to produce a [Process]
+pub struct Environment<'a> {
+  /// [Interner] pseudo-global
+  pub i: &'a Interner,
+  systems: Vec<System<'a>>,
+}
+impl<'a> Environment<'a> {
+  /// Initialize a new environment
+  pub fn new(i: &'a Interner) -> Self {
+    Self { i, systems: Vec::new() }
+  }
+
+  /// Register a new system in the environment
+  pub fn add_system<'b: 'a>(mut self, is: impl IntoSystem<'b> + 'b) -> Self {
+    self.systems.push(Box::new(is).into_system(self.i));
+    self
+  }
+
+  /// Compile the environment from the set of systems and return it directly.
+  /// See [#load_dir]
+  pub fn compile(self) -> ProjectResult<CompiledEnv<'a>> {
+    let Self { i, systems, .. } = self;
+    let mut tree = from_const_tree(HashMap::new(), &[i.i("none")]);
+    for sys in systems.iter() {
+      let system_tree = from_const_tree(sys.constants.clone(), &sys.vname(i));
+      tree = ProjectTree(tree.0.overlay(system_tree.0));
+    }
+    let mut prelude = vec![];
+    for sys in systems.iter() {
+      if !sys.code.is_empty() {
+        tree = parse_layer(
+          sys.code.keys().map(|sym| &sym[..]),
+          &|k| sys.load_file(k),
+          &tree,
+          &prelude,
+          i,
+        )?;
+      }
+      prelude.extend_from_slice(&sys.prelude);
+    }
+    Ok(CompiledEnv { prelude, tree, systems })
+  }
+
+  /// Load a directory from the local file system as an Orchid project.
+  pub fn load_dir(
+    self,
+    dir: &Path,
+    target: &[Stok],
+  ) -> ProjectResult<PreMacro<'a>> {
+    let i = self.i;
+    let CompiledEnv { prelude, systems, tree } = self.compile()?;
+    let file_cache = file_loader::mk_dir_cache(dir.to_path_buf(), i);
+    let vname_tree = parse_layer(
+      iter::once(target),
+      &|path| file_cache.find(path),
+      &tree,
+      &prelude,
+      i,
+    )?;
+    let tree = vname_to_sym_tree(vname_tree, i);
+    PreMacro::new(tree, systems, i)
+  }
+}
+
+/// Compiled environment waiting for usercode. An intermediate step between
+/// [Environment] and [Process]
+pub struct CompiledEnv<'a> {
+  /// Namespace tree for pre-defined symbols with symbols at the leaves and
+  /// rules defined on the nodes
+  pub tree: ProjectTree<VName>,
+  /// Lines prepended to each usercode file
+  pub prelude: Vec<FileEntry>,
+  /// List of systems to source handlers for the interpreter
+  pub systems: Vec<System<'a>>,
+}

src/facade/mod.rs

@@ -0,0 +1,12 @@
+//! A simplified set of commands each grouping a large subset of the operations
+//! exposed by Orchid to make writing embeddings faster in the typical case.
+
+mod environment;
+mod pre_macro;
+mod process;
+mod system;
+
+pub use environment::{CompiledEnv, Environment};
+pub use pre_macro::{MacroTimeout, PreMacro};
+pub use process::Process;
+pub use system::{IntoSystem, MissingSystemCode, System};

src/facade/pre_macro.rs

@@ -0,0 +1,134 @@
+use std::iter;
+use std::rc::Rc;
+
+use hashbrown::HashMap;
+
+use super::{Process, System};
+use crate::error::{ErrorPosition, ProjectError, ProjectResult};
+use crate::interpreter::HandlerTable;
+use crate::rule::Repo;
+use crate::utils::iter::box_once;
+use crate::utils::BoxedIter;
+use crate::{
+  ast, ast_to_interpreted, collect_consts, collect_rules, rule, Interner,
+  Location, ProjectTree, Sym,
+};
+
+/// Everything needed for macro execution, and constructing the process
+pub struct PreMacro<'a> {
+  /// Optimized catalog of substitution rules
+  pub repo: Repo,
+  /// Runtime code containing macro invocations
+  pub consts: HashMap<Sym, (ast::Expr<Sym>, Location)>,
+  /// Libraries and plug-ins
+  pub systems: Vec<System<'a>>,
+  /// [Interner] pseudo-global
+  pub i: &'a Interner,
+}
+impl<'a> PreMacro<'a> {
+  /// Build a [PreMacro] from a source tree and system list
+  pub fn new(
+    tree: ProjectTree<Sym>,
+    systems: Vec<System<'a>>,
+    i: &'a Interner,
+  ) -> ProjectResult<Self> {
+    let consts = collect_consts(&tree, i);
+    let rules = collect_rules(&tree);
+    let repo = match rule::Repo::new(rules, i) {
+      Ok(r) => r,
+      Err((rule, error)) => {
+        return Err(error.to_project_error(&rule));
+      },
+    };
+    Ok(Self {
+      repo,
+      consts: (consts.into_iter())
+        .map(|(name, expr)| {
+          let location = (i.r(name).split_last())
+            .and_then(|(_, path)| {
+              let origin = (tree.0.walk_ref(path, false))
+                .expect("path sourced from symbol names");
+              origin.extra.file.as_ref().map(|path| i.extern_all(&path[..]))
+            })
+            .map(|p| Location::File(Rc::new(p)))
+            .unwrap_or(Location::Unknown);
+          (name, (expr, location))
+        })
+        .collect(),
+      i,
+      systems,
+    })
+  }
+
+  /// Run all macros to termination or the optional timeout. If a timeout does
+  /// not occur, returns a process which can execute Orchid code
+  pub fn build_process(
+    self,
+    timeout: Option<usize>,
+  ) -> ProjectResult<Process<'a>> {
+    let Self { i, systems, repo, consts } = self;
+    let mut symbols = HashMap::new();
+    for (name, (source, source_location)) in consts.iter() {
+      let unmatched = if let Some(limit) = timeout {
+        let (unmatched, steps_left) = repo.long_step(source, limit + 1);
+        if steps_left == 0 {
+          return Err(
+            MacroTimeout {
+              location: source_location.clone(),
+              symbol: *name,
+              limit,
+            }
+            .rc(),
+          );
+        } else {
+          unmatched
+        }
+      } else {
+        repo.pass(source).unwrap_or_else(|| source.clone())
+      };
+      let runtree = ast_to_interpreted(&unmatched).map_err(|e| e.rc())?;
+      symbols.insert(*name, runtree);
+    }
+    Ok(Process {
+      symbols,
+      i,
+      handlers: (systems.into_iter())
+        .fold(HandlerTable::new(), |tbl, sys| tbl.combine(sys.handlers)),
+    })
+  }
+
+  /// Obtain an iterator that steps through the preprocessing of a constant
+  /// for debugging macros
+  pub fn step(&self, sym: Sym) -> impl Iterator<Item = ast::Expr<Sym>> + '_ {
+    let mut target = self.consts.get(&sym).expect("Target not found").0.clone();
+    iter::from_fn(move || {
+      target = self.repo.step(&target)?;
+      Some(target.clone())
+    })
+  }
+}
+
+/// Error raised when a macro runs too long
+#[derive(Debug)]
+pub struct MacroTimeout {
+  location: Location,
+  symbol: Sym,
+  limit: usize,
+}
+impl ProjectError for MacroTimeout {
+  fn description(&self) -> &str {
+    "Macro execution has not halted"
+  }
+
+  fn message(&self, i: &Interner) -> String {
+    format!(
+      "Macro execution during the processing of {} took more than {} steps",
+      i.extern_vec(self.symbol).join("::"),
+      self.limit
+    )
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.location.clone(), message: None })
+  }
+}

src/facade/process.rs

@@ -0,0 +1,90 @@
+use hashbrown::HashMap;
+
+use crate::error::{ErrorPosition, ProjectError, ProjectResult};
+use crate::interpreted::{self, ExprInst};
+#[allow(unused)] // for doc
+use crate::interpreter;
+use crate::interpreter::{
+  run_handler, Context, HandlerTable, Return, RuntimeError,
+};
+use crate::utils::iter::box_once;
+use crate::utils::BoxedIter;
+use crate::{Interner, Location, Sym};
+
+/// This struct ties the state of systems to loaded code, and allows to call
+/// Orchid-defined functions
+pub struct Process<'a> {
+  pub(crate) symbols: HashMap<Sym, ExprInst>,
+  pub(crate) handlers: HandlerTable<'a>,
+  pub(crate) i: &'a Interner,
+}
+impl<'a> Process<'a> {
+  /// Execute the given command in this process. If gas is specified, at most as
+  /// many steps will be executed and then the partial result returned.
+  ///
+  /// This is useful to catch infinite loops or ensure that a tenant program
+  /// yields
+  pub fn run(
+    &mut self,
+    prompt: ExprInst,
+    gas: Option<usize>,
+  ) -> Result<Return, RuntimeError> {
+    let ctx = Context { gas, interner: self.i, symbols: &self.symbols };
+    run_handler(prompt, &mut self.handlers, ctx)
+  }
+
+  /// Find all unbound constant names in a symbol. This is often useful to
+  /// identify dynamic loading targets.
+  pub fn unbound_refs(&self, key: Sym) -> Vec<(Sym, Location)> {
+    let mut errors = Vec::new();
+    let sym = self.symbols.get(&key).expect("symbol must exist");
+    sym.search_all(&mut |s: &ExprInst| {
+      let expr = s.expr();
+      if let interpreted::Clause::Constant(sym) = expr.clause {
+        if !self.symbols.contains_key(&sym) {
+          errors.push((sym, expr.location.clone()))
+        }
+      }
+      None::<()>
+    });
+    errors
+  }
+
+  /// Assert that, unless [interpreted::Clause::Constant]s are created
+  /// procedurally, a [interpreter::RuntimeError::MissingSymbol] cannot be
+  /// produced
+  pub fn validate_refs(&self) -> ProjectResult<()> {
+    for key in self.symbols.keys() {
+      if let Some((symbol, location)) = self.unbound_refs(*key).pop() {
+        return Err(MissingSymbol { location, referrer: *key, symbol }.rc());
+      }
+    }
+    Ok(())
+  }
+}
+
+#[derive(Debug)]
+pub struct MissingSymbol {
+  referrer: Sym,
+  location: Location,
+  symbol: Sym,
+}
+impl ProjectError for MissingSymbol {
+  fn description(&self) -> &str {
+    "A name not referring to a known symbol was found in the source after \
+     macro execution. This can either mean that a symbol name was mistyped, or \
+     that macro execution didn't correctly halt."
+  }
+
+  fn message(&self, i: &Interner) -> String {
+    format!(
+      "The symbol {} referenced in {} does not exist",
+      i.extern_vec(self.symbol).join("::"),
+      i.extern_vec(self.referrer).join("::")
+    )
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.location.clone(), message: None })
+  }
+}

src/facade/system.rs

@@ -0,0 +1,72 @@
+use hashbrown::HashMap;
+
+use crate::error::{ErrorPosition, ProjectError};
+use crate::interpreter::HandlerTable;
+use crate::pipeline::file_loader::{IOResult, Loaded};
+use crate::sourcefile::FileEntry;
+use crate::utils::iter::box_empty;
+use crate::utils::BoxedIter;
+use crate::{ConstTree, Interner, Tok, VName};
+
+/// A description of every point where an external library can hook into Orchid.
+/// Intuitively, this can be thought of as a plugin
+pub struct System<'a> {
+  /// An identifier for the system used eg. in error reporting.
+  pub name: Vec<String>,
+  /// External functions and other constant values defined in AST form
+  pub constants: HashMap<Tok<String>, ConstTree>,
+  /// Orchid libraries defined by this system
+  pub code: HashMap<Vec<Tok<String>>, Loaded>,
+  /// Prelude lines to be added to **subsequent** systems and usercode to
+  /// expose the functionality of this system. The prelude is not added during
+  /// the loading of this system
+  pub prelude: Vec<FileEntry>,
+  /// Handlers for actions defined in this system
+  pub handlers: HandlerTable<'a>,
+}
+impl<'a> System<'a> {
+  /// Intern the name of the system so that it can be used as an Orchid
+  /// namespace
+  pub fn vname(&self, i: &Interner) -> Vec<Tok<String>> {
+    self.name.iter().map(|s| i.i(s)).collect::<Vec<_>>()
+  }
+
+  /// Load a file from the system
+  pub fn load_file(&self, path: &[Tok<String>]) -> IOResult {
+    (self.code.get(path)).cloned().ok_or_else(|| {
+      let err =
+        MissingSystemCode { path: path.to_vec(), system: self.name.clone() };
+      err.rc()
+    })
+  }
+}
+
+/// An error raised when a system fails to load a path. This usually means that
+/// another system the current one depends on did not get loaded
+#[derive(Debug)]
+pub struct MissingSystemCode {
+  path: VName,
+  system: Vec<String>,
+}
+impl ProjectError for MissingSystemCode {
+  fn description(&self) -> &str {
+    "A system tried to import a path that doesn't exist"
+  }
+  fn message(&self, i: &Interner) -> String {
+    format!(
+      "Path {} is not defined by {} or any system before it",
+      i.extern_all(&self.path).join("::"),
+      self.system.join("::")
+    )
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_empty()
+  }
+}
+
+/// Trait for objects that can be converted into a [System] in the presence
+/// of an [Interner].
+pub trait IntoSystem<'a>: 'a {
+  /// Convert this object into a system using an interner
+  fn into_system(self, i: &Interner) -> System<'a>;
+}

src/foreign/atom.rs

@@ -1,19 +1,12 @@
-//! Interaction with foreign code
-//!
-//! Structures and traits used in the exposure of external functions and values
-//! to Orchid code
 use std::any::Any;
-use std::error::Error;
+use std::fmt::Debug;
-use std::fmt::{Debug, Display};
-use std::hash::Hash;
-use std::rc::Rc;
 
 use dyn_clone::DynClone;
 
+use crate::interpreted::ExprInst;
 use crate::interpreter::{Context, RuntimeError};
-pub use crate::representations::interpreted::Clause;
+use crate::representations::interpreted::Clause;
-use crate::representations::interpreted::ExprInst;
+use crate::Primitive;
-use crate::representations::Primitive;
 
 /// Information returned by [Atomic::run]. This mirrors
 /// [crate::interpreter::Return] but with a clause instead of an Expr.
@@ -29,73 +22,12 @@ pub struct AtomicReturn {
 impl AtomicReturn {
   /// Wrap an inert atomic for delivery to the supervisor
   pub fn from_data<D: Atomic>(d: D, c: Context) -> Self {
-    AtomicReturn { clause: d.to_atom_cls(), gas: c.gas, inert: false }
+    AtomicReturn { clause: d.atom_cls(), gas: c.gas, inert: false }
   }
 }
 
-/// A type-erased error in external code
-pub type RcError = Rc<dyn ExternError>;
 /// Returned by [Atomic::run]
 pub type AtomicResult = Result<AtomicReturn, RuntimeError>;
-/// Returned by [ExternFn::apply]
-pub type XfnResult = Result<Clause, RcError>;
-
-/// Errors produced by external code
-pub trait ExternError: Display {
-  /// Convert into trait object
-  fn into_extern(self) -> Rc<dyn ExternError>
-  where
-    Self: 'static + Sized,
-  {
-    Rc::new(self)
-  }
-}
-
-impl Debug for dyn ExternError {
-  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-    write!(f, "{self}")
-  }
-}
-
-impl Error for dyn ExternError {}
-
-/// Represents an externally defined function from the perspective of
-/// the executor. Since Orchid lacks basic numerical operations,
-/// these are also external functions.
-pub trait ExternFn: DynClone {
-  /// Display name of the function
-  fn name(&self) -> &str;
-  /// Combine the function with an argument to produce a new clause
-  fn apply(&self, arg: ExprInst, ctx: Context) -> XfnResult;
-  /// Hash the name to get a somewhat unique hash.
-  fn hash(&self, mut state: &mut dyn std::hash::Hasher) {
-    self.name().hash(&mut state)
-  }
-  /// Wrap this function in a clause to be placed in an [AtomicResult].
-  fn to_xfn_cls(self) -> Clause
-  where
-    Self: Sized + 'static,
-  {
-    Clause::P(Primitive::ExternFn(Box::new(self)))
-  }
-}
-
-impl Eq for dyn ExternFn {}
-impl PartialEq for dyn ExternFn {
-  fn eq(&self, other: &Self) -> bool {
-    self.name() == other.name()
-  }
-}
-impl Hash for dyn ExternFn {
-  fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-    self.name().hash(state)
-  }
-}
-impl Debug for dyn ExternFn {
-  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-    write!(f, "##EXTERN[{}]##", self.name())
-  }
-}
 
 /// Functionality the interpreter needs to handle a value
 pub trait Atomic: Any + Debug + DynClone
@@ -106,17 +38,27 @@ where
   /// during introspection by other external code. There is no other way to
   /// interact with values of unknown types at the moment.
   fn as_any(&self) -> &dyn Any;
+
   /// Attempt to normalize this value. If it wraps a value, this should report
   /// inert. If it wraps a computation, it should execute one logical step of
   /// the computation and return a structure representing the ntext.
   fn run(&self, ctx: Context) -> AtomicResult;
+
   /// Wrap the atom in a clause to be placed in an [AtomicResult].
-  fn to_atom_cls(self) -> Clause
+  fn atom_cls(self) -> Clause
   where
     Self: Sized,
   {
     Clause::P(Primitive::Atom(Atom(Box::new(self))))
   }
+
+  /// Wrap the atom in a new expression instance to be placed in a tree
+  fn atom_exi(self) -> ExprInst
+  where
+    Self: Sized,
+  {
+    self.atom_cls().wrap()
+  }
 }
 
 /// Represents a black box unit of code with its own normalization steps.

src/foreign/cps_box.rs

@@ -0,0 +1,122 @@
+//! Automated wrappers to make working with CPS commands easier.
+
+use std::fmt::Debug;
+use std::iter;
+
+use trait_set::trait_set;
+
+use super::{Atomic, AtomicResult, AtomicReturn, ExternFn, XfnResult};
+use crate::interpreted::{Clause, ExprInst};
+use crate::interpreter::{Context, HandlerRes};
+use crate::{atomic_defaults, ConstTree};
+
+trait_set! {
+  /// A "well behaved" type that can be used as payload in a CPS box
+  pub trait CPSPayload = Clone + Debug + 'static;
+  /// A function to handle a CPS box with a specific payload
+  pub trait CPSHandler<T: CPSPayload> = FnMut(&T, &ExprInst) -> HandlerRes;
+}
+
+/// The pre-argument version of CPSBox
+#[derive(Debug, Clone)]
+struct CPSFn<T: CPSPayload> {
+  pub argc: usize,
+  pub continuations: Vec<ExprInst>,
+  pub payload: T,
+}
+impl<T: CPSPayload> CPSFn<T> {
+  fn new(argc: usize, payload: T) -> Self {
+    debug_assert!(
+      argc > 0,
+      "Null-ary CPS functions are invalid, use an Atom instead"
+    );
+    Self { argc, continuations: Vec::new(), payload }
+  }
+}
+impl<T: CPSPayload> ExternFn for CPSFn<T> {
+  fn name(&self) -> &str {
+    "CPS function without argument"
+  }
+  fn apply(&self, arg: ExprInst, _ctx: Context) -> XfnResult {
+    let payload = self.payload.clone();
+    let continuations = (self.continuations.iter())
+      .cloned()
+      .chain(iter::once(arg))
+      .collect::<Vec<_>>();
+    if self.argc == 1 {
+      Ok(CPSBox { payload, continuations }.atom_cls())
+    } else {
+      Ok(CPSFn { argc: self.argc - 1, payload, continuations }.xfn_cls())
+    }
+  }
+}
+
+/// An inert Orchid Atom value encapsulating a payload and a continuation point
+#[derive(Debug, Clone)]
+pub struct CPSBox<T: CPSPayload> {
+  /// Details about the command
+  pub payload: T,
+  /// Possible continuations, in the order they were provided
+  pub continuations: Vec<ExprInst>,
+}
+impl<T: CPSPayload> CPSBox<T> {
+  /// Assert that the command was instantiated with the correct number of
+  /// possible continuations. This is decided by the native bindings, not user
+  /// code, therefore this error may be uncovered by usercode but can never be
+  /// produced at will.
+  pub fn assert_count(&self, expect: usize) {
+    let real = self.continuations.len();
+    debug_assert!(
+      real == expect,
+      "Tried to read {expect} argument(s) but {real} were provided for {:?}",
+      self.payload
+    )
+  }
+  /// Unpack the wrapped command and the continuation
+  pub fn unpack1(&self) -> (&T, &ExprInst) {
+    self.assert_count(1);
+    (&self.payload, &self.continuations[0])
+  }
+  /// Unpack the wrapped command and 2 continuations (usually an async and a
+  /// sync)
+  pub fn unpack2(&self) -> (&T, &ExprInst, &ExprInst) {
+    self.assert_count(2);
+    (&self.payload, &self.continuations[0], &self.continuations[1])
+  }
+  /// Unpack the wrapped command and 3 continuations (usually an async success,
+  /// an async fail and a sync)
+  pub fn unpack3(&self) -> (&T, &ExprInst, &ExprInst, &ExprInst) {
+    self.assert_count(3);
+    (
+      &self.payload,
+      &self.continuations[0],
+      &self.continuations[1],
+      &self.continuations[2],
+    )
+  }
+}
+
+impl<T: CPSPayload> Atomic for CPSBox<T> {
+  atomic_defaults!();
+  fn run(&self, ctx: Context) -> AtomicResult {
+    Ok(AtomicReturn {
+      clause: self.clone().atom_cls(),
+      gas: ctx.gas,
+      inert: true,
+    })
+  }
+}
+
+/// Like [init_cps] but wrapped in a [ConstTree] for init-time usage
+pub fn mk_const<T: CPSPayload>(argc: usize, payload: T) -> ConstTree {
+  ConstTree::xfn(CPSFn::new(argc, payload))
+}
+
+/// Construct a CPS function which takes an argument and then acts inert
+/// so that command executors can receive it.
+///
+/// This function is meant to be used in an external function defined with
+/// [crate::define_fn]. For usage in a [ConstTree], see [mk_const]
+pub fn init_cps<T: CPSPayload>(argc: usize, payload: T) -> Clause {
+  CPSFn::new(argc, payload).xfn_cls()
+}

src/foreign/extern_fn.rs

@@ -0,0 +1,71 @@
+use std::error::Error;
+use std::fmt::{Debug, Display};
+use std::hash::Hash;
+use std::rc::Rc;
+
+use dyn_clone::DynClone;
+
+use crate::interpreted::ExprInst;
+use crate::interpreter::Context;
+use crate::representations::interpreted::Clause;
+use crate::Primitive;
+
+/// Returned by [ExternFn::apply]
+pub type XfnResult = Result<Clause, Rc<dyn ExternError>>;
+
+/// Errors produced by external code
+pub trait ExternError: Display {
+  /// Convert into trait object
+  fn into_extern(self) -> Rc<dyn ExternError>
+  where
+    Self: 'static + Sized,
+  {
+    Rc::new(self)
+  }
+}
+
+impl Debug for dyn ExternError {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    write!(f, "{self}")
+  }
+}
+
+impl Error for dyn ExternError {}
+
+/// Represents an externally defined function from the perspective of
+/// the executor. Since Orchid lacks basic numerical operations,
+/// these are also external functions.
+pub trait ExternFn: DynClone {
+  /// Display name of the function
+  fn name(&self) -> &str;
+  /// Combine the function with an argument to produce a new clause
+  fn apply(&self, arg: ExprInst, ctx: Context) -> XfnResult;
+  /// Hash the name to get a somewhat unique hash.
+  fn hash(&self, mut state: &mut dyn std::hash::Hasher) {
+    self.name().hash(&mut state)
+  }
+  /// Wrap this function in a clause to be placed in an [AtomicResult].
+  fn xfn_cls(self) -> Clause
+  where
+    Self: Sized + 'static,
+  {
+    Clause::P(Primitive::ExternFn(Box::new(self)))
+  }
+}
+
+impl Eq for dyn ExternFn {}
+impl PartialEq for dyn ExternFn {
+  fn eq(&self, other: &Self) -> bool {
+    self.name() == other.name()
+  }
+}
+impl Hash for dyn ExternFn {
+  fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+    self.name().hash(state)
+  }
+}
+impl Debug for dyn ExternFn {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    write!(f, "##EXTERN[{}]##", self.name())
+  }
+}

src/foreign/mod.rs

@@ -0,0 +1,17 @@
+//! Interaction with foreign code
+//!
+//! Structures and traits used in the exposure of external functions and values
+//! to Orchid code
+mod atom;
+pub mod cps_box;
+mod extern_fn;
+
+use std::rc::Rc;
+
+pub use atom::{Atom, Atomic, AtomicResult, AtomicReturn};
+pub use extern_fn::{ExternError, ExternFn, XfnResult};
+
+pub use crate::representations::interpreted::Clause;
+
+/// A type-erased error in external code
+pub type RcError = Rc<dyn ExternError>;

src/foreign_macros/atomic_impl.rs

@@ -74,7 +74,7 @@ macro_rules! atomic_impl {
   ($typ:ident) => {
     $crate::atomic_impl! {$typ, |this: &Self, _: $crate::interpreter::Context| {
       use $crate::foreign::ExternFn;
-      Ok(this.clone().to_xfn_cls())
+      Ok(this.clone().xfn_cls())
     }}
   };
   ($typ:ident, $next_phase:expr) => {
@@ -108,7 +108,7 @@ macro_rules! atomic_impl {
             Err(e) => return Err($crate::interpreter::RuntimeError::Extern(e)),
           }
         } else {
-          next_self.to_atom_cls()
+          next_self.atom_cls()
         };
         // package and return
         Ok($crate::foreign::AtomicReturn { clause, gas, inert: false })

src/foreign_macros/atomic_inert.rs

@@ -14,7 +14,7 @@ use crate::foreign::Atomic;
 /// on [Any], [Debug] and [DynClone].
 #[macro_export]
 macro_rules! atomic_inert {
-  ($typ:ident) => {
+  ($typ:ident, $typename:expr) => {
     impl $crate::foreign::Atomic for $typ {
       $crate::atomic_defaults! {}
 
@@ -23,11 +23,25 @@ macro_rules! atomic_inert {
         ctx: $crate::interpreter::Context,
       ) -> $crate::foreign::AtomicResult {
         Ok($crate::foreign::AtomicReturn {
-          clause: self.clone().to_atom_cls(),
+          clause: self.clone().atom_cls(),
           gas: ctx.gas,
           inert: true,
         })
       }
     }
+
+    impl TryFrom<&ExprInst> for $typ {
+      type Error = std::rc::Rc<dyn $crate::foreign::ExternError>;
+
+      fn try_from(
+        value: &$crate::interpreted::ExprInst,
+      ) -> Result<Self, Self::Error> {
+        $crate::systems::cast_exprinst::with_atom(
+          value,
+          $typename,
+          |a: &$typ| Ok(a.clone()),
+        )
+      }
+    }
   };
 }

src/foreign_macros/define_fn.rs

@@ -74,7 +74,8 @@ use crate::write_fn_step;
 #[macro_export]
 macro_rules! define_fn {
   // Unary function entry
-  ($( #[ $attr:meta ] )* $qual:vis $name:ident = $body:expr) => {paste::paste!{
+  ($( #[ $attr:meta ] )* $qual:vis $name:ident = |$x:ident| $body:expr) => {
+    paste::paste!{
       $crate::write_fn_step!(
         $( #[ $attr ] )* $qual $name
         >
@@ -85,14 +86,15 @@ macro_rules! define_fn {
         {}
         out = expr => Ok(expr);
         {
-        let lambda = $body;
+          let lambda = |$x: &$crate::interpreted::ExprInst| $body;
           lambda(out)
         }
       );
-  }};
+    }
+  };
   // xname is optional only if every conversion is implicit
   ($( #[ $attr:meta ] )* $qual:vis $name:ident {
-    $( $arg:ident: $typ:ty ),+
+    $( $arg:ident: $typ:ty ),+  $(,)?
   } => $body:expr) => {
     $crate::define_fn!{expr=expr in
       $( #[ $attr ] )* $qual $name {
@@ -105,7 +107,7 @@ macro_rules! define_fn {
     $( #[ $attr:meta ] )*
     $qual:vis $name:ident {
       $arg0:ident: $typ0:ty $( as $parse0:expr )?
-      $(, $arg:ident: $typ:ty $( as $parse:expr )? )*
+      $(, $arg:ident: $typ:ty $( as $parse:expr )? )* $(,)?
     } => $body:expr
   ) => {paste::paste!{
     // Generate initial state

src/interner/multitype.rs

@@ -8,6 +8,7 @@ use hashbrown::HashMap;
 
 use super::monotype::TypedInterner;
 use super::token::Tok;
+use super::InternedDisplay;
 
 /// A collection of interners based on their type. Allows to intern any object
 /// that implements [ToOwned]. Objects of the same type are stored together in a
@@ -59,6 +60,29 @@ impl Interner {
   ) -> Vec<T> {
     s.iter().map(|t| self.r(*t)).cloned().collect()
   }
+
+  /// A variant of `unwrap` using [InternedDisplay] to circumvent `unwrap`'s
+  /// dependencyon [Debug]. For clarity, [expect] should be preferred.
+  pub fn unwrap<T, E: InternedDisplay>(&self, result: Result<T, E>) -> T {
+    result.unwrap_or_else(|e| {
+      println!("Unwrapped Error: {}", e.bundle(self));
+      panic!("Unwrapped an error");
+    })
+  }
+
+  /// A variant of `expect` using  [InternedDisplay] to circumvent `expect`'s
+  /// depeendency on [Debug].
+  pub fn expect<T, E: InternedDisplay>(
+    &self,
+    result: Result<T, E>,
+    msg: &str,
+  ) -> T {
+    result.unwrap_or_else(|e| {
+      println!("Expectation failed: {msg}");
+      println!("Error: {}", e.bundle(self));
+      panic!("Expected an error");
+    })
+  }
 }
 
 impl Default for Interner {

src/interner/traits.rs

@@ -1,5 +1,6 @@
-use core::fmt::Formatter;
+use core::fmt::{self, Display, Formatter};
-use std::fmt::Display;
+use core::ops::Deref;
+use std::rc::Rc;
 
 use crate::interner::Interner;
 
@@ -29,16 +30,13 @@ pub trait InternedDisplay {
   }
 }
 
-impl<T> InternedDisplay for T
+// Special loophole for Rc<dyn ProjectError>
+impl<T: ?Sized> InternedDisplay for Rc<T>
 where
-  T: Display,
+  T: InternedDisplay,
 {
-  fn fmt_i(
+  fn fmt_i(&self, f: &mut Formatter<'_>, i: &Interner) -> fmt::Result {
-    &self,
+    self.deref().fmt_i(f, i)
-    f: &mut std::fmt::Formatter<'_>,
-    _i: &Interner,
-  ) -> std::fmt::Result {
-    <Self as Display>::fmt(self, f)
   }
 }
 

src/interpreter/apply.rs

@@ -15,7 +15,7 @@ fn map_at<E>(
   mapper: &mut impl FnMut(&Clause) -> Result<Clause, E>,
 ) -> Result<ExprInst, E> {
   source
-    .try_update(|value| {
+    .try_update(|value, _loc| {
       // Pass right through lambdas
       if let Clause::Lambda { args, body } = value {
         return Ok((
@@ -87,7 +87,7 @@ pub fn apply(
   x: ExprInst,
   ctx: Context,
 ) -> Result<Return, RuntimeError> {
-  let (state, (gas, inert)) = f.try_update(|clause| match clause {
+  let (state, (gas, inert)) = f.try_update(|clause, loc| match clause {
     // apply an ExternFn or an internal function
     Clause::P(Primitive::ExternFn(f)) => {
       let clause =
@@ -104,16 +104,11 @@ pub fn apply(
     } else {
       (body.expr().clause.clone(), (ctx.gas, false))
     }),
-    Clause::Constant(name) => {
+    Clause::Constant(name) =>
-      let symval = if let Some(sym) = ctx.symbols.get(name) {
+      if let Some(sym) = ctx.symbols.get(name) {
-        sym.clone()
+        Ok((Clause::Apply { f: sym.clone(), x }, (ctx.gas, false)))
       } else {
-        panic!(
+        Err(RuntimeError::MissingSymbol(*name, loc.clone()))
-          "missing symbol for function {}",
-          ctx.interner.extern_vec(*name).join("::")
-        )
-      };
-      Ok((Clause::Apply { f: symval, x }, (ctx.gas, false)))
       },
     Clause::P(Primitive::Atom(atom)) => {
       // take a step in expanding atom

src/interpreter/error.rs

@@ -1,8 +1,9 @@
-use std::fmt::Display;
 use std::rc::Rc;
 
 use crate::foreign::ExternError;
+use crate::interner::InternedDisplay;
 use crate::representations::interpreted::ExprInst;
+use crate::{Location, Sym};
 
 /// Problems in the process of execution
 #[derive(Clone, Debug)]
@@ -11,6 +12,8 @@ pub enum RuntimeError {
   Extern(Rc<dyn ExternError>),
   /// Primitive applied as function
   NonFunctionApplication(ExprInst),
+  /// Symbol not in context
+  MissingSymbol(Sym, Location),
 }
 
 impl From<Rc<dyn ExternError>> for RuntimeError {
@@ -19,12 +22,23 @@ impl From<Rc<dyn ExternError>> for RuntimeError {
   }
 }
 
-impl Display for RuntimeError {
+impl InternedDisplay for RuntimeError {
-  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+  fn fmt_i(
+    &self,
+    f: &mut std::fmt::Formatter<'_>,
+    i: &crate::Interner,
+  ) -> std::fmt::Result {
     match self {
       Self::Extern(e) => write!(f, "Error in external function: {e}"),
-      Self::NonFunctionApplication(loc) => {
+      Self::NonFunctionApplication(expr) => {
-        write!(f, "Primitive applied as function at {loc:?}")
+        write!(f, "Primitive applied as function at {}", expr.expr().location)
+      },
+      Self::MissingSymbol(sym, loc) => {
+        write!(
+          f,
+          "{}, called at {loc} is not loaded",
+          i.extern_vec(*sym).join("::")
+        )
       },
     }
   }

src/interpreter/handler.rs

@@ -0,0 +1,84 @@
+use std::any::{Any, TypeId};
+use std::mem;
+use std::rc::Rc;
+
+use hashbrown::HashMap;
+use trait_set::trait_set;
+
+use super::{run, Context, Return, RuntimeError};
+use crate::foreign::ExternError;
+use crate::interpreted::{Clause, ExprInst};
+use crate::utils::unwrap_or;
+use crate::Primitive;
+
+trait_set! {
+  trait Handler = for<'b> FnMut(&'b dyn Any) -> HandlerRes;
+}
+
+/// A table of command handlers
+#[derive(Default)]
+pub struct HandlerTable<'a> {
+  handlers: HashMap<TypeId, Box<dyn Handler + 'a>>,
+}
+impl<'a> HandlerTable<'a> {
+  /// Create a new [HandlerTable]
+  pub fn new() -> Self {
+    Self { handlers: HashMap::new() }
+  }
+
+  /// Add a handler function to interpret a type of atom and decide what happens
+  /// next. This function can be impure.
+  pub fn register<T: 'static>(
+    &mut self,
+    mut f: impl for<'b> FnMut(&'b T) -> HandlerRes + 'a,
+  ) {
+    let cb = move |a: &dyn Any| f(a.downcast_ref().expect("found by TypeId"));
+    let prev = self.handlers.insert(TypeId::of::<T>(), Box::new(cb));
+    assert!(prev.is_none(), "A handler for this type is already registered");
+  }
+
+  /// Find and execute the corresponding handler for this type
+  pub fn dispatch(&mut self, arg: &dyn Any) -> Option<HandlerRes> {
+    self.handlers.get_mut(&arg.type_id()).map(|f| f(arg))
+  }
+
+  /// Combine two non-overlapping handler sets
+  pub fn combine(mut self, other: Self) -> Self {
+    for (key, value) in other.handlers {
+      let prev = self.handlers.insert(key, value);
+      assert!(prev.is_none(), "Duplicate handlers")
+    }
+    self
+  }
+}
+
+/// Various possible outcomes of a [Handler] execution. Ok returns control to
+/// the interpreter. The meaning of Err is decided by the value in it.
+pub type HandlerRes = Result<ExprInst, Rc<dyn ExternError>>;
+
+/// [run] orchid code, executing any commands it returns using the specified
+/// [Handler]s.
+pub fn run_handler(
+  mut expr: ExprInst,
+  handlers: &mut HandlerTable,
+  mut ctx: Context,
+) -> Result<Return, RuntimeError> {
+  loop {
+    let ret = run(expr.clone(), ctx.clone())?;
+    if ret.gas == Some(0) {
+      return Ok(ret);
+    }
+    let state_ex = ret.state.expr();
+    let a = if let Clause::P(Primitive::Atom(a)) = &state_ex.clause {
+      a
+    } else {
+      mem::drop(state_ex);
+      return Ok(ret);
+    };
+    expr = unwrap_or!(handlers.dispatch(a.0.as_any()); {
+      mem::drop(state_ex);
+      return Ok(ret)
+    })?;
+    ctx.gas = ret.gas;
+  }
+}

src/interpreter/mod.rs

@@ -2,8 +2,10 @@
 mod apply;
 mod context;
 mod error;
+mod handler;
 mod run;
 
 pub use context::{Context, Return};
 pub use error::RuntimeError;
-pub use run::{run, run_handler, Handler, HandlerErr, HandlerParm, HandlerRes};
+pub use handler::{run_handler, HandlerRes, HandlerTable};
+pub use run::run;

src/interpreter/run.rs

@@ -1,17 +1,14 @@
-use std::mem;
-use std::rc::Rc;
-
 use super::apply::apply;
 use super::context::{Context, Return};
 use super::error::RuntimeError;
-use crate::foreign::{Atom, Atomic, AtomicReturn, ExternError};
+use crate::foreign::AtomicReturn;
 use crate::representations::interpreted::{Clause, ExprInst};
 use crate::representations::Primitive;
 
 /// Normalize an expression using beta reduction with memoization
 pub fn run(expr: ExprInst, mut ctx: Context) -> Result<Return, RuntimeError> {
   let (state, (gas, inert)) =
-    expr.try_normalize(|cls| -> Result<(Clause, _), RuntimeError> {
+    expr.try_normalize(|cls, loc| -> Result<(Clause, _), RuntimeError> {
       let mut i = cls.clone();
       while ctx.gas.map(|g| g > 0).unwrap_or(true) {
         match &i {
@@ -33,7 +30,8 @@ pub fn run(expr: ExprInst, mut ctx: Context) -> Result<Return, RuntimeError> {
             i = clause.clone();
           },
           Clause::Constant(c) => {
-            let symval = ctx.symbols.get(c).expect("missing symbol for value");
+            let symval = (ctx.symbols.get(c))
+              .ok_or_else(|| RuntimeError::MissingSymbol(*c, loc.clone()))?;
             ctx.gas = ctx.gas.map(|g| g - 1); // cost of lookup
             i = symval.expr().clause.clone();
           },
@@ -46,111 +44,3 @@ pub fn run(expr: ExprInst, mut ctx: Context) -> Result<Return, RuntimeError> {
     })?;
   Ok(Return { state, gas, inert })
 }
-
-/// Opaque inert data that may encode a command to a [Handler]
-pub type HandlerParm = Box<dyn Atomic>;
-
-/// Reasons why a [Handler] could not interpret a command. Convertible from
-/// either variant
-pub enum HandlerErr {
-  /// The command was addressed to us but its execution resulted in an error
-  Extern(Rc<dyn ExternError>),
-  /// This handler is not applicable, either because the [HandlerParm] is not a
-  /// command or because it's meant for some other handler
-  NA(HandlerParm),
-}
-impl From<Rc<dyn ExternError>> for HandlerErr {
-  fn from(value: Rc<dyn ExternError>) -> Self {
-    Self::Extern(value)
-  }
-}
-impl<T> From<T> for HandlerErr
-where
-  T: ExternError + 'static,
-{
-  fn from(value: T) -> Self {
-    Self::Extern(value.into_extern())
-  }
-}
-impl From<HandlerParm> for HandlerErr {
-  fn from(value: HandlerParm) -> Self {
-    Self::NA(value)
-  }
-}
-
-/// Various possible outcomes of a [Handler] execution. Ok returns control to
-/// the interpreter. The meaning of Err is decided by the value in it.
-pub type HandlerRes = Result<ExprInst, HandlerErr>;
-
-/// A trait for things that may be able to handle commands returned by Orchid
-/// code. This trait is implemented for `FnMut(HandlerParm) -> HandlerRes` and
-/// `(Handler, Handler)`, users are not supposed to implement it themselves.
-///
-/// A handler receives an arbitrary inert [Atomic] and uses [Atomic::as_any]
-/// then downcast_ref of [std::any::Any] to obtain a known type. If this fails,
-/// it returns the box in [HandlerErr::NA] which will be passed to the next
-/// handler.
-pub trait Handler {
-  /// Attempt to resolve a command with this handler.
-  fn resolve(&mut self, data: HandlerParm) -> HandlerRes;
-
-  /// If this handler isn't applicable, try the other one.
-  fn or<T: Handler>(self, t: T) -> (Self, T)
-  where
-    Self: Sized,
-  {
-    (self, t)
-  }
-}
-
-impl<F> Handler for F
-where
-  F: FnMut(HandlerParm) -> HandlerRes,
-{
-  fn resolve(&mut self, data: HandlerParm) -> HandlerRes {
-    self(data)
-  }
-}
-
-impl<T: Handler, U: Handler> Handler for (T, U) {
-  fn resolve(&mut self, data: HandlerParm) -> HandlerRes {
-    match self.0.resolve(data) {
-      Err(HandlerErr::NA(data)) => self.1.resolve(data),
-      x => x,
-    }
-  }
-}
-
-/// [run] orchid code, executing any commands it returns using the specified
-/// [Handler]s.
-pub fn run_handler(
-  mut expr: ExprInst,
-  mut handler: impl Handler,
-  mut ctx: Context,
-) -> Result<Return, RuntimeError> {
-  loop {
-    let ret = run(expr.clone(), ctx.clone())?;
-    if ret.gas == Some(0) {
-      return Ok(ret);
-    }
-    let state_ex = ret.state.expr();
-    let a = if let Clause::P(Primitive::Atom(a)) = &state_ex.clause {
-      a
-    } else {
-      mem::drop(state_ex);
-      return Ok(ret);
-    };
-    let boxed = a.clone().0;
-    expr = match handler.resolve(boxed) {
-      Ok(expr) => expr,
-      Err(HandlerErr::Extern(ext)) => Err(ext)?,
-      Err(HandlerErr::NA(atomic)) =>
-        return Ok(Return {
-          gas: ret.gas,
-          inert: ret.inert,
-          state: Clause::P(Primitive::Atom(Atom(atomic))).wrap(),
-        }),
-    };
-    ctx.gas = ret.gas;
-  }
-}

src/lib.rs

@@ -1,4 +1,4 @@
-#![deny(missing_docs)]
+#![warn(missing_docs)]
 #![doc(
   html_logo_url = "https://raw.githubusercontent.com/lbfalvy/orchid/master/icon.svg"
 )]
@@ -7,6 +7,8 @@
 )]
 //! Orchid is a lazy, pure scripting language to be embedded in Rust
 //! applications. Check out the repo for examples and other links.
+pub mod error;
+pub mod facade;
 pub mod foreign;
 mod foreign_macros;
 pub mod interner;
@@ -15,7 +17,7 @@ mod parse;
 pub mod pipeline;
 mod representations;
 pub mod rule;
-pub mod stl;
+pub mod systems;
 mod utils;
 
 pub use interner::{Interner, Tok};
@@ -32,4 +34,4 @@ pub use representations::{
   ast, from_const_tree, interpreted, sourcefile, tree, ConstTree, Literal,
   Location, PathSet, Primitive,
 };
-pub use utils::{Side, Substack};
+pub use utils::{Side, Substack, ThreadPool};

src/parse/decls.rs

@@ -1,7 +1,6 @@
 use std::hash::Hash;
 
 use chumsky::prelude::Simple;
-use chumsky::recursive::Recursive;
 use chumsky::{BoxedParser, Parser};
 use trait_set::trait_set;
 
@@ -12,6 +11,3 @@ trait_set! {
 }
 /// Boxed version of [SimpleParser]
 pub type BoxedSimpleParser<'a, I, O> = BoxedParser<'a, I, O, Simple<I>>;
-/// [Recursive] specialization of [SimpleParser] to parameterize calls to
-/// [chumsky::recursive::recursive]
-pub type SimpleRecursive<'a, I, O> = Recursive<'a, I, O, Simple<I>>;

src/parse/enum_filter.rs

@@ -1,51 +0,0 @@
-/// Produces filter_mapping functions for enum types:
-/// ```rs
-/// enum_parser!(Foo::Bar | "Some error!")
-/// // Foo::Bar(T) into T
-/// enum_parser!(Foo::Bar)
-/// // same as above but with the default error "Expected Foo::Bar"
-/// enum_parser!(Foo >> Quz; Bar, Baz)
-/// // Foo::Bar(T) into Quz::Bar(T)
-/// // Foo::Baz(U) into Quz::Baz(U)
-/// ```
-macro_rules! enum_filter {
-  ($p:path | $m:tt) => {
-    {
-      |l| {
-        if let $p(x) = l { Ok(x) }
-        else { Err($m) }
-      }
-    }
-  };
-  ($p:path >> $q:path; $i:ident | $m:tt) => {
-    {
-      use $p as srcpath;
-      use $q as tgtpath;
-      let base = enum_filter!(srcpath::$i | $m);
-      move |l| base(l).map(tgtpath::$i)
-    }
-  };
-  ($p:path >> $q:path; $i:ident) => {
-    enum_filter!($p >> $q; $i | {concat!("Expected ", stringify!($i))})
-  };
-  ($p:path >> $q:path; $($i:ident),+ | $m:tt) => {
-    {
-      use $p as srcpath;
-      use $q as tgtpath;
-      |l| match l {
-        $( srcpath::$i(x) => Ok(tgtpath::$i(x)), )+
-        _ => Err($m)
-      }
-    }
-  };
-  ($p:path >> $q:path; $($i:ident),+) => {
-    enum_filter!($p >> $q; $($i),+ | {
-      concat!("Expected one of ", $(stringify!($i), " "),+)
-    })
-  };
-  ($p:path) => {
-    enum_filter!($p | {concat!("Expected ", stringify!($p))})
-  };
-}
-
-pub(crate) use enum_filter;

src/parse/errors.rs

@@ -0,0 +1,241 @@
+use std::rc::Rc;
+
+use chumsky::prelude::Simple;
+use itertools::Itertools;
+
+use super::{Entry, Lexeme};
+use crate::error::{ErrorPosition, ProjectError};
+use crate::interner::InternedDisplay;
+use crate::utils::iter::box_once;
+use crate::utils::BoxedIter;
+use crate::{Interner, Location, Tok};
+
+#[derive(Debug)]
+pub struct LineNeedsPrefix {
+  pub entry: Entry,
+}
+impl ProjectError for LineNeedsPrefix {
+  fn description(&self) -> &str {
+    "This linetype requires a prefix"
+  }
+  fn message(&self, i: &Interner) -> String {
+    format!("{} cannot appear at the beginning of a line", self.entry.bundle(i))
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { message: None, location: self.entry.location() })
+  }
+}
+
+#[derive(Debug)]
+pub struct UnexpectedEOL {
+  /// Last entry before EOL
+  pub entry: Entry,
+}
+impl ProjectError for UnexpectedEOL {
+  fn description(&self) -> &str {
+    "The line ended abruptly"
+  }
+
+  fn message(&self, _i: &Interner) -> String {
+    "The line ends unexpectedly here. In Orchid, all line breaks outside \
+     parentheses start a new declaration"
+      .to_string()
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { message: None, location: self.entry.location() })
+  }
+}
+
+pub struct ExpectedEOL {
+  pub location: Location,
+}
+impl ProjectError for ExpectedEOL {
+  fn description(&self) -> &str {
+    "Expected the end of the line"
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.location.clone(), message: None })
+  }
+}
+
+#[derive(Debug)]
+pub struct ExpectedName {
+  pub entry: Entry,
+}
+impl ExpectedName {
+  pub fn expect(entry: &Entry) -> Result<Tok<String>, Rc<dyn ProjectError>> {
+    match entry.lexeme {
+      Lexeme::Name(n) => Ok(n),
+      _ => Err(Self { entry: entry.clone() }.rc()),
+    }
+  }
+}
+impl ProjectError for ExpectedName {
+  fn description(&self) -> &str {
+    "A name was expected here, but something else was found"
+  }
+
+  fn message(&self, i: &Interner) -> String {
+    if self.entry.is_keyword() {
+      format!(
+        "{} is a restricted keyword and cannot be used as a name",
+        self.entry.bundle(i)
+      )
+    } else {
+      format!("Expected a name, found {}", self.entry.bundle(i))
+    }
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.entry.location(), message: None })
+  }
+}
+
+#[derive()]
+pub struct Expected {
+  pub expected: Vec<Lexeme>,
+  pub or_name: bool,
+  pub found: Entry,
+}
+impl Expected {
+  pub fn expect(l: Lexeme, e: &Entry) -> Result<(), Rc<dyn ProjectError>> {
+    if e.lexeme != l {
+      return Err(
+        Self { expected: vec![l], or_name: false, found: e.clone() }.rc(),
+      );
+    }
+    Ok(())
+  }
+}
+impl ProjectError for Expected {
+  fn description(&self) -> &str {
+    "A concrete token was expected but something else was found"
+  }
+  fn message(&self, i: &Interner) -> String {
+    let list = match &self.expected[..] {
+      &[] => return "Unsatisfiable expectation".to_string(),
+      [only] => only.to_string_i(i),
+      [a, b] => format!("either {} or {}", a.bundle(i), b.bundle(i)),
+      [variants @ .., last] => format!(
+        "any of {} or {}",
+        variants.iter().map(|l| l.to_string_i(i)).join(", "),
+        last.bundle(i)
+      ),
+    };
+    let or_name = if self.or_name { " or a name" } else { "" };
+    format!("Expected {}{} but found {}", list, or_name, self.found.bundle(i))
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.found.location(), message: None })
+  }
+}
+
+pub struct ReservedToken {
+  pub entry: Entry,
+}
+impl ProjectError for ReservedToken {
+  fn description(&self) -> &str {
+    "A token reserved for future use was found in the code"
+  }
+
+  fn message(&self, i: &Interner) -> String {
+    format!("{} is a reserved token", self.entry.bundle(i))
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.entry.location(), message: None })
+  }
+}
+
+pub struct BadTokenInRegion {
+  pub entry: Entry,
+  pub region: &'static str,
+}
+impl ProjectError for BadTokenInRegion {
+  fn description(&self) -> &str {
+    "A token was found in a region where it should not appear"
+  }
+
+  fn message(&self, i: &Interner) -> String {
+    format!("{} cannot appear in {}", self.entry.bundle(i), self.region)
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.entry.location(), message: None })
+  }
+}
+
+pub struct NotFound {
+  pub expected: &'static str,
+  pub location: Location,
+}
+impl ProjectError for NotFound {
+  fn description(&self) -> &str {
+    "A specific lexeme was expected but not found in the given range"
+  }
+
+  fn message(&self, _i: &Interner) -> String {
+    format!("{} was expected", self.expected)
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.location.clone(), message: None })
+  }
+}
+
+pub struct LeadingNS {
+  pub location: Location,
+}
+impl ProjectError for LeadingNS {
+  fn description(&self) -> &str {
+    ":: can only follow a name token"
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.location.clone(), message: None })
+  }
+}
+
+pub struct MisalignedParen {
+  pub entry: Entry,
+}
+impl ProjectError for MisalignedParen {
+  fn description(&self) -> &str {
+    "Parentheses (), [] and {} must always pair up"
+  }
+  fn message(&self, i: &Interner) -> String {
+    format!("This {} has no pair", self.entry.bundle(i))
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.entry.location(), message: None })
+  }
+}
+
+pub struct NamespacedExport {
+  pub location: Location,
+}
+impl ProjectError for NamespacedExport {
+  fn description(&self) -> &str {
+    "Exports can only refer to unnamespaced names in the local namespace"
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.location.clone(), message: None })
+  }
+}
+
+pub struct LexError {
+  pub errors: Vec<Simple<char>>,
+  pub file: Rc<Vec<String>>,
+}
+impl ProjectError for LexError {
+  fn description(&self) -> &str {
+    "An error occured during tokenization"
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    let file = self.file.clone();
+    Box::new(self.errors.iter().map(move |s| ErrorPosition {
+      location: Location::Range { file: file.clone(), range: s.span() },
+      message: Some(format!("{}", s)),
+    }))
+  }
+}

src/parse/expression.rs

@@ -1,107 +0,0 @@
-use std::ops::Range;
-use std::rc::Rc;
-
-use chumsky::prelude::*;
-use chumsky::{self, Parser};
-
-use super::context::Context;
-use super::decls::SimpleParser;
-use super::enum_filter::enum_filter;
-use super::lexer::{filter_map_lex, Entry, Lexeme};
-use crate::representations::ast::{Clause, Expr};
-use crate::representations::location::Location;
-use crate::representations::{Primitive, VName};
-
-/// Parses any number of expr wrapped in (), [] or {}
-fn sexpr_parser(
-  expr: impl SimpleParser<Entry, Expr<VName>> + Clone,
-) -> impl SimpleParser<Entry, (Clause<VName>, Range<usize>)> + Clone {
-  let body = expr.repeated();
-  choice((
-    Lexeme::LP('(').parser().then(body.clone()).then(Lexeme::RP('(').parser()),
-    Lexeme::LP('[').parser().then(body.clone()).then(Lexeme::RP('[').parser()),
-    Lexeme::LP('{').parser().then(body).then(Lexeme::RP('{').parser()),
-  ))
-  .map(|((lp, body), rp)| {
-    let Entry { lexeme, range: Range { start, .. } } = lp;
-    let end = rp.range.end;
-    let char = if let Lexeme::LP(c) = lexeme {
-      c
-    } else {
-      unreachable!("The parser only matches Lexeme::LP")
-    };
-    (Clause::S(char, Rc::new(body)), start..end)
-  })
-  .labelled("S-expression")
-}
-
-/// Parses `\name.body` or `\name:type.body` where name is any valid name
-/// and type and body are both expressions. Comments are allowed
-/// and ignored everywhere in between the tokens
-fn lambda_parser<'a>(
-  expr: impl SimpleParser<Entry, Expr<VName>> + Clone + 'a,
-  ctx: impl Context + 'a,
-) -> impl SimpleParser<Entry, (Clause<VName>, Range<usize>)> + Clone + 'a {
-  Lexeme::BS
-    .parser()
-    .ignore_then(expr.clone())
-    .then_ignore(Lexeme::Name(ctx.interner().i(".")).parser())
-    .then(expr.repeated().at_least(1))
-    .map_with_span(move |(arg, body), span| {
-      (Clause::Lambda(Rc::new(arg), Rc::new(body)), span)
-    })
-    .labelled("Lambda")
-}
-
-/// Parses a sequence of names separated by :: <br/>
-/// Comments and line breaks are allowed and ignored in between
-pub fn ns_name_parser<'a>()
--> impl SimpleParser<Entry, (VName, Range<usize>)> + Clone + 'a {
-  filter_map_lex(enum_filter!(Lexeme::Name))
-    .separated_by(Lexeme::NS.parser())
-    .at_least(1)
-    .map(move |elements| {
-      let start = elements.first().expect("can never be empty").1.start;
-      let end = elements.last().expect("can never be empty").1.end;
-      let tokens = (elements.iter().map(|(t, _)| *t)).collect::<Vec<_>>();
-      (tokens, start..end)
-    })
-    .labelled("Namespaced name")
-}
-
-pub fn namelike_parser<'a>()
--> impl SimpleParser<Entry, (Clause<VName>, Range<usize>)> + Clone + 'a {
-  choice((
-    filter_map_lex(enum_filter!(Lexeme::PH))
-      .map(|(ph, range)| (Clause::Placeh(ph), range)),
-    ns_name_parser().map(|(token, range)| (Clause::Name(token), range)),
-  ))
-}
-
-pub fn clause_parser<'a>(
-  expr: impl SimpleParser<Entry, Expr<VName>> + Clone + 'a,
-  ctx: impl Context + 'a,
-) -> impl SimpleParser<Entry, (Clause<VName>, Range<usize>)> + Clone + 'a {
-  choice((
-    filter_map_lex(enum_filter!(Lexeme >> Primitive; Literal))
-      .map(|(p, s)| (Clause::P(p), s))
-      .labelled("Literal"),
-    sexpr_parser(expr.clone()),
-    lambda_parser(expr, ctx),
-    namelike_parser(),
-  ))
-  .labelled("Clause")
-}
-
-/// Parse an expression
-pub fn xpr_parser<'a>(
-  ctx: impl Context + 'a,
-) -> impl SimpleParser<Entry, Expr<VName>> + 'a {
-  recursive(move |expr| {
-    clause_parser(expr, ctx.clone()).map(move |(value, range)| Expr {
-      value,
-      location: Location::Range { file: ctx.file(), range },
-    })
-  })
-  .labelled("Expression")
-}

src/parse/facade.rs

@@ -1,59 +1,22 @@
-use std::fmt::Debug;
-
-use chumsky::prelude::*;
 use chumsky::Parser;
-use thiserror::Error;
 
 use super::context::Context;
-use super::{lexer, line_parser, Entry};
+use super::errors::LexError;
-use crate::parse::sourcefile::split_lines;
+use super::lexer;
+use super::sourcefile::parse_module_body;
+use super::stream::Stream;
+use crate::error::{ParseErrorWithTokens, ProjectError, ProjectResult};
 use crate::representations::sourcefile::FileEntry;
 
-#[derive(Error, Debug, Clone)]
+pub fn parse2(data: &str, ctx: impl Context) -> ProjectResult<Vec<FileEntry>> {
-pub enum ParseError {
-  #[error("Could not tokenize {0:?}")]
-  Lex(Vec<Simple<char>>),
-  #[error(
-    "Could not parse {:?} on line {}",
-    .0.first().unwrap().1.span(),
-    .0.first().unwrap().0
-  )]
-  Ast(Vec<(usize, Simple<Entry>)>),
-}
-
-/// Parse a string of code into a collection of module elements;
-/// imports, exports, comments, declarations, etc.
-///
-/// Notice that because the lexer splits operators based on the provided
-/// list, the output will only be correct if operator list already
-/// contains all operators defined or imported by this module.
-pub fn parse(
-  data: &str,
-  ctx: impl Context,
-) -> Result<Vec<FileEntry>, ParseError> {
-  // TODO: wrap `i`, `ops` and `prefix` in a parsing context
   let lexie = lexer(ctx.clone());
-  let token_batchv = lexie.parse(data).map_err(ParseError::Lex)?;
+  let tokens = (lexie.parse(data))
-  let parsr = line_parser(ctx).then_ignore(end());
+    .map_err(|errors| LexError { errors, file: ctx.file() }.rc())?;
-  let (parsed_lines, errors_per_line) = split_lines(&token_batchv)
+  if tokens.is_empty() {
-    .enumerate()
+    Ok(Vec::new())
-    .map(|(i, entv)| {
-      (i, entv.iter().filter(|e| !e.is_filler()).cloned().collect::<Vec<_>>())
-    })
-    .filter(|(_, l)| !l.is_empty())
-    .map(|(i, l)| (i, parsr.parse(l)))
-    .map(|(i, res)| match res {
-      Ok(r) => (Some(r), (i, vec![])),
-      Err(e) => (None, (i, e)),
-    })
-    .unzip::<_, _, Vec<_>, Vec<_>>();
-  let total_err = errors_per_line
-    .into_iter()
-    .flat_map(|(i, v)| v.into_iter().map(move |e| (i, e)))
-    .collect::<Vec<_>>();
-  if !total_err.is_empty() {
-    Err(ParseError::Ast(total_err))
   } else {
-    Ok(parsed_lines.into_iter().map(Option::unwrap).collect())
+    parse_module_body(Stream::from_slice(&tokens), ctx).map_err(|error| {
+      ParseErrorWithTokens { error, full_source: data.to_string(), tokens }.rc()
+    })
   }
 }

src/parse/import.rs

@@ -1,93 +0,0 @@
-use chumsky::prelude::*;
-use chumsky::Parser;
-use itertools::Itertools;
-
-use super::context::Context;
-use super::decls::{SimpleParser, SimpleRecursive};
-use super::enum_filter::enum_filter;
-use super::lexer::{filter_map_lex, Lexeme};
-use super::Entry;
-use crate::interner::Tok;
-use crate::representations::sourcefile::Import;
-use crate::utils::iter::{
-  box_chain, box_flatten, box_once, into_boxed_iter, BoxedIterIter,
-};
-
-/// initialize an iterator of iterator with a single element.
-fn init_table(name: Tok<String>) -> BoxedIterIter<'static, Tok<String>> {
-  box_once(box_once(name))
-}
-
-/// Parse an import command
-/// Syntax is same as Rust's `use` except the verb is import, no trailing
-/// semi and the delimiters are plain parentheses. Namespaces should
-/// preferably contain crossplatform filename-legal characters but the
-/// symbols are explicitly allowed to go wild.
-/// There's a blacklist in [crate::parse::name::NOT_NAME_CHAR]
-pub fn import_parser<'a>(
-  ctx: impl Context + 'a,
-) -> impl SimpleParser<Entry, Vec<Import>> + 'a {
-  // TODO: this algorithm isn't cache friendly and copies a lot
-  recursive({
-    let ctx = ctx.clone();
-    move |expr: SimpleRecursive<Entry, BoxedIterIter<Tok<String>>>| {
-      filter_map_lex(enum_filter!(Lexeme::Name))
-        .map(|(t, _)| t)
-        .separated_by(Lexeme::NS.parser())
-        .then(
-          Lexeme::NS
-            .parser()
-            .ignore_then(choice((
-              expr
-                .clone()
-                .separated_by(Lexeme::Name(ctx.interner().i(",")).parser())
-                .delimited_by(
-                  Lexeme::LP('(').parser(),
-                  Lexeme::RP('(').parser(),
-                )
-                .map(|v| box_flatten(v.into_iter()))
-                .labelled("import group"),
-              // Each expr returns a list of imports, flatten into common list
-              Lexeme::Name(ctx.interner().i("*"))
-                .parser()
-                .map(move |_| init_table(ctx.interner().i("*")))
-                .labelled("wildcard import"), // Just a *, wrapped
-              filter_map_lex(enum_filter!(Lexeme::Name))
-                .map(|(t, _)| init_table(t))
-                .labelled("import terminal"), // Just a name, wrapped
-            )))
-            .or_not(),
-        )
-        .map(
-          |(name, opt_post): (
-            Vec<Tok<String>>,
-            Option<BoxedIterIter<Tok<String>>>,
-          )|
-           -> BoxedIterIter<Tok<String>> {
-            if let Some(post) = opt_post {
-              Box::new(
-                post.map(move |el| box_chain!(name.clone().into_iter(), el)),
-              )
-            } else {
-              box_once(into_boxed_iter(name))
-            }
-          },
-        )
-    }
-  })
-  .map(move |paths| {
-    paths
-      .filter_map(|namespaces| {
-        let mut path = namespaces.collect_vec();
-        let name = path.pop()?;
-        Some(Import {
-          path: ctx.interner().i(&path),
-          name: {
-            if name == ctx.interner().i("*") { None } else { Some(name) }
-          },
-        })
-      })
-      .collect()
-  })
-  .labelled("import")
-}

src/parse/lexer.rs

@@ -1,27 +1,53 @@
 use std::fmt;
 use std::ops::Range;
+use std::rc::Rc;
 
 use chumsky::prelude::*;
 use chumsky::text::keyword;
-use chumsky::{Parser, Span};
+use chumsky::Parser;
 use ordered_float::NotNan;
 
 use super::context::Context;
 use super::decls::SimpleParser;
+use super::number::print_nat16;
 use super::{comment, name, number, placeholder, string};
 use crate::ast::{PHClass, Placeholder};
 use crate::interner::{InternedDisplay, Interner, Tok};
 use crate::representations::Literal;
+use crate::Location;
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct Entry {
   pub lexeme: Lexeme,
-  pub range: Range<usize>,
+  pub location: Location,
 }
 impl Entry {
+  /// Checks if the lexeme is a comment or line break
   pub fn is_filler(&self) -> bool {
-    matches!(self.lexeme, Lexeme::Comment(_))
+    matches!(self.lexeme, Lexeme::Comment(_) | Lexeme::BR)
-      || matches!(self.lexeme, Lexeme::BR)
+  }
+
+  pub fn is_keyword(&self) -> bool {
+    matches!(
+      self.lexeme,
+      Lexeme::Const
+        | Lexeme::Export
+        | Lexeme::Import
+        | Lexeme::Macro
+        | Lexeme::Module
+    )
+  }
+
+  pub fn location(&self) -> Location {
+    self.location.clone()
+  }
+
+  pub fn range(&self) -> Range<usize> {
+    self.location.range().expect("An Entry can only have a known location")
+  }
+
+  pub fn file(&self) -> Rc<Vec<String>> {
+    self.location.file().expect("An Entry can only have a range location")
   }
 }
 
@@ -35,27 +61,33 @@ impl InternedDisplay for Entry {
   }
 }
 
-impl From<Entry> for (Lexeme, Range<usize>) {
+// impl From<Entry> for (Lexeme, Range<usize>) {
-  fn from(ent: Entry) -> Self {
+//   fn from(ent: Entry) -> Self {
-    (ent.lexeme, ent.range)
+//     (ent.lexeme.clone(), ent.range())
-  }
+//   }
-}
+// }
 
-impl Span for Entry {
+// impl Span for Entry {
-  type Context = Lexeme;
+//   type Context = (Lexeme, Rc<Vec<String>>);
-  type Offset = usize;
+//   type Offset = usize;
 
-  fn context(&self) -> Self::Context {
+//   fn context(&self) -> Self::Context {
-    self.lexeme.clone()
+//     (self.lexeme.clone(), self.file())
-  }
+//   }
-  fn start(&self) -> Self::Offset {
+//   fn start(&self) -> Self::Offset {
-    self.range.start()
+//     self.range().start()
-  }
+//   }
-  fn end(&self) -> Self::Offset {
+//   fn end(&self) -> Self::Offset {
-    self.range.end()
+//     self.range().end()
-  }
+//   }
-  fn new(context: Self::Context, range: Range<Self::Offset>) -> Self {
+//   fn new((lexeme, file): Self::Context, range: Range<Self::Offset>) -> Self {
-    Self { lexeme: context, range }
+//     Self { lexeme, location: Location::Range { file, range } }
+//   }
+// }
+
+impl AsRef<Location> for Entry {
+  fn as_ref(&self) -> &Location {
+    &self.location
   }
 }
 
@@ -63,9 +95,9 @@ impl Span for Entry {
 pub enum Lexeme {
   Literal(Literal),
   Name(Tok<String>),
-  Rule(NotNan<f64>),
+  Arrow(NotNan<f64>),
   /// Walrus operator (formerly shorthand macro)
-  Const,
+  Walrus,
   /// Line break
   BR,
   /// Namespace separator
@@ -77,12 +109,15 @@ pub enum Lexeme {
   /// Backslash
   BS,
   At,
+  Dot,
   Type, // type operator
   Comment(String),
   Export,
   Import,
-  Namespace,
+  Module,
-  PH(Placeholder),
+  Macro,
+  Const,
+  Placeh(Placeholder),
 }
 
 impl InternedDisplay for Lexeme {
@@ -94,8 +129,8 @@ impl InternedDisplay for Lexeme {
     match self {
       Self::Literal(l) => write!(f, "{:?}", l),
       Self::Name(token) => write!(f, "{}", i.r(*token)),
-      Self::Const => write!(f, ":="),
+      Self::Walrus => write!(f, ":="),
-      Self::Rule(prio) => write!(f, "={}=>", prio),
+      Self::Arrow(prio) => write!(f, "={}=>", print_nat16(*prio)),
       Self::NS => write!(f, "::"),
       Self::LP(l) => write!(f, "{}", l),
       Self::RP(l) => match l {
@@ -107,12 +142,15 @@ impl InternedDisplay for Lexeme {
       Self::BR => writeln!(f),
       Self::BS => write!(f, "\\"),
       Self::At => write!(f, "@"),
+      Self::Dot => write!(f, "."),
       Self::Type => write!(f, ":"),
       Self::Comment(text) => write!(f, "--[{}]--", text),
       Self::Export => write!(f, "export"),
       Self::Import => write!(f, "import"),
-      Self::Namespace => write!(f, "namespace"),
+      Self::Module => write!(f, "module"),
-      Self::PH(Placeholder { name, class }) => match *class {
+      Self::Const => write!(f, "const"),
+      Self::Macro => write!(f, "macro"),
+      Self::Placeh(Placeholder { name, class }) => match *class {
         PHClass::Scalar => write!(f, "${}", i.r(*name)),
         PHClass::Vec { nonzero, prio } => {
           if nonzero { write!(f, "...") } else { write!(f, "..") }?;
@@ -129,7 +167,9 @@ impl InternedDisplay for Lexeme {
 
 impl Lexeme {
   pub fn rule(prio: impl Into<f64>) -> Self {
-    Lexeme::Rule(NotNan::new(prio.into()).expect("Rule priority cannot be NaN"))
+    Lexeme::Arrow(
+      NotNan::new(prio.into()).expect("Rule priority cannot be NaN"),
+    )
   }
 
   pub fn parser<E: chumsky::Error<Entry>>(
@@ -179,38 +219,37 @@ pub fn lexer<'a>(
     .collect::<Vec<_>>();
   choice((
     keyword("export").to(Lexeme::Export),
-    keyword("module").to(Lexeme::Namespace),
+    keyword("module").to(Lexeme::Module),
     keyword("import").to(Lexeme::Import),
+    keyword("macro").to(Lexeme::Macro),
+    keyword("const").to(Lexeme::Const),
     paren_parser('(', ')'),
     paren_parser('[', ']'),
     paren_parser('{', '}'),
-    just(":=").to(Lexeme::Const),
+    just(":=").to(Lexeme::Walrus),
     just("=")
       .ignore_then(number::float_parser())
       .then_ignore(just("=>"))
       .map(Lexeme::rule),
     comment::comment_parser().map(Lexeme::Comment),
+    placeholder::placeholder_parser(ctx.clone()).map(Lexeme::Placeh),
     just("::").to(Lexeme::NS),
     just('\\').to(Lexeme::BS),
     just('@').to(Lexeme::At),
     just(':').to(Lexeme::Type),
     just('\n').to(Lexeme::BR),
-    placeholder::placeholder_parser(ctx.clone()).map(Lexeme::PH),
+    just('.').to(Lexeme::Dot),
     literal_parser().map(Lexeme::Literal),
-    name::name_parser(&all_ops)
+    name::name_parser(&all_ops).map({
-      .map(move |n| Lexeme::Name(ctx.interner().i(&n))),
+      let ctx = ctx.clone();
+      move |n| Lexeme::Name(ctx.interner().i(&n))
+    }),
   ))
-  .map_with_span(|lexeme, range| Entry { lexeme, range })
+  .map_with_span(move |lexeme, range| Entry {
+    lexeme,
+    location: Location::Range { range, file: ctx.file() },
+  })
   .padded_by(one_of(" \t").repeated())
   .repeated()
   .then_ignore(end())
 }
-
-pub fn filter_map_lex<'a, O, M: ToString>(
-  f: impl Fn(Lexeme) -> Result<O, M> + Clone + 'a,
-) -> impl SimpleParser<Entry, (O, Range<usize>)> + Clone + 'a {
-  filter_map(move |s: Range<usize>, e: Entry| {
-    let out = f(e.lexeme).map_err(|msg| Simple::custom(s.clone(), msg))?;
-    Ok((out, s))
-  })
-}

src/parse/mod.rs

@@ -1,20 +1,19 @@
 mod comment;
 mod context;
 mod decls;
-mod enum_filter;
+mod errors;
-mod expression;
 mod facade;
-mod import;
 mod lexer;
+mod multiname;
 mod name;
 mod number;
 mod placeholder;
 mod sourcefile;
+mod stream;
 mod string;
 
 pub use context::ParsingContext;
-pub use facade::{parse, ParseError};
+pub use facade::parse2;
 pub use lexer::{lexer, Entry, Lexeme};
 pub use name::is_op;
 pub use number::{float_parser, int_parser};
-pub use sourcefile::line_parser;

src/parse/multiname.rs

@@ -0,0 +1,88 @@
+use std::collections::VecDeque;
+use std::iter;
+
+use super::context::Context;
+use super::errors::{Expected, ExpectedName};
+use super::stream::Stream;
+use super::Lexeme;
+use crate::error::{ProjectError, ProjectResult};
+use crate::utils::iter::{box_chain, box_once, BoxedIterIter};
+use crate::Tok;
+
+fn parse_multiname_branch(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<(BoxedIterIter<Tok<String>>, Stream<'_>)> {
+  let comma = ctx.interner().i(",");
+  let (subnames, cursor) = parse_multiname_rec(cursor, ctx.clone())?;
+  let (delim, cursor) = cursor.trim().pop()?;
+  match delim.lexeme {
+    Lexeme::Name(n) if n == comma => {
+      let (tail, cont) = parse_multiname_branch(cursor, ctx)?;
+      Ok((box_chain!(subnames, tail), cont))
+    },
+    Lexeme::RP('(') => Ok((subnames, cursor)),
+    _ => Err(
+      Expected {
+        expected: vec![Lexeme::Name(comma), Lexeme::RP('(')],
+        or_name: false,
+        found: delim.clone(),
+      }
+      .rc(),
+    ),
+  }
+}
+
+pub fn parse_multiname_rec(
+  curosr: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<(BoxedIterIter<Tok<String>>, Stream<'_>)> {
+  let comma = ctx.interner().i(",");
+  let (head, cursor) = curosr.trim().pop()?;
+  match &head.lexeme {
+    Lexeme::LP('(') => parse_multiname_branch(cursor, ctx),
+    Lexeme::LP('[') => {
+      let (op_ent, cursor) = cursor.trim().pop()?;
+      let op = ExpectedName::expect(op_ent)?;
+      let (rp_ent, cursor) = cursor.trim().pop()?;
+      Expected::expect(Lexeme::RP('['), rp_ent)?;
+      Ok((box_once(box_once(op)), cursor))
+    },
+    Lexeme::Name(n) if *n != comma => {
+      let cursor = cursor.trim();
+      if cursor.get(0).ok().map(|e| &e.lexeme) == Some(&Lexeme::NS) {
+        let cursor = cursor.step()?;
+        let (out, cursor) = parse_multiname_rec(cursor, ctx)?;
+        let out = Box::new(out.map(|i| box_chain!(i, iter::once(*n))));
+        Ok((out, cursor))
+      } else {
+        Ok((box_once(box_once(*n)), cursor))
+      }
+    },
+    _ => Err(
+      Expected {
+        expected: vec![Lexeme::LP('(')],
+        or_name: true,
+        found: head.clone(),
+      }
+      .rc(),
+    ),
+  }
+}
+
+pub fn parse_multiname(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<(Vec<Vec<Tok<String>>>, Stream<'_>)> {
+  let (output, cont) = parse_multiname_rec(cursor, ctx)?;
+  let output = output
+    .map(|it| {
+      let mut deque = VecDeque::with_capacity(it.size_hint().0);
+      for item in it {
+        deque.push_front(item)
+      }
+      deque.into()
+    })
+    .collect();
+  Ok((output, cont))
+}

src/parse/number.rs

@@ -123,3 +123,9 @@ pub fn float_parser() -> impl SimpleParser<char, NotNan<f64>> {
   ))
   .labelled("float")
 }
+
+pub fn print_nat16(num: NotNan<f64>) -> String {
+  let exp = num.log(16.0).floor();
+  let man = num / 16_f64.powf(exp);
+  format!("{man}p{exp:.0}")
+}

src/parse/sourcefile.rs

@@ -1,138 +1,28 @@
 use std::iter;
 use std::rc::Rc;
 
-use chumsky::prelude::*;
-use chumsky::Parser;
 use itertools::Itertools;
 
 use super::context::Context;
-use super::decls::{SimpleParser, SimpleRecursive};
+use super::errors::{
-use super::enum_filter::enum_filter;
+  BadTokenInRegion, Expected, ExpectedName, LeadingNS, MisalignedParen,
-use super::expression::xpr_parser;
+  NamespacedExport, ReservedToken, UnexpectedEOL,
-use super::import::import_parser;
+};
-use super::lexer::{filter_map_lex, Lexeme};
+use super::lexer::Lexeme;
+use super::multiname::parse_multiname;
+use super::stream::Stream;
 use super::Entry;
 use crate::ast::{Clause, Constant, Expr, Rule};
+use crate::error::{ProjectError, ProjectResult};
 use crate::representations::location::Location;
-use crate::representations::sourcefile::{FileEntry, Member, Namespace};
+use crate::representations::sourcefile::{FileEntry, Member, ModuleBlock};
 use crate::representations::VName;
+use crate::sourcefile::Import;
+use crate::Primitive;
 
-fn rule_parser<'a>(
+pub fn split_lines(module: Stream<'_>) -> impl Iterator<Item = Stream<'_>> {
-  ctx: impl Context + 'a,
+  let mut source = module.data.iter().enumerate();
-) -> impl SimpleParser<Entry, Rule<VName>> + 'a {
+  let mut fallback = module.fallback;
-  xpr_parser(ctx.clone())
-    .repeated()
-    .at_least(1)
-    .then(filter_map_lex(enum_filter!(Lexeme::Rule)))
-    .then(xpr_parser(ctx).repeated().at_least(1))
-    .map(|((p, (prio, _)), t)| Rule { pattern: p, prio, template: t })
-    .labelled("Rule")
-}
-
-fn const_parser<'a>(
-  ctx: impl Context + 'a,
-) -> impl SimpleParser<Entry, Constant> + 'a {
-  filter_map_lex(enum_filter!(Lexeme::Name))
-    .then_ignore(Lexeme::Const.parser())
-    .then(xpr_parser(ctx.clone()).repeated().at_least(1))
-    .map(move |((name, _), value)| Constant {
-      name,
-      value: if let Ok(ex) = value.iter().exactly_one() {
-        ex.clone()
-      } else {
-        let start = value
-          .first()
-          .expect("value cannot be empty")
-          .location
-          .range()
-          .expect("all locations in parsed source are known")
-          .start;
-        let end = value
-          .last()
-          .expect("asserted right above")
-          .location
-          .range()
-          .expect("all locations in parsed source are known")
-          .end;
-        Expr {
-          location: Location::Range { file: ctx.file(), range: start..end },
-          value: Clause::S('(', Rc::new(value)),
-        }
-      },
-    })
-}
-
-pub fn collect_errors<T, E: chumsky::Error<T>>(e: Vec<E>) -> E {
-  e.into_iter()
-    .reduce(chumsky::Error::merge)
-    .expect("Error list must be non_enmpty")
-}
-
-fn namespace_parser<'a>(
-  line: impl SimpleParser<Entry, FileEntry> + 'a,
-) -> impl SimpleParser<Entry, Namespace> + 'a {
-  Lexeme::Namespace
-    .parser()
-    .ignore_then(filter_map_lex(enum_filter!(Lexeme::Name)))
-    .then(
-      any()
-        .repeated()
-        .delimited_by(Lexeme::LP('(').parser(), Lexeme::RP('(').parser())
-        .try_map(move |body, _| {
-          split_lines(&body)
-            .map(|l| line.parse(l))
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(collect_errors)
-        }),
-    )
-    .map(move |((name, _), body)| Namespace { name, body })
-}
-
-fn member_parser<'a>(
-  line: impl SimpleParser<Entry, FileEntry> + 'a,
-  ctx: impl Context + 'a,
-) -> impl SimpleParser<Entry, Member> + 'a {
-  choice((
-    namespace_parser(line).map(Member::Namespace),
-    rule_parser(ctx.clone()).map(Member::Rule),
-    const_parser(ctx).map(Member::Constant),
-  ))
-}
-
-pub fn line_parser<'a>(
-  ctx: impl Context + 'a,
-) -> impl SimpleParser<Entry, FileEntry> + 'a {
-  recursive(|line: SimpleRecursive<Entry, FileEntry>| {
-    choice((
-      // In case the usercode wants to parse doc
-      filter_map_lex(enum_filter!(Lexeme >> FileEntry; Comment))
-        .map(|(ent, _)| ent),
-      // plain old imports
-      Lexeme::Import
-        .parser()
-        .ignore_then(import_parser(ctx.clone()).map(FileEntry::Import)),
-      Lexeme::Export.parser().ignore_then(choice((
-        // token collection
-        Lexeme::NS
-          .parser()
-          .ignore_then(
-            filter_map_lex(enum_filter!(Lexeme::Name))
-              .map(|(e, _)| e)
-              .separated_by(Lexeme::Name(ctx.interner().i(",")).parser())
-              .delimited_by(Lexeme::LP('(').parser(), Lexeme::RP('(').parser()),
-          )
-          .map(FileEntry::Export),
-        // public declaration
-        member_parser(line.clone(), ctx.clone()).map(FileEntry::Exported),
-      ))),
-      // This could match almost anything so it has to go last
-      member_parser(line, ctx).map(FileEntry::Internal),
-    ))
-  })
-}
-
-pub fn split_lines(data: &[Entry]) -> impl Iterator<Item = &[Entry]> {
-  let mut source = data.iter().enumerate();
   let mut last_slice = 0;
   let mut finished = false;
   iter::from_fn(move || {
@@ -144,7 +34,9 @@ pub fn split_lines(data: &[Entry]) -> impl Iterator<Item = &[Entry]> {
         Lexeme::BR if paren_count == 0 => {
           let begin = last_slice;
           last_slice = i + 1;
-          return Some(&data[begin..i]);
+          let cur_prev = fallback;
+          fallback = &module.data[i];
+          return Some(Stream::new(cur_prev, &module.data[begin..i]));
         },
         _ => (),
       }
@@ -152,9 +44,242 @@ pub fn split_lines(data: &[Entry]) -> impl Iterator<Item = &[Entry]> {
     // Include last line even without trailing newline
     if !finished {
       finished = true;
-      return Some(&data[last_slice..]);
+      return Some(Stream::new(fallback, &module.data[last_slice..]));
     }
     None
   })
-  .filter(|s| !s.is_empty())
+}
+
+pub fn parse_module_body(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<Vec<FileEntry>> {
+  split_lines(cursor)
+    .map(Stream::trim)
+    .filter(|l| !l.data.is_empty())
+    .map(|l| parse_line(l, ctx.clone()))
+    .collect()
+}
+
+pub fn parse_line(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<FileEntry> {
+  match cursor.get(0)?.lexeme {
+    Lexeme::BR | Lexeme::Comment(_) => parse_line(cursor.step()?, ctx),
+    Lexeme::Export => parse_export_line(cursor.step()?, ctx),
+    Lexeme::Const | Lexeme::Macro | Lexeme::Module =>
+      Ok(FileEntry::Internal(parse_member(cursor, ctx)?)),
+    Lexeme::Import => {
+      let globstar = ctx.interner().i("*");
+      let (names, cont) = parse_multiname(cursor.step()?, ctx.clone())?;
+      cont.expect_empty()?;
+      let imports = (names.into_iter())
+        .map(|mut nsname| {
+          let name = nsname.pop().expect("multinames cannot be zero-length");
+          Import {
+            path: ctx.interner().i(&nsname),
+            name: if name == globstar { None } else { Some(name) },
+          }
+        })
+        .collect();
+      Ok(FileEntry::Import(imports))
+    },
+    _ => {
+      let err = BadTokenInRegion {
+        entry: cursor.get(0)?.clone(),
+        region: "start of line",
+      };
+      Err(err.rc())
+    },
+  }
+}
+
+pub fn parse_export_line(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<FileEntry> {
+  let cursor = cursor.trim();
+  match cursor.get(0)?.lexeme {
+    Lexeme::NS => {
+      let (names, cont) = parse_multiname(cursor.step()?, ctx)?;
+      cont.expect_empty()?;
+      let names = (names.into_iter())
+        .map(|i| if i.len() == 1 { Some(i[0]) } else { None })
+        .collect::<Option<Vec<_>>>()
+        .ok_or_else(|| NamespacedExport { location: cursor.location() }.rc())?;
+      Ok(FileEntry::Export(names))
+    },
+    Lexeme::Const | Lexeme::Macro | Lexeme::Module =>
+      Ok(FileEntry::Exported(parse_member(cursor, ctx)?)),
+    _ => {
+      let err = BadTokenInRegion {
+        entry: cursor.get(0)?.clone(),
+        region: "exported line",
+      };
+      Err(err.rc())
+    },
+  }
+}
+
+fn parse_member(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<Member> {
+  let (typemark, cursor) = cursor.trim().pop()?;
+  match typemark.lexeme {
+    Lexeme::Const => {
+      let constant = parse_const(cursor, ctx)?;
+      Ok(Member::Constant(constant))
+    },
+    Lexeme::Macro => {
+      let rule = parse_rule(cursor, ctx)?;
+      Ok(Member::Rule(rule))
+    },
+    Lexeme::Module => {
+      let module = parse_module(cursor, ctx)?;
+      Ok(Member::Module(module))
+    },
+    _ => {
+      let err =
+        BadTokenInRegion { entry: typemark.clone(), region: "member type" };
+      Err(err.rc())
+    },
+  }
+}
+
+fn parse_rule(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<Rule<VName>> {
+  let (pattern, prio, template) = cursor.find_map("arrow", |a| match a {
+    Lexeme::Arrow(p) => Some(*p),
+    _ => None,
+  })?;
+  let (pattern, _) = parse_exprv(pattern, None, ctx.clone())?;
+  let (template, _) = parse_exprv(template, None, ctx)?;
+  Ok(Rule { pattern, prio, template })
+}
+
+fn parse_const(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<Constant> {
+  let (name_ent, cursor) = cursor.trim().pop()?;
+  let name = ExpectedName::expect(name_ent)?;
+  let (walrus_ent, cursor) = cursor.trim().pop()?;
+  Expected::expect(Lexeme::Walrus, walrus_ent)?;
+  let (body, _) = parse_exprv(cursor, None, ctx)?;
+  Ok(Constant { name, value: vec_to_single(walrus_ent, body)? })
+}
+
+fn parse_module(
+  cursor: Stream<'_>,
+  ctx: impl Context,
+) -> ProjectResult<ModuleBlock> {
+  let (name_ent, cursor) = cursor.trim().pop()?;
+  let name = ExpectedName::expect(name_ent)?;
+  let (lp_ent, cursor) = cursor.trim().pop()?;
+  Expected::expect(Lexeme::LP('('), lp_ent)?;
+  let (last, cursor) = cursor.pop_back()?;
+  Expected::expect(Lexeme::RP('('), last)?;
+  let body = parse_module_body(cursor, ctx)?;
+  Ok(ModuleBlock { name, body })
+}
+
+fn parse_exprv(
+  mut cursor: Stream<'_>,
+  paren: Option<char>,
+  ctx: impl Context,
+) -> ProjectResult<(Vec<Expr<VName>>, Stream<'_>)> {
+  let mut output = Vec::new();
+  cursor = cursor.trim();
+  while let Ok(current) = cursor.get(0) {
+    match &current.lexeme {
+      Lexeme::BR | Lexeme::Comment(_) => unreachable!("Fillers skipped"),
+      Lexeme::At | Lexeme::Type =>
+        return Err(ReservedToken { entry: current.clone() }.rc()),
+      Lexeme::Literal(l) => {
+        output.push(Expr {
+          value: Clause::P(Primitive::Literal(l.clone())),
+          location: current.location(),
+        });
+        cursor = cursor.step()?;
+      },
+      Lexeme::Placeh(ph) => {
+        output.push(Expr {
+          value: Clause::Placeh(*ph),
+          location: current.location(),
+        });
+        cursor = cursor.step()?;
+      },
+      Lexeme::Name(n) => {
+        let location = cursor.location();
+        let mut fullname = vec![*n];
+        while cursor.get(1).ok().map(|e| &e.lexeme) == Some(&Lexeme::NS) {
+          fullname.push(ExpectedName::expect(cursor.get(2)?)?);
+          cursor = cursor.step()?.step()?;
+        }
+        output.push(Expr { value: Clause::Name(fullname), location });
+        cursor = cursor.step()?;
+      },
+      Lexeme::NS =>
+        return Err(LeadingNS { location: current.location() }.rc()),
+      Lexeme::RP(c) =>
+        return if Some(*c) == paren {
+          Ok((output, cursor.step()?))
+        } else {
+          Err(MisalignedParen { entry: cursor.get(0)?.clone() }.rc())
+        },
+      Lexeme::LP(c) => {
+        let (result, leftover) =
+          parse_exprv(cursor.step()?, Some(*c), ctx.clone())?;
+        output.push(Expr {
+          value: Clause::S(*c, Rc::new(result)),
+          location: cursor.get(0)?.location().to(leftover.fallback.location()),
+        });
+        cursor = leftover;
+      },
+      Lexeme::BS => {
+        let (arg, body) =
+          cursor.step()?.find("A '.'", |l| l == &Lexeme::Dot)?;
+        let (arg, _) = parse_exprv(arg, None, ctx.clone())?;
+        let (body, leftover) = parse_exprv(body, paren, ctx)?;
+        output.push(Expr {
+          location: cursor.location(),
+          value: Clause::Lambda(Rc::new(arg), Rc::new(body)),
+        });
+        return Ok((output, leftover));
+      },
+      _ => {
+        let err = BadTokenInRegion {
+          entry: cursor.get(0)?.clone(),
+          region: "expression",
+        };
+        return Err(err.rc());
+      },
+    }
+    cursor = cursor.trim();
+  }
+  Ok((output, Stream::new(cursor.fallback, &[])))
+}
+
+fn vec_to_single(
+  fallback: &Entry,
+  v: Vec<Expr<VName>>,
+) -> ProjectResult<Expr<VName>> {
+  match v.len() {
+    0 => return Err(UnexpectedEOL { entry: fallback.clone() }.rc()),
+    1 => Ok(v.into_iter().exactly_one().unwrap()),
+    _ => Ok(Expr {
+      location: expr_slice_location(&v),
+      value: Clause::S('(', Rc::new(v)),
+    }),
+  }
+}
+
+pub fn expr_slice_location(v: &[impl AsRef<Location>]) -> Location {
+  v.first()
+    .map(|l| l.as_ref().clone().to(v.last().unwrap().as_ref().clone()))
+    .unwrap_or(Location::Unknown)
 }

src/parse/stream.rs

@@ -0,0 +1,119 @@
+use super::errors::{ExpectedEOL, NotFound, UnexpectedEOL};
+use super::{Entry, Lexeme};
+use crate::error::{ProjectError, ProjectResult};
+use crate::Location;
+
+/// Represents a slice which may or may not contain items, and a fallback entry
+/// used for error reporting whenever the errant stream is empty.
+#[derive(Clone, Copy)]
+pub struct Stream<'a> {
+  pub fallback: &'a Entry,
+  pub data: &'a [Entry],
+}
+impl<'a> Stream<'a> {
+  pub fn new(fallback: &'a Entry, data: &'a [Entry]) -> Self {
+    Self { fallback, data }
+  }
+
+  pub fn trim(self) -> Self {
+    let Self { data, fallback } = self;
+    let front = data.iter().take_while(|e| e.is_filler()).count();
+    let (_, right) = data.split_at(front);
+    let back = right.iter().rev().take_while(|e| e.is_filler()).count();
+    let (data, _) = right.split_at(right.len() - back);
+    Self { fallback, data }
+  }
+
+  pub fn step(self) -> ProjectResult<Self> {
+    let (fallback, data) = (self.data.split_first())
+      .ok_or_else(|| UnexpectedEOL { entry: self.fallback.clone() }.rc())?;
+    Ok(Stream { data, fallback })
+  }
+
+  pub fn pop(self) -> ProjectResult<(&'a Entry, Stream<'a>)> {
+    Ok((self.get(0)?, self.step()?))
+  }
+
+  /// Retrieve an index from a slice or raise an [UnexpectedEOL].
+  pub fn get(self, idx: usize) -> ProjectResult<&'a Entry> {
+    self.data.get(idx).ok_or_else(|| {
+      let entry = self.data.last().unwrap_or(self.fallback).clone();
+      UnexpectedEOL { entry }.rc()
+    })
+  }
+
+  pub fn location(self) -> Location {
+    self.data.first().map_or_else(
+      || self.fallback.location(),
+      |f| f.location().to(self.data.last().unwrap().location()),
+    )
+  }
+
+  pub fn find_map<T>(
+    self,
+    expected: &'static str,
+    mut f: impl FnMut(&'a Lexeme) -> Option<T>,
+  ) -> ProjectResult<(Self, T, Self)> {
+    let Self { data, fallback } = self;
+    let (dot_idx, output) = skip_parenthesized(data.iter())
+      .find_map(|(i, e)| f(&e.lexeme).map(|t| (i, t)))
+      .ok_or_else(|| NotFound { expected, location: self.location() }.rc())?;
+    let (left, not_left) = data.split_at(dot_idx);
+    let (middle_ent, right) = not_left.split_first().unwrap();
+    Ok((Self::new(fallback, left), output, Self::new(middle_ent, right)))
+  }
+
+  pub fn find(
+    self,
+    expected: &'static str,
+    mut f: impl FnMut(&Lexeme) -> bool,
+  ) -> ProjectResult<(Self, Self)> {
+    let (left, _, right) =
+      self.find_map(expected, |l| if f(l) { Some(()) } else { None })?;
+    Ok((left, right))
+  }
+
+  pub fn pop_back(self) -> ProjectResult<(&'a Entry, Self)> {
+    let Self { data, fallback } = self;
+    let (last, data) = (data.split_last())
+      .ok_or_else(|| UnexpectedEOL { entry: fallback.clone() }.rc())?;
+    Ok((last, Self { fallback, data }))
+  }
+
+  /// # Panics
+  ///
+  /// If the slice is empty
+  pub fn from_slice(data: &'a [Entry]) -> Self {
+    let fallback =
+      (data.first()).expect("Empty slice cannot be converted into a parseable");
+    Self { data, fallback }
+  }
+
+  pub fn expect_empty(self) -> ProjectResult<()> {
+    if let Some(x) = self.data.first() {
+      Err(ExpectedEOL { location: x.location() }.rc())
+    } else {
+      Ok(())
+    }
+  }
+}
+
+// impl<'a> From<(&'a Entry, &'a [Entry])> for Stream<'a> {
+//   fn from((fallback, data): (&'a Entry, &'a [Entry])) -> Self {
+//     Self::new(fallback, data)
+//   }
+// }
+
+pub fn skip_parenthesized<'a>(
+  it: impl Iterator<Item = &'a Entry>,
+) -> impl Iterator<Item = (usize, &'a Entry)> {
+  let mut paren_lvl = 1;
+  it.enumerate().filter(move |(_, e)| {
+    match e.lexeme {
+      Lexeme::LP(_) => paren_lvl += 1,
+      Lexeme::RP(_) => paren_lvl -= 1,
+      _ => (),
+    }
+    paren_lvl <= 1
+  })
+}

src/pipeline/error/parse_error_with_path.rs

@@ -1,45 +0,0 @@
-use std::rc::Rc;
-
-use super::{ErrorPosition, ProjectError};
-use crate::parse::ParseError;
-use crate::representations::location::Location;
-use crate::utils::BoxedIter;
-
-/// Produced by stages that parse text when it fails.
-#[derive(Debug)]
-pub struct ParseErrorWithPath {
-  /// The complete source of the faulty file
-  pub full_source: String,
-  /// The path to the faulty file
-  pub path: Vec<String>,
-  /// The parse error produced by Chumsky
-  pub error: ParseError,
-}
-impl ProjectError for ParseErrorWithPath {
-  fn description(&self) -> &str {
-    "Failed to parse code"
-  }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
-    match &self.error {
-      ParseError::Lex(lex) => Box::new(lex.iter().map(|s| ErrorPosition {
-        location: Location::Range {
-          file: Rc::new(self.path.clone()),
-          range: s.span(),
-        },
-        message: Some(s.to_string()),
-      })),
-      ParseError::Ast(ast) => Box::new(ast.iter().map(|(_i, s)| {
-        ErrorPosition {
-          location: s
-            .found()
-            .map(|e| Location::Range {
-              file: Rc::new(self.path.clone()),
-              range: e.range.clone(),
-            })
-            .unwrap_or_else(|| Location::File(Rc::new(self.path.clone()))),
-          message: Some(s.label().unwrap_or("Parse error").to_string()),
-        }
-      })),
-    }
-  }
-}

src/pipeline/file_loader.rs

@@ -3,11 +3,13 @@ use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use std::{fs, io};
 
-use chumsky::text::Character;
+use hashbrown::{HashMap, HashSet};
 use rust_embed::RustEmbed;
 
+use crate::error::{ErrorPosition, ProjectError, ProjectResult};
+#[allow(unused)] // for doc
+use crate::facade::System;
 use crate::interner::Interner;
-use crate::pipeline::error::{ErrorPosition, ProjectError};
 use crate::utils::iter::box_once;
 use crate::utils::{BoxedIter, Cache};
 use crate::{Stok, VName};
@@ -23,10 +25,10 @@ impl ProjectError for FileLoadingError {
   fn description(&self) -> &str {
     "Neither a file nor a directory could be read from the requested path"
   }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
     box_once(ErrorPosition::just_file(self.path.clone()))
   }
-  fn message(&self) -> String {
+  fn message(&self, _i: &Interner) -> String {
     format!("File: {}\nDirectory: {}", self.file, self.dir)
   }
 }
@@ -49,7 +51,7 @@ impl Loaded {
 }
 
 /// Returned by any source loading callback
-pub type IOResult = Result<Loaded, Rc<dyn ProjectError>>;
+pub type IOResult = ProjectResult<Loaded>;
 
 /// Load a file from a path expressed in Rust strings, but relative to
 /// a root expressed as an OS Path.
@@ -103,7 +105,8 @@ pub fn mk_dir_cache(root: PathBuf, i: &Interner) -> Cache<VName, IOResult> {
 pub fn load_embed<T: 'static + RustEmbed>(path: &str, ext: &str) -> IOResult {
   let file_path = path.to_string() + ext;
   if let Some(file) = T::get(&file_path) {
-    let s = file.data.iter().map(|c| c.to_char()).collect::<String>();
+    let s =
+      String::from_utf8(file.data.to_vec()).expect("Embed must be valid UTF-8");
     Ok(Loaded::Code(Rc::new(s)))
   } else {
     let entries = T::iter()
@@ -137,3 +140,38 @@ pub fn mk_embed_cache<'a, T: 'static + RustEmbed>(
     load_embed::<T>(&path, ext)
   })
 }
+
+/// Load all files from an embed and convert them into a map usable in a
+/// [System]
+pub fn embed_to_map<T: 'static + RustEmbed>(
+  suffix: &str,
+  i: &Interner,
+) -> HashMap<Vec<Stok>, Loaded> {
+  let mut files = HashMap::new();
+  let mut dirs = HashMap::new();
+  for path in T::iter() {
+    let vpath = path
+      .strip_suffix(suffix)
+      .expect("the embed must be filtered for suffix")
+      .split('/')
+      .map(|s| s.to_string())
+      .collect::<Vec<_>>();
+    let tokvpath = vpath.iter().map(|segment| i.i(segment)).collect::<Vec<_>>();
+    let data = T::get(&path).expect("path from iterator").data;
+    let text =
+      String::from_utf8(data.to_vec()).expect("code embeds must be utf-8");
+    files.insert(tokvpath.clone(), text);
+    for (lvl, subname) in vpath.iter().enumerate() {
+      let dirname = tokvpath.split_at(lvl).0;
+      let (_, entries) = (dirs.raw_entry_mut().from_key(dirname))
+        .or_insert_with(|| (dirname.to_vec(), HashSet::new()));
+      entries.get_or_insert_with(subname, Clone::clone);
+    }
+  }
+  (files.into_iter())
+    .map(|(k, s)| (k, Loaded::Code(Rc::new(s))))
+    .chain((dirs.into_iter()).map(|(k, entv)| {
+      (k, Loaded::Collection(Rc::new(entv.into_iter().collect())))
+    }))
+    .collect()
+}

src/pipeline/import_abs_path.rs

@@ -1,6 +1,4 @@
-use std::rc::Rc;
+use crate::error::{ProjectError, ProjectResult, TooManySupers};
-
-use super::error::{ProjectError, TooManySupers};
 use crate::interner::{Interner, Tok};
 use crate::representations::sourcefile::absolute_path;
 use crate::utils::Substack;
@@ -10,7 +8,7 @@ pub fn import_abs_path(
   mod_stack: Substack<Tok<String>>,
   import_path: &[Tok<String>],
   i: &Interner,
-) -> Result<Vec<Tok<String>>, Rc<dyn ProjectError>> {
+) -> ProjectResult<Vec<Tok<String>>> {
   // path of module within file
   let mod_pathv = mod_stack.iter().rev_vec_clone();
   // path of module within compilation
@@ -23,9 +21,9 @@ pub fn import_abs_path(
   // preload-target path within compilation
   absolute_path(&abs_pathv, import_path, i).map_err(|_| {
     TooManySupers {
-      path: import_path.iter().map(|t| i.r(*t)).cloned().collect(),
+      path: import_path.to_vec(),
-      offender_file: src_path.iter().map(|t| i.r(*t)).cloned().collect(),
+      offender_file: src_path.to_vec(),
-      offender_mod: mod_pathv.iter().map(|t| i.r(*t)).cloned().collect(),
+      offender_mod: mod_pathv,
     }
     .rc()
   })

src/pipeline/import_resolution/collect_aliases.rs

@@ -1,9 +1,7 @@
-use std::rc::Rc;
-
 use super::alias_map::AliasMap;
 use super::decls::UpdatedFn;
-use crate::interner::{Interner, Tok};
+use crate::error::{NotExported, NotFound, ProjectError, ProjectResult};
-use crate::pipeline::error::{NotExported, NotFound, ProjectError};
+use crate::interner::Tok;
 use crate::pipeline::project_tree::split_path;
 use crate::representations::project::{ProjectModule, ProjectTree};
 use crate::representations::tree::{ModMember, WalkErrorKind};
@@ -15,8 +13,7 @@ fn assert_visible(
   source: &[Tok<String>], // must point to a file or submodule
   target: &[Tok<String>], // may point to a symbol or module of any kind
   project: &ProjectTree<VName>,
-  i: &Interner,
+) -> ProjectResult<()> {
-) -> Result<(), Rc<dyn ProjectError>> {
   let (tgt_item, tgt_path) = unwrap_or!(target.split_last(); return Ok(()));
   let shared_len =
     source.iter().zip(tgt_path.iter()).take_while(|(a, b)| a == b).count();
@@ -27,11 +24,11 @@ fn assert_visible(
       WalkErrorKind::Private =>
         unreachable!("visibility is not being checked here"),
       WalkErrorKind::Missing => NotFound::from_walk_error(
+        source,
         &[],
         &tgt_path[..vis_ignored_len],
         &project.0,
         e,
-        i,
       )
       .rc(),
     })?;
@@ -39,37 +36,51 @@ fn assert_visible(
     .walk_ref(&tgt_path[vis_ignored_len..], true)
     .map_err(|e| match e.kind {
       WalkErrorKind::Missing => NotFound::from_walk_error(
+        source,
         &tgt_path[..vis_ignored_len],
         &tgt_path[vis_ignored_len..],
         &project.0,
         e,
-        i,
       )
       .rc(),
       WalkErrorKind::Private => {
         let full_path = &tgt_path[..shared_len + e.pos];
-        let (file, sub) = split_path(full_path, project);
+        // These errors are encountered during error reporting but they're more
-        let (ref_file, ref_sub) = split_path(source, project);
+        // fundamental / higher prio than the error to be raised and would
+        // emerge nonetheless so they take over and the original error is
+        // swallowed
+        match split_path(full_path, project) {
+          Err(e) =>
+            NotFound::from_walk_error(source, &[], full_path, &project.0, e)
+              .rc(),
+          Ok((file, sub)) => {
+            let (ref_file, ref_sub) = split_path(source, project)
+              .expect("Source path assumed to be valid");
             NotExported {
-          file: i.extern_all(file),
+              file: file.to_vec(),
-          subpath: i.extern_all(sub),
+              subpath: sub.to_vec(),
-          referrer_file: i.extern_all(ref_file),
+              referrer_file: ref_file.to_vec(),
-          referrer_subpath: i.extern_all(ref_sub),
+              referrer_subpath: ref_sub.to_vec(),
             }
             .rc()
           },
+        }
+      },
     })?;
   let tgt_item_exported = direct_parent.extra.exports.contains_key(tgt_item);
   let target_prefixes_source = shared_len == tgt_path.len();
   if !tgt_item_exported && !target_prefixes_source {
-    let (file, sub) = split_path(target, project);
+    let (file, sub) = split_path(target, project).map_err(|e| {
-    let (ref_file, ref_sub) = split_path(source, project);
+      NotFound::from_walk_error(source, &[], target, &project.0, e).rc()
+    })?;
+    let (ref_file, ref_sub) = split_path(source, project)
+      .expect("The source path is assumed to be valid");
     Err(
       NotExported {
-        file: i.extern_all(file),
+        file: file.to_vec(),
-        subpath: i.extern_all(sub),
+        subpath: sub.to_vec(),
-        referrer_file: i.extern_all(ref_file),
+        referrer_file: ref_file.to_vec(),
-        referrer_subpath: i.extern_all(ref_sub),
+        referrer_subpath: ref_sub.to_vec(),
       }
       .rc(),
     )
@@ -84,9 +95,8 @@ fn collect_aliases_rec(
   module: &ProjectModule<VName>,
   project: &ProjectTree<VName>,
   alias_map: &mut AliasMap,
-  i: &Interner,
   updated: &impl UpdatedFn,
-) -> Result<(), Rc<dyn ProjectError>> {
+) -> ProjectResult<()> {
   // Assume injected module has been alias-resolved
   let mod_path_v = path.iter().rev_vec_clone();
   if !updated(&mod_path_v) {
@@ -94,20 +104,18 @@ fn collect_aliases_rec(
   };
   for (&name, target_mod_name) in module.extra.imports_from.iter() {
     let target_sym_v = pushed(target_mod_name, name);
-    assert_visible(&mod_path_v, &target_sym_v, project, i)?;
+    assert_visible(&mod_path_v, &target_sym_v, project)?;
     let sym_path_v = pushed(&mod_path_v, name);
     let target_mod = (project.0.walk_ref(target_mod_name, false))
       .expect("checked above in assert_visible");
-    let target_sym = target_mod
+    let target_sym = (target_mod.extra.exports.get(&name))
-      .extra
-      .exports
-      .get(&name)
       .ok_or_else(|| {
         let file_len =
           target_mod.extra.file.as_ref().unwrap_or(target_mod_name).len();
         NotFound {
-          file: i.extern_all(&target_mod_name[..file_len]),
+          source: Some(mod_path_v.clone()),
-          subpath: i.extern_all(&target_sym_v[file_len..]),
+          file: target_mod_name[..file_len].to_vec(),
+          subpath: target_sym_v[file_len..].to_vec(),
         }
         .rc()
       })?
@@ -121,7 +129,6 @@ fn collect_aliases_rec(
       submodule,
       project,
       alias_map,
-      i,
       updated,
     )?
   }
@@ -133,8 +140,7 @@ pub fn collect_aliases(
   module: &ProjectModule<VName>,
   project: &ProjectTree<VName>,
   alias_map: &mut AliasMap,
-  i: &Interner,
   updated: &impl UpdatedFn,
-) -> Result<(), Rc<dyn ProjectError>> {
+) -> ProjectResult<()> {
-  collect_aliases_rec(Substack::Bottom, module, project, alias_map, i, updated)
+  collect_aliases_rec(Substack::Bottom, module, project, alias_map, updated)
 }

src/pipeline/import_resolution/resolve_imports.rs

@@ -1,11 +1,8 @@
-use std::rc::Rc;
-
 use super::alias_map::AliasMap;
 use super::apply_aliases::apply_aliases;
 use super::collect_aliases::collect_aliases;
 use super::decls::{InjectedAsFn, UpdatedFn};
-use crate::interner::Interner;
+use crate::error::ProjectResult;
-use crate::pipeline::error::ProjectError;
 use crate::representations::project::ProjectTree;
 use crate::representations::VName;
 
@@ -13,12 +10,11 @@ use crate::representations::VName;
 /// replace these aliases with the original names throughout the tree
 pub fn resolve_imports(
   project: ProjectTree<VName>,
-  i: &Interner,
   injected_as: &impl InjectedAsFn,
   updated: &impl UpdatedFn,
-) -> Result<ProjectTree<VName>, Rc<dyn ProjectError>> {
+) -> ProjectResult<ProjectTree<VName>> {
   let mut map = AliasMap::new();
-  collect_aliases(&project.0, &project, &mut map, i, updated)?;
+  collect_aliases(&project.0, &project, &mut map, updated)?;
   let new_mod = apply_aliases(&project.0, &map, injected_as, updated);
   Ok(ProjectTree(new_mod))
 }

src/pipeline/mod.rs

@@ -1,5 +1,4 @@
 //! Loading Orchid modules from source
-pub mod error;
 pub mod file_loader;
 mod import_abs_path;
 mod import_resolution;

src/pipeline/parse_layer.rs

@@ -1,8 +1,8 @@
 use std::rc::Rc;
 
-use super::error::ProjectError;
 use super::file_loader::IOResult;
 use super::{import_resolution, project_tree, source_loader};
+use crate::error::ProjectResult;
 use crate::interner::{Interner, Tok};
 use crate::representations::sourcefile::FileEntry;
 use crate::representations::VName;
@@ -22,7 +22,7 @@ pub fn parse_layer<'a>(
   environment: &'a ProjectTree<VName>,
   prelude: &[FileEntry],
   i: &Interner,
-) -> Result<ProjectTree<VName>, Rc<dyn ProjectError>> {
+) -> ProjectResult<ProjectTree<VName>> {
   // A path is injected if it is walkable in the injected tree
   let injected_as = |path: &[Tok<String>]| {
     let (item, modpath) = path.split_last()?;
@@ -40,7 +40,7 @@ pub fn parse_layer<'a>(
   let tree = project_tree::build_tree(source, i, prelude, &injected_names)?;
   let sum = ProjectTree(environment.0.clone().overlay(tree.0.clone()));
   let resolvd =
-    import_resolution::resolve_imports(sum, i, &injected_as, &|path| {
+    import_resolution::resolve_imports(sum, &injected_as, &|path| {
       tree.0.walk_ref(path, false).is_ok()
     })?;
   // Addition among modules favours the left hand side.

src/pipeline/project_tree/add_prelude.rs

@@ -1,5 +1,5 @@
 use crate::interner::Tok;
-use crate::representations::sourcefile::{FileEntry, Member, Namespace};
+use crate::representations::sourcefile::{FileEntry, Member, ModuleBlock};
 
 fn member_rec(
   // object
@@ -9,9 +9,9 @@ fn member_rec(
   prelude: &[FileEntry],
 ) -> Member {
   match member {
-    Member::Namespace(Namespace { name, body }) => {
+    Member::Module(ModuleBlock { name, body }) => {
       let new_body = entv_rec(body, path, prelude);
-      Member::Namespace(Namespace { name, body: new_body })
+      Member::Module(ModuleBlock { name, body: new_body })
     },
     any => any,
   }

src/pipeline/project_tree/build_tree.rs

@@ -1,19 +1,18 @@
-use std::rc::Rc;
-
 use hashbrown::HashMap;
 use itertools::Itertools;
 
 use super::collect_ops;
 use super::collect_ops::InjectedOperatorsFn;
 use super::parse_file::parse_file;
-use crate::ast::{Constant, Expr};
+use crate::ast::{Clause, Constant, Expr};
+use crate::error::{ProjectError, ProjectResult, TooManySupers};
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::{ProjectError, TooManySupers};
 use crate::pipeline::source_loader::{LoadedSource, LoadedSourceTable};
 use crate::representations::project::{ProjectExt, ProjectTree};
 use crate::representations::sourcefile::{absolute_path, FileEntry, Member};
 use crate::representations::tree::{ModEntry, ModMember, Module};
 use crate::representations::{NameLike, VName};
+use crate::tree::{WalkError, WalkErrorKind};
 use crate::utils::iter::{box_empty, box_once};
 use crate::utils::{pushed, unwrap_or, Substack};
 
@@ -26,25 +25,27 @@ struct ParsedSource<'a> {
 
 /// Split a path into file- and subpath in knowledge
 ///
-/// # Panics
+/// # Errors
 ///
 /// if the path is invalid
+#[allow(clippy::type_complexity)] // bit too sensitive here IMO
 pub fn split_path<'a>(
   path: &'a [Tok<String>],
   proj: &'a ProjectTree<impl NameLike>,
-) -> (&'a [Tok<String>], &'a [Tok<String>]) {
+) -> Result<(&'a [Tok<String>], &'a [Tok<String>]), WalkError> {
   let (end, body) = unwrap_or!(path.split_last(); {
-    return (&[], &[])
+    return Ok((&[], &[]))
   });
-  let mut module = (proj.0.walk_ref(body, false))
+  let mut module = (proj.0.walk_ref(body, false))?;
-    .expect("invalid path can't have been split above");
+  let entry = (module.items.get(end))
-  if let ModMember::Sub(m) = &module.items[end].member {
+    .ok_or(WalkError { pos: path.len() - 1, kind: WalkErrorKind::Missing })?;
+  if let ModMember::Sub(m) = &entry.member {
     module = m;
   }
   let file =
     module.extra.file.as_ref().map(|s| &path[..s.len()]).unwrap_or(path);
   let subpath = &path[file.len()..];
-  (file, subpath)
+  Ok((file, subpath))
 }
 
 /// Convert normalized, prefixed source into a module
@@ -63,7 +64,7 @@ fn source_to_module(
   // context
   i: &Interner,
   filepath_len: usize,
-) -> Result<Module<Expr<VName>, ProjectExt<VName>>, Rc<dyn ProjectError>> {
+) -> ProjectResult<Module<Expr<VName>, ProjectExt<VName>>> {
   let path_v = path.iter().rev_vec_clone();
   let imports = (data.iter())
     .filter_map(|ent| {
@@ -73,16 +74,16 @@ fn source_to_module(
     .cloned()
     .collect::<Vec<_>>();
   let imports_from = (imports.iter())
-    .map(|imp| -> Result<_, Rc<dyn ProjectError>> {
+    .map(|imp| -> ProjectResult<_> {
       let mut imp_path_v = i.r(imp.path).clone();
       imp_path_v.push(imp.name.expect("glob imports had just been resolved"));
       let mut abs_path = absolute_path(&path_v, &imp_path_v, i)
         .expect("should have failed in preparsing");
       let name = abs_path.pop().ok_or_else(|| {
         TooManySupers {
-          offender_file: i.extern_all(&path_v[..filepath_len]),
+          offender_file: path_v[..filepath_len].to_vec(),
-          offender_mod: i.extern_all(&path_v[filepath_len..]),
+          offender_mod: path_v[filepath_len..].to_vec(),
-          path: i.extern_all(&imp_path_v),
+          path: imp_path_v,
         }
         .rc()
       })?;
@@ -96,15 +97,18 @@ fn source_to_module(
         FileEntry::Export(names) => Box::new(names.iter().copied().map(mk_ent)),
         FileEntry::Exported(mem) => match mem {
           Member::Constant(constant) => box_once(mk_ent(constant.name)),
-          Member::Namespace(ns) => box_once(mk_ent(ns.name)),
+          Member::Module(ns) => box_once(mk_ent(ns.name)),
           Member::Rule(rule) => {
             let mut names = Vec::new();
             for e in rule.pattern.iter() {
-              e.visit_names(Substack::Bottom, &mut |n| {
+              e.search_all(&mut |e| {
+                if let Clause::Name(n) = &e.value {
                   if let Some([name]) = n.strip_prefix(&path_v[..]) {
                     names.push((*name, n.clone()))
                   }
-              })
+                }
+                None::<()>
+              });
             }
             Box::new(names.into_iter())
           },
@@ -124,7 +128,7 @@ fn source_to_module(
   let items = (data.into_iter())
     .filter_map(|ent| {
       let member_to_item = |exported, member| match member {
-        Member::Namespace(ns) => {
+        Member::Module(ns) => {
           let new_prep = unwrap_or!(
             &preparsed.items[&ns.name].member => ModMember::Sub;
             panic!("Preparsed should include entries for all submodules")
@@ -171,7 +175,7 @@ fn files_to_module(
   path: Substack<Tok<String>>,
   files: Vec<ParsedSource>,
   i: &Interner,
-) -> Result<Module<Expr<VName>, ProjectExt<VName>>, Rc<dyn ProjectError>> {
+) -> ProjectResult<Module<Expr<VName>, ProjectExt<VName>>> {
   let lvl = path.len();
   debug_assert!(
     files.iter().map(|f| f.path.len()).max().unwrap() >= lvl,
@@ -190,7 +194,7 @@ fn files_to_module(
   let items = (files.into_iter())
     .group_by(|f| f.path[lvl])
     .into_iter()
-    .map(|(namespace, files)| -> Result<_, Rc<dyn ProjectError>> {
+    .map(|(namespace, files)| -> ProjectResult<_> {
       let subpath = path.push(namespace);
       let files_v = files.collect::<Vec<_>>();
       let module = files_to_module(subpath, files_v, i)?;
@@ -217,7 +221,7 @@ pub fn build_tree(
   i: &Interner,
   prelude: &[FileEntry],
   injected: &impl InjectedOperatorsFn,
-) -> Result<ProjectTree<VName>, Rc<dyn ProjectError>> {
+) -> ProjectResult<ProjectTree<VName>> {
   assert!(!files.is_empty(), "A tree requires at least one module");
   let ops_cache = collect_ops::mk_cache(&files, i, injected);
   let mut entries = files
@@ -225,7 +229,7 @@ pub fn build_tree(
     .map(|(path, loaded)| {
       Ok((path, loaded, parse_file(path, &files, &ops_cache, i, prelude)?))
     })
-    .collect::<Result<Vec<_>, Rc<dyn ProjectError>>>()?;
+    .collect::<ProjectResult<Vec<_>>>()?;
   // sort by similarity, then longest-first
   entries.sort_unstable_by(|a, b| a.0.cmp(b.0).reverse());
   let files = entries

src/pipeline/project_tree/collect_ops/exported_ops.rs

@@ -3,14 +3,14 @@ use std::rc::Rc;
 use hashbrown::HashSet;
 use trait_set::trait_set;
 
+use crate::error::{NotFound, ProjectError, ProjectResult};
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::{NotFound, ProjectError};
 use crate::pipeline::source_loader::LoadedSourceTable;
 use crate::representations::tree::WalkErrorKind;
 use crate::utils::{split_max_prefix, unwrap_or, Cache};
 use crate::Sym;
 
-pub type OpsResult = Result<Rc<HashSet<Tok<String>>>, Rc<dyn ProjectError>>;
+pub type OpsResult = ProjectResult<Rc<HashSet<Tok<String>>>>;
 pub type ExportedOpsCache<'a> = Cache<'a, Sym, OpsResult>;
 
 trait_set! {
@@ -54,12 +54,9 @@ pub fn collect_exported_ops(
           unreachable!("visibility is not being checked here")
         },
         WalkErrorKind::Missing => NotFound {
-          file: i.extern_all(fpath),
+          source: None,
-          subpath: (subpath.iter())
+          file: fpath.to_vec(),
-            .take(walk_err.pos)
+          subpath: subpath[..walk_err.pos].to_vec(),
-            .map(|t| i.r(*t))
-            .cloned()
-            .collect(),
         }
         .rc(),
       },

src/pipeline/project_tree/collect_ops/ops_for.rs

@@ -3,9 +3,9 @@ use std::rc::Rc;
 use hashbrown::HashSet;
 
 use super::exported_ops::{ExportedOpsCache, OpsResult};
+use crate::error::ProjectResult;
 use crate::interner::{Interner, Tok};
 use crate::parse::is_op;
-use crate::pipeline::error::ProjectError;
 use crate::pipeline::import_abs_path::import_abs_path;
 use crate::pipeline::source_loader::LoadedSourceTable;
 use crate::representations::tree::{ModMember, Module};
@@ -39,16 +39,17 @@ pub fn collect_ops_for(
   let tree = &loaded[file].preparsed.0;
   let mut ret = HashSet::new();
   tree_all_ops(tree, &mut ret);
-  tree.visit_all_imports(&mut |modpath, _module, import| {
+  tree.visit_all_imports(&mut |modpath, _m, import| -> ProjectResult<()> {
     if let Some(n) = import.name {
       ret.insert(n);
     } else {
-      let path = import_abs_path(file, modpath, &i.r(import.path)[..], i)
+      let path = i.expect(
-        .expect("This error should have been caught during loading");
+        import_abs_path(file, modpath, &i.r(import.path)[..], i),
+        "This error should have been caught during loading",
+      );
       ret.extend(ops_cache.find(&i.i(&path))?.iter().copied());
     }
-    Ok::<_, Rc<dyn ProjectError>>(())
+    Ok(())
   })?;
-  ret.drain_filter(|t| !is_op(i.r(*t)));
+  Ok(Rc::new(ret.into_iter().filter(|t| is_op(i.r(*t))).collect()))
-  Ok(Rc::new(ret))
 }

src/pipeline/project_tree/normalize_imports.rs

@@ -2,7 +2,7 @@ use super::collect_ops::ExportedOpsCache;
 use crate::interner::{Interner, Tok};
 use crate::pipeline::import_abs_path::import_abs_path;
 use crate::representations::sourcefile::{
-  FileEntry, Import, Member, Namespace,
+  FileEntry, Import, Member, ModuleBlock,
 };
 use crate::representations::tree::{ModMember, Module};
 use crate::utils::iter::box_once;
@@ -20,14 +20,14 @@ fn member_rec(
   i: &Interner,
 ) -> Member {
   match member {
-    Member::Namespace(Namespace { name, body }) => {
+    Member::Module(ModuleBlock { name, body }) => {
       let subprep = unwrap_or!(
         &preparsed.items[&name].member => ModMember::Sub;
         unreachable!("This name must point to a namespace")
       );
       let new_body =
         entv_rec(mod_stack.push(name), subprep, body, path, ops_cache, i);
-      Member::Namespace(Namespace { name, body: new_body })
+      Member::Module(ModuleBlock { name, body: new_body })
     },
     any => any,
   }
@@ -58,10 +58,14 @@ fn entv_rec(
           .into_iter()
           .flat_map(|import| {
             if let Import { name: None, path } = import {
-              let p = import_abs_path(mod_path, mod_stack, &i.r(path)[..], i)
+              let p = i.expect(
-                .expect("Should have emerged in preparsing");
+                import_abs_path(mod_path, mod_stack, &i.r(path)[..], i),
-              let names = (ops_cache.find(&i.i(&p)))
+                "Should have emerged in preparsing",
-                .expect("Should have emerged in second parsing");
+              );
+              let names = i.expect(
+                ops_cache.find(&i.i(&p)),
+                "Should have emerged in second parsing",
+              );
               let imports = (names.iter())
                 .map(move |&n| Import { name: Some(n), path })
                 .collect::<Vec<_>>();

src/pipeline/project_tree/parse_file.rs

@@ -4,9 +4,9 @@ use super::add_prelude::add_prelude;
 use super::collect_ops::{collect_ops_for, ExportedOpsCache};
 use super::normalize_imports::normalize_imports;
 use super::prefix::prefix;
+use crate::error::ProjectResult;
 use crate::interner::{Interner, Tok};
 use crate::parse;
-use crate::pipeline::error::ProjectError;
 use crate::pipeline::source_loader::LoadedSourceTable;
 use crate::representations::sourcefile::{normalize_namespaces, FileEntry};
 
@@ -24,7 +24,7 @@ pub fn parse_file(
   ops_cache: &ExportedOpsCache,
   i: &Interner,
   prelude: &[FileEntry],
-) -> Result<Vec<FileEntry>, Rc<dyn ProjectError>> {
+) -> ProjectResult<Vec<FileEntry>> {
   let ld = &loaded[path];
   // let ops_cache = collect_ops::mk_cache(loaded, i);
   let ops = collect_ops_for(path, loaded, ops_cache, i)?;
@@ -34,8 +34,10 @@ pub fn parse_file(
     ops: &ops_vec,
     file: Rc::new(i.extern_all(path)),
   };
-  let entries = parse::parse(ld.text.as_str(), ctx)
+  let entries = i.expect(
-    .expect("This error should have been caught during loading");
+    parse::parse2(ld.text.as_str(), ctx),
+    "This error should have been caught during loading",
+  );
   let with_prelude = add_prelude(entries, path, prelude);
   let impnormalized =
     normalize_imports(&ld.preparsed.0, with_prelude, path, ops_cache, i);

src/pipeline/project_tree/prefix.rs

@@ -1,7 +1,7 @@
 use super::collect_ops::ExportedOpsCache;
 use crate::ast::{Constant, Rule};
 use crate::interner::{Interner, Tok};
-use crate::representations::sourcefile::{FileEntry, Member, Namespace};
+use crate::representations::sourcefile::{FileEntry, Member, ModuleBlock};
 use crate::utils::Substack;
 
 fn member_rec(
@@ -19,9 +19,9 @@ fn member_rec(
     .chain(mod_stack.iter().rev_vec_clone().into_iter())
     .collect::<Vec<_>>();
   match data {
-    Member::Namespace(Namespace { name, body }) => {
+    Member::Module(ModuleBlock { name, body }) => {
       let new_body = entv_rec(mod_stack.push(name), body, path, ops_cache, i);
-      Member::Namespace(Namespace { name, body: new_body })
+      Member::Module(ModuleBlock { name, body: new_body })
     },
     Member::Constant(constant) => Member::Constant(Constant {
       name: constant.name,

src/pipeline/source_loader/load_source.rs

@@ -1,10 +1,11 @@
 use std::iter;
-use std::rc::Rc;
 
 use super::loaded_source::{LoadedSource, LoadedSourceTable};
 use super::preparse::preparse;
+use crate::error::{
+  NoTargets, ProjectError, ProjectResult, UnexpectedDirectory,
+};
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::{ProjectError, UnexpectedDirectory};
 use crate::pipeline::file_loader::{IOResult, Loaded};
 use crate::pipeline::import_abs_path::import_abs_path;
 use crate::representations::sourcefile::FileEntry;
@@ -19,7 +20,7 @@ fn load_abs_path_rec(
   i: &Interner,
   get_source: &impl Fn(&[Tok<String>]) -> IOResult,
   is_injected_module: &impl Fn(&[Tok<String>]) -> bool,
-) -> Result<(), Rc<dyn ProjectError>> {
+) -> ProjectResult<()> {
   // # Termination
   //
   // Every recursion of this function either
@@ -40,7 +41,7 @@ fn load_abs_path_rec(
   if let Some((filename, _)) = name_split {
     // if the filename is valid, load, preparse and record this file
     let text = unwrap_or!(get_source(filename)? => Loaded::Code; {
-      return Err(UnexpectedDirectory { path: i.extern_all(filename) }.rc())
+      return Err(UnexpectedDirectory { path: filename.to_vec() }.rc())
     });
     let preparsed = preparse(
       filename.iter().map(|t| i.r(*t)).cloned().collect(),
@@ -56,6 +57,9 @@ fn load_abs_path_rec(
     preparsed.0.visit_all_imports(&mut |modpath, _module, import| {
       let abs_pathv =
         import_abs_path(filename, modpath, &import.nonglob_path(i), i)?;
+      if abs_path.starts_with(&abs_pathv) {
+        return Ok(());
+      }
       // recurse on imported module
       load_abs_path_rec(
         &abs_pathv,
@@ -111,9 +115,11 @@ pub fn load_source<'a>(
   i: &Interner,
   get_source: &impl Fn(&[Tok<String>]) -> IOResult,
   is_injected_module: &impl Fn(&[Tok<String>]) -> bool,
-) -> Result<LoadedSourceTable, Rc<dyn ProjectError>> {
+) -> ProjectResult<LoadedSourceTable> {
   let mut table = LoadedSourceTable::new();
+  let mut any_target = false;
   for target in targets {
+    any_target |= true;
     load_abs_path_rec(
       target,
       &mut table,
@@ -123,5 +129,5 @@ pub fn load_source<'a>(
       is_injected_module,
     )?
   }
-  Ok(table)
+  if any_target { Ok(table) } else { Err(NoTargets.rc()) }
 }

src/pipeline/source_loader/preparse.rs

@@ -4,11 +4,9 @@ use std::rc::Rc;
 use hashbrown::HashMap;
 
 use crate::ast::Constant;
+use crate::error::{ProjectError, ProjectResult, VisibilityMismatch};
 use crate::interner::Interner;
 use crate::parse::{self, ParsingContext};
-use crate::pipeline::error::{
-  ParseErrorWithPath, ProjectError, VisibilityMismatch,
-};
 use crate::representations::sourcefile::{
   imports, normalize_namespaces, FileEntry, Member,
 };
@@ -38,12 +36,12 @@ fn to_module(src: &[FileEntry], prelude: &[FileEntry]) -> Module<(), ()> {
   let imports = imports(all_src()).cloned().collect::<Vec<_>>();
   let mut items = all_src()
     .filter_map(|ent| match ent {
-      FileEntry::Internal(Member::Namespace(ns)) => {
+      FileEntry::Internal(Member::Module(ns)) => {
         let member = ModMember::Sub(to_module(&ns.body, prelude));
         let entry = ModEntry { exported: false, member };
         Some((ns.name, entry))
       },
-      FileEntry::Exported(Member::Namespace(ns)) => {
+      FileEntry::Exported(Member::Module(ns)) => {
         let member = ModMember::Sub(to_module(&ns.body, prelude));
         let entry = ModEntry { exported: true, member };
         Some((ns.name, entry))
@@ -55,8 +53,8 @@ fn to_module(src: &[FileEntry], prelude: &[FileEntry]) -> Module<(), ()> {
     match file_entry {
       FileEntry::Comment(_)
       | FileEntry::Import(_)
-      | FileEntry::Internal(Member::Namespace(_))
+      | FileEntry::Internal(Member::Module(_))
-      | FileEntry::Exported(Member::Namespace(_)) => (),
+      | FileEntry::Exported(Member::Module(_)) => (),
       FileEntry::Export(tokv) =>
         for tok in tokv {
           add_export(&mut items, *tok)
@@ -89,24 +87,13 @@ pub fn preparse(
   source: &str,
   prelude: &[FileEntry],
   i: &Interner,
-) -> Result<Preparsed, Rc<dyn ProjectError>> {
+) -> ProjectResult<Preparsed> {
   // Parse with no operators
   let ctx = ParsingContext::<&str>::new(&[], i, Rc::new(file.clone()));
-  let entries = parse::parse(source, ctx).map_err(|error| {
+  let entries = parse::parse2(source, ctx)?;
-    ParseErrorWithPath {
-      full_source: source.to_string(),
-      error,
-      path: file.clone(),
-    }
-    .rc()
-  })?;
   let normalized = normalize_namespaces(Box::new(entries.into_iter()))
-    .map_err(|ns| {
+    .map_err(|namespace| {
-      VisibilityMismatch {
+      VisibilityMismatch { namespace, file: Rc::new(file.clone()) }.rc()
-        namespace: ns.into_iter().map(|t| i.r(t)).cloned().collect(),
-        file: Rc::new(file.clone()),
-      }
-      .rc()
     })?;
   Ok(Preparsed(to_module(&normalized, prelude)))
 }

src/representations/ast.rs

@@ -6,14 +6,17 @@
 use std::hash::Hash;
 use std::rc::Rc;
 
+use hashbrown::HashSet;
 use itertools::Itertools;
 use ordered_float::NotNan;
 
+#[allow(unused)] // for doc
+use super::interpreted;
 use super::location::Location;
 use super::namelike::{NameLike, VName};
 use super::primitive::Primitive;
 use crate::interner::{InternedDisplay, Interner, Tok};
-use crate::utils::{map_rc, Substack};
+use crate::utils::map_rc;
 
 /// A [Clause] with associated metadata
 #[derive(Clone, Debug, PartialEq)]
@@ -25,17 +28,6 @@ pub struct Expr<N: NameLike> {
 }
 
 impl<N: NameLike> Expr<N> {
-  /// Obtain the contained clause
-  pub fn into_clause(self) -> Clause<N> {
-    self.value
-  }
-
-  /// Call the function on every name in this expression
-  pub fn visit_names(&self, binds: Substack<&N>, cb: &mut impl FnMut(&N)) {
-    let Expr { value, .. } = self;
-    value.visit_names(binds, cb);
-  }
-
   /// Process all names with the given mapper.
   /// Return a new object if anything was processed
   pub fn map_names(&self, pred: &impl Fn(&N) -> Option<N>) -> Option<Self> {
@@ -49,6 +41,32 @@ impl<N: NameLike> Expr<N> {
   pub fn transform_names<O: NameLike>(self, pred: &impl Fn(N) -> O) -> Expr<O> {
     Expr { value: self.value.transform_names(pred), location: self.location }
   }
+
+  /// Visit all expressions in the tree. The search can be exited early by
+  /// returning [Some]
+  ///
+  /// See also [interpreted::ExprInst::search_all]
+  pub fn search_all<T>(
+    &self,
+    f: &mut impl FnMut(&Self) -> Option<T>,
+  ) -> Option<T> {
+    f(self).or_else(|| self.value.search_all(f))
+  }
+}
+
+impl<N: NameLike> AsRef<Location> for Expr<N> {
+  fn as_ref(&self) -> &Location {
+    &self.location
+  }
+}
+
+/// Visit all expression sequences including this sequence itself. Otherwise
+/// works exactly like [Expr::search_all_slcs]
+pub fn search_all_slcs<N: NameLike, T>(
+  this: &[Expr<N>],
+  f: &mut impl FnMut(&[Expr<N>]) -> Option<T>,
+) -> Option<T> {
+  f(this).or_else(|| this.iter().find_map(|expr| expr.value.search_all_slcs(f)))
 }
 
 impl Expr<VName> {
@@ -130,7 +148,7 @@ pub enum Clause<N: NameLike> {
   /// eg. `(print out "hello")`, `[1, 2, 3]`, `{Some(t) => t}`
   S(char, Rc<Vec<Expr<N>>>),
   /// A function expression, eg. `\x. x + 1`
-  Lambda(Rc<Expr<N>>, Rc<Vec<Expr<N>>>),
+  Lambda(Rc<Vec<Expr<N>>>, Rc<Vec<Expr<N>>>),
   /// A placeholder for macros, eg. `$name`, `...$body`, `...$lhs:1`
   Placeh(Placeholder),
 }
@@ -162,7 +180,7 @@ impl<N: NameLike> Clause<N> {
     if exprs.is_empty() {
       None
     } else if exprs.len() == 1 {
-      Some(exprs[0].clone().into_clause())
+      Some(exprs[0].value.clone())
     } else {
       Some(Self::S('(', Rc::new(exprs.to_vec())))
     }
@@ -176,30 +194,22 @@ impl<N: NameLike> Clause<N> {
     }
   }
 
-  /// Recursively iterate through all "names" in an expression.
+  /// Collect all names that appear in this expression.
-  /// It also finds a lot of things that aren't names, such as all
+  /// NOTICE: this isn't the total set of unbound names, it's mostly useful to
-  /// bound parameters. Generally speaking, this is not a very
+  /// make weak statements for optimization.
-  /// sophisticated search.
+  pub fn collect_names(&self) -> HashSet<N> {
-  pub fn visit_names(&self, binds: Substack<&N>, cb: &mut impl FnMut(&N)) {
+    if let Self::Name(n) = self {
-    match self {
+      return HashSet::from([n.clone()]);
-      Clause::Lambda(arg, body) => {
-        arg.visit_names(binds, cb);
-        let new_binds =
-          if let Clause::Name(n) = &arg.value { binds.push(n) } else { binds };
-        for x in body.iter() {
-          x.visit_names(new_binds, cb)
     }
-      },
+    let mut glossary = HashSet::new();
-      Clause::S(_, body) =>
+    let result = self.search_all(&mut |e| {
-        for x in body.iter() {
+      if let Clause::Name(n) = &e.value {
-          x.visit_names(binds, cb)
+        glossary.insert(n.clone());
-        },
-      Clause::Name(name) =>
-        if binds.iter().all(|x| x != &name) {
-          cb(name)
-        },
-      _ => (),
       }
+      None::<()>
+    });
+    assert!(result.is_none(), "Callback never returns Some, wtf???");
+    glossary
   }
 
   /// Process all names with the given mapper.
@@ -221,10 +231,15 @@ impl<N: NameLike> Clause<N> {
         if any_some { Some(Clause::S(*c, Rc::new(new_body))) } else { None }
       },
       Clause::Lambda(arg, body) => {
-        let new_arg = arg.map_names(pred);
+        let mut any_some = false;
-        let mut any_some = new_arg.is_some();
+        let new_arg = (arg.iter())
-        let new_body = body
+          .map(|e| {
-          .iter()
+            let val = e.map_names(pred);
+            any_some |= val.is_some();
+            val.unwrap_or_else(|| e.clone())
+          })
+          .collect();
+        let new_body = (body.iter())
           .map(|e| {
             let val = e.map_names(pred);
             any_some |= val.is_some();
@@ -232,10 +247,7 @@ impl<N: NameLike> Clause<N> {
           })
           .collect();
         if any_some {
-          Some(Clause::Lambda(
+          Some(Clause::Lambda(Rc::new(new_arg), Rc::new(new_body)))
-            new_arg.map(Rc::new).unwrap_or_else(|| arg.clone()),
-            Rc::new(new_body),
-          ))
         } else {
           None
         }
@@ -253,7 +265,7 @@ impl<N: NameLike> Clause<N> {
       Self::Placeh(p) => Clause::Placeh(p),
       Self::P(p) => Clause::P(p),
       Self::Lambda(n, b) => Clause::Lambda(
-        map_rc(n, |n| n.transform_names(pred)),
+        map_rc(n, |n| n.into_iter().map(|e| e.transform_names(pred)).collect()),
         map_rc(b, |b| b.into_iter().map(|e| e.transform_names(pred)).collect()),
       ),
       Self::S(c, b) => Clause::S(
@@ -262,6 +274,32 @@ impl<N: NameLike> Clause<N> {
       ),
     }
   }
+
+  /// Pair of [Expr::search_all]
+  pub fn search_all<T>(
+    &self,
+    f: &mut impl FnMut(&Expr<N>) -> Option<T>,
+  ) -> Option<T> {
+    match self {
+      Clause::Lambda(arg, body) =>
+        arg.iter().chain(body.iter()).find_map(|expr| expr.search_all(f)),
+      Clause::Name(_) | Clause::P(_) | Clause::Placeh(_) => None,
+      Clause::S(_, body) => body.iter().find_map(|expr| expr.search_all(f)),
+    }
+  }
+
+  /// Pair of [Expr::search_all_slcs]
+  pub fn search_all_slcs<T>(
+    &self,
+    f: &mut impl FnMut(&[Expr<N>]) -> Option<T>,
+  ) -> Option<T> {
+    match self {
+      Clause::Lambda(arg, body) =>
+        search_all_slcs(arg, f).or_else(|| search_all_slcs(body, f)),
+      Clause::Name(_) | Clause::P(_) | Clause::Placeh(_) => None,
+      Clause::S(_, body) => search_all_slcs(body, f),
+    }
+  }
 }
 
 impl Clause<VName> {
@@ -319,7 +357,7 @@ impl<N: NameLike> InternedDisplay for Clause<N> {
       },
       Self::Lambda(arg, body) => {
         f.write_str("\\")?;
-        arg.fmt_i(f, i)?;
+        fmt_expr_seq(&mut arg.iter(), f, i)?;
         f.write_str(".")?;
         fmt_expr_seq(&mut body.iter(), f, i)
       },
@@ -360,10 +398,13 @@ impl Rule<VName> {
   pub fn collect_single_names(&self) -> Vec<Tok<String>> {
     let mut names = Vec::new();
     for e in self.pattern.iter() {
-      e.visit_names(Substack::Bottom, &mut |ns_name| {
+      e.search_all(&mut |e| {
+        if let Clause::Name(ns_name) = &e.value {
           if ns_name.len() == 1 {
             names.push(ns_name[0])
           }
+        }
+        None::<()>
       });
     }
     names

src/representations/ast_to_postmacro.rs

@@ -1,13 +1,14 @@
-use std::fmt::Display;
 use std::rc::Rc;
 
 use super::location::Location;
 use super::{ast, postmacro};
-use crate::utils::Substack;
+use crate::error::{ErrorPosition, ProjectError};
-use crate::Sym;
+use crate::utils::iter::box_once;
+use crate::utils::{BoxedIter, Substack};
+use crate::{Interner, Sym};
 
-#[derive(Clone)]
+#[derive(Debug, Clone)]
-pub enum Error {
+pub enum ErrorKind {
   /// `()` as a clause is meaningless in lambda calculus
   EmptyS,
   /// Only `(...)` may be converted to typed lambdas. `[...]` and `{...}`
@@ -16,28 +17,43 @@ pub enum Error {
   /// Placeholders shouldn't even occur in the code during macro
   /// execution. Something is clearly terribly wrong
   Placeholder,
-  /// Arguments can only be [ast::Clause::Name]
+  /// Arguments can only be a single [ast::Clause::Name]
   InvalidArg,
 }
 
-impl Display for Error {
+#[derive(Debug, Clone)]
-  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+pub struct Error {
-    match self {
+  pub location: Location,
-      Error::EmptyS => {
+  pub kind: ErrorKind,
-        write!(f, "`()` as a clause is meaningless in lambda calculus")
-      },
-      Error::BadGroup(_) => write!(
-        f,
-        "Only `(...)` may be converted to typed lambdas. `[...]` and \
-         `{{...}}` left in the code are signs of incomplete macro execution"
-      ),
-      Error::Placeholder => write!(
-        f,
-        "Placeholders shouldn't even appear in the code during macro \
-         execution, this is likely a compiler bug"
-      ),
-      Error::InvalidArg => write!(f, "Arguments can only be Name nodes"),
 }
+impl Error {
+  pub fn new(kind: ErrorKind, location: &Location) -> Self {
+    Self { location: location.clone(), kind }
+  }
+}
+impl ProjectError for Error {
+  fn description(&self) -> &str {
+    match self.kind {
+      ErrorKind::BadGroup(_) =>
+        "Only `(...)` may be converted to typed lambdas. `[...]` and `{{...}}` \
+         left in the code are signs of incomplete macro execution",
+      ErrorKind::EmptyS => "`()` as a clause is meaningless in lambda calculus",
+      ErrorKind::InvalidArg => "Argument names can only be Name nodes",
+      ErrorKind::Placeholder =>
+        "Placeholders shouldn't even appear in the code during macro \
+         execution,this is likely a compiler bug",
+    }
+  }
+
+  fn message(&self, _i: &Interner) -> String {
+    match self.kind {
+      ErrorKind::BadGroup(char) => format!("{} block found in the code", char),
+      _ => self.description().to_string(),
+    }
+  }
+
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition { location: self.location.clone(), message: None })
   }
 }
 
@@ -66,14 +82,16 @@ impl<'a> Context<'a> {
 
 /// Process an expression sequence
 fn exprv_rec<'a>(
+  location: &'a Location,
   v: &'a [ast::Expr<Sym>],
   ctx: Context<'a>,
 ) -> Result<postmacro::Expr, Error> {
-  let (last, rest) = v.split_last().ok_or(Error::EmptyS)?;
+  let (last, rest) =
+    (v.split_last()).ok_or_else(|| Error::new(ErrorKind::EmptyS, location))?;
   if rest.is_empty() {
     return expr_rec(&v[0], ctx);
   }
-  let f = exprv_rec(rest, ctx)?;
+  let f = exprv_rec(location, rest, ctx)?;
   let x = expr_rec(last, ctx)?;
   let value = postmacro::Clause::Apply(Rc::new(f), Rc::new(x));
   Ok(postmacro::Expr { value, location: Location::Unknown })
@@ -86,52 +104,44 @@ fn expr_rec<'a>(
 ) -> Result<postmacro::Expr, Error> {
   if let ast::Clause::S(paren, body) = value {
     if *paren != '(' {
-      return Err(Error::BadGroup(*paren));
+      return Err(Error::new(ErrorKind::BadGroup(*paren), location));
     }
-    let expr = exprv_rec(body.as_ref(), ctx)?;
+    let expr = exprv_rec(location, body.as_ref(), ctx)?;
     Ok(postmacro::Expr { value: expr.value, location: location.clone() })
   } else {
-    let value = clause_rec(value, ctx)?;
+    let value = match value {
-    Ok(postmacro::Expr { value, location: location.clone() })
+      ast::Clause::P(p) => postmacro::Clause::P(p.clone()),
-  }
+      ast::Clause::Lambda(arg, b) => {
-}
+        let name = match &arg[..] {
-
+          [ast::Expr { value: ast::Clause::Name(name), .. }] => name,
-/// Process a clause
+          [ast::Expr { value: ast::Clause::Placeh { .. }, .. }] =>
-fn clause_rec<'a>(
+            return Err(Error::new(ErrorKind::Placeholder, location)),
-  cls: &'a ast::Clause<Sym>,
+          _ => return Err(Error::new(ErrorKind::InvalidArg, location)),
-  ctx: Context<'a>,
-) -> Result<postmacro::Clause, Error> {
-  match cls {
-    ast::Clause::P(p) => Ok(postmacro::Clause::P(p.clone())),
-    ast::Clause::Lambda(expr, b) => {
-      let name = match expr.value {
-        ast::Clause::Name(name) => name,
-        ast::Clause::Placeh { .. } => return Err(Error::Placeholder),
-        _ => return Err(Error::InvalidArg),
         };
-      let body_ctx = ctx.w_name(name);
+        let body_ctx = ctx.w_name(*name);
-      let body = exprv_rec(b.as_ref(), body_ctx)?;
+        let body = exprv_rec(location, b.as_ref(), body_ctx)?;
-      Ok(postmacro::Clause::Lambda(Rc::new(body)))
+        postmacro::Clause::Lambda(Rc::new(body))
       },
       ast::Clause::Name(name) => {
-      let lvl_opt = ctx
+        let lvl_opt = (ctx.names.iter())
-        .names
-        .iter()
           .enumerate()
           .find(|(_, n)| *n == name)
           .map(|(lvl, _)| lvl);
-      Ok(match lvl_opt {
+        match lvl_opt {
           Some(lvl) => postmacro::Clause::LambdaArg(lvl),
           None => postmacro::Clause::Constant(*name),
-      })
+        }
       },
       ast::Clause::S(paren, entries) => {
         if *paren != '(' {
-        return Err(Error::BadGroup(*paren));
+          return Err(Error::new(ErrorKind::BadGroup(*paren), location));
         }
-      let expr = exprv_rec(entries.as_ref(), ctx)?;
+        let expr = exprv_rec(location, entries.as_ref(), ctx)?;
-      Ok(expr.value)
+        expr.value
       },
-    ast::Clause::Placeh { .. } => Err(Error::Placeholder),
+      ast::Clause::Placeh { .. } =>
+        return Err(Error::new(ErrorKind::Placeholder, location)),
+    };
+    Ok(postmacro::Expr { value, location: location.clone() })
   }
 }

src/representations/const_tree.rs

@@ -14,6 +14,7 @@ use crate::utils::{pushed, Substack};
 /// A lightweight module tree that can be built declaratively by hand to
 /// describe libraries of external functions in Rust. It implements [Add] for
 /// added convenience
+#[derive(Clone, Debug)]
 pub enum ConstTree {
   /// A function or constant
   Const(Expr<VName>),
@@ -40,6 +41,29 @@ impl ConstTree {
   pub fn tree(arr: impl IntoIterator<Item = (Tok<String>, Self)>) -> Self {
     Self::Tree(arr.into_iter().collect())
   }
+  /// Namespace the tree with the list of names
+  pub fn namespace(
+    pref: impl IntoIterator<Item = Tok<String>>,
+    data: Self,
+  ) -> Self {
+    let mut iter = pref.into_iter();
+    if let Some(ns) = iter.next() {
+      Self::tree([(ns, Self::namespace(iter, data))])
+    } else {
+      data
+    }
+  }
+  /// Unwrap the table of subtrees from a tree
+  ///
+  /// # Panics
+  ///
+  /// If this is a leaf node aka. constant and not a namespace
+  pub fn unwrap_tree(self) -> HashMap<Tok<String>, Self> {
+    match self {
+      Self::Tree(map) => map,
+      _ => panic!("Attempted to unwrap leaf as tree"),
+    }
+  }
 }
 impl Add for ConstTree {
   type Output = ConstTree;

src/representations/interpreted.rs

@@ -7,6 +7,8 @@ use std::fmt::Debug;
 use std::ops::{Deref, DerefMut};
 use std::rc::Rc;
 
+#[allow(unused)] // for doc
+use super::ast;
 use super::location::Location;
 use super::path_set::PathSet;
 use super::primitive::Primitive;
@@ -83,9 +85,11 @@ impl ExprInst {
   /// across the tree.
   pub fn try_normalize<T, E>(
     &self,
-    mapper: impl FnOnce(&Clause) -> Result<(Clause, T), E>,
+    mapper: impl FnOnce(&Clause, &Location) -> Result<(Clause, T), E>,
   ) -> Result<(Self, T), E> {
-    let (new_clause, extra) = mapper(&self.expr().clause)?;
+    let expr = self.expr();
+    let (new_clause, extra) = mapper(&expr.clause, &expr.location)?;
+    drop(expr);
     self.expr_mut().clause = new_clause;
     Ok((self.clone(), extra))
   }
@@ -95,10 +99,10 @@ impl ExprInst {
   /// the original but is normalized independently.
   pub fn try_update<T, E>(
     &self,
-    mapper: impl FnOnce(&Clause) -> Result<(Clause, T), E>,
+    mapper: impl FnOnce(&Clause, &Location) -> Result<(Clause, T), E>,
   ) -> Result<(Self, T), E> {
     let expr = self.expr();
-    let (clause, extra) = mapper(&expr.clause)?;
+    let (clause, extra) = mapper(&expr.clause, &expr.location)?;
     let new_expr = Expr { clause, location: expr.location.clone() };
     Ok((Self(Rc::new(RefCell::new(new_expr))), extra))
   }
@@ -123,6 +127,25 @@ impl ExprInst {
       Err(NotALiteral)
     }
   }
+
+  /// Visit all expressions in the tree. The search can be exited early by
+  /// returning [Some]
+  ///
+  /// See also [ast::Expr::search_all]
+  pub fn search_all<T>(
+    &self,
+    predicate: &mut impl FnMut(&Self) -> Option<T>,
+  ) -> Option<T> {
+    if let Some(t) = predicate(self) {
+      return Some(t);
+    }
+    self.inspect(|c| match c {
+      Clause::Apply { f, x } =>
+        f.search_all(predicate).or_else(|| x.search_all(predicate)),
+      Clause::Lambda { body, .. } => body.search_all(predicate),
+      Clause::Constant(_) | Clause::LambdaArg | Clause::P(_) => None,
+    })
+  }
 }
 
 impl Debug for ExprInst {

src/representations/location.rs

@@ -39,6 +39,36 @@ impl Location {
       None
     }
   }
+
+  /// If the two locations are ranges in the same file, connect them.
+  /// Otherwise choose the more accurate, preferring lhs if equal.
+  pub fn to(self, other: Self) -> Self {
+    match self {
+      Location::Unknown => other,
+      Location::File(f) => match other {
+        Location::Range { .. } => other,
+        _ => Location::File(f),
+      },
+      Location::Range { file: f1, range: r1 } => Location::Range {
+        range: match other {
+          Location::Range { file: f2, range: r2 } if f1 == f2 =>
+            r1.start..r2.end,
+          _ => r1,
+        },
+        file: f1,
+      },
+    }
+  }
+
+  /// Choose one of the two locations, preferring better accuracy, or lhs if
+  /// equal
+  pub fn or(self, alt: Self) -> Self {
+    match (&self, &alt) {
+      (Self::Unknown, _) => alt,
+      (Self::File(_), Self::Range { .. }) => alt,
+      _ => self,
+    }
+  }
 }
 
 impl Display for Location {

src/representations/namelike.rs

@@ -1,3 +1,5 @@
+use std::hash::Hash;
+
 use crate::interner::{Interner, Tok};
 
 /// A mutable representation of a namespaced identifier.
@@ -16,7 +18,7 @@ pub type Sym = Tok<Vec<Tok<String>>>;
 
 /// An abstraction over tokenized vs non-tokenized names so that they can be
 /// handled together in datastructures
-pub trait NameLike: 'static + Clone + Eq {
+pub trait NameLike: 'static + Clone + Eq + Hash {
   /// Fully resolve the name for printing
   fn to_strv(&self, i: &Interner) -> Vec<String>;
 }

src/representations/sourcefile.rs

@@ -40,7 +40,7 @@ impl Import {
 
 /// A namespace block
 #[derive(Debug, Clone)]
-pub struct Namespace {
+pub struct ModuleBlock {
   /// Name prefixed to all names in the block
   pub name: Tok<String>,
   /// Prefixed entries
@@ -56,7 +56,7 @@ pub enum Member {
   /// A constant (or function) associated with a name
   Constant(Constant),
   /// A prefixed set of other entries
-  Namespace(Namespace),
+  Module(ModuleBlock),
 }
 
 /// Anything we might encounter in a file
@@ -94,8 +94,8 @@ pub fn normalize_namespaces(
 ) -> Result<Vec<FileEntry>, Vec<Tok<String>>> {
   let (mut namespaces, mut rest) = src
     .partition_map::<Vec<_>, Vec<_>, _, _, _>(|ent| match ent {
-      FileEntry::Exported(Member::Namespace(ns)) => Either::Left((true, ns)),
+      FileEntry::Exported(Member::Module(ns)) => Either::Left((true, ns)),
-      FileEntry::Internal(Member::Namespace(ns)) => Either::Left((false, ns)),
+      FileEntry::Internal(Member::Module(ns)) => Either::Left((false, ns)),
       other => Either::Right(other),
     });
   // Combine namespace blocks with the same name
@@ -123,7 +123,7 @@ pub fn normalize_namespaces(
         e.push(name);
         e
       })?;
-      let member = Member::Namespace(Namespace { name, body });
+      let member = Member::Module(ModuleBlock { name, body });
       match (any_exported, any_internal) {
         (true, true) => Err(vec![name]),
         (true, false) => Ok(FileEntry::Exported(member)),

src/rule/matcher_vectree/build.rs

@@ -112,7 +112,7 @@ fn mk_scalar(pattern: &RuleExpr) -> ScalMatcher {
     },
     Clause::S(c, body) => ScalMatcher::S(*c, Box::new(mk_any(body))),
     Clause::Lambda(arg, body) =>
-      ScalMatcher::Lambda(Box::new(mk_scalar(arg)), Box::new(mk_any(body))),
+      ScalMatcher::Lambda(Box::new(mk_any(arg)), Box::new(mk_any(body))),
   }
 }
 

src/rule/matcher_vectree/scal_match.rs

@@ -16,7 +16,7 @@ pub fn scal_match<'a>(
     (ScalMatcher::S(c1, b_mat), Clause::S(c2, body)) if c1 == c2 =>
       any_match(b_mat, &body[..]),
     (ScalMatcher::Lambda(arg_mat, b_mat), Clause::Lambda(arg, body)) => {
-      let mut state = scal_match(arg_mat, arg)?;
+      let mut state = any_match(arg_mat, arg)?;
       state.extend(any_match(b_mat, body)?);
       Some(state)
     },

src/rule/matcher_vectree/shared.rs

@@ -14,7 +14,7 @@ pub enum ScalMatcher {
   P(Primitive),
   Name(Sym),
   S(char, Box<AnyMatcher>),
-  Lambda(Box<ScalMatcher>, Box<AnyMatcher>),
+  Lambda(Box<AnyMatcher>, Box<AnyMatcher>),
   Placeh(Tok<String>),
 }
 

src/rule/prepare_rule.rs

@@ -68,7 +68,7 @@ fn check_rec_expr(
         if !in_template {
           Err(RuleError::Multiple(*name))
         } else if known != typ {
-          Err(RuleError::TypeMismatch(*name))
+          Err(RuleError::ArityMismatch(*name))
         } else {
           Ok(())
         }
@@ -79,7 +79,7 @@ fn check_rec_expr(
       }
     },
     Clause::Lambda(arg, body) => {
-      check_rec_expr(arg.as_ref(), types, in_template)?;
+      check_rec_exprv(arg, types, in_template)?;
       check_rec_exprv(body, types, in_template)
     },
     Clause::S(_, body) => check_rec_exprv(body, types, in_template),

src/rule/repository.rs

@@ -3,6 +3,7 @@ use std::format;
 use std::rc::Rc;
 
 use hashbrown::HashSet;
+use itertools::Itertools;
 use ordered_float::NotNan;
 
 use super::matcher::{Matcher, RuleExpr};
@@ -11,7 +12,6 @@ use super::state::apply_exprv;
 use super::{update_first_seq, RuleError, VectreeMatcher};
 use crate::ast::Rule;
 use crate::interner::{InternedDisplay, Interner};
-use crate::utils::Substack;
 use crate::Sym;
 
 #[derive(Debug)]
@@ -60,9 +60,7 @@ impl<M: Matcher> Repository<M> {
         let rule = prepare_rule(r.clone(), i).map_err(|e| (r, e))?;
         let mut glossary = HashSet::new();
         for e in rule.pattern.iter() {
-          e.visit_names(Substack::Bottom, &mut |op| {
+          glossary.extend(e.value.collect_names().into_iter());
-            glossary.insert(*op);
-          })
         }
         let matcher = M::new(Rc::new(rule.pattern.clone()));
         let prep = CachedRule {
@@ -78,10 +76,7 @@ impl<M: Matcher> Repository<M> {
 
   /// Attempt to run each rule in priority order once
   pub fn step(&self, code: &RuleExpr) -> Option<RuleExpr> {
-    let mut glossary = HashSet::new();
+    let glossary = code.value.collect_names();
-    code.visit_names(Substack::Bottom, &mut |op| {
-      glossary.insert(*op);
-    });
     for (rule, deps, _) in self.cache.iter() {
       if !deps.is_subset(&glossary) {
         continue;
@@ -164,8 +159,13 @@ impl<M: InternedDisplay + Matcher> InternedDisplay for Repository<M> {
     i: &Interner,
   ) -> std::fmt::Result {
     writeln!(f, "Repository[")?;
-    for (item, _, p) in self.cache.iter() {
+    for (item, deps, p) in self.cache.iter() {
-      write!(f, "\t{}", fmt_hex(f64::from(*p)))?;
+      write!(
+        f,
+        "  priority: {}\tdependencies: [{}]\n    ",
+        fmt_hex(f64::from(*p)),
+        deps.iter().map(|t| i.extern_vec(*t).join("::")).join(", ")
+      )?;
       item.fmt_i(f, i)?;
       writeln!(f)?;
     }

src/rule/rule_error.rs

@@ -1,6 +1,13 @@
 use std::fmt;
+use std::rc::Rc;
 
+use hashbrown::HashSet;
+
+use crate::ast::{self, search_all_slcs, PHClass, Placeholder, Rule};
+use crate::error::{ErrorPosition, ProjectError};
 use crate::interner::{InternedDisplay, Interner, Tok};
+use crate::utils::BoxedIter;
+use crate::{Location, Sym};
 
 /// Various reasons why a substitution rule may be invalid
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -8,12 +15,23 @@ pub enum RuleError {
   /// A key is present in the template but not the pattern
   Missing(Tok<String>),
   /// A key uses a different arity in the template and in the pattern
-  TypeMismatch(Tok<String>),
+  ArityMismatch(Tok<String>),
   /// Multiple occurences of a placeholder in a pattern
   Multiple(Tok<String>),
   /// Two vectorial placeholders are next to each other
   VecNeighbors(Tok<String>, Tok<String>),
 }
+impl RuleError {
+  /// Convert into a unified error trait object shared by all Orchid errors
+  pub fn to_project_error(self, rule: &Rule<Sym>) -> Rc<dyn ProjectError> {
+    match self {
+      RuleError::Missing(name) => Missing::new(rule, name).rc(),
+      RuleError::Multiple(name) => Multiple::new(rule, name).rc(),
+      RuleError::ArityMismatch(name) => ArityMismatch::new(rule, name).rc(),
+      RuleError::VecNeighbors(n1, n2) => VecNeighbors::new(rule, n1, n2).rc(),
+    }
+  }
+}
 
 impl InternedDisplay for RuleError {
   fn fmt_i(&self, f: &mut fmt::Formatter<'_>, i: &Interner) -> fmt::Result {
@@ -21,7 +39,7 @@ impl InternedDisplay for RuleError {
       Self::Missing(key) => {
         write!(f, "Key {:?} not in match pattern", i.r(key))
       },
-      Self::TypeMismatch(key) => write!(
+      Self::ArityMismatch(key) => write!(
         f,
         "Key {:?} used inconsistently with and without ellipsis",
         i.r(key)
@@ -38,3 +56,181 @@ impl InternedDisplay for RuleError {
     }
   }
 }
+
+/// A key is present in the template but not the pattern of a rule
+#[derive(Debug)]
+pub struct Missing {
+  locations: HashSet<Location>,
+  name: Tok<String>,
+}
+impl Missing {
+  pub fn new(rule: &ast::Rule<Sym>, name: Tok<String>) -> Self {
+    let mut locations = HashSet::new();
+    for expr in rule.template.iter() {
+      expr.search_all(&mut |e| {
+        if let ast::Clause::Placeh(ph) = &e.value {
+          if ph.name == name {
+            locations.insert(e.location.clone());
+          }
+        }
+        None::<()>
+      });
+    }
+    Self { locations, name }
+  }
+}
+impl ProjectError for Missing {
+  fn description(&self) -> &str {
+    "A key appears in the template but not the pattern of a rule"
+  }
+  fn message(&self, i: &Interner) -> String {
+    format!(
+      "The key {} appears in the template but not the pattern of this rule",
+      i.r(self.name)
+    )
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    Box::new(
+      (self.locations.iter())
+        .cloned()
+        .map(|location| ErrorPosition { location, message: None }),
+    )
+  }
+}
+
+/// A key is present multiple times in the pattern of a rule
+#[derive(Debug)]
+pub struct Multiple {
+  locations: HashSet<Location>,
+  name: Tok<String>,
+}
+impl Multiple {
+  pub fn new(rule: &ast::Rule<Sym>, name: Tok<String>) -> Self {
+    let mut locations = HashSet::new();
+    for expr in rule.template.iter() {
+      expr.search_all(&mut |e| {
+        if let ast::Clause::Placeh(ph) = &e.value {
+          if ph.name == name {
+            locations.insert(e.location.clone());
+          }
+        }
+        None::<()>
+      });
+    }
+    Self { locations, name }
+  }
+}
+impl ProjectError for Multiple {
+  fn description(&self) -> &str {
+    "A key appears multiple times in the pattern of a rule"
+  }
+  fn message(&self, i: &Interner) -> String {
+    format!("The key {} appears multiple times in this pattern", i.r(self.name))
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    Box::new(
+      (self.locations.iter())
+        .cloned()
+        .map(|location| ErrorPosition { location, message: None }),
+    )
+  }
+}
+
+/// A key is present multiple times in the pattern of a rule
+#[derive(Debug)]
+pub struct ArityMismatch {
+  locations: HashSet<(Location, ast::PHClass)>,
+  name: Tok<String>,
+}
+impl ArityMismatch {
+  pub fn new(rule: &ast::Rule<Sym>, name: Tok<String>) -> Self {
+    let mut locations = HashSet::new();
+    for expr in rule.template.iter() {
+      expr.search_all(&mut |e| {
+        if let ast::Clause::Placeh(ph) = &e.value {
+          if ph.name == name {
+            locations.insert((e.location.clone(), ph.class));
+          }
+        }
+        None::<()>
+      });
+    }
+    Self { locations, name }
+  }
+}
+impl ProjectError for ArityMismatch {
+  fn description(&self) -> &str {
+    "A key appears with different arities in a rule"
+  }
+  fn message(&self, i: &Interner) -> String {
+    format!(
+      "The key {} appears multiple times with different arities in this rule",
+      i.r(self.name)
+    )
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    Box::new((self.locations.iter()).cloned().map(|(location, class)| {
+      ErrorPosition {
+        location,
+        message: Some(
+          "This instance represents ".to_string()
+            + match class {
+              ast::PHClass::Scalar => "one clause",
+              ast::PHClass::Vec { nonzero: true, .. } => "one or more clauses",
+              ast::PHClass::Vec { nonzero: false, .. } =>
+                "any number of clauses",
+            },
+        ),
+      }
+    }))
+  }
+}
+
+/// Two vectorial placeholders appear next to each other
+#[derive(Debug)]
+pub struct VecNeighbors {
+  locations: HashSet<Location>,
+  n1: Tok<String>,
+  n2: Tok<String>,
+}
+impl VecNeighbors {
+  pub fn new(rule: &ast::Rule<Sym>, n1: Tok<String>, n2: Tok<String>) -> Self {
+    let mut locations = HashSet::new();
+    search_all_slcs(&rule.template[..], &mut |ev| {
+      for pair in ev.windows(2) {
+        let (a, b) = (&pair[0], &pair[1]);
+        let a_vec = matches!(a.value, ast::Clause::Placeh(
+          Placeholder{ class: PHClass::Vec { .. }, name }
+        ) if name == n1);
+        let b_vec = matches!(b.value, ast::Clause::Placeh(
+          Placeholder{ class: PHClass::Vec { .. }, name }
+        ) if name == n2);
+        if a_vec && b_vec {
+          locations.insert(a.location.clone());
+          locations.insert(b.location.clone());
+        }
+      }
+      None::<()>
+    });
+    Self { locations, n1, n2 }
+  }
+}
+impl ProjectError for VecNeighbors {
+  fn description(&self) -> &str {
+    "Two vectorial placeholders appear next to each other"
+  }
+  fn message(&self, i: &Interner) -> String {
+    format!(
+      "The keys {} and {} appear next to each other with a vectorial arity",
+      i.r(self.n1),
+      i.r(self.n2)
+    )
+  }
+  fn positions(&self, _i: &Interner) -> BoxedIter<ErrorPosition> {
+    Box::new(
+      (self.locations.iter())
+        .cloned()
+        .map(|location| ErrorPosition { location, message: None }),
+    )
+  }
+}

src/rule/state.rs

@@ -1,7 +1,6 @@
 use std::rc::Rc;
 
 use hashbrown::HashMap;
-use itertools::Itertools;
 
 use super::matcher::RuleExpr;
 use crate::ast::{Clause, Expr, PHClass, Placeholder};
@@ -44,12 +43,7 @@ pub fn apply_expr(template: &RuleExpr, state: &State) -> Vec<RuleExpr> {
     Clause::Lambda(arg, body) => vec![Expr {
       location: location.clone(),
       value: Clause::Lambda(
-        Rc::new(
+        Rc::new(apply_exprv(arg, state)),
-          apply_expr(arg.as_ref(), state)
-            .into_iter()
-            .exactly_one()
-            .expect("Lambda arguments can only ever be scalar"),
-        ),
         Rc::new(apply_exprv(&body[..], state)),
       ),
     }],

src/rule/update_first_seq.rs

@@ -34,8 +34,8 @@ pub fn clause<F: FnMut(Rc<Vec<RuleExpr>>) -> Option<Rc<Vec<RuleExpr>>>>(
   match c {
     Clause::P(_) | Clause::Placeh { .. } | Clause::Name { .. } => None,
     Clause::Lambda(arg, body) =>
-      if let Some(arg) = expr(arg.as_ref(), pred) {
+      if let Some(arg) = exprv(arg.clone(), pred) {
-        Some(Clause::Lambda(Rc::new(arg), body.clone()))
+        Some(Clause::Lambda(arg, body.clone()))
       } else {
         exprv(body.clone(), pred).map(|body| Clause::Lambda(arg.clone(), body))
       },

src/stl/bool.orc

@@ -1,6 +0,0 @@
-export not := \bool. if bool then false else true
-export ...$a != ...$b =0x3p36=> (not (...$a == ...$b))
-export ...$a == ...$b =0x3p36=> (equals (...$a) (...$b))
-export if ...$cond then ...$true else ...$false:1 =0x1p84=> (
-  ifthenelse (...$cond) (...$true) (...$false)
-)

src/stl/fn.orc

@@ -1,26 +0,0 @@
-import super::known::*
-
---[ Do nothing. Especially useful as a passive cps operation ]--
-export identity := \x.x
---[
-  Apply the function to the given value. Can be used to assign a
-  concrete value in a cps assignment statement.
-]--
-export pass := \val.\cont. cont val
---[
-  Apply the function to the given pair of values. Mainly useful to assign
-  a concrete pair of values in a cps multi-assignment statement
-]--
-export pass2 := \a.\b.\cont. cont a b
---[
-  A function that returns the given value for any input. Also useful as a
-  "break" statement in a "do" block.
-]--
-export const := \a. \b.a
-
-export ...$prefix $ ...$suffix:1 =0x1p38=> ...$prefix (...$suffix)
-export ...$prefix |> $fn ..$suffix:1 =0x2p32=> $fn (...$prefix) ..$suffix
-
-export ($name) => ...$body =0x2p129=> (\$name. ...$body)
-export ($name, ...$argv) => ...$body =0x2p129=> (\$name. (...$argv) => ...$body)
-$name => ...$body =0x1p129=> (\$name. ...$body)

src/stl/io/command.rs

@@ -1,45 +0,0 @@
-use std::io::{self, Write};
-
-use super::super::runtime_error::RuntimeError;
-use crate::atomic_inert;
-use crate::interpreter::{HandlerParm, HandlerRes};
-use crate::representations::interpreted::{Clause, ExprInst};
-use crate::representations::{Literal, Primitive};
-use crate::utils::unwrap_or;
-
-
-
-/// An IO command to be handled by the host application.
-#[derive(Clone, Debug)]
-pub enum IO {
-  /// Print a string to standard output and resume execution
-  Print(String, ExprInst),
-  /// Read a line from standard input and pass it to the calback
-  Readline(ExprInst),
-}
-atomic_inert!(IO);
-
-/// Default xommand handler for IO actions
-pub fn handle(effect: HandlerParm) -> HandlerRes {
-  // Downcast command
-  let io: &IO = unwrap_or!(effect.as_any().downcast_ref(); Err(effect)?);
-  // Interpret and execute
-  Ok(match io {
-    IO::Print(str, cont) => {
-      print!("{}", str);
-      io::stdout()
-        .flush()
-        .map_err(|e| RuntimeError::ext(e.to_string(), "writing to stdout"))?;
-      cont.clone()
-    },
-    IO::Readline(cont) => {
-      let mut buf = String::new();
-      io::stdin()
-        .read_line(&mut buf)
-        .map_err(|e| RuntimeError::ext(e.to_string(), "reading from stdin"))?;
-      buf.pop();
-      let x = Clause::P(Primitive::Literal(Literal::Str(buf))).wrap();
-      Clause::Apply { f: cont.clone(), x }.wrap()
-    },
-  })
-}

src/stl/io/mod.rs

@@ -1,30 +0,0 @@
-use crate::interner::Interner;
-use crate::ConstTree;
-
-mod command;
-mod inspect;
-mod panic;
-mod print;
-mod readline;
-
-pub use command::{handle, IO};
-
-pub fn io(i: &Interner, allow_impure: bool) -> ConstTree {
-  let pure = ConstTree::tree([(
-    i.i("io"),
-    ConstTree::tree([
-      (i.i("print"), ConstTree::xfn(print::Print)),
-      (i.i("readline"), ConstTree::xfn(readline::ReadLn)),
-      (i.i("panic"), ConstTree::xfn(panic::Panic)),
-    ]),
-  )]);
-  if !allow_impure {
-    pure
-  } else {
-    pure
-      + ConstTree::tree([(
-        i.i("io"),
-        ConstTree::tree([(i.i("debug"), ConstTree::xfn(inspect::Inspect))]),
-      )])
-  }
-}

src/stl/io/print.rs

@@ -1,31 +0,0 @@
-use std::fmt::Debug;
-
-use super::super::inspect::with_str;
-use super::command::IO;
-use crate::foreign::{Atomic, AtomicResult, AtomicReturn};
-use crate::interpreter::Context;
-use crate::representations::interpreted::ExprInst;
-use crate::{atomic_defaults, write_fn_step};
-
-write_fn_step! {
-  /// Wrap a string and the continuation into an [IO] event to be evaluated by
-  /// the embedder.
-  pub Print > Print1
-}
-write_fn_step! {
-  Print1 {}
-  Print0 where message = x => with_str(x, |s| Ok(s.clone()));
-}
-
-#[derive(Debug, Clone)]
-struct Print0 {
-  message: String,
-  expr_inst: ExprInst,
-}
-impl Atomic for Print0 {
-  atomic_defaults!();
-  fn run(&self, ctx: Context) -> AtomicResult {
-    let command = IO::Print(self.message.clone(), self.expr_inst.clone());
-    Ok(AtomicReturn::from_data(command, ctx))
-  }
-}

src/stl/io/readline.rs

@@ -1,25 +0,0 @@
-use std::fmt::Debug;
-
-use super::command::IO;
-use crate::foreign::{Atomic, AtomicResult, AtomicReturn};
-use crate::interpreter::Context;
-use crate::representations::interpreted::ExprInst;
-use crate::{atomic_defaults, write_fn_step};
-
-write_fn_step! {
-  /// Create an [IO] event that reads a line form standard input and calls the
-  /// continuation with it.
-  pub ReadLn > ReadLn1
-}
-
-#[derive(Debug, Clone)]
-struct ReadLn1 {
-  expr_inst: ExprInst,
-}
-impl Atomic for ReadLn1 {
-  atomic_defaults!();
-  fn run(&self, ctx: Context) -> AtomicResult {
-    let command = IO::Readline(self.expr_inst.clone());
-    Ok(AtomicReturn::from_data(command, ctx))
-  }
-}

src/stl/known.orc

@@ -1 +0,0 @@
-export ::(,)

src/stl/mk_stl.rs

@@ -1,70 +0,0 @@
-use hashbrown::HashMap;
-use rust_embed::RustEmbed;
-
-use super::bin::bin;
-use super::bool::bool;
-use super::conv::conv;
-use super::io::io;
-use super::num::num;
-use super::str::str;
-use crate::interner::Interner;
-use crate::pipeline::file_loader::mk_embed_cache;
-use crate::pipeline::parse_layer;
-use crate::representations::VName;
-use crate::sourcefile::{FileEntry, Import};
-use crate::{from_const_tree, ProjectTree};
-
-/// Feature flags for the STL.
-#[derive(Default)]
-pub struct StlOptions {
-  /// Whether impure functions (such as io::debug) are allowed. An embedder
-  /// would typically disable this flag
-  pub impure: bool,
-}
-
-#[derive(RustEmbed)]
-#[folder = "src/stl"]
-#[prefix = "std/"]
-#[include = "*.orc"]
-struct StlEmbed;
-
-// TODO: fix all orc modules to not rely on prelude
-
-/// Build the standard library used by the interpreter by combining the other
-/// libraries
-pub fn mk_stl(i: &Interner, options: StlOptions) -> ProjectTree<VName> {
-  let const_tree = from_const_tree(
-    HashMap::from([(
-      i.i("std"),
-      io(i, options.impure) + conv(i) + bool(i) + str(i) + num(i) + bin(i),
-    )]),
-    &[i.i("std")],
-  );
-  let ld_cache = mk_embed_cache::<StlEmbed>(".orc", i);
-  let targets = StlEmbed::iter()
-    .map(|path| {
-      path
-        .strip_suffix(".orc")
-        .expect("the embed is filtered for suffix")
-        .split('/')
-        .map(|segment| i.i(segment))
-        .collect::<Vec<_>>()
-    })
-    .collect::<Vec<_>>();
-  parse_layer(
-    targets.iter().map(|v| &v[..]),
-    &|p| ld_cache.find(p),
-    &const_tree,
-    &[],
-    i,
-  )
-  .expect("Parse error in STL")
-}
-
-/// Generate prelude lines to be injected to every module compiled with the STL
-pub fn mk_prelude(i: &Interner) -> Vec<FileEntry> {
-  vec![FileEntry::Import(vec![Import {
-    path: i.i(&[i.i("std"), i.i("prelude")][..]),
-    name: None,
-  }])]
-}

src/stl/mod.rs

@@ -1,22 +0,0 @@
-//! Constants exposed to usercode by the interpreter
-mod arithmetic_error;
-mod assertion_error;
-mod bool;
-mod conv;
-mod io;
-pub mod inspect;
-mod mk_stl;
-mod num;
-mod runtime_error;
-mod str;
-pub mod codegen;
-mod bin;
-
-pub use arithmetic_error::ArithmeticError;
-pub use assertion_error::AssertionError;
-pub use io::{handle as handleIO, IO};
-pub use mk_stl::{mk_prelude, mk_stl, StlOptions};
-pub use runtime_error::RuntimeError;
-
-pub use self::bool::Boolean;
-pub use self::num::Numeric;

src/stl/num.orc

@@ -1,5 +0,0 @@
-export ...$a + ...$b =0x2p36=> (add (...$a) (...$b))
-export ...$a - ...$b:1 =0x2p36=> (subtract (...$a) (...$b))
-export ...$a * ...$b =0x1p36=> (multiply (...$a) (...$b))
-export ...$a % ...$b:1 =0x1p36=> (remainder (...$a) (...$b))
-export ...$a / ...$b:1 =0x1p36=> (divide (...$a) (...$b))

src/stl/option.orc

@@ -1,9 +0,0 @@
-import std::io::panic
-
-export some := \v. \d.\f. f v
-export none := \d.\f. d
-
-export map := \option.\f. option none f
-export flatten := \option. option none \opt. opt
-export flatmap := \option.\f. option none \opt. map opt f
-export unwrap := \option. option (panic "value expected") \x.x

src/stl/proc.orc

@@ -1,16 +0,0 @@
-import super::fn::=>
-
--- remove duplicate ;-s
-export do { ...$statement ; ; ...$rest:1 } =0x3p130=> do { ...$statement ; ...$rest }
-export do { ...$statement ; ...$rest:1 } =0x2p130=> statement (...$statement) do { ...$rest }
-export do { ...$return } =0x1p130=> ...$return
-
-export statement (let $name = ...$value) ...$next =0x1p230=> (
-  ( \$name. ...$next) (...$value)
-)
-export statement (cps ...$names = ...$operation:1) ...$next =0x2p230=> (
-  (...$operation) ( (...$names) => ...$next )
-)
-export statement (cps ...$operation) ...$next =0x1p230=> (
-  (...$operation) (...$next)
-)

src/stl/str.orc

@@ -1,10 +0,0 @@
-import super::(proc::*, bool::*, io::panic)
-
-export ...$a ++ ...$b =0x4p36=> (concat (...$a) (...$b))
-
-export char_at := \s.\i. do{
-  let slc = slice s i 1;
-  if len slc == 1
-  then slc
-  else panic "Character index out of bounds"
-}

src/systems/asynch/mod.rs

@@ -0,0 +1,5 @@
+mod system;
+mod types;
+
+pub use system::{AsynchConfig, InfiniteBlock};
+pub use types::{Asynch, MessagePort};

src/systems/asynch/system.rs

@@ -0,0 +1,183 @@
+use std::any::{type_name, Any, TypeId};
+use std::cell::RefCell;
+use std::fmt::{Debug, Display};
+use std::rc::Rc;
+use std::sync::mpsc::Sender;
+use std::time::Duration;
+
+use hashbrown::HashMap;
+use ordered_float::NotNan;
+
+use super::types::MessagePort;
+use super::Asynch;
+use crate::facade::{IntoSystem, System};
+use crate::foreign::cps_box::{init_cps, CPSBox};
+use crate::foreign::ExternError;
+use crate::interpreted::ExprInst;
+use crate::interpreter::HandlerTable;
+use crate::systems::codegen::call;
+use crate::systems::stl::Boolean;
+use crate::utils::{unwrap_or, PollEvent, Poller};
+use crate::{atomic_inert, define_fn, ConstTree, Interner};
+
+#[derive(Debug, Clone)]
+struct Timer {
+  recurring: Boolean,
+  duration: NotNan<f64>,
+}
+define_fn! {expr=x in
+  SetTimer {
+    recurring: Boolean,
+    duration: NotNan<f64>
+  } => Ok(init_cps(2, Timer{
+    recurring: *recurring,
+    duration: *duration
+  }))
+}
+
+#[derive(Clone)]
+struct CancelTimer(Rc<dyn Fn()>);
+impl Debug for CancelTimer {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    write!(f, "opaque cancel operation")
+  }
+}
+
+#[derive(Clone, Debug)]
+struct Yield;
+atomic_inert!(Yield, "a yield command");
+
+/// Error indicating a yield command when all event producers and timers had
+/// exited
+pub struct InfiniteBlock;
+impl ExternError for InfiniteBlock {}
+impl Display for InfiniteBlock {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    static MSG: &str = "User code yielded, but there are no timers or event \
+                        producers to wake it up in the future";
+    write!(f, "{}", MSG)
+  }
+}
+
+impl MessagePort for Sender<Box<dyn Any + Send>> {
+  fn send<T: Send + 'static>(&mut self, message: T) {
+    let _ = Self::send(self, Box::new(message));
+  }
+}
+
+impl<F> MessagePort for F
+where
+  F: FnMut(Box<dyn Any + Send>) + Send + Clone + 'static,
+{
+  fn send<T: Send + 'static>(&mut self, message: T) {
+    self(Box::new(message))
+  }
+}
+
+type AnyHandler<'a> = Box<dyn FnMut(Box<dyn Any>) -> Option<ExprInst> + 'a>;
+
+/// Datastructures the asynch system will eventually be constructed from
+pub struct AsynchConfig<'a> {
+  poller: Poller<Box<dyn Any + Send>, ExprInst, ExprInst>,
+  sender: Sender<Box<dyn Any + Send>>,
+  handlers: HashMap<TypeId, AnyHandler<'a>>,
+}
+impl<'a> AsynchConfig<'a> {
+  /// Create a new async event loop that allows registering handlers and taking
+  /// references to the port before it's converted into a [System]
+  pub fn new() -> Self {
+    let (sender, poller) = Poller::new();
+    Self { poller, sender, handlers: HashMap::new() }
+  }
+}
+impl<'a> Asynch for AsynchConfig<'a> {
+  type Port = Sender<Box<dyn Any + Send>>;
+
+  fn register<T: 'static>(
+    &mut self,
+    mut f: impl FnMut(Box<T>) -> Option<ExprInst> + 'a,
+  ) {
+    let cb = move |a: Box<dyn Any>| f(a.downcast().expect("keyed by TypeId"));
+    let prev = self.handlers.insert(TypeId::of::<T>(), Box::new(cb));
+    assert!(
+      prev.is_none(),
+      "Duplicate handlers for async event {}",
+      type_name::<T>()
+    )
+  }
+
+  fn get_port(&self) -> Self::Port {
+    self.sender.clone()
+  }
+}
+
+impl<'a> Default for AsynchConfig<'a> {
+  fn default() -> Self {
+    Self::new()
+  }
+}
+
+impl<'a> IntoSystem<'a> for AsynchConfig<'a> {
+  fn into_system(self, i: &Interner) -> System<'a> {
+    let Self { mut handlers, poller, .. } = self;
+    let mut handler_table = HandlerTable::new();
+    let polly = Rc::new(RefCell::new(poller));
+    handler_table.register({
+      let polly = polly.clone();
+      move |t: &CPSBox<Timer>| {
+        let mut polly = polly.borrow_mut();
+        let (timeout, action, cont) = t.unpack2();
+        let duration = Duration::from_secs_f64(*timeout.duration);
+        let cancel_timer = if timeout.recurring.0 {
+          CancelTimer(Rc::new(polly.set_interval(duration, action.clone())))
+        } else {
+          CancelTimer(Rc::new(polly.set_timeout(duration, action.clone())))
+        };
+        Ok(call(cont.clone(), [init_cps(1, cancel_timer).wrap()]).wrap())
+      }
+    });
+    handler_table.register(move |t: &CPSBox<CancelTimer>| {
+      let (command, cont) = t.unpack1();
+      command.0.as_ref()();
+      Ok(cont.clone())
+    });
+    handler_table.register({
+      let polly = polly.clone();
+      move |_: &Yield| {
+        let mut polly = polly.borrow_mut();
+        loop {
+          let next = unwrap_or!(polly.run();
+            return Err(InfiniteBlock.into_extern())
+          );
+          match next {
+            PollEvent::Once(expr) => return Ok(expr),
+            PollEvent::Recurring(expr) => return Ok(expr),
+            PollEvent::Event(ev) => {
+              let handler = (handlers.get_mut(&ev.as_ref().type_id()))
+                .unwrap_or_else(|| {
+                  panic!("Unhandled messgae type: {:?}", ev.type_id())
+                });
+              if let Some(expr) = handler(ev) {
+                return Ok(expr);
+              }
+            },
+          }
+        }
+      }
+    });
+    System {
+      name: vec!["system".to_string(), "asynch".to_string()],
+      constants: ConstTree::namespace(
+        [i.i("system"), i.i("async")],
+        ConstTree::tree([
+          (i.i("set_timer"), ConstTree::xfn(SetTimer)),
+          (i.i("yield"), ConstTree::atom(Yield)),
+        ]),
+      )
+      .unwrap_tree(),
+      code: HashMap::new(),
+      prelude: Vec::new(),
+      handlers: handler_table,
+    }
+  }
+}

src/systems/asynch/types.rs

@@ -0,0 +1,30 @@
+use crate::interpreted::ExprInst;
+
+/// A thread-safe handle that can be used to send events of any type
+pub trait MessagePort: Send + Clone + 'static {
+  /// Send an event. Any type is accepted, handlers are dispatched by type ID
+  fn send<T: Send + 'static>(&mut self, message: T);
+}
+
+pub trait Asynch {
+  /// A thread-safe handle that can be used to push events into the dispatcher
+  type Port: MessagePort;
+
+  /// Register a function that will be called synchronously when an event of the
+  /// accepted type is dispatched. Only one handler may be specified for each
+  /// event type. The handler may choose to process the event autonomously, or
+  /// return an Orchid thunk for the interpreter to execute.
+  ///
+  /// # Panics
+  ///
+  /// When the function is called with an argument type it was previously called
+  /// with
+  fn register<T: 'static>(
+    &mut self,
+    f: impl FnMut(Box<T>) -> Option<ExprInst> + 'static,
+  );
+
+  /// Return a handle that can be passed to worker threads and used to push
+  /// events onto the dispatcher
+  fn get_port(&self) -> Self::Port;
+}