Various Improvements

- removed many panics from the pipeline
- extracted project and const tree to representations
- extended STL list support
- improved loops

examples/list-processing/main.orc

@@ -6,7 +6,8 @@ export main := do{
   let sum = bar
     |> list::skip 2
     |> list::take 3
-    |> list::reduce 0 (a b) => a + b;
+    |> list::reduce (\a.\b. a + b)
+    |> option::unwrap;
   cps print $ to_string sum ++ "\n";
   0
 }

src/bin/main.rs

@@ -7,9 +7,11 @@ use std::{iter, process};
 use clap::Parser;
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchidlang::interner::{InternedDisplay, Interner};
+use orchidlang::interner::InternedDisplay;
 use orchidlang::{
-  ast, ast_to_interpreted, interpreter, pipeline, rule, stl, Stok, Sym, VName,
+  ast, ast_to_interpreted, collect_consts, collect_rules, interpreter,
+  pipeline, rule, stl, vname_to_sym_tree, Interner, ProjectTree, Stok, Sym,
+  VName,
 };
 
 use crate::cli::cmd_prompt;
@@ -68,11 +70,7 @@ impl Args {
 /// Load and parse all source related to the symbol `target` or all symbols
 /// in the namespace `target` in the context of the STL. All sourcefiles must
 /// reside within `dir`.
-fn load_dir(
+fn load_dir(dir: &Path, target: &[Stok], i: &Interner) -> ProjectTree<VName> {
-  dir: &Path,
-  target: &[Stok],
-  i: &Interner,
-) -> pipeline::ProjectTree<VName> {
   let file_cache = pipeline::file_loader::mk_dir_cache(dir.to_path_buf(), i);
   let library = stl::mk_stl(i, stl::StlOptions::default());
   pipeline::parse_layer(
@@ -126,9 +124,9 @@ pub fn main() {
   let dir = PathBuf::try_from(args.dir).unwrap();
   let i = Interner::new();
   let main = to_vname(&args.main, &i);
-  let project = pipeline::vname_to_sym_tree(load_dir(&dir, &main, &i), &i);
+  let project = vname_to_sym_tree(load_dir(&dir, &main, &i), &i);
-  let rules = pipeline::collect_rules(&project);
+  let rules = collect_rules(&project);
-  let consts = pipeline::collect_consts(&project, &i);
+  let consts = collect_consts(&project, &i);
   let repo = rule::Repo::new(rules, &i).unwrap_or_else(|(rule, error)| {
     panic!(
       "Rule error: {}

src/lib.rs

@@ -18,12 +18,18 @@ pub mod rule;
 pub mod stl;
 mod utils;
 
-use interner::Tok;
+pub use interner::{Interner, Tok};
+pub use pipeline::file_loader::{mk_dir_cache, mk_embed_cache};
+pub use pipeline::parse_layer;
 pub use representations::{NameLike, Sym, VName};
 /// Element of VName and a common occurrence in the API
 pub type Stok = Tok<String>;
 pub use representations::ast_to_interpreted::ast_to_interpreted;
+pub use representations::project::{
+  collect_consts, collect_rules, vname_to_sym_tree, ProjectTree,
+};
 pub use representations::{
-  ast, interpreted, sourcefile, tree, Literal, Location, PathSet, Primitive,
+  ast, from_const_tree, interpreted, sourcefile, tree, ConstTree, Literal,
+  Location, PathSet, Primitive,
 };
 pub use utils::{Side, Substack};

src/pipeline/error/import_all.rs

@@ -0,0 +1,30 @@
+use std::rc::Rc;
+
+use super::{ErrorPosition, ProjectError};
+use crate::representations::location::Location;
+use crate::utils::iter::box_once;
+use crate::utils::BoxedIter;
+
+/// Error produced for the statement `import *`
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct ImportAll {
+  /// The file containing the offending import
+  pub offender_file: Vec<String>,
+  /// The module containing the offending import
+  pub offender_mod: Vec<String>,
+}
+impl ProjectError for ImportAll {
+  fn description(&self) -> &str {
+    "a top-level glob import was used"
+  }
+  fn message(&self) -> String {
+    format!("{} imports *", self.offender_mod.join("::"))
+  }
+
+  fn positions(&self) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition {
+      location: Location::File(Rc::new(self.offender_file.clone())),
+      message: Some(format!("{} imports *", self.offender_mod.join("::"))),
+    })
+  }
+}

src/pipeline/error/mod.rs

@@ -1,14 +1,16 @@
 //! Various errors the pipeline can produce
-mod module_not_found;
+mod import_all;
 mod not_exported;
+mod not_found;
 mod parse_error_with_path;
 mod project_error;
 mod too_many_supers;
 mod unexpected_directory;
 mod visibility_mismatch;
 
-pub use module_not_found::ModuleNotFound;
+pub use import_all::ImportAll;
 pub use not_exported::NotExported;
+pub use not_found::NotFound;
 pub use parse_error_with_path::ParseErrorWithPath;
 pub use project_error::{ErrorPosition, ProjectError};
 pub use too_many_supers::TooManySupers;

src/pipeline/error/module_not_found.rs

@@ -1,27 +0,0 @@
-use super::{ErrorPosition, ProjectError};
-use crate::utils::iter::box_once;
-use crate::utils::BoxedIter;
-
-/// Error produced when an import refers to a nonexistent module
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub struct ModuleNotFound {
-  /// The file containing the invalid import
-  pub file: Vec<String>,
-  /// The invalid import path
-  pub subpath: Vec<String>,
-}
-impl ProjectError for ModuleNotFound {
-  fn description(&self) -> &str {
-    "an import refers to a nonexistent module"
-  }
-  fn message(&self) -> String {
-    format!(
-      "module {} in {} was not found",
-      self.subpath.join("::"),
-      self.file.join("/"),
-    )
-  }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
-    box_once(ErrorPosition::just_file(self.file.clone()))
-  }
-}

src/pipeline/error/not_found.rs

@@ -0,0 +1,64 @@
+use super::{ErrorPosition, ProjectError};
+use crate::representations::project::ProjectModule;
+#[allow(unused)] // For doc
+use crate::tree::Module;
+use crate::tree::WalkError;
+use crate::utils::iter::box_once;
+use crate::utils::BoxedIter;
+use crate::{Interner, NameLike, Tok};
+
+/// Error produced when an import refers to a nonexistent module
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct NotFound {
+  /// The file containing the invalid import
+  pub file: Vec<String>,
+  /// The invalid import path
+  pub subpath: Vec<String>,
+}
+impl NotFound {
+  /// Produce this error from the parameters of [Module]`::walk_ref` and a
+  /// [WalkError]
+  ///
+  /// # Panics
+  ///
+  /// - if `path` is shorter than the `pos` of the error
+  /// - if a walk up to but not including `pos` fails
+  ///
+  /// Basically, if `e` was not produced by the `walk*` methods called on
+  /// `path`.
+  pub fn from_walk_error(
+    prefix: &[Tok<String>],
+    path: &[Tok<String>],
+    orig: &ProjectModule<impl NameLike>,
+    e: WalkError,
+    i: &Interner,
+  ) -> Self {
+    let last_mod =
+      orig.walk_ref(&path[..e.pos], false).expect("error occured on next step");
+    let mut whole_path =
+      prefix.iter().chain(path.iter()).map(|t| i.r(*t)).cloned();
+    if let Some(file) = &last_mod.extra.file {
+      Self {
+        file: whole_path.by_ref().take(file.len()).collect(),
+        subpath: whole_path.collect(),
+      }
+    } else {
+      Self { file: whole_path.collect(), subpath: Vec::new() }
+    }
+  }
+}
+impl ProjectError for NotFound {
+  fn description(&self) -> &str {
+    "an import refers to a nonexistent module"
+  }
+  fn message(&self) -> String {
+    format!(
+      "module {} in {} was not found",
+      self.subpath.join("::"),
+      self.file.join("/"),
+    )
+  }
+  fn positions(&self) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition::just_file(self.file.clone()))
+  }
+}

src/pipeline/file_loader.rs

@@ -96,6 +96,10 @@ pub fn mk_dir_cache(root: PathBuf, i: &Interner) -> Cache<VName, IOResult> {
 }
 
 /// Load a file from the specified path from an embed table
+///
+/// # Panics
+///
+/// if the `RustEmbed` includes files that do not end in `ext`
 pub fn load_embed<T: 'static + RustEmbed>(path: &str, ext: &str) -> IOResult {
   let file_path = path.to_string() + ext;
   if let Some(file) = T::get(&file_path) {

src/pipeline/import_resolution/apply_aliases.rs

@@ -4,7 +4,7 @@ use super::alias_map::AliasMap;
 use super::decls::{InjectedAsFn, UpdatedFn};
 use crate::ast::{Expr, Rule};
 use crate::interner::Tok;
-use crate::pipeline::{ProjectExt, ProjectModule};
+use crate::representations::project::{ProjectExt, ProjectModule};
 use crate::representations::tree::{ModEntry, ModMember};
 use crate::representations::VName;
 use crate::utils::Substack;

src/pipeline/import_resolution/collect_aliases.rs

@@ -1,11 +1,11 @@
-use core::panic;
 use std::rc::Rc;
 
 use super::alias_map::AliasMap;
 use super::decls::UpdatedFn;
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::{NotExported, ProjectError};
+use crate::pipeline::error::{NotExported, NotFound, ProjectError};
-use crate::pipeline::project_tree::{split_path, ProjectModule, ProjectTree};
+use crate::pipeline::project_tree::split_path;
+use crate::representations::project::{ProjectModule, ProjectTree};
 use crate::representations::tree::{ModMember, WalkErrorKind};
 use crate::representations::VName;
 use crate::utils::{pushed, unwrap_or, Substack};
@@ -23,20 +23,29 @@ fn assert_visible(
   let vis_ignored_len = usize::min(tgt_path.len(), shared_len + 1);
   let private_root = (project.0)
     .walk_ref(&tgt_path[..vis_ignored_len], false)
-    .unwrap_or_else(|e| {
+    .map_err(|e| match e.kind {
-      let path_slc = &tgt_path[..vis_ignored_len];
+      WalkErrorKind::Private =>
-      let bad_path = i.extern_all(path_slc).join("::");
+        unreachable!("visibility is not being checked here"),
-      eprintln!(
+      WalkErrorKind::Missing => NotFound::from_walk_error(
-        "Error while walking {bad_path}; {:?} on step {}",
+        &[],
-        e.kind, e.pos
+        &tgt_path[..vis_ignored_len],
-      );
+        &project.0,
-      eprintln!("looking from {}", i.extern_all(source).join("::"));
+        e,
-      panic!("")
+        i,
-    });
+      )
+      .rc(),
+    })?;
   let direct_parent = private_root
     .walk_ref(&tgt_path[vis_ignored_len..], true)
     .map_err(|e| match e.kind {
-      WalkErrorKind::Missing => panic!("checked in parsing"),
+      WalkErrorKind::Missing => NotFound::from_walk_error(
+        &tgt_path[..vis_ignored_len],
+        &tgt_path[vis_ignored_len..],
+        &project.0,
+        e,
+        i,
+      )
+      .rc(),
       WalkErrorKind::Private => {
         let full_path = &tgt_path[..shared_len + e.pos];
         let (file, sub) = split_path(full_path, project);
@@ -93,14 +102,15 @@ fn collect_aliases_rec(
       .extra
       .exports
       .get(&name)
-      .unwrap_or_else(|| {
+      .ok_or_else(|| {
-        panic!(
+        let file_len =
-          "error in {}, {} has no member {}",
+          target_mod.extra.file.as_ref().unwrap_or(target_mod_name).len();
-          i.extern_all(&mod_path_v).join("::"),
+        NotFound {
-          i.extern_all(target_mod_name).join("::"),
+          file: i.extern_all(&target_mod_name[..file_len]),
-          i.r(name)
+          subpath: i.extern_all(&target_sym_v[file_len..]),
-        )
+        }
-      })
+        .rc()
+      })?
       .clone();
     alias_map.link(sym_path_v, target_sym);
   }

src/pipeline/import_resolution/resolve_imports.rs

@@ -6,7 +6,7 @@ use super::collect_aliases::collect_aliases;
 use super::decls::{InjectedAsFn, UpdatedFn};
 use crate::interner::Interner;
 use crate::pipeline::error::ProjectError;
-use crate::pipeline::project_tree::ProjectTree;
+use crate::representations::project::ProjectTree;
 use crate::representations::VName;
 
 /// Follow import chains to locate the original name of all tokens, then

src/pipeline/mod.rs

@@ -8,7 +8,3 @@ mod project_tree;
 mod source_loader;
 
 pub use parse_layer::parse_layer;
-pub use project_tree::{
-  collect_consts, collect_rules, from_const_tree, vname_to_sym_tree, ConstTree,
-  ProjectExt, ProjectModule, ProjectTree,
-};

src/pipeline/parse_layer.rs

@@ -2,10 +2,11 @@ use std::rc::Rc;
 
 use super::error::ProjectError;
 use super::file_loader::IOResult;
-use super::{import_resolution, project_tree, source_loader, ProjectTree};
+use super::{import_resolution, project_tree, source_loader};
 use crate::interner::{Interner, Tok};
 use crate::representations::sourcefile::FileEntry;
 use crate::representations::VName;
+use crate::ProjectTree;
 
 /// Using an IO callback, produce a project tree that includes the given
 /// target symbols or files if they're defined.

src/pipeline/project_tree/build_tree.rs

@@ -3,13 +3,14 @@ use std::rc::Rc;
 use hashbrown::HashMap;
 use itertools::Itertools;
 
+use super::collect_ops;
 use super::collect_ops::InjectedOperatorsFn;
 use super::parse_file::parse_file;
-use super::{collect_ops, ProjectExt, ProjectTree};
 use crate::ast::{Constant, Expr};
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::ProjectError;
+use crate::pipeline::error::{ProjectError, TooManySupers};
 use crate::pipeline::source_loader::{LoadedSource, LoadedSourceTable};
+use crate::representations::project::{ProjectExt, ProjectTree};
 use crate::representations::sourcefile::{absolute_path, FileEntry, Member};
 use crate::representations::tree::{ModEntry, ModMember, Module};
 use crate::representations::{NameLike, VName};
@@ -23,17 +24,20 @@ struct ParsedSource<'a> {
   parsed: Vec<FileEntry>,
 }
 
+/// Split a path into file- and subpath in knowledge
+///
+/// # Panics
+///
+/// if the path is invalid
 pub fn split_path<'a>(
   path: &'a [Tok<String>],
   proj: &'a ProjectTree<impl NameLike>,
 ) -> (&'a [Tok<String>], &'a [Tok<String>]) {
-  let (end, body) = if let Some(s) = path.split_last() {
+  let (end, body) = unwrap_or!(path.split_last(); {
-    s
+    return (&[], &[])
-  } else {
+  });
-    return (&[], &[]);
+  let mut module = (proj.0.walk_ref(body, false))
-  };
+    .expect("invalid path can't have been split above");
-  let mut module =
-    proj.0.walk_ref(body, false).expect("invalid path cannot be split");
   if let ModMember::Sub(m) = &module.items[end].member {
     module = m;
   }
@@ -44,6 +48,12 @@ pub fn split_path<'a>(
 }
 
 /// Convert normalized, prefixed source into a module
+///
+/// # Panics
+///
+/// - if there are imports with too many "super" prefixes (this is normally
+///   detected in preparsing)
+/// - if the preparsed tree is missing a module that exists in the source
 fn source_to_module(
   // level
   path: Substack<Tok<String>>,
@@ -53,29 +63,33 @@ fn source_to_module(
   // context
   i: &Interner,
   filepath_len: usize,
-) -> Module<Expr<VName>, ProjectExt<VName>> {
+) -> Result<Module<Expr<VName>, ProjectExt<VName>>, Rc<dyn ProjectError>> {
   let path_v = path.iter().rev_vec_clone();
-  let imports = data
+  let imports = (data.iter())
-    .iter()
     .filter_map(|ent| {
       if let FileEntry::Import(impv) = ent { Some(impv.iter()) } else { None }
     })
     .flatten()
     .cloned()
     .collect::<Vec<_>>();
-  let imports_from = imports
+  let imports_from = (imports.iter())
-    .iter()
+    .map(|imp| -> Result<_, Rc<dyn ProjectError>> {
-    .map(|imp| {
       let mut imp_path_v = i.r(imp.path).clone();
-      imp_path_v.push(imp.name.expect("imports normalized"));
+      imp_path_v.push(imp.name.expect("glob imports had just been resolved"));
-      let mut abs_path =
+      let mut abs_path = absolute_path(&path_v, &imp_path_v, i)
-        absolute_path(&path_v, &imp_path_v, i).expect("tested in preparsing");
+        .expect("should have failed in preparsing");
-      let name = abs_path.pop().expect("importing the global context");
+      let name = abs_path.pop().ok_or_else(|| {
-      (name, abs_path)
+        TooManySupers {
+          offender_file: i.extern_all(&path_v[..filepath_len]),
+          offender_mod: i.extern_all(&path_v[filepath_len..]),
+          path: i.extern_all(&imp_path_v),
+        }
+        .rc()
+      })?;
+      Ok((name, abs_path))
     })
-    .collect::<HashMap<_, _>>();
+    .collect::<Result<HashMap<_, _>, _>>()?;
-  let exports = data
+  let exports = (data.iter())
-    .iter()
     .flat_map(|ent| {
       let mk_ent = |name| (name, pushed(&path_v, name));
       match ent {
@@ -99,8 +113,7 @@ fn source_to_module(
       }
     })
     .collect::<HashMap<_, _>>();
-  let rules = data
+  let rules = (data.iter())
-    .iter()
     .filter_map(|ent| match ent {
       FileEntry::Exported(Member::Rule(rule)) => Some(rule),
       FileEntry::Internal(Member::Rule(rule)) => Some(rule),
@@ -108,28 +121,30 @@ fn source_to_module(
     })
     .cloned()
     .collect::<Vec<_>>();
-  let items = data
+  let items = (data.into_iter())
-    .into_iter()
     .filter_map(|ent| {
       let member_to_item = |exported, member| match member {
         Member::Namespace(ns) => {
           let new_prep = unwrap_or!(
             &preparsed.items[&ns.name].member => ModMember::Sub;
-            panic!("preparsed missing a submodule")
+            panic!("Preparsed should include entries for all submodules")
           );
-          let module = source_to_module(
+          let module = match source_to_module(
             path.push(ns.name),
             new_prep,
             ns.body,
             i,
             filepath_len,
-          );
+          ) {
+            Err(e) => return Some(Err(e)),
+            Ok(t) => t,
+          };
           let member = ModMember::Sub(module);
-          Some((ns.name, ModEntry { exported, member }))
+          Some(Ok((ns.name, ModEntry { exported, member })))
         },
         Member::Constant(Constant { name, value }) => {
           let member = ModMember::Item(value);
-          Some((name, ModEntry { exported, member }))
+          Some(Ok((name, ModEntry { exported, member })))
         },
         _ => None,
       };
@@ -139,8 +154,8 @@ fn source_to_module(
         _ => None,
       }
     })
-    .collect::<HashMap<_, _>>();
+    .collect::<Result<HashMap<_, _>, _>>()?;
-  Module {
+  Ok(Module {
     imports,
     items,
     extra: ProjectExt {
@@ -149,14 +164,14 @@ fn source_to_module(
       rules,
       file: Some(path_v[..filepath_len].to_vec()),
     },
-  }
+  })
 }
 
 fn files_to_module(
   path: Substack<Tok<String>>,
   files: Vec<ParsedSource>,
   i: &Interner,
-) -> Module<Expr<VName>, ProjectExt<VName>> {
+) -> Result<Module<Expr<VName>, ProjectExt<VName>>, Rc<dyn ProjectError>> {
   let lvl = path.len();
   debug_assert!(
     files.iter().map(|f| f.path.len()).max().unwrap() >= lvl,
@@ -172,21 +187,20 @@ fn files_to_module(
       path.len(),
     );
   }
-  let items = files
+  let items = (files.into_iter())
-    .into_iter()
     .group_by(|f| f.path[lvl])
     .into_iter()
-    .map(|(namespace, files)| {
+    .map(|(namespace, files)| -> Result<_, Rc<dyn ProjectError>> {
       let subpath = path.push(namespace);
       let files_v = files.collect::<Vec<_>>();
-      let module = files_to_module(subpath, files_v, i);
+      let module = files_to_module(subpath, files_v, i)?;
       let member = ModMember::Sub(module);
-      (namespace, ModEntry { exported: true, member })
+      Ok((namespace, ModEntry { exported: true, member }))
     })
-    .collect::<HashMap<_, _>>();
+    .collect::<Result<HashMap<_, _>, _>>()?;
   let exports: HashMap<_, _> =
     items.keys().copied().map(|name| (name, pushed(&path_v, name))).collect();
-  Module {
+  Ok(Module {
     items,
     imports: vec![],
     extra: ProjectExt {
@@ -195,7 +209,7 @@ fn files_to_module(
       rules: vec![],
       file: None,
     },
-  }
+  })
 }
 
 pub fn build_tree(
@@ -222,5 +236,5 @@ pub fn build_tree(
       path: path.clone(),
     })
     .collect::<Vec<_>>();
-  Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)))
+  Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)?))
 }

src/pipeline/project_tree/collect_ops/exported_ops.rs

@@ -4,7 +4,7 @@ use hashbrown::HashSet;
 use trait_set::trait_set;
 
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::{ModuleNotFound, ProjectError};
+use crate::pipeline::error::{NotFound, ProjectError};
 use crate::pipeline::source_loader::LoadedSourceTable;
 use crate::representations::tree::WalkErrorKind;
 use crate::utils::{split_max_prefix, unwrap_or, Cache};
@@ -53,7 +53,7 @@ pub fn collect_exported_ops(
         WalkErrorKind::Private => {
           unreachable!("visibility is not being checked here")
         },
-        WalkErrorKind::Missing => ModuleNotFound {
+        WalkErrorKind::Missing => NotFound {
           file: i.extern_all(fpath),
           subpath: (subpath.iter())
             .take(walk_err.pos)

src/pipeline/project_tree/collect_ops/ops_for.rs

@@ -24,7 +24,12 @@ fn tree_all_ops(
   }
 }
 
-/// Collect all names imported in this file
+/// Collect all names visible in this file
+///
+/// # Panics
+///
+/// if any import contains too many Super calls. This should be caught during
+/// preparsing
 pub fn collect_ops_for(
   file: &[Tok<String>],
   loaded: &LoadedSourceTable,

src/pipeline/project_tree/mod.rs

@@ -16,16 +16,9 @@
 mod add_prelude;
 mod build_tree;
 mod collect_ops;
-mod const_tree;
 mod normalize_imports;
 mod parse_file;
 mod prefix;
-mod tree;
 
 pub use build_tree::{build_tree, split_path};
 pub use collect_ops::InjectedOperatorsFn;
-pub use const_tree::{from_const_tree, ConstTree};
-pub use tree::{
-  collect_consts, collect_rules, vname_to_sym_tree, ProjectExt, ProjectModule,
-  ProjectTree,
-};

src/pipeline/project_tree/normalize_imports.rs

@@ -33,6 +33,12 @@ fn member_rec(
   }
 }
 
+/// Normalize imports in the FileEntry list recursively
+///
+/// # Panics
+///
+/// - if a path contains too many "super" prefixes
+/// - if the exported operators in a module cannot be determined
 fn entv_rec(
   // level
   mod_stack: Substack<Tok<String>>,
@@ -54,11 +60,9 @@ fn entv_rec(
             if let Import { name: None, path } = import {
               let p = import_abs_path(mod_path, mod_stack, &i.r(path)[..], i)
                 .expect("Should have emerged in preparsing");
-              let names = ops_cache
+              let names = (ops_cache.find(&i.i(&p)))
-                .find(&i.i(&p))
                 .expect("Should have emerged in second parsing");
-              let imports = names
+              let imports = (names.iter())
-                .iter()
                 .map(move |&n| Import { name: Some(n), path })
                 .collect::<Vec<_>>();
               Box::new(imports.into_iter()) as BoxedIter<Import>

src/pipeline/project_tree/parse_file.rs

@@ -10,6 +10,14 @@ use crate::pipeline::error::ProjectError;
 use crate::pipeline::source_loader::LoadedSourceTable;
 use crate::representations::sourcefile::{normalize_namespaces, FileEntry};
 
+/// Parses a file with the correct operator set
+///
+/// # Panics
+///
+/// - on syntax error
+/// - if namespaces are exported inconsistently
+///
+/// These are both checked in the preparsing stage
 pub fn parse_file(
   path: &[Tok<String>],
   loaded: &LoadedSourceTable,

src/pipeline/source_loader/load_source.rs

@@ -73,6 +73,7 @@ fn load_abs_path_rec(
       Ok(Loaded::Code(_)) =>
         unreachable!("split_name returned None but the path is a file"),
       Err(e) => {
+        // todo: if this can actually be produced, return Err(ImportAll) instead
         let parent = abs_path.split_last().expect("import path nonzero").1;
         // exit without error if it was injected, or raise any IO error that was
         // previously swallowed

src/representations/const_tree.rs

@@ -2,11 +2,11 @@ use std::ops::Add;
 
 use hashbrown::HashMap;
 
-use super::{ProjectExt, ProjectModule, ProjectTree};
 use crate::ast::{Clause, Expr};
 use crate::foreign::{Atom, Atomic, ExternFn};
 use crate::interner::Tok;
 use crate::representations::location::Location;
+use crate::representations::project::{ProjectExt, ProjectModule, ProjectTree};
 use crate::representations::tree::{ModEntry, ModMember, Module};
 use crate::representations::{Primitive, VName};
 use crate::utils::{pushed, Substack};

src/representations/mod.rs

@@ -1,6 +1,7 @@
 pub mod ast;
 pub mod ast_to_interpreted;
 pub mod ast_to_postmacro;
+mod const_tree;
 pub mod interpreted;
 pub mod literal;
 pub mod location;
@@ -9,9 +10,11 @@ pub mod path_set;
 pub mod postmacro;
 pub mod postmacro_to_interpreted;
 pub mod primitive;
+pub mod project;
 pub mod sourcefile;
 pub mod tree;
 
+pub use const_tree::{from_const_tree, ConstTree};
 pub use literal::Literal;
 pub use location::Location;
 pub use namelike::{NameLike, Sym, VName};

src/stl/bool.orc

@@ -1,3 +1,5 @@
+export not := \bool. if bool then false else true
+export ...$a != ...$b =0x3p36=> (not (...$a == ...$b))
 export ...$a == ...$b =0x3p36=> (equals (...$a) (...$b))
 export if ...$cond then ...$true else ...$false:1 =0x1p84=> (
   ifthenelse (...$cond) (...$true) (...$false)

src/stl/bool.rs

@@ -2,12 +2,11 @@ use std::rc::Rc;
 
 use crate::foreign::Atom;
 use crate::interner::Interner;
-use crate::pipeline::ConstTree;
 use crate::representations::interpreted::{Clause, ExprInst};
 use crate::representations::Primitive;
 use crate::stl::litconv::with_lit;
 use crate::stl::AssertionError;
-use crate::{atomic_inert, define_fn, Literal, PathSet};
+use crate::{atomic_inert, define_fn, ConstTree, Literal, PathSet};
 
 /// Booleans exposed to Orchid
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]

src/stl/conv.rs

@@ -6,8 +6,7 @@ use super::{ArithmeticError, AssertionError};
 use crate::foreign::ExternError;
 use crate::interner::Interner;
 use crate::parse::{float_parser, int_parser};
-use crate::pipeline::ConstTree;
+use crate::{define_fn, ConstTree, Literal};
-use crate::{define_fn, Literal};
 
 define_fn! {
   /// parse a number. Accepts the same syntax Orchid does.

src/stl/fn.orc

@@ -1,15 +1,26 @@
-export Y := \f.(\x.f (x x))(\x.f (x x))
+import super::known::*
 
-export loop $r on (..$parameters) with ...$tail =0x5p129=> Y (\$r.
+--[ Do nothing. Especially useful as a passive cps operation ]--
-  bind_names (..$parameters) (...$tail)
+export identity := \x.x
-) ..$parameters
+--[
-
+  Apply the function to the given value. Can be used to assign a
--- bind each of the names in the first argument as a parameter for the second argument
+  concrete value in a cps assignment statement.
-bind_names ($name ..$rest) $payload =0x1p250=> \$name. bind_names (..$rest) $payload
+]--
-bind_names () (...$payload) =0x1p250=> ...$payload
+export pass := \val.\cont. cont val
+--[
+  Apply the function to the given pair of values. Mainly useful to assign
+  a concrete pair of values in a cps multi-assignment statement
+]--
+export pass2 := \a.\b.\cont. cont a b
+--[
+  A function that returns the given value for any input. Also useful as a
+  "break" statement in a "do" block.
+]--
+export const := \a. \b.a
 
 export ...$prefix $ ...$suffix:1 =0x1p38=> ...$prefix (...$suffix)
 export ...$prefix |> $fn ..$suffix:1 =0x2p32=> $fn (...$prefix) ..$suffix
 
-export (...$argv) => ...$body =0x2p129=> (bind_names (...$argv) (...$body))
+export ($name) => ...$body =0x2p129=> (\$name. ...$body)
+export ($name, ...$argv) => ...$body =0x2p129=> (\$name. (...$argv) => ...$body)
 $name => ...$body =0x1p129=> (\$name. ...$body)

src/stl/io/mod.rs

@@ -1,5 +1,5 @@
 use crate::interner::Interner;
-use crate::pipeline::ConstTree;
+use crate::ConstTree;
 
 mod command;
 mod inspect;

src/stl/list.orc

@@ -1,4 +1,4 @@
-import super::(option, fn::*, bool::*, known::*, num::*,)
+import super::(option, fn::*, proc::*, loop::*, bool::*, known::*, num::*)
 
 pair := \a.\b. \f. f a b
 
@@ -11,33 +11,93 @@ export pop := \list.\default.\f.list default \cons.cons f
 
 -- Operators
 
-export reduce := \list.\acc.\f. (
+--[
-  loop r on (list acc) with
+  Fold each element into an accumulator using an `acc -> el -> acc`.
-    pop list acc \head.\tail. r tail (f acc head)
+  This evaluates the entire list, and is always tail recursive.
+]--
+export fold := \list.\acc.\f. (
+  loop_over (list, acc) {
+    cps head, list = pop list acc;
+    let acc = f acc head;
+  }
 )
 
+--[
+  Fold each element into an accumulator in reverse order.
+  This evaulates the entire list, and is never tail recursive.
+]--
+export rfold := \list.\acc.\f. (
+  recursive r (list)
+    pop list acc \head.\tail.
+      f (r tail) head
+)
+
+--[
+  Fold each element into a shared element with an `el -> el -> el`.
+  This evaluates the entire list, and is never tail recursive.
+]--
+export reduce := \list.\f. do{
+  cps head, list = pop list option::none;
+  option::some $ fold list head f
+}
+
+--[
+  Return a new list that contains only the elements from the input list
+  for which the function returns true. This operation is lazy.
+]--
+export filter := \list.\f. (
+  pop list end \head.\tail.
+    if (f el)
+    then cons el (filter tail f)
+    else filter tail f
+)
+
+--[
+  Transform each element of the list with an `el -> any`.
+]--
 export map := \list.\f. (
-  loop r on (list) with
+  recursive r (list)
-    pop list end \head.\tail. cons (f head) (r tail)
+    pop list end \head.\tail.
+      cons (f head) (r tail)
 )
 
+--[
+  Skip `n` elements from the list and return the tail
+  If `n` is not an integer, this returns `end`.
+]--
 export skip := \list.\n. (
-  loop r on (list n) with
+  loop_over (list, n) {
-    if n == 0 then list
+    cps _head, list = if n == 0
-    else pop list end \head.\tail. r tail (n - 1)
+      then const list
+      else pop list end;
+    let n = n - 1;
+  }
 )
 
+--[
+  Return `n` elements from the list and discard the rest.
+  This operation is lazy.
+]--
 export take := \list.\n. (
-  loop r on (list n) with
+  recursive r (list, n)
-    if n == 0 then end
+    if n == 0
-    else pop list end \head.\tail. cons head $ r tail $ n - 1
+    then end
+    else pop list end \head.\tail.
+      cons head $ r tail $ n - 1
 )
 
+--[
+  Return the `n`th element from the list.
+  This operation is tail recursive.
+]--
 export get := \list.\n. (
-  loop r on (list n) with
+  loop_over (list, n) {
-    pop list option::none \head.\tail.
+    cps head, list = pop list option::none;
-      if n == 0 then option::some head
+    cps if n == 0
-      else r tail (n - 1)
+      then const (option::some head)
+      else identity;
+    let n = n - 1;
+  }
 )
 
 new[...$item, ...$rest:1] =0x2p84=> (cons (...$item) new[...$rest])

src/stl/loop.orc

@@ -0,0 +1,63 @@
+import super::proc::(;, do, =)
+import super::known::*
+
+--[
+  Bare fixpoint combinator. Due to its many pitfalls, usercode is
+  recommended to use one of the wrappers such as [recursive] or
+  [loop_over] instead.
+]--
+export Y := \f.(\x.f (x x))(\x.f (x x))
+
+--[
+  A syntax construct that encapsulates the Y combinator and encourages
+  single tail recursion. It's possible to use this for multiple or
+  non-tail recursion by using cps statements, but it's more ergonomic
+  than [Y] and more flexible than [std::list::fold].
+
+  To break out of the loop, use [std::fn::const] in a cps statement
+]--
+export loop_over (..$binds) {
+  ...$body
+} =0x5p129=> Y (\r. 
+  def_binds parse_binds (..$binds) do{
+    ...$body;
+    r apply_binds parse_binds (..$binds)
+  }
+) init_binds parse_binds (..$binds)
+
+-- parse_binds builds a conslist
+parse_binds (...$item, ...$tail:1) =0x2p250=> (
+  parse_bind (...$item)
+  parse_binds (...$tail)
+)
+parse_binds (...$item) =0x1p250=> (
+  parse_bind (...$item)
+  ()
+)
+
+-- parse_bind converts items to pairs
+parse_bind ($name) =0x1p250=> ($name bind_no_value)
+parse_bind ($name = ...$value) =0x1p250=> ($name (...$value))
+
+-- def_binds creates name bindings for everything
+def_binds ( ($name $value) $tail ) ...$body =0x1p250=> (
+  \$name. def_binds $tail ...$body
+)
+def_binds () ...$body =0x1p250=> ...$body
+
+-- init_binds passes the value for initializers
+init_binds ( ($name bind_no_value) $tail ) =0x2p250=> $name init_binds $tail
+init_binds ( ($name $value) $tail ) =0x1p250=> $value init_binds $tail
+-- avoid empty templates by assuming that there is a previous token
+$fn init_binds () =0x1p250=> $fn
+
+-- apply_binds passes the name for initializers
+apply_binds ( ($name $_value) $tail ) =0x1p250=> $name apply_binds $tail
+$fn apply_binds () =0x1p250=> $fn
+
+--[
+  Alias for the Y-combinator to avoid some universal pitfalls
+]--
+export recursive $name (..$binds) ...$body =0x5p129=> Y (\$name.
+  def_binds parse_binds (..$binds) ...$body
+) init_binds parse_binds (..$binds)

src/stl/map.orc

@@ -1,4 +1,4 @@
-import super::(bool::*, fn::*, known::*, list, option, proc::*)
+import super::(bool::*, fn::*, known::*, list, option, loop::*, proc::*)
 import std::io::panic
 
 -- utilities for using lists as pairs
@@ -26,19 +26,20 @@ export add := \m.\k.\v. (
 -- queries
 
 -- return the last occurrence of a key if exists
-export get := \m.\k. (
+export get := \m.\key. (
-  loop r on (m) with
+  loop_over (m) {
-    list::pop m option::none \head.\tail. 
+    cps record, m = list::pop m option::none;
-      if fst head == k
+    cps if fst record == key
-      then option::some $ snd head
+      then const $ option::some $ snd record
-      else r tail
+      else identity;
+  }
 )
 
 -- commands
 
 -- remove one occurrence of a key
 export del := \m.\k. (
-  loop r on (m) with
+  recursive r (m)
     list::pop m list::end \head.\tail.
       if fst head == k then tail
       else list::cons head $ r tail
@@ -46,10 +47,7 @@ export del := \m.\k. (
 
 -- remove all occurrences of a key
 export delall := \m.\k. (
-  loop r on (m) with
+  list::filter m \record. fst record != k
-    list::pop m list::end \head.\tail.
-      if (fst head) == k then r tail
-      else list::cons head $ r tail
 )
 
 -- replace at most one occurrence of a key
@@ -60,12 +58,11 @@ export set := \m.\k.\v. (
 )
 
 -- ensure that there's only one instance of each key in the map
-export normalize := \m. do{
+export normalize := \m. (
-  let normal = empty
+  recursive r (m, normal=empty) with
-  loop r on (m normal) with
     list::pop m normal \head.\tail.
       r tail $ set normal (fst head) (snd head)
-}
+)
 
 new[...$tail:2, ...$key = ...$value:1] =0x2p84=> (
   set new[...$tail] (...$key) (...$value)

src/stl/mk_stl.rs

@@ -8,9 +8,10 @@ use super::num::num;
 use super::str::str;
 use crate::interner::Interner;
 use crate::pipeline::file_loader::mk_embed_cache;
-use crate::pipeline::{from_const_tree, parse_layer, ProjectTree};
+use crate::pipeline::parse_layer;
 use crate::representations::VName;
 use crate::sourcefile::{FileEntry, Import};
+use crate::{from_const_tree, ProjectTree};
 
 /// Feature flags for the STL.
 #[derive(Default)]

src/stl/num.rs

@@ -4,12 +4,10 @@ use ordered_float::NotNan;
 
 use super::litconv::with_lit;
 use super::{ArithmeticError, AssertionError};
-use crate::define_fn;
 use crate::foreign::ExternError;
-use crate::interner::Interner;
-use crate::pipeline::ConstTree;
 use crate::representations::interpreted::{Clause, ExprInst};
 use crate::representations::{Literal, Primitive};
+use crate::{define_fn, ConstTree, Interner};
 
 // region:  Numeric, type to handle floats and uints together
 

src/stl/prelude.orc

@@ -3,13 +3,15 @@ export ::(+, -, *, /, %)
 import std::str::*
 export ::(++)
 import std::bool::*
-export ::(==, if, then, else)
+export ::(==, if, then, else, true, false)
 import std::fn::*
-export ::(loop, on, with, $, |>, =>)
+export ::($, |>, =>, identity, pass, pass2, const)
 import std::list
 import std::map
 import std::option
 export ::(list, map, option)
+import std::loop::*
+export ::(loop_over, recursive)
 
 import std::known::*
 export ::(,)

src/stl/proc.orc

@@ -1,11 +1,15 @@
+import super::fn::=>
+
+-- remove duplicate ;-s
+export do { ...$statement ; ; ...$rest:1 } =0x3p130=> do { ...$statement ; ...$rest }
 export do { ...$statement ; ...$rest:1 } =0x2p130=> statement (...$statement) do { ...$rest }
 export do { ...$return } =0x1p130=> ...$return
 
 export statement (let $name = ...$value) ...$next =0x1p230=> (
   ( \$name. ...$next) (...$value)
 )
-export statement (cps $name = ...$operation) ...$next =0x2p230=> (
+export statement (cps ...$names = ...$operation:1) ...$next =0x2p230=> (
-  (...$operation) \$name. ...$next
+  (...$operation) ( (...$names) => ...$next )
 )
 export statement (cps ...$operation) ...$next =0x1p230=> (
   (...$operation) (...$next)

src/stl/str.rs

@@ -1,8 +1,7 @@
 use super::litconv::{with_str, with_uint};
 use super::RuntimeError;
 use crate::interner::Interner;
-use crate::pipeline::ConstTree;
+use crate::{define_fn, ConstTree, Literal};
-use crate::{define_fn, Literal};
 
 define_fn! {expr=x in
   /// Append a string to another