Various Improvements

- removed many panics from the pipeline
- extracted project and const tree to representations
- extended STL list support
- improved loops

src/pipeline/error/import_all.rs

@@ -0,0 +1,30 @@
+use std::rc::Rc;
+
+use super::{ErrorPosition, ProjectError};
+use crate::representations::location::Location;
+use crate::utils::iter::box_once;
+use crate::utils::BoxedIter;
+
+/// Error produced for the statement `import *`
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct ImportAll {
+  /// The file containing the offending import
+  pub offender_file: Vec<String>,
+  /// The module containing the offending import
+  pub offender_mod: Vec<String>,
+}
+impl ProjectError for ImportAll {
+  fn description(&self) -> &str {
+    "a top-level glob import was used"
+  }
+  fn message(&self) -> String {
+    format!("{} imports *", self.offender_mod.join("::"))
+  }
+
+  fn positions(&self) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition {
+      location: Location::File(Rc::new(self.offender_file.clone())),
+      message: Some(format!("{} imports *", self.offender_mod.join("::"))),
+    })
+  }
+}

src/pipeline/error/mod.rs

@@ -1,14 +1,16 @@
 //! Various errors the pipeline can produce
-mod module_not_found;
+mod import_all;
 mod not_exported;
+mod not_found;
 mod parse_error_with_path;
 mod project_error;
 mod too_many_supers;
 mod unexpected_directory;
 mod visibility_mismatch;
 
-pub use module_not_found::ModuleNotFound;
+pub use import_all::ImportAll;
 pub use not_exported::NotExported;
+pub use not_found::NotFound;
 pub use parse_error_with_path::ParseErrorWithPath;
 pub use project_error::{ErrorPosition, ProjectError};
 pub use too_many_supers::TooManySupers;

src/pipeline/error/module_not_found.rs

@@ -1,27 +0,0 @@
-use super::{ErrorPosition, ProjectError};
-use crate::utils::iter::box_once;
-use crate::utils::BoxedIter;
-
-/// Error produced when an import refers to a nonexistent module
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-pub struct ModuleNotFound {
-  /// The file containing the invalid import
-  pub file: Vec<String>,
-  /// The invalid import path
-  pub subpath: Vec<String>,
-}
-impl ProjectError for ModuleNotFound {
-  fn description(&self) -> &str {
-    "an import refers to a nonexistent module"
-  }
-  fn message(&self) -> String {
-    format!(
-      "module {} in {} was not found",
-      self.subpath.join("::"),
-      self.file.join("/"),
-    )
-  }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
-    box_once(ErrorPosition::just_file(self.file.clone()))
-  }
-}

src/pipeline/error/not_found.rs

@@ -0,0 +1,64 @@
+use super::{ErrorPosition, ProjectError};
+use crate::representations::project::ProjectModule;
+#[allow(unused)] // For doc
+use crate::tree::Module;
+use crate::tree::WalkError;
+use crate::utils::iter::box_once;
+use crate::utils::BoxedIter;
+use crate::{Interner, NameLike, Tok};
+
+/// Error produced when an import refers to a nonexistent module
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct NotFound {
+  /// The file containing the invalid import
+  pub file: Vec<String>,
+  /// The invalid import path
+  pub subpath: Vec<String>,
+}
+impl NotFound {
+  /// Produce this error from the parameters of [Module]`::walk_ref` and a
+  /// [WalkError]
+  ///
+  /// # Panics
+  ///
+  /// - if `path` is shorter than the `pos` of the error
+  /// - if a walk up to but not including `pos` fails
+  ///
+  /// Basically, if `e` was not produced by the `walk*` methods called on
+  /// `path`.
+  pub fn from_walk_error(
+    prefix: &[Tok<String>],
+    path: &[Tok<String>],
+    orig: &ProjectModule<impl NameLike>,
+    e: WalkError,
+    i: &Interner,
+  ) -> Self {
+    let last_mod =
+      orig.walk_ref(&path[..e.pos], false).expect("error occured on next step");
+    let mut whole_path =
+      prefix.iter().chain(path.iter()).map(|t| i.r(*t)).cloned();
+    if let Some(file) = &last_mod.extra.file {
+      Self {
+        file: whole_path.by_ref().take(file.len()).collect(),
+        subpath: whole_path.collect(),
+      }
+    } else {
+      Self { file: whole_path.collect(), subpath: Vec::new() }
+    }
+  }
+}
+impl ProjectError for NotFound {
+  fn description(&self) -> &str {
+    "an import refers to a nonexistent module"
+  }
+  fn message(&self) -> String {
+    format!(
+      "module {} in {} was not found",
+      self.subpath.join("::"),
+      self.file.join("/"),
+    )
+  }
+  fn positions(&self) -> BoxedIter<ErrorPosition> {
+    box_once(ErrorPosition::just_file(self.file.clone()))
+  }
+}

src/pipeline/file_loader.rs

@@ -96,6 +96,10 @@ pub fn mk_dir_cache(root: PathBuf, i: &Interner) -> Cache<VName, IOResult> {
 }
 
 /// Load a file from the specified path from an embed table
+///
+/// # Panics
+///
+/// if the `RustEmbed` includes files that do not end in `ext`
 pub fn load_embed<T: 'static + RustEmbed>(path: &str, ext: &str) -> IOResult {
   let file_path = path.to_string() + ext;
   if let Some(file) = T::get(&file_path) {

src/pipeline/import_resolution/apply_aliases.rs

@@ -4,7 +4,7 @@ use super::alias_map::AliasMap;
 use super::decls::{InjectedAsFn, UpdatedFn};
 use crate::ast::{Expr, Rule};
 use crate::interner::Tok;
-use crate::pipeline::{ProjectExt, ProjectModule};
+use crate::representations::project::{ProjectExt, ProjectModule};
 use crate::representations::tree::{ModEntry, ModMember};
 use crate::representations::VName;
 use crate::utils::Substack;

src/pipeline/import_resolution/collect_aliases.rs

@@ -1,11 +1,11 @@
-use core::panic;
 use std::rc::Rc;
 
 use super::alias_map::AliasMap;
 use super::decls::UpdatedFn;
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::{NotExported, ProjectError};
-use crate::pipeline::project_tree::{split_path, ProjectModule, ProjectTree};
+use crate::pipeline::error::{NotExported, NotFound, ProjectError};
+use crate::pipeline::project_tree::split_path;
+use crate::representations::project::{ProjectModule, ProjectTree};
 use crate::representations::tree::{ModMember, WalkErrorKind};
 use crate::representations::VName;
 use crate::utils::{pushed, unwrap_or, Substack};
@@ -23,20 +23,29 @@ fn assert_visible(
   let vis_ignored_len = usize::min(tgt_path.len(), shared_len + 1);
   let private_root = (project.0)
     .walk_ref(&tgt_path[..vis_ignored_len], false)
-    .unwrap_or_else(|e| {
-      let path_slc = &tgt_path[..vis_ignored_len];
-      let bad_path = i.extern_all(path_slc).join("::");
-      eprintln!(
-        "Error while walking {bad_path}; {:?} on step {}",
-        e.kind, e.pos
-      );
-      eprintln!("looking from {}", i.extern_all(source).join("::"));
-      panic!("")
-    });
+    .map_err(|e| match e.kind {
+      WalkErrorKind::Private =>
+        unreachable!("visibility is not being checked here"),
+      WalkErrorKind::Missing => NotFound::from_walk_error(
+        &[],
+        &tgt_path[..vis_ignored_len],
+        &project.0,
+        e,
+        i,
+      )
+      .rc(),
+    })?;
   let direct_parent = private_root
     .walk_ref(&tgt_path[vis_ignored_len..], true)
     .map_err(|e| match e.kind {
-      WalkErrorKind::Missing => panic!("checked in parsing"),
+      WalkErrorKind::Missing => NotFound::from_walk_error(
+        &tgt_path[..vis_ignored_len],
+        &tgt_path[vis_ignored_len..],
+        &project.0,
+        e,
+        i,
+      )
+      .rc(),
       WalkErrorKind::Private => {
         let full_path = &tgt_path[..shared_len + e.pos];
         let (file, sub) = split_path(full_path, project);
@@ -93,14 +102,15 @@ fn collect_aliases_rec(
       .extra
       .exports
       .get(&name)
-      .unwrap_or_else(|| {
-        panic!(
-          "error in {}, {} has no member {}",
-          i.extern_all(&mod_path_v).join("::"),
-          i.extern_all(target_mod_name).join("::"),
-          i.r(name)
-        )
-      })
+      .ok_or_else(|| {
+        let file_len =
+          target_mod.extra.file.as_ref().unwrap_or(target_mod_name).len();
+        NotFound {
+          file: i.extern_all(&target_mod_name[..file_len]),
+          subpath: i.extern_all(&target_sym_v[file_len..]),
+        }
+        .rc()
+      })?
       .clone();
     alias_map.link(sym_path_v, target_sym);
   }

src/pipeline/import_resolution/resolve_imports.rs

@@ -6,7 +6,7 @@ use super::collect_aliases::collect_aliases;
 use super::decls::{InjectedAsFn, UpdatedFn};
 use crate::interner::Interner;
 use crate::pipeline::error::ProjectError;
-use crate::pipeline::project_tree::ProjectTree;
+use crate::representations::project::ProjectTree;
 use crate::representations::VName;
 
 /// Follow import chains to locate the original name of all tokens, then

src/pipeline/mod.rs

@@ -8,7 +8,3 @@ mod project_tree;
 mod source_loader;
 
 pub use parse_layer::parse_layer;
-pub use project_tree::{
-  collect_consts, collect_rules, from_const_tree, vname_to_sym_tree, ConstTree,
-  ProjectExt, ProjectModule, ProjectTree,
-};

src/pipeline/parse_layer.rs

@@ -2,10 +2,11 @@ use std::rc::Rc;
 
 use super::error::ProjectError;
 use super::file_loader::IOResult;
-use super::{import_resolution, project_tree, source_loader, ProjectTree};
+use super::{import_resolution, project_tree, source_loader};
 use crate::interner::{Interner, Tok};
 use crate::representations::sourcefile::FileEntry;
 use crate::representations::VName;
+use crate::ProjectTree;
 
 /// Using an IO callback, produce a project tree that includes the given
 /// target symbols or files if they're defined.

src/pipeline/project_tree/build_tree.rs

@@ -3,13 +3,14 @@ use std::rc::Rc;
 use hashbrown::HashMap;
 use itertools::Itertools;
 
+use super::collect_ops;
 use super::collect_ops::InjectedOperatorsFn;
 use super::parse_file::parse_file;
-use super::{collect_ops, ProjectExt, ProjectTree};
 use crate::ast::{Constant, Expr};
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::ProjectError;
+use crate::pipeline::error::{ProjectError, TooManySupers};
 use crate::pipeline::source_loader::{LoadedSource, LoadedSourceTable};
+use crate::representations::project::{ProjectExt, ProjectTree};
 use crate::representations::sourcefile::{absolute_path, FileEntry, Member};
 use crate::representations::tree::{ModEntry, ModMember, Module};
 use crate::representations::{NameLike, VName};
@@ -23,17 +24,20 @@ struct ParsedSource<'a> {
   parsed: Vec<FileEntry>,
 }
 
+/// Split a path into file- and subpath in knowledge
+///
+/// # Panics
+///
+/// if the path is invalid
 pub fn split_path<'a>(
   path: &'a [Tok<String>],
   proj: &'a ProjectTree<impl NameLike>,
 ) -> (&'a [Tok<String>], &'a [Tok<String>]) {
-  let (end, body) = if let Some(s) = path.split_last() {
-    s
-  } else {
-    return (&[], &[]);
-  };
-  let mut module =
-    proj.0.walk_ref(body, false).expect("invalid path cannot be split");
+  let (end, body) = unwrap_or!(path.split_last(); {
+    return (&[], &[])
+  });
+  let mut module = (proj.0.walk_ref(body, false))
+    .expect("invalid path can't have been split above");
   if let ModMember::Sub(m) = &module.items[end].member {
     module = m;
   }
@@ -44,6 +48,12 @@ pub fn split_path<'a>(
 }
 
 /// Convert normalized, prefixed source into a module
+///
+/// # Panics
+///
+/// - if there are imports with too many "super" prefixes (this is normally
+///   detected in preparsing)
+/// - if the preparsed tree is missing a module that exists in the source
 fn source_to_module(
   // level
   path: Substack<Tok<String>>,
@@ -53,29 +63,33 @@ fn source_to_module(
   // context
   i: &Interner,
   filepath_len: usize,
-) -> Module<Expr<VName>, ProjectExt<VName>> {
+) -> Result<Module<Expr<VName>, ProjectExt<VName>>, Rc<dyn ProjectError>> {
   let path_v = path.iter().rev_vec_clone();
-  let imports = data
-    .iter()
+  let imports = (data.iter())
     .filter_map(|ent| {
       if let FileEntry::Import(impv) = ent { Some(impv.iter()) } else { None }
     })
     .flatten()
     .cloned()
     .collect::<Vec<_>>();
-  let imports_from = imports
-    .iter()
-    .map(|imp| {
+  let imports_from = (imports.iter())
+    .map(|imp| -> Result<_, Rc<dyn ProjectError>> {
       let mut imp_path_v = i.r(imp.path).clone();
-      imp_path_v.push(imp.name.expect("imports normalized"));
-      let mut abs_path =
-        absolute_path(&path_v, &imp_path_v, i).expect("tested in preparsing");
-      let name = abs_path.pop().expect("importing the global context");
-      (name, abs_path)
+      imp_path_v.push(imp.name.expect("glob imports had just been resolved"));
+      let mut abs_path = absolute_path(&path_v, &imp_path_v, i)
+        .expect("should have failed in preparsing");
+      let name = abs_path.pop().ok_or_else(|| {
+        TooManySupers {
+          offender_file: i.extern_all(&path_v[..filepath_len]),
+          offender_mod: i.extern_all(&path_v[filepath_len..]),
+          path: i.extern_all(&imp_path_v),
+        }
+        .rc()
+      })?;
+      Ok((name, abs_path))
     })
-    .collect::<HashMap<_, _>>();
-  let exports = data
-    .iter()
+    .collect::<Result<HashMap<_, _>, _>>()?;
+  let exports = (data.iter())
     .flat_map(|ent| {
       let mk_ent = |name| (name, pushed(&path_v, name));
       match ent {
@@ -99,8 +113,7 @@ fn source_to_module(
       }
     })
     .collect::<HashMap<_, _>>();
-  let rules = data
-    .iter()
+  let rules = (data.iter())
     .filter_map(|ent| match ent {
       FileEntry::Exported(Member::Rule(rule)) => Some(rule),
       FileEntry::Internal(Member::Rule(rule)) => Some(rule),
@@ -108,28 +121,30 @@ fn source_to_module(
     })
     .cloned()
     .collect::<Vec<_>>();
-  let items = data
-    .into_iter()
+  let items = (data.into_iter())
     .filter_map(|ent| {
       let member_to_item = |exported, member| match member {
         Member::Namespace(ns) => {
           let new_prep = unwrap_or!(
             &preparsed.items[&ns.name].member => ModMember::Sub;
-            panic!("preparsed missing a submodule")
+            panic!("Preparsed should include entries for all submodules")
           );
-          let module = source_to_module(
+          let module = match source_to_module(
             path.push(ns.name),
             new_prep,
             ns.body,
             i,
             filepath_len,
-          );
+          ) {
+            Err(e) => return Some(Err(e)),
+            Ok(t) => t,
+          };
           let member = ModMember::Sub(module);
-          Some((ns.name, ModEntry { exported, member }))
+          Some(Ok((ns.name, ModEntry { exported, member })))
         },
         Member::Constant(Constant { name, value }) => {
           let member = ModMember::Item(value);
-          Some((name, ModEntry { exported, member }))
+          Some(Ok((name, ModEntry { exported, member })))
         },
         _ => None,
       };
@@ -139,8 +154,8 @@ fn source_to_module(
         _ => None,
       }
     })
-    .collect::<HashMap<_, _>>();
-  Module {
+    .collect::<Result<HashMap<_, _>, _>>()?;
+  Ok(Module {
     imports,
     items,
     extra: ProjectExt {
@@ -149,14 +164,14 @@ fn source_to_module(
       rules,
       file: Some(path_v[..filepath_len].to_vec()),
     },
-  }
+  })
 }
 
 fn files_to_module(
   path: Substack<Tok<String>>,
   files: Vec<ParsedSource>,
   i: &Interner,
-) -> Module<Expr<VName>, ProjectExt<VName>> {
+) -> Result<Module<Expr<VName>, ProjectExt<VName>>, Rc<dyn ProjectError>> {
   let lvl = path.len();
   debug_assert!(
     files.iter().map(|f| f.path.len()).max().unwrap() >= lvl,
@@ -172,21 +187,20 @@ fn files_to_module(
       path.len(),
     );
   }
-  let items = files
-    .into_iter()
+  let items = (files.into_iter())
     .group_by(|f| f.path[lvl])
     .into_iter()
-    .map(|(namespace, files)| {
+    .map(|(namespace, files)| -> Result<_, Rc<dyn ProjectError>> {
       let subpath = path.push(namespace);
       let files_v = files.collect::<Vec<_>>();
-      let module = files_to_module(subpath, files_v, i);
+      let module = files_to_module(subpath, files_v, i)?;
       let member = ModMember::Sub(module);
-      (namespace, ModEntry { exported: true, member })
+      Ok((namespace, ModEntry { exported: true, member }))
     })
-    .collect::<HashMap<_, _>>();
+    .collect::<Result<HashMap<_, _>, _>>()?;
   let exports: HashMap<_, _> =
     items.keys().copied().map(|name| (name, pushed(&path_v, name))).collect();
-  Module {
+  Ok(Module {
     items,
     imports: vec![],
     extra: ProjectExt {
@@ -195,7 +209,7 @@ fn files_to_module(
       rules: vec![],
       file: None,
     },
-  }
+  })
 }
 
 pub fn build_tree(
@@ -222,5 +236,5 @@ pub fn build_tree(
       path: path.clone(),
     })
     .collect::<Vec<_>>();
-  Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)))
+  Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)?))
 }

src/pipeline/project_tree/collect_ops/exported_ops.rs

@@ -4,7 +4,7 @@ use hashbrown::HashSet;
 use trait_set::trait_set;
 
 use crate::interner::{Interner, Tok};
-use crate::pipeline::error::{ModuleNotFound, ProjectError};
+use crate::pipeline::error::{NotFound, ProjectError};
 use crate::pipeline::source_loader::LoadedSourceTable;
 use crate::representations::tree::WalkErrorKind;
 use crate::utils::{split_max_prefix, unwrap_or, Cache};
@@ -53,7 +53,7 @@ pub fn collect_exported_ops(
         WalkErrorKind::Private => {
           unreachable!("visibility is not being checked here")
         },
-        WalkErrorKind::Missing => ModuleNotFound {
+        WalkErrorKind::Missing => NotFound {
           file: i.extern_all(fpath),
           subpath: (subpath.iter())
             .take(walk_err.pos)

src/pipeline/project_tree/collect_ops/ops_for.rs

@@ -24,7 +24,12 @@ fn tree_all_ops(
   }
 }
 
-/// Collect all names imported in this file
+/// Collect all names visible in this file
+///
+/// # Panics
+///
+/// if any import contains too many Super calls. This should be caught during
+/// preparsing
 pub fn collect_ops_for(
   file: &[Tok<String>],
   loaded: &LoadedSourceTable,

src/pipeline/project_tree/const_tree.rs

@@ -1,103 +0,0 @@
-use std::ops::Add;
-
-use hashbrown::HashMap;
-
-use super::{ProjectExt, ProjectModule, ProjectTree};
-use crate::ast::{Clause, Expr};
-use crate::foreign::{Atom, Atomic, ExternFn};
-use crate::interner::Tok;
-use crate::representations::location::Location;
-use crate::representations::tree::{ModEntry, ModMember, Module};
-use crate::representations::{Primitive, VName};
-use crate::utils::{pushed, Substack};
-
-/// A lightweight module tree that can be built declaratively by hand to
-/// describe libraries of external functions in Rust. It implements [Add] for
-/// added convenience
-pub enum ConstTree {
-  /// A function or constant
-  Const(Expr<VName>),
-  /// A submodule
-  Tree(HashMap<Tok<String>, ConstTree>),
-}
-impl ConstTree {
-  /// Describe a [Primitive]
-  pub fn primitive(primitive: Primitive) -> Self {
-    Self::Const(Expr {
-      location: Location::Unknown,
-      value: Clause::P(primitive),
-    })
-  }
-  /// Describe an [ExternFn]
-  pub fn xfn(xfn: impl ExternFn + 'static) -> Self {
-    Self::primitive(Primitive::ExternFn(Box::new(xfn)))
-  }
-  /// Describe an [Atomic]
-  pub fn atom(atom: impl Atomic + 'static) -> Self {
-    Self::primitive(Primitive::Atom(Atom(Box::new(atom))))
-  }
-  /// Describe a module
-  pub fn tree(arr: impl IntoIterator<Item = (Tok<String>, Self)>) -> Self {
-    Self::Tree(arr.into_iter().collect())
-  }
-}
-impl Add for ConstTree {
-  type Output = ConstTree;
-
-  fn add(self, rhs: ConstTree) -> Self::Output {
-    if let (Self::Tree(t1), Self::Tree(mut t2)) = (self, rhs) {
-      let mut product = HashMap::new();
-      for (key, i1) in t1 {
-        if let Some(i2) = t2.remove(&key) {
-          product.insert(key, i1 + i2);
-        } else {
-          product.insert(key, i1);
-        }
-      }
-      product.extend(t2.into_iter());
-      Self::Tree(product)
-    } else {
-      panic!("cannot combine tree and value fields")
-    }
-  }
-}
-
-fn from_const_tree_rec(
-  path: Substack<Tok<String>>,
-  consts: HashMap<Tok<String>, ConstTree>,
-  file: &[Tok<String>],
-) -> ProjectModule<VName> {
-  let mut items = HashMap::new();
-  let path_v = path.iter().rev_vec_clone();
-  for (name, item) in consts {
-    items.insert(name, ModEntry {
-      exported: true,
-      member: match item {
-        ConstTree::Const(c) => ModMember::Item(c),
-        ConstTree::Tree(t) =>
-          ModMember::Sub(from_const_tree_rec(path.push(name), t, file)),
-      },
-    });
-  }
-  let exports =
-    items.keys().map(|name| (*name, pushed(&path_v, *name))).collect();
-  Module {
-    items,
-    imports: vec![],
-    extra: ProjectExt {
-      exports,
-      file: Some(file.to_vec()),
-      ..Default::default()
-    },
-  }
-}
-
-/// Convert a map of [ConstTree] into a [ProjectTree] that can be used with the
-/// layered parsing system
-pub fn from_const_tree(
-  consts: HashMap<Tok<String>, ConstTree>,
-  file: &[Tok<String>],
-) -> ProjectTree<VName> {
-  let module = from_const_tree_rec(Substack::Bottom, consts, file);
-  ProjectTree(module)
-}

src/pipeline/project_tree/mod.rs

@@ -16,16 +16,9 @@
 mod add_prelude;
 mod build_tree;
 mod collect_ops;
-mod const_tree;
 mod normalize_imports;
 mod parse_file;
 mod prefix;
-mod tree;
 
 pub use build_tree::{build_tree, split_path};
 pub use collect_ops::InjectedOperatorsFn;
-pub use const_tree::{from_const_tree, ConstTree};
-pub use tree::{
-  collect_consts, collect_rules, vname_to_sym_tree, ProjectExt, ProjectModule,
-  ProjectTree,
-};

src/pipeline/project_tree/normalize_imports.rs

@@ -33,6 +33,12 @@ fn member_rec(
   }
 }
 
+/// Normalize imports in the FileEntry list recursively
+///
+/// # Panics
+///
+/// - if a path contains too many "super" prefixes
+/// - if the exported operators in a module cannot be determined
 fn entv_rec(
   // level
   mod_stack: Substack<Tok<String>>,
@@ -54,11 +60,9 @@ fn entv_rec(
             if let Import { name: None, path } = import {
               let p = import_abs_path(mod_path, mod_stack, &i.r(path)[..], i)
                 .expect("Should have emerged in preparsing");
-              let names = ops_cache
-                .find(&i.i(&p))
+              let names = (ops_cache.find(&i.i(&p)))
                 .expect("Should have emerged in second parsing");
-              let imports = names
-                .iter()
+              let imports = (names.iter())
                 .map(move |&n| Import { name: Some(n), path })
                 .collect::<Vec<_>>();
               Box::new(imports.into_iter()) as BoxedIter<Import>

src/pipeline/project_tree/parse_file.rs

@@ -10,6 +10,14 @@ use crate::pipeline::error::ProjectError;
 use crate::pipeline::source_loader::LoadedSourceTable;
 use crate::representations::sourcefile::{normalize_namespaces, FileEntry};
 
+/// Parses a file with the correct operator set
+///
+/// # Panics
+///
+/// - on syntax error
+/// - if namespaces are exported inconsistently
+///
+/// These are both checked in the preparsing stage
 pub fn parse_file(
   path: &[Tok<String>],
   loaded: &LoadedSourceTable,

src/pipeline/project_tree/tree.rs

@@ -1,144 +0,0 @@
-use std::ops::Add;
-
-use hashbrown::HashMap;
-
-use crate::ast::{Expr, Rule};
-use crate::interner::{Interner, Tok};
-use crate::representations::tree::{ModMember, Module};
-use crate::representations::NameLike;
-use crate::tree::ModEntry;
-use crate::utils::Substack;
-use crate::{Sym, VName};
-
-/// Additional data about a loaded module beyond the list of constants and
-/// submodules
-#[derive(Clone, Debug, Default)]
-pub struct ProjectExt<N: NameLike> {
-  /// Pairs each foreign token to the module it was imported from
-  pub imports_from: HashMap<Tok<String>, N>,
-  /// Pairs each exported token to its original full name
-  pub exports: HashMap<Tok<String>, N>,
-  /// All rules defined in this module, exported or not
-  pub rules: Vec<Rule<N>>,
-  /// Filename, if known, for error reporting
-  pub file: Option<Vec<Tok<String>>>,
-}
-
-impl<N: NameLike> Add for ProjectExt<N> {
-  type Output = Self;
-
-  fn add(mut self, rhs: Self) -> Self::Output {
-    let ProjectExt { imports_from, exports, rules, file } = rhs;
-    self.imports_from.extend(imports_from.into_iter());
-    self.exports.extend(exports.into_iter());
-    self.rules.extend(rules.into_iter());
-    if file.is_some() {
-      self.file = file
-    }
-    self
-  }
-}
-
-/// A node in the tree describing the project
-pub type ProjectModule<N> = Module<Expr<N>, ProjectExt<N>>;
-
-/// Module corresponding to the root of a project
-#[derive(Debug, Clone)]
-pub struct ProjectTree<N: NameLike>(pub ProjectModule<N>);
-
-fn collect_rules_rec<N: NameLike>(
-  bag: &mut Vec<Rule<N>>,
-  module: &ProjectModule<N>,
-) {
-  bag.extend(module.extra.rules.iter().cloned());
-  for item in module.items.values() {
-    if let ModMember::Sub(module) = &item.member {
-      collect_rules_rec(bag, module);
-    }
-  }
-}
-
-/// Collect the complete list of rules to be used by the rule repository from
-/// the [ProjectTree]
-pub fn collect_rules<N: NameLike>(project: &ProjectTree<N>) -> Vec<Rule<N>> {
-  let mut rules = Vec::new();
-  collect_rules_rec(&mut rules, &project.0);
-  rules
-}
-
-fn collect_consts_rec<N: NameLike>(
-  path: Substack<Tok<String>>,
-  bag: &mut HashMap<Sym, Expr<N>>,
-  module: &ProjectModule<N>,
-  i: &Interner,
-) {
-  for (key, entry) in module.items.iter() {
-    match &entry.member {
-      ModMember::Item(expr) => {
-        let mut name = path.iter().rev_vec_clone();
-        name.push(*key);
-        bag.insert(i.i(&name), expr.clone());
-      },
-      ModMember::Sub(module) =>
-        collect_consts_rec(path.push(*key), bag, module, i),
-    }
-  }
-}
-
-/// Extract the symbol table from a [ProjectTree]
-pub fn collect_consts<N: NameLike>(
-  project: &ProjectTree<N>,
-  i: &Interner,
-) -> HashMap<Sym, Expr<N>> {
-  let mut consts = HashMap::new();
-  collect_consts_rec(Substack::Bottom, &mut consts, &project.0, i);
-  consts
-}
-
-fn vname_to_sym_tree_rec(
-  tree: ProjectModule<VName>,
-  i: &Interner,
-) -> ProjectModule<Sym> {
-  let process_expr = |ex: Expr<VName>| ex.transform_names(&|n| i.i(&n));
-  ProjectModule {
-    imports: tree.imports,
-    items: (tree.items.into_iter())
-      .map(|(k, ModEntry { exported, member })| {
-        (k, ModEntry {
-          exported,
-          member: match member {
-            ModMember::Sub(module) =>
-              ModMember::Sub(vname_to_sym_tree_rec(module, i)),
-            ModMember::Item(ex) => ModMember::Item(process_expr(ex)),
-          },
-        })
-      })
-      .collect(),
-    extra: ProjectExt {
-      imports_from: (tree.extra.imports_from.into_iter())
-        .map(|(k, v)| (k, i.i(&v)))
-        .collect(),
-      exports: (tree.extra.exports.into_iter())
-        .map(|(k, v)| (k, i.i(&v)))
-        .collect(),
-      rules: (tree.extra.rules.into_iter())
-        .map(|Rule { pattern, prio, template }| Rule {
-          pattern: pattern.into_iter().map(process_expr).collect(),
-          prio,
-          template: template.into_iter().map(process_expr).collect(),
-        })
-        .collect(),
-      file: tree.extra.file,
-    },
-  }
-}
-
-/// Convert a flexible vname-based tree to a more rigid but faster symbol-based
-/// tree. The pipeline works with vnames, but the macro executor works with
-/// symbols.
-pub fn vname_to_sym_tree(
-  tree: ProjectTree<VName>,
-  i: &Interner,
-) -> ProjectTree<Sym> {
-  ProjectTree(vname_to_sym_tree_rec(tree.0, i))
-}

src/pipeline/source_loader/load_source.rs

@@ -73,6 +73,7 @@ fn load_abs_path_rec(
       Ok(Loaded::Code(_)) =>
         unreachable!("split_name returned None but the path is a file"),
       Err(e) => {
+        // todo: if this can actually be produced, return Err(ImportAll) instead
         let parent = abs_path.split_last().expect("import path nonzero").1;
         // exit without error if it was injected, or raise any IO error that was
         // previously swallowed