The pipeline is finally reasonably clean

src/pipeline/dealias/alias_cache.rs

@@ -0,0 +1,59 @@
+use std::slice;
+
+use chumsky::primitive::Container;
+use hashbrown::HashMap;
+
+use crate::representations::project::{ProjectMod, ItemKind, ProjectEntry};
+use crate::tree::ModMember;
+use crate::utils::{pushed, unwrap_or};
+use crate::{ProjectTree, VName, Tok, NameLike};
+
+use super::walk_with_links::{walk_with_links, Target};
+
+pub struct AliasCache {
+  data: HashMap<Vec<Tok<String>>, Option<Vec<Tok<String>>>>,
+}
+impl AliasCache {
+  pub fn new() -> Self {
+    Self { data: HashMap::new() }
+  }
+
+  /// Finds the absolute nsname corresponding to the given name in the given
+  /// context, if it's imported. If the name is defined locally, returns None
+  /// to avoid allocating several vectors for every local variable.
+  pub fn resolv_name<'a>(
+    &'a mut self,
+    root: &ProjectMod<VName>,
+    location: &[Tok<String>],
+    name: Tok<String>
+  ) -> Option<&'a [Tok<String>]> {
+    let full_path = pushed(location, name);
+    if let Some(result) = self.data.get(&full_path) {
+      return result.as_deref();
+    }
+    let (ent, finalp) = walk_with_links(root, location.iter().cloned())
+      .expect("This path should be valid");
+    let m = unwrap_or!{ent => Target::Mod; panic!("Must be a module")};
+    let result = m.extra.imports_from.get(&name).map(|next| {
+      self.resolv_name(root, &next, name).unwrap_or(&next)
+    });
+    self.data.insert(full_path, result.map(|s| s.to_vec()));
+    return result;
+  }
+
+  /// Find the absolute target of a 
+  pub fn resolv_vec<'a>(
+    &'a mut self,
+    root: &ProjectMod<VName>,
+    modname: &[Tok<String>],
+    vname: &[Tok<String>],
+  ) -> Option<&'a [Tok<String>]> {
+    let (name, ns) = vname.split_last().expect("name cannot be empty");
+    if ns.is_empty() {
+      self.resolv_name(modname, name)
+    } else {
+      let origin = self.resolv_vec(modname, ns)?;
+      self.resolv_name(origin, name)
+    }
+  }
+}

src/pipeline/dealias/mod.rs

@@ -0,0 +1,5 @@
+// mod alias_cache;
+mod resolve_aliases;
+mod walk_with_links;
+
+pub use resolve_aliases::resolve_aliases;

src/pipeline/dealias/resolve_aliases.rs

@@ -0,0 +1,95 @@
+use std::iter;
+
+use itertools::Itertools;
+
+use super::walk_with_links::walk_with_links;
+use crate::ast::{Expr, Rule};
+use crate::representations::project::{
+  ItemKind, ProjectExt, ProjectItem, ProjectMod,
+};
+use crate::tree::{ModEntry, ModMember, Module};
+use crate::utils::pushed::pushed;
+use crate::{Interner, ProjectTree, Tok, VName};
+
+fn resolve_aliases_rec(
+  root: &ProjectMod<VName>,
+  module: &ProjectMod<VName>,
+  updated: &impl Fn(&[Tok<String>]) -> bool,
+  is_root: bool,
+) -> ProjectMod<VName> {
+  if !is_root && !updated(&module.extra.path) {
+    return module.clone();
+  }
+  let process_expr = |expr: &Expr<VName>| {
+    expr
+      .map_names(&|n| {
+        let full_name = (module.extra.path.iter()).chain(n.iter()).cloned();
+        match walk_with_links(root, full_name, false) {
+          Ok(rep) => Some(rep.abs_path),
+          // Ok(_) => None,
+          Err(e) => {
+            let leftovers = e.tail.collect::<Vec<_>>();
+            if !leftovers.is_empty() {
+              let full_name = (module.extra.path.iter())
+                .chain(n.iter())
+                .cloned()
+                .collect::<Vec<_>>();
+              let _ = walk_with_links(root, full_name.iter().cloned(), true);
+              panic!(
+                "Invalid path {} while resolving {} should have been noticed \
+                 earlier",
+                (e.abs_path.into_iter())
+                  .chain(iter::once(e.name))
+                  .chain(leftovers.into_iter())
+                  .join("::"),
+                Interner::extern_all(&full_name).join("::"),
+              );
+            }
+            Some(pushed(e.abs_path, e.name))
+          },
+        }
+      })
+      .unwrap_or_else(|| expr.clone())
+  };
+  Module {
+    extra: ProjectExt {
+      path: module.extra.path.clone(),
+      file: module.extra.file.clone(),
+      imports_from: module.extra.imports_from.clone(),
+      rules: (module.extra.rules.iter())
+        .map(|Rule { pattern, prio, template }| Rule {
+          pattern: pattern.iter().map(process_expr).collect(),
+          template: template.iter().map(process_expr).collect(),
+          prio: *prio,
+        })
+        .collect(),
+    },
+    entries: module
+      .entries
+      .iter()
+      .map(|(k, v)| {
+        (k.clone(), ModEntry {
+          exported: v.exported,
+          member: match &v.member {
+            ModMember::Sub(module) =>
+              ModMember::Sub(resolve_aliases_rec(root, module, updated, false)),
+            ModMember::Item(item) => ModMember::Item(ProjectItem {
+              is_op: item.is_op,
+              kind: match &item.kind {
+                ItemKind::Const(value) => ItemKind::Const(process_expr(value)),
+                other => other.clone(),
+              },
+            }),
+          },
+        })
+      })
+      .collect(),
+  }
+}
+
+pub fn resolve_aliases(
+  project: ProjectTree<VName>,
+  updated: &impl Fn(&[Tok<String>]) -> bool,
+) -> ProjectTree<VName> {
+  ProjectTree(resolve_aliases_rec(&project.0, &project.0, updated, true))
+}

src/pipeline/dealias/walk_with_links.rs

@@ -0,0 +1,124 @@
+#[allow(unused)] // for doc
+use crate::representations::project::ProjectEntry;
+use crate::representations::project::{ItemKind, ProjectItem, ProjectMod};
+use crate::tree::ModMember;
+use crate::utils::{unwrap_or, BoxedIter};
+use crate::{Interner, NameLike, Tok, VName};
+
+/// The destination of a linked walk. [ProjectEntry] cannot be used for this
+/// purpose because it might be the project root.
+pub enum Target<'a, N: NameLike> {
+  Mod(&'a ProjectMod<N>),
+  Leaf(&'a ProjectItem<N>),
+}
+
+pub struct WalkReport<'a, N: NameLike> {
+  pub target: Target<'a, N>,
+  pub abs_path: VName,
+  pub aliased: bool,
+}
+
+pub struct LinkWalkError<'a> {
+  /// The last known valid path
+  pub abs_path: VName,
+  /// The name that wasn't found
+  pub name: Tok<String>,
+  /// Leftover steps
+  pub tail: BoxedIter<'a, Tok<String>>,
+  /// Whether an alias was ever encountered
+  pub aliased: bool,
+}
+
+fn walk_with_links_rec<'a, 'b, N: NameLike>(
+  mut abs_path: VName,
+  root: &'a ProjectMod<N>,
+  cur: &'a ProjectMod<N>,
+  prev_tgt: Target<'a, N>,
+  aliased: bool,
+  mut path: impl Iterator<Item = Tok<String>> + 'b,
+  l: bool,
+) -> Result<WalkReport<'a, N>, LinkWalkError<'b>> {
+  let name = unwrap_or! {path.next();
+    // ends on this module
+    return Ok(WalkReport{ target: prev_tgt, abs_path, aliased })
+  };
+  if l {
+    eprintln!(
+      "Resolving {} in {}",
+      name,
+      Interner::extern_all(&abs_path).join("::")
+    )
+  }
+  let entry = unwrap_or! {cur.entries.get(&name); {
+    // panic!("No entry {name} on {}", Interner::extern_all(&cur.extra.path).join("::"));
+    // leads into a missing branch
+    return Err(LinkWalkError{ abs_path, aliased, name, tail: Box::new(path) })
+  }};
+  match &entry.member {
+    ModMember::Sub(m) => {
+      // leads into submodule
+      abs_path.push(name);
+      walk_with_links_rec(abs_path, root, m, Target::Mod(m), aliased, path, l)
+    },
+    ModMember::Item(item) => match &item.kind {
+      ItemKind::Alias(alias) => {
+        // leads into alias (reset acc, cur, cur_entry)
+        if l {
+          eprintln!(
+            "{} points to {}",
+            Interner::extern_all(&abs_path).join("::"),
+            Interner::extern_all(alias).join("::")
+          )
+        }
+        abs_path.clone_from(alias);
+        abs_path.extend(path);
+        let path_acc = Vec::with_capacity(abs_path.len());
+        let new_path = abs_path.into_iter();
+        let tgt = Target::Mod(root);
+        walk_with_links_rec(path_acc, root, root, tgt, true, new_path, l)
+      },
+      ItemKind::Const(_) | ItemKind::None => {
+        abs_path.push(name);
+        match path.next() {
+          Some(name) => {
+            // leads into leaf
+            let tail = Box::new(path);
+            Err(LinkWalkError { abs_path, aliased, name, tail })
+          },
+          None => {
+            // ends on leaf
+            let target = Target::Leaf(item);
+            Ok(WalkReport { target, abs_path, aliased })
+          },
+        }
+      },
+    },
+  }
+}
+
+/// Execute a walk down the tree, following aliases.
+/// If the path ends on an alias, that alias is also resolved.
+/// If the path leads out of the tree, the shortest failing path is returned
+pub fn walk_with_links<'a, N: NameLike>(
+  root: &ProjectMod<N>,
+  path: impl Iterator<Item = Tok<String>> + 'a,
+  l: bool,
+) -> Result<WalkReport<'_, N>, LinkWalkError<'a>> {
+  let path_acc = path.size_hint().1.map_or_else(Vec::new, Vec::with_capacity);
+  let mut result = walk_with_links_rec(
+    path_acc,
+    root,
+    root,
+    Target::Mod(root),
+    false,
+    path,
+    l,
+  );
+  // cut off excess preallocated space within normal vector growth policy
+  let abs_path = match &mut result {
+    Ok(rep) => &mut rep.abs_path,
+    Err(err) => &mut err.abs_path,
+  };
+  abs_path.shrink_to(abs_path.len().next_power_of_two());
+  result
+}

src/pipeline/file_loader.rs

@@ -11,14 +11,14 @@ use crate::error::{ProjectError, ProjectResult};
 use crate::facade::System;
 use crate::interner::Interner;
 use crate::utils::Cache;
-use crate::{Location, Stok, VName};
+use crate::{Location, Stok, Tok, VName};
 
 /// All the data available about a failed source load call
 #[derive(Debug)]
 pub struct FileLoadingError {
   file: io::Error,
   dir: io::Error,
-  path: Vec<String>,
+  path: VName,
 }
 impl ProjectError for FileLoadingError {
   fn description(&self) -> &str {
@@ -54,8 +54,8 @@ pub type IOResult = ProjectResult<Loaded>;
 
 /// Load a file from a path expressed in Rust strings, but relative to
 /// a root expressed as an OS Path.
-pub fn load_file(root: &Path, path: &[impl AsRef<str>]) -> IOResult {
-  let full_path = path.iter().fold(root.to_owned(), |p, s| p.join(s.as_ref()));
+pub fn load_file(root: &Path, path: &[Tok<String>]) -> IOResult {
+  let full_path = path.iter().fold(root.to_owned(), |p, t| p.join(t.as_str()));
   let file_path = full_path.with_extension("orc");
   let file_error = match fs::read_to_string(file_path) {
     Ok(string) => return Ok(Loaded::Code(Rc::new(string))),
@@ -68,7 +68,7 @@ pub fn load_file(root: &Path, path: &[impl AsRef<str>]) -> IOResult {
         FileLoadingError {
           file: file_error,
           dir: dir_error,
-          path: path.iter().map(|s| s.as_ref().to_string()).collect(),
+          path: path.to_vec(),
         }
         .rc(),
       ),
@@ -90,10 +90,7 @@ pub fn load_file(root: &Path, path: &[impl AsRef<str>]) -> IOResult {
 
 /// Generates a cached file loader for a directory
 pub fn mk_dir_cache(root: PathBuf) -> Cache<'static, VName, IOResult> {
-  Cache::new(move |vname: VName, _this| -> IOResult {
-    let path = vname.iter().map(|t| t.as_str()).collect::<Vec<_>>();
-    load_file(&root, &path)
-  })
+  Cache::new(move |vname: VName, _this| load_file(&root, &vname))
 }
 
 /// Load a file from the specified path from an embed table

src/pipeline/import_abs_path.rs

@@ -1,29 +1,22 @@
-use crate::error::{ProjectError, ProjectResult, TooManySupers};
+use crate::error::ProjectResult;
 use crate::interner::{Interner, Tok};
 use crate::representations::sourcefile::absolute_path;
 use crate::utils::Substack;
+use crate::{Location, VName};
 
 pub fn import_abs_path(
   src_path: &[Tok<String>],
   mod_stack: Substack<Tok<String>>,
   import_path: &[Tok<String>],
   i: &Interner,
-) -> ProjectResult<Vec<Tok<String>>> {
+  location: &Location,
+) -> ProjectResult<VName> {
   // path of module within file
   let mod_pathv = mod_stack.iter().rev_vec_clone();
   // path of module within compilation
-  let abs_pathv = (src_path.iter())
-    .chain(mod_pathv.iter())
-    .cloned()
-    .collect::<Vec<_>>();
+  let abs_pathv =
+    (src_path.iter()).chain(mod_pathv.iter()).cloned().collect::<Vec<_>>();
   // preload-target path relative to module
   // preload-target path within compilation
-  absolute_path(&abs_pathv, import_path, i).map_err(|_| {
-    TooManySupers {
-      path: import_path.to_vec(),
-      offender_file: src_path.to_vec(),
-      offender_mod: mod_pathv,
-    }
-    .rc()
-  })
+  absolute_path(&abs_pathv, import_path, i, location)
 }

src/pipeline/import_resolution/alias_map.rs

@@ -2,19 +2,19 @@ use std::hash::Hash;
 
 use hashbrown::{HashMap, HashSet};
 
-use crate::interner::Tok;
+use crate::{interner::Tok, VName};
 
 #[derive(Clone, Debug, Default)]
 pub struct AliasMap {
-  pub targets: HashMap<Vec<Tok<String>>, Vec<Tok<String>>>,
-  pub aliases: HashMap<Vec<Tok<String>>, HashSet<Vec<Tok<String>>>>,
+  pub targets: HashMap<VName, VName>,
+  pub aliases: HashMap<VName, HashSet<VName>>,
 }
 impl AliasMap {
   pub fn new() -> Self {
     Self::default()
   }
 
-  pub fn link(&mut self, alias: Vec<Tok<String>>, target: Vec<Tok<String>>) {
+  pub fn link(&mut self, alias: VName, target: VName) {
     let prev = self.targets.insert(alias.clone(), target.clone());
     debug_assert!(prev.is_none(), "Alias already has a target");
     multimap_entry(&mut self.aliases, &target).insert(alias.clone());
@@ -36,7 +36,7 @@ impl AliasMap {
     }
   }
 
-  pub fn resolve(&self, alias: &[Tok<String>]) -> Option<&Vec<Tok<String>>> {
+  pub fn resolve(&self, alias: &[Tok<String>]) -> Option<&VName> {
     self.targets.get(alias)
   }
 }

src/pipeline/import_resolution/apply_aliases.rs

@@ -1,18 +1,16 @@
-use hashbrown::HashMap;
-
 use super::alias_map::AliasMap;
 use super::decls::{InjectedAsFn, UpdatedFn};
 use crate::ast::{Expr, Rule};
 use crate::interner::Tok;
-use crate::representations::project::{ProjectExt, ProjectModule};
-use crate::representations::tree::{ModEntry, ModMember};
+use crate::representations::project::{ItemKind, ProjectMod};
+use crate::representations::tree::ModMember;
 use crate::representations::VName;
 use crate::utils::Substack;
 
 fn resolve_rec(
   namespace: &[Tok<String>],
   alias_map: &AliasMap,
-) -> Option<Vec<Tok<String>>> {
+) -> Option<VName> {
   if let Some(alias) = alias_map.resolve(namespace) {
     Some(alias.clone())
   } else if let Some((foot, body)) = namespace.split_last() {
@@ -28,7 +26,7 @@ fn resolve(
   namespace: &[Tok<String>],
   alias_map: &AliasMap,
   injected_as: &impl InjectedAsFn,
-) -> Option<Vec<Tok<String>>> {
+) -> Option<VName> {
   injected_as(namespace).or_else(|| {
     let next_v = resolve_rec(namespace, alias_map)?;
     Some(injected_as(&next_v).unwrap_or(next_v))
@@ -45,69 +43,44 @@ fn process_expr(
     .unwrap_or_else(|| expr.clone())
 }
 
-// TODO: replace is_injected with injected_as
 /// Replace all aliases with the name they're originally defined as
 fn apply_aliases_rec(
   path: Substack<Tok<String>>,
-  module: &ProjectModule<VName>,
+  module: &mut ProjectMod<VName>,
   alias_map: &AliasMap,
   injected_as: &impl InjectedAsFn,
   updated: &impl UpdatedFn,
-) -> ProjectModule<VName> {
-  let items = (module.items.iter())
-    .map(|(name, ent)| {
-      let ModEntry { exported, member } = ent;
-      let member = match member {
-        ModMember::Item(expr) =>
-          ModMember::Item(process_expr(expr, alias_map, injected_as)),
-        ModMember::Sub(module) => {
-          let subpath = path.push(name.clone());
-          let new_mod = if !updated(&subpath.iter().rev_vec_clone()) {
-            module.clone()
-          } else {
-            apply_aliases_rec(subpath, module, alias_map, injected_as, updated)
-          };
-          ModMember::Sub(new_mod)
-        },
-      };
-      (name.clone(), ModEntry { exported: *exported, member })
-    })
-    .collect::<HashMap<_, _>>();
-  let rules = (module.extra.rules.iter())
-    .map(|rule| {
-      let Rule { pattern, prio, template } = rule;
-      Rule {
-        prio: *prio,
-        pattern: (pattern.iter())
-          .map(|expr| process_expr(expr, alias_map, injected_as))
-          .collect::<Vec<_>>(),
-        template: (template.iter())
-          .map(|expr| process_expr(expr, alias_map, injected_as))
-          .collect::<Vec<_>>(),
-      }
-    })
-    .collect::<Vec<_>>();
-  ProjectModule {
-    items,
-    imports: module.imports.clone(),
-    extra: ProjectExt {
-      rules,
-      exports: (module.extra.exports.iter())
-        .map(|(k, v)| {
-          (k.clone(), resolve(v, alias_map, injected_as).unwrap_or(v.clone()))
-        })
-        .collect(),
-      file: module.extra.file.clone(),
-      imports_from: module.extra.imports_from.clone(),
-    },
+) {
+  for (name, entry) in module.entries.iter_mut() {
+    match &mut entry.member {
+      ModMember::Sub(sub) => {
+        let subpath = path.push(name.clone());
+        apply_aliases_rec(subpath, sub, alias_map, injected_as, updated)
+      },
+      ModMember::Item(it) => match &mut it.kind {
+        ItemKind::None => (),
+        ItemKind::Const(expr) =>
+          *expr = process_expr(expr, alias_map, injected_as),
+        ItemKind::Alias(name) =>
+          if let Some(alt) = alias_map.resolve(&name) {
+            *name = alt.clone()
+          },
+      },
+      _ => (),
+    }
+  }
+  for Rule { pattern, prio, template } in module.extra.rules.iter_mut() {
+    for expr in pattern.iter_mut().chain(template.iter_mut()) {
+      *expr = process_expr(expr, alias_map, injected_as)
+    }
   }
 }
 
 pub fn apply_aliases(
-  module: &ProjectModule<VName>,
+  module: &mut ProjectMod<VName>,
   alias_map: &AliasMap,
   injected_as: &impl InjectedAsFn,
   updated: &impl UpdatedFn,
-) -> ProjectModule<VName> {
+) {
   apply_aliases_rec(Substack::Bottom, module, alias_map, injected_as, updated)
 }

src/pipeline/import_resolution/collect_aliases.rs

@@ -1,134 +1,32 @@
 use super::alias_map::AliasMap;
 use super::decls::UpdatedFn;
-use crate::error::{NotExported, NotFound, ProjectError, ProjectResult};
+use crate::error::ProjectResult;
 use crate::interner::Tok;
-use crate::pipeline::project_tree::split_path;
-use crate::representations::project::{ProjectModule, ProjectTree};
-use crate::representations::tree::{ModMember, WalkErrorKind};
+use crate::representations::project::{ProjectMod, ProjectTree};
+use crate::representations::tree::ModMember;
 use crate::representations::VName;
-use crate::utils::{pushed, unwrap_or, Substack};
-
-/// Assert that a module identified by a path can see a given symbol
-fn assert_visible(
-  source: &[Tok<String>], // must point to a file or submodule
-  target: &[Tok<String>], // may point to a symbol or module of any kind
-  project: &ProjectTree<VName>,
-) -> ProjectResult<()> {
-  let (tgt_item, tgt_path) = unwrap_or! {
-    target.split_last();
-    return Ok(())
-  };
-  let shared_len =
-    source.iter().zip(tgt_path.iter()).take_while(|(a, b)| a == b).count();
-  let vis_ignored_len = usize::min(tgt_path.len(), shared_len + 1);
-  let private_root = (project.0)
-    .walk_ref(&tgt_path[..vis_ignored_len], false)
-    .map_err(|e| match e.kind {
-      WalkErrorKind::Private =>
-        unreachable!("visibility is not being checked here"),
-      WalkErrorKind::Missing => NotFound::from_walk_error(
-        source,
-        &[],
-        &tgt_path[..vis_ignored_len],
-        &project.0,
-        e,
-      )
-      .rc(),
-    })?;
-  let direct_parent = private_root
-    .walk_ref(&tgt_path[vis_ignored_len..], true)
-    .map_err(|e| match e.kind {
-      WalkErrorKind::Missing => NotFound::from_walk_error(
-        source,
-        &tgt_path[..vis_ignored_len],
-        &tgt_path[vis_ignored_len..],
-        &project.0,
-        e,
-      )
-      .rc(),
-      WalkErrorKind::Private => {
-        let full_path = &tgt_path[..shared_len + e.pos];
-        // These errors are encountered during error reporting but they're more
-        // fundamental / higher prio than the error to be raised and would
-        // emerge nonetheless so they take over and the original error is
-        // swallowed
-        match split_path(full_path, project) {
-          Err(e) =>
-            NotFound::from_walk_error(source, &[], full_path, &project.0, e)
-              .rc(),
-          Ok((file, sub)) => {
-            let (ref_file, ref_sub) = split_path(source, project)
-              .expect("Source path assumed to be valid");
-            NotExported {
-              file: file.to_vec(),
-              subpath: sub.to_vec(),
-              referrer_file: ref_file.to_vec(),
-              referrer_subpath: ref_sub.to_vec(),
-            }
-            .rc()
-          },
-        }
-      },
-    })?;
-  let tgt_item_exported = direct_parent.extra.exports.contains_key(tgt_item);
-  let target_prefixes_source = shared_len == tgt_path.len();
-  if !tgt_item_exported && !target_prefixes_source {
-    let (file, sub) = split_path(target, project).map_err(|e| {
-      NotFound::from_walk_error(source, &[], target, &project.0, e).rc()
-    })?;
-    let (ref_file, ref_sub) = split_path(source, project)
-      .expect("The source path is assumed to be valid");
-    Err(
-      NotExported {
-        file: file.to_vec(),
-        subpath: sub.to_vec(),
-        referrer_file: ref_file.to_vec(),
-        referrer_subpath: ref_sub.to_vec(),
-      }
-      .rc(),
-    )
-  } else {
-    Ok(())
-  }
-}
+use crate::utils::{pushed, unwrap_or};
 
 /// Populate target and alias maps from the module tree recursively
 fn collect_aliases_rec(
-  path: Substack<Tok<String>>,
-  module: &ProjectModule<VName>,
+  path: Vec<Tok<String>>,
+  module: &ProjectMod<VName>,
   project: &ProjectTree<VName>,
   alias_map: &mut AliasMap,
   updated: &impl UpdatedFn,
 ) -> ProjectResult<()> {
   // Assume injected module has been alias-resolved
-  let mod_path_v = path.iter().rev_vec_clone();
-  if !updated(&mod_path_v) {
+  if !updated(&path) {
     return Ok(());
   };
-  for (name, target_mod_name) in module.extra.imports_from.iter() {
-    let target_sym_v = pushed(target_mod_name, name.clone());
-    assert_visible(&mod_path_v, &target_sym_v, project)?;
-    let sym_path_v = pushed(&mod_path_v, name.clone());
-    let target_mod = (project.0.walk_ref(target_mod_name, false))
-      .expect("checked above in assert_visible");
-    let target_sym = (target_mod.extra.exports.get(name))
-      .ok_or_else(|| {
-        let file_len =
-          target_mod.extra.file.as_ref().unwrap_or(target_mod_name).len();
-        NotFound {
-          source: Some(mod_path_v.clone()),
-          file: target_mod_name[..file_len].to_vec(),
-          subpath: target_sym_v[file_len..].to_vec(),
-        }
-        .rc()
-      })?
-      .clone();
-    alias_map.link(sym_path_v, target_sym);
+  for (name, target_sym_v) in module.extra.imports_from.iter() {
+    let sym_path_v = pushed(&path, name.clone());
+    alias_map.link(sym_path_v, target_sym_v.clone());
   }
-  for (name, entry) in module.items.iter() {
+  for (name, entry) in module.entries.iter() {
     let submodule = unwrap_or!(&entry.member => ModMember::Sub; continue);
     collect_aliases_rec(
-      path.push(name.clone()),
+      pushed(&path, name.clone()),
       submodule,
       project,
       alias_map,
@@ -140,10 +38,10 @@ fn collect_aliases_rec(
 
 /// Populate target and alias maps from the module tree
 pub fn collect_aliases(
-  module: &ProjectModule<VName>,
+  module: &ProjectMod<VName>,
   project: &ProjectTree<VName>,
   alias_map: &mut AliasMap,
   updated: &impl UpdatedFn,
 ) -> ProjectResult<()> {
-  collect_aliases_rec(Substack::Bottom, module, project, alias_map, updated)
+  collect_aliases_rec(Vec::new(), module, project, alias_map, updated)
 }

src/pipeline/import_resolution/decls.rs

@@ -1,8 +1,8 @@
 use trait_set::trait_set;
 
-use crate::interner::Tok;
+use crate::{interner::Tok, VName};
 
 trait_set! {
-  pub trait InjectedAsFn = Fn(&[Tok<String>]) -> Option<Vec<Tok<String>>>;
+  pub trait InjectedAsFn = Fn(&[Tok<String>]) -> Option<VName>;
   pub trait UpdatedFn = Fn(&[Tok<String>]) -> bool;
 }

src/pipeline/import_resolution/mod.rs

@@ -3,5 +3,6 @@ mod apply_aliases;
 mod collect_aliases;
 mod decls;
 mod resolve_imports;
+mod alias_cache;
 
 pub use resolve_imports::resolve_imports;

src/pipeline/import_resolution/resolve_imports.rs

@@ -1,3 +1,4 @@
+use super::alias_cache::AliasCache;
 use super::alias_map::AliasMap;
 use super::apply_aliases::apply_aliases;
 use super::collect_aliases::collect_aliases;
@@ -9,12 +10,13 @@ use crate::representations::VName;
 /// Follow import chains to locate the original name of all tokens, then
 /// replace these aliases with the original names throughout the tree
 pub fn resolve_imports(
-  project: ProjectTree<VName>,
+  mut project: ProjectTree<VName>,
   injected_as: &impl InjectedAsFn,
   updated: &impl UpdatedFn,
 ) -> ProjectResult<ProjectTree<VName>> {
-  let mut map = AliasMap::new();
-  collect_aliases(&project.0, &project, &mut map, updated)?;
-  let new_mod = apply_aliases(&project.0, &map, injected_as, updated);
-  Ok(ProjectTree(new_mod))
+  let mut cache = AliasCache::new(&project);
+  // let mut map = AliasMap::new();
+  // collect_aliases(&project.0, &project, &mut map, updated)?;
+  // apply_aliases(&mut project.0, &map, injected_as, updated);
+  Ok(project)
 }

src/pipeline/mod.rs

@@ -1,7 +1,8 @@
 //! Loading Orchid modules from source
 pub mod file_loader;
 mod import_abs_path;
-mod import_resolution;
+// mod import_resolution;
+mod dealias;
 mod parse_layer;
 mod project_tree;
 mod source_loader;

src/pipeline/parse_layer.rs

@@ -1,11 +1,11 @@
-use std::rc::Rc;
-
+use super::dealias::resolve_aliases;
 use super::file_loader::IOResult;
-use super::{import_resolution, project_tree, source_loader};
+use super::{project_tree, source_loader};
 use crate::error::ProjectResult;
 use crate::interner::{Interner, Tok};
 use crate::representations::sourcefile::FileEntry;
 use crate::representations::VName;
+use crate::utils::never;
 use crate::ProjectTree;
 
 /// Using an IO callback, produce a project tree that includes the given
@@ -23,26 +23,17 @@ pub fn parse_layer<'a>(
   prelude: &[FileEntry],
   i: &Interner,
 ) -> ProjectResult<ProjectTree<VName>> {
-  // A path is injected if it is walkable in the injected tree
-  let injected_as = |path: &[Tok<String>]| {
-    let (item, modpath) = path.split_last()?;
-    let module = environment.0.walk_ref(modpath, false).ok()?;
-    module.extra.exports.get(item).cloned()
-  };
-  let injected_names = |path: Tok<Vec<Tok<String>>>| {
-    let module = environment.0.walk_ref(&path, false).ok()?;
-    Some(Rc::new(module.extra.exports.keys().cloned().collect()))
-  };
-  let source =
+  let (preparsed, source) =
     source_loader::load_source(targets, prelude, i, loader, &|path| {
-      environment.0.walk_ref(path, false).is_ok()
+      environment.0.walk_ref(&[], path, false).is_ok()
     })?;
-  let tree = project_tree::build_tree(source, i, prelude, &injected_names)?;
-  let sum = ProjectTree(environment.0.clone().overlay(tree.0.clone()));
+  let tree =
+    project_tree::rebuild_tree(&source, preparsed, environment, prelude, i)?;
+  let sum = ProjectTree(never::unwrap_always(
+    environment.0.clone().overlay(tree.0.clone()),
+  ));
   let resolvd =
-    import_resolution::resolve_imports(sum, &injected_as, &|path| {
-      tree.0.walk_ref(path, false).is_ok()
-    })?;
+    resolve_aliases(sum, &|path| tree.0.walk1_ref(&[], path, false).is_ok());
   // Addition among modules favours the left hand side.
   Ok(resolvd)
 }

src/pipeline/project_tree/add_prelude.rs

@@ -1,46 +0,0 @@
-use crate::interner::Tok;
-use crate::representations::sourcefile::{FileEntry, Member, ModuleBlock};
-
-fn member_rec(
-  // object
-  member: Member,
-  // context
-  path: &[Tok<String>],
-  prelude: &[FileEntry],
-) -> Member {
-  match member {
-    Member::Module(ModuleBlock { name, body }) => {
-      let new_body = entv_rec(body, path, prelude);
-      Member::Module(ModuleBlock { name, body: new_body })
-    },
-    any => any,
-  }
-}
-
-fn entv_rec(
-  // object
-  data: Vec<FileEntry>,
-  // context
-  mod_path: &[Tok<String>],
-  prelude: &[FileEntry],
-) -> Vec<FileEntry> {
-  prelude
-    .iter()
-    .cloned()
-    .chain(data.into_iter().map(|ent| match ent {
-      FileEntry::Exported(mem) =>
-        FileEntry::Exported(member_rec(mem, mod_path, prelude)),
-      FileEntry::Internal(mem) =>
-        FileEntry::Internal(member_rec(mem, mod_path, prelude)),
-      any => any,
-    }))
-    .collect()
-}
-
-pub fn add_prelude(
-  data: Vec<FileEntry>,
-  path: &[Tok<String>],
-  prelude: &[FileEntry],
-) -> Vec<FileEntry> {
-  entv_rec(data, path, prelude)
-}

src/pipeline/project_tree/build_tree.rs

@@ -1,246 +1,144 @@
 use hashbrown::HashMap;
-use itertools::Itertools;
+use itertools::{Either, Itertools};
 
-use super::collect_ops;
-use super::collect_ops::InjectedOperatorsFn;
-use super::parse_file::parse_file;
-use crate::ast::{Clause, Constant, Expr};
-use crate::error::{ProjectError, ProjectResult, TooManySupers};
-use crate::interner::{Interner, Tok};
-use crate::pipeline::source_loader::{LoadedSource, LoadedSourceTable};
-use crate::representations::project::{ProjectExt, ProjectTree};
-use crate::representations::sourcefile::{absolute_path, FileEntry, Member};
-use crate::representations::tree::{ModEntry, ModMember, Module};
-use crate::representations::{NameLike, VName};
-use crate::tree::{WalkError, WalkErrorKind};
-use crate::utils::iter::{box_empty, box_once};
-use crate::utils::{pushed, unwrap_or, Substack};
+use super::import_tree::ImpMod;
+use crate::ast::{Constant, Rule};
+use crate::error::{ConflictingRoles, ProjectError, ProjectResult};
+use crate::pipeline::source_loader::{PreItem, PreMod};
+use crate::representations::project::{
+  ImpReport, ItemKind, ProjectEntry, ProjectExt, ProjectItem,
+};
+use crate::sourcefile::{
+  FileEntry, FileEntryKind, Member, MemberKind, ModuleBlock,
+};
+use crate::tree::{ModEntry, ModMember, Module};
+use crate::utils::get_or_default;
+use crate::utils::pushed::pushed_ref;
+use crate::{Tok, VName};
 
-#[derive(Debug)]
-struct ParsedSource<'a> {
-  path: Vec<Tok<String>>,
-  loaded: &'a LoadedSource,
-  parsed: Vec<FileEntry>,
-}
-
-/// Split a path into file- and subpath in knowledge
-///
-/// # Errors
-///
-/// if the path is invalid
-#[allow(clippy::type_complexity)] // bit too sensitive here IMO
-pub fn split_path<'a>(
-  path: &'a [Tok<String>],
-  proj: &'a ProjectTree<impl NameLike>,
-) -> Result<(&'a [Tok<String>], &'a [Tok<String>]), WalkError> {
-  let (end, body) = unwrap_or!(path.split_last(); {
-    return Ok((&[], &[]))
-  });
-  let mut module = (proj.0.walk_ref(body, false))?;
-  let entry = (module.items.get(end))
-    .ok_or(WalkError { pos: path.len() - 1, kind: WalkErrorKind::Missing })?;
-  if let ModMember::Sub(m) = &entry.member {
-    module = m;
-  }
-  let file =
-    module.extra.file.as_ref().map(|s| &path[..s.len()]).unwrap_or(path);
-  let subpath = &path[file.len()..];
-  Ok((file, subpath))
-}
-
-/// Convert normalized, prefixed source into a module
-///
-/// # Panics
-///
-/// - if there are imports with too many "super" prefixes (this is normally
-///   detected in preparsing)
-/// - if the preparsed tree is missing a module that exists in the source
-fn source_to_module(
-  // level
-  path: Substack<Tok<String>>,
-  preparsed: &Module<impl Clone, impl Clone>,
-  // data
-  data: Vec<FileEntry>,
-  // context
-  i: &Interner,
-  filepath_len: usize,
-) -> ProjectResult<Module<Expr<VName>, ProjectExt<VName>>> {
-  let path_v = path.iter().rev_vec_clone();
-  let imports = (data.iter())
-    .filter_map(|ent| {
-      if let FileEntry::Import(impv) = ent { Some(impv.iter()) } else { None }
-    })
-    .flatten()
-    .cloned()
-    .collect::<Vec<_>>();
-  let imports_from = (imports.iter())
-    .map(|imp| -> ProjectResult<_> {
-      let mut imp_path_v = imp.path.clone();
-      imp_path_v
-        .push(imp.name.clone().expect("glob imports had just been resolved"));
-      let mut abs_path = absolute_path(&path_v, &imp_path_v, i)
-        .expect("should have failed in preparsing");
-      let name = abs_path.pop().ok_or_else(|| {
-        TooManySupers {
-          offender_file: path_v[..filepath_len].to_vec(),
-          offender_mod: path_v[filepath_len..].to_vec(),
-          path: imp_path_v,
-        }
-        .rc()
-      })?;
-      Ok((name, abs_path))
-    })
-    .collect::<Result<HashMap<_, _>, _>>()?;
-  let exports = (data.iter())
-    .flat_map(|ent| {
-      let mk_ent = |name: Tok<String>| (name.clone(), pushed(&path_v, name));
-      match ent {
-        FileEntry::Export(names) => Box::new(names.iter().cloned().map(mk_ent)),
-        FileEntry::Exported(mem) => match mem {
-          Member::Constant(constant) => box_once(mk_ent(constant.name.clone())),
-          Member::Module(ns) => box_once(mk_ent(ns.name.clone())),
-          Member::Rule(rule) => {
-            let mut names = Vec::new();
-            for e in rule.pattern.iter() {
-              e.search_all(&mut |e| {
-                if let Clause::Name(n) = &e.value {
-                  if let Some([name]) = n.strip_prefix(&path_v[..]) {
-                    names.push((name.clone(), n.clone()))
-                  }
-                }
-                None::<()>
-              });
-            }
-            Box::new(names.into_iter())
-          },
-        },
-        _ => box_empty(),
-      }
-    })
-    .collect::<HashMap<_, _>>();
-  let rules = (data.iter())
-    .filter_map(|ent| match ent {
-      FileEntry::Exported(Member::Rule(rule)) => Some(rule),
-      FileEntry::Internal(Member::Rule(rule)) => Some(rule),
-      _ => None,
-    })
-    .cloned()
-    .collect::<Vec<_>>();
-  let items = (data.into_iter())
-    .filter_map(|ent| {
-      let member_to_item = |exported, member| match member {
-        Member::Module(ns) => {
-          let new_prep = unwrap_or!(
-            &preparsed.items[&ns.name].member => ModMember::Sub;
-            panic!("Preparsed should include entries for all submodules")
-          );
-          let module = match source_to_module(
-            path.push(ns.name.clone()),
-            new_prep,
-            ns.body,
-            i,
-            filepath_len,
-          ) {
-            Err(e) => return Some(Err(e)),
-            Ok(t) => t,
-          };
-          let member = ModMember::Sub(module);
-          Some(Ok((ns.name.clone(), ModEntry { exported, member })))
-        },
-        Member::Constant(Constant { name, value }) => {
-          let member = ModMember::Item(value);
-          Some(Ok((name, ModEntry { exported, member })))
-        },
-        _ => None,
-      };
-      match ent {
-        FileEntry::Exported(member) => member_to_item(true, member),
-        FileEntry::Internal(member) => member_to_item(false, member),
-        _ => None,
-      }
-    })
-    .collect::<Result<HashMap<_, _>, _>>()?;
-  Ok(Module {
-    imports,
-    items,
-    extra: ProjectExt {
-      imports_from,
-      exports,
-      rules,
-      file: Some(path_v[..filepath_len].to_vec()),
-    },
-  })
-}
-
-fn files_to_module(
-  path: Substack<Tok<String>>,
-  files: Vec<ParsedSource>,
-  i: &Interner,
-) -> ProjectResult<Module<Expr<VName>, ProjectExt<VName>>> {
-  let lvl = path.len();
-  debug_assert!(
-    files.iter().map(|f| f.path.len()).max().unwrap() >= lvl,
-    "path is longer than any of the considered file paths"
-  );
-  let path_v = path.iter().rev_vec_clone();
-  if files.len() == 1 && files[0].path.len() == lvl {
-    return source_to_module(
-      path.clone(),
-      &files[0].loaded.preparsed.0,
-      files[0].parsed.clone(),
-      i,
-      path.len(),
-    );
-  }
-  let items = (files.into_iter())
-    .group_by(|f| f.path[lvl].clone())
-    .into_iter()
-    .map(|(namespace, files)| -> ProjectResult<_> {
-      let subpath = path.push(namespace.clone());
-      let files_v = files.collect::<Vec<_>>();
-      let module = files_to_module(subpath, files_v, i)?;
-      let member = ModMember::Sub(module);
-      Ok((namespace, ModEntry { exported: true, member }))
-    })
-    .collect::<Result<HashMap<_, _>, _>>()?;
-  let exports: HashMap<_, _> = (items.keys())
-    .map(|name| (name.clone(), pushed(&path_v, name.clone())))
-    .collect();
-  Ok(Module {
-    items,
-    imports: vec![],
-    extra: ProjectExt {
-      exports,
-      imports_from: HashMap::new(),
-      rules: vec![],
-      file: None,
-    },
-  })
+pub struct TreeReport {
+  pub entries: HashMap<Tok<String>, ProjectEntry<VName>>,
+  pub rules: Vec<Rule<VName>>,
+  /// Maps imported symbols to the absolute paths of the modules they are
+  /// imported from
+  pub imports_from: HashMap<Tok<String>, ImpReport<VName>>,
 }
 
 pub fn build_tree(
-  files: LoadedSourceTable,
-  i: &Interner,
+  path: &VName,
+  source: Vec<FileEntry>,
+  Module { entries, extra }: PreMod,
+  imports: ImpMod,
   prelude: &[FileEntry],
-  injected: &impl InjectedOperatorsFn,
-) -> ProjectResult<ProjectTree<VName>> {
-  assert!(!files.is_empty(), "A tree requires at least one module");
-  let ops_cache = collect_ops::mk_cache(&files, injected);
-  let mut entries = files
-    .iter()
-    .map(|(path, loaded)| {
-      Ok((path, loaded, parse_file(path, &files, &ops_cache, i, prelude)?))
+) -> ProjectResult<TreeReport> {
+  let source =
+    source.into_iter().chain(prelude.iter().cloned()).collect::<Vec<_>>();
+  let (imports_from, mut submod_imports) = (imports.entries.into_iter())
+    .partition_map::<HashMap<_, _>, HashMap<_, _>, _, _, _>(
+      |(n, ent)| match ent.member {
+        ModMember::Item(it) => Either::Left((n, it)),
+        ModMember::Sub(s) => Either::Right((n, s)),
+      },
+    );
+  let mut rule_fragments = Vec::new();
+  let mut submodules = HashMap::<_, Vec<_>>::new();
+  let mut consts = HashMap::new();
+  for FileEntry { kind, locations: _ } in source {
+    match kind {
+      FileEntryKind::Import(_) => (),
+      FileEntryKind::Comment(_) => (),
+      FileEntryKind::Export(_) => (),
+      FileEntryKind::Member(Member { kind, .. }) => match kind {
+        MemberKind::Module(ModuleBlock { body, name }) => {
+          get_or_default(&mut submodules, &name).extend(body.into_iter());
+        },
+        MemberKind::Constant(Constant { name, value }) => {
+          consts.insert(name, value /* .prefix(path, &|_| false) */);
+        },
+        MemberKind::Operators(_) => (),
+        MemberKind::Rule(rule) => rule_fragments.push(rule),
+      },
+    }
+  }
+  let mod_details = extra.details().expect("Directories handled elsewhere");
+  let rules = (mod_details.patterns.iter())
+    .zip(rule_fragments.into_iter())
+    .map(|(p, Rule { prio, template: t, .. })| {
+      // let p = p.iter().map(|e| e.prefix(path, &|_| false)).collect();
+      // let t = t.into_iter().map(|e| e.prefix(path, &|_| false)).collect();
+      Rule { pattern: p.clone(), prio, template: t }
     })
-    .collect::<ProjectResult<Vec<_>>>()?;
-  // sort by similarity, then longest-first
-  entries.sort_unstable_by(|a, b| a.0.cmp(b.0).reverse());
-  let files = entries
-    .into_iter()
-    .map(|(path, loaded, parsed)| ParsedSource {
-      loaded,
-      parsed,
-      path: path.clone(),
+    .collect();
+  let (pre_subs, pre_items) = (entries.into_iter())
+    .partition_map::<HashMap<_, _>, HashMap<_, _>, _, _, _>(
+      |(k, ModEntry { exported, member })| match member {
+        ModMember::Sub(s) => Either::Left((k, (exported, s))),
+        ModMember::Item(it) => Either::Right((k, (exported, it))),
+      },
+    );
+  let mut entries = (pre_subs.into_iter())
+    .map(|(k, (exported, pre_member))| {
+      let impmod = (submod_imports.remove(&k))
+        .expect("Imports and preparsed should line up");
+      (k, exported, pre_member, impmod)
     })
-    .collect::<Vec<_>>();
-  Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)?))
+    .map(|(k, exported, pre, imp)| {
+      let source = submodules
+        .remove(&k)
+        .expect("Submodules should not disappear after reparsing");
+      (k, exported, pre, imp, source)
+    })
+    .map(|(k, exported, pre, imp, source)| {
+      let path = pushed_ref(path, k.clone());
+      let TreeReport { entries, imports_from, rules } =
+        build_tree(&path, source, pre, imp, prelude)?;
+      let extra = ProjectExt { path, file: None, imports_from, rules };
+      let member = ModMember::Sub(Module { entries, extra });
+      Ok((k, ModEntry { exported, member }))
+    })
+    .chain((pre_items.into_iter()).map(
+      |(k, (exported, PreItem { has_value, is_op, location }))| {
+        let item = match imports_from.get(&k) {
+          Some(_) if has_value => {
+            // Local value cannot be assigned to imported key
+            let const_loc =
+              consts.remove(&k).expect("has_value is true").location;
+            let err = ConflictingRoles {
+              locations: vec![location, const_loc],
+              name: pushed_ref(path, k),
+            };
+            return Err(err.rc());
+          },
+          None => {
+            let k = consts.remove(&k).map_or(ItemKind::None, ItemKind::Const);
+            ProjectItem { is_op, kind: k }
+          },
+          Some(report) => ProjectItem {
+            is_op: is_op | report.is_op,
+            kind: ItemKind::Alias(report.source.clone()),
+          },
+        };
+        Ok((k, ModEntry { exported, member: ModMember::Item(item) }))
+      },
+    ))
+    .collect::<Result<HashMap<_, _>, _>>()?;
+  for (k, from) in imports_from.iter() {
+    let (_, ent) = entries.raw_entry_mut().from_key(k).or_insert_with(|| {
+      (k.clone(), ModEntry {
+        exported: false,
+        member: ModMember::Item(ProjectItem {
+          kind: ItemKind::Alias(from.source.clone()),
+          is_op: from.is_op,
+        }),
+      })
+    });
+    debug_assert!(
+      matches!(
+        ent.member,
+        ModMember::Item(ProjectItem { kind: ItemKind::Alias(_), .. })
+      ),
+      "Should have emerged in the processing of pre_items"
+    )
+  }
+  Ok(TreeReport { entries, rules, imports_from })
 }

src/pipeline/project_tree/collect_ops/exported_ops.rs

@@ -1,79 +0,0 @@
-use std::rc::Rc;
-
-use hashbrown::HashSet;
-use trait_set::trait_set;
-
-use crate::error::{NotFound, ProjectError, ProjectResult};
-use crate::interner::Tok;
-use crate::pipeline::source_loader::LoadedSourceTable;
-use crate::representations::tree::WalkErrorKind;
-use crate::utils::{split_max_prefix, Cache};
-use crate::Sym;
-
-pub type OpsResult = ProjectResult<Rc<HashSet<Tok<String>>>>;
-pub type ExportedOpsCache<'a> = Cache<'a, Sym, OpsResult>;
-
-trait_set! {
-  pub trait InjectedOperatorsFn = Fn(Sym) -> Option<Rc<HashSet<Tok<String>>>>;
-}
-
-fn coprefix<T: Eq>(
-  l: impl Iterator<Item = T>,
-  r: impl Iterator<Item = T>,
-) -> usize {
-  l.zip(r).take_while(|(a, b)| a == b).count()
-}
-
-/// Collect all names exported by the module at the specified path
-pub fn collect_exported_ops(
-  path: Sym,
-  loaded: &LoadedSourceTable,
-  injected: &impl InjectedOperatorsFn,
-) -> OpsResult {
-  let injected =
-    injected(path.clone()).unwrap_or_else(|| Rc::new(HashSet::new()));
-  match split_max_prefix(&path, &|n| loaded.contains_key(n)) {
-    None => {
-      let ops = (loaded.keys())
-        .filter_map(|modname| {
-          if path.len() == coprefix(path.iter(), modname.iter()) {
-            Some(modname[path.len()].clone())
-          } else {
-            None
-          }
-        })
-        .chain(injected.iter().cloned())
-        .collect::<HashSet<_>>();
-      Ok(Rc::new(ops))
-    },
-    Some((fpath, subpath)) => {
-      let preparsed = &loaded[fpath].preparsed;
-      let module = preparsed.0.walk_ref(subpath, false).map_err(
-        |walk_err| match walk_err.kind {
-          WalkErrorKind::Private => {
-            unreachable!("visibility is not being checked here")
-          },
-          WalkErrorKind::Missing => NotFound {
-            source: None,
-            file: fpath.to_vec(),
-            subpath: subpath[..walk_err.pos].to_vec(),
-          }
-          .rc(),
-        },
-      )?;
-      let out = (module.items.iter())
-        .filter(|(_, v)| v.exported)
-        .map(|(k, _)| k.clone())
-        .chain(injected.iter().cloned())
-        .collect::<HashSet<_>>();
-      Ok(Rc::new(out))
-    },
-  }
-}
-
-pub fn mk_cache<'a>(
-  loaded: &'a LoadedSourceTable,
-  injected: &'a impl InjectedOperatorsFn,
-) -> ExportedOpsCache<'a> {
-  Cache::new(|path, _this| collect_exported_ops(path, loaded, injected))
-}

src/pipeline/project_tree/collect_ops/mod.rs

@@ -1,8 +0,0 @@
-mod exported_ops;
-mod ops_for;
-
-pub use exported_ops::{
-  collect_exported_ops, mk_cache, ExportedOpsCache, InjectedOperatorsFn,
-  OpsResult,
-};
-pub use ops_for::collect_ops_for;

src/pipeline/project_tree/collect_ops/ops_for.rs

@@ -1,53 +0,0 @@
-use std::rc::Rc;
-
-use hashbrown::HashSet;
-
-use super::exported_ops::{ExportedOpsCache, OpsResult};
-use crate::error::ProjectResult;
-use crate::interner::{Interner, Tok};
-use crate::parse::is_op;
-use crate::pipeline::import_abs_path::import_abs_path;
-use crate::pipeline::source_loader::LoadedSourceTable;
-use crate::representations::tree::{ModMember, Module};
-
-/// Collect all operators and names, exported or local, defined in this
-/// tree.
-fn tree_all_ops(
-  module: &Module<impl Clone, impl Clone>,
-  ops: &mut HashSet<Tok<String>>,
-) {
-  ops.extend(module.items.keys().cloned());
-  for ent in module.items.values() {
-    if let ModMember::Sub(m) = &ent.member {
-      tree_all_ops(m, ops);
-    }
-  }
-}
-
-/// Collect all names visible in this file
-///
-/// # Panics
-///
-/// if any import contains too many Super calls. This should be caught during
-/// preparsing
-pub fn collect_ops_for(
-  file: &[Tok<String>],
-  loaded: &LoadedSourceTable,
-  ops_cache: &ExportedOpsCache,
-  i: &Interner,
-) -> OpsResult {
-  let tree = &loaded[file].preparsed.0;
-  let mut ret = HashSet::new();
-  tree_all_ops(tree, &mut ret);
-  tree.visit_all_imports(&mut |modpath, _m, import| -> ProjectResult<()> {
-    if let Some(n) = &import.name {
-      ret.insert(n.clone());
-    } else {
-      let path = import_abs_path(file, modpath, &import.path, i)
-        .expect("This error should have been caught during loading");
-      ret.extend(ops_cache.find(&i.i(&path))?.iter().cloned());
-    }
-    Ok(())
-  })?;
-  Ok(Rc::new(ret.into_iter().filter(|t| is_op(&**t)).collect()))
-}

src/pipeline/project_tree/import_tree.rs

@@ -0,0 +1,169 @@
+use std::cmp::Ordering;
+use std::fmt::Display;
+use std::rc::Rc;
+
+use hashbrown::HashMap;
+use itertools::Itertools;
+
+use crate::error::{ProjectError, ProjectResult};
+use crate::pipeline::source_loader::{PreMod, Preparsed};
+use crate::representations::project::ImpReport;
+use crate::sourcefile::{absolute_path, Import};
+use crate::tree::{ErrKind, ModEntry, ModMember, Module, WalkError};
+use crate::utils::iter::{box_chain, box_once};
+use crate::utils::pushed::pushed_ref;
+use crate::utils::{unwrap_or, BoxedIter};
+use crate::{Interner, ProjectTree, Tok, VName};
+
+pub type ImpMod = Module<ImpReport<VName>, ()>;
+
+/// Assert that a module identified by a path can see a given symbol
+fn assert_visible<'a>(
+  source: &'a [Tok<String>], // must point to a file or submodule
+  target: &'a [Tok<String>],
+  root: &'a Module<impl Clone + Display, impl Clone + Display>,
+) -> Result<(), WalkError<'a>> {
+  if target.split_last().map_or(true, |(_, m)| source.starts_with(m)) {
+    // The global module (empty path) is always visible
+    return Ok(()); // Symbols in ancestor modules are always visible
+  }
+  // walk the first section where visibility is ignored
+  let shared_len =
+    source.iter().zip(target.iter()).take_while(|(a, b)| a == b).count();
+  let (shared_path, deep_path) = target.split_at(shared_len + 1);
+  let private_root = root.walk_ref(&[], shared_path, false)?;
+  // walk the second part where visibility matters
+  private_root.walk1_ref(shared_path, deep_path, true)?;
+  Ok(())
+}
+
+pub fn assert_visible_overlay<'a>(
+  source: &'a [Tok<String>], // must point to a file or submodule
+  target: &'a [Tok<String>],
+  first: &'a Module<impl Clone + Display, impl Clone + Display>,
+  fallback: &'a Module<impl Clone + Display, impl Clone + Display>,
+) -> Result<(), WalkError<'a>> {
+  assert_visible(source, target, first).or_else(|e1| {
+    if e1.kind == ErrKind::Missing {
+      match assert_visible(source, target, fallback) {
+        // if both are walk errors, report the longer of the two
+        Err(mut e2) if e2.kind == ErrKind::Missing =>
+          Err(match e1.depth().cmp(&e2.depth()) {
+            Ordering::Less => e2,
+            Ordering::Greater => e1,
+            Ordering::Equal => {
+              e2.options = box_chain!(e2.options, e1.options);
+              e2
+            },
+          }),
+        // otherwise return the parent's result
+        x => x,
+      }
+    } else {
+      Err(e1)
+    }
+  })
+}
+
+pub fn process_donor_module<'a, TItem: Clone>(
+  module: &'a Module<TItem, impl Clone>,
+  abs_path: Rc<VName>,
+  is_op: impl Fn(&TItem) -> bool + 'a,
+) -> impl Iterator<Item = (Tok<String>, VName, bool)> + 'a {
+  (module.entries.iter()).filter(|(_, ent)| ent.exported).map(
+    move |(n, ent)| {
+      let is_op = ent.item().map_or(false, &is_op);
+      (n.clone(), pushed_ref(abs_path.as_ref(), n.clone()), is_op)
+    },
+  )
+}
+
+pub fn import_tree(
+  modpath: VName,
+  pre: &PreMod,
+  root: &Preparsed,
+  prev_root: &ProjectTree<VName>,
+  i: &Interner,
+) -> ProjectResult<ImpMod> {
+  let imports = pre.extra.details().map_or(&[][..], |e| &e.imports[..]);
+  let entries = (imports.iter())
+    // imports become leaf sets
+    .map(|Import { name, path, location }| -> ProjectResult<BoxedIter<_>> {
+      let mut abs_path = absolute_path(&modpath, path, i, location)?;
+      Ok(if let Some(name) = name {
+        // named imports are validated and translated 1->1
+        abs_path.push(name.clone());
+        assert_visible_overlay(&modpath, &abs_path, &root.0, &prev_root.0)
+          .map_err(|e| -> Rc<dyn ProjectError> {
+            println!("Current root: {}", &root.0);
+            // println!("Old root: {:#?}", &prev_root.0);
+            panic!("{}", e.at(location))
+          })?;
+        let is_op = (root.0.walk1_ref(&[], &abs_path, false))
+          .map(|(ent, _)| ent.item().map_or(false, |i| i.is_op))
+          .or_else(|e| if e.kind == ErrKind::Missing {
+            (prev_root.0.walk1_ref(&[], &abs_path, false))
+              .map(|(ent, _)| ent.item().map_or(false, |i| i.is_op))
+          } else {Err(e)})
+          .map_err(|e| e.at(location))?;
+        box_once((name.clone(), abs_path, is_op))
+      } else {
+        let rc_path = Rc::new(abs_path);
+        // wildcard imports are validated
+        assert_visible_overlay(&modpath, &rc_path, &root.0, &prev_root.0)
+          .map_err(|e| e.at(location))?;
+        // and instantiated for all exports of the target 1->n
+        let new_imports = match (root.0).walk_ref(&[], &rc_path, false) {
+          Err(e) if e.kind == ErrKind::Missing => Err(e),
+          Err(e) => return Err(e.at(location)),
+          Ok(module)
+            => Ok(process_donor_module(module, rc_path.clone(), |i| i.is_op))
+        };
+        let old_m = match (prev_root.0).walk_ref(&[], &rc_path, false) {
+          Err(e) if e.kind != ErrKind::Missing => return Err(e.at(location)),
+          Err(e1) => match new_imports {
+            Ok(it) => return Ok(Box::new(it)),
+            Err(mut e2) => return Err(match e1.depth().cmp(&e2.depth()) {
+              Ordering::Less => e2.at(location),
+              Ordering::Greater => e1.at(location),
+              Ordering::Equal => {
+                e2.options = box_chain!(e2.options, e1.options);
+                e2.at(location)
+              },
+            }),
+          },
+          Ok(old_m) => old_m,
+        };
+        let it1 = process_donor_module(old_m, rc_path.clone(), |i| i.is_op);
+        match new_imports {
+          Err(_) => Box::new(it1),
+          Ok(it2) => box_chain!(it1, it2)
+        }
+      })
+    })
+    // leaf sets flattened to leaves
+    .flatten_ok()
+    // translated to entries
+    .map_ok(|(name, source, is_op)| {
+      (name, ModEntry {
+        exported: false, // this is irrelevant but needed
+        member: ModMember::Item(ImpReport { source, is_op }),
+      })
+    })
+    .chain(
+      (pre.entries.iter())
+        // recurse on submodules
+        .filter_map(|(k, v)| {
+          Some((k, v, unwrap_or!(&v.member => ModMember::Sub; return None)))
+        })
+        .map(|(k, v, pre)| {
+          let path = pushed_ref(&modpath, k.clone());
+          Ok((k.clone(), ModEntry {
+            exported: v.exported,
+            member: ModMember::Sub(import_tree(path, pre, root, prev_root, i)?),
+          }))
+        }),
+    )
+    .collect::<Result<HashMap<_, _>, _>>()?;
+  Ok(Module { extra: (), entries })
+}

src/pipeline/project_tree/mod.rs

@@ -1,24 +1,5 @@
-// FILE SEPARATION BOUNDARY
-//
-// Collect all operators accessible in each file, parse the files with
-// correct tokenization, resolve glob imports, convert expressions to
-// refer to tokens with (local) absolute path, and connect them into a
-// single tree.
-//
-// The module checks for imports from missing modules (including
-// submodules). All other errors must be checked later.
-//
-// Injection strategy:
-// Return all items of the given module in the injected tree for
-// `injected` The output of this stage is a tree, which can simply be
-// overlaid with the injected tree
-
-mod add_prelude;
 mod build_tree;
-mod collect_ops;
-mod normalize_imports;
-mod parse_file;
-mod prefix;
+mod import_tree;
+mod rebuild_tree;
 
-pub use build_tree::{build_tree, split_path};
-pub use collect_ops::InjectedOperatorsFn;
+pub use rebuild_tree::rebuild_tree;

src/pipeline/project_tree/normalize_imports.rs

@@ -1,94 +0,0 @@
-use super::collect_ops::ExportedOpsCache;
-use crate::interner::{Interner, Tok};
-use crate::pipeline::import_abs_path::import_abs_path;
-use crate::representations::sourcefile::{
-  FileEntry, Import, Member, ModuleBlock,
-};
-use crate::representations::tree::{ModMember, Module};
-use crate::utils::iter::box_once;
-use crate::utils::{unwrap_or, BoxedIter, Substack};
-
-fn member_rec(
-  // level
-  mod_stack: Substack<Tok<String>>,
-  preparsed: &Module<impl Clone, impl Clone>,
-  // object
-  member: Member,
-  // context
-  path: &[Tok<String>],
-  ops_cache: &ExportedOpsCache,
-  i: &Interner,
-) -> Member {
-  match member {
-    Member::Module(ModuleBlock { name, body }) => {
-      let subprep = unwrap_or!(
-        &preparsed.items[&name].member => ModMember::Sub;
-        unreachable!("This name must point to a namespace")
-      );
-      let new_stack = mod_stack.push(name.clone());
-      let new_body = entv_rec(new_stack, subprep, body, path, ops_cache, i);
-      Member::Module(ModuleBlock { name, body: new_body })
-    },
-    any => any,
-  }
-}
-
-/// Normalize imports in the FileEntry list recursively
-///
-/// # Panics
-///
-/// - if a path contains too many "super" prefixes
-/// - if the exported operators in a module cannot be determined
-fn entv_rec(
-  // level
-  mod_stack: Substack<Tok<String>>,
-  preparsed: &Module<impl Clone, impl Clone>,
-  // object
-  data: Vec<FileEntry>,
-  // context
-  mod_path: &[Tok<String>],
-  ops_cache: &ExportedOpsCache,
-  i: &Interner,
-) -> Vec<FileEntry> {
-  data
-    .into_iter()
-    .map(|ent| match ent {
-      FileEntry::Import(imps) => FileEntry::Import(
-        imps
-          .into_iter()
-          .flat_map(|import| {
-            if let Import { name: None, path } = import {
-              let p = import_abs_path(mod_path, mod_stack.clone(), &path, i)
-                .expect("Should have emerged in preparsing");
-              let names = (ops_cache.find(&i.i(&p)))
-                .expect("Should have emerged in second parsing");
-              let imports = (names.iter())
-                .map(|n| Import { name: Some(n.clone()), path: path.clone() })
-                .collect::<Vec<_>>();
-              Box::new(imports.into_iter()) as BoxedIter<Import>
-            } else {
-              box_once(import)
-            }
-          })
-          .collect(),
-      ),
-      FileEntry::Exported(mem) => FileEntry::Exported(member_rec(
-        mod_stack.clone(), preparsed, mem, mod_path, ops_cache, i,
-      )),
-      FileEntry::Internal(mem) => FileEntry::Internal(member_rec(
-        mod_stack.clone(), preparsed, mem, mod_path, ops_cache, i,
-      )),
-      any => any,
-    })
-    .collect()
-}
-
-pub fn normalize_imports(
-  preparsed: &Module<impl Clone, impl Clone>,
-  data: Vec<FileEntry>,
-  path: &[Tok<String>],
-  ops_cache: &ExportedOpsCache,
-  i: &Interner,
-) -> Vec<FileEntry> {
-  entv_rec(Substack::Bottom, preparsed, data, path, ops_cache, i)
-}

src/pipeline/project_tree/parse_file.rs

@@ -1,46 +0,0 @@
-use std::rc::Rc;
-
-use super::add_prelude::add_prelude;
-use super::collect_ops::{collect_ops_for, ExportedOpsCache};
-use super::normalize_imports::normalize_imports;
-use super::prefix::prefix;
-use crate::error::ProjectResult;
-use crate::interner::{Interner, Tok};
-use crate::parse;
-use crate::pipeline::source_loader::LoadedSourceTable;
-use crate::representations::sourcefile::{normalize_namespaces, FileEntry};
-
-/// Parses a file with the correct operator set
-///
-/// # Panics
-///
-/// - on syntax error
-/// - if namespaces are exported inconsistently
-///
-/// These are both checked in the preparsing stage
-pub fn parse_file(
-  path: &[Tok<String>],
-  loaded: &LoadedSourceTable,
-  ops_cache: &ExportedOpsCache,
-  i: &Interner,
-  prelude: &[FileEntry],
-) -> ProjectResult<Vec<FileEntry>> {
-  let ld = &loaded[path];
-  // let ops_cache = collect_ops::mk_cache(loaded, i);
-  let ops = collect_ops_for(path, loaded, ops_cache, i)?;
-  let ops_vec = ops.iter().map(|t| (**t).clone()).collect::<Vec<_>>();
-  let ctx = parse::ParsingContext {
-    interner: i,
-    ops: &ops_vec,
-    file: Rc::new(Interner::extern_all(path)),
-  };
-  let entries = parse::parse2(ld.text.as_str(), ctx)
-    .expect("This error should have been caught during loading");
-  let with_prelude = add_prelude(entries, path, prelude);
-  let impnormalized =
-    normalize_imports(&ld.preparsed.0, with_prelude, path, ops_cache, i);
-  let nsnormalized = normalize_namespaces(Box::new(impnormalized.into_iter()))
-    .expect("This error should have been caught during preparsing");
-  let prefixed = prefix(nsnormalized, path, ops_cache, i);
-  Ok(prefixed)
-}

src/pipeline/project_tree/prefix.rs

@@ -1,73 +0,0 @@
-use super::collect_ops::ExportedOpsCache;
-use crate::ast::{Constant, Rule};
-use crate::interner::{Interner, Tok};
-use crate::representations::sourcefile::{FileEntry, Member, ModuleBlock};
-use crate::utils::Substack;
-
-fn member_rec(
-  // level
-  mod_stack: Substack<Tok<String>>,
-  // object
-  data: Member,
-  // context
-  path: &[Tok<String>],
-  ops_cache: &ExportedOpsCache,
-  i: &Interner,
-) -> Member {
-  let prefix = (path.iter())
-    .cloned()
-    .chain(mod_stack.iter().rev_vec_clone().into_iter())
-    .collect::<Vec<_>>();
-  match data {
-    Member::Module(ModuleBlock { name, body }) => {
-      let new_stack = mod_stack.push(name.clone());
-      let new_body = entv_rec(new_stack, body, path, ops_cache, i);
-      Member::Module(ModuleBlock { name, body: new_body })
-    },
-    Member::Constant(constant) => Member::Constant(Constant {
-      name: constant.name,
-      value: constant.value.prefix(&prefix, &|_| false),
-    }),
-    Member::Rule(rule) => Member::Rule(Rule {
-      prio: rule.prio,
-      pattern: (rule.pattern.into_iter())
-        .map(|e| e.prefix(&prefix, &|_| false))
-        .collect(),
-      template: (rule.template.into_iter())
-        .map(|e| e.prefix(&prefix, &|_| false))
-        .collect(),
-    }),
-  }
-}
-
-fn entv_rec(
-  // level
-  mod_stack: Substack<Tok<String>>,
-  // object
-  data: Vec<FileEntry>,
-  // context
-  path: &[Tok<String>],
-  ops_cache: &ExportedOpsCache,
-  i: &Interner,
-) -> Vec<FileEntry> {
-  (data.into_iter())
-    .map(|fe| {
-      let (mem, wrapper): (Member, fn(Member) -> FileEntry) = match fe {
-        FileEntry::Exported(mem) => (mem, FileEntry::Exported),
-        FileEntry::Internal(mem) => (mem, FileEntry::Internal),
-        // XXX should [FileEntry::Export] be prefixed?
-        any => return any,
-      };
-      wrapper(member_rec(mod_stack.clone(), mem, path, ops_cache, i))
-    })
-    .collect()
-}
-
-pub fn prefix(
-  data: Vec<FileEntry>,
-  path: &[Tok<String>],
-  ops_cache: &ExportedOpsCache,
-  i: &Interner,
-) -> Vec<FileEntry> {
-  entv_rec(Substack::Bottom, data, path, ops_cache, i)
-}

src/pipeline/project_tree/rebuild_tree.rs

@@ -0,0 +1,131 @@
+use std::rc::Rc;
+
+use hashbrown::HashMap;
+use itertools::Itertools;
+
+use super::build_tree::{build_tree, TreeReport};
+use super::import_tree::{import_tree, ImpMod};
+use crate::error::ProjectResult;
+use crate::pipeline::source_loader::{
+  LoadedSourceTable, PreExtra, PreItem, PreMod, Preparsed,
+};
+use crate::representations::project::{ImpReport, ProjectExt, ProjectMod};
+use crate::sourcefile::FileEntry;
+use crate::tree::{ModEntry, ModMember, Module};
+use crate::utils::never::{always, unwrap_always};
+use crate::utils::pushed::pushed_ref;
+use crate::utils::unwrap_or;
+use crate::{parse, Interner, ProjectTree, Tok, VName};
+
+pub fn rebuild_file(
+  path: Vec<Tok<String>>,
+  pre: PreMod,
+  imports: ImpMod,
+  source: &LoadedSourceTable,
+  prelude: &[FileEntry],
+  i: &Interner,
+) -> ProjectResult<ProjectMod<VName>> {
+  let file = match &pre.extra {
+    PreExtra::Dir => panic!("Dir should not hand this node off"),
+    PreExtra::Submod(_) => panic!("should not have received this"),
+    PreExtra::File(f) => f,
+  };
+  let mut ops = Vec::new();
+  unwrap_always(imports.search_all((), &mut |_, module, ()| {
+    ops.extend(
+      (module.entries.iter())
+        .filter(|(_, ent)| {
+          matches!(ent.member, ModMember::Item(ImpReport { is_op: true, .. }))
+        })
+        .map(|(name, _)| name.clone()),
+    );
+    always(())
+  }));
+  unwrap_always(pre.search_all((), &mut |_, module, ()| {
+    ops.extend(
+      (module.entries.iter())
+        .filter(|(_, ent)| {
+          matches!(ent.member, ModMember::Item(PreItem { is_op: true, .. }))
+        })
+        .map(|(name, _)| name.clone()),
+    );
+    always(())
+  }));
+  let ctx = parse::ParsingContext::new(&ops, i, Rc::new(path.clone()));
+  let src = source.get(&file.name).unwrap_or_else(|| {
+    panic!(
+      "{} should have been preparsed already. Preparsed files are {}",
+      Interner::extern_all(&file.name).join("/") + ".orc",
+      source
+        .keys()
+        .map(|f| Interner::extern_all(f).join("/") + ".orc")
+        .join(", ")
+    )
+  });
+  let entries = parse::parse2(&src.text, ctx)?;
+  let TreeReport { entries: items, rules, imports_from } =
+    build_tree(&path, entries, pre, imports, prelude)?;
+  Ok(Module {
+    entries: items,
+    extra: ProjectExt { file: Some(path.clone()), path, imports_from, rules },
+  })
+}
+
+pub fn rebuild_dir(
+  path: Vec<Tok<String>>,
+  pre: PreMod,
+  mut imports: ImpMod,
+  source: &LoadedSourceTable,
+  prelude: &[FileEntry],
+  i: &Interner,
+) -> ProjectResult<ProjectMod<VName>> {
+  match pre.extra {
+    PreExtra::Dir => (),
+    PreExtra::File(_) =>
+      return rebuild_file(path, pre, imports, source, prelude, i),
+    PreExtra::Submod(_) => panic!("Dirs contain dirs and files"),
+  }
+  let items = (pre.entries.into_iter())
+    .map(|(name, entry)| {
+      match imports.entries.remove(&name).map(|e| e.member) {
+        Some(ModMember::Sub(impmod)) => (name, entry, impmod),
+        _ => panic!("Imports must line up with modules"),
+      }
+    })
+    .map(|(name, ModEntry { member, exported }, impmod)| -> ProjectResult<_> {
+      let path = pushed_ref(&path, name.clone());
+      let pre = unwrap_or!(member => ModMember::Sub;
+        panic!("Dirs can only contain submodules")
+      );
+      Ok((name, ModEntry {
+        exported,
+        member: ModMember::Sub(rebuild_dir(
+          path, pre, impmod, source, prelude, i,
+        )?),
+      }))
+    })
+    .collect::<Result<HashMap<_, _>, _>>()?;
+  Ok(Module {
+    extra: ProjectExt {
+      path,
+      imports_from: HashMap::new(),
+      rules: Vec::new(),
+      file: None,
+    },
+    entries: items,
+  })
+}
+
+/// Rebuild the entire tree
+pub fn rebuild_tree(
+  source: &LoadedSourceTable,
+  preparsed: Preparsed,
+  prev_root: &ProjectTree<VName>,
+  prelude: &[FileEntry],
+  i: &Interner,
+) -> ProjectResult<ProjectTree<VName>> {
+  let imports =
+    import_tree(Vec::new(), &preparsed.0, &preparsed, prev_root, i)?;
+  rebuild_dir(Vec::new(), preparsed.0, imports, source, prelude, i)
+    .map(ProjectTree)
+}

src/pipeline/source_loader/load_source.rs

@@ -1,7 +1,8 @@
-use std::iter;
+use hashbrown::HashMap;
 
 use super::loaded_source::{LoadedSource, LoadedSourceTable};
 use super::preparse::preparse;
+use super::{PreExtra, Preparsed};
 use crate::error::{
   NoTargets, ProjectError, ProjectResult, UnexpectedDirectory,
 };
@@ -9,67 +10,79 @@ use crate::interner::{Interner, Tok};
 use crate::pipeline::file_loader::{IOResult, Loaded};
 use crate::pipeline::import_abs_path::import_abs_path;
 use crate::representations::sourcefile::FileEntry;
+use crate::tree::Module;
+use crate::utils::pushed::pushed_ref;
 use crate::utils::{split_max_prefix, unwrap_or};
+use crate::Location;
 
 /// Load the source at the given path or all within if it's a collection,
 /// and all sources imported from these.
 fn load_abs_path_rec(
   abs_path: &[Tok<String>],
-  table: &mut LoadedSourceTable,
+  mut all: Preparsed,
+  source: &mut LoadedSourceTable,
   prelude: &[FileEntry],
   i: &Interner,
   get_source: &impl Fn(&[Tok<String>]) -> IOResult,
   is_injected_module: &impl Fn(&[Tok<String>]) -> bool,
-) -> ProjectResult<()> {
+) -> ProjectResult<Preparsed> {
   // # Termination
   //
   // Every recursion of this function either
-  // - adds one of the files in the source directory to `table` or
+  // - adds one of the files in the source directory to `visited` or
   // - recursively traverses a directory tree
   // therefore eventually the function exits, assuming that the directory tree
   // contains no cycles.
 
-  // Termination: exit if entry already visited
-  if table.contains_key(abs_path) {
-    return Ok(());
-  }
-
   // try splitting the path to file, swallowing any IO errors
   let name_split = split_max_prefix(abs_path, &|p| {
     get_source(p).map(|l| l.is_code()).unwrap_or(false)
   });
   if let Some((filename, _)) = name_split {
+    // Termination: exit if entry already visited
+    if source.contains_key(filename) {
+      return Ok(all);
+    }
     // if the filename is valid, load, preparse and record this file
     let text = unwrap_or!(get_source(filename)? => Loaded::Code; {
       return Err(UnexpectedDirectory { path: filename.to_vec() }.rc())
     });
-    let preparsed = preparse(
-      Interner::extern_all(filename),
-      text.as_str(),
-      prelude,
-      i,
-    )?;
-    table.insert(filename.to_vec(), LoadedSource {
-      text,
-      preparsed: preparsed.clone(),
-    });
+    source.insert(filename.to_vec(), LoadedSource { text: text.clone() });
+    let preparsed = preparse(filename.to_vec(), text.as_str(), prelude, i)?;
     // recurse on all imported modules
-    preparsed.0.visit_all_imports(&mut |modpath, _module, import| {
-      let abs_pathv =
-        import_abs_path(filename, modpath, &import.nonglob_path(), i)?;
-      if abs_path.starts_with(&abs_pathv) {
-        return Ok(());
+    // will be taken and returned by the closure. None iff an error is thrown
+    all = preparsed.0.search_all(all, &mut |modpath,
+                                             module,
+                                             mut all|
+     -> ProjectResult<_> {
+      let details = unwrap_or!(module.extra.details(); return Ok(all));
+      for import in &details.imports {
+        let origin = &Location::Unknown;
+        let abs_pathv = import_abs_path(
+          filename,
+          modpath.clone(),
+          &import.nonglob_path(),
+          i,
+          origin,
+        )?;
+        if abs_path.starts_with(&abs_pathv) {
+          continue;
+        }
+        // recurse on imported module
+        all = load_abs_path_rec(
+          &abs_pathv,
+          all,
+          source,
+          prelude,
+          i,
+          get_source,
+          is_injected_module,
+        )?;
       }
-      // recurse on imported module
-      load_abs_path_rec(
-        &abs_pathv,
-        table,
-        prelude,
-        i,
-        get_source,
-        is_injected_module,
-      )
-    })
+      Ok(all)
+    })?;
+    // Combine the trees
+    all.0.overlay(preparsed.0).map(Preparsed)
   } else {
     // If the path is not within a file, load it as directory
     let coll = match get_source(abs_path) {
@@ -81,25 +94,23 @@ fn load_abs_path_rec(
         let parent = abs_path.split_last().expect("import path nonzero").1;
         // exit without error if it was injected, or raise any IO error that was
         // previously swallowed
-        return if is_injected_module(parent) { Ok(()) } else { Err(e) };
+        return if is_injected_module(parent) { Ok(all) } else { Err(e) };
       },
     };
     // recurse on all files and folders within
     for item in coll.iter() {
-      let abs_subpath = (abs_path.iter())
-        .cloned()
-        .chain(iter::once(i.i(item)))
-        .collect::<Vec<_>>();
-      load_abs_path_rec(
+      let abs_subpath = pushed_ref(abs_path, i.i(item));
+      all = load_abs_path_rec(
         &abs_subpath,
-        table,
+        all,
+        source,
         prelude,
         i,
         get_source,
         is_injected_module,
-      )?
+      )?;
     }
-    Ok(())
+    Ok(all)
   }
 }
 
@@ -115,19 +126,22 @@ pub fn load_source<'a>(
   i: &Interner,
   get_source: &impl Fn(&[Tok<String>]) -> IOResult,
   is_injected_module: &impl Fn(&[Tok<String>]) -> bool,
-) -> ProjectResult<LoadedSourceTable> {
+) -> ProjectResult<(Preparsed, LoadedSourceTable)> {
   let mut table = LoadedSourceTable::new();
+  let mut all =
+    Preparsed(Module { extra: PreExtra::Dir, entries: HashMap::new() });
   let mut any_target = false;
   for target in targets {
     any_target |= true;
-    load_abs_path_rec(
+    all = load_abs_path_rec(
       target,
+      all,
       &mut table,
       prelude,
       i,
       get_source,
       is_injected_module,
-    )?
+    )?;
   }
-  if any_target { Ok(table) } else { Err(NoTargets.rc()) }
+  if any_target { Ok((all, table)) } else { Err(NoTargets.rc()) }
 }

src/pipeline/source_loader/loaded_source.rs

@@ -1,13 +1,11 @@
 use std::collections::HashMap;
 use std::rc::Rc;
 
-use super::preparse::Preparsed;
 use crate::representations::VName;
 
 #[derive(Debug)]
 pub struct LoadedSource {
   pub text: Rc<String>,
-  pub preparsed: Preparsed,
 }
 
 pub type LoadedSourceTable = HashMap<VName, LoadedSource>;

src/pipeline/source_loader/mod.rs

@@ -18,7 +18,8 @@
 mod load_source;
 mod loaded_source;
 mod preparse;
+mod types;
 
 pub use load_source::load_source;
 pub use loaded_source::{LoadedSource, LoadedSourceTable};
-pub use preparse::Preparsed;
+pub use types::{PreExtra, PreFileExt, PreItem, PreMod, PreSubExt, Preparsed};

src/pipeline/source_loader/preparse.rs

@@ -1,99 +1,169 @@
-use std::hash::Hash;
 use std::rc::Rc;
 
 use hashbrown::HashMap;
+use itertools::Itertools;
 
-use crate::ast::Constant;
-use crate::error::{ProjectError, ProjectResult, VisibilityMismatch};
+use super::types::{PreFileExt, PreItem, PreSubExt};
+use super::{PreExtra, Preparsed};
+use crate::ast::{Clause, Constant, Expr};
+use crate::error::{
+  ConflictingRoles, ProjectError, ProjectResult, VisibilityMismatch,
+};
 use crate::interner::Interner;
 use crate::parse::{self, ParsingContext};
-use crate::representations::sourcefile::{
-  imports, normalize_namespaces, FileEntry, Member,
-};
+use crate::representations::sourcefile::{FileEntry, MemberKind};
 use crate::representations::tree::{ModEntry, ModMember, Module};
+use crate::sourcefile::{FileEntryKind, Import, Member, ModuleBlock};
+use crate::utils::pushed::pushed;
+use crate::utils::{get_or_default, get_or_make};
+use crate::{Location, Tok, VName};
 
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Preparsed(pub Module<(), ()>);
-
-/// Add an internal flat name if it does not exist yet
-fn add_intern<K: Eq + Hash>(map: &mut HashMap<K, ModEntry<(), ()>>, k: K) {
-  let _ = map
-    .try_insert(k, ModEntry { exported: false, member: ModMember::Item(()) });
-}
-
-/// Add an exported flat name or export any existing entry
-fn add_export<K: Eq + Hash>(map: &mut HashMap<K, ModEntry<(), ()>>, k: K) {
-  if let Some(entry) = map.get_mut(&k) {
-    entry.exported = true
-  } else {
-    map.insert(k, ModEntry { exported: true, member: ModMember::Item(()) });
-  }
+struct FileReport {
+  entries: HashMap<Tok<String>, ModEntry<PreItem, PreExtra>>,
+  patterns: Vec<Vec<Expr<VName>>>,
+  imports: Vec<Import>,
 }
 
 /// Convert source lines into a module
-fn to_module(src: &[FileEntry], prelude: &[FileEntry]) -> Module<(), ()> {
-  let all_src = || src.iter().chain(prelude.iter());
-  let imports = imports(all_src()).cloned().collect::<Vec<_>>();
-  let mut items = all_src()
-    .filter_map(|ent| match ent {
-      FileEntry::Internal(Member::Module(ns)) => {
-        let member = ModMember::Sub(to_module(&ns.body, prelude));
-        let entry = ModEntry { exported: false, member };
-        Some((ns.name.clone(), entry))
-      },
-      FileEntry::Exported(Member::Module(ns)) => {
-        let member = ModMember::Sub(to_module(&ns.body, prelude));
-        let entry = ModEntry { exported: true, member };
-        Some((ns.name.clone(), entry))
-      },
-      _ => None,
-    })
-    .collect::<HashMap<_, _>>();
-  for file_entry in all_src() {
-    match file_entry {
-      FileEntry::Comment(_)
-      | FileEntry::Import(_)
-      | FileEntry::Internal(Member::Module(_))
-      | FileEntry::Exported(Member::Module(_)) => (),
-      FileEntry::Export(tokv) =>
-        for tok in tokv {
-          add_export(&mut items, tok.clone())
+fn to_module(
+  file: &[Tok<String>],
+  path: VName,
+  src: Vec<FileEntry>,
+  prelude: &[FileEntry],
+) -> ProjectResult<FileReport> {
+  let mut imports = Vec::new();
+  let mut patterns = Vec::new();
+  let mut items = HashMap::<Tok<String>, (bool, PreItem)>::new();
+  let mut to_export = HashMap::<Tok<String>, Vec<Location>>::new();
+  let mut submods =
+    HashMap::<Tok<String>, (bool, Vec<Location>, Vec<FileEntry>)>::new();
+  let entries = prelude.iter().cloned().chain(src.into_iter());
+  for FileEntry { kind, locations } in entries {
+    match kind {
+      FileEntryKind::Import(imp) => imports.extend(imp.into_iter()),
+      FileEntryKind::Export(names) =>
+        for (t, l) in names {
+          get_or_default(&mut to_export, &t).push(l)
+        },
+      FileEntryKind::Member(Member { exported, kind }) => match kind {
+        MemberKind::Constant(Constant { name, .. }) => {
+          let (prev_exported, it) = get_or_default(&mut items, &name);
+          if it.has_value {
+            let err = ConflictingRoles { name: pushed(path, name), locations };
+            return Err(err.rc());
+          }
+          if let Some(loc) = locations.get(0) {
+            it.location = it.location.clone().or(loc.clone())
+          };
+          it.has_value = true;
+          *prev_exported |= exported;
+        },
+        MemberKind::Module(ModuleBlock { name, body }) => {
+          if let Some((prev_exported, locv, entv)) = submods.get_mut(&name) {
+            if *prev_exported != exported {
+              let mut namespace = path;
+              namespace.push(name.clone());
+              let err = VisibilityMismatch { namespace, file: file.to_vec() };
+              return Err(err.rc());
+            }
+            locv.extend(locations.into_iter());
+            entv.extend(body.into_iter())
+          } else {
+            submods.insert(name.clone(), (exported, locations, body.clone()));
+          }
+        },
+        MemberKind::Operators(ops) =>
+          for op in ops {
+            let (prev_exported, it) = get_or_default(&mut items, &op);
+            if let Some(loc) = locations.get(0) {
+              it.location = it.location.clone().or(loc.clone())
+            }
+            *prev_exported |= exported;
+            it.is_op = true;
+          },
+        MemberKind::Rule(r) => {
+          patterns.push(r.pattern.clone());
+          if exported {
+            for ex in r.pattern {
+              ex.search_all(&mut |ex| {
+                if let Clause::Name(vname) = &ex.value {
+                  if let Ok(name) = vname.iter().exactly_one() {
+                    get_or_default(&mut to_export, name)
+                      .push(ex.location.clone());
+                  }
+                }
+                None::<()>
+              });
+            }
+          }
         },
-      FileEntry::Internal(Member::Constant(Constant { name, .. })) =>
-        add_intern(&mut items, name.clone()),
-      FileEntry::Exported(Member::Constant(Constant { name, .. })) =>
-        add_export(&mut items, name.clone()),
-      FileEntry::Internal(Member::Rule(rule)) => {
-        let names = rule.collect_single_names();
-        for name in names {
-          add_intern(&mut items, name)
-        }
-      },
-      FileEntry::Exported(Member::Rule(rule)) => {
-        let names = rule.collect_single_names();
-        for name in names {
-          add_export(&mut items, name)
-        }
       },
+      _ => (),
     }
   }
-  Module { imports, items, extra: () }
+  let mut entries = HashMap::with_capacity(items.len() + submods.len());
+  entries.extend(items.into_iter().map(|(name, (exported, it))| {
+    (name, ModEntry { member: ModMember::Item(it), exported })
+  }));
+  for (subname, (exported, locations, body)) in submods {
+    let mut name = path.clone();
+    entries
+      .try_insert(subname.clone(), ModEntry {
+        member: ModMember::Sub({
+          name.push(subname);
+          let FileReport { imports, entries: items, patterns } =
+            to_module(file, name.clone(), body, prelude)?;
+          Module {
+            entries: items,
+            extra: PreExtra::Submod(PreSubExt { imports, patterns }),
+          }
+        }),
+        exported,
+      })
+      .map_err(|_| ConflictingRoles { locations, name }.rc())?;
+  }
+  for (item, locations) in to_export {
+    get_or_make(&mut entries, &item, || ModEntry {
+      member: ModMember::Item(PreItem {
+        is_op: false,
+        has_value: false,
+        location: locations[0].clone(),
+      }),
+      exported: true,
+    })
+    .exported = true
+  }
+  Ok(FileReport { entries, imports, patterns })
 }
 
 /// Preparse the module. At this stage, only the imports and
 /// names defined by the module can be parsed
 pub fn preparse(
-  file: Vec<String>,
+  file: VName,
   source: &str,
   prelude: &[FileEntry],
   i: &Interner,
 ) -> ProjectResult<Preparsed> {
   // Parse with no operators
-  let ctx = ParsingContext::<&str>::new(&[], i, Rc::new(file.clone()));
+  let ctx = ParsingContext::new(&[], i, Rc::new(file.clone()));
   let entries = parse::parse2(source, ctx)?;
-  let normalized = normalize_namespaces(Box::new(entries.into_iter()))
-    .map_err(|namespace| {
-      VisibilityMismatch { namespace, file: Rc::new(file.clone()) }.rc()
-    })?;
-  Ok(Preparsed(to_module(&normalized, prelude)))
+  let FileReport { entries, imports, patterns } =
+    to_module(&file, file.clone(), entries, prelude)?;
+  let mut module = Module {
+    entries,
+    extra: PreExtra::File(PreFileExt {
+      details: PreSubExt { patterns, imports },
+      name: file.clone(),
+    }),
+  };
+  for name in file.iter().rev() {
+    module = Module {
+      extra: PreExtra::Dir,
+      entries: HashMap::from([(name.clone(), ModEntry {
+        exported: true,
+        member: ModMember::Sub(module),
+      })]),
+    };
+  }
+  Ok(Preparsed(module))
 }

src/pipeline/source_loader/types.rs

@@ -0,0 +1,92 @@
+use std::fmt::Display;
+use std::ops::Add;
+
+use crate::ast::Expr;
+use crate::error::ProjectResult;
+use crate::sourcefile::Import;
+use crate::tree::Module;
+use crate::{Interner, Location, VName};
+
+#[derive(Debug, Clone)]
+pub struct PreItem {
+  pub is_op: bool,
+  pub has_value: bool,
+  pub location: Location,
+}
+
+impl Display for PreItem {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    let Self { has_value, is_op, location } = self;
+    let description = match (is_op, has_value) {
+      (true, true) => "operator with value",
+      (true, false) => "operator",
+      (false, true) => "value",
+      (false, false) => "keyword",
+    };
+    write!(f, "{description} {location}")
+  }
+}
+
+impl Default for PreItem {
+  fn default() -> Self {
+    PreItem { is_op: false, has_value: false, location: Location::Unknown }
+  }
+}
+
+#[derive(Debug, Clone)]
+pub struct PreSubExt {
+  pub imports: Vec<Import>,
+  pub patterns: Vec<Vec<Expr<VName>>>,
+}
+
+#[derive(Debug, Clone)]
+pub struct PreFileExt {
+  pub name: VName,
+  pub details: PreSubExt,
+}
+
+#[derive(Debug, Clone)]
+pub enum PreExtra {
+  File(PreFileExt),
+  Submod(PreSubExt),
+  Dir,
+}
+
+impl PreExtra {
+  /// If the module is not a directory, returns the source-only details
+  pub fn details(&self) -> Option<&PreSubExt> {
+    match self {
+      Self::Submod(sub) => Some(sub),
+      Self::File(PreFileExt { details, .. }) => Some(details),
+      Self::Dir => None,
+    }
+  }
+}
+
+impl Add for PreExtra {
+  type Output = ProjectResult<Self>;
+
+  fn add(self, rhs: Self) -> Self::Output {
+    match (self, rhs) {
+      (alt, Self::Dir) | (Self::Dir, alt) => Ok(alt),
+      (Self::File(_) | Self::Submod(_), Self::File(_) | Self::Submod(_)) =>
+        panic!("Each file should be parsed once."),
+    }
+  }
+}
+
+impl Display for PreExtra {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    match self {
+      Self::Dir => write!(f, "Directory"),
+      Self::File(PreFileExt { name, .. }) =>
+        write!(f, "File({}.orc)", Interner::extern_all(name).join("/")),
+      Self::Submod(_) => write!(f, "Submodule"),
+    }
+  }
+}
+
+pub type PreMod = Module<PreItem, PreExtra>;
+
+#[derive(Debug, Clone)]
+pub struct Preparsed(pub PreMod);