Orchid/orchid
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

The pipeline is finally reasonably clean

Browse Source
This commit is contained in:
Lawrence Bethlenfalvy
2023-09-12 01:26:46 +01:00
parent 6693d93944
commit 8c866967a9
86 changed files with 1959 additions and 1393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/pipeline/project_tree/add_prelude.rs
View File

@@ -1,46 +0,0 @@
use crate::interner::Tok;
use crate::representations::sourcefile::{FileEntry, Member, ModuleBlock};
fn member_rec(
// object
member: Member,
// context
path: &[Tok<String>],
prelude: &[FileEntry],
) -> Member {
match member {
Member::Module(ModuleBlock { name, body }) => {
let new_body = entv_rec(body, path, prelude);
Member::Module(ModuleBlock { name, body: new_body })
},
any => any,
}
}
fn entv_rec(
// object
data: Vec<FileEntry>,
// context
mod_path: &[Tok<String>],
prelude: &[FileEntry],
) -> Vec<FileEntry> {
prelude
.iter()
.cloned()
.chain(data.into_iter().map(|ent| match ent {
FileEntry::Exported(mem) =>
FileEntry::Exported(member_rec(mem, mod_path, prelude)),
FileEntry::Internal(mem) =>
FileEntry::Internal(member_rec(mem, mod_path, prelude)),
any => any,
}))
.collect()
}
pub fn add_prelude(
data: Vec<FileEntry>,
path: &[Tok<String>],
prelude: &[FileEntry],
) -> Vec<FileEntry> {
entv_rec(data, path, prelude)
}

370
src/pipeline/project_tree/build_tree.rs
View File

@@ -1,246 +1,144 @@
use hashbrown::HashMap;
use itertools::Itertools;
use itertools::{Either, Itertools};
use super::collect_ops;
use super::collect_ops::InjectedOperatorsFn;
use super::parse_file::parse_file;
use crate::ast::{Clause, Constant, Expr};
use crate::error::{ProjectError, ProjectResult, TooManySupers};
use crate::interner::{Interner, Tok};
use crate::pipeline::source_loader::{LoadedSource, LoadedSourceTable};
use crate::representations::project::{ProjectExt, ProjectTree};
use crate::representations::sourcefile::{absolute_path, FileEntry, Member};
use crate::representations::tree::{ModEntry, ModMember, Module};
use crate::representations::{NameLike, VName};
use crate::tree::{WalkError, WalkErrorKind};
use crate::utils::iter::{box_empty, box_once};
use crate::utils::{pushed, unwrap_or, Substack};
use super::import_tree::ImpMod;
use crate::ast::{Constant, Rule};
use crate::error::{ConflictingRoles, ProjectError, ProjectResult};
use crate::pipeline::source_loader::{PreItem, PreMod};
use crate::representations::project::{
ImpReport, ItemKind, ProjectEntry, ProjectExt, ProjectItem,
};
use crate::sourcefile::{
FileEntry, FileEntryKind, Member, MemberKind, ModuleBlock,
};
use crate::tree::{ModEntry, ModMember, Module};
use crate::utils::get_or_default;
use crate::utils::pushed::pushed_ref;
use crate::{Tok, VName};
#[derive(Debug)]
struct ParsedSource<'a> {
path: Vec<Tok<String>>,
loaded: &'a LoadedSource,
parsed: Vec<FileEntry>,
}
/// Split a path into file- and subpath in knowledge
///
/// # Errors
///
/// if the path is invalid
#[allow(clippy::type_complexity)] // bit too sensitive here IMO
pub fn split_path<'a>(
path: &'a [Tok<String>],
proj: &'a ProjectTree<impl NameLike>,
) -> Result<(&'a [Tok<String>], &'a [Tok<String>]), WalkError> {
let (end, body) = unwrap_or!(path.split_last(); {
return Ok((&[], &[]))
});
let mut module = (proj.0.walk_ref(body, false))?;
let entry = (module.items.get(end))
.ok_or(WalkError { pos: path.len() - 1, kind: WalkErrorKind::Missing })?;
if let ModMember::Sub(m) = &entry.member {
module = m;
}
let file =
module.extra.file.as_ref().map(|s| &path[..s.len()]).unwrap_or(path);
let subpath = &path[file.len()..];
Ok((file, subpath))
}
/// Convert normalized, prefixed source into a module
///
/// # Panics
///
/// - if there are imports with too many "super" prefixes (this is normally
/// detected in preparsing)
/// - if the preparsed tree is missing a module that exists in the source
fn source_to_module(
// level
path: Substack<Tok<String>>,
preparsed: &Module<impl Clone, impl Clone>,
// data
data: Vec<FileEntry>,
// context
i: &Interner,
filepath_len: usize,
) -> ProjectResult<Module<Expr<VName>, ProjectExt<VName>>> {
let path_v = path.iter().rev_vec_clone();
let imports = (data.iter())
.filter_map(|ent| {
if let FileEntry::Import(impv) = ent { Some(impv.iter()) } else { None }
})
.flatten()
.cloned()
.collect::<Vec<_>>();
let imports_from = (imports.iter())
.map(|imp| -> ProjectResult<_> {
let mut imp_path_v = imp.path.clone();
imp_path_v
.push(imp.name.clone().expect("glob imports had just been resolved"));
let mut abs_path = absolute_path(&path_v, &imp_path_v, i)
.expect("should have failed in preparsing");
let name = abs_path.pop().ok_or_else(|| {
TooManySupers {
offender_file: path_v[..filepath_len].to_vec(),
offender_mod: path_v[filepath_len..].to_vec(),
path: imp_path_v,
}
.rc()
})?;
Ok((name, abs_path))
})
.collect::<Result<HashMap<_, _>, _>>()?;
let exports = (data.iter())
.flat_map(|ent| {
let mk_ent = |name: Tok<String>| (name.clone(), pushed(&path_v, name));
match ent {
FileEntry::Export(names) => Box::new(names.iter().cloned().map(mk_ent)),
FileEntry::Exported(mem) => match mem {
Member::Constant(constant) => box_once(mk_ent(constant.name.clone())),
Member::Module(ns) => box_once(mk_ent(ns.name.clone())),
Member::Rule(rule) => {
let mut names = Vec::new();
for e in rule.pattern.iter() {
e.search_all(&mut |e| {
if let Clause::Name(n) = &e.value {
if let Some([name]) = n.strip_prefix(&path_v[..]) {
names.push((name.clone(), n.clone()))
}
}
None::<()>
});
}
Box::new(names.into_iter())
},
},
_ => box_empty(),
}
})
.collect::<HashMap<_, _>>();
let rules = (data.iter())
.filter_map(|ent| match ent {
FileEntry::Exported(Member::Rule(rule)) => Some(rule),
FileEntry::Internal(Member::Rule(rule)) => Some(rule),
_ => None,
})
.cloned()
.collect::<Vec<_>>();
let items = (data.into_iter())
.filter_map(|ent| {
let member_to_item = |exported, member| match member {
Member::Module(ns) => {
let new_prep = unwrap_or!(
&preparsed.items[&ns.name].member => ModMember::Sub;
panic!("Preparsed should include entries for all submodules")
);
let module = match source_to_module(
path.push(ns.name.clone()),
new_prep,
ns.body,
i,
filepath_len,
) {
Err(e) => return Some(Err(e)),
Ok(t) => t,
};
let member = ModMember::Sub(module);
Some(Ok((ns.name.clone(), ModEntry { exported, member })))
},
Member::Constant(Constant { name, value }) => {
let member = ModMember::Item(value);
Some(Ok((name, ModEntry { exported, member })))
},
_ => None,
};
match ent {
FileEntry::Exported(member) => member_to_item(true, member),
FileEntry::Internal(member) => member_to_item(false, member),
_ => None,
}
})
.collect::<Result<HashMap<_, _>, _>>()?;
Ok(Module {
imports,
items,
extra: ProjectExt {
imports_from,
exports,
rules,
file: Some(path_v[..filepath_len].to_vec()),
},
})
}
fn files_to_module(
path: Substack<Tok<String>>,
files: Vec<ParsedSource>,
i: &Interner,
) -> ProjectResult<Module<Expr<VName>, ProjectExt<VName>>> {
let lvl = path.len();
debug_assert!(
files.iter().map(|f| f.path.len()).max().unwrap() >= lvl,
"path is longer than any of the considered file paths"
);
let path_v = path.iter().rev_vec_clone();
if files.len() == 1 && files[0].path.len() == lvl {
return source_to_module(
path.clone(),
&files[0].loaded.preparsed.0,
files[0].parsed.clone(),
i,
path.len(),
);
}
let items = (files.into_iter())
.group_by(|f| f.path[lvl].clone())
.into_iter()
.map(|(namespace, files)| -> ProjectResult<_> {
let subpath = path.push(namespace.clone());
let files_v = files.collect::<Vec<_>>();
let module = files_to_module(subpath, files_v, i)?;
let member = ModMember::Sub(module);
Ok((namespace, ModEntry { exported: true, member }))
})
.collect::<Result<HashMap<_, _>, _>>()?;
let exports: HashMap<_, _> = (items.keys())
.map(|name| (name.clone(), pushed(&path_v, name.clone())))
.collect();
Ok(Module {
items,
imports: vec![],
extra: ProjectExt {
exports,
imports_from: HashMap::new(),
rules: vec![],
file: None,
},
})
pub struct TreeReport {
pub entries: HashMap<Tok<String>, ProjectEntry<VName>>,
pub rules: Vec<Rule<VName>>,
/// Maps imported symbols to the absolute paths of the modules they are
/// imported from
pub imports_from: HashMap<Tok<String>, ImpReport<VName>>,
}
pub fn build_tree(
files: LoadedSourceTable,
i: &Interner,
path: &VName,
source: Vec<FileEntry>,
Module { entries, extra }: PreMod,
imports: ImpMod,
prelude: &[FileEntry],
injected: &impl InjectedOperatorsFn,
) -> ProjectResult<ProjectTree<VName>> {
assert!(!files.is_empty(), "A tree requires at least one module");
let ops_cache = collect_ops::mk_cache(&files, injected);
let mut entries = files
.iter()
.map(|(path, loaded)| {
Ok((path, loaded, parse_file(path, &files, &ops_cache, i, prelude)?))
) -> ProjectResult<TreeReport> {
let source =
source.into_iter().chain(prelude.iter().cloned()).collect::<Vec<_>>();
let (imports_from, mut submod_imports) = (imports.entries.into_iter())
.partition_map::<HashMap<_, _>, HashMap<_, _>, _, _, _>(
|(n, ent)| match ent.member {
ModMember::Item(it) => Either::Left((n, it)),
ModMember::Sub(s) => Either::Right((n, s)),
},
);
let mut rule_fragments = Vec::new();
let mut submodules = HashMap::<_, Vec<_>>::new();
let mut consts = HashMap::new();
for FileEntry { kind, locations: _ } in source {
match kind {
FileEntryKind::Import(_) => (),
FileEntryKind::Comment(_) => (),
FileEntryKind::Export(_) => (),
FileEntryKind::Member(Member { kind, .. }) => match kind {
MemberKind::Module(ModuleBlock { body, name }) => {
get_or_default(&mut submodules, &name).extend(body.into_iter());
},
MemberKind::Constant(Constant { name, value }) => {
consts.insert(name, value /* .prefix(path, &|_| false) */);
},
MemberKind::Operators(_) => (),
MemberKind::Rule(rule) => rule_fragments.push(rule),
},
}
}
let mod_details = extra.details().expect("Directories handled elsewhere");
let rules = (mod_details.patterns.iter())
.zip(rule_fragments.into_iter())
.map(|(p, Rule { prio, template: t, .. })| {
// let p = p.iter().map(|e| e.prefix(path, &|_| false)).collect();
// let t = t.into_iter().map(|e| e.prefix(path, &|_| false)).collect();
Rule { pattern: p.clone(), prio, template: t }
})
.collect::<ProjectResult<Vec<_>>>()?;
// sort by similarity, then longest-first
entries.sort_unstable_by(|a, b| a.0.cmp(b.0).reverse());
let files = entries
.into_iter()
.map(|(path, loaded, parsed)| ParsedSource {
loaded,
parsed,
path: path.clone(),
.collect();
let (pre_subs, pre_items) = (entries.into_iter())
.partition_map::<HashMap<_, _>, HashMap<_, _>, _, _, _>(
|(k, ModEntry { exported, member })| match member {
ModMember::Sub(s) => Either::Left((k, (exported, s))),
ModMember::Item(it) => Either::Right((k, (exported, it))),
},
);
let mut entries = (pre_subs.into_iter())
.map(|(k, (exported, pre_member))| {
let impmod = (submod_imports.remove(&k))
.expect("Imports and preparsed should line up");
(k, exported, pre_member, impmod)
})
.collect::<Vec<_>>();
Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)?))
.map(|(k, exported, pre, imp)| {
let source = submodules
.remove(&k)
.expect("Submodules should not disappear after reparsing");
(k, exported, pre, imp, source)
})
.map(|(k, exported, pre, imp, source)| {
let path = pushed_ref(path, k.clone());
let TreeReport { entries, imports_from, rules } =
build_tree(&path, source, pre, imp, prelude)?;
let extra = ProjectExt { path, file: None, imports_from, rules };
let member = ModMember::Sub(Module { entries, extra });
Ok((k, ModEntry { exported, member }))
})
.chain((pre_items.into_iter()).map(
|(k, (exported, PreItem { has_value, is_op, location }))| {
let item = match imports_from.get(&k) {
Some(_) if has_value => {
// Local value cannot be assigned to imported key
let const_loc =
consts.remove(&k).expect("has_value is true").location;
let err = ConflictingRoles {
locations: vec![location, const_loc],
name: pushed_ref(path, k),
};
return Err(err.rc());
},
None => {
let k = consts.remove(&k).map_or(ItemKind::None, ItemKind::Const);
ProjectItem { is_op, kind: k }
},
Some(report) => ProjectItem {
is_op: is_op | report.is_op,
kind: ItemKind::Alias(report.source.clone()),
},
};
Ok((k, ModEntry { exported, member: ModMember::Item(item) }))
},
))
.collect::<Result<HashMap<_, _>, _>>()?;
for (k, from) in imports_from.iter() {
let (_, ent) = entries.raw_entry_mut().from_key(k).or_insert_with(|| {
(k.clone(), ModEntry {
exported: false,
member: ModMember::Item(ProjectItem {
kind: ItemKind::Alias(from.source.clone()),
is_op: from.is_op,
}),
})
});
debug_assert!(
matches!(
ent.member,
ModMember::Item(ProjectItem { kind: ItemKind::Alias(_), .. })
),
"Should have emerged in the processing of pre_items"
)
}
Ok(TreeReport { entries, rules, imports_from })
}

79
src/pipeline/project_tree/collect_ops/exported_ops.rs
View File

@@ -1,79 +0,0 @@
use std::rc::Rc;
use hashbrown::HashSet;
use trait_set::trait_set;
use crate::error::{NotFound, ProjectError, ProjectResult};
use crate::interner::Tok;
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::representations::tree::WalkErrorKind;
use crate::utils::{split_max_prefix, Cache};
use crate::Sym;
pub type OpsResult = ProjectResult<Rc<HashSet<Tok<String>>>>;
pub type ExportedOpsCache<'a> = Cache<'a, Sym, OpsResult>;
trait_set! {
pub trait InjectedOperatorsFn = Fn(Sym) -> Option<Rc<HashSet<Tok<String>>>>;
}
fn coprefix<T: Eq>(
l: impl Iterator<Item = T>,
r: impl Iterator<Item = T>,
) -> usize {
l.zip(r).take_while(|(a, b)| a == b).count()
}
/// Collect all names exported by the module at the specified path
pub fn collect_exported_ops(
path: Sym,
loaded: &LoadedSourceTable,
injected: &impl InjectedOperatorsFn,
) -> OpsResult {
let injected =
injected(path.clone()).unwrap_or_else(|| Rc::new(HashSet::new()));
match split_max_prefix(&path, &|n| loaded.contains_key(n)) {
None => {
let ops = (loaded.keys())
.filter_map(|modname| {
if path.len() == coprefix(path.iter(), modname.iter()) {
Some(modname[path.len()].clone())
} else {
None
}
})
.chain(injected.iter().cloned())
.collect::<HashSet<_>>();
Ok(Rc::new(ops))
},
Some((fpath, subpath)) => {
let preparsed = &loaded[fpath].preparsed;
let module = preparsed.0.walk_ref(subpath, false).map_err(
|walk_err| match walk_err.kind {
WalkErrorKind::Private => {
unreachable!("visibility is not being checked here")
},
WalkErrorKind::Missing => NotFound {
source: None,
file: fpath.to_vec(),
subpath: subpath[..walk_err.pos].to_vec(),
}
.rc(),
},
)?;
let out = (module.items.iter())
.filter(|(_, v)| v.exported)
.map(|(k, _)| k.clone())
.chain(injected.iter().cloned())
.collect::<HashSet<_>>();
Ok(Rc::new(out))
},
}
}
pub fn mk_cache<'a>(
loaded: &'a LoadedSourceTable,
injected: &'a impl InjectedOperatorsFn,
) -> ExportedOpsCache<'a> {
Cache::new(|path, _this| collect_exported_ops(path, loaded, injected))
}

8
src/pipeline/project_tree/collect_ops/mod.rs
View File

@@ -1,8 +0,0 @@
mod exported_ops;
mod ops_for;
pub use exported_ops::{
collect_exported_ops, mk_cache, ExportedOpsCache, InjectedOperatorsFn,
OpsResult,
};
pub use ops_for::collect_ops_for;

53
src/pipeline/project_tree/collect_ops/ops_for.rs
View File

@@ -1,53 +0,0 @@
use std::rc::Rc;
use hashbrown::HashSet;
use super::exported_ops::{ExportedOpsCache, OpsResult};
use crate::error::ProjectResult;
use crate::interner::{Interner, Tok};
use crate::parse::is_op;
use crate::pipeline::import_abs_path::import_abs_path;
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::representations::tree::{ModMember, Module};
/// Collect all operators and names, exported or local, defined in this
/// tree.
fn tree_all_ops(
module: &Module<impl Clone, impl Clone>,
ops: &mut HashSet<Tok<String>>,
) {
ops.extend(module.items.keys().cloned());
for ent in module.items.values() {
if let ModMember::Sub(m) = &ent.member {
tree_all_ops(m, ops);
}
}
}
/// Collect all names visible in this file
///
/// # Panics
///
/// if any import contains too many Super calls. This should be caught during
/// preparsing
pub fn collect_ops_for(
file: &[Tok<String>],
loaded: &LoadedSourceTable,
ops_cache: &ExportedOpsCache,
i: &Interner,
) -> OpsResult {
let tree = &loaded[file].preparsed.0;
let mut ret = HashSet::new();
tree_all_ops(tree, &mut ret);
tree.visit_all_imports(&mut |modpath, _m, import| -> ProjectResult<()> {
if let Some(n) = &import.name {
ret.insert(n.clone());
} else {
let path = import_abs_path(file, modpath, &import.path, i)
.expect("This error should have been caught during loading");
ret.extend(ops_cache.find(&i.i(&path))?.iter().cloned());
}
Ok(())
})?;
Ok(Rc::new(ret.into_iter().filter(|t| is_op(&**t)).collect()))
}

169
src/pipeline/project_tree/import_tree.rs Normal file
View File

@@ -0,0 +1,169 @@
use std::cmp::Ordering;
use std::fmt::Display;
use std::rc::Rc;
use hashbrown::HashMap;
use itertools::Itertools;
use crate::error::{ProjectError, ProjectResult};
use crate::pipeline::source_loader::{PreMod, Preparsed};
use crate::representations::project::ImpReport;
use crate::sourcefile::{absolute_path, Import};
use crate::tree::{ErrKind, ModEntry, ModMember, Module, WalkError};
use crate::utils::iter::{box_chain, box_once};
use crate::utils::pushed::pushed_ref;
use crate::utils::{unwrap_or, BoxedIter};
use crate::{Interner, ProjectTree, Tok, VName};
pub type ImpMod = Module<ImpReport<VName>, ()>;
/// Assert that a module identified by a path can see a given symbol
fn assert_visible<'a>(
source: &'a [Tok<String>], // must point to a file or submodule
target: &'a [Tok<String>],
root: &'a Module<impl Clone + Display, impl Clone + Display>,
) -> Result<(), WalkError<'a>> {
if target.split_last().map_or(true, |(_, m)| source.starts_with(m)) {
// The global module (empty path) is always visible
return Ok(()); // Symbols in ancestor modules are always visible
}
// walk the first section where visibility is ignored
let shared_len =
source.iter().zip(target.iter()).take_while(|(a, b)| a == b).count();
let (shared_path, deep_path) = target.split_at(shared_len + 1);
let private_root = root.walk_ref(&[], shared_path, false)?;
// walk the second part where visibility matters
private_root.walk1_ref(shared_path, deep_path, true)?;
Ok(())
}
pub fn assert_visible_overlay<'a>(
source: &'a [Tok<String>], // must point to a file or submodule
target: &'a [Tok<String>],
first: &'a Module<impl Clone + Display, impl Clone + Display>,
fallback: &'a Module<impl Clone + Display, impl Clone + Display>,
) -> Result<(), WalkError<'a>> {
assert_visible(source, target, first).or_else(|e1| {
if e1.kind == ErrKind::Missing {
match assert_visible(source, target, fallback) {
// if both are walk errors, report the longer of the two
Err(mut e2) if e2.kind == ErrKind::Missing =>
Err(match e1.depth().cmp(&e2.depth()) {
Ordering::Less => e2,
Ordering::Greater => e1,
Ordering::Equal => {
e2.options = box_chain!(e2.options, e1.options);
e2
},
}),
// otherwise return the parent's result
x => x,
}
} else {
Err(e1)
}
})
}
pub fn process_donor_module<'a, TItem: Clone>(
module: &'a Module<TItem, impl Clone>,
abs_path: Rc<VName>,
is_op: impl Fn(&TItem) -> bool + 'a,
) -> impl Iterator<Item = (Tok<String>, VName, bool)> + 'a {
(module.entries.iter()).filter(|(_, ent)| ent.exported).map(
move |(n, ent)| {
let is_op = ent.item().map_or(false, &is_op);
(n.clone(), pushed_ref(abs_path.as_ref(), n.clone()), is_op)
},
)
}
pub fn import_tree(
modpath: VName,
pre: &PreMod,
root: &Preparsed,
prev_root: &ProjectTree<VName>,
i: &Interner,
) -> ProjectResult<ImpMod> {
let imports = pre.extra.details().map_or(&[][..], |e| &e.imports[..]);
let entries = (imports.iter())
// imports become leaf sets
.map(|Import { name, path, location }| -> ProjectResult<BoxedIter<_>> {
let mut abs_path = absolute_path(&modpath, path, i, location)?;
Ok(if let Some(name) = name {
// named imports are validated and translated 1->1
abs_path.push(name.clone());
assert_visible_overlay(&modpath, &abs_path, &root.0, &prev_root.0)
.map_err(|e| -> Rc<dyn ProjectError> {
println!("Current root: {}", &root.0);
// println!("Old root: {:#?}", &prev_root.0);
panic!("{}", e.at(location))
})?;
let is_op = (root.0.walk1_ref(&[], &abs_path, false))
.map(|(ent, _)| ent.item().map_or(false, |i| i.is_op))
.or_else(|e| if e.kind == ErrKind::Missing {
(prev_root.0.walk1_ref(&[], &abs_path, false))
.map(|(ent, _)| ent.item().map_or(false, |i| i.is_op))
} else {Err(e)})
.map_err(|e| e.at(location))?;
box_once((name.clone(), abs_path, is_op))
} else {
let rc_path = Rc::new(abs_path);
// wildcard imports are validated
assert_visible_overlay(&modpath, &rc_path, &root.0, &prev_root.0)
.map_err(|e| e.at(location))?;
// and instantiated for all exports of the target 1->n
let new_imports = match (root.0).walk_ref(&[], &rc_path, false) {
Err(e) if e.kind == ErrKind::Missing => Err(e),
Err(e) => return Err(e.at(location)),
Ok(module)
=> Ok(process_donor_module(module, rc_path.clone(), |i| i.is_op))
};
let old_m = match (prev_root.0).walk_ref(&[], &rc_path, false) {
Err(e) if e.kind != ErrKind::Missing => return Err(e.at(location)),
Err(e1) => match new_imports {
Ok(it) => return Ok(Box::new(it)),
Err(mut e2) => return Err(match e1.depth().cmp(&e2.depth()) {
Ordering::Less => e2.at(location),
Ordering::Greater => e1.at(location),
Ordering::Equal => {
e2.options = box_chain!(e2.options, e1.options);
e2.at(location)
},
}),
},
Ok(old_m) => old_m,
};
let it1 = process_donor_module(old_m, rc_path.clone(), |i| i.is_op);
match new_imports {
Err(_) => Box::new(it1),
Ok(it2) => box_chain!(it1, it2)
}
})
})
// leaf sets flattened to leaves
.flatten_ok()
// translated to entries
.map_ok(|(name, source, is_op)| {
(name, ModEntry {
exported: false, // this is irrelevant but needed
member: ModMember::Item(ImpReport { source, is_op }),
})
})
.chain(
(pre.entries.iter())
// recurse on submodules
.filter_map(|(k, v)| {
Some((k, v, unwrap_or!(&v.member => ModMember::Sub; return None)))
})
.map(|(k, v, pre)| {
let path = pushed_ref(&modpath, k.clone());
Ok((k.clone(), ModEntry {
exported: v.exported,
member: ModMember::Sub(import_tree(path, pre, root, prev_root, i)?),
}))
}),
)
.collect::<Result<HashMap<_, _>, _>>()?;
Ok(Module { extra: (), entries })
}

25
src/pipeline/project_tree/mod.rs
View File

@@ -1,24 +1,5 @@
// FILE SEPARATION BOUNDARY
//
// Collect all operators accessible in each file, parse the files with
// correct tokenization, resolve glob imports, convert expressions to
// refer to tokens with (local) absolute path, and connect them into a
// single tree.
//
// The module checks for imports from missing modules (including
// submodules). All other errors must be checked later.
//
// Injection strategy:
// Return all items of the given module in the injected tree for
// `injected` The output of this stage is a tree, which can simply be
// overlaid with the injected tree
mod add_prelude;
mod build_tree;
mod collect_ops;
mod normalize_imports;
mod parse_file;
mod prefix;
mod import_tree;
mod rebuild_tree;
pub use build_tree::{build_tree, split_path};
pub use collect_ops::InjectedOperatorsFn;
pub use rebuild_tree::rebuild_tree;

94
src/pipeline/project_tree/normalize_imports.rs
View File

@@ -1,94 +0,0 @@
use super::collect_ops::ExportedOpsCache;
use crate::interner::{Interner, Tok};
use crate::pipeline::import_abs_path::import_abs_path;
use crate::representations::sourcefile::{
FileEntry, Import, Member, ModuleBlock,
};
use crate::representations::tree::{ModMember, Module};
use crate::utils::iter::box_once;
use crate::utils::{unwrap_or, BoxedIter, Substack};
fn member_rec(
// level
mod_stack: Substack<Tok<String>>,
preparsed: &Module<impl Clone, impl Clone>,
// object
member: Member,
// context
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner,
) -> Member {
match member {
Member::Module(ModuleBlock { name, body }) => {
let subprep = unwrap_or!(
&preparsed.items[&name].member => ModMember::Sub;
unreachable!("This name must point to a namespace")
);
let new_stack = mod_stack.push(name.clone());
let new_body = entv_rec(new_stack, subprep, body, path, ops_cache, i);
Member::Module(ModuleBlock { name, body: new_body })
},
any => any,
}
}
/// Normalize imports in the FileEntry list recursively
///
/// # Panics
///
/// - if a path contains too many "super" prefixes
/// - if the exported operators in a module cannot be determined
fn entv_rec(
// level
mod_stack: Substack<Tok<String>>,
preparsed: &Module<impl Clone, impl Clone>,
// object
data: Vec<FileEntry>,
// context
mod_path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner,
) -> Vec<FileEntry> {
data
.into_iter()
.map(|ent| match ent {
FileEntry::Import(imps) => FileEntry::Import(
imps
.into_iter()
.flat_map(|import| {
if let Import { name: None, path } = import {
let p = import_abs_path(mod_path, mod_stack.clone(), &path, i)
.expect("Should have emerged in preparsing");
let names = (ops_cache.find(&i.i(&p)))
.expect("Should have emerged in second parsing");
let imports = (names.iter())
.map(|n| Import { name: Some(n.clone()), path: path.clone() })
.collect::<Vec<_>>();
Box::new(imports.into_iter()) as BoxedIter<Import>
} else {
box_once(import)
}
})
.collect(),
),
FileEntry::Exported(mem) => FileEntry::Exported(member_rec(
mod_stack.clone(), preparsed, mem, mod_path, ops_cache, i,
)),
FileEntry::Internal(mem) => FileEntry::Internal(member_rec(
mod_stack.clone(), preparsed, mem, mod_path, ops_cache, i,
)),
any => any,
})
.collect()
}
pub fn normalize_imports(
preparsed: &Module<impl Clone, impl Clone>,
data: Vec<FileEntry>,
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner,
) -> Vec<FileEntry> {
entv_rec(Substack::Bottom, preparsed, data, path, ops_cache, i)
}

46
src/pipeline/project_tree/parse_file.rs
View File

@@ -1,46 +0,0 @@
use std::rc::Rc;
use super::add_prelude::add_prelude;
use super::collect_ops::{collect_ops_for, ExportedOpsCache};
use super::normalize_imports::normalize_imports;
use super::prefix::prefix;
use crate::error::ProjectResult;
use crate::interner::{Interner, Tok};
use crate::parse;
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::representations::sourcefile::{normalize_namespaces, FileEntry};
/// Parses a file with the correct operator set
///
/// # Panics
///
/// - on syntax error
/// - if namespaces are exported inconsistently
///
/// These are both checked in the preparsing stage
pub fn parse_file(
path: &[Tok<String>],
loaded: &LoadedSourceTable,
ops_cache: &ExportedOpsCache,
i: &Interner,
prelude: &[FileEntry],
) -> ProjectResult<Vec<FileEntry>> {
let ld = &loaded[path];
// let ops_cache = collect_ops::mk_cache(loaded, i);
let ops = collect_ops_for(path, loaded, ops_cache, i)?;
let ops_vec = ops.iter().map(|t| (**t).clone()).collect::<Vec<_>>();
let ctx = parse::ParsingContext {
interner: i,
ops: &ops_vec,
file: Rc::new(Interner::extern_all(path)),
};
let entries = parse::parse2(ld.text.as_str(), ctx)
.expect("This error should have been caught during loading");
let with_prelude = add_prelude(entries, path, prelude);
let impnormalized =
normalize_imports(&ld.preparsed.0, with_prelude, path, ops_cache, i);
let nsnormalized = normalize_namespaces(Box::new(impnormalized.into_iter()))
.expect("This error should have been caught during preparsing");
let prefixed = prefix(nsnormalized, path, ops_cache, i);
Ok(prefixed)
}

73
src/pipeline/project_tree/prefix.rs
View File

@@ -1,73 +0,0 @@
use super::collect_ops::ExportedOpsCache;
use crate::ast::{Constant, Rule};
use crate::interner::{Interner, Tok};
use crate::representations::sourcefile::{FileEntry, Member, ModuleBlock};
use crate::utils::Substack;
fn member_rec(
// level
mod_stack: Substack<Tok<String>>,
// object
data: Member,
// context
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner,
) -> Member {
let prefix = (path.iter())
.cloned()
.chain(mod_stack.iter().rev_vec_clone().into_iter())
.collect::<Vec<_>>();
match data {
Member::Module(ModuleBlock { name, body }) => {
let new_stack = mod_stack.push(name.clone());
let new_body = entv_rec(new_stack, body, path, ops_cache, i);
Member::Module(ModuleBlock { name, body: new_body })
},
Member::Constant(constant) => Member::Constant(Constant {
name: constant.name,
value: constant.value.prefix(&prefix, &|_| false),
}),
Member::Rule(rule) => Member::Rule(Rule {
prio: rule.prio,
pattern: (rule.pattern.into_iter())
.map(|e| e.prefix(&prefix, &|_| false))
.collect(),
template: (rule.template.into_iter())
.map(|e| e.prefix(&prefix, &|_| false))
.collect(),
}),
}
}
fn entv_rec(
// level
mod_stack: Substack<Tok<String>>,
// object
data: Vec<FileEntry>,
// context
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner,
) -> Vec<FileEntry> {
(data.into_iter())
.map(|fe| {
let (mem, wrapper): (Member, fn(Member) -> FileEntry) = match fe {
FileEntry::Exported(mem) => (mem, FileEntry::Exported),
FileEntry::Internal(mem) => (mem, FileEntry::Internal),
// XXX should [FileEntry::Export] be prefixed?
any => return any,
};
wrapper(member_rec(mod_stack.clone(), mem, path, ops_cache, i))
})
.collect()
}
pub fn prefix(
data: Vec<FileEntry>,
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner,
) -> Vec<FileEntry> {
entv_rec(Substack::Bottom, data, path, ops_cache, i)
}

131
src/pipeline/project_tree/rebuild_tree.rs Normal file
View File

@@ -0,0 +1,131 @@
use std::rc::Rc;
use hashbrown::HashMap;
use itertools::Itertools;
use super::build_tree::{build_tree, TreeReport};
use super::import_tree::{import_tree, ImpMod};
use crate::error::ProjectResult;
use crate::pipeline::source_loader::{
LoadedSourceTable, PreExtra, PreItem, PreMod, Preparsed,
};
use crate::representations::project::{ImpReport, ProjectExt, ProjectMod};
use crate::sourcefile::FileEntry;
use crate::tree::{ModEntry, ModMember, Module};
use crate::utils::never::{always, unwrap_always};
use crate::utils::pushed::pushed_ref;
use crate::utils::unwrap_or;
use crate::{parse, Interner, ProjectTree, Tok, VName};
pub fn rebuild_file(
path: Vec<Tok<String>>,
pre: PreMod,
imports: ImpMod,
source: &LoadedSourceTable,
prelude: &[FileEntry],
i: &Interner,
) -> ProjectResult<ProjectMod<VName>> {
let file = match &pre.extra {
PreExtra::Dir => panic!("Dir should not hand this node off"),
PreExtra::Submod(_) => panic!("should not have received this"),
PreExtra::File(f) => f,
};
let mut ops = Vec::new();
unwrap_always(imports.search_all((), &mut |_, module, ()| {
ops.extend(
(module.entries.iter())
.filter(|(_, ent)| {
matches!(ent.member, ModMember::Item(ImpReport { is_op: true, .. }))
})
.map(|(name, _)| name.clone()),
);
always(())
}));
unwrap_always(pre.search_all((), &mut |_, module, ()| {
ops.extend(
(module.entries.iter())
.filter(|(_, ent)| {
matches!(ent.member, ModMember::Item(PreItem { is_op: true, .. }))
})
.map(|(name, _)| name.clone()),
);
always(())
}));
let ctx = parse::ParsingContext::new(&ops, i, Rc::new(path.clone()));
let src = source.get(&file.name).unwrap_or_else(|| {
panic!(
"{} should have been preparsed already. Preparsed files are {}",
Interner::extern_all(&file.name).join("/") + ".orc",
source
.keys()
.map(|f| Interner::extern_all(f).join("/") + ".orc")
.join(", ")
)
});
let entries = parse::parse2(&src.text, ctx)?;
let TreeReport { entries: items, rules, imports_from } =
build_tree(&path, entries, pre, imports, prelude)?;
Ok(Module {
entries: items,
extra: ProjectExt { file: Some(path.clone()), path, imports_from, rules },
})
}
pub fn rebuild_dir(
path: Vec<Tok<String>>,
pre: PreMod,
mut imports: ImpMod,
source: &LoadedSourceTable,
prelude: &[FileEntry],
i: &Interner,
) -> ProjectResult<ProjectMod<VName>> {
match pre.extra {
PreExtra::Dir => (),
PreExtra::File(_) =>
return rebuild_file(path, pre, imports, source, prelude, i),
PreExtra::Submod(_) => panic!("Dirs contain dirs and files"),
}
let items = (pre.entries.into_iter())
.map(|(name, entry)| {
match imports.entries.remove(&name).map(|e| e.member) {
Some(ModMember::Sub(impmod)) => (name, entry, impmod),
_ => panic!("Imports must line up with modules"),
}
})
.map(|(name, ModEntry { member, exported }, impmod)| -> ProjectResult<_> {
let path = pushed_ref(&path, name.clone());
let pre = unwrap_or!(member => ModMember::Sub;
panic!("Dirs can only contain submodules")
);
Ok((name, ModEntry {
exported,
member: ModMember::Sub(rebuild_dir(
path, pre, impmod, source, prelude, i,
)?),
}))
})
.collect::<Result<HashMap<_, _>, _>>()?;
Ok(Module {
extra: ProjectExt {
path,
imports_from: HashMap::new(),
rules: Vec::new(),
file: None,
},
entries: items,
})
}
/// Rebuild the entire tree
pub fn rebuild_tree(
source: &LoadedSourceTable,
preparsed: Preparsed,
prev_root: &ProjectTree<VName>,
prelude: &[FileEntry],
i: &Interner,
) -> ProjectResult<ProjectTree<VName>> {
let imports =
import_tree(Vec::new(), &preparsed.0, &preparsed, prev_root, i)?;
rebuild_dir(Vec::new(), preparsed.0, imports, source, prelude, i)
.map(ProjectTree)
}