Orchid/orchid
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Various Improvements

Browse Source 
- removed many panics from the pipeline
- extracted project and const tree to representations
- extended STL list support
- improved loops
This commit is contained in:
Lawrence Bethlenfalvy
2023-06-28 22:29:58 +01:00
parent 79e28883db
commit cce4b8f11c
36 changed files with 436 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
src/pipeline/project_tree/build_tree.rs
View File

@@ -3,13 +3,14 @@ use std::rc::Rc;
use hashbrown::HashMap;
use itertools::Itertools;
use super::collect_ops;
use super::collect_ops::InjectedOperatorsFn;
use super::parse_file::parse_file;
use super::{collect_ops, ProjectExt, ProjectTree};
use crate::ast::{Constant, Expr};
use crate::interner::{Interner, Tok};
use crate::pipeline::error::ProjectError;
use crate::pipeline::error::{ProjectError, TooManySupers};
use crate::pipeline::source_loader::{LoadedSource, LoadedSourceTable};
use crate::representations::project::{ProjectExt, ProjectTree};
use crate::representations::sourcefile::{absolute_path, FileEntry, Member};
use crate::representations::tree::{ModEntry, ModMember, Module};
use crate::representations::{NameLike, VName};
@@ -23,17 +24,20 @@ struct ParsedSource<'a> {
parsed: Vec<FileEntry>,
}
/// Split a path into file- and subpath in knowledge
///
/// # Panics
///
/// if the path is invalid
pub fn split_path<'a>(
path: &'a [Tok<String>],
proj: &'a ProjectTree<impl NameLike>,
) -> (&'a [Tok<String>], &'a [Tok<String>]) {
let (end, body) = if let Some(s) = path.split_last() {
s
} else {
return (&[], &[]);
};
let mut module =
proj.0.walk_ref(body, false).expect("invalid path cannot be split");
let (end, body) = unwrap_or!(path.split_last(); {
return (&[], &[])
});
let mut module = (proj.0.walk_ref(body, false))
.expect("invalid path can't have been split above");
if let ModMember::Sub(m) = &module.items[end].member {
module = m;
}
@@ -44,6 +48,12 @@ pub fn split_path<'a>(
}
/// Convert normalized, prefixed source into a module
///
/// # Panics
///
/// - if there are imports with too many "super" prefixes (this is normally
/// detected in preparsing)
/// - if the preparsed tree is missing a module that exists in the source
fn source_to_module(
// level
path: Substack<Tok<String>>,
@@ -53,29 +63,33 @@ fn source_to_module(
// context
i: &Interner,
filepath_len: usize,
) -> Module<Expr<VName>, ProjectExt<VName>> {
) -> Result<Module<Expr<VName>, ProjectExt<VName>>, Rc<dyn ProjectError>> {
let path_v = path.iter().rev_vec_clone();
let imports = data
.iter()
let imports = (data.iter())
.filter_map(|ent| {
if let FileEntry::Import(impv) = ent { Some(impv.iter()) } else { None }
})
.flatten()
.cloned()
.collect::<Vec<_>>();
let imports_from = imports
.iter()
.map(|imp| {
let imports_from = (imports.iter())
.map(|imp| -> Result<_, Rc<dyn ProjectError>> {
let mut imp_path_v = i.r(imp.path).clone();
imp_path_v.push(imp.name.expect("imports normalized"));
let mut abs_path =
absolute_path(&path_v, &imp_path_v, i).expect("tested in preparsing");
let name = abs_path.pop().expect("importing the global context");
(name, abs_path)
imp_path_v.push(imp.name.expect("glob imports had just been resolved"));
let mut abs_path = absolute_path(&path_v, &imp_path_v, i)
.expect("should have failed in preparsing");
let name = abs_path.pop().ok_or_else(|| {
TooManySupers {
offender_file: i.extern_all(&path_v[..filepath_len]),
offender_mod: i.extern_all(&path_v[filepath_len..]),
path: i.extern_all(&imp_path_v),
}
.rc()
})?;
Ok((name, abs_path))
})
.collect::<HashMap<_, _>>();
let exports = data
.iter()
.collect::<Result<HashMap<_, _>, _>>()?;
let exports = (data.iter())
.flat_map(|ent| {
let mk_ent = |name| (name, pushed(&path_v, name));
match ent {
@@ -99,8 +113,7 @@ fn source_to_module(
}
})
.collect::<HashMap<_, _>>();
let rules = data
.iter()
let rules = (data.iter())
.filter_map(|ent| match ent {
FileEntry::Exported(Member::Rule(rule)) => Some(rule),
FileEntry::Internal(Member::Rule(rule)) => Some(rule),
@@ -108,28 +121,30 @@ fn source_to_module(
})
.cloned()
.collect::<Vec<_>>();
let items = data
.into_iter()
let items = (data.into_iter())
.filter_map(|ent| {
let member_to_item = |exported, member| match member {
Member::Namespace(ns) => {
let new_prep = unwrap_or!(
&preparsed.items[&ns.name].member => ModMember::Sub;
panic!("preparsed missing a submodule")
panic!("Preparsed should include entries for all submodules")
);
let module = source_to_module(
let module = match source_to_module(
path.push(ns.name),
new_prep,
ns.body,
i,
filepath_len,
);
) {
Err(e) => return Some(Err(e)),
Ok(t) => t,
};
let member = ModMember::Sub(module);
Some((ns.name, ModEntry { exported, member }))
Some(Ok((ns.name, ModEntry { exported, member })))
},
Member::Constant(Constant { name, value }) => {
let member = ModMember::Item(value);
Some((name, ModEntry { exported, member }))
Some(Ok((name, ModEntry { exported, member })))
},
_ => None,
};
@@ -139,8 +154,8 @@ fn source_to_module(
_ => None,
}
})
.collect::<HashMap<_, _>>();
Module {
.collect::<Result<HashMap<_, _>, _>>()?;
Ok(Module {
imports,
items,
extra: ProjectExt {
@@ -149,14 +164,14 @@ fn source_to_module(
rules,
file: Some(path_v[..filepath_len].to_vec()),
},
}
})
}
fn files_to_module(
path: Substack<Tok<String>>,
files: Vec<ParsedSource>,
i: &Interner,
) -> Module<Expr<VName>, ProjectExt<VName>> {
) -> Result<Module<Expr<VName>, ProjectExt<VName>>, Rc<dyn ProjectError>> {
let lvl = path.len();
debug_assert!(
files.iter().map(|f| f.path.len()).max().unwrap() >= lvl,
@@ -172,21 +187,20 @@ fn files_to_module(
path.len(),
);
}
let items = files
.into_iter()
let items = (files.into_iter())
.group_by(|f| f.path[lvl])
.into_iter()
.map(|(namespace, files)| {
.map(|(namespace, files)| -> Result<_, Rc<dyn ProjectError>> {
let subpath = path.push(namespace);
let files_v = files.collect::<Vec<_>>();
let module = files_to_module(subpath, files_v, i);
let module = files_to_module(subpath, files_v, i)?;
let member = ModMember::Sub(module);
(namespace, ModEntry { exported: true, member })
Ok((namespace, ModEntry { exported: true, member }))
})
.collect::<HashMap<_, _>>();
.collect::<Result<HashMap<_, _>, _>>()?;
let exports: HashMap<_, _> =
items.keys().copied().map(|name| (name, pushed(&path_v, name))).collect();
Module {
Ok(Module {
items,
imports: vec![],
extra: ProjectExt {
@@ -195,7 +209,7 @@ fn files_to_module(
rules: vec![],
file: None,
},
}
})
}
pub fn build_tree(
@@ -222,5 +236,5 @@ pub fn build_tree(
path: path.clone(),
})
.collect::<Vec<_>>();
Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)))
Ok(ProjectTree(files_to_module(Substack::Bottom, files, i)?))
}

4
src/pipeline/project_tree/collect_ops/exported_ops.rs
View File

@@ -4,7 +4,7 @@ use hashbrown::HashSet;
use trait_set::trait_set;
use crate::interner::{Interner, Tok};
use crate::pipeline::error::{ModuleNotFound, ProjectError};
use crate::pipeline::error::{NotFound, ProjectError};
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::representations::tree::WalkErrorKind;
use crate::utils::{split_max_prefix, unwrap_or, Cache};
@@ -53,7 +53,7 @@ pub fn collect_exported_ops(
WalkErrorKind::Private => {
unreachable!("visibility is not being checked here")
},
WalkErrorKind::Missing => ModuleNotFound {
WalkErrorKind::Missing => NotFound {
file: i.extern_all(fpath),
subpath: (subpath.iter())
.take(walk_err.pos)

7
src/pipeline/project_tree/collect_ops/ops_for.rs
View File

@@ -24,7 +24,12 @@ fn tree_all_ops(
}
}
/// Collect all names imported in this file
/// Collect all names visible in this file
///
/// # Panics
///
/// if any import contains too many Super calls. This should be caught during
/// preparsing
pub fn collect_ops_for(
file: &[Tok<String>],
loaded: &LoadedSourceTable,

103
src/pipeline/project_tree/const_tree.rs
View File

@@ -1,103 +0,0 @@
use std::ops::Add;
use hashbrown::HashMap;
use super::{ProjectExt, ProjectModule, ProjectTree};
use crate::ast::{Clause, Expr};
use crate::foreign::{Atom, Atomic, ExternFn};
use crate::interner::Tok;
use crate::representations::location::Location;
use crate::representations::tree::{ModEntry, ModMember, Module};
use crate::representations::{Primitive, VName};
use crate::utils::{pushed, Substack};
/// A lightweight module tree that can be built declaratively by hand to
/// describe libraries of external functions in Rust. It implements [Add] for
/// added convenience
pub enum ConstTree {
/// A function or constant
Const(Expr<VName>),
/// A submodule
Tree(HashMap<Tok<String>, ConstTree>),
}
impl ConstTree {
/// Describe a [Primitive]
pub fn primitive(primitive: Primitive) -> Self {
Self::Const(Expr {
location: Location::Unknown,
value: Clause::P(primitive),
})
}
/// Describe an [ExternFn]
pub fn xfn(xfn: impl ExternFn + 'static) -> Self {
Self::primitive(Primitive::ExternFn(Box::new(xfn)))
}
/// Describe an [Atomic]
pub fn atom(atom: impl Atomic + 'static) -> Self {
Self::primitive(Primitive::Atom(Atom(Box::new(atom))))
}
/// Describe a module
pub fn tree(arr: impl IntoIterator<Item = (Tok<String>, Self)>) -> Self {
Self::Tree(arr.into_iter().collect())
}
}
impl Add for ConstTree {
type Output = ConstTree;
fn add(self, rhs: ConstTree) -> Self::Output {
if let (Self::Tree(t1), Self::Tree(mut t2)) = (self, rhs) {
let mut product = HashMap::new();
for (key, i1) in t1 {
if let Some(i2) = t2.remove(&key) {
product.insert(key, i1 + i2);
} else {
product.insert(key, i1);
}
}
product.extend(t2.into_iter());
Self::Tree(product)
} else {
panic!("cannot combine tree and value fields")
}
}
}
fn from_const_tree_rec(
path: Substack<Tok<String>>,
consts: HashMap<Tok<String>, ConstTree>,
file: &[Tok<String>],
) -> ProjectModule<VName> {
let mut items = HashMap::new();
let path_v = path.iter().rev_vec_clone();
for (name, item) in consts {
items.insert(name, ModEntry {
exported: true,
member: match item {
ConstTree::Const(c) => ModMember::Item(c),
ConstTree::Tree(t) =>
ModMember::Sub(from_const_tree_rec(path.push(name), t, file)),
},
});
}
let exports =
items.keys().map(|name| (*name, pushed(&path_v, *name))).collect();
Module {
items,
imports: vec![],
extra: ProjectExt {
exports,
file: Some(file.to_vec()),
..Default::default()
},
}
}
/// Convert a map of [ConstTree] into a [ProjectTree] that can be used with the
/// layered parsing system
pub fn from_const_tree(
consts: HashMap<Tok<String>, ConstTree>,
file: &[Tok<String>],
) -> ProjectTree<VName> {
let module = from_const_tree_rec(Substack::Bottom, consts, file);
ProjectTree(module)
}

7
src/pipeline/project_tree/mod.rs
View File

@@ -16,16 +16,9 @@
mod add_prelude;
mod build_tree;
mod collect_ops;
mod const_tree;
mod normalize_imports;
mod parse_file;
mod prefix;
mod tree;
pub use build_tree::{build_tree, split_path};
pub use collect_ops::InjectedOperatorsFn;
pub use const_tree::{from_const_tree, ConstTree};
pub use tree::{
collect_consts, collect_rules, vname_to_sym_tree, ProjectExt, ProjectModule,
ProjectTree,
};

12
src/pipeline/project_tree/normalize_imports.rs
View File

@@ -33,6 +33,12 @@ fn member_rec(
}
}
/// Normalize imports in the FileEntry list recursively
///
/// # Panics
///
/// - if a path contains too many "super" prefixes
/// - if the exported operators in a module cannot be determined
fn entv_rec(
// level
mod_stack: Substack<Tok<String>>,
@@ -54,11 +60,9 @@ fn entv_rec(
if let Import { name: None, path } = import {
let p = import_abs_path(mod_path, mod_stack, &i.r(path)[..], i)
.expect("Should have emerged in preparsing");
let names = ops_cache
.find(&i.i(&p))
let names = (ops_cache.find(&i.i(&p)))
.expect("Should have emerged in second parsing");
let imports = names
.iter()
let imports = (names.iter())
.map(move |&n| Import { name: Some(n), path })
.collect::<Vec<_>>();
Box::new(imports.into_iter()) as BoxedIter<Import>

8
src/pipeline/project_tree/parse_file.rs
View File

@@ -10,6 +10,14 @@ use crate::pipeline::error::ProjectError;
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::representations::sourcefile::{normalize_namespaces, FileEntry};
/// Parses a file with the correct operator set
///
/// # Panics
///
/// - on syntax error
/// - if namespaces are exported inconsistently
///
/// These are both checked in the preparsing stage
pub fn parse_file(
path: &[Tok<String>],
loaded: &LoadedSourceTable,

144
src/pipeline/project_tree/tree.rs
View File

@@ -1,144 +0,0 @@
use std::ops::Add;
use hashbrown::HashMap;
use crate::ast::{Expr, Rule};
use crate::interner::{Interner, Tok};
use crate::representations::tree::{ModMember, Module};
use crate::representations::NameLike;
use crate::tree::ModEntry;
use crate::utils::Substack;
use crate::{Sym, VName};
/// Additional data about a loaded module beyond the list of constants and
/// submodules
#[derive(Clone, Debug, Default)]
pub struct ProjectExt<N: NameLike> {
/// Pairs each foreign token to the module it was imported from
pub imports_from: HashMap<Tok<String>, N>,
/// Pairs each exported token to its original full name
pub exports: HashMap<Tok<String>, N>,
/// All rules defined in this module, exported or not
pub rules: Vec<Rule<N>>,
/// Filename, if known, for error reporting
pub file: Option<Vec<Tok<String>>>,
}
impl<N: NameLike> Add for ProjectExt<N> {
type Output = Self;
fn add(mut self, rhs: Self) -> Self::Output {
let ProjectExt { imports_from, exports, rules, file } = rhs;
self.imports_from.extend(imports_from.into_iter());
self.exports.extend(exports.into_iter());
self.rules.extend(rules.into_iter());
if file.is_some() {
self.file = file
}
self
}
}
/// A node in the tree describing the project
pub type ProjectModule<N> = Module<Expr<N>, ProjectExt<N>>;
/// Module corresponding to the root of a project
#[derive(Debug, Clone)]
pub struct ProjectTree<N: NameLike>(pub ProjectModule<N>);
fn collect_rules_rec<N: NameLike>(
bag: &mut Vec<Rule<N>>,
module: &ProjectModule<N>,
) {
bag.extend(module.extra.rules.iter().cloned());
for item in module.items.values() {
if let ModMember::Sub(module) = &item.member {
collect_rules_rec(bag, module);
}
}
}
/// Collect the complete list of rules to be used by the rule repository from
/// the [ProjectTree]
pub fn collect_rules<N: NameLike>(project: &ProjectTree<N>) -> Vec<Rule<N>> {
let mut rules = Vec::new();
collect_rules_rec(&mut rules, &project.0);
rules
}
fn collect_consts_rec<N: NameLike>(
path: Substack<Tok<String>>,
bag: &mut HashMap<Sym, Expr<N>>,
module: &ProjectModule<N>,
i: &Interner,
) {
for (key, entry) in module.items.iter() {
match &entry.member {
ModMember::Item(expr) => {
let mut name = path.iter().rev_vec_clone();
name.push(*key);
bag.insert(i.i(&name), expr.clone());
},
ModMember::Sub(module) =>
collect_consts_rec(path.push(*key), bag, module, i),
}
}
}
/// Extract the symbol table from a [ProjectTree]
pub fn collect_consts<N: NameLike>(
project: &ProjectTree<N>,
i: &Interner,
) -> HashMap<Sym, Expr<N>> {
let mut consts = HashMap::new();
collect_consts_rec(Substack::Bottom, &mut consts, &project.0, i);
consts
}
fn vname_to_sym_tree_rec(
tree: ProjectModule<VName>,
i: &Interner,
) -> ProjectModule<Sym> {
let process_expr = |ex: Expr<VName>| ex.transform_names(&|n| i.i(&n));
ProjectModule {
imports: tree.imports,
items: (tree.items.into_iter())
.map(|(k, ModEntry { exported, member })| {
(k, ModEntry {
exported,
member: match member {
ModMember::Sub(module) =>
ModMember::Sub(vname_to_sym_tree_rec(module, i)),
ModMember::Item(ex) => ModMember::Item(process_expr(ex)),
},
})
})
.collect(),
extra: ProjectExt {
imports_from: (tree.extra.imports_from.into_iter())
.map(|(k, v)| (k, i.i(&v)))
.collect(),
exports: (tree.extra.exports.into_iter())
.map(|(k, v)| (k, i.i(&v)))
.collect(),
rules: (tree.extra.rules.into_iter())
.map(|Rule { pattern, prio, template }| Rule {
pattern: pattern.into_iter().map(process_expr).collect(),
prio,
template: template.into_iter().map(process_expr).collect(),
})
.collect(),
file: tree.extra.file,
},
}
}
/// Convert a flexible vname-based tree to a more rigid but faster symbol-based
/// tree. The pipeline works with vnames, but the macro executor works with
/// symbols.
pub fn vname_to_sym_tree(
tree: ProjectTree<VName>,
i: &Interner,
) -> ProjectTree<Sym> {
ProjectTree(vname_to_sym_tree_rec(tree.0, i))
}