Orchid/orchid
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preparation for sharing

Browse Source 
- rustfmt
- clippy
- comments
- README
This commit is contained in:
Lawrence Bethlenfalvy
2023-05-25 19:14:24 +01:00
parent e99ade92ba
commit bc2714aad8
144 changed files with 3734 additions and 3243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/pipeline/error/mod.rs
View File

@@ -1,15 +1,15 @@
mod project_error;
mod parse_error_with_path;
mod unexpected_directory;
mod module_not_found;
mod not_exported;
mod parse_error_with_path;
mod project_error;
mod too_many_supers;
mod unexpected_directory;
mod visibility_mismatch;
pub use project_error::{ErrorPosition, ProjectError};
pub use parse_error_with_path::ParseErrorWithPath;
pub use unexpected_directory::UnexpectedDirectory;
pub use module_not_found::ModuleNotFound;
pub use not_exported::NotExported;
pub use parse_error_with_path::ParseErrorWithPath;
pub use project_error::{ErrorPosition, ProjectError};
pub use too_many_supers::TooManySupers;
pub use visibility_mismatch::VisibilityMismatch;
pub use unexpected_directory::UnexpectedDirectory;
pub use visibility_mismatch::VisibilityMismatch;

12
src/pipeline/error/module_not_found.rs
View File

@@ -1,16 +1,16 @@
use crate::utils::{BoxedIter, iter::box_once};
use super::{ProjectError, ErrorPosition};
use super::{ErrorPosition, ProjectError};
use crate::utils::iter::box_once;
use crate::utils::BoxedIter;
/// Error produced when an import refers to a nonexistent module
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct ModuleNotFound {
pub file: Vec<String>,
pub subpath: Vec<String>
pub subpath: Vec<String>,
}
impl ProjectError for ModuleNotFound {
fn description(&self) -> &str {
"an import refers to a nonexistent module"
"an import refers to a nonexistent module"
}
fn message(&self) -> String {
format!(
@@ -22,4 +22,4 @@ impl ProjectError for ModuleNotFound {
fn positions(&self) -> BoxedIter<ErrorPosition> {
box_once(ErrorPosition::just_file(self.file.clone()))
}
}
}

40
src/pipeline/error/not_exported.rs
View File

@@ -1,8 +1,8 @@
use std::rc::Rc;
use crate::{utils::BoxedIter, representations::location::Location};
use super::{ProjectError, ErrorPosition};
use super::{ErrorPosition, ProjectError};
use crate::representations::location::Location;
use crate::utils::BoxedIter;
#[derive(Debug)]
pub struct NotExported {
@@ -16,21 +16,21 @@ impl ProjectError for NotExported {
"An import refers to a symbol that exists but isn't exported"
}
fn positions(&self) -> BoxedIter<ErrorPosition> {
Box::new([
ErrorPosition{
location: Location::File(Rc::new(self.file.clone())),
message: Some(format!(
"{} isn't exported",
self.subpath.join("::")
)),
},
ErrorPosition{
location: Location::File(Rc::new(self.referrer_file.clone())),
message: Some(format!(
"{} cannot see this symbol",
self.referrer_subpath.join("::")
)),
}
].into_iter())
Box::new(
[
ErrorPosition {
location: Location::File(Rc::new(self.file.clone())),
message: Some(format!("{} isn't exported", self.subpath.join("::"))),
},
ErrorPosition {
location: Location::File(Rc::new(self.referrer_file.clone())),
message: Some(format!(
"{} cannot see this symbol",
self.referrer_subpath.join("::")
)),
},
]
.into_iter(),
)
}
}
}

33
src/pipeline/error/parse_error_with_path.rs
View File

@@ -1,21 +1,21 @@
use std::rc::Rc;
use super::{ErrorPosition, ProjectError};
use crate::parse::ParseError;
use crate::representations::location::Location;
use crate::utils::BoxedIter;
use crate::parse::ParseError;
use super::ErrorPosition;
use super::ProjectError;
/// Produced by stages that parse text when it fails.
#[derive(Debug)]
pub struct ParseErrorWithPath {
pub full_source: String,
pub path: Vec<String>,
pub error: ParseError
pub error: ParseError,
}
impl ProjectError for ParseErrorWithPath {
fn description(&self) -> &str {"Failed to parse code"}
fn description(&self) -> &str {
"Failed to parse code"
}
fn positions(&self) -> BoxedIter<ErrorPosition> {
match &self.error {
ParseError::Lex(lex) => Box::new(lex.iter().map(|s| ErrorPosition {
@@ -23,15 +23,20 @@ impl ProjectError for ParseErrorWithPath {
file: Rc::new(self.path.clone()),
range: s.span(),
},
message: Some(s.to_string())
message: Some(s.to_string()),
})),
ParseError::Ast(ast) => Box::new(ast.iter().map(|(_i, s)| ErrorPosition {
location: s.found().map(|e| Location::Range {
file: Rc::new(self.path.clone()),
range: e.range.clone()
}).unwrap_or_else(|| Location::File(Rc::new(self.path.clone()))),
message: Some(s.label().unwrap_or("Parse error").to_string())
ParseError::Ast(ast) => Box::new(ast.iter().map(|(_i, s)| {
ErrorPosition {
location: s
.found()
.map(|e| Location::Range {
file: Rc::new(self.path.clone()),
range: e.range.clone(),
})
.unwrap_or_else(|| Location::File(Rc::new(self.path.clone()))),
message: Some(s.label().unwrap_or("Parse error").to_string()),
}
})),
}
}
}
}

17
src/pipeline/error/project_error.rs
View File

@@ -8,7 +8,7 @@ use crate::utils::BoxedIter;
/// processing got stuck, a command that is likely to be incorrect
pub struct ErrorPosition {
pub location: Location,
pub message: Option<String>
pub message: Option<String>,
}
impl ErrorPosition {
@@ -24,12 +24,17 @@ pub trait ProjectError: Debug {
/// A general description of this type of error
fn description(&self) -> &str;
/// A formatted message that includes specific parameters
fn message(&self) -> String {String::new()}
fn message(&self) -> String {
String::new()
}
/// Code positions relevant to this error
fn positions(&self) -> BoxedIter<ErrorPosition>;
/// Convert the error into an [Rc<dyn ProjectError>] to be able to
/// handle various errors together
fn rc(self) -> Rc<dyn ProjectError> where Self: Sized + 'static {
fn rc(self) -> Rc<dyn ProjectError>
where
Self: Sized + 'static,
{
Rc::new(self)
}
}
@@ -41,10 +46,12 @@ impl Display for dyn ProjectError {
let positions = self.positions();
write!(f, "Problem with the project: {description}; {message}")?;
for ErrorPosition { location, message } in positions {
write!(f, "@{location}: {}",
write!(
f,
"@{location}: {}",
message.unwrap_or("location of interest".to_string())
)?
}
Ok(())
}
}
}

17
src/pipeline/error/too_many_supers.rs
View File

@@ -1,8 +1,9 @@
use std::rc::Rc;
use crate::{utils::{BoxedIter, iter::box_once}, representations::location::Location};
use super::{ProjectError, ErrorPosition};
use super::{ErrorPosition, ProjectError};
use crate::representations::location::Location;
use crate::utils::iter::box_once;
use crate::utils::BoxedIter;
/// Error produced when an import path starts with more `super` segments
/// than the current module's absolute path
@@ -10,12 +11,12 @@ use super::{ProjectError, ErrorPosition};
pub struct TooManySupers {
pub path: Vec<String>,
pub offender_file: Vec<String>,
pub offender_mod: Vec<String>
pub offender_mod: Vec<String>,
}
impl ProjectError for TooManySupers {
fn description(&self) -> &str {
"an import path starts with more `super` segments than \
the current module's absolute path"
"an import path starts with more `super` segments than the current \
module's absolute path"
}
fn message(&self) -> String {
format!(
@@ -32,7 +33,7 @@ impl ProjectError for TooManySupers {
"path {} in {} contains too many `super` steps.",
self.path.join("::"),
self.offender_mod.join("::")
))
)),
})
}
}
}

15
src/pipeline/error/unexpected_directory.rs
View File

@@ -1,18 +1,17 @@
use crate::utils::{BoxedIter, iter::box_once};
use super::ErrorPosition;
use super::ProjectError;
use super::{ErrorPosition, ProjectError};
use crate::utils::iter::box_once;
use crate::utils::BoxedIter;
/// Produced when a stage that deals specifically with code encounters
/// a path that refers to a directory
#[derive(Debug)]
pub struct UnexpectedDirectory {
pub path: Vec<String>
pub path: Vec<String>,
}
impl ProjectError for UnexpectedDirectory {
fn description(&self) -> &str {
"A stage that deals specifically with code encountered a path \
that refers to a directory"
"A stage that deals specifically with code encountered a path that refers \
to a directory"
}
fn positions(&self) -> BoxedIter<ErrorPosition> {
box_once(ErrorPosition::just_file(self.path.clone()))
@@ -23,4 +22,4 @@ impl ProjectError for UnexpectedDirectory {
self.path.join("/")
)
}
}
}

13
src/pipeline/error/visibility_mismatch.rs
View File

@@ -1,13 +1,14 @@
use std::rc::Rc;
use crate::representations::location::Location;
use crate::utils::{BoxedIter, iter::box_once};
use super::project_error::{ProjectError, ErrorPosition};
use super::project_error::{ErrorPosition, ProjectError};
use crate::representations::location::Location;
use crate::utils::iter::box_once;
use crate::utils::BoxedIter;
#[derive(Debug)]
pub struct VisibilityMismatch{
pub struct VisibilityMismatch {
pub namespace: Vec<String>,
pub file: Rc<Vec<String>>
pub file: Rc<Vec<String>>,
}
impl ProjectError for VisibilityMismatch {
fn description(&self) -> &str {
@@ -19,7 +20,7 @@ impl ProjectError for VisibilityMismatch {
message: Some(format!(
"{} is opened multiple times with different visibilities",
self.namespace.join("::")
))
)),
})
}
}

96
src/pipeline/file_loader.rs
View File

@@ -1,25 +1,23 @@
use std::path::Path;
use std::path::{Path, PathBuf};
use std::rc::Rc;
use std::path::PathBuf;
use std::io;
use std::fs;
use std::{fs, io};
use crate::interner::{Interner, Sym};
use crate::pipeline::error::{
ErrorPosition, ProjectError, UnexpectedDirectory,
};
use crate::utils::iter::box_once;
use crate::utils::{Cache, BoxedIter};
use crate::interner::{Interner, Token};
use crate::pipeline::error::UnexpectedDirectory;
use crate::pipeline::error::{ProjectError, ErrorPosition};
use crate::utils::{BoxedIter, Cache};
#[derive(Debug)]
pub struct FileLoadingError{
pub struct FileLoadingError {
file: io::Error,
dir: io::Error,
path: Vec<String>
path: Vec<String>,
}
impl ProjectError for FileLoadingError {
fn description(&self) -> &str {
"Neither a file nor a directory could be read from \
the requested path"
"Neither a file nor a directory could be read from the requested path"
}
fn positions(&self) -> BoxedIter<ErrorPosition> {
box_once(ErrorPosition::just_file(self.path.clone()))
@@ -37,57 +35,55 @@ pub enum Loaded {
Collection(Rc<Vec<String>>),
}
impl Loaded {
pub fn is_code(&self) -> bool {matches!(self, Loaded::Code(_))}
pub fn is_code(&self) -> bool {
matches!(self, Loaded::Code(_))
}
}
pub type IOResult = Result<Loaded, Rc<dyn ProjectError>>;
pub type FileCache<'a> = Cache<'a, Token<Vec<Token<String>>>, IOResult>;
pub type FileCache<'a> = Cache<'a, Sym, IOResult>;
/// Load a file from a path expressed in Rust strings, but relative to
/// a root expressed as an OS Path.
pub fn load_file(root: &Path, path: &[impl AsRef<str>]) -> IOResult {
// let os_path = path.into_iter()
// .map_into::<OsString>()
// .collect::<Vec<_>>();
let full_path = path.iter().fold(
root.to_owned(),
|p, s| p.join(s.as_ref())
);
let full_path = path.iter().fold(root.to_owned(), |p, s| p.join(s.as_ref()));
let file_path = full_path.with_extension("orc");
let file_error = match fs::read_to_string(&file_path) {
let file_error = match fs::read_to_string(file_path) {
Ok(string) => return Ok(Loaded::Code(Rc::new(string))),
Err(err) => err
Err(err) => err,
};
let dir = match fs::read_dir(&full_path) {
Ok(dir) => dir,
Err(dir_error) => {
return Err(FileLoadingError {
file: file_error,
dir: dir_error,
path: path.iter()
.map(|s| s.as_ref().to_string())
.collect(),
}.rc())
}
Err(dir_error) =>
return Err(
FileLoadingError {
file: file_error,
dir: dir_error,
path: path.iter().map(|s| s.as_ref().to_string()).collect(),
}
.rc(),
),
};
let names = dir.filter_map(Result::ok)
let names = dir
.filter_map(Result::ok)
.filter_map(|ent| {
let fname = ent.file_name().into_string().ok()?;
let ftyp = ent.metadata().ok()?.file_type();
Some(if ftyp.is_dir() {fname} else {
Some(if ftyp.is_dir() {
fname
} else {
fname.strip_suffix(".or")?.to_string()
})
}).collect();
})
.collect();
Ok(Loaded::Collection(Rc::new(names)))
}
/// Generates a cached file loader for a directory
pub fn mk_cache(root: PathBuf, i: &Interner) -> FileCache {
Cache::new(move |token: Token<Vec<Token<String>>>, _this| -> IOResult {
let path = i.r(token).iter()
.map(|t| i.r(*t).as_str())
.collect::<Vec<_>>();
Cache::new(move |token: Sym, _this| -> IOResult {
let path = i.r(token).iter().map(|t| i.r(*t).as_str()).collect::<Vec<_>>();
load_file(&root, &path)
})
}
@@ -95,12 +91,18 @@ pub fn mk_cache(root: PathBuf, i: &Interner) -> FileCache {
/// Loads the string contents of a file at the given location.
/// If the path points to a directory, raises an error.
pub fn load_text(
path: Token<Vec<Token<String>>>,
load_file: &impl Fn(Token<Vec<Token<String>>>) -> IOResult,
i: &Interner
path: Sym,
load_file: &impl Fn(Sym) -> IOResult,
i: &Interner,
) -> Result<Rc<String>, Rc<dyn ProjectError>> {
if let Loaded::Code(s) = load_file(path)? {Ok(s)}
else {Err(UnexpectedDirectory{
path: i.r(path).iter().map(|t| i.r(*t)).cloned().collect()
}.rc())}
}
if let Loaded::Code(s) = load_file(path)? {
Ok(s)
} else {
Err(
UnexpectedDirectory {
path: i.r(path).iter().map(|t| i.r(*t)).cloned().collect(),
}
.rc(),
)
}
}

32
src/pipeline/import_abs_path.rs
View File

@@ -1,32 +1,32 @@
use std::rc::Rc;
use crate::representations::tree::Module;
use crate::representations::sourcefile::absolute_path;
use crate::utils::{Substack};
use crate::interner::{Token, Interner};
use super::error::{ProjectError, TooManySupers};
use crate::interner::{Interner, Tok};
use crate::representations::sourcefile::absolute_path;
use crate::utils::Substack;
pub fn import_abs_path(
src_path: &[Token<String>],
mod_stack: Substack<Token<String>>,
module: &Module<impl Clone, impl Clone>,
import_path: &[Token<String>],
src_path: &[Tok<String>],
mod_stack: Substack<Tok<String>>,
import_path: &[Tok<String>],
i: &Interner,
) -> Result<Vec<Token<String>>, Rc<dyn ProjectError>> {
) -> Result<Vec<Tok<String>>, Rc<dyn ProjectError>> {
// path of module within file
let mod_pathv = mod_stack.iter().rev_vec_clone();
// path of module within compilation
let abs_pathv = src_path.iter().copied()
let abs_pathv = src_path
.iter()
.copied()
.chain(mod_pathv.iter().copied())
.collect::<Vec<_>>();
// preload-target path relative to module
// preload-target path within compilation
absolute_path(&abs_pathv, import_path, i, &|n| {
module.items.contains_key(&n)
}).map_err(|_| TooManySupers{
absolute_path(&abs_pathv, import_path, i).map_err(|_| {
TooManySupers {
path: import_path.iter().map(|t| i.r(*t)).cloned().collect(),
offender_file: src_path.iter().map(|t| i.r(*t)).cloned().collect(),
offender_mod: mod_pathv.iter().map(|t| i.r(*t)).cloned().collect(),
}.rc())
}
}
.rc()
})
}

31
src/pipeline/import_resolution/alias_map.rs
View File

@@ -1,18 +1,20 @@
use hashbrown::{HashMap, HashSet};
use std::hash::Hash;
use crate::interner::Token;
use hashbrown::{HashMap, HashSet};
use crate::interner::Sym;
#[derive(Clone, Debug, Default)]
pub struct AliasMap{
pub targets: HashMap<Token<Vec<Token<String>>>, Token<Vec<Token<String>>>>,
pub aliases: HashMap<Token<Vec<Token<String>>>, HashSet<Token<Vec<Token<String>>>>>,
pub struct AliasMap {
pub targets: HashMap<Sym, Sym>,
pub aliases: HashMap<Sym, HashSet<Sym>>,
}
impl AliasMap {
pub fn new() -> Self {Self::default()}
pub fn new() -> Self {
Self::default()
}
pub fn link(&mut self, alias: Token<Vec<Token<String>>>, target: Token<Vec<Token<String>>>) {
pub fn link(&mut self, alias: Sym, target: Sym) {
let prev = self.targets.insert(alias, target);
debug_assert!(prev.is_none(), "Alias already has a target");
multimap_entry(&mut self.aliases, &target).insert(alias);
@@ -21,9 +23,7 @@ impl AliasMap {
for alt in alts {
// Assert that this step has always been done in the past
debug_assert!(
self.aliases.get(&alt)
.map(HashSet::is_empty)
.unwrap_or(true),
self.aliases.get(&alt).map(HashSet::is_empty).unwrap_or(true),
"Alias set of alias not empty"
);
debug_assert!(
@@ -35,7 +35,7 @@ impl AliasMap {
}
}
pub fn resolve(&self, alias: Token<Vec<Token<String>>>) -> Option<Token<Vec<Token<String>>>> {
pub fn resolve(&self, alias: Sym) -> Option<Sym> {
self.targets.get(&alias).copied()
}
}
@@ -44,10 +44,11 @@ impl AliasMap {
/// map-to-set (aka. multimap)
fn multimap_entry<'a, K: Eq + Hash + Clone, V>(
map: &'a mut HashMap<K, HashSet<V>>,
key: &'_ K
key: &'_ K,
) -> &'a mut HashSet<V> {
map.raw_entry_mut()
map
.raw_entry_mut()
.from_key(key)
.or_insert_with(|| (key.clone(), HashSet::new()))
.1
}
}

137
src/pipeline/import_resolution/apply_aliases.rs
View File

@@ -2,22 +2,28 @@ use std::rc::Rc;
use hashbrown::HashMap;
use crate::{utils::Substack, interner::{Token, Interner}, pipeline::{ProjectModule, ProjectExt}, representations::tree::{ModEntry, ModMember}, ast::{Rule, Expr}};
use super::{alias_map::AliasMap, decls::InjectedAsFn};
use super::alias_map::AliasMap;
use super::decls::InjectedAsFn;
use crate::ast::{Expr, Rule};
use crate::interner::{Interner, Sym, Tok};
use crate::pipeline::{ProjectExt, ProjectModule};
use crate::representations::tree::{ModEntry, ModMember};
use crate::utils::Substack;
fn resolve(
token: Token<Vec<Token<String>>>,
token: Sym,
alias_map: &AliasMap,
i: &Interner,
) -> Option<Vec<Token<String>>> {
) -> Option<Vec<Tok<String>>> {
if let Some(alias) = alias_map.resolve(token) {
Some(i.r(alias).clone())
} else if let Some((foot, body)) = i.r(token).split_last() {
let mut new_beginning = resolve(i.i(body), alias_map, i)?;
new_beginning.push(*foot);
Some(new_beginning)
} else {None}
} else {
None
}
}
fn process_expr(
@@ -26,74 +32,91 @@ fn process_expr(
injected_as: &impl InjectedAsFn,
i: &Interner,
) -> Expr {
expr.map_names(&|n| {
injected_as(&i.r(n)[..]).or_else(|| {
let next_v = resolve(n, alias_map, i)?;
// println!("Resolved alias {} to {}",
// i.extern_vec(n).join("::"),
// i.extern_all(&next_v).join("::")
// );
Some(
injected_as(&next_v)
.unwrap_or_else(|| i.i(&next_v))
)
expr
.map_names(&|n| {
injected_as(&i.r(n)[..]).or_else(|| {
let next_v = resolve(n, alias_map, i)?;
// println!("Resolved alias {} to {}",
// i.extern_vec(n).join("::"),
// i.extern_all(&next_v).join("::")
// );
Some(injected_as(&next_v).unwrap_or_else(|| i.i(&next_v)))
})
})
}).unwrap_or_else(|| expr.clone())
.unwrap_or_else(|| expr.clone())
}
// TODO: replace is_injected with injected_as
/// Replace all aliases with the name they're originally defined as
fn apply_aliases_rec(
path: Substack<Token<String>>,
path: Substack<Tok<String>>,
module: &ProjectModule,
alias_map: &AliasMap,
i: &Interner,
injected_as: &impl InjectedAsFn,
) -> ProjectModule {
let items = module.items.iter().map(|(name, ent)| {
let ModEntry{ exported, member } = ent;
let member = match member {
ModMember::Item(expr) => ModMember::Item(
process_expr(expr, alias_map, injected_as, i)
),
ModMember::Sub(module) => {
let subpath = path.push(*name);
let is_ignored = injected_as(&subpath.iter().rev_vec_clone()).is_some();
let new_mod = if is_ignored {module.clone()} else {
let module = module.as_ref();
Rc::new(apply_aliases_rec(
subpath, module,
alias_map, i, injected_as
))
};
ModMember::Sub(new_mod)
let items = module
.items
.iter()
.map(|(name, ent)| {
let ModEntry { exported, member } = ent;
let member = match member {
ModMember::Item(expr) =>
ModMember::Item(process_expr(expr, alias_map, injected_as, i)),
ModMember::Sub(module) => {
let subpath = path.push(*name);
let is_ignored =
injected_as(&subpath.iter().rev_vec_clone()).is_some();
let new_mod = if is_ignored {
module.clone()
} else {
let module = module.as_ref();
Rc::new(apply_aliases_rec(
subpath,
module,
alias_map,
i,
injected_as,
))
};
ModMember::Sub(new_mod)
},
};
(*name, ModEntry { exported: *exported, member })
})
.collect::<HashMap<_, _>>();
let rules = module
.extra
.rules
.iter()
.map(|rule| {
let Rule { source, prio, target } = rule;
Rule {
prio: *prio,
source: Rc::new(
source
.iter()
.map(|expr| process_expr(expr, alias_map, injected_as, i))
.collect::<Vec<_>>(),
),
target: Rc::new(
target
.iter()
.map(|expr| process_expr(expr, alias_map, injected_as, i))
.collect::<Vec<_>>(),
),
}
};
(*name, ModEntry{ exported: *exported, member })
}).collect::<HashMap<_, _>>();
let rules = module.extra.rules.iter().map(|rule| {
let Rule{ source, prio, target } = rule;
Rule{
prio: *prio,
source: Rc::new(source.iter()
.map(|expr| process_expr(expr, alias_map, injected_as, i))
.collect::<Vec<_>>()
),
target: Rc::new(target.iter()
.map(|expr| process_expr(expr, alias_map, injected_as, i))
.collect::<Vec<_>>()
),
}
}).collect::<Vec<_>>();
ProjectModule{
})
.collect::<Vec<_>>();
ProjectModule {
items,
imports: module.imports.clone(),
extra: ProjectExt{
extra: ProjectExt {
rules,
exports: module.extra.exports.clone(),
file: module.extra.file.clone(),
imports_from: module.extra.imports_from.clone(),
}
},
}
}
@@ -104,4 +127,4 @@ pub fn apply_aliases(
injected_as: &impl InjectedAsFn,
) -> ProjectModule {
apply_aliases_rec(Substack::Bottom, module, alias_map, i, injected_as)
}
}

117
src/pipeline/import_resolution/collect_aliases.rs
View File

@@ -1,62 +1,70 @@
use std::rc::Rc;
use crate::representations::tree::{WalkErrorKind, ModMember};
use crate::pipeline::error::{ProjectError, NotExported};
use crate::pipeline::project_tree::{ProjectTree, split_path, ProjectModule};
use crate::interner::{Token, Interner};
use crate::utils::{Substack, pushed};
use super::alias_map::AliasMap;
use super::decls::InjectedAsFn;
use crate::interner::{Interner, Tok};
use crate::pipeline::error::{NotExported, ProjectError};
use crate::pipeline::project_tree::{split_path, ProjectModule, ProjectTree};
use crate::representations::tree::{ModMember, WalkErrorKind};
use crate::utils::{pushed, Substack};
/// Assert that a module identified by a path can see a given symbol
fn assert_visible(
source: &[Token<String>], // must point to a file or submodule
target: &[Token<String>], // may point to a symbol or module of any kind
source: &[Tok<String>], // must point to a file or submodule
target: &[Tok<String>], // may point to a symbol or module of any kind
project: &ProjectTree,
i: &Interner
i: &Interner,
) -> Result<(), Rc<dyn ProjectError>> {
let (tgt_item, tgt_path) = if let Some(s) = target.split_last() {s}
else {return Ok(())};
let shared_len = source.iter()
.zip(tgt_path.iter())
.take_while(|(a, b)| a == b)
.count();
let shared_root = project.0.walk(&tgt_path[..shared_len], false)
.expect("checked in parsing");
let direct_parent = shared_root.walk(&tgt_path[shared_len..], true)
.map_err(|e| match e.kind {
WalkErrorKind::Missing => panic!("checked in parsing"),
WalkErrorKind::Private => {
let full_path = &tgt_path[..shared_len + e.pos];
let (file, sub) = split_path(full_path, &project);
let (ref_file, ref_sub) = split_path(source, &project);
NotExported{
file: i.extern_all(file),
subpath: i.extern_all(sub),
referrer_file: i.extern_all(ref_file),
referrer_subpath: i.extern_all(ref_sub),
}.rc()
let (tgt_item, tgt_path) = if let Some(s) = target.split_last() {
s
} else {
return Ok(());
};
let shared_len =
source.iter().zip(tgt_path.iter()).take_while(|(a, b)| a == b).count();
let shared_root =
project.0.walk(&tgt_path[..shared_len], false).expect("checked in parsing");
let direct_parent =
shared_root.walk(&tgt_path[shared_len..], true).map_err(|e| {
match e.kind {
WalkErrorKind::Missing => panic!("checked in parsing"),
WalkErrorKind::Private => {
let full_path = &tgt_path[..shared_len + e.pos];
let (file, sub) = split_path(full_path, project);
let (ref_file, ref_sub) = split_path(source, project);
NotExported {
file: i.extern_all(file),
subpath: i.extern_all(sub),
referrer_file: i.extern_all(ref_file),
referrer_subpath: i.extern_all(ref_sub),
}
.rc()
},
}
})?;
let tgt_item_exported = direct_parent.extra.exports.contains_key(tgt_item);
let target_prefixes_source = shared_len == tgt_path.len()
&& source.get(shared_len) == Some(tgt_item);
let target_prefixes_source =
shared_len == tgt_path.len() && source.get(shared_len) == Some(tgt_item);
if !tgt_item_exported && !target_prefixes_source {
let (file, sub) = split_path(target, &project);
let (ref_file, ref_sub) = split_path(source, &project);
Err(NotExported{
file: i.extern_all(file),
subpath: i.extern_all(sub),
referrer_file: i.extern_all(ref_file),
referrer_subpath: i.extern_all(ref_sub),
}.rc())
} else {Ok(())}
let (file, sub) = split_path(target, project);
let (ref_file, ref_sub) = split_path(source, project);
Err(
NotExported {
file: i.extern_all(file),
subpath: i.extern_all(sub),
referrer_file: i.extern_all(ref_file),
referrer_subpath: i.extern_all(ref_sub),
}
.rc(),
)
} else {
Ok(())
}
}
/// Populate target and alias maps from the module tree recursively
fn collect_aliases_rec(
path: Substack<Token<String>>,
path: Substack<Tok<String>>,
module: &ProjectModule,
project: &ProjectTree,
alias_map: &mut AliasMap,
@@ -65,7 +73,9 @@ fn collect_aliases_rec(
) -> Result<(), Rc<dyn ProjectError>> {
// Assume injected module has been alias-resolved
let mod_path_v = path.iter().rev_vec_clone();
if injected_as(&mod_path_v).is_some() {return Ok(())};
if injected_as(&mod_path_v).is_some() {
return Ok(());
};
for (&name, &target_mod) in module.extra.imports_from.iter() {
let target_mod_v = i.r(target_mod);
let target_sym_v = pushed(target_mod_v, name);
@@ -78,11 +88,16 @@ fn collect_aliases_rec(
for (&name, entry) in module.items.iter() {
let submodule = if let ModMember::Sub(s) = &entry.member {
s.as_ref()
} else {continue};
} else {
continue;
};
collect_aliases_rec(
path.push(name),
submodule, project, alias_map,
i, injected_as,
submodule,
project,
alias_map,
i,
injected_as,
)?
}
Ok(())
@@ -97,7 +112,11 @@ pub fn collect_aliases(
injected_as: &impl InjectedAsFn,
) -> Result<(), Rc<dyn ProjectError>> {
collect_aliases_rec(
Substack::Bottom, module, project, alias_map,
i, injected_as
Substack::Bottom,
module,
project,
alias_map,
i,
injected_as,
)
}
}

6
src/pipeline/import_resolution/decls.rs
View File

@@ -1,5 +1,3 @@
use crate::interner::Token;
use crate::interner::{Sym, Tok};
pub trait InjectedAsFn = Fn(
&[Token<String>]
) -> Option<Token<Vec<Token<String>>>>;
pub trait InjectedAsFn = Fn(&[Tok<String>]) -> Option<Sym>;

4
src/pipeline/import_resolution/mod.rs
View File

@@ -1,7 +1,7 @@
mod alias_map;
mod collect_aliases;
mod apply_aliases;
mod resolve_imports;
mod collect_aliases;
mod decls;
mod resolve_imports;
pub use resolve_imports::resolve_imports;

28
src/pipeline/import_resolution/resolve_imports.rs
View File

@@ -2,16 +2,14 @@ use std::rc::Rc;
use itertools::Itertools;
use super::alias_map::AliasMap;
use super::apply_aliases::apply_aliases;
use super::collect_aliases::collect_aliases;
use super::decls::InjectedAsFn;
use crate::interner::Interner;
use crate::pipeline::error::ProjectError;
use crate::pipeline::project_tree::ProjectTree;
use super::alias_map::AliasMap;
use super::collect_aliases::collect_aliases;
use super::apply_aliases::apply_aliases;
use super::decls::InjectedAsFn;
/// Follow import chains to locate the original name of all tokens, then
/// replace these aliases with the original names throughout the tree
pub fn resolve_imports(
@@ -20,14 +18,14 @@ pub fn resolve_imports(
injected_as: &impl InjectedAsFn,
) -> Result<ProjectTree, Rc<dyn ProjectError>> {
let mut map = AliasMap::new();
collect_aliases(
project.0.as_ref(),
&project, &mut map,
i, injected_as
)?;
println!("Aliases: {{{:?}}}",
map.targets.iter()
.map(|(kt, vt)| format!("{} => {}",
collect_aliases(project.0.as_ref(), &project, &mut map, i, injected_as)?;
println!(
"Aliases: {{{:?}}}",
map
.targets
.iter()
.map(|(kt, vt)| format!(
"{} => {}",
i.extern_vec(*kt).join("::"),
i.extern_vec(*vt).join("::")
))
@@ -35,4 +33,4 @@ pub fn resolve_imports(
);
let new_mod = apply_aliases(project.0.as_ref(), &map, i, injected_as);
Ok(ProjectTree(Rc::new(new_mod)))
}
}

17
src/pipeline/mod.rs
View File

@@ -1,19 +1,14 @@
pub mod error;
pub mod file_loader;
mod import_abs_path;
mod import_resolution;
mod parse_layer;
mod project_tree;
mod source_loader;
mod import_abs_path;
mod split_name;
mod import_resolution;
pub mod file_loader;
mod parse_layer;
pub use parse_layer::parse_layer;
pub use project_tree::{
ConstTree, ProjectExt, ProjectModule, ProjectTree, from_const_tree,
collect_consts, collect_rules,
collect_consts, collect_rules, from_const_tree, ConstTree, ProjectExt,
ProjectModule, ProjectTree,
};
// pub use file_loader::{Loaded, FileLoadingError, IOResult};
// pub use error::{
// ErrorPosition, ModuleNotFound, NotExported, ParseErrorWithPath,
// ProjectError, TooManySupers, UnexpectedDirectory
// };

44
src/pipeline/parse_layer.rs
View File

@@ -1,52 +1,46 @@
use std::rc::Rc;
use crate::representations::sourcefile::FileEntry;
use crate::interner::{Token, Interner};
use super::{project_tree, import_resolution};
use super::source_loader;
use super::file_loader::IOResult;
use super::error::ProjectError;
use super::ProjectTree;
use super::file_loader::IOResult;
use super::{import_resolution, project_tree, source_loader, ProjectTree};
use crate::interner::{Interner, Sym, Tok};
use crate::representations::sourcefile::FileEntry;
/// Using an IO callback, produce a project tree that includes the given
/// target symbols or files if they're defined.
///
///
/// The environment accessible to the loaded source can be specified with
/// a pre-existing tree which will be merged with the loaded data, and a
/// prelude which will be prepended to each individual file. Since the
/// prelude gets compiled with each file, normally it should be a glob
/// import pointing to a module in the environment.
pub fn parse_layer<'a>(
targets: &[Token<Vec<Token<String>>>],
loader: &impl Fn(Token<Vec<Token<String>>>) -> IOResult,
environment: &'a ProjectTree,
pub fn parse_layer(
targets: &[Sym],
loader: &impl Fn(Sym) -> IOResult,
environment: &ProjectTree,
prelude: &[FileEntry],
i: &Interner,
) -> Result<ProjectTree, Rc<dyn ProjectError>> {
// A path is injected if it is walkable in the injected tree
let injected_as = |path: &[Token<String>]| {
let injected_as = |path: &[Tok<String>]| {
let (item, modpath) = path.split_last()?;
let module = environment.0.walk(modpath, false).ok()?;
let inj = module.extra.exports.get(item).copied()?;
Some(inj)
};
let injected_names = |path: Token<Vec<Token<String>>>| {
let pathv = &i.r(path)[..];
let module = environment.0.walk(&pathv, false).ok()?;
Some(Rc::new(
module.extra.exports.keys().copied().collect()
))
let injected_names = |path: Tok<Vec<Tok<String>>>| {
let module = environment.0.walk(&i.r(path)[..], false).ok()?;
Some(Rc::new(module.extra.exports.keys().copied().collect()))
};
let source = source_loader::load_source(
targets, prelude, i, loader, &|path| injected_as(path).is_some()
)?;
let source =
source_loader::load_source(targets, prelude, i, loader, &|path| {
injected_as(path).is_some()
})?;
let tree = project_tree::build_tree(source, i, prelude, &injected_names)?;
let sum = ProjectTree(Rc::new(
environment.0.as_ref().clone()
+ tree.0.as_ref().clone()
environment.0.as_ref().clone() + tree.0.as_ref().clone(),
));
let resolvd = import_resolution::resolve_imports(sum, i, &injected_as)?;
// Addition among modules favours the left hand side.
Ok(resolvd)
}
}

46
src/pipeline/project_tree/add_prelude.rs
View File

@@ -1,23 +1,19 @@
use crate::representations::sourcefile::{Member, FileEntry};
use crate::interner::Token;
use crate::interner::Tok;
use crate::representations::sourcefile::{FileEntry, Member, Namespace};
fn member_rec(
// object
member: Member,
// context
path: &[Token<String>],
path: &[Tok<String>],
prelude: &[FileEntry],
) -> Member {
match member {
Member::Namespace(name, body) => {
let new_body = entv_rec(
body,
path,
prelude
);
Member::Namespace(name, new_body)
Member::Namespace(Namespace { name, body }) => {
let new_body = entv_rec(body, path, prelude);
Member::Namespace(Namespace { name, body: new_body })
},
any => any
any => any,
}
}
@@ -25,28 +21,26 @@ fn entv_rec(
// object
data: Vec<FileEntry>,
// context
mod_path: &[Token<String>],
mod_path: &[Tok<String>],
prelude: &[FileEntry],
) -> Vec<FileEntry> {
prelude.iter().cloned()
.chain(data.into_iter()
.map(|ent| match ent {
FileEntry::Exported(mem) => FileEntry::Exported(member_rec(
mem, mod_path, prelude
)),
FileEntry::Internal(mem) => FileEntry::Internal(member_rec(
mem, mod_path, prelude
)),
any => any
})
)
prelude
.iter()
.cloned()
.chain(data.into_iter().map(|ent| match ent {
FileEntry::Exported(mem) =>
FileEntry::Exported(member_rec(mem, mod_path, prelude)),
FileEntry::Internal(mem) =>
FileEntry::Internal(member_rec(mem, mod_path, prelude)),
any => any,
}))
.collect()
}
pub fn add_prelude(
data: Vec<FileEntry>,
path: &[Token<String>],
path: &[Tok<String>],
prelude: &[FileEntry],
) -> Vec<FileEntry> {
entv_rec(data, path, prelude)
}
}

205
src/pipeline/project_tree/build_tree.rs
View File

@@ -2,38 +2,41 @@ use std::rc::Rc;
use hashbrown::HashMap;
use crate::pipeline::error::ProjectError;
use crate::interner::{Token, Interner};
use crate::utils::iter::{box_once, box_empty};
use crate::utils::{Substack, pushed};
use crate::ast::{Expr, Constant};
use crate::pipeline::source_loader::{LoadedSourceTable, LoadedSource};
use crate::representations::tree::{Module, ModMember, ModEntry};
use crate::representations::sourcefile::{FileEntry, Member, absolute_path};
use super::collect_ops::InjectedOperatorsFn;
use super::{collect_ops, ProjectTree, ProjectExt};
use super::parse_file::parse_file;
use super::{collect_ops, ProjectExt, ProjectTree};
use crate::ast::{Constant, Expr};
use crate::interner::{Interner, Tok};
use crate::pipeline::error::ProjectError;
use crate::pipeline::source_loader::{LoadedSource, LoadedSourceTable};
use crate::representations::sourcefile::{absolute_path, FileEntry, Member};
use crate::representations::tree::{ModEntry, ModMember, Module};
use crate::utils::iter::{box_empty, box_once};
use crate::utils::{pushed, Substack};
#[derive(Debug)]
struct ParsedSource<'a> {
path: Vec<Token<String>>,
path: Vec<Tok<String>>,
loaded: &'a LoadedSource,
parsed: Vec<FileEntry>
parsed: Vec<FileEntry>,
}
pub fn split_path<'a>(path: &'a [Token<String>], proj: &'a ProjectTree)
-> (&'a [Token<String>], &'a [Token<String>])
{
let (end, body) = if let Some(s) = path.split_last() {s}
else {return (&[], &[])};
let mut module = proj.0.walk(body, false).expect("invalid path cannot be split");
pub fn split_path<'a>(
path: &'a [Tok<String>],
proj: &'a ProjectTree,
) -> (&'a [Tok<String>], &'a [Tok<String>]) {
let (end, body) = if let Some(s) = path.split_last() {
s
} else {
return (&[], &[]);
};
let mut module =
proj.0.walk(body, false).expect("invalid path cannot be split");
if let ModMember::Sub(m) = &module.items[end].member {
module = m.clone();
}
let file = module.extra.file.as_ref()
.map(|s| &path[..s.len()])
.unwrap_or(&path[..]);
let file =
module.extra.file.as_ref().map(|s| &path[..s.len()]).unwrap_or(path);
let subpath = &path[file.len()..];
(file, subpath)
}
@@ -41,7 +44,7 @@ pub fn split_path<'a>(path: &'a [Token<String>], proj: &'a ProjectTree)
/// Convert normalized, prefixed source into a module
fn source_to_module(
// level
path: Substack<Token<String>>,
path: Substack<Tok<String>>,
preparsed: &Module<impl Clone, impl Clone>,
// data
data: Vec<FileEntry>,
@@ -50,35 +53,38 @@ fn source_to_module(
filepath_len: usize,
) -> Rc<Module<Expr, ProjectExt>> {
let path_v = path.iter().rev_vec_clone();
let imports = data.iter()
.filter_map(|ent| if let FileEntry::Import(impv) = ent {
Some(impv.iter())
} else {None})
let imports = data
.iter()
.filter_map(|ent| {
if let FileEntry::Import(impv) = ent {
Some(impv.iter())
} else {
None
}
})
.flatten()
.cloned()
.collect::<Vec<_>>();
let imports_from = imports.iter()
let imports_from = imports
.iter()
.map(|imp| {
let mut imp_path_v = i.r(imp.path).clone();
imp_path_v.push(imp.name.expect("imports normalized"));
let mut abs_path = absolute_path(
&path_v,
&imp_path_v,
i, &|n| preparsed.items.contains_key(&n)
).expect("tested in preparsing");
let mut abs_path =
absolute_path(&path_v, &imp_path_v, i).expect("tested in preparsing");
let name = abs_path.pop().expect("importing the global context");
(name, i.i(&abs_path))
})
.collect::<HashMap<_, _>>();
let exports = data.iter()
let exports = data
.iter()
.flat_map(|ent| {
let mk_ent = |name| (name, i.i(&pushed(&path_v, name)));
match ent {
FileEntry::Export(names)
=> Box::new(names.iter().copied().map(mk_ent)),
FileEntry::Export(names) => Box::new(names.iter().copied().map(mk_ent)),
FileEntry::Exported(mem) => match mem {
Member::Constant(constant) => box_once(mk_ent(constant.name)),
Member::Namespace(name, _) => box_once(mk_ent(*name)),
Member::Namespace(ns) => box_once(mk_ent(ns.name)),
Member::Rule(rule) => {
let mut names = Vec::new();
for e in rule.source.iter() {
@@ -89,13 +95,14 @@ fn source_to_module(
})
}
Box::new(names.into_iter())
}
}
_ => box_empty()
},
},
_ => box_empty(),
}
})
.collect::<HashMap<_, _>>();
let rules = data.iter()
let rules = data
.iter()
.filter_map(|ent| match ent {
FileEntry::Exported(Member::Rule(rule)) => Some(rule),
FileEntry::Internal(Member::Rule(rule)) => Some(rule),
@@ -103,38 +110,51 @@ fn source_to_module(
})
.cloned()
.collect::<Vec<_>>();
let items = data.into_iter()
let items = data
.into_iter()
.filter_map(|ent| match ent {
FileEntry::Exported(Member::Namespace(name, body)) => {
let prep_member = &preparsed.items[&name].member;
let new_prep = if let ModMember::Sub(s) = prep_member {s.as_ref()}
else { panic!("preparsed missing a submodule") };
FileEntry::Exported(Member::Namespace(ns)) => {
let prep_member = &preparsed.items[&ns.name].member;
let new_prep = if let ModMember::Sub(s) = prep_member {
s.as_ref()
} else {
panic!("preparsed missing a submodule")
};
let module = source_to_module(
path.push(name),
new_prep, body, i, filepath_len
path.push(ns.name),
new_prep,
ns.body,
i,
filepath_len,
);
let member = ModMember::Sub(module);
Some((name, ModEntry{ exported: true, member }))
}
FileEntry::Internal(Member::Namespace(name, body)) => {
let prep_member = &preparsed.items[&name].member;
let new_prep = if let ModMember::Sub(s) = prep_member {s.as_ref()}
else { panic!("preparsed missing a submodule") };
Some((ns.name, ModEntry { exported: true, member }))
},
FileEntry::Internal(Member::Namespace(ns)) => {
let prep_member = &preparsed.items[&ns.name].member;
let new_prep = if let ModMember::Sub(s) = prep_member {
s.as_ref()
} else {
panic!("preparsed missing a submodule")
};
let module = source_to_module(
path.push(name),
new_prep, body, i, filepath_len
path.push(ns.name),
new_prep,
ns.body,
i,
filepath_len,
);
let member = ModMember::Sub(module);
Some((name, ModEntry{ exported: false, member }))
}
FileEntry::Exported(Member::Constant(Constant{ name, value })) => {
Some((ns.name, ModEntry { exported: false, member }))
},
FileEntry::Exported(Member::Constant(Constant { name, value })) => {
let member = ModMember::Item(value);
Some((name, ModEntry{ exported: true, member }))
}
FileEntry::Internal(Member::Constant(Constant{ name, value })) => {
Some((name, ModEntry { exported: true, member }))
},
FileEntry::Internal(Member::Constant(Constant { name, value })) => {
let member = ModMember::Item(value);
Some((name, ModEntry{ exported: false, member }))
}
Some((name, ModEntry { exported: false, member }))
},
_ => None,
})
.collect::<HashMap<_, _>>();
@@ -150,15 +170,15 @@ fn source_to_module(
imports_from,
exports,
rules,
file: Some(path_v[..filepath_len].to_vec())
}
file: Some(path_v[..filepath_len].to_vec()),
},
})
}
fn files_to_module(
path: Substack<Token<String>>,
path: Substack<Tok<String>>,
files: &[ParsedSource],
i: &Interner
i: &Interner,
) -> Rc<Module<Expr, ProjectExt>> {
let lvl = path.len();
let path_v = path.iter().rev_vec_clone();
@@ -167,19 +187,22 @@ fn files_to_module(
path,
files[0].loaded.preparsed.0.as_ref(),
files[0].parsed.clone(),
i, path.len()
)
i,
path.len(),
);
}
let items = files.group_by(|a, b| a.path[lvl] == b.path[lvl]).into_iter()
let items = files
.group_by(|a, b| a.path[lvl] == b.path[lvl])
.map(|files| {
let namespace = files[0].path[lvl];
let subpath = path.push(namespace);
let module = files_to_module(subpath, files, i);
let member = ModMember::Sub(module);
(namespace, ModEntry{ exported: true, member })
(namespace, ModEntry { exported: true, member })
})
.collect::<HashMap<_, _>>();
let exports: HashMap<_, _> = items.keys()
let exports: HashMap<_, _> = items
.keys()
.copied()
.map(|name| (name, i.i(&pushed(&path_v, name))))
.collect();
@@ -188,38 +211,44 @@ fn files_to_module(
// i.extern_all(&path_v[..]).join("::"),
// exports.keys().map(|t| i.r(*t)).join(", ")
// );
Rc::new(Module{
Rc::new(Module {
items,
imports: vec![],
extra: ProjectExt {
exports,
imports_from: HashMap::new(),
rules: vec![], file: None,
}
rules: vec![],
file: None,
},
})
}
pub fn build_tree<'a>(
pub fn build_tree(
files: LoadedSourceTable,
i: &Interner,
prelude: &[FileEntry],
injected: &impl InjectedOperatorsFn,
) -> Result<ProjectTree, Rc<dyn ProjectError>> {
let ops_cache = collect_ops::mk_cache(&files, i, injected);
let mut entries = files.iter()
.map(|(path, loaded)| Ok((
i.r(*path),
loaded,
parse_file(*path, &files, &ops_cache, i, prelude)?
)))
let mut entries = files
.iter()
.map(|(path, loaded)| {
Ok((
i.r(*path),
loaded,
parse_file(*path, &files, &ops_cache, i, prelude)?,
))
})
.collect::<Result<Vec<_>, Rc<dyn ProjectError>>>()?;
// sort by similarity, then longest-first
entries.sort_unstable_by(|a, b| a.0.cmp(&b.0).reverse());
let files = entries.into_iter()
.map(|(path, loaded, parsed)| ParsedSource{
loaded, parsed,
path: path.clone()
entries.sort_unstable_by(|a, b| a.0.cmp(b.0).reverse());
let files = entries
.into_iter()
.map(|(path, loaded, parsed)| ParsedSource {
loaded,
parsed,
path: path.clone(),
})
.collect::<Vec<_>>();
Ok(ProjectTree(files_to_module(Substack::Bottom, &files, i)))
}
}

85
src/pipeline/project_tree/collect_ops/exported_ops.rs
View File

@@ -4,73 +4,80 @@ use std::rc::Rc;
use hashbrown::HashSet;
use itertools::Itertools;
use crate::representations::tree::WalkErrorKind;
use crate::interner::{Interner, Sym, Tok};
use crate::pipeline::error::{ModuleNotFound, ProjectError};
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::pipeline::error::{ProjectError, ModuleNotFound};
use crate::interner::{Token, Interner};
use crate::utils::Cache;
use crate::pipeline::split_name::split_name;
use crate::representations::tree::WalkErrorKind;
use crate::utils::Cache;
pub type OpsResult = Result<Rc<HashSet<Token<String>>>, Rc<dyn ProjectError>>;
pub type ExportedOpsCache<'a> = Cache<'a, Token<Vec<Token<String>>>, OpsResult>;
pub type OpsResult = Result<Rc<HashSet<Tok<String>>>, Rc<dyn ProjectError>>;
pub type ExportedOpsCache<'a> = Cache<'a, Sym, OpsResult>;
pub trait InjectedOperatorsFn = Fn(
Token<Vec<Token<String>>>
) -> Option<Rc<HashSet<Token<String>>>>;
pub trait InjectedOperatorsFn = Fn(Sym) -> Option<Rc<HashSet<Tok<String>>>>;
fn coprefix<T: Eq>(
l: impl Iterator<Item = T>,
r: impl Iterator<Item = T>
r: impl Iterator<Item = T>,
) -> usize {
l.zip(r).take_while(|(a, b)| a == b).count()
}
/// Collect all names exported by the module at the specified path
pub fn collect_exported_ops(
path: Token<Vec<Token<String>>>,
path: Sym,
loaded: &LoadedSourceTable,
i: &Interner,
injected: &impl InjectedOperatorsFn
injected: &impl InjectedOperatorsFn,
) -> OpsResult {
if let Some(ops) = injected(path) {
if path == i.i(&[i.i("prelude")][..]) {
println!("%%% Prelude exported ops %%%");
println!("{}", ops.iter().map(|t| i.r(*t)).join(", "));
}
return Ok(ops)
return Ok(ops);
}
let is_file = |n: &[Token<String>]| loaded.contains_key(&i.i(n));
let is_file = |n: &[Tok<String>]| loaded.contains_key(&i.i(n));
let path_s = &i.r(path)[..];
let name_split = split_name(path_s, &is_file);
let (fpath_v, subpath_v) = if let Some(f) = name_split {f} else {
return Ok(Rc::new(loaded.keys().copied()
.filter_map(|modname| {
let modname_s = i.r(modname);
if path_s.len() == coprefix(path_s.iter(), modname_s.iter()) {
Some(modname_s[path_s.len()])
} else {None}
})
.collect::<HashSet<_>>()
))
let (fpath_v, subpath_v) = if let Some(f) = name_split {
f
} else {
return Ok(Rc::new(
loaded
.keys()
.copied()
.filter_map(|modname| {
let modname_s = i.r(modname);
if path_s.len() == coprefix(path_s.iter(), modname_s.iter()) {
Some(modname_s[path_s.len()])
} else {
None
}
})
.collect::<HashSet<_>>(),
));
};
let fpath = i.i(fpath_v);
let preparsed = &loaded[&fpath].preparsed;
let module = preparsed.0.walk(&subpath_v, false)
.map_err(|walk_err| match walk_err.kind {
WalkErrorKind::Private => unreachable!("visibility is not being checked here"),
WalkErrorKind::Missing => ModuleNotFound{
let module = preparsed.0.walk(subpath_v, false).map_err(|walk_err| {
match walk_err.kind {
WalkErrorKind::Private =>
unreachable!("visibility is not being checked here"),
WalkErrorKind::Missing => ModuleNotFound {
file: i.extern_vec(fpath),
subpath: subpath_v.into_iter()
subpath: subpath_v
.iter()
.take(walk_err.pos)
.map(|t| i.r(*t))
.cloned()
.collect()
}.rc(),
})?;
let out: HashSet<_> = module.items.iter()
.filter(|(_, v)| v.exported)
.map(|(k, _)| *k)
.collect();
.collect(),
}
.rc(),
}
})?;
let out: HashSet<_> =
module.items.iter().filter(|(_, v)| v.exported).map(|(k, _)| *k).collect();
if path == i.i(&[i.i("prelude")][..]) {
println!("%%% Prelude exported ops %%%");
println!("{}", out.iter().map(|t| i.r(*t)).join(", "));
@@ -83,7 +90,5 @@ pub fn mk_cache<'a>(
i: &'a Interner,
injected: &'a impl InjectedOperatorsFn,
) -> ExportedOpsCache<'a> {
Cache::new(|path, _this| {
collect_exported_ops(path, loaded, i, injected)
})
}
Cache::new(|path, _this| collect_exported_ops(path, loaded, i, injected))
}

6
src/pipeline/project_tree/collect_ops/mod.rs
View File

@@ -2,7 +2,7 @@ mod exported_ops;
mod ops_for;
pub use exported_ops::{
ExportedOpsCache, OpsResult, InjectedOperatorsFn,
collect_exported_ops, mk_cache
collect_exported_ops, mk_cache, ExportedOpsCache, InjectedOperatorsFn,
OpsResult,
};
pub use ops_for::collect_ops_for;
pub use ops_for::collect_ops_for;

28
src/pipeline/project_tree/collect_ops/ops_for.rs
View File

@@ -3,20 +3,19 @@ use std::rc::Rc;
use hashbrown::HashSet;
use itertools::Itertools;
use super::exported_ops::{ExportedOpsCache, OpsResult};
use crate::interner::{Interner, Tok};
use crate::parse::is_op;
use crate::pipeline::error::ProjectError;
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::interner::{Token, Interner};
use crate::representations::tree::{Module, ModMember};
use crate::pipeline::import_abs_path::import_abs_path;
use super::exported_ops::{ExportedOpsCache, OpsResult};
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::representations::tree::{ModMember, Module};
/// Collect all operators and names, exported or local, defined in this
/// tree.
fn tree_all_ops(
module: &Module<impl Clone, impl Clone>,
ops: &mut HashSet<Token<String>>
ops: &mut HashSet<Tok<String>>,
) {
ops.extend(module.items.keys().copied());
for ent in module.items.values() {
@@ -28,21 +27,22 @@ fn tree_all_ops(
/// Collect all names imported in this file
pub fn collect_ops_for(
file: &[Token<String>],
file: &[Tok<String>],
loaded: &LoadedSourceTable,
ops_cache: &ExportedOpsCache,
i: &Interner
i: &Interner,
) -> OpsResult {
let tree = &loaded[&i.i(file)].preparsed.0;
let mut ret = HashSet::new();
println!("collecting ops for {}", i.extern_all(file).join("::"));
tree_all_ops(tree.as_ref(), &mut ret);
tree.visit_all_imports(&mut |modpath, module, import| {
if let Some(n) = import.name { ret.insert(n); } else {
tree.visit_all_imports(&mut |modpath, _module, import| {
if let Some(n) = import.name {
ret.insert(n);
} else {
println!("\tglob import from {}", i.extern_vec(import.path).join("::"));
let path = import_abs_path(
&file, modpath, module, &i.r(import.path)[..], i
).expect("This error should have been caught during loading");
let path = import_abs_path(file, modpath, &i.r(import.path)[..], i)
.expect("This error should have been caught during loading");
ret.extend(ops_cache.find(&i.i(&path))?.iter().copied());
}
Ok::<_, Rc<dyn ProjectError>>(())
@@ -53,4 +53,4 @@ pub fn collect_ops_for(
println!("{}", ret.iter().map(|t| i.r(*t)).join(", "))
}
Ok(Rc::new(ret))
}
}

58
src/pipeline/project_tree/const_tree.rs
View File

@@ -1,26 +1,26 @@
use std::{ops::Add, rc::Rc};
use std::ops::Add;
use std::rc::Rc;
use hashbrown::HashMap;
use super::{ProjectExt, ProjectModule, ProjectTree};
use crate::ast::{Clause, Expr};
use crate::foreign::{Atom, Atomic, ExternFn};
use crate::interner::{Interner, Tok};
use crate::representations::location::Location;
use crate::representations::tree::{ModEntry, ModMember, Module};
use crate::representations::Primitive;
use crate::representations::location::Location;
use crate::foreign::{ExternFn, Atomic, Atom};
use crate::interner::{Token, Interner};
use crate::ast::{Expr, Clause};
use crate::utils::{Substack, pushed};
use super::{ProjectModule, ProjectExt, ProjectTree};
use crate::utils::{pushed, Substack};
pub enum ConstTree {
Const(Expr),
Tree(HashMap<Token<String>, ConstTree>)
Tree(HashMap<Tok<String>, ConstTree>),
}
impl ConstTree {
pub fn primitive(primitive: Primitive) -> Self {
Self::Const(Expr{
Self::Const(Expr {
location: Location::Unknown,
value: Clause::P(primitive)
value: Clause::P(primitive),
})
}
pub fn xfn(xfn: impl ExternFn + 'static) -> Self {
@@ -29,9 +29,7 @@ impl ConstTree {
pub fn atom(atom: impl Atomic + 'static) -> Self {
Self::primitive(Primitive::Atom(Atom(Box::new(atom))))
}
pub fn tree(
arr: impl IntoIterator<Item = (Token<String>, Self)>
) -> Self {
pub fn tree(arr: impl IntoIterator<Item = (Tok<String>, Self)>) -> Self {
Self::Tree(arr.into_iter().collect())
}
}
@@ -57,27 +55,29 @@ impl Add for ConstTree {
}
fn from_const_tree_rec(
path: Substack<Token<String>>,
consts: HashMap<Token<String>, ConstTree>,
file: &[Token<String>],
path: Substack<Tok<String>>,
consts: HashMap<Tok<String>, ConstTree>,
file: &[Tok<String>],
i: &Interner,
) -> ProjectModule {
let mut items = HashMap::new();
let path_v = path.iter().rev_vec_clone();
for (name, item) in consts {
items.insert(name, ModEntry{
items.insert(name, ModEntry {
exported: true,
member: match item {
ConstTree::Const(c) => ModMember::Item(c),
ConstTree::Tree(t) => ModMember::Sub(Rc::new(
from_const_tree_rec(path.push(name), t, file, i)
)),
}
ConstTree::Tree(t) => ModMember::Sub(Rc::new(from_const_tree_rec(
path.push(name),
t,
file,
i,
))),
},
});
}
let exports = items.keys()
.map(|name| (*name, i.i(&pushed(&path_v, *name))))
.collect();
let exports =
items.keys().map(|name| (*name, i.i(&pushed(&path_v, *name)))).collect();
Module {
items,
imports: vec![],
@@ -85,15 +85,15 @@ fn from_const_tree_rec(
exports,
file: Some(file.to_vec()),
..Default::default()
}
},
}
}
pub fn from_const_tree(
consts: HashMap<Token<String>, ConstTree>,
file: &[Token<String>],
consts: HashMap<Tok<String>, ConstTree>,
file: &[Tok<String>],
i: &Interner,
) -> ProjectTree {
let module = from_const_tree_rec(Substack::Bottom, consts, file, i);
ProjectTree(Rc::new(module))
}
}

50
src/pipeline/project_tree/mod.rs
View File

@@ -1,38 +1,30 @@
/* FILE SEPARATION BOUNDARY
// FILE SEPARATION BOUNDARY
//
// Collect all operators accessible in each file, parse the files with
// correct tokenization, resolve glob imports, convert expressions to
// refer to tokens with (local) absolute path, and connect them into a
// single tree.
//
// The module checks for imports from missing modules (including
// submodules). All other errors must be checked later.
//
// Injection strategy:
// Return all items of the given module in the injected tree for
// `injected` The output of this stage is a tree, which can simply be
// overlaid with the injected tree
Collect all operators accessible in each file, parse the files with
correct tokenization, resolve glob imports, convert expressions to
refer to tokens with (local) absolute path, and connect them into a
single tree.
The module checks for imports from missing modules (including submodules).
All other errors must be checked later.
Injection strategy:
Return all items of the given module in the injected tree for `injected`
The output of this stage is a tree, which can simply be overlaid with
the injected tree
*/
mod collect_ops;
mod parse_file;
mod add_prelude;
mod build_tree;
mod collect_ops;
mod const_tree;
mod normalize_imports;
mod parse_file;
mod prefix;
mod tree;
mod const_tree;
mod add_prelude;
pub use build_tree::{build_tree, split_path};
pub use collect_ops::InjectedOperatorsFn;
pub use const_tree::{
ConstTree, from_const_tree,
};
pub use const_tree::{from_const_tree, ConstTree};
pub use tree::{
ProjectExt, ProjectModule, ProjectTree, collect_consts, collect_rules
collect_consts, collect_rules, ProjectExt, ProjectModule, ProjectTree,
};
pub use build_tree::{
build_tree, split_path
};

92
src/pipeline/project_tree/normalize_imports.rs
View File

@@ -1,74 +1,88 @@
use crate::representations::tree::{Module, ModMember};
use crate::representations::sourcefile::{Member, FileEntry, Import};
use crate::utils::BoxedIter;
use crate::utils::{Substack, iter::box_once};
use crate::interner::{Interner, Token};
use crate::pipeline::import_abs_path::import_abs_path;
use super::collect_ops::ExportedOpsCache;
use crate::interner::{Interner, Tok};
use crate::pipeline::import_abs_path::import_abs_path;
use crate::representations::sourcefile::{
FileEntry, Import, Member, Namespace,
};
use crate::representations::tree::{ModMember, Module};
use crate::utils::iter::box_once;
use crate::utils::{BoxedIter, Substack};
fn member_rec(
// level
mod_stack: Substack<Token<String>>,
mod_stack: Substack<Tok<String>>,
preparsed: &Module<impl Clone, impl Clone>,
// object
member: Member,
// context
path: &[Token<String>],
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner
i: &Interner,
) -> Member {
match member {
Member::Namespace(name, body) => {
Member::Namespace(Namespace { name, body }) => {
let prepmember = &preparsed.items[&name].member;
let subprep = if let ModMember::Sub(m) = prepmember {m.clone()}
else {unreachable!("This name must point to a namespace")};
let subprep = if let ModMember::Sub(m) = prepmember {
m.clone()
} else {
unreachable!("This name must point to a namespace")
};
let new_body = entv_rec(
mod_stack.push(name),
subprep.as_ref(),
body,
path, ops_cache, i
path,
ops_cache,
i,
);
Member::Namespace(name, new_body)
Member::Namespace(Namespace { name, body: new_body })
},
any => any
any => any,
}
}
fn entv_rec(
// level
mod_stack: Substack<Token<String>>,
mod_stack: Substack<Tok<String>>,
preparsed: &Module<impl Clone, impl Clone>,
// object
data: Vec<FileEntry>,
// context
mod_path: &[Token<String>],
mod_path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner
i: &Interner,
) -> Vec<FileEntry> {
data.into_iter()
data
.into_iter()
.map(|ent| match ent {
FileEntry::Import(imps) => FileEntry::Import(imps.into_iter()
.flat_map(|import| if let Import{ name: None, path } = import {
let p = import_abs_path(
mod_path, mod_stack, preparsed, &i.r(path)[..], i
).expect("Should have emerged in preparsing");
let names = ops_cache.find(&i.i(&p))
.expect("Should have emerged in second parsing");
let imports = names.iter()
.map(move |&n| Import{ name: Some(n), path })
.collect::<Vec<_>>();
Box::new(imports.into_iter()) as BoxedIter<Import>
} else {box_once(import)})
.collect()
FileEntry::Import(imps) => FileEntry::Import(
imps
.into_iter()
.flat_map(|import| {
if let Import { name: None, path } = import {
let p = import_abs_path(mod_path, mod_stack, &i.r(path)[..], i)
.expect("Should have emerged in preparsing");
let names = ops_cache
.find(&i.i(&p))
.expect("Should have emerged in second parsing");
let imports = names
.iter()
.map(move |&n| Import { name: Some(n), path })
.collect::<Vec<_>>();
Box::new(imports.into_iter()) as BoxedIter<Import>
} else {
box_once(import)
}
})
.collect(),
),
FileEntry::Exported(mem) => FileEntry::Exported(member_rec(
mod_stack, preparsed, mem, mod_path, ops_cache, i
mod_stack, preparsed, mem, mod_path, ops_cache, i,
)),
FileEntry::Internal(mem) => FileEntry::Internal(member_rec(
mod_stack, preparsed, mem, mod_path, ops_cache, i
mod_stack, preparsed, mem, mod_path, ops_cache, i,
)),
any => any
any => any,
})
.collect()
}
@@ -76,9 +90,9 @@ fn entv_rec(
pub fn normalize_imports(
preparsed: &Module<impl Clone, impl Clone>,
data: Vec<FileEntry>,
path: &[Token<String>],
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner
i: &Interner,
) -> Vec<FileEntry> {
entv_rec(Substack::Bottom, preparsed, data, path, ops_cache, i)
}
}

37
src/pipeline/project_tree/parse_file.rs
View File

@@ -1,18 +1,17 @@
use std::rc::Rc;
use crate::parse;
use crate::pipeline::error::ProjectError;
use crate::representations::sourcefile::{FileEntry, normalize_namespaces};
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::interner::{Token, Interner};
use super::add_prelude::add_prelude;
use super::collect_ops::{ExportedOpsCache, collect_ops_for};
use super::collect_ops::{collect_ops_for, ExportedOpsCache};
use super::normalize_imports::normalize_imports;
use super::prefix::prefix;
use crate::interner::{Interner, Sym};
use crate::parse;
use crate::pipeline::error::ProjectError;
use crate::pipeline::source_loader::LoadedSourceTable;
use crate::representations::sourcefile::{normalize_namespaces, FileEntry};
pub fn parse_file(
path: Token<Vec<Token<String>>>,
path: Sym,
loaded: &LoadedSourceTable,
ops_cache: &ExportedOpsCache,
i: &Interner,
@@ -21,24 +20,24 @@ pub fn parse_file(
let ld = &loaded[&path];
// let ops_cache = collect_ops::mk_cache(loaded, i);
let ops = collect_ops_for(&i.r(path)[..], loaded, ops_cache, i)?;
let ops_vec = ops.iter()
.map(|t| i.r(*t))
.cloned()
.collect::<Vec<_>>();
let ctx = parse::ParsingContext{
let ops_vec = ops.iter().map(|t| i.r(*t)).cloned().collect::<Vec<_>>();
let ctx = parse::ParsingContext {
interner: i,
ops: &ops_vec,
file: Rc::new(i.extern_vec(path))
file: Rc::new(i.extern_vec(path)),
};
let entries = parse::parse(ld.text.as_str(), ctx)
.expect("This error should have been caught during loading");
let with_prelude = add_prelude(entries, &i.r(path)[..], prelude);
let impnormalized = normalize_imports(
&ld.preparsed.0, with_prelude, &i.r(path)[..], ops_cache, i
&ld.preparsed.0,
with_prelude,
&i.r(path)[..],
ops_cache,
i,
);
let nsnormalized = normalize_namespaces(
Box::new(impnormalized.into_iter()), i
).expect("This error should have been caught during preparsing");
let nsnormalized = normalize_namespaces(Box::new(impnormalized.into_iter()))
.expect("This error should have been caught during preparsing");
let prefixed = prefix(nsnormalized, &i.r(path)[..], ops_cache, i);
Ok(prefixed)
}
}

82
src/pipeline/project_tree/prefix.rs
View File

@@ -1,82 +1,78 @@
use std::rc::Rc;
use crate::ast::{Constant, Rule};
use crate::interner::{Token, Interner};
use crate::utils::Substack;
use crate::representations::sourcefile::{Member, FileEntry};
use super::collect_ops::ExportedOpsCache;
use crate::ast::{Constant, Rule};
use crate::interner::{Interner, Tok};
use crate::representations::sourcefile::{FileEntry, Member, Namespace};
use crate::utils::Substack;
fn member_rec(
// level
mod_stack: Substack<Token<String>>,
mod_stack: Substack<Tok<String>>,
// object
data: Member,
// context
path: &[Token<String>],
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner
i: &Interner,
) -> Member {
// let except = |op| imported.contains(&op);
let except = |_| false;
let prefix_v = path.iter().copied()
let prefix_v = path
.iter()
.copied()
.chain(mod_stack.iter().rev_vec_clone().into_iter())
.collect::<Vec<_>>();
let prefix = i.i(&prefix_v);
match data {
Member::Namespace(name, body) => {
let new_body = entv_rec(
mod_stack.push(name),
body,
path, ops_cache, i
);
Member::Namespace(name, new_body)
}
Member::Constant(constant) => Member::Constant(Constant{
Member::Namespace(Namespace { name, body }) => {
let new_body = entv_rec(mod_stack.push(name), body, path, ops_cache, i);
Member::Namespace(Namespace { name, body: new_body })
},
Member::Constant(constant) => Member::Constant(Constant {
name: constant.name,
value: constant.value.prefix(prefix, i, &except)
value: constant.value.prefix(prefix, i, &except),
}),
Member::Rule(rule) => Member::Rule(Rule{
Member::Rule(rule) => Member::Rule(Rule {
prio: rule.prio,
source: Rc::new(rule.source.iter()
.map(|e| e.prefix(prefix, i, &except))
.collect()
source: Rc::new(
rule.source.iter().map(|e| e.prefix(prefix, i, &except)).collect(),
),
target: Rc::new(rule.target.iter()
.map(|e| e.prefix(prefix, i, &except))
.collect()
target: Rc::new(
rule.target.iter().map(|e| e.prefix(prefix, i, &except)).collect(),
),
})
}),
}
}
fn entv_rec(
// level
mod_stack: Substack<Token<String>>,
mod_stack: Substack<Tok<String>>,
// object
data: Vec<FileEntry>,
// context
path: &[Token<String>],
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner
i: &Interner,
) -> Vec<FileEntry> {
data.into_iter().map(|fe| match fe {
FileEntry::Exported(mem) => FileEntry::Exported(member_rec(
mod_stack, mem, path, ops_cache, i
)),
FileEntry::Internal(mem) => FileEntry::Internal(member_rec(
mod_stack, mem, path, ops_cache, i
)),
// XXX should [FileEntry::Export] be prefixed?
any => any
}).collect()
data
.into_iter()
.map(|fe| match fe {
FileEntry::Exported(mem) =>
FileEntry::Exported(member_rec(mod_stack, mem, path, ops_cache, i)),
FileEntry::Internal(mem) =>
FileEntry::Internal(member_rec(mod_stack, mem, path, ops_cache, i)),
// XXX should [FileEntry::Export] be prefixed?
any => any,
})
.collect()
}
pub fn prefix(
data: Vec<FileEntry>,
path: &[Token<String>],
path: &[Tok<String>],
ops_cache: &ExportedOpsCache,
i: &Interner
i: &Interner,
) -> Vec<FileEntry> {
entv_rec(Substack::Bottom, data, path, ops_cache, i)
}
}

55
src/pipeline/project_tree/tree.rs
View File

@@ -1,33 +1,36 @@
use std::{ops::Add, rc::Rc};
use std::ops::Add;
use std::rc::Rc;
use hashbrown::HashMap;
use crate::representations::tree::{Module, ModMember};
use crate::ast::{Rule, Expr};
use crate::interner::{Token, Interner};
use crate::ast::{Expr, Rule};
use crate::interner::{Interner, Sym, Tok};
use crate::representations::tree::{ModMember, Module};
use crate::utils::Substack;
#[derive(Clone, Debug, Default)]
pub struct ProjectExt{
pub struct ProjectExt {
/// Pairs each foreign token to the module it was imported from
pub imports_from: HashMap<Token<String>, Token<Vec<Token<String>>>>,
pub imports_from: HashMap<Tok<String>, Sym>,
/// Pairs each exported token to its original full name.
pub exports: HashMap<Token<String>, Token<Vec<Token<String>>>>,
pub exports: HashMap<Tok<String>, Sym>,
/// All rules defined in this module, exported or not
pub rules: Vec<Rule>,
/// Filename, if known, for error reporting
pub file: Option<Vec<Token<String>>>
pub file: Option<Vec<Tok<String>>>,
}
impl Add for ProjectExt {
type Output = Self;
fn add(mut self, rhs: Self) -> Self::Output {
let ProjectExt{ imports_from, exports, rules, file } = rhs;
let ProjectExt { imports_from, exports, rules, file } = rhs;
self.imports_from.extend(imports_from.into_iter());
self.exports.extend(exports.into_iter());
self.rules.extend(rules.into_iter());
if file.is_some() { self.file = file }
if file.is_some() {
self.file = file
}
self
}
}
@@ -51,10 +54,10 @@ pub fn collect_rules(project: &ProjectTree) -> Vec<Rule> {
}
fn collect_consts_rec(
path: Substack<Token<String>>,
bag: &mut HashMap<Token<Vec<Token<String>>>, Expr>,
path: Substack<Tok<String>>,
bag: &mut HashMap<Sym, Expr>,
module: &ProjectModule,
i: &Interner
i: &Interner,
) {
for (key, entry) in module.items.iter() {
match &entry.member {
@@ -62,26 +65,18 @@ fn collect_consts_rec(
let mut name = path.iter().rev_vec_clone();
name.push(*key);
bag.insert(i.i(&name), expr.clone());
}
ModMember::Sub(module) => {
collect_consts_rec(
path.push(*key),
bag, module, i
)
}
},
ModMember::Sub(module) =>
collect_consts_rec(path.push(*key), bag, module, i),
}
}
}
pub fn collect_consts(project: &ProjectTree, i: &Interner)
-> HashMap<Token<Vec<Token<String>>>, Expr>
{
pub fn collect_consts(
project: &ProjectTree,
i: &Interner,
) -> HashMap<Sym, Expr> {
let mut consts = HashMap::new();
collect_consts_rec(
Substack::Bottom,
&mut consts,
project.0.as_ref(),
i
);
collect_consts_rec(Substack::Bottom, &mut consts, project.0.as_ref(), i);
consts
}
}

81
src/pipeline/source_loader/load_source.rs
View File

@@ -1,47 +1,58 @@
use std::iter;
use std::rc::Rc;
use super::loaded_source::{LoadedSource, LoadedSourceTable};
use super::preparse::preparse;
use crate::interner::{Interner, Sym, Tok};
use crate::pipeline::error::ProjectError;
use crate::pipeline::file_loader::{load_text, IOResult, Loaded};
use crate::pipeline::import_abs_path::import_abs_path;
use crate::pipeline::split_name::split_name;
use crate::interner::{Token, Interner};
use crate::pipeline::file_loader::{Loaded, load_text, IOResult};
use crate::representations::sourcefile::FileEntry;
use super::loaded_source::{LoadedSourceTable, LoadedSource};
use super::preparse::preparse;
/// Load the source at the given path or all within if it's a collection,
/// and all sources imported from these.
fn load_abs_path_rec(
abs_path: Token<Vec<Token<String>>>,
abs_path: Sym,
table: &mut LoadedSourceTable,
prelude: &[FileEntry],
i: &Interner,
get_source: &impl Fn(Token<Vec<Token<String>>>) -> IOResult,
is_injected: &impl Fn(&[Token<String>]) -> bool
get_source: &impl Fn(Sym) -> IOResult,
is_injected: &impl Fn(&[Tok<String>]) -> bool,
) -> Result<(), Rc<dyn ProjectError>> {
let abs_pathv = i.r(abs_path);
// short-circuit if this import is defined externally or already known
if is_injected(&abs_pathv) | table.contains_key(&abs_path) {
return Ok(())
if is_injected(abs_pathv) | table.contains_key(&abs_path) {
return Ok(());
}
// try splitting the path to file, swallowing any IO errors
let is_file = |p| (get_source)(p).map(|l| l.is_code()).unwrap_or(false);
let name_split = split_name(&abs_pathv, &|p| is_file(i.i(p)));
let filename = if let Some((f, _)) = name_split {f} else {
let name_split = split_name(abs_pathv, &|p| is_file(i.i(p)));
let filename = if let Some((f, _)) = name_split {
f
} else {
// If the path could not be split to file, load it as directory
let coll = if let Loaded::Collection(c) = (get_source)(abs_path)? {c}
// ^^ raise any IO error that was previously swallowed
else {panic!("split_name returned None but the path is a file")};
let coll = if let Loaded::Collection(c) = (get_source)(abs_path)? {
c
}
// ^^ raise any IO error that was previously swallowed
else {
panic!("split_name returned None but the path is a file")
};
// recurse on all files and folders within
for item in coll.iter() {
let abs_subpath = abs_pathv.iter()
let abs_subpath = abs_pathv
.iter()
.copied()
.chain(iter::once(i.i(item)))
.collect::<Vec<_>>();
load_abs_path_rec(
i.i(&abs_subpath), table, prelude, i, get_source, is_injected
i.i(&abs_subpath),
table,
prelude,
i,
get_source,
is_injected,
)?
}
return Ok(());
@@ -50,18 +61,23 @@ fn load_abs_path_rec(
let text = load_text(i.i(filename), &get_source, i)?;
let preparsed = preparse(
filename.iter().map(|t| i.r(*t)).cloned().collect(),
text.as_str(), prelude, i
text.as_str(),
prelude,
i,
)?;
table.insert(abs_path, LoadedSource{ text, preparsed: preparsed.clone() });
table.insert(abs_path, LoadedSource { text, preparsed: preparsed.clone() });
// recurse on all imported modules
preparsed.0.visit_all_imports(&mut |modpath, module, import| {
let abs_pathv = import_abs_path(
&filename, modpath,
module, &import.nonglob_path(i), i
)?;
preparsed.0.visit_all_imports(&mut |modpath, _module, import| {
let abs_pathv =
import_abs_path(filename, modpath, &import.nonglob_path(i), i)?;
// recurse on imported module
load_abs_path_rec(
i.i(&abs_pathv), table, prelude, i, get_source, is_injected
i.i(&abs_pathv),
table,
prelude,
i,
get_source,
is_injected,
)
})
}
@@ -69,20 +85,15 @@ fn load_abs_path_rec(
/// Load and preparse all files reachable from the load targets via
/// imports that aren't injected.
pub fn load_source(
targets: &[Token<Vec<Token<String>>>],
targets: &[Sym],
prelude: &[FileEntry],
i: &Interner,
get_source: &impl Fn(Token<Vec<Token<String>>>) -> IOResult,
is_injected: &impl Fn(&[Token<String>]) -> bool,
get_source: &impl Fn(Sym) -> IOResult,
is_injected: &impl Fn(&[Tok<String>]) -> bool,
) -> Result<LoadedSourceTable, Rc<dyn ProjectError>> {
let mut table = LoadedSourceTable::new();
for target in targets {
load_abs_path_rec(
*target,
&mut table,
prelude,
i, get_source, is_injected
)?
load_abs_path_rec(*target, &mut table, prelude, i, get_source, is_injected)?
}
Ok(table)
}
}

8
src/pipeline/source_loader/loaded_source.rs
View File

@@ -1,8 +1,8 @@
use std::{rc::Rc, collections::HashMap};
use crate::interner::Token;
use std::collections::HashMap;
use std::rc::Rc;
use super::preparse::Preparsed;
use crate::interner::Sym;
#[derive(Debug)]
pub struct LoadedSource {
@@ -10,4 +10,4 @@ pub struct LoadedSource {
pub preparsed: Preparsed,
}
pub type LoadedSourceTable = HashMap<Token<Vec<Token<String>>>, LoadedSource>;
pub type LoadedSourceTable = HashMap<Sym, LoadedSource>;

37
src/pipeline/source_loader/mod.rs
View File

@@ -1,25 +1,24 @@
/* PULL LOGISTICS BOUNDARY
Specifying exactly what this module should be doing was an unexpectedly
hard challenge. It is intended to encapsulate all pull logistics, but
this definition is apparently prone to scope creep.
Load files, preparse them to obtain a list of imports, follow these.
Preparsing also returns the module tree and list of exported synbols
for free, which is needed later so the output of preparsing is also
attached to the module output.
The module checks for IO errors, syntax errors, malformed imports and
imports from missing files. All other errors must be checked later.
Injection strategy:
see whether names are valid in the injected tree for is_injected
*/
// PULL LOGISTICS BOUNDARY
//
// Specifying exactly what this module should be doing was an unexpectedly
// hard challenge. It is intended to encapsulate all pull logistics, but
// this definition is apparently prone to scope creep.
//
// Load files, preparse them to obtain a list of imports, follow these.
// Preparsing also returns the module tree and list of exported synbols
// for free, which is needed later so the output of preparsing is also
// attached to the module output.
//
// The module checks for IO errors, syntax errors, malformed imports and
// imports from missing files. All other errors must be checked later.
//
// Injection strategy:
// see whether names are valid in the injected tree for is_injected
mod load_source;
mod loaded_source;
mod preparse;
pub use loaded_source::{LoadedSource, LoadedSourceTable};
pub use load_source::load_source;
pub use preparse::Preparsed;
pub use loaded_source::{LoadedSource, LoadedSourceTable};
pub use preparse::Preparsed;

138
src/pipeline/source_loader/preparse.rs
View File

@@ -1,39 +1,34 @@
use hashbrown::HashMap;
use std::hash::Hash;
use std::rc::Rc;
use hashbrown::HashMap;
use crate::ast::Constant;
use crate::pipeline::error::{ProjectError, ParseErrorWithPath, VisibilityMismatch};
use crate::representations::sourcefile::{normalize_namespaces, Member};
use crate::representations::tree::{ModEntry, ModMember};
use crate::interner::Interner;
use crate::parse::{self, ParsingContext};
use crate::representations::{sourcefile::{FileEntry, imports}, tree::Module};
use crate::pipeline::error::{
ParseErrorWithPath, ProjectError, VisibilityMismatch,
};
use crate::representations::sourcefile::{
imports, normalize_namespaces, FileEntry, Member,
};
use crate::representations::tree::{ModEntry, ModMember, Module};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Preparsed(pub Rc<Module<(), ()>>);
/// Add an internal flat name if it does not exist yet
fn add_intern<K: Eq + Hash>(
map: &mut HashMap<K, ModEntry<(), ()>>, k: K
) {
let _ = map.try_insert(k, ModEntry {
exported: false,
member: ModMember::Item(()),
});
fn add_intern<K: Eq + Hash>(map: &mut HashMap<K, ModEntry<(), ()>>, k: K) {
let _ = map
.try_insert(k, ModEntry { exported: false, member: ModMember::Item(()) });
}
/// Add an exported flat name or export any existing entry
fn add_export<K: Eq + Hash>(
map: &mut HashMap<K, ModEntry<(), ()>>, k: K
) {
fn add_export<K: Eq + Hash>(map: &mut HashMap<K, ModEntry<(), ()>>, k: K) {
if let Some(entry) = map.get_mut(&k) {
entry.exported = true
} else {
map.insert(k, ModEntry {
exported: true,
member: ModMember::Item(()),
});
map.insert(k, ModEntry { exported: true, member: ModMember::Item(()) });
}
}
@@ -41,47 +36,53 @@ fn add_export<K: Eq + Hash>(
fn to_module(
src: &[FileEntry],
prelude: &[FileEntry],
i: &Interner
i: &Interner,
) -> Rc<Module<(), ()>> {
let all_src = || src.iter().chain(prelude.iter());
let imports = imports(all_src()).cloned().collect::<Vec<_>>();
let mut items = all_src().filter_map(|ent| match ent {
FileEntry::Internal(Member::Namespace(name, data)) => {
let member = ModMember::Sub(to_module(data, prelude, i));
let entry = ModEntry{ exported: false, member };
Some((*name, entry))
let mut items = all_src()
.filter_map(|ent| match ent {
FileEntry::Internal(Member::Namespace(ns)) => {
let member = ModMember::Sub(to_module(&ns.body, prelude, i));
let entry = ModEntry { exported: false, member };
Some((ns.name, entry))
},
FileEntry::Exported(Member::Namespace(ns)) => {
let member = ModMember::Sub(to_module(&ns.body, prelude, i));
let entry = ModEntry { exported: true, member };
Some((ns.name, entry))
},
_ => None,
})
.collect::<HashMap<_, _>>();
for file_entry in all_src() {
match file_entry {
FileEntry::Comment(_)
| FileEntry::Import(_)
| FileEntry::Internal(Member::Namespace(_))
| FileEntry::Exported(Member::Namespace(_)) => (),
FileEntry::Export(tokv) =>
for tok in tokv {
add_export(&mut items, *tok)
},
FileEntry::Internal(Member::Constant(Constant { name, .. })) =>
add_intern(&mut items, *name),
FileEntry::Exported(Member::Constant(Constant { name, .. })) =>
add_export(&mut items, *name),
FileEntry::Internal(Member::Rule(rule)) => {
let names = rule.collect_single_names(i);
for name in names {
add_intern(&mut items, name)
}
},
FileEntry::Exported(Member::Rule(rule)) => {
let names = rule.collect_single_names(i);
for name in names {
add_export(&mut items, name)
}
},
}
FileEntry::Exported(Member::Namespace(name, data)) => {
let member = ModMember::Sub(to_module(data, prelude, i));
let entry = ModEntry{ exported: true, member };
Some((*name, entry))
}
_ => None
}).collect::<HashMap<_, _>>();
for file_entry in all_src() { match file_entry {
FileEntry::Comment(_) | FileEntry::Import(_)
| FileEntry::Internal(Member::Namespace(..))
| FileEntry::Exported(Member::Namespace(..)) => (),
FileEntry::Export(tokv) => for tok in tokv {
add_export(&mut items, *tok)
}
FileEntry::Internal(Member::Constant(Constant{ name, .. }))
=> add_intern(&mut items, *name),
FileEntry::Exported(Member::Constant(Constant{ name, .. }))
=> add_export(&mut items, *name),
FileEntry::Internal(Member::Rule(rule)) => {
let names = rule.collect_single_names(i);
for name in names {
add_intern(&mut items, name)
}
}
FileEntry::Exported(Member::Rule(rule)) => {
let names = rule.collect_single_names(i);
for name in names {
add_export(&mut items, name)
}
}
}}
}
Rc::new(Module { imports, items, extra: () })
}
@@ -95,16 +96,21 @@ pub fn preparse(
) -> Result<Preparsed, Rc<dyn ProjectError>> {
// Parse with no operators
let ctx = ParsingContext::<&str>::new(&[], i, Rc::new(file.clone()));
let entries = parse::parse(source, ctx)
.map_err(|error| ParseErrorWithPath{
let entries = parse::parse(source, ctx).map_err(|error| {
ParseErrorWithPath {
full_source: source.to_string(),
error,
path: file.clone()
}.rc())?;
let normalized = normalize_namespaces(Box::new(entries.into_iter()), i)
.map_err(|ns| VisibilityMismatch{
namespace: ns.into_iter().map(|t| i.r(t)).cloned().collect(),
file: Rc::new(file.clone())
}.rc())?;
path: file.clone(),
}
.rc()
})?;
let normalized = normalize_namespaces(Box::new(entries.into_iter()))
.map_err(|ns| {
VisibilityMismatch {
namespace: ns.into_iter().map(|t| i.r(t)).cloned().collect(),
file: Rc::new(file.clone()),
}
.rc()
})?;
Ok(Preparsed(to_module(&normalized, prelude, i)))
}
}

14
src/pipeline/split_name.rs
View File

@@ -1,14 +1,16 @@
use crate::interner::Token;
use crate::interner::Tok;
#[allow(clippy::type_complexity)]
// FIXME couldn't find a good factoring
pub fn split_name<'a>(
path: &'a [Token<String>],
is_valid: &impl Fn(&[Token<String>]) -> bool
) -> Option<(&'a [Token<String>], &'a [Token<String>])> {
path: &'a [Tok<String>],
is_valid: &impl Fn(&[Tok<String>]) -> bool,
) -> Option<(&'a [Tok<String>], &'a [Tok<String>])> {
for split in (0..=path.len()).rev() {
let (filename, subpath) = path.split_at(split);
if is_valid(filename) {
return Some((filename, subpath))
return Some((filename, subpath));
}
}
None
}
}