diff --git a/orchid-api/src/parser.rs b/orchid-api/src/parser.rs index 7f0dd64..7102d65 100644 --- a/orchid-api/src/parser.rs +++ b/orchid-api/src/parser.rs @@ -1,9 +1,10 @@ +use core::ops::Range; use std::num::NonZeroU64; use orchid_api_derive::{Coding, Hierarchy}; use orchid_api_traits::Request; -use crate::{Comment, HostExtReq, OrcResult, SysId, TStrv, TokenTree}; +use crate::{HostExtReq, OrcResult, SysId, TStr, TStrv, TokenTree}; #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)] pub struct ParsId(pub NonZeroU64); @@ -20,3 +21,9 @@ pub struct ParseLine { impl Request for ParseLine { type Response = OrcResult>; } + +#[derive(Clone, Debug, Coding)] +pub struct Comment { + pub text: TStr, + pub range: Range, +} diff --git a/orchid-api/src/tree.rs b/orchid-api/src/tree.rs index 3e8837f..ae788a1 100644 --- a/orchid-api/src/tree.rs +++ b/orchid-api/src/tree.rs @@ -1,14 +1,12 @@ use std::collections::HashMap; use std::num::NonZeroU64; use std::ops::Range; -use std::sync::Arc; +use std::rc::Rc; use orchid_api_derive::{Coding, Hierarchy}; use orchid_api_traits::Request; -use crate::{ - ExprTicket, Expression, ExtHostReq, HostExtReq, Location, OrcError, SysId, TStr, TStrv, -}; +use crate::{ExprTicket, Expression, ExtHostReq, HostExtReq, OrcError, SysId, TStr, TStrv}; /// A token tree from a lexer recursion request. Its lifetime is the lex call, /// the lexer can include it in its output or discard it by implication. @@ -49,7 +47,7 @@ pub enum Token { /// NewExpr(Bottom) because it fails in dead branches too. Bottom(Vec), /// A comment - Comment(Arc), + Comment(Rc), } #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)] @@ -62,42 +60,25 @@ pub enum Paren { #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)] pub struct TreeId(pub NonZeroU64); -#[derive(Clone, Debug, Coding)] -pub struct Item { - pub location: Location, - pub comments: Vec, - pub kind: ItemKind, -} - -#[derive(Clone, Debug, Coding)] -pub enum ItemKind { - Member(Member), - Export(TStr), - Import(TStrv), -} - -#[derive(Clone, Debug, Coding)] -pub struct Comment { - pub text: TStr, - pub location: Location, -} - #[derive(Clone, Debug, Coding)] pub struct Member { pub name: TStr, + pub exported: bool, pub kind: MemberKind, + pub comments: Vec, } #[derive(Clone, Debug, Coding)] pub enum MemberKind { Const(Expression), Module(Module), + Import(TStrv), Lazy(TreeId), } #[derive(Clone, Debug, Coding)] pub struct Module { - pub items: Vec, + pub members: Vec, } /// Evaluate a lazy member. This call will only be issued to each system once. diff --git a/orchid-base/src/parse.rs b/orchid-base/src/parse.rs index 0c8cf92..a07fa61 100644 --- a/orchid-base/src/parse.rs +++ b/orchid-base/src/parse.rs @@ -113,14 +113,23 @@ pub fn strip_fluff(tt: &TokTree) -> Option, - pub pos: Pos, + pub range: Range, } impl Comment { - pub fn to_api(&self) -> api::Comment { - api::Comment { location: self.pos.to_api(), text: self.text.to_api() } + pub async fn from_api(c: &api::Comment, i: &Interner) -> Self { + Self { text: i.ex(c.text).await, range: c.range.clone() } } - pub async fn from_api(api: &api::Comment, i: &Interner) -> Self { - Self { pos: Pos::from_api(&api.location, i).await, text: Tok::from_api(api.text, i).await } + pub async fn from_tk(tk: &TokTree, i: &Interner) -> Option { + match &tk.tok { + Token::Comment(text) => Some(Self { text: i.i(&**text).await, range: tk.range.clone() }), + _ => None, + } + } + pub fn to_tk(&self) -> TokTree { + TokTree { tok: Token::Comment(self.text.rc().clone()), range: self.range.clone() } + } + pub fn to_api(&self) -> api::Comment { + api::Comment { range: self.range.clone(), text: self.text.to_api() } } } @@ -145,11 +154,7 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>( Some(i) => { let (cmts, tail) = line.split_at(i); let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async { - match &t.tok { - Token::Comment(c) => - Comment { text: ctx.i().i(&**c).await, pos: Pos::Range(t.range.clone()) }, - _ => unreachable!("All are comments checked above"), - } + Comment::from_tk(t, ctx.i()).await.expect("All are comments checked above") })) .await; items.push(Parsed { output: comments, tail }); diff --git a/orchid-base/src/tree.rs b/orchid-base/src/tree.rs index 87e53a6..4f72992 100644 --- a/orchid-base/src/tree.rs +++ b/orchid-base/src/tree.rs @@ -4,7 +4,6 @@ use std::future::Future; use std::marker::PhantomData; use std::ops::Range; use std::rc::Rc; -use std::sync::Arc; use async_stream::stream; use futures::future::join_all; @@ -216,7 +215,7 @@ pub enum Token { /// Information about the code addressed to the human reader or dev tooling /// It has no effect on the behaviour of the program unless it's explicitly /// read via reflection - Comment(Arc), + Comment(Rc), /// The part of a lambda between `\` and `.` enclosing the argument. The body /// stretches to the end of the enclosing parens or the end of the const line LambdaHead(Vec>), diff --git a/orchid-extension/src/entrypoint.rs b/orchid-extension/src/entrypoint.rs index 6119e48..69c8339 100644 --- a/orchid-extension/src/entrypoint.rs +++ b/orchid-extension/src/entrypoint.rs @@ -34,7 +34,7 @@ use crate::fs::VirtFS; use crate::lexer::{LexContext, err_cascade, err_not_applicable}; use crate::system::{SysCtx, atom_by_idx}; use crate::system_ctor::{CtedObj, DynSystemCtor}; -use crate::tree::{GenItemKind, GenTok, GenTokTree, LazyMemberFactory, TreeIntoApiCtxImpl}; +use crate::tree::{GenTok, GenTokTree, LazyMemberFactory, TreeIntoApiCtxImpl}; pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>; pub type ExtReqNot = ReqNot; @@ -197,9 +197,6 @@ pub fn extension_init( let lazy_mems = Mutex::new(HashMap::new()); let ctx = init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await; let const_root = stream::from_iter(cted.inst().dyn_env()) - .filter_map( - async |i| if let GenItemKind::Member(m) = i.kind { Some(m) } else { None }, - ) .then(|mem| { let (req, lazy_mems) = (&hand, &lazy_mems); clone!(i, ctx; async move { diff --git a/orchid-extension/src/func_atom.rs b/orchid-extension/src/func_atom.rs index 32a69ed..bee0419 100644 --- a/orchid-extension/src/func_atom.rs +++ b/orchid-extension/src/func_atom.rs @@ -17,7 +17,7 @@ use orchid_base::format::{FmtCtx, FmtUnit}; use orchid_base::name::Sym; use trait_set::trait_set; -use crate::atom::{Atomic, MethodSetBuilder}; +use crate::atom::Atomic; use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant}; use crate::conv::ToExpr; use crate::expr::Expr; diff --git a/orchid-extension/src/system.rs b/orchid-extension/src/system.rs index 200ac7d..a61b0f9 100644 --- a/orchid-extension/src/system.rs +++ b/orchid-extension/src/system.rs @@ -23,7 +23,7 @@ use crate::func_atom::Fun; use crate::lexer::LexerObj; use crate::parser::ParserObj; use crate::system_ctor::{CtedObj, SystemCtor}; -use crate::tree::GenItem; +use crate::tree::GenMember; /// System as consumed by foreign code pub trait SystemCard: Default + Send + Sync + 'static { @@ -82,7 +82,7 @@ impl DynSystemCard for T { /// System as defined by author pub trait System: Send + Sync + SystemCard + 'static { - fn env() -> Vec; + fn env() -> Vec; fn vfs() -> DeclFs; fn lexers() -> Vec; fn parsers() -> Vec; @@ -90,7 +90,7 @@ pub trait System: Send + Sync + SystemCard + 'static { } pub trait DynSystem: Send + Sync + DynSystemCard + 'static { - fn dyn_env(&self) -> Vec; + fn dyn_env(&self) -> Vec; fn dyn_vfs(&self) -> DeclFs; fn dyn_lexers(&self) -> Vec; fn dyn_parsers(&self) -> Vec; @@ -99,7 +99,7 @@ pub trait DynSystem: Send + Sync + DynSystemCard + 'static { } impl DynSystem for T { - fn dyn_env(&self) -> Vec { Self::env() } + fn dyn_env(&self) -> Vec { Self::env() } fn dyn_vfs(&self) -> DeclFs { Self::vfs() } fn dyn_lexers(&self) -> Vec { Self::lexers() } fn dyn_parsers(&self) -> Vec { Self::parsers() } diff --git a/orchid-extension/src/tree.rs b/orchid-extension/src/tree.rs index bbeedbe..94bf7be 100644 --- a/orchid-extension/src/tree.rs +++ b/orchid-extension/src/tree.rs @@ -1,10 +1,11 @@ use std::num::NonZero; use std::rc::Rc; +use async_stream::stream; use dyn_clone::{DynClone, clone_box}; -use futures::FutureExt; use futures::future::{LocalBoxFuture, join_all}; -use hashbrown::{HashMap, HashSet}; +use futures::{FutureExt, StreamExt}; +use hashbrown::HashMap; use itertools::Itertools; use orchid_base::interner::{Interner, Tok}; use orchid_base::location::Pos; @@ -64,60 +65,27 @@ impl TokenVariant for Expr { } } -fn with_export(mem: GenMember, public: bool) -> Vec { - (public.then(|| GenItemKind::Export(mem.name.clone())).into_iter()) - .chain([GenItemKind::Member(mem)]) - .map(|kind| GenItem { comments: vec![], kind }) - .collect() -} - -pub struct GenItem { - pub kind: GenItemKind, - pub comments: Vec, -} -impl GenItem { - pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Item { - let kind = match self.kind { - GenItemKind::Export(n) => api::ItemKind::Export(ctx.sys().i().i::(&n).await.to_api()), - GenItemKind::Member(mem) => api::ItemKind::Member(mem.into_api(ctx).await), - GenItemKind::Import(cn) => api::ItemKind::Import( - Sym::parse(&cn, ctx.sys().i()).await.expect("Import path empty string").to_api(), - ), - }; - let comments = join_all(self.comments.iter().map(|c| async { - api::Comment { - location: api::Location::Inherit, - text: ctx.sys().i().i::(c).await.to_api(), - } - })) - .await; - api::Item { location: api::Location::Inherit, comments, kind } - } -} - -pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec { - with_export(GenMember { name: name.to_string(), kind: MemKind::Const(value.to_expr()) }, public) -} -pub fn import(public: bool, path: &str) -> Vec { - let mut out = vec![GenItemKind::Import(path.to_string())]; - if public { - out.push(GenItemKind::Export(path.split("::").last().unwrap().to_string())); - } - out.into_iter().map(|kind| GenItem { comments: vec![], kind }).collect() +pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec { + vec![GenMember { + name: name.to_string(), + kind: MemKind::Const(value.to_expr()), + comments: vec![], + public, + }] } pub fn module( public: bool, name: &str, - items: impl IntoIterator>, -) -> Vec { - let (name, kind) = root_mod(name, items); - with_export(GenMember { name, kind }, public) + mems: impl IntoIterator>, +) -> Vec { + let (name, kind) = root_mod(name, mems); + vec![GenMember { name, kind, public, comments: vec![] }] } -pub fn root_mod(name: &str, items: impl IntoIterator>) -> (String, MemKind) { - let kind = MemKind::Mod { items: items.into_iter().flatten().collect() }; +pub fn root_mod(name: &str, mems: impl IntoIterator>) -> (String, MemKind) { + let kind = MemKind::Mod { members: mems.into_iter().flatten().collect() }; (name.to_string(), kind) } -pub fn fun(exported: bool, name: &str, xf: impl ExprFunc) -> Vec { +pub fn fun(public: bool, name: &str, xf: impl ExprFunc) -> Vec { let fac = LazyMemberFactory::new(move |sym, ctx| async { return MemKind::Const(build_lambdas(Fun::new(sym, ctx, xf).await, 0)); fn build_lambdas(fun: Fun, i: u64) -> GExpr { @@ -132,9 +100,9 @@ pub fn fun(exported: bool, name: &str, xf: impl ExprFunc) -> Vec>) -> Vec { +pub fn prefix(path: &str, items: impl IntoIterator>) -> Vec { let mut items = items.into_iter().flatten().collect_vec(); for step in path.split("::").collect_vec().into_iter().rev() { items = module(true, step, [items]); @@ -144,8 +112,8 @@ pub fn prefix(path: &str, items: impl IntoIterator>) -> Vec< pub fn comments<'a>( cmts: impl IntoIterator, - mut val: Vec, -) -> Vec { + mut val: Vec, +) -> Vec { let cmts = cmts.into_iter().map(|c| c.to_string()).collect_vec(); for v in val.iter_mut() { v.comments.extend(cmts.iter().cloned()); @@ -159,50 +127,35 @@ pub fn comments<'a>( /// - Comments on exports and submodules are combined /// - Duplicate constants result in an error /// - A combination of lazy and anything results in an error -pub fn merge_trivial(trees: impl IntoIterator>) -> Vec { - let mut imported = HashSet::::new(); - let mut exported = HashMap::>::new(); - let mut members = HashMap::)>::new(); - for item in trees.into_iter().flatten() { - match item.kind { - GenItemKind::Import(sym) => { - imported.insert(sym); +pub fn merge_trivial(trees: impl IntoIterator>) -> Vec { + let mut all_members = HashMap::)>::new(); + for mem in trees.into_iter().flatten() { + assert!(mem.public, "Non-trivial merge in {}", mem.name); + match mem.kind { + unit @ (MemKind::Const(_) | MemKind::Lazy(_)) => { + let prev = all_members.insert(mem.name.clone(), (unit, mem.comments.into_iter().collect())); + assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name); }, - GenItemKind::Export(e) => - exported.entry(e.clone()).or_insert(HashSet::new()).extend(item.comments.iter().cloned()), - GenItemKind::Member(mem) => match mem.kind { - unit @ (MemKind::Const(_) | MemKind::Lazy(_)) => { - let prev = members.insert(mem.name.clone(), (unit, item.comments.into_iter().collect())); - assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name); + MemKind::Mod { members } => match all_members.entry(mem.name.clone()) { + hashbrown::hash_map::Entry::Vacant(slot) => { + slot.insert((MemKind::Mod { members }, mem.comments.into_iter().collect())); }, - MemKind::Mod { items } => match members.entry(mem.name.clone()) { - hashbrown::hash_map::Entry::Vacant(slot) => { - slot.insert((MemKind::Mod { items }, item.comments.into_iter().collect())); - }, - hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() { - (MemKind::Mod { items: old_items }, old_cmts) => { - let mut swap = vec![]; - std::mem::swap(&mut swap, old_items); - *old_items = merge_trivial([swap, items]); - old_cmts.extend(item.comments); - }, - _ => panic!("Conflict in trivial merge on {}", mem.name), + hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() { + (MemKind::Mod { members: old_items, .. }, old_cmts) => { + let mut swap = vec![]; + std::mem::swap(&mut swap, old_items); + *old_items = merge_trivial([swap, members]); + old_cmts.extend(mem.comments); }, + _ => panic!("non-trivial merge on {}", mem.name), }, }, } } - (imported.into_iter().map(|txt| GenItem { comments: vec![], kind: GenItemKind::Import(txt) })) - .chain(exported.into_iter().map(|(k, cmtv)| GenItem { - comments: cmtv.into_iter().collect(), - kind: GenItemKind::Export(k), - })) - .chain(members.into_iter().map(|(name, (kind, cmtv))| GenItem { - comments: cmtv.into_iter().collect(), - kind: GenItemKind::Member(GenMember { name, kind }), - })) - .collect() + (all_members.into_iter()) + .map(|(name, (kind, comments))| GenMember { comments, kind, name, public: true }) + .collect_vec() } trait_set! { @@ -220,29 +173,25 @@ impl Clone for LazyMemberFactory { fn clone(&self) -> Self { Self(clone_box(&*self.0)) } } -pub enum GenItemKind { - Member(GenMember), - Export(String), - Import(String), -} - pub struct GenMember { pub name: String, pub kind: MemKind, + pub public: bool, + pub comments: Vec, } impl GenMember { pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Member { let name = ctx.sys().i().i::(&self.name).await; - api::Member { - kind: self.kind.into_api(&mut ctx.push_path(name.clone())).await, - name: name.to_api(), - } + let kind = self.kind.into_api(&mut ctx.push_path(name.clone())).await; + let comments = + join_all(self.comments.iter().map(|cmt| async { ctx.sys().i().i(cmt).await.to_api() })).await; + api::Member { kind, name: name.to_api(), comments, exported: self.public } } } pub enum MemKind { Const(GExpr), - Mod { items: Vec }, + Mod { members: Vec }, Lazy(LazyMemberFactory), } impl MemKind { @@ -250,13 +199,10 @@ impl MemKind { match self { Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)), Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys(), ctx.req()).await), - Self::Mod { items } => { - let mut api_items = Vec::new(); - for i in items { - api_items.push(i.into_api(ctx).boxed_local().await) - } - api::MemberKind::Module(api::Module { items: api_items }) - }, + Self::Mod { members } => api::MemberKind::Module(api::Module { + members: Box::pin(stream! { for m in members { yield m.into_api(ctx).await } }.collect()) + .await, + }), } } } diff --git a/orchid-host/src/extension.rs b/orchid-host/src/extension.rs index ec842f4..46cd143 100644 --- a/orchid-host/src/extension.rs +++ b/orchid-host/src/extension.rs @@ -10,7 +10,7 @@ use async_std::sync::Mutex; use async_stream::stream; use derive_destructure::destructure; use futures::future::{join, join_all}; -use futures::{FutureExt, StreamExt, stream, stream_select}; +use futures::{StreamExt, stream}; use hashbrown::HashMap; use itertools::Itertools; use orchid_api::{HostMsgSet, LsModule}; diff --git a/orchid-host/src/lex.rs b/orchid-host/src/lex.rs index a5c6215..0764e50 100644 --- a/orchid-host/src/lex.rs +++ b/orchid-host/src/lex.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::rc::Rc; use async_std::sync::Mutex; use futures::FutureExt; @@ -107,11 +107,11 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes { )); }; ctx.set_tail(tail); - ParsTok::Comment(Arc::new(cmt.to_string())) + ParsTok::Comment(Rc::new(cmt.to_string())) } else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) { let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1); ctx.push_pos(end as u32); - ParsTok::Comment(Arc::new(tail[2..end].to_string())) + ParsTok::Comment(Rc::new(tail[2..end].to_string())) } else if ctx.strip_char('\\') { let mut arg = Vec::new(); ctx.trim_ws(); diff --git a/orchid-host/src/parse.rs b/orchid-host/src/parse.rs index 2f9be6e..69df744 100644 --- a/orchid-host/src/parse.rs +++ b/orchid-host/src/parse.rs @@ -141,10 +141,10 @@ pub async fn parse_exportable_item<'a>( let path_sym = Sym::new(path.unreverse(), ctx.i()).await.expect("Files should have a namespace"); let kind = if discr == ctx.i().i("mod").await { let (name, body) = parse_module(ctx, path, tail).await?; - ItemKind::Member(ParsedMember::new(name, path_sym, ParsedMemberKind::Mod(body))) + ItemKind::Member(ParsedMember { name, full_name: path_sym, kind: ParsedMemberKind::Mod(body) }) } else if discr == ctx.i().i("const").await { let name = parse_const(ctx, tail, path.clone()).await?; - ItemKind::Member(ParsedMember::new(name, path_sym, ParsedMemberKind::Const)) + ItemKind::Member(ParsedMember { name, full_name: path_sym, kind: ParsedMemberKind::Const }) } else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) { let line = sys.parse(path_sym, tail.to_vec(), exported, comments).await?; return parse_items(ctx, path, Snippet::new(tail.prev(), &line)).await; diff --git a/orchid-host/src/parsed.rs b/orchid-host/src/parsed.rs index e4d4a41..76a6f69 100644 --- a/orchid-host/src/parsed.rs +++ b/orchid-host/src/parsed.rs @@ -90,36 +90,17 @@ pub enum ItemKind { impl ItemKind { pub fn at(self, pos: Pos) -> Item { Item { comments: vec![], pos, kind: self } } } - -impl Item { - pub async fn from_api<'a>(tree: api::Item, ctx: &mut ParsedFromApiCx<'a>) -> Self { - let kind = match tree.kind { - api::ItemKind::Member(m) => ItemKind::Member(ParsedMember::from_api(m, ctx).await), - api::ItemKind::Import(name) => ItemKind::Import(Import { - path: Sym::from_api(name, &ctx.sys.ctx().i).await.iter().collect(), - name: None, - }), - api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &ctx.sys.ctx().i).await), - }; - let mut comments = Vec::new(); - for comment in tree.comments.iter() { - comments.push(Comment::from_api(comment, &ctx.sys.ctx().i).await) - } - Self { pos: Pos::from_api(&tree.location, &ctx.sys.ctx().i).await, comments, kind } - } -} impl Format for Item { async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit { let comment_text = self.comments.iter().join("\n"); let item_text = match &self.kind { ItemKind::Import(i) => format!("import {i}").into(), ItemKind::Export(e) => format!("export {e}").into(), - ItemKind::Member(mem) => match mem.kind.get() { - None => format!("lazy {}", mem.name).into(), - Some(ParsedMemberKind::Const) => + ItemKind::Member(mem) => match &mem.kind { + ParsedMemberKind::Const => tl_cache!(Rc: Rc::new(Variants::default().bounded("const {0}"))) .units([mem.name.rc().into()]), - Some(ParsedMemberKind::Mod(module)) => + ParsedMemberKind::Mod(module) => tl_cache!(Rc: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}"))) .units([mem.name.rc().into(), module.print(c).boxed_local().await]), }, @@ -130,71 +111,12 @@ impl Format for Item { } pub struct ParsedMember { - name: Tok, - full_name: Sym, - kind: OnceCell, - lazy: Mutex>, + pub name: Tok, + pub full_name: Sym, + pub kind: ParsedMemberKind, } -// TODO: this one should own but not execute the lazy handle. -// Lazy handles should run -// - in the tree converter function as needed to resolve imports -// - in the tree itself when a constant is loaded -// - when a different lazy subtree references them in a wildcard import and -// forces the enumeration. -// -// do we actually need to allow wildcard imports in lazy trees? maybe a -// different kind of import is sufficient. Source code never becomes a lazy -// tree. What does? -// - Systems subtrees rarely reference each other at all. They can't use macros -// and they usually point to constants with an embedded expr. -// - Compiled libraries on the long run. The code as written may reference -// constants by indirect path. But this is actually the same as the above, -// they also wouldn't use regular imports because they are distributed as -// exprs. -// -// Everything is distributed either as source code or as exprs. Line parsers -// also operate on tokens. -// -// TODO: The trees produced by systems can be safely changed -// to the new kind of tree. This datastructure does not need to support the lazy -// handle. impl ParsedMember { pub fn name(&self) -> Tok { self.name.clone() } - pub async fn kind(&self, consts: &mut HashMap) -> &ParsedMemberKind { - (self.kind.get_or_init(async { - let handle = self.lazy.lock().await.take().expect("Neither known nor lazy"); - handle.run(consts).await - })) - .await - } - pub async fn kind_mut(&mut self, consts: &mut HashMap) -> &mut ParsedMemberKind { - self.kind(consts).await; - self.kind.get_mut().expect("kind() already filled the cell") - } - pub async fn from_api<'a>(api: api::Member, ctx: &'_ mut ParsedFromApiCx<'a>) -> Self { - let name = Tok::from_api(api.name, &ctx.sys.ctx().i).await; - let mut ctx: ParsedFromApiCx<'_> = (&mut *ctx).push(name.clone()).await; - let path_sym = Sym::from_tok(ctx.path.clone()).expect("We just pushed on to this"); - let kind = match api.kind { - api::MemberKind::Lazy(id) => { - let handle = LazyMemberHandle { id, sys: ctx.sys.clone(), path: path_sym.clone() }; - return handle.into_member(name.clone()).await; - }, - api::MemberKind::Const(c) => { - let mut pctx = - ExprParseCtx { ctx: ctx.sys.ctx().clone(), exprs: ctx.sys.ext().exprs().clone() }; - let expr = Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await; - ctx.consts.insert(path_sym.clone(), expr); - ParsedMemberKind::Const - }, - api::MemberKind::Module(m) => - ParsedMemberKind::Mod(ParsedModule::from_api(m, &mut ctx).await), - }; - ParsedMember { name, full_name: path_sym, kind: OnceCell::from(kind), lazy: Mutex::default() } - } - pub fn new(name: Tok, full_name: Sym, kind: ParsedMemberKind) -> Self { - ParsedMember { name, full_name, kind: OnceCell::from(kind), lazy: Mutex::default() } - } } impl Debug for ParsedMember { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -233,18 +155,10 @@ impl ParsedModule { std::mem::swap(self, &mut swap); *self = ParsedModule::new(swap.items.into_iter().chain(other.items)) } - pub async fn from_api<'a>(m: api::Module, ctx: &mut ParsedFromApiCx<'a>) -> Self { - Self::new( - stream! { for item in m.items { yield Item::from_api(item, ctx).boxed_local().await } } - .collect::>() - .await, - ) - } pub async fn walk<'a>( &self, allow_private: bool, path: impl IntoIterator>, - consts: &mut HashMap, ) -> Result<&ParsedModule, WalkError> { let mut cur = self; for (pos, step) in path.into_iter().enumerate() { @@ -257,7 +171,7 @@ impl ParsedModule { if !allow_private && !cur.exports.contains(&step) { return Err(WalkError { pos, kind: WalkErrorKind::Private }); } - match member.kind(consts).await { + match &member.kind { ParsedMemberKind::Const => return Err(WalkError { pos, kind: WalkErrorKind::Constant }), ParsedMemberKind::Mod(m) => cur = m, } @@ -286,41 +200,6 @@ impl Format for ParsedModule { } } -pub struct LazyMemberHandle { - id: api::TreeId, - sys: System, - path: Sym, -} -impl LazyMemberHandle { - pub async fn run(self, consts: &mut HashMap) -> ParsedMemberKind { - match self.sys.get_tree(self.id).await { - api::MemberKind::Const(c) => { - let mut pctx = - ExprParseCtx { ctx: self.sys.ctx().clone(), exprs: self.sys.ext().exprs().clone() }; - consts.insert(self.path, Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await); - ParsedMemberKind::Const - }, - api::MemberKind::Module(m) => ParsedMemberKind::Mod( - ParsedModule::from_api(m, &mut ParsedFromApiCx { - sys: &self.sys, - consts, - path: self.path.tok(), - }) - .await, - ), - api::MemberKind::Lazy(id) => Self { id, ..self }.run(consts).boxed_local().await, - } - } - pub async fn into_member(self, name: Tok) -> ParsedMember { - ParsedMember { - name, - full_name: self.path.clone(), - kind: OnceCell::new(), - lazy: Mutex::new(Some(self)), - } - } -} - /// TODO: /// /// idea, does the host need an IR here or can we figure out a way to transcribe @@ -356,8 +235,7 @@ impl Root { return Ok(val.clone()); } let (cn, mp) = name.split_last(); - let consts_mut = &mut *self.consts.write().await; - let module = self.tree.walk(true, mp.iter().cloned(), consts_mut).await.unwrap(); + let module = self.tree.walk(true, mp.iter().cloned()).await.unwrap(); let member = (module.items.iter()) .filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None }) .find(|m| m.name() == cn); @@ -367,14 +245,14 @@ impl Root { format!("{name} does not refer to a constant"), [pos.clone().into()], )), - Some(mem) => match mem.kind(consts_mut).await { + Some(mem) => match &mem.kind { ParsedMemberKind::Mod(_) => Err(mk_errv( ctx.i.i("module used as constant").await, format!("{name} is a module, not a constant"), [pos.clone().into()], )), ParsedMemberKind::Const => Ok( - (consts_mut.get(&name).cloned()) + (self.consts.read().await.get(&name).cloned()) .expect("Tree says the path is correct but no value was found"), ), }, diff --git a/orchid-host/src/system.rs b/orchid-host/src/system.rs index 780ab3b..184a96c 100644 --- a/orchid-host/src/system.rs +++ b/orchid-host/src/system.rs @@ -26,7 +26,8 @@ use crate::api; use crate::ctx::Ctx; use crate::expr::{Expr, ExprParseCtx}; use crate::extension::{Extension, WeakExtension}; -use crate::parsed::{ItemKind, ParsedMember, ParsedModule, ParsTokTree, ParsedFromApiCx, Root}; +use crate::parsed::{ItemKind, ParsTokTree, ParsedFromApiCx, ParsedMember, ParsedModule, Root}; +use crate::tree::{Member, Module}; #[derive(destructure)] struct SystemInstData { @@ -137,7 +138,7 @@ impl SystemCtor { &self, depends: impl IntoIterator, consts: &mut HashMap, - ) -> (ParsedModule, System) { + ) -> (Module, System) { let depends = depends.into_iter().map(|si| si.id()).collect_vec(); debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided"); let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension"); @@ -152,10 +153,17 @@ impl SystemCtor { .await, id, })); + let const_root = Module::from_api((sys_inst.const_root.into_iter()).map(|k, v| api::Member { + name: k, + kind: v, + comments: vec![], + exported: true, + })) + .await; let const_root = clone!(data, ext; stream! { for (k, v) in sys_inst.const_root { - yield ParsedMember::from_api( - api::Member { name: k, kind: v }, + yield Member::from_api( + , &mut ParsedFromApiCx { consts, path: ext.ctx().i.i(&[]).await, diff --git a/orchid-host/src/tree.rs b/orchid-host/src/tree.rs index dbf4952..adf26eb 100644 --- a/orchid-host/src/tree.rs +++ b/orchid-host/src/tree.rs @@ -2,23 +2,40 @@ use std::cell::RefCell; use std::rc::{Rc, Weak}; use async_once_cell::OnceCell; +use async_stream::stream; +use futures::StreamExt; use hashbrown::HashMap; use orchid_base::interner::Tok; use orchid_base::name::Sym; +use crate::api; +use crate::ctx::Ctx; use crate::expr::Expr; -use crate::parsed::{LazyMemberHandle, ParsedMemberKind, ParsedModule}; +use crate::parsed::{ParsedMemberKind, ParsedModule}; +use crate::system::System; -pub struct Tree(Rc); +pub struct Tree(Rc>); -pub struct WeakTree(Weak); +pub struct WeakTree(Weak>); pub struct Module { pub members: HashMap, Rc>, } impl Module { - async fn from_parsed(parsed: &ParsedModule, root: &ParsedModule) -> Self { - let imports = + pub async fn from_api( + api: api::Module, + consts: &mut HashMap, + sys: System, + path: &mut Vec> + ) -> Self { + let mut members = HashMap::new(); + for mem in api.members { + let (lazy, kind) = match mem.kind { + orchid_api::MemberKind::Lazy(id) => (Some(LazyMemberHandle{ id, sys: sys.clone(), path: })) + } + members.insert(sys.ctx().i.ex(mem.name).await, member); + } + Self { members } } } @@ -57,3 +74,63 @@ impl MemberKind { } } } + +pub struct LazyMemberHandle { + id: api::TreeId, + sys: System, + path: Sym, +} +impl LazyMemberHandle { + pub async fn run(self, consts: &mut HashMap) -> ParsedMemberKind { + match self.sys.get_tree(self.id).await { + api::MemberKind::Const(c) => { + let mut pctx = + ExprParseCtx { ctx: self.sys.ctx().clone(), exprs: self.sys.ext().exprs().clone() }; + consts.insert(self.path, Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await); + ParsedMemberKind::Const + }, + api::MemberKind::Module(m) => ParsedMemberKind::Mod( + ParsedModule::from_api(m, &mut ParsedFromApiCx { + sys: &self.sys, + consts, + path: self.path.tok(), + }) + .await, + ), + api::MemberKind::Lazy(id) => Self { id, ..self }.run(consts).boxed_local().await, + } + } + pub async fn into_member(self, public: bool, name: Tok) -> Member { + Member { + name, + public, + canonical_path: self.path.clone(), + kind: OnceCell::new(), + lazy: Mutex::new(Some(self)), + } + } +} + +// TODO: this one should own but not execute the lazy handle. +// Lazy handles should run +// - in the tree converter function as needed to resolve imports +// - in the tree itself when a constant is loaded +// - when a different lazy subtree references them in a wildcard import and +// forces the enumeration. +// +// do we actually need to allow wildcard imports in lazy trees? maybe a +// different kind of import is sufficient. Source code never becomes a lazy +// tree. What does? +// - Systems subtrees rarely reference each other at all. They can't use macros +// and they usually point to constants with an embedded expr. +// - Compiled libraries on the long run. The code as written may reference +// constants by indirect path. But this is actually the same as the above, +// they also wouldn't use regular imports because they are distributed as +// exprs. +// +// Everything is distributed either as source code or as exprs. Line parsers +// also operate on tokens. +// +// TODO: The trees produced by systems can be safely changed +// to the new kind of tree. This datastructure does not need to support the lazy +// handle.