sync commit

orchid-api/src/parser.rs

@@ -1,9 +1,10 @@
+use core::ops::Range;
 use std::num::NonZeroU64;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{Comment, HostExtReq, OrcResult, SysId, TStrv, TokenTree};
+use crate::{HostExtReq, OrcResult, SysId, TStr, TStrv, TokenTree};
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ParsId(pub NonZeroU64);
@@ -20,3 +21,9 @@ pub struct ParseLine {
 impl Request for ParseLine {
 	type Response = OrcResult<Vec<TokenTree>>;
 }
+
+#[derive(Clone, Debug, Coding)]
+pub struct Comment {
+	pub text: TStr,
+	pub range: Range<u32>,
+}

orchid-api/src/tree.rs

@@ -1,14 +1,12 @@
 use std::collections::HashMap;
 use std::num::NonZeroU64;
 use std::ops::Range;
-use std::sync::Arc;
+use std::rc::Rc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{
+use crate::{ExprTicket, Expression, ExtHostReq, HostExtReq, OrcError, SysId, TStr, TStrv};
-	ExprTicket, Expression, ExtHostReq, HostExtReq, Location, OrcError, SysId, TStr, TStrv,
-};
 
 /// A token tree from a lexer recursion request. Its lifetime is the lex call,
 /// the lexer can include it in its output or discard it by implication.
@@ -49,7 +47,7 @@ pub enum Token {
 	/// NewExpr(Bottom) because it fails in dead branches too.
 	Bottom(Vec<OrcError>),
 	/// A comment
-	Comment(Arc<String>),
+	Comment(Rc<String>),
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]
@@ -62,42 +60,25 @@ pub enum Paren {
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct TreeId(pub NonZeroU64);
 
-#[derive(Clone, Debug, Coding)]
-pub struct Item {
-	pub location: Location,
-	pub comments: Vec<Comment>,
-	pub kind: ItemKind,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub enum ItemKind {
-	Member(Member),
-	Export(TStr),
-	Import(TStrv),
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct Comment {
-	pub text: TStr,
-	pub location: Location,
-}
-
 #[derive(Clone, Debug, Coding)]
 pub struct Member {
 	pub name: TStr,
+	pub exported: bool,
 	pub kind: MemberKind,
+	pub comments: Vec<TStr>,
 }
 
 #[derive(Clone, Debug, Coding)]
 pub enum MemberKind {
 	Const(Expression),
 	Module(Module),
+	Import(TStrv),
 	Lazy(TreeId),
 }
 
 #[derive(Clone, Debug, Coding)]
 pub struct Module {
-	pub items: Vec<Item>,
+	pub members: Vec<Member>,
 }
 
 /// Evaluate a lazy member. This call will only be issued to each system once.

orchid-base/src/parse.rs

@@ -113,14 +113,23 @@ pub fn strip_fluff<A: ExprRepr, X: ExtraTok>(tt: &TokTree<A, X>) -> Option<TokTr
 #[derive(Clone, Debug)]
 pub struct Comment {
 	pub text: Tok<String>,
-	pub pos: Pos,
+	pub range: Range<u32>,
 }
 impl Comment {
-	pub fn to_api(&self) -> api::Comment {
+	pub async fn from_api(c: &api::Comment, i: &Interner) -> Self {
-		api::Comment { location: self.pos.to_api(), text: self.text.to_api() }
+		Self { text: i.ex(c.text).await, range: c.range.clone() }
 	}
-	pub async fn from_api(api: &api::Comment, i: &Interner) -> Self {
+	pub async fn from_tk(tk: &TokTree<impl ExprRepr, impl ExtraTok>, i: &Interner) -> Option<Self> {
-		Self { pos: Pos::from_api(&api.location, i).await, text: Tok::from_api(api.text, i).await }
+		match &tk.tok {
+			Token::Comment(text) => Some(Self { text: i.i(&**text).await, range: tk.range.clone() }),
+			_ => None,
+		}
+	}
+	pub fn to_tk<R: ExprRepr, X: ExtraTok>(&self) -> TokTree<R, X> {
+		TokTree { tok: Token::Comment(self.text.rc().clone()), range: self.range.clone() }
+	}
+	pub fn to_api(&self) -> api::Comment {
+		api::Comment { range: self.range.clone(), text: self.text.to_api() }
 	}
 }
 
@@ -145,11 +154,7 @@ pub async fn line_items<'a, A: ExprRepr, X: ExtraTok>(
 			Some(i) => {
 				let (cmts, tail) = line.split_at(i);
 				let comments = join_all(comments.drain(..).chain(cmts.cur).map(|t| async {
-					match &t.tok {
+					Comment::from_tk(t, ctx.i()).await.expect("All are comments checked above")
-						Token::Comment(c) =>
-							Comment { text: ctx.i().i(&**c).await, pos: Pos::Range(t.range.clone()) },
-						_ => unreachable!("All are comments checked above"),
-					}
 				}))
 				.await;
 				items.push(Parsed { output: comments, tail });

orchid-base/src/tree.rs

@@ -4,7 +4,6 @@ use std::future::Future;
 use std::marker::PhantomData;
 use std::ops::Range;
 use std::rc::Rc;
-use std::sync::Arc;
 
 use async_stream::stream;
 use futures::future::join_all;
@@ -216,7 +215,7 @@ pub enum Token<H: ExprRepr, X: ExtraTok> {
 	/// Information about the code addressed to the human reader or dev tooling
 	/// It has no effect on the behaviour of the program unless it's explicitly
 	/// read via reflection
-	Comment(Arc<String>),
+	Comment(Rc<String>),
 	/// The part of a lambda between `\` and `.` enclosing the argument. The body
 	/// stretches to the end of the enclosing parens or the end of the const line
 	LambdaHead(Vec<TokTree<H, X>>),

orchid-extension/src/entrypoint.rs

@@ -34,7 +34,7 @@ use crate::fs::VirtFS;
 use crate::lexer::{LexContext, err_cascade, err_not_applicable};
 use crate::system::{SysCtx, atom_by_idx};
 use crate::system_ctor::{CtedObj, DynSystemCtor};
-use crate::tree::{GenItemKind, GenTok, GenTokTree, LazyMemberFactory, TreeIntoApiCtxImpl};
+use crate::tree::{GenTok, GenTokTree, LazyMemberFactory, TreeIntoApiCtxImpl};
 
 pub type ExtReq<'a> = RequestHandle<'a, api::ExtMsgSet>;
 pub type ExtReqNot = ReqNot<api::ExtMsgSet>;
@@ -197,9 +197,6 @@ pub fn extension_init(
 							let lazy_mems = Mutex::new(HashMap::new());
 							let ctx = init_ctx(new_sys.id, cted.clone(), hand.reqnot()).await;
 							let const_root = stream::from_iter(cted.inst().dyn_env())
-								.filter_map(
-									async |i| if let GenItemKind::Member(m) = i.kind { Some(m) } else { None },
-								)
 								.then(|mem| {
 									let (req, lazy_mems) = (&hand, &lazy_mems);
 									clone!(i, ctx; async move {

orchid-extension/src/func_atom.rs

@@ -17,7 +17,7 @@ use orchid_base::format::{FmtCtx, FmtUnit};
 use orchid_base::name::Sym;
 use trait_set::trait_set;
 
-use crate::atom::{Atomic, MethodSetBuilder};
+use crate::atom::Atomic;
 use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use crate::conv::ToExpr;
 use crate::expr::Expr;

orchid-extension/src/system.rs

@@ -23,7 +23,7 @@ use crate::func_atom::Fun;
 use crate::lexer::LexerObj;
 use crate::parser::ParserObj;
 use crate::system_ctor::{CtedObj, SystemCtor};
-use crate::tree::GenItem;
+use crate::tree::GenMember;
 
 /// System as consumed by foreign code
 pub trait SystemCard: Default + Send + Sync + 'static {
@@ -82,7 +82,7 @@ impl<T: SystemCard> DynSystemCard for T {
 
 /// System as defined by author
 pub trait System: Send + Sync + SystemCard + 'static {
-	fn env() -> Vec<GenItem>;
+	fn env() -> Vec<GenMember>;
 	fn vfs() -> DeclFs;
 	fn lexers() -> Vec<LexerObj>;
 	fn parsers() -> Vec<ParserObj>;
@@ -90,7 +90,7 @@ pub trait System: Send + Sync + SystemCard + 'static {
 }
 
 pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
-	fn dyn_env(&self) -> Vec<GenItem>;
+	fn dyn_env(&self) -> Vec<GenMember>;
 	fn dyn_vfs(&self) -> DeclFs;
 	fn dyn_lexers(&self) -> Vec<LexerObj>;
 	fn dyn_parsers(&self) -> Vec<ParserObj>;
@@ -99,7 +99,7 @@ pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
 }
 
 impl<T: System> DynSystem for T {
-	fn dyn_env(&self) -> Vec<GenItem> { Self::env() }
+	fn dyn_env(&self) -> Vec<GenMember> { Self::env() }
 	fn dyn_vfs(&self) -> DeclFs { Self::vfs() }
 	fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
 	fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }

orchid-extension/src/tree.rs

@@ -1,10 +1,11 @@
 use std::num::NonZero;
 use std::rc::Rc;
 
+use async_stream::stream;
 use dyn_clone::{DynClone, clone_box};
-use futures::FutureExt;
 use futures::future::{LocalBoxFuture, join_all};
-use hashbrown::{HashMap, HashSet};
+use futures::{FutureExt, StreamExt};
+use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_base::interner::{Interner, Tok};
 use orchid_base::location::Pos;
@@ -64,60 +65,27 @@ impl TokenVariant<api::ExprTicket> for Expr {
 	}
 }
 
-fn with_export(mem: GenMember, public: bool) -> Vec<GenItem> {
+pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenMember> {
-	(public.then(|| GenItemKind::Export(mem.name.clone())).into_iter())
+	vec![GenMember {
-		.chain([GenItemKind::Member(mem)])
+		name: name.to_string(),
-		.map(|kind| GenItem { comments: vec![], kind })
+		kind: MemKind::Const(value.to_expr()),
-		.collect()
+		comments: vec![],
-}
+		public,
-
+	}]
-pub struct GenItem {
-	pub kind: GenItemKind,
-	pub comments: Vec<String>,
-}
-impl GenItem {
-	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Item {
-		let kind = match self.kind {
-			GenItemKind::Export(n) => api::ItemKind::Export(ctx.sys().i().i::<String>(&n).await.to_api()),
-			GenItemKind::Member(mem) => api::ItemKind::Member(mem.into_api(ctx).await),
-			GenItemKind::Import(cn) => api::ItemKind::Import(
-				Sym::parse(&cn, ctx.sys().i()).await.expect("Import path empty string").to_api(),
-			),
-		};
-		let comments = join_all(self.comments.iter().map(|c| async {
-			api::Comment {
-				location: api::Location::Inherit,
-				text: ctx.sys().i().i::<String>(c).await.to_api(),
-			}
-		}))
-		.await;
-		api::Item { location: api::Location::Inherit, comments, kind }
-	}
-}
-
-pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenItem> {
-	with_export(GenMember { name: name.to_string(), kind: MemKind::Const(value.to_expr()) }, public)
-}
-pub fn import(public: bool, path: &str) -> Vec<GenItem> {
-	let mut out = vec![GenItemKind::Import(path.to_string())];
-	if public {
-		out.push(GenItemKind::Export(path.split("::").last().unwrap().to_string()));
-	}
-	out.into_iter().map(|kind| GenItem { comments: vec![], kind }).collect()
 }
 pub fn module(
 	public: bool,
 	name: &str,
-	items: impl IntoIterator<Item = Vec<GenItem>>,
+	mems: impl IntoIterator<Item = Vec<GenMember>>,
-) -> Vec<GenItem> {
+) -> Vec<GenMember> {
-	let (name, kind) = root_mod(name, items);
+	let (name, kind) = root_mod(name, mems);
-	with_export(GenMember { name, kind }, public)
+	vec![GenMember { name, kind, public, comments: vec![] }]
 }
-pub fn root_mod(name: &str, items: impl IntoIterator<Item = Vec<GenItem>>) -> (String, MemKind) {
+pub fn root_mod(name: &str, mems: impl IntoIterator<Item = Vec<GenMember>>) -> (String, MemKind) {
-	let kind = MemKind::Mod { items: items.into_iter().flatten().collect() };
+	let kind = MemKind::Mod { members: mems.into_iter().flatten().collect() };
 	(name.to_string(), kind)
 }
-pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenItem> {
+pub fn fun<I, O>(public: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenMember> {
 	let fac = LazyMemberFactory::new(move |sym, ctx| async {
 		return MemKind::Const(build_lambdas(Fun::new(sym, ctx, xf).await, 0));
 		fn build_lambdas(fun: Fun, i: u64) -> GExpr {
@@ -132,9 +100,9 @@ pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<Gen
 			)
 		}
 	});
-	with_export(GenMember { name: name.to_string(), kind: MemKind::Lazy(fac) }, exported)
+	vec![GenMember { name: name.to_string(), kind: MemKind::Lazy(fac), public, comments: vec![] }]
 }
-pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenItem>>) -> Vec<GenItem> {
+pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
 	let mut items = items.into_iter().flatten().collect_vec();
 	for step in path.split("::").collect_vec().into_iter().rev() {
 		items = module(true, step, [items]);
@@ -144,8 +112,8 @@ pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenItem>>) -> Vec<
 
 pub fn comments<'a>(
 	cmts: impl IntoIterator<Item = &'a str>,
-	mut val: Vec<GenItem>,
+	mut val: Vec<GenMember>,
-) -> Vec<GenItem> {
+) -> Vec<GenMember> {
 	let cmts = cmts.into_iter().map(|c| c.to_string()).collect_vec();
 	for v in val.iter_mut() {
 		v.comments.extend(cmts.iter().cloned());
@@ -159,50 +127,35 @@ pub fn comments<'a>(
 /// - Comments on exports and submodules are combined
 /// - Duplicate constants result in an error
 /// - A combination of lazy and anything results in an error
-pub fn merge_trivial(trees: impl IntoIterator<Item = Vec<GenItem>>) -> Vec<GenItem> {
+pub fn merge_trivial(trees: impl IntoIterator<Item = Vec<GenMember>>) -> Vec<GenMember> {
-	let mut imported = HashSet::<String>::new();
+	let mut all_members = HashMap::<String, (MemKind, Vec<String>)>::new();
-	let mut exported = HashMap::<String, HashSet<String>>::new();
+	for mem in trees.into_iter().flatten() {
-	let mut members = HashMap::<String, (MemKind, HashSet<String>)>::new();
+		assert!(mem.public, "Non-trivial merge in {}", mem.name);
-	for item in trees.into_iter().flatten() {
+		match mem.kind {
-		match item.kind {
-			GenItemKind::Import(sym) => {
-				imported.insert(sym);
-			},
-			GenItemKind::Export(e) =>
-				exported.entry(e.clone()).or_insert(HashSet::new()).extend(item.comments.iter().cloned()),
-			GenItemKind::Member(mem) => match mem.kind {
 			unit @ (MemKind::Const(_) | MemKind::Lazy(_)) => {
-					let prev = members.insert(mem.name.clone(), (unit, item.comments.into_iter().collect()));
+				let prev = all_members.insert(mem.name.clone(), (unit, mem.comments.into_iter().collect()));
 				assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name);
 			},
-				MemKind::Mod { items } => match members.entry(mem.name.clone()) {
+			MemKind::Mod { members } => match all_members.entry(mem.name.clone()) {
 				hashbrown::hash_map::Entry::Vacant(slot) => {
-						slot.insert((MemKind::Mod { items }, item.comments.into_iter().collect()));
+					slot.insert((MemKind::Mod { members }, mem.comments.into_iter().collect()));
 				},
 				hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() {
-						(MemKind::Mod { items: old_items }, old_cmts) => {
+					(MemKind::Mod { members: old_items, .. }, old_cmts) => {
 						let mut swap = vec![];
 						std::mem::swap(&mut swap, old_items);
-							*old_items = merge_trivial([swap, items]);
+						*old_items = merge_trivial([swap, members]);
-							old_cmts.extend(item.comments);
+						old_cmts.extend(mem.comments);
-						},
-						_ => panic!("Conflict in trivial merge on {}", mem.name),
 					},
+					_ => panic!("non-trivial merge on {}", mem.name),
 				},
 			},
 		}
 	}
 
-	(imported.into_iter().map(|txt| GenItem { comments: vec![], kind: GenItemKind::Import(txt) }))
+	(all_members.into_iter())
-		.chain(exported.into_iter().map(|(k, cmtv)| GenItem {
+		.map(|(name, (kind, comments))| GenMember { comments, kind, name, public: true })
-			comments: cmtv.into_iter().collect(),
+		.collect_vec()
-			kind: GenItemKind::Export(k),
-		}))
-		.chain(members.into_iter().map(|(name, (kind, cmtv))| GenItem {
-			comments: cmtv.into_iter().collect(),
-			kind: GenItemKind::Member(GenMember { name, kind }),
-		}))
-		.collect()
 }
 
 trait_set! {
@@ -220,29 +173,25 @@ impl Clone for LazyMemberFactory {
 	fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
 }
 
-pub enum GenItemKind {
-	Member(GenMember),
-	Export(String),
-	Import(String),
-}
-
 pub struct GenMember {
 	pub name: String,
 	pub kind: MemKind,
+	pub public: bool,
+	pub comments: Vec<String>,
 }
 impl GenMember {
 	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Member {
 		let name = ctx.sys().i().i::<String>(&self.name).await;
-		api::Member {
+		let kind = self.kind.into_api(&mut ctx.push_path(name.clone())).await;
-			kind: self.kind.into_api(&mut ctx.push_path(name.clone())).await,
+		let comments =
-			name: name.to_api(),
+			join_all(self.comments.iter().map(|cmt| async { ctx.sys().i().i(cmt).await.to_api() })).await;
-		}
+		api::Member { kind, name: name.to_api(), comments, exported: self.public }
 	}
 }
 
 pub enum MemKind {
 	Const(GExpr),
-	Mod { items: Vec<GenItem> },
+	Mod { members: Vec<GenMember> },
 	Lazy(LazyMemberFactory),
 }
 impl MemKind {
@@ -250,13 +199,10 @@ impl MemKind {
 		match self {
 			Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
 			Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys(), ctx.req()).await),
-			Self::Mod { items } => {
+			Self::Mod { members } => api::MemberKind::Module(api::Module {
-				let mut api_items = Vec::new();
+				members: Box::pin(stream! { for m in members { yield m.into_api(ctx).await } }.collect())
-				for i in items {
+					.await,
-					api_items.push(i.into_api(ctx).boxed_local().await)
+			}),
-				}
-				api::MemberKind::Module(api::Module { items: api_items })
-			},
 		}
 	}
 }

orchid-host/src/extension.rs

@@ -10,7 +10,7 @@ use async_std::sync::Mutex;
 use async_stream::stream;
 use derive_destructure::destructure;
 use futures::future::{join, join_all};
-use futures::{FutureExt, StreamExt, stream, stream_select};
+use futures::{StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_api::{HostMsgSet, LsModule};

orchid-host/src/lex.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::rc::Rc;
 
 use async_std::sync::Mutex;
 use futures::FutureExt;
@@ -107,11 +107,11 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 			));
 		};
 		ctx.set_tail(tail);
-		ParsTok::Comment(Arc::new(cmt.to_string()))
+		ParsTok::Comment(Rc::new(cmt.to_string()))
 	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
 		ctx.push_pos(end as u32);
-		ParsTok::Comment(Arc::new(tail[2..end].to_string()))
+		ParsTok::Comment(Rc::new(tail[2..end].to_string()))
 	} else if ctx.strip_char('\\') {
 		let mut arg = Vec::new();
 		ctx.trim_ws();

orchid-host/src/parse.rs

@@ -141,10 +141,10 @@ pub async fn parse_exportable_item<'a>(
 	let path_sym = Sym::new(path.unreverse(), ctx.i()).await.expect("Files should have a namespace");
 	let kind = if discr == ctx.i().i("mod").await {
 		let (name, body) = parse_module(ctx, path, tail).await?;
-		ItemKind::Member(ParsedMember::new(name, path_sym, ParsedMemberKind::Mod(body)))
+		ItemKind::Member(ParsedMember { name, full_name: path_sym, kind: ParsedMemberKind::Mod(body) })
 	} else if discr == ctx.i().i("const").await {
 		let name = parse_const(ctx, tail, path.clone()).await?;
-		ItemKind::Member(ParsedMember::new(name, path_sym, ParsedMemberKind::Const))
+		ItemKind::Member(ParsedMember { name, full_name: path_sym, kind: ParsedMemberKind::Const })
 	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
 		let line = sys.parse(path_sym, tail.to_vec(), exported, comments).await?;
 		return parse_items(ctx, path, Snippet::new(tail.prev(), &line)).await;

orchid-host/src/parsed.rs

@@ -90,36 +90,17 @@ pub enum ItemKind {
 impl ItemKind {
 	pub fn at(self, pos: Pos) -> Item { Item { comments: vec![], pos, kind: self } }
 }
-
-impl Item {
-	pub async fn from_api<'a>(tree: api::Item, ctx: &mut ParsedFromApiCx<'a>) -> Self {
-		let kind = match tree.kind {
-			api::ItemKind::Member(m) => ItemKind::Member(ParsedMember::from_api(m, ctx).await),
-			api::ItemKind::Import(name) => ItemKind::Import(Import {
-				path: Sym::from_api(name, &ctx.sys.ctx().i).await.iter().collect(),
-				name: None,
-			}),
-			api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &ctx.sys.ctx().i).await),
-		};
-		let mut comments = Vec::new();
-		for comment in tree.comments.iter() {
-			comments.push(Comment::from_api(comment, &ctx.sys.ctx().i).await)
-		}
-		Self { pos: Pos::from_api(&tree.location, &ctx.sys.ctx().i).await, comments, kind }
-	}
-}
 impl Format for Item {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		let comment_text = self.comments.iter().join("\n");
 		let item_text = match &self.kind {
 			ItemKind::Import(i) => format!("import {i}").into(),
 			ItemKind::Export(e) => format!("export {e}").into(),
-			ItemKind::Member(mem) => match mem.kind.get() {
+			ItemKind::Member(mem) => match &mem.kind {
-				None => format!("lazy {}", mem.name).into(),
+				ParsedMemberKind::Const =>
-				Some(ParsedMemberKind::Const) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0}")))
 						.units([mem.name.rc().into()]),
-				Some(ParsedMemberKind::Mod(module)) =>
+				ParsedMemberKind::Mod(module) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
 						.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
 			},
@@ -130,71 +111,12 @@ impl Format for Item {
 }
 
 pub struct ParsedMember {
-	name: Tok<String>,
+	pub name: Tok<String>,
-	full_name: Sym,
+	pub full_name: Sym,
-	kind: OnceCell<ParsedMemberKind>,
+	pub kind: ParsedMemberKind,
-	lazy: Mutex<Option<LazyMemberHandle>>,
 }
-// TODO: this one should own but not execute the lazy handle.
-// Lazy handles should run
-// - in the tree converter function as needed to resolve imports
-// - in the tree itself when a constant is loaded
-// - when a different lazy subtree references them in a wildcard import and
-//   forces the enumeration.
-//
-// do we actually need to allow wildcard imports in lazy trees? maybe a
-// different kind of import is sufficient. Source code never becomes a lazy
-// tree. What does?
-// - Systems subtrees rarely reference each other at all. They can't use macros
-//   and they usually point to constants with an embedded expr.
-// - Compiled libraries on the long run. The code as written may reference
-//   constants by indirect path. But this is actually the same as the above,
-//   they also wouldn't use regular imports because they are distributed as
-//   exprs.
-//
-// Everything is distributed either as source code or as exprs. Line parsers
-// also operate on tokens.
-//
-// TODO: The trees produced by systems can be safely changed
-// to the new kind of tree. This datastructure does not need to support the lazy
-// handle.
 impl ParsedMember {
 	pub fn name(&self) -> Tok<String> { self.name.clone() }
-	pub async fn kind(&self, consts: &mut HashMap<Sym, Expr>) -> &ParsedMemberKind {
-		(self.kind.get_or_init(async {
-			let handle = self.lazy.lock().await.take().expect("Neither known nor lazy");
-			handle.run(consts).await
-		}))
-		.await
-	}
-	pub async fn kind_mut(&mut self, consts: &mut HashMap<Sym, Expr>) -> &mut ParsedMemberKind {
-		self.kind(consts).await;
-		self.kind.get_mut().expect("kind() already filled the cell")
-	}
-	pub async fn from_api<'a>(api: api::Member, ctx: &'_ mut ParsedFromApiCx<'a>) -> Self {
-		let name = Tok::from_api(api.name, &ctx.sys.ctx().i).await;
-		let mut ctx: ParsedFromApiCx<'_> = (&mut *ctx).push(name.clone()).await;
-		let path_sym = Sym::from_tok(ctx.path.clone()).expect("We just pushed on to this");
-		let kind = match api.kind {
-			api::MemberKind::Lazy(id) => {
-				let handle = LazyMemberHandle { id, sys: ctx.sys.clone(), path: path_sym.clone() };
-				return handle.into_member(name.clone()).await;
-			},
-			api::MemberKind::Const(c) => {
-				let mut pctx =
-					ExprParseCtx { ctx: ctx.sys.ctx().clone(), exprs: ctx.sys.ext().exprs().clone() };
-				let expr = Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await;
-				ctx.consts.insert(path_sym.clone(), expr);
-				ParsedMemberKind::Const
-			},
-			api::MemberKind::Module(m) =>
-				ParsedMemberKind::Mod(ParsedModule::from_api(m, &mut ctx).await),
-		};
-		ParsedMember { name, full_name: path_sym, kind: OnceCell::from(kind), lazy: Mutex::default() }
-	}
-	pub fn new(name: Tok<String>, full_name: Sym, kind: ParsedMemberKind) -> Self {
-		ParsedMember { name, full_name, kind: OnceCell::from(kind), lazy: Mutex::default() }
-	}
 }
 impl Debug for ParsedMember {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -233,18 +155,10 @@ impl ParsedModule {
 		std::mem::swap(self, &mut swap);
 		*self = ParsedModule::new(swap.items.into_iter().chain(other.items))
 	}
-	pub async fn from_api<'a>(m: api::Module, ctx: &mut ParsedFromApiCx<'a>) -> Self {
-		Self::new(
-			stream! { for item in m.items { yield Item::from_api(item, ctx).boxed_local().await } }
-				.collect::<Vec<_>>()
-				.await,
-		)
-	}
 	pub async fn walk<'a>(
 		&self,
 		allow_private: bool,
 		path: impl IntoIterator<Item = Tok<String>>,
-		consts: &mut HashMap<Sym, Expr>,
 	) -> Result<&ParsedModule, WalkError> {
 		let mut cur = self;
 		for (pos, step) in path.into_iter().enumerate() {
@@ -257,7 +171,7 @@ impl ParsedModule {
 			if !allow_private && !cur.exports.contains(&step) {
 				return Err(WalkError { pos, kind: WalkErrorKind::Private });
 			}
-			match member.kind(consts).await {
+			match &member.kind {
 				ParsedMemberKind::Const => return Err(WalkError { pos, kind: WalkErrorKind::Constant }),
 				ParsedMemberKind::Mod(m) => cur = m,
 			}
@@ -286,41 +200,6 @@ impl Format for ParsedModule {
 	}
 }
 
-pub struct LazyMemberHandle {
-	id: api::TreeId,
-	sys: System,
-	path: Sym,
-}
-impl LazyMemberHandle {
-	pub async fn run(self, consts: &mut HashMap<Sym, Expr>) -> ParsedMemberKind {
-		match self.sys.get_tree(self.id).await {
-			api::MemberKind::Const(c) => {
-				let mut pctx =
-					ExprParseCtx { ctx: self.sys.ctx().clone(), exprs: self.sys.ext().exprs().clone() };
-				consts.insert(self.path, Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await);
-				ParsedMemberKind::Const
-			},
-			api::MemberKind::Module(m) => ParsedMemberKind::Mod(
-				ParsedModule::from_api(m, &mut ParsedFromApiCx {
-					sys: &self.sys,
-					consts,
-					path: self.path.tok(),
-				})
-				.await,
-			),
-			api::MemberKind::Lazy(id) => Self { id, ..self }.run(consts).boxed_local().await,
-		}
-	}
-	pub async fn into_member(self, name: Tok<String>) -> ParsedMember {
-		ParsedMember {
-			name,
-			full_name: self.path.clone(),
-			kind: OnceCell::new(),
-			lazy: Mutex::new(Some(self)),
-		}
-	}
-}
-
 /// TODO:
 ///
 /// idea, does the host need an IR here or can we figure out a way to transcribe
@@ -356,8 +235,7 @@ impl Root {
 			return Ok(val.clone());
 		}
 		let (cn, mp) = name.split_last();
-		let consts_mut = &mut *self.consts.write().await;
+		let module = self.tree.walk(true, mp.iter().cloned()).await.unwrap();
-		let module = self.tree.walk(true, mp.iter().cloned(), consts_mut).await.unwrap();
 		let member = (module.items.iter())
 			.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
 			.find(|m| m.name() == cn);
@@ -367,14 +245,14 @@ impl Root {
 				format!("{name} does not refer to a constant"),
 				[pos.clone().into()],
 			)),
-			Some(mem) => match mem.kind(consts_mut).await {
+			Some(mem) => match &mem.kind {
 				ParsedMemberKind::Mod(_) => Err(mk_errv(
 					ctx.i.i("module used as constant").await,
 					format!("{name} is a module, not a constant"),
 					[pos.clone().into()],
 				)),
 				ParsedMemberKind::Const => Ok(
-					(consts_mut.get(&name).cloned())
+					(self.consts.read().await.get(&name).cloned())
 						.expect("Tree says the path is correct but no value was found"),
 				),
 			},

orchid-host/src/system.rs

@@ -26,7 +26,8 @@ use crate::api;
 use crate::ctx::Ctx;
 use crate::expr::{Expr, ExprParseCtx};
 use crate::extension::{Extension, WeakExtension};
-use crate::parsed::{ItemKind, ParsedMember, ParsedModule, ParsTokTree, ParsedFromApiCx, Root};
+use crate::parsed::{ItemKind, ParsTokTree, ParsedFromApiCx, ParsedMember, ParsedModule, Root};
+use crate::tree::{Member, Module};
 
 #[derive(destructure)]
 struct SystemInstData {
@@ -137,7 +138,7 @@ impl SystemCtor {
 		&self,
 		depends: impl IntoIterator<Item = &'a System>,
 		consts: &mut HashMap<Sym, Expr>,
-	) -> (ParsedModule, System) {
+	) -> (Module, System) {
 		let depends = depends.into_iter().map(|si| si.id()).collect_vec();
 		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
 		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
@@ -152,10 +153,17 @@ impl SystemCtor {
 				.await,
 			id,
 		}));
+		let const_root = Module::from_api((sys_inst.const_root.into_iter()).map(|k, v| api::Member {
+			name: k,
+			kind: v,
+			comments: vec![],
+			exported: true,
+		}))
+		.await;
 		let const_root = clone!(data, ext; stream! {
 			for (k, v) in sys_inst.const_root {
-				yield ParsedMember::from_api(
+				yield Member::from_api(
-					api::Member { name: k, kind: v },
+					,
 					&mut ParsedFromApiCx {
 						consts,
 						path: ext.ctx().i.i(&[]).await,

orchid-host/src/tree.rs

@@ -2,23 +2,40 @@ use std::cell::RefCell;
 use std::rc::{Rc, Weak};
 
 use async_once_cell::OnceCell;
+use async_stream::stream;
+use futures::StreamExt;
 use hashbrown::HashMap;
 use orchid_base::interner::Tok;
 use orchid_base::name::Sym;
 
+use crate::api;
+use crate::ctx::Ctx;
 use crate::expr::Expr;
-use crate::parsed::{LazyMemberHandle, ParsedMemberKind, ParsedModule};
+use crate::parsed::{ParsedMemberKind, ParsedModule};
+use crate::system::System;
 
-pub struct Tree(Rc<Module>);
+pub struct Tree(Rc<RefCell<Module>>);
 
-pub struct WeakTree(Weak<Module>);
+pub struct WeakTree(Weak<RefCell<Module>>);
 
 pub struct Module {
 	pub members: HashMap<Tok<String>, Rc<Member>>,
 }
 impl Module {
-	async fn from_parsed(parsed: &ParsedModule, root: &ParsedModule) -> Self {
+	pub async fn from_api(
-		let imports = 
+		api: api::Module,
+		consts: &mut HashMap<Sym, Expr>,
+		sys: System,
+		path: &mut Vec<Tok<String>>
+	) -> Self {
+		let mut members = HashMap::new();
+		for mem in api.members {
+			let (lazy, kind) = match mem.kind {
+				orchid_api::MemberKind::Lazy(id) => (Some(LazyMemberHandle{ id, sys: sys.clone(), path:  }))
+			}
+			members.insert(sys.ctx().i.ex(mem.name).await, member);
+		}
+		Self { members }
 	}
 }
 
@@ -57,3 +74,63 @@ impl MemberKind {
 		}
 	}
 }
+
+pub struct LazyMemberHandle {
+	id: api::TreeId,
+	sys: System,
+	path: Sym,
+}
+impl LazyMemberHandle {
+	pub async fn run(self, consts: &mut HashMap<Sym, Expr>) -> ParsedMemberKind {
+		match self.sys.get_tree(self.id).await {
+			api::MemberKind::Const(c) => {
+				let mut pctx =
+					ExprParseCtx { ctx: self.sys.ctx().clone(), exprs: self.sys.ext().exprs().clone() };
+				consts.insert(self.path, Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await);
+				ParsedMemberKind::Const
+			},
+			api::MemberKind::Module(m) => ParsedMemberKind::Mod(
+				ParsedModule::from_api(m, &mut ParsedFromApiCx {
+					sys: &self.sys,
+					consts,
+					path: self.path.tok(),
+				})
+				.await,
+			),
+			api::MemberKind::Lazy(id) => Self { id, ..self }.run(consts).boxed_local().await,
+		}
+	}
+	pub async fn into_member(self, public: bool, name: Tok<String>) -> Member {
+		Member {
+			name,
+			public,
+			canonical_path: self.path.clone(),
+			kind: OnceCell::new(),
+			lazy: Mutex::new(Some(self)),
+		}
+	}
+}
+
+// TODO: this one should own but not execute the lazy handle.
+// Lazy handles should run
+// - in the tree converter function as needed to resolve imports
+// - in the tree itself when a constant is loaded
+// - when a different lazy subtree references them in a wildcard import and
+//   forces the enumeration.
+//
+// do we actually need to allow wildcard imports in lazy trees? maybe a
+// different kind of import is sufficient. Source code never becomes a lazy
+// tree. What does?
+// - Systems subtrees rarely reference each other at all. They can't use macros
+//   and they usually point to constants with an embedded expr.
+// - Compiled libraries on the long run. The code as written may reference
+//   constants by indirect path. But this is actually the same as the above,
+//   they also wouldn't use regular imports because they are distributed as
+//   exprs.
+//
+// Everything is distributed either as source code or as exprs. Line parsers
+// also operate on tokens.
+//
+// TODO: The trees produced by systems can be safely changed
+// to the new kind of tree. This datastructure does not need to support the lazy
+// handle.