sync commit

orchid-host/src/extension.rs

@@ -10,7 +10,7 @@ use async_std::sync::Mutex;
 use async_stream::stream;
 use derive_destructure::destructure;
 use futures::future::{join, join_all};
-use futures::{FutureExt, StreamExt, stream, stream_select};
+use futures::{StreamExt, stream};
 use hashbrown::HashMap;
 use itertools::Itertools;
 use orchid_api::{HostMsgSet, LsModule};

orchid-host/src/lex.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::rc::Rc;
 
 use async_std::sync::Mutex;
 use futures::FutureExt;
@@ -107,11 +107,11 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 			));
 		};
 		ctx.set_tail(tail);
-		ParsTok::Comment(Arc::new(cmt.to_string()))
+		ParsTok::Comment(Rc::new(cmt.to_string()))
 	} else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
 		ctx.push_pos(end as u32);
-		ParsTok::Comment(Arc::new(tail[2..end].to_string()))
+		ParsTok::Comment(Rc::new(tail[2..end].to_string()))
 	} else if ctx.strip_char('\\') {
 		let mut arg = Vec::new();
 		ctx.trim_ws();

orchid-host/src/parse.rs

@@ -141,10 +141,10 @@ pub async fn parse_exportable_item<'a>(
 	let path_sym = Sym::new(path.unreverse(), ctx.i()).await.expect("Files should have a namespace");
 	let kind = if discr == ctx.i().i("mod").await {
 		let (name, body) = parse_module(ctx, path, tail).await?;
-		ItemKind::Member(ParsedMember::new(name, path_sym, ParsedMemberKind::Mod(body)))
+		ItemKind::Member(ParsedMember { name, full_name: path_sym, kind: ParsedMemberKind::Mod(body) })
 	} else if discr == ctx.i().i("const").await {
 		let name = parse_const(ctx, tail, path.clone()).await?;
-		ItemKind::Member(ParsedMember::new(name, path_sym, ParsedMemberKind::Const))
+		ItemKind::Member(ParsedMember { name, full_name: path_sym, kind: ParsedMemberKind::Const })
 	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
 		let line = sys.parse(path_sym, tail.to_vec(), exported, comments).await?;
 		return parse_items(ctx, path, Snippet::new(tail.prev(), &line)).await;

orchid-host/src/parsed.rs

@@ -90,36 +90,17 @@ pub enum ItemKind {
 impl ItemKind {
 	pub fn at(self, pos: Pos) -> Item { Item { comments: vec![], pos, kind: self } }
 }
-
-impl Item {
-	pub async fn from_api<'a>(tree: api::Item, ctx: &mut ParsedFromApiCx<'a>) -> Self {
-		let kind = match tree.kind {
-			api::ItemKind::Member(m) => ItemKind::Member(ParsedMember::from_api(m, ctx).await),
-			api::ItemKind::Import(name) => ItemKind::Import(Import {
-				path: Sym::from_api(name, &ctx.sys.ctx().i).await.iter().collect(),
-				name: None,
-			}),
-			api::ItemKind::Export(e) => ItemKind::Export(Tok::from_api(e, &ctx.sys.ctx().i).await),
-		};
-		let mut comments = Vec::new();
-		for comment in tree.comments.iter() {
-			comments.push(Comment::from_api(comment, &ctx.sys.ctx().i).await)
-		}
-		Self { pos: Pos::from_api(&tree.location, &ctx.sys.ctx().i).await, comments, kind }
-	}
-}
 impl Format for Item {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
 		let comment_text = self.comments.iter().join("\n");
 		let item_text = match &self.kind {
 			ItemKind::Import(i) => format!("import {i}").into(),
 			ItemKind::Export(e) => format!("export {e}").into(),
-			ItemKind::Member(mem) => match mem.kind.get() {
-				None => format!("lazy {}", mem.name).into(),
-				Some(ParsedMemberKind::Const) =>
+			ItemKind::Member(mem) => match &mem.kind {
+				ParsedMemberKind::Const =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0}")))
 						.units([mem.name.rc().into()]),
-				Some(ParsedMemberKind::Mod(module)) =>
+				ParsedMemberKind::Mod(module) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("module {0} {{\n\t{1}\n}}")))
 						.units([mem.name.rc().into(), module.print(c).boxed_local().await]),
 			},
@@ -130,71 +111,12 @@ impl Format for Item {
 }
 
 pub struct ParsedMember {
-	name: Tok<String>,
-	full_name: Sym,
-	kind: OnceCell<ParsedMemberKind>,
-	lazy: Mutex<Option<LazyMemberHandle>>,
+	pub name: Tok<String>,
+	pub full_name: Sym,
+	pub kind: ParsedMemberKind,
 }
-// TODO: this one should own but not execute the lazy handle.
-// Lazy handles should run
-// - in the tree converter function as needed to resolve imports
-// - in the tree itself when a constant is loaded
-// - when a different lazy subtree references them in a wildcard import and
-//   forces the enumeration.
-//
-// do we actually need to allow wildcard imports in lazy trees? maybe a
-// different kind of import is sufficient. Source code never becomes a lazy
-// tree. What does?
-// - Systems subtrees rarely reference each other at all. They can't use macros
-//   and they usually point to constants with an embedded expr.
-// - Compiled libraries on the long run. The code as written may reference
-//   constants by indirect path. But this is actually the same as the above,
-//   they also wouldn't use regular imports because they are distributed as
-//   exprs.
-//
-// Everything is distributed either as source code or as exprs. Line parsers
-// also operate on tokens.
-//
-// TODO: The trees produced by systems can be safely changed
-// to the new kind of tree. This datastructure does not need to support the lazy
-// handle.
 impl ParsedMember {
 	pub fn name(&self) -> Tok<String> { self.name.clone() }
-	pub async fn kind(&self, consts: &mut HashMap<Sym, Expr>) -> &ParsedMemberKind {
-		(self.kind.get_or_init(async {
-			let handle = self.lazy.lock().await.take().expect("Neither known nor lazy");
-			handle.run(consts).await
-		}))
-		.await
-	}
-	pub async fn kind_mut(&mut self, consts: &mut HashMap<Sym, Expr>) -> &mut ParsedMemberKind {
-		self.kind(consts).await;
-		self.kind.get_mut().expect("kind() already filled the cell")
-	}
-	pub async fn from_api<'a>(api: api::Member, ctx: &'_ mut ParsedFromApiCx<'a>) -> Self {
-		let name = Tok::from_api(api.name, &ctx.sys.ctx().i).await;
-		let mut ctx: ParsedFromApiCx<'_> = (&mut *ctx).push(name.clone()).await;
-		let path_sym = Sym::from_tok(ctx.path.clone()).expect("We just pushed on to this");
-		let kind = match api.kind {
-			api::MemberKind::Lazy(id) => {
-				let handle = LazyMemberHandle { id, sys: ctx.sys.clone(), path: path_sym.clone() };
-				return handle.into_member(name.clone()).await;
-			},
-			api::MemberKind::Const(c) => {
-				let mut pctx =
-					ExprParseCtx { ctx: ctx.sys.ctx().clone(), exprs: ctx.sys.ext().exprs().clone() };
-				let expr = Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await;
-				ctx.consts.insert(path_sym.clone(), expr);
-				ParsedMemberKind::Const
-			},
-			api::MemberKind::Module(m) =>
-				ParsedMemberKind::Mod(ParsedModule::from_api(m, &mut ctx).await),
-		};
-		ParsedMember { name, full_name: path_sym, kind: OnceCell::from(kind), lazy: Mutex::default() }
-	}
-	pub fn new(name: Tok<String>, full_name: Sym, kind: ParsedMemberKind) -> Self {
-		ParsedMember { name, full_name, kind: OnceCell::from(kind), lazy: Mutex::default() }
-	}
 }
 impl Debug for ParsedMember {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -233,18 +155,10 @@ impl ParsedModule {
 		std::mem::swap(self, &mut swap);
 		*self = ParsedModule::new(swap.items.into_iter().chain(other.items))
 	}
-	pub async fn from_api<'a>(m: api::Module, ctx: &mut ParsedFromApiCx<'a>) -> Self {
-		Self::new(
-			stream! { for item in m.items { yield Item::from_api(item, ctx).boxed_local().await } }
-				.collect::<Vec<_>>()
-				.await,
-		)
-	}
 	pub async fn walk<'a>(
 		&self,
 		allow_private: bool,
 		path: impl IntoIterator<Item = Tok<String>>,
-		consts: &mut HashMap<Sym, Expr>,
 	) -> Result<&ParsedModule, WalkError> {
 		let mut cur = self;
 		for (pos, step) in path.into_iter().enumerate() {
@@ -257,7 +171,7 @@ impl ParsedModule {
 			if !allow_private && !cur.exports.contains(&step) {
 				return Err(WalkError { pos, kind: WalkErrorKind::Private });
 			}
-			match member.kind(consts).await {
+			match &member.kind {
 				ParsedMemberKind::Const => return Err(WalkError { pos, kind: WalkErrorKind::Constant }),
 				ParsedMemberKind::Mod(m) => cur = m,
 			}
@@ -286,41 +200,6 @@ impl Format for ParsedModule {
 	}
 }
 
-pub struct LazyMemberHandle {
-	id: api::TreeId,
-	sys: System,
-	path: Sym,
-}
-impl LazyMemberHandle {
-	pub async fn run(self, consts: &mut HashMap<Sym, Expr>) -> ParsedMemberKind {
-		match self.sys.get_tree(self.id).await {
-			api::MemberKind::Const(c) => {
-				let mut pctx =
-					ExprParseCtx { ctx: self.sys.ctx().clone(), exprs: self.sys.ext().exprs().clone() };
-				consts.insert(self.path, Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await);
-				ParsedMemberKind::Const
-			},
-			api::MemberKind::Module(m) => ParsedMemberKind::Mod(
-				ParsedModule::from_api(m, &mut ParsedFromApiCx {
-					sys: &self.sys,
-					consts,
-					path: self.path.tok(),
-				})
-				.await,
-			),
-			api::MemberKind::Lazy(id) => Self { id, ..self }.run(consts).boxed_local().await,
-		}
-	}
-	pub async fn into_member(self, name: Tok<String>) -> ParsedMember {
-		ParsedMember {
-			name,
-			full_name: self.path.clone(),
-			kind: OnceCell::new(),
-			lazy: Mutex::new(Some(self)),
-		}
-	}
-}
-
 /// TODO:
 ///
 /// idea, does the host need an IR here or can we figure out a way to transcribe
@@ -356,8 +235,7 @@ impl Root {
 			return Ok(val.clone());
 		}
 		let (cn, mp) = name.split_last();
-		let consts_mut = &mut *self.consts.write().await;
-		let module = self.tree.walk(true, mp.iter().cloned(), consts_mut).await.unwrap();
+		let module = self.tree.walk(true, mp.iter().cloned()).await.unwrap();
 		let member = (module.items.iter())
 			.filter_map(|it| if let ItemKind::Member(m) = &it.kind { Some(m) } else { None })
 			.find(|m| m.name() == cn);
@@ -367,14 +245,14 @@ impl Root {
 				format!("{name} does not refer to a constant"),
 				[pos.clone().into()],
 			)),
-			Some(mem) => match mem.kind(consts_mut).await {
+			Some(mem) => match &mem.kind {
 				ParsedMemberKind::Mod(_) => Err(mk_errv(
 					ctx.i.i("module used as constant").await,
 					format!("{name} is a module, not a constant"),
 					[pos.clone().into()],
 				)),
 				ParsedMemberKind::Const => Ok(
-					(consts_mut.get(&name).cloned())
+					(self.consts.read().await.get(&name).cloned())
 						.expect("Tree says the path is correct but no value was found"),
 				),
 			},

orchid-host/src/system.rs

@@ -26,7 +26,8 @@ use crate::api;
 use crate::ctx::Ctx;
 use crate::expr::{Expr, ExprParseCtx};
 use crate::extension::{Extension, WeakExtension};
-use crate::parsed::{ItemKind, ParsedMember, ParsedModule, ParsTokTree, ParsedFromApiCx, Root};
+use crate::parsed::{ItemKind, ParsTokTree, ParsedFromApiCx, ParsedMember, ParsedModule, Root};
+use crate::tree::{Member, Module};
 
 #[derive(destructure)]
 struct SystemInstData {
@@ -137,7 +138,7 @@ impl SystemCtor {
 		&self,
 		depends: impl IntoIterator<Item = &'a System>,
 		consts: &mut HashMap<Sym, Expr>,
-	) -> (ParsedModule, System) {
+	) -> (Module, System) {
 		let depends = depends.into_iter().map(|si| si.id()).collect_vec();
 		debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
 		let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
@@ -152,10 +153,17 @@ impl SystemCtor {
 				.await,
 			id,
 		}));
+		let const_root = Module::from_api((sys_inst.const_root.into_iter()).map(|k, v| api::Member {
+			name: k,
+			kind: v,
+			comments: vec![],
+			exported: true,
+		}))
+		.await;
 		let const_root = clone!(data, ext; stream! {
 			for (k, v) in sys_inst.const_root {
-				yield ParsedMember::from_api(
-					api::Member { name: k, kind: v },
+				yield Member::from_api(
+					,
 					&mut ParsedFromApiCx {
 						consts,
 						path: ext.ctx().i.i(&[]).await,

orchid-host/src/tree.rs

@@ -2,23 +2,40 @@ use std::cell::RefCell;
 use std::rc::{Rc, Weak};
 
 use async_once_cell::OnceCell;
+use async_stream::stream;
+use futures::StreamExt;
 use hashbrown::HashMap;
 use orchid_base::interner::Tok;
 use orchid_base::name::Sym;
 
+use crate::api;
+use crate::ctx::Ctx;
 use crate::expr::Expr;
-use crate::parsed::{LazyMemberHandle, ParsedMemberKind, ParsedModule};
+use crate::parsed::{ParsedMemberKind, ParsedModule};
+use crate::system::System;
 
-pub struct Tree(Rc<Module>);
+pub struct Tree(Rc<RefCell<Module>>);
 
-pub struct WeakTree(Weak<Module>);
+pub struct WeakTree(Weak<RefCell<Module>>);
 
 pub struct Module {
 	pub members: HashMap<Tok<String>, Rc<Member>>,
 }
 impl Module {
-	async fn from_parsed(parsed: &ParsedModule, root: &ParsedModule) -> Self {
-		let imports = 
+	pub async fn from_api(
+		api: api::Module,
+		consts: &mut HashMap<Sym, Expr>,
+		sys: System,
+		path: &mut Vec<Tok<String>>
+	) -> Self {
+		let mut members = HashMap::new();
+		for mem in api.members {
+			let (lazy, kind) = match mem.kind {
+				orchid_api::MemberKind::Lazy(id) => (Some(LazyMemberHandle{ id, sys: sys.clone(), path:  }))
+			}
+			members.insert(sys.ctx().i.ex(mem.name).await, member);
+		}
+		Self { members }
 	}
 }
 
@@ -57,3 +74,63 @@ impl MemberKind {
 		}
 	}
 }
+
+pub struct LazyMemberHandle {
+	id: api::TreeId,
+	sys: System,
+	path: Sym,
+}
+impl LazyMemberHandle {
+	pub async fn run(self, consts: &mut HashMap<Sym, Expr>) -> ParsedMemberKind {
+		match self.sys.get_tree(self.id).await {
+			api::MemberKind::Const(c) => {
+				let mut pctx =
+					ExprParseCtx { ctx: self.sys.ctx().clone(), exprs: self.sys.ext().exprs().clone() };
+				consts.insert(self.path, Expr::from_api(&c, PathSetBuilder::new(), &mut pctx).await);
+				ParsedMemberKind::Const
+			},
+			api::MemberKind::Module(m) => ParsedMemberKind::Mod(
+				ParsedModule::from_api(m, &mut ParsedFromApiCx {
+					sys: &self.sys,
+					consts,
+					path: self.path.tok(),
+				})
+				.await,
+			),
+			api::MemberKind::Lazy(id) => Self { id, ..self }.run(consts).boxed_local().await,
+		}
+	}
+	pub async fn into_member(self, public: bool, name: Tok<String>) -> Member {
+		Member {
+			name,
+			public,
+			canonical_path: self.path.clone(),
+			kind: OnceCell::new(),
+			lazy: Mutex::new(Some(self)),
+		}
+	}
+}
+
+// TODO: this one should own but not execute the lazy handle.
+// Lazy handles should run
+// - in the tree converter function as needed to resolve imports
+// - in the tree itself when a constant is loaded
+// - when a different lazy subtree references them in a wildcard import and
+//   forces the enumeration.
+//
+// do we actually need to allow wildcard imports in lazy trees? maybe a
+// different kind of import is sufficient. Source code never becomes a lazy
+// tree. What does?
+// - Systems subtrees rarely reference each other at all. They can't use macros
+//   and they usually point to constants with an embedded expr.
+// - Compiled libraries on the long run. The code as written may reference
+//   constants by indirect path. But this is actually the same as the above,
+//   they also wouldn't use regular imports because they are distributed as
+//   exprs.
+//
+// Everything is distributed either as source code or as exprs. Line parsers
+// also operate on tokens.
+//
+// TODO: The trees produced by systems can be safely changed
+// to the new kind of tree. This datastructure does not need to support the lazy
+// handle.