commit before easter break

orchid-extension/src/tree.rs

@@ -1,34 +1,67 @@
 use std::num::NonZero;
-use std::ops::Range;
 use std::rc::Rc;
 
 use dyn_clone::{DynClone, clone_box};
 use futures::FutureExt;
 use futures::future::{LocalBoxFuture, join_all};
-use hashbrown::HashMap;
+use hashbrown::{HashMap, HashSet};
 use itertools::Itertools;
-use orchid_base::interner::Tok;
+use orchid_base::interner::{Interner, Tok};
+use orchid_base::location::Pos;
 use orchid_base::name::Sym;
 use orchid_base::reqnot::ReqHandlish;
-use orchid_base::tree::{TokTree, Token};
-use ordered_float::NotNan;
+use orchid_base::tree::{TokTree, Token, TokenVariant};
 use substack::Substack;
 use trait_set::trait_set;
 
 use crate::api;
-use crate::atom::{AtomFactory, ForeignAtom};
 use crate::conv::ToExpr;
 use crate::entrypoint::MemberRecord;
+use crate::expr::{Expr, ExprHandle};
 use crate::func_atom::{ExprFunc, Fun};
 use crate::gen_expr::{GExpr, arg, call, lambda, seq};
-use crate::macros::Rule;
 use crate::system::SysCtx;
 
-pub type GenTokTree<'a> = TokTree<'a, ForeignAtom<'a>, AtomFactory>;
-pub type GenTok<'a> = Token<'a, ForeignAtom<'a>, AtomFactory>;
+pub type GenTokTree = TokTree<Expr, GExpr>;
+pub type GenTok = Token<Expr, GExpr>;
 
-pub async fn do_extra(f: &AtomFactory, r: Range<u32>, ctx: SysCtx) -> api::TokenTree {
-	api::TokenTree { range: r, token: api::Token::Atom(f.clone().build(ctx).await) }
+impl TokenVariant<api::Expression> for GExpr {
+	type FromApiCtx<'a> = ();
+	type ToApiCtx<'a> = (SysCtx, &'a dyn ReqHandlish);
+	async fn from_api(
+		_: &api::Expression,
+		_: &mut Self::FromApiCtx<'_>,
+		_: Pos,
+		_: &Interner,
+	) -> Self {
+		panic!("Received new expression from host")
+	}
+	async fn into_api(self, (ctx, hand): &mut Self::ToApiCtx<'_>) -> api::Expression {
+		self.api_return(ctx.clone(), hand).await
+	}
+}
+
+impl TokenVariant<api::ExprTicket> for Expr {
+	type FromApiCtx<'a> = SysCtx;
+	async fn from_api(
+		api: &api::ExprTicket,
+		ctx: &mut Self::FromApiCtx<'_>,
+		_: Pos,
+		_: &Interner,
+	) -> Self {
+		// SAFETY: receiving trees from sublexers implies ownership transfer
+		Expr::from_handle(Rc::new(unsafe { ExprHandle::from_args(ctx.clone(), *api) }))
+	}
+	type ToApiCtx<'a> = ();
+	async fn into_api(self, (): &mut Self::ToApiCtx<'_>) -> api::ExprTicket {
+		let hand = self.handle();
+		std::mem::drop(self);
+		let h = match Rc::try_unwrap(hand) {
+			Ok(h) => h,
+			Err(h) => h.as_ref().clone().await,
+		};
+		h.into_tk()
+	}
 }
 
 fn with_export(mem: GenMember, public: bool) -> Vec<GenItem> {
@@ -47,14 +80,9 @@ impl GenItem {
 		let kind = match self.kind {
 			GenItemKind::Export(n) => api::ItemKind::Export(ctx.sys().i().i::<String>(&n).await.to_api()),
 			GenItemKind::Member(mem) => api::ItemKind::Member(mem.into_api(ctx).await),
-			GenItemKind::Import(cn) => api::ItemKind::Import(cn.tok().to_api()),
-			GenItemKind::Macro(priority, gen_rules) => {
-				let mut rules = Vec::with_capacity(gen_rules.len());
-				for rule in gen_rules {
-					rules.push(rule.into_api(ctx).await)
-				}
-				api::ItemKind::Macro(api::MacroBlock { priority, rules })
-			},
+			GenItemKind::Import(cn) => api::ItemKind::Import(
+				Sym::parse(&cn, ctx.sys().i()).await.expect("Import path empty string").to_api(),
+			),
 		};
 		let comments = join_all(self.comments.iter().map(|c| async {
 			api::Comment {
@@ -70,24 +98,23 @@ impl GenItem {
 pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> Vec<GenItem> {
 	with_export(GenMember { name: name.to_string(), kind: MemKind::Const(value.to_expr()) }, public)
 }
+pub fn import(public: bool, path: &str) -> Vec<GenItem> {
+	let mut out = vec![GenItemKind::Import(path.to_string())];
+	if public {
+		out.push(GenItemKind::Export(path.split("::").last().unwrap().to_string()));
+	}
+	out.into_iter().map(|kind| GenItem { comments: vec![], kind }).collect()
+}
 pub fn module(
 	public: bool,
 	name: &str,
-	imports: impl IntoIterator<Item = Sym>,
 	items: impl IntoIterator<Item = Vec<GenItem>>,
 ) -> Vec<GenItem> {
-	let (name, kind) = root_mod(name, imports, items);
+	let (name, kind) = root_mod(name, items);
 	with_export(GenMember { name, kind }, public)
 }
-pub fn root_mod(
-	name: &str,
-	imports: impl IntoIterator<Item = Sym>,
-	items: impl IntoIterator<Item = Vec<GenItem>>,
-) -> (String, MemKind) {
-	let kind = MemKind::Mod {
-		imports: imports.into_iter().collect(),
-		items: items.into_iter().flatten().collect(),
-	};
+pub fn root_mod(name: &str, items: impl IntoIterator<Item = Vec<GenItem>>) -> (String, MemKind) {
+	let kind = MemKind::Mod { items: items.into_iter().flatten().collect() };
 	(name.to_string(), kind)
 }
 pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<GenItem> {
@@ -107,12 +134,12 @@ pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> Vec<Gen
 	});
 	with_export(GenMember { name: name.to_string(), kind: MemKind::Lazy(fac) }, exported)
 }
-pub fn macro_block(prio: Option<f64>, rules: impl IntoIterator<Item = Rule>) -> Vec<GenItem> {
-	let prio = prio.map(|p| NotNan::new(p).unwrap());
-	vec![GenItem {
-		kind: GenItemKind::Macro(prio, rules.into_iter().collect_vec()),
-		comments: vec![],
-	}]
+pub fn prefix(path: &str, items: impl IntoIterator<Item = Vec<GenItem>>) -> Vec<GenItem> {
+	let mut items = items.into_iter().flatten().collect_vec();
+	for step in path.split("::").collect_vec().into_iter().rev() {
+		items = module(true, step, [items]);
+	}
+	items
 }
 
 pub fn comments<'a>(
@@ -126,6 +153,58 @@ pub fn comments<'a>(
 	val
 }
 
+/// Trivially merge a gen tree. Behaviours were chosen to make this simple.
+///
+/// - Comments on imports are discarded
+/// - Comments on exports and submodules are combined
+/// - Duplicate constants result in an error
+/// - A combination of lazy and anything results in an error
+pub fn merge_trivial(trees: impl IntoIterator<Item = Vec<GenItem>>) -> Vec<GenItem> {
+	let mut imported = HashSet::<String>::new();
+	let mut exported = HashMap::<String, HashSet<String>>::new();
+	let mut members = HashMap::<String, (MemKind, HashSet<String>)>::new();
+	for item in trees.into_iter().flatten() {
+		match item.kind {
+			GenItemKind::Import(sym) => {
+				imported.insert(sym);
+			},
+			GenItemKind::Export(e) =>
+				exported.entry(e.clone()).or_insert(HashSet::new()).extend(item.comments.iter().cloned()),
+			GenItemKind::Member(mem) => match mem.kind {
+				unit @ (MemKind::Const(_) | MemKind::Lazy(_)) => {
+					let prev = members.insert(mem.name.clone(), (unit, item.comments.into_iter().collect()));
+					assert!(prev.is_none(), "Conflict in trivial tree merge on {}", mem.name);
+				},
+				MemKind::Mod { items } => match members.entry(mem.name.clone()) {
+					hashbrown::hash_map::Entry::Vacant(slot) => {
+						slot.insert((MemKind::Mod { items }, item.comments.into_iter().collect()));
+					},
+					hashbrown::hash_map::Entry::Occupied(mut old) => match old.get_mut() {
+						(MemKind::Mod { items: old_items }, old_cmts) => {
+							let mut swap = vec![];
+							std::mem::swap(&mut swap, old_items);
+							*old_items = merge_trivial([swap, items]);
+							old_cmts.extend(item.comments);
+						},
+						_ => panic!("Conflict in trivial merge on {}", mem.name),
+					},
+				},
+			},
+		}
+	}
+
+	(imported.into_iter().map(|txt| GenItem { comments: vec![], kind: GenItemKind::Import(txt) }))
+		.chain(exported.into_iter().map(|(k, cmtv)| GenItem {
+			comments: cmtv.into_iter().collect(),
+			kind: GenItemKind::Export(k),
+		}))
+		.chain(members.into_iter().map(|(name, (kind, cmtv))| GenItem {
+			comments: cmtv.into_iter().collect(),
+			kind: GenItemKind::Member(GenMember { name, kind }),
+		}))
+		.collect()
+}
+
 trait_set! {
 	trait LazyMemberCallback =
 		FnOnce(Sym, SysCtx) -> LocalBoxFuture<'static, MemKind> + DynClone
@@ -144,13 +223,12 @@ impl Clone for LazyMemberFactory {
 pub enum GenItemKind {
 	Member(GenMember),
 	Export(String),
-	Import(Sym),
-	Macro(Option<NotNan<f64>>, Vec<Rule>),
+	Import(String),
 }
 
 pub struct GenMember {
-	name: String,
-	kind: MemKind,
+	pub name: String,
+	pub kind: MemKind,
 }
 impl GenMember {
 	pub async fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Member {
@@ -164,7 +242,7 @@ impl GenMember {
 
 pub enum MemKind {
 	Const(GExpr),
-	Mod { imports: Vec<Sym>, items: Vec<GenItem> },
+	Mod { items: Vec<GenItem> },
 	Lazy(LazyMemberFactory),
 }
 impl MemKind {
@@ -172,15 +250,12 @@ impl MemKind {
 		match self {
 			Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
 			Self::Const(c) => api::MemberKind::Const(c.api_return(ctx.sys(), ctx.req()).await),
-			Self::Mod { imports, items } => {
-				let all_items = (imports.into_iter())
-					.map(|t| GenItem { comments: vec![], kind: GenItemKind::Import(t) })
-					.chain(items);
-				let mut items = Vec::new();
-				for i in all_items {
-					items.push(i.into_api(ctx).boxed_local().await)
+			Self::Mod { items } => {
+				let mut api_items = Vec::new();
+				for i in items {
+					api_items.push(i.into_api(ctx).boxed_local().await)
 				}
-				api::MemberKind::Module(api::Module { items })
+				api::MemberKind::Module(api::Module { items: api_items })
 			},
 		}
 	}
@@ -189,7 +264,6 @@ impl MemKind {
 pub trait TreeIntoApiCtx {
 	fn sys(&self) -> SysCtx;
 	fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
-	fn with_rule(&mut self, rule: Rc<Rule>) -> api::MacroId;
 	fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
 	fn req(&self) -> &impl ReqHandlish;
 }
@@ -199,7 +273,6 @@ pub struct TreeIntoApiCtxImpl<'a, 'b, RH: ReqHandlish> {
 	pub basepath: &'a [Tok<String>],
 	pub path: Substack<'a, Tok<String>>,
 	pub lazy_members: &'b mut HashMap<api::TreeId, MemberRecord>,
-	pub rules: &'b mut HashMap<api::MacroId, Rc<Rule>>,
 	pub req: &'a RH,
 }
 
@@ -209,7 +282,6 @@ impl<RH: ReqHandlish> TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_, RH> {
 		TreeIntoApiCtxImpl {
 			req: self.req,
 			lazy_members: self.lazy_members,
-			rules: self.rules,
 			sys: self.sys.clone(),
 			basepath: self.basepath,
 			path: self.path.push(seg),
@@ -221,10 +293,5 @@ impl<RH: ReqHandlish> TreeIntoApiCtx for TreeIntoApiCtxImpl<'_, '_, RH> {
 		self.lazy_members.insert(id, MemberRecord::Gen(path, fac));
 		id
 	}
-	fn with_rule(&mut self, rule: Rc<Rule>) -> orchid_api::MacroId {
-		let id = api::MacroId(NonZero::new((self.lazy_members.len() + 1) as u64).unwrap());
-		self.rules.insert(id, rule);
-		id
-	}
 	fn req(&self) -> &impl ReqHandlish { self.req }
 }