Various progress, doesnt compile

Added prelude, made lambdas a single-token prefix like NS, made progress on implementations, removed const line type

orchid-host/src/extension.rs

@@ -293,6 +293,7 @@ impl Extension {
 	pub fn downgrade(&self) -> WeakExtension { WeakExtension(Rc::downgrade(&self.0)) }
 }
 
+#[derive(Clone)]
 pub struct WeakExtension(Weak<ExtensionData>);
 impl WeakExtension {
 	#[must_use]

orchid-host/src/lex.rs

@@ -125,21 +125,12 @@ pub async fn lex_once(ctx: &mut LexCtx<'_>) -> OrcRes<ParsTokTree> {
 		let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
 		ctx.push_pos(end as u32);
 		ParsTok::Comment(Rc::new(tail[2..end].to_string()))
-	} else if ctx.strip_char('\\') {
-		let mut arg = Vec::new();
+	} else if let Some(tail) = ctx.tail.strip_prefix('\\').filter(|t| t.starts_with(name_start)) {
+		// fanciness like \$placeh in templates is resolved in the macro engine.
+		ctx.set_tail(tail);
+		let arg = lex_once(ctx).boxed_local().await?;
 		ctx.trim_ws();
-		while !ctx.strip_char('.') {
-			if ctx.tail.is_empty() {
-				return Err(mk_errv(
-					ctx.ctx.i.i("Unclosed lambda").await,
-					"Lambdae started with \\ should separate arguments from body with .",
-					[SrcRange::new(start..start + 1, ctx.path)],
-				));
-			}
-			arg.push(lex_once(ctx).boxed_local().await?);
-			ctx.trim_ws();
-		}
-		ParsTok::LambdaHead(arg)
+		ParsTok::LambdaHead(Box::new(arg))
 	} else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
 		let mut body = Vec::new();
 		ctx.trim_ws();

orchid-host/src/parse.rs

@@ -1,10 +1,9 @@
-use futures::FutureExt;
 use futures::future::join_all;
 use itertools::Itertools;
 use orchid_base::error::{OrcRes, Reporter, mk_errv};
 use orchid_base::format::fmt;
 use orchid_base::interner::{Interner, Tok};
-use orchid_base::name::{Sym, VPath};
+use orchid_base::name::Sym;
 use orchid_base::parse::{
 	Comment, Import, ParseCtx, Parsed, Snippet, expect_end, line_items, parse_multiname,
 	try_pop_no_fluff,
@@ -13,7 +12,7 @@ use orchid_base::tree::{Paren, TokTree, Token};
 use substack::Substack;
 
 use crate::ctx::Ctx;
-use crate::expr::{Expr, ExprKind, PathSetBuilder};
+use crate::expr::Expr;
 use crate::parsed::{Item, ItemKind, ParsedMember, ParsedMemberKind, ParsedModule};
 use crate::system::System;
 
@@ -110,9 +109,6 @@ pub async fn parse_exportable_item<'a>(
 	let kind = if discr == ctx.i().i("mod").await {
 		let (name, body) = parse_module(ctx, path, tail).await?;
 		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::Mod(body) })
-	} else if discr == ctx.i().i("const").await {
-		let (name, expr) = parse_const(ctx, tail, path.clone()).await?;
-		ItemKind::Member(ParsedMember { name, exported, kind: ParsedMemberKind::ParsedConst(expr) })
 	} else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
 		return sys
 			.parse(path, tail.to_vec(), exported, comments, &mut async |stack, lines| {
@@ -156,107 +152,5 @@ pub async fn parse_module<'a>(
 		));
 	};
 	let path = path.push(name.clone());
-	Ok((name, ParsedModule::new(parse_items(ctx, path, body).await?)))
-}
-
-pub async fn parse_const<'a>(
-	ctx: &impl HostParseCtx,
-	tail: ParsSnippet<'a>,
-	path: Substack<'_, Tok<String>>,
-) -> OrcRes<(Tok<String>, Expr)> {
-	let Parsed { output, tail } = try_pop_no_fluff(ctx, tail).await?;
-	let Some(name) = output.as_name() else {
-		return Err(mk_errv(
-			ctx.i().i("Missing module name").await,
-			format!("A name was expected, {} was found", fmt(output, ctx.i()).await),
-			[output.sr()],
-		));
-	};
-	let Parsed { output, tail } = try_pop_no_fluff(ctx, tail).await?;
-	if !output.is_kw(ctx.i().i("=").await) {
-		return Err(mk_errv(
-			ctx.i().i("Missing = separator").await,
-			format!("Expected = , found {}", fmt(output, ctx.i()).await),
-			[output.sr()],
-		));
-	}
-	try_pop_no_fluff(ctx, tail).await?;
-	// ctx.save_const(path, tail[..].to_vec()).await;
-	let final_path =
-		VPath::new(path.unreverse()).name_with_suffix(name.clone()).to_sym(ctx.i()).await;
-	let val = parse_expr(ctx, final_path, PathSetBuilder::new(), tail).await?;
-	Ok((name, val))
-}
-
-pub async fn parse_expr(
-	ctx: &impl HostParseCtx,
-	path: Sym,
-	psb: PathSetBuilder<'_, Tok<String>>,
-	tail: ParsSnippet<'_>,
-) -> OrcRes<Expr> {
-	let Some((last_idx, _)) = (tail.iter().enumerate().find(|(_, tt)| tt.as_lambda().is_some()))
-		.or_else(|| tail.iter().enumerate().rev().find(|(_, tt)| !tt.is_fluff()))
-	else {
-		return Err(mk_errv(ctx.i().i("Empty expression").await, "Expression ends abruptly here", [
-			tail.sr(),
-		]));
-	};
-	let (function, value) = tail.split_at(last_idx as u32);
-	let pos = tail.sr().pos();
-	if !function.iter().all(TokTree::is_fluff) {
-		let (f_psb, x_psb) = psb.split();
-		let x_expr = parse_expr(ctx, path.clone(), x_psb, value).boxed_local().await?;
-		let f_expr = parse_expr(ctx, path, f_psb, function).boxed_local().await?;
-		return Ok(ExprKind::Call(f_expr, x_expr).at(pos));
-	}
-	let Parsed { output: head, tail } = try_pop_no_fluff(ctx, value).await?;
-	match &head.tok {
-		Token::BR | Token::Comment(_) => panic!("Fluff skipped"),
-		Token::Bottom(b) => Ok(ExprKind::Bottom(b.clone()).at(pos.clone())),
-		Token::Handle(expr) => Ok(expr.clone()),
-		Token::NS(n, nametail) => {
-			let mut nametail = nametail;
-			let mut segments = vec![n.clone()];
-			while let Token::NS(n, newtail) = &nametail.tok {
-				segments.push(n.clone());
-				nametail = newtail;
-			}
-			let Token::Name(n) = &nametail.tok else {
-				return Err(mk_errv(
-					ctx.i().i("Loose namespace prefix in constant").await,
-					"Namespace prefixes in constants must be followed by names",
-					[pos],
-				));
-			};
-			segments.push(n.clone());
-			Ok(ExprKind::Const(Sym::new(segments, ctx.i()).await.unwrap()).at(pos.clone()))
-		},
-		Token::LambdaHead(h) => {
-			let [TokTree { tok: Token::Name(arg), .. }] = &h[..] else {
-				return Err(mk_errv(
-					ctx.i().i("Complex lambda binding in constant").await,
-					"Lambda args in constants must be identified by a single name",
-					[pos],
-				));
-			};
-			let lambda_builder = psb.lambda(arg);
-			let body = parse_expr(ctx, path.clone(), lambda_builder.stack(), tail).boxed_local().await?;
-			Ok(ExprKind::Lambda(lambda_builder.collect(), body).at(pos.clone()))
-		},
-		Token::S(Paren::Round, body) =>
-			parse_expr(ctx, path, psb, Snippet::new(head, body)).boxed_local().await,
-		Token::S(..) =>
-			return Err(mk_errv(
-				ctx.i().i("Constants may only contain (), not [] or {}").await,
-				"It seems like you are trying to call a macro. Consider a 'let' line",
-				[pos],
-			)),
-		Token::Name(n) =>
-			if psb.register_arg(n) {
-				Ok(ExprKind::Arg.at(pos))
-			} else {
-				Ok(ExprKind::Const(Sym::new([n.clone()], ctx.i()).await.unwrap()).at(pos))
-			},
-		Token::NewExpr(ex) => Ok(ex.clone()),
-	}
+	Ok((name, ParsedModule::new(true, parse_items(ctx, path, body).await?)))
 }

orchid-host/src/parsed.rs

@@ -1,8 +1,8 @@
-use std::fmt::Debug;
+use std::fmt::{self, Debug};
 use std::rc::Rc;
 
 use futures::FutureExt;
-use futures::future::join_all;
+use futures::future::{LocalBoxFuture, join_all};
 use hashbrown::HashSet;
 use itertools::Itertools;
 use orchid_base::format::{FmtCtx, FmtUnit, Format, Variants};
@@ -26,6 +26,11 @@ pub struct Item {
 	pub comments: Vec<Comment>,
 	pub kind: ItemKind,
 }
+impl Item {
+	pub fn new(sr: SrcRange, kind: impl Into<ItemKind>) -> Self {
+		Self { sr, comments: Vec::new(), kind: kind.into() }
+	}
+}
 
 #[derive(Debug)]
 pub enum ItemKind {
@@ -36,6 +41,12 @@ impl ItemKind {
 	#[must_use]
 	pub fn at(self, sr: SrcRange) -> Item { Item { comments: vec![], sr, kind: self } }
 }
+impl From<ParsedMember> for ItemKind {
+	fn from(value: ParsedMember) -> Self { Self::Member(value) }
+}
+impl From<Import> for ItemKind {
+	fn from(value: Import) -> Self { Self::Import(value) }
+}
 
 impl Format for Item {
 	async fn print<'a>(&'a self, c: &'a (impl FmtCtx + ?Sized + 'a)) -> FmtUnit {
@@ -43,9 +54,6 @@ impl Format for Item {
 		let item_text = match &self.kind {
 			ItemKind::Import(i) => format!("import {i}").into(),
 			ItemKind::Member(mem) => match &mem.kind {
-				ParsedMemberKind::ParsedConst(expr) =>
-					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} = {1l}")))
-						.units([mem.name.rc().into(), expr.print(c).await]),
 				ParsedMemberKind::DeferredConst(_, sys) =>
 					tl_cache!(Rc<Variants>: Rc::new(Variants::default().bounded("const {0} via {1}")))
 						.units([mem.name.rc().into(), sys.print(c).await]),
@@ -67,6 +75,9 @@ pub struct ParsedMember {
 impl ParsedMember {
 	#[must_use]
 	pub fn name(&self) -> Tok<String> { self.name.clone() }
+	pub fn new(exported: bool, name: Tok<String>, kind: impl Into<ParsedMemberKind>) -> Self {
+		Self { exported, name, kind: kind.into() }
+	}
 }
 impl Debug for ParsedMember {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -77,36 +88,53 @@ impl Debug for ParsedMember {
 	}
 }
 
+pub(crate) type ParsedExprCallback =
+	Rc<dyn for<'a> Fn(&'a [Tok<String>]) -> LocalBoxFuture<'a, Expr>>;
+
+pub struct ParsedExpr {
+	pub(crate) debug: String,
+	pub(crate) callback: ParsedExprCallback,
+}
+impl ParsedExpr {
+	pub async fn run(self, imported_names: &[Tok<String>]) -> Expr {
+		(self.callback)(imported_names).await
+	}
+}
+impl fmt::Debug for ParsedExpr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.debug) }
+}
+
 #[derive(Debug)]
 pub enum ParsedMemberKind {
 	DeferredConst(api::ParsedConstId, System),
-	ParsedConst(Expr),
 	Mod(ParsedModule),
 }
-
-// TODO: cannot determine alias origin at this stage; parsed tree is never
-// walkable!
+impl From<ParsedModule> for ParsedMemberKind {
+	fn from(value: ParsedModule) -> Self { Self::Mod(value) }
+}
 
 #[derive(Debug, Default)]
 pub struct ParsedModule {
 	pub exports: Vec<Tok<String>>,
 	pub items: Vec<Item>,
+	pub use_prelude: bool,
 }
 impl ParsedModule {
 	#[must_use]
-	pub fn new(items: impl IntoIterator<Item = Item>) -> Self {
+	pub fn new(use_prelude: bool, items: impl IntoIterator<Item = Item>) -> Self {
 		let items = items.into_iter().collect_vec();
 		let exports = (items.iter())
 			.filter_map(|i| if let ItemKind::Member(m) = &i.kind { Some(m) } else { None })
 			.filter(|m| m.exported)
 			.map(|m| m.name.clone())
 			.collect_vec();
-		Self { exports, items }
+		Self { exports, items, use_prelude }
 	}
 	pub fn merge(&mut self, other: ParsedModule) {
 		let mut swap = ParsedModule::default();
 		std::mem::swap(self, &mut swap);
-		*self = ParsedModule::new(swap.items.into_iter().chain(other.items))
+		assert_eq!(self.use_prelude, other.use_prelude, "merging modules that disagree on prelude");
+		*self = ParsedModule::new(self.use_prelude, swap.items.into_iter().chain(other.items))
 	}
 	#[must_use]
 	pub fn get_imports(&self) -> impl IntoIterator<Item = &Import> {
@@ -134,8 +162,7 @@ impl Tree for ParsedModule {
 			.find(|m| m.name == key)
 		{
 			match &member.kind {
-				ParsedMemberKind::DeferredConst(..) | ParsedMemberKind::ParsedConst(_) =>
-					return ChildResult::Err(ChildErrorKind::Constant),
+				ParsedMemberKind::DeferredConst(..) => return ChildResult::Err(ChildErrorKind::Constant),
 				ParsedMemberKind::Mod(m) => return ChildResult::Ok(m),
 			}
 		}

orchid-host/src/system.rs

@@ -39,6 +39,7 @@ struct SystemInstData {
 	lex_filter: api::CharFilter,
 	id: api::SysId,
 	line_types: Vec<Tok<String>>,
+	prelude: Vec<Sym>,
 	pub(crate) const_paths: MemoMap<api::ParsedConstId, Sym>,
 }
 impl Drop for SystemInstData {
@@ -69,6 +70,11 @@ impl System {
 	#[must_use]
 	pub fn deps(&self) -> &[System] { &self.0.deps }
 	#[must_use]
+	pub fn ctor(&self) -> SystemCtor {
+		(self.0.ext.system_ctors().find(|c| c.decl.id == self.0.decl_id).cloned())
+			.expect("Ctor was used to create ext")
+	}
+	#[must_use]
 	pub(crate) fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { self.0.ext.reqnot() }
 	#[must_use]
 	pub async fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
@@ -78,6 +84,8 @@ impl System {
 	pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
 	#[must_use]
 	pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
+	#[must_use]
+	pub fn prelude(&self) -> Vec<Sym> { self.0.prelude.clone() }
 	/// Have this system lex a part of the source. It is assumed that
 	/// [Self::can_lex] was called and returned true.
 	pub async fn lex<F: Future<Output = Option<api::SubLexed>>>(
@@ -147,10 +155,10 @@ impl System {
 						};
 						let name = ctx.i.ex(name).await;
 						let mkind = match kind {
-							api::ParsedMemberKind::Module(items) => {
+							api::ParsedMemberKind::Module { lines, use_prelude } => {
 								let items =
-									conv(items, module.push(name.clone()), callback, ctx).boxed_local().await?;
-								ParsedMemberKind::Mod(ParsedModule::new(items))
+									conv(lines, module.push(name.clone()), callback, ctx).boxed_local().await?;
+								ParsedMemberKind::Mod(ParsedModule::new(use_prelude, items))
 							},
 							api::ParsedMemberKind::Constant(cid) =>
 								ParsedMemberKind::DeferredConst(cid, ctx.sys.clone()),
@@ -199,7 +207,7 @@ impl System {
 		let orig = self.0.const_paths.get(&orig).expect("origin for find_names invalid").clone();
 		let ctx = self.0.ctx.clone();
 		async move |rel| {
-			let cwd = orig.split_last().1;
+			let cwd = orig.split_last_seg().1;
 			let abs = absolute_path(cwd, rel, &ctx.i).await.ok()?;
 			let root_data = &mut *root.0.write().await;
 			let walk_ctx = &mut (ctx.clone(), &mut root_data.consts);
@@ -221,6 +229,7 @@ impl WeakSystem {
 	pub fn upgrade(&self) -> Option<System> { self.0.upgrade().map(System) }
 }
 
+#[derive(Clone)]
 pub struct SystemCtor {
 	pub(crate) decl: api::SystemDecl,
 	pub(crate) ext: WeakExtension,
@@ -228,6 +237,10 @@ pub struct SystemCtor {
 impl SystemCtor {
 	#[must_use]
 	pub fn name(&self) -> &str { &self.decl.name }
+	pub async fn name_tok(&self) -> Sym {
+		(Sym::parse(&self.decl.name, &self.ext.upgrade().expect("ext dropped early").ctx().i).await)
+			.expect("System cannot have empty name")
+	}
 	#[must_use]
 	pub fn priority(&self) -> NotNan<f64> { self.decl.priority }
 	#[must_use]
@@ -252,6 +265,7 @@ impl SystemCtor {
 			line_types: join_all(sys_inst.line_types.iter().map(|m| Tok::from_api(*m, &ext.ctx().i)))
 				.await,
 			id,
+			prelude: join_all(sys_inst.prelude.iter().map(|tok| Sym::from_api(*tok, &ext.ctx().i))).await,
 			const_paths: MemoMap::new(),
 		}));
 		let api_module_root = api::Module {

orchid-host/src/tree.rs

@@ -9,12 +9,11 @@ use futures::{FutureExt, StreamExt, stream};
 use hashbrown::HashMap;
 use hashbrown::hash_map::Entry;
 use itertools::Itertools;
-use orchid_api::FetchParsedConst;
 use orchid_base::clone;
 use orchid_base::error::{OrcRes, Reporter, mk_err, mk_errv};
 use orchid_base::interner::Tok;
-use orchid_base::location::{Pos, SrcRange};
-use orchid_base::name::{Sym, VPath};
+use orchid_base::location::{CodeGenInfo, Pos};
+use orchid_base::name::{NameLike, Sym, VPath};
 use orchid_base::reqnot::Requester;
 
 use crate::api;
@@ -60,11 +59,12 @@ impl Root {
 		let mut ref_this = self.0.write().await;
 		let this = &mut *ref_this;
 		let mut deferred_consts = HashMap::new();
+		let mut consts = this.consts.clone();
 		let mut tfpctx = FromParsedCtx {
 			pars_root: parsed,
 			deferred_consts: &mut deferred_consts,
+			consts: &mut consts,
 			pars_prefix: pars_prefix.clone(),
-			consts: &mut this.consts,
 			root: &this.root,
 			ctx: &this.ctx,
 			rep,
@@ -78,14 +78,13 @@ impl Root {
 			)]);
 			module = Module { imports: HashMap::new(), members }
 		}
-		let mut consts = this.consts.clone();
 		let root = (this.root.merge(&module, this.ctx.clone(), &mut consts).await)
 			.expect("Merge conflict between parsed and existing module");
 		let new = Root(Rc::new(RwLock::new(RootData { root, consts, ctx: this.ctx.clone() })));
 		*this.ctx.root.write().await = new.downgrade();
 		for (path, (sys_id, pc_id)) in deferred_consts {
 			let sys = this.ctx.system_inst(sys_id).await.expect("System dropped since parsing");
-			let api_expr = sys.reqnot().request(FetchParsedConst { id: pc_id, sys: sys.id() }).await;
+			let api_expr = sys.reqnot().request(api::FetchParsedConst { id: pc_id, sys: sys.id() }).await;
 			let mut xp_ctx = ExprParseCtx { ctx: &this.ctx, exprs: sys.ext().exprs() };
 			let expr = Expr::from_api(&api_expr, PathSetBuilder::new(), &mut xp_ctx).await;
 			new.0.write().await.consts.insert(path, expr);
@@ -154,7 +153,7 @@ impl<'a> TreeFromApiCtx<'a> {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ResolvedImport {
 	target: Sym,
-	sr: SrcRange,
+	pos: Pos,
 }
 
 #[derive(Clone, Default)]
@@ -234,6 +233,22 @@ impl Module {
 			}
 		}
 		let mut imports = HashMap::new();
+		if parsed.use_prelude {
+			let systems = ctx.ctx.systems.read().await;
+			for sys in systems.values().flat_map(|weak| weak.upgrade()) {
+				for prelude_item in sys.prelude() {
+					imports.insert(
+						prelude_item.last_seg(),
+						Ok(ResolvedImport {
+							target: prelude_item,
+							pos: CodeGenInfo::new_details(sys.ctor().name_tok().await, "In prelude", &ctx.ctx.i)
+								.await
+								.pos(),
+						}),
+					);
+				}
+			}
+		}
 		let conflicting_imports_msg = ctx.ctx.i.i("Conflicting imports").await;
 		for (key, values) in imports_by_name {
 			if values.len() == 1 {
@@ -243,8 +258,10 @@ impl Module {
 				match abs_path_res {
 					Err(e) => ctx.rep.report(e.err_obj(&ctx.ctx.i, sr.pos(), &import.to_string()).await),
 					Ok(abs_path) => {
-						imports
-							.insert(key, Ok(ResolvedImport { target: abs_path.to_sym(&ctx.ctx.i).await, sr }));
+						imports.insert(
+							key,
+							Ok(ResolvedImport { target: abs_path.to_sym(&ctx.ctx.i).await, pos: sr.pos() }),
+						);
 					},
 				}
 			} else {
@@ -263,7 +280,10 @@ impl Module {
 				let values = stream::iter(values)
 					.then(|(n, sr)| {
 						clone!(key; async move {
-							ResolvedImport { target: n.to_vname().suffix([key.clone()]).to_sym(i).await, sr }
+							ResolvedImport {
+								target: n.to_vname().suffix([key.clone()]).to_sym(i).await,
+								pos: sr.pos(),
+							}
 						})
 					})
 					.collect::<Vec<_>>()
@@ -278,7 +298,7 @@ impl Module {
 				ctx.rep.report(mk_err(
 					self_referential_msg.clone(),
 					format!("import {} points to itself or a path within itself", &import.target),
-					[import.sr.pos().into()],
+					[import.pos.clone().into()],
 				));
 			}
 		}
@@ -363,9 +383,9 @@ pub struct FromParsedCtx<'a> {
 	pars_prefix: Sym,
 	pars_root: &'a ParsedModule,
 	root: &'a Module,
-	consts: &'a mut HashMap<Sym, Expr>,
 	rep: &'a Reporter,
 	ctx: &'a Ctx,
+	consts: &'a mut HashMap<Sym, Expr>,
 	deferred_consts: &'a mut HashMap<Sym, (api::SysId, api::ParsedConstId)>,
 }
 
@@ -417,10 +437,6 @@ impl MemberKind {
 	#[must_use]
 	async fn from_parsed(parsed: &ParsedMemberKind, path: Sym, ctx: &mut FromParsedCtx<'_>) -> Self {
 		match parsed {
-			ParsedMemberKind::ParsedConst(expr) => {
-				ctx.consts.insert(path, expr.clone());
-				MemberKind::Const
-			},
 			ParsedMemberKind::DeferredConst(id, sys) => {
 				ctx.deferred_consts.insert(path, (sys.id(), *id));
 				MemberKind::Const