commit before easter break

orchid-api/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "orchid-api"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

orchid-api/src/lexer.rs

@@ -3,7 +3,7 @@ use std::ops::RangeInclusive;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TokenTree, TreeTicket};
+use crate::{ExtHostReq, HostExtReq, OrcResult, ParsId, SysId, TStr, TokenTree};
 
 /// - All ranges contain at least one character
 /// - All ranges are in increasing characeter order
@@ -42,5 +42,5 @@ impl Request for SubLex {
 #[derive(Clone, Debug, Coding)]
 pub struct SubLexed {
 	pub pos: u32,
-	pub ticket: TreeTicket,
+	pub tree: TokenTree,
 }

orchid-api/src/lib.rs

@@ -2,8 +2,6 @@ mod lexer;
 pub use lexer::*;
 mod format;
 pub use format::*;
-mod macros;
-pub use macros::*;
 mod atom;
 pub use atom::*;
 mod error;

orchid-api/src/macros.rs

@@ -1,86 +0,0 @@
-use std::collections::HashMap;
-use std::num::NonZeroU64;
-
-use orchid_api_derive::{Coding, Hierarchy};
-use orchid_api_traits::Request;
-use ordered_float::NotNan;
-
-use crate::{
-	Atom, Comment, ExtHostReq, HostExtReq, Location, OrcResult, Paren, ParsId, SysId, TStr, TStrv,
-};
-
-#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct MacroTreeId(pub NonZeroU64);
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroTree {
-	pub location: Location,
-	pub token: MacroToken,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub enum MacroToken {
-	S(Paren, Vec<MacroTree>),
-	Name(TStrv),
-	Slot(MacroTreeId),
-	Lambda(Vec<MacroTree>, Vec<MacroTree>),
-	Ph(Placeholder),
-	Atom(Atom),
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroBlock {
-	pub priority: Option<NotNan<f64>>,
-	pub rules: Vec<MacroRule>,
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct MacroRule {
-	pub location: Location,
-	pub comments: Vec<Comment>,
-	pub pattern: Vec<MacroTree>,
-	pub id: MacroId,
-}
-
-/// A specific macro rule with a specific pattern across invocations
-#[derive(Clone, Copy, Debug, Coding, PartialEq, Eq, Hash)]
-pub struct MacroId(pub NonZeroU64);
-
-/// After a pattern matches, this call executes the body of the macro. This
-/// request returns None if an inner nested request raised an exception
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtReq)]
-pub struct ApplyMacro {
-	pub sys: SysId,
-	pub id: MacroId,
-	/// Recursion token
-	pub run_id: ParsId,
-	/// Must contain exactly the keys that were specified as placeholders in the
-	/// pattern
-	pub params: HashMap<TStr, Vec<MacroTree>>,
-}
-impl Request for ApplyMacro {
-	type Response = Option<OrcResult<Vec<MacroTree>>>;
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostReq)]
-pub struct RunMacros {
-	pub run_id: ParsId,
-	pub query: Vec<MacroTree>,
-}
-impl Request for RunMacros {
-	type Response = Option<Vec<MacroTree>>;
-}
-
-#[derive(Clone, Debug, Coding)]
-pub struct Placeholder {
-	pub name: TStr,
-	pub kind: PhKind,
-}
-
-#[derive(Clone, Copy, Debug, Coding)]
-pub enum PhKind {
-	Scalar,
-	Vector { priority: u8, at_least_one: bool },
-}

orchid-api/src/parser.rs

@@ -1,24 +1,18 @@
 use std::num::NonZeroU64;
 
-use orchid_api_derive::{Coding, Decode, Encode, Hierarchy};
+use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::{Comment, HostExtReq, OrcResult, SysId, TokenTree};
+use crate::{Comment, HostExtReq, OrcResult, SysId, TStrv, TokenTree};
 
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)]
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct ParsId(pub NonZeroU64);
 
-// impl orchid_api_traits::Decode for ParsId {
-// 	async fn decode<R: async_std::io::Read + ?Sized>(mut read:
-// std::pin::Pin<&mut R>) -> Self {
-// 		Self(orchid_api_traits::Decode::decode(read.as_mut()).await)
-// 	}
-// }
-
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct ParseLine {
 	pub sys: SysId,
+	pub module: TStrv,
 	pub comments: Vec<Comment>,
 	pub exported: bool,
 	pub line: Vec<TokenTree>,

orchid-api/src/proto.rs

@@ -28,7 +28,7 @@ use async_std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{Channel, Decode, Encode, MsgSet, Request, read_exact, write_exact};
 
-use crate::{atom, expr, interner, lexer, logging, macros, parser, system, tree, vfs};
+use crate::{atom, expr, interner, lexer, logging, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader {
@@ -88,7 +88,7 @@ pub enum ExtHostReq {
 	SysFwd(system::SysFwd),
 	ExprReq(expr::ExprReq),
 	SubLex(lexer::SubLex),
-	RunMacros(macros::RunMacros),
+	LsModule(tree::LsModule),
 }
 
 /// Notifications sent from the extension to the host
@@ -119,7 +119,6 @@ pub enum HostExtReq {
 	ParseLine(parser::ParseLine),
 	GetMember(tree::GetMember),
 	VfsReq(vfs::VfsReq),
-	ApplyMacro(macros::ApplyMacro),
 }
 
 /// Notifications sent from the host to the extension

orchid-api/src/system.rs

@@ -52,11 +52,11 @@ pub struct NewSystem {
 	pub depends: Vec<SysId>,
 }
 impl Request for NewSystem {
-	type Response = SystemInst;
+	type Response = NewSystemResponse;
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct SystemInst {
+pub struct NewSystemResponse {
 	/// The set of possible starting characters of tokens the lexer of this system
 	/// can process. The lexer will notify this system if it encounters one of
 	/// these characters.9

orchid-api/src/tree.rs

@@ -1,13 +1,13 @@
+use std::collections::HashMap;
 use std::num::NonZeroU64;
 use std::ops::Range;
 use std::sync::Arc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
-use ordered_float::NotNan;
 
 use crate::{
-	Atom, Expression, HostExtReq, Location, MacroBlock, OrcError, Placeholder, SysId, TStr, TStrv,
+	ExprTicket, Expression, ExtHostReq, HostExtReq, Location, OrcError, SysId, TStr, TStrv,
 };
 
 /// A token tree from a lexer recursion request. Its lifetime is the lex call,
@@ -32,27 +32,24 @@ pub enum Token {
 	LambdaHead(Vec<TokenTree>),
 	/// A name segment or an operator.
 	Name(TStr),
-	/// An absolute name
-	Reference(TStrv),
+	/// A newly generated expression. The last place this is supposed to happen is
+	/// in lexers, parsers and macros should have enumerable many outputs
+	/// expressed as function calls.
+	NewExpr(Expression),
+	/// A pre-existing expression
+	Handle(ExprTicket),
 	/// ::
-	NS,
+	NS(TStr, Box<TokenTree>),
 	/// Line break.
 	BR,
 	/// ( Round parens ), [ Square brackets ] or { Curly braces }
 	S(Paren, Vec<TokenTree>),
-	/// A new atom
-	Atom(Atom),
-	/// Anchor to insert a subtree
-	Slot(TreeTicket),
 	/// A static compile-time error returned by failing lexers if
-	/// the rest of the source is likely still meaningful
+	/// the rest of the source is likely still meaningful. This is distinct from
+	/// NewExpr(Bottom) because it fails in dead branches too.
 	Bottom(Vec<OrcError>),
 	/// A comment
 	Comment(Arc<String>),
-	/// Placeholder
-	Ph(Placeholder),
-	/// Macro block head
-	Macro(Option<NotNan<f64>>),
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, Coding)]
@@ -75,7 +72,6 @@ pub struct Item {
 #[derive(Clone, Debug, Coding)]
 pub enum ItemKind {
 	Member(Member),
-	Macro(MacroBlock),
 	Export(TStr),
 	Import(TStrv),
 }
@@ -104,9 +100,53 @@ pub struct Module {
 	pub items: Vec<Item>,
 }
 
+/// Evaluate a lazy member. This call will only be issued to each system once.
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct GetMember(pub SysId, pub TreeId);
 impl Request for GetMember {
 	type Response = MemberKind;
 }
+
+/// This request can only be issued while the interpreter is running, so during
+/// an atom call.
+#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+pub struct LsModule(pub SysId, pub TStrv);
+impl Request for LsModule {
+	type Response = Result<ModuleInfo, LsModuleError>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum LsModuleError {
+	InvalidPath,
+	IsConstant,
+	TreeUnavailable,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct ModuleInfo {
+	/// If the name isn't a canonical name, returns the true name.
+	pub canonical: Option<TStrv>,
+	/// List the names defined in this module
+	pub members: HashMap<TStr, MemberInfo>,
+}
+
+#[derive(Clone, Copy, Debug, Coding)]
+pub struct MemberInfo {
+	/// true if the name is exported
+	pub exported: bool,
+	/// If it's imported, you can find the canonical name here
+	pub canonical: Option<TStrv>,
+	/// Whether the tree item is a constant value or a module
+	pub kind: MemberInfoKind,
+}
+
+/// Indicates what kind of node a name refers to
+#[derive(Clone, Copy, Debug, Coding)]
+pub enum MemberInfoKind {
+	/// has children obtained with [crate::LsModule]
+	Module,
+	/// has a value retrievable in [crate::ExpressionKind::Const]
+	Constant,
+}