Changes in api and upwards

- Removed out-of-stack error reporting
- Revised module system to match previous Orchid system
- Errors are now in a Vec everywhere
- Implemented atoms and lexer
- Started implementation of line parser
- Tree is now ephemeral to avoid copying Atoms held inside
- Moved numbers into std and the shared parser into base
- Started implementation of Commands

orchid-api/src/atom.rs

@@ -15,6 +15,9 @@ pub struct LocalAtom {
   pub drop: bool,
   pub data: AtomData,
 }
+impl LocalAtom {
+  pub fn associate(self, owner: SysId) -> Atom { Atom { owner, drop: self.drop, data: self.data } }
+}
 
 /// An atom representation that can be serialized and sent around. Atoms
 /// represent the smallest increment of work.
@@ -101,6 +104,7 @@ impl Request for Command {
 #[extends(HostExtNotif)]
 pub struct AtomDrop(pub Atom);
 
+/// Requests that apply to an existing atom instance
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 #[extendable]
@@ -111,3 +115,16 @@ pub enum AtomReq {
   Fwded(Fwded),
   Command(Command),
 }
+impl AtomReq {
+  /// Obtain the first [Atom] argument of the request. All requests in this
+  /// subclass have at least one atom argument.
+  pub fn get_atom(&self) -> &Atom {
+    match self {
+      Self::AtomSame(AtomSame(a, ..))
+      | Self::CallRef(CallRef(a, ..))
+      | Self::Command(Command(a))
+      | Self::FinalCall(FinalCall(a, ..))
+      | Self::Fwded(Fwded(a, ..)) => a,
+    }
+  }
+}

orchid-api/src/error.rs

@@ -6,10 +6,10 @@ use orchid_api_traits::Request;
 
 use crate::interner::TStr;
 use crate::location::Location;
-use crate::proto::{ExtHostNotif, ExtHostReq};
-use crate::system::SysId;
+use crate::proto::ExtHostReq;
 
-pub type ProjErrId = NonZeroU16;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct ProjErrId(pub NonZeroU16);
 
 #[derive(Clone, Debug, Coding)]
 pub struct ProjErrLocation {
@@ -37,18 +37,8 @@ pub struct ProjErr {
   pub locations: Vec<ProjErrLocation>,
 }
 
-/// When the interpreter encounters an error while serving a system's request,
-/// it sends this error as an ID back to the system to save bandwidth.
-/// The lifetime of this ID is the request being served, the receiving system
-/// can return it and query its details with [GetDetails].
-#[derive(Clone, Debug, Coding)]
-pub enum ProjErrOrRef {
-  New(ProjErr),
-  Known(ProjErrId),
-}
-
 /// If this is an [`Err`] then the [`Vec`] must not be empty.
-pub type ProjResult<T> = Result<T, Vec<ProjErrOrRef>>;
+pub type ProjResult<T> = Result<T, Vec<ProjErr>>;
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ProjErrReq, ExtHostReq)]
@@ -57,26 +47,9 @@ impl Request for GetErrorDetails {
   type Response = ProjErr;
 }
 
-/// Notify the host about an error without being forced to return said error.
-/// This will still count as an error, but later operations that depend on the
-/// value returned by the currently running function will get to run
-///
-/// The error is not connected to the place it was raised, since multiple calls
-/// can be issued to a system at the same time
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ErrNotif, ExtHostNotif)]
-pub struct ReportError(pub SysId, pub ProjErrOrRef);
-
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 #[extendable]
 pub enum ProjErrReq {
   GetErrorDetails(GetErrorDetails),
 }
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostNotif)]
-#[extendable]
-pub enum ErrNotif {
-  ReportError(ReportError),
-}

orchid-api/src/expr.rs

@@ -15,7 +15,8 @@ use crate::system::SysId;
 /// [Acquire].
 ///
 /// The ID is globally unique within its lifetime, but may be reused.
-pub type ExprTicket = NonZeroU64;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct ExprTicket(pub NonZeroU64);
 
 /// Acquire a strong reference to an expression. This keeps it alive until a
 /// corresponding [Release] is emitted. The number of times a system has
@@ -84,7 +85,7 @@ pub enum Clause {
   /// A reference to a constant
   Const(TStrv),
   /// A static runtime error.
-  Bottom(ProjErr),
+  Bottom(Vec<ProjErr>),
 }
 
 #[derive(Clone, Debug, Coding)]

orchid-api/src/interner.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::proto::{ExtHostReq, HostExtReq};
+use crate::proto::{ExtHostNotif, ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.
@@ -80,6 +80,16 @@ impl Request for Sweep {
   type Response = Retained;
 }
 
+/// A notification from the extension to the host, that the extension would benefit from a sweep
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostNotif)]
+pub struct AdviseSweep(SweepReason);
+
+#[derive(Clone, Debug, Coding)]
+pub enum SweepReason {
+  None
+}
+
 /// List of keys in this replica that couldn't be sweeped because local
 /// datastructures reference their value.
 #[derive(Clone, Debug, Coding)]

orchid-api/src/parser.rs

@@ -10,7 +10,8 @@ use crate::proto::{ExtHostReq, HostExtReq};
 use crate::system::SysId;
 use crate::tree::{TokenTree, TreeTicket};
 
-pub type LexId = NonZeroU64;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct ParsId(pub NonZeroU64);
 
 /// - All ranges contain at least one character
 /// - All ranges are in increasing characeter order
@@ -22,35 +23,35 @@ pub struct CharFilter(pub Vec<RangeInclusive<char>>);
 #[extends(HostExtReq)]
 #[extendable]
 pub enum ParserReq {
-  Lex(Lex),
+  LexExpr(LexExpr),
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ParserReq, HostExtReq)]
-pub struct Lex {
+pub struct LexExpr {
   pub sys: SysId,
-  pub id: LexId,
+  pub id: ParsId,
   pub text: TStr,
   pub pos: u32,
 }
-impl Request for Lex {
-  type Response = Option<ProjResult<Lexed>>;
+impl Request for LexExpr {
+  type Response = Option<ProjResult<LexedExpr>>;
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct Lexed {
+pub struct LexedExpr {
   pub pos: u32,
-  pub data: TokenTree,
+  pub expr: TokenTree,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct SubLex {
-  pub id: LexId,
+  pub id: ParsId,
   pub pos: u32,
 }
 impl Request for SubLex {
-  type Response = ProjResult<SubLexed>;
+  type Response = Option<SubLexed>;
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -59,4 +60,11 @@ pub struct SubLexed {
   pub ticket: TreeTicket,
 }
 
-pub struct ParseLine {}
+#[derive(Clone, Debug, Coding)]
+pub struct ParseLine {
+  pub sys: SysId,
+  pub line: Vec<TokenTree>,
+}
+impl Request for ParseLine {
+  type Response = Vec<TokenTree>;
+}

orchid-api/src/proto.rs

@@ -82,7 +82,7 @@ pub enum ExtHostReq {
 #[extendable]
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
-  ErrNotif(error::ErrNotif),
+  AdviseSweep(interner::AdviseSweep),
 }
 
 pub struct ExtHostChannel;
@@ -100,7 +100,7 @@ pub enum HostExtReq {
   Sweep(interner::Sweep),
   AtomReq(atom::AtomReq),
   ParserReq(parser::ParserReq),
-  GetConstTree(tree::GetConstTree),
+  GetMember(tree::GetMember),
   VfsReq(vfs::VfsReq),
 }
 

orchid-api/src/system.rs

@@ -1,20 +1,25 @@
+use std::collections::HashMap;
 use std::num::NonZeroU16;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
+use crate::interner::TStr;
 use crate::parser::CharFilter;
 use crate::proto::{HostExtNotif, HostExtReq};
-use crate::tree::TreeId;
+use crate::tree::MemberKind;
 
 /// ID of a system type
-pub type SysDeclId = NonZeroU16;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct SysDeclId(pub NonZeroU16);
 
 /// ID of a system instance
-pub type SysId = NonZeroU16;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct SysId(pub NonZeroU16);
 
-/// Details about a system provided by this library
+/// Details about a system provided by this library. This is included in the extension header,
+/// so it cannot rely on the interner. 
 #[derive(Debug, Clone, Coding)]
 pub struct SystemDecl {
   /// ID of the system, unique within the library
@@ -59,7 +64,8 @@ pub struct SystemInst {
   /// can process. The lexer will notify this system if it encounters one of
   /// these characters.9
   pub lex_filter: CharFilter,
-  pub const_root_id: TreeId,
+  pub parses_lines: Vec<TStr>,
+  pub const_root: HashMap<TStr, MemberKind>,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/tree.rs

@@ -1,4 +1,5 @@
-use std::collections::HashMap;
+use crate::interner::TStrv;
+use crate::location::Location;
 use std::num::NonZeroU64;
 use std::ops::Range;
 
@@ -7,7 +8,7 @@ use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
 use crate::atom::LocalAtom;
-use crate::error::ProjErrOrRef;
+use crate::error::ProjErr;
 use crate::expr::Expr;
 use crate::interner::TStr;
 use crate::proto::HostExtReq;
@@ -17,8 +18,11 @@ use crate::system::SysId;
 /// the lexer can include it in its output or discard it by implication.
 ///
 /// Similar to [crate::expr::ExprTicket] in that it represents a token tree the
-/// lifetime of which is managed by the interpreter.
-pub type TreeTicket = NonZeroU64;
+/// lifetime of which is managed by the interpreter, and as such should probably
+/// not be exposed to libraries directly but rather wrapped in a
+/// lifetime-controlled abstraction.
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct TreeTicket(pub NonZeroU64);
 
 #[derive(Clone, Debug, Coding)]
 pub struct TokenTree {
@@ -28,18 +32,26 @@ pub struct TokenTree {
 
 #[derive(Clone, Debug, Coding)]
 pub enum Token {
-  /// Lambda function. The number operates as an argument name
-  Lambda(Vec<TokenTree>, Vec<TokenTree>),
-  Name(Vec<TStr>),
+  /// Lambda function head, from the opening \ until the beginning of the body.
+  Lambda(Vec<TokenTree>),
+  /// A name segment or an operator.
+  Name(TStr),
+  /// ::
+  NS,
+  /// Line break.
+  BR,
+  /// ( Round parens ), [ Square brackets ] or { Curly braces }
   S(Paren, Vec<TokenTree>),
   /// A placeholder in a macro. This variant is forbidden everywhere outside
   /// line parser output
   Ph(Placeholder),
+  /// A new atom
   Atom(LocalAtom),
+  /// Anchor to insert a subtree
   Slot(TreeTicket),
-  /// A static compile-time error returned by erroring lexers if
+  /// A static compile-time error returned by failing lexers if
   /// the rest of the source is likely still meaningful
-  Bottom(ProjErrOrRef),
+  Bottom(Vec<ProjErr>),
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -63,30 +75,52 @@ pub enum Paren {
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct MacroRule {
+pub struct Macro {
   pub pattern: Vec<TokenTree>,
   pub priority: NotNan<f64>,
   pub template: Vec<TokenTree>,
 }
 
-pub type TreeId = NonZeroU64;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct TreeId(pub NonZeroU64);
+
 
 #[derive(Clone, Debug, Coding)]
-pub enum Tree {
+pub struct Item {
+  pub location: Location,
+  pub kind: ItemKind,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum ItemKind {
+  Member(Member),
+  Raw(Vec<TokenTree>),
+  Rule(Macro),
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Member {
+  pub name: TStr,
+  pub public: bool,
+  pub kind: MemberKind,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum MemberKind {
   Const(Expr),
-  Mod(TreeModule),
-  Rule(MacroRule),
+  Module(Module),
   Lazy(TreeId),
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct TreeModule {
-  pub children: HashMap<String, Tree>,
+pub struct Module {
+  pub imports: Vec<TStrv>,
+  pub items: Vec<Item>,
 }
 
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
-pub struct GetConstTree(pub SysId, pub TreeId);
-impl Request for GetConstTree {
-  type Response = Tree;
+pub struct GetMember(pub SysId, pub TreeId);
+impl Request for GetMember {
+  type Response = MemberKind;
 }

orchid-api/src/vfs.rs

@@ -9,7 +9,8 @@ use crate::interner::TStr;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
 
-pub type VfsId = NonZeroU16;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct VfsId(pub NonZeroU16);
 
 #[derive(Clone, Debug, Coding)]
 pub enum Loaded {