very elegant extension API and parts of it used in std as POC

orchid-api/src/atom.rs

@@ -1,6 +1,7 @@
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
+use crate::error::ProjResult;
 use crate::expr::{Expr, ExprTicket};
 use crate::proto::{ExtHostReq, HostExtNotif, HostExtReq};
 use crate::system::SysId;
@@ -81,6 +82,18 @@ impl Request for Fwd {
   type Response = Vec<u8>;
 }
 
+#[derive(Clone, Debug, Coding)]
+pub enum NextStep {
+  Continue(Expr),
+  Halt,
+}
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct Command(pub Atom);
+impl Request for Command {
+  type Response = ProjResult<NextStep>;
+}
+
 /// Notification that an atom is being dropped because its associated expression
 /// isn't referenced anywhere. This should have no effect if the atom's `drop`
 /// flag is false.
@@ -96,4 +109,5 @@ pub enum AtomReq {
   FinalCall(FinalCall),
   AtomSame(AtomSame),
   Fwded(Fwded),
+  Command(Command),
 }

orchid-api/src/error.rs

@@ -1,17 +1,23 @@
-use orchid_api_derive::Coding;
+use std::num::NonZeroU16;
+use std::sync::Arc;
 
-use crate::intern::TStr;
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::interner::TStr;
 use crate::location::Location;
+use crate::proto::{ExtHostNotif, ExtHostReq};
+use crate::system::SysId;
 
-pub type ProjErrId = u16;
+pub type ProjErrId = NonZeroU16;
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct ProjErrLocation {
   /// Description of the relation of this location to the error. If not used,
   /// set to empty string
-  message: String,
+  pub message: Arc<String>,
   /// Location in code where the error emerged. This is usually [Location::Gen].
-  location: Location,
+  pub location: Location,
 }
 
 /// Programming errors raised by extensions. At runtime these produce the
@@ -19,17 +25,58 @@ pub struct ProjErrLocation {
 /// fallible operations should return an Orchid result and not a bottom.
 /// For example, file reading produces result::err when the file doesn't exist,
 /// and a bottom if the file name isn't a string.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct ProjErr {
   /// General description of the kind of error.
-  description: TStr,
+  pub description: TStr,
   /// Specific information about the exact error, preferably containing concrete
   /// values.
-  message: String,
+  pub message: Arc<String>,
   /// Specific code fragments that have contributed to the emergence of the
   /// error.
-  locations: Vec<ProjErrLocation>,
+  pub locations: Vec<ProjErrLocation>,
+}
+
+/// When the interpreter encounters an error while serving a system's request,
+/// it sends this error as an ID back to the system to save bandwidth.
+/// The lifetime of this ID is the request being served, the receiving system
+/// can return it and query its details with [GetDetails].
+#[derive(Clone, Debug, Coding)]
+pub enum ProjErrOrRef {
+  New(ProjErr),
+  Known(ProjErrId),
 }
 
 /// If this is an [`Err`] then the [`Vec`] must not be empty.
-pub type ProjResult<T> = Result<T, Vec<ProjErr>>;
+pub type ProjResult<T> = Result<T, Vec<ProjErrOrRef>>;
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ProjErrReq, ExtHostReq)]
+pub struct GetErrorDetails(pub ProjErrId);
+impl Request for GetErrorDetails {
+  type Response = ProjErr;
+}
+
+/// Notify the host about an error without being forced to return said error.
+/// This will still count as an error, but later operations that depend on the
+/// value returned by the currently running function will get to run
+///
+/// The error is not connected to the place it was raised, since multiple calls
+/// can be issued to a system at the same time
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ErrNotif, ExtHostNotif)]
+pub struct ReportError(pub SysId, pub ProjErrOrRef);
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+#[extendable]
+pub enum ProjErrReq {
+  GetErrorDetails(GetErrorDetails),
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostNotif)]
+#[extendable]
+pub enum ErrNotif {
+  ReportError(ReportError),
+}

orchid-api/src/expr.rs

@@ -1,18 +1,20 @@
+use std::num::NonZeroU64;
+
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::atom::LocalAtom;
-use crate::intern::{TStr, TStrv};
+use crate::atom::{Atom, LocalAtom};
+use crate::interner::TStrv;
 use crate::location::Location;
 use crate::proto::{ExtHostNotif, ExtHostReq};
 use crate::system::SysId;
 
 /// An arbitrary ID associated with an expression on the host side. Incoming
 /// tickets always come with some lifetime guarantee, which can be extended with
-/// [AcquireExpr].
+/// [Acquire].
 ///
 /// The ID is globally unique within its lifetime, but may be reused.
-pub type ExprTicket = u64;
+pub type ExprTicket = NonZeroU64;
 
 /// Acquire a strong reference to an expression. This keeps it alive until a
 /// corresponding [Release] is emitted. The number of times a system has
@@ -52,15 +54,15 @@ pub struct Relocate {
 /// A description of a new expression. It is used as the return value of
 /// [crate::atom::Call] or [crate::atom::CallRef], or a constant in the
 /// [crate::tree::Tree].
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub enum Clause {
   /// Apply the lhs as a function to the rhs
   Call(Box<Expr>, Box<Expr>),
   /// Lambda function. The number operates as an argument name
-  Lambda(TStr, Box<Expr>),
+  Lambda(u64, Box<Expr>),
   /// Binds the argument passed to the lambda with the same ID in the same
   /// template
-  Arg(TStr),
+  Arg(u64),
   /// Insert the specified host-expression in the template here. When the clause
   /// is used in the const tree, this variant is forbidden.
   Slot(ExprTicket),
@@ -73,22 +75,34 @@ pub enum Clause {
   /// Because the atom is newly constructed, it also must belong to this system.
   /// For convenience, [SysId::MAX] is also accepted as referring to this
   /// system.
-  Atom(LocalAtom),
+  NewAtom(LocalAtom),
+  /// An atom, specifically an atom that already exists. This form is only ever
+  /// returned from [Inspect], and it's borrowed from the expression being
+  /// inspected.
+  Atom(ExprTicket, Atom),
   /// A reference to a constant
   Const(TStrv),
+  /// A static runtime error.
+  Bottom(String),
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct Expr {
   pub clause: Clause,
   pub location: Location,
 }
 
+#[derive(Clone, Debug, Coding)]
+pub struct Details {
+  pub expr: Expr,
+  pub refcount: u32,
+}
+
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(ExprReq, ExtHostReq)]
 pub struct Inspect(pub ExprTicket);
 impl Request for Inspect {
-  type Response = Clause;
+  type Response = Details;
 }
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]

orchid-api/src/interner.rs

@@ -8,7 +8,7 @@ use crate::proto::{ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 #[extendable]
 pub enum IntReq {
@@ -18,9 +18,7 @@ pub enum IntReq {
   ExternStrv(ExternStrv),
 }
 
-/// replica -> master to intern a string on the master.
-///
-/// Repeatable.
+/// replica -> master to intern a string on the master. Repeatable.
 ///
 /// See [IntReq]
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
@@ -35,7 +33,7 @@ impl Request for InternStr {
 /// Repeatable.
 ///
 /// See [IntReq]
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(IntReq, ExtHostReq)]
 pub struct ExternStr(pub TStr);
 impl Request for ExternStr {
@@ -46,7 +44,7 @@ impl Request for ExternStr {
 /// Repeatable.
 ///
 /// See [IntReq]
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(IntReq, ExtHostReq)]
 pub struct InternStrv(pub Arc<Vec<TStr>>);
 impl Request for InternStrv {
@@ -58,7 +56,7 @@ impl Request for InternStrv {
 /// Repeatable.
 ///
 /// See [IntReq]
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(IntReq, ExtHostReq)]
 pub struct ExternStrv(pub TStrv);
 impl Request for ExternStrv {
@@ -75,7 +73,7 @@ pub struct TStrv(pub NonZeroU64);
 
 /// A request to sweep the replica. The master will not be sweeped until all
 /// replicas respond, as it must retain everything the replicas retained
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct Sweep;
 impl Request for Sweep {

orchid-api/src/lib.rs

@@ -1,7 +1,7 @@
 pub mod atom;
 pub mod error;
 pub mod expr;
-pub mod intern;
+pub mod interner;
 pub mod location;
 pub mod parser;
 pub mod proto;

orchid-api/src/location.rs

@@ -2,26 +2,29 @@ use std::ops::Range;
 
 use orchid_api_derive::Coding;
 
-use crate::intern::{TStr, TStrv};
+use crate::interner::TStrv;
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub enum Location {
   None,
   /// Used in functions to denote the generated code that carries on the
   /// location of the call. Not allowed in the const tree.
   Inherit,
   Gen(CodeGenInfo),
-  Range(SourceRange),
+  /// Range and file
+  SourceRange(SourceRange),
+  /// Range only, file implied. Most notably used by parsers
+  Range(Range<u32>),
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct SourceRange {
   pub path: TStrv,
   pub range: Range<u32>,
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct CodeGenInfo {
-  pub generator: TStr,
-  pub details: TStr,
+  pub generator: TStrv,
+  pub details: String,
 }

orchid-api/src/parser.rs

@@ -1,13 +1,16 @@
+use std::num::NonZeroU64;
 use std::ops::RangeInclusive;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::error::ProjResult;
-use crate::intern::TStr;
+use crate::interner::TStr;
 use crate::proto::{ExtHostReq, HostExtReq};
 use crate::system::SysId;
-use crate::tree::TokenTree;
+use crate::tree::{TokenTree, TreeTicket};
+
+pub type LexId = NonZeroU64;
 
 /// - All ranges contain at least one character
 /// - All ranges are in increasing characeter order
@@ -26,6 +29,7 @@ pub enum ParserReq {
 #[extends(ParserReq, HostExtReq)]
 pub struct Lex {
   pub sys: SysId,
+  pub id: LexId,
   pub text: TStr,
   pub pos: u32,
 }
@@ -42,11 +46,17 @@ pub struct Lexed {
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct SubLex {
-  pub text: TStr,
+  pub id: LexId,
   pub pos: u32,
 }
 impl Request for SubLex {
-  type Response = ProjResult<Lexed>;
+  type Response = ProjResult<SubLexed>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct SubLexed {
+  pub pos: u32,
+  pub ticket: TreeTicket,
 }
 
 pub struct ParseLine {}

orchid-api/src/proto.rs

@@ -27,7 +27,7 @@ use std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{read_exact, write_exact, Channel, Decode, Encode, MsgSet, Request};
 
-use crate::{atom, expr, intern, parser, system, tree, vfs};
+use crate::{atom, error, expr, interner, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader;
@@ -69,10 +69,11 @@ impl Request for Ping {
 #[extendable]
 pub enum ExtHostReq {
   Ping(Ping),
-  IntReq(intern::IntReq),
+  IntReq(interner::IntReq),
   Fwd(atom::Fwd),
   ExprReq(expr::ExprReq),
   SubLex(parser::SubLex),
+  ProjErrReq(error::ProjErrReq),
 }
 
 /// Notifications sent from the extension to the host
@@ -81,6 +82,7 @@ pub enum ExtHostReq {
 #[extendable]
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
+  ErrNotif(error::ErrNotif),
 }
 
 pub struct ExtHostChannel;
@@ -95,7 +97,7 @@ impl Channel for ExtHostChannel {
 pub enum HostExtReq {
   Ping(Ping),
   NewSystem(system::NewSystem),
-  Sweep(intern::Sweep),
+  Sweep(interner::Sweep),
   AtomReq(atom::AtomReq),
   ParserReq(parser::ParserReq),
   GetConstTree(tree::GetConstTree),

orchid-api/src/system.rs

@@ -1,15 +1,18 @@
+use std::num::NonZeroU16;
+
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
 use crate::parser::CharFilter;
 use crate::proto::{HostExtNotif, HostExtReq};
+use crate::tree::TreeId;
 
 /// ID of a system type
-pub type SysDeclId = u16;
+pub type SysDeclId = NonZeroU16;
 
 /// ID of a system instance
-pub type SysId = u16;
+pub type SysId = NonZeroU16;
 
 /// Details about a system provided by this library
 #[derive(Debug, Clone, Coding)]
@@ -56,6 +59,7 @@ pub struct SystemInst {
   /// can process. The lexer will notify this system if it encounters one of
   /// these characters.9
   pub lex_filter: CharFilter,
+  pub const_root_id: TreeId,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/tree.rs

@@ -1,41 +1,53 @@
 use std::collections::HashMap;
+use std::num::NonZeroU64;
+use std::ops::Range;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::atom::Atom;
+use crate::atom::LocalAtom;
 use crate::expr::Expr;
-use crate::intern::TStr;
-use crate::location::SourceRange;
+use crate::interner::TStr;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+/// A token tree from a lexer recursion request. Its lifetime is the lex call,
+/// the lexer can include it in its output or discard it by implication.
+///
+/// Similar to [crate::expr::ExprTicket] in that it represents a token tree the
+/// lifetime of which is managed by the interpreter.
+pub type TreeTicket = NonZeroU64;
+
+#[derive(Clone, Debug, Coding)]
 pub struct TokenTree {
-  token: Token,
-  location: SourceRange,
+  pub token: Token,
+  pub range: Range<u32>,
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub enum Token {
   /// Lambda function. The number operates as an argument name
-  Lambda(TStr, Vec<TokenTree>),
+  Lambda(Vec<TokenTree>, Vec<TokenTree>),
   Name(Vec<TStr>),
   S(Paren, Vec<TokenTree>),
   /// A placeholder in a macro. This variant is forbidden everywhere outside
   /// line parser output
   Ph(Placeholder),
-  Atom(Atom),
+  Atom(LocalAtom),
+  Slot(TreeTicket),
+  /// A static compile-time error returned by erroring lexers if
+  /// the rest of the source is likely still meaningful
+  Bottom(String),
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct Placeholder {
-  name: TStr,
-  kind: PlaceholderKind,
+  pub name: TStr,
+  pub kind: PlaceholderKind,
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub enum PlaceholderKind {
   Scalar,
   Name,
@@ -56,11 +68,14 @@ pub struct MacroRule {
   pub template: Vec<TokenTree>,
 }
 
+pub type TreeId = NonZeroU64;
+
 #[derive(Clone, Debug, Coding)]
 pub enum Tree {
   Const(Expr),
   Mod(TreeModule),
   Rule(MacroRule),
+  Lazy(TreeId),
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -68,9 +83,9 @@ pub struct TreeModule {
   pub children: HashMap<String, Tree>,
 }
 
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
-pub struct GetConstTree(pub SysId);
+pub struct GetConstTree(pub SysId, pub TreeId);
 impl Request for GetConstTree {
-  type Response = TreeModule;
+  type Response = Tree;
 }

orchid-api/src/vfs.rs

@@ -1,14 +1,15 @@
 use std::collections::HashMap;
+use std::num::NonZeroU16;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::error::ProjResult;
-use crate::intern::TStr;
+use crate::interner::TStr;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
 
-pub type VfsId = u16;
+pub type VfsId = NonZeroU16;
 
 #[derive(Clone, Debug, Coding)]
 pub enum Loaded {