very elegant extension API and parts of it used in std as POC

.gitignore

@@ -10,4 +10,5 @@
 *.gz
 *.html
 *.png
+/.VSCodeCounter
 !notes/papers/report/template/Figures/**

.vscode/settings.json

@@ -1,4 +1,8 @@
 {
     "rust-analyzer.check.command": "check",
-    "rust-analyzer.rustfmt.extraArgs": ["+nightly"]
+    "rust-analyzer.rustfmt.extraArgs": ["+nightly"],
+    "[rust]": {
+        "editor.tabSize": 2,
+        "editor.insertSpaces": true,
+    }
 }

Cargo.lock

@@ -42,6 +42,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "const_panic"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6051f239ecec86fde3410901ab7860d458d160371533842974fc61f96d15879b"
+
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -193,13 +199,40 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "itertools"
-version = "0.12.1"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
 dependencies = [
  "either",
 ]
 
+[[package]]
+name = "konst"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50a0ba6de5f7af397afff922f22c149ff605c766cd3269cf6c1cd5e466dbe3b9"
+dependencies = [
+ "const_panic",
+ "konst_kernel",
+ "konst_proc_macros",
+ "typewit",
+]
+
+[[package]]
+name = "konst_kernel"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be0a455a1719220fd6adf756088e1c69a85bf14b6a9e24537a5cc04f503edb2b"
+dependencies = [
+ "typewit",
+]
+
+[[package]]
+name = "konst_proc_macros"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e28ab1dc35e09d60c2b8c90d12a9a8d9666c876c10a3739a3196db0103b6043"
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -288,13 +321,17 @@ version = "0.1.0"
 dependencies = [
  "ahash",
  "derive_destructure",
+ "dyn-clone",
  "hashbrown",
  "itertools",
+ "konst",
  "never",
  "orchid-api",
+ "orchid-api-derive",
  "orchid-api-traits",
  "orchid-base",
  "ordered-float",
+ "paste",
  "substack",
  "trait-set",
  "typeid",
@@ -317,6 +354,14 @@ dependencies = [
 [[package]]
 name = "orchid-std"
 version = "0.1.0"
+dependencies = [
+ "itertools",
+ "orchid-api",
+ "orchid-api-derive",
+ "orchid-api-traits",
+ "orchid-base",
+ "orchid-extension",
+]
 
 [[package]]
 name = "orcx"
@@ -331,6 +376,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "proc-macro2"
 version = "0.4.30"
@@ -508,6 +559,21 @@ version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
+[[package]]
+name = "typewit"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6fb9ae6a3cafaf0a5d14c2302ca525f9ae8e07a0f0e6949de88d882c37a6e24"
+dependencies = [
+ "typewit_proc_macros",
+]
+
+[[package]]
+name = "typewit_proc_macros"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e36a83ea2b3c704935a01b4642946aadd445cea40b10935e3f8bd8052b8193d6"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.12"

Cargo.toml

@@ -2,8 +2,13 @@
 resolver = "2"
 
 members = [
-  # "orchidlang",
   "orcx",
+  "orchid-std",
+  "orchid-host",
+  "orchid-extension",
+  "orchid-base",
   "orchid-api",
-  "orchid-std", "orchid-base", "orchid-api-derive", "orchid-api-traits", "stdio-perftest", "orchid-extension", "orchid-host",
+  "orchid-api-derive",
+  "orchid-api-traits",
+  "stdio-perftest",
 ]

orchid-api-derive/Cargo.toml

@@ -14,4 +14,4 @@ syn = { version = "2.0.52" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 proc-macro2 = "1.0.78"
 darling = "0.20.8"
-itertools = "0.12.1"
+itertools = "0.13.0"

orchid-api-derive/src/encode.rs

@@ -8,7 +8,7 @@ use crate::common::{add_trait_bounds, destructure, pos_field_name};
 pub fn derive(input: TokenStream) -> TokenStream {
   // Parse the input tokens into a syntax tree
   let input = parse_macro_input!(input as syn::DeriveInput);
-  let e_generics = add_trait_bounds(input.generics, parse_quote!(orchid_api_traits::Decode));
+  let e_generics = add_trait_bounds(input.generics, parse_quote!(orchid_api_traits::Encode));
   let (e_impl_generics, e_ty_generics, e_where_clause) = e_generics.split_for_impl();
   let name = input.ident;
   let encode = encode_body(&input.data);

orchid-api-traits/src/coding.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::collections::HashMap;
 use std::hash::Hash;
 use std::io::{Read, Write};
@@ -282,6 +283,15 @@ smart_ptr!(Arc);
 smart_ptr!(Rc);
 smart_ptr!(Box);
 
+impl<'a, T: ?Sized + ToOwned> Decode for Cow<'a, T>
+where T::Owned: Decode
+{
+  fn decode<R: Read + ?Sized>(read: &mut R) -> Self { Cow::Owned(T::Owned::decode(read)) }
+}
+impl<'a, T: ?Sized + Encode + ToOwned> Encode for Cow<'a, T> {
+  fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
+}
+
 impl Decode for char {
   fn decode<R: Read + ?Sized>(read: &mut R) -> Self { char::from_u32(u32::decode(read)).unwrap() }
 }

orchid-api/src/atom.rs

@@ -1,6 +1,7 @@
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
+use crate::error::ProjResult;
 use crate::expr::{Expr, ExprTicket};
 use crate::proto::{ExtHostReq, HostExtNotif, HostExtReq};
 use crate::system::SysId;
@@ -81,6 +82,18 @@ impl Request for Fwd {
   type Response = Vec<u8>;
 }
 
+#[derive(Clone, Debug, Coding)]
+pub enum NextStep {
+  Continue(Expr),
+  Halt,
+}
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct Command(pub Atom);
+impl Request for Command {
+  type Response = ProjResult<NextStep>;
+}
+
 /// Notification that an atom is being dropped because its associated expression
 /// isn't referenced anywhere. This should have no effect if the atom's `drop`
 /// flag is false.
@@ -96,4 +109,5 @@ pub enum AtomReq {
   FinalCall(FinalCall),
   AtomSame(AtomSame),
   Fwded(Fwded),
+  Command(Command),
 }

orchid-api/src/error.rs

@@ -1,17 +1,23 @@
-use orchid_api_derive::Coding;
+use std::num::NonZeroU16;
+use std::sync::Arc;
 
-use crate::intern::TStr;
+use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_traits::Request;
+
+use crate::interner::TStr;
 use crate::location::Location;
+use crate::proto::{ExtHostNotif, ExtHostReq};
+use crate::system::SysId;
 
-pub type ProjErrId = u16;
+pub type ProjErrId = NonZeroU16;
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct ProjErrLocation {
   /// Description of the relation of this location to the error. If not used,
   /// set to empty string
-  message: String,
+  pub message: Arc<String>,
   /// Location in code where the error emerged. This is usually [Location::Gen].
-  location: Location,
+  pub location: Location,
 }
 
 /// Programming errors raised by extensions. At runtime these produce the
@@ -19,17 +25,58 @@ pub struct ProjErrLocation {
 /// fallible operations should return an Orchid result and not a bottom.
 /// For example, file reading produces result::err when the file doesn't exist,
 /// and a bottom if the file name isn't a string.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct ProjErr {
   /// General description of the kind of error.
-  description: TStr,
+  pub description: TStr,
   /// Specific information about the exact error, preferably containing concrete
   /// values.
-  message: String,
+  pub message: Arc<String>,
   /// Specific code fragments that have contributed to the emergence of the
   /// error.
-  locations: Vec<ProjErrLocation>,
+  pub locations: Vec<ProjErrLocation>,
+}
+
+/// When the interpreter encounters an error while serving a system's request,
+/// it sends this error as an ID back to the system to save bandwidth.
+/// The lifetime of this ID is the request being served, the receiving system
+/// can return it and query its details with [GetDetails].
+#[derive(Clone, Debug, Coding)]
+pub enum ProjErrOrRef {
+  New(ProjErr),
+  Known(ProjErrId),
 }
 
 /// If this is an [`Err`] then the [`Vec`] must not be empty.
-pub type ProjResult<T> = Result<T, Vec<ProjErr>>;
+pub type ProjResult<T> = Result<T, Vec<ProjErrOrRef>>;
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ProjErrReq, ExtHostReq)]
+pub struct GetErrorDetails(pub ProjErrId);
+impl Request for GetErrorDetails {
+  type Response = ProjErr;
+}
+
+/// Notify the host about an error without being forced to return said error.
+/// This will still count as an error, but later operations that depend on the
+/// value returned by the currently running function will get to run
+///
+/// The error is not connected to the place it was raised, since multiple calls
+/// can be issued to a system at the same time
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ErrNotif, ExtHostNotif)]
+pub struct ReportError(pub SysId, pub ProjErrOrRef);
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostReq)]
+#[extendable]
+pub enum ProjErrReq {
+  GetErrorDetails(GetErrorDetails),
+}
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostNotif)]
+#[extendable]
+pub enum ErrNotif {
+  ReportError(ReportError),
+}

orchid-api/src/expr.rs

@@ -1,18 +1,20 @@
+use std::num::NonZeroU64;
+
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::atom::LocalAtom;
+use crate::atom::{Atom, LocalAtom};
-use crate::intern::{TStr, TStrv};
+use crate::interner::TStrv;
 use crate::location::Location;
 use crate::proto::{ExtHostNotif, ExtHostReq};
 use crate::system::SysId;
 
 /// An arbitrary ID associated with an expression on the host side. Incoming
 /// tickets always come with some lifetime guarantee, which can be extended with
-/// [AcquireExpr].
+/// [Acquire].
 ///
 /// The ID is globally unique within its lifetime, but may be reused.
-pub type ExprTicket = u64;
+pub type ExprTicket = NonZeroU64;
 
 /// Acquire a strong reference to an expression. This keeps it alive until a
 /// corresponding [Release] is emitted. The number of times a system has
@@ -52,15 +54,15 @@ pub struct Relocate {
 /// A description of a new expression. It is used as the return value of
 /// [crate::atom::Call] or [crate::atom::CallRef], or a constant in the
 /// [crate::tree::Tree].
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub enum Clause {
   /// Apply the lhs as a function to the rhs
   Call(Box<Expr>, Box<Expr>),
   /// Lambda function. The number operates as an argument name
-  Lambda(TStr, Box<Expr>),
+  Lambda(u64, Box<Expr>),
   /// Binds the argument passed to the lambda with the same ID in the same
   /// template
-  Arg(TStr),
+  Arg(u64),
   /// Insert the specified host-expression in the template here. When the clause
   /// is used in the const tree, this variant is forbidden.
   Slot(ExprTicket),
@@ -73,22 +75,34 @@ pub enum Clause {
   /// Because the atom is newly constructed, it also must belong to this system.
   /// For convenience, [SysId::MAX] is also accepted as referring to this
   /// system.
-  Atom(LocalAtom),
+  NewAtom(LocalAtom),
+  /// An atom, specifically an atom that already exists. This form is only ever
+  /// returned from [Inspect], and it's borrowed from the expression being
+  /// inspected.
+  Atom(ExprTicket, Atom),
   /// A reference to a constant
   Const(TStrv),
+  /// A static runtime error.
+  Bottom(String),
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct Expr {
   pub clause: Clause,
   pub location: Location,
 }
 
+#[derive(Clone, Debug, Coding)]
+pub struct Details {
+  pub expr: Expr,
+  pub refcount: u32,
+}
+
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(ExprReq, ExtHostReq)]
 pub struct Inspect(pub ExprTicket);
 impl Request for Inspect {
-  type Response = Clause;
+  type Response = Details;
 }
 
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]

orchid-api/src/interner.rs

@@ -8,7 +8,7 @@ use crate::proto::{ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 #[extendable]
 pub enum IntReq {
@@ -18,9 +18,7 @@ pub enum IntReq {
   ExternStrv(ExternStrv),
 }
 
-/// replica -> master to intern a string on the master.
+/// replica -> master to intern a string on the master. Repeatable.
-///
-/// Repeatable.
 ///
 /// See [IntReq]
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
@@ -35,7 +33,7 @@ impl Request for InternStr {
 /// Repeatable.
 ///
 /// See [IntReq]
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(IntReq, ExtHostReq)]
 pub struct ExternStr(pub TStr);
 impl Request for ExternStr {
@@ -46,7 +44,7 @@ impl Request for ExternStr {
 /// Repeatable.
 ///
 /// See [IntReq]
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(IntReq, ExtHostReq)]
 pub struct InternStrv(pub Arc<Vec<TStr>>);
 impl Request for InternStrv {
@@ -58,7 +56,7 @@ impl Request for InternStrv {
 /// Repeatable.
 ///
 /// See [IntReq]
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(IntReq, ExtHostReq)]
 pub struct ExternStrv(pub TStrv);
 impl Request for ExternStrv {
@@ -75,7 +73,7 @@ pub struct TStrv(pub NonZeroU64);
 
 /// A request to sweep the replica. The master will not be sweeped until all
 /// replicas respond, as it must retain everything the replicas retained
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct Sweep;
 impl Request for Sweep {

orchid-api/src/lib.rs

@@ -1,7 +1,7 @@
 pub mod atom;
 pub mod error;
 pub mod expr;
-pub mod intern;
+pub mod interner;
 pub mod location;
 pub mod parser;
 pub mod proto;

orchid-api/src/location.rs

@@ -2,26 +2,29 @@ use std::ops::Range;
 
 use orchid_api_derive::Coding;
 
-use crate::intern::{TStr, TStrv};
+use crate::interner::TStrv;
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub enum Location {
   None,
   /// Used in functions to denote the generated code that carries on the
   /// location of the call. Not allowed in the const tree.
   Inherit,
   Gen(CodeGenInfo),
-  Range(SourceRange),
+  /// Range and file
+  SourceRange(SourceRange),
+  /// Range only, file implied. Most notably used by parsers
+  Range(Range<u32>),
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct SourceRange {
   pub path: TStrv,
   pub range: Range<u32>,
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct CodeGenInfo {
-  pub generator: TStr,
+  pub generator: TStrv,
-  pub details: TStr,
+  pub details: String,
 }

orchid-api/src/parser.rs

@@ -1,13 +1,16 @@
+use std::num::NonZeroU64;
 use std::ops::RangeInclusive;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::error::ProjResult;
-use crate::intern::TStr;
+use crate::interner::TStr;
 use crate::proto::{ExtHostReq, HostExtReq};
 use crate::system::SysId;
-use crate::tree::TokenTree;
+use crate::tree::{TokenTree, TreeTicket};
+
+pub type LexId = NonZeroU64;
 
 /// - All ranges contain at least one character
 /// - All ranges are in increasing characeter order
@@ -26,6 +29,7 @@ pub enum ParserReq {
 #[extends(ParserReq, HostExtReq)]
 pub struct Lex {
   pub sys: SysId,
+  pub id: LexId,
   pub text: TStr,
   pub pos: u32,
 }
@@ -42,11 +46,17 @@ pub struct Lexed {
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct SubLex {
-  pub text: TStr,
+  pub id: LexId,
   pub pos: u32,
 }
 impl Request for SubLex {
-  type Response = ProjResult<Lexed>;
+  type Response = ProjResult<SubLexed>;
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct SubLexed {
+  pub pos: u32,
+  pub ticket: TreeTicket,
 }
 
 pub struct ParseLine {}

orchid-api/src/proto.rs

@@ -27,7 +27,7 @@ use std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{read_exact, write_exact, Channel, Decode, Encode, MsgSet, Request};
 
-use crate::{atom, expr, intern, parser, system, tree, vfs};
+use crate::{atom, error, expr, interner, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader;
@@ -69,10 +69,11 @@ impl Request for Ping {
 #[extendable]
 pub enum ExtHostReq {
   Ping(Ping),
-  IntReq(intern::IntReq),
+  IntReq(interner::IntReq),
   Fwd(atom::Fwd),
   ExprReq(expr::ExprReq),
   SubLex(parser::SubLex),
+  ProjErrReq(error::ProjErrReq),
 }
 
 /// Notifications sent from the extension to the host
@@ -81,6 +82,7 @@ pub enum ExtHostReq {
 #[extendable]
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
+  ErrNotif(error::ErrNotif),
 }
 
 pub struct ExtHostChannel;
@@ -95,7 +97,7 @@ impl Channel for ExtHostChannel {
 pub enum HostExtReq {
   Ping(Ping),
   NewSystem(system::NewSystem),
-  Sweep(intern::Sweep),
+  Sweep(interner::Sweep),
   AtomReq(atom::AtomReq),
   ParserReq(parser::ParserReq),
   GetConstTree(tree::GetConstTree),

orchid-api/src/system.rs

@@ -1,15 +1,18 @@
+use std::num::NonZeroU16;
+
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
 use crate::parser::CharFilter;
 use crate::proto::{HostExtNotif, HostExtReq};
+use crate::tree::TreeId;
 
 /// ID of a system type
-pub type SysDeclId = u16;
+pub type SysDeclId = NonZeroU16;
 
 /// ID of a system instance
-pub type SysId = u16;
+pub type SysId = NonZeroU16;
 
 /// Details about a system provided by this library
 #[derive(Debug, Clone, Coding)]
@@ -56,6 +59,7 @@ pub struct SystemInst {
   /// can process. The lexer will notify this system if it encounters one of
   /// these characters.9
   pub lex_filter: CharFilter,
+  pub const_root_id: TreeId,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/tree.rs

@@ -1,41 +1,53 @@
 use std::collections::HashMap;
+use std::num::NonZeroU64;
+use std::ops::Range;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::atom::Atom;
+use crate::atom::LocalAtom;
 use crate::expr::Expr;
-use crate::intern::TStr;
+use crate::interner::TStr;
-use crate::location::SourceRange;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+/// A token tree from a lexer recursion request. Its lifetime is the lex call,
+/// the lexer can include it in its output or discard it by implication.
+///
+/// Similar to [crate::expr::ExprTicket] in that it represents a token tree the
+/// lifetime of which is managed by the interpreter.
+pub type TreeTicket = NonZeroU64;
+
+#[derive(Clone, Debug, Coding)]
 pub struct TokenTree {
-  token: Token,
+  pub token: Token,
-  location: SourceRange,
+  pub range: Range<u32>,
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub enum Token {
   /// Lambda function. The number operates as an argument name
-  Lambda(TStr, Vec<TokenTree>),
+  Lambda(Vec<TokenTree>, Vec<TokenTree>),
   Name(Vec<TStr>),
   S(Paren, Vec<TokenTree>),
   /// A placeholder in a macro. This variant is forbidden everywhere outside
   /// line parser output
   Ph(Placeholder),
-  Atom(Atom),
+  Atom(LocalAtom),
+  Slot(TreeTicket),
+  /// A static compile-time error returned by erroring lexers if
+  /// the rest of the source is likely still meaningful
+  Bottom(String),
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub struct Placeholder {
-  name: TStr,
+  pub name: TStr,
-  kind: PlaceholderKind,
+  pub kind: PlaceholderKind,
 }
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
+#[derive(Clone, Debug, Coding)]
 pub enum PlaceholderKind {
   Scalar,
   Name,
@@ -56,11 +68,14 @@ pub struct MacroRule {
   pub template: Vec<TokenTree>,
 }
 
+pub type TreeId = NonZeroU64;
+
 #[derive(Clone, Debug, Coding)]
 pub enum Tree {
   Const(Expr),
   Mod(TreeModule),
   Rule(MacroRule),
+  Lazy(TreeId),
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -68,9 +83,9 @@ pub struct TreeModule {
   pub children: HashMap<String, Tree>,
 }
 
-#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
-pub struct GetConstTree(pub SysId);
+pub struct GetConstTree(pub SysId, pub TreeId);
 impl Request for GetConstTree {
-  type Response = TreeModule;
+  type Response = Tree;
 }

orchid-api/src/vfs.rs

@@ -1,14 +1,15 @@
 use std::collections::HashMap;
+use std::num::NonZeroU16;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
 use crate::error::ProjResult;
-use crate::intern::TStr;
+use crate::interner::TStr;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
 
-pub type VfsId = u16;
+pub type VfsId = NonZeroU16;
 
 #[derive(Clone, Debug, Coding)]
 pub enum Loaded {

orchid-base/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 derive_destructure = "1.0.0"
 dyn-clone = "1.0.17"
 hashbrown = "0.14.3"
-itertools = "0.12.1"
+itertools = "0.13.0"
 lazy_static = "1.4.0"
 never = "0.1.0"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }

orchid-base/src/box_cow.rs

@@ -0,0 +1,29 @@
+use std::borrow::Borrow;
+use std::ops::Deref;
+use std::sync::Arc;
+
+pub enum ArcCow<'a, T: ?Sized + ToOwned> {
+  Borrowed(&'a T),
+  Owned(Arc<T::Owned>),
+}
+impl<'a, T: ?Sized + ToOwned> ArcCow<'a, T> {
+  pub fn owned(value: T::Owned) -> Self { Self::Owned(Arc::new(value)) }
+}
+impl<'a, T: ?Sized + ToOwned> Clone for ArcCow<'a, T> {
+  fn clone(&self) -> Self {
+    match self {
+      Self::Borrowed(r) => Self::Borrowed(r),
+      Self::Owned(b) => Self::Owned(b.clone()),
+    }
+  }
+}
+
+impl<'a, T: ?Sized + ToOwned> Deref for ArcCow<'a, T> {
+  type Target = T;
+  fn deref(&self) -> &Self::Target {
+    match self {
+      Self::Borrowed(t) => t,
+      Self::Owned(b) => b.as_ref().borrow(),
+    }
+  }
+}

orchid-base/src/boxed_iter.rs

@@ -11,13 +11,12 @@ pub fn box_once<'a, T: 'a>(t: T) -> BoxedIter<'a, T> { Box::new(iter::once(t)) }
 pub fn box_empty<'a, T: 'a>() -> BoxedIter<'a, T> { Box::new(iter::empty()) }
 
 /// Chain various iterators into a [BoxedIter]
+#[macro_export]
 macro_rules! box_chain {
   ($curr:expr) => {
-    Box::new($curr) as BoxedIter<_>
+    Box::new($curr) as $crate::boxed_iter::BoxedIter<_>
   };
   ($curr:expr, $($rest:expr),*) => {
-    Box::new($curr$(.chain($rest))*) as $crate::utils::boxed_iter::BoxedIter<_>
+    Box::new($curr$(.chain($rest))*) as $crate::boxed_iter::BoxedIter<_>
   };
 }
-
-pub(crate) use box_chain;

orchid-base/src/error.rs

@@ -0,0 +1,48 @@
+use std::sync::Arc;
+
+use orchid_api::error::{ProjErr, ProjErrLocation};
+
+use crate::interner::{deintern, Tok};
+use crate::location::Pos;
+
+/// A point of interest in resolving the error, such as the point where
+/// processing got stuck, a command that is likely to be incorrect
+#[derive(Clone)]
+pub struct ErrorPosition {
+  /// The suspected origin
+  pub position: Pos,
+  /// Any information about the role of this origin
+  pub message: Option<Arc<String>>,
+}
+impl ErrorPosition {
+  pub fn from_api(pel: &ProjErrLocation) -> Self {
+    Self {
+      message: Some(pel.message.clone()).filter(|s| !s.is_empty()),
+      position: Pos::from_api(&pel.location),
+    }
+  }
+  pub fn to_api(&self) -> ProjErrLocation {
+    ProjErrLocation {
+      message: self.message.clone().unwrap_or_default(),
+      location: self.position.to_api(),
+    }
+  }
+}
+impl From<Pos> for ErrorPosition {
+  fn from(origin: Pos) -> Self { Self { position: origin, message: None } }
+}
+
+pub struct ErrorDetails {
+  pub description: Tok<String>,
+  pub message: Arc<String>,
+  pub locations: Vec<ErrorPosition>,
+}
+impl ErrorDetails {
+  pub fn from_api(err: &ProjErr) -> Self {
+    Self {
+      description: deintern(err.description),
+      message: err.message.clone(),
+      locations: err.locations.iter().map(ErrorPosition::from_api).collect(),
+    }
+  }
+}

orchid-base/src/event.rs

@@ -61,3 +61,7 @@ impl<T, U> Event<T, U> {
     }
   }
 }
+
+impl<T, U> Default for Event<T, U> {
+  fn default() -> Self { Self::new() }
+}

orchid-base/src/id_store.rs

@@ -1,4 +1,3 @@
-use std::fmt::Debug;
 use std::num::NonZeroU64;
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::{AtomicU64, Ordering};
@@ -12,17 +11,12 @@ pub struct IdStore<T> {
 }
 impl<T> IdStore<T> {
   pub const fn new() -> Self { Self { table: OnceLock::new(), id: AtomicU64::new(1) } }
-  pub fn add<R>(&self, t: T) -> R
+  pub fn add(&self, t: T) -> IdRecord<'_, T> {
-  where
-    NonZeroU64: TryInto<R>,
-    <NonZeroU64 as TryInto<R>>::Error: Debug,
-  {
     let tbl = self.table.get_or_init(Mutex::default);
     let mut tbl_g = tbl.lock().unwrap();
-    let id64: NonZeroU64 = self.id.fetch_add(1, Ordering::Relaxed).try_into().unwrap();
+    let id: NonZeroU64 = self.id.fetch_add(1, Ordering::Relaxed).try_into().unwrap();
-    let id: R = id64.try_into().expect("Keyspace exhausted");
+    assert!(tbl_g.insert(id, t).is_none(), "atom ID wraparound");
-    assert!(tbl_g.insert(id64, t).is_none(), "atom ID wraparound");
+    IdRecord(id, tbl_g)
-    id
   }
   pub fn get(&self, id: impl Into<NonZeroU64>) -> Option<IdRecord<'_, T>> {
     let tbl = self.table.get_or_init(Mutex::default);
@@ -38,6 +32,7 @@ impl<T> Default for IdStore<T> {
 
 pub struct IdRecord<'a, T>(NonZeroU64, MutexGuard<'a, HashMap<NonZeroU64, T>>);
 impl<'a, T> IdRecord<'a, T> {
+  pub fn id(&self) -> NonZeroU64 { self.0 }
   pub fn remove(mut self) -> T { self.1.remove(&self.0).unwrap() }
 }
 impl<'a, T> Deref for IdRecord<'a, T> {

orchid-base/src/interner.rs

@@ -7,56 +7,62 @@ use std::{fmt, hash};
 
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools as _;
-use orchid_api::intern::{
+use orchid_api::interner::{
   ExternStr, ExternStrv, IntReq, InternStr, InternStrv, Retained, TStr, TStrv,
 };
 use orchid_api_traits::Request;
 
 use crate::reqnot::{DynRequester, Requester};
 
+/// Clippy crashes while verifying `Tok: Sized` without this and I cba to create
+/// a minimal example
 #[derive(Clone)]
-pub struct Token<T: ?Sized + Interned> {
+struct ForceSized<T>(T);
+
+#[derive(Clone)]
+pub struct Tok<T: Interned> {
   data: Arc<T>,
-  marker: T::Marker,
+  marker: ForceSized<T::Marker>,
 }
-impl<T: Interned + ?Sized> Token<T> {
+impl<T: Interned> Tok<T> {
-  pub fn marker(&self) -> T::Marker { self.marker }
+  pub fn new(data: Arc<T>, marker: T::Marker) -> Self { Self { data, marker: ForceSized(marker) } }
+  pub fn marker(&self) -> T::Marker { self.marker.0 }
   pub fn arc(&self) -> Arc<T> { self.data.clone() }
 }
-impl<T: Interned + ?Sized> Deref for Token<T> {
+impl<T: Interned> Deref for Tok<T> {
   type Target = T;
 
   fn deref(&self) -> &Self::Target { self.data.as_ref() }
 }
-impl<T: Interned + ?Sized> Ord for Token<T> {
+impl<T: Interned> Ord for Tok<T> {
   fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.marker().cmp(&other.marker()) }
 }
-impl<T: Interned + ?Sized> PartialOrd for Token<T> {
+impl<T: Interned> PartialOrd for Tok<T> {
   fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { Some(self.cmp(other)) }
 }
-impl<T: Interned + ?Sized> Eq for Token<T> {}
+impl<T: Interned> Eq for Tok<T> {}
-impl<T: Interned + ?Sized> PartialEq for Token<T> {
+impl<T: Interned> PartialEq for Tok<T> {
   fn eq(&self, other: &Self) -> bool { self.cmp(other).is_eq() }
 }
-impl<T: Interned + ?Sized> hash::Hash for Token<T> {
+impl<T: Interned> hash::Hash for Tok<T> {
   fn hash<H: hash::Hasher>(&self, state: &mut H) { self.marker().hash(state) }
 }
-impl<T: Interned + ?Sized + fmt::Display> fmt::Display for Token<T> {
+impl<T: Interned + fmt::Display> fmt::Display for Tok<T> {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     write!(f, "{}", &*self.data)
   }
 }
-impl<T: Interned + ?Sized + fmt::Debug> fmt::Debug for Token<T> {
+impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     write!(f, "Token({} -> {:?})", self.marker().get_id(), self.data.as_ref())
   }
 }
 
 pub trait Interned: Eq + hash::Hash + Clone {
-  type Marker: InternMarker<Interned = Self>;
+  type Marker: InternMarker<Interned = Self> + Sized;
   fn intern(self: Arc<Self>, req: &(impl DynRequester<Transfer = IntReq> + ?Sized))
   -> Self::Marker;
-  fn bimap(interner: &mut Interner) -> &mut Bimap<Self>;
+  fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
 }
 
 pub trait Internable {
@@ -64,9 +70,9 @@ pub trait Internable {
   fn get_owned(&self) -> Arc<Self::Interned>;
 }
 
-pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash {
+pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash + Sized {
   type Interned: Interned<Marker = Self>;
-  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Token<Self::Interned>;
+  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Tok<Self::Interned>;
   fn get_id(self) -> NonZeroU64;
   fn from_id(id: NonZeroU64) -> Self;
 }
@@ -79,13 +85,13 @@ impl Interned for String {
   ) -> Self::Marker {
     req.request(InternStr(self))
   }
-  fn bimap(interner: &mut Interner) -> &mut Bimap<Self> { &mut interner.strings }
+  fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.strings }
 }
 
 impl InternMarker for TStr {
   type Interned = String;
-  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Token<Self::Interned> {
+  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Tok<Self::Interned> {
-    Token { marker: self, data: req.request(ExternStr(self)) }
+    Tok::new(req.request(ExternStr(self)), self)
   }
   fn get_id(self) -> NonZeroU64 { self.0 }
   fn from_id(id: NonZeroU64) -> Self { Self(id) }
@@ -101,7 +107,7 @@ impl Internable for String {
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_string()) }
 }
 
-impl Interned for Vec<Token<String>> {
+impl Interned for Vec<Tok<String>> {
   type Marker = TStrv;
   fn intern(
     self: Arc<Self>,
@@ -109,38 +115,38 @@ impl Interned for Vec<Token<String>> {
   ) -> Self::Marker {
     req.request(InternStrv(Arc::new(self.iter().map(|t| t.marker()).collect())))
   }
-  fn bimap(interner: &mut Interner) -> &mut Bimap<Self> { &mut interner.vecs }
+  fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.vecs }
 }
 
 impl InternMarker for TStrv {
-  type Interned = Vec<Token<String>>;
+  type Interned = Vec<Tok<String>>;
-  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Token<Self::Interned> {
+  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Tok<Self::Interned> {
     let data = Arc::new(req.request(ExternStrv(self)).iter().map(|m| deintern(*m)).collect_vec());
-    Token { marker: self, data }
+    Tok::new(data, self)
   }
   fn get_id(self) -> NonZeroU64 { self.0 }
   fn from_id(id: NonZeroU64) -> Self { Self(id) }
 }
 
-impl Internable for [Token<String>] {
+impl Internable for [Tok<String>] {
-  type Interned = Vec<Token<String>>;
+  type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_vec()) }
 }
 
-impl Internable for Vec<Token<String>> {
+impl Internable for Vec<Tok<String>> {
-  type Interned = Vec<Token<String>>;
+  type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_vec()) }
 }
 
 impl Internable for Vec<TStr> {
-  type Interned = Vec<Token<String>>;
+  type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> {
     Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
   }
 }
 
 impl Internable for [TStr] {
-  type Interned = Vec<Token<String>>;
+  type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> {
     Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
   }
@@ -151,27 +157,25 @@ const BASE_RC: usize = 3;
 
 #[test]
 fn base_rc_correct() {
-  let tok = Token { marker: TStr(1.try_into().unwrap()), data: Arc::new("foo".to_string()) };
+  let tok = Tok::new(Arc::new("foo".to_string()), TStr(1.try_into().unwrap()));
   let mut bimap = Bimap::default();
   bimap.insert(tok.clone());
   assert_eq!(Arc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the current instance");
 }
 
-pub struct Bimap<T: Interned + ?Sized> {
+pub struct Bimap<T: Interned> {
-  intern: HashMap<Arc<T>, Token<T>>,
+  intern: HashMap<Arc<T>, Tok<T>>,
-  by_id: HashMap<T::Marker, Token<T>>,
+  by_id: HashMap<T::Marker, Tok<T>>,
 }
-impl<T: Interned + ?Sized> Bimap<T> {
+impl<T: Interned> Bimap<T> {
-  pub fn insert(&mut self, token: Token<T>) {
+  pub fn insert(&mut self, token: Tok<T>) {
     self.intern.insert(token.data.clone(), token.clone());
     self.by_id.insert(token.marker(), token);
   }
 
-  pub fn by_marker(&self, marker: T::Marker) -> Option<Token<T>> {
+  pub fn by_marker(&self, marker: T::Marker) -> Option<Tok<T>> { self.by_id.get(&marker).cloned() }
-    self.by_id.get(&marker).cloned()
-  }
 
-  pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Token<T>>
+  pub fn by_value<Q: Eq + hash::Hash>(&self, q: &Q) -> Option<Tok<T>>
   where T: Borrow<Q> {
     (self.intern.raw_entry())
       .from_hash(self.intern.hasher().hash_one(q), |k| k.as_ref().borrow() == q)
@@ -193,7 +197,7 @@ impl<T: Interned + ?Sized> Bimap<T> {
   }
 }
 
-impl<T: Interned + ?Sized> Default for Bimap<T> {
+impl<T: Interned> Default for Bimap<T> {
   fn default() -> Self { Self { by_id: HashMap::new(), intern: HashMap::new() } }
 }
 
@@ -202,9 +206,14 @@ pub trait UpComm {
 }
 
 #[derive(Default)]
-pub struct Interner {
+pub struct TypedInterners {
   strings: Bimap<String>,
-  vecs: Bimap<Vec<Token<String>>>,
+  vecs: Bimap<Vec<Tok<String>>>,
+}
+
+#[derive(Default)]
+pub struct Interner {
+  interners: TypedInterners,
   master: Option<Box<dyn DynRequester<Transfer = IntReq>>>,
 }
 
@@ -231,32 +240,36 @@ pub fn init_replica(req: impl DynRequester<Transfer = IntReq> + 'static) {
   let mut g = INTERNER.lock().unwrap();
   assert!(g.is_none(), "Attempted to initialize replica interner after first use");
   *g = Some(Interner {
-    strings: Bimap::default(),
-    vecs: Bimap::default(),
     master: Some(Box::new(req)),
+    interners: TypedInterners { strings: Bimap::default(), vecs: Bimap::default() },
   })
 }
 
-pub fn intern<T: Interned>(t: &(impl Internable<Interned = T> + ?Sized)) -> Token<T> {
+pub fn intern<T: Interned>(t: &(impl Internable<Interned = T> + ?Sized)) -> Tok<T> {
   let mut g = interner();
   let data = t.get_owned();
-  let marker = (g.master.as_mut()).map_or_else(
+  let typed = T::bimap(&mut g.interners);
-    || T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
+  if let Some(tok) = typed.by_value(&data) {
-    |c| data.clone().intern(&**c),
+    return tok;
-  );
+  }
-  let tok = Token { marker, data };
+  let marker = match &mut g.master {
-  T::bimap(&mut g).insert(tok.clone());
+    Some(c) => data.clone().intern(&**c),
+    None =>
+      T::Marker::from_id(NonZeroU64::new(ID.fetch_add(1, atomic::Ordering::Relaxed)).unwrap()),
+  };
+  let tok = Tok::new(data, marker);
+  T::bimap(&mut g.interners).insert(tok.clone());
   tok
 }
 
-pub fn deintern<M: InternMarker>(marker: M) -> Token<M::Interned> {
+pub fn deintern<M: InternMarker>(marker: M) -> Tok<M::Interned> {
   let mut g = interner();
-  if let Some(tok) = M::Interned::bimap(&mut g).by_marker(marker) {
+  if let Some(tok) = M::Interned::bimap(&mut g.interners).by_marker(marker) {
     return tok;
   }
   let master = g.master.as_mut().expect("ID not in local interner and this is master");
   let token = marker.resolve(&**master);
-  M::Interned::bimap(&mut g).insert(token.clone());
+  M::Interned::bimap(&mut g.interners).insert(token.clone());
   token
 }
 
@@ -268,14 +281,7 @@ pub fn merge_retained(into: &mut Retained, from: &Retained) {
 pub fn sweep_replica() -> Retained {
   let mut g = interner();
   assert!(g.master.is_some(), "Not a replica");
-  Retained { strings: g.strings.sweep_replica(), vecs: g.vecs.sweep_replica() }
+  Retained { strings: g.interners.strings.sweep_replica(), vecs: g.interners.vecs.sweep_replica() }
-}
-
-pub fn sweep_master(retained: Retained) {
-  let mut g = interner();
-  assert!(g.master.is_none(), "Not master");
-  g.strings.sweep_master(retained.strings.into_iter().collect());
-  g.vecs.sweep_master(retained.vecs.into_iter().collect());
 }
 
 /// Create a thread-local token instance and copy it. This ensures that the
@@ -286,17 +292,33 @@ pub fn sweep_master(retained: Retained) {
 macro_rules! intern {
   ($ty:ty : $expr:expr) => {{
     thread_local! {
-      static VALUE: $crate::intern::Token<<$ty as $crate::intern::Internable>::Interned>
+      static VALUE: $crate::interner::Tok<<$ty as $crate::interner::Internable>::Interned>
-        = $crate::intern::intern($expr as &$ty);
+        = $crate::interner::intern::<
+            <$ty as $crate::interner::Internable>::Interned
+          >($expr as &$ty);
     }
     VALUE.with(|v| v.clone())
   }};
 }
 
-#[allow(unused)]
+pub fn sweep_master(retained: Retained) {
+  eprintln!(
+    "Hello, {:?}",
+    intern!([Tok<String>]: &[
+      intern!(str: "bar"),
+      intern!(str: "baz")
+    ])
+  );
+  let mut g = interner();
+  assert!(g.master.is_none(), "Not master");
+  g.interners.strings.sweep_master(retained.strings.into_iter().collect());
+  g.interners.vecs.sweep_master(retained.vecs.into_iter().collect());
+}
+
+#[test]
 fn test_i() {
-  let _: Token<String> = intern!(str: "foo");
+  let _: Tok<String> = intern!(str: "foo");
-  let _: Token<Vec<Token<String>>> = intern!([Token<String>]: &[
+  let _: Tok<Vec<Tok<String>>> = intern!([Tok<String>]: &[
     intern!(str: "bar"),
     intern!(str: "baz")
   ]);

orchid-base/src/lib.rs

@@ -5,14 +5,15 @@ pub mod event;
 pub mod msg;
 // pub mod gen;
 pub mod api_utils;
+pub mod box_cow;
 pub mod char_filter;
+pub mod error;
 pub mod id_store;
-pub mod intern;
+pub mod interner;
 pub mod join;
 pub mod location;
 pub mod name;
-pub mod proj_error;
 pub mod reqnot;
 pub mod sequence;
 pub mod tree;
-pub mod virt_fs;
+// pub mod virt_fs;

orchid-base/src/location.rs

@@ -5,88 +5,99 @@ use std::hash::Hash;
 use std::ops::Range;
 use std::sync::Arc;
 
-use itertools::Itertools;
+use orchid_api::location as api;
+use trait_set::trait_set;
 
+use crate::interner::{deintern, Tok};
 use crate::name::Sym;
 use crate::sym;
 
-/// A full source code unit, such as a source file
+trait_set! {
-#[derive(Clone, Eq)]
+  pub trait GetSrc = FnMut(&Sym) -> Tok<String>;
-pub struct SourceCode {
-  pub(crate) path: Sym,
-  pub(crate) text: Arc<String>,
 }
-impl SourceCode {
+
-  /// Create a new source file description
+#[derive(Debug, Clone)]
-  pub fn new(path: Sym, source: Arc<String>) -> Self { Self { path, text: source } }
+pub enum Pos {
-  /// Location the source code was loaded from in the virtual tree
+  None,
-  pub fn path(&self) -> Sym { self.path.clone() }
+  /// Used in functions to denote the generated code that carries on the
-  /// Raw source code string
+  /// location of the call. Not allowed in the const tree.
-  pub fn text(&self) -> Arc<String> { self.text.clone() }
+  Inherit,
+  Gen(CodeGenInfo),
+  /// Range and file
+  SourceRange(SourceRange),
+  /// Range only, file implied. Most notably used by parsers
+  Range(Range<u32>),
 }
-impl PartialEq for SourceCode {
+impl Pos {
-  fn eq(&self, other: &Self) -> bool { self.path == other.path }
+  pub fn to_api(&self) -> api::Location {
-}
+    match self {
-impl Hash for SourceCode {
+      Self::Inherit => api::Location::Inherit,
-  fn hash<H: std::hash::Hasher>(&self, state: &mut H) { self.path.hash(state) }
+      Self::None => api::Location::None,
-}
+      Self::Range(r) => api::Location::Range(r.clone()),
-impl fmt::Debug for SourceCode {
+      Self::Gen(cgi) => api::Location::Gen(cgi.to_api()),
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeInfo({self})") }
+      Self::SourceRange(sr) => api::Location::SourceRange(sr.to_api()),
-}
+    }
-impl fmt::Display for SourceCode {
+  }
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+  pub fn from_api(loc: &api::Location) -> Self {
-    write!(f, "{}.orc", self.path.str_iter().join("/"))
+    match loc {
+      api::Location::Inherit => Self::Inherit,
+      api::Location::None => Self::None,
+      api::Location::Range(r) => Self::Range(r.clone()),
+      api::Location::Gen(cgi) => CodeGenInfo::from_api(cgi).location(),
+      api::Location::SourceRange(sr) => SourceRange::from_api(sr).location(),
+    }
+  }
+  pub fn pretty_print(&self, get_src: &mut impl GetSrc) -> String {
+    match self {
+      Self::Gen(g) => g.to_string(),
+      Self::SourceRange(sr) => sr.pretty_print(&get_src(&sr.path)),
+      // Can't pretty print partial and meta-location
+      other => format!("{other:?}"),
+    }
   }
-}
-impl AsRef<str> for SourceCode {
-  fn as_ref(&self) -> &str { &self.text }
 }
 
 /// Exact source code location. Includes where the code was loaded from, what
 /// the original source code was, and a byte range.
-#[derive(Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct SourceRange {
-  pub(crate) code: SourceCode,
+  pub(crate) path: Sym,
-  pub(crate) range: Range<usize>,
+  pub(crate) range: Range<u32>,
 }
 impl SourceRange {
-  /// Create a new instance.
+  pub fn new(range: &Range<u32>, path: &Sym) -> Self {
-  pub fn new(range: Range<usize>, code: SourceCode) -> Self { Self { code, range } }
+    Self { range: range.clone(), path: path.clone() }
+  }
+  pub fn to_api(&self) -> api::SourceRange {
+    api::SourceRange { path: self.path.tok().marker(), range: self.range.clone() }
+  }
+  pub fn from_api(sr: &api::SourceRange) -> Self {
+    Self { path: Sym::from_tok(deintern(sr.path)).unwrap(), range: sr.range.clone() }
+  }
   /// Create a dud [SourceRange] for testing. Its value is unspecified and
   /// volatile.
-  pub fn mock() -> Self { Self::new(0..1, SourceCode::new(sym!(test), Arc::new(String::new()))) }
+  pub fn mock() -> Self { Self { range: 0..1, path: sym!(test) } }
-  /// Source code
-  pub fn code(&self) -> SourceCode { self.code.clone() }
-  /// Source text
-  pub fn text(&self) -> Arc<String> { self.code.text.clone() }
   /// Path the source text was loaded from
-  pub fn path(&self) -> Sym { self.code.path.clone() }
+  pub fn path(&self) -> Sym { self.path.clone() }
   /// Byte range
-  pub fn range(&self) -> Range<usize> { self.range.clone() }
+  pub fn range(&self) -> Range<u32> { self.range.clone() }
   /// 0-based index of first byte
-  pub fn start(&self) -> usize { self.range.start }
+  pub fn start(&self) -> u32 { self.range.start }
   /// 0-based index of last byte + 1
-  pub fn end(&self) -> usize { self.range.end }
+  pub fn end(&self) -> u32 { self.range.end }
   /// Syntactic location
-  pub fn origin(&self) -> CodeOrigin { CodeOrigin::Source(self.clone()) }
+  pub fn location(&self) -> Pos { Pos::SourceRange(self.clone()) }
   /// Transform the numeric byte range
-  pub fn map_range(&self, map: impl FnOnce(Range<usize>) -> Range<usize>) -> Self {
+  pub fn map_range(&self, map: impl FnOnce(Range<u32>) -> Range<u32>) -> Self {
-    Self::new(map(self.range()), self.code())
+    Self { range: map(self.range()), path: self.path() }
   }
-}
+  pub fn pretty_print(&self, src: &str) -> String {
-impl fmt::Debug for SourceRange {
+    let (sl, sc) = pos2lc(src, self.range.start);
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeRange({self})") }
+    let (el, ec) = pos2lc(src, self.range.end);
-}
+    match (el == sl, ec <= sc + 1) {
-impl fmt::Display for SourceRange {
+      (true, true) => format!("{sl}:{sc}"),
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+      (true, false) => format!("{sl}:{sc}..{ec}"),
-    let Self { code, range } = self;
+      (false, _) => format!("{sl}:{sc}..{el}:{ec}"),
-    let (sl, sc) = pos2lc(code.text.as_str(), range.start);
-    let (el, ec) = pos2lc(code.text.as_str(), range.end);
-    write!(f, "{code} {sl}:{sc}")?;
-    if el == sl {
-      if sc + 1 == ec { Ok(()) } else { write!(f, "..{ec}") }
-    } else {
-      write!(f, "..{el}:{ec}")
     }
   }
 }
@@ -106,6 +117,17 @@ impl CodeGenInfo {
   pub fn details(generator: Sym, details: impl AsRef<str>) -> Self {
     Self { generator, details: Arc::new(details.as_ref().to_string()) }
   }
+  /// Syntactic location
+  pub fn location(&self) -> Pos { Pos::Gen(self.clone()) }
+  pub fn to_api(&self) -> api::CodeGenInfo {
+    api::CodeGenInfo { generator: self.generator.tok().marker(), details: self.details.to_string() }
+  }
+  pub fn from_api(cgi: &api::CodeGenInfo) -> Self {
+    Self {
+      generator: Sym::from_tok(deintern(cgi.generator)).unwrap(),
+      details: Arc::new(cgi.details.clone()),
+    }
+  }
 }
 impl fmt::Debug for CodeGenInfo {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeGenInfo({self})") }
@@ -117,72 +139,9 @@ impl fmt::Display for CodeGenInfo {
   }
 }
 
-/// identifies a sequence of characters that contributed to the enclosing
-/// construct or the reason the code was generated for generated code.
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub enum CodeOrigin {
-  /// Character sequence
-  Source(SourceRange),
-  /// Generated construct
-  Gen(CodeGenInfo),
-}
-
-impl fmt::Display for CodeOrigin {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-    match self {
-      Self::Gen(info) => write!(f, "{info}"),
-      Self::Source(cr) => write!(f, "{cr}"),
-    }
-  }
-}
-
-impl fmt::Debug for CodeOrigin {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeOrigin({self})") }
-}
-
-/// Location data associated with any code fragment. Identifies where the code
-/// came from and where it resides in the tree.
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub struct CodeLocation {
-  pub(crate) origin: CodeOrigin,
-  pub(crate) module: Sym,
-}
-impl CodeLocation {
-  pub(crate) fn new_src(range: SourceRange, module: Sym) -> Self {
-    Self { origin: CodeOrigin::Source(range), module }
-  }
-  /// Create a location for generated code. The generator string must not be
-  /// empty. For code, the generator string must contain at least one `::`
-  pub fn new_gen(gen: CodeGenInfo) -> Self {
-    Self { module: gen.generator.clone(), origin: CodeOrigin::Gen(gen) }
-  }
-
-  /// Get the syntactic location
-  pub fn origin(&self) -> CodeOrigin { self.origin.clone() }
-  /// Get the name of the containing module
-  pub fn module(&self) -> Sym { self.module.clone() }
-}
-
-impl fmt::Display for CodeLocation {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-    if self.module[..].is_empty() {
-      write!(f, "global {}", self.origin)
-    } else {
-      write!(f, "{} in {}", self.origin, self.module)
-    }
-  }
-}
-
-impl fmt::Debug for CodeLocation {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CodeLocation({self})") }
-}
-
 #[must_use]
-fn pos2lc(s: &str, i: usize) -> (usize, usize) {
+fn pos2lc(s: &str, i: u32) -> (u32, u32) {
-  s.chars().take(i).fold(
+  s.chars()
-    (1, 1),
+    .take(i.try_into().unwrap())
-    |(line, col), char| {
+    .fold((1, 1), |(line, col), char| if char == '\n' { (line + 1, 1) } else { (line, col + 1) })
-      if char == '\n' { (line + 1, 1) } else { (line, col + 1) }
-    },
-  )
 }

orchid-base/src/name.rs

@@ -11,23 +11,23 @@ use std::{fmt, slice, vec};
 use itertools::Itertools;
 use trait_set::trait_set;
 
-use crate::intern::{intern, InternMarker, Token};
+use crate::interner::{intern, InternMarker, Tok};
 
 trait_set! {
   /// Traits that all name iterators should implement
-  pub trait NameIter = Iterator<Item = Token<String>> + DoubleEndedIterator + ExactSizeIterator;
+  pub trait NameIter = Iterator<Item = Tok<String>> + DoubleEndedIterator + ExactSizeIterator;
 }
 
 /// A borrowed name fragment which can be empty. See [VPath] for the owned
 /// variant.
 #[derive(Hash, PartialEq, Eq)]
 #[repr(transparent)]
-pub struct PathSlice([Token<String>]);
+pub struct PathSlice([Tok<String>]);
 impl PathSlice {
   /// Create a new [PathSlice]
-  pub fn new(slice: &[Token<String>]) -> &PathSlice {
+  pub fn new(slice: &[Tok<String>]) -> &PathSlice {
     // SAFETY: This is ok because PathSlice is #[repr(transparent)]
-    unsafe { &*(slice as *const [Token<String>] as *const PathSlice) }
+    unsafe { &*(slice as *const [Tok<String>] as *const PathSlice) }
   }
   /// Convert to an owned name fragment
   pub fn to_vpath(&self) -> VPath { VPath(self.0.to_vec()) }
@@ -57,7 +57,7 @@ impl PathSlice {
   pub fn is_empty(&self) -> bool { self.len() == 0 }
   /// Obtain a reference to the held slice. With all indexing traits shadowed,
   /// this is better done explicitly
-  pub fn as_slice(&self) -> &[Token<String>] { self }
+  pub fn as_slice(&self) -> &[Tok<String>] { self }
   /// Global empty path slice
   pub fn empty() -> &'static Self { PathSlice::new(&[]) }
 }
@@ -69,12 +69,12 @@ impl fmt::Display for PathSlice {
     write!(f, "{}", self.str_iter().join("::"))
   }
 }
-impl Borrow<[Token<String>]> for PathSlice {
+impl Borrow<[Tok<String>]> for PathSlice {
-  fn borrow(&self) -> &[Token<String>] { &self.0 }
+  fn borrow(&self) -> &[Tok<String>] { &self.0 }
 }
 impl<'a> IntoIterator for &'a PathSlice {
-  type IntoIter = Cloned<slice::Iter<'a, Token<String>>>;
+  type IntoIter = Cloned<slice::Iter<'a, Tok<String>>>;
-  type Item = Token<String>;
+  type Item = Tok<String>;
   fn into_iter(self) -> Self::IntoIter { self.0.iter().cloned() }
 }
 
@@ -82,10 +82,10 @@ mod idx_impls {
   use std::ops;
 
   use super::PathSlice;
-  use crate::intern::Token;
+  use crate::interner::Tok;
 
   impl ops::Index<usize> for PathSlice {
-    type Output = Token<String>;
+    type Output = Tok<String>;
     fn index(&self, index: usize) -> &Self::Output { &self.0[index] }
   }
   macro_rules! impl_range_index_for_pathslice {
@@ -106,27 +106,27 @@ mod idx_impls {
 }
 
 impl Deref for PathSlice {
-  type Target = [Token<String>];
+  type Target = [Tok<String>];
 
   fn deref(&self) -> &Self::Target { &self.0 }
 }
-impl Borrow<PathSlice> for [Token<String>] {
+impl Borrow<PathSlice> for [Tok<String>] {
   fn borrow(&self) -> &PathSlice { PathSlice::new(self) }
 }
-impl<const N: usize> Borrow<PathSlice> for [Token<String>; N] {
+impl<const N: usize> Borrow<PathSlice> for [Tok<String>; N] {
   fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
 }
-impl Borrow<PathSlice> for Vec<Token<String>> {
+impl Borrow<PathSlice> for Vec<Tok<String>> {
   fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
 }
 
 /// A token path which may be empty. [VName] is the non-empty,
 /// [PathSlice] is the borrowed version
 #[derive(Clone, Default, Hash, PartialEq, Eq)]
-pub struct VPath(pub Vec<Token<String>>);
+pub struct VPath(pub Vec<Tok<String>>);
 impl VPath {
   /// Collect segments into a vector
-  pub fn new(items: impl IntoIterator<Item = Token<String>>) -> Self {
+  pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Self {
     Self(items.into_iter().collect())
   }
   /// Number of path segments
@@ -135,11 +135,11 @@ impl VPath {
   /// valid name
   pub fn is_empty(&self) -> bool { self.len() == 0 }
   /// Prepend some tokens to the path
-  pub fn prefix(self, items: impl IntoIterator<Item = Token<String>>) -> Self {
+  pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
     Self(items.into_iter().chain(self.0).collect())
   }
   /// Append some tokens to the path
-  pub fn suffix(self, items: impl IntoIterator<Item = Token<String>>) -> Self {
+  pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
     Self(self.0.into_iter().chain(items).collect())
   }
   /// Partition the string by `::` namespace separators
@@ -154,12 +154,12 @@ impl VPath {
   pub fn into_name(self) -> Result<VName, EmptyNameError> { VName::new(self.0) }
   /// Add a token to the path. Since now we know that it can't be empty, turn it
   /// into a name.
-  pub fn name_with_prefix(self, name: Token<String>) -> VName {
+  pub fn name_with_prefix(self, name: Tok<String>) -> VName {
     VName(self.into_iter().chain([name]).collect())
   }
   /// Add a token to the beginning of the. Since now we know that it can't be
   /// empty, turn it into a name.
-  pub fn name_with_suffix(self, name: Token<String>) -> VName {
+  pub fn name_with_suffix(self, name: Tok<String>) -> VName {
     VName([name].into_iter().chain(self).collect())
   }
 
@@ -182,18 +182,18 @@ impl fmt::Display for VPath {
     write!(f, "{}", self.str_iter().join("::"))
   }
 }
-impl FromIterator<Token<String>> for VPath {
+impl FromIterator<Tok<String>> for VPath {
-  fn from_iter<T: IntoIterator<Item = Token<String>>>(iter: T) -> Self {
+  fn from_iter<T: IntoIterator<Item = Tok<String>>>(iter: T) -> Self {
     Self(iter.into_iter().collect())
   }
 }
 impl IntoIterator for VPath {
-  type Item = Token<String>;
+  type Item = Tok<String>;
   type IntoIter = vec::IntoIter<Self::Item>;
   fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
-impl Borrow<[Token<String>]> for VPath {
+impl Borrow<[Tok<String>]> for VPath {
-  fn borrow(&self) -> &[Token<String>] { self.0.borrow() }
+  fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
 }
 impl Borrow<PathSlice> for VPath {
   fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
@@ -218,39 +218,40 @@ where PathSlice: Index<T>
 /// See also [Sym] for the immutable representation, and [VPath] for possibly
 /// empty values
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct VName(Vec<Token<String>>);
+pub struct VName(Vec<Tok<String>>);
 impl VName {
   /// Assert that the sequence isn't empty and wrap it in [VName] to represent
   /// this invariant
-  pub fn new(items: impl IntoIterator<Item = Token<String>>) -> Result<Self, EmptyNameError> {
+  pub fn new(items: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
     let data: Vec<_> = items.into_iter().collect();
     if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
   }
   /// Unwrap the enclosed vector
-  pub fn into_vec(self) -> Vec<Token<String>> { self.0 }
+  pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
   /// Get a reference to the enclosed vector
-  pub fn vec(&self) -> &Vec<Token<String>> { &self.0 }
+  pub fn vec(&self) -> &Vec<Tok<String>> { &self.0 }
   /// Mutable access to the underlying vector. To ensure correct results, this
   /// must never be empty.
-  pub fn vec_mut(&mut self) -> &mut Vec<Token<String>> { &mut self.0 }
+  pub fn vec_mut(&mut self) -> &mut Vec<Tok<String>> { &mut self.0 }
   /// Intern the name and return a [Sym]
   pub fn to_sym(&self) -> Sym { Sym(intern(&self.0[..])) }
   /// If this name has only one segment, return it
-  pub fn as_root(&self) -> Option<Token<String>> { self.0.iter().exactly_one().ok().cloned() }
+  pub fn as_root(&self) -> Option<Tok<String>> { self.0.iter().exactly_one().ok().cloned() }
   /// Prepend the segments to this name
   #[must_use = "This is a pure function"]
-  pub fn prefix(self, items: impl IntoIterator<Item = Token<String>>) -> Self {
+  pub fn prefix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
     Self(items.into_iter().chain(self.0).collect())
   }
   /// Append the segments to this name
   #[must_use = "This is a pure function"]
-  pub fn suffix(self, items: impl IntoIterator<Item = Token<String>>) -> Self {
+  pub fn suffix(self, items: impl IntoIterator<Item = Tok<String>>) -> Self {
     Self(self.0.into_iter().chain(items).collect())
   }
   /// Read a `::` separated namespaced name
   pub fn parse(s: &str) -> Result<Self, EmptyNameError> { Self::new(VPath::parse(s)) }
+  pub fn literal(s: &'static str) -> Self { Self::parse(s).expect("empty literal !?") }
   /// Obtain an iterator over the segments of the name
-  pub fn iter(&self) -> impl Iterator<Item = Token<String>> + '_ { self.0.iter().cloned() }
+  pub fn iter(&self) -> impl Iterator<Item = Tok<String>> + '_ { self.0.iter().cloned() }
 }
 impl fmt::Debug for VName {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "VName({self})") }
@@ -261,7 +262,7 @@ impl fmt::Display for VName {
   }
 }
 impl IntoIterator for VName {
-  type Item = Token<String>;
+  type Item = Tok<String>;
   type IntoIter = vec::IntoIter<Self::Item>;
   fn into_iter(self) -> Self::IntoIter { self.0.into_iter() }
 }
@@ -272,8 +273,8 @@ where PathSlice: Index<T>
 
   fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[Token<String>]> for VName {
+impl Borrow<[Tok<String>]> for VName {
-  fn borrow(&self) -> &[Token<String>] { self.0.borrow() }
+  fn borrow(&self) -> &[Tok<String>] { self.0.borrow() }
 }
 impl Borrow<PathSlice> for VName {
   fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
@@ -287,9 +288,9 @@ impl Deref for VName {
 /// empty sequence
 #[derive(Debug, Copy, Clone, Default, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct EmptyNameError;
-impl TryFrom<&[Token<String>]> for VName {
+impl TryFrom<&[Tok<String>]> for VName {
   type Error = EmptyNameError;
-  fn try_from(value: &[Token<String>]) -> Result<Self, Self::Error> {
+  fn try_from(value: &[Tok<String>]) -> Result<Self, Self::Error> {
     Self::new(value.iter().cloned())
   }
 }
@@ -300,11 +301,11 @@ impl TryFrom<&[Token<String>]> for VName {
 ///
 /// See also [VName]
 #[derive(Clone, Hash, PartialEq, Eq)]
-pub struct Sym(Token<Vec<Token<String>>>);
+pub struct Sym(Tok<Vec<Tok<String>>>);
 impl Sym {
   /// Assert that the sequence isn't empty, intern it and wrap it in a [Sym] to
   /// represent this invariant
-  pub fn new(v: impl IntoIterator<Item = Token<String>>) -> Result<Self, EmptyNameError> {
+  pub fn new(v: impl IntoIterator<Item = Tok<String>>) -> Result<Self, EmptyNameError> {
     let items = v.into_iter().collect_vec();
     Self::from_tok(intern(&items[..]))
   }
@@ -313,11 +314,11 @@ impl Sym {
     Ok(Sym(intern(&VName::parse(s)?.into_vec()[..])))
   }
   /// Assert that a token isn't empty, and wrap it in a [Sym]
-  pub fn from_tok(t: Token<Vec<Token<String>>>) -> Result<Self, EmptyNameError> {
+  pub fn from_tok(t: Tok<Vec<Tok<String>>>) -> Result<Self, EmptyNameError> {
     if t.is_empty() { Err(EmptyNameError) } else { Ok(Self(t)) }
   }
   /// Grab the interner token
-  pub fn tok(&self) -> Token<Vec<Token<String>>> { self.0.clone() }
+  pub fn tok(&self) -> Tok<Vec<Tok<String>>> { self.0.clone() }
   /// Get a number unique to this name suitable for arbitrary ordering.
   pub fn id(&self) -> NonZeroU64 { self.0.marker().get_id() }
   /// Extern the sym for editing
@@ -338,8 +339,8 @@ where PathSlice: Index<T>
 
   fn index(&self, index: T) -> &Self::Output { &self.deref()[index] }
 }
-impl Borrow<[Token<String>]> for Sym {
+impl Borrow<[Tok<String>]> for Sym {
-  fn borrow(&self) -> &[Token<String>] { &self.0[..] }
+  fn borrow(&self) -> &[Tok<String>] { &self.0[..] }
 }
 impl Borrow<PathSlice> for Sym {
   fn borrow(&self) -> &PathSlice { PathSlice::new(&self.0[..]) }
@@ -356,7 +357,7 @@ pub trait NameLike:
   'static + Clone + Eq + Hash + fmt::Debug + fmt::Display + Borrow<PathSlice>
 {
   /// Convert into held slice
-  fn as_slice(&self) -> &[Token<String>] { Borrow::<PathSlice>::borrow(self) }
+  fn as_slice(&self) -> &[Tok<String>] { Borrow::<PathSlice>::borrow(self) }
   /// Get iterator over tokens
   fn iter(&self) -> impl NameIter + '_ { self.as_slice().iter().cloned() }
   /// Get iterator over string segments
@@ -373,19 +374,19 @@ pub trait NameLike:
     NonZeroUsize::try_from(self.iter().count()).expect("NameLike never empty")
   }
   /// Like slice's `split_first` except we know that it always returns Some
-  fn split_first(&self) -> (Token<String>, &PathSlice) {
+  fn split_first(&self) -> (Tok<String>, &PathSlice) {
     let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
     (foot.clone(), PathSlice::new(torso))
   }
   /// Like slice's `split_last` except we know that it always returns Some
-  fn split_last(&self) -> (Token<String>, &PathSlice) {
+  fn split_last(&self) -> (Tok<String>, &PathSlice) {
     let (foot, torso) = self.as_slice().split_last().expect("NameLike never empty");
     (foot.clone(), PathSlice::new(torso))
   }
   /// Get the first element
-  fn first(&self) -> Token<String> { self.split_first().0 }
+  fn first(&self) -> Tok<String> { self.split_first().0 }
   /// Get the last element
-  fn last(&self) -> Token<String> { self.split_last().0 }
+  fn last(&self) -> Tok<String> { self.split_last().0 }
 }
 
 impl NameLike for Sym {}
@@ -399,7 +400,7 @@ impl NameLike for VName {}
 #[macro_export]
 macro_rules! sym {
   ($seg1:tt $( :: $seg:tt)*) => {
-    $crate::name::Sym::from_tok($crate::intern!([$crate::intern::Token<String>]: &[
+    $crate::name::Sym::from_tok($crate::intern!([$crate::interner::Tok<String>]: &[
       $crate::intern!(str: stringify!($seg1))
       $( , $crate::intern!(str: stringify!($seg)) )*
     ])).unwrap()
@@ -457,16 +458,16 @@ mod test {
   use std::borrow::Borrow;
 
   use super::{PathSlice, Sym, VName};
-  use crate::intern::{intern, Token};
+  use crate::interner::{intern, Tok};
   use crate::name::VPath;
 
   #[test]
   fn recur() {
     let myname = vname!(foo::bar);
-    let _borrowed_slice: &[Token<String>] = myname.borrow();
+    let _borrowed_slice: &[Tok<String>] = myname.borrow();
     let _borrowed_pathslice: &PathSlice = myname.borrow();
     let _deref_pathslice: &PathSlice = &myname;
-    let _as_slice_out: &[Token<String>] = myname.as_slice();
+    let _as_slice_out: &[Tok<String>] = myname.as_slice();
   }
 
   #[test]

orchid-base/src/reqnot.rs

@@ -12,9 +12,9 @@ use trait_set::trait_set;
 
 trait_set! {
   pub trait SendFn<T: MsgSet> = for<'a> FnMut(&'a [u8], ReqNot<T>) + DynClone + Send + 'static;
-  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + Send + 'static;
+  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + DynClone + Send + 'static;
   pub trait NotifFn<T: MsgSet> =
-    for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) + Send + Sync + 'static;
+    for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) + DynClone + Send + Sync + 'static;
 }
 
 fn get_id(message: &[u8]) -> (u64, &[u8]) {
@@ -39,7 +39,7 @@ impl<MS: MsgSet + 'static> RequestHandle<MS> {
   }
   pub fn handle<T: Request>(&self, _: &T, rep: &T::Response) { self.respond(rep) }
   pub fn will_handle_as<T: Request>(&self, _: &T) -> ReqTypToken<T> { ReqTypToken(PhantomData) }
-  pub fn handle_as<T: Request>(&self, token: ReqTypToken<T>, rep: &T::Response) {
+  pub fn handle_as<T: Request>(&self, _token: ReqTypToken<T>, rep: &T::Response) {
     self.respond(rep)
   }
 }
@@ -86,13 +86,17 @@ impl<T: MsgSet> ReqNot<T> {
     let mut g = self.0.lock().unwrap();
     let (id, payload) = get_id(&message[..]);
     if id == 0 {
-      (g.notif)(<T::In as Channel>::Notif::decode(&mut &payload[..]), self.clone())
+      let mut notif = clone_box(&*g.notif);
+      mem::drop(g);
+      notif(<T::In as Channel>::Notif::decode(&mut &payload[..]), self.clone())
     } else if 0 < id.bitand(1 << 63) {
       let sender = g.responses.remove(&!id).expect("Received response for invalid message");
       sender.send(message).unwrap();
     } else {
       let message = <T::In as Channel>::Req::decode(&mut &payload[..]);
-      (g.req)(RequestHandle { id, message, fulfilled: false.into(), parent: self.clone() })
+      let mut req = clone_box(&*g.req);
+      mem::drop(g);
+      req(RequestHandle { id, message, fulfilled: false.into(), parent: self.clone() })
     }
   }
 

orchid-base/src/sequence.rs

@@ -20,7 +20,7 @@ impl<'a, T: 'a> Sequence<'a, T> {
     Self(Rc::new(move || Box::new(f().into_iter())))
   }
   /// Get an iterator from the function
-  pub fn iter(&self) -> impl Iterator<Item = T> + '_ { (self.0)() }
+  pub fn iter(&self) -> BoxedIter<'_, T> { (self.0)() }
 }
 impl<'a, T: 'a> Clone for Sequence<'a, T> {
   fn clone(&self) -> Self { Self(self.0.clone()) }

orchid-base/src/tree.rs

@@ -4,18 +4,15 @@
 use std::fmt;
 
 use hashbrown::HashMap;
-use itertools::Itertools as _;
 use never::Never;
 use substack::Substack;
 use trait_set::trait_set;
 
 use crate::boxed_iter::BoxedIter;
 use crate::combine::Combine;
-use crate::intern::{intern, Token};
+use crate::interner::{intern, Tok};
 use crate::join::try_join_maps;
-use crate::location::CodeOrigin;
 use crate::name::{VName, VPath};
-use crate::proj_error::{ProjectError, ProjectErrorObj};
 use crate::sequence::Sequence;
 
 /// An umbrella trait for operations you can carry out on any part of the tree
@@ -34,18 +31,18 @@ pub trait TreeTransforms: Sized {
   /// Implementation for [TreeTransforms::map_data]
   fn map_data_rec<T, U, V>(
     self,
-    item: &mut impl FnMut(Substack<Token<String>>, Self::Item) -> T,
+    item: &mut impl FnMut(Substack<Tok<String>>, Self::Item) -> T,
-    module: &mut impl FnMut(Substack<Token<String>>, Self::XMod) -> U,
+    module: &mut impl FnMut(Substack<Tok<String>>, Self::XMod) -> U,
-    entry: &mut impl FnMut(Substack<Token<String>>, Self::XEnt) -> V,
+    entry: &mut impl FnMut(Substack<Tok<String>>, Self::XEnt) -> V,
-    path: Substack<Token<String>>,
+    path: Substack<Tok<String>>,
   ) -> Self::SelfType<T, U, V>;
 
   /// Transform all the data in the tree without changing its structure
   fn map_data<T, U, V>(
     self,
-    mut item: impl FnMut(Substack<Token<String>>, Self::Item) -> T,
+    mut item: impl FnMut(Substack<Tok<String>>, Self::Item) -> T,
-    mut module: impl FnMut(Substack<Token<String>>, Self::XMod) -> U,
+    mut module: impl FnMut(Substack<Tok<String>>, Self::XMod) -> U,
-    mut entry: impl FnMut(Substack<Token<String>>, Self::XEnt) -> V,
+    mut entry: impl FnMut(Substack<Tok<String>>, Self::XEnt) -> V,
   ) -> Self::SelfType<T, U, V> {
     self.map_data_rec(&mut item, &mut module, &mut entry, Substack::Bottom)
   }
@@ -55,14 +52,14 @@ pub trait TreeTransforms: Sized {
   ///
   /// * init - can be used for reduce, otherwise pass `()`
   /// * callback - a callback applied on every module.
-  ///   * [`Substack<Token<String>>`] - the walked path
+  ///   * [`Substack<Tok<String>>`] - the walked path
   ///   * [Module] - the current module
   ///   * `T` - data for reduce.
   fn search_all<'a, T>(
     &'a self,
     init: T,
     mut callback: impl FnMut(
-      Substack<Token<String>>,
+      Substack<Tok<String>>,
       ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
       T,
     ) -> T,
@@ -77,14 +74,14 @@ pub trait TreeTransforms: Sized {
   /// * init - can be used for reduce, otherwise pass `()`
   /// * callback - a callback applied on every module. Can return [Err] to
   ///   short-circuit the walk
-  ///   * [`Substack<Token<String>>`] - the walked path
+  ///   * [`Substack<Tok<String>>`] - the walked path
   ///   * [Module] - the current module
   ///   * `T` - data for reduce.
   fn search<'a, T, E>(
     &'a self,
     init: T,
     mut callback: impl FnMut(
-      Substack<Token<String>>,
+      Substack<Tok<String>>,
       ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
       T,
     ) -> Result<T, E>,
@@ -96,9 +93,9 @@ pub trait TreeTransforms: Sized {
   fn search_rec<'a, T, E>(
     &'a self,
     state: T,
-    stack: Substack<Token<String>>,
+    stack: Substack<Tok<String>>,
     callback: &mut impl FnMut(
-      Substack<Token<String>>,
+      Substack<Tok<String>>,
       ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
       T,
     ) -> Result<T, E>,
@@ -122,10 +119,10 @@ impl<Item, XMod, XEnt> TreeTransforms for ModMember<Item, XMod, XEnt> {
 
   fn map_data_rec<T, U, V>(
     self,
-    item: &mut impl FnMut(Substack<Token<String>>, Item) -> T,
+    item: &mut impl FnMut(Substack<Tok<String>>, Item) -> T,
-    module: &mut impl FnMut(Substack<Token<String>>, XMod) -> U,
+    module: &mut impl FnMut(Substack<Tok<String>>, XMod) -> U,
-    entry: &mut impl FnMut(Substack<Token<String>>, XEnt) -> V,
+    entry: &mut impl FnMut(Substack<Tok<String>>, XEnt) -> V,
-    path: Substack<Token<String>>,
+    path: Substack<Tok<String>>,
   ) -> Self::SelfType<T, U, V> {
     match self {
       Self::Item(it) => ModMember::Item(item(path, it)),
@@ -136,9 +133,9 @@ impl<Item, XMod, XEnt> TreeTransforms for ModMember<Item, XMod, XEnt> {
   fn search_rec<'a, T, E>(
     &'a self,
     state: T,
-    stack: Substack<Token<String>>,
+    stack: Substack<Tok<String>>,
     callback: &mut impl FnMut(
-      Substack<Token<String>>,
+      Substack<Tok<String>>,
       ModMemberRef<'a, Item, XMod, XEnt>,
       T,
     ) -> Result<T, E>,
@@ -209,7 +206,7 @@ pub struct TreeConflict<Item: Combine, XMod: Combine, XEnt: Combine> {
 impl<Item: Combine, XMod: Combine, XEnt: Combine> TreeConflict<Item, XMod, XEnt> {
   fn new(kind: ConflictKind<Item, XMod, XEnt>) -> Self { Self { path: VPath::new([]), kind } }
 
-  fn push(self, seg: Token<String>) -> Self {
+  fn push(self, seg: Tok<String>) -> Self {
     Self { path: self.path.prefix([seg]), kind: self.kind }
   }
 }
@@ -280,10 +277,10 @@ impl<Item, XMod, XEnt> TreeTransforms for ModEntry<Item, XMod, XEnt> {
 
   fn map_data_rec<T, U, V>(
     self,
-    item: &mut impl FnMut(Substack<Token<String>>, Item) -> T,
+    item: &mut impl FnMut(Substack<Tok<String>>, Item) -> T,
-    module: &mut impl FnMut(Substack<Token<String>>, XMod) -> U,
+    module: &mut impl FnMut(Substack<Tok<String>>, XMod) -> U,
-    entry: &mut impl FnMut(Substack<Token<String>>, XEnt) -> V,
+    entry: &mut impl FnMut(Substack<Tok<String>>, XEnt) -> V,
-    path: Substack<Token<String>>,
+    path: Substack<Tok<String>>,
   ) -> Self::SelfType<T, U, V> {
     ModEntry {
       member: self.member.map_data_rec(item, module, entry, path.clone()),
@@ -294,9 +291,9 @@ impl<Item, XMod, XEnt> TreeTransforms for ModEntry<Item, XMod, XEnt> {
   fn search_rec<'a, T, E>(
     &'a self,
     state: T,
-    stack: Substack<Token<String>>,
+    stack: Substack<Tok<String>>,
     callback: &mut impl FnMut(
-      Substack<Token<String>>,
+      Substack<Tok<String>>,
       ModMemberRef<'a, Item, XMod, XEnt>,
       T,
     ) -> Result<T, E>,
@@ -357,7 +354,7 @@ impl<Item, XMod: Default, XEnt: Default> ModEntry<Item, XMod, XEnt> {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Module<Item, XMod, XEnt> {
   /// Submodules and items by name
-  pub entries: HashMap<Token<String>, ModEntry<Item, XMod, XEnt>>,
+  pub entries: HashMap<Tok<String>, ModEntry<Item, XMod, XEnt>>,
   /// Additional fields
   pub x: XMod,
 }
@@ -369,7 +366,7 @@ trait_set! {
 }
 
 /// A line in a [Module]
-pub type Record<Item, XMod, XEnt> = (Token<String>, ModEntry<Item, XMod, XEnt>);
+pub type Record<Item, XMod, XEnt> = (Tok<String>, ModEntry<Item, XMod, XEnt>);
 
 impl<Item, XMod, XEnt> Module<Item, XMod, XEnt> {
   /// Returns child names for which the value matches a filter
@@ -377,15 +374,15 @@ impl<Item, XMod, XEnt> Module<Item, XMod, XEnt> {
   pub fn keys<'a>(
     &'a self,
     filter: impl for<'b> Fn(&'b ModEntry<Item, XMod, XEnt>) -> bool + 'a,
-  ) -> BoxedIter<Token<String>> {
+  ) -> BoxedIter<Tok<String>> {
     Box::new((self.entries.iter()).filter(move |(_, v)| filter(v)).map(|(k, _)| k.clone()))
   }
 
   /// Return the module at the end of the given path
   pub fn walk_ref<'a: 'b, 'b>(
     &'a self,
-    prefix: &'b [Token<String>],
+    prefix: &'b [Tok<String>],
-    path: &'b [Token<String>],
+    path: &'b [Tok<String>],
     filter: impl Filter<'b, Item, XMod, XEnt>,
   ) -> Result<&'a Self, WalkError<'b>> {
     let mut module = self;
@@ -412,8 +409,8 @@ impl<Item, XMod, XEnt> Module<Item, XMod, XEnt> {
   /// if path is empty, since the reference cannot be forwarded that way
   pub fn walk1_ref<'a: 'b, 'b>(
     &'a self,
-    prefix: &'b [Token<String>],
+    prefix: &'b [Tok<String>],
-    path: &'b [Token<String>],
+    path: &'b [Tok<String>],
     filter: impl Filter<'b, Item, XMod, XEnt>,
   ) -> Result<(&'a ModEntry<Item, XMod, XEnt>, &'a Self), WalkError<'b>> {
     let (last, parent) = path.split_last().expect("Path cannot be empty");
@@ -436,8 +433,8 @@ impl<Item, XMod, XEnt> Module<Item, XMod, XEnt> {
   /// If the target is the root node
   pub fn inner_walk<'a: 'b, 'b>(
     &'a self,
-    origin: &[Token<String>],
+    origin: &[Tok<String>],
-    target: &'b [Token<String>],
+    target: &'b [Tok<String>],
     is_exported: impl for<'c> Fn(&'c ModEntry<Item, XMod, XEnt>) -> bool + Clone + 'b,
   ) -> Result<(&'a ModEntry<Item, XMod, XEnt>, &'a Self), WalkError<'b>> {
     let ignore_vis_len = 1 + origin.iter().zip(target).take_while(|(a, b)| a == b).count();
@@ -464,10 +461,10 @@ impl<Item, XMod, XEnt> TreeTransforms for Module<Item, XMod, XEnt> {
 
   fn map_data_rec<T, U, V>(
     self,
-    item: &mut impl FnMut(Substack<Token<String>>, Item) -> T,
+    item: &mut impl FnMut(Substack<Tok<String>>, Item) -> T,
-    module: &mut impl FnMut(Substack<Token<String>>, XMod) -> U,
+    module: &mut impl FnMut(Substack<Tok<String>>, XMod) -> U,
-    entry: &mut impl FnMut(Substack<Token<String>>, XEnt) -> V,
+    entry: &mut impl FnMut(Substack<Tok<String>>, XEnt) -> V,
-    path: Substack<Token<String>>,
+    path: Substack<Tok<String>>,
   ) -> Self::SelfType<T, U, V> {
     Module {
       x: module(path.clone(), self.x),
@@ -480,9 +477,9 @@ impl<Item, XMod, XEnt> TreeTransforms for Module<Item, XMod, XEnt> {
   fn search_rec<'a, T, E>(
     &'a self,
     mut state: T,
-    stack: Substack<Token<String>>,
+    stack: Substack<Tok<String>>,
     callback: &mut impl FnMut(
-      Substack<Token<String>>,
+      Substack<Tok<String>>,
       ModMemberRef<'a, Item, XMod, XEnt>,
       T,
     ) -> Result<T, E>,
@@ -538,6 +535,15 @@ pub enum ErrKind {
   /// The path leads into a leaf node
   NotModule,
 }
+impl ErrKind {
+  pub const fn msg(&self) -> &'static str {
+    match self {
+      Self::Filtered => "The path leads into a private module",
+      Self::Missing => "Nonexistent path",
+      Self::NotModule => "The path leads into a leaf",
+    }
+  }
+}
 
 #[derive(Clone)]
 /// All details about a failed tree-walk
@@ -545,42 +551,31 @@ pub struct WalkError<'a> {
   /// Failure mode
   kind: ErrKind,
   /// Path to the module where the walk started
-  prefix: &'a [Token<String>],
+  prefix: &'a [Tok<String>],
   /// Planned walk path
-  path: &'a [Token<String>],
+  path: &'a [Tok<String>],
   /// Index into walked path where the error occurred
   pos: usize,
   /// Alternatives to the failed steps
-  options: Sequence<'a, Token<String>>,
+  options: Sequence<'a, Tok<String>>,
 }
 impl<'a> WalkError<'a> {
   /// Total length of the path represented by this error
   #[must_use]
   pub fn depth(&self) -> usize { self.prefix.len() + self.pos + 1 }
 
-  /// Attach a location to the error and convert into trait object for reporting
+  pub fn alternatives(&self) -> BoxedIter<Tok<String>> { self.options.iter() }
-  #[must_use]
+
-  pub fn at(self, origin: &CodeOrigin) -> ProjectErrorObj {
+  /// Get the total path including the step that caused the error
-    let details = WalkErrorDetails {
+  pub fn full_path(&self) -> VName {
-      origin: origin.clone(),
+    VName::new((self.prefix.iter()).chain(self.path.iter().take(self.pos + 1)).cloned())
-      path: VName::new((self.prefix.iter()).chain(self.path.iter().take(self.pos + 1)).cloned())
+      .expect("empty paths don't cause an error")
-        .expect("empty paths don't cause an error"),
-      options: self.options.iter().collect(),
-    };
-    match self.kind {
-      ErrKind::Filtered => FilteredError(details).pack(),
-      ErrKind::Missing => MissingError(details).pack(),
-      ErrKind::NotModule => NotModuleError(details).pack(),
-    }
   }
+
   /// Construct an error for the very last item in a slice. This is often done
   /// outside [super::tree] so it gets a function rather than exposing the
   /// fields of [WalkError]
-  pub fn last(
+  pub fn last(path: &'a [Tok<String>], kind: ErrKind, options: Sequence<'a, Tok<String>>) -> Self {
-    path: &'a [Token<String>],
-    kind: ErrKind,
-    options: Sequence<'a, Token<String>>,
-  ) -> Self {
     WalkError { kind, path, options, pos: path.len() - 1, prefix: &[] }
   }
 }
@@ -594,37 +589,3 @@ impl<'a> fmt::Debug for WalkError<'a> {
       .finish_non_exhaustive()
   }
 }
-
-struct WalkErrorDetails {
-  path: VName,
-  options: Vec<Token<String>>,
-  origin: CodeOrigin,
-}
-impl WalkErrorDetails {
-  fn print_options(&self) -> String { format!("options are {}", self.options.iter().join(", ")) }
-}
-
-struct FilteredError(WalkErrorDetails);
-impl ProjectError for FilteredError {
-  const DESCRIPTION: &'static str = "The path leads into a private module";
-  fn one_position(&self) -> CodeOrigin { self.0.origin.clone() }
-  fn message(&self) -> String { format!("{} is private, {}", self.0.path, self.0.print_options()) }
-}
-
-struct MissingError(WalkErrorDetails);
-impl ProjectError for MissingError {
-  const DESCRIPTION: &'static str = "Nonexistent path";
-  fn one_position(&self) -> CodeOrigin { self.0.origin.clone() }
-  fn message(&self) -> String {
-    format!("{} does not exist, {}", self.0.path, self.0.print_options())
-  }
-}
-
-struct NotModuleError(WalkErrorDetails);
-impl ProjectError for NotModuleError {
-  const DESCRIPTION: &'static str = "The path leads into a leaf";
-  fn one_position(&self) -> CodeOrigin { self.0.origin.clone() }
-  fn message(&self) -> String {
-    format!("{} is not a module, {}", self.0.path, self.0.print_options())
-  }
-}

orchid-base/src/virt_fs/common.rs

@@ -13,13 +13,13 @@ pub enum Loaded {
   Code(Arc<String>),
   /// Conceptually equivalent to the list of *.orc files in a folder, without
   /// the extension
-  Collection(Arc<Vec<Token<String>>>),
+  Collection(Arc<Vec<Tok<String>>>),
 }
 impl Loaded {
   /// Is the loaded item source code (not a collection)?
   pub fn is_code(&self) -> bool { matches!(self, Loaded::Code(_)) }
   /// Collect the elements in a collection rreport
-  pub fn collection(items: impl IntoIterator<Item = Token<String>>) -> Self {
+  pub fn collection(items: impl IntoIterator<Item = Tok<String>>) -> Self {
     Self::Collection(Arc::new(items.into_iter().collect()))
   }
 }
@@ -55,7 +55,7 @@ impl ErrorSansOrigin for CodeNotFound {
 /// formats and other sources for libraries and dependencies.
 pub trait VirtFS {
   /// Implementation of [VirtFS::read]
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult;
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult;
   /// Discover information about a path without reading it.
   ///
   /// Implement this if your vfs backend can do expensive operations
@@ -68,7 +68,7 @@ pub trait VirtFS {
   }
   /// Convert a path into a human-readable string that is meaningful in the
   /// target context.
-  fn display(&self, path: &[Token<String>]) -> Option<String>;
+  fn display(&self, path: &[Tok<String>]) -> Option<String>;
   /// Convert the FS handler into a type-erased version of itself for packing in
   /// a tree.
   fn rc(self) -> Rc<dyn VirtFS>
@@ -81,15 +81,15 @@ pub trait VirtFS {
 }
 
 impl VirtFS for &dyn VirtFS {
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     (*self).get(path, full_path)
   }
-  fn display(&self, path: &[Token<String>]) -> Option<String> { (*self).display(path) }
+  fn display(&self, path: &[Tok<String>]) -> Option<String> { (*self).display(path) }
 }
 
 impl<T: VirtFS + ?Sized> VirtFS for Rc<T> {
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     (**self).get(path, full_path)
   }
-  fn display(&self, path: &[Token<String>]) -> Option<String> { (**self).display(path) }
+  fn display(&self, path: &[Tok<String>]) -> Option<String> { (**self).display(path) }
 }

orchid-base/src/virt_fs/decl.rs

@@ -32,7 +32,7 @@ impl<'a> Combine for &'a dyn VirtFS {
 pub type DeclTree = ModEntry<Rc<dyn VirtFS>, (), ()>;
 
 impl VirtFS for DeclTree {
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     match &self.member {
       ModMember::Item(it) => it.get(path, full_path),
       ModMember::Sub(module) => match path.split_first() {
@@ -44,7 +44,7 @@ impl VirtFS for DeclTree {
     }
   }
 
-  fn display(&self, path: &[Token<String>]) -> Option<String> {
+  fn display(&self, path: &[Tok<String>]) -> Option<String> {
     let (head, tail) = path.split_first()?;
     match &self.member {
       ModMember::Item(it) => it.display(path),
@@ -54,16 +54,16 @@ impl VirtFS for DeclTree {
 }
 
 impl VirtFS for String {
-  fn display(&self, _: &[Token<String>]) -> Option<String> { None }
+  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.as_str().to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }
 }
 
 impl<'a> VirtFS for &'a str {
-  fn display(&self, _: &[Token<String>]) -> Option<String> { None }
+  fn display(&self, _: &[Tok<String>]) -> Option<String> { None }
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     (path.is_empty().then(|| Loaded::Code(Arc::new(self.to_string()))))
       .ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())
   }

orchid-base/src/virt_fs/dir.rs

@@ -99,14 +99,14 @@ impl DirNode {
     }
   }
 
-  fn mk_pathbuf(&self, path: &[Token<String>]) -> PathBuf {
+  fn mk_pathbuf(&self, path: &[Tok<String>]) -> PathBuf {
     let mut fpath = self.root.clone();
     path.iter().for_each(|seg| fpath.push(seg.as_str()));
     fpath
   }
 }
 impl VirtFS for DirNode {
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     let fpath = self.mk_pathbuf(path);
     let mut binding = self.cached.borrow_mut();
     let (_, res) = (binding.raw_entry_mut().from_key(&fpath))
@@ -114,7 +114,7 @@ impl VirtFS for DirNode {
     res.clone()
   }
 
-  fn display(&self, path: &[Token<String>]) -> Option<String> {
+  fn display(&self, path: &[Tok<String>]) -> Option<String> {
     let pathbuf = self.mk_pathbuf(path).with_extension(self.ext());
     Some(pathbuf.to_string_lossy().to_string())
   }

orchid-base/src/virt_fs/embed.rs

@@ -56,7 +56,7 @@ impl EmbeddedFS {
 }
 
 impl VirtFS for EmbeddedFS {
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> FSResult {
     if path.is_empty() {
       return Ok(Loaded::collection(self.tree.keys(|_| true)));
     }
@@ -67,7 +67,7 @@ impl VirtFS for EmbeddedFS {
       ModMember::Sub(sub) => Loaded::collection(sub.keys(|_| true)),
     })
   }
-  fn display(&self, path: &[Token<String>]) -> Option<String> {
+  fn display(&self, path: &[Tok<String>]) -> Option<String> {
     let Self { gen, suffix, .. } = self;
     Some(format!("{}{suffix} in {gen}", path.iter().join("/")))
   }

orchid-base/src/virt_fs/prefix.rs

@@ -21,18 +21,18 @@ impl<'a> PrefixFS<'a> {
       add: VPath::parse(add.as_ref()),
     }
   }
-  fn proc_path(&self, path: &[Token<String>]) -> Option<Vec<Token<String>>> {
+  fn proc_path(&self, path: &[Tok<String>]) -> Option<Vec<Tok<String>>> {
     let path = path.strip_prefix(self.remove.as_slice())?;
     Some(self.add.0.iter().chain(path).cloned().collect_vec())
   }
 }
 impl<'a> VirtFS for PrefixFS<'a> {
-  fn get(&self, path: &[Token<String>], full_path: &PathSlice) -> super::FSResult {
+  fn get(&self, path: &[Tok<String>], full_path: &PathSlice) -> super::FSResult {
     let path =
       self.proc_path(path).ok_or_else(|| CodeNotFound::new(full_path.to_vpath()).pack())?;
     self.wrapped.get(&path, full_path)
   }
-  fn display(&self, path: &[Token<String>]) -> Option<String> {
+  fn display(&self, path: &[Tok<String>]) -> Option<String> {
     self.wrapped.display(&self.proc_path(path)?)
   }
 }

orchid-extension/Cargo.toml

@@ -8,13 +8,17 @@ edition = "2021"
 [dependencies]
 ahash = "0.8.11"
 derive_destructure = "1.0.0"
+dyn-clone = "1.0.17"
 hashbrown = "0.14.5"
-itertools = "0.12.1"
+itertools = "0.13.0"
+konst = "0.3.9"
 never = "0.1.0"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
+orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 ordered-float = "4.2.0"
+paste = "1.0.15"
 substack = "1.1.0"
 trait-set = "0.3.0"
 typeid = "1.0.0"

orchid-extension/src/atom.rs

@@ -1,44 +1,43 @@
 use std::any::{type_name, Any};
+use std::borrow::Cow;
+use std::fmt;
 use std::io::{Read, Write};
 use std::num::NonZeroU64;
 use std::ops::Deref;
-use std::sync::Arc;
-use std::{fmt, mem};
 
-use derive_destructure::destructure;
+use dyn_clone::{clone_box, DynClone};
-use orchid_api::atom::{Atom, Fwd};
+use orchid_api::atom::{Atom, Fwd, LocalAtom};
-use orchid_api::expr::{ExprTicket, Release};
+use orchid_api::expr::ExprTicket;
 use orchid_api::system::SysId;
 use orchid_api_traits::{Coding, Decode, Encode, Request};
 use orchid_base::id_store::{IdRecord, IdStore};
-use orchid_base::reqnot::Requester as _;
+use orchid_base::location::Pos;
+use orchid_base::reqnot::Requester;
+use trait_set::trait_set;
 use typeid::ConstTypeId;
 
-use crate::expr::GenClause;
+use crate::expr::{bot, ExprHandle, GenClause};
-use crate::other_system::SystemHandle;
+use crate::system::{atom_info_for, DynSystem, DynSystemCard, SysCtx};
-use crate::system::{DynSystemCard, SystemCard};
 
 pub trait AtomCard: 'static + Sized {
-  type Owner: SystemCard;
+  // type Owner: SystemCard;
   type Data: Clone + Coding + Sized;
   type Req: Coding;
 }
 
-pub fn get_info<A: AtomCard>(sys: &(impl DynSystemCard + ?Sized)) -> (usize, &AtomInfo) {
+pub fn get_info<A: AtomCard>(sys: &(impl DynSystemCard + ?Sized)) -> (u64, &AtomInfo) {
-  sys.atom_info_for(ConstTypeId::of::<A>()).unwrap_or_else(|| {
+  atom_info_for(sys, ConstTypeId::of::<A>()).unwrap_or_else(|| {
     panic!("Atom {} not associated with system {}", type_name::<A>(), sys.name())
   })
 }
 
 pub fn encode_atom_nodrop<A: AtomCard>(
-  sys_id: SysId,
   sys: &(impl DynSystemCard + ?Sized),
   data: &A::Data,
-) -> Atom {
+) -> LocalAtom {
-  let (info_pos, _) = get_info::<A>(sys);
+  let mut buf = get_info::<A>(sys).0.enc_vec();
-  let mut buf = (info_pos as u64).enc_vec();
   data.encode(&mut buf);
-  Atom { owner: sys_id, drop: false, data: buf }
+  LocalAtom { drop: false, data: buf }
 }
 
 pub fn encode_atom_drop<A: AtomCard>(
@@ -47,8 +46,7 @@ pub fn encode_atom_drop<A: AtomCard>(
   atom_id: u64,
   data: &A::Data,
 ) -> Atom {
-  let (info_pos, _) = get_info::<A>(sys);
+  let mut buf = get_info::<A>(sys).0.enc_vec();
-  let mut buf = (info_pos as u64).enc_vec();
   atom_id.encode(&mut buf);
   data.encode(&mut buf);
   Atom { owner: sys_id, drop: true, data: buf }
@@ -56,119 +54,175 @@ pub fn encode_atom_drop<A: AtomCard>(
 
 pub fn decode_atom<A: AtomCard>(
   sys: &(impl DynSystemCard + ?Sized),
-  Atom { data, drop, owner: _ }: &Atom,
+  Atom { data, drop: _, owner: _ }: &Atom,
 ) -> Option<A::Data> {
   let (info_pos, info) = get_info::<A>(sys);
   let mut data = &data[..];
-  if u64::decode(&mut data) != info_pos as u64 {
+  if u64::decode(&mut data) != info_pos {
     return None;
   }
   let val = (info.decode)(data);
   Some(*val.downcast().expect("The type-id checked out, the decode should've worked"))
 }
 
-#[derive(destructure)]
+#[derive(Clone)]
-pub struct ForeignAtom<A: AtomCard> {
+pub struct ForeignAtom {
-  pub(crate) sys: SystemHandle<A::Owner>,
+  pub expr: ExprHandle,
-  pub(crate) ticket: ExprTicket,
+  pub atom: Atom,
-  pub(crate) api: Atom,
+  pub position: Pos,
-  pub(crate) value: A::Data,
 }
-impl<A: AtomCard> ForeignAtom<A> {
+impl ForeignAtom {}
-  /// Unpack the object, returning the held atom and expr ticket. This is in
+
-  /// contrast to dropping the atom which releases the expr ticket.
+#[derive(Clone)]
-  pub fn unpack(self) -> (A::Data, ExprTicket, Atom) {
+pub struct TypAtom<A: AtomCard> {
-    let (_, ticket, api, value) = self.destructure();
+  pub data: ForeignAtom,
-    (value, ticket, api)
+  pub value: A::Data,
-  }
+}
-  pub fn ticket(&self) -> ExprTicket { self.ticket }
+impl<A: AtomCard> TypAtom<A> {
   pub fn request<R: Coding + Into<A::Req> + Request>(&self, req: R) -> R::Response {
-    R::Response::decode(&mut &self.sys.reqnot.request(Fwd(self.api.clone(), req.enc_vec()))[..])
+    R::Response::decode(
+      &mut &self.data.expr.ctx.reqnot.request(Fwd(self.data.atom.clone(), req.enc_vec()))[..],
+    )
   }
 }
-impl<A: AtomCard> Deref for ForeignAtom<A> {
+impl<A: AtomCard> Deref for TypAtom<A> {
   type Target = A::Data;
   fn deref(&self) -> &Self::Target { &self.value }
 }
-impl<A: AtomCard> Drop for ForeignAtom<A> {
-  fn drop(&mut self) { self.sys.reqnot.notify(Release(self.sys.id(), self.ticket)) }
-}
 
 pub struct AtomInfo {
   pub tid: ConstTypeId,
   pub decode: fn(&[u8]) -> Box<dyn Any>,
-  pub call: fn(&[u8], ExprTicket) -> GenClause,
+  pub call: fn(&[u8], SysCtx, ExprTicket) -> GenClause,
-  pub call_ref: fn(&[u8], ExprTicket) -> GenClause,
+  pub call_ref: fn(&[u8], SysCtx, ExprTicket) -> GenClause,
-  pub same: fn(&[u8], &[u8]) -> bool,
+  pub same: fn(&[u8], SysCtx, &[u8]) -> bool,
-  pub handle_req: fn(&[u8], &mut dyn Read, &mut dyn Write),
+  pub handle_req: fn(&[u8], SysCtx, &mut dyn Read, &mut dyn Write),
-  pub drop: fn(&[u8]),
+  pub drop: fn(&[u8], SysCtx),
 }
 
 pub trait ThinAtom: AtomCard<Data = Self> + Coding + fmt::Debug {
-  fn call(&self, arg: ExprTicket) -> GenClause;
+  #[allow(unused_variables)]
-  fn same(&self, other: &Self) -> bool;
+  fn call(&self, arg: ExprHandle) -> GenClause { bot("This atom is not callable") }
-  fn handle_req(&self, req: Self::Req, rep: &mut (impl Write + ?Sized));
+  #[allow(unused_variables)]
+  fn same(&self, ctx: SysCtx, other: &Self) -> bool {
+    eprintln!(
+      "Override ThinAtom::same for {} if it can be generated during parsing",
+      type_name::<Self>()
+    );
+    false
+  }
+  fn handle_req(&self, ctx: SysCtx, req: Self::Req, rep: &mut (impl Write + ?Sized));
+  fn factory(self) -> AtomFactory {
+    AtomFactory::new(move |sys| encode_atom_nodrop::<Self>(sys.dyn_card(), &self))
+  }
 }
 
 pub const fn thin_atom_info<T: ThinAtom>() -> AtomInfo {
   AtomInfo {
     tid: ConstTypeId::of::<T>(),
     decode: |mut b| Box::new(T::decode(&mut b)),
-    call: |mut b, extk| T::decode(&mut b).call(extk),
+    call: |mut b, ctx, extk| T::decode(&mut b).call(ExprHandle::from_args(ctx, extk)),
-    call_ref: |mut b, extk| T::decode(&mut b).call(extk),
+    call_ref: |mut b, ctx, extk| T::decode(&mut b).call(ExprHandle::from_args(ctx, extk)),
-    handle_req: |mut b, req, rep| T::decode(&mut b).handle_req(Decode::decode(req), rep),
+    handle_req: |mut b, ctx, req, rep| T::decode(&mut b).handle_req(ctx, Decode::decode(req), rep),
-    same: |mut b1, mut b2| T::decode(&mut b1).same(&T::decode(&mut b2)),
+    same: |mut b1, ctx, mut b2| T::decode(&mut b1).same(ctx, &T::decode(&mut b2)),
-    drop: |mut b1| eprintln!("Received drop signal for non-drop atom {:?}", T::decode(&mut b1)),
+    drop: |mut b1, _| eprintln!("Received drop signal for non-drop atom {:?}", T::decode(&mut b1)),
   }
 }
 
 /// Atoms that have a [Drop]
-pub trait OwnedAtom: AtomCard + Deref<Target = Self::Data> + Send + Sync + Any + 'static {
+pub trait OwnedAtom: AtomCard + Send + Sync + Any + Clone + 'static {
-  fn call_ref(&self, arg: ExprTicket) -> GenClause;
+  fn val(&self) -> Cow<'_, Self::Data>;
-  fn call(self, arg: ExprTicket) -> GenClause;
+  #[allow(unused_variables)]
-  fn same(&self, other: &Self) -> bool;
+  fn call_ref(&self, arg: ExprHandle) -> GenClause { bot("This atom is not callable") }
-  fn handle_req(&self, req: Self::Req, rep: &mut (impl Write + ?Sized));
+  fn call(self, arg: ExprHandle) -> GenClause {
+    let ctx = arg.get_ctx();
+    let gcl = self.call_ref(arg);
+    self.free(ctx);
+    gcl
+  }
+  #[allow(unused_variables)]
+  fn same(&self, ctx: SysCtx, other: &Self) -> bool {
+    eprintln!(
+      "Override OwnedAtom::same for {} if it can be generated during parsing",
+      type_name::<Self>()
+    );
+    false
+  }
+  fn handle_req(&self, ctx: SysCtx, req: Self::Req, rep: &mut (impl Write + ?Sized));
+  #[allow(unused_variables)]
+  fn free(self, ctx: SysCtx) {}
+  #[allow(unused_variables)]
+  fn factory(self) -> AtomFactory {
+    AtomFactory::new(move |sys| {
+      let rec = OBJ_STORE.add(Box::new(self));
+      let mut data = atom_info_for(sys.dyn_card(), rec.atom_tid()).expect("obj exists").0.enc_vec();
+      rec.id().encode(&mut data);
+      rec.encode(&mut data);
+      LocalAtom { drop: true, data }
+    })
+  }
 }
 
 pub trait DynOwnedAtom: Send + Sync + 'static {
   fn atom_tid(&self) -> ConstTypeId;
   fn as_any_ref(&self) -> &dyn Any;
-  fn dyn_call_ref(&self, arg: ExprTicket) -> GenClause;
+  fn encode(&self, buffer: &mut dyn Write);
-  fn dyn_call(self: Box<Self>, arg: ExprTicket) -> GenClause;
+  fn dyn_call_ref(&self, ctx: SysCtx, arg: ExprTicket) -> GenClause;
-  fn dyn_same(&self, other: &dyn DynOwnedAtom) -> bool;
+  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: ExprTicket) -> GenClause;
-  fn dyn_handle_req(&self, req: &mut dyn Read, rep: &mut dyn Write);
+  fn dyn_same(&self, ctx: SysCtx, other: &dyn DynOwnedAtom) -> bool;
+  fn dyn_handle_req(&self, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write);
+  fn dyn_free(self: Box<Self>, ctx: SysCtx);
 }
-
 impl<T: OwnedAtom> DynOwnedAtom for T {
   fn atom_tid(&self) -> ConstTypeId { ConstTypeId::of::<T>() }
   fn as_any_ref(&self) -> &dyn Any { self }
-  fn dyn_call_ref(&self, arg: ExprTicket) -> GenClause { self.call_ref(arg) }
+  fn encode(&self, buffer: &mut dyn Write) { self.val().as_ref().encode(buffer) }
-  fn dyn_call(self: Box<Self>, arg: ExprTicket) -> GenClause { self.call(arg) }
+  fn dyn_call_ref(&self, ctx: SysCtx, arg: ExprTicket) -> GenClause {
-  fn dyn_same(&self, other: &dyn DynOwnedAtom) -> bool {
+    self.call_ref(ExprHandle::from_args(ctx, arg))
+  }
+  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: ExprTicket) -> GenClause {
+    self.call(ExprHandle::from_args(ctx, arg))
+  }
+  fn dyn_same(&self, ctx: SysCtx, other: &dyn DynOwnedAtom) -> bool {
     if ConstTypeId::of::<Self>() != other.as_any_ref().type_id() {
       return false;
     }
     let other_self = other.as_any_ref().downcast_ref().expect("The type_ids are the same");
-    self.same(other_self)
+    self.same(ctx, other_self)
   }
-  fn dyn_handle_req(&self, req: &mut dyn Read, rep: &mut dyn Write) {
+  fn dyn_handle_req(&self, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write) {
-    self.handle_req(<Self as AtomCard>::Req::decode(req), rep)
+    self.handle_req(ctx, <Self as AtomCard>::Req::decode(req), rep)
   }
+  fn dyn_free(self: Box<Self>, ctx: SysCtx) { self.free(ctx) }
 }
 
 pub(crate) static OBJ_STORE: IdStore<Box<dyn DynOwnedAtom>> = IdStore::new();
 
-const fn owned_atom_info<T: OwnedAtom>() -> AtomInfo {
+pub const fn owned_atom_info<T: OwnedAtom>() -> AtomInfo {
   fn with_atom<U>(mut b: &[u8], f: impl FnOnce(IdRecord<'_, Box<dyn DynOwnedAtom>>) -> U) -> U {
     f(OBJ_STORE.get(NonZeroU64::decode(&mut b)).expect("Received invalid atom ID"))
   }
   AtomInfo {
     tid: ConstTypeId::of::<T>(),
     decode: |mut b| Box::new(T::Data::decode(&mut b)),
-    call: |b, arg| with_atom(b, |a| a.remove().dyn_call(arg)),
+    call: |b, ctx, arg| with_atom(b, |a| a.remove().dyn_call(ctx, arg)),
-    call_ref: |b, arg| with_atom(b, |a| a.dyn_call_ref(arg)),
+    call_ref: |b, ctx, arg| with_atom(b, |a| a.dyn_call_ref(ctx, arg)),
-    same: |b1, b2| with_atom(b1, |a1| with_atom(b2, |a2| a1.dyn_same(&**a2))),
+    same: |b1, ctx, b2| with_atom(b1, |a1| with_atom(b2, |a2| a1.dyn_same(ctx, &**a2))),
-    handle_req: |b, req, rep| with_atom(b, |a| a.dyn_handle_req(req, rep)),
+    handle_req: |b, ctx, req, rep| with_atom(b, |a| a.dyn_handle_req(ctx, req, rep)),
-    drop: |b| mem::drop(with_atom(b, |a| a.remove())),
+    drop: |b, ctx| with_atom(b, |a| a.remove().dyn_free(ctx)),
   }
 }
+
+trait_set! {
+  pub trait AtomFactoryFn = FnOnce(&dyn DynSystem) -> LocalAtom + DynClone;
+}
+pub struct AtomFactory(Box<dyn AtomFactoryFn>);
+impl AtomFactory {
+  pub fn new(f: impl FnOnce(&dyn DynSystem) -> LocalAtom + Clone + 'static) -> Self {
+    Self(Box::new(f))
+  }
+  pub fn build(self, sys: &dyn DynSystem) -> LocalAtom { (self.0)(sys) }
+}
+impl Clone for AtomFactory {
+  fn clone(&self) -> Self { AtomFactory(clone_box(&*self.0)) }
+}

orchid-extension/src/entrypoint.rs

@@ -1,92 +1,138 @@
+use std::num::{NonZeroU16, NonZeroU64};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::{mem, thread};
 
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::atom::{Atom, AtomReq, AtomSame, CallRef, FinalCall, Fwded};
+use orchid_api::atom::{Atom, AtomDrop, AtomReq, AtomSame, CallRef, Command, FinalCall, Fwded};
-use orchid_api::parser::{CharFilter, Lex, Lexed, ParserReq, SubLex};
+use orchid_api::interner::Sweep;
-use orchid_api::proto::{ExtMsgSet, ExtensionHeader, HostExtNotif, HostExtReq, HostHeader};
+use orchid_api::parser::{CharFilter, Lex, Lexed, ParserReq};
-use orchid_api::system::{SysId, SystemInst};
+use orchid_api::proto::{ExtMsgSet, ExtensionHeader, HostExtNotif, HostExtReq, HostHeader, Ping};
-use orchid_api::vfs::{EagerVfs, VfsId, VfsRead, VfsReq};
+use orchid_api::system::{SysId, SystemDrop, SystemInst};
+use orchid_api::tree::{GetConstTree, Tree, TreeId};
+use orchid_api::vfs::{EagerVfs, GetVfs, VfsId, VfsRead, VfsReq};
 use orchid_api_traits::{Decode, Encode};
 use orchid_base::char_filter::{char_filter_union, mk_char_filter};
 use orchid_base::clone;
-use orchid_base::intern::{deintern, init_replica, sweep_replica};
+use orchid_base::interner::{deintern, init_replica, sweep_replica};
 use orchid_base::name::PathSlice;
 use orchid_base::reqnot::{ReqNot, Requester};
 
 use crate::atom::AtomInfo;
+use crate::error::{err_to_api, unpack_err};
 use crate::fs::VirtFS;
+use crate::lexer::LexContext;
 use crate::msg::{recv_parent_msg, send_parent_msg};
-use crate::system::DynSystem;
+use crate::system::{atom_by_idx, SysCtx};
-use crate::system_ctor::DynSystemCtor;
+use crate::system_ctor::{CtedObj, DynSystemCtor};
+use crate::tree::LazyTreeFactory;
 
 pub struct ExtensionData {
-  pub systems: Vec<Box<dyn DynSystemCtor>>,
+  pub systems: &'static [&'static dyn DynSystemCtor],
+}
+
+pub enum TreeRecord {
+  Gen(LazyTreeFactory),
+  Res(Tree),
 }
 
 pub struct SystemRecord {
-  instance: Box<dyn DynSystem>,
+  cted: CtedObj,
-  vfses: HashMap<VfsId, Arc<dyn VirtFS>>,
+  vfses: HashMap<VfsId, &'static dyn VirtFS>,
   declfs: EagerVfs,
+  tree: Tree,
+  subtrees: HashMap<TreeId, TreeRecord>,
 }
 
 pub fn with_atom_record<T>(
   systems: &Mutex<HashMap<SysId, SystemRecord>>,
   atom: &Atom,
-  cb: impl FnOnce(&AtomInfo, &[u8]) -> T,
+  cb: impl FnOnce(&AtomInfo, CtedObj, &[u8]) -> T,
 ) -> T {
   let mut data = &atom.data[..];
   let systems_g = systems.lock().unwrap();
-  let sys = &systems_g[&atom.owner].instance;
+  let cted = &systems_g[&atom.owner].cted;
-  let atom_record =
+  let sys = cted.inst();
-    (sys.card().atoms()[u64::decode(&mut data) as usize].as_ref()).expect("Atom ID reserved");
+  let atom_record = atom_by_idx(sys.dyn_card(), u64::decode(&mut data)).expect("Atom ID reserved");
-  cb(atom_record, data)
+  cb(atom_record, cted.clone(), data)
 }
 
-pub fn main(data: ExtensionData) {
+pub fn extension_main(data: ExtensionData) {
   HostHeader::decode(&mut &recv_parent_msg().unwrap()[..]);
   let mut buf = Vec::new();
-  let decls = data.systems.iter().map(|sys| sys.decl()).collect_vec();
+  let decls = (data.systems.iter().enumerate())
+    .map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
+    .map(|(id, sys)| sys.decl(NonZeroU16::new(id + 1).unwrap()))
+    .collect_vec();
   let systems = Arc::new(Mutex::new(HashMap::<SysId, SystemRecord>::new()));
   ExtensionHeader { systems: decls.clone() }.encode(&mut buf);
   send_parent_msg(&buf).unwrap();
   let exiting = Arc::new(AtomicBool::new(false));
   let rn = ReqNot::<ExtMsgSet>::new(
     |a, _| send_parent_msg(a).unwrap(),
-    clone!(systems, exiting; move |n, _| match n {
+    clone!(systems, exiting; move |n, reqnot| match n {
       HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
-      HostExtNotif::SystemDrop(sys) => mem::drop(systems.lock().unwrap().remove(&sys.0)),
+      HostExtNotif::SystemDrop(SystemDrop(sys_id)) =>
-      HostExtNotif::AtomDrop(atom) =>
+        mem::drop(systems.lock().unwrap().remove(&sys_id)),
-        with_atom_record(&systems, &atom.0, |rec, data| (rec.drop)(data)),
+      HostExtNotif::AtomDrop(AtomDrop(atom)) => {
-    }),
+        with_atom_record(&systems, &atom, |rec, cted, data| {
-    clone!(systems; move |req| match req.req() {
+          (rec.drop)(data, SysCtx{ reqnot, id: atom.owner, cted })
-      HostExtReq::Ping(ping) => req.handle(ping, &()),
-      HostExtReq::Sweep(sweep) => req.handle(sweep, &sweep_replica()),
-      HostExtReq::NewSystem(new_sys) => {
-        let i = decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system).unwrap().0;
-        let system = data.systems[i].new_system(new_sys, req.reqnot());
-        let mut vfses = HashMap::new();
-        let lex_filter = system.lexers().iter().fold(CharFilter(vec![]), |cf, lx| {
-          char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
-        });
-        systems.lock().unwrap().insert(new_sys.id, SystemRecord {
-          declfs: system.source().to_api_rec(&mut vfses),
-          vfses,
-          instance: system,
-        });
-        req.handle(new_sys, &SystemInst {
-          lex_filter
         })
       }
-      HostExtReq::GetConstTree(get_tree) => {
+    }),
-        let systems_g = systems.lock().unwrap();
+    clone!(systems; move |req| match req.req() {
-        req.handle(get_tree, &systems_g[&get_tree.0].instance.env())
+      HostExtReq::Ping(ping@Ping) => req.handle(ping, &()),
+      HostExtReq::Sweep(sweep@Sweep) => req.handle(sweep, &sweep_replica()),
+      HostExtReq::NewSystem(new_sys) => {
+        let i = decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system).unwrap().0;
+        let cted = data.systems[i].new_system(new_sys);
+        let mut vfses = HashMap::new();
+        let lex_filter = cted.inst().dyn_lexers().iter().fold(CharFilter(vec![]), |cf, lx| {
+          char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
+        });
+        let mut subtrees = HashMap::new();
+        systems.lock().unwrap().insert(new_sys.id, SystemRecord {
+          declfs: cted.inst().dyn_vfs().to_api_rec(&mut vfses),
+          vfses,
+          tree: cted.inst().dyn_env().into_api(&*cted.inst(), &mut |gen| {
+            let id = TreeId::new((subtrees.len() + 2) as u64).unwrap();
+            subtrees.insert(id, TreeRecord::Gen(gen.clone()));
+            id
+          }),
+          cted,
+          subtrees
+        });
+        req.handle(new_sys, &SystemInst {
+          lex_filter, const_root_id: NonZeroU64::new(1).unwrap()
+        })
       }
-      HostExtReq::VfsReq(VfsReq::GetVfs(get_vfs)) => {
+      HostExtReq::GetConstTree(get_tree@GetConstTree(sys_id, tree_id)) => {
+        let mut systems_g = systems.lock().unwrap();
+        let sys = systems_g.get_mut(sys_id).expect("System not found");
+        if tree_id.get() == 1 {
+          req.handle(get_tree, &sys.tree);
+        } else {
+          let subtrees = &mut sys.subtrees;
+          let tree_rec = subtrees.get_mut(tree_id).expect("Tree for ID not found");
+          match tree_rec {
+            TreeRecord::Res(tree) => req.handle(get_tree, tree),
+            TreeRecord::Gen(cb) => {
+              let tree = cb.build();
+              let reply_tree = tree.into_api(&*sys.cted.inst(), &mut |cb| {
+                let id = NonZeroU64::new((subtrees.len() + 2) as u64).unwrap();
+                subtrees.insert(id, TreeRecord::Gen(cb.clone()));
+                id
+              });
+              req.handle(get_tree, &reply_tree);
+              subtrees.insert(*tree_id, TreeRecord::Res(reply_tree));
+            }
+          }
+        }
+      }
+      HostExtReq::VfsReq(VfsReq::GetVfs(get_vfs@GetVfs(sys_id))) => {
         let systems_g = systems.lock().unwrap();
-        req.handle(get_vfs, &systems_g[&get_vfs.0].declfs)
+        req.handle(get_vfs, &systems_g[sys_id].declfs)
       }
       HostExtReq::VfsReq(VfsReq::VfsRead(vfs_read@VfsRead(sys_id, vfs_id, path))) => {
         let systems_g = systems.lock().unwrap();
@@ -94,30 +140,30 @@ pub fn main(data: ExtensionData) {
         req.handle(vfs_read, &systems_g[sys_id].vfses[vfs_id].load(PathSlice::new(&path)))
       }
       HostExtReq::ParserReq(ParserReq::Lex(lex)) => {
+        let Lex{ sys, text, pos, id } = *lex;
         let systems_g = systems.lock().unwrap();
-        let Lex{ sys, text, pos } = *lex;
+        let lexers = systems_g[&sys].cted.inst().dyn_lexers();
-        let lexers = systems_g[&sys].instance.lexers();
         mem::drop(systems_g);
-        let source = deintern(text);
+        let text = deintern(text);
         let tk = req.will_handle_as(lex);
-        thread::spawn(move || {
+        thread::spawn(clone!(systems; move || {
-          let reqnot = req.reqnot();
+          let ctx = LexContext { sys, id, pos, reqnot: req.reqnot(), text: &text };
-          let mut recurse = |tail: &str| {
+          let lex_res = lexers.iter().find_map(|lx| lx.lex(&text[pos as usize..], &ctx));
-            let pos = (source.len() - tail.len()) as u32;
+          req.handle_as(tk, &lex_res.map(|r| match r {
-            let lexed = reqnot.request(SubLex{ pos, text })?;
+            Ok((s, data)) => {
-            Ok((&source[lexed.pos as usize..], lexed.data))
+              let systems_g = systems.lock().unwrap();
-          };
+              let data = data.into_api(&*systems_g[&sys].cted.inst());
-          let lex_res = lexers.iter().find_map(|lx| lx.lex(&source[pos as usize..], &mut recurse));
+              Ok(Lexed { data, pos: (text.len() - s.len()) as u32 })
-          req.handle_as(tk, &lex_res.map(|r| r.map(|(s, data)| {
+            },
-            let pos = (source.len() - s.len()) as u32;
+            Err(e) => Err(unpack_err(e).into_iter().map(err_to_api).collect_vec())
-            Lexed { data, pos }
+          }))
-          })))
+        }));
-        });
       },
       HostExtReq::AtomReq(AtomReq::AtomSame(same@AtomSame(l, r))) => todo!("subsys nimpl"),
       HostExtReq::AtomReq(AtomReq::Fwded(call@Fwded(atom, req))) => todo!("subsys nimpl"),
       HostExtReq::AtomReq(AtomReq::CallRef(call@CallRef(atom, arg))) => todo!("subsys nimpl"),
       HostExtReq::AtomReq(AtomReq::FinalCall(call@FinalCall(atom, arg))) => todo!("subsys nimpl"),
+      HostExtReq::AtomReq(AtomReq::Command(cmd@Command(atom))) => todo!("subsys impl"),
     }),
   );
   init_replica(rn.clone().map());

orchid-extension/src/error.rs

@@ -1,31 +1,19 @@
-//! Abstractions for handling various code-related errors under a common trait
-//! object.
-
 use std::any::Any;
+use std::borrow::Cow;
 use std::cell::RefCell;
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
-use std::{fmt, process};
+use std::{fmt, iter};
 
 use dyn_clone::{clone_box, DynClone};
 use itertools::Itertools;
-
+use orchid_api::error::{GetErrorDetails, ProjErr, ProjErrId, ProjErrOrRef};
-use crate::boxed_iter::{box_once, BoxedIter};
+use orchid_api::proto::ExtMsgSet;
-use crate::location::CodeOrigin;
+use orchid_base::boxed_iter::{box_once, BoxedIter};
-#[allow(unused)] // for doc
+use orchid_base::clone;
-use crate::virt_fs::CodeNotFound;
+use orchid_base::error::{ErrorDetails, ErrorPosition};
-
+use orchid_base::interner::{deintern, intern};
-/// A point of interest in resolving the error, such as the point where
+use orchid_base::location::{GetSrc, Pos};
-/// processing got stuck, a command that is likely to be incorrect
+use orchid_base::reqnot::{ReqNot, Requester};
-#[derive(Clone)]
-pub struct ErrorPosition {
-  /// The suspected origin
-  pub origin: CodeOrigin,
-  /// Any information about the role of this origin
-  pub message: Option<String>,
-}
-impl From<CodeOrigin> for ErrorPosition {
-  fn from(origin: CodeOrigin) -> Self { Self { origin, message: None } }
-}
 
 /// Errors addressed to the developer which are to be resolved with
 /// code changes
@@ -39,12 +27,14 @@ pub trait ProjectError: Sized + Send + Sync + 'static {
   /// must implement [ProjectError::one_position]
   #[must_use]
   fn positions(&self) -> impl IntoIterator<Item = ErrorPosition> + '_ {
-    box_once(ErrorPosition { origin: self.one_position(), message: None })
+    box_once(ErrorPosition { position: self.one_position(), message: None })
   }
   /// Short way to provide a single origin. If you don't implement this, you
   /// must implement [ProjectError::positions]
   #[must_use]
-  fn one_position(&self) -> CodeOrigin { unimplemented!() }
+  fn one_position(&self) -> Pos {
+    unimplemented!("Error type did not implement either positions or one_position")
+  }
   /// Convert the error into an `Arc<dyn DynProjectError>` to be able to
   /// handle various errors together
   #[must_use]
@@ -62,7 +52,7 @@ pub trait DynProjectError: Send + Sync {
   fn into_packed(self: Arc<Self>) -> ProjectErrorObj;
   /// A general description of this type of error
   #[must_use]
-  fn description(&self) -> &str;
+  fn description(&self) -> Cow<'_, str>;
   /// A formatted message that includes specific parameters
   #[must_use]
   fn message(&self) -> String { self.description().to_string() }
@@ -76,43 +66,36 @@ where T: ProjectError
 {
   fn as_any_ref(&self) -> &dyn Any { self }
   fn into_packed(self: Arc<Self>) -> ProjectErrorObj { self }
-  fn description(&self) -> &str { T::DESCRIPTION }
+  fn description(&self) -> Cow<'_, str> { Cow::Borrowed(T::DESCRIPTION) }
   fn message(&self) -> String { ProjectError::message(self) }
   fn positions(&self) -> BoxedIter<ErrorPosition> {
     Box::new(ProjectError::positions(self).into_iter())
   }
 }
 
-impl DynProjectError for ProjectErrorObj {
+pub fn pretty_print(err: &dyn DynProjectError, get_src: &mut impl GetSrc) -> String {
-  fn as_any_ref(&self) -> &dyn Any { (**self).as_any_ref() }
+  let description = err.description();
-  fn description(&self) -> &str { (**self).description() }
+  let message = err.message();
-  fn into_packed(self: Arc<Self>) -> ProjectErrorObj { (*self).clone() }
+  let positions = err.positions().collect::<Vec<_>>();
-  fn message(&self) -> String { (**self).message() }
+  let head = format!("Project error: {description}\n{message}");
-  fn positions(&self) -> BoxedIter<'_, ErrorPosition> { (**self).positions() }
+  if positions.is_empty() {
-}
+    head + "No origins specified"
-
+  } else {
-impl fmt::Display for dyn DynProjectError {
+    iter::once(head)
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+      .chain(positions.iter().map(|ErrorPosition { position: origin, message }| match message {
-    let description = self.description();
+        None => format!("@{}", origin.pretty_print(get_src)),
-    let message = self.message();
+        Some(msg) => format!("@{}: {msg}", origin.pretty_print(get_src)),
-    let positions = self.positions().collect::<Vec<_>>();
+      }))
-    writeln!(f, "Project error: {description}\n{message}")?;
+      .join("\n")
-    if positions.is_empty() {
-      writeln!(f, "No origins specified")?;
-    } else {
-      for ErrorPosition { origin, message } in positions {
-        match message {
-          None => writeln!(f, "@{origin}"),
-          Some(msg) => writeln!(f, "@{origin}: {msg}"),
-        }?
-      }
-    }
-    Ok(())
   }
 }
 
-impl fmt::Debug for dyn DynProjectError {
+impl DynProjectError for ProjectErrorObj {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") }
+  fn as_any_ref(&self) -> &dyn Any { (**self).as_any_ref() }
+  fn description(&self) -> Cow<'_, str> { (**self).description() }
+  fn into_packed(self: Arc<Self>) -> ProjectErrorObj { (*self).clone() }
+  fn message(&self) -> String { (**self).message() }
+  fn positions(&self) -> BoxedIter<'_, ErrorPosition> { (**self).positions() }
 }
 
 /// Type-erased [ProjectError] implementor through the [DynProjectError]
@@ -135,7 +118,7 @@ pub trait ErrorSansOrigin: Clone + Sized + Send + Sync + 'static {
   fn pack(self) -> ErrorSansOriginObj { Box::new(self) }
   /// A shortcut to streamline switching code between [ErrorSansOriginObj] and
   /// concrete types
-  fn bundle(self, origin: &CodeOrigin) -> ProjectErrorObj { self.pack().bundle(origin) }
+  fn bundle(self, origin: &Pos) -> ProjectErrorObj { self.pack().bundle(origin) }
 }
 
 /// Object-safe equivalent to [ErrorSansOrigin]. Implement that one instead of
@@ -148,11 +131,11 @@ pub trait DynErrorSansOrigin: Any + Send + Sync + DynClone {
   /// Regularize the type
   fn into_packed(self: Box<Self>) -> ErrorSansOriginObj;
   /// Generic description of the error condition
-  fn description(&self) -> &str;
+  fn description(&self) -> Cow<'_, str>;
   /// Specific description of this particular error
   fn message(&self) -> String;
   /// Add an origin
-  fn bundle(self: Box<Self>, origin: &CodeOrigin) -> ProjectErrorObj;
+  fn bundle(self: Box<Self>, origin: &Pos) -> ProjectErrorObj;
 }
 
 /// Type-erased [ErrorSansOrigin] implementor through the object-trait
@@ -163,11 +146,11 @@ pub type ErrorSansOriginObj = Box<dyn DynErrorSansOrigin>;
 pub type ResultSansOrigin<T> = Result<T, ErrorSansOriginObj>;
 
 impl<T: ErrorSansOrigin + 'static> DynErrorSansOrigin for T {
-  fn description(&self) -> &str { Self::DESCRIPTION }
+  fn description(&self) -> Cow<'_, str> { Cow::Borrowed(Self::DESCRIPTION) }
   fn message(&self) -> String { (*self).message() }
   fn as_any_ref(&self) -> &dyn Any { self }
   fn into_packed(self: Box<Self>) -> ErrorSansOriginObj { (*self).pack() }
-  fn bundle(self: Box<Self>, origin: &CodeOrigin) -> ProjectErrorObj {
+  fn bundle(self: Box<Self>, origin: &Pos) -> ProjectErrorObj {
     Arc::new(OriginBundle(origin.clone(), *self))
   }
 }
@@ -175,11 +158,11 @@ impl Clone for ErrorSansOriginObj {
   fn clone(&self) -> Self { clone_box(&**self) }
 }
 impl DynErrorSansOrigin for ErrorSansOriginObj {
-  fn description(&self) -> &str { (**self).description() }
+  fn description(&self) -> Cow<'_, str> { (**self).description() }
   fn message(&self) -> String { (**self).message() }
   fn as_any_ref(&self) -> &dyn Any { (**self).as_any_ref() }
   fn into_packed(self: Box<Self>) -> ErrorSansOriginObj { *self }
-  fn bundle(self: Box<Self>, origin: &CodeOrigin) -> ProjectErrorObj { (*self).bundle(origin) }
+  fn bundle(self: Box<Self>, origin: &Pos) -> ProjectErrorObj { (*self).bundle(origin) }
 }
 impl fmt::Display for ErrorSansOriginObj {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -190,14 +173,14 @@ impl fmt::Debug for ErrorSansOriginObj {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") }
 }
 
-struct OriginBundle<T: ErrorSansOrigin>(CodeOrigin, T);
+struct OriginBundle<T: ErrorSansOrigin>(Pos, T);
 impl<T: ErrorSansOrigin> DynProjectError for OriginBundle<T> {
   fn as_any_ref(&self) -> &dyn Any { self.1.as_any_ref() }
   fn into_packed(self: Arc<Self>) -> ProjectErrorObj { self }
-  fn description(&self) -> &str { self.1.description() }
+  fn description(&self) -> Cow<'_, str> { self.1.description() }
   fn message(&self) -> String { self.1.message() }
   fn positions(&self) -> BoxedIter<ErrorPosition> {
-    box_once(ErrorPosition { origin: self.0.clone(), message: None })
+    box_once(ErrorPosition { position: self.0.clone(), message: None })
   }
 }
 
@@ -233,30 +216,6 @@ impl Reporter {
   pub fn fallback<T>(&self, res: ProjectResult<T>, cb: impl FnOnce(ProjectErrorObj) -> T) -> T {
     res.inspect_err(|e| self.report(e.clone())).unwrap_or_else(cb)
   }
-  /// Panic if there were errors
-  pub fn assert(&self) { self.unwrap(Ok(())) }
-  /// Exit with code -1 if there were errors
-  pub fn assert_exit(&self) { self.unwrap_exit(Ok(())) }
-  /// Panic with descriptive messages if there were errors. If there were no
-  /// errors, unwrap the result
-  pub fn unwrap<T>(&self, res: ProjectResult<T>) -> T {
-    if self.failing() {
-      panic!("Errors were encountered: \n{}", self.0.borrow().iter().join("\n"));
-    }
-    res.unwrap()
-  }
-  /// Print errors and exit if any occurred.  If there were no errors, unwrap
-  /// the result
-  pub fn unwrap_exit<T>(&self, res: ProjectResult<T>) -> T {
-    if self.failing() {
-      eprintln!("Errors were encountered: \n{}", self.0.borrow().iter().join("\n"));
-      process::exit(-1)
-    }
-    res.unwrap_or_else(|e| {
-      eprintln!("{e}");
-      process::exit(-1)
-    })
-  }
   /// Take the errors out of the reporter
   #[must_use]
   pub fn into_errors(self) -> Option<Vec<ProjectErrorObj>> {
@@ -267,7 +226,7 @@ impl Reporter {
   pub fn bind(self) -> ProjectResult<()> {
     match self.into_errors() {
       None => Ok(()),
-      Some(v) if v.len() == 1 => Err(v.into_iter().exactly_one().unwrap()),
+      Some(v) if v.len() == 1 => Err(v.into_iter().next().unwrap()),
       Some(v) => Err(MultiError(v).pack()),
     }
   }
@@ -277,6 +236,25 @@ impl Default for Reporter {
   fn default() -> Self { Self::new() }
 }
 
+fn unpack_into(err: impl DynProjectError, res: &mut Vec<ProjectErrorObj>) {
+  match err.as_any_ref().downcast_ref::<MultiError>() {
+    Some(multi) => multi.0.iter().for_each(|e| unpack_into(e.clone(), res)),
+    None => res.push(Arc::new(err).into_packed()),
+  }
+}
+
+pub fn unpack_err(err: ProjectErrorObj) -> Vec<ProjectErrorObj> {
+  let mut out = Vec::new();
+  unpack_into(err, &mut out);
+  out
+}
+
+pub fn pack_err<E: DynProjectError>(iter: impl IntoIterator<Item = E>) -> ProjectErrorObj {
+  let mut errors = Vec::new();
+  iter.into_iter().for_each(|e| unpack_into(e, &mut errors));
+  if errors.len() == 1 { errors.into_iter().next().unwrap() } else { MultiError(errors).pack() }
+}
+
 struct MultiError(Vec<ProjectErrorObj>);
 impl ProjectError for MultiError {
   const DESCRIPTION: &'static str = "Multiple errors occurred";
@@ -290,8 +268,56 @@ impl ProjectError for MultiError {
           Some(s) if s.is_empty() => emsg,
           Some(pmsg) => format!("{emsg}: {pmsg}"),
         };
-        ErrorPosition { origin: pos.origin, message: Some(msg) }
+        ErrorPosition { position: pos.position, message: Some(Arc::new(msg)) }
       })
     })
   }
 }
+
+pub fn err_to_api(err: ProjectErrorObj) -> ProjErrOrRef {
+  match err.as_any_ref().downcast_ref() {
+    Some(RelayedError { id: Some(id), .. }) => ProjErrOrRef::Known(*id),
+    _ => ProjErrOrRef::New(ProjErr {
+      description: intern(&*err.description()).marker(),
+      message: Arc::new(err.message()),
+      locations: err.positions().map(|e| e.to_api()).collect_vec(),
+    }),
+  }
+}
+
+pub fn err_from_api(err: &ProjErrOrRef, reqnot: ReqNot<ExtMsgSet>) -> ProjectErrorObj {
+  Arc::new(match err {
+    ProjErrOrRef::Known(id) => RelayedError { id: Some(*id), reqnot, details: OnceLock::default() },
+    ProjErrOrRef::New(err) =>
+      RelayedError { id: None, reqnot, details: ErrorDetails::from_api(err).into() },
+  })
+}
+
+struct RelayedError {
+  pub id: Option<ProjErrId>,
+  pub reqnot: ReqNot<ExtMsgSet>,
+  pub details: OnceLock<ErrorDetails>,
+}
+impl RelayedError {
+  fn details(&self) -> &ErrorDetails {
+    let Self { id, reqnot, details: data } = self;
+    data.get_or_init(clone!(reqnot; move || {
+      let id = id.expect("Either data or ID must be initialized");
+      let projerr = reqnot.request(GetErrorDetails(id));
+      ErrorDetails {
+        description: deintern(projerr.description),
+        message: projerr.message,
+        locations: projerr.locations.iter().map(ErrorPosition::from_api).collect_vec(),
+      }
+    }))
+  }
+}
+impl DynProjectError for RelayedError {
+  fn description(&self) -> Cow<'_, str> { Cow::Borrowed(self.details().description.as_str()) }
+  fn message(&self) -> String { self.details().message.to_string() }
+  fn as_any_ref(&self) -> &dyn std::any::Any { self }
+  fn into_packed(self: Arc<Self>) -> ProjectErrorObj { self }
+  fn positions(&self) -> BoxedIter<'_, ErrorPosition> {
+    Box::new(self.details().locations.iter().cloned())
+  }
+}

orchid-extension/src/expr.rs

@@ -1,53 +1,173 @@
+use std::ops::Deref;
+use std::sync::OnceLock;
+
+use derive_destructure::destructure;
 use orchid_api::atom::Atom;
-use orchid_api::expr::ExprTicket;
+use orchid_api::expr::{Acquire, Clause, Expr, ExprTicket, Inspect, Release};
-use orchid_api::system::SysId;
+use orchid_base::interner::{deintern, Tok};
-use orchid_base::id_store::IdStore;
+use orchid_base::location::Pos;
-use orchid_base::intern::Token;
+use orchid_base::reqnot::Requester;
 
-use crate::atom::{encode_atom_nodrop, DynOwnedAtom, OwnedAtom, ThinAtom, OBJ_STORE};
+use crate::atom::{AtomFactory, ForeignAtom, OwnedAtom, ThinAtom};
-use crate::system::DynSystem;
+use crate::system::{DynSystem, SysCtx};
 
+#[derive(destructure)]
+pub struct ExprHandle {
+  pub tk: ExprTicket,
+  pub ctx: SysCtx,
+}
+impl ExprHandle {
+  pub(crate) fn from_args(ctx: SysCtx, tk: ExprTicket) -> Self { Self { ctx, tk } }
+  pub(crate) fn into_tk(self) -> ExprTicket {
+    let (tk, ..) = self.destructure();
+    tk
+  }
+  pub(crate) fn get_tk(&self) -> ExprTicket { self.tk }
+  pub fn get_ctx(&self) -> SysCtx { self.ctx.clone() }
+}
+impl Clone for ExprHandle {
+  fn clone(&self) -> Self {
+    self.ctx.reqnot.notify(Acquire(self.ctx.id, self.tk));
+    Self { ctx: self.ctx.clone(), tk: self.tk }
+  }
+}
+impl Drop for ExprHandle {
+  fn drop(&mut self) { self.ctx.reqnot.notify(Release(self.ctx.id, self.tk)) }
+}
+
+#[derive(Clone, destructure)]
+pub struct OwnedExpr {
+  pub handle: ExprHandle,
+  pub val: OnceLock<Box<GenExpr>>,
+}
+impl OwnedExpr {
+  pub fn new(handle: ExprHandle) -> Self { Self { handle, val: OnceLock::new() } }
+  pub fn get_data(&self) -> &GenExpr {
+    self.val.get_or_init(|| {
+      Box::new(GenExpr::from_api(
+        self.handle.ctx.reqnot.request(Inspect(self.handle.tk)).expr,
+        self.handle.get_ctx(),
+      ))
+    })
+  }
+  pub fn foreign_atom(self) -> Result<ForeignAtom, Self> {
+    if let GenExpr { clause: GenClause::Atom(_, atom), position } = self.get_data() {
+      let (atom, position) = (atom.clone(), position.clone());
+      return Ok(ForeignAtom { expr: self.handle, atom, position });
+    }
+    Err(self)
+  }
+}
+impl Deref for OwnedExpr {
+  type Target = GenExpr;
+  fn deref(&self) -> &Self::Target { self.get_data() }
+}
+
+#[derive(Clone)]
+pub struct GenExpr {
+  pub position: Pos,
+  pub clause: GenClause,
+}
+impl GenExpr {
+  pub fn to_api(&self, sys: &dyn DynSystem) -> Expr {
+    Expr { location: self.position.to_api(), clause: self.clause.to_api(sys) }
+  }
+  pub fn into_api(self, sys: &dyn DynSystem) -> Expr {
+    Expr { location: self.position.to_api(), clause: self.clause.into_api(sys) }
+  }
+  pub fn from_api(api: Expr, ctx: SysCtx) -> Self {
+    Self { position: Pos::from_api(&api.location), clause: GenClause::from_api(api.clause, ctx) }
+  }
+}
+
+#[derive(Clone)]
 pub enum GenClause {
-  Call(Box<GenClause>, Box<GenClause>),
+  Call(Box<GenExpr>, Box<GenExpr>),
-  Lambda(Token<String>, Box<GenClause>),
+  Lambda(u64, Box<GenExpr>),
-  Arg(Token<String>),
+  Arg(u64),
-  Slot(ExprTicket),
+  Slot(OwnedExpr),
-  Seq(Box<GenClause>, Box<GenClause>),
+  Seq(Box<GenExpr>, Box<GenExpr>),
-  Const(Token<Vec<Token<String>>>),
+  Const(Tok<Vec<Tok<String>>>),
-  ThinAtom(Box<dyn Fn(SysId, &dyn DynSystem) -> Atom>),
+  NewAtom(AtomFactory),
-  OwnedAtom(u64),
+  Atom(ExprTicket, Atom),
+  Bottom(String),
 }
-
+impl GenClause {
-pub fn cnst(path: Token<Vec<Token<String>>>) -> GenClause { GenClause::Const(path) }
+  pub fn to_api(&self, sys: &dyn DynSystem) -> Clause {
-pub fn val<A: ThinAtom>(atom: A) -> GenClause {
+    match self {
-  GenClause::ThinAtom(Box::new(move |id, sys| encode_atom_nodrop::<A>(id, sys.card(), &atom)))
+      Self::Call(f, x) => Clause::Call(Box::new(f.to_api(sys)), Box::new(x.to_api(sys))),
+      Self::Seq(a, b) => Clause::Seq(Box::new(a.to_api(sys)), Box::new(b.to_api(sys))),
+      Self::Lambda(arg, body) => Clause::Lambda(*arg, Box::new(body.to_api(sys))),
+      Self::Arg(arg) => Clause::Arg(*arg),
+      Self::Const(name) => Clause::Const(name.marker()),
+      Self::Bottom(msg) => Clause::Bottom(msg.clone()),
+      Self::NewAtom(fac) => Clause::NewAtom(fac.clone().build(sys)),
+      Self::Atom(tk, atom) => Clause::Atom(*tk, atom.clone()),
+      Self::Slot(_) => panic!("Slot is forbidden in const tree"),
+    }
+  }
+  pub fn into_api(self, sys: &dyn DynSystem) -> Clause {
+    match self {
+      Self::Call(f, x) => Clause::Call(Box::new(f.into_api(sys)), Box::new(x.into_api(sys))),
+      Self::Seq(a, b) => Clause::Seq(Box::new(a.into_api(sys)), Box::new(b.into_api(sys))),
+      Self::Lambda(arg, body) => Clause::Lambda(arg, Box::new(body.into_api(sys))),
+      Self::Arg(arg) => Clause::Arg(arg),
+      Self::Slot(extk) => Clause::Slot(extk.handle.into_tk()),
+      Self::Const(name) => Clause::Const(name.marker()),
+      Self::Bottom(msg) => Clause::Bottom(msg.clone()),
+      Self::NewAtom(fac) => Clause::NewAtom(fac.clone().build(sys)),
+      Self::Atom(tk, atom) => Clause::Atom(tk, atom),
+    }
+  }
+  pub fn from_api(api: Clause, ctx: SysCtx) -> Self {
+    match api {
+      Clause::Arg(id) => Self::Arg(id),
+      Clause::Lambda(arg, body) => Self::Lambda(arg, Box::new(GenExpr::from_api(*body, ctx))),
+      Clause::NewAtom(_) => panic!("Clause::NewAtom should never be received, only sent"),
+      Clause::Bottom(s) => Self::Bottom(s),
+      Clause::Call(f, x) => Self::Call(
+        Box::new(GenExpr::from_api(*f, ctx.clone())),
+        Box::new(GenExpr::from_api(*x, ctx)),
+      ),
+      Clause::Seq(a, b) => Self::Seq(
+        Box::new(GenExpr::from_api(*a, ctx.clone())),
+        Box::new(GenExpr::from_api(*b, ctx)),
+      ),
+      Clause::Const(name) => Self::Const(deintern(name)),
+      Clause::Slot(exi) => Self::Slot(OwnedExpr::new(ExprHandle::from_args(ctx, exi))),
+      Clause::Atom(tk, atom) => Self::Atom(tk, atom),
+    }
+  }
 }
+fn inherit(clause: GenClause) -> GenExpr { GenExpr { position: Pos::Inherit, clause } }
 
-pub fn obj<A: OwnedAtom>(atom: A) -> GenClause {
+pub fn cnst(path: Tok<Vec<Tok<String>>>) -> GenExpr { inherit(GenClause::Const(path)) }
-  GenClause::OwnedAtom(OBJ_STORE.add(Box::new(atom)))
+pub fn val<A: ThinAtom>(atom: A) -> GenExpr { inherit(GenClause::NewAtom(atom.factory())) }
-}
+pub fn obj<A: OwnedAtom>(atom: A) -> GenExpr { inherit(GenClause::NewAtom(atom.factory())) }
 
-pub fn seq(ops: impl IntoIterator<Item = GenClause>) -> GenClause {
+pub fn seq(ops: impl IntoIterator<Item = GenExpr>) -> GenExpr {
-  fn recur(mut ops: impl Iterator<Item = GenClause>) -> Option<GenClause> {
+  fn recur(mut ops: impl Iterator<Item = GenExpr>) -> Option<GenExpr> {
     let op = ops.next()?;
     Some(match recur(ops) {
       None => op,
-      Some(rec) => GenClause::Seq(Box::new(op), Box::new(rec)),
+      Some(rec) => inherit(GenClause::Seq(Box::new(op), Box::new(rec))),
     })
   }
   recur(ops.into_iter()).expect("Empty list provided to seq!")
 }
 
-pub fn slot(extk: ExprTicket) -> GenClause { GenClause::Slot(extk) }
+pub fn slot(extk: OwnedExpr) -> GenClause { GenClause::Slot(extk) }
 
-pub fn arg(n: Token<String>) -> GenClause { GenClause::Arg(n) }
+pub fn arg(n: u64) -> GenClause { GenClause::Arg(n) }
 
-pub fn lambda(n: Token<String>, b: impl IntoIterator<Item = GenClause>) -> GenClause {
+pub fn lambda(n: u64, b: impl IntoIterator<Item = GenExpr>) -> GenExpr {
-  GenClause::Lambda(n, Box::new(call(b)))
+  inherit(GenClause::Lambda(n, Box::new(call(b))))
 }
 
-pub fn call(v: impl IntoIterator<Item = GenClause>) -> GenClause {
+pub fn call(v: impl IntoIterator<Item = GenExpr>) -> GenExpr {
   v.into_iter()
-    .reduce(|f, x| GenClause::Call(Box::new(f), Box::new(x)))
+    .reduce(|f, x| inherit(GenClause::Call(Box::new(f), Box::new(x))))
     .expect("Empty call expression")
 }
+
+pub fn bot(msg: &str) -> GenClause { GenClause::Bottom(msg.to_string()) }

orchid-extension/src/fs.rs

@@ -1,47 +1,30 @@
-use std::sync::Arc;
+use std::num::NonZeroU16;
 
 use hashbrown::HashMap;
 use orchid_api::error::ProjResult;
 use orchid_api::vfs::{EagerVfs, Loaded, VfsId};
-use orchid_base::intern::{intern, Token};
+use orchid_base::interner::intern;
 use orchid_base::name::PathSlice;
-use substack::Substack;
-use trait_set::trait_set;
 
 pub trait VirtFS: Send + Sync + 'static {
   fn load(&self, path: &PathSlice) -> ProjResult<Loaded>;
 }
 
-trait_set! {
-  pub trait RecFsHandler<E> = FnMut(Substack<Token<String>>, &Arc<dyn VirtFS>) -> Result<(), E>;
-}
-
 pub enum DeclFs {
-  Lazy(Arc<dyn VirtFS>),
+  Lazy(&'static dyn VirtFS),
-  Mod(HashMap<Token<String>, DeclFs>),
+  Mod(&'static [(&'static str, DeclFs)]),
 }
 impl DeclFs {
-  pub fn module(entries: impl IntoIterator<Item = (&'static str, Self)>) -> Self {
+  pub fn to_api_rec(&self, vfses: &mut HashMap<VfsId, &'static dyn VirtFS>) -> EagerVfs {
-    Self::Mod(entries.into_iter().map(|(k, v)| (intern(k), v)).collect())
-  }
-  fn rec<E>(&self, path: Substack<Token<String>>, f: &mut impl RecFsHandler<E>) -> Result<(), E> {
-    match self {
-      DeclFs::Lazy(fs) => f(path, fs),
-      DeclFs::Mod(entries) => entries.iter().try_for_each(|(k, v)| v.rec(path.push(k.clone()), f)),
-    }
-  }
-  pub fn recurse<E>(&self, f: &mut impl RecFsHandler<E>) -> Result<(), E> {
-    self.rec(Substack::Bottom, f)
-  }
-  pub fn to_api_rec(&self, vfses: &mut HashMap<VfsId, Arc<dyn VirtFS>>) -> EagerVfs {
     match self {
       DeclFs::Lazy(fs) => {
-        let id = vfses.len() as VfsId;
+        let vfsc: u16 = vfses.len().try_into().expect("too many vfses (more than u16::MAX)");
-        vfses.insert(id, fs.clone());
+        let id: VfsId = NonZeroU16::new(vfsc + 1).unwrap();
+        vfses.insert(id, *fs);
         EagerVfs::Lazy(id)
       },
       DeclFs::Mod(children) => EagerVfs::Eager(
-        children.into_iter().map(|(k, v)| (k.marker(), v.to_api_rec(vfses))).collect(),
+        children.iter().map(|(k, v)| (intern(*k).marker(), v.to_api_rec(vfses))).collect(),
       ),
     }
   }

orchid-extension/src/fun.rs

@@ -0,0 +1,35 @@
+use std::borrow::Cow;
+
+use dyn_clone::{clone_box, DynClone};
+use never::Never;
+use trait_set::trait_set;
+
+use crate::atom::{AtomCard, OwnedAtom};
+use crate::expr::{ExprHandle, GenClause};
+use crate::system::SysCtx;
+
+trait_set! {
+  trait FunCB = FnOnce(ExprHandle) -> GenClause + DynClone + Send + Sync + 'static;
+}
+
+pub struct Fun(Box<dyn FunCB>);
+impl Fun {
+  pub fn new(f: impl FnOnce(ExprHandle) -> GenClause + Clone + Send + Sync + 'static) -> Self {
+    Self(Box::new(f))
+  }
+}
+impl Clone for Fun {
+  fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
+}
+impl AtomCard for Fun {
+  type Data = ();
+  type Req = Never;
+}
+impl OwnedAtom for Fun {
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+  fn call_ref(&self, arg: ExprHandle) -> GenClause { self.clone().call(arg) }
+  fn call(self, arg: ExprHandle) -> GenClause { (self.0)(arg) }
+  fn handle_req(&self, _ctx: SysCtx, req: Self::Req, _rep: &mut (impl std::io::Write + ?Sized)) {
+    match req {}
+  }
+}

orchid-extension/src/lexer.rs

@@ -1,14 +1,53 @@
-use std::ops::RangeInclusive;
+use std::ops::{Range, RangeInclusive};
 
-use orchid_api::error::ProjResult;
+use orchid_api::error::ReportError;
-use orchid_api::tree::TokenTree;
+use orchid_api::parser::{LexId, SubLex};
+use orchid_api::proto::ExtMsgSet;
+use orchid_api::system::SysId;
+use orchid_base::interner::Tok;
+use orchid_base::reqnot::{ReqNot, Requester};
+
+use crate::error::{
+  err_from_api, err_to_api, pack_err, unpack_err, ProjectErrorObj, ProjectResult,
+};
+use crate::tree::{GenTok, GenTokTree};
+
+pub struct LexContext<'a> {
+  pub text: &'a Tok<String>,
+  pub sys: SysId,
+  pub id: LexId,
+  pub pos: u32,
+  pub reqnot: ReqNot<ExtMsgSet>,
+}
+impl<'a> LexContext<'a> {
+  pub fn recurse(&self, tail: &'a str) -> ProjectResult<(&'a str, GenTokTree)> {
+    let start = self.pos(tail);
+    self
+      .reqnot
+      .request(SubLex { pos: start, id: self.id })
+      .map_err(|e| pack_err(e.iter().map(|e| err_from_api(e, self.reqnot.clone()))))
+      .map(|lx| (&self.text[lx.pos as usize..], GenTok::Slot(lx.ticket).at(start..lx.pos)))
+  }
+
+  pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }
+
+  pub fn tok_ran(&self, len: u32, tail: &'a str) -> Range<u32> {
+    self.pos(tail) - len..self.pos(tail)
+  }
+
+  pub fn report(&self, e: ProjectErrorObj) {
+    for e in unpack_err(e) {
+      self.reqnot.notify(ReportError(self.sys, err_to_api(e)))
+    }
+  }
+}
 
 pub trait Lexer: Send + Sync + Sized + Default + 'static {
   const CHAR_FILTER: &'static [RangeInclusive<char>];
   fn lex<'a>(
     tail: &'a str,
-    recur: impl FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+    ctx: &'a LexContext<'a>,
-  ) -> Option<ProjResult<(&'a str, TokenTree)>>;
+  ) -> Option<ProjectResult<(&'a str, GenTokTree)>>;
 }
 
 pub trait DynLexer: Send + Sync + 'static {
@@ -16,8 +55,8 @@ pub trait DynLexer: Send + Sync + 'static {
   fn lex<'a>(
     &self,
     tail: &'a str,
-    recur: &mut dyn FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+    ctx: &'a LexContext<'a>,
-  ) -> Option<ProjResult<(&'a str, TokenTree)>>;
+  ) -> Option<ProjectResult<(&'a str, GenTokTree)>>;
 }
 
 impl<T: Lexer> DynLexer for T {
@@ -25,9 +64,9 @@ impl<T: Lexer> DynLexer for T {
   fn lex<'a>(
     &self,
     tail: &'a str,
-    recur: &mut dyn FnMut(&'a str) -> ProjResult<(&'a str, TokenTree)>,
+    ctx: &'a LexContext<'a>,
-  ) -> Option<ProjResult<(&'a str, TokenTree)>> {
+  ) -> Option<ProjectResult<(&'a str, GenTokTree)>> {
-    T::lex(tail, recur)
+    T::lex(tail, ctx)
   }
 }
 

orchid-extension/src/lib.rs

@@ -1,9 +1,13 @@
 pub mod atom;
 pub mod entrypoint;
+pub mod error;
 pub mod expr;
 pub mod fs;
+pub mod fun;
 pub mod lexer;
 pub mod msg;
 pub mod other_system;
 pub mod system;
 pub mod system_ctor;
+pub mod tree;
+pub mod try_from_expr;

orchid-extension/src/other_system.rs

@@ -1,37 +1,37 @@
 use std::marker::PhantomData;
+use std::mem::size_of;
 
-use orchid_api::atom::Atom;
-use orchid_api::expr::ExprTicket;
-use orchid_api::proto::ExtMsgSet;
 use orchid_api::system::SysId;
-use orchid_base::reqnot::ReqNot;
 
-use crate::atom::{decode_atom, AtomCard, ForeignAtom};
+use crate::system::{DynSystemCard, SystemCard};
-use crate::system::SystemCard;
 
 pub struct SystemHandle<C: SystemCard> {
   pub(crate) _card: PhantomData<C>,
   pub(crate) id: SysId,
-  pub(crate) reqnot: ReqNot<ExtMsgSet>,
 }
-impl<T: SystemCard> SystemHandle<T> {
+impl<C: SystemCard> SystemHandle<C> {
-  pub(crate) fn new(id: SysId, reqnot: ReqNot<ExtMsgSet>) -> Self {
+  pub(crate) fn new(id: SysId) -> Self { Self { _card: PhantomData, id } }
-    Self { _card: PhantomData, id, reqnot }
-  }
   pub fn id(&self) -> SysId { self.id }
-  pub fn wrap_atom<A: AtomCard<Owner = T>>(
+}
-    &self,
+impl<C: SystemCard> Clone for SystemHandle<C> {
-    api: Atom,
+  fn clone(&self) -> Self { Self::new(self.id) }
-    ticket: ExprTicket,
+}
-  ) -> Result<ForeignAtom<A>, Atom> {
+
-    if api.owner == self.id {
+pub trait DynSystemHandle {
-      if let Some(value) = decode_atom::<A>(&T::default(), &api) {
+  fn id(&self) -> SysId;
-        return Ok(ForeignAtom { ticket, sys: self.clone(), value, api });
+  fn get_card(&self) -> &dyn DynSystemCard;
-      }
+}
+
+pub fn leak_card<T: Default>() -> &'static T {
+  const {
+    if 0 != size_of::<T>() {
+      panic!("Attempted to leak positively sized Card. Card types must always be zero-sized");
     }
-    Err(api)
   }
+  Box::leak(Box::default())
 }
-impl<T: SystemCard> Clone for SystemHandle<T> {
+
-  fn clone(&self) -> Self { Self { reqnot: self.reqnot.clone(), _card: PhantomData, id: self.id } }
+impl<C: SystemCard> DynSystemHandle for SystemHandle<C> {
+  fn id(&self) -> SysId { self.id }
+  fn get_card(&self) -> &'static dyn DynSystemCard { leak_card::<C>() }
 }

orchid-extension/src/proj_error.rs

@@ -0,0 +1,18 @@
+//! Abstractions for handling various code-related errors under a common trait
+//! object.
+
+use std::any::Any;
+use std::borrow::Cow;
+use std::cell::RefCell;
+use std::sync::Arc;
+use std::{fmt, iter};
+
+use dyn_clone::{clone_box, DynClone};
+use itertools::Itertools;
+use orchid_api::error::{ProjErr, ProjErrLocation};
+
+use crate::boxed_iter::{box_once, BoxedIter};
+use crate::intern::{deintern, intern, Token};
+use crate::location::{GetSrc, Position};
+#[allow(unused)] // for doc
+use crate::virt_fs::CodeNotFound;

orchid-extension/src/system.rs

@@ -1,53 +1,90 @@
-use std::any::Any;
+use orchid_api::proto::ExtMsgSet;
-use std::io::{Read, Write};
+use orchid_api::system::SysId;
-
+use orchid_base::reqnot::ReqNot;
-use orchid_api::expr::ExprTicket;
-use orchid_api::tree::TreeModule;
 use typeid::ConstTypeId;
 
-use crate::atom::AtomInfo;
+use crate::atom::{decode_atom, owned_atom_info, AtomCard, AtomInfo, ForeignAtom, TypAtom};
-use crate::expr::GenClause;
 use crate::fs::DeclFs;
+use crate::fun::Fun;
 use crate::lexer::LexerObj;
+use crate::system_ctor::{CtedObj, SystemCtor};
+use crate::tree::GenTree;
 
 /// System as consumed by foreign code
 pub trait SystemCard: Default + Send + Sync + 'static {
-  const NAME: &'static str;
+  type Ctor: SystemCtor;
   const ATOM_DEFS: &'static [Option<AtomInfo>];
 }
 
 pub trait DynSystemCard: Send + Sync + 'static {
   fn name(&self) -> &'static str;
+  /// Atoms explicitly defined by the system card. Do not rely on this for
+  /// querying atoms as it doesn't include the general atom types
   fn atoms(&self) -> &'static [Option<AtomInfo>];
-  fn atom_info_for(&self, tid: ConstTypeId) -> Option<(usize, &AtomInfo)> {
+}
-    (self.atoms().iter().enumerate())
+
-      .filter_map(|(i, o)| o.as_ref().map(|a| (i, a)))
+/// Atoms supported by this package which may appear in all extensions.
-      .find(|ent| ent.1.tid == tid)
+/// The indices of these are bitwise negated, such that the MSB of an atom index
+/// marks whether it belongs to this package (0) or the importer (1)
+const GENERAL_ATOMS: &[Option<AtomInfo>] = &[Some(owned_atom_info::<Fun>())];
+
+pub fn atom_info_for(
+  sys: &(impl DynSystemCard + ?Sized),
+  tid: ConstTypeId,
+) -> Option<(u64, &AtomInfo)> {
+  (sys.atoms().iter().enumerate().map(|(i, o)| (i as u64, o)))
+    .chain(GENERAL_ATOMS.iter().enumerate().map(|(i, o)| (!(i as u64), o)))
+    .filter_map(|(i, o)| o.as_ref().map(|a| (i, a)))
+    .find(|ent| ent.1.tid == tid)
+}
+
+pub fn atom_by_idx(sys: &(impl DynSystemCard + ?Sized), tid: u64) -> Option<&AtomInfo> {
+  if (tid >> (u64::BITS - 1)) & 1 == 1 {
+    GENERAL_ATOMS[!tid as usize].as_ref()
+  } else {
+    sys.atoms()[tid as usize].as_ref()
   }
 }
 
 impl<T: SystemCard> DynSystemCard for T {
-  fn name(&self) -> &'static str { Self::NAME }
+  fn name(&self) -> &'static str { T::Ctor::NAME }
   fn atoms(&self) -> &'static [Option<AtomInfo>] { Self::ATOM_DEFS }
 }
 
 /// System as defined by author
-pub trait System: Send + Sync + SystemCard {
+pub trait System: Send + Sync + SystemCard + 'static {
-  fn env() -> TreeModule;
+  fn env() -> GenTree;
-  fn source() -> DeclFs;
+  fn vfs() -> DeclFs;
-  const LEXERS: &'static [LexerObj];
+  fn lexers() -> Vec<LexerObj>;
 }
 
 pub trait DynSystem: Send + Sync + 'static {
-  fn env(&self) -> TreeModule;
+  fn dyn_env(&self) -> GenTree;
-  fn source(&self) -> DeclFs;
+  fn dyn_vfs(&self) -> DeclFs;
-  fn lexers(&self) -> &'static [LexerObj];
+  fn dyn_lexers(&self) -> Vec<LexerObj>;
-  fn card(&self) -> &dyn DynSystemCard;
+  fn dyn_card(&self) -> &dyn DynSystemCard;
 }
 
 impl<T: System> DynSystem for T {
-  fn env(&self) -> TreeModule { <Self as System>::env() }
+  fn dyn_env(&self) -> GenTree { Self::env() }
-  fn source(&self) -> DeclFs { <Self as System>::source() }
+  fn dyn_vfs(&self) -> DeclFs { Self::vfs() }
-  fn lexers(&self) -> &'static [LexerObj] { Self::LEXERS }
+  fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
-  fn card(&self) -> &dyn DynSystemCard { self }
+  fn dyn_card(&self) -> &dyn DynSystemCard { self }
+}
+
+pub fn downcast_atom<A: AtomCard>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom> {
+  match (foreign.expr.get_ctx().cted.deps())
+    .find(|s| s.id() == foreign.atom.owner)
+    .and_then(|sys| decode_atom::<A>(sys.get_card(), &foreign.atom))
+  {
+    None => Err(foreign),
+    Some(value) => Ok(TypAtom { value, data: foreign }),
+  }
+}
+
+#[derive(Clone)]
+pub struct SysCtx {
+  pub reqnot: ReqNot<ExtMsgSet>,
+  pub id: SysId,
+  pub cted: CtedObj,
 }

orchid-extension/src/system_ctor.rs

@@ -1,94 +1,136 @@
-use std::hash::{Hash as _, Hasher as _};
+use std::any::Any;
+use std::num::NonZeroU16;
+use std::sync::Arc;
 
-use itertools::Itertools as _;
-use orchid_api::proto::ExtMsgSet;
 use orchid_api::system::{NewSystem, SysId, SystemDecl};
-use orchid_base::reqnot::ReqNot;
+use orchid_base::boxed_iter::{box_empty, box_once, BoxedIter};
 use ordered_float::NotNan;
-use typeid::ConstTypeId;
 
-use crate::other_system::SystemHandle;
+use crate::other_system::{DynSystemHandle, SystemHandle};
 use crate::system::{DynSystem, System, SystemCard};
 
-pub struct SystemParams<Ctor: SystemCtor + ?Sized> {
+pub struct Cted<Ctor: SystemCtor + ?Sized> {
-  pub deps: <Ctor::Deps as DepSet>::Sat,
+  pub deps: <Ctor::Deps as DepDef>::Sat,
-  pub id: SysId,
+  pub inst: Arc<Ctor::Instance>,
-  pub reqnot: ReqNot<ExtMsgSet>,
+}
+impl<C: SystemCtor + ?Sized> Clone for Cted<C> {
+  fn clone(&self) -> Self { Self { deps: self.deps.clone(), inst: self.inst.clone() } }
+}
+pub trait DynCted: Send + Sync + 'static {
+  fn as_any(&self) -> &dyn Any;
+  fn deps<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)>;
+  fn inst(&self) -> Arc<dyn DynSystem>;
+}
+impl<C: SystemCtor + ?Sized> DynCted for Cted<C> {
+  fn as_any(&self) -> &dyn Any { self }
+  fn deps<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)> { self.deps.iter() }
+  fn inst(&self) -> Arc<dyn DynSystem> { self.inst.clone() }
+}
+pub type CtedObj = Arc<dyn DynCted>;
+
+pub trait DepSat: Clone + Send + Sync + 'static {
+  fn iter<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)>;
 }
 
-pub trait DepSet {
+pub trait DepDef {
-  type Sat;
+  type Sat: DepSat;
   fn report(names: &mut impl FnMut(&'static str));
-  fn create(take: &mut impl FnMut() -> SysId, reqnot: ReqNot<ExtMsgSet>) -> Self::Sat;
+  fn create(take: &mut impl FnMut() -> SysId) -> Self::Sat;
 }
 
-impl<T: SystemCard> DepSet for T {
+impl<T: SystemCard> DepSat for SystemHandle<T> {
+  fn iter<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)> { box_once(self) }
+}
+
+impl<T: SystemCard> DepDef for T {
   type Sat = SystemHandle<Self>;
-  fn report(names: &mut impl FnMut(&'static str)) { names(T::NAME) }
+  fn report(names: &mut impl FnMut(&'static str)) { names(T::Ctor::NAME) }
-  fn create(take: &mut impl FnMut() -> SysId, reqnot: ReqNot<ExtMsgSet>) -> Self::Sat {
+  fn create(take: &mut impl FnMut() -> SysId) -> Self::Sat { SystemHandle::new(take()) }
-    SystemHandle::new(take(), reqnot)
-  }
 }
 
-pub trait SystemCtor: Send + 'static {
+impl DepSat for () {
-  type Deps: DepSet;
+  fn iter<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)> { box_empty() }
+}
+
+impl DepDef for () {
+  type Sat = ();
+  fn create(_: &mut impl FnMut() -> SysId) -> Self::Sat {}
+  fn report(_: &mut impl FnMut(&'static str)) {}
+}
+
+pub trait SystemCtor: Send + Sync + 'static {
+  type Deps: DepDef;
   type Instance: System;
   const NAME: &'static str;
   const VERSION: f64;
-  #[allow(clippy::new_ret_no_self)]
+  fn inst() -> Option<Self::Instance>;
-  fn new(params: SystemParams<Self>) -> Self::Instance;
 }
 
-pub trait DynSystemCtor: Send + 'static {
+pub trait DynSystemCtor: Send + Sync + 'static {
-  fn decl(&self) -> SystemDecl;
+  fn decl(&self, id: NonZeroU16) -> SystemDecl;
-  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn DynSystem>;
+  fn new_system(&self, new: &NewSystem) -> CtedObj;
 }
 
 impl<T: SystemCtor> DynSystemCtor for T {
-  fn decl(&self) -> SystemDecl {
+  fn decl(&self, id: NonZeroU16) -> SystemDecl {
     // Version is equivalent to priority for all practical purposes
     let priority = NotNan::new(T::VERSION).unwrap();
     // aggregate depends names
     let mut depends = Vec::new();
     T::Deps::report(&mut |n| depends.push(n.to_string()));
-    // generate definitely unique id
-    let mut ahash = ahash::AHasher::default();
-    ConstTypeId::of::<T>().hash(&mut ahash);
-    let id = (ahash.finish().to_be_bytes().into_iter().tuples())
-      .map(|(l, b)| u16::from_be_bytes([l, b]))
-      .fold(0, |a, b| a ^ b);
     SystemDecl { name: T::NAME.to_string(), depends, id, priority }
   }
-  fn new_system(&self, new: &NewSystem, reqnot: ReqNot<ExtMsgSet>) -> Box<dyn DynSystem> {
+  fn new_system(&self, NewSystem { system: _, id: _, depends }: &NewSystem) -> CtedObj {
-    let mut ids = new.depends.iter().copied();
+    let mut ids = depends.iter().copied();
-    Box::new(T::new(SystemParams {
+    let inst = Arc::new(T::inst().expect("Constructor did not create system"));
-      deps: T::Deps::create(&mut || ids.next().unwrap(), reqnot.clone()),
+    let deps = T::Deps::create(&mut || ids.next().unwrap());
-      id: new.id,
+    Arc::new(Cted::<T> { deps, inst })
-      reqnot,
-    }))
   }
 }
 
 mod dep_set_tuple_impls {
-  use orchid_api::proto::ExtMsgSet;
   use orchid_api::system::SysId;
-  use orchid_base::reqnot::ReqNot;
+  use orchid_base::box_chain;
+  use orchid_base::boxed_iter::BoxedIter;
+  use paste::paste;
 
-  use super::DepSet;
+  use super::{DepDef, DepSat};
+  use crate::system_ctor::DynSystemHandle;
 
   macro_rules! dep_set_tuple_impl {
     ($($name:ident),*) => {
-      impl<$( $name :DepSet ),*> DepSet for ( $( $name , )* ) {
+      impl<$( $name :DepSat ),*> DepSat for ( $( $name , )* ) {
+        fn iter<'a>(&'a self) -> BoxedIter<'a, &'a (dyn DynSystemHandle + 'a)> {
+          // we're using the Paste crate to convert the names to lowercase,
+          // so `dep_set_tuple_impl!(A, B, C)` generates `let (a, b, c,) = self;`
+          // This step isn't really required for correctness, but Rust warns about uppercase
+          // variable names.
+          paste!{
+            let (
+              $(
+                [< $name :lower >] ,
+              )*
+            ) = self;
+            box_chain! (
+              $(
+                [< $name :lower >] .iter()
+              ),*
+            )
+          }
+        }
+      }
+
+      impl<$( $name :DepDef ),*> DepDef for ( $( $name , )* ) {
         type Sat = ( $( $name ::Sat , )* );
         fn report(names: &mut impl FnMut(&'static str)) {
           $(
             $name ::report(names);
           )*
         }
-        fn create(take: &mut impl FnMut() -> SysId, reqnot: ReqNot<ExtMsgSet>) -> Self::Sat {
+        fn create(take: &mut impl FnMut() -> SysId) -> Self::Sat {
           (
             $(
-              $name ::create(take, reqnot.clone()),
+              $name ::create(take),
             )*
           )
         }

orchid-extension/src/trait_obj_coder.rs

@@ -1,44 +0,0 @@
-pub struct TraitObject<T: ?Sized + Coding + 'static>(Box<T>, Arc<dyn Fn(&mut dyn Read) -> Self>);
-impl<T: ?Sized + Coding + 'static> TraitObject<T> {
-  fn inner_type_id(&self) -> ConstTypeId { self.0.as_ref().type_id() }
-  fn get_decoder(&self) -> Arc<dyn Fn(&mut dyn Read) -> Self> { self.1.clone() }
-}
-pub trait AsTraitObject<T: ?Sized + Coding + 'static>: 'static {
-  fn trait_box(self) -> Box<T>;
-  fn into_trait_object(self) -> TraitObject<T>
-  where Self: Sized + Coding {
-    let decoder = Self::get_decoder(Self::into_trait_object);
-    TraitObject(self.trait_box(), Arc::new(decoder))
-  }
-}
-
-pub struct TraitObjectCoder<T: ?Sized + Coding + 'static> {
-  entries: HashMap<u64, Box<dyn Fn(&mut dyn Read) -> TraitObject<T>>>,
-}
-impl<T: ?Sized + Coding + 'static> TraitObjectCoder<T> {
-  pub fn add_type<U: AsTraitObject<T> + Coding>(&mut self, tid_hash: u64) {
-    self.entries.entry(tid_hash).or_insert_with(|| Box::new(|b| U::decode(b).into_trait_object()));
-  }
-  pub fn add_obj(&mut self, tid_hash: u64, obj: &TraitObject<T>) {
-    self.entries.entry(tid_hash).or_insert_with(|| {
-      let decoder = obj.get_decoder();
-      Box::new(move |b| decoder(b))
-    });
-  }
-  pub fn encode<U: AsTraitObject<T> + Coding>(&mut self, data: U, out: &mut impl Write) {
-    let tid = hash_tid(ConstTypeId::of::<U>());
-    tid.encode(out);
-    self.add_type::<U>(tid);
-    data.encode(out);
-  }
-  pub fn encode_obj(&mut self, data: &TraitObject<T>, out: &mut impl Write) {
-    let tid = hash_tid(data.inner_type_id());
-    self.add_obj(tid, data);
-    tid.encode(out);
-    data.0.as_ref().encode(out);
-  }
-  pub fn decode(&mut self, src: &mut impl Read) -> TraitObject<T> {
-    let tid = u64::decode(src);
-    (self.entries.get(&tid).expect("Received object of unknown ConstTypeId"))(src)
-  }
-}

orchid-extension/src/tree.rs

@@ -0,0 +1,178 @@
+use std::ops::Range;
+
+use ahash::HashMap;
+use dyn_clone::{clone_box, DynClone};
+use itertools::Itertools;
+use orchid_api::tree::{
+  MacroRule, Paren, Placeholder, PlaceholderKind, Token, TokenTree, Tree, TreeId, TreeModule,
+  TreeTicket,
+};
+use orchid_base::interner::{intern, Tok};
+use orchid_base::location::Pos;
+use orchid_base::name::VName;
+use ordered_float::NotNan;
+use trait_set::trait_set;
+
+use crate::atom::AtomFactory;
+use crate::expr::GenExpr;
+use crate::system::DynSystem;
+
+#[derive(Clone)]
+pub struct GenPh {
+  pub name: Tok<String>,
+  pub kind: PlaceholderKind,
+}
+
+#[derive(Clone)]
+pub struct GenTokTree {
+  pub tok: GenTok,
+  pub range: Range<u32>,
+}
+impl GenTokTree {
+  pub fn into_api(self, sys: &dyn DynSystem) -> TokenTree {
+    TokenTree { token: self.tok.into_api(sys), range: self.range }
+  }
+}
+
+pub fn ph(s: &str) -> GenPh {
+  match s.strip_prefix("..") {
+    Some(v_tail) => {
+      let (mid, priority) = match v_tail.split_once(':') {
+        Some((h, t)) => (h, t.parse().expect("priority not an u8")),
+        None => (v_tail, 0),
+      };
+      let (name, nonzero) = match mid.strip_prefix(".$") {
+        Some(name) => (name, true),
+        None => (mid.strip_prefix('$').expect("Invalid placeholder"), false),
+      };
+      if konst::string::starts_with(name, "_") {
+        panic!("Names starting with an underscore indicate a single-name scalar placeholder")
+      }
+      GenPh { name: intern(name), kind: PlaceholderKind::Vector { nonzero, priority } }
+    },
+    None => match konst::string::strip_prefix(s, "$_") {
+      Some(name) => GenPh { name: intern(name), kind: PlaceholderKind::Name },
+      None => match konst::string::strip_prefix(s, "$") {
+        None => panic!("Invalid placeholder"),
+        Some(name) => GenPh { name: intern(name), kind: PlaceholderKind::Scalar },
+      },
+    },
+  }
+}
+
+#[derive(Clone)]
+pub enum GenTok {
+  Lambda(Vec<GenTokTree>, Vec<GenTokTree>),
+  Name(VName),
+  S(Paren, Vec<GenTokTree>),
+  Atom(AtomFactory),
+  Slot(TreeTicket),
+  Ph(GenPh),
+}
+impl GenTok {
+  pub fn at(self, range: Range<u32>) -> GenTokTree { GenTokTree { tok: self, range } }
+  pub fn into_api(self, sys: &dyn DynSystem) -> Token {
+    match self {
+      Self::Lambda(x, body) => Token::Lambda(
+        x.into_iter().map(|tt| tt.into_api(sys)).collect_vec(),
+        body.into_iter().map(|tt| tt.into_api(sys)).collect_vec(),
+      ),
+      Self::Name(n) => Token::Name(n.into_iter().map(|t| t.marker()).collect_vec()),
+      Self::Ph(GenPh { name, kind }) => Token::Ph(Placeholder { name: name.marker(), kind }),
+      Self::S(p, body) => Token::S(p, body.into_iter().map(|tt| tt.into_api(sys)).collect_vec()),
+      Self::Slot(tk) => Token::Slot(tk),
+      Self::Atom(at) => Token::Atom(at.build(sys)),
+    }
+  }
+}
+
+#[derive(Clone)]
+pub struct GenMacro {
+  pub pattern: Vec<GenTokTree>,
+  pub priority: NotNan<f64>,
+  pub template: Vec<GenTokTree>,
+}
+
+pub fn tokv_into_api(
+  tokv: impl IntoIterator<Item = GenTokTree>,
+  sys: &dyn DynSystem,
+) -> Vec<TokenTree> {
+  tokv.into_iter().map(|tok| tok.into_api(sys)).collect_vec()
+}
+
+pub fn wrap_tokv(items: Vec<GenTokTree>, range: Range<u32>) -> GenTokTree {
+  match items.len() {
+    1 => items.into_iter().next().unwrap(),
+    _ => GenTok::S(Paren::Round, items).at(range),
+  }
+}
+
+#[derive(Clone)]
+pub struct GenTree {
+  pub item: GenItem,
+  pub location: Pos,
+}
+impl GenTree {
+  pub fn cnst(gc: GenExpr) -> Self { GenItem::Const(gc).at(Pos::Inherit) }
+  pub fn module<'a>(entries: impl IntoIterator<Item = (&'a str, GenTree)>) -> Self {
+    GenItem::Mod(entries.into_iter().map(|(k, v)| (k.to_string(), v)).collect()).at(Pos::Inherit)
+  }
+  pub fn rule(
+    prio: f64,
+    pat: impl IntoIterator<Item = GenTokTree>,
+    tpl: impl IntoIterator<Item = GenTokTree>,
+  ) -> Self {
+    GenItem::Rule(GenMacro {
+      pattern: pat.into_iter().collect(),
+      priority: NotNan::new(prio).expect("expected to be static"),
+      template: tpl.into_iter().collect(),
+    })
+    .at(Pos::Inherit)
+  }
+  pub fn into_api(
+    self,
+    sys: &dyn DynSystem,
+    with_lazy: &mut impl FnMut(LazyTreeFactory) -> TreeId,
+  ) -> Tree {
+    match self.item {
+      GenItem::Const(gc) => Tree::Const(gc.into_api(sys)),
+      GenItem::Rule(GenMacro { pattern, priority, template }) => Tree::Rule(MacroRule {
+        pattern: tokv_into_api(pattern, sys),
+        priority,
+        template: tokv_into_api(template, sys),
+      }),
+      GenItem::Mod(entv) => Tree::Mod(TreeModule {
+        children: entv
+          .into_iter()
+          .map(|(name, tree)| (name.to_string(), tree.into_api(sys, with_lazy)))
+          .collect(),
+      }),
+      GenItem::Lazy(cb) => Tree::Lazy(with_lazy(cb)),
+    }
+  }
+}
+
+trait_set! {
+  trait LazyTreeCallback = FnMut() -> GenTree + Send + Sync + DynClone
+}
+pub struct LazyTreeFactory(Box<dyn LazyTreeCallback>);
+impl LazyTreeFactory {
+  pub fn new(cb: impl FnMut() -> GenTree + Send + Sync + Clone + 'static) -> Self {
+    Self(Box::new(cb))
+  }
+  pub fn build(&mut self) -> GenTree { (self.0)() }
+}
+impl Clone for LazyTreeFactory {
+  fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
+}
+
+#[derive(Clone)]
+pub enum GenItem {
+  Const(GenExpr),
+  Mod(HashMap<String, GenTree>),
+  Rule(GenMacro),
+  Lazy(LazyTreeFactory),
+}
+impl GenItem {
+  pub fn at(self, position: Pos) -> GenTree { GenTree { item: self, location: position } }
+}

orchid-extension/src/try_from_expr.rs

@@ -0,0 +1,16 @@
+use crate::error::ProjectResult;
+use crate::expr::{ExprHandle, OwnedExpr};
+
+pub trait TryFromExpr: Sized {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self>;
+}
+
+impl TryFromExpr for OwnedExpr {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> { Ok(OwnedExpr::new(expr)) }
+}
+
+impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    Ok((T::try_from_expr(expr.clone())?, U::try_from_expr(expr)?))
+  }
+}

orchid-host/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 [dependencies]
 derive_destructure = "1.0.0"
 hashbrown = "0.14.5"
-itertools = "0.12.1"
+itertools = "0.13.0"
 lazy_static = "1.4.0"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }

orchid-host/src/expr.rs

@@ -1,3 +1,4 @@
+use std::num::NonZeroU64;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 
@@ -15,7 +16,10 @@ pub struct RtExpr {
 impl RtExpr {
   pub fn as_atom(&self) -> Option<AtomHand> { todo!() }
   pub fn strong_count(&self) -> usize { todo!() }
-  pub fn id(&self) -> u64 { self.data.as_ref() as *const () as usize as u64 }
+  pub fn id(&self) -> ExprTicket {
+    NonZeroU64::new(self.data.as_ref() as *const () as usize as u64)
+      .expect("this is a ref, it cannot be null")
+  }
   pub fn canonicalize(&self) -> ExprTicket {
     if !self.is_canonical.swap(true, Ordering::Relaxed) {
       KNOWN_EXPRS.write().unwrap().entry(self.id()).or_insert_with(|| self.clone());

orchid-host/src/extension.rs

@@ -1,5 +1,6 @@
 use std::io::Write as _;
 use std::sync::atomic::{AtomicU16, AtomicU32, Ordering};
+use std::sync::mpsc::{channel, Receiver, Sender};
 use std::sync::{Arc, Mutex, RwLock, Weak};
 use std::{fmt, io, process, thread};
 
@@ -8,18 +9,19 @@ use hashbrown::HashMap;
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use orchid_api::atom::{Atom, AtomDrop, AtomSame, CallRef, FinalCall, Fwd, Fwded};
+use orchid_api::error::{ErrNotif, ProjErr, ProjErrOrRef, ProjResult, ReportError};
 use orchid_api::expr::{Acquire, Expr, ExprNotif, ExprTicket, Release, Relocate};
-use orchid_api::intern::IntReq;
+use orchid_api::interner::IntReq;
 use orchid_api::parser::CharFilter;
 use orchid_api::proto::{
-  ExtHostNotif, ExtHostReq, ExtensionHeader, HostExtNotif, HostHeader, HostMsgSet,
+  ExtHostNotif, ExtHostReq, ExtensionHeader, HostExtNotif, HostExtReq, HostHeader, HostMsgSet
 };
 use orchid_api::system::{NewSystem, SysDeclId, SysId, SystemDecl, SystemDrop};
-use orchid_api::tree::{GetConstTree, TreeModule};
+use orchid_api::tree::{GetConstTree, Tree, TreeId};
-use orchid_api_traits::{Decode, Encode};
+use orchid_api_traits::{Coding, Decode, Encode, Request};
 use orchid_base::char_filter::char_filter_match;
 use orchid_base::clone;
-use orchid_base::intern::{deintern, intern};
+use orchid_base::interner::{deintern, intern};
 use orchid_base::reqnot::{ReqNot, Requester as _};
 use ordered_float::NotNan;
 
@@ -128,6 +130,9 @@ impl Extension {
             acq_expr(inc, expr);
             rel_expr(dec, expr);
           },
+          ExtHostNotif::ErrNotif(ErrNotif::ReportError(ReportError(sys, err))) => {
+            System::resolve(sys).unwrap().0.err_send.send(err).unwrap();
+          },
         },
         |req| match req.req() {
           ExtHostReq::Ping(ping) => req.handle(ping, &()),
@@ -166,14 +171,18 @@ impl SystemCtor {
     let depends = depends.into_iter().map(|si| si.0.id).collect_vec();
     debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
     let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
-    static NEXT_ID: AtomicU16 = AtomicU16::new(0);
+    static NEXT_ID: AtomicU16 = AtomicU16::new(1);
-    let id = NEXT_ID.fetch_add(1, Ordering::Relaxed);
+    let id = SysId::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped");
     let sys_inst = ext.reqnot.request(NewSystem { depends, id, system: self.decl.id });
+    let (err_send, err_rec) = channel();
     let data = System(Arc::new(SystemInstData {
       decl_id: self.decl.id,
       ext: Extension(ext),
       exprs: RwLock::default(),
       lex_filter: sys_inst.lex_filter,
+      const_root_id: sys_inst.const_root_id,
+      err_send,
+      err_rec: Mutex::new(err_rec),
       id,
     }));
     inst_g.insert(id, data.clone());
@@ -182,7 +191,7 @@ impl SystemCtor {
 }
 
 lazy_static! {
-  static ref SYSTEM_INSTS: RwLock<HashMap<u16, System>> = RwLock::default();
+  static ref SYSTEM_INSTS: RwLock<HashMap<SysId, System>> = RwLock::default();
 }
 
 #[derive(destructure)]
@@ -191,7 +200,10 @@ pub struct SystemInstData {
   ext: Extension,
   decl_id: SysDeclId,
   lex_filter: CharFilter,
-  id: u16,
+  id: SysId,
+  const_root_id: TreeId,
+  err_rec: Mutex<Receiver<ProjErrOrRef>>,
+  err_send: Sender<ProjErrOrRef>,
 }
 impl Drop for SystemInstData {
   fn drop(&mut self) {
@@ -204,7 +216,7 @@ impl Drop for SystemInstData {
 #[derive(Clone)]
 pub struct System(Arc<SystemInstData>);
 impl System {
-  fn resolve(id: u16) -> Option<System> { SYSTEM_INSTS.read().unwrap().get(&id).cloned() }
+  fn resolve(id: SysId) -> Option<System> { SYSTEM_INSTS.read().unwrap().get(&id).cloned() }
   fn give_expr(&self, ticket: ExprTicket, get_expr: impl FnOnce() -> RtExpr) -> ExprTicket {
     let mut exprs = self.0.exprs.write().unwrap();
     exprs
@@ -215,7 +227,25 @@ impl System {
       .or_insert((AtomicU32::new(1), get_expr()));
     ticket
   }
-  pub fn const_tree(&self) -> TreeModule { self.0.ext.0.reqnot.request(GetConstTree(self.0.id)) }
+  pub fn const_tree(&self) -> Tree {
+    self.0.ext.0.reqnot.request(GetConstTree(self.0.id, self.0.const_root_id))
+  }
+  pub fn request<R: Coding>(&self, req: impl Request<Response = ProjResult<R>> + Into<HostExtReq>) -> ProjResult<R> {
+    let mut errors = Vec::new();
+    if let Ok(err) = self.0.err_rec.lock().unwrap().try_recv() {
+      eprintln!("Errors left in queue");
+      errors.push(err);
+    }
+    let value = self.0.ext.0.reqnot.request(req).inspect_err(|e| errors.extend(e.iter().cloned()));
+    while let Ok(err) = self.0.err_rec.lock().unwrap().try_recv() {
+      errors.push(err);
+    }
+    if !errors.is_empty() {
+      Err(errors)
+    } else {
+      value
+    }
+  }
   pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
   pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
 }

orchid-std/Cargo.toml

@@ -3,6 +3,10 @@ name = "orchid-std"
 version = "0.1.0"
 edition = "2021"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
+itertools = "0.13.0"
+orchid-api = { version = "0.1.0", path = "../orchid-api" }
+orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
+orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
+orchid-base = { version = "0.1.0", path = "../orchid-base" }
+orchid-extension = { version = "0.1.0", path = "../orchid-extension" }

orchid-std/src/lib.rs

@@ -1,12 +1,6 @@
-pub fn add(left: usize, right: usize) -> usize { left + right }
+mod std;
+mod string;
 
-#[cfg(test)]
+pub use std::StdSystem;
-mod tests {
-  use super::*;
 
-  #[test]
+pub use string::str_atom::OrcString;
-  fn it_works() {
-    let result = add(2, 2);
-    assert_eq!(result, 4);
-  }
-}

orchid-std/src/main.rs

@@ -0,0 +1,4 @@
+use orchid_extension::entrypoint::{extension_main, ExtensionData};
+use orchid_std::StdSystem;
+
+pub fn main() { extension_main(ExtensionData { systems: &[&StdSystem] }) }

orchid-std/src/std.rs

@@ -0,0 +1,30 @@
+use orchid_extension::atom::owned_atom_info;
+use orchid_extension::fs::DeclFs;
+use orchid_extension::system::{System, SystemCard};
+use orchid_extension::system_ctor::SystemCtor;
+use orchid_extension::tree::GenTree;
+
+use crate::string::str_atom::StringAtom;
+use crate::string::str_leer::StringLexer;
+
+#[derive(Default)]
+pub struct StdSystem;
+impl SystemCtor for StdSystem {
+  type Deps = ();
+  type Instance = Self;
+  const NAME: &'static str = "orchid::std";
+  const VERSION: f64 = 0.00_01;
+  fn inst() -> Option<Self::Instance> { Some(StdSystem) }
+}
+impl SystemCard for StdSystem {
+  type Ctor = Self;
+  const ATOM_DEFS: &'static [Option<orchid_extension::atom::AtomInfo>] =
+    &[Some(owned_atom_info::<StringAtom>())];
+}
+impl System for StdSystem {
+  fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer] }
+  fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
+  fn env() -> GenTree {
+    GenTree::module([("std", GenTree::module([("string", GenTree::module([]))]))])
+  }
+}

orchid-std/src/string/mod.rs

@@ -0,0 +1,2 @@
+pub mod str_atom;
+pub mod str_leer;

orchid-std/src/string/str_atom.rs

@@ -0,0 +1,101 @@
+use std::borrow::Cow;
+use std::num::NonZeroU64;
+use std::sync::Arc;
+
+use orchid_api::interner::TStr;
+use orchid_api_derive::Coding;
+use orchid_api_traits::{Encode, Request};
+use orchid_base::id_store::IdStore;
+use orchid_base::interner::{deintern, Tok};
+use orchid_base::location::Pos;
+use orchid_extension::atom::{AtomCard, OwnedAtom, TypAtom};
+use orchid_extension::error::{ProjectError, ProjectResult};
+use orchid_extension::expr::{ExprHandle, OwnedExpr};
+use orchid_extension::system::{downcast_atom, SysCtx};
+use orchid_extension::try_from_expr::TryFromExpr;
+
+pub static STR_REPO: IdStore<Arc<String>> = IdStore::new();
+
+#[derive(Clone, Coding)]
+pub(crate) enum StringVal {
+  Val(NonZeroU64),
+  Int(TStr),
+}
+#[derive(Copy, Clone, Coding)]
+pub(crate) struct StringGetVal;
+impl Request for StringGetVal {
+  type Response = String;
+}
+
+pub(crate) enum StringAtom {
+  Val(NonZeroU64),
+  Int(Tok<String>),
+}
+impl AtomCard for StringAtom {
+  type Data = StringVal;
+  type Req = StringGetVal;
+}
+impl StringAtom {
+  pub(crate) fn new_int(tok: Tok<String>) -> Self { Self::Int(tok) }
+  pub(crate) fn new(str: Arc<String>) -> Self { Self::Val(STR_REPO.add(str).id()) }
+}
+impl Clone for StringAtom {
+  fn clone(&self) -> Self {
+    match &self {
+      Self::Int(t) => Self::Int(t.clone()),
+      Self::Val(v) => Self::Val(STR_REPO.add(STR_REPO.get(*v).unwrap().clone()).id()),
+    }
+  }
+}
+impl StringAtom {
+  fn try_local_value(&self) -> Option<Arc<String>> {
+    match self {
+      Self::Int(tok) => Some(tok.arc()),
+      Self::Val(id) => STR_REPO.get(*id).map(|r| r.clone()),
+    }
+  }
+  fn get_value(&self) -> Arc<String> { self.try_local_value().expect("no string found for ID") }
+}
+impl OwnedAtom for StringAtom {
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(match self {
+    Self::Int(tok) => StringVal::Int(tok.marker()),
+    Self::Val(id) => StringVal::Val(*id),
+  }) }
+  fn same(&self, _ctx: SysCtx, other: &Self) -> bool { self.get_value() == other.get_value() }
+  fn handle_req(
+    &self,
+    _ctx: SysCtx,
+    StringGetVal: Self::Req,
+    rep: &mut (impl std::io::Write + ?Sized),
+  ) {
+    self.get_value().encode(rep)
+  }
+}
+
+pub struct OrcString(TypAtom<StringAtom>);
+impl OrcString {
+  pub fn get_string(&self) -> Arc<String> {
+    match &self.0.value {
+      StringVal::Int(tok) => deintern(*tok).arc(),
+      StringVal::Val(id) => match STR_REPO.get(*id) {
+        Some(rec) => rec.clone(),
+        None => Arc::new(self.0.request(StringGetVal)),
+      },
+    }
+  }
+}
+pub struct NotString(Pos);
+impl ProjectError for NotString {
+  const DESCRIPTION: &'static str = "A string was expected";
+  fn one_position(&self) -> Pos {
+      self.0.clone()
+  }
+}
+impl TryFromExpr for OrcString {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    (OwnedExpr::new(expr).foreign_atom().map_err(|expr| expr.position.clone()))
+      .and_then(|fatom| downcast_atom(fatom).map_err(|f| f.position))
+      .map_err(|p| NotString(p).pack())
+      .map(OrcString)
+  }
+}

orchid-std/src/string/str_leer.rs

@@ -0,0 +1,171 @@
+use itertools::Itertools;
+use orchid_base::interner::intern;
+use orchid_base::location::Pos;
+use orchid_base::name::VName;
+use orchid_base::vname;
+use orchid_extension::atom::OwnedAtom;
+use orchid_extension::error::{ErrorSansOrigin, ProjectErrorObj, ProjectResult};
+use orchid_extension::lexer::{LexContext, Lexer};
+use orchid_extension::tree::{wrap_tokv, GenTok, GenTokTree};
+
+use super::str_atom::StringAtom;
+
+/// Reasons why [parse_string] might fail. See [StringError]
+#[derive(Clone)]
+enum StringErrorKind {
+  /// A unicode escape sequence wasn't followed by 4 hex digits
+  NotHex,
+  /// A unicode escape sequence contained an unassigned code point
+  BadCodePoint,
+  /// An unrecognized escape sequence was found
+  BadEscSeq,
+}
+
+/// Error produced by [parse_string]
+#[derive(Clone)]
+struct StringError {
+  /// Character where the error occured
+  pos: u32,
+  /// Reason for the error
+  kind: StringErrorKind,
+}
+
+#[derive(Clone)]
+struct NotHex;
+impl ErrorSansOrigin for NotHex {
+  const DESCRIPTION: &'static str = "Expected a hex digit";
+}
+
+#[derive(Clone)]
+struct BadCodePoint;
+impl ErrorSansOrigin for BadCodePoint {
+  const DESCRIPTION: &'static str = "The specified number is not a Unicode code point";
+}
+
+#[derive(Clone)]
+struct BadEscapeSequence;
+impl ErrorSansOrigin for BadEscapeSequence {
+  const DESCRIPTION: &'static str = "Unrecognized escape sequence";
+}
+
+impl StringError {
+  /// Convert into project error for reporting
+  pub fn into_proj(self, pos: u32) -> ProjectErrorObj {
+    let start = pos + self.pos;
+    let pos = Pos::Range(start..start + 1);
+    match self.kind {
+      StringErrorKind::NotHex => NotHex.bundle(&pos),
+      StringErrorKind::BadCodePoint => BadCodePoint.bundle(&pos),
+      StringErrorKind::BadEscSeq => BadEscapeSequence.bundle(&pos),
+    }
+  }
+}
+
+/// Process escape sequences in a string literal
+fn parse_string(str: &str) -> Result<String, StringError> {
+  let mut target = String::new();
+  let mut iter = str.char_indices().map(|(i, c)| (i as u32, c));
+  while let Some((_, c)) = iter.next() {
+    if c != '\\' {
+      target.push(c);
+      continue;
+    }
+    let (mut pos, code) = iter.next().expect("lexer would have continued");
+    let next = match code {
+      c @ ('\\' | '"' | '$') => c,
+      'b' => '\x08',
+      'f' => '\x0f',
+      'n' => '\n',
+      'r' => '\r',
+      't' => '\t',
+      '\n' => 'skipws: loop {
+        match iter.next() {
+          None => return Ok(target),
+          Some((_, c)) =>
+            if !c.is_whitespace() {
+              break 'skipws c;
+            },
+        }
+      },
+      'u' => {
+        let acc = ((0..4).rev())
+          .map(|radical| {
+            let (j, c) = (iter.next()).ok_or(StringError { pos, kind: StringErrorKind::NotHex })?;
+            pos = j;
+            let b = u32::from_str_radix(&String::from(c), 16)
+              .map_err(|_| StringError { pos, kind: StringErrorKind::NotHex })?;
+            Ok(16u32.pow(radical) + b)
+          })
+          .fold_ok(0, u32::wrapping_add)?;
+        char::from_u32(acc).ok_or(StringError { pos, kind: StringErrorKind::BadCodePoint })?
+      },
+      _ => return Err(StringError { pos, kind: StringErrorKind::BadEscSeq }),
+    };
+    target.push(next);
+  }
+  Ok(target)
+}
+
+#[derive(Clone)]
+pub struct NoStringEnd;
+impl ErrorSansOrigin for NoStringEnd {
+  const DESCRIPTION: &'static str = "String never terminated with \"";
+}
+
+#[derive(Default)]
+pub struct StringLexer;
+impl Lexer for StringLexer {
+  const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"'];
+  fn lex<'a>(
+    full_string: &'a str,
+    ctx: &'a LexContext<'a>,
+  ) -> Option<ProjectResult<(&'a str, GenTokTree)>> {
+    full_string.strip_prefix('"').map(|mut tail| {
+      let mut parts = vec![];
+      let mut cur = String::new();
+      let commit_str = |str: &mut String, tail: &str, parts: &mut Vec<GenTokTree>| {
+        let str_val = parse_string(str)
+          .inspect_err(|e| ctx.report(e.clone().into_proj(ctx.pos(tail) - str.len() as u32)))
+          .unwrap_or_default();
+        let tok = GenTok::Atom(StringAtom::new_int(intern(&str_val)).factory());
+        parts.push(tok.at(ctx.tok_ran(str.len() as u32, tail)));
+        *str = String::new();
+      };
+      loop {
+        if let Some(rest) = tail.strip_prefix('"') {
+          commit_str(&mut cur, tail, &mut parts);
+          return Ok((rest, wrap_tokv(parts, ctx.pos(full_string)..ctx.pos(rest))));
+        } else if let Some(rest) = tail.strip_prefix('$') {
+          commit_str(&mut cur, tail, &mut parts);
+          parts.push(GenTok::Name(VName::literal("++")).at(ctx.tok_ran(1, rest)));
+          parts.push(GenTok::Name(vname!(std::string::convert)).at(ctx.tok_ran(1, rest)));
+          match ctx.recurse(rest) {
+            Ok((new_tail, tree)) => {
+              tail = new_tail;
+              parts.push(tree);
+            },
+            Err(e) => {
+              ctx.report(e.clone());
+              return Ok(("", wrap_tokv(parts, ctx.pos(full_string)..ctx.pos(rest))));
+            },
+          }
+        } else if tail.starts_with('\\') {
+          // parse_string will deal with it, we just have to make sure we skip the next
+          // char
+          tail = &tail[2..];
+        } else {
+          let mut ch = tail.chars();
+          if let Some(c) = ch.next() {
+            cur.push(c);
+            tail = ch.as_str();
+          } else {
+            let range = ctx.pos(full_string)..ctx.pos("");
+            commit_str(&mut cur, tail, &mut parts);
+            ctx.report(NoStringEnd.bundle(&Pos::Range(range.clone())));
+            return Ok(("", wrap_tokv(parts, range)));
+          }
+        }
+      }
+    })
+  }
+}

stdio-perftest/src/main.rs

@@ -18,7 +18,7 @@ fn main() {
       }
     }
   } else {
-    let steps = 10_000;
+    let steps = 1_000_000;
     let mut child = process::Command::new(args().next().unwrap())
       .arg("child")
       .stdin(process::Stdio::piped())