Changes in api and upwards

- Removed out-of-stack error reporting
- Revised module system to match previous Orchid system
- Errors are now in a Vec everywhere
- Implemented atoms and lexer
- Started implementation of line parser
- Tree is now ephemeral to avoid copying Atoms held inside
- Moved numbers into std and the shared parser into base
- Started implementation of Commands

Cargo.lock

@@ -476,6 +476,7 @@ dependencies = [
  "hashbrown 0.14.5",
  "itertools",
  "lazy_static",
+ "never",
  "orchid-api",
  "orchid-api-traits",
  "orchid-base",
@@ -487,12 +488,14 @@ name = "orchid-std"
 version = "0.1.0"
 dependencies = [
  "itertools",
+ "never",
  "once_cell",
  "orchid-api",
  "orchid-api-derive",
  "orchid-api-traits",
  "orchid-base",
  "orchid-extension",
+ "ordered-float",
 ]
 
 [[package]]
@@ -501,9 +504,9 @@ version = "0.1.0"
 
 [[package]]
 name = "ordered-float"
-version = "4.2.0"
+version = "4.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e"
+checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be"
 dependencies = [
  "num-traits",
 ]

orchid-api/src/atom.rs

@@ -15,6 +15,9 @@ pub struct LocalAtom {
   pub drop: bool,
   pub data: AtomData,
 }
+impl LocalAtom {
+  pub fn associate(self, owner: SysId) -> Atom { Atom { owner, drop: self.drop, data: self.data } }
+}
 
 /// An atom representation that can be serialized and sent around. Atoms
 /// represent the smallest increment of work.
@@ -101,6 +104,7 @@ impl Request for Command {
 #[extends(HostExtNotif)]
 pub struct AtomDrop(pub Atom);
 
+/// Requests that apply to an existing atom instance
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 #[extendable]
@@ -111,3 +115,16 @@ pub enum AtomReq {
   Fwded(Fwded),
   Command(Command),
 }
+impl AtomReq {
+  /// Obtain the first [Atom] argument of the request. All requests in this
+  /// subclass have at least one atom argument.
+  pub fn get_atom(&self) -> &Atom {
+    match self {
+      Self::AtomSame(AtomSame(a, ..))
+      | Self::CallRef(CallRef(a, ..))
+      | Self::Command(Command(a))
+      | Self::FinalCall(FinalCall(a, ..))
+      | Self::Fwded(Fwded(a, ..)) => a,
+    }
+  }
+}

orchid-api/src/error.rs

@@ -6,10 +6,10 @@ use orchid_api_traits::Request;
 
 use crate::interner::TStr;
 use crate::location::Location;
-use crate::proto::{ExtHostNotif, ExtHostReq};
+use crate::proto::ExtHostReq;
-use crate::system::SysId;
 
-pub type ProjErrId = NonZeroU16;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct ProjErrId(pub NonZeroU16);
 
 #[derive(Clone, Debug, Coding)]
 pub struct ProjErrLocation {
@@ -37,18 +37,8 @@ pub struct ProjErr {
   pub locations: Vec<ProjErrLocation>,
 }
 
-/// When the interpreter encounters an error while serving a system's request,
-/// it sends this error as an ID back to the system to save bandwidth.
-/// The lifetime of this ID is the request being served, the receiving system
-/// can return it and query its details with [GetDetails].
-#[derive(Clone, Debug, Coding)]
-pub enum ProjErrOrRef {
-  New(ProjErr),
-  Known(ProjErrId),
-}
-
 /// If this is an [`Err`] then the [`Vec`] must not be empty.
-pub type ProjResult<T> = Result<T, Vec<ProjErrOrRef>>;
+pub type ProjResult<T> = Result<T, Vec<ProjErr>>;
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ProjErrReq, ExtHostReq)]
@@ -57,26 +47,9 @@ impl Request for GetErrorDetails {
   type Response = ProjErr;
 }
 
-/// Notify the host about an error without being forced to return said error.
-/// This will still count as an error, but later operations that depend on the
-/// value returned by the currently running function will get to run
-///
-/// The error is not connected to the place it was raised, since multiple calls
-/// can be issued to a system at the same time
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ErrNotif, ExtHostNotif)]
-pub struct ReportError(pub SysId, pub ProjErrOrRef);
-
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 #[extendable]
 pub enum ProjErrReq {
   GetErrorDetails(GetErrorDetails),
 }
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostNotif)]
-#[extendable]
-pub enum ErrNotif {
-  ReportError(ReportError),
-}

orchid-api/src/expr.rs

@@ -15,7 +15,8 @@ use crate::system::SysId;
 /// [Acquire].
 ///
 /// The ID is globally unique within its lifetime, but may be reused.
-pub type ExprTicket = NonZeroU64;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct ExprTicket(pub NonZeroU64);
 
 /// Acquire a strong reference to an expression. This keeps it alive until a
 /// corresponding [Release] is emitted. The number of times a system has
@@ -84,7 +85,7 @@ pub enum Clause {
   /// A reference to a constant
   Const(TStrv),
   /// A static runtime error.
-  Bottom(ProjErr),
+  Bottom(Vec<ProjErr>),
 }
 
 #[derive(Clone, Debug, Coding)]

orchid-api/src/interner.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::proto::{ExtHostReq, HostExtReq};
+use crate::proto::{ExtHostNotif, ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.
@@ -80,6 +80,16 @@ impl Request for Sweep {
   type Response = Retained;
 }
 
+/// A notification from the extension to the host, that the extension would benefit from a sweep
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ExtHostNotif)]
+pub struct AdviseSweep(SweepReason);
+
+#[derive(Clone, Debug, Coding)]
+pub enum SweepReason {
+  None
+}
+
 /// List of keys in this replica that couldn't be sweeped because local
 /// datastructures reference their value.
 #[derive(Clone, Debug, Coding)]

orchid-api/src/parser.rs

@@ -10,7 +10,8 @@ use crate::proto::{ExtHostReq, HostExtReq};
 use crate::system::SysId;
 use crate::tree::{TokenTree, TreeTicket};
 
-pub type LexId = NonZeroU64;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct ParsId(pub NonZeroU64);
 
 /// - All ranges contain at least one character
 /// - All ranges are in increasing characeter order
@@ -22,35 +23,35 @@ pub struct CharFilter(pub Vec<RangeInclusive<char>>);
 #[extends(HostExtReq)]
 #[extendable]
 pub enum ParserReq {
-  Lex(Lex),
+  LexExpr(LexExpr),
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ParserReq, HostExtReq)]
-pub struct Lex {
+pub struct LexExpr {
   pub sys: SysId,
-  pub id: LexId,
+  pub id: ParsId,
   pub text: TStr,
   pub pos: u32,
 }
-impl Request for Lex {
+impl Request for LexExpr {
-  type Response = Option<ProjResult<Lexed>>;
+  type Response = Option<ProjResult<LexedExpr>>;
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct Lexed {
+pub struct LexedExpr {
   pub pos: u32,
-  pub data: TokenTree,
+  pub expr: TokenTree,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(ExtHostReq)]
 pub struct SubLex {
-  pub id: LexId,
+  pub id: ParsId,
   pub pos: u32,
 }
 impl Request for SubLex {
-  type Response = ProjResult<SubLexed>;
+  type Response = Option<SubLexed>;
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -59,4 +60,11 @@ pub struct SubLexed {
   pub ticket: TreeTicket,
 }
 
-pub struct ParseLine {}
+#[derive(Clone, Debug, Coding)]
+pub struct ParseLine {
+  pub sys: SysId,
+  pub line: Vec<TokenTree>,
+}
+impl Request for ParseLine {
+  type Response = Vec<TokenTree>;
+}

orchid-api/src/proto.rs

@@ -82,7 +82,7 @@ pub enum ExtHostReq {
 #[extendable]
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
-  ErrNotif(error::ErrNotif),
+  AdviseSweep(interner::AdviseSweep),
 }
 
 pub struct ExtHostChannel;
@@ -100,7 +100,7 @@ pub enum HostExtReq {
   Sweep(interner::Sweep),
   AtomReq(atom::AtomReq),
   ParserReq(parser::ParserReq),
-  GetConstTree(tree::GetConstTree),
+  GetMember(tree::GetMember),
   VfsReq(vfs::VfsReq),
 }
 

orchid-api/src/system.rs

@@ -1,20 +1,25 @@
+use std::collections::HashMap;
 use std::num::NonZeroU16;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
+use crate::interner::TStr;
 use crate::parser::CharFilter;
 use crate::proto::{HostExtNotif, HostExtReq};
-use crate::tree::TreeId;
+use crate::tree::MemberKind;
 
 /// ID of a system type
-pub type SysDeclId = NonZeroU16;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct SysDeclId(pub NonZeroU16);
 
 /// ID of a system instance
-pub type SysId = NonZeroU16;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct SysId(pub NonZeroU16);
 
-/// Details about a system provided by this library
+/// Details about a system provided by this library. This is included in the extension header,
+/// so it cannot rely on the interner. 
 #[derive(Debug, Clone, Coding)]
 pub struct SystemDecl {
   /// ID of the system, unique within the library
@@ -59,7 +64,8 @@ pub struct SystemInst {
   /// can process. The lexer will notify this system if it encounters one of
   /// these characters.9
   pub lex_filter: CharFilter,
-  pub const_root_id: TreeId,
+  pub parses_lines: Vec<TStr>,
+  pub const_root: HashMap<TStr, MemberKind>,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/tree.rs

@@ -1,4 +1,5 @@
-use std::collections::HashMap;
+use crate::interner::TStrv;
+use crate::location::Location;
 use std::num::NonZeroU64;
 use std::ops::Range;
 
@@ -7,7 +8,7 @@ use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
 use crate::atom::LocalAtom;
-use crate::error::ProjErrOrRef;
+use crate::error::ProjErr;
 use crate::expr::Expr;
 use crate::interner::TStr;
 use crate::proto::HostExtReq;
@@ -17,8 +18,11 @@ use crate::system::SysId;
 /// the lexer can include it in its output or discard it by implication.
 ///
 /// Similar to [crate::expr::ExprTicket] in that it represents a token tree the
-/// lifetime of which is managed by the interpreter.
+/// lifetime of which is managed by the interpreter, and as such should probably
-pub type TreeTicket = NonZeroU64;
+/// not be exposed to libraries directly but rather wrapped in a
+/// lifetime-controlled abstraction.
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct TreeTicket(pub NonZeroU64);
 
 #[derive(Clone, Debug, Coding)]
 pub struct TokenTree {
@@ -28,18 +32,26 @@ pub struct TokenTree {
 
 #[derive(Clone, Debug, Coding)]
 pub enum Token {
-  /// Lambda function. The number operates as an argument name
+  /// Lambda function head, from the opening \ until the beginning of the body.
-  Lambda(Vec<TokenTree>, Vec<TokenTree>),
+  Lambda(Vec<TokenTree>),
-  Name(Vec<TStr>),
+  /// A name segment or an operator.
+  Name(TStr),
+  /// ::
+  NS,
+  /// Line break.
+  BR,
+  /// ( Round parens ), [ Square brackets ] or { Curly braces }
   S(Paren, Vec<TokenTree>),
   /// A placeholder in a macro. This variant is forbidden everywhere outside
   /// line parser output
   Ph(Placeholder),
+  /// A new atom
   Atom(LocalAtom),
+  /// Anchor to insert a subtree
   Slot(TreeTicket),
-  /// A static compile-time error returned by erroring lexers if
+  /// A static compile-time error returned by failing lexers if
   /// the rest of the source is likely still meaningful
-  Bottom(ProjErrOrRef),
+  Bottom(Vec<ProjErr>),
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -63,30 +75,52 @@ pub enum Paren {
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct MacroRule {
+pub struct Macro {
   pub pattern: Vec<TokenTree>,
   pub priority: NotNan<f64>,
   pub template: Vec<TokenTree>,
 }
 
-pub type TreeId = NonZeroU64;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct TreeId(pub NonZeroU64);
+
 
 #[derive(Clone, Debug, Coding)]
-pub enum Tree {
+pub struct Item {
+  pub location: Location,
+  pub kind: ItemKind,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum ItemKind {
+  Member(Member),
+  Raw(Vec<TokenTree>),
+  Rule(Macro),
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Member {
+  pub name: TStr,
+  pub public: bool,
+  pub kind: MemberKind,
+}
+
+#[derive(Clone, Debug, Coding)]
+pub enum MemberKind {
   Const(Expr),
-  Mod(TreeModule),
+  Module(Module),
-  Rule(MacroRule),
   Lazy(TreeId),
 }
 
 #[derive(Clone, Debug, Coding)]
-pub struct TreeModule {
+pub struct Module {
-  pub children: HashMap<String, Tree>,
+  pub imports: Vec<TStrv>,
+  pub items: Vec<Item>,
 }
 
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
-pub struct GetConstTree(pub SysId, pub TreeId);
+pub struct GetMember(pub SysId, pub TreeId);
-impl Request for GetConstTree {
+impl Request for GetMember {
-  type Response = Tree;
+  type Response = MemberKind;
 }

orchid-api/src/vfs.rs

@@ -9,7 +9,8 @@ use crate::interner::TStr;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
 
-pub type VfsId = NonZeroU16;
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct VfsId(pub NonZeroU16);
 
 #[derive(Clone, Debug, Coding)]
 pub enum Loaded {

orchid-base/src/as_api.rs

@@ -1,10 +0,0 @@
-use orchid_api_traits::Coding;
-
-pub trait AsApi: Sized {
-  type Api: Sized;
-  type Ctx<'a>;
-  fn to_api(&self, ctx: Self::Ctx<'_>) -> Self::Api;
-  fn into_api(self, ctx: Self::Ctx<'_>) -> Self::Api { self.to_api(ctx) }
-  fn from_api_ref(api: &Self::Api, ctx: Self::Ctx<'_>) -> Self;
-  fn from_api(api: Self::Api, ctx: Self::Ctx<'_>) -> Self { Self::from_api_ref(&api, ctx) }
-}

orchid-base/src/char_filter.rs

@@ -5,6 +5,16 @@ use orchid_api::parser::CharFilter;
 
 pub type CRange = RangeInclusive<char>;
 
+pub trait ICFilter {
+  fn ranges(&self) -> &[RangeInclusive<char>];
+}
+impl ICFilter for [RangeInclusive<char>] {
+  fn ranges(&self) -> &[RangeInclusive<char>] { self }
+}
+impl ICFilter for CharFilter {
+  fn ranges(&self) -> &[RangeInclusive<char>] { &self.0 }
+}
+
 fn try_merge_char_ranges(left: CRange, right: CRange) -> Result<CRange, (CRange, CRange)> {
   match *left.end() as u32 + 1 < *right.start() as u32 {
     true => Err((left, right)),
@@ -25,19 +35,19 @@ pub fn mk_char_filter(items: impl IntoIterator<Item = CRange>) -> CharFilter {
 }
 
 /// Decide whether a char filter matches a character via binary search
-pub fn char_filter_match(cf: &CharFilter, c: char) -> bool {
+pub fn char_filter_match(cf: &(impl ICFilter + ?Sized), c: char) -> bool {
-  match cf.0.binary_search_by_key(&c, |l| *l.end()) {
+  match cf.ranges().binary_search_by_key(&c, |l| *l.end()) {
     Ok(_) => true,                      // c is the end of a range
-    Err(i) if i == cf.0.len() => false, // all ranges end before c
+    Err(i) if i == cf.ranges().len() => false, // all ranges end before c
-    Err(i) => cf.0[i].contains(&c),     // c between cf.0[i-1]?.end and cf.0[i].end, check [i]
+    Err(i) => cf.ranges()[i].contains(&c),     // c between cf.0[i-1]?.end and cf.0[i].end, check [i]
   }
 }
 
 /// Merge two char filters into a filter that matches if either of the
 /// constituents would match.
-pub fn char_filter_union(l: &CharFilter, r: &CharFilter) -> CharFilter {
+pub fn char_filter_union(l: &(impl ICFilter + ?Sized), r: &(impl ICFilter + ?Sized)) -> CharFilter {
   CharFilter(
-    (l.0.iter().merge_by(&r.0, |l, r| l.start() <= r.start()))
+    (l.ranges().iter().merge_by(r.ranges(), |l, r| l.start() <= r.start()))
       .cloned()
       .coalesce(try_merge_char_ranges)
       .collect_vec(),

orchid-base/src/error.rs

@@ -7,7 +7,7 @@ use crate::location::Pos;
 
 /// A point of interest in resolving the error, such as the point where
 /// processing got stuck, a command that is likely to be incorrect
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct ErrorPosition {
   /// The suspected origin
   pub position: Pos,
@@ -35,7 +35,7 @@ impl From<Pos> for ErrorPosition {
   fn from(origin: Pos) -> Self { Self { position: origin, message: None } }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct OwnedError {
   pub description: Tok<String>,
   pub message: Arc<String>,
@@ -49,4 +49,7 @@ impl OwnedError {
       positions: err.locations.iter().map(ErrorPosition::from_api).collect(),
     }
   }
+  pub fn from_apiv(err: Vec<ProjErr>) -> Vec<Self> {
+    err.into_iter().map(|e| Self::from_api(&e)).collect()
+  }
 }

orchid-base/src/lib.rs

@@ -5,7 +5,6 @@ pub mod event;
 pub mod msg;
 // pub mod gen;
 pub mod api_utils;
-pub mod as_api;
 pub mod box_cow;
 pub mod char_filter;
 pub mod error;

orchid-base/src/name.rs

@@ -9,9 +9,10 @@ use std::path::Path;
 use std::{fmt, slice, vec};
 
 use itertools::Itertools;
+use orchid_api::interner::TStr;
 use trait_set::trait_set;
 
-use crate::interner::{intern, InternMarker, Tok};
+use crate::interner::{deintern, intern, InternMarker, Tok};
 
 trait_set! {
   /// Traits that all name iterators should implement
@@ -226,6 +227,9 @@ impl VName {
     let data: Vec<_> = items.into_iter().collect();
     if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
   }
+  pub fn deintern(items: impl IntoIterator<Item = TStr>) -> Result<Self, EmptyNameError> {
+    Self::new(items.into_iter().map(deintern))
+  }
   /// Unwrap the enclosed vector
   pub fn into_vec(self) -> Vec<Tok<String>> { self.0 }
   /// Get a reference to the enclosed vector

orchid-base/src/tokens.rs

@@ -2,7 +2,7 @@ use orchid_api::tree::{Placeholder, PlaceholderKind};
 
 use crate::interner::{deintern, Tok};
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct OwnedPh {
   pub name: Tok<String>,
   pub kind: PlaceholderKind,

orchid-extension/src/atom.rs

@@ -1,8 +1,10 @@
 use std::any::{type_name, Any, TypeId};
 use std::io::{Read, Write};
 use std::ops::Deref;
+use std::sync::OnceLock;
 
 use dyn_clone::{clone_box, DynClone};
+use never::Never;
 use orchid_api::atom::{Atom, Fwd, LocalAtom};
 use orchid_api::expr::ExprTicket;
 use orchid_api_traits::{Coding, Decode, Request};
@@ -10,9 +12,9 @@ use orchid_base::location::Pos;
 use orchid_base::reqnot::Requester;
 use trait_set::trait_set;
 
-use crate::error::ProjectError;
+use crate::error::{ProjectError, ProjectResult};
-use crate::expr::{ExprHandle, GenExpr};
+use crate::expr::{ExprHandle, GenClause, GenExpr, OwnedExpr};
-use crate::system::{atom_info_for, DynSystem, DynSystemCard, SysCtx};
+use crate::system::{atom_info_for, downcast_atom, DynSystem, DynSystemCard, SysCtx};
 
 pub trait AtomCard: 'static + Sized {
   type Data: Clone + Coding + Sized;
@@ -40,7 +42,7 @@ pub trait AtomicFeaturesImpl<Variant: AtomicVariant> {
   type _Info: AtomDynfo;
   const _INFO: &'static Self::_Info;
 }
-impl<A: Atomic + AtomicFeaturesImpl<A::Variant>> AtomicFeatures for A {
+impl<A: Atomic + AtomicFeaturesImpl<A::Variant> + ?Sized> AtomicFeatures for A {
   fn factory(self) -> AtomFactory { self._factory() }
   type Info = <Self as AtomicFeaturesImpl<A::Variant>>::_Info;
   const INFO: &'static Self::Info = Self::_INFO;
@@ -58,41 +60,65 @@ pub struct ForeignAtom {
   pub atom: Atom,
   pub pos: Pos,
 }
-impl ForeignAtom {}
+impl ForeignAtom {
+  pub fn oex(self) -> OwnedExpr {
+    let gen_expr = GenExpr { pos: self.pos, clause: GenClause::Atom(self.expr.tk, self.atom) };
+    OwnedExpr { handle: self.expr, val: OnceLock::from(Box::new(gen_expr)) }
+  }
+}
+
+pub struct NotTypAtom(pub Pos, pub OwnedExpr, pub &'static dyn AtomDynfo);
+impl ProjectError for NotTypAtom {
+  const DESCRIPTION: &'static str = "Not the expected type";
+  fn message(&self) -> String { format!("This expression is not a {}", self.2.name()) }
+}
 
 #[derive(Clone)]
-pub struct TypAtom<A: AtomCard> {
+pub struct TypAtom<A: AtomicFeatures> {
   pub data: ForeignAtom,
   pub value: A::Data,
 }
-impl<A: AtomCard> TypAtom<A> {
+impl<A: AtomicFeatures> TypAtom<A> {
+  pub fn downcast(expr: ExprHandle) -> Result<Self, NotTypAtom> {
+    match OwnedExpr::new(expr).foreign_atom() {
+      Err(oe) => Err(NotTypAtom(oe.get_data().pos.clone(), oe, A::INFO)),
+      Ok(atm) => match downcast_atom::<A>(atm) {
+        Err(fa) => Err(NotTypAtom(fa.pos.clone(), fa.oex(), A::INFO)),
+        Ok(tatom) => Ok(tatom),
+      },
+    }
+  }
   pub fn request<R: Coding + Into<A::Req> + Request>(&self, req: R) -> R::Response {
     R::Response::decode(
       &mut &self.data.expr.ctx.reqnot.request(Fwd(self.data.atom.clone(), req.enc_vec()))[..],
     )
   }
 }
-impl<A: AtomCard> Deref for TypAtom<A> {
+impl<A: AtomicFeatures> Deref for TypAtom<A> {
   type Target = A::Data;
   fn deref(&self) -> &Self::Target { &self.value }
 }
 
+pub struct AtomCtx<'a>(pub &'a [u8], pub SysCtx);
+
 pub trait AtomDynfo: Send + Sync + 'static {
   fn tid(&self) -> TypeId;
-  fn decode(&self, data: &[u8]) -> Box<dyn Any>;
+  fn name(&self) -> &'static str;
-  fn call(&self, buf: &[u8], ctx: SysCtx, arg: ExprTicket) -> GenExpr;
+  fn decode(&self, ctx: AtomCtx<'_>) -> Box<dyn Any>;
-  fn call_ref(&self, buf: &[u8], ctx: SysCtx, arg: ExprTicket) -> GenExpr;
+  fn call(&self, ctx: AtomCtx<'_>, arg: ExprTicket) -> GenExpr;
-  fn same(&self, buf: &[u8], ctx: SysCtx, buf2: &[u8]) -> bool;
+  fn call_ref(&self, ctx: AtomCtx<'_>, arg: ExprTicket) -> GenExpr;
-  fn handle_req(&self, buf: &[u8], ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write);
+  fn same(&self, ctx: AtomCtx<'_>, buf2: &[u8]) -> bool;
-  fn drop(&self, buf: &[u8], ctx: SysCtx);
+  fn handle_req(&self, ctx: AtomCtx<'_>, req: &mut dyn Read, rep: &mut dyn Write);
+  fn command(&self, ctx: AtomCtx<'_>) -> ProjectResult<Option<GenExpr>>;
+  fn drop(&self, ctx: AtomCtx<'_>);
 }
 
 trait_set! {
-  pub trait AtomFactoryFn = FnOnce(&dyn DynSystem) -> LocalAtom + DynClone;
+  pub trait AtomFactoryFn = FnOnce(&dyn DynSystem) -> LocalAtom + DynClone + Send + Sync;
 }
 pub struct AtomFactory(Box<dyn AtomFactoryFn>);
 impl AtomFactory {
-  pub fn new(f: impl FnOnce(&dyn DynSystem) -> LocalAtom + Clone + 'static) -> Self {
+  pub fn new(f: impl FnOnce(&dyn DynSystem) -> LocalAtom + Clone + Send + Sync + 'static) -> Self {
     Self(Box::new(f))
   }
   pub fn build(self, sys: &dyn DynSystem) -> LocalAtom { (self.0)(sys) }
@@ -105,3 +131,27 @@ pub struct ErrorNotCallable;
 impl ProjectError for ErrorNotCallable {
   const DESCRIPTION: &'static str = "This atom is not callable";
 }
+
+pub struct ErrorNotCommand;
+impl ProjectError for ErrorNotCommand {
+  const DESCRIPTION: &'static str = "This atom is not a command";
+}
+
+pub trait ReqPck<T: AtomCard + ?Sized>: Sized {
+  type W: Write + ?Sized;
+  fn unpack<'a>(self) -> (T::Req, &'a mut Self::W)
+  where Self: 'a;
+  fn never(self)
+  where T: AtomCard<Req = Never> {
+  }
+}
+
+pub struct RequestPack<'a, T: AtomCard + ?Sized, W: Write + ?Sized>(pub T::Req, pub &'a mut W);
+
+impl<'a, T: AtomCard + ?Sized, W: Write + ?Sized> ReqPck<T> for RequestPack<'a, T, W> {
+  type W = W;
+  fn unpack<'b>(self) -> (<T as AtomCard>::Req, &'b mut Self::W)
+  where 'a: 'b {
+    (self.0, self.1)
+  }
+}

orchid-extension/src/atom_owned.rs

@@ -3,6 +3,7 @@ use std::borrow::Cow;
 use std::io::{Read, Write};
 use std::marker::PhantomData;
 use std::num::NonZeroU64;
+use std::sync::Arc;
 
 use orchid_api::atom::LocalAtom;
 use orchid_api::expr::ExprTicket;
@@ -10,8 +11,10 @@ use orchid_api_traits::{Decode, Encode};
 use orchid_base::id_store::{IdRecord, IdStore};
 
 use crate::atom::{
-  AtomCard, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, ErrorNotCallable,
+  AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant,
+  ErrorNotCallable, ErrorNotCommand, ReqPck, RequestPack,
 };
+use crate::error::ProjectResult;
 use crate::expr::{bot, ExprHandle, GenExpr};
 use crate::system::{atom_info_for, SysCtx};
 
@@ -38,22 +41,26 @@ fn with_atom<U>(mut b: &[u8], f: impl FnOnce(IdRecord<'_, Box<dyn DynOwnedAtom>>
 pub struct OwnedAtomDynfo<T: OwnedAtom>(PhantomData<T>);
 impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
   fn tid(&self) -> TypeId { TypeId::of::<T>() }
-  fn decode(&self, data: &[u8]) -> Box<dyn Any> {
+  fn name(&self) -> &'static str { type_name::<T>() }
+  fn decode(&self, AtomCtx(data, _): AtomCtx) -> Box<dyn Any> {
     Box::new(<T as AtomCard>::Data::decode(&mut &data[..]))
   }
-  fn call(&self, buf: &[u8], ctx: SysCtx, arg: ExprTicket) -> GenExpr {
+  fn call(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
     with_atom(buf, |a| a.remove().dyn_call(ctx, arg))
   }
-  fn call_ref(&self, buf: &[u8], ctx: SysCtx, arg: ExprTicket) -> GenExpr {
+  fn call_ref(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
     with_atom(buf, |a| a.dyn_call_ref(ctx, arg))
   }
-  fn same(&self, buf: &[u8], ctx: SysCtx, buf2: &[u8]) -> bool {
+  fn same(&self, AtomCtx(buf, ctx): AtomCtx, buf2: &[u8]) -> bool {
     with_atom(buf, |a1| with_atom(buf2, |a2| a1.dyn_same(ctx, &**a2)))
   }
-  fn handle_req(&self, buf: &[u8], ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write) {
+  fn handle_req(&self, AtomCtx(buf, ctx): AtomCtx, req: &mut dyn Read, rep: &mut dyn Write) {
     with_atom(buf, |a| a.dyn_handle_req(ctx, req, rep))
   }
-  fn drop(&self, buf: &[u8], ctx: SysCtx) { with_atom(buf, |a| a.remove().dyn_free(ctx)) }
+  fn command(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> ProjectResult<Option<GenExpr>> {
+    with_atom(buf, |a| a.remove().dyn_command(ctx))
+  }
+  fn drop(&self, AtomCtx(buf, ctx): AtomCtx) { with_atom(buf, |a| a.remove().dyn_free(ctx)) }
 }
 
 /// Atoms that have a [Drop]
@@ -75,7 +82,9 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone
     );
     false
   }
-  fn handle_req(&self, ctx: SysCtx, req: Self::Req, rep: &mut (impl Write + ?Sized));
+  fn handle_req(&self, ctx: SysCtx, pck: impl ReqPck<Self>);
+  #[allow(unused_variables)]
+  fn command(self, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> { Err(Arc::new(ErrorNotCommand)) }
   #[allow(unused_variables)]
   fn free(self, ctx: SysCtx) {}
 }
@@ -87,6 +96,7 @@ pub trait DynOwnedAtom: Send + Sync + 'static {
   fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: ExprTicket) -> GenExpr;
   fn dyn_same(&self, ctx: SysCtx, other: &dyn DynOwnedAtom) -> bool;
   fn dyn_handle_req(&self, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write);
+  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> ProjectResult<Option<GenExpr>>;
   fn dyn_free(self: Box<Self>, ctx: SysCtx);
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
@@ -107,7 +117,10 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
     self.same(ctx, other_self)
   }
   fn dyn_handle_req(&self, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write) {
-    self.handle_req(ctx, <Self as AtomCard>::Req::decode(req), rep)
+    self.handle_req(ctx, RequestPack::<T, dyn Write>(<Self as AtomCard>::Req::decode(req), rep))
+  }
+  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> {
+    self.command(ctx)
   }
   fn dyn_free(self: Box<Self>, ctx: SysCtx) { self.free(ctx) }
 }

orchid-extension/src/atom_thin.rs

@@ -2,15 +2,17 @@ use std::any::{type_name, Any, TypeId};
 use std::fmt;
 use std::io::Write;
 use std::marker::PhantomData;
+use std::sync::Arc;
 
 use orchid_api::atom::LocalAtom;
 use orchid_api::expr::ExprTicket;
 use orchid_api_traits::{Coding, Decode, Encode};
 
 use crate::atom::{
-  get_info, AtomCard, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant,
+  get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant,
-  ErrorNotCallable,
+  ErrorNotCallable, ReqPck, RequestPack,
 };
+use crate::error::ProjectResult;
 use crate::expr::{bot, ExprHandle, GenExpr};
 use crate::system::SysCtx;
 
@@ -31,20 +33,31 @@ impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant
 pub struct ThinAtomDynfo<T: ThinAtom>(PhantomData<T>);
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
   fn tid(&self) -> TypeId { TypeId::of::<T>() }
-  fn decode(&self, mut data: &[u8]) -> Box<dyn Any> { Box::new(T::decode(&mut data)) }
+  fn name(&self) -> &'static str { type_name::<T>() }
-  fn call(&self, buf: &[u8], ctx: SysCtx, arg: ExprTicket) -> GenExpr {
+  fn decode(&self, AtomCtx(data, _): AtomCtx) -> Box<dyn Any> {
+    Box::new(T::decode(&mut &data[..]))
+  }
+  fn call(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
     T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg))
   }
-  fn call_ref(&self, buf: &[u8], ctx: SysCtx, arg: ExprTicket) -> GenExpr {
+  fn call_ref(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
     T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg))
   }
-  fn handle_req(&self, buf: &[u8], ctx: SysCtx, req: &mut dyn std::io::Read, rep: &mut dyn Write) {
+  fn handle_req(
-    T::decode(&mut &buf[..]).handle_req(ctx, Decode::decode(req), rep)
+    &self,
+    AtomCtx(buf, ctx): AtomCtx,
+    req: &mut dyn std::io::Read,
+    rep: &mut dyn Write,
+  ) {
+    T::decode(&mut &buf[..]).handle_req(ctx, RequestPack::<T, dyn Write>(Decode::decode(req), rep))
   }
-  fn same(&self, buf: &[u8], ctx: SysCtx, buf2: &[u8]) -> bool {
+  fn same(&self, AtomCtx(buf, ctx): AtomCtx, buf2: &[u8]) -> bool {
     T::decode(&mut &buf[..]).same(ctx, &T::decode(&mut &buf2[..]))
   }
-  fn drop(&self, buf: &[u8], _ctx: SysCtx) {
+  fn command(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> ProjectResult<Option<GenExpr>> {
+    T::decode(&mut &buf[..]).command(ctx)
+  }
+  fn drop(&self, AtomCtx(buf, _): AtomCtx) {
     eprintln!("Received drop signal for non-drop atom {:?}", T::decode(&mut &buf[..]))
   }
 }
@@ -60,5 +73,9 @@ pub trait ThinAtom: AtomCard<Data = Self> + Coding + fmt::Debug + Send + Sync +
     );
     false
   }
-  fn handle_req(&self, ctx: SysCtx, req: Self::Req, rep: &mut (impl Write + ?Sized));
+  fn handle_req(&self, ctx: SysCtx, pck: impl ReqPck<Self>);
+  #[allow(unused_variables)]
+  fn command(&self, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> {
+    Err(Arc::new(ErrorNotCallable))
+  }
 }

orchid-extension/src/entrypoint.rs

@@ -1,48 +1,49 @@
-use std::num::{NonZeroU16, NonZeroU64};
+use std::num::NonZero;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::{mem, thread};
 
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::atom::{Atom, AtomDrop, AtomReq, AtomSame, CallRef, Command, FinalCall, Fwded};
+use orchid_api::atom::{
+  Atom, AtomDrop, AtomReq, AtomSame, CallRef, Command, FinalCall, Fwded, NextStep,
+};
 use orchid_api::interner::Sweep;
-use orchid_api::parser::{CharFilter, Lex, Lexed, ParserReq};
+use orchid_api::parser::{CharFilter, LexExpr, LexedExpr, ParserReq};
 use orchid_api::proto::{ExtMsgSet, ExtensionHeader, HostExtNotif, HostExtReq, HostHeader, Ping};
-use orchid_api::system::{SysId, SystemDrop, SystemInst};
+use orchid_api::system::{SysDeclId, SysId, SystemDrop, SystemInst};
-use orchid_api::tree::{GetConstTree, Tree, TreeId};
+use orchid_api::tree::{GetMember, TreeId};
 use orchid_api::vfs::{EagerVfs, GetVfs, VfsId, VfsRead, VfsReq};
 use orchid_api_traits::{Decode, Encode};
-use orchid_base::char_filter::{char_filter_union, mk_char_filter};
+use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
 use orchid_base::clone;
 use orchid_base::interner::{deintern, init_replica, sweep_replica};
 use orchid_base::name::PathSlice;
 use orchid_base::reqnot::{ReqNot, Requester};
 
-use crate::atom::AtomDynfo;
+use crate::atom::{AtomCtx, AtomDynfo};
-use crate::error::{err_or_ref_to_api, unpack_err};
+use crate::error::errv_to_apiv;
 use crate::fs::VirtFS;
-use crate::lexer::LexContext;
+use crate::lexer::{CascadingError, LexContext, NotApplicableLexerError};
 use crate::msg::{recv_parent_msg, send_parent_msg};
-use crate::system::{atom_by_idx, SysCtx};
+use crate::system::{atom_by_idx, resolv_atom, SysCtx};
 use crate::system_ctor::{CtedObj, DynSystemCtor};
-use crate::tree::LazyTreeFactory;
+use crate::tree::{LazyMemberFactory, TIACtxImpl};
 
 pub struct ExtensionData {
   pub systems: &'static [&'static dyn DynSystemCtor],
 }
 
-pub enum TreeRecord {
+pub enum MemberRecord {
-  Gen(LazyTreeFactory),
+  Gen(LazyMemberFactory),
-  Res(Tree),
+  Res,
 }
 
 pub struct SystemRecord {
   cted: CtedObj,
   vfses: HashMap<VfsId, &'static dyn VirtFS>,
   declfs: EagerVfs,
-  tree: Tree,
+  lazy_members: HashMap<TreeId, MemberRecord>,
-  subtrees: HashMap<TreeId, TreeRecord>,
 }
 
 pub fn with_atom_record<T>(
@@ -63,7 +64,7 @@ pub fn extension_main(data: ExtensionData) {
   let mut buf = Vec::new();
   let decls = (data.systems.iter().enumerate())
     .map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
-    .map(|(id, sys)| sys.decl(NonZeroU16::new(id + 1).unwrap()))
+    .map(|(id, sys)| sys.decl(SysDeclId(NonZero::new(id + 1).unwrap())))
     .collect_vec();
   let systems = Arc::new(Mutex::new(HashMap::<SysId, SystemRecord>::new()));
   ExtensionHeader { systems: decls.clone() }.encode(&mut buf);
@@ -77,7 +78,7 @@ pub fn extension_main(data: ExtensionData) {
         mem::drop(systems.lock().unwrap().remove(&sys_id)),
       HostExtNotif::AtomDrop(AtomDrop(atom)) => {
         with_atom_record(&systems, &atom, |rec, cted, data| {
-          rec.drop(data, SysCtx{ reqnot, id: atom.owner, cted })
+          rec.drop(AtomCtx(data, SysCtx{ reqnot, id: atom.owner, cted }))
         })
       }
     }),
@@ -91,44 +92,36 @@ pub fn extension_main(data: ExtensionData) {
         let lex_filter = cted.inst().dyn_lexers().iter().fold(CharFilter(vec![]), |cf, lx| {
           char_filter_union(&cf, &mk_char_filter(lx.char_filter().iter().cloned()))
         });
-        let mut subtrees = HashMap::new();
+        let mut lazy_mems = HashMap::new();
+        let const_root = (cted.inst().dyn_env().into_iter())
+          .map(|(k, v)| {
+            (k.marker(), v.into_api(&mut TIACtxImpl{ lazy: &mut lazy_mems, sys: &*cted.inst()}))
+          })
+          .collect();
         systems.lock().unwrap().insert(new_sys.id, SystemRecord {
           declfs: cted.inst().dyn_vfs().to_api_rec(&mut vfses),
           vfses,
-          tree: cted.inst().dyn_env().into_api(&*cted.inst(), &mut |gen| {
-            let id = TreeId::new((subtrees.len() + 2) as u64).unwrap();
-            subtrees.insert(id, TreeRecord::Gen(gen.clone()));
-            id
-          }),
           cted,
-          subtrees
+          lazy_members: lazy_mems
         });
         req.handle(new_sys, &SystemInst {
-          lex_filter, const_root_id: NonZeroU64::new(1).unwrap()
+          lex_filter,
+          const_root,
+          parses_lines: vec!()
         })
       }
-      HostExtReq::GetConstTree(get_tree@GetConstTree(sys_id, tree_id)) => {
+      HostExtReq::GetMember(get_tree@GetMember(sys_id, tree_id)) => {
         let mut systems_g = systems.lock().unwrap();
         let sys = systems_g.get_mut(sys_id).expect("System not found");
-        if tree_id.get() == 1 {
+        let lazy = &mut sys.lazy_members;
-          req.handle(get_tree, &sys.tree);
+        let cb = match lazy.insert(*tree_id, MemberRecord::Res) {
-        } else {
+          None => panic!("Tree for ID not found"),
-          let subtrees = &mut sys.subtrees;
+          Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
-          let tree_rec = subtrees.get_mut(tree_id).expect("Tree for ID not found");
+          Some(MemberRecord::Gen(cb)) => cb,
-          match tree_rec {
+        };
-            TreeRecord::Res(tree) => req.handle(get_tree, tree),
+        let tree = cb.build();
-            TreeRecord::Gen(cb) => {
+        let reply_tree = tree.into_api(&mut TIACtxImpl{ sys: &*sys.cted.inst(), lazy });
-              let tree = cb.build();
+        req.handle(get_tree, &reply_tree);
-              let reply_tree = tree.into_api(&*sys.cted.inst(), &mut |cb| {
-                let id = NonZeroU64::new((subtrees.len() + 2) as u64).unwrap();
-                subtrees.insert(id, TreeRecord::Gen(cb.clone()));
-                id
-              });
-              req.handle(get_tree, &reply_tree);
-              subtrees.insert(*tree_id, TreeRecord::Res(reply_tree));
-            }
-          }
-        }
       }
       HostExtReq::VfsReq(VfsReq::GetVfs(get_vfs@GetVfs(sys_id))) => {
         let systems_g = systems.lock().unwrap();
@@ -139,8 +132,8 @@ pub fn extension_main(data: ExtensionData) {
         let path = path.iter().map(|t| deintern(*t)).collect_vec();
         req.handle(vfs_read, &systems_g[sys_id].vfses[vfs_id].load(PathSlice::new(&path)))
       }
-      HostExtReq::ParserReq(ParserReq::Lex(lex)) => {
+      HostExtReq::ParserReq(ParserReq::LexExpr(lex)) => {
-        let Lex{ sys, text, pos, id } = *lex;
+        let LexExpr{ sys, text, pos, id } = *lex;
         let systems_g = systems.lock().unwrap();
         let lexers = systems_g[&sys].cted.inst().dyn_lexers();
         mem::drop(systems_g);
@@ -148,22 +141,56 @@ pub fn extension_main(data: ExtensionData) {
         let tk = req.will_handle_as(lex);
         thread::spawn(clone!(systems; move || {
           let ctx = LexContext { sys, id, pos, reqnot: req.reqnot(), text: &text };
-          let lex_res = lexers.iter().find_map(|lx| lx.lex(&text[pos as usize..], &ctx));
+          let first_char = text.chars().next().unwrap();
-          req.handle_as(tk, &lex_res.map(|r| match r {
+          for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), first_char)) {
-            Ok((s, data)) => {
+            match lx.lex(&text[pos as usize..], &ctx) {
-              let systems_g = systems.lock().unwrap();
+              Err(e) if e.as_any_ref().is::<NotApplicableLexerError>() => continue,
-              let data = data.into_api(&*systems_g[&sys].cted.inst());
+              Err(e) if e.as_any_ref().is::<CascadingError>() => return req.handle_as(tk, &None),
-              Ok(Lexed { data, pos: (text.len() - s.len()) as u32 })
+              Err(e) => return req.handle_as(tk, &Some(Err(errv_to_apiv([e])))),
-            },
+              Ok((s, expr)) => {
-            Err(e) => Err(unpack_err(e).into_iter().map(err_or_ref_to_api).collect_vec())
+                let systems_g = systems.lock().unwrap();
-          }))
+                let expr = expr.into_api(&*systems_g[&sys].cted.inst());
+                let pos = (text.len() - s.len()) as u32;
+                return req.handle_as(tk, &Some(Ok(LexedExpr{ pos, expr })))
+              }
+            }
+          }
+          req.handle_as(tk, &None)
         }));
       },
-      HostExtReq::AtomReq(AtomReq::AtomSame(same@AtomSame(l, r))) => todo!("subsys nimpl"),
+      HostExtReq::AtomReq(atom_req) => {
-      HostExtReq::AtomReq(AtomReq::Fwded(call@Fwded(atom, req))) => todo!("subsys nimpl"),
+        let systems_g = systems.lock().unwrap();
-      HostExtReq::AtomReq(AtomReq::CallRef(call@CallRef(atom, arg))) => todo!("subsys nimpl"),
+        let atom = atom_req.get_atom();
-      HostExtReq::AtomReq(AtomReq::FinalCall(call@FinalCall(atom, arg))) => todo!("subsys nimpl"),
+        let sys = &systems_g[&atom.owner];
-      HostExtReq::AtomReq(AtomReq::Command(cmd@Command(atom))) => todo!("subsys impl"),
+        let ctx = SysCtx { cted: sys.cted.clone(), id: atom.owner, reqnot: req.reqnot() };
+        let dynfo = resolv_atom(&*sys.cted.inst(), atom);
+        let actx = AtomCtx(&atom.data[8..], ctx);
+        match atom_req {
+          AtomReq::AtomSame(same@AtomSame(_, r)) => {
+            // different systems or different type tags
+            if atom.owner != r.owner || atom.data[..8] != r.data[..8] {
+              return req.handle(same, &false)
+            }
+            req.handle(same, &dynfo.same(actx, &r.data[8..]))
+          },
+          AtomReq::Fwded(fwded@Fwded(_, payload)) => {
+            let mut reply = Vec::new();
+            dynfo.handle_req(actx, &mut &payload[..], &mut reply);
+            req.handle(fwded, &reply)
+          }
+          AtomReq::CallRef(call@CallRef(_, arg))
+            => req.handle(call, &dynfo.call_ref(actx, *arg).to_api(&*sys.cted.inst())),
+          AtomReq::FinalCall(call@FinalCall(_, arg))
+            => req.handle(call, &dynfo.call(actx, *arg).to_api(&*sys.cted.inst())),
+          AtomReq::Command(cmd@Command(_)) => req.handle(cmd, &match dynfo.command(actx) {
+            Err(e) => Err(errv_to_apiv([e])),
+            Ok(opt) => Ok(match opt {
+              Some(cont) => NextStep::Continue(cont.into_api(&*sys.cted.inst())),
+              None => NextStep::Halt,
+            })
+          })
+        }
+      }
     }),
   );
   init_replica(rn.clone().map());

orchid-extension/src/error.rs

@@ -6,7 +6,7 @@ use std::{fmt, iter};
 
 use dyn_clone::{clone_box, DynClone};
 use itertools::Itertools;
-use orchid_api::error::{GetErrorDetails, ProjErr, ProjErrId, ProjErrOrRef};
+use orchid_api::error::{GetErrorDetails, ProjErr, ProjErrId};
 use orchid_api::proto::ExtMsgSet;
 use orchid_base::boxed_iter::{box_once, BoxedIter};
 use orchid_base::clone;
@@ -274,7 +274,7 @@ impl ProjectError for MultiError {
   }
 }
 
-pub fn err_to_api(err: ProjectErrorObj) -> ProjErr {
+fn err_to_api(err: ProjectErrorObj) -> ProjErr {
   ProjErr {
     description: intern(&*err.description()).marker(),
     message: Arc::new(err.message()),
@@ -282,26 +282,18 @@ pub fn err_to_api(err: ProjectErrorObj) -> ProjErr {
   }
 }
 
-pub(crate) fn err_or_ref_to_api(err: ProjectErrorObj) -> ProjErrOrRef {
+pub fn errv_to_apiv(errv: impl IntoIterator<Item = ProjectErrorObj>) -> Vec<ProjErr> {
-  match err.as_any_ref().downcast_ref() {
+  errv.into_iter().flat_map(unpack_err).map(err_to_api).collect_vec()
-    Some(RelayedError { id: Some(id), .. }) => ProjErrOrRef::Known(*id),
-    _ => ProjErrOrRef::New(err_to_api(err)),
-  }
 }
 
-pub fn err_from_api(err: &ProjErr, reqnot: ReqNot<ExtMsgSet>) -> ProjectErrorObj {
+pub fn err_from_apiv<'a>(
-  Arc::new(RelayedError { id: None, reqnot, details: OwnedError::from_api(err).into() })
+  err: impl IntoIterator<Item = &'a ProjErr>,
-}
+  reqnot: &ReqNot<ExtMsgSet>
-
-pub(crate) fn err_from_api_or_ref(
-  err: &ProjErrOrRef,
-  reqnot: ReqNot<ExtMsgSet>,
 ) -> ProjectErrorObj {
-  match err {
+  pack_err(err.into_iter().map(|e| {
-    ProjErrOrRef::Known(id) =>
+    let details: OnceLock<_> = OwnedError::from_api(e).into();
-      Arc::new(RelayedError { id: Some(*id), reqnot, details: OnceLock::default() }),
+    Arc::new(RelayedError { id: None, reqnot: reqnot.clone(), details }).into_packed()
-    ProjErrOrRef::New(err) => err_from_api(err, reqnot),
+  }))
-  }
 }
 
 struct RelayedError {

orchid-extension/src/expr.rs

@@ -9,7 +9,7 @@ use orchid_base::location::Pos;
 use orchid_base::reqnot::Requester;
 
 use crate::atom::{AtomFactory, AtomicFeatures, ForeignAtom};
-use crate::error::{err_from_api, err_to_api, DynProjectError, ProjectErrorObj};
+use crate::error::{err_from_apiv, errv_to_apiv, DynProjectError, ProjectErrorObj};
 use crate::system::{DynSystem, SysCtx};
 
 #[derive(destructure)]
@@ -46,7 +46,7 @@ impl OwnedExpr {
     self.val.get_or_init(|| {
       Box::new(GenExpr::from_api(
         self.handle.ctx.reqnot.request(Inspect(self.handle.tk)).expr,
-        self.handle.get_ctx(),
+        &self.handle.ctx,
       ))
     })
   }
@@ -75,7 +75,7 @@ impl GenExpr {
   pub fn into_api(self, sys: &dyn DynSystem) -> Expr {
     Expr { location: self.pos.to_api(), clause: self.clause.into_api(sys) }
   }
-  pub fn from_api(api: Expr, ctx: SysCtx) -> Self {
+  pub fn from_api(api: Expr, ctx: &SysCtx) -> Self {
     Self { pos: Pos::from_api(&api.location), clause: GenClause::from_api(api.clause, ctx) }
   }
 }
@@ -100,7 +100,7 @@ impl GenClause {
       Self::Lambda(arg, body) => Clause::Lambda(*arg, Box::new(body.to_api(sys))),
       Self::Arg(arg) => Clause::Arg(*arg),
       Self::Const(name) => Clause::Const(name.marker()),
-      Self::Bottom(msg) => Clause::Bottom(err_to_api(msg.clone())),
+      Self::Bottom(err) => Clause::Bottom(errv_to_apiv([err.clone()])),
       Self::NewAtom(fac) => Clause::NewAtom(fac.clone().build(sys)),
       Self::Atom(tk, atom) => Clause::Atom(*tk, atom.clone()),
       Self::Slot(_) => panic!("Slot is forbidden in const tree"),
@@ -114,34 +114,34 @@ impl GenClause {
       Self::Arg(arg) => Clause::Arg(arg),
       Self::Slot(extk) => Clause::Slot(extk.handle.into_tk()),
       Self::Const(name) => Clause::Const(name.marker()),
-      Self::Bottom(msg) => Clause::Bottom(err_to_api(msg)),
+      Self::Bottom(err) => Clause::Bottom(errv_to_apiv([err])),
       Self::NewAtom(fac) => Clause::NewAtom(fac.clone().build(sys)),
       Self::Atom(tk, atom) => Clause::Atom(tk, atom),
     }
   }
-  pub fn from_api(api: Clause, ctx: SysCtx) -> Self {
+  pub fn from_api(api: Clause, ctx: &SysCtx) -> Self {
     match api {
       Clause::Arg(id) => Self::Arg(id),
       Clause::Lambda(arg, body) => Self::Lambda(arg, Box::new(GenExpr::from_api(*body, ctx))),
       Clause::NewAtom(_) => panic!("Clause::NewAtom should never be received, only sent"),
-      Clause::Bottom(s) => Self::Bottom(err_from_api(&s, ctx.reqnot)),
+      Clause::Bottom(s) => Self::Bottom(err_from_apiv(&s, &ctx.reqnot)),
       Clause::Call(f, x) => Self::Call(
-        Box::new(GenExpr::from_api(*f, ctx.clone())),
+        Box::new(GenExpr::from_api(*f, ctx)),
         Box::new(GenExpr::from_api(*x, ctx)),
       ),
       Clause::Seq(a, b) => Self::Seq(
-        Box::new(GenExpr::from_api(*a, ctx.clone())),
+        Box::new(GenExpr::from_api(*a, ctx)),
         Box::new(GenExpr::from_api(*b, ctx)),
       ),
       Clause::Const(name) => Self::Const(deintern(name)),
-      Clause::Slot(exi) => Self::Slot(OwnedExpr::new(ExprHandle::from_args(ctx, exi))),
+      Clause::Slot(exi) => Self::Slot(OwnedExpr::new(ExprHandle::from_args(ctx.clone(), exi))),
       Clause::Atom(tk, atom) => Self::Atom(tk, atom),
     }
   }
 }
 fn inherit(clause: GenClause) -> GenExpr { GenExpr { pos: Pos::Inherit, clause } }
 
-pub fn cnst(path: Tok<Vec<Tok<String>>>) -> GenExpr { inherit(GenClause::Const(path)) }
+pub fn sym_ref(path: Tok<Vec<Tok<String>>>) -> GenExpr { inherit(GenClause::Const(path)) }
 pub fn atom<A: AtomicFeatures>(atom: A) -> GenExpr { inherit(GenClause::NewAtom(atom.factory())) }
 
 pub fn seq(ops: impl IntoIterator<Item = GenExpr>) -> GenExpr {

orchid-extension/src/fs.rs

@@ -1,4 +1,4 @@
-use std::num::NonZeroU16;
+use std::num::NonZero;
 
 use hashbrown::HashMap;
 use orchid_api::error::ProjResult;
@@ -19,7 +19,7 @@ impl DeclFs {
     match self {
       DeclFs::Lazy(fs) => {
         let vfsc: u16 = vfses.len().try_into().expect("too many vfses (more than u16::MAX)");
-        let id: VfsId = NonZeroU16::new(vfsc + 1).unwrap();
+        let id = VfsId(NonZero::new(vfsc + 1).unwrap());
         vfses.insert(id, *fs);
         EagerVfs::Lazy(id)
       },

orchid-extension/src/fun.rs

@@ -4,7 +4,7 @@ use dyn_clone::{clone_box, DynClone};
 use never::Never;
 use trait_set::trait_set;
 
-use crate::atom::Atomic;
+use crate::atom::{Atomic, ReqPck};
 use crate::atom_owned::{OwnedAtom, OwnedVariant};
 use crate::conv::{ToExpr, TryFromExpr};
 use crate::expr::{ExprHandle, GenExpr};
@@ -34,7 +34,5 @@ impl OwnedAtom for Fun {
   fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
   fn call_ref(&self, arg: ExprHandle) -> GenExpr { self.clone().call(arg) }
   fn call(self, arg: ExprHandle) -> GenExpr { (self.0)(arg) }
-  fn handle_req(&self, _ctx: SysCtx, req: Self::Req, _rep: &mut (impl std::io::Write + ?Sized)) {
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
-    match req {}
-  }
 }

orchid-extension/src/lexer.rs

@@ -1,32 +1,46 @@
 use std::ops::{Range, RangeInclusive};
 
-use orchid_api::error::ReportError;
+use orchid_api::parser::{ParsId, SubLex};
-use orchid_api::parser::{LexId, SubLex};
 use orchid_api::proto::ExtMsgSet;
 use orchid_api::system::SysId;
 use orchid_base::interner::Tok;
+use orchid_base::location::Pos;
 use orchid_base::reqnot::{ReqNot, Requester};
 
 use crate::error::{
-  err_from_api_or_ref, err_or_ref_to_api, pack_err, unpack_err, ProjectErrorObj, ProjectResult,
+  ProjectError, ProjectResult
 };
-use crate::tree::{OwnedTok, OwnedTokTree};
+use crate::tree::{GenTok, GenTokTree};
+
+pub struct CascadingError;
+impl ProjectError for CascadingError {
+  const DESCRIPTION: &'static str = "An error cascading from a recursive sublexer";
+  fn message(&self) -> String {
+    "This error should not surface. If you are seeing it, something is wrong".to_string()
+  }
+  fn one_position(&self) -> Pos { Pos::None }
+}
+
+pub struct NotApplicableLexerError;
+impl ProjectError for NotApplicableLexerError {
+  const DESCRIPTION: &'static str = "Pseudo-error to communicate that the lexer doesn't apply";
+  fn message(&self) -> String { CascadingError.message() }
+  fn one_position(&self) -> Pos { Pos::None }
+}
 
 pub struct LexContext<'a> {
   pub text: &'a Tok<String>,
   pub sys: SysId,
-  pub id: LexId,
+  pub id: ParsId,
   pub pos: u32,
   pub reqnot: ReqNot<ExtMsgSet>,
 }
 impl<'a> LexContext<'a> {
-  pub fn recurse(&self, tail: &'a str) -> ProjectResult<(&'a str, OwnedTokTree)> {
+  pub fn recurse(&self, tail: &'a str) -> ProjectResult<(&'a str, GenTokTree)> {
     let start = self.pos(tail);
-    self
+    let lx = (self.reqnot.request(SubLex { pos: start, id: self.id }))
-      .reqnot
+      .ok_or_else(|| CascadingError.pack())?;
-      .request(SubLex { pos: start, id: self.id })
+    Ok((&self.text[lx.pos as usize..], GenTok::Slot(lx.ticket).at(start..lx.pos)))
-      .map_err(|e| pack_err(e.iter().map(|e| err_from_api_or_ref(e, self.reqnot.clone()))))
-      .map(|lx| (&self.text[lx.pos as usize..], OwnedTok::Slot(lx.ticket).at(start..lx.pos)))
   }
 
   pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }
@@ -34,12 +48,6 @@ impl<'a> LexContext<'a> {
   pub fn tok_ran(&self, len: u32, tail: &'a str) -> Range<u32> {
     self.pos(tail) - len..self.pos(tail)
   }
-
-  pub fn report(&self, e: ProjectErrorObj) {
-    for e in unpack_err(e) {
-      self.reqnot.notify(ReportError(self.sys, err_or_ref_to_api(e)))
-    }
-  }
 }
 
 pub trait Lexer: Send + Sync + Sized + Default + 'static {
@@ -47,7 +55,7 @@ pub trait Lexer: Send + Sync + Sized + Default + 'static {
   fn lex<'a>(
     tail: &'a str,
     ctx: &'a LexContext<'a>,
-  ) -> Option<ProjectResult<(&'a str, OwnedTokTree)>>;
+  ) -> ProjectResult<(&'a str, GenTokTree)>;
 }
 
 pub trait DynLexer: Send + Sync + 'static {
@@ -56,7 +64,7 @@ pub trait DynLexer: Send + Sync + 'static {
     &self,
     tail: &'a str,
     ctx: &'a LexContext<'a>,
-  ) -> Option<ProjectResult<(&'a str, OwnedTokTree)>>;
+  ) -> ProjectResult<(&'a str, GenTokTree)>;
 }
 
 impl<T: Lexer> DynLexer for T {
@@ -65,7 +73,7 @@ impl<T: Lexer> DynLexer for T {
     &self,
     tail: &'a str,
     ctx: &'a LexContext<'a>,
-  ) -> Option<ProjectResult<(&'a str, OwnedTokTree)>> {
+  ) -> ProjectResult<(&'a str, GenTokTree)> {
     T::lex(tail, ctx)
   }
 }

orchid-extension/src/proj_error.rs

@@ -1,18 +0,0 @@
-//! Abstractions for handling various code-related errors under a common trait
-//! object.
-
-use std::any::Any;
-use std::borrow::Cow;
-use std::cell::RefCell;
-use std::sync::Arc;
-use std::{fmt, iter};
-
-use dyn_clone::{clone_box, DynClone};
-use itertools::Itertools;
-use orchid_api::error::{ProjErr, ProjErrLocation};
-
-use crate::boxed_iter::{box_once, BoxedIter};
-use crate::intern::{deintern, intern, Token};
-use crate::location::{GetSrc, Position};
-#[allow(unused)] // for doc
-use crate::virt_fs::CodeNotFound;

orchid-extension/src/system.rs

@@ -1,16 +1,19 @@
 use std::any::TypeId;
 
+use hashbrown::HashMap;
+use orchid_api::atom::Atom;
 use orchid_api::proto::ExtMsgSet;
 use orchid_api::system::SysId;
 use orchid_api_traits::Decode;
+use orchid_base::interner::Tok;
 use orchid_base::reqnot::ReqNot;
 
-use crate::atom::{get_info, AtomCard, AtomDynfo, AtomicFeatures, ForeignAtom, TypAtom};
+use crate::atom::{get_info, AtomCtx, AtomDynfo, AtomicFeatures, ForeignAtom, TypAtom};
 use crate::fs::DeclFs;
 use crate::fun::Fun;
 use crate::lexer::LexerObj;
 use crate::system_ctor::{CtedObj, SystemCtor};
-use crate::tree::GenTree;
+use crate::tree::GenMemberKind;
 
 /// System as consumed by foreign code
 pub trait SystemCard: Default + Send + Sync + 'static {
@@ -51,6 +54,11 @@ pub fn atom_by_idx(
   }
 }
 
+pub fn resolv_atom(sys: &(impl DynSystemCard + ?Sized), atom: &Atom) -> &'static dyn AtomDynfo {
+  let tid = u64::decode(&mut &atom.data[..8]);
+  atom_by_idx(sys, tid).expect("Value of nonexistent type found")
+}
+
 impl<T: SystemCard> DynSystemCard for T {
   fn name(&self) -> &'static str { T::Ctor::NAME }
   fn atoms(&self) -> &'static [Option<&'static dyn AtomDynfo>] { Self::ATOM_DEFS }
@@ -58,26 +66,26 @@ impl<T: SystemCard> DynSystemCard for T {
 
 /// System as defined by author
 pub trait System: Send + Sync + SystemCard + 'static {
-  fn env() -> GenTree;
+  fn env() -> Vec<(Tok<String>, GenMemberKind)>;
   fn vfs() -> DeclFs;
   fn lexers() -> Vec<LexerObj>;
 }
 
-pub trait DynSystem: Send + Sync + 'static {
+pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
-  fn dyn_env(&self) -> GenTree;
+  fn dyn_env(&self) -> HashMap<Tok<String>, GenMemberKind>;
   fn dyn_vfs(&self) -> DeclFs;
   fn dyn_lexers(&self) -> Vec<LexerObj>;
   fn dyn_card(&self) -> &dyn DynSystemCard;
 }
 
 impl<T: System> DynSystem for T {
-  fn dyn_env(&self) -> GenTree { Self::env() }
+  fn dyn_env(&self) -> HashMap<Tok<String>, GenMemberKind> { Self::env().into_iter().collect() }
   fn dyn_vfs(&self) -> DeclFs { Self::vfs() }
   fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
   fn dyn_card(&self) -> &dyn DynSystemCard { self }
 }
 
-pub fn downcast_atom<A: AtomCard>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom> {
+pub fn downcast_atom<A: AtomicFeatures>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom> {
   let mut data = &foreign.atom.data[..];
   let ctx = foreign.expr.get_ctx();
   let info_ent = (ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
@@ -86,7 +94,7 @@ pub fn downcast_atom<A: AtomCard>(foreign: ForeignAtom) -> Result<TypAtom<A>, Fo
   match info_ent {
     None => Err(foreign),
     Some((_, info)) => {
-      let val = info.decode(data);
+      let val = info.decode(AtomCtx(data, ctx));
       let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
       Ok(TypAtom { value, data: foreign })
     },

orchid-extension/src/system_ctor.rs

@@ -1,8 +1,7 @@
 use std::any::Any;
-use std::num::NonZeroU16;
 use std::sync::Arc;
 
-use orchid_api::system::{NewSystem, SysId, SystemDecl};
+use orchid_api::system::{NewSystem, SysDeclId, SysId, SystemDecl};
 use orchid_base::boxed_iter::{box_empty, box_once, BoxedIter};
 use ordered_float::NotNan;
 
@@ -67,12 +66,12 @@ pub trait SystemCtor: Send + Sync + 'static {
 }
 
 pub trait DynSystemCtor: Send + Sync + 'static {
-  fn decl(&self, id: NonZeroU16) -> SystemDecl;
+  fn decl(&self, id: SysDeclId) -> SystemDecl;
   fn new_system(&self, new: &NewSystem) -> CtedObj;
 }
 
 impl<T: SystemCtor> DynSystemCtor for T {
-  fn decl(&self, id: NonZeroU16) -> SystemDecl {
+  fn decl(&self, id: SysDeclId) -> SystemDecl {
     // Version is equivalent to priority for all practical purposes
     let priority = NotNan::new(T::VERSION).unwrap();
     // aggregate depends names

orchid-extension/src/tree.rs

@@ -1,31 +1,33 @@
+use std::iter;
+use std::num::NonZero;
 use std::ops::Range;
 
-use ahash::HashMap;
+use hashbrown::HashMap;
 use dyn_clone::{clone_box, DynClone};
 use itertools::Itertools;
 use orchid_api::tree::{
-  MacroRule, Paren, PlaceholderKind, Token, TokenTree, Tree, TreeId, TreeModule,
+  Macro, Paren, PlaceholderKind, Token, TokenTree, Item, TreeId, ItemKind, Member, MemberKind, Module, TreeTicket
-  TreeTicket,
 };
-use orchid_base::interner::intern;
+use orchid_base::interner::{intern, Tok};
 use orchid_base::location::Pos;
-use orchid_base::name::VName;
+use orchid_base::name::{NameLike, Sym, VName};
 use orchid_base::tokens::OwnedPh;
 use ordered_float::NotNan;
 use trait_set::trait_set;
 
 use crate::atom::AtomFactory;
 use crate::conv::ToExpr;
-use crate::error::{err_or_ref_to_api, ProjectErrorObj};
+use crate::entrypoint::MemberRecord;
+use crate::error::{errv_to_apiv, ProjectErrorObj};
 use crate::expr::GenExpr;
 use crate::system::DynSystem;
 
 #[derive(Clone)]
-pub struct OwnedTokTree {
+pub struct GenTokTree {
-  pub tok: OwnedTok,
+  pub tok: GenTok,
   pub range: Range<u32>,
 }
-impl OwnedTokTree {
+impl GenTokTree {
   pub fn into_api(self, sys: &dyn DynSystem) -> TokenTree {
     TokenTree { token: self.tok.into_api(sys), range: self.range }
   }
@@ -58,120 +60,185 @@ pub fn ph(s: &str) -> OwnedPh {
 }
 
 #[derive(Clone)]
-pub enum OwnedTok {
+pub enum GenTok {
-  Lambda(Vec<OwnedTokTree>, Vec<OwnedTokTree>),
+  Lambda(Vec<GenTokTree>),
-  Name(VName),
+  Name(Tok<String>),
-  S(Paren, Vec<OwnedTokTree>),
+  NS,
+  BR,
+  S(Paren, Vec<GenTokTree>),
   Atom(AtomFactory),
   Slot(TreeTicket),
   Ph(OwnedPh),
   Bottom(ProjectErrorObj),
 }
-impl OwnedTok {
+impl GenTok {
-  pub fn at(self, range: Range<u32>) -> OwnedTokTree { OwnedTokTree { tok: self, range } }
+  pub fn at(self, range: Range<u32>) -> GenTokTree { GenTokTree { tok: self, range } }
   pub fn into_api(self, sys: &dyn DynSystem) -> Token {
     match self {
-      Self::Lambda(x, body) => Token::Lambda(
+      Self::Lambda(x) => Token::Lambda(x.into_iter().map(|tt| tt.into_api(sys)).collect()),
-        x.into_iter().map(|tt| tt.into_api(sys)).collect_vec(),
+      Self::Name(n) => Token::Name(n.marker()),
-        body.into_iter().map(|tt| tt.into_api(sys)).collect_vec(),
+      Self::NS => Token::NS,
-      ),
+      Self::BR => Token::BR,
-      Self::Name(n) => Token::Name(n.into_iter().map(|t| t.marker()).collect_vec()),
       Self::Ph(ph) => Token::Ph(ph.to_api()),
       Self::S(p, body) => Token::S(p, body.into_iter().map(|tt| tt.into_api(sys)).collect_vec()),
       Self::Slot(tk) => Token::Slot(tk),
       Self::Atom(at) => Token::Atom(at.build(sys)),
-      Self::Bottom(err) => Token::Bottom(err_or_ref_to_api(err)),
+      Self::Bottom(err) => Token::Bottom(errv_to_apiv([err])),
     }
   }
+  pub fn vname(name: &VName) -> impl Iterator<Item = GenTok> + '_ {
+    let (head, tail) = name.split_first();
+    iter::once(Self::Name(head)).chain(tail.iter().flat_map(|t| [Self::NS, Self::Name(t)]))
+  }
 }
 
 #[derive(Clone)]
 pub struct GenMacro {
-  pub pattern: Vec<OwnedTokTree>,
+  pub pattern: Vec<GenTokTree>,
   pub priority: NotNan<f64>,
-  pub template: Vec<OwnedTokTree>,
+  pub template: Vec<GenTokTree>,
 }
 
 pub fn tokv_into_api(
-  tokv: impl IntoIterator<Item = OwnedTokTree>,
+  tokv: impl IntoIterator<Item = GenTokTree>,
   sys: &dyn DynSystem,
 ) -> Vec<TokenTree> {
   tokv.into_iter().map(|tok| tok.into_api(sys)).collect_vec()
 }
 
-pub fn wrap_tokv(items: Vec<OwnedTokTree>, range: Range<u32>) -> OwnedTokTree {
+pub fn wrap_tokv(items: Vec<GenTokTree>, range: Range<u32>) -> GenTokTree {
   match items.len() {
     1 => items.into_iter().next().unwrap(),
-    _ => OwnedTok::S(Paren::Round, items).at(range),
+    _ => GenTok::S(Paren::Round, items).at(range),
   }
 }
 
-#[derive(Clone)]
+pub struct GenItem {
-pub struct GenTree {
+  pub item: GenItemKind,
-  pub item: GenItem,
+  pub pos: Pos,
-  pub location: Pos,
 }
-impl GenTree {
+impl GenItem {
-  pub fn cnst(gc: impl ToExpr) -> Self { GenItem::Const(gc.to_expr()).at(Pos::Inherit) }
+  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> Item {
-  pub fn module<'a>(entries: impl IntoIterator<Item = (&'a str, GenTree)>) -> Self {
+    let kind = match self.item {
-    GenItem::Mod(entries.into_iter().map(|(k, v)| (k.to_string(), v)).collect()).at(Pos::Inherit)
+      GenItemKind::Rule(GenMacro { pattern, priority, template }) => ItemKind::Rule(Macro {
-  }
+        pattern: tokv_into_api(pattern, ctx.sys()),
-  pub fn rule(
-    prio: f64,
-    pat: impl IntoIterator<Item = OwnedTokTree>,
-    tpl: impl IntoIterator<Item = OwnedTokTree>,
-  ) -> Self {
-    GenItem::Rule(GenMacro {
-      pattern: pat.into_iter().collect(),
-      priority: NotNan::new(prio).expect("expected to be static"),
-      template: tpl.into_iter().collect(),
-    })
-    .at(Pos::Inherit)
-  }
-  pub fn into_api(
-    self,
-    sys: &dyn DynSystem,
-    with_lazy: &mut impl FnMut(LazyTreeFactory) -> TreeId,
-  ) -> Tree {
-    match self.item {
-      GenItem::Const(gc) => Tree::Const(gc.into_api(sys)),
-      GenItem::Rule(GenMacro { pattern, priority, template }) => Tree::Rule(MacroRule {
-        pattern: tokv_into_api(pattern, sys),
         priority,
-        template: tokv_into_api(template, sys),
+        template: tokv_into_api(template, ctx.sys()),
       }),
-      GenItem::Mod(entv) => Tree::Mod(TreeModule {
+      GenItemKind::Raw(item) => ItemKind::Raw(item.into_iter().map(|t| t.into_api(ctx.sys())).collect_vec()),
-        children: entv
+      GenItemKind::Member(mem) => ItemKind::Member(mem.into_api(ctx))
-          .into_iter()
+    };
-          .map(|(name, tree)| (name.to_string(), tree.into_api(sys, with_lazy)))
+    Item { location: self.pos.to_api(), kind }
-          .collect(),
+  }
+}
+
+pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> GenItem {
+  let kind = GenMemberKind::Const(value.to_expr());
+  GenItemKind::Member(GenMember { public, name: intern(name), kind }).at(Pos::Inherit)
+}
+pub fn module(
+  public: bool,
+  name: &str,
+  imports: impl IntoIterator<Item = Sym>,
+  items: impl IntoIterator<Item = GenItem>
+) -> GenItem {
+  let (name, kind) = root_mod(name, imports, items);
+  GenItemKind::Member(GenMember { public, name, kind }).at(Pos::Inherit)
+}
+pub fn root_mod(
+  name: &str, 
+  imports: impl IntoIterator<Item = Sym>,
+  items: impl IntoIterator<Item = GenItem>
+) -> (Tok<String>, GenMemberKind) {
+  let kind = GenMemberKind::Mod {
+    imports: imports.into_iter().collect(),
+    items: items.into_iter().collect()
+  };
+  (intern(name), kind)
+}
+pub fn rule(
+  prio: f64,
+  pat: impl IntoIterator<Item = GenTokTree>,
+  tpl: impl IntoIterator<Item = GenTokTree>,
+) -> GenItem {
+  GenItemKind::Rule(GenMacro {
+    pattern: pat.into_iter().collect(),
+    priority: NotNan::new(prio).expect("expected to be static"),
+    template: tpl.into_iter().collect(),
+  })
+  .at(Pos::Inherit)
+}
+
+trait_set! {
+  trait LazyMemberCallback = FnOnce() -> GenMemberKind + Send + Sync + DynClone
+}
+pub struct LazyMemberFactory(Box<dyn LazyMemberCallback>);
+impl LazyMemberFactory {
+  pub fn new(cb: impl FnOnce() -> GenMemberKind + Send + Sync + Clone + 'static) -> Self {
+    Self(Box::new(cb))
+  }
+  pub fn build(self) -> GenMemberKind { (self.0)() }
+}
+impl Clone for LazyMemberFactory {
+  fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
+}
+
+pub enum GenItemKind {
+  Member(GenMember),
+  Raw(Vec<GenTokTree>),
+  Rule(GenMacro),
+}
+impl GenItemKind {
+  pub fn at(self, position: Pos) -> GenItem { GenItem { item: self, pos: position } }
+}
+
+pub struct GenMember {
+  public: bool,
+  name: Tok<String>,
+  kind: GenMemberKind,
+}
+impl GenMember {
+  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> Member {
+    Member { name: self.name.marker(), public: self.public, kind: self.kind.into_api(ctx) }
+  }
+}
+
+pub enum GenMemberKind {
+  Const(GenExpr),
+  Mod{
+    imports: Vec<Sym>,
+    items: Vec<GenItem>,
+  },
+  Lazy(LazyMemberFactory)
+}
+impl GenMemberKind {
+  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> MemberKind {
+    match self {
+      Self::Lazy(lazy) => MemberKind::Lazy(ctx.with_lazy(lazy)),
+      Self::Const(c) => MemberKind::Const(c.into_api(ctx.sys())),
+      Self::Mod { imports, items } => MemberKind::Module(Module {
+        imports: imports.into_iter().map(|t| t.tok().marker()).collect(),
+        items: items.into_iter().map(|i| i.into_api(ctx)).collect_vec()
       }),
-      GenItem::Lazy(cb) => Tree::Lazy(with_lazy(cb)),
     }
   }
 }
 
-trait_set! {
+pub trait TreeIntoApiCtx {
-  trait LazyTreeCallback = FnMut() -> GenTree + Send + Sync + DynClone
+  fn sys(&self) -> &dyn DynSystem;
-}
+  fn with_lazy(&mut self, fac: LazyMemberFactory) -> TreeId;
-pub struct LazyTreeFactory(Box<dyn LazyTreeCallback>);
-impl LazyTreeFactory {
-  pub fn new(cb: impl FnMut() -> GenTree + Send + Sync + Clone + 'static) -> Self {
-    Self(Box::new(cb))
-  }
-  pub fn build(&mut self) -> GenTree { (self.0)() }
-}
-impl Clone for LazyTreeFactory {
-  fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
 }
 
-#[derive(Clone)]
+pub struct TIACtxImpl<'a> {
-pub enum GenItem {
+  pub sys: &'a dyn DynSystem,
-  Const(GenExpr),
+  pub lazy: &'a mut HashMap<TreeId, MemberRecord>
-  Mod(HashMap<String, GenTree>),
-  Rule(GenMacro),
-  Lazy(LazyTreeFactory),
 }
-impl GenItem {
+
-  pub fn at(self, position: Pos) -> GenTree { GenTree { item: self, location: position } }
+impl<'a> TreeIntoApiCtx for TIACtxImpl<'a> {
+  fn sys(&self) -> &dyn DynSystem { self.sys }
+  fn with_lazy(&mut self, fac: LazyMemberFactory) -> TreeId {
+    let id = TreeId(NonZero::new((self.lazy.len() + 2) as u64).unwrap());
+    self.lazy.insert(id, MemberRecord::Gen(fac));
+    id
+  }
 }

orchid-host/Cargo.toml

@@ -10,6 +10,7 @@ derive_destructure = "1.0.0"
 hashbrown = "0.14.5"
 itertools = "0.13.0"
 lazy_static = "1.4.0"
+never = "0.1.0"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }

orchid-host/src/expr.rs

@@ -4,9 +4,9 @@ use std::sync::{Arc, RwLock};
 
 use hashbrown::HashMap;
 use lazy_static::lazy_static;
-use orchid_api::expr::ExprTicket;
+use orchid_api::expr::{Expr, ExprTicket};
 
-use crate::extension::AtomHand;
+use crate::extension::{AtomHand, System};
 
 #[derive(Clone, Debug)]
 pub struct RtExpr {
@@ -17,8 +17,10 @@ impl RtExpr {
   pub fn as_atom(&self) -> Option<AtomHand> { todo!() }
   pub fn strong_count(&self) -> usize { todo!() }
   pub fn id(&self) -> ExprTicket {
-    NonZeroU64::new(self.data.as_ref() as *const () as usize as u64)
+    ExprTicket(
-      .expect("this is a ref, it cannot be null")
+      NonZeroU64::new(self.data.as_ref() as *const () as usize as u64)
+        .expect("this is a ref, it cannot be null")
+    )
   }
   pub fn canonicalize(&self) -> ExprTicket {
     if !self.is_canonical.swap(true, Ordering::Relaxed) {
@@ -27,6 +29,7 @@ impl RtExpr {
     self.id()
   }
   pub fn resolve(tk: ExprTicket) -> Option<Self> { KNOWN_EXPRS.read().unwrap().get(&tk).cloned() }
+  pub fn from_api(api: Expr, sys: &System) -> Self { todo!() }
 }
 impl Drop for RtExpr {
   fn drop(&mut self) {

orchid-host/src/extension.rs

@@ -1,25 +1,27 @@
 use std::io::Write as _;
+use std::num::NonZero;
 use std::ops::Deref;
-use std::sync::atomic::{AtomicU16, AtomicU32, Ordering};
+use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering};
-use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::mpsc::{sync_channel, SyncSender};
-use std::sync::{Arc, Mutex, RwLock, Weak};
+use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
 use std::{fmt, io, process, thread};
 
 use derive_destructure::destructure;
+use hashbrown::hash_map::Entry;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use orchid_api::atom::{Atom, AtomDrop, AtomSame, CallRef, FinalCall, Fwd, Fwded};
-use orchid_api::error::{ErrNotif, ProjErrOrRef, ProjResult, ReportError};
+use orchid_api::error::ProjResult;
 use orchid_api::expr::{Acquire, Expr, ExprNotif, ExprTicket, Release, Relocate};
 use orchid_api::interner::IntReq;
-use orchid_api::parser::{CharFilter, Lexed, SubLexed};
+use orchid_api::parser::{CharFilter, LexExpr, LexedExpr, ParsId, SubLex, SubLexed};
 use orchid_api::proto::{
   ExtHostNotif, ExtHostReq, ExtensionHeader, HostExtNotif, HostHeader, HostMsgSet,
 };
 use orchid_api::system::{NewSystem, SysDeclId, SysId, SystemDecl, SystemDrop};
-use orchid_api::tree::{GetConstTree, Tree, TreeId};
+use orchid_api::tree::{GetMember, Member, MemberKind, TreeId};
-use orchid_api_traits::{Coding, Decode, Encode};
+use orchid_api_traits::{Decode, Encode, Request};
 use orchid_base::char_filter::char_filter_match;
 use orchid_base::clone;
 use orchid_base::interner::{deintern, intern, Tok};
@@ -27,6 +29,7 @@ use orchid_base::reqnot::{ReqNot, Requester as _};
 use ordered_float::NotNan;
 
 use crate::expr::RtExpr;
+use crate::tree::OwnedMember;
 
 #[derive(Debug, destructure)]
 pub struct AtomData {
@@ -37,16 +40,16 @@ pub struct AtomData {
 impl AtomData {
   fn api(self) -> Atom {
     let (owner, drop, data) = self.destructure();
-    Atom { data, drop, owner: owner.0.id }
+    Atom { data, drop, owner: owner.id() }
   }
   fn api_ref(&self) -> Atom {
-    Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.0.id }
+    Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
   }
 }
 impl Drop for AtomData {
   fn drop(&mut self) {
-    self.owner.0.ext.0.reqnot.notify(AtomDrop(Atom {
+    self.owner.reqnot().notify(AtomDrop(Atom {
-      owner: self.owner.0.id,
+      owner: self.owner.id(),
       data: self.data.clone(),
       drop: true,
     }))
@@ -62,22 +65,22 @@ impl AtomHand {
   }
   pub fn call(self, arg: RtExpr) -> Expr {
     let owner_sys = self.0.owner.clone();
-    let ext = &owner_sys.0.ext;
+    let reqnot = owner_sys.reqnot();
     let ticket = owner_sys.give_expr(arg.canonicalize(), || arg);
     match Arc::try_unwrap(self.0) {
-      Ok(data) => ext.0.reqnot.request(FinalCall(data.api(), ticket)),
+      Ok(data) => reqnot.request(FinalCall(data.api(), ticket)),
-      Err(hand) => ext.0.reqnot.request(CallRef(hand.api_ref(), ticket)),
+      Err(hand) => reqnot.request(CallRef(hand.api_ref(), ticket)),
     }
   }
   pub fn same(&self, other: &AtomHand) -> bool {
-    let owner = self.0.owner.0.id;
+    let owner = self.0.owner.id();
-    if other.0.owner.0.id != owner {
+    if other.0.owner.id() != owner {
       return false;
     }
-    self.0.owner.0.ext.0.reqnot.request(AtomSame(self.0.api_ref(), other.0.api_ref()))
+    self.0.owner.reqnot().request(AtomSame(self.0.api_ref(), other.0.api_ref()))
   }
   pub fn req(&self, req: Vec<u8>) -> Vec<u8> {
-    self.0.owner.0.ext.0.reqnot.request(Fwded(self.0.api_ref(), req))
+    self.0.owner.reqnot().request(Fwded(self.0.api_ref(), req))
   }
   pub fn api_ref(&self) -> Atom { self.0.api_ref() }
 }
@@ -131,9 +134,7 @@ impl Extension {
             acq_expr(inc, expr);
             rel_expr(dec, expr);
           },
-          ExtHostNotif::ErrNotif(ErrNotif::ReportError(ReportError(sys, err))) => {
+          ExtHostNotif::AdviseSweep(_advice) => eprintln!("Sweep advice is unsupported")
-            System::resolve(sys).unwrap().0.err_send.send(err).unwrap();
-          },
         },
         |req| match req.req() {
           ExtHostReq::Ping(ping) => req.handle(ping, &()),
@@ -145,9 +146,16 @@ impl Extension {
           ExtHostReq::Fwd(fw @ Fwd(atom, _body)) => {
             let sys = System::resolve(atom.owner).unwrap();
             thread::spawn(clone!(fw; move || {
-              req.handle(&fw, &sys.0.ext.0.reqnot.request(Fwded(fw.0.clone(), fw.1.clone())))
+              req.handle(&fw, &sys.reqnot().request(Fwded(fw.0.clone(), fw.1.clone())))
             }));
           },
+          ExtHostReq::SubLex(sl) => {
+            let lex_g = LEX_RECUR.lock().unwrap();
+            let (rep_in, rep_out) = sync_channel(0);
+            let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
+            req_in.send(ReqPair(sl.clone(), rep_in)).unwrap();
+            req.handle(sl, &rep_out.recv().unwrap())
+          },
           _ => todo!(),
         },
       ),
@@ -169,23 +177,24 @@ impl SystemCtor {
   }
   pub fn run<'a>(&self, depends: impl IntoIterator<Item = &'a System>) -> System {
     let mut inst_g = SYSTEM_INSTS.write().unwrap();
-    let depends = depends.into_iter().map(|si| si.0.id).collect_vec();
+    let depends = depends.into_iter().map(|si| si.id()).collect_vec();
     debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
     let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
     static NEXT_ID: AtomicU16 = AtomicU16::new(1);
-    let id = SysId::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped");
+    let id = SysId(NonZero::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped"));
     let sys_inst = ext.reqnot.request(NewSystem { depends, id, system: self.decl.id });
-    let (err_send, err_rec) = channel();
     let data = System(Arc::new(SystemInstData {
       decl_id: self.decl.id,
       ext: Extension(ext),
       exprs: RwLock::default(),
       lex_filter: sys_inst.lex_filter,
-      const_root_id: sys_inst.const_root_id,
+      const_root: OnceLock::new(),
-      err_send,
-      err_rec: Mutex::new(err_rec),
       id,
     }));
+    let root = (sys_inst.const_root.into_iter())
+        .map(|(k, v)| OwnedMember::from_api(Member { public: true, name: k, kind: v }, &data))
+        .collect_vec();
+      data.0.const_root.set(root).unwrap();
     inst_g.insert(id, data.clone());
     data
   }
@@ -193,8 +202,11 @@ impl SystemCtor {
 
 lazy_static! {
   static ref SYSTEM_INSTS: RwLock<HashMap<SysId, System>> = RwLock::default();
+  static ref LEX_RECUR: Mutex<HashMap<ParsId, SyncSender<ReqPair<SubLex>>>> = Mutex::default();
 }
 
+pub struct ReqPair<R: Request>(R, pub SyncSender<R::Response>);
+
 #[derive(destructure)]
 pub struct SystemInstData {
   exprs: RwLock<HashMap<ExprTicket, (AtomicU32, RtExpr)>>,
@@ -202,9 +214,7 @@ pub struct SystemInstData {
   decl_id: SysDeclId,
   lex_filter: CharFilter,
   id: SysId,
-  const_root_id: TreeId,
+  const_root: OnceLock<Vec<OwnedMember>>,
-  err_rec: Mutex<Receiver<ProjErrOrRef>>,
-  err_send: Sender<ProjErrOrRef>,
 }
 impl Drop for SystemInstData {
   fn drop(&mut self) {
@@ -217,48 +227,59 @@ impl Drop for SystemInstData {
 #[derive(Clone)]
 pub struct System(Arc<SystemInstData>);
 impl System {
+  pub fn id(&self) -> SysId { self.id }
   fn resolve(id: SysId) -> Option<System> { SYSTEM_INSTS.read().unwrap().get(&id).cloned() }
+  fn reqnot(&self) -> &ReqNot<HostMsgSet> { &self.0.ext.0.reqnot }
   fn give_expr(&self, ticket: ExprTicket, get_expr: impl FnOnce() -> RtExpr) -> ExprTicket {
-    let mut exprs = self.0.exprs.write().unwrap();
+    match self.0.exprs.write().unwrap().entry(ticket) {
-    exprs
+      Entry::Occupied(mut oe) => {
-      .entry(ticket)
+        oe.get_mut().0.fetch_add(1, Ordering::Relaxed);
-      .and_modify(|(c, _)| {
+      },
-        c.fetch_add(1, Ordering::Relaxed);
+      Entry::Vacant(v) => {
-      })
+        v.insert((AtomicU32::new(1), get_expr()));
-      .or_insert((AtomicU32::new(1), get_expr()));
+      }
+    }
     ticket
   }
-  pub fn const_tree(&self) -> Tree {
+  pub fn get_tree(&self, id: TreeId) -> MemberKind {
-    self.0.ext.0.reqnot.request(GetConstTree(self.0.id, self.0.const_root_id))
+    self.reqnot().request(GetMember(self.0.id, id))
-  }
-  pub fn catch_err<R: Coding>(&self, cb: impl FnOnce() -> ProjResult<R>) -> ProjResult<R> {
-    let mut errors = Vec::new();
-    if let Ok(err) = self.0.err_rec.lock().unwrap().try_recv() {
-      eprintln!("Errors left in queue");
-      errors.push(err);
-    }
-    let value = cb().inspect_err(|e| errors.extend(e.iter().cloned()));
-    while let Ok(err) = self.0.err_rec.lock().unwrap().try_recv() {
-      errors.push(err);
-    }
-    if !errors.is_empty() { Err(errors) } else { value }
   }
   pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
   pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
+  /// Have this system lex a part of the source. It is assumed that
+  /// [Self::can_lex] was called and returned true.
   pub fn lex(
     &self,
     source: Tok<String>,
-    pos: usize,
+    pos: u32,
-    r: impl FnMut(usize) -> ProjResult<SubLexed>,
+    mut r: impl FnMut(u32) -> Option<SubLexed> + Send,
-  ) -> ProjResult<Lexed> {
+  ) -> ProjResult<Option<LexedExpr>> {
-    todo!()
+    // get unique lex ID
+    static LEX_ID: AtomicU64 = AtomicU64::new(1);
+    let id = ParsId(NonZero::new(LEX_ID.fetch_add(1, Ordering::Relaxed)).unwrap());
+    thread::scope(|s| {
+      // create and register channel
+      let (req_in, req_out) = sync_channel(0);
+      LEX_RECUR.lock().unwrap().insert(id, req_in); // LEX_RECUR released
+      // spawn recursion handler which will exit when the sender is collected
+      s.spawn(move || {
+        while let Ok(ReqPair(sublex, rep_in)) = req_out.recv() {
+          rep_in.send(r(sublex.pos)).unwrap()
+        }
+      });
+      // Pass control to extension
+      let ret = self.reqnot().request(LexExpr { id, pos, sys: self.id(), text: source.marker() });
+      // collect sender to unblock recursion handler thread before returning
+      LEX_RECUR.lock().unwrap().remove(&id);
+      ret.transpose()
+    }) // exit recursion handler thread
   }
 }
 impl fmt::Debug for System {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     let ctor = (self.0.ext.0.systems.iter().find(|c| c.decl.id == self.0.decl_id))
       .expect("System instance with no associated constructor");
-    write!(f, "System({} @ {} #{}, ", ctor.decl.name, ctor.decl.priority, self.0.id)?;
+    write!(f, "System({} @ {} #{}, ", ctor.decl.name, ctor.decl.priority, self.0.id.0)?;
     match self.0.exprs.read() {
       Err(_) => write!(f, "expressions unavailable"),
       Ok(r) => {

orchid-host/src/lex.rs

@@ -1,44 +1,70 @@
-use orchid_api::tree::Paren;
+use std::num::NonZeroU64;
-use orchid_base::intern;
-use orchid_base::interner::Tok;
-use orchid_base::location::Pos;
-use orchid_base::number::parse_num;
 
-use crate::extension::System;
+use hashbrown::HashMap;
-use crate::results::{mk_err, num_to_err, OwnedResult};
+use orchid_api::parser::SubLexed;
+use orchid_api::system::SysId;
+use orchid_api::tree::{Paren, Token, TokenTree, TreeTicket};
+use orchid_base::error::OwnedError;
+use orchid_base::intern;
+use orchid_base::interner::{deintern, intern, Tok};
+use orchid_base::location::Pos;
+use orchid_base::tokens::OwnedPh;
+
+use crate::extension::{AtomHand, System};
+use crate::results::{mk_err, OwnedResult};
 use crate::tree::{OwnedTok, OwnedTokTree};
 
 pub struct LexCtx<'a> {
   pub systems: &'a [System],
-  pub source: Tok<String>,
+  pub source: &'a Tok<String>,
-  pub src: &'a str,
+  pub tail: &'a str,
+  pub sub_trees: &'a mut HashMap<TreeTicket, OwnedTokTree>,
 }
 impl<'a> LexCtx<'a> {
-  pub fn get_pos(&self) -> u32 { self.source.len() as u32 - self.src.len() as u32 }
+  pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
+  where 'a: 'b {
+    LexCtx {
+      source: self.source,
+      tail: &self.source[pos as usize..],
+      systems: self.systems,
+      sub_trees: &mut *self.sub_trees,
+    }
+  }
+  pub fn get_pos(&self) -> u32 { self.end_pos() - self.tail.len() as u32 }
+  pub fn end_pos(&self) -> u32 { self.source.len() as u32 }
+  pub fn set_pos(&mut self, pos: u32) { self.tail = &self.source[pos as usize..] }
+  pub fn push_pos(&mut self, delta: u32) { self.set_pos(self.get_pos() + delta) }
+  pub fn set_tail(&mut self, tail: &'a str) { self.tail = tail }
   pub fn strip_prefix(&mut self, tgt: &str) -> bool {
-    if let Some(src) = self.src.strip_prefix(tgt) {
+    if let Some(src) = self.tail.strip_prefix(tgt) {
-      self.src = src;
+      self.tail = src;
       return true;
     }
     false
   }
+  pub fn add_subtree(&mut self, subtree: OwnedTokTree) -> TreeTicket {
+    let next_idx = TreeTicket(NonZeroU64::new(self.sub_trees.len() as u64 + 1).unwrap());
+    self.sub_trees.insert(next_idx, subtree);
+    next_idx
+  }
+  pub fn rm_subtree(&mut self, ticket: TreeTicket) -> OwnedTokTree {
+    self.sub_trees.remove(&ticket).unwrap()
+  }
   pub fn strip_char(&mut self, tgt: char) -> bool {
-    if let Some(src) = self.src.strip_prefix(tgt) {
+    if let Some(src) = self.tail.strip_prefix(tgt) {
-      self.src = src;
+      self.tail = src;
       return true;
     }
     false
   }
   pub fn trim(&mut self, filter: impl Fn(char) -> bool) {
-    self.src = self.src.trim_start_matches(filter);
+    self.tail = self.tail.trim_start_matches(filter);
-  }
-  pub fn trim_ws(&mut self, br: bool) {
-    self.trim(|c| c.is_whitespace() && br || !"\r\n".contains(c))
   }
+  pub fn trim_ws(&mut self) { self.trim(|c| c.is_whitespace() && !"\r\n".contains(c)) }
   pub fn get_start_matches(&mut self, filter: impl Fn(char) -> bool) -> &'a str {
-    let rest = self.src.trim_start_matches(filter);
+    let rest = self.tail.trim_start_matches(filter);
-    let matches = &self.src[..self.src.len() - rest.len()];
+    let matches = &self.tail[..self.tail.len() - rest.len()];
-    self.src = rest;
+    self.tail = rest;
     matches
   }
 }
@@ -46,73 +72,129 @@ impl<'a> LexCtx<'a> {
 const PARENS: &[(char, char, Paren)] =
   &[('(', ')', Paren::Round), ('[', ']', Paren::Square), ('{', '}', Paren::Curly)];
 
-pub fn lex_tok(ctx: &mut LexCtx, br: bool) -> OwnedResult<OwnedTokTree> {
+pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
+  let start = ctx.get_pos();
   assert!(
-    !ctx.src.is_empty() && !ctx.src.starts_with(char::is_whitespace),
+    !ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space),
-    "Lexing empty string or whitespace to token! Invocations of lex_tok should check for empty string"
+    "Lexing empty string or whitespace to token!\n\
+    Invocations of lex_tok should check for empty string"
   );
-  for (open, close, paren) in PARENS {
+  let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
-    let paren_pos = ctx.get_pos();
+    OwnedTok::BR
-    if ctx.strip_char(*open) {
+  } else if ctx.strip_prefix("::") {
-      let mut body = Vec::new();
+    OwnedTok::NS
-      return loop {
+  } else if ctx.strip_prefix("--[") {
-        ctx.trim_ws(true);
+    let (cmt, tail) = ctx.tail.split_once("]--").ok_or_else(|| {
-        if ctx.strip_char(*close) {
+      vec![mk_err(
-          break Ok(OwnedTokTree {
+        intern!(str: "Unterminated block comment"),
-            tok: OwnedTok::S(paren.clone(), body),
+        "This block comment has no ending ]--",
-            range: paren_pos..ctx.get_pos(),
+        [Pos::Range(start..start + 3).into()],
-          });
+      )]
-        } else if ctx.src.is_empty() {
+    })?;
-          return Err(vec![mk_err(
+    ctx.set_tail(tail);
-            intern!(str: "unclosed paren"),
+    OwnedTok::Comment(cmt.to_string())
-            format!("this {open} has no matching {close}"),
+  } else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
-            [Pos::Range(paren_pos..paren_pos + 1).into()],
+    let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
-          )]);
+    ctx.push_pos(end as u32);
-        }
+    OwnedTok::Comment(tail[2..end].to_string())
-        body.push(lex_tok(ctx, true)?);
+  } else if ctx.strip_char('\\') {
-      };
-    }
-  }
-  if ctx.strip_char('\\') {
-    let bs_pos = ctx.get_pos() - 1;
     let mut arg = Vec::new();
-    loop {
+    ctx.trim_ws();
-      ctx.trim_ws(true);
+    while !ctx.strip_char('.') {
-      if ctx.strip_char('.') {
+      if ctx.tail.is_empty() {
-        break;
-      } else if ctx.src.is_empty() {
         return Err(vec![mk_err(
           intern!(str: "Unclosed lambda"),
           "Lambdae started with \\ should separate arguments from body with .",
-          [Pos::Range(bs_pos..bs_pos + 1).into()],
+          [Pos::Range(start..start + 1).into()],
         )]);
       }
-      arg.push(lex_tok(ctx, true)?);
+      arg.push(lex_once(ctx)?);
+      ctx.trim_ws();
     }
+    OwnedTok::Lambda(arg)
+  } else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
     let mut body = Vec::new();
-    return loop {
+    ctx.trim_ws();
-      ctx.trim_ws(br);
+    while !ctx.strip_char(*rp) {
-      let pos_before_end = ctx.get_pos();
+      if ctx.tail.is_empty() {
-      if !br && ctx.strip_char('\n')
+        return Err(vec![mk_err(
-        || PARENS.iter().any(|(_, e, _)| ctx.strip_char(*e))
+          intern!(str: "unclosed paren"),
-        || ctx.src.is_empty()
+          format!("this {lp} has no matching {rp}"),
-      {
+          [Pos::Range(start..start + 1).into()],
-        break Ok(OwnedTokTree { tok: OwnedTok::Lambda(arg, body), range: bs_pos..pos_before_end });
+        )]);
       }
-      body.push(lex_tok(ctx, br)?);
+      body.push(lex_once(ctx)?);
-    };
+      ctx.trim_ws();
-  }
+    }
-  if ctx.src.starts_with(char::is_numeric) {
+    OwnedTok::S(paren.clone(), body)
-    let num_pos = ctx.get_pos();
+  } else {
-    let num_str = ctx.get_start_matches(|c| c.is_alphanumeric() || "._".contains(c));
+    for sys in ctx.systems {
-    return Ok(OwnedTokTree {
+      let mut errors = Vec::new();
-      range: num_pos..ctx.get_pos(),
+      if ctx.tail.starts_with(|c| sys.can_lex(c)) {
-      tok: match parse_num(num_str) {
+        let lexed = sys.lex(ctx.source.clone(), ctx.get_pos(), |pos| {
-        Err(e) => OwnedTok::Bottom(num_to_err(e, num_pos)),
+          let mut sub_ctx = ctx.push(pos);
-        Ok(v) => todo!(),
+          let ott = lex_once(&mut sub_ctx).inspect_err(|e| errors.extend(e.iter().cloned())).ok()?;
-      },
+          Some(SubLexed { pos: sub_ctx.get_pos(), ticket: sub_ctx.add_subtree(ott) })
-    });
+        });
-  }
+        match lexed {
-  for sys in ctx.systems {}
+          Ok(None) if errors.is_empty() => continue,
-  todo!()
+          Ok(None) => return Err(errors),
+          Err(e) => return Err(e.into_iter().map(|e| OwnedError::from_api(&e)).collect()),
+          Ok(Some(lexed)) => {
+            ctx.set_pos(lexed.pos);
+            return Ok(tt_to_owned(&lexed.expr, sys.id(), ctx))
+          },
+        }
+      }
+    }
+    if ctx.tail.starts_with(name_start) {
+      OwnedTok::Name(intern(ctx.get_start_matches(name_char)))
+    } else if ctx.tail.starts_with(op_char) {
+      OwnedTok::Name(intern(ctx.get_start_matches(op_char)))
+    } else {
+      return Err(vec![mk_err(
+        intern!(str: "Unrecognized character"),
+        "The following syntax is meaningless.",
+        [Pos::Range(start..start + 1).into()],
+      )]);
+    }
+  };
+  Ok(OwnedTokTree { tok, range: start..ctx.get_pos() })
 }
+
+fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
+fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
+fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}:\\".contains(c) }
+fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
+
+fn tt_to_owned(api: &TokenTree, sys: SysId, ctx: &mut LexCtx<'_>) -> OwnedTokTree {
+  let tok = match &api.token {
+    Token::Atom(atom) => OwnedTok::Atom(AtomHand::from_api(atom.clone().associate(sys))),
+    Token::Ph(ph) => OwnedTok::Ph(OwnedPh::from_api(ph.clone())),
+    Token::Bottom(err) => OwnedTok::Bottom(err.iter().map(OwnedError::from_api).collect()),
+    Token::Lambda(arg) => OwnedTok::Lambda(arg.iter().map(|t| tt_to_owned(t, sys, ctx)).collect()),
+    Token::Name(name) => OwnedTok::Name(deintern(*name)),
+    Token::S(p, b) => OwnedTok::S(p.clone(), b.iter().map(|t| tt_to_owned(t, sys, ctx)).collect()),
+    Token::Slot(id) => return ctx.rm_subtree(*id),
+    Token::BR => OwnedTok::BR,
+    Token::NS => OwnedTok::NS,
+  };
+  OwnedTokTree { range: api.range.clone(), tok }
+}
+
+pub fn lex(text: Tok<String>, systems: &[System]) -> OwnedResult<Vec<OwnedTokTree>> {
+  let mut sub_trees = HashMap::new();
+  let mut ctx = LexCtx {
+    source: &text,
+    sub_trees: &mut sub_trees,
+    tail: &text[..],
+    systems,
+  };
+  let mut tokv = Vec::new();
+  ctx.trim(unrep_space);
+  while !ctx.tail.is_empty() {
+    tokv.push(lex_once(&mut ctx)?);
+    ctx.trim(unrep_space);
+  }
+  Ok(tokv)
+}

orchid-host/src/lib.rs

@@ -4,3 +4,4 @@ pub mod extension;
 pub mod lex;
 pub mod results;
 pub mod tree;
+pub mod parse;

orchid-host/src/parse.rs

@@ -0,0 +1,33 @@
+use orchid_base::interner::Tok;
+
+use crate::tree::{OwnedItem, OwnedModule, OwnedTok, OwnedTokTree};
+
+pub struct ParseCtx<'a> {
+  tokens: &'a [OwnedTokTree]
+}
+
+pub fn split_br(ctx: ParseCtx) -> impl Iterator<Item = ParseCtx> {
+  ctx.tokens.split(|t| matches!(t.tok, OwnedTok::BR)).map(|tokens| ParseCtx { tokens })
+}
+
+pub fn strip_br(tt: &OwnedTokTree) -> Option<OwnedTokTree> {
+  let tok = match &tt.tok {
+    OwnedTok::BR => return None,
+    OwnedTok::Lambda(arg) => OwnedTok::Lambda(arg.iter().filter_map(strip_br).collect()),
+    OwnedTok::S(p, b) => OwnedTok::S(p.clone(), b.iter().filter_map(strip_br).collect()),
+    t => t.clone(),
+  };
+  Some(OwnedTokTree { tok, range: tt.range.clone() })
+}
+
+pub fn parse_items(ctx: ParseCtx) -> Vec<OwnedItem> {
+  todo!()
+}
+
+pub fn parse_item(ctx: ParseCtx) -> OwnedItem {
+  todo!()
+}
+
+pub fn parse_module(ctx: ParseCtx) -> (Tok<String>, OwnedModule) {
+  todo!()
+}

orchid-host/src/tree.rs

@@ -1,24 +1,159 @@
+use std::borrow::Borrow;
 use std::ops::Range;
+use std::sync::{Mutex, OnceLock};
 
-use orchid_api::tree::Paren;
+use itertools::Itertools;
+use never::Never;
+use orchid_api::tree::{Item, ItemKind, Macro, Member, MemberKind, Module, Paren, Token, TokenTree, TreeId, TreeTicket};
 use orchid_base::error::OwnedError;
-use orchid_base::name::VName;
+use orchid_base::interner::{deintern, Tok};
+use orchid_base::location::Pos;
+use orchid_base::name::Sym;
 use orchid_base::tokens::OwnedPh;
+use ordered_float::NotNan;
 
-use crate::extension::AtomHand;
+use crate::expr::RtExpr;
+use crate::extension::{AtomHand, System};
+use crate::results::OwnedResult;
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct OwnedTokTree {
   pub tok: OwnedTok,
   pub range: Range<u32>,
 }
+impl OwnedTokTree {
+  pub fn from_api<E>(
+    tt: &TokenTree,
+    sys: &System,
+    do_slot: &mut impl FnMut(&TreeTicket, Range<u32>) -> Result<Self, E>,
+  ) -> Result<Self, E> {
+    let tok = match &tt.token {
+      Token::Atom(a) => OwnedTok::Atom(AtomHand::from_api(a.clone().associate(sys.id()))),
+      Token::BR => OwnedTok::BR,
+      Token::NS => OwnedTok::NS,
+      Token::Bottom(e) => OwnedTok::Bottom(e.iter().map(OwnedError::from_api).collect_vec()),
+      Token::Lambda(arg) => OwnedTok::Lambda(Self::v_from_api(arg, sys, do_slot)?),
+      Token::Name(name) => OwnedTok::Name(deintern(*name)),
+      Token::Ph(ph) => OwnedTok::Ph(OwnedPh::from_api(ph.clone())),
+      Token::S(par, b) => OwnedTok::S(par.clone(), Self::v_from_api(b, sys, do_slot)?),
+      Token::Slot(id) => return do_slot(id, tt.range.clone()),
+    };
+    Ok(Self { range: tt.range.clone(), tok })
+  }
 
-#[derive(Clone)]
+  pub fn v_from_api<E>(
+    tokv: impl IntoIterator<Item: Borrow<TokenTree>>,
+    sys: &System,
+    do_slot: &mut impl FnMut(&TreeTicket, Range<u32>) -> Result<Self, E>,
+  ) -> Result<Vec<Self>, E> {
+    tokv.into_iter().map(|t| Self::from_api(t.borrow(), sys, do_slot)).collect()
+  }
+}
+
+#[derive(Clone, Debug)]
 pub enum OwnedTok {
-  Lambda(Vec<OwnedTokTree>, Vec<OwnedTokTree>),
+  Comment(String),
-  Name(VName),
+  Lambda(Vec<OwnedTokTree>),
+  Name(Tok<String>),
+  NS,
+  BR,
   S(Paren, Vec<OwnedTokTree>),
   Atom(AtomHand),
   Ph(OwnedPh),
-  Bottom(OwnedError),
+  Bottom(Vec<OwnedError>),
 }
+
+#[derive(Debug)]
+pub struct OwnedItem {
+  pub pos: Pos,
+  pub kind: OwnedItemKind,
+}
+
+#[derive(Debug)]
+pub enum OwnedItemKind {
+  Raw(Vec<OwnedTokTree>),
+  Member(OwnedMember),
+  Rule(OwnedMacro),
+}
+
+fn slot_panic(_: &TreeTicket, _: Range<u32>) -> Result<OwnedTokTree, Never> {
+  panic!("No slots allowed at item stage")
+}
+
+impl OwnedItem {
+  pub fn from_api(tree: Item, sys: &System) -> Self {
+    let kind = match tree.kind {
+      ItemKind::Raw(tokv) =>
+        OwnedItemKind::Raw(OwnedTokTree::v_from_api::<Never>(tokv, sys, &mut slot_panic).unwrap()),
+      ItemKind::Member(m) => OwnedItemKind::Member(OwnedMember::from_api(m, sys)),
+      ItemKind::Rule(r) => OwnedItemKind::Rule(OwnedMacro::from_api(r, sys)),
+    };
+    Self { pos: Pos::from_api(&tree.location), kind }
+  }
+}
+
+#[derive(Debug)]
+pub struct OwnedMember {
+  pub public: bool,
+  pub name: Tok<String>,
+  pub kind: OnceLock<OMemKind>,
+  pub lazy: Mutex<Option<LazyMemberHandle>>,
+}
+impl OwnedMember {
+  pub fn from_api(Member{ public, name, kind }: Member, sys: &System) -> Self {
+    let (kind, lazy) = match kind {
+      MemberKind::Const(c) => (OnceLock::from(OMemKind::Const(RtExpr::from_api(c, sys))), None),
+      MemberKind::Module(m) => (OnceLock::from(OMemKind::Mod(OwnedModule::from_api(m, sys))), None),
+      MemberKind::Lazy(id) => (OnceLock::new(), Some(LazyMemberHandle(id, sys.clone())))
+    };
+    OwnedMember { public, name: deintern(name), kind, lazy: Mutex::new(lazy) }
+  }
+}
+
+#[derive(Debug)]
+pub enum OMemKind {
+  Const(RtExpr),
+  Mod(OwnedModule),
+}
+
+#[derive(Debug)]
+pub struct OwnedModule {
+  pub imports: Vec<Sym>,
+  pub items: Vec<OwnedItem>,
+}
+impl OwnedModule {
+  pub fn from_api(m: Module, sys: &System) -> Self {
+    Self {
+      imports: m.imports.into_iter().map(|m| Sym::from_tok(deintern(m)).unwrap()).collect_vec(),
+      items: m.items.into_iter().map(|i| OwnedItem::from_api(i, sys)).collect_vec(),
+    }
+  }
+}
+
+#[derive(Debug)]
+pub struct OwnedMacro {
+  pub priority: NotNan<f64>,
+  pub pattern: Vec<OwnedTokTree>,
+  pub template: Vec<OwnedTokTree>,
+}
+impl OwnedMacro {
+  pub fn from_api(m: Macro, sys: &System) -> Self {
+    Self {
+      priority: m.priority,
+      pattern: OwnedTokTree::v_from_api(m.pattern, sys, &mut slot_panic).unwrap(),
+      template: OwnedTokTree::v_from_api(m.template, sys, &mut slot_panic).unwrap(),
+    }
+  }
+}
+
+#[derive(Debug)]
+pub struct LazyMemberHandle(TreeId, System);
+impl LazyMemberHandle {
+  pub fn run(self) -> OwnedResult<OMemKind> {
+    match self.1.get_tree(self.0) {
+      MemberKind::Const(c) => Ok(OMemKind::Const(RtExpr::from_api(c, &self.1))),
+      MemberKind::Module(m) => Ok(OMemKind::Mod(OwnedModule::from_api(m, &self.1))),
+      MemberKind::Lazy(id) => Self(id, self.1).run()
+    }
+  }
+}

orchid-std/Cargo.toml

@@ -5,9 +5,11 @@ edition = "2021"
 
 [dependencies]
 itertools = "0.13.0"
+never = "0.1.0"
 once_cell = "1.19.0"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-derive = { version = "0.1.0", path = "../orchid-api-derive" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 orchid-extension = { version = "0.1.0", path = "../orchid-extension" }
+ordered-float = "4.2.1"

orchid-std/src/lib.rs

@@ -1,3 +1,4 @@
+mod number;
 mod std;
 mod string;
 

orchid-std/src/number/mod.rs

@@ -0,0 +1,2 @@
+pub mod num_atom;
+pub mod num_lexer;

orchid-std/src/number/num_atom.rs

@@ -0,0 +1,61 @@
+use never::Never;
+use orchid_api_derive::Coding;
+use orchid_extension::atom::{Atomic, ReqPck, TypAtom};
+use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
+use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::error::{pack_err, ProjectResult};
+use orchid_extension::expr::{ExprHandle, GenExpr};
+use orchid_extension::system::SysCtx;
+use ordered_float::NotNan;
+
+#[derive(Clone, Debug, Coding)]
+pub struct Int(pub i64);
+impl Atomic for Int {
+  type Variant = ThinVariant;
+  type Data = Self;
+  type Req = Never;
+}
+impl ThinAtom for Int {
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+}
+impl TryFromExpr for Int {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    TypAtom::<Int>::try_from_expr(expr).map(|t| t.value)
+  }
+}
+
+#[derive(Clone, Debug, Coding)]
+pub struct Float(pub NotNan<f64>);
+impl Atomic for Float {
+  type Variant = ThinVariant;
+  type Data = Self;
+  type Req = Never;
+}
+impl ThinAtom for Float {
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+}
+impl TryFromExpr for Float {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    TypAtom::<Float>::try_from_expr(expr).map(|t| t.value)
+  }
+}
+
+pub enum Numeric {
+  Int(i64),
+  Float(NotNan<f64>),
+}
+impl TryFromExpr for Numeric {
+  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+    Int::try_from_expr(expr.clone()).map(|t| Numeric::Int(t.0)).or_else(|e| {
+      Float::try_from_expr(expr).map(|t| Numeric::Float(t.0)).map_err(|e2| pack_err([e, e2]))
+    })
+  }
+}
+impl ToExpr for Numeric {
+  fn to_expr(self) -> GenExpr {
+    match self {
+      Self::Float(f) => Float(f).to_expr(),
+      Self::Int(i) => Int(i).to_expr(),
+    }
+  }
+}

orchid-std/src/number/num_lexer.rs

@@ -0,0 +1,44 @@
+use std::ops::RangeInclusive;
+
+use orchid_base::location::Pos;
+use orchid_base::number::{parse_num, NumError, NumErrorKind, Numeric};
+use orchid_extension::atom::AtomicFeatures;
+use orchid_extension::error::{ProjectError, ProjectResult};
+use orchid_extension::lexer::{LexContext, Lexer};
+use orchid_extension::tree::{GenTok, GenTokTree};
+use ordered_float::NotNan;
+
+use super::num_atom::{Float, Int};
+
+struct NumProjError(u32, NumError);
+impl ProjectError for NumProjError {
+  const DESCRIPTION: &'static str = "Failed to parse number";
+  fn message(&self) -> String {
+    match self.1.kind {
+      NumErrorKind::InvalidDigit => "This character is not meaningful in this base",
+      NumErrorKind::NaN => "Number somehow evaluated to NaN",
+      NumErrorKind::Overflow => "Number literal overflowed its enclosing type",
+    }
+    .to_string()
+  }
+  fn one_position(&self) -> Pos {
+    Pos::Range(self.0 + self.1.range.start as u32..self.0 + self.1.range.end as u32)
+  }
+}
+
+#[derive(Default)]
+pub struct NumLexer;
+impl Lexer for NumLexer {
+  const CHAR_FILTER: &'static [RangeInclusive<char>] = &['0'..='9'];
+  fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> ProjectResult<(&'a str, GenTokTree)> {
+    let ends_at = all.find(|c: char| !c.is_ascii_hexdigit() && !"xX._pP".contains(c));
+    let (chars, tail) = all.split_at(ends_at.unwrap_or(all.len()));
+    let fac = match parse_num(chars) {
+      Ok(Numeric::Float(f)) => Float(f).factory(),
+      Ok(Numeric::Uint(uint)) => Int(uint.try_into().unwrap()).factory(),
+      Ok(Numeric::Decimal(dec)) => Float(NotNan::new(dec.try_into().unwrap()).unwrap()).factory(),
+      Err(e) => return Err(NumProjError(ctx.pos(all), e).pack()),
+    };
+    Ok((tail, GenTok::Atom(fac).at(ctx.pos(all)..ctx.pos(tail))))
+  }
+}

orchid-std/src/std.rs

@@ -1,14 +1,16 @@
 use std::sync::Arc;
 
+use orchid_base::interner::Tok;
 use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
 use orchid_extension::fs::DeclFs;
 use orchid_extension::fun::Fun;
 use orchid_extension::system::{System, SystemCard};
 use orchid_extension::system_ctor::SystemCtor;
-use orchid_extension::tree::GenTree;
+use orchid_extension::tree::{cnst, module, root_mod, GenMemberKind};
 
-use crate::string::str_atom::StringAtom;
+use crate::number::num_atom::{Float, Int};
-use crate::string::str_leer::StringLexer;
+use crate::string::str_atom::StrAtom;
+use crate::string::str_lexer::StringLexer;
 use crate::OrcString;
 
 #[derive(Default)]
@@ -22,25 +24,23 @@ impl SystemCtor for StdSystem {
 }
 impl SystemCard for StdSystem {
   type Ctor = Self;
-  const ATOM_DEFS: &'static [Option<&'static dyn AtomDynfo>] = &[Some(StringAtom::INFO)];
+  const ATOM_DEFS: &'static [Option<&'static dyn AtomDynfo>] =
+    &[Some(Int::INFO), Some(Float::INFO), Some(StrAtom::INFO)];
 }
 impl System for StdSystem {
   fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer] }
   fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
-  fn env() -> GenTree {
+  fn env() -> Vec<(Tok<String>, GenMemberKind)> {
-    GenTree::module([(
+    vec![
-      "std",
+      root_mod("std", [], [
-      GenTree::module([(
+        module(true, "string", [], [
-        "string",
+          cnst(true, "concat", Fun::new(|left: OrcString| {
-        GenTree::module([(
-          "concat",
-          GenTree::cnst(Fun::new(|left: OrcString| {
             Fun::new(move |right: OrcString| {
-              StringAtom::new(Arc::new(left.get_string().to_string() + &right.get_string()))
+              StrAtom::new(Arc::new(left.get_string().to_string() + &right.get_string()))
             })
-          })),
+          }))
-        )]),
+        ]),
-      )]),
+      ])
-    )])
+    ]
   }
 }

orchid-std/src/string/mod.rs

@@ -1,2 +1,2 @@
 pub mod str_atom;
-pub mod str_leer;
+pub mod str_lexer;

orchid-std/src/string/str_atom.rs

@@ -2,93 +2,87 @@ use std::borrow::Cow;
 use std::num::NonZeroU64;
 use std::sync::Arc;
 
+use never::Never;
 use orchid_api::interner::TStr;
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Encode, Request};
 use orchid_base::id_store::IdStore;
 use orchid_base::interner::{deintern, Tok};
 use orchid_base::location::Pos;
-use orchid_extension::atom::{Atomic, TypAtom};
+use orchid_extension::atom::{Atomic, ReqPck, TypAtom};
 use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
 use orchid_extension::conv::TryFromExpr;
 use orchid_extension::error::{ProjectError, ProjectResult};
-use orchid_extension::expr::{ExprHandle, OwnedExpr};
+use orchid_extension::expr::ExprHandle;
-use orchid_extension::system::{downcast_atom, SysCtx};
+use orchid_extension::system::SysCtx;
 
 pub static STR_REPO: IdStore<Arc<String>> = IdStore::new();
 
-#[derive(Clone, Coding)]
-pub(crate) enum StringVal {
-  Val(NonZeroU64),
-  Int(TStr),
-}
 #[derive(Copy, Clone, Coding)]
-pub(crate) struct StringGetVal;
+pub struct StringGetVal;
 impl Request for StringGetVal {
   type Response = String;
 }
 
-pub(crate) enum StringAtom {
+pub struct StrAtom(NonZeroU64);
-  Val(NonZeroU64),
+impl Atomic for StrAtom {
-  Int(Tok<String>),
-}
-impl Atomic for StringAtom {
   type Variant = OwnedVariant;
-  type Data = StringVal;
+  type Data = NonZeroU64;
   type Req = StringGetVal;
 }
-impl StringAtom {
+impl StrAtom {
-  pub(crate) fn new_int(tok: Tok<String>) -> Self { Self::Int(tok) }
+  pub fn new(str: Arc<String>) -> Self { Self(STR_REPO.add(str).id()) }
-  pub(crate) fn new(str: Arc<String>) -> Self { Self::Val(STR_REPO.add(str).id()) }
 }
-impl Clone for StringAtom {
+impl Clone for StrAtom {
-  fn clone(&self) -> Self {
+  fn clone(&self) -> Self { Self(STR_REPO.add(STR_REPO.get(self.0).unwrap().clone()).id()) }
-    match &self {
-      Self::Int(t) => Self::Int(t.clone()),
-      Self::Val(v) => Self::Val(STR_REPO.add(STR_REPO.get(*v).unwrap().clone()).id()),
-    }
-  }
 }
-impl StringAtom {
+impl StrAtom {
-  fn try_local_value(&self) -> Option<Arc<String>> {
+  fn try_local_value(&self) -> Option<Arc<String>> { STR_REPO.get(self.0).map(|r| r.clone()) }
-    match self {
+  fn local_value(&self) -> Arc<String> { self.try_local_value().expect("no string found for ID") }
-      Self::Int(tok) => Some(tok.arc()),
-      Self::Val(id) => STR_REPO.get(*id).map(|r| r.clone()),
-    }
-  }
-  fn get_value(&self) -> Arc<String> { self.try_local_value().expect("no string found for ID") }
 }
-impl OwnedAtom for StringAtom {
+impl OwnedAtom for StrAtom {
-  fn val(&self) -> Cow<'_, Self::Data> {
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0) }
-    Cow::Owned(match self {
+  fn same(&self, _ctx: SysCtx, other: &Self) -> bool { self.local_value() == other.local_value() }
-      Self::Int(tok) => StringVal::Int(tok.marker()),
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) {
-      Self::Val(id) => StringVal::Val(*id),
+    self.local_value().encode(pck.unpack().1)
-    })
-  }
-  fn same(&self, _ctx: SysCtx, other: &Self) -> bool { self.get_value() == other.get_value() }
-  fn handle_req(
-    &self,
-    _ctx: SysCtx,
-    StringGetVal: Self::Req,
-    rep: &mut (impl std::io::Write + ?Sized),
-  ) {
-    self.get_value().encode(rep)
   }
 }
 
+#[derive(Debug, Clone)]
+pub struct IntStrAtom(Tok<String>);
+impl Atomic for IntStrAtom {
+  type Variant = OwnedVariant;
+  type Data = TStr;
+  type Req = Never;
+}
+impl From<Tok<String>> for IntStrAtom {
+  fn from(value: Tok<String>) -> Self { Self(value) }
+}
+impl OwnedAtom for IntStrAtom {
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.marker()) }
+  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+}
+
 #[derive(Clone)]
-pub struct OrcString(TypAtom<StringAtom>);
+pub enum OrcString {
+  Val(TypAtom<StrAtom>),
+  Int(Tok<String>),
+}
 impl OrcString {
   pub fn get_string(&self) -> Arc<String> {
-    match &self.0.value {
+    match &self {
-      StringVal::Int(tok) => deintern(*tok).arc(),
+      Self::Int(tok) => tok.arc(),
-      StringVal::Val(id) => match STR_REPO.get(*id) {
+      Self::Val(atom) => match STR_REPO.get(**atom) {
         Some(rec) => rec.clone(),
-        None => Arc::new(self.0.request(StringGetVal)),
+        None => Arc::new(atom.request(StringGetVal)),
       },
     }
   }
 }
+impl From<Tok<String>> for OrcString {
+  fn from(value: Tok<String>) -> Self { OrcString::Int(value) }
+}
+
 pub struct NotString(Pos);
 impl ProjectError for NotString {
   const DESCRIPTION: &'static str = "A string was expected";
@@ -96,9 +90,12 @@ impl ProjectError for NotString {
 }
 impl TryFromExpr for OrcString {
   fn try_from_expr(expr: ExprHandle) -> ProjectResult<OrcString> {
-    (OwnedExpr::new(expr).foreign_atom().map_err(|expr| expr.pos.clone()))
+    if let Ok(v) = TypAtom::<StrAtom>::downcast(expr.clone()) {
-      .and_then(|fatom| downcast_atom(fatom).map_err(|f| f.pos))
+      return Ok(OrcString::Val(v));
-      .map_err(|p| NotString(p).pack())
+    }
-      .map(OrcString)
+    match TypAtom::<IntStrAtom>::downcast(expr) {
+      Ok(t) => Ok(OrcString::Int(deintern(*t))),
+      Err(e) => Err(NotString(e.0).pack()),
+    }
   }
 }

orchid-std/src/string/str_lexer.rs

@@ -1,14 +1,14 @@
+use orchid_base::intern;
 use itertools::Itertools;
 use orchid_base::interner::intern;
 use orchid_base::location::Pos;
-use orchid_base::name::VName;
 use orchid_base::vname;
 use orchid_extension::atom::AtomicFeatures;
-use orchid_extension::error::{ErrorSansOrigin, ProjectErrorObj, ProjectResult};
+use orchid_extension::error::{ErrorSansOrigin, ProjectError, ProjectErrorObj, ProjectResult};
-use orchid_extension::lexer::{LexContext, Lexer};
+use orchid_extension::lexer::{LexContext, Lexer, NotApplicableLexerError};
-use orchid_extension::tree::{wrap_tokv, OwnedTok, OwnedTokTree};
+use orchid_extension::tree::{wrap_tokv, GenTok, GenTokTree};
 
-use super::str_atom::StringAtom;
+use super::str_atom::IntStrAtom;
 
 /// Reasons why [parse_string] might fail. See [StringError]
 #[derive(Clone)]
@@ -117,55 +117,48 @@ pub struct StringLexer;
 impl Lexer for StringLexer {
   const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"'];
   fn lex<'a>(
-    full_string: &'a str,
+    all: &'a str,
     ctx: &'a LexContext<'a>,
-  ) -> Option<ProjectResult<(&'a str, OwnedTokTree)>> {
+  ) -> ProjectResult<(&'a str, GenTokTree)> {
-    full_string.strip_prefix('"').map(|mut tail| {
+    let mut tail = all.strip_prefix('"').ok_or_else(|| NotApplicableLexerError.pack())?;
-      let mut parts = vec![];
+    let mut parts = vec![];
-      let mut cur = String::new();
+    let mut cur = String::new();
-      let commit_str = |str: &mut String, tail: &str, parts: &mut Vec<OwnedTokTree>| {
+    let mut errors = vec![];
-        let str_val = parse_string(str)
+    let commit_str = |str: &mut String, tail: &str, err: &mut Vec<ProjectErrorObj>, parts: &mut Vec<GenTokTree>| {
-          .inspect_err(|e| ctx.report(e.clone().into_proj(ctx.pos(tail) - str.len() as u32)))
+      let str_val = parse_string(str)
-          .unwrap_or_default();
+        .inspect_err(|e| err.push(e.clone().into_proj(ctx.pos(tail) - str.len() as u32)))
-        let tok = OwnedTok::Atom(StringAtom::new_int(intern(&str_val)).factory());
+        .unwrap_or_default();
-        parts.push(tok.at(ctx.tok_ran(str.len() as u32, tail)));
+      let tok = GenTok::Atom(IntStrAtom::from(intern(&*str_val)).factory());
-        *str = String::new();
+      parts.push(tok.at(ctx.tok_ran(str.len() as u32, tail)));
-      };
+      *str = String::new();
-      loop {
+    };
-        if let Some(rest) = tail.strip_prefix('"') {
+    loop {
-          commit_str(&mut cur, tail, &mut parts);
+      if let Some(rest) = tail.strip_prefix('"') {
-          return Ok((rest, wrap_tokv(parts, ctx.pos(full_string)..ctx.pos(rest))));
+        commit_str(&mut cur, tail, &mut errors, &mut parts);
-        } else if let Some(rest) = tail.strip_prefix('$') {
+        return Ok((rest, wrap_tokv(parts, ctx.pos(all)..ctx.pos(rest))));
-          commit_str(&mut cur, tail, &mut parts);
+      } else if let Some(rest) = tail.strip_prefix('$') {
-          parts.push(OwnedTok::Name(VName::literal("++")).at(ctx.tok_ran(1, rest)));
+        commit_str(&mut cur, tail, &mut errors, &mut parts);
-          parts.push(OwnedTok::Name(vname!(std::string::convert)).at(ctx.tok_ran(1, rest)));
+        parts.push(GenTok::Name(intern!(str: "++")).at(ctx.tok_ran(1, rest)));
-          match ctx.recurse(rest) {
+        parts.extend(GenTok::vname(&vname!(std::string::convert))
-            Ok((new_tail, tree)) => {
+          .map(|t| t.at(ctx.tok_ran(1, rest))));
-              tail = new_tail;
+        let (new_tail, tree) = ctx.recurse(rest)?;
-              parts.push(tree);
+        tail = new_tail;
-            },
+        parts.push(tree);
-            Err(e) => {
+      } else if tail.starts_with('\\') {
-              ctx.report(e.clone());
+        // parse_string will deal with it, we just have to make sure we skip the next
-              return Ok(("", wrap_tokv(parts, ctx.pos(full_string)..ctx.pos(rest))));
+        // char
-            },
+        tail = &tail[2..];
-          }
+      } else {
-        } else if tail.starts_with('\\') {
+        let mut ch = tail.chars();
-          // parse_string will deal with it, we just have to make sure we skip the next
+        if let Some(c) = ch.next() {
-          // char
+          cur.push(c);
-          tail = &tail[2..];
+          tail = ch.as_str();
         } else {
-          let mut ch = tail.chars();
+          let range = ctx.pos(all)..ctx.pos("");
-          if let Some(c) = ch.next() {
+          commit_str(&mut cur, tail, &mut errors, &mut parts);
-            cur.push(c);
+          return Err(NoStringEnd.bundle(&Pos::Range(range.clone())));
-            tail = ch.as_str();
-          } else {
-            let range = ctx.pos(full_string)..ctx.pos("");
-            commit_str(&mut cur, tail, &mut parts);
-            ctx.report(NoStringEnd.bundle(&Pos::Range(range.clone())));
-            return Ok(("", wrap_tokv(parts, range)));
-          }
         }
       }
-    })
+    }
   }
 }

orchid.code-workspace

@@ -5,6 +5,9 @@
 		}
 	],
 	"settings": {
+		"editor.rulers": [
+			100 // Important; for accessibility reasons, code cannot be wider than 100ch
+		],
 		"[markdown]": {
 			"editor.unicodeHighlight.ambiguousCharacters": false,
 			"editor.unicodeHighlight.invisibleCharacters": false,
@@ -23,11 +26,6 @@
 			"editor.formatOnSave": true,
 			"editor.formatOnType": true,
 		},
-		"[rust]": {
-			"editor.rulers": [
-				100
-			],
-		},
 		"rust-analyzer.showUnlinkedFileNotification": false,
 		"rust-analyzer.checkOnSave": true,
 		"rust-analyzer.check.command": "clippy",
@@ -42,12 +40,12 @@
 	"extensions": {
 		"recommendations": [
 			"maptz.regionfolder",
-			"serayuzgur.crates",
 			"tamasfe.even-better-toml",
 			"yzhang.markdown-all-in-one",
 			"gruntfuggly.todo-tree",
 			"vadimcn.vscode-lldb",
-			"rust-lang.rust-analyzer"
+			"rust-lang.rust-analyzer",
+			"fill-labs.dependi"
 		]
 	},
 }