New plans for macros

About to move them completely to stdlib

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --quiet --package xtask --"

Cargo.lock

@@ -472,9 +472,9 @@ checksum = "4e28ab1dc35e09d60c2b8c90d12a9a8d9666c876c10a3739a3196db0103b6043"
 
 [[package]]
 name = "lazy_static"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "libc"
@@ -569,6 +569,7 @@ dependencies = [
  "hashbrown 0.14.5",
  "itertools",
  "konst",
+ "lazy_static",
  "never",
  "once_cell",
  "orchid-api",
@@ -594,6 +595,7 @@ dependencies = [
  "orchid-api-traits",
  "orchid-base",
  "ordered-float",
+ "paste",
  "substack",
 ]
 
@@ -1245,6 +1247,13 @@ dependencies = [
  "tap",
 ]
 
+[[package]]
+name = "xtask"
+version = "0.1.0"
+dependencies = [
+ "clap",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.7.32"

Cargo.toml

@@ -10,5 +10,5 @@ members = [
   "orchid-api",
   "orchid-api-derive",
   "orchid-api-traits",
-  "stdio-perftest",
+  "stdio-perftest", "xtask",
 ]

examples/hello-world/main.orc

@@ -1 +1,35 @@
 const main := println "Hello World!" exit_status::success
+
+macro (
+	rule match ...$expr { ...$body } => '(
+		fn::pass (...$expr) \match::value. ...$(
+			fn::pass (quote::split body ';) \cases.
+			fn::pass (list::map cases \x. (
+				fn::pass (quote::split_once x '=>) \pair.
+				tuple::destr pair 2 \req. \handler.
+				fn::pass (macro::run '(match::request (...$key))) \match_res.
+				quote::match '(match::response $decoder (...$bindings)) match_res \match_res_match.
+				fn::pass (option::expect match_res_match "Invalid pattern ${key}") \res_parts.
+				fn::pass (map::get_unwrap res_parts "decoder") \decoder.
+				fn::pass (map::get_unwrap res_parts "bindings") \bindings.
+				fn::pass (quote::to_list bindings) \binding_names.
+				fn::pass (list::rfold handler \tail. \name. '( \ $name . $tail )) \success.
+				'( $decoder $success )
+			)) \case_fns.
+			list::append case_fns '( panic "No cases match" )
+		)
+	)
+)
+
+--[
+	Conceptually, all matches are compared.
+		1. Within a macro, the top rule wins
+		2. If two winning matches are not within the same macro,
+			the one that matches the outermost, first token wins,
+			including tokens that are immediately captured, such that a rule starting with .. is
+			beaten by the same rule parenthesized and any rule starting with a scalar is beaten
+			by the same rule prefixed with a vectorial
+		3. If two winning matches start with the same token, an ambiguity error is raised
+
+	
+]--

notes/comm_arch.md

@@ -4,7 +4,7 @@
 
 - **Protocol definition** plain objects that define nothing beside serialization/deserialization and English descriptions of invariants (`orchid-api`)
 
-- **Active objects** smart objects that represent resources and communicate their modification through the protocol, sorted into 3 categories for asymmetric communication: common/extension/host. Some ext/host logic is defined in `orchid-base` to ensure that other behaviour within it can rely on certain global functionality. ext/host also manage their respective connection state (`orchid-base`, `orchid-extension`, `orchid-host`)
+- **Active objects** smart objects that represent resources and communicate commands and queries through the protocol, sorted into 3 categories for asymmetric communication: common/extension/host. Some ext/host logic is defined in `orchid-base` to ensure that other behaviour within it can rely on certain global functionality. ext/host also manage their respective connection state (`orchid-base`, `orchid-extension`, `orchid-host`)
 
 - **Application** (client, server) binaries that use active objects to implement actual application behaviour (`orcx`, `orchid-std`)
 

notes/expr_refs.md

@@ -0,0 +1,18 @@
+# Reference hierarchy of the host
+
+Reference loops are resource leaks. There are two primary ways to avoid reference loops; a strict hierarchy between types is the easiest. The expression tree uses a less obvious hierarchy where expressions are only able to reference expressions that are older than them.
+
+- Trees reference Constants
+- Constants reference their constituent Expressions
+- Expressions reference Atoms
+- During evaluation, Constants replace their unbound names with Constants
+  - There is a reference cycle here, but it always goes through a Constant.
+    > **todo** A potential fix may be to update all Constants to point to a dummy value before freeing Trees
+- Atoms reference the Systems that implement them
+- Atoms may reference Expressions that are not younger than them
+  - This link is managed by the System but tied to Atom and not System lifecycle
+  - Atoms can technically be applied to themselves, but it's a copying apply so it probably isn't a risk factor
+- Systems reference the Extension that contains them
+- Extensions reference the Port that connects them
+  - The Extension signals the remote peer to disconnect on drop
+  - The port is also referenced in a loose receiver thread, which always eventually tries to find the Extension or polls for ingress so it always eventually exits after the Extension's drop handler is called

orchid-api-derive/src/lib.rs

@@ -25,3 +25,5 @@ pub fn hierarchy(input: TokenStream) -> TokenStream { hierarchy::derive(input) }
 pub fn coding(input: TokenStream) -> TokenStream {
   decode(input.clone()).into_iter().chain(encode(input)).collect()
 }
+// TODO: Figure out how serialize/deserialize can be elegantly implemented
+// consider adding a context argument to encode/decode that can just be forwarded

orchid-api-traits/src/coding.rs

@@ -20,11 +20,6 @@ pub trait Decode {
 pub trait Encode {
   /// Append an instance of the struct to the buffer
   fn encode<W: Write + ?Sized>(&self, write: &mut W);
-  fn enc_vec(&self) -> Vec<u8> {
-    let mut vec = Vec::new();
-    self.encode(&mut vec);
-    vec
-  }
 }
 pub trait Coding: Encode + Decode + Clone {
   fn get_decoder<T>(map: impl Fn(Self) -> T + 'static) -> impl Fn(&mut dyn Read) -> T {
@@ -87,7 +82,7 @@ nonzero_impl!(std::num::NonZeroI32);
 nonzero_impl!(std::num::NonZeroI64);
 nonzero_impl!(std::num::NonZeroI128);
 
-impl<'a, T: Encode + 'a> Encode for &'a T {
+impl<'a, T: Encode + ?Sized> Encode for &'a T {
   fn encode<W: Write + ?Sized>(&self, write: &mut W) { (**self).encode(write) }
 }
 impl<T: Decode + FloatCore> Decode for NotNan<T> {

orchid-api-traits/src/helpers.rs

@@ -16,3 +16,9 @@ pub fn read_exact<R: Read + ?Sized>(read: &mut R, bytes: &'static [u8]) {
   read.read_exact(&mut data).expect("Failed to read bytes");
   assert_eq!(&data, bytes, "Wrong bytes")
 }
+
+pub fn enc_vec(enc: &impl Encode) -> Vec<u8> {
+  let mut vec = Vec::new();
+  enc.encode(&mut vec);
+  vec
+}

orchid-api-traits/src/lib.rs

@@ -4,6 +4,6 @@ mod hierarchy;
 mod relations;
 
 pub use coding::{Coding, Decode, Encode};
-pub use helpers::{encode_enum, read_exact, write_exact};
+pub use helpers::{encode_enum, read_exact, write_exact, enc_vec};
 pub use hierarchy::{Extends, InHierarchy, TLBool, TLFalse, TLTrue, UnderRoot};
 pub use relations::{Channel, MsgSet, Request};

orchid-api-traits/src/relations.rs

@@ -1,8 +1,10 @@
-use super::coding::{Coding, Encode};
+use crate::helpers::enc_vec;
+
+use super::coding::Coding;
 
 pub trait Request: Coding + Sized + Send + 'static {
   type Response: Coding + Send + 'static;
-  fn respond(&self, rep: Self::Response) -> Vec<u8> { rep.enc_vec() }
+  fn respond(&self, rep: Self::Response) -> Vec<u8> { enc_vec(&rep) }
 }
 
 pub trait Channel: 'static {

orchid-api/src/atom.rs

@@ -1,18 +1,24 @@
+use std::num::NonZeroU64;
+
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::error::ProjResult;
+use crate::error::OrcResult;
 use crate::expr::{Expr, ExprTicket};
 use crate::proto::{ExtHostReq, HostExtNotif, HostExtReq};
 use crate::system::SysId;
 
 pub type AtomData = Vec<u8>;
 
+/// Unique ID associated with atoms that have an identity
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
+pub struct AtomId(pub NonZeroU64);
+
 /// An atom owned by an implied system. Usually used in responses from a system.
 /// This has the same semantics as [Atom] except in that the owner is implied.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding)]
 pub struct LocalAtom {
-  pub drop: bool,
+  pub drop: Option<AtomId>,
   pub data: AtomData,
 }
 impl LocalAtom {
@@ -25,10 +31,10 @@ impl LocalAtom {
 pub struct Atom {
   /// Instance ID of the system that created the atom
   pub owner: SysId,
-  /// Indicates whether the owner should be notified when this atom is dropped.
+  /// Indicates how the owner should be notified when this atom is dropped.
   /// Construction is always explicit and atoms are never cloned.
   ///
-  /// Atoms with `drop == false` are also known as trivial, they can be
+  /// Atoms with `drop == None` are also known as trivial, they can be
   /// duplicated and stored with no regard to expression lifetimes. NOTICE
   /// that this only applies to the atom. If it's referenced with an
   /// [ExprTicket], the ticket itself can still expire.
@@ -36,7 +42,7 @@ pub struct Atom {
   /// Notice also that the atoms still expire when the system is dropped, and
   /// are not portable across instances of the same system, so this doesn't
   /// imply that the atom is serializable.
-  pub drop: bool,
+  pub drop: Option<AtomId>,
   /// Data stored in the atom. This could be a key into a map, or the raw data
   /// of the atom if it isn't too big.
   pub data: AtomData,
@@ -60,6 +66,20 @@ impl Request for FinalCall {
   type Response = Expr;
 }
 
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(AtomReq, HostExtReq)]
+pub struct SerializeAtom(pub Atom);
+impl Request for SerializeAtom {
+  type Response = (Vec<u8>, Vec<ExprTicket>);
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+pub struct DeserAtom(pub SysId, pub Vec<u8>, pub Vec<ExprTicket>);
+impl Request for DeserAtom {
+  type Response = Atom;
+}
+
 /// Determine whether two atoms are identical for the purposes of macro
 /// application. If a given atom is never generated by macros or this relation
 /// is difficult to define, the module can return false
@@ -94,7 +114,7 @@ pub enum NextStep {
 #[extends(AtomReq, HostExtReq)]
 pub struct Command(pub Atom);
 impl Request for Command {
-  type Response = ProjResult<NextStep>;
+  type Response = OrcResult<NextStep>;
 }
 
 /// Notification that an atom is being dropped because its associated expression
@@ -102,7 +122,7 @@ impl Request for Command {
 /// flag is false.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(HostExtNotif)]
-pub struct AtomDrop(pub Atom);
+pub struct AtomDrop(pub SysId, pub AtomId);
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(AtomReq, HostExtReq)]
@@ -122,6 +142,7 @@ pub enum AtomReq {
   Fwded(Fwded),
   Command(Command),
   AtomPrint(AtomPrint),
+  SerializeAtom(SerializeAtom),
 }
 impl AtomReq {
   /// Obtain the first [Atom] argument of the request. All requests in this
@@ -133,7 +154,8 @@ impl AtomReq {
       | Self::Command(Command(a))
       | Self::FinalCall(FinalCall(a, ..))
       | Self::Fwded(Fwded(a, ..))
-      | Self::AtomPrint(AtomPrint(a)) => a,
+      | Self::AtomPrint(AtomPrint(a))
+      | Self::SerializeAtom(SerializeAtom(a)) => a,
     }
   }
 }

orchid-api/src/error.rs

@@ -1,18 +1,16 @@
 use std::num::NonZeroU16;
 use std::sync::Arc;
 
-use orchid_api_derive::{Coding, Hierarchy};
+use orchid_api_derive::Coding;
-use orchid_api_traits::Request;
 
 use crate::interner::TStr;
 use crate::location::Location;
-use crate::proto::ExtHostReq;
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
-pub struct ProjErrId(pub NonZeroU16);
+pub struct ErrId(pub NonZeroU16);
 
 #[derive(Clone, Debug, Coding)]
-pub struct ProjErrLocation {
+pub struct ErrLocation {
   /// Description of the relation of this location to the error. If not used,
   /// set to empty string
   pub message: Arc<String>,
@@ -26,7 +24,7 @@ pub struct ProjErrLocation {
 /// For example, file reading produces result::err when the file doesn't exist,
 /// and a bottom if the file name isn't a string.
 #[derive(Clone, Debug, Coding)]
-pub struct ProjErr {
+pub struct OrcError {
   /// General description of the kind of error.
   pub description: TStr,
   /// Specific information about the exact error, preferably containing concrete
@@ -34,22 +32,8 @@ pub struct ProjErr {
   pub message: Arc<String>,
   /// Specific code fragments that have contributed to the emergence of the
   /// error.
-  pub locations: Vec<ProjErrLocation>,
+  pub locations: Vec<ErrLocation>,
 }
 
 /// If this is an [`Err`] then the [`Vec`] must not be empty.
-pub type ProjResult<T> = Result<T, Vec<ProjErr>>;
+pub type OrcResult<T> = Result<T, Vec<OrcError>>;
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ProjErrReq, ExtHostReq)]
-pub struct GetErrorDetails(pub ProjErrId);
-impl Request for GetErrorDetails {
-  type Response = ProjErr;
-}
-
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostReq)]
-#[extendable]
-pub enum ProjErrReq {
-  GetErrorDetails(GetErrorDetails),
-}

orchid-api/src/expr.rs

@@ -3,8 +3,8 @@ use std::num::NonZeroU64;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::atom::{Atom, LocalAtom};
+use crate::atom::Atom;
-use crate::error::ProjErr;
+use crate::error::OrcError;
 use crate::interner::TStrv;
 use crate::location::Location;
 use crate::proto::{ExtHostNotif, ExtHostReq};
@@ -47,7 +47,7 @@ pub struct Release(pub SysId, pub ExprTicket);
 /// [Release].
 #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, Coding, Hierarchy)]
 #[extends(ExprNotif, ExtHostNotif)]
-pub struct Relocate {
+pub struct Move {
   pub dec: SysId,
   pub inc: SysId,
   pub expr: ExprTicket,
@@ -77,7 +77,7 @@ pub enum Clause {
   /// Because the atom is newly constructed, it also must belong to this system.
   /// For convenience, [SysId::MAX] is also accepted as referring to this
   /// system.
-  NewAtom(LocalAtom),
+  NewAtom(Atom),
   /// An atom, specifically an atom that already exists. This form is only ever
   /// returned from [Inspect], and it's borrowed from the expression being
   /// inspected.
@@ -85,7 +85,7 @@ pub enum Clause {
   /// A reference to a constant
   Const(TStrv),
   /// A static runtime error.
-  Bottom(Vec<ProjErr>),
+  Bottom(Vec<OrcError>),
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -120,5 +120,5 @@ pub enum ExprReq {
 pub enum ExprNotif {
   Acquire(Acquire),
   Release(Release),
-  Relocate(Relocate),
+  Move(Move),
 }

orchid-api/src/interner.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::proto::{ExtHostNotif, ExtHostReq, HostExtReq};
+use crate::proto::{ExtHostReq, HostExtReq};
 
 /// Intern requests sent by the replica to the master. These requests are
 /// repeatable.
@@ -80,16 +80,6 @@ impl Request for Sweep {
   type Response = Retained;
 }
 
-/// A notification from the extension to the host, that the extension would benefit from a sweep
-#[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(ExtHostNotif)]
-pub struct AdviseSweep(SweepReason);
-
-#[derive(Clone, Debug, Coding)]
-pub enum SweepReason {
-  None
-}
-
 /// List of keys in this replica that couldn't be sweeped because local
 /// datastructures reference their value.
 #[derive(Clone, Debug, Coding)]

orchid-api/src/lib.rs

@@ -1,11 +1,35 @@
-pub mod atom;
+mod atom;
-pub mod error;
+pub use atom::{
-pub mod expr;
+  Atom, AtomData, AtomDrop, AtomId, AtomPrint, AtomReq, AtomSame, CallRef, Command, FinalCall, Fwd,
-pub mod interner;
+  Fwded, LocalAtom, NextStep, DeserAtom, SerializeAtom
-pub mod location;
+};
-pub mod parser;
+mod error;
-pub mod proto;
+pub use error::{ErrId, ErrLocation, OrcError, OrcResult};
-pub mod system;
+mod expr;
-pub mod tree;
+pub use expr::{
-pub mod vfs;
+  Acquire, Clause, Details, Expr, ExprNotif, ExprReq, ExprTicket, Inspect, Move, Release,
-pub mod logging;
+};
+mod interner;
+pub use interner::{
+  ExternStr, ExternStrv, IntReq, InternStr, InternStrv, Retained, Sweep, TStr, TStrv,
+};
+mod location;
+pub use location::{CodeGenInfo, Location, SourceRange};
+mod logging;
+pub use logging::{Log, LogStrategy};
+mod parser;
+pub use parser::{CharFilter, LexExpr, LexedExpr, ParsId, ParseLine, ParserReq, SubLex, SubLexed};
+mod proto;
+pub use proto::{
+  ExtHostChannel, ExtHostNotif, ExtHostReq, ExtMsgSet, ExtensionHeader, HostExtChannel,
+  HostExtNotif, HostExtReq, HostHeader, HostMsgSet, Ping,
+};
+mod system;
+pub use system::{SysReq, NewSystem, SysDeclId, SysId, SystemDecl, SystemDrop, SystemInst};
+mod tree;
+pub use tree::{
+  CompName, GetMember, Item, ItemKind, Macro, Member, MemberKind, Module, Paren, Placeholder,
+  PlaceholderKind, Token, TokenTree, TreeId, TreeTicket,
+};
+mod vfs;
+pub use vfs::{EagerVfs, GetVfs, Loaded, VfsId, VfsRead, VfsReq};

orchid-api/src/logging.rs

@@ -5,7 +5,7 @@ use crate::proto::ExtHostNotif;
 #[derive(Clone, Debug, Coding)]
 pub enum LogStrategy {
   StdErr,
-  File(String)
+  File(String),
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]

orchid-api/src/parser.rs

@@ -4,7 +4,7 @@ use std::ops::RangeInclusive;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::error::ProjResult;
+use crate::error::OrcResult;
 use crate::interner::TStr;
 use crate::proto::{ExtHostReq, HostExtReq};
 use crate::system::SysId;
@@ -24,6 +24,7 @@ pub struct CharFilter(pub Vec<RangeInclusive<char>>);
 #[extendable]
 pub enum ParserReq {
   LexExpr(LexExpr),
+  ParseLine(ParseLine),
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
@@ -35,7 +36,7 @@ pub struct LexExpr {
   pub pos: u32,
 }
 impl Request for LexExpr {
-  type Response = Option<ProjResult<LexedExpr>>;
+  type Response = Option<OrcResult<LexedExpr>>;
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -60,11 +61,12 @@ pub struct SubLexed {
   pub ticket: TreeTicket,
 }
 
-#[derive(Clone, Debug, Coding)]
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(ParserReq, HostExtReq)]
 pub struct ParseLine {
   pub sys: SysId,
   pub line: Vec<TokenTree>,
 }
 impl Request for ParseLine {
-  type Response = Vec<TokenTree>;
+  type Response = OrcResult<Vec<TokenTree>>;
 }

orchid-api/src/proto.rs

@@ -27,7 +27,8 @@ use std::io::{Read, Write};
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::{read_exact, write_exact, Channel, Decode, Encode, MsgSet, Request};
 
-use crate::{atom, error, expr, interner, logging::{self, LogStrategy}, parser, system, tree, vfs};
+use crate::logging::{self, LogStrategy};
+use crate::{atom, expr, interner, parser, system, tree, vfs};
 
 static HOST_INTRO: &[u8] = b"Orchid host, binary API v0\n";
 pub struct HostHeader {
@@ -48,17 +49,19 @@ impl Encode for HostHeader {
 
 static EXT_INTRO: &[u8] = b"Orchid extension, binary API v0\n";
 pub struct ExtensionHeader {
+  pub name: String,
   pub systems: Vec<system::SystemDecl>,
 }
 impl Decode for ExtensionHeader {
   fn decode<R: Read + ?Sized>(read: &mut R) -> Self {
     read_exact(read, EXT_INTRO);
-    Self { systems: Vec::decode(read) }
+    Self { name: String::decode(read), systems: Vec::decode(read) }
   }
 }
 impl Encode for ExtensionHeader {
   fn encode<W: Write + ?Sized>(&self, write: &mut W) {
     write_exact(write, EXT_INTRO);
+    self.name.encode(write);
     self.systems.encode(write)
   }
 }
@@ -78,7 +81,6 @@ pub enum ExtHostReq {
   Fwd(atom::Fwd),
   ExprReq(expr::ExprReq),
   SubLex(parser::SubLex),
-  ProjErrReq(error::ProjErrReq),
 }
 
 /// Notifications sent from the extension to the host
@@ -87,7 +89,6 @@ pub enum ExtHostReq {
 #[extendable]
 pub enum ExtHostNotif {
   ExprNotif(expr::ExprNotif),
-  AdviseSweep(interner::AdviseSweep),
   Log(logging::Log),
 }
 
@@ -102,9 +103,10 @@ impl Channel for ExtHostChannel {
 #[extendable]
 pub enum HostExtReq {
   Ping(Ping),
-  NewSystem(system::NewSystem),
+  SysReq(system::SysReq),
   Sweep(interner::Sweep),
   AtomReq(atom::AtomReq),
+  DeserAtom(atom::DeserAtom),
   ParserReq(parser::ParserReq),
   GetMember(tree::GetMember),
   VfsReq(vfs::VfsReq),
@@ -143,6 +145,7 @@ impl MsgSet for HostMsgSet {
 
 #[cfg(test)]
 mod tests {
+  use orchid_api_traits::enc_vec;
 use ordered_float::NotNan;
   use system::{SysDeclId, SystemDecl};
 
@@ -151,7 +154,7 @@ mod tests {
   #[test]
   fn host_header_enc() {
     let hh = HostHeader { log_strategy: LogStrategy::File("SomeFile".to_string()) };
-      let mut enc = &hh.enc_vec()[..];
+    let mut enc = &enc_vec(&hh)[..];
     eprintln!("Encoded to {enc:?}");
     HostHeader::decode(&mut enc);
     assert_eq!(enc, []);
@@ -160,16 +163,15 @@ mod tests {
   #[test]
   fn ext_header_enc() {
     let eh = ExtensionHeader {
-        systems: vec![
+      name: "my_extension".to_string(),
-          SystemDecl {
+      systems: vec![SystemDecl {
         id: SysDeclId(1.try_into().unwrap()),
         name: "misc".to_string(),
         depends: vec!["std".to_string()],
-            priority: NotNan::new(1f64).unwrap()
+        priority: NotNan::new(1f64).unwrap(),
-          }
+      }],
-        ]
     };
-      let mut enc = &eh.enc_vec()[..];
+    let mut enc = &enc_vec(&eh)[..];
     eprintln!("Encoded to {enc:?}");
     ExtensionHeader::decode(&mut enc);
     assert_eq!(enc, [])

orchid-api/src/system.rs

@@ -18,8 +18,8 @@ pub struct SysDeclId(pub NonZeroU16);
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct SysId(pub NonZeroU16);
 
-/// Details about a system provided by this library. This is included in the extension header,
+/// Details about a system provided by this library. This is included in the
-/// so it cannot rely on the interner. 
+/// extension header, so it cannot rely on the interner.
 #[derive(Debug, Clone, Coding)]
 pub struct SystemDecl {
   /// ID of the system, unique within the library
@@ -44,7 +44,7 @@ pub struct SystemDecl {
 /// essential that any resource associated with a system finds its system by the
 /// ID in a global map.
 #[derive(Clone, Debug, Coding, Hierarchy)]
-#[extends(HostExtReq)]
+#[extends(SysReq, HostExtReq)]
 pub struct NewSystem {
   /// ID of the system
   pub system: SysDeclId,
@@ -64,10 +64,17 @@ pub struct SystemInst {
   /// can process. The lexer will notify this system if it encounters one of
   /// these characters.9
   pub lex_filter: CharFilter,
-  pub parses_lines: Vec<TStr>,
+  pub line_types: Vec<TStr>,
   pub const_root: HashMap<TStr, MemberKind>,
 }
 
 #[derive(Clone, Debug, Coding, Hierarchy)]
 #[extends(HostExtNotif)]
 pub struct SystemDrop(pub SysId);
+
+#[derive(Clone, Debug, Coding, Hierarchy)]
+#[extends(HostExtReq)]
+#[extendable]
+pub enum SysReq {
+  NewSystem(NewSystem),
+}

orchid-api/src/tree.rs

@@ -1,18 +1,17 @@
-use crate::interner::TStrv;
-use crate::location::Location;
 use std::num::NonZeroU64;
 use std::ops::Range;
+use std::sync::Arc;
 
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 use ordered_float::NotNan;
 
-use crate::atom::LocalAtom;
+use crate::error::OrcError;
-use crate::error::ProjErr;
+use crate::interner::{TStr, TStrv};
-use crate::expr::Expr;
+use crate::location::Location;
-use crate::interner::TStr;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
+use crate::{Atom, Expr};
 
 /// A token tree from a lexer recursion request. Its lifetime is the lex call,
 /// the lexer can include it in its output or discard it by implication.
@@ -46,12 +45,14 @@ pub enum Token {
   /// line parser output
   Ph(Placeholder),
   /// A new atom
-  Atom(LocalAtom),
+  Atom(Atom),
   /// Anchor to insert a subtree
   Slot(TreeTicket),
   /// A static compile-time error returned by failing lexers if
   /// the rest of the source is likely still meaningful
-  Bottom(Vec<ProjErr>),
+  Bottom(Vec<OrcError>),
+  /// A comment
+  Comment(Arc<String>),
 }
 
 #[derive(Clone, Debug, Coding)]
@@ -84,10 +85,10 @@ pub struct Macro {
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Coding)]
 pub struct TreeId(pub NonZeroU64);
 
-
 #[derive(Clone, Debug, Coding)]
 pub struct Item {
   pub location: Location,
+  pub comments: Vec<(Arc<String>, Location)>,
   pub kind: ItemKind,
 }
 
@@ -95,13 +96,15 @@ pub struct Item {
 pub enum ItemKind {
   Member(Member),
   Raw(Vec<TokenTree>),
+  Export(TStr),
   Rule(Macro),
+  Import(CompName),
 }
 
 #[derive(Clone, Debug, Coding)]
 pub struct Member {
   pub name: TStr,
-  pub public: bool,
+  pub exported: bool,
   pub kind: MemberKind,
 }
 
@@ -118,6 +121,13 @@ pub struct Module {
   pub items: Vec<Item>,
 }
 
+#[derive(Clone, Debug, Coding)]
+pub struct CompName {
+  pub path: Vec<TStr>,
+  pub name: Option<TStr>,
+  pub location: Location,
+}
+
 #[derive(Clone, Copy, Debug, Coding, Hierarchy)]
 #[extends(HostExtReq)]
 pub struct GetMember(pub SysId, pub TreeId);

orchid-api/src/vfs.rs

@@ -4,7 +4,7 @@ use std::num::NonZeroU16;
 use orchid_api_derive::{Coding, Hierarchy};
 use orchid_api_traits::Request;
 
-use crate::error::ProjResult;
+use crate::error::OrcResult;
 use crate::interner::TStr;
 use crate::proto::HostExtReq;
 use crate::system::SysId;
@@ -22,7 +22,7 @@ pub enum Loaded {
 #[extends(VfsReq, HostExtReq)]
 pub struct VfsRead(pub SysId, pub VfsId, pub Vec<TStr>);
 impl Request for VfsRead {
-  type Response = ProjResult<Loaded>;
+  type Response = OrcResult<Loaded>;
 }
 
 #[derive(Clone, Debug, Coding)]

orchid-base/src/char_filter.rs

@@ -1,7 +1,9 @@
-use std::{fmt, ops::RangeInclusive};
+use std::fmt;
+use std::ops::RangeInclusive;
 
 use itertools::Itertools;
-use orchid_api::parser::CharFilter;
+
+use crate::api;
 
 pub type CRange = RangeInclusive<char>;
 
@@ -11,7 +13,7 @@ pub trait ICFilter: fmt::Debug {
 impl ICFilter for [RangeInclusive<char>] {
   fn ranges(&self) -> &[RangeInclusive<char>] { self }
 }
-impl ICFilter for CharFilter {
+impl ICFilter for api::CharFilter {
   fn ranges(&self) -> &[RangeInclusive<char>] { &self.0 }
 }
 
@@ -24,8 +26,8 @@ fn try_merge_char_ranges(left: CRange, right: CRange) -> Result<CRange, (CRange,
 
 /// Process the character ranges to make them adhere to the structural
 /// requirements of [CharFilter]
-pub fn mk_char_filter(items: impl IntoIterator<Item = CRange>) -> CharFilter {
+pub fn mk_char_filter(items: impl IntoIterator<Item = CRange>) -> api::CharFilter {
-  CharFilter(
+  api::CharFilter(
     (items.into_iter())
       .filter(|r| *r.start() as u32 <= *r.end() as u32)
       .sorted_by_key(|r| *r.start() as u32)
@@ -39,14 +41,18 @@ pub fn char_filter_match(cf: &(impl ICFilter + ?Sized), c: char) -> bool {
   match cf.ranges().binary_search_by_key(&c, |l| *l.end()) {
     Ok(_) => true,                             // c is the end of a range
     Err(i) if i == cf.ranges().len() => false, // all ranges end before c
-    Err(i) => cf.ranges()[i].contains(&c),     // c between cf.0[i-1]?.end and cf.0[i].end, check [i]
+    Err(i) => cf.ranges()[i].contains(&c),     /* c between cf.0[i-1]?.end and cf.0[i].end,
+                                                 * check [i] */
   }
 }
 
 /// Merge two char filters into a filter that matches if either of the
 /// constituents would match.
-pub fn char_filter_union(l: &(impl ICFilter + ?Sized), r: &(impl ICFilter + ?Sized)) -> CharFilter {
+pub fn char_filter_union(
-  CharFilter(
+  l: &(impl ICFilter + ?Sized),
+  r: &(impl ICFilter + ?Sized),
+) -> api::CharFilter {
+  api::CharFilter(
     (l.ranges().iter().merge_by(r.ranges(), |l, r| l.start() <= r.start()))
       .cloned()
       .coalesce(try_merge_char_ranges)

orchid-base/src/error.rs

@@ -1,28 +1,29 @@
 use std::sync::Arc;
 
-use orchid_api::error::{ProjErr, ProjErrLocation};
+use itertools::Itertools;
 
 use crate::interner::{deintern, Tok};
 use crate::location::Pos;
+use crate::api;
 
 /// A point of interest in resolving the error, such as the point where
 /// processing got stuck, a command that is likely to be incorrect
 #[derive(Clone, Debug)]
-pub struct ErrorPosition {
+pub struct ErrPos {
   /// The suspected origin
   pub position: Pos,
   /// Any information about the role of this origin
   pub message: Option<Arc<String>>,
 }
-impl ErrorPosition {
+impl ErrPos {
-  pub fn from_api(pel: &ProjErrLocation) -> Self {
+  pub fn from_api(pel: &api::ErrLocation) -> Self {
     Self {
       message: Some(pel.message.clone()).filter(|s| !s.is_empty()),
       position: Pos::from_api(&pel.location),
     }
   }
-  pub fn to_api(&self) -> ProjErrLocation {
+  pub fn to_api(&self) -> api::ErrLocation {
-    ProjErrLocation {
+    api::ErrLocation {
       message: self.message.clone().unwrap_or_default(),
       location: self.position.to_api(),
     }
@@ -31,25 +32,62 @@ impl ErrorPosition {
     Self { message: Some(Arc::new(msg.to_string())), position }
   }
 }
-impl From<Pos> for ErrorPosition {
+impl From<Pos> for ErrPos {
   fn from(origin: Pos) -> Self { Self { position: origin, message: None } }
 }
 
 #[derive(Clone, Debug)]
-pub struct OwnedError {
+pub struct OrcErr {
   pub description: Tok<String>,
   pub message: Arc<String>,
-  pub positions: Vec<ErrorPosition>,
+  pub positions: Vec<ErrPos>,
 }
-impl OwnedError {
+impl OrcErr {
-  pub fn from_api(err: &ProjErr) -> Self {
+  pub fn from_api(err: &api::OrcError) -> Self {
     Self {
       description: deintern(err.description),
       message: err.message.clone(),
-      positions: err.locations.iter().map(ErrorPosition::from_api).collect(),
+      positions: err.locations.iter().map(ErrPos::from_api).collect(),
     }
   }
-  pub fn from_apiv(err: Vec<ProjErr>) -> Vec<Self> {
+  pub fn to_api(&self) -> api::OrcError {
-    err.into_iter().map(|e| Self::from_api(&e)).collect()
+    api::OrcError {
+      description: self.description.marker(),
+      message: self.message.clone(),
+      locations: self.positions.iter().map(ErrPos::to_api).collect(),
     }
   }
+}
+impl Eq for OrcErr {}
+impl PartialEq for OrcErr {
+  fn eq(&self, other: &Self) -> bool { self.description == other.description }
+}
+impl From<OrcErr> for Vec<OrcErr> {
+  fn from(value: OrcErr) -> Self { vec![value] }
+}
+
+pub fn errv_to_apiv<'a>(errv: impl IntoIterator<Item = &'a OrcErr>) -> Vec<api::OrcError> {
+  errv.into_iter().map(OrcErr::to_api).collect_vec()
+}
+
+pub fn errv_from_apiv<'a>(err: impl IntoIterator<Item = &'a api::OrcError>) -> Vec<OrcErr> {
+  err.into_iter().map(OrcErr::from_api).collect()
+}
+
+pub type OrcRes<T> = Result<T, Vec<OrcErr>>;
+
+pub fn mk_err(
+  description: Tok<String>,
+  message: impl AsRef<str>,
+  posv: impl IntoIterator<Item = ErrPos>,
+) -> OrcErr {
+  OrcErr {
+    description,
+    message: Arc::new(message.as_ref().to_string()),
+    positions: posv.into_iter().collect(),
+  }
+}
+
+pub trait Reporter {
+  fn report(&self, e: OrcErr);
+}

orchid-base/src/gen/impls.rs

@@ -1,8 +1,7 @@
 use std::any::TypeId;
 
 use itertools::Itertools;
-use orchid_api::expr::{Clause, Expr};
+use crate::api;
-use orchid_api::location::Location;
 
 use super::traits::{GenClause, Generable};
 use crate::intern::{deintern, intern};

orchid-base/src/gen/tpl.rs

@@ -1,7 +1,7 @@
 //! Various elemental components to build expression trees that all implement
 //! [GenClause].
 
-use orchid_api::atom::Atom;
+use crate::api;
 
 use super::traits::{GenClause, Generable};
 

orchid-base/src/gen/traits.rs

@@ -5,7 +5,7 @@ use std::cell::RefCell;
 use std::collections::VecDeque;
 use std::fmt;
 
-use orchid_api::atom::Atom;
+use crate::api;
 
 /// Representations of the Orchid expression tree that can describe basic
 /// language elements.

orchid-base/src/gen/tree.rs

@@ -4,8 +4,7 @@
 use std::fmt;
 
 use dyn_clone::{clone_box, DynClone};
-use orchid_api::atom::Atom;
+use crate::api;
-use orchid_api::expr::Expr;
 use trait_set::trait_set;
 
 use super::tpl;

orchid-base/src/id_store.rs

@@ -25,9 +25,7 @@ impl<T> IdStore<T> {
     if tbl_g.contains_key(&id64) { Some(IdRecord(id64, tbl_g)) } else { None }
   }
   pub fn is_empty(&self) -> bool { self.len() == 0 }
-  pub fn len(&self) -> usize {
+  pub fn len(&self) -> usize { self.table.get().map(|t| t.lock().unwrap().len()).unwrap_or(0) }
-    self.table.get().map(|t| t.lock().unwrap().len()).unwrap_or(0)
-  }
 }
 
 impl<T> Default for IdStore<T> {

orchid-base/src/interner.rs

@@ -7,11 +7,9 @@ use std::{fmt, hash};
 
 use hashbrown::{HashMap, HashSet};
 use itertools::Itertools as _;
-use orchid_api::interner::{
+use orchid_api_traits::{Encode, Decode, Request};
-  ExternStr, ExternStrv, IntReq, InternStr, InternStrv, Retained, TStr, TStrv,
-};
-use orchid_api_traits::Request;
 
+use crate::api;
 use crate::reqnot::{DynRequester, Requester};
 
 /// Clippy crashes while verifying `Tok: Sized` without this and I cba to create
@@ -57,11 +55,21 @@ impl<T: Interned + fmt::Debug> fmt::Debug for Tok<T> {
     write!(f, "Token({} -> {:?})", self.marker().get_id(), self.data.as_ref())
   }
 }
+impl<T: Interned + Encode> Encode for Tok<T> {
+  fn encode<W: std::io::Write + ?Sized>(&self, write: &mut W) { self.data.encode(write) }
+}
+impl<T: Interned + Decode> Decode for Tok<T> {
+  fn decode<R: std::io::Read + ?Sized>(read: &mut R) -> Self {
+      intern(&T::decode(read))
+  }
+}
 
-pub trait Interned: Eq + hash::Hash + Clone {
+pub trait Interned: Eq + hash::Hash + Clone + Internable<Interned = Self> {
   type Marker: InternMarker<Interned = Self> + Sized;
-  fn intern(self: Arc<Self>, req: &(impl DynRequester<Transfer = IntReq> + ?Sized))
+  fn intern(
-  -> Self::Marker;
+    self: Arc<Self>,
+    req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+  ) -> Self::Marker;
   fn bimap(interner: &mut TypedInterners) -> &mut Bimap<Self>;
 }
 
@@ -72,26 +80,32 @@ pub trait Internable {
 
 pub trait InternMarker: Copy + PartialEq + Eq + PartialOrd + Ord + hash::Hash + Sized {
   type Interned: Interned<Marker = Self>;
-  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Tok<Self::Interned>;
+  fn resolve(
+    self,
+    req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+  ) -> Tok<Self::Interned>;
   fn get_id(self) -> NonZeroU64;
   fn from_id(id: NonZeroU64) -> Self;
 }
 
 impl Interned for String {
-  type Marker = TStr;
+  type Marker = api::TStr;
   fn intern(
     self: Arc<Self>,
-    req: &(impl DynRequester<Transfer = IntReq> + ?Sized),
+    req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
   ) -> Self::Marker {
-    req.request(InternStr(self))
+    req.request(api::InternStr(self))
   }
   fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.strings }
 }
 
-impl InternMarker for TStr {
+impl InternMarker for api::TStr {
   type Interned = String;
-  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Tok<Self::Interned> {
+  fn resolve(
-    Tok::new(req.request(ExternStr(self)), self)
+    self,
+    req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+  ) -> Tok<Self::Interned> {
+    Tok::new(req.request(api::ExternStr(self)), self)
   }
   fn get_id(self) -> NonZeroU64 { self.0 }
   fn from_id(id: NonZeroU64) -> Self { Self(id) }
@@ -108,20 +122,24 @@ impl Internable for String {
 }
 
 impl Interned for Vec<Tok<String>> {
-  type Marker = TStrv;
+  type Marker = api::TStrv;
   fn intern(
     self: Arc<Self>,
-    req: &(impl DynRequester<Transfer = IntReq> + ?Sized),
+    req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
   ) -> Self::Marker {
-    req.request(InternStrv(Arc::new(self.iter().map(|t| t.marker()).collect())))
+    req.request(api::InternStrv(Arc::new(self.iter().map(|t| t.marker()).collect())))
   }
   fn bimap(interners: &mut TypedInterners) -> &mut Bimap<Self> { &mut interners.vecs }
 }
 
-impl InternMarker for TStrv {
+impl InternMarker for api::TStrv {
   type Interned = Vec<Tok<String>>;
-  fn resolve(self, req: &(impl DynRequester<Transfer = IntReq> + ?Sized)) -> Tok<Self::Interned> {
+  fn resolve(
-    let data = Arc::new(req.request(ExternStrv(self)).iter().map(|m| deintern(*m)).collect_vec());
+    self,
+    req: &(impl DynRequester<Transfer = api::IntReq> + ?Sized),
+  ) -> Tok<Self::Interned> {
+    let data =
+      Arc::new(req.request(api::ExternStrv(self)).iter().map(|m| deintern(*m)).collect_vec());
     Tok::new(data, self)
   }
   fn get_id(self) -> NonZeroU64 { self.0 }
@@ -138,14 +156,14 @@ impl Internable for Vec<Tok<String>> {
   fn get_owned(&self) -> Arc<Self::Interned> { Arc::new(self.to_vec()) }
 }
 
-impl Internable for Vec<TStr> {
+impl Internable for Vec<api::TStr> {
   type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> {
     Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
   }
 }
 
-impl Internable for [TStr] {
+impl Internable for [api::TStr] {
   type Interned = Vec<Tok<String>>;
   fn get_owned(&self) -> Arc<Self::Interned> {
     Arc::new(self.iter().map(|ts| deintern(*ts)).collect())
@@ -157,7 +175,7 @@ const BASE_RC: usize = 3;
 
 #[test]
 fn base_rc_correct() {
-  let tok = Tok::new(Arc::new("foo".to_string()), TStr(1.try_into().unwrap()));
+  let tok = Tok::new(Arc::new("foo".to_string()), api::TStr(1.try_into().unwrap()));
   let mut bimap = Bimap::default();
   bimap.insert(tok.clone());
   assert_eq!(Arc::strong_count(&tok.data), BASE_RC + 1, "the bimap plus the current instance");
@@ -214,7 +232,7 @@ pub struct TypedInterners {
 #[derive(Default)]
 pub struct Interner {
   interners: TypedInterners,
-  master: Option<Box<dyn DynRequester<Transfer = IntReq>>>,
+  master: Option<Box<dyn DynRequester<Transfer = api::IntReq>>>,
 }
 
 static ID: atomic::AtomicU64 = atomic::AtomicU64::new(1);
@@ -236,7 +254,7 @@ pub fn interner() -> impl DerefMut<Target = Interner> {
   G(g)
 }
 
-pub fn init_replica(req: impl DynRequester<Transfer = IntReq> + 'static) {
+pub fn init_replica(req: impl DynRequester<Transfer = api::IntReq> + 'static) {
   let mut g = INTERNER.lock().unwrap();
   assert!(g.is_none(), "Attempted to initialize replica interner after first use");
   *g = Some(Interner {
@@ -273,15 +291,18 @@ pub fn deintern<M: InternMarker>(marker: M) -> Tok<M::Interned> {
   token
 }
 
-pub fn merge_retained(into: &mut Retained, from: &Retained) {
+pub fn merge_retained(into: &mut api::Retained, from: &api::Retained) {
   into.strings = into.strings.iter().chain(&from.strings).copied().unique().collect();
   into.vecs = into.vecs.iter().chain(&from.vecs).copied().unique().collect();
 }
 
-pub fn sweep_replica() -> Retained {
+pub fn sweep_replica() -> api::Retained {
   let mut g = interner();
   assert!(g.master.is_some(), "Not a replica");
-  Retained { strings: g.interners.strings.sweep_replica(), vecs: g.interners.vecs.sweep_replica() }
+  api::Retained {
+    strings: g.interners.strings.sweep_replica(),
+    vecs: g.interners.vecs.sweep_replica(),
+  }
 }
 
 /// Create a thread-local token instance and copy it. This ensures that the
@@ -301,14 +322,7 @@ macro_rules! intern {
   }};
 }
 
-pub fn sweep_master(retained: Retained) {
+pub fn sweep_master(retained: api::Retained) {
-  eprintln!(
-    "Hello, {:?}",
-    intern!([Tok<String>]: &[
-      intern!(str: "bar"),
-      intern!(str: "baz")
-    ])
-  );
   let mut g = interner();
   assert!(g.master.is_none(), "Not master");
   g.interners.strings.sweep_master(retained.strings.into_iter().collect());
@@ -317,12 +331,12 @@ pub fn sweep_master(retained: Retained) {
 
 #[cfg(test)]
 mod test {
-  use super::*;
-
   use std::num::NonZero;
 
-    use orchid_api::interner::TStr;
+  use orchid_api_traits::{enc_vec, Decode};
-    use orchid_api_traits::{Decode, Encode};
+
+  use super::*;
+  use crate::api;
 
   #[test]
   fn test_i() {
@@ -335,10 +349,9 @@ mod test {
 
   #[test]
   fn test_coding() {
-    let coded = TStr(NonZero::new(3u64).unwrap());
+    let coded = api::TStr(NonZero::new(3u64).unwrap());
-    let mut enc = &coded.enc_vec()[..];
+    let mut enc = &enc_vec(&coded)[..];
-    eprintln!("Coded {enc:?}");
+    api::TStr::decode(&mut enc);
-    TStr::decode(&mut enc);
+    assert_eq!(enc, [], "Did not consume all of {enc:?}")
-    assert_eq!(enc, [])
   }
 }

orchid-base/src/lib.rs

@@ -1,3 +1,5 @@
+use orchid_api as api;
+
 pub mod boxed_iter;
 pub mod clone;
 pub mod combine;
@@ -12,10 +14,11 @@ pub mod id_store;
 pub mod interner;
 pub mod join;
 pub mod location;
+pub mod logging;
 pub mod name;
 pub mod number;
+pub mod parse;
 pub mod reqnot;
 pub mod sequence;
 pub mod tokens;
 pub mod tree;
-pub mod logging;

orchid-base/src/location.rs

@@ -5,12 +5,11 @@ use std::hash::Hash;
 use std::ops::Range;
 use std::sync::Arc;
 
-use orchid_api::location as api;
 use trait_set::trait_set;
 
 use crate::interner::{deintern, Tok};
 use crate::name::Sym;
-use crate::sym;
+use crate::{api, sym};
 
 trait_set! {
   pub trait GetSrc = FnMut(&Sym) -> Tok<String>;

orchid-base/src/logging.rs

@@ -1,16 +1,30 @@
-use std::{fs::File, io::Write};
+use std::fmt::Arguments;
+use std::fs::File;
+use std::io::{stderr, Write};
 
-pub use orchid_api::logging::LogStrategy;
+pub use api::LogStrategy;
+use itertools::Itertools;
 
-pub struct Logger(LogStrategy);
+use crate::api;
+
+#[derive(Clone)]
+pub struct Logger(api::LogStrategy);
 impl Logger {
-  pub fn new(strat: LogStrategy) -> Self { Self(strat) }
+  pub fn new(strat: api::LogStrategy) -> Self { Self(strat) }
-  pub fn log(&self, msg: String) {
+  pub fn log(&self, msg: impl AsRef<str>) { writeln!(self, "{}", msg.as_ref()) }
+  pub fn strat(&self) -> api::LogStrategy { self.0.clone() }
+  pub fn log_buf(&self, event: impl AsRef<str>, buf: &[u8]) {
+    if !std::env::var("ORCHID_LOG_BUFFERS").unwrap().is_empty() {
+      writeln!(self, "{}: [{}]", event.as_ref(), buf.iter().map(|b| format!("{b:02x}")).join(" "))
+    }
+  }
+  pub fn write_fmt(&self, fmt: Arguments) {
     match &self.0 {
-      LogStrategy::StdErr => eprintln!("{msg}"),
+      api::LogStrategy::StdErr => stderr().write_fmt(fmt).expect("Could not write to stderr!"),
-      LogStrategy::File(f) => writeln!(File::open(f).unwrap(), "{msg}").unwrap(),
+      api::LogStrategy::File(f) => {
+        let mut file = File::open(f).expect("Could not open logfile");
+        file.write_fmt(fmt).expect("Could not write to logfile");
+      },
     }
   }
-  pub fn strat(&self) -> LogStrategy { self.0.clone() }
 }
-

orchid-base/src/msg.rs

@@ -1,7 +1,6 @@
-use orchid_api_traits::Decode;
 use std::io;
 
-use orchid_api_traits::Encode;
+use orchid_api_traits::{Decode, Encode};
 
 pub fn send_msg(write: &mut impl io::Write, msg: &[u8]) -> io::Result<()> {
   u32::try_from(msg.len()).unwrap().encode(write);

orchid-base/src/name.rs

@@ -4,14 +4,15 @@ use std::borrow::Borrow;
 use std::hash::Hash;
 use std::iter::Cloned;
 use std::num::{NonZeroU64, NonZeroUsize};
-use std::ops::{Deref, Index};
+use std::ops::{Bound, Deref, Index, RangeBounds};
 use std::path::Path;
 use std::{fmt, slice, vec};
 
 use itertools::Itertools;
-use orchid_api::interner::TStr;
+use orchid_api::TStrv;
 use trait_set::trait_set;
 
+use crate::api;
 use crate::interner::{deintern, intern, InternMarker, Tok};
 
 trait_set! {
@@ -40,11 +41,11 @@ impl PathSlice {
   }
   /// Find the longest shared prefix of this name and another sequence
   pub fn coprefix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
-    &self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count()]
+    &self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count() as u16]
   }
   /// Find the longest shared suffix of this name and another sequence
   pub fn cosuffix<'a>(&'a self, other: &PathSlice) -> &'a PathSlice {
-    &self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count()]
+    &self[0..self.iter().zip(other.iter()).take_while(|(l, r)| l == r).count() as u16]
   }
   /// Remove another
   pub fn strip_prefix<'a>(&'a self, other: &PathSlice) -> Option<&'a PathSlice> {
@@ -52,7 +53,8 @@ impl PathSlice {
     (shared == other.len()).then_some(PathSlice::new(&self[shared..]))
   }
   /// Number of path segments
-  pub fn len(&self) -> usize { self.0.len() }
+  pub fn len(&self) -> u16 { self.0.len().try_into().expect("Too long name!") }
+  pub fn get<I: NameIndex>(&self, index: I) -> Option<&I::Output> { index.get(self) }
   /// Whether there are any path segments. In other words, whether this is a
   /// valid name
   pub fn is_empty(&self) -> bool { self.len() == 0 }
@@ -79,31 +81,47 @@ impl<'a> IntoIterator for &'a PathSlice {
   fn into_iter(self) -> Self::IntoIter { self.0.iter().cloned() }
 }
 
+pub trait NameIndex {
+  type Output: ?Sized;
+  fn get(self, name: &PathSlice) -> Option<&Self::Output>;
+}
+impl<T: NameIndex> Index<T> for PathSlice {
+  type Output = T::Output;
+  fn index(&self, index: T) -> &Self::Output { index.get(self).expect("Index out of bounds") }
+}
+
 mod idx_impls {
   use std::ops;
 
-  use super::PathSlice;
+  use super::{NameIndex, PathSlice, conv_range};
   use crate::interner::Tok;
 
-  impl ops::Index<usize> for PathSlice {
+  impl NameIndex for u16 {
     type Output = Tok<String>;
-    fn index(&self, index: usize) -> &Self::Output { &self.0[index] }
+    fn get(self, name: &PathSlice) -> Option<&Self::Output> { name.0.get(self as usize) }
   }
+
+  impl NameIndex for ops::RangeFull {
+    type Output = PathSlice;
+    fn get(self, name: &PathSlice) -> Option<&Self::Output> { Some(name) }
+  }
+
   macro_rules! impl_range_index_for_pathslice {
-    ($range:ty) => {
+    ($range:ident) => {
-      impl ops::Index<$range> for PathSlice {
+      impl ops::Index<ops::$range<u16>> for PathSlice {
         type Output = Self;
-        fn index(&self, index: $range) -> &Self::Output { Self::new(&self.0[index]) }
+        fn index(&self, index: ops::$range<u16>) -> &Self::Output {
+          Self::new(&self.0[conv_range::<u16, usize>(index)])
+        }
       }
     };
   }
 
-  impl_range_index_for_pathslice!(ops::RangeFull);
+  impl_range_index_for_pathslice!(RangeFrom);
-  impl_range_index_for_pathslice!(ops::RangeFrom<usize>);
+  impl_range_index_for_pathslice!(RangeTo);
-  impl_range_index_for_pathslice!(ops::RangeTo<usize>);
+  impl_range_index_for_pathslice!(Range);
-  impl_range_index_for_pathslice!(ops::Range<usize>);
+  impl_range_index_for_pathslice!(RangeInclusive);
-  impl_range_index_for_pathslice!(ops::RangeInclusive<usize>);
+  impl_range_index_for_pathslice!(RangeToInclusive);
-  impl_range_index_for_pathslice!(ops::RangeToInclusive<usize>);
 }
 
 impl Deref for PathSlice {
@@ -120,6 +138,18 @@ impl<const N: usize> Borrow<PathSlice> for [Tok<String>; N] {
 impl Borrow<PathSlice> for Vec<Tok<String>> {
   fn borrow(&self) -> &PathSlice { PathSlice::new(&self[..]) }
 }
+pub fn conv_bound<T: Into<U> + Clone, U>(bound: Bound<&T>) -> Bound<U> {
+  match bound {
+    Bound::Included(i) => Bound::Included(i.clone().into()),
+    Bound::Excluded(i) => Bound::Excluded(i.clone().into()),
+    Bound::Unbounded => Bound::Unbounded,
+  }
+}
+pub fn conv_range<'a, T: Into<U> + Clone + 'a, U: 'a>(
+  range: impl RangeBounds<T>
+) -> (Bound<U>, Bound<U>) {
+  (conv_bound(range.start_bound()), conv_bound(range.end_bound()))
+}
 
 /// A token path which may be empty. [VName] is the non-empty,
 /// [PathSlice] is the borrowed version
@@ -227,7 +257,7 @@ impl VName {
     let data: Vec<_> = items.into_iter().collect();
     if data.is_empty() { Err(EmptyNameError) } else { Ok(Self(data)) }
   }
-  pub fn deintern(items: impl IntoIterator<Item = TStr>) -> Result<Self, EmptyNameError> {
+  pub fn deintern(items: impl IntoIterator<Item = api::TStr>) -> Result<Self, EmptyNameError> {
     Self::new(items.into_iter().map(deintern))
   }
   /// Unwrap the enclosed vector
@@ -327,6 +357,9 @@ impl Sym {
   pub fn id(&self) -> NonZeroU64 { self.0.marker().get_id() }
   /// Extern the sym for editing
   pub fn to_vname(&self) -> VName { VName(self[..].to_vec()) }
+  pub fn deintern(marker: TStrv) -> Sym {
+    Self::from_tok(deintern(marker)).expect("Empty sequence found for serialized Sym")
+  }
 }
 impl fmt::Debug for Sym {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Sym({self})") }

orchid-base/src/number.rs

@@ -4,6 +4,10 @@ use std::ops::Range;
 use ordered_float::NotNan;
 use rust_decimal::Decimal;
 
+use crate::error::{mk_err, OrcErr};
+use crate::intern;
+use crate::location::Pos;
+
 /// A number, either floating point or unsigned int, parsed by Orchid.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum Numeric {
@@ -48,6 +52,18 @@ pub struct NumError {
   pub kind: NumErrorKind,
 }
 
+pub fn num_to_err(NumError { kind, range }: NumError, offset: u32) -> OrcErr {
+  mk_err(
+    intern!(str: "Failed to parse number"),
+    match kind {
+      NumErrorKind::NaN => "NaN emerged during parsing",
+      NumErrorKind::InvalidDigit => "non-digit character encountered",
+      NumErrorKind::Overflow => "The number being described is too large or too accurate",
+    },
+    [Pos::Range(offset + range.start as u32..offset + range.end as u32).into()],
+  )
+}
+
 /// Parse a numbre literal out of text
 pub fn parse_num(string: &str) -> Result<Numeric, NumError> {
   let overflow_err = NumError { range: 0..string.len(), kind: NumErrorKind::Overflow };

orchid-base/src/parse.rs

@@ -0,0 +1,286 @@
+use std::ops::{Deref, Range};
+use std::sync::Arc;
+use std::{fmt, iter};
+
+use itertools::Itertools;
+
+use crate::error::{mk_err, OrcRes, Reporter};
+use crate::interner::{deintern, Tok};
+use crate::location::Pos;
+use crate::name::VPath;
+use crate::tree::{AtomInTok, Paren, TokTree, Token};
+use crate::{api, intern};
+
+pub fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
+pub fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
+pub fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}\\".contains(c) }
+pub fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
+
+#[derive(Debug)]
+pub struct Snippet<'a, 'b, A: AtomInTok, X> {
+  prev: &'a TokTree<'b, A, X>,
+  cur: &'a [TokTree<'b, A, X>],
+}
+impl<'a, 'b, A: AtomInTok, X> Snippet<'a, 'b, A, X> {
+  pub fn new(prev: &'a TokTree<'b, A, X>, cur: &'a [TokTree<'b, A, X>]) -> Self {
+    Self { prev, cur }
+  }
+  pub fn split_at(self, pos: u32) -> (Self, Self) {
+    let fst = Self { prev: self.prev, cur: &self.cur[..pos as usize] };
+    let new_prev = if pos == 0 { self.prev } else { &self.cur[pos as usize - 1] };
+    let snd = Self { prev: new_prev, cur: &self.cur[pos as usize..] };
+    (fst, snd)
+  }
+  pub fn find_idx(self, mut f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<u32> {
+    self.cur.iter().position(|t| f(&t.tok)).map(|t| t as u32)
+  }
+  pub fn get(self, idx: u32) -> Option<&'a TokTree<'b, A, X>> { self.cur.get(idx as usize) }
+  pub fn len(self) -> u32 { self.cur.len() as u32 }
+  pub fn prev(self) -> &'a TokTree<'b, A, X> { self.prev }
+  pub fn pos(self) -> Range<u32> {
+    (self.cur.first().map(|f| f.range.start..self.cur.last().unwrap().range.end))
+      .unwrap_or(self.prev.range.clone())
+  }
+  pub fn pop_front(self) -> Option<(&'a TokTree<'b, A, X>, Self)> {
+    self.cur.first().map(|r| (r, self.split_at(1).1))
+  }
+  pub fn split_once(self, f: impl FnMut(&Token<'b, A, X>) -> bool) -> Option<(Self, Self)> {
+    let idx = self.find_idx(f)?;
+    Some((self.split_at(idx).0, self.split_at(idx + 1).1))
+  }
+  pub fn split(
+    mut self,
+    mut f: impl FnMut(&Token<'b, A, X>) -> bool,
+  ) -> impl Iterator<Item = Self> {
+    iter::from_fn(move || {
+      self.is_empty().then_some(())?;
+      let (ret, next) = self.split_once(&mut f).unwrap_or(self.split_at(self.len()));
+      self = next;
+      Some(ret)
+    })
+  }
+  pub fn is_empty(self) -> bool { self.len() == 0 }
+  pub fn skip_fluff(self) -> Self {
+    let non_fluff_start = self.find_idx(|t| !matches!(t, Token::NS | Token::Comment(_)));
+    self.split_at(non_fluff_start.unwrap_or(self.len())).1
+  }
+}
+impl<'a, 'b, A: AtomInTok, X> Copy for Snippet<'a, 'b, A, X> {}
+impl<'a, 'b, A: AtomInTok, X> Clone for Snippet<'a, 'b, A, X> {
+  fn clone(&self) -> Self { *self }
+}
+impl<'a, 'b, A: AtomInTok, X> Deref for Snippet<'a, 'b, A, X> {
+  type Target = [TokTree<'b, A, X>];
+  fn deref(&self) -> &Self::Target { self.cur }
+}
+
+/// Remove tokens that aren't meaningful in expression context, such as comments
+/// or line breaks
+pub fn strip_fluff<'a, A: AtomInTok, X: Clone>(
+  tt: &TokTree<'a, A, X>,
+) -> Option<TokTree<'a, A, X>> {
+  let tok = match &tt.tok {
+    Token::BR => return None,
+    Token::Comment(_) => return None,
+    Token::LambdaHead(arg) => Token::LambdaHead(arg.iter().filter_map(strip_fluff).collect()),
+    Token::S(p, b) => Token::S(p.clone(), b.iter().filter_map(strip_fluff).collect()),
+    t => t.clone(),
+  };
+  Some(TokTree { tok, range: tt.range.clone() })
+}
+
+#[derive(Clone, Debug)]
+pub struct Comment {
+  pub text: Arc<String>,
+  pub pos: Pos,
+}
+
+pub fn line_items<'a, 'b, A: AtomInTok, X>(
+  snip: Snippet<'a, 'b, A, X>,
+) -> Vec<(Vec<Comment>, Snippet<'a, 'b, A, X>)> {
+  let mut items = Vec::new();
+  let mut comments = Vec::new();
+  for mut line in snip.split(|t| matches!(t, Token::BR)) {
+    match &line.cur {
+      [TokTree { tok: Token::S(Paren::Round, tokens), .. }] => line.cur = tokens,
+      [] => continue,
+      _ => (),
+    }
+    match line.find_idx(|t| !matches!(t, Token::Comment(_))) {
+      None => comments.extend(line.cur),
+      Some(i) => {
+        let (cmts, line) = line.split_at(i);
+        let comments = Vec::from_iter(comments.drain(..).chain(cmts.cur).map(|t| match &t.tok {
+          Token::Comment(c) => Comment { text: c.clone(), pos: Pos::Range(t.range.clone()) },
+          _ => unreachable!("All are comments checked above"),
+        }));
+        items.push((comments, line));
+      },
+    }
+  }
+  items
+}
+
+pub fn try_pop_no_fluff<'a, 'b, A: AtomInTok, X>(
+  snip: Snippet<'a, 'b, A, X>,
+) -> OrcRes<(&'a TokTree<'b, A, X>, Snippet<'a, 'b, A, X>)> {
+  snip.skip_fluff().pop_front().ok_or_else(|| {
+    vec![mk_err(intern!(str: "Unexpected end"), "Pattern ends abruptly", [
+      Pos::Range(snip.pos()).into()
+    ])]
+  })
+}
+
+pub fn expect_end(snip: Snippet<'_, '_, impl AtomInTok, impl Sized>) -> OrcRes<()> {
+  match snip.skip_fluff().get(0) {
+    Some(surplus) => Err(vec![mk_err(
+      intern!(str: "Extra code after end of line"),
+      "Code found after the end of the line",
+      [Pos::Range(surplus.range.clone()).into()],
+    )]),
+    None => Ok(()),
+  }
+}
+
+pub fn expect_tok<'a, 'b, A: AtomInTok, X: fmt::Display>(
+  snip: Snippet<'a, 'b, A, X>, tok: Tok<String>
+) -> OrcRes<Snippet<'a, 'b, A, X>> {
+  let (head, tail) = try_pop_no_fluff(snip)?;
+  match &head.tok {
+    Token::Name(n) if *n == tok => Ok(tail),
+    t => Err(vec![mk_err(
+      intern!(str: "Expected specific keyword"), 
+      format!("Expected {tok} but found {t}"),
+      [Pos::Range(head.range.clone()).into()]
+    )])
+  }
+}
+
+pub fn parse_multiname<'a, 'b, A: AtomInTok, X: fmt::Display>(
+  ctx: &impl Reporter,
+  tail: Snippet<'a, 'b, A, X>,
+) -> OrcRes<(Vec<CompName>, Snippet<'a, 'b, A, X>)> {
+  let ret = rec(ctx, tail);
+  #[allow(clippy::type_complexity)] // it's an internal function
+  pub fn rec<'a, 'b, A: AtomInTok, X: fmt::Display>(
+    ctx: &impl Reporter,
+    tail: Snippet<'a, 'b, A, X>,
+  ) -> OrcRes<(Vec<(Vec<Tok<String>>, Option<Tok<String>>, Pos)>, Snippet<'a, 'b, A, X>)> {
+    let comma = intern!(str: ",");
+    let globstar = intern!(str: "*");
+    let (name, tail) = tail.skip_fluff().pop_front().ok_or_else(|| {
+      vec![mk_err(
+        intern!(str: "Expected name"),
+        "Expected a name, a list of names, or a globstar.",
+        [Pos::Range(tail.pos()).into()],
+      )]
+    })?;
+    if let Some((Token::NS, tail)) = tail.skip_fluff().pop_front().map(|(tt, s)| (&tt.tok, s)) {
+      let n = match &name.tok {
+        Token::Name(n) if n.starts_with(name_start) => Ok(n),
+        _ => Err(mk_err(intern!(str: "Unexpected name prefix"), "Only names can precede ::", [
+          Pos::Range(name.range.clone()).into(),
+        ])),
+      };
+      match (rec(ctx, tail), n) {
+        (Err(ev), n) => Err(Vec::from_iter(ev.into_iter().chain(n.err()))),
+        (Ok((_, tail)), Err(e)) => {
+          ctx.report(e);
+          Ok((vec![], tail))
+        },
+        (Ok((n, tail)), Ok(pre)) =>
+          Ok((n.into_iter().update(|i| i.0.push(pre.clone())).collect_vec(), tail)),
+      }
+    } else {
+      let names = match &name.tok {
+        Token::Name(ntok) => {
+          let nopt = match ntok {
+            n if *n == globstar => None,
+            n if n.starts_with(op_char) =>
+              return Err(vec![mk_err(
+                intern!(str: "Unescaped operator in multiname"),
+                "Operators in multinames should be enclosed in []",
+                [Pos::Range(name.range.clone()).into()],
+              )]),
+            n => Some(n.clone()),
+          };
+          vec![(vec![], nopt, Pos::Range(name.range.clone()))]
+        },
+        Token::S(Paren::Square, b) => {
+          let mut ok = Vec::new();
+          b.iter().for_each(|tt| match &tt.tok {
+            Token::Name(n) if n.starts_with(op_char) =>
+              ok.push((vec![], Some(n.clone()), Pos::Range(tt.range.clone()))),
+            Token::BR | Token::Comment(_) => (),
+            _ => ctx.report(mk_err(
+              intern!(str: "Non-operator in escapement in multiname"),
+              "In multinames, [] functions as a literal name list reserved for operators",
+              [Pos::Range(name.range.clone()).into()],
+            )),
+          });
+          ok
+        },
+        Token::S(Paren::Round, b) => {
+          let mut ok = Vec::new();
+          let body = Snippet::new(name, b);
+          for csent in body.split(|n| matches!(n, Token::Name(n) if *n == comma)) {
+            match rec(ctx, csent) {
+              Err(e) => e.into_iter().for_each(|e| ctx.report(e)),
+              Ok((v, surplus)) => match surplus.get(0) {
+                None => ok.extend(v),
+                Some(t) => ctx.report(mk_err(
+                  intern!(str: "Unexpected token in multiname group"),
+                  "Unexpected token. Likely missing a :: or , or wanted [] instead of ()",
+                  [Pos::Range(t.range.clone()).into()],
+                )),
+              },
+            }
+          }
+          ok
+        },
+        t =>
+          return Err(vec![mk_err(
+            intern!(str: "Unrecognized name end"),
+            format!("Names cannot end with {t} tokens"),
+            [Pos::Range(name.range.clone()).into()],
+          )]),
+      };
+      Ok((names, tail))
+    }
+  }
+  ret.map(|(i, tail)| {
+    let i = Vec::from_iter((i.into_iter()).map(|(p, name, pos)| CompName {
+      path: VPath::new(p.into_iter().rev()),
+      name,
+      pos,
+    }));
+    (i, tail)
+  })
+}
+
+/// A compound name, possibly ending with a globstar
+#[derive(Debug, Clone)]
+pub struct CompName {
+  pub path: VPath,
+  pub name: Option<Tok<String>>,
+  pub pos: Pos,
+}
+impl CompName {
+  pub fn from_api(i: api::CompName) -> Self {
+    Self {
+      path: VPath::new(i.path.into_iter().map(deintern)),
+      name: i.name.map(deintern),
+      pos: Pos::from_api(&i.location),
+    }
+  }
+}
+
+#[cfg(test)]
+mod test {
+    use never::Never;
+
+    use super::Snippet;
+
+  fn _covary_snip_a<'a, 'b>(x: Snippet<'static, 'b, Never, ()>) -> Snippet<'a, 'b, Never, ()> { x }
+  fn _covary_snip_b<'a, 'b>(x: Snippet<'a, 'static, Never, ()>) -> Snippet<'a, 'b, Never, ()> { x }
+}

orchid-base/src/reqnot.rs

@@ -1,18 +1,20 @@
 use std::marker::PhantomData;
-use std::{mem, thread};
 use std::ops::{BitAnd, Deref};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::{sync_channel, SyncSender};
 use std::sync::{Arc, Mutex};
+use std::{mem, thread};
 
 use dyn_clone::{clone_box, DynClone};
 use hashbrown::HashMap;
 use orchid_api_traits::{Channel, Coding, Decode, Encode, MsgSet, Request};
 use trait_set::trait_set;
 
+pub struct ReplyToken;
+
 trait_set! {
   pub trait SendFn<T: MsgSet> = for<'a> FnMut(&'a [u8], ReqNot<T>) + DynClone + Send + 'static;
-  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) + DynClone + Send + Sync + 'static;
+  pub trait ReqFn<T: MsgSet> = FnMut(RequestHandle<T>) -> ReplyToken + DynClone + Send + Sync + 'static;
   pub trait NotifFn<T: MsgSet> =
     for<'a> FnMut(<T::In as Channel>::Notif, ReqNot<T>) + DynClone + Send + Sync + 'static;
 }
@@ -30,16 +32,17 @@ pub struct RequestHandle<T: MsgSet> {
 impl<MS: MsgSet + 'static> RequestHandle<MS> {
   pub fn reqnot(&self) -> ReqNot<MS> { self.parent.clone() }
   pub fn req(&self) -> &<MS::In as Channel>::Req { &self.message }
-  fn respond(&self, response: &impl Encode) {
+  fn respond(&self, response: &impl Encode) -> ReplyToken {
     assert!(!self.fulfilled.swap(true, Ordering::Relaxed), "Already responded to {}", self.id);
     let mut buf = (!self.id).to_be_bytes().to_vec();
     response.encode(&mut buf);
     let mut send = clone_box(&*self.reqnot().0.lock().unwrap().send);
     (send)(&buf, self.parent.clone());
+    ReplyToken
   }
-  pub fn handle<T: Request>(&self, _: &T, rep: &T::Response) { self.respond(rep) }
+  pub fn handle<T: Request>(&self, _: &T, rep: &T::Response) -> ReplyToken { self.respond(rep) }
   pub fn will_handle_as<T: Request>(&self, _: &T) -> ReqTypToken<T> { ReqTypToken(PhantomData) }
-  pub fn handle_as<T: Request>(&self, _token: ReqTypToken<T>, rep: &T::Response) {
+  pub fn handle_as<T: Request>(&self, _token: ReqTypToken<T>, rep: &T::Response) -> ReplyToken {
     self.respond(rep)
   }
 }
@@ -236,7 +239,7 @@ mod test {
       |_, _| panic!("Not receiving notifs"),
       |req| {
         assert_eq!(req.req(), &TestReq(5));
-        req.respond(&6u8);
+        req.respond(&6u8)
       },
     ));
     let response = sender.request(TestReq(5));

orchid-base/src/tokens.rs

@@ -1,7 +1,8 @@
 use std::fmt::Display;
 
-use orchid_api::tree::{Paren, Placeholder, PlaceholderKind};
+pub use api::Paren;
 
+use crate::api;
 use crate::interner::{deintern, Tok};
 
 pub const PARENS: &[(char, char, Paren)] =
@@ -10,24 +11,24 @@ pub const PARENS: &[(char, char, Paren)] =
 #[derive(Clone, Debug)]
 pub struct OwnedPh {
   pub name: Tok<String>,
-  pub kind: PlaceholderKind,
+  pub kind: api::PlaceholderKind,
 }
 impl OwnedPh {
-  pub fn to_api(&self) -> Placeholder {
+  pub fn to_api(&self) -> api::Placeholder {
-    Placeholder { name: self.name.marker(), kind: self.kind.clone() }
+    api::Placeholder { name: self.name.marker(), kind: self.kind.clone() }
   }
-  pub fn from_api(ph: Placeholder) -> Self { Self { name: deintern(ph.name), kind: ph.kind } }
+  pub fn from_api(ph: api::Placeholder) -> Self { Self { name: deintern(ph.name), kind: ph.kind } }
 }
 
 impl Display for OwnedPh {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     match self.kind {
-      PlaceholderKind::Name => write!(f, "$_{}", self.name),
+      api::PlaceholderKind::Name => write!(f, "$_{}", self.name),
-      PlaceholderKind::Scalar => write!(f, "${}", self.name),
+      api::PlaceholderKind::Scalar => write!(f, "${}", self.name),
-      PlaceholderKind::Vector { nz: false, prio: 0 } => write!(f, "..${}", self.name),
+      api::PlaceholderKind::Vector { nz: false, prio: 0 } => write!(f, "..${}", self.name),
-      PlaceholderKind::Vector { nz: true, prio: 0 } => write!(f, "...${}", self.name),
+      api::PlaceholderKind::Vector { nz: true, prio: 0 } => write!(f, "...${}", self.name),
-      PlaceholderKind::Vector { nz: false, prio } => write!(f, "..${}:{prio}", self.name),
+      api::PlaceholderKind::Vector { nz: false, prio } => write!(f, "..${}:{prio}", self.name),
-      PlaceholderKind::Vector { nz: true, prio } => write!(f, "...${}:{prio}", self.name),
+      api::PlaceholderKind::Vector { nz: true, prio } => write!(f, "...${}:{prio}", self.name),
     }
   }
 }

orchid-base/src/tree.rs

@@ -1,591 +1,266 @@
-//! Generic module tree structure
+use std::borrow::Borrow;
-//!
+use std::cell::RefCell;
-//! Used by various stages of the pipeline with different parameters
+use std::fmt::{self, Display, Write};
-use std::fmt;
+use std::iter;
+use std::marker::PhantomData;
+use std::ops::Range;
+use std::sync::Arc;
 
-use hashbrown::HashMap;
+use itertools::Itertools;
 use never::Never;
-use substack::Substack;
 use trait_set::trait_set;
 
-use crate::boxed_iter::BoxedIter;
+use crate::api;
-use crate::combine::Combine;
+use crate::error::OrcErr;
-use crate::interner::{intern, Tok};
+use crate::interner::{deintern, intern, Tok};
-use crate::join::try_join_maps;
+use crate::name::{NameLike, VName};
-use crate::name::{VName, VPath};
+use crate::tokens::{OwnedPh, PARENS};
-use crate::sequence::Sequence;
-
-/// An umbrella trait for operations you can carry out on any part of the tree
-/// structure
-pub trait TreeTransforms: Sized {
-  /// Data held at the leaves of the tree
-  type Item;
-  /// Data associated with modules
-  type XMod;
-  /// Data associated with entries inside modules
-  type XEnt;
-  /// Recursive type to enable [TreeTransforms::map_data] to transform the whole
-  /// tree
-  type SelfType<T, U, V>: TreeTransforms<Item = T, XMod = U, XEnt = V>;
-
-  /// Implementation for [TreeTransforms::map_data]
-  fn map_data_rec<T, U, V>(
-    self,
-    item: &mut impl FnMut(Substack<Tok<String>>, Self::Item) -> T,
-    module: &mut impl FnMut(Substack<Tok<String>>, Self::XMod) -> U,
-    entry: &mut impl FnMut(Substack<Tok<String>>, Self::XEnt) -> V,
-    path: Substack<Tok<String>>,
-  ) -> Self::SelfType<T, U, V>;
-
-  /// Transform all the data in the tree without changing its structure
-  fn map_data<T, U, V>(
-    self,
-    mut item: impl FnMut(Substack<Tok<String>>, Self::Item) -> T,
-    mut module: impl FnMut(Substack<Tok<String>>, Self::XMod) -> U,
-    mut entry: impl FnMut(Substack<Tok<String>>, Self::XEnt) -> V,
-  ) -> Self::SelfType<T, U, V> {
-    self.map_data_rec(&mut item, &mut module, &mut entry, Substack::Bottom)
-  }
-
-  /// Visit all elements in the tree. This is like [TreeTransforms::search] but
-  /// without the early exit
-  ///
-  /// * init - can be used for reduce, otherwise pass `()`
-  /// * callback - a callback applied on every module.
-  ///   * [`Substack<Tok<String>>`] - the walked path
-  ///   * [Module] - the current module
-  ///   * `T` - data for reduce.
-  fn search_all<'a, T>(
-    &'a self,
-    init: T,
-    mut callback: impl FnMut(
-      Substack<Tok<String>>,
-      ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
-      T,
-    ) -> T,
-  ) -> T {
-    let res =
-      self.search(init, |stack, member, state| Ok::<T, Never>(callback(stack, member, state)));
-    res.unwrap_or_else(|e| match e {})
-  }
-
-  /// Visit elements in the tree depth first with the provided function
-  ///
-  /// * init - can be used for reduce, otherwise pass `()`
-  /// * callback - a callback applied on every module. Can return [Err] to
-  ///   short-circuit the walk
-  ///   * [`Substack<Tok<String>>`] - the walked path
-  ///   * [Module] - the current module
-  ///   * `T` - data for reduce.
-  fn search<'a, T, E>(
-    &'a self,
-    init: T,
-    mut callback: impl FnMut(
-      Substack<Tok<String>>,
-      ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
-      T,
-    ) -> Result<T, E>,
-  ) -> Result<T, E> {
-    self.search_rec(init, Substack::Bottom, &mut callback)
-  }
-
-  /// Internal version of [TreeTransforms::search_all]
-  fn search_rec<'a, T, E>(
-    &'a self,
-    state: T,
-    stack: Substack<Tok<String>>,
-    callback: &mut impl FnMut(
-      Substack<Tok<String>>,
-      ModMemberRef<'a, Self::Item, Self::XMod, Self::XEnt>,
-      T,
-    ) -> Result<T, E>,
-  ) -> Result<T, E>;
-}
-
-/// The member in a [ModEntry] which is associated with a name in a [Module]
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum ModMember<Item, XMod, XEnt> {
-  /// Arbitrary data
-  Item(Item),
-  /// A child module
-  Sub(Module<Item, XMod, XEnt>),
-}
-
-impl<Item, XMod, XEnt> TreeTransforms for ModMember<Item, XMod, XEnt> {
-  type Item = Item;
-  type XEnt = XEnt;
-  type XMod = XMod;
-  type SelfType<T, U, V> = ModMember<T, U, V>;
-
-  fn map_data_rec<T, U, V>(
-    self,
-    item: &mut impl FnMut(Substack<Tok<String>>, Item) -> T,
-    module: &mut impl FnMut(Substack<Tok<String>>, XMod) -> U,
-    entry: &mut impl FnMut(Substack<Tok<String>>, XEnt) -> V,
-    path: Substack<Tok<String>>,
-  ) -> Self::SelfType<T, U, V> {
-    match self {
-      Self::Item(it) => ModMember::Item(item(path, it)),
-      Self::Sub(sub) => ModMember::Sub(sub.map_data_rec(item, module, entry, path)),
-    }
-  }
-
-  fn search_rec<'a, T, E>(
-    &'a self,
-    state: T,
-    stack: Substack<Tok<String>>,
-    callback: &mut impl FnMut(
-      Substack<Tok<String>>,
-      ModMemberRef<'a, Item, XMod, XEnt>,
-      T,
-    ) -> Result<T, E>,
-  ) -> Result<T, E> {
-    match self {
-      Self::Item(it) => callback(stack, ModMemberRef::Item(it), state),
-      Self::Sub(m) => m.search_rec(state, stack, callback),
-    }
-  }
-}
-
-/// Reasons why merging trees might fail
-pub enum ConflictKind<Item: Combine, XMod: Combine, XEnt: Combine> {
-  /// Error during the merging of items
-  Item(Item::Error),
-  /// Error during the merging of module metadata
-  Module(XMod::Error),
-  /// Error during the merging of entry metadata
-  XEnt(XEnt::Error),
-  /// An item appeared in one tree where the other contained a submodule
-  ItemModule,
-}
-
-macro_rules! impl_for_conflict {
-  ($target:ty, ($($deps:tt)*), $for:ty, $body:tt) => {
-    impl<Item: Combine, XMod: Combine, XEnt: Combine> $target
-    for $for
-    where
-      Item::Error: $($deps)*,
-      XMod::Error: $($deps)*,
-      XEnt::Error: $($deps)*,
-    $body
-  };
-}
-
-impl_for_conflict!(Clone, (Clone), ConflictKind<Item, XMod, XEnt>, {
-  fn clone(&self) -> Self {
-    match self {
-      ConflictKind::Item(it_e) => ConflictKind::Item(it_e.clone()),
-      ConflictKind::Module(mod_e) => ConflictKind::Module(mod_e.clone()),
-      ConflictKind::XEnt(ent_e) => ConflictKind::XEnt(ent_e.clone()),
-      ConflictKind::ItemModule => ConflictKind::ItemModule,
-    }
-  }
-});
-
-impl_for_conflict!(fmt::Debug, (fmt::Debug), ConflictKind<Item, XMod, XEnt>, {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-    match self {
-      ConflictKind::Item(it_e) =>
-        f.debug_tuple("TreeCombineErr::Item").field(it_e).finish(),
-      ConflictKind::Module(mod_e) =>
-        f.debug_tuple("TreeCombineErr::Module").field(mod_e).finish(),
-      ConflictKind::XEnt(ent_e) =>
-        f.debug_tuple("TreeCombineErr::XEnt").field(ent_e).finish(),
-      ConflictKind::ItemModule => write!(f, "TreeCombineErr::Item2Module"),
-    }
-  }
-});
-
-/// Error produced when two trees cannot be merged
-pub struct TreeConflict<Item: Combine, XMod: Combine, XEnt: Combine> {
-  /// Which subtree caused the failure
-  pub path: VPath,
-  /// What type of failure occurred
-  pub kind: ConflictKind<Item, XMod, XEnt>,
-}
-impl<Item: Combine, XMod: Combine, XEnt: Combine> TreeConflict<Item, XMod, XEnt> {
-  fn new(kind: ConflictKind<Item, XMod, XEnt>) -> Self { Self { path: VPath::new([]), kind } }
-
-  fn push(self, seg: Tok<String>) -> Self {
-    Self { path: self.path.prefix([seg]), kind: self.kind }
-  }
-}
-
-impl_for_conflict!(Clone, (Clone), TreeConflict<Item, XMod, XEnt>, {
-  fn clone(&self) -> Self {
-    Self { path: self.path.clone(), kind: self.kind.clone() }
-  }
-});
-
-impl_for_conflict!(fmt::Debug, (fmt::Debug), TreeConflict<Item, XMod, XEnt>, {
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-    f.debug_struct("TreeConflict")
-      .field("path", &self.path)
-      .field("kind", &self.kind)
-      .finish()
-  }
-});
-
-impl<Item: Combine, XMod: Combine, XEnt: Combine> Combine for ModMember<Item, XMod, XEnt> {
-  type Error = TreeConflict<Item, XMod, XEnt>;
-
-  fn combine(self, other: Self) -> Result<Self, Self::Error> {
-    match (self, other) {
-      (Self::Item(i1), Self::Item(i2)) => match i1.combine(i2) {
-        Ok(i) => Ok(Self::Item(i)),
-        Err(e) => Err(TreeConflict::new(ConflictKind::Item(e))),
-      },
-      (Self::Sub(m1), Self::Sub(m2)) => m1.combine(m2).map(Self::Sub),
-      (..) => Err(TreeConflict::new(ConflictKind::ItemModule)),
-    }
-  }
-}
-
-/// Data about a name in a [Module]
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ModEntry<Item, XMod, XEnt> {
-  /// The submodule or item
-  pub member: ModMember<Item, XMod, XEnt>,
-  /// Additional fields
-  pub x: XEnt,
-}
-impl<Item: Combine, XMod: Combine, XEnt: Combine> Combine for ModEntry<Item, XMod, XEnt> {
-  type Error = TreeConflict<Item, XMod, XEnt>;
-  fn combine(self, other: Self) -> Result<Self, Self::Error> {
-    match self.x.combine(other.x) {
-      Err(e) => Err(TreeConflict::new(ConflictKind::XEnt(e))),
-      Ok(x) => Ok(Self { x, member: self.member.combine(other.member)? }),
-    }
-  }
-}
-impl<Item, XMod, XEnt> ModEntry<Item, XMod, XEnt> {
-  /// Returns the item in this entry if it contains one.
-  #[must_use]
-  pub fn item(&self) -> Option<&Item> {
-    match &self.member {
-      ModMember::Item(it) => Some(it),
-      ModMember::Sub(_) => None,
-    }
-  }
-}
-
-impl<Item, XMod, XEnt> TreeTransforms for ModEntry<Item, XMod, XEnt> {
-  type Item = Item;
-  type XEnt = XEnt;
-  type XMod = XMod;
-  type SelfType<T, U, V> = ModEntry<T, U, V>;
-
-  fn map_data_rec<T, U, V>(
-    self,
-    item: &mut impl FnMut(Substack<Tok<String>>, Item) -> T,
-    module: &mut impl FnMut(Substack<Tok<String>>, XMod) -> U,
-    entry: &mut impl FnMut(Substack<Tok<String>>, XEnt) -> V,
-    path: Substack<Tok<String>>,
-  ) -> Self::SelfType<T, U, V> {
-    ModEntry {
-      member: self.member.map_data_rec(item, module, entry, path.clone()),
-      x: entry(path, self.x),
-    }
-  }
-
-  fn search_rec<'a, T, E>(
-    &'a self,
-    state: T,
-    stack: Substack<Tok<String>>,
-    callback: &mut impl FnMut(
-      Substack<Tok<String>>,
-      ModMemberRef<'a, Item, XMod, XEnt>,
-      T,
-    ) -> Result<T, E>,
-  ) -> Result<T, E> {
-    self.member.search_rec(state, stack, callback)
-  }
-}
-impl<Item, XMod, XEnt: Default> ModEntry<Item, XMod, XEnt> {
-  /// Wrap a member directly with trivial metadata
-  pub fn wrap(member: ModMember<Item, XMod, XEnt>) -> Self { Self { member, x: XEnt::default() } }
-  /// Wrap an item directly with trivial metadata
-  pub fn leaf(item: Item) -> Self { Self::wrap(ModMember::Item(item)) }
-}
-impl<Item, XMod: Default, XEnt: Default> ModEntry<Item, XMod, XEnt> {
-  /// Create an empty submodule
-  pub fn empty() -> Self { Self::wrap(ModMember::Sub(Module::wrap([]))) }
-
-  /// Create a module
-  #[must_use]
-  pub fn tree<K: AsRef<str>>(arr: impl IntoIterator<Item = (K, Self)>) -> Self {
-    Self::wrap(ModMember::Sub(Module::wrap(arr.into_iter().map(|(k, v)| (intern(k.as_ref()), v)))))
-  }
-
-  /// Create a record in the list passed to [ModEntry#tree] which describes a
-  /// submodule. This mostly exists to deal with strange rustfmt block
-  /// breaking behaviour
-  pub fn tree_ent<K: AsRef<str>>(key: K, arr: impl IntoIterator<Item = (K, Self)>) -> (K, Self) {
-    (key, Self::tree(arr))
-  }
-
-  /// Namespace the tree with the list of names
-  ///
-  /// The unarray is used to trick rustfmt into breaking the sub-item
-  /// into a block without breaking anything else.
-  #[must_use]
-  pub fn ns(name: impl AsRef<str>, [mut end]: [Self; 1]) -> Self {
-    let elements = name.as_ref().split("::").collect::<Vec<_>>();
-    for name in elements.into_iter().rev() {
-      end = Self::tree([(name, end)]);
-    }
-    end
-  }
-
-  fn not_mod_panic<T>() -> T { panic!("Expected module but found leaf") }
-
-  /// Return the wrapped module. Panic if the entry wraps an item
-  pub fn unwrap_mod(self) -> Module<Item, XMod, XEnt> {
-    if let ModMember::Sub(m) = self.member { m } else { Self::not_mod_panic() }
-  }
-
-  /// Return the wrapped module. Panic if the entry wraps an item
-  pub fn unwrap_mod_ref(&self) -> &Module<Item, XMod, XEnt> {
-    if let ModMember::Sub(m) = &self.member { m } else { Self::not_mod_panic() }
-  }
-}
-
-/// A module, containing imports,
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Module<Item, XMod, XEnt> {
-  /// Submodules and items by name
-  pub entries: HashMap<Tok<String>, ModEntry<Item, XMod, XEnt>>,
-  /// Additional fields
-  pub x: XMod,
-}
 
 trait_set! {
-  /// A filter applied to a module tree
+  pub trait RecurCB<'a, A: AtomInTok, X> = Fn(TokTree<'a, A, X>) -> TokTree<'a, A, X>;
-  pub trait Filter<'a, Item, XMod, XEnt> =
-    for<'b> Fn(&'b ModEntry<Item, XMod, XEnt>) -> bool + Clone + 'a
 }
 
-/// A line in a [Module]
+pub fn recur<'a, A: AtomInTok, X>(
-pub type Record<Item, XMod, XEnt> = (Tok<String>, ModEntry<Item, XMod, XEnt>);
+  tt: TokTree<'a, A, X>,
-
+  f: &impl Fn(TokTree<'a, A, X>, &dyn RecurCB<'a, A, X>) -> TokTree<'a, A, X>,
-impl<Item, XMod, XEnt> Module<Item, XMod, XEnt> {
+) -> TokTree<'a, A, X> {
-  /// Returns child names for which the value matches a filter
+  f(tt, &|TokTree { range, tok }| {
-  #[must_use]
+    let tok = match tok {
-  pub fn keys<'a>(
+      tok @ (Token::Atom(_) | Token::BR | Token::Bottom(_) | Token::Comment(_) | Token::NS) => tok,
-    &'a self,
+      tok @ (Token::Name(_) | Token::Ph(_) | Token::Slot(_) | Token::X(_)) => tok,
-    filter: impl for<'b> Fn(&'b ModEntry<Item, XMod, XEnt>) -> bool + 'a,
+      Token::LambdaHead(arg) =>
-  ) -> BoxedIter<Tok<String>> {
+        Token::LambdaHead(arg.into_iter().map(|tt| recur(tt, f)).collect_vec()),
-    Box::new((self.entries.iter()).filter(move |(_, v)| filter(v)).map(|(k, _)| k.clone()))
+      Token::S(p, b) => Token::S(p, b.into_iter().map(|tt| recur(tt, f)).collect_vec()),
+    };
+    TokTree { range, tok }
+  })
 }
 
-  /// Return the module at the end of the given path
+pub trait AtomInTok: Display + Clone {
-  pub fn walk_ref<'a: 'b, 'b>(
+  type Context: ?Sized;
-    &'a self,
+  fn from_api(atom: &api::Atom, pos: Range<u32>, ctx: &mut Self::Context) -> Self;
-    prefix: &'b [Tok<String>],
+  fn to_api(&self) -> api::Atom;
-    path: &'b [Tok<String>],
+}
-    filter: impl Filter<'b, Item, XMod, XEnt>,
+impl AtomInTok for Never {
-  ) -> Result<&'a Self, WalkError<'b>> {
+  type Context = Never;
-    let mut module = self;
+  fn from_api(_: &api::Atom, _: Range<u32>, _: &mut Self::Context) -> Self { panic!() }
-    for (pos, step) in path.iter().enumerate() {
+  fn to_api(&self) -> orchid_api::Atom { match *self {} }
-      let kind = match module.entries.get(step) {
+}
-        None => ErrKind::Missing,
+
-        Some(ent) if !filter(ent) => ErrKind::Filtered,
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
-        Some(ModEntry { member: ModMember::Item(_), .. }) => ErrKind::NotModule,
+pub struct TreeHandle<'a>(api::TreeTicket, PhantomData<&'a ()>);
-        Some(ModEntry { member: ModMember::Sub(next), .. }) => {
+impl TreeHandle<'static> {
-          module = next;
+  pub fn new(tt: api::TreeTicket) -> Self { TreeHandle(tt, PhantomData) }
-          continue;
+}
+impl<'a> TreeHandle<'a> {
+  pub fn ticket(self) -> api::TreeTicket { self.0 }
+}
+impl<'a> Display for TreeHandle<'a> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Handle({})", self.0.0) }
+}
+
+#[derive(Clone, Debug)]
+pub struct TokTree<'a, A: AtomInTok, X> {
+  pub tok: Token<'a, A, X>,
+  pub range: Range<u32>,
+}
+impl<'a, A: AtomInTok, X> TokTree<'a, A, X> {
+  pub fn from_api(tt: &api::TokenTree, ctx: &mut A::Context) -> Self {
+    let tok = match &tt.token {
+      api::Token::Atom(a) => Token::Atom(A::from_api(a, tt.range.clone(), ctx)),
+      api::Token::BR => Token::BR,
+      api::Token::NS => Token::NS,
+      api::Token::Bottom(e) => Token::Bottom(e.iter().map(OrcErr::from_api).collect()),
+      api::Token::Lambda(arg) => Token::LambdaHead(ttv_from_api(arg, ctx)),
+      api::Token::Name(name) => Token::Name(deintern(*name)),
+      api::Token::Ph(ph) => Token::Ph(OwnedPh::from_api(ph.clone())),
+      api::Token::S(par, b) => Token::S(par.clone(), ttv_from_api(b, ctx)),
+      api::Token::Comment(c) => Token::Comment(c.clone()),
+      api::Token::Slot(id) => Token::Slot(TreeHandle::new(*id)),
+    };
+    Self { range: tt.range.clone(), tok }
+  }
+
+  pub fn to_api(
+    &self,
+    do_extra: &mut impl FnMut(&X, Range<u32>) -> api::TokenTree,
+  ) -> api::TokenTree {
+    let token = match &self.tok {
+      Token::Atom(a) => api::Token::Atom(a.to_api()),
+      Token::BR => api::Token::BR,
+      Token::NS => api::Token::NS,
+      Token::Bottom(e) => api::Token::Bottom(e.iter().map(OrcErr::to_api).collect()),
+      Token::Comment(c) => api::Token::Comment(c.clone()),
+      Token::LambdaHead(arg) =>
+        api::Token::Lambda(arg.iter().map(|t| t.to_api(do_extra)).collect_vec()),
+      Token::Name(n) => api::Token::Name(n.marker()),
+      Token::Ph(ph) => api::Token::Ph(ph.to_api()),
+      Token::Slot(tt) => api::Token::Slot(tt.ticket()),
+      Token::S(p, b) => api::Token::S(p.clone(), b.iter().map(|t| t.to_api(do_extra)).collect()),
+      Token::X(x) => return do_extra(x, self.range.clone()),
+    };
+    api::TokenTree { range: self.range.clone(), token }
+  }
+}
+impl<'a, A: AtomInTok + Display, X: Display> Display for TokTree<'a, A, X> {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
+}
+
+pub fn ttv_from_api<A: AtomInTok, X>(
+  tokv: impl IntoIterator<Item: Borrow<api::TokenTree>>,
+  ctx: &mut A::Context,
+) -> Vec<TokTree<'static, A, X>> {
+  tokv.into_iter().map(|t| TokTree::<A, X>::from_api(t.borrow(), ctx)).collect()
+}
+
+pub fn ttv_to_api<'a, A: AtomInTok, X>(
+  tokv: impl IntoIterator<Item: Borrow<TokTree<'a, A, X>>>,
+  do_extra: &mut impl FnMut(&X, Range<u32>) -> api::TokenTree,
+) -> Vec<api::TokenTree> {
+  tokv
+    .into_iter()
+    .map(|tok| {
+      let tt: &TokTree<A, X> = tok.borrow();
+      tt.to_api(do_extra)
+    })
+    .collect_vec()
+}
+
+pub fn vname_tv<'a: 'b, 'b, A: AtomInTok + 'a, X: 'a>(
+  name: &'b VName,
+  ran: Range<u32>,
+) -> impl Iterator<Item = TokTree<'a, A, X>> + 'b {
+  let (head, tail) = name.split_first();
+  iter::once(Token::Name(head))
+    .chain(tail.iter().flat_map(|t| [Token::NS, Token::Name(t)]))
+    .map(move |t| t.at(ran.clone()))
+}
+
+pub fn wrap_tokv<'a, A: AtomInTok + 'a, X: 'a>(
+  items: Vec<TokTree<'a, A, X>>,
+  range: Range<u32>,
+) -> TokTree<'a, A, X> {
+  match items.len() {
+    1 => items.into_iter().next().unwrap(),
+    _ => Token::S(api::Paren::Round, items).at(range),
+  }
+}
+
+pub fn ph(s: &str) -> OwnedPh {
+  match s.strip_prefix("..") {
+    Some(v_tail) => {
+      let (mid, priority) = match v_tail.split_once(':') {
+        Some((h, t)) => (h, t.parse().expect("priority not an u8")),
+        None => (v_tail, 0),
+      };
+      let (name, nonzero) = match mid.strip_prefix(".$") {
+        Some(name) => (name, true),
+        None => (mid.strip_prefix('$').expect("Invalid placeholder"), false),
+      };
+      if name.starts_with("_") {
+        panic!("Names starting with an underscore indicate a single-name scalar placeholder")
+      }
+      OwnedPh {
+        name: intern(name),
+        kind: api::PlaceholderKind::Vector { nz: nonzero, prio: priority },
+      }
+    },
+    None => match s.strip_prefix("$_") {
+      Some(name) => OwnedPh { name: intern(name), kind: api::PlaceholderKind::Name },
+      None => match s.strip_prefix("$") {
+        None => panic!("Invalid placeholder"),
+        Some(name) => OwnedPh { name: intern(name), kind: api::PlaceholderKind::Scalar },
+      },
     },
-      };
-      let options = Sequence::new(move || module.keys(filter.clone()));
-      return Err(WalkError { kind, prefix, path, pos, options });
-    }
-    Ok(module)
-  }
-
-  /// Return the member at the end of the given path
-  ///
-  /// # Panics
-  ///
-  /// if path is empty, since the reference cannot be forwarded that way
-  pub fn walk1_ref<'a: 'b, 'b>(
-    &'a self,
-    prefix: &'b [Tok<String>],
-    path: &'b [Tok<String>],
-    filter: impl Filter<'b, Item, XMod, XEnt>,
-  ) -> Result<(&'a ModEntry<Item, XMod, XEnt>, &'a Self), WalkError<'b>> {
-    let (last, parent) = path.split_last().expect("Path cannot be empty");
-    let pos = path.len() - 1;
-    let module = self.walk_ref(prefix, parent, filter.clone())?;
-    let err_kind = match &module.entries.get(last) {
-      Some(entry) if filter(entry) => return Ok((entry, module)),
-      Some(_) => ErrKind::Filtered,
-      None => ErrKind::Missing,
-    };
-    let options = Sequence::new(move || module.keys(filter.clone()));
-    Err(WalkError { kind: err_kind, options, prefix, path, pos })
-  }
-
-  /// Walk from one node to another in a tree, asserting that the origin can see
-  /// the target.
-  ///
-  /// # Panics
-  ///
-  /// If the target is the root node
-  pub fn inner_walk<'a: 'b, 'b>(
-    &'a self,
-    origin: &[Tok<String>],
-    target: &'b [Tok<String>],
-    is_exported: impl for<'c> Fn(&'c ModEntry<Item, XMod, XEnt>) -> bool + Clone + 'b,
-  ) -> Result<(&'a ModEntry<Item, XMod, XEnt>, &'a Self), WalkError<'b>> {
-    let ignore_vis_len = 1 + origin.iter().zip(target).take_while(|(a, b)| a == b).count();
-    if target.len() <= ignore_vis_len {
-      return self.walk1_ref(&[], target, |_| true);
-    }
-    let (ignore_vis_path, hidden_path) = target.split_at(ignore_vis_len);
-    let first_divergence = self.walk_ref(&[], ignore_vis_path, |_| true)?;
-    first_divergence.walk1_ref(ignore_vis_path, hidden_path, is_exported)
-  }
-
-  /// Wrap entry table in a module with trivial metadata
-  pub fn wrap(entries: impl IntoIterator<Item = Record<Item, XMod, XEnt>>) -> Self
-  where XMod: Default {
-    Self { entries: entries.into_iter().collect(), x: XMod::default() }
   }
 }
 
-impl<Item, XMod, XEnt> TreeTransforms for Module<Item, XMod, XEnt> {
+pub use api::Paren;
-  type Item = Item;
-  type XEnt = XEnt;
-  type XMod = XMod;
-  type SelfType<T, U, V> = Module<T, U, V>;
 
-  fn map_data_rec<T, U, V>(
+#[derive(Clone, Debug)]
-    self,
+pub enum Token<'a, A: AtomInTok, X> {
-    item: &mut impl FnMut(Substack<Tok<String>>, Item) -> T,
+  Comment(Arc<String>),
-    module: &mut impl FnMut(Substack<Tok<String>>, XMod) -> U,
+  LambdaHead(Vec<TokTree<'a, A, X>>),
-    entry: &mut impl FnMut(Substack<Tok<String>>, XEnt) -> V,
+  Name(Tok<String>),
-    path: Substack<Tok<String>>,
+  NS,
-  ) -> Self::SelfType<T, U, V> {
+  BR,
-    Module {
+  S(Paren, Vec<TokTree<'a, A, X>>),
-      x: module(path.clone(), self.x),
+  Atom(A),
-      entries: (self.entries.into_iter())
+  Ph(OwnedPh),
-        .map(|(k, e)| (k.clone(), e.map_data_rec(item, module, entry, path.push(k))))
+  Bottom(Vec<OrcErr>),
-        .collect(),
+  Slot(TreeHandle<'a>),
+  X(X),
 }
+impl<'a, A: AtomInTok, X> Token<'a, A, X> {
+  pub fn at(self, range: Range<u32>) -> TokTree<'a, A, X> { TokTree { range, tok: self } }
 }
-
+impl<'a, A: AtomInTok + Display, X: Display> Display for Token<'a, A, X> {
-  fn search_rec<'a, T, E>(
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-    &'a self,
+    thread_local! {
-    mut state: T,
+      static PAREN_LEVEL: RefCell<usize> = 0.into();
-    stack: Substack<Tok<String>>,
-    callback: &mut impl FnMut(
-      Substack<Tok<String>>,
-      ModMemberRef<'a, Item, XMod, XEnt>,
-      T,
-    ) -> Result<T, E>,
-  ) -> Result<T, E> {
-    state = callback(stack.clone(), ModMemberRef::Mod(self), state)?;
-    for (key, value) in &self.entries {
-      state = value.search_rec(state, stack.push(key.clone()), callback)?;
     }
-    Ok(state)
+    fn get_indent() -> usize { PAREN_LEVEL.with_borrow(|t| *t) }
+    fn with_indent<T>(f: impl FnOnce() -> T) -> T {
+      PAREN_LEVEL.with_borrow_mut(|t| *t += 1);
+      let r = f();
+      PAREN_LEVEL.with_borrow_mut(|t| *t -= 1);
+      r
     }
-}
-
-impl<Item: Combine, XMod: Combine, XEnt: Combine> Combine for Module<Item, XMod, XEnt> {
-  type Error = TreeConflict<Item, XMod, XEnt>;
-  fn combine(self, Self { entries, x }: Self) -> Result<Self, Self::Error> {
-    let entries =
-      try_join_maps(self.entries, entries, |k, l, r| l.combine(r).map_err(|e| e.push(k.clone())))?;
-    let x = (self.x.combine(x)).map_err(|e| TreeConflict::new(ConflictKind::Module(e)))?;
-    Ok(Self { x, entries })
-  }
-}
-
-impl<Item: fmt::Display, TExt: fmt::Display, XEnt: fmt::Display> fmt::Display
-  for Module<Item, TExt, XEnt>
-{
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-    write!(f, "module {{")?;
-    for (name, ModEntry { member, x: extra }) in &self.entries {
-      match member {
-        ModMember::Sub(module) => write!(f, "\n{name} {extra} = {module}"),
-        ModMember::Item(item) => write!(f, "\n{name} {extra} = {item}"),
-      }?;
-    }
-    write!(f, "\n\n{}\n}}", &self.x)
-  }
-}
-
-/// A non-owning version of [ModMember]. Either an item-ref or a module-ref.
-pub enum ModMemberRef<'a, Item, XMod, XEnt> {
-  /// Leaf
-  Item(&'a Item),
-  /// Node
-  Mod(&'a Module<Item, XMod, XEnt>),
-}
-
-/// Possible causes why the path could not be walked
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum ErrKind {
-  /// `require_exported` was set to `true` and a module wasn't exported
-  Filtered,
-  /// A module was not found
-  Missing,
-  /// The path leads into a leaf node
-  NotModule,
-}
-impl ErrKind {
-  pub const fn msg(&self) -> &'static str {
     match self {
-      Self::Filtered => "The path leads into a private module",
+      Self::Atom(a) => f.write_str(&indent(&format!("{a}"), get_indent(), false)),
-      Self::Missing => "Nonexistent path",
+      Self::BR => write!(f, "\n{}", "  ".repeat(get_indent())),
-      Self::NotModule => "The path leads into a leaf",
+      Self::Bottom(err) => write!(
+        f,
+        "Botttom({})",
+        err.iter().map(|e| format!("{}: {}", e.description, e.message)).join(", ")
+      ),
+      Self::Comment(c) => write!(f, "--[{c}]--"),
+      Self::LambdaHead(arg) => with_indent(|| write!(f, "\\ {} .", ttv_fmt(arg))),
+      Self::NS => f.write_str("::"),
+      Self::Name(n) => f.write_str(n),
+      Self::Ph(ph) => write!(f, "{ph}"),
+      Self::Slot(th) => write!(f, "{th}"),
+      Self::S(p, b) => {
+        let (lp, rp, _) = PARENS.iter().find(|(_, _, par)| par == p).unwrap();
+        f.write_char(*lp)?;
+        with_indent(|| f.write_str(&ttv_fmt(b)))?;
+        f.write_char(*rp)
+      },
+      Self::X(x) => write!(f, "{x}"),
     }
   }
 }
 
-#[derive(Clone)]
+pub fn ttv_fmt<'a>(
-/// All details about a failed tree-walk
+  ttv: impl IntoIterator<Item = &'a TokTree<'a, impl AtomInTok + 'a, impl Display + 'a>>,
-pub struct WalkError<'a> {
+) -> String {
-  /// Failure mode
+  ttv.into_iter().join(" ")
-  kind: ErrKind,
-  /// Path to the module where the walk started
-  prefix: &'a [Tok<String>],
-  /// Planned walk path
-  path: &'a [Tok<String>],
-  /// Index into walked path where the error occurred
-  pos: usize,
-  /// Alternatives to the failed steps
-  options: Sequence<'a, Tok<String>>,
-}
-impl<'a> WalkError<'a> {
-  /// Total length of the path represented by this error
-  #[must_use]
-  pub fn depth(&self) -> usize { self.prefix.len() + self.pos + 1 }
-
-  pub fn alternatives(&self) -> BoxedIter<Tok<String>> { self.options.iter() }
-
-  /// Get the total path including the step that caused the error
-  pub fn full_path(&self) -> VName {
-    VName::new((self.prefix.iter()).chain(self.path.iter().take(self.pos + 1)).cloned())
-      .expect("empty paths don't cause an error")
 }
 
-  /// Construct an error for the very last item in a slice. This is often done
+pub fn indent(s: &str, lvl: usize, first: bool) -> String {
-  /// outside [super::tree] so it gets a function rather than exposing the
+  if first {
-  /// fields of [WalkError]
+    s.replace("\n", &("\n".to_string() + &"  ".repeat(lvl)))
-  pub fn last(path: &'a [Tok<String>], kind: ErrKind, options: Sequence<'a, Tok<String>>) -> Self {
+  } else if let Some((fst, rest)) = s.split_once('\n') {
-    WalkError { kind, path, options, pos: path.len() - 1, prefix: &[] }
+    fst.to_string() + "\n" + &indent(rest, lvl, true)
+  } else {
+    s.to_string()
   }
 }
-impl<'a> fmt::Debug for WalkError<'a> {
+
-  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+#[cfg(test)]
-    f.debug_struct("WalkError")
+mod test {
-      .field("kind", &self.kind)
+  use super::*;
-      .field("prefix", &self.prefix)
+
-      .field("path", &self.path)
+  #[test]
-      .field("pos", &self.pos)
+  fn test_covariance() {
-      .finish_non_exhaustive()
+    fn _f<'a>(x: Token<'static, Never, ()>) -> Token<'a, Never, ()> { x }
+  }
+
+  #[test]
+  fn fail_covariance() {
+    // this fails to compile
+    // fn _f<'a, 'b>(x: &'a mut &'static ()) -> &'a mut &'b () { x }
+    // this passes because it's covariant
+    fn _f<'a, 'b>(x: &'a fn() -> &'static ()) -> &'a fn() -> &'b () { x }
   }
 }

orchid-extension/Cargo.toml

@@ -12,6 +12,7 @@ dyn-clone = "1.0.17"
 hashbrown = "0.14.5"
 itertools = "0.13.0"
 konst = "0.3.9"
+lazy_static = "1.5.0"
 never = "0.1.0"
 once_cell = "1.19.0"
 orchid-api = { version = "0.1.0", path = "../orchid-api" }

orchid-extension/src/atom.rs

@@ -1,20 +1,25 @@
 use std::any::{type_name, Any, TypeId};
+use std::fmt;
 use std::io::{Read, Write};
-use std::ops::Deref;
+use std::marker::PhantomData;
+use std::ops::{Deref, Range};
 use std::sync::OnceLock;
 
 use dyn_clone::{clone_box, DynClone};
 use never::Never;
-use orchid_api::atom::{Atom, Fwd, LocalAtom};
+use orchid_api::ExprTicket;
-use orchid_api::expr::ExprTicket;
+use orchid_api_traits::{enc_vec, Coding, Decode, Request};
-use orchid_api_traits::{Coding, Decode, Request};
+use orchid_base::error::{mk_err, OrcErr, OrcRes};
+use orchid_base::intern;
 use orchid_base::location::Pos;
 use orchid_base::reqnot::Requester;
+use orchid_base::tree::AtomInTok;
 use trait_set::trait_set;
 
-use crate::error::{ProjectError, ProjectResult};
+use crate::api;
+// use crate::error::{ProjectError, ProjectResult};
 use crate::expr::{ExprHandle, GenClause, GenExpr, OwnedExpr};
-use crate::system::{atom_info_for, downcast_atom, DynSystem, DynSystemCard, SysCtx};
+use crate::system::{atom_info_for, downcast_atom, DynSystemCard, SysCtx};
 
 pub trait AtomCard: 'static + Sized {
   type Data: Clone + Coding + Sized;
@@ -37,6 +42,15 @@ pub trait AtomicFeatures: Atomic {
   type Info: AtomDynfo;
   const INFO: &'static Self::Info;
 }
+pub trait ToAtom {
+  fn to_atom_factory(self) -> AtomFactory;
+}
+impl<A: AtomicFeatures> ToAtom for A {
+  fn to_atom_factory(self) -> AtomFactory { self.factory() }
+}
+impl ToAtom for AtomFactory {
+  fn to_atom_factory(self) -> AtomFactory { self }
+}
 pub trait AtomicFeaturesImpl<Variant: AtomicVariant> {
   fn _factory(self) -> AtomFactory;
   type _Info: AtomDynfo;
@@ -48,37 +62,69 @@ impl<A: Atomic + AtomicFeaturesImpl<A::Variant> + ?Sized> AtomicFeatures for A {
   const INFO: &'static Self::Info = Self::_INFO;
 }
 
-pub fn get_info<A: AtomCard>(sys: &(impl DynSystemCard + ?Sized)) -> (u64, &'static dyn AtomDynfo) {
+pub fn get_info<A: AtomCard>(
+  sys: &(impl DynSystemCard + ?Sized)
+) -> (api::AtomId, &'static dyn AtomDynfo) {
   atom_info_for(sys, TypeId::of::<A>()).unwrap_or_else(|| {
     panic!("Atom {} not associated with system {}", type_name::<A>(), sys.name())
   })
 }
 
 #[derive(Clone)]
-pub struct ForeignAtom {
+pub struct ForeignAtom<'a> {
-  pub expr: ExprHandle,
+  pub expr: Option<ExprHandle>,
-  pub atom: Atom,
+  pub char_marker: PhantomData<&'a ()>,
+  pub ctx: SysCtx,
+  pub atom: api::Atom,
   pub pos: Pos,
 }
-impl ForeignAtom {
+impl<'a> ForeignAtom<'a> {
-  pub fn oex(self) -> OwnedExpr {
+  pub fn oex_opt(self) -> Option<OwnedExpr> {
-    let gen_expr = GenExpr { pos: self.pos, clause: GenClause::Atom(self.expr.tk, self.atom) };
+    self.expr.map(|handle| {
-    OwnedExpr { handle: self.expr, val: OnceLock::from(Box::new(gen_expr)) }
+      let gen_expr = GenExpr { pos: self.pos, clause: GenClause::Atom(handle.tk, self.atom) };
+      OwnedExpr { handle, val: OnceLock::from(Box::new(gen_expr)) }
+    })
   }
 }
+impl ForeignAtom<'static> {
+  pub fn oex(self) -> OwnedExpr { self.oex_opt().unwrap() }
+}
+impl<'a> fmt::Display for ForeignAtom<'a> {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{}::{:?}", if self.expr.is_some() { "Clause" } else { "Tok" }, self.atom)
+  }
+}
+impl<'a> AtomInTok for ForeignAtom<'a> {
+  type Context = SysCtx;
+  fn from_api(atom: &api::Atom, pos: Range<u32>, ctx: &mut Self::Context) -> Self {
+    Self {
+      atom: atom.clone(),
+      char_marker: PhantomData,
+      ctx: ctx.clone(),
+      expr: None,
+      pos: Pos::Range(pos),
+    }
+  }
+  fn to_api(&self) -> orchid_api::Atom { self.atom.clone() }
+}
 
 pub struct NotTypAtom(pub Pos, pub OwnedExpr, pub &'static dyn AtomDynfo);
-impl ProjectError for NotTypAtom {
+impl NotTypAtom {
-  const DESCRIPTION: &'static str = "Not the expected type";
+  pub fn mk_err(&self) -> OrcErr {
-  fn message(&self) -> String { format!("This expression is not a {}", self.2.name()) }
+    mk_err(
+      intern!(str: "Not the expected type"),
+      format!("This expression is not a {}", self.2.name()),
+      [self.0.clone().into()],
+    )
+  }
 }
 
 #[derive(Clone)]
-pub struct TypAtom<A: AtomicFeatures> {
+pub struct TypAtom<'a, A: AtomicFeatures> {
-  pub data: ForeignAtom,
+  pub data: ForeignAtom<'a>,
   pub value: A::Data,
 }
-impl<A: AtomicFeatures> TypAtom<A> {
+impl<A: AtomicFeatures> TypAtom<'static, A> {
   pub fn downcast(expr: ExprHandle) -> Result<Self, NotTypAtom> {
     match OwnedExpr::new(expr).foreign_atom() {
       Err(oe) => Err(NotTypAtom(oe.get_data().pos.clone(), oe, A::INFO)),
@@ -88,71 +134,77 @@ impl<A: AtomicFeatures> TypAtom<A> {
       },
     }
   }
+}
+impl<'a, A: AtomicFeatures> TypAtom<'a, A> {
   pub fn request<R: Coding + Into<A::Req> + Request>(&self, req: R) -> R::Response {
     R::Response::decode(
-      &mut &self.data.expr.ctx.reqnot.request(Fwd(self.data.atom.clone(), req.enc_vec()))[..],
+      &mut &self.data.ctx.reqnot.request(api::Fwd(self.data.atom.clone(), enc_vec(&req)))[..],
     )
   }
 }
-impl<A: AtomicFeatures> Deref for TypAtom<A> {
+impl<'a, A: AtomicFeatures> Deref for TypAtom<'a, A> {
   type Target = A::Data;
   fn deref(&self) -> &Self::Target { &self.value }
 }
 
-pub struct AtomCtx<'a>(pub &'a [u8], pub SysCtx);
+pub struct AtomCtx<'a>(pub &'a [u8], pub Option<api::AtomId>, pub SysCtx);
 
 pub trait AtomDynfo: Send + Sync + 'static {
   fn tid(&self) -> TypeId;
   fn name(&self) -> &'static str;
   fn decode(&self, ctx: AtomCtx<'_>) -> Box<dyn Any>;
-  fn call(&self, ctx: AtomCtx<'_>, arg: ExprTicket) -> GenExpr;
+  fn call(&self, ctx: AtomCtx<'_>, arg: api::ExprTicket) -> GenExpr;
-  fn call_ref(&self, ctx: AtomCtx<'_>, arg: ExprTicket) -> GenExpr;
+  fn call_ref(&self, ctx: AtomCtx<'_>, arg: api::ExprTicket) -> GenExpr;
-  fn same(&self, ctx: AtomCtx<'_>, buf2: &[u8]) -> bool;
+  fn same(&self, ctx: AtomCtx<'_>, other: &api::Atom) -> bool;
   fn print(&self, ctx: AtomCtx<'_>) -> String;
   fn handle_req(&self, ctx: AtomCtx<'_>, req: &mut dyn Read, rep: &mut dyn Write);
-  fn command(&self, ctx: AtomCtx<'_>) -> ProjectResult<Option<GenExpr>>;
+  fn command(&self, ctx: AtomCtx<'_>) -> OrcRes<Option<GenExpr>>;
+  fn serialize(&self, ctx: AtomCtx<'_>, write: &mut dyn Write) -> Vec<ExprTicket>;
+  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[ExprTicket]) -> api::Atom;
   fn drop(&self, ctx: AtomCtx<'_>);
 }
 
 trait_set! {
-  pub trait AtomFactoryFn = FnOnce(&dyn DynSystem) -> LocalAtom + DynClone + Send + Sync;
+  pub trait AtomFactoryFn = FnOnce(SysCtx) -> api::Atom + DynClone + Send + Sync;
 }
 pub struct AtomFactory(Box<dyn AtomFactoryFn>);
 impl AtomFactory {
-  pub fn new(f: impl FnOnce(&dyn DynSystem) -> LocalAtom + Clone + Send + Sync + 'static) -> Self {
+  pub fn new(f: impl FnOnce(SysCtx) -> api::Atom + Clone + Send + Sync + 'static) -> Self {
     Self(Box::new(f))
   }
-  pub fn build(self, sys: &dyn DynSystem) -> LocalAtom { (self.0)(sys) }
+  pub fn build(self, ctx: SysCtx) -> api::Atom { (self.0)(ctx) }
 }
 impl Clone for AtomFactory {
   fn clone(&self) -> Self { AtomFactory(clone_box(&*self.0)) }
 }
 
-pub struct ErrNotCallable;
+pub fn err_not_callable() -> OrcErr {
-impl ProjectError for ErrNotCallable {
+  mk_err(intern!(str: "This atom is not callable"), "Attempted to apply value as function", [])
-  const DESCRIPTION: &'static str = "This atom is not callable";
 }
 
-pub struct ErrorNotCommand;
+pub fn err_not_command() -> OrcErr {
-impl ProjectError for ErrorNotCommand {
+  mk_err(intern!(str: "This atom is not a command"), "Settled on an inactionable value", [])
-  const DESCRIPTION: &'static str = "This atom is not a command";
 }
 
 pub trait ReqPck<T: AtomCard + ?Sized>: Sized {
   type W: Write + ?Sized;
-  fn unpack<'a>(self) -> (T::Req, &'a mut Self::W)
+  fn unpack<'a>(self) -> (T::Req, &'a mut Self::W, SysCtx)
   where Self: 'a;
   fn never(self)
   where T: AtomCard<Req = Never> {
   }
 }
 
-pub struct RequestPack<'a, T: AtomCard + ?Sized, W: Write + ?Sized>(pub T::Req, pub &'a mut W);
+pub(crate) struct RequestPack<'a, T: AtomCard + ?Sized, W: Write + ?Sized>{
+  pub req: T::Req,
+  pub write: &'a mut W,
+  pub sys: SysCtx
+}
 
 impl<'a, T: AtomCard + ?Sized, W: Write + ?Sized> ReqPck<T> for RequestPack<'a, T, W> {
   type W = W;
-  fn unpack<'b>(self) -> (<T as AtomCard>::Req, &'b mut Self::W)
+  fn unpack<'b>(self) -> (<T as AtomCard>::Req, &'b mut Self::W, SysCtx)
   where 'a: 'b {
-    (self.0, self.1)
+    (self.req, self.write, self.sys)
   }
 }

orchid-extension/src/atom_owned.rs

@@ -2,18 +2,17 @@ use std::any::{type_name, Any, TypeId};
 use std::borrow::Cow;
 use std::io::{Read, Write};
 use std::marker::PhantomData;
-use std::num::NonZeroU64;
-use std::sync::Arc;
 
-use orchid_api::atom::LocalAtom;
+use itertools::Itertools;
-use orchid_api::expr::ExprTicket;
+use orchid_api_traits::{enc_vec, Decode, Encode};
-use orchid_api_traits::{Decode, Encode};
+use orchid_base::error::OrcRes;
 use orchid_base::id_store::{IdRecord, IdStore};
 
+use crate::api;
 use crate::atom::{
-  get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant, ErrNotCallable, ErrorNotCommand, ReqPck, RequestPack
+  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic,
+  AtomicFeaturesImpl, AtomicVariant, ReqPck, RequestPack,
 };
-use crate::error::ProjectResult;
 use crate::expr::{bot, ExprHandle, GenExpr};
 use crate::system::SysCtx;
 
@@ -21,56 +20,114 @@ pub struct OwnedVariant;
 impl AtomicVariant for OwnedVariant {}
 impl<A: OwnedAtom + Atomic<Variant = OwnedVariant>> AtomicFeaturesImpl<OwnedVariant> for A {
   fn _factory(self) -> AtomFactory {
-    AtomFactory::new(move |sys| {
+    AtomFactory::new(move |ctx| {
       let rec = OBJ_STORE.add(Box::new(self));
-      let mut data = get_info::<A>(sys.dyn_card()).0.enc_vec();
+      let (id, _) = get_info::<A>(ctx.cted.inst().card());
-      rec.id().encode(&mut data);
+      let mut data = enc_vec(&id);
       rec.encode(&mut data);
-      LocalAtom { drop: true, data }
+      api::Atom { drop: Some(api::AtomId(rec.id())), data, owner: ctx.id }
     })
   }
   type _Info = OwnedAtomDynfo<A>;
   const _INFO: &'static Self::_Info = &OwnedAtomDynfo(PhantomData);
 }
 
-fn with_atom<U>(mut b: &[u8], f: impl FnOnce(IdRecord<'_, Box<dyn DynOwnedAtom>>) -> U) -> U {
+fn with_atom<U>(id: api::AtomId, f: impl FnOnce(IdRecord<'_, Box<dyn DynOwnedAtom>>) -> U) -> U {
-  let id = NonZeroU64::decode(&mut b);
+  f(OBJ_STORE.get(id.0).unwrap_or_else(|| panic!("Received invalid atom ID: {}", id.0)))
-  f(OBJ_STORE.get(id).unwrap_or_else(|| panic!("Received invalid atom ID: {id}")))
 }
 
 pub struct OwnedAtomDynfo<T: OwnedAtom>(PhantomData<T>);
 impl<T: OwnedAtom> AtomDynfo for OwnedAtomDynfo<T> {
-  fn print(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> String {
+  fn print(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> String {
-    with_atom(buf, |a| a.dyn_print(ctx))
+    with_atom(id.unwrap(), |a| a.dyn_print(ctx))
   }
   fn tid(&self) -> TypeId { TypeId::of::<T>() }
   fn name(&self) -> &'static str { type_name::<T>() }
-  fn decode(&self, AtomCtx(data, _): AtomCtx) -> Box<dyn Any> {
+  fn decode(&self, AtomCtx(data, ..): AtomCtx) -> Box<dyn Any> {
     Box::new(<T as AtomCard>::Data::decode(&mut &data[..]))
   }
-  fn call(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
+  fn call(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> GenExpr {
-    with_atom(buf, |a| a.remove().dyn_call(ctx, arg))
+    with_atom(id.unwrap(), |a| a.remove().dyn_call(ctx, arg))
   }
-  fn call_ref(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
+  fn call_ref(&self, AtomCtx(_, id, ctx): AtomCtx, arg: api::ExprTicket) -> GenExpr {
-    with_atom(buf, |a| a.dyn_call_ref(ctx, arg))
+    with_atom(id.unwrap(), |a| a.dyn_call_ref(ctx, arg))
   }
-  fn same(&self, AtomCtx(buf, ctx): AtomCtx, buf2: &[u8]) -> bool {
+  fn same(&self, AtomCtx(_, id, ctx): AtomCtx, a2: &api::Atom) -> bool {
-    with_atom(buf, |a1| with_atom(buf2, |a2| a1.dyn_same(ctx, &**a2)))
+    with_atom(id.unwrap(), |a1| with_atom(a2.drop.unwrap(), |a2| a1.dyn_same(ctx, &**a2)))
   }
-  fn handle_req(&self, AtomCtx(buf, ctx): AtomCtx, req: &mut dyn Read, rep: &mut dyn Write) {
+  fn handle_req(&self, AtomCtx(_, id, ctx): AtomCtx, req: &mut dyn Read, rep: &mut dyn Write) {
-    with_atom(buf, |a| a.dyn_handle_req(ctx, req, rep))
+    with_atom(id.unwrap(), |a| a.dyn_handle_req(ctx, req, rep))
   }
-  fn command(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> ProjectResult<Option<GenExpr>> {
+  fn command(&self, AtomCtx(_, id, ctx): AtomCtx<'_>) -> OrcRes<Option<GenExpr>> {
-    with_atom(buf, |a| a.remove().dyn_command(ctx))
+    with_atom(id.unwrap(), |a| a.remove().dyn_command(ctx))
+  }
+  fn drop(&self, AtomCtx(_, id, ctx): AtomCtx) {
+    with_atom(id.unwrap(), |a| a.remove().dyn_free(ctx))
+  }
+  fn serialize(&self, AtomCtx(_, id, ctx): AtomCtx<'_>, write: &mut dyn Write) -> Vec<api::ExprTicket> {
+    let id = id.unwrap();
+    id.encode(write);
+    with_atom(id, |a| a.dyn_serialize(ctx, write)).into_iter().map(|t| t.into_tk()).collect_vec()
+  }
+  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[api::ExprTicket]) -> orchid_api::Atom {
+    let refs = refs.iter().map(|tk| ExprHandle::from_args(ctx.clone(), *tk));
+    let obj = T::deserialize(DeserCtxImpl(data, &ctx), T::Refs::from_iter(refs));
+    obj._factory().build(ctx)
+  }
+}
+
+pub trait DeserializeCtx: Sized {
+  fn read<T: Decode>(&mut self) -> T;
+  fn is_empty(&self) -> bool;
+  fn assert_empty(self) { assert!(self.is_empty(), "Bytes found after decoding") }
+  fn decode<T: Decode>(mut self) -> T {
+    let t = self.read();
+    self.assert_empty();
+    t
+  }
+  fn sys(&self) -> SysCtx;
+}
+
+struct DeserCtxImpl<'a>(&'a [u8], &'a SysCtx);
+impl<'a> DeserializeCtx for DeserCtxImpl<'a> {
+  fn read<T: Decode>(&mut self) -> T { T::decode(&mut self.0) }
+  fn is_empty(&self) -> bool { self.0.is_empty() }
+  fn sys(&self) -> SysCtx { self.1.clone() }
+}
+
+pub trait RefSet {
+  fn from_iter<I: Iterator<Item = ExprHandle> + ExactSizeIterator>(refs: I) -> Self;
+  fn to_vec(self) -> Vec<ExprHandle>;
+}
+
+impl RefSet for () {
+  fn to_vec(self) -> Vec<ExprHandle> { Vec::new() }
+  fn from_iter<I: Iterator<Item = ExprHandle> + ExactSizeIterator>(refs: I) -> Self {
+    assert_eq!(refs.len(), 0, "Expected no refs")
+  }
+}
+
+impl RefSet for Vec<ExprHandle> {
+  fn from_iter<I: Iterator<Item = ExprHandle> + ExactSizeIterator>(refs: I) -> Self {
+    refs.collect_vec()
+  }
+  fn to_vec(self) -> Vec<ExprHandle> { self }
+}
+
+impl<const N: usize> RefSet for [ExprHandle; N] {
+  fn to_vec(self) -> Vec<ExprHandle> { self.into_iter().collect_vec() }
+  fn from_iter<I: Iterator<Item = ExprHandle> + ExactSizeIterator>(refs: I) -> Self {
+    assert_eq!(refs.len(), N, "Wrong number of refs provided");
+    refs.collect_vec().try_into().unwrap_or_else(|_: Vec<_>| unreachable!())
   }
-  fn drop(&self, AtomCtx(buf, ctx): AtomCtx) { with_atom(buf, |a| a.remove().dyn_free(ctx)) }
 }
 
 /// Atoms that have a [Drop]
 pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone + 'static {
+  type Refs: RefSet;
   fn val(&self) -> Cow<'_, Self::Data>;
   #[allow(unused_variables)]
-  fn call_ref(&self, arg: ExprHandle) -> GenExpr { bot(ErrNotCallable) }
+  fn call_ref(&self, arg: ExprHandle) -> GenExpr { bot(err_not_callable()) }
   fn call(self, arg: ExprHandle) -> GenExpr {
     let ctx = arg.get_ctx();
     let gcl = self.call_ref(arg);
@@ -79,40 +136,42 @@ pub trait OwnedAtom: Atomic<Variant = OwnedVariant> + Send + Sync + Any + Clone
   }
   #[allow(unused_variables)]
   fn same(&self, ctx: SysCtx, other: &Self) -> bool {
-    eprintln!(
+    let tname = type_name::<Self>();
-      "Override OwnedAtom::same for {} if it can be generated during parsing",
+    writeln!(ctx.logger, "Override OwnedAtom::same for {tname} if it can appear in macro input");
-      type_name::<Self>()
-    );
     false
   }
-  fn handle_req(&self, ctx: SysCtx, pck: impl ReqPck<Self>);
+  fn handle_req(&self, pck: impl ReqPck<Self>);
   #[allow(unused_variables)]
-  fn command(self, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> { Err(Arc::new(ErrorNotCommand)) }
+  fn command(self, ctx: SysCtx) -> OrcRes<Option<GenExpr>> { Err(vec![err_not_command()]) }
   #[allow(unused_variables)]
   fn free(self, ctx: SysCtx) {}
   #[allow(unused_variables)]
   fn print(&self, ctx: SysCtx) -> String { format!("OwnedAtom({})", type_name::<Self>()) }
+  #[allow(unused_variables)]
+  fn serialize(&self, ctx: SysCtx, write: &mut (impl Write + ?Sized)) -> Self::Refs;
+  fn deserialize(ctx: impl DeserializeCtx, refs: Self::Refs) -> Self;
 }
 pub trait DynOwnedAtom: Send + Sync + 'static {
   fn atom_tid(&self) -> TypeId;
   fn as_any_ref(&self) -> &dyn Any;
   fn encode(&self, buffer: &mut dyn Write);
-  fn dyn_call_ref(&self, ctx: SysCtx, arg: ExprTicket) -> GenExpr;
+  fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> GenExpr;
-  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: ExprTicket) -> GenExpr;
+  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket) -> GenExpr;
   fn dyn_same(&self, ctx: SysCtx, other: &dyn DynOwnedAtom) -> bool;
   fn dyn_handle_req(&self, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write);
-  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> ProjectResult<Option<GenExpr>>;
+  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<GenExpr>>;
   fn dyn_free(self: Box<Self>, ctx: SysCtx);
   fn dyn_print(&self, ctx: SysCtx) -> String;
+  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<ExprHandle>;
 }
 impl<T: OwnedAtom> DynOwnedAtom for T {
   fn atom_tid(&self) -> TypeId { TypeId::of::<T>() }
   fn as_any_ref(&self) -> &dyn Any { self }
   fn encode(&self, buffer: &mut dyn Write) { self.val().as_ref().encode(buffer) }
-  fn dyn_call_ref(&self, ctx: SysCtx, arg: ExprTicket) -> GenExpr {
+  fn dyn_call_ref(&self, ctx: SysCtx, arg: api::ExprTicket) -> GenExpr {
     self.call_ref(ExprHandle::from_args(ctx, arg))
   }
-  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: ExprTicket) -> GenExpr {
+  fn dyn_call(self: Box<Self>, ctx: SysCtx, arg: api::ExprTicket) -> GenExpr {
     self.call(ExprHandle::from_args(ctx, arg))
   }
   fn dyn_same(&self, ctx: SysCtx, other: &dyn DynOwnedAtom) -> bool {
@@ -122,14 +181,16 @@ impl<T: OwnedAtom> DynOwnedAtom for T {
     let other_self = other.as_any_ref().downcast_ref().expect("The type_ids are the same");
     self.same(ctx, other_self)
   }
-  fn dyn_handle_req(&self, ctx: SysCtx, req: &mut dyn Read, rep: &mut dyn Write) {
+  fn dyn_handle_req(&self, sys: SysCtx, req: &mut dyn Read, write: &mut dyn Write) {
-    self.handle_req(ctx, RequestPack::<T, dyn Write>(<Self as AtomCard>::Req::decode(req), rep))
+    let pack = RequestPack::<T, dyn Write>{ req: <Self as AtomCard>::Req::decode(req), write, sys };
-  }
+    self.handle_req(pack)
-  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> {
-    self.command(ctx)
   }
+  fn dyn_command(self: Box<Self>, ctx: SysCtx) -> OrcRes<Option<GenExpr>> { self.command(ctx) }
   fn dyn_free(self: Box<Self>, ctx: SysCtx) { self.free(ctx) }
   fn dyn_print(&self, ctx: SysCtx) -> String { self.print(ctx) }
+  fn dyn_serialize(&self, ctx: SysCtx, sink: &mut dyn Write) -> Vec<ExprHandle> {
+    self.serialize(ctx, sink).to_vec()
+  }
 }
 
 pub(crate) static OBJ_STORE: IdStore<Box<dyn DynOwnedAtom>> = IdStore::new();

orchid-extension/src/atom_thin.rs

@@ -1,17 +1,16 @@
 use std::any::{type_name, Any, TypeId};
 use std::io::Write;
 use std::marker::PhantomData;
-use std::sync::Arc;
 
-use orchid_api::atom::LocalAtom;
+use orchid_api::ExprTicket;
-use orchid_api::expr::ExprTicket;
+use orchid_api_traits::{enc_vec, Coding, Decode};
-use orchid_api_traits::{Coding, Decode, Encode};
+use orchid_base::error::OrcRes;
 
+use crate::api;
 use crate::atom::{
-  get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic, AtomicFeaturesImpl, AtomicVariant,
+  err_not_callable, err_not_command, get_info, AtomCard, AtomCtx, AtomDynfo, AtomFactory, Atomic,
-  ErrNotCallable, ReqPck, RequestPack,
+  AtomicFeaturesImpl, AtomicVariant, ReqPck, RequestPack,
 };
-use crate::error::ProjectResult;
 use crate::expr::{bot, ExprHandle, GenExpr};
 use crate::system::SysCtx;
 
@@ -19,10 +18,11 @@ pub struct ThinVariant;
 impl AtomicVariant for ThinVariant {}
 impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant> for A {
   fn _factory(self) -> AtomFactory {
-    AtomFactory::new(move |sys| {
+    AtomFactory::new(move |ctx| {
-      let mut buf = get_info::<A>(sys.dyn_card()).0.enc_vec();
+      let (id, _) = get_info::<A>(ctx.cted.inst().card());
+      let mut buf = enc_vec(&id);
       self.encode(&mut buf);
-      LocalAtom { drop: false, data: buf }
+      api::Atom { drop: None, data: buf, owner: ctx.id }
     })
   }
   type _Info = ThinAtomDynfo<Self>;
@@ -31,48 +31,59 @@ impl<A: ThinAtom + Atomic<Variant = ThinVariant>> AtomicFeaturesImpl<ThinVariant
 
 pub struct ThinAtomDynfo<T: ThinAtom>(PhantomData<T>);
 impl<T: ThinAtom> AtomDynfo for ThinAtomDynfo<T> {
-  fn print(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> String { T::decode(&mut &buf[..]).print(ctx) }
+  fn print(&self, AtomCtx(buf, _, ctx): AtomCtx<'_>) -> String {
+    T::decode(&mut &buf[..]).print(ctx)
+  }
   fn tid(&self) -> TypeId { TypeId::of::<T>() }
   fn name(&self) -> &'static str { type_name::<T>() }
-  fn decode(&self, AtomCtx(buf, _): AtomCtx) -> Box<dyn Any> { Box::new(T::decode(&mut &buf[..])) }
+  fn decode(&self, AtomCtx(buf, ..): AtomCtx) -> Box<dyn Any> { Box::new(T::decode(&mut &buf[..])) }
-  fn call(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
+  fn call(&self, AtomCtx(buf, _, ctx): AtomCtx, arg: api::ExprTicket) -> GenExpr {
     T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg))
   }
-  fn call_ref(&self, AtomCtx(buf, ctx): AtomCtx, arg: ExprTicket) -> GenExpr {
+  fn call_ref(&self, AtomCtx(buf, _, ctx): AtomCtx, arg: api::ExprTicket) -> GenExpr {
     T::decode(&mut &buf[..]).call(ExprHandle::from_args(ctx, arg))
   }
   fn handle_req(
     &self,
-    AtomCtx(buf, ctx): AtomCtx,
+    AtomCtx(buf, _, sys): AtomCtx,
     req: &mut dyn std::io::Read,
-    rep: &mut dyn Write,
+    write: &mut dyn Write,
   ) {
-    T::decode(&mut &buf[..]).handle_req(ctx, RequestPack::<T, dyn Write>(Decode::decode(req), rep))
+    let pack = RequestPack::<T, dyn Write>{ req: Decode::decode(req), write, sys };
+    T::decode(&mut &buf[..]).handle_req(pack)
   }
-  fn same(&self, AtomCtx(buf, ctx): AtomCtx, buf2: &[u8]) -> bool {
+  fn same(&self, AtomCtx(buf, _, ctx): AtomCtx, a2: &api::Atom) -> bool {
-    T::decode(&mut &buf[..]).same(ctx, &T::decode(&mut &buf2[..]))
+    T::decode(&mut &buf[..]).same(ctx, &T::decode(&mut &a2.data[8..]))
   }
-  fn command(&self, AtomCtx(buf, ctx): AtomCtx<'_>) -> ProjectResult<Option<GenExpr>> {
+  fn command(&self, AtomCtx(buf, _, ctx): AtomCtx<'_>) -> OrcRes<Option<GenExpr>> {
     T::decode(&mut &buf[..]).command(ctx)
   }
-  fn drop(&self, AtomCtx(buf, ctx): AtomCtx) {
+  fn serialize(&self, AtomCtx(buf, _, _): AtomCtx<'_>, write: &mut dyn Write) -> Vec<ExprTicket> {
-    let string_self = T::decode(&mut &buf[..]).print(ctx);
+    T::decode(&mut &buf[..]).encode(write);
-    eprintln!("Received drop signal for non-drop atom {string_self:?}")
+    Vec::new()
+  }
+  fn deserialize(&self, ctx: SysCtx, data: &[u8], refs: &[ExprTicket]) -> api::Atom {
+    assert!(refs.is_empty(), "Refs found when deserializing thin atom");
+    T::decode(&mut &data[..])._factory().build(ctx)
+  }
+  fn drop(&self, AtomCtx(buf, _, ctx): AtomCtx) {
+    let string_self = T::decode(&mut &buf[..]).print(ctx.clone());
+    writeln!(ctx.logger, "Received drop signal for non-drop atom {string_self:?}");
   }
 }
 
-pub trait ThinAtom: AtomCard<Data = Self> + Coding + Send + Sync + 'static {
+pub trait ThinAtom: AtomCard<Data = Self> + Atomic<Variant = ThinVariant> + Coding + Send + Sync + 'static {
   #[allow(unused_variables)]
-  fn call(&self, arg: ExprHandle) -> GenExpr { bot(ErrNotCallable) }
+  fn call(&self, arg: ExprHandle) -> GenExpr { bot(err_not_callable()) }
   #[allow(unused_variables)]
   fn same(&self, ctx: SysCtx, other: &Self) -> bool {
     let tname = type_name::<Self>();
-    eprintln!("Override ThinAtom::same for {tname} if it can be generated during parsing");
+    writeln!(ctx.logger, "Override ThinAtom::same for {tname} if it can appear in macro input");
     false
   }
-  fn handle_req(&self, ctx: SysCtx, pck: impl ReqPck<Self>);
+  fn handle_req(&self, pck: impl ReqPck<Self>);
   #[allow(unused_variables)]
-  fn command(&self, ctx: SysCtx) -> ProjectResult<Option<GenExpr>> { Err(Arc::new(ErrNotCallable)) }
+  fn command(&self, ctx: SysCtx) -> OrcRes<Option<GenExpr>> { Err(vec![err_not_command()]) }
   #[allow(unused_variables)]
   fn print(&self, ctx: SysCtx) -> String { format!("ThinAtom({})", type_name::<Self>()) }
 }

orchid-extension/src/conv.rs

@@ -1,42 +1,39 @@
+use orchid_base::error::{mk_err, OrcErr, OrcRes};
+use orchid_base::intern;
 use orchid_base::location::Pos;
 
-use crate::atom::{AtomicFeatures, TypAtom};
+use crate::atom::{AtomicFeatures, ToAtom, TypAtom};
-use crate::error::{ProjectError, ProjectResult};
+use crate::expr::{atom, botv, ExprHandle, GenExpr, OwnedExpr};
-use crate::expr::{atom, bot_obj, ExprHandle, GenExpr, OwnedExpr};
 use crate::system::downcast_atom;
 
 pub trait TryFromExpr: Sized {
-  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self>;
+  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self>;
 }
 
 impl TryFromExpr for OwnedExpr {
-  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> { Ok(OwnedExpr::new(expr)) }
+  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> { Ok(OwnedExpr::new(expr)) }
 }
 
 impl<T: TryFromExpr, U: TryFromExpr> TryFromExpr for (T, U) {
-  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
     Ok((T::try_from_expr(expr.clone())?, U::try_from_expr(expr)?))
   }
 }
 
-pub struct ErrorNotAtom(Pos);
+fn err_not_atom(pos: Pos) -> OrcErr {
-impl ProjectError for ErrorNotAtom {
+  mk_err(intern!(str: "Expected an atom"), "This expression is not an atom", [pos.into()])
-  const DESCRIPTION: &'static str = "Expected an atom";
-  fn one_position(&self) -> Pos { self.0.clone() }
 }
 
-pub struct ErrorUnexpectedType(Pos);
+fn err_type(pos: Pos) -> OrcErr {
-impl ProjectError for ErrorUnexpectedType {
+  mk_err(intern!(str: "Type error"), "The atom is a different type than expected", [pos.into()])
-  const DESCRIPTION: &'static str = "Type error";
-  fn one_position(&self) -> Pos { self.0.clone() }
 }
 
-impl<A: AtomicFeatures> TryFromExpr for TypAtom<A> {
+impl<'a, A: AtomicFeatures> TryFromExpr for TypAtom<'a, A> {
-  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
     OwnedExpr::new(expr)
       .foreign_atom()
-      .map_err(|ex| ErrorNotAtom(ex.pos.clone()).pack())
+      .map_err(|ex| vec![err_not_atom(ex.pos.clone())])
-      .and_then(|f| downcast_atom(f).map_err(|f| ErrorUnexpectedType(f.pos).pack()))
+      .and_then(|f| downcast_atom(f).map_err(|f| vec![err_type(f.pos)]))
   }
 }
 
@@ -44,15 +41,19 @@ pub trait ToExpr {
   fn to_expr(self) -> GenExpr;
 }
 
-impl<T: ToExpr> ToExpr for ProjectResult<T> {
+impl ToExpr for GenExpr {
+  fn to_expr(self) -> GenExpr { self }
+}
+
+impl<T: ToExpr> ToExpr for OrcRes<T> {
   fn to_expr(self) -> GenExpr {
     match self {
-      Err(e) => bot_obj(e),
+      Err(e) => botv(e),
       Ok(t) => t.to_expr(),
     }
   }
 }
 
-impl<A: AtomicFeatures> ToExpr for A {
+impl<A: ToAtom> ToExpr for A {
   fn to_expr(self) -> GenExpr { atom(self) }
 }

orchid-extension/src/entrypoint.rs

@@ -6,120 +6,133 @@ use std::{mem, process, thread};
 
 use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::atom::{
+use orchid_api::DeserAtom;
-  Atom, AtomDrop, AtomPrint, AtomReq, AtomSame, CallRef, Command, FinalCall, Fwded, NextStep
+use orchid_api_traits::{enc_vec, Decode, Encode};
-};
-use orchid_api::interner::Sweep;
-use orchid_api::parser::{CharFilter, LexExpr, LexedExpr, ParserReq};
-use orchid_api::proto::{ExtMsgSet, ExtensionHeader, HostExtNotif, HostExtReq, HostHeader, Ping};
-use orchid_api::system::{SysDeclId, SysId, SystemDrop, SystemInst};
-use orchid_api::tree::{GetMember, TreeId};
-use orchid_api::vfs::{EagerVfs, GetVfs, VfsId, VfsRead, VfsReq};
-use orchid_api_traits::{Decode, Encode};
 use orchid_base::char_filter::{char_filter_match, char_filter_union, mk_char_filter};
 use orchid_base::clone;
+use orchid_base::error::errv_to_apiv;
 use orchid_base::interner::{deintern, init_replica, sweep_replica};
 use orchid_base::logging::Logger;
-use orchid_base::name::PathSlice;
+use orchid_base::name::{PathSlice, Sym};
+use orchid_base::parse::Snippet;
 use orchid_base::reqnot::{ReqNot, Requester};
+use orchid_base::tree::{ttv_from_api, ttv_to_api};
+use substack::Substack;
 
+use crate::api;
 use crate::atom::{AtomCtx, AtomDynfo};
-use crate::error::errv_to_apiv;
+use crate::atom_owned::OBJ_STORE;
 use crate::fs::VirtFS;
-use crate::lexer::{CascadingError, LexContext, NotApplicableLexerError};
+use crate::lexer::{err_cascade, err_lexer_na, LexContext};
 use crate::msg::{recv_parent_msg, send_parent_msg};
-use crate::system::{atom_by_idx, resolv_atom, SysCtx};
+use crate::system::{atom_by_idx, SysCtx};
 use crate::system_ctor::{CtedObj, DynSystemCtor};
-use crate::tree::{LazyMemberFactory, TIACtxImpl};
+use crate::tree::{do_extra, GenTok, GenTokTree, LazyMemberFactory, TIACtxImpl};
 
 pub struct ExtensionData {
-  pub thread_name: &'static str,
+  pub name: &'static str,
   pub systems: &'static [&'static dyn DynSystemCtor],
 }
 impl ExtensionData {
-  pub fn new(thread_name: &'static str, systems: &'static [&'static dyn DynSystemCtor]) -> Self {
+  pub fn new(name: &'static str, systems: &'static [&'static dyn DynSystemCtor]) -> Self {
-    Self { thread_name, systems }
+    Self { name, systems }
-  }
-  pub fn main(self) {
-    extension_main(self)
   }
+  pub fn main(self) { extension_main(self) }
 }
 
 pub enum MemberRecord {
-  Gen(LazyMemberFactory),
+  Gen(Sym, LazyMemberFactory),
   Res,
 }
 
 pub struct SystemRecord {
   cted: CtedObj,
-  vfses: HashMap<VfsId, &'static dyn VirtFS>,
+  vfses: HashMap<api::VfsId, &'static dyn VirtFS>,
-  declfs: EagerVfs,
+  declfs: api::EagerVfs,
-  lazy_members: HashMap<TreeId, MemberRecord>,
+  lazy_members: HashMap<api::TreeId, MemberRecord>,
 }
 
 pub fn with_atom_record<T>(
-  systems: &Mutex<HashMap<SysId, SystemRecord>>,
+  get_sys_ctx: &impl Fn(api::SysId, ReqNot<api::ExtMsgSet>) -> SysCtx,
-  atom: &Atom,
+  reqnot: ReqNot<api::ExtMsgSet>,
-  cb: impl FnOnce(&'static dyn AtomDynfo, CtedObj, &[u8]) -> T,
+  atom: &api::Atom,
+  cb: impl FnOnce(&'static dyn AtomDynfo, SysCtx, api::AtomId, &[u8]) -> T,
 ) -> T {
   let mut data = &atom.data[..];
-  let systems_g = systems.lock().unwrap();
+  let ctx = get_sys_ctx(atom.owner, reqnot);
-  let cted = &systems_g[&atom.owner].cted;
+  let inst = ctx.cted.inst();
-  let sys = cted.inst();
+  let id = api::AtomId::decode(&mut data);
-  let atom_record = atom_by_idx(sys.dyn_card(), u64::decode(&mut data)).expect("Atom ID reserved");
+  let atom_record = atom_by_idx(inst.card(), id).expect("Atom ID reserved");
-  cb(atom_record, cted.clone(), data)
+  cb(atom_record, ctx, id, data)
 }
 
 pub fn extension_main(data: ExtensionData) {
-  if thread::Builder::new().name(data.thread_name.to_string()).spawn(|| extension_main_logic(data)).unwrap().join().is_err() {
+  if thread::Builder::new()
+    .name(format!("ext-main:{}", data.name))
+    .spawn(|| extension_main_logic(data))
+    .unwrap()
+    .join()
+    .is_err()
+  {
     process::exit(-1)
   }
 }
 
 fn extension_main_logic(data: ExtensionData) {
-  let HostHeader{ log_strategy } = HostHeader::decode(&mut std::io::stdin().lock());
+  let api::HostHeader { log_strategy } = api::HostHeader::decode(&mut std::io::stdin().lock());
   let mut buf = Vec::new();
   let decls = (data.systems.iter().enumerate())
     .map(|(id, sys)| (u16::try_from(id).expect("more than u16max system ctors"), sys))
-    .map(|(id, sys)| sys.decl(SysDeclId(NonZero::new(id + 1).unwrap())))
+    .map(|(id, sys)| sys.decl(api::SysDeclId(NonZero::new(id + 1).unwrap())))
     .collect_vec();
-  let systems = Arc::new(Mutex::new(HashMap::<SysId, SystemRecord>::new()));
+  let systems = Arc::new(Mutex::new(HashMap::<api::SysId, SystemRecord>::new()));
-  ExtensionHeader { systems: decls.clone() }.encode(&mut buf);
+  api::ExtensionHeader { name: data.name.to_string(), systems: decls.clone() }.encode(&mut buf);
   std::io::stdout().write_all(&buf).unwrap();
   std::io::stdout().flush().unwrap();
   let exiting = Arc::new(AtomicBool::new(false));
   let logger = Arc::new(Logger::new(log_strategy));
-  let rn = ReqNot::<ExtMsgSet>::new(
+  let mk_ctx = clone!(logger, systems; move |id: api::SysId, reqnot: ReqNot<api::ExtMsgSet>| {
-    |a, _| {
+    let cted = systems.lock().unwrap()[&id].cted.clone();
-      eprintln!("Upsending {:?}", a);
+    SysCtx { id, cted, logger: logger.clone(), reqnot }
+  });
+  let rn = ReqNot::<api::ExtMsgSet>::new(
+    clone!(logger; move |a, _| {
+      logger.log_buf("Upsending", a);
       send_parent_msg(a).unwrap()
-    },
+    }),
-    clone!(systems, exiting, logger; move |n, reqnot| match n {
+    clone!(systems, exiting, mk_ctx; move |n, reqnot| match n {
-      HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
+      api::HostExtNotif::Exit => exiting.store(true, Ordering::Relaxed),
-      HostExtNotif::SystemDrop(SystemDrop(sys_id)) =>
+      api::HostExtNotif::SystemDrop(api::SystemDrop(sys_id)) =>
         mem::drop(systems.lock().unwrap().remove(&sys_id)),
-      HostExtNotif::AtomDrop(AtomDrop(atom)) => {
+      api::HostExtNotif::AtomDrop(api::AtomDrop(sys_id, atom)) =>
-        with_atom_record(&systems, &atom, |rec, cted, data| {
+        OBJ_STORE.get(atom.0).unwrap().remove().dyn_free(mk_ctx(sys_id, reqnot)),
-          rec.drop(AtomCtx(data, SysCtx{ reqnot, logger: logger.clone(), id: atom.owner, cted }))
-        })
-      }
     }),
     clone!(systems, logger; move |req| match req.req() {
-      HostExtReq::Ping(ping@Ping) => req.handle(ping, &()),
+      api::HostExtReq::Ping(ping@api::Ping) => req.handle(ping, &()),
-      HostExtReq::Sweep(sweep@Sweep) => req.handle(sweep, &sweep_replica()),
+      api::HostExtReq::Sweep(sweep@api::Sweep) => req.handle(sweep, &sweep_replica()),
-      HostExtReq::NewSystem(new_sys) => {
+      api::HostExtReq::SysReq(api::SysReq::NewSystem(new_sys)) => {
         let i = decls.iter().enumerate().find(|(_, s)| s.id == new_sys.system).unwrap().0;
         let cted = data.systems[i].new_system(new_sys);
         let mut vfses = HashMap::new();
-        let lex_filter = cted.inst().dyn_lexers().iter().fold(CharFilter(vec![]), |cf, lx| {
+        let lex_filter = cted.inst().dyn_lexers().iter().fold(api::CharFilter(vec![]), |cf, lx| {
           let lxcf = mk_char_filter(lx.char_filter().iter().cloned());
           char_filter_union(&cf, &lxcf)
         });
         let mut lazy_mems = HashMap::new();
+        let ctx = SysCtx{
+          cted: cted.clone(),
+          id: new_sys.id,
+          logger: logger.clone(),
+          reqnot: req.reqnot()
+        };
+        let mut tia_ctx = TIACtxImpl{
+          lazy: &mut lazy_mems,
+          ctx: ctx.clone(),
+          basepath: &[],
+          path: Substack::Bottom,
+        };
         let const_root = (cted.inst().dyn_env().into_iter())
-          .map(|(k, v)| {
+          .map(|(k, v)| (k.marker(), v.into_api(&mut tia_ctx)))
-            (k.marker(), v.into_api(&mut TIACtxImpl{ lazy: &mut lazy_mems, sys: &*cted.inst()}))
-          })
           .collect();
         systems.lock().unwrap().insert(new_sys.id, SystemRecord {
           declfs: cted.inst().dyn_vfs().to_api_rec(&mut vfses),
@@ -127,106 +140,128 @@ fn extension_main_logic(data: ExtensionData) {
           cted,
           lazy_members: lazy_mems
         });
-        req.handle(new_sys, &SystemInst {
+        req.handle(new_sys, &api::SystemInst {
           lex_filter,
           const_root,
-          parses_lines: vec!()
+          line_types: vec![]
         })
       }
-      HostExtReq::GetMember(get_tree@GetMember(sys_id, tree_id)) => {
+      api::HostExtReq::GetMember(get_tree@api::GetMember(sys_id, tree_id)) => {
         let mut systems_g = systems.lock().unwrap();
         let sys = systems_g.get_mut(sys_id).expect("System not found");
         let lazy = &mut sys.lazy_members;
-        let cb = match lazy.insert(*tree_id, MemberRecord::Res) {
+        let (path, cb) = match lazy.insert(*tree_id, MemberRecord::Res) {
           None => panic!("Tree for ID not found"),
           Some(MemberRecord::Res) => panic!("This tree has already been transmitted"),
-          Some(MemberRecord::Gen(cb)) => cb,
+          Some(MemberRecord::Gen(path, cb)) => (path, cb),
         };
-        let tree = cb.build();
+        let tree = cb.build(path.clone());
-        let reply_tree = tree.into_api(&mut TIACtxImpl{ sys: &*sys.cted.inst(), lazy });
+        let ctx = SysCtx::new(*sys_id, &sys.cted, &logger, req.reqnot());
-        req.handle(get_tree, &reply_tree);
+        let reply_tree = tree.into_api(&mut TIACtxImpl{ ctx: ctx.clone(), lazy, path: Substack::Bottom, basepath: &path });
+        req.handle(get_tree, &reply_tree)
       }
-      HostExtReq::VfsReq(VfsReq::GetVfs(get_vfs@GetVfs(sys_id))) => {
+      api::HostExtReq::VfsReq(api::VfsReq::GetVfs(get_vfs@api::GetVfs(sys_id))) => {
         let systems_g = systems.lock().unwrap();
         req.handle(get_vfs, &systems_g[sys_id].declfs)
       }
-      HostExtReq::VfsReq(VfsReq::VfsRead(vfs_read@VfsRead(sys_id, vfs_id, path))) => {
+      api::HostExtReq::VfsReq(api::VfsReq::VfsRead(vfs_read)) => {
+        let api::VfsRead(sys_id, vfs_id, path) = vfs_read;
         let systems_g = systems.lock().unwrap();
         let path = path.iter().map(|t| deintern(*t)).collect_vec();
         req.handle(vfs_read, &systems_g[sys_id].vfses[vfs_id].load(PathSlice::new(&path)))
       }
-      HostExtReq::ParserReq(ParserReq::LexExpr(lex)) => {
+      api::HostExtReq::ParserReq(api::ParserReq::LexExpr(lex)) => {
-        let LexExpr{ sys, text, pos, id } = *lex;
+        let api::LexExpr{ sys, text, pos, id } = *lex;
         let systems_g = systems.lock().unwrap();
         let lexers = systems_g[&sys].cted.inst().dyn_lexers();
         mem::drop(systems_g);
         let text = deintern(text);
-        let tk = req.will_handle_as(lex);
-        thread::spawn(clone!(systems; move || {
         let ctx = LexContext { sys, id, pos, reqnot: req.reqnot(), text: &text };
         let trigger_char = text.chars().nth(pos as usize).unwrap();
         for lx in lexers.iter().filter(|l| char_filter_match(l.char_filter(), trigger_char)) {
           match lx.lex(&text[pos as usize..], &ctx) {
-              Err(e) if e.as_any_ref().is::<NotApplicableLexerError>() => continue,
+            Err(e) if e.iter().any(|e| *e == err_lexer_na()) => continue,
-              Err(e) if e.as_any_ref().is::<CascadingError>() => return req.handle_as(tk, &None),
+            Err(e) => {
-              Err(e) => return req.handle_as(tk, &Some(Err(errv_to_apiv([e])))),
+              let errv = errv_to_apiv(e.iter().filter(|e| **e == err_cascade()));
-              Ok((s, expr)) => {
+              return req.handle(lex, &if errv.is_empty() { None } else { Some(Err(errv))})
-                let systems_g = systems.lock().unwrap();
-                let expr = expr.into_api(&*systems_g[&sys].cted.inst());
-                let pos = (text.len() - s.len()) as u32;
-                return req.handle_as(tk, &Some(Ok(LexedExpr{ pos, expr })))
-              }
-            }
-          }
-          eprintln!("Got notified about n/a character '{trigger_char}'");
-          req.handle_as(tk, &None)
-        }));
             },
-      HostExtReq::AtomReq(atom_req) => {
+            Ok((s, expr)) => {
-        let systems_g = systems.lock().unwrap();
+              let ctx = mk_ctx(sys, req.reqnot());
-        let atom = atom_req.get_atom();
+              let expr = expr.to_api(&mut |f, r| do_extra(f, r, ctx.clone()));
-        let sys = &systems_g[&atom.owner];
+              let pos = (text.len() - s.len()) as u32;
-        let ctx = SysCtx {
+              return req.handle(lex, &Some(Ok(api::LexedExpr{ pos, expr })))
-          cted: sys.cted.clone(),
+            }
-          id: atom.owner,
+          }
-          logger: logger.clone(),
+        }
-          reqnot: req.reqnot()
+        writeln!(logger, "Got notified about n/a character '{trigger_char}'");
+        req.handle(lex, &None)
+      },
+      api::HostExtReq::ParserReq(api::ParserReq::ParseLine(pline@api::ParseLine{ sys, line })) => {
+        let mut ctx = mk_ctx(*sys, req.reqnot());
+        let parsers = ctx.cted.inst().dyn_parsers();
+        let line: Vec<GenTokTree> = ttv_from_api(line, &mut ctx);
+        let snip = Snippet::new(line.first().expect("Empty line"), &line);
+        let (head, tail) = snip.pop_front().unwrap();
+        let name = if let GenTok::Name(n) = &head.tok { n } else { panic!("No line head") };
+        let parser = parsers.iter().find(|p| p.line_head() == **name).expect("No parser candidate");
+        let o_line = match parser.parse(tail) {
+          Err(e) => Err(errv_to_apiv(e.iter())),
+          Ok(t) => Ok(ttv_to_api(t, &mut |f, range| {
+            api::TokenTree{ range, token: api::Token::Atom(f.clone().build(ctx.clone())) }
+          })),
         };
-        let dynfo = resolv_atom(&*sys.cted.inst(), atom);
+        req.handle(pline, &o_line)
-        let actx = AtomCtx(&atom.data[8..], ctx);
+      }
+      api::HostExtReq::AtomReq(atom_req) => {
+        let atom = atom_req.get_atom();
+        with_atom_record(&mk_ctx, req.reqnot(), atom, |nfo, ctx, id, buf| {
+          let actx = AtomCtx(buf, atom.drop, ctx.clone());
           match atom_req {
-          AtomReq::AtomPrint(print@AtomPrint(_)) => req.handle(print, &dynfo.print(actx)),
+            api::AtomReq::SerializeAtom(ser) => {
-          AtomReq::AtomSame(same@AtomSame(_, r)) => {
+              let mut buf = enc_vec(&id);
+              let refs = nfo.serialize(actx, &mut buf);
+              req.handle(ser, &(buf, refs))
+            }
+            api::AtomReq::AtomPrint(print@api::AtomPrint(_)) => req.handle(print, &nfo.print(actx)),
+            api::AtomReq::AtomSame(same@api::AtomSame(_, r)) => {
               // different systems or different type tags
-            if atom.owner != r.owner || atom.data[..8] != r.data[..8] {
+              if atom.owner != r.owner || buf != &r.data[..8] {
                 return req.handle(same, &false)
               }
-            req.handle(same, &dynfo.same(actx, &r.data[8..]))
+              req.handle(same, &nfo.same(actx, r))
             },
-          AtomReq::Fwded(fwded@Fwded(_, payload)) => {
+            api::AtomReq::Fwded(fwded@api::Fwded(_, payload)) => {
               let mut reply = Vec::new();
-            dynfo.handle_req(actx, &mut &payload[..], &mut reply);
+              nfo.handle_req(actx, &mut &payload[..], &mut reply);
               req.handle(fwded, &reply)
             }
-          AtomReq::CallRef(call@CallRef(_, arg))
+            api::AtomReq::CallRef(call@api::CallRef(_, arg))
-            => req.handle(call, &dynfo.call_ref(actx, *arg).to_api(&*sys.cted.inst())),
+              => req.handle(call, &nfo.call_ref(actx, *arg).to_api(ctx.clone())),
-          AtomReq::FinalCall(call@FinalCall(_, arg))
+            api::AtomReq::FinalCall(call@api::FinalCall(_, arg))
-            => req.handle(call, &dynfo.call(actx, *arg).to_api(&*sys.cted.inst())),
+              => req.handle(call, &nfo.call(actx, *arg).to_api(ctx.clone())),
-          AtomReq::Command(cmd@Command(_)) => req.handle(cmd, &match dynfo.command(actx) {
+            api::AtomReq::Command(cmd@api::Command(_)) => req.handle(cmd, &match nfo.command(actx) {
-            Err(e) => Err(errv_to_apiv([e])),
+              Err(e) => Err(errv_to_apiv(e.iter())),
               Ok(opt) => Ok(match opt {
-              Some(cont) => NextStep::Continue(cont.into_api(&*sys.cted.inst())),
+                Some(cont) => api::NextStep::Continue(cont.into_api(ctx.clone())),
-              None => NextStep::Halt,
+                None => api::NextStep::Halt,
               })
             })
           }
+        })
+      },
+      api::HostExtReq::DeserAtom(deser@DeserAtom(sys, buf, refs)) => {
+        let mut read = &mut &buf[..];
+        let ctx = mk_ctx(*sys, req.reqnot());
+        let id = api::AtomId::decode(&mut read);
+        let inst = ctx.cted.inst();
+        let nfo = atom_by_idx(inst.card(), id).expect("Deserializing atom with invalid ID");
+        req.handle(deser, &nfo.deserialize(ctx.clone(), read, refs))
       }
     }),
   );
   init_replica(rn.clone().map());
   while !exiting.load(Ordering::Relaxed) {
     let rcvd = recv_parent_msg().unwrap();
-    // eprintln!("Downsent {rcvd:?}");
     rn.receive(rcvd)
   }
 }

orchid-extension/src/error.rs

@@ -6,15 +6,15 @@ use std::{fmt, iter};
 
 use dyn_clone::{clone_box, DynClone};
 use itertools::Itertools;
-use orchid_api::error::{GetErrorDetails, ProjErr, ProjErrId};
-use orchid_api::proto::ExtMsgSet;
 use orchid_base::boxed_iter::{box_once, BoxedIter};
 use orchid_base::clone;
-use orchid_base::error::{ErrorPosition, OwnedError};
+use orchid_base::error::{ErrPos, OrcError};
 use orchid_base::interner::{deintern, intern};
 use orchid_base::location::{GetSrc, Pos};
 use orchid_base::reqnot::{ReqNot, Requester};
 
+use crate::api;
+
 /// Errors addressed to the developer which are to be resolved with
 /// code changes
 pub trait ProjectError: Sized + Send + Sync + 'static {
@@ -26,8 +26,8 @@ pub trait ProjectError: Sized + Send + Sync + 'static {
   /// Code positions relevant to this error. If you don't implement this, you
   /// must implement [ProjectError::one_position]
   #[must_use]
-  fn positions(&self) -> impl IntoIterator<Item = ErrorPosition> + '_ {
+  fn positions(&self) -> impl IntoIterator<Item = ErrPos> + '_ {
-    box_once(ErrorPosition { position: self.one_position(), message: None })
+    box_once(ErrPos { position: self.one_position(), message: None })
   }
   /// Short way to provide a single origin. If you don't implement this, you
   /// must implement [ProjectError::positions]
@@ -58,7 +58,7 @@ pub trait DynProjectError: Send + Sync + 'static {
   fn message(&self) -> String { self.description().to_string() }
   /// Code positions relevant to this error.
   #[must_use]
-  fn positions(&self) -> BoxedIter<'_, ErrorPosition>;
+  fn positions(&self) -> BoxedIter<'_, ErrPos>;
 }
 
 impl<T> DynProjectError for T
@@ -68,9 +68,7 @@ where T: ProjectError
   fn into_packed(self: Arc<Self>) -> ProjectErrorObj { self }
   fn description(&self) -> Cow<'_, str> { Cow::Borrowed(T::DESCRIPTION) }
   fn message(&self) -> String { ProjectError::message(self) }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self) -> BoxedIter<ErrPos> { Box::new(ProjectError::positions(self).into_iter()) }
-    Box::new(ProjectError::positions(self).into_iter())
-  }
 }
 
 pub fn pretty_print(err: &dyn DynProjectError, get_src: &mut impl GetSrc) -> String {
@@ -82,7 +80,7 @@ pub fn pretty_print(err: &dyn DynProjectError, get_src: &mut impl GetSrc) -> Str
     head + "No origins specified"
   } else {
     iter::once(head)
-      .chain(positions.iter().map(|ErrorPosition { position: origin, message }| match message {
+      .chain(positions.iter().map(|ErrPos { position: origin, message }| match message {
         None => format!("@{}", origin.pretty_print(get_src)),
         Some(msg) => format!("@{}: {msg}", origin.pretty_print(get_src)),
       }))
@@ -95,7 +93,7 @@ impl DynProjectError for ProjectErrorObj {
   fn description(&self) -> Cow<'_, str> { (**self).description() }
   fn into_packed(self: Arc<Self>) -> ProjectErrorObj { (*self).clone() }
   fn message(&self) -> String { (**self).message() }
-  fn positions(&self) -> BoxedIter<'_, ErrorPosition> { (**self).positions() }
+  fn positions(&self) -> BoxedIter<'_, ErrPos> { (**self).positions() }
 }
 
 /// Type-erased [ProjectError] implementor through the [DynProjectError]
@@ -179,8 +177,8 @@ impl<T: ErrorSansOrigin> DynProjectError for OriginBundle<T> {
   fn into_packed(self: Arc<Self>) -> ProjectErrorObj { self }
   fn description(&self) -> Cow<'_, str> { self.1.description() }
   fn message(&self) -> String { self.1.message() }
-  fn positions(&self) -> BoxedIter<ErrorPosition> {
+  fn positions(&self) -> BoxedIter<ErrPos> {
-    box_once(ErrorPosition { position: self.0.clone(), message: None })
+    box_once(ErrPos { position: self.0.clone(), message: None })
   }
 }
 
@@ -259,7 +257,7 @@ struct MultiError(Vec<ProjectErrorObj>);
 impl ProjectError for MultiError {
   const DESCRIPTION: &'static str = "Multiple errors occurred";
   fn message(&self) -> String { format!("{} errors occurred", self.0.len()) }
-  fn positions(&self) -> impl IntoIterator<Item = ErrorPosition> + '_ {
+  fn positions(&self) -> impl IntoIterator<Item = ErrPos> + '_ {
     self.0.iter().flat_map(|e| {
       e.positions().map(|pos| {
         let emsg = e.message();
@@ -268,49 +266,35 @@ impl ProjectError for MultiError {
           Some(s) if s.is_empty() => emsg,
           Some(pmsg) => format!("{emsg}: {pmsg}"),
         };
-        ErrorPosition { position: pos.position, message: Some(Arc::new(msg)) }
+        ErrPos { position: pos.position, message: Some(Arc::new(msg)) }
       })
     })
   }
 }
 
-fn err_to_api(err: ProjectErrorObj) -> ProjErr {
+fn err_to_api(err: ProjectErrorObj) -> api::OrcErr {
-  ProjErr {
+  api::OrcErr {
     description: intern(&*err.description()).marker(),
     message: Arc::new(err.message()),
     locations: err.positions().map(|e| e.to_api()).collect_vec(),
   }
 }
 
-pub fn errv_to_apiv(errv: impl IntoIterator<Item = ProjectErrorObj>) -> Vec<ProjErr> {
-  errv.into_iter().flat_map(unpack_err).map(err_to_api).collect_vec()
-}
-
-pub fn err_from_apiv<'a>(
-  err: impl IntoIterator<Item = &'a ProjErr>,
-  reqnot: &ReqNot<ExtMsgSet>
-) -> ProjectErrorObj {
-  pack_err(err.into_iter().map(|e| {
-    let details: OnceLock<_> = OwnedError::from_api(e).into();
-    Arc::new(RelayedError { id: None, reqnot: reqnot.clone(), details }).into_packed()
-  }))
-}
-
 struct RelayedError {
-  pub id: Option<ProjErrId>,
+  pub id: Option<api::ErrId>,
-  pub reqnot: ReqNot<ExtMsgSet>,
+  pub reqnot: ReqNot<api::ExtMsgSet>,
-  pub details: OnceLock<OwnedError>,
+  pub details: OnceLock<OrcError>,
 }
 impl RelayedError {
-  fn details(&self) -> &OwnedError {
+  fn details(&self) -> &OrcError {
     let Self { id, reqnot, details: data } = self;
     data.get_or_init(clone!(reqnot; move || {
       let id = id.expect("Either data or ID must be initialized");
-      let projerr = reqnot.request(GetErrorDetails(id));
+      let projerr = reqnot.request(api::GetErrorDetails(id));
-      OwnedError {
+      OrcError {
         description: deintern(projerr.description),
         message: projerr.message,
-        positions: projerr.locations.iter().map(ErrorPosition::from_api).collect_vec(),
+        positions: projerr.locations.iter().map(ErrPos::from_api).collect_vec(),
       }
     }))
   }
@@ -320,7 +304,7 @@ impl DynProjectError for RelayedError {
   fn message(&self) -> String { self.details().message.to_string() }
   fn as_any_ref(&self) -> &dyn std::any::Any { self }
   fn into_packed(self: Arc<Self>) -> ProjectErrorObj { self }
-  fn positions(&self) -> BoxedIter<'_, ErrorPosition> {
+  fn positions(&self) -> BoxedIter<'_, ErrPos> {
     Box::new(self.details().positions.iter().cloned())
   }
 }

orchid-extension/src/expr.rs

@@ -1,25 +1,25 @@
+use std::marker::PhantomData;
 use std::ops::Deref;
-use std::sync::{Arc, OnceLock};
+use std::sync::OnceLock;
 
 use derive_destructure::destructure;
-use orchid_api::atom::Atom;
+use orchid_base::error::{errv_from_apiv, errv_to_apiv, OrcErr};
-use orchid_api::expr::{Acquire, Clause, Expr, ExprTicket, Inspect, Release};
 use orchid_base::interner::{deintern, Tok};
 use orchid_base::location::Pos;
 use orchid_base::reqnot::Requester;
 
-use crate::atom::{AtomFactory, AtomicFeatures, ForeignAtom};
+use crate::api;
-use crate::error::{err_from_apiv, errv_to_apiv, DynProjectError, ProjectErrorObj};
+use crate::atom::{AtomFactory, ForeignAtom, ToAtom};
-use crate::system::{DynSystem, SysCtx};
+use crate::system::SysCtx;
 
 #[derive(destructure)]
 pub struct ExprHandle {
-  pub tk: ExprTicket,
+  pub tk: api::ExprTicket,
   pub ctx: SysCtx,
 }
 impl ExprHandle {
-  pub(crate) fn from_args(ctx: SysCtx, tk: ExprTicket) -> Self { Self { ctx, tk } }
+  pub(crate) fn from_args(ctx: SysCtx, tk: api::ExprTicket) -> Self { Self { ctx, tk } }
-  pub(crate) fn into_tk(self) -> ExprTicket {
+  pub(crate) fn into_tk(self) -> api::ExprTicket {
     let (tk, ..) = self.destructure();
     tk
   }
@@ -27,12 +27,12 @@ impl ExprHandle {
 }
 impl Clone for ExprHandle {
   fn clone(&self) -> Self {
-    self.ctx.reqnot.notify(Acquire(self.ctx.id, self.tk));
+    self.ctx.reqnot.notify(api::Acquire(self.ctx.id, self.tk));
     Self { ctx: self.ctx.clone(), tk: self.tk }
   }
 }
 impl Drop for ExprHandle {
-  fn drop(&mut self) { self.ctx.reqnot.notify(Release(self.ctx.id, self.tk)) }
+  fn drop(&mut self) { self.ctx.reqnot.notify(api::Release(self.ctx.id, self.tk)) }
 }
 
 #[derive(Clone, destructure)]
@@ -45,15 +45,21 @@ impl OwnedExpr {
   pub fn get_data(&self) -> &GenExpr {
     self.val.get_or_init(|| {
       Box::new(GenExpr::from_api(
-        self.handle.ctx.reqnot.request(Inspect(self.handle.tk)).expr,
+        self.handle.ctx.reqnot.request(api::Inspect(self.handle.tk)).expr,
         &self.handle.ctx,
       ))
     })
   }
-  pub fn foreign_atom(self) -> Result<ForeignAtom, Self> {
+  pub fn foreign_atom(self) -> Result<ForeignAtom<'static>, Self> {
     if let GenExpr { clause: GenClause::Atom(_, atom), pos: position } = self.get_data() {
       let (atom, position) = (atom.clone(), position.clone());
-      return Ok(ForeignAtom { expr: self.handle, atom, pos: position });
+      return Ok(ForeignAtom {
+        ctx: self.handle.ctx.clone(),
+        expr: Some(self.handle),
+        char_marker: PhantomData,
+        pos: position,
+        atom,
+      });
     }
     Err(self)
   }
@@ -69,13 +75,13 @@ pub struct GenExpr {
   pub clause: GenClause,
 }
 impl GenExpr {
-  pub fn to_api(&self, sys: &dyn DynSystem) -> Expr {
+  pub fn to_api(&self, ctx: SysCtx) -> api::Expr {
-    Expr { location: self.pos.to_api(), clause: self.clause.to_api(sys) }
+    api::Expr { location: self.pos.to_api(), clause: self.clause.to_api(ctx) }
   }
-  pub fn into_api(self, sys: &dyn DynSystem) -> Expr {
+  pub fn into_api(self, ctx: SysCtx) -> api::Expr {
-    Expr { location: self.pos.to_api(), clause: self.clause.into_api(sys) }
+    api::Expr { location: self.pos.to_api(), clause: self.clause.into_api(ctx) }
   }
-  pub fn from_api(api: Expr, ctx: &SysCtx) -> Self {
+  pub fn from_api(api: api::Expr, ctx: &SysCtx) -> Self {
     Self { pos: Pos::from_api(&api.location), clause: GenClause::from_api(api.clause, ctx) }
   }
 }
@@ -89,60 +95,59 @@ pub enum GenClause {
   Seq(Box<GenExpr>, Box<GenExpr>),
   Const(Tok<Vec<Tok<String>>>),
   NewAtom(AtomFactory),
-  Atom(ExprTicket, Atom),
+  Atom(api::ExprTicket, api::Atom),
-  Bottom(ProjectErrorObj),
+  Bottom(Vec<OrcErr>),
 }
 impl GenClause {
-  pub fn to_api(&self, sys: &dyn DynSystem) -> Clause {
+  pub fn to_api(&self, ctx: SysCtx) -> api::Clause {
     match self {
-      Self::Call(f, x) => Clause::Call(Box::new(f.to_api(sys)), Box::new(x.to_api(sys))),
+      Self::Call(f, x) =>
-      Self::Seq(a, b) => Clause::Seq(Box::new(a.to_api(sys)), Box::new(b.to_api(sys))),
+        api::Clause::Call(Box::new(f.to_api(ctx.clone())), Box::new(x.to_api(ctx))),
-      Self::Lambda(arg, body) => Clause::Lambda(*arg, Box::new(body.to_api(sys))),
+      Self::Seq(a, b) => api::Clause::Seq(Box::new(a.to_api(ctx.clone())), Box::new(b.to_api(ctx))),
-      Self::Arg(arg) => Clause::Arg(*arg),
+      Self::Lambda(arg, body) => api::Clause::Lambda(*arg, Box::new(body.to_api(ctx))),
-      Self::Const(name) => Clause::Const(name.marker()),
+      Self::Arg(arg) => api::Clause::Arg(*arg),
-      Self::Bottom(err) => Clause::Bottom(errv_to_apiv([err.clone()])),
+      Self::Const(name) => api::Clause::Const(name.marker()),
-      Self::NewAtom(fac) => Clause::NewAtom(fac.clone().build(sys)),
+      Self::Bottom(err) => api::Clause::Bottom(errv_to_apiv(err)),
-      Self::Atom(tk, atom) => Clause::Atom(*tk, atom.clone()),
+      Self::NewAtom(fac) => api::Clause::NewAtom(fac.clone().build(ctx)),
+      Self::Atom(tk, atom) => api::Clause::Atom(*tk, atom.clone()),
       Self::Slot(_) => panic!("Slot is forbidden in const tree"),
     }
   }
-  pub fn into_api(self, sys: &dyn DynSystem) -> Clause {
+  pub fn into_api(self, ctx: SysCtx) -> api::Clause {
     match self {
-      Self::Call(f, x) => Clause::Call(Box::new(f.into_api(sys)), Box::new(x.into_api(sys))),
+      Self::Call(f, x) =>
-      Self::Seq(a, b) => Clause::Seq(Box::new(a.into_api(sys)), Box::new(b.into_api(sys))),
+        api::Clause::Call(Box::new(f.into_api(ctx.clone())), Box::new(x.into_api(ctx))),
-      Self::Lambda(arg, body) => Clause::Lambda(arg, Box::new(body.into_api(sys))),
+      Self::Seq(a, b) =>
-      Self::Arg(arg) => Clause::Arg(arg),
+        api::Clause::Seq(Box::new(a.into_api(ctx.clone())), Box::new(b.into_api(ctx))),
-      Self::Slot(extk) => Clause::Slot(extk.handle.into_tk()),
+      Self::Lambda(arg, body) => api::Clause::Lambda(arg, Box::new(body.into_api(ctx))),
-      Self::Const(name) => Clause::Const(name.marker()),
+      Self::Arg(arg) => api::Clause::Arg(arg),
-      Self::Bottom(err) => Clause::Bottom(errv_to_apiv([err])),
+      Self::Slot(extk) => api::Clause::Slot(extk.handle.into_tk()),
-      Self::NewAtom(fac) => Clause::NewAtom(fac.clone().build(sys)),
+      Self::Const(name) => api::Clause::Const(name.marker()),
-      Self::Atom(tk, atom) => Clause::Atom(tk, atom),
+      Self::Bottom(err) => api::Clause::Bottom(errv_to_apiv(err.iter())),
+      Self::NewAtom(fac) => api::Clause::NewAtom(fac.clone().build(ctx)),
+      Self::Atom(tk, atom) => api::Clause::Atom(tk, atom),
     }
   }
-  pub fn from_api(api: Clause, ctx: &SysCtx) -> Self {
+  pub fn from_api(api: api::Clause, ctx: &SysCtx) -> Self {
     match api {
-      Clause::Arg(id) => Self::Arg(id),
+      api::Clause::Arg(id) => Self::Arg(id),
-      Clause::Lambda(arg, body) => Self::Lambda(arg, Box::new(GenExpr::from_api(*body, ctx))),
+      api::Clause::Lambda(arg, body) => Self::Lambda(arg, Box::new(GenExpr::from_api(*body, ctx))),
-      Clause::NewAtom(_) => panic!("Clause::NewAtom should never be received, only sent"),
+      api::Clause::NewAtom(_) => panic!("Clause::NewAtom should never be received, only sent"),
-      Clause::Bottom(s) => Self::Bottom(err_from_apiv(&s, &ctx.reqnot)),
+      api::Clause::Bottom(s) => Self::Bottom(errv_from_apiv(&s)),
-      Clause::Call(f, x) => Self::Call(
+      api::Clause::Call(f, x) =>
-        Box::new(GenExpr::from_api(*f, ctx)),
+        Self::Call(Box::new(GenExpr::from_api(*f, ctx)), Box::new(GenExpr::from_api(*x, ctx))),
-        Box::new(GenExpr::from_api(*x, ctx)),
+      api::Clause::Seq(a, b) =>
-      ),
+        Self::Seq(Box::new(GenExpr::from_api(*a, ctx)), Box::new(GenExpr::from_api(*b, ctx))),
-      Clause::Seq(a, b) => Self::Seq(
+      api::Clause::Const(name) => Self::Const(deintern(name)),
-        Box::new(GenExpr::from_api(*a, ctx)),
+      api::Clause::Slot(exi) => Self::Slot(OwnedExpr::new(ExprHandle::from_args(ctx.clone(), exi))),
-        Box::new(GenExpr::from_api(*b, ctx)),
+      api::Clause::Atom(tk, atom) => Self::Atom(tk, atom),
-      ),
-      Clause::Const(name) => Self::Const(deintern(name)),
-      Clause::Slot(exi) => Self::Slot(OwnedExpr::new(ExprHandle::from_args(ctx.clone(), exi))),
-      Clause::Atom(tk, atom) => Self::Atom(tk, atom),
     }
   }
 }
 fn inherit(clause: GenClause) -> GenExpr { GenExpr { pos: Pos::Inherit, clause } }
 
 pub fn sym_ref(path: Tok<Vec<Tok<String>>>) -> GenExpr { inherit(GenClause::Const(path)) }
-pub fn atom<A: AtomicFeatures>(atom: A) -> GenExpr { inherit(GenClause::NewAtom(atom.factory())) }
+pub fn atom<A: ToAtom>(atom: A) -> GenExpr { inherit(GenClause::NewAtom(atom.to_atom_factory())) }
 
 pub fn seq(ops: impl IntoIterator<Item = GenExpr>) -> GenExpr {
   fn recur(mut ops: impl Iterator<Item = GenExpr>) -> Option<GenExpr> {
@@ -169,5 +174,5 @@ pub fn call(v: impl IntoIterator<Item = GenExpr>) -> GenExpr {
     .expect("Empty call expression")
 }
 
-pub fn bot<E: DynProjectError>(msg: E) -> GenExpr { inherit(GenClause::Bottom(Arc::new(msg))) }
+pub fn bot(e: OrcErr) -> GenExpr { botv(vec![e]) }
-pub fn bot_obj(e: ProjectErrorObj) -> GenExpr { inherit(GenClause::Bottom(e)) }
+pub fn botv(ev: Vec<OrcErr>) -> GenExpr { inherit(GenClause::Bottom(ev)) }

orchid-extension/src/fs.rs

@@ -1,13 +1,13 @@
 use std::num::NonZero;
 
 use hashbrown::HashMap;
-use orchid_api::error::ProjResult;
-use orchid_api::vfs::{EagerVfs, Loaded, VfsId};
 use orchid_base::interner::intern;
 use orchid_base::name::PathSlice;
 
+use crate::api;
+
 pub trait VirtFS: Send + Sync + 'static {
-  fn load(&self, path: &PathSlice) -> ProjResult<Loaded>;
+  fn load(&self, path: &PathSlice) -> api::OrcResult<api::Loaded>;
 }
 
 pub enum DeclFs {
@@ -15,15 +15,15 @@ pub enum DeclFs {
   Mod(&'static [(&'static str, DeclFs)]),
 }
 impl DeclFs {
-  pub fn to_api_rec(&self, vfses: &mut HashMap<VfsId, &'static dyn VirtFS>) -> EagerVfs {
+  pub fn to_api_rec(&self, vfses: &mut HashMap<api::VfsId, &'static dyn VirtFS>) -> api::EagerVfs {
     match self {
       DeclFs::Lazy(fs) => {
         let vfsc: u16 = vfses.len().try_into().expect("too many vfses (more than u16::MAX)");
-        let id = VfsId(NonZero::new(vfsc + 1).unwrap());
+        let id = api::VfsId(NonZero::new(vfsc + 1).unwrap());
         vfses.insert(id, *fs);
-        EagerVfs::Lazy(id)
+        api::EagerVfs::Lazy(id)
       },
-      DeclFs::Mod(children) => EagerVfs::Eager(
+      DeclFs::Mod(children) => api::EagerVfs::Eager(
         children.iter().map(|(k, v)| (intern(*k).marker(), v.to_api_rec(vfses))).collect(),
       ),
     }

orchid-extension/src/fun.rs

@@ -1,38 +0,0 @@
-use std::borrow::Cow;
-
-use dyn_clone::{clone_box, DynClone};
-use never::Never;
-use trait_set::trait_set;
-
-use crate::atom::{Atomic, ReqPck};
-use crate::atom_owned::{OwnedAtom, OwnedVariant};
-use crate::conv::{ToExpr, TryFromExpr};
-use crate::expr::{ExprHandle, GenExpr};
-use crate::system::SysCtx;
-
-trait_set! {
-  trait FunCB = FnOnce(ExprHandle) -> GenExpr + DynClone + Send + Sync + 'static;
-}
-
-pub struct Fun(Box<dyn FunCB>);
-impl Fun {
-  pub fn new<I: TryFromExpr, O: ToExpr>(
-    f: impl FnOnce(I) -> O + Clone + Send + Sync + 'static,
-  ) -> Self {
-    Self(Box::new(|eh| I::try_from_expr(eh).map(f).to_expr()))
-  }
-}
-impl Clone for Fun {
-  fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
-}
-impl Atomic for Fun {
-  type Data = ();
-  type Req = Never;
-  type Variant = OwnedVariant;
-}
-impl OwnedAtom for Fun {
-  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
-  fn call_ref(&self, arg: ExprHandle) -> GenExpr { self.clone().call(arg) }
-  fn call(self, arg: ExprHandle) -> GenExpr { (self.0)(arg) }
-  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
-}

orchid-extension/src/func_atom.rs

@@ -0,0 +1,123 @@
+use orchid_base::interner::Tok;
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::io;
+use std::sync::{Arc, Mutex};
+
+use itertools::Itertools;
+use lazy_static::lazy_static;
+use never::Never;
+use orchid_api_traits::Encode;
+use orchid_base::error::OrcRes;
+use orchid_base::name::Sym;
+use trait_set::trait_set;
+
+use crate::atom::{Atomic, ReqPck};
+use crate::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
+use crate::conv::ToExpr;
+use crate::expr::{ExprHandle, GenExpr};
+use crate::system::SysCtx;
+
+trait_set! {
+  trait FunCB = Fn(Vec<ExprHandle>) -> OrcRes<GenExpr> + Send + Sync + 'static;
+}
+
+pub trait ExprFunc<I, O>: Clone + Send + Sync + 'static {
+  const ARITY: u8;
+  fn apply(&self, v: Vec<ExprHandle>) -> OrcRes<GenExpr>;
+}
+
+lazy_static!{
+  static ref FUNS: Mutex<HashMap<Sym, (u8, Arc<dyn FunCB>)>> = Mutex::default();
+}
+
+#[derive(Clone)]
+pub(crate) struct Fun{
+  path: Sym,
+  args: Vec<ExprHandle>,
+  arity: u8,
+  fun: Arc<dyn FunCB>,
+}
+impl Fun {
+  pub fn new<I, O, F: ExprFunc<I, O>>(path: Sym, f: F) -> Self {
+    let mut fung = FUNS.lock().unwrap();
+    let fun = if let Some(x) = fung.get(&path) {
+      x.1.clone()
+    } else {
+      let fun = Arc::new(move |v| f.apply(v));
+      fung.insert(path.clone(), (F::ARITY, fun.clone()));
+      fun
+    };
+    Self { args: vec![], arity: F::ARITY, path, fun }
+  }
+}
+impl Atomic for Fun {
+  type Data = ();
+  type Req = Never;
+  type Variant = OwnedVariant;
+}
+impl OwnedAtom for Fun {
+  type Refs = Vec<ExprHandle>;
+  fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(()) }
+  fn call_ref(&self, arg: ExprHandle) -> GenExpr {
+    let new_args = self.args.iter().cloned().chain([arg]).collect_vec();
+    if new_args.len() == self.arity.into() {
+      (self.fun)(new_args).to_expr()
+    } else {
+      Self {
+        args: new_args, arity: self.arity, fun: self.fun.clone(), path: self.path.clone() }.to_expr()
+    }
+  }
+  fn call(self, arg: ExprHandle) -> GenExpr { self.call_ref(arg) }
+  fn handle_req(&self, pck: impl ReqPck<Self>) { pck.never() }
+  fn serialize(&self, _: SysCtx, sink: &mut (impl io::Write + ?Sized)) -> Self::Refs {
+    self.path.encode(sink);
+    self.args.clone()
+  }
+  fn deserialize(ctx: impl DeserializeCtx, args: Self::Refs) -> Self {
+    let path = Sym::new(ctx.decode::<Vec<Tok<String>>>()).unwrap();
+    let (arity, fun) = FUNS.lock().unwrap().get(&path).unwrap().clone();
+    Self { args, arity, path, fun }
+  }
+}
+
+mod expr_func_derives {
+  use orchid_base::error::OrcRes;
+  use crate::func_atom::GenExpr;
+  use super::ExprFunc;
+  use crate::conv::{TryFromExpr, ToExpr};
+  use crate::func_atom::ExprHandle;
+
+  macro_rules! expr_func_derive {
+    ($arity: tt, $($t:ident),*) => {
+      paste::paste!{
+        impl<
+          $($t: TryFromExpr, )*
+          Out: ToExpr,
+          Func: Fn($($t,)*) -> Out + Clone + Send + Sync + 'static
+        > ExprFunc<($($t,)*), Out> for Func {
+          const ARITY: u8 = $arity;
+          fn apply(&self, v: Vec<ExprHandle>) -> OrcRes<GenExpr> {
+            assert_eq!(v.len(), Self::ARITY.into(), "Arity mismatch");
+            let [$([< $t:lower >],)*] = v.try_into().unwrap_or_else(|_| panic!("Checked above"));
+            Ok(self($($t::try_from_expr([< $t:lower >])?,)*).to_expr())
+          }
+        }
+      }
+    };
+  }
+  expr_func_derive!(1,  A);
+  expr_func_derive!(2,  A, B);
+  expr_func_derive!(3,  A, B, C);
+  expr_func_derive!(4,  A, B, C, D);
+  expr_func_derive!(5,  A, B, C, D, E);
+  expr_func_derive!(6,  A, B, C, D, E, F);
+  expr_func_derive!(7,  A, B, C, D, E, F, G);
+  expr_func_derive!(8,  A, B, C, D, E, F, G, H);
+  expr_func_derive!(9,  A, B, C, D, E, F, G, H, I);
+  expr_func_derive!(10, A, B, C, D, E, F, G, H, I, J);
+  expr_func_derive!(11, A, B, C, D, E, F, G, H, I, J, K);
+  expr_func_derive!(12, A, B, C, D, E, F, G, H, I, J, K, L);
+  expr_func_derive!(13, A, B, C, D, E, F, G, H, I, J, K, L, M);
+  expr_func_derive!(14, A, B, C, D, E, F, G, H, I, J, K, L, M, N);
+}

orchid-extension/src/lexer.rs

@@ -1,46 +1,44 @@
 use std::ops::{Range, RangeInclusive};
 
-use orchid_api::parser::{ParsId, SubLex};
+use orchid_base::error::{mk_err, OrcErr, OrcRes};
-use orchid_api::proto::ExtMsgSet;
+use orchid_base::intern;
-use orchid_api::system::SysId;
 use orchid_base::interner::Tok;
 use orchid_base::location::Pos;
 use orchid_base::reqnot::{ReqNot, Requester};
+use orchid_base::tree::TreeHandle;
 
-use crate::error::{
+use crate::api;
-  ProjectError, ProjectResult
-};
 use crate::tree::{GenTok, GenTokTree};
 
-pub struct CascadingError;
+pub fn err_cascade() -> OrcErr {
-impl ProjectError for CascadingError {
+  mk_err(
-  const DESCRIPTION: &'static str = "An error cascading from a recursive sublexer";
+    intern!(str: "An error cascading from a recursive sublexer"),
-  fn message(&self) -> String {
+    "This error should not surface. If you are seeing it, something is wrong",
-    "This error should not surface. If you are seeing it, something is wrong".to_string()
+    [Pos::None.into()],
-  }
+  )
-  fn one_position(&self) -> Pos { Pos::None }
 }
 
-pub struct NotApplicableLexerError;
+pub fn err_lexer_na() -> OrcErr {
-impl ProjectError for NotApplicableLexerError {
+  mk_err(
-  const DESCRIPTION: &'static str = "Pseudo-error to communicate that the lexer doesn't apply";
+    intern!(str: "Pseudo-error to communicate that the lexer doesn't apply"),
-  fn message(&self) -> String { CascadingError.message() }
+    &*err_cascade().message,
-  fn one_position(&self) -> Pos { Pos::None }
+    [Pos::None.into()],
+  )
 }
 
 pub struct LexContext<'a> {
   pub text: &'a Tok<String>,
-  pub sys: SysId,
+  pub sys: api::SysId,
-  pub id: ParsId,
+  pub id: api::ParsId,
   pub pos: u32,
-  pub reqnot: ReqNot<ExtMsgSet>,
+  pub reqnot: ReqNot<api::ExtMsgSet>,
 }
 impl<'a> LexContext<'a> {
-  pub fn recurse(&self, tail: &'a str) -> ProjectResult<(&'a str, GenTokTree)> {
+  pub fn recurse(&self, tail: &'a str) -> OrcRes<(&'a str, GenTokTree<'a>)> {
     let start = self.pos(tail);
-    let lx = (self.reqnot.request(SubLex { pos: start, id: self.id }))
+    let lx =
-      .ok_or_else(|| CascadingError.pack())?;
+      self.reqnot.request(api::SubLex { pos: start, id: self.id }).ok_or_else(err_cascade)?;
-    Ok((&self.text[lx.pos as usize..], GenTok::Slot(lx.ticket).at(start..lx.pos)))
+    Ok((&self.text[lx.pos as usize..], GenTok::Slot(TreeHandle::new(lx.ticket)).at(start..lx.pos)))
   }
 
   pub fn pos(&self, tail: &'a str) -> u32 { (self.text.len() - tail.len()) as u32 }
@@ -52,19 +50,13 @@ impl<'a> LexContext<'a> {
 
 pub trait Lexer: Send + Sync + Sized + Default + 'static {
   const CHAR_FILTER: &'static [RangeInclusive<char>];
-  fn lex<'a>(
+  fn lex<'a>(tail: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)>;
-    tail: &'a str,
-    ctx: &'a LexContext<'a>,
-  ) -> ProjectResult<(&'a str, GenTokTree)>;
 }
 
 pub trait DynLexer: Send + Sync + 'static {
   fn char_filter(&self) -> &'static [RangeInclusive<char>];
-  fn lex<'a>(
+  fn lex<'a>(&self, tail: &'a str, ctx: &'a LexContext<'a>)
-    &self,
+  -> OrcRes<(&'a str, GenTokTree<'a>)>;
-    tail: &'a str,
-    ctx: &'a LexContext<'a>,
-  ) -> ProjectResult<(&'a str, GenTokTree)>;
 }
 
 impl<T: Lexer> DynLexer for T {
@@ -73,7 +65,7 @@ impl<T: Lexer> DynLexer for T {
     &self,
     tail: &'a str,
     ctx: &'a LexContext<'a>,
-  ) -> ProjectResult<(&'a str, GenTokTree)> {
+  ) -> OrcRes<(&'a str, GenTokTree<'a>)> {
     T::lex(tail, ctx)
   }
 }

orchid-extension/src/lib.rs

@@ -1,15 +1,18 @@
+use orchid_api as api;
+
 pub mod atom;
 pub mod atom_owned;
 pub mod atom_thin;
 pub mod conv;
 pub mod entrypoint;
-pub mod error;
+// pub mod error;
 pub mod expr;
 pub mod fs;
-pub mod fun;
+pub mod func_atom;
 pub mod lexer;
 pub mod msg;
 pub mod other_system;
+pub mod parser;
 pub mod system;
 pub mod system_ctor;
 pub mod tree;

orchid-extension/src/other_system.rs

@@ -1,24 +1,23 @@
 use std::marker::PhantomData;
 use std::mem::size_of;
 
-use orchid_api::system::SysId;
+use crate::api;
-
 use crate::system::{DynSystemCard, SystemCard};
 
 pub struct SystemHandle<C: SystemCard> {
   pub(crate) _card: PhantomData<C>,
-  pub(crate) id: SysId,
+  pub(crate) id: api::SysId,
 }
 impl<C: SystemCard> SystemHandle<C> {
-  pub(crate) fn new(id: SysId) -> Self { Self { _card: PhantomData, id } }
+  pub(crate) fn new(id: api::SysId) -> Self { Self { _card: PhantomData, id } }
-  pub fn id(&self) -> SysId { self.id }
+  pub fn id(&self) -> api::SysId { self.id }
 }
 impl<C: SystemCard> Clone for SystemHandle<C> {
   fn clone(&self) -> Self { Self::new(self.id) }
 }
 
 pub trait DynSystemHandle {
-  fn id(&self) -> SysId;
+  fn id(&self) -> api::SysId;
   fn get_card(&self) -> &dyn DynSystemCard;
 }
 
@@ -32,6 +31,6 @@ pub fn leak_card<T: Default>() -> &'static T {
 }
 
 impl<C: SystemCard> DynSystemHandle for SystemHandle<C> {
-  fn id(&self) -> SysId { self.id }
+  fn id(&self) -> api::SysId { self.id }
   fn get_card(&self) -> &'static dyn DynSystemCard { leak_card::<C>() }
 }

orchid-extension/src/parser.rs

@@ -0,0 +1,24 @@
+use orchid_base::error::OrcRes;
+use orchid_base::parse::Snippet;
+
+use crate::atom::{AtomFactory, ForeignAtom};
+use crate::tree::GenTokTree;
+
+pub type GenSnippet<'a> = Snippet<'a, 'a, ForeignAtom<'a>, AtomFactory>;
+
+pub trait Parser: Send + Sync + Sized + Default + 'static {
+  const LINE_HEAD: &'static str;
+  fn parse(line: GenSnippet<'_>) -> OrcRes<Vec<GenTokTree<'_>>>;
+}
+
+pub trait DynParser: Send + Sync + 'static {
+  fn line_head(&self) -> &'static str;
+  fn parse<'a>(&self, line: GenSnippet<'a>) -> OrcRes<Vec<GenTokTree<'a>>>;
+}
+
+impl<T: Parser> DynParser for T {
+  fn line_head(&self) -> &'static str { Self::LINE_HEAD }
+  fn parse<'a>(&self, line: GenSnippet<'a>) -> OrcRes<Vec<GenTokTree<'a>>> { Self::parse(line) }
+}
+
+pub type ParserObj = &'static dyn DynParser;

orchid-extension/src/system.rs

@@ -1,19 +1,20 @@
 use std::any::TypeId;
+use std::num::NonZero;
 use std::sync::Arc;
 
 use hashbrown::HashMap;
-use orchid_api::atom::Atom;
+use orchid_api::AtomId;
-use orchid_api::proto::ExtMsgSet;
-use orchid_api::system::SysId;
 use orchid_api_traits::Decode;
 use orchid_base::interner::Tok;
 use orchid_base::logging::Logger;
 use orchid_base::reqnot::ReqNot;
 
+use crate::api;
 use crate::atom::{get_info, AtomCtx, AtomDynfo, AtomicFeatures, ForeignAtom, TypAtom};
 use crate::fs::DeclFs;
-use crate::fun::Fun;
+// use crate::fun::Fun;
 use crate::lexer::LexerObj;
+use crate::parser::ParserObj;
 use crate::system_ctor::{CtedObj, SystemCtor};
 use crate::tree::GenMemberKind;
 
@@ -33,31 +34,34 @@ pub trait DynSystemCard: Send + Sync + 'static {
 /// Atoms supported by this package which may appear in all extensions.
 /// The indices of these are bitwise negated, such that the MSB of an atom index
 /// marks whether it belongs to this package (0) or the importer (1)
-fn general_atoms() -> &'static [Option<&'static dyn AtomDynfo>] { &[Some(Fun::INFO)] }
+fn general_atoms() -> &'static [Option<&'static dyn AtomDynfo>] { &[/*Some(Fun::INFO)*/] }
 
 pub fn atom_info_for(
   sys: &(impl DynSystemCard + ?Sized),
   tid: TypeId,
-) -> Option<(u64, &'static dyn AtomDynfo)> {
+) -> Option<(api::AtomId, &'static dyn AtomDynfo)> {
-  (sys.atoms().iter().enumerate().map(|(i, o)| (i as u64, o)))
+  (sys.atoms().iter().enumerate().map(|(i, o)| (NonZero::new(i as u64 + 1).unwrap(), o)))
-    .chain(general_atoms().iter().enumerate().map(|(i, o)| (!(i as u64), o)))
+    .chain(general_atoms().iter().enumerate().map(|(i, o)| (NonZero::new(!(i as u64)).unwrap(), o)))
-    .filter_map(|(i, o)| o.as_ref().map(|a| (i, *a)))
+    .filter_map(|(i, o)| o.as_ref().map(|a| (api::AtomId(i), *a)))
     .find(|ent| ent.1.tid() == tid)
 }
 
 pub fn atom_by_idx(
   sys: &(impl DynSystemCard + ?Sized),
-  tid: u64,
+  tid: api::AtomId,
 ) -> Option<&'static dyn AtomDynfo> {
-  if (tid >> (u64::BITS - 1)) & 1 == 1 {
+  if (u64::from(tid.0) >> (u64::BITS - 1)) & 1 == 1 {
-    general_atoms()[!tid as usize]
+    general_atoms()[!u64::from(tid.0) as usize]
   } else {
-    sys.atoms()[tid as usize]
+    sys.atoms()[u64::from(tid.0) as usize - 1]
   }
 }
 
-pub fn resolv_atom(sys: &(impl DynSystemCard + ?Sized), atom: &Atom) -> &'static dyn AtomDynfo {
+pub fn resolv_atom(
-  let tid = u64::decode(&mut &atom.data[..8]);
+  sys: &(impl DynSystemCard + ?Sized),
+  atom: &api::Atom,
+) -> &'static dyn AtomDynfo {
+  let tid = api::AtomId::decode(&mut &atom.data[..8]);
   atom_by_idx(sys, tid).expect("Value of nonexistent type found")
 }
 
@@ -71,32 +75,35 @@ pub trait System: Send + Sync + SystemCard + 'static {
   fn env() -> Vec<(Tok<String>, GenMemberKind)>;
   fn vfs() -> DeclFs;
   fn lexers() -> Vec<LexerObj>;
+  fn parsers() -> Vec<ParserObj>;
 }
 
 pub trait DynSystem: Send + Sync + DynSystemCard + 'static {
   fn dyn_env(&self) -> HashMap<Tok<String>, GenMemberKind>;
   fn dyn_vfs(&self) -> DeclFs;
   fn dyn_lexers(&self) -> Vec<LexerObj>;
-  fn dyn_card(&self) -> &dyn DynSystemCard;
+  fn dyn_parsers(&self) -> Vec<ParserObj>;
+  fn card(&self) -> &dyn DynSystemCard;
 }
 
 impl<T: System> DynSystem for T {
   fn dyn_env(&self) -> HashMap<Tok<String>, GenMemberKind> { Self::env().into_iter().collect() }
   fn dyn_vfs(&self) -> DeclFs { Self::vfs() }
   fn dyn_lexers(&self) -> Vec<LexerObj> { Self::lexers() }
-  fn dyn_card(&self) -> &dyn DynSystemCard { self }
+  fn dyn_parsers(&self) -> Vec<ParserObj> { Self::parsers() }
+  fn card(&self) -> &dyn DynSystemCard { self }
 }
 
 pub fn downcast_atom<A: AtomicFeatures>(foreign: ForeignAtom) -> Result<TypAtom<A>, ForeignAtom> {
   let mut data = &foreign.atom.data[..];
-  let ctx = foreign.expr.get_ctx();
+  let ctx = foreign.ctx.clone();
   let info_ent = (ctx.cted.deps().find(|s| s.id() == foreign.atom.owner))
     .map(|sys| get_info::<A>(sys.get_card()))
-    .filter(|(pos, _)| u64::decode(&mut data) == *pos);
+    .filter(|(pos, _)| AtomId::decode(&mut data) == *pos);
   match info_ent {
     None => Err(foreign),
     Some((_, info)) => {
-      let val = info.decode(AtomCtx(data, ctx));
+      let val = info.decode(AtomCtx(data, foreign.atom.drop, ctx));
       let value = *val.downcast::<A::Data>().expect("atom decode returned wrong type");
       Ok(TypAtom { value, data: foreign })
     },
@@ -105,8 +112,18 @@ pub fn downcast_atom<A: AtomicFeatures>(foreign: ForeignAtom) -> Result<TypAtom<
 
 #[derive(Clone)]
 pub struct SysCtx {
-  pub reqnot: ReqNot<ExtMsgSet>,
+  pub reqnot: ReqNot<api::ExtMsgSet>,
-  pub id: SysId,
+  pub id: api::SysId,
   pub cted: CtedObj,
   pub logger: Arc<Logger>,
 }
+impl SysCtx {
+  pub fn new(
+    id: api::SysId,
+    cted: &CtedObj,
+    logger: &Arc<Logger>,
+    reqnot: ReqNot<api::ExtMsgSet>,
+  ) -> Self {
+    Self { cted: cted.clone(), id, logger: logger.clone(), reqnot }
+  }
+}

orchid-extension/src/system_ctor.rs

@@ -1,10 +1,10 @@
 use std::any::Any;
 use std::sync::Arc;
 
-use orchid_api::system::{NewSystem, SysDeclId, SysId, SystemDecl};
 use orchid_base::boxed_iter::{box_empty, box_once, BoxedIter};
 use ordered_float::NotNan;
 
+use crate::api;
 use crate::other_system::{DynSystemHandle, SystemHandle};
 use crate::system::{DynSystem, System, SystemCard};
 
@@ -34,7 +34,7 @@ pub trait DepSat: Clone + Send + Sync + 'static {
 pub trait DepDef {
   type Sat: DepSat;
   fn report(names: &mut impl FnMut(&'static str));
-  fn create(take: &mut impl FnMut() -> SysId) -> Self::Sat;
+  fn create(take: &mut impl FnMut() -> api::SysId) -> Self::Sat;
 }
 
 impl<T: SystemCard> DepSat for SystemHandle<T> {
@@ -44,7 +44,7 @@ impl<T: SystemCard> DepSat for SystemHandle<T> {
 impl<T: SystemCard> DepDef for T {
   type Sat = SystemHandle<Self>;
   fn report(names: &mut impl FnMut(&'static str)) { names(T::Ctor::NAME) }
-  fn create(take: &mut impl FnMut() -> SysId) -> Self::Sat { SystemHandle::new(take()) }
+  fn create(take: &mut impl FnMut() -> api::SysId) -> Self::Sat { SystemHandle::new(take()) }
 }
 
 impl DepSat for () {
@@ -53,7 +53,7 @@ impl DepSat for () {
 
 impl DepDef for () {
   type Sat = ();
-  fn create(_: &mut impl FnMut() -> SysId) -> Self::Sat {}
+  fn create(_: &mut impl FnMut() -> api::SysId) -> Self::Sat {}
   fn report(_: &mut impl FnMut(&'static str)) {}
 }
 
@@ -66,20 +66,20 @@ pub trait SystemCtor: Send + Sync + 'static {
 }
 
 pub trait DynSystemCtor: Send + Sync + 'static {
-  fn decl(&self, id: SysDeclId) -> SystemDecl;
+  fn decl(&self, id: api::SysDeclId) -> api::SystemDecl;
-  fn new_system(&self, new: &NewSystem) -> CtedObj;
+  fn new_system(&self, new: &api::NewSystem) -> CtedObj;
 }
 
 impl<T: SystemCtor> DynSystemCtor for T {
-  fn decl(&self, id: SysDeclId) -> SystemDecl {
+  fn decl(&self, id: api::SysDeclId) -> api::SystemDecl {
     // Version is equivalent to priority for all practical purposes
     let priority = NotNan::new(T::VERSION).unwrap();
     // aggregate depends names
     let mut depends = Vec::new();
     T::Deps::report(&mut |n| depends.push(n.to_string()));
-    SystemDecl { name: T::NAME.to_string(), depends, id, priority }
+    api::SystemDecl { name: T::NAME.to_string(), depends, id, priority }
   }
-  fn new_system(&self, NewSystem { system: _, id: _, depends }: &NewSystem) -> CtedObj {
+  fn new_system(&self, api::NewSystem { system: _, id: _, depends }: &api::NewSystem) -> CtedObj {
     let mut ids = depends.iter().copied();
     let inst = Arc::new(T::inst().expect("Constructor did not create system"));
     let deps = T::Deps::create(&mut || ids.next().unwrap());
@@ -88,12 +88,12 @@ impl<T: SystemCtor> DynSystemCtor for T {
 }
 
 mod dep_set_tuple_impls {
-  use orchid_api::system::SysId;
   use orchid_base::box_chain;
   use orchid_base::boxed_iter::BoxedIter;
   use paste::paste;
 
   use super::{DepDef, DepSat};
+  use crate::api;
   use crate::system_ctor::DynSystemHandle;
 
   macro_rules! dep_set_tuple_impl {
@@ -126,7 +126,7 @@ mod dep_set_tuple_impls {
             $name ::report(names);
           )*
         }
-        fn create(take: &mut impl FnMut() -> SysId) -> Self::Sat {
+        fn create(take: &mut impl FnMut() -> api::SysId) -> Self::Sat {
           (
             $(
               $name ::create(take),

orchid-extension/src/tree.rs

@@ -1,183 +1,119 @@
-use std::iter;
 use std::num::NonZero;
 use std::ops::Range;
+use std::sync::Arc;
 
-use hashbrown::HashMap;
 use dyn_clone::{clone_box, DynClone};
+use hashbrown::HashMap;
 use itertools::Itertools;
-use orchid_api::tree::{
-  Macro, Paren, PlaceholderKind, Token, TokenTree, Item, TreeId, ItemKind, Member, MemberKind, Module, TreeTicket
-};
 use orchid_base::interner::{intern, Tok};
 use orchid_base::location::Pos;
-use orchid_base::name::{NameLike, Sym, VName};
+use orchid_base::name::Sym;
-use orchid_base::tokens::OwnedPh;
+use orchid_base::tree::{ttv_to_api, TokTree, Token};
 use ordered_float::NotNan;
+use substack::Substack;
 use trait_set::trait_set;
 
-use crate::atom::AtomFactory;
+use crate::api;
+use crate::atom::{AtomFactory, ForeignAtom};
 use crate::conv::ToExpr;
 use crate::entrypoint::MemberRecord;
-use crate::error::{errv_to_apiv, ProjectErrorObj};
 use crate::expr::GenExpr;
-use crate::system::DynSystem;
+use crate::func_atom::{ExprFunc, Fun};
+use crate::system::SysCtx;
 
-#[derive(Clone)]
+pub type GenTokTree<'a> = TokTree<'a, ForeignAtom<'a>, AtomFactory>;
-pub struct GenTokTree {
+pub type GenTok<'a> = Token<'a, ForeignAtom<'a>, AtomFactory>;
-  pub tok: GenTok,
-  pub range: Range<u32>,
-}
-impl GenTokTree {
-  pub fn into_api(self, sys: &dyn DynSystem) -> TokenTree {
-    TokenTree { token: self.tok.into_api(sys), range: self.range }
-  }
-}
 
-pub fn ph(s: &str) -> OwnedPh {
+pub fn do_extra(f: &AtomFactory, r: Range<u32>, ctx: SysCtx) -> api::TokenTree {
-  match s.strip_prefix("..") {
+  api::TokenTree { range: r, token: api::Token::Atom(f.clone().build(ctx)) }
-    Some(v_tail) => {
-      let (mid, priority) = match v_tail.split_once(':') {
-        Some((h, t)) => (h, t.parse().expect("priority not an u8")),
-        None => (v_tail, 0),
-      };
-      let (name, nonzero) = match mid.strip_prefix(".$") {
-        Some(name) => (name, true),
-        None => (mid.strip_prefix('$').expect("Invalid placeholder"), false),
-      };
-      if konst::string::starts_with(name, "_") {
-        panic!("Names starting with an underscore indicate a single-name scalar placeholder")
-      }
-      OwnedPh { name: intern(name), kind: PlaceholderKind::Vector { nz: nonzero, prio: priority } }
-    },
-    None => match konst::string::strip_prefix(s, "$_") {
-      Some(name) => OwnedPh { name: intern(name), kind: PlaceholderKind::Name },
-      None => match konst::string::strip_prefix(s, "$") {
-        None => panic!("Invalid placeholder"),
-        Some(name) => OwnedPh { name: intern(name), kind: PlaceholderKind::Scalar },
-      },
-    },
-  }
-}
-
-#[derive(Clone)]
-pub enum GenTok {
-  Lambda(Vec<GenTokTree>),
-  Name(Tok<String>),
-  NS,
-  BR,
-  S(Paren, Vec<GenTokTree>),
-  Atom(AtomFactory),
-  Slot(TreeTicket),
-  Ph(OwnedPh),
-  Bottom(ProjectErrorObj),
-}
-impl GenTok {
-  pub fn at(self, range: Range<u32>) -> GenTokTree { GenTokTree { tok: self, range } }
-  pub fn into_api(self, sys: &dyn DynSystem) -> Token {
-    match self {
-      Self::Lambda(x) => Token::Lambda(x.into_iter().map(|tt| tt.into_api(sys)).collect()),
-      Self::Name(n) => Token::Name(n.marker()),
-      Self::NS => Token::NS,
-      Self::BR => Token::BR,
-      Self::Ph(ph) => Token::Ph(ph.to_api()),
-      Self::S(p, body) => Token::S(p, body.into_iter().map(|tt| tt.into_api(sys)).collect_vec()),
-      Self::Slot(tk) => Token::Slot(tk),
-      Self::Atom(at) => Token::Atom(at.build(sys)),
-      Self::Bottom(err) => Token::Bottom(errv_to_apiv([err])),
-    }
-  }
-  pub fn vname(name: &VName) -> impl Iterator<Item = GenTok> + '_ {
-    let (head, tail) = name.split_first();
-    iter::once(Self::Name(head)).chain(tail.iter().flat_map(|t| [Self::NS, Self::Name(t)]))
-  }
 }
 
 #[derive(Clone)]
 pub struct GenMacro {
-  pub pattern: Vec<GenTokTree>,
+  pub pattern: Vec<GenTokTree<'static>>,
   pub priority: NotNan<f64>,
-  pub template: Vec<GenTokTree>,
+  pub template: Vec<GenTokTree<'static>>,
-}
-
-pub fn tokv_into_api(
-  tokv: impl IntoIterator<Item = GenTokTree>,
-  sys: &dyn DynSystem,
-) -> Vec<TokenTree> {
-  tokv.into_iter().map(|tok| tok.into_api(sys)).collect_vec()
-}
-
-pub fn wrap_tokv(items: Vec<GenTokTree>, range: Range<u32>) -> GenTokTree {
-  match items.len() {
-    1 => items.into_iter().next().unwrap(),
-    _ => GenTok::S(Paren::Round, items).at(range),
-  }
 }
 
 pub struct GenItem {
   pub item: GenItemKind,
+  pub comments: Vec<(String, Pos)>,
   pub pos: Pos,
 }
 impl GenItem {
-  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> Item {
+  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Item {
     let kind = match self.item {
-      GenItemKind::Rule(GenMacro { pattern, priority, template }) => ItemKind::Rule(Macro {
+      GenItemKind::Rule(m) => api::ItemKind::Rule(api::Macro {
-        pattern: tokv_into_api(pattern, ctx.sys()),
+        pattern: ttv_to_api(m.pattern, &mut |f, r| do_extra(f, r, ctx.sys())),
-        priority,
+        priority: m.priority,
-        template: tokv_into_api(template, ctx.sys()),
+        template: ttv_to_api(m.template, &mut |f, r| do_extra(f, r, ctx.sys())),
       }),
-      GenItemKind::Raw(item) => ItemKind::Raw(item.into_iter().map(|t| t.into_api(ctx.sys())).collect_vec()),
+      GenItemKind::Raw(item) => api::ItemKind::Raw(Vec::from_iter(
-      GenItemKind::Member(mem) => ItemKind::Member(mem.into_api(ctx))
+        item.into_iter().map(|t| t.to_api(&mut |f, r| do_extra(f, r, ctx.sys()))),
+      )),
+      GenItemKind::Member(mem) => api::ItemKind::Member(mem.into_api(ctx)),
     };
-    Item { location: self.pos.to_api(), kind }
+    let comments = self.comments.into_iter().map(|(s, p)| (Arc::new(s), p.to_api())).collect_vec();
+    api::Item { location: self.pos.to_api(), comments, kind }
   }
 }
 
 pub fn cnst(public: bool, name: &str, value: impl ToExpr) -> GenItem {
   let kind = GenMemberKind::Const(value.to_expr());
-  GenItemKind::Member(GenMember { public, name: intern(name), kind }).at(Pos::Inherit)
+  GenItemKind::Member(GenMember { exported: public, name: intern(name), kind }).at(Pos::Inherit)
 }
 pub fn module(
   public: bool,
   name: &str,
   imports: impl IntoIterator<Item = Sym>,
-  items: impl IntoIterator<Item = GenItem>
+  items: impl IntoIterator<Item = GenItem>,
 ) -> GenItem {
   let (name, kind) = root_mod(name, imports, items);
-  GenItemKind::Member(GenMember { public, name, kind }).at(Pos::Inherit)
+  GenItemKind::Member(GenMember { exported: public, name, kind }).at(Pos::Inherit)
 }
 pub fn root_mod(
   name: &str,
   imports: impl IntoIterator<Item = Sym>,
-  items: impl IntoIterator<Item = GenItem>
+  items: impl IntoIterator<Item = GenItem>,
 ) -> (Tok<String>, GenMemberKind) {
   let kind = GenMemberKind::Mod {
     imports: imports.into_iter().collect(),
-    items: items.into_iter().collect()
+    items: items.into_iter().collect(),
   };
   (intern(name), kind)
 }
+pub fn fun<I, O>(exported: bool, name: &str, xf: impl ExprFunc<I, O>) -> GenItem {
+  let fac = LazyMemberFactory::new(move |sym| GenMemberKind::Const(Fun::new(sym, xf).to_expr()));
+  let mem = GenMember{ exported, name: intern(name), kind: GenMemberKind::Lazy(fac) };
+  GenItemKind::Member(mem).at(Pos::Inherit)
+}
 pub fn rule(
-  prio: f64,
+  priority: f64,
-  pat: impl IntoIterator<Item = GenTokTree>,
+  pat: impl IntoIterator<Item = GenTokTree<'static>>,
-  tpl: impl IntoIterator<Item = GenTokTree>,
+  tpl: impl IntoIterator<Item = GenTokTree<'static>>,
 ) -> GenItem {
   GenItemKind::Rule(GenMacro {
     pattern: pat.into_iter().collect(),
-    priority: NotNan::new(prio).expect("expected to be static"),
+    priority: NotNan::new(priority).expect("Rule created with NaN prio"),
     template: tpl.into_iter().collect(),
   })
   .at(Pos::Inherit)
 }
 
+pub fn comments<'a>(cmts: impl IntoIterator<Item = &'a str>, mut val: GenItem) -> GenItem {
+  val.comments.extend(cmts.into_iter().map(|c| (c.to_string(), Pos::Inherit)));
+  val
+}
+
 trait_set! {
-  trait LazyMemberCallback = FnOnce() -> GenMemberKind + Send + Sync + DynClone
+  trait LazyMemberCallback = FnOnce(Sym) -> GenMemberKind + Send + Sync + DynClone
 }
 pub struct LazyMemberFactory(Box<dyn LazyMemberCallback>);
 impl LazyMemberFactory {
-  pub fn new(cb: impl FnOnce() -> GenMemberKind + Send + Sync + Clone + 'static) -> Self {
+  pub fn new(cb: impl FnOnce(Sym) -> GenMemberKind + Send + Sync + Clone + 'static) -> Self {
     Self(Box::new(cb))
   }
-  pub fn build(self) -> GenMemberKind { (self.0)() }
+  pub fn build(self, path: Sym) -> GenMemberKind { (self.0)(path) }
 }
 impl Clone for LazyMemberFactory {
   fn clone(&self) -> Self { Self(clone_box(&*self.0)) }
@@ -185,60 +121,75 @@ impl Clone for LazyMemberFactory {
 
 pub enum GenItemKind {
   Member(GenMember),
-  Raw(Vec<GenTokTree>),
+  Raw(Vec<GenTokTree<'static>>),
   Rule(GenMacro),
 }
 impl GenItemKind {
-  pub fn at(self, position: Pos) -> GenItem { GenItem { item: self, pos: position } }
+  pub fn at(self, position: Pos) -> GenItem {
+    GenItem { item: self, comments: vec![], pos: position }
+  }
 }
 
 pub struct GenMember {
-  public: bool,
+  exported: bool,
   name: Tok<String>,
   kind: GenMemberKind,
 }
 impl GenMember {
-  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> Member {
+  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::Member {
-    Member { name: self.name.marker(), public: self.public, kind: self.kind.into_api(ctx) }
+    api::Member {
+      name: self.name.marker(),
+      exported: self.exported,
+      kind: self.kind.into_api(&mut ctx.push_path(self.name))
+    }
   }
 }
 
 pub enum GenMemberKind {
   Const(GenExpr),
-  Mod{
+  Mod { imports: Vec<Sym>, items: Vec<GenItem> },
-    imports: Vec<Sym>,
+  Lazy(LazyMemberFactory),
-    items: Vec<GenItem>,
-  },
-  Lazy(LazyMemberFactory)
 }
 impl GenMemberKind {
-  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> MemberKind {
+  pub fn into_api(self, ctx: &mut impl TreeIntoApiCtx) -> api::MemberKind {
     match self {
-      Self::Lazy(lazy) => MemberKind::Lazy(ctx.with_lazy(lazy)),
+      Self::Lazy(lazy) => api::MemberKind::Lazy(ctx.with_lazy(lazy)),
-      Self::Const(c) => MemberKind::Const(c.into_api(ctx.sys())),
+      Self::Const(c) => api::MemberKind::Const(c.into_api(ctx.sys())),
-      Self::Mod { imports, items } => MemberKind::Module(Module {
+      Self::Mod { imports, items } => api::MemberKind::Module(api::Module {
         imports: imports.into_iter().map(|t| t.tok().marker()).collect(),
-        items: items.into_iter().map(|i| i.into_api(ctx)).collect_vec()
+        items: items.into_iter().map(|i| i.into_api(ctx)).collect_vec(),
       }),
     }
   }
 }
 
 pub trait TreeIntoApiCtx {
-  fn sys(&self) -> &dyn DynSystem;
+  fn sys(&self) -> SysCtx;
-  fn with_lazy(&mut self, fac: LazyMemberFactory) -> TreeId;
+  fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId;
+  fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx;
 }
 
-pub struct TIACtxImpl<'a> {
+pub struct TIACtxImpl<'a, 'b> {
-  pub sys: &'a dyn DynSystem,
+  pub ctx: SysCtx,
-  pub lazy: &'a mut HashMap<TreeId, MemberRecord>
+  pub basepath: &'a [Tok<String>],
+  pub path: Substack<'a, Tok<String>>,
+  pub lazy: &'b mut HashMap<api::TreeId, MemberRecord>,
 }
 
-impl<'a> TreeIntoApiCtx for TIACtxImpl<'a> {
+impl<'a, 'b> TreeIntoApiCtx for TIACtxImpl<'a, 'b> {
-  fn sys(&self) -> &dyn DynSystem { self.sys }
+  fn sys(&self) -> SysCtx { self.ctx.clone() }
-  fn with_lazy(&mut self, fac: LazyMemberFactory) -> TreeId {
+  fn push_path(&mut self, seg: Tok<String>) -> impl TreeIntoApiCtx {
-    let id = TreeId(NonZero::new((self.lazy.len() + 2) as u64).unwrap());
+    TIACtxImpl {
-    self.lazy.insert(id, MemberRecord::Gen(fac));
+      ctx: self.ctx.clone(),
+      lazy: self.lazy,
+      basepath: self.basepath,
+      path: self.path.push(seg)
+    }
+  }
+  fn with_lazy(&mut self, fac: LazyMemberFactory) -> api::TreeId {
+    let id = api::TreeId(NonZero::new((self.lazy.len() + 2) as u64).unwrap());
+    let path = Sym::new(self.basepath.iter().cloned().chain(self.path.unreverse())).unwrap();
+    self.lazy.insert(id, MemberRecord::Gen(path, fac));
     id
   }
 }

orchid-host/Cargo.toml

@@ -15,4 +15,5 @@ orchid-api = { version = "0.1.0", path = "../orchid-api" }
 orchid-api-traits = { version = "0.1.0", path = "../orchid-api-traits" }
 orchid-base = { version = "0.1.0", path = "../orchid-base" }
 ordered-float = "4.2.0"
+paste = "1.0.15"
 substack = "1.1.0"

orchid-host/src/child.rs

@@ -10,8 +10,12 @@ pub struct SharedChild {
   debug: Option<(String, Mutex<Box<dyn fmt::Write>>)>,
 }
 impl SharedChild {
-  pub fn new(command: &mut process::Command, debug: Option<(&str, impl fmt::Write + 'static)>) -> io::Result<Self> {
+  pub fn new(
-    let mut child = command.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
+    command: &mut process::Command,
+    debug: Option<(&str, impl fmt::Write + 'static)>,
+  ) -> io::Result<Self> {
+    let mut child =
+      command.stdin(process::Stdio::piped()).stdout(process::Stdio::piped()).spawn()?;
     let stdin = Mutex::new(child.stdin.take().expect("Piped stdin above"));
     let stdout = Mutex::new(child.stdout.take().expect("Piped stdout above"));
     let debug = debug.map(|(n, w)| (n.to_string(), Mutex::new(Box::new(w) as Box<dyn fmt::Write>)));

orchid-host/src/expr.rs

@@ -4,8 +4,8 @@ use std::sync::{Arc, RwLock};
 
 use hashbrown::HashMap;
 use lazy_static::lazy_static;
-use orchid_api::expr::{Expr, ExprTicket};
 
+use crate::api;
 use crate::extension::{AtomHand, System};
 
 #[derive(Clone, Debug)]
@@ -16,20 +16,22 @@ pub struct RtExpr {
 impl RtExpr {
   pub fn as_atom(&self) -> Option<AtomHand> { todo!() }
   pub fn strong_count(&self) -> usize { todo!() }
-  pub fn id(&self) -> ExprTicket {
+  pub fn id(&self) -> api::ExprTicket {
-    ExprTicket(
+    api::ExprTicket(
       NonZeroU64::new(self.data.as_ref() as *const () as usize as u64)
-        .expect("this is a ref, it cannot be null")
+        .expect("this is a ref, it cannot be null"),
     )
   }
-  pub fn canonicalize(&self) -> ExprTicket {
+  pub fn canonicalize(&self) -> api::ExprTicket {
     if !self.is_canonical.swap(true, Ordering::Relaxed) {
       KNOWN_EXPRS.write().unwrap().entry(self.id()).or_insert_with(|| self.clone());
     }
     self.id()
   }
-  pub fn resolve(tk: ExprTicket) -> Option<Self> { KNOWN_EXPRS.read().unwrap().get(&tk).cloned() }
+  pub fn resolve(tk: api::ExprTicket) -> Option<Self> {
-  pub fn from_api(api: Expr, sys: &System) -> Self {
+    KNOWN_EXPRS.read().unwrap().get(&tk).cloned()
+  }
+  pub fn from_api(api: api::Expr, sys: &System) -> Self {
     Self { data: Arc::default(), is_canonical: Arc::default() }
   }
 }
@@ -46,5 +48,5 @@ impl Drop for RtExpr {
 }
 
 lazy_static! {
-  static ref KNOWN_EXPRS: RwLock<HashMap<ExprTicket, RtExpr>> = RwLock::default();
+  static ref KNOWN_EXPRS: RwLock<HashMap<api::ExprTicket, RtExpr>> = RwLock::default();
 }

orchid-host/src/extension.rs

@@ -1,82 +1,89 @@
-use orchid_api::logging::Log;
+use orchid_base::intern;
-use orchid_base::logging::Logger;
-use orchid_base::msg::{recv_msg, send_msg};
-use substack::{Stackframe, Substack};
 use std::collections::VecDeque;
-use std::io::{stderr, BufRead, BufReader, Write as _};
 use std::num::NonZero;
-use std::ops::Deref;
+use std::ops::{Deref, Range};
-use std::path::PathBuf;
-use std::process::ChildStdin;
 use std::sync::atomic::{AtomicU16, AtomicU32, AtomicU64, Ordering};
 use std::sync::mpsc::{sync_channel, SyncSender};
 use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
-use std::{fmt, io, process, thread};
+use std::{fmt, io, thread};
 
 use derive_destructure::destructure;
 use hashbrown::hash_map::Entry;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use lazy_static::lazy_static;
-use orchid_api::atom::{Atom, AtomDrop, AtomPrint, AtomSame, CallRef, FinalCall, Fwd, Fwded};
+use orchid_api_traits::{enc_vec, Decode, Request};
-use orchid_api::error::ProjResult;
-use orchid_api::expr::{Acquire, Expr, ExprNotif, ExprTicket, Release, Relocate};
-use orchid_api::interner::IntReq;
-use orchid_api::parser::{CharFilter, LexExpr, LexedExpr, ParsId, SubLex, SubLexed};
-use orchid_api::proto::{
-  ExtHostNotif, ExtHostReq, ExtensionHeader, HostExtNotif, HostHeader, HostMsgSet,
-};
-use orchid_api::system::{NewSystem, SysDeclId, SysId, SystemDecl, SystemDrop};
-use orchid_api::tree::{GetMember, Member, MemberKind, TreeId};
-use orchid_api_traits::{Decode, Encode, Request};
 use orchid_base::char_filter::char_filter_match;
 use orchid_base::clone;
+use orchid_base::error::{errv_from_apiv, mk_err, OrcRes};
 use orchid_base::interner::{deintern, intern, Tok};
+use orchid_base::logging::Logger;
 use orchid_base::reqnot::{ReqNot, Requester as _};
+use orchid_base::tree::{ttv_from_api, AtomInTok};
 use ordered_float::NotNan;
+use substack::{Stackframe, Substack};
 
+use crate::api;
 use crate::expr::RtExpr;
-use crate::tree::OwnedMember;
+use crate::tree::{Member, ParsTokTree};
 
 #[derive(Debug, destructure)]
 pub struct AtomData {
   owner: System,
-  drop: bool,
+  drop: Option<api::AtomId>,
   data: Vec<u8>,
 }
 impl AtomData {
-  fn api(self) -> Atom {
+  fn api(self) -> api::Atom {
     let (owner, drop, data) = self.destructure();
-    Atom { data, drop, owner: owner.id() }
+    api::Atom { data, drop, owner: owner.id() }
   }
-  fn api_ref(&self) -> Atom {
+  fn api_ref(&self) -> api::Atom {
-    Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
+    api::Atom { data: self.data.clone(), drop: self.drop, owner: self.owner.id() }
   }
 }
 impl Drop for AtomData {
   fn drop(&mut self) {
-    self.owner.reqnot().notify(AtomDrop(Atom {
+    if let Some(id) = self.drop {
-      owner: self.owner.id(),
+      self.owner.reqnot().notify(api::AtomDrop(self.owner.id(), id))
-      data: self.data.clone(),
+    }
-      drop: true,
-    }))
   }
 }
 
 #[derive(Clone, Debug)]
 pub struct AtomHand(Arc<AtomData>);
 impl AtomHand {
-  pub fn from_api(Atom { data, drop, owner }: Atom) -> Self {
+  fn create_new(api::Atom { data, drop, owner }: api::Atom) -> Self {
     let owner = System::resolve(owner).expect("Atom owned by non-existing system");
     Self(Arc::new(AtomData { data, drop, owner }))
   }
-  pub fn call(self, arg: RtExpr) -> Expr {
+  pub fn from_api(atom: api::Atom) -> Self {
+    if let Some(id) = atom.drop {
+      lazy_static! {
+        static ref OWNED_ATOMS: Mutex<HashMap<(api::SysId, api::AtomId), Weak<AtomData>>> =
+          Mutex::default();
+      }
+      let owner = atom.owner;
+      let mut owned_g = OWNED_ATOMS.lock().unwrap();
+      if let Some(data) = owned_g.get(&(owner, id)) {
+        if let Some(atom) = data.upgrade() {
+          return Self(atom);
+        }
+      }
+      let new = Self::create_new(atom);
+      owned_g.insert((owner, id), Arc::downgrade(&new.0));
+      new
+    } else {
+      Self::create_new(atom)
+    }
+  }
+  pub fn call(self, arg: RtExpr) -> api::Expr {
     let owner_sys = self.0.owner.clone();
     let reqnot = owner_sys.reqnot();
     let ticket = owner_sys.give_expr(arg.canonicalize(), || arg);
     match Arc::try_unwrap(self.0) {
-      Ok(data) => reqnot.request(FinalCall(data.api(), ticket)),
+      Ok(data) => reqnot.request(api::FinalCall(data.api(), ticket)),
-      Err(hand) => reqnot.request(CallRef(hand.api_ref(), ticket)),
+      Err(hand) => reqnot.request(api::CallRef(hand.api_ref(), ticket)),
     }
   }
   pub fn same(&self, other: &AtomHand) -> bool {
@@ -84,13 +91,34 @@ impl AtomHand {
     if other.0.owner.id() != owner {
       return false;
     }
-    self.0.owner.reqnot().request(AtomSame(self.0.api_ref(), other.0.api_ref()))
+    self.0.owner.reqnot().request(api::AtomSame(self.0.api_ref(), other.0.api_ref()))
   }
   pub fn req(&self, req: Vec<u8>) -> Vec<u8> {
-    self.0.owner.reqnot().request(Fwded(self.0.api_ref(), req))
+    self.0.owner.reqnot().request(api::Fwded(self.0.api_ref(), req))
   }
-  pub fn api_ref(&self) -> Atom { self.0.api_ref() }
+  pub fn api_ref(&self) -> api::Atom { self.0.api_ref() }
-  pub fn print(&self) -> String { self.0.owner.reqnot().request(AtomPrint(self.0.api_ref())) }
+  pub fn print(&self) -> String { self.0.owner.reqnot().request(api::AtomPrint(self.0.api_ref())) }
+}
+impl AtomInTok for AtomHand {
+  type Context = ();
+  fn from_api(atom: &orchid_api::Atom, _: Range<u32>, (): &mut Self::Context) -> Self {
+    Self::from_api(atom.clone())
+  }
+  fn to_api(&self) -> orchid_api::Atom { self.api_ref() }
+}
+impl fmt::Display for AtomHand {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(&self.print()) }
+}
+
+/// The 3 primary contact points with an extension are
+/// - send a message
+/// - wait for a message to arrive
+/// - wait for the extension to stop after exit (this is the implicit Drop)
+/// 
+/// There are no ordering guarantees about these
+pub trait ExtensionPort: Send + Sync {
+  fn send(&self, msg: &[u8]);
+  fn receive(&self) -> Option<Vec<u8>>;
 }
 
 /// Data held about an Extension. This is refcounted within [Extension]. It's
@@ -99,25 +127,25 @@ impl AtomHand {
 /// upgrading fails.
 #[derive(destructure)]
 pub struct ExtensionData {
-  child: Mutex<process::Child>,
+  port: Arc<dyn ExtensionPort>,
-  child_stdin: Mutex<ChildStdin>,
+  // child: Mutex<process::Child>,
-  reqnot: ReqNot<HostMsgSet>,
+  // child_stdin: Mutex<ChildStdin>,
+  reqnot: ReqNot<api::HostMsgSet>,
   systems: Vec<SystemCtor>,
   logger: Logger,
 }
 impl Drop for ExtensionData {
   fn drop(&mut self) {
-    self.reqnot.notify(HostExtNotif::Exit);
+    self.reqnot.notify(api::HostExtNotif::Exit);
-    self.child.lock().unwrap().wait().expect("Extension exited with error");
   }
 }
 
-fn acq_expr(sys: SysId, extk: ExprTicket) {
+fn acq_expr(sys: api::SysId, extk: api::ExprTicket) {
   (System::resolve(sys).expect("Expr acq'd by invalid system"))
     .give_expr(extk, || RtExpr::resolve(extk).expect("Invalid expr acq'd"));
 }
 
-fn rel_expr(sys: SysId, extk: ExprTicket) {
+fn rel_expr(sys: api::SysId, extk: api::ExprTicket) {
   let sys = System::resolve(sys).unwrap();
   let mut exprs = sys.0.exprs.write().unwrap();
   exprs.entry(extk).and_replace_entry_with(|_, (rc, rt)| {
@@ -128,89 +156,88 @@ fn rel_expr(sys: SysId, extk: ExprTicket) {
 #[derive(Clone)]
 pub struct Extension(Arc<ExtensionData>);
 impl Extension {
-  pub fn new(mut cmd: process::Command, logger: Logger) -> io::Result<Self> {
+  
-    let mut child = cmd
+  pub fn new_process(port: Arc<dyn ExtensionPort>, logger: Logger) -> io::Result<Self> {
-      .stdin(process::Stdio::piped())
+    port.send(&enc_vec(&api::HostHeader { log_strategy: logger.strat() }));
-      .stdout(process::Stdio::piped())
+    let header_reply = port.receive().expect("Extension exited immediately");
-      .stderr(process::Stdio::piped())
+    let eh = api::ExtensionHeader::decode(&mut &header_reply[..]);
-      .spawn()?;
-    let mut child_stdin = child.stdin.take().unwrap();
-    let mut child_stdout = child.stdout.take().unwrap();
-    let child_stderr = child.stderr.take().unwrap();
-    thread::Builder::new().name("stderr forwarder".to_string()).spawn(|| {
-      let mut reader = BufReader::new(child_stderr);
-      loop {
-        let mut buf = String::new();
-        if 0 == reader.read_line(&mut buf).unwrap() {
-          break;
-        }
-        stderr().write_all(buf.as_bytes()).unwrap();
-      }
-    }).unwrap();
-    HostHeader{ log_strategy: logger.strat() }.encode(&mut child_stdin);
-    let eh = ExtensionHeader::decode(&mut child_stdout);
     let ret = Arc::new_cyclic(|weak: &Weak<ExtensionData>| ExtensionData {
       logger,
-      child: Mutex::new(child),
+      port: port.clone(),
-      child_stdin: Mutex::new(child_stdin),
       reqnot: ReqNot::new(
         clone!(weak; move |sfn, _| {
-          eprintln!("Downsending {:?}", sfn);
+          let data = weak.upgrade().unwrap();
-          send_msg(&mut *weak.upgrade().unwrap().child_stdin.lock().unwrap(), sfn).unwrap();
+          data.logger.log_buf("Downsending", sfn);
+          data.port.send(sfn);
         }),
         clone!(weak; move |notif, _| match notif {
-          ExtHostNotif::ExprNotif(ExprNotif::Acquire(Acquire(sys, extk))) => acq_expr(sys, extk),
+          api::ExtHostNotif::ExprNotif(api::ExprNotif::Acquire(acq)) => acq_expr(acq.0, acq.1),
-          ExtHostNotif::ExprNotif(ExprNotif::Release(Release(sys, extk))) => rel_expr(sys, extk),
+          api::ExtHostNotif::ExprNotif(api::ExprNotif::Release(rel)) => rel_expr(rel.0, rel.1),
-          ExtHostNotif::ExprNotif(ExprNotif::Relocate(Relocate { dec, inc, expr })) => {
+          api::ExtHostNotif::ExprNotif(api::ExprNotif::Move(mov)) => {
-            acq_expr(inc, expr);
+            acq_expr(mov.inc, mov.expr);
-            rel_expr(dec, expr);
+            rel_expr(mov.dec, mov.expr);
           },
-          ExtHostNotif::AdviseSweep(_advice) => eprintln!("Sweep advice is unsupported"),
+          api::ExtHostNotif::Log(api::Log(str)) => weak.upgrade().unwrap().logger.log(str),
-          ExtHostNotif::Log(Log(str)) => weak.upgrade().unwrap().logger.log(str),
         }),
         |req| match req.req() {
-          ExtHostReq::Ping(ping) => req.handle(ping, &()),
+          api::ExtHostReq::Ping(ping) => req.handle(ping, &()),
-          ExtHostReq::IntReq(IntReq::InternStr(s)) => req.handle(s, &intern(&**s.0).marker()),
+          api::ExtHostReq::IntReq(intreq) => match intreq {
-          ExtHostReq::IntReq(IntReq::InternStrv(v)) => req.handle(v, &intern(&*v.0).marker()),
+            api::IntReq::InternStr(s) => req.handle(s, &intern(&**s.0).marker()),
-          ExtHostReq::IntReq(IntReq::ExternStr(si)) => req.handle(si, &deintern(si.0).arc()),
+            api::IntReq::InternStrv(v) => req.handle(v, &intern(&*v.0).marker()),
-          ExtHostReq::IntReq(IntReq::ExternStrv(vi)) =>
+            api::IntReq::ExternStr(si) => req.handle(si, &deintern(si.0).arc()),
+            api::IntReq::ExternStrv(vi) =>
               req.handle(vi, &Arc::new(deintern(vi.0).iter().map(|t| t.marker()).collect_vec())),
-          ExtHostReq::Fwd(fw @ Fwd(atom, _body)) => {
+          }
+          api::ExtHostReq::Fwd(fw @ api::Fwd(atom, _body)) => {
             let sys = System::resolve(atom.owner).unwrap();
-            thread::spawn(clone!(fw; move || {
+            req.handle(fw, &sys.reqnot().request(api::Fwded(fw.0.clone(), fw.1.clone())))
-              req.handle(&fw, &sys.reqnot().request(Fwded(fw.0.clone(), fw.1.clone())))
-            }));
           },
-          ExtHostReq::SubLex(sl) => {
+          api::ExtHostReq::SubLex(sl) => {
-            let lex_g = LEX_RECUR.lock().unwrap();
             let (rep_in, rep_out) = sync_channel(0);
+            let lex_g = LEX_RECUR.lock().unwrap();
             let req_in = lex_g.get(&sl.id).expect("Sublex for nonexistent lexid");
             req_in.send(ReqPair(sl.clone(), rep_in)).unwrap();
             req.handle(sl, &rep_out.recv().unwrap())
           },
-          _ => todo!(),
+          api::ExtHostReq::ExprReq(api::ExprReq::Inspect(ins@api::Inspect(tk))) => {
+            let expr = RtExpr::resolve(*tk);
+            req.handle(ins, &api::Details{
+              refcount: 1,
+              expr: api::Expr{
+                location: api::Location::None,
+                clause: api::Clause::Bottom(vec![
+                  mk_err(
+                    intern!(str: "Unsupported"), 
+                    "Inspecting clauses is unsupported at the moment", 
+                    []
+                  )
+                  .to_api()
+                ])
+              }
+            })
+          }
         },
       ),
       systems: eh.systems.into_iter().map(|decl| SystemCtor { decl, ext: weak.clone() }).collect(),
     });
     let weak = Arc::downgrade(&ret);
-    let prog_pbuf = PathBuf::from(cmd.get_program());
+    thread::Builder::new()
-    let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy();
+      .name(format!("host-end:{}", eh.name))
-    thread::Builder::new().name(format!("host-end:{}", prog)).spawn(move || {
+      .spawn::<_, Option<()>>(move || loop {
-      loop {
+        // thread will exit if either the peer exits or the extension object is dropped.
-        let ingress = recv_msg(&mut child_stdout).expect("could not receive");
+        // It holds a strong reference to the port so the port's destructor will not be called
-        if let Some(sys) = weak.upgrade() {
+        // until the 
-          sys.reqnot.receive(ingress);
+        let msg = port.receive()?;
-        }
+        weak.upgrade()?.reqnot.receive(msg);
-      }
+      })
-    }).unwrap();
+      .unwrap();
     Ok(Self(ret))
   }
   pub fn systems(&self) -> impl Iterator<Item = &SystemCtor> { self.0.systems.iter() }
 }
 
 pub struct SystemCtor {
-  decl: SystemDecl,
+  decl: api::SystemDecl,
   ext: Weak<ExtensionData>,
 }
 impl SystemCtor {
@@ -225,18 +252,20 @@ impl SystemCtor {
     debug_assert_eq!(depends.len(), self.decl.depends.len(), "Wrong number of deps provided");
     let ext = self.ext.upgrade().expect("SystemCtor should be freed before Extension");
     static NEXT_ID: AtomicU16 = AtomicU16::new(1);
-    let id = SysId(NonZero::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped"));
+    let id =
-    let sys_inst = ext.reqnot.request(NewSystem { depends, id, system: self.decl.id });
+      api::SysId(NonZero::new(NEXT_ID.fetch_add(1, Ordering::Relaxed)).expect("next_id wrapped"));
+    let sys_inst = ext.reqnot.request(api::NewSystem { depends, id, system: self.decl.id });
     let data = System(Arc::new(SystemInstData {
       decl_id: self.decl.id,
       ext: Extension(ext),
       exprs: RwLock::default(),
       lex_filter: sys_inst.lex_filter,
       const_root: OnceLock::new(),
+      line_types: sys_inst.line_types.into_iter().map(deintern).collect(),
       id,
     }));
     let root = (sys_inst.const_root.into_iter())
-        .map(|(k, v)| OwnedMember::from_api(Member { public: true, name: k, kind: v }, &data))
+      .map(|(k, v)| Member::from_api(api::Member { exported: true, name: k, kind: v }, &data))
       .collect_vec();
     data.0.const_root.set(root).unwrap();
     inst_g.insert(id, data.clone());
@@ -245,24 +274,26 @@ impl SystemCtor {
 }
 
 lazy_static! {
-  static ref SYSTEM_INSTS: RwLock<HashMap<SysId, System>> = RwLock::default();
+  static ref SYSTEM_INSTS: RwLock<HashMap<api::SysId, System>> = RwLock::default();
-  static ref LEX_RECUR: Mutex<HashMap<ParsId, SyncSender<ReqPair<SubLex>>>> = Mutex::default();
+  static ref LEX_RECUR: Mutex<HashMap<api::ParsId, SyncSender<ReqPair<api::SubLex>>>> =
+    Mutex::default();
 }
 
 pub struct ReqPair<R: Request>(R, pub SyncSender<R::Response>);
 
 #[derive(destructure)]
 pub struct SystemInstData {
-  exprs: RwLock<HashMap<ExprTicket, (AtomicU32, RtExpr)>>,
+  exprs: RwLock<HashMap<api::ExprTicket, (AtomicU32, RtExpr)>>,
   ext: Extension,
-  decl_id: SysDeclId,
+  decl_id: api::SysDeclId,
-  lex_filter: CharFilter,
+  lex_filter: api::CharFilter,
-  id: SysId,
+  id: api::SysId,
-  const_root: OnceLock<Vec<OwnedMember>>,
+  const_root: OnceLock<Vec<Member>>,
+  line_types: Vec<Tok<String>>,
 }
 impl Drop for SystemInstData {
   fn drop(&mut self) {
-    self.ext.0.reqnot.notify(SystemDrop(self.id));
+    self.ext.0.reqnot.notify(api::SystemDrop(self.id));
     if let Ok(mut g) = SYSTEM_INSTS.write() {
       g.remove(&self.id);
     }
@@ -271,22 +302,26 @@ impl Drop for SystemInstData {
 #[derive(Clone)]
 pub struct System(Arc<SystemInstData>);
 impl System {
-  pub fn id(&self) -> SysId { self.id }
+  pub fn id(&self) -> api::SysId { self.id }
-  fn resolve(id: SysId) -> Option<System> { SYSTEM_INSTS.read().unwrap().get(&id).cloned() }
+  fn resolve(id: api::SysId) -> Option<System> { SYSTEM_INSTS.read().unwrap().get(&id).cloned() }
-  fn reqnot(&self) -> &ReqNot<HostMsgSet> { &self.0.ext.0.reqnot }
+  fn reqnot(&self) -> &ReqNot<api::HostMsgSet> { &self.0.ext.0.reqnot }
-  fn give_expr(&self, ticket: ExprTicket, get_expr: impl FnOnce() -> RtExpr) -> ExprTicket {
+  fn give_expr(
+    &self,
+    ticket: api::ExprTicket,
+    get_expr: impl FnOnce() -> RtExpr,
+  ) -> api::ExprTicket {
     match self.0.exprs.write().unwrap().entry(ticket) {
       Entry::Occupied(mut oe) => {
         oe.get_mut().0.fetch_add(1, Ordering::Relaxed);
       },
       Entry::Vacant(v) => {
         v.insert((AtomicU32::new(1), get_expr()));
-      }
+      },
     }
     ticket
   }
-  pub fn get_tree(&self, id: TreeId) -> MemberKind {
+  pub fn get_tree(&self, id: api::TreeId) -> api::MemberKind {
-    self.reqnot().request(GetMember(self.0.id, id))
+    self.reqnot().request(api::GetMember(self.0.id, id))
   }
   pub fn has_lexer(&self) -> bool { !self.0.lex_filter.0.is_empty() }
   pub fn can_lex(&self, c: char) -> bool { char_filter_match(&self.0.lex_filter, c) }
@@ -296,11 +331,11 @@ impl System {
     &self,
     source: Tok<String>,
     pos: u32,
-    mut r: impl FnMut(u32) -> Option<SubLexed> + Send,
+    mut r: impl FnMut(u32) -> Option<api::SubLexed> + Send,
-  ) -> ProjResult<Option<LexedExpr>> {
+  ) -> api::OrcResult<Option<api::LexedExpr>> {
     // get unique lex ID
     static LEX_ID: AtomicU64 = AtomicU64::new(1);
-    let id = ParsId(NonZero::new(LEX_ID.fetch_add(1, Ordering::Relaxed)).unwrap());
+    let id = api::ParsId(NonZero::new(LEX_ID.fetch_add(1, Ordering::Relaxed)).unwrap());
     thread::scope(|s| {
       // create and register channel
       let (req_in, req_out) = sync_channel(0);
@@ -312,12 +347,23 @@ impl System {
         }
       });
       // Pass control to extension
-      let ret = self.reqnot().request(LexExpr { id, pos, sys: self.id(), text: source.marker() });
+      let ret =
+        self.reqnot().request(api::LexExpr { id, pos, sys: self.id(), text: source.marker() });
       // collect sender to unblock recursion handler thread before returning
       LEX_RECUR.lock().unwrap().remove(&id);
       ret.transpose()
     }) // exit recursion handler thread
   }
+  pub fn can_parse(&self, line_type: Tok<String>) -> bool { self.line_types.contains(&line_type) }
+  pub fn line_types(&self) -> impl Iterator<Item = Tok<String>> + '_ {
+    self.line_types.iter().cloned()
+  }
+  pub fn parse(&self, line: Vec<ParsTokTree>) -> OrcRes<Vec<ParsTokTree>> {
+    let line = line.iter().map(|t| t.to_api(&mut |n, _| match *n {})).collect_vec();
+    let parsed = (self.reqnot().request(api::ParseLine { sys: self.id(), line }))
+      .map_err(|e| errv_from_apiv(e.iter()))?;
+    Ok(ttv_from_api(parsed, &mut ()))
+  }
 }
 impl fmt::Debug for System {
   fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -341,12 +387,12 @@ impl Deref for System {
 #[derive(Debug, Clone)]
 pub enum SysResolvErr {
   Loop(Vec<String>),
-  Missing(String)
+  Missing(String),
 }
 
 pub fn init_systems(tgts: &[String], exts: &[Extension]) -> Result<Vec<System>, SysResolvErr> {
   let mut to_load = HashMap::<&str, &SystemCtor>::new();
-  let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque::<&str>>();
+  let mut to_find = tgts.iter().map(|s| s.as_str()).collect::<VecDeque<&str>>();
   while let Some(target) = to_find.pop_front() {
     if to_load.contains_key(target) {
       continue;
@@ -360,17 +406,18 @@ pub fn init_systems(tgts: &[String], exts: &[Extension]) -> Result<Vec<System>,
   }
   let mut to_load_ordered = Vec::new();
   fn walk_deps<'a>(
-    graph: &mut HashMap::<&str, &'a SystemCtor>,
+    graph: &mut HashMap<&str, &'a SystemCtor>,
     list: &mut Vec<&'a SystemCtor>,
-    chain: Stackframe<&str>
+    chain: Stackframe<&str>,
   ) -> Result<(), SysResolvErr> {
     if let Some(ctor) = graph.remove(chain.item) {
-      // if the above is none, the system is already queued. Missing systems are detected above
+      // if the above is none, the system is already queued. Missing systems are
+      // detected above
       for dep in ctor.decl.depends.iter() {
         if Substack::Frame(chain).iter().any(|c| c == dep) {
           let mut circle = vec![dep.to_string()];
           circle.extend(Substack::Frame(chain).iter().map(|s| s.to_string()));
-          return Err(SysResolvErr::Loop(circle))
+          return Err(SysResolvErr::Loop(circle));
         }
         walk_deps(graph, list, Substack::Frame(chain).new_frame(dep))?
       }

orchid-host/src/lex.rs

@@ -1,24 +1,23 @@
 use std::num::NonZeroU64;
+use std::sync::Arc;
 
 use hashbrown::HashMap;
-use orchid_api::parser::SubLexed;
+use orchid_base::error::{mk_err, OrcErr, OrcRes};
-use orchid_api::system::SysId;
-use orchid_api::tree::{Token, TokenTree, TreeTicket};
-use orchid_base::error::OwnedError;
 use orchid_base::intern;
 use orchid_base::interner::{deintern, intern, Tok};
 use orchid_base::location::Pos;
+use orchid_base::parse::{name_char, name_start, op_char, unrep_space};
 use orchid_base::tokens::{OwnedPh, PARENS};
 
+use crate::api;
 use crate::extension::{AtomHand, System};
-use crate::results::{mk_err, OwnedResult};
+use crate::tree::{ParsTok, ParsTokTree};
-use crate::tree::{OwnedTok, OwnedTokTree};
 
 pub struct LexCtx<'a> {
   pub systems: &'a [System],
   pub source: &'a Tok<String>,
   pub tail: &'a str,
-  pub sub_trees: &'a mut HashMap<TreeTicket, OwnedTokTree>,
+  pub sub_trees: &'a mut HashMap<api::TreeTicket, ParsTokTree>,
 }
 impl<'a> LexCtx<'a> {
   pub fn push<'b>(&'b mut self, pos: u32) -> LexCtx<'b>
@@ -42,12 +41,12 @@ impl<'a> LexCtx<'a> {
     }
     false
   }
-  pub fn add_subtree(&mut self, subtree: OwnedTokTree) -> TreeTicket {
+  pub fn add_subtree(&mut self, subtree: ParsTokTree) -> api::TreeTicket {
-    let next_idx = TreeTicket(NonZeroU64::new(self.sub_trees.len() as u64 + 1).unwrap());
+    let next_idx = api::TreeTicket(NonZeroU64::new(self.sub_trees.len() as u64 + 1).unwrap());
     self.sub_trees.insert(next_idx, subtree);
     next_idx
   }
-  pub fn rm_subtree(&mut self, ticket: TreeTicket) -> OwnedTokTree {
+  pub fn rm_subtree(&mut self, ticket: api::TreeTicket) -> ParsTokTree {
     self.sub_trees.remove(&ticket).unwrap()
   }
   pub fn strip_char(&mut self, tgt: char) -> bool {
@@ -69,7 +68,7 @@ impl<'a> LexCtx<'a> {
   }
 }
 
-pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
+pub fn lex_once(ctx: &mut LexCtx) -> OrcRes<ParsTokTree> {
   let start = ctx.get_pos();
   assert!(
     !ctx.tail.is_empty() && !ctx.tail.starts_with(unrep_space),
@@ -77,9 +76,9 @@ pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
     Invocations of lex_tok should check for empty string"
   );
   let tok = if ctx.strip_prefix("\r\n") || ctx.strip_prefix("\r") || ctx.strip_prefix("\n") {
-    OwnedTok::BR
+    ParsTok::BR
   } else if ctx.strip_prefix("::") {
-    OwnedTok::NS
+    ParsTok::NS
   } else if ctx.strip_prefix("--[") {
     let (cmt, tail) = ctx.tail.split_once("]--").ok_or_else(|| {
       vec![mk_err(
@@ -89,11 +88,11 @@ pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
       )]
     })?;
     ctx.set_tail(tail);
-    OwnedTok::Comment(cmt.to_string())
+    ParsTok::Comment(Arc::new(cmt.to_string()))
   } else if let Some(tail) = ctx.tail.strip_prefix("--").filter(|t| !t.starts_with(op_char)) {
     let end = tail.find(['\n', '\r']).map_or(tail.len(), |n| n - 1);
     ctx.push_pos(end as u32);
-    OwnedTok::Comment(tail[2..end].to_string())
+    ParsTok::Comment(Arc::new(tail[2..end].to_string()))
   } else if ctx.strip_char('\\') {
     let mut arg = Vec::new();
     ctx.trim_ws();
@@ -108,7 +107,7 @@ pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
       arg.push(lex_once(ctx)?);
       ctx.trim_ws();
     }
-    OwnedTok::Lambda(arg)
+    ParsTok::LambdaHead(arg)
   } else if let Some((lp, rp, paren)) = PARENS.iter().find(|(lp, ..)| ctx.strip_char(*lp)) {
     let mut body = Vec::new();
     ctx.trim_ws();
@@ -123,31 +122,32 @@ pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
       body.push(lex_once(ctx)?);
       ctx.trim_ws();
     }
-    OwnedTok::S(paren.clone(), body)
+    ParsTok::S(paren.clone(), body)
   } else {
     for sys in ctx.systems {
       let mut errors = Vec::new();
       if ctx.tail.starts_with(|c| sys.can_lex(c)) {
         let lexed = sys.lex(ctx.source.clone(), ctx.get_pos(), |pos| {
           let mut sub_ctx = ctx.push(pos);
-          let ott = lex_once(&mut sub_ctx).inspect_err(|e| errors.extend(e.iter().cloned())).ok()?;
+          let ott =
-          Some(SubLexed { pos: sub_ctx.get_pos(), ticket: sub_ctx.add_subtree(ott) })
+            lex_once(&mut sub_ctx).inspect_err(|e| errors.extend(e.iter().cloned())).ok()?;
+          Some(api::SubLexed { pos: sub_ctx.get_pos(), ticket: sub_ctx.add_subtree(ott) })
         });
         match lexed {
           Ok(None) if errors.is_empty() => continue,
           Ok(None) => return Err(errors),
-          Err(e) => return Err(e.into_iter().map(|e| OwnedError::from_api(&e)).collect()),
+          Err(e) => return Err(e.into_iter().map(|e| OrcErr::from_api(&e)).collect()),
           Ok(Some(lexed)) => {
             ctx.set_pos(lexed.pos);
-            return Ok(tt_to_owned(&lexed.expr, sys.id(), ctx))
+            return Ok(tt_to_owned(&lexed.expr, ctx));
           },
         }
       }
     }
     if ctx.tail.starts_with(name_start) {
-      OwnedTok::Name(intern(ctx.get_start_matches(name_char)))
+      ParsTok::Name(intern(ctx.get_start_matches(name_char)))
     } else if ctx.tail.starts_with(op_char) {
-      OwnedTok::Name(intern(ctx.get_start_matches(op_char)))
+      ParsTok::Name(intern(ctx.get_start_matches(op_char)))
     } else {
       return Err(vec![mk_err(
         intern!(str: "Unrecognized character"),
@@ -156,37 +156,29 @@ pub fn lex_once(ctx: &mut LexCtx) -> OwnedResult<OwnedTokTree> {
       )]);
     }
   };
-  Ok(OwnedTokTree { tok, range: start..ctx.get_pos() })
+  Ok(ParsTokTree { tok, range: start..ctx.get_pos() })
 }
 
-fn name_start(c: char) -> bool { c.is_alphabetic() || c == '_' }
+fn tt_to_owned(api: &api::TokenTree, ctx: &mut LexCtx<'_>) -> ParsTokTree {
-fn name_char(c: char) -> bool { name_start(c) || c.is_numeric() }
-fn op_char(c: char) -> bool { !name_char(c) && !c.is_whitespace() && !"()[]{}\\".contains(c) }
-fn unrep_space(c: char) -> bool { c.is_whitespace() && !"\r\n".contains(c) }
-
-fn tt_to_owned(api: &TokenTree, sys: SysId, ctx: &mut LexCtx<'_>) -> OwnedTokTree {
   let tok = match &api.token {
-    Token::Atom(atom) => OwnedTok::Atom(AtomHand::from_api(atom.clone().associate(sys))),
+    api::Token::Atom(atom) => ParsTok::Atom(AtomHand::from_api(atom.clone())),
-    Token::Ph(ph) => OwnedTok::Ph(OwnedPh::from_api(ph.clone())),
+    api::Token::Ph(ph) => ParsTok::Ph(OwnedPh::from_api(ph.clone())),
-    Token::Bottom(err) => OwnedTok::Bottom(err.iter().map(OwnedError::from_api).collect()),
+    api::Token::Bottom(err) => ParsTok::Bottom(err.iter().map(OrcErr::from_api).collect()),
-    Token::Lambda(arg) => OwnedTok::Lambda(arg.iter().map(|t| tt_to_owned(t, sys, ctx)).collect()),
+    api::Token::Lambda(arg) =>
-    Token::Name(name) => OwnedTok::Name(deintern(*name)),
+      ParsTok::LambdaHead(arg.iter().map(|t| tt_to_owned(t, ctx)).collect()),
-    Token::S(p, b) => OwnedTok::S(p.clone(), b.iter().map(|t| tt_to_owned(t, sys, ctx)).collect()),
+    api::Token::Name(name) => ParsTok::Name(deintern(*name)),
-    Token::Slot(id) => return ctx.rm_subtree(*id),
+    api::Token::S(p, b) => ParsTok::S(p.clone(), b.iter().map(|t| tt_to_owned(t, ctx)).collect()),
-    Token::BR => OwnedTok::BR,
+    api::Token::Slot(id) => return ctx.rm_subtree(*id),
-    Token::NS => OwnedTok::NS,
+    api::Token::BR => ParsTok::BR,
+    api::Token::NS => ParsTok::NS,
+    api::Token::Comment(c) => ParsTok::Comment(c.clone()),
   };
-  OwnedTokTree { range: api.range.clone(), tok }
+  ParsTokTree { range: api.range.clone(), tok }
 }
 
-pub fn lex(text: Tok<String>, systems: &[System]) -> OwnedResult<Vec<OwnedTokTree>> {
+pub fn lex(text: Tok<String>, systems: &[System]) -> OrcRes<Vec<ParsTokTree>> {
   let mut sub_trees = HashMap::new();
-  let mut ctx = LexCtx {
+  let mut ctx = LexCtx { source: &text, sub_trees: &mut sub_trees, tail: &text[..], systems };
-    source: &text,
-    sub_trees: &mut sub_trees,
-    tail: &text[..],
-    systems,
-  };
   let mut tokv = Vec::new();
   ctx.trim(unrep_space);
   while !ctx.tail.is_empty() {

orchid-host/src/lib.rs

@@ -1,7 +1,9 @@
+use orchid_api as api;
+
 pub mod child;
 pub mod expr;
 pub mod extension;
 pub mod lex;
-pub mod results;
-pub mod tree;
 pub mod parse;
+pub mod tree;
+pub mod subprocess;

orchid-host/src/parse.rs

@@ -1,33 +1,187 @@
+use std::{iter, thread};
+
+use itertools::Itertools;
+use never::Never;
+use orchid_base::error::{mk_err, OrcErr, OrcRes, Reporter};
+use orchid_base::intern;
 use orchid_base::interner::Tok;
-
+use orchid_base::location::Pos;
-use crate::tree::{OwnedItem, OwnedModule, OwnedTok, OwnedTokTree};
+use orchid_base::parse::{
-
+  expect_end, expect_tok, line_items, parse_multiname, strip_fluff, try_pop_no_fluff, Comment, CompName, Snippet
-pub struct ParseCtx<'a> {
-  tokens: &'a [OwnedTokTree]
-}
-
-pub fn split_br(ctx: ParseCtx) -> impl Iterator<Item = ParseCtx> {
-  ctx.tokens.split(|t| matches!(t.tok, OwnedTok::BR)).map(|tokens| ParseCtx { tokens })
-}
-
-pub fn strip_br(tt: &OwnedTokTree) -> Option<OwnedTokTree> {
-  let tok = match &tt.tok {
-    OwnedTok::BR => return None,
-    OwnedTok::Lambda(arg) => OwnedTok::Lambda(arg.iter().filter_map(strip_br).collect()),
-    OwnedTok::S(p, b) => OwnedTok::S(p.clone(), b.iter().filter_map(strip_br).collect()),
-    t => t.clone(),
 };
-  Some(OwnedTokTree { tok, range: tt.range.clone() })
+use orchid_base::tree::{Paren, TokTree, Token};
+
+use crate::extension::{AtomHand, System};
+use crate::tree::{Item, ItemKind, Macro, Member, MemberKind, Module, ParsTokTree};
+
+type ParsSnippet<'a> = Snippet<'a, 'static, AtomHand, Never>;
+
+pub trait ParseCtx: Send + Sync {
+  fn systems(&self) -> impl Iterator<Item = &System>;
+  fn reporter(&self) -> &impl Reporter;
 }
 
-pub fn parse_items(ctx: ParseCtx) -> Vec<OwnedItem> {
+pub fn parse_items(ctx: &impl ParseCtx, items: ParsSnippet) -> OrcRes<Vec<Item>> {
-  todo!()
+  let lines = line_items(items);
+  let mut ok = iter::from_fn(|| None).take(lines.len()).collect_vec();
+  thread::scope(|s| {
+    let mut threads = Vec::new();
+    for (slot, (cmts, item)) in ok.iter_mut().zip(lines.into_iter()) {
+      threads.push(s.spawn(move || {
+        *slot = Some(parse_item(ctx, cmts, item)?);
+        Ok::<(), Vec<OrcErr>>(())
+      }))
+    }
+    for t in threads {
+      t.join().unwrap().err().into_iter().flatten().for_each(|e| ctx.reporter().report(e))
+    }
+  });
+  Ok(ok.into_iter().flatten().flatten().collect_vec())
 }
 
-pub fn parse_item(ctx: ParseCtx) -> OwnedItem {
+pub fn parse_item(
-  todo!()
+  ctx: &impl ParseCtx,
+  comments: Vec<Comment>,
+  item: ParsSnippet,
+) -> OrcRes<Vec<Item>> {
+  match item.pop_front() {
+    Some((TokTree { tok: Token::Name(n), .. }, postdisc)) => match n {
+      n if *n == intern!(str: "export") => match try_pop_no_fluff(postdisc)? {
+        (TokTree { tok: Token::Name(n), .. }, postdisc) =>
+          parse_item_2(ctx, comments, true, n.clone(), postdisc),
+        (TokTree { tok: Token::NS, .. }, postdisc) => {
+          let (exports, surplus) = parse_multiname(ctx.reporter(), postdisc)?;
+          let mut ok = Vec::new();
+          exports.into_iter().for_each(|e| match (&e.path.as_slice(), e.name) {
+            ([], Some(n)) => ok.push(Item {
+              comments: comments.clone(),
+              pos: e.pos.clone(),
+              kind: ItemKind::Export(n),
+            }),
+            (_, Some(_)) => ctx.reporter().report(mk_err(
+              intern!(str: "Compound export"),
+              "Cannot export compound names (names containing the :: separator)",
+              [e.pos.into()],
+            )),
+            (_, None) => ctx.reporter().report(mk_err(
+              intern!(str: "Wildcard export"),
+              "Exports cannot contain the globstar *",
+              [e.pos.into()],
+            )),
+          });
+          expect_end(surplus)?;
+          Ok(ok)
+        },
+        (bogus, _) => Err(vec![mk_err(
+          intern!(str: "Malformed export"),
+          "`export` can either prefix other lines or list names inside ::( ) or ::[ ]",
+          [Pos::Range(bogus.range.clone()).into()],
+        )]),
+      },
+      n if *n == intern!(str: "import") => parse_import(ctx, postdisc).map(|v| {
+        Vec::from_iter(v.into_iter().map(|t| Item {
+          comments: comments.clone(),
+          pos: Pos::Range(postdisc.pos()),
+          kind: ItemKind::Import(t),
+        }))
+      }),
+      n => parse_item_2(ctx, comments, false, n.clone(), postdisc),
+    },
+    Some(_) => Err(vec![mk_err(
+      intern!(str: "Expected a line type"),
+      "All lines must begin with a keyword",
+      [Pos::Range(item.pos()).into()],
+    )]),
+    None => unreachable!("These lines are filtered and aggregated in earlier stages"),
+  }
 }
 
-pub fn parse_module(ctx: ParseCtx) -> (Tok<String>, OwnedModule) {
+pub fn parse_import(ctx: &impl ParseCtx, tail: ParsSnippet) -> OrcRes<Vec<CompName>> {
-  todo!()
+  let (imports, surplus) = parse_multiname(ctx.reporter(), tail)?;
+  expect_end(surplus)?;
+  Ok(imports)
+}
+
+pub fn parse_item_2(
+  ctx: &impl ParseCtx,
+  comments: Vec<Comment>,
+  exported: bool,
+  discr: Tok<String>,
+  tail: ParsSnippet,
+) -> OrcRes<Vec<Item>> {
+  let kind = if discr == intern!(str: "mod") {
+    let (name, body) = parse_module(ctx, tail)?;
+    ItemKind::Member(Member::new(exported, name, MemberKind::Mod(body)))
+  } else if discr == intern!(str: "const") {
+    let (name, val) = parse_const(tail)?;
+    ItemKind::Member(Member::new(exported, name, MemberKind::Const(val)))
+  } else if discr == intern!(str: "macro") {
+    ItemKind::Rule(parse_macro(tail)?)
+  } else if let Some(sys) = ctx.systems().find(|s| s.can_parse(discr.clone())) {
+    let line = sys.parse(tail.to_vec())?;
+    return parse_items(ctx, Snippet::new(tail.prev(), &line))
+  } else {
+    let ext_lines = ctx.systems().flat_map(System::line_types).join(", ");
+    return Err(vec![mk_err(
+      intern!(str: "Unrecognized line type"),
+      format!("Line types are: const, mod, macro, grammar, {ext_lines}"),
+      [Pos::Range(tail.prev().range.clone()).into()]
+    )])
+  };
+  Ok(vec![Item { comments, pos: Pos::Range(tail.pos()), kind }])
+}
+
+pub fn parse_module(
+  ctx: &impl ParseCtx,
+  tail: ParsSnippet,
+) -> OrcRes<(Tok<String>, Module)> {
+  let (name, tail) = match try_pop_no_fluff(tail)? {
+    (TokTree { tok: Token::Name(n), .. }, tail) => (n.clone(), tail),
+    (tt, _) =>
+      return Err(vec![mk_err(
+        intern!(str: "Missing module name"),
+        format!("A name was expected, {tt} was found"),
+        [Pos::Range(tt.range.clone()).into()],
+      )]),
+  };
+  let (body, surplus) = match try_pop_no_fluff(tail)? {
+    (TokTree { tok: Token::S(Paren::Round, b), .. }, tail) => (b, tail),
+    (tt, _) =>
+      return Err(vec![mk_err(
+        intern!(str: "Expected module body"),
+        format!("A ( block ) was expected, {tt} was found"),
+        [Pos::Range(tt.range.clone()).into()],
+      )]),
+  };
+  let items = parse_items(ctx, ParsSnippet::new(surplus.prev(), body))?;
+  Ok((name, Module { imports: vec![], items }))
+}
+
+pub fn parse_const(tail: ParsSnippet) -> OrcRes<(Tok<String>, Vec<ParsTokTree>)> {
+  let (name, tail) = match try_pop_no_fluff(tail)? {
+    (TokTree { tok: Token::Name(n), .. }, tail) => (n.clone(), tail),
+    (tt, _) =>
+      return Err(vec![mk_err(
+        intern!(str: "Missing module name"),
+        format!("A name was expected, {tt} was found"),
+        [Pos::Range(tt.range.clone()).into()],
+      )]),
+  };
+  let tail = match try_pop_no_fluff(tail)? {
+    (TokTree { tok: Token::Name(n), .. }, tail) if *n == intern!(str: ":=") => tail,
+    (tt, _) =>
+      return Err(vec![mk_err(
+        intern!(str: "Missing walrus := separator"),
+        format!("Expected operator := , found {tt}"),
+        [Pos::Range(tt.range.clone()).into()],
+      )]),
+  };
+  try_pop_no_fluff(tail)?;
+  Ok((name, tail.iter().flat_map(strip_fluff).collect_vec()))
+}
+
+pub fn parse_macro(tail: ParsSnippet) -> OrcRes<Macro> {
+  let tail = expect_tok(tail, intern!(str: "prio"))?;
+  let (prio, tail) = ((), ());
+  todo!();
 }

orchid-host/src/results.rs

@@ -1,36 +0,0 @@
-use std::sync::Arc;
-
-use orchid_base::error::{ErrorPosition, OwnedError};
-use orchid_base::intern;
-use orchid_base::interner::Tok;
-use orchid_base::location::Pos;
-use orchid_base::number::{NumError, NumErrorKind};
-
-pub type OwnedResult<T> = Result<T, Vec<OwnedError>>;
-
-pub fn mk_err(
-  description: Tok<String>,
-  message: impl AsRef<str>,
-  posv: impl IntoIterator<Item = ErrorPosition>,
-) -> OwnedError {
-  OwnedError {
-    description,
-    message: Arc::new(message.as_ref().to_string()),
-    positions: posv.into_iter().collect(),
-  }
-}
-
-pub fn num_to_err(NumError { kind, range }: NumError, offset: u32) -> OwnedError {
-  OwnedError {
-    description: intern!(str: "Failed to parse number"),
-    message: Arc::new(
-      match kind {
-        NumErrorKind::NaN => "NaN emerged during parsing",
-        NumErrorKind::InvalidDigit => "non-digit character encountered",
-        NumErrorKind::Overflow => "The number being described is too large or too accurate",
-      }
-      .to_string(),
-    ),
-    positions: vec![Pos::Range(offset + range.start as u32..offset + range.end as u32).into()],
-  }
-}

orchid-host/src/subprocess.rs

@@ -0,0 +1,53 @@
+use std::{io::{self, BufRead as _}, path::PathBuf, process, sync::Mutex, thread};
+
+use orchid_base::{logging::Logger, msg::{recv_msg, send_msg}};
+
+use crate::extension::ExtensionPort;
+
+pub struct Subprocess {
+  child: Mutex<process::Child>,
+  stdin: Mutex<process::ChildStdin>,
+  stdout: Mutex<process::ChildStdout>,
+}
+impl Subprocess {
+  pub fn new(mut cmd: process::Command, logger: Logger) -> io::Result<Self> {
+    let prog_pbuf = PathBuf::from(cmd.get_program());
+    let prog = prog_pbuf.file_stem().unwrap_or(cmd.get_program()).to_string_lossy().to_string();
+    let mut child = cmd
+      .stdin(process::Stdio::piped())
+      .stdout(process::Stdio::piped())
+      .stderr(process::Stdio::piped())
+      .spawn()?;
+    let stdin = child.stdin.take().unwrap();
+    let stdout = child.stdout.take().unwrap();
+    let child_stderr = child.stderr.take().unwrap();
+    thread::Builder::new()
+      .name(format!("stderr-fwd:{prog}"))
+      .spawn(move || {
+        let mut reader = io::BufReader::new(child_stderr);
+        loop {
+          let mut buf = String::new();
+          if 0 == reader.read_line(&mut buf).unwrap() {
+            break;
+          }
+          logger.log(buf);
+        }
+      })?;
+    Ok(Self{ child: Mutex::new(child), stdin: Mutex::new(stdin), stdout: Mutex::new(stdout) })
+  }
+}
+impl Drop for Subprocess {
+  fn drop(&mut self) {
+    self.child.lock().unwrap().wait().expect("Extension exited with error");
+  }
+}
+impl ExtensionPort for Subprocess {
+  fn send(&self, msg: &[u8]) { send_msg(&mut *self.stdin.lock().unwrap(), msg).unwrap() }
+  fn receive(&self) -> Option<Vec<u8>> {
+    match recv_msg(&mut *self.stdout.lock().unwrap()) {
+      Ok(msg) => Some(msg),
+      Err(e) if e.kind() == io::ErrorKind::BrokenPipe => None,
+      Err(e) => panic!("Failed to read from stdout: {}, {e}", e.kind()),
+    }
+  }
+}

orchid-host/src/tree.rs

@@ -1,210 +1,124 @@
-use std::borrow::Borrow;
-use std::cell::RefCell;
-use std::fmt::{Display, Write};
-use std::ops::Range;
 use std::sync::{Mutex, OnceLock};
 
 use itertools::Itertools;
 use never::Never;
-use orchid_api::tree::{Item, ItemKind, Macro, Member, MemberKind, Module, Paren, Token, TokenTree, TreeId, TreeTicket};
+use orchid_base::error::OrcRes;
-use orchid_base::error::OwnedError;
 use orchid_base::interner::{deintern, Tok};
 use orchid_base::location::Pos;
 use orchid_base::name::Sym;
-use orchid_base::tokens::{OwnedPh, PARENS};
+use orchid_base::parse::{Comment, CompName};
+use orchid_base::tree::{ttv_from_api, TokTree, Token};
 use ordered_float::NotNan;
 
+use crate::api;
 use crate::expr::RtExpr;
 use crate::extension::{AtomHand, System};
-use crate::results::OwnedResult;
 
-#[derive(Clone, Debug)]
+pub type ParsTokTree = TokTree<'static, AtomHand, Never>;
-pub struct OwnedTokTree {
+pub type ParsTok = Token<'static, AtomHand, Never>;
-  pub tok: OwnedTok,
-  pub range: Range<u32>,
-}
-impl OwnedTokTree {
-  pub fn from_api<E>(
-    tt: &TokenTree,
-    sys: &System,
-    do_slot: &mut impl FnMut(&TreeTicket, Range<u32>) -> Result<Self, E>,
-  ) -> Result<Self, E> {
-    let tok = match &tt.token {
-      Token::Atom(a) => OwnedTok::Atom(AtomHand::from_api(a.clone().associate(sys.id()))),
-      Token::BR => OwnedTok::BR,
-      Token::NS => OwnedTok::NS,
-      Token::Bottom(e) => OwnedTok::Bottom(e.iter().map(OwnedError::from_api).collect_vec()),
-      Token::Lambda(arg) => OwnedTok::Lambda(Self::v_from_api(arg, sys, do_slot)?),
-      Token::Name(name) => OwnedTok::Name(deintern(*name)),
-      Token::Ph(ph) => OwnedTok::Ph(OwnedPh::from_api(ph.clone())),
-      Token::S(par, b) => OwnedTok::S(par.clone(), Self::v_from_api(b, sys, do_slot)?),
-      Token::Slot(id) => return do_slot(id, tt.range.clone()),
-    };
-    Ok(Self { range: tt.range.clone(), tok })
-  }
-
-  pub fn v_from_api<E>(
-    tokv: impl IntoIterator<Item: Borrow<TokenTree>>,
-    sys: &System,
-    do_slot: &mut impl FnMut(&TreeTicket, Range<u32>) -> Result<Self, E>,
-  ) -> Result<Vec<Self>, E> {
-    tokv.into_iter().map(|t| Self::from_api(t.borrow(), sys, do_slot)).collect()
-  }
-}
-impl Display for OwnedTokTree {
-  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.tok) }
-}
-
-#[derive(Clone, Debug)]
-pub enum OwnedTok {
-  Comment(String),
-  Lambda(Vec<OwnedTokTree>),
-  Name(Tok<String>),
-  NS,
-  BR,
-  S(Paren, Vec<OwnedTokTree>),
-  Atom(AtomHand),
-  Ph(OwnedPh),
-  Bottom(Vec<OwnedError>),
-}
-impl Display for OwnedTok {
-  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-    thread_local! {
-      static PAREN_LEVEL: RefCell<usize> = 0.into();
-    }
-    fn get_indent() -> usize { PAREN_LEVEL.with_borrow(|t| *t) }
-    fn with_indent<T>(f: impl FnOnce() -> T) -> T {
-      PAREN_LEVEL.with_borrow_mut(|t| *t += 1);
-      let r = f();
-      PAREN_LEVEL.with_borrow_mut(|t| *t -= 1);
-      r
-    }
-    match self {
-      Self::Atom(ah) => f.write_str(&indent(&ah.print(), get_indent(), false)),
-      Self::BR => write!(f, "\n{}", "  ".repeat(get_indent())),
-      Self::Bottom(err) => write!(f, "Botttom({})",
-        err.iter().map(|e| format!("{}: {}", e.description, e.message)).join(", ")
-      ),
-      Self::Comment(c) => write!(f, "--[{c}]--"),
-      Self::Lambda(arg) => with_indent(|| write!(f, "\\ {} .", fmt_tt_v(arg))),
-      Self::NS => f.write_str("::"),
-      Self::Name(n) => f.write_str(n),
-      Self::Ph(ph) => write!(f, "{ph}"),
-      Self::S(p, b) => {
-        let (lp, rp, _) = PARENS.iter().find(|(_, _, par)| par == p).unwrap();
-        f.write_char(*lp)?;
-        with_indent(|| f.write_str(&fmt_tt_v(b)))?;
-        f.write_char(*rp)
-      }
-    }
-  }
-}
-
-pub fn fmt_tt_v<'a>(ttv: impl IntoIterator<Item = &'a OwnedTokTree>) -> String {
-  ttv.into_iter().join(" ")
-}
-
-pub fn indent(s: &str, lvl: usize, first: bool) -> String {
-  if first {
-    s.replace("\n", &("\n".to_string() + &"  ".repeat(lvl)))
-  } else if let Some((fst, rest)) = s.split_once('\n') {
-    fst.to_string() + "\n" + &indent(rest, lvl, true)
-  } else {
-    s.to_string()
-  }
-}
 
 #[derive(Debug)]
-pub struct OwnedItem {
+pub struct Item {
   pub pos: Pos,
-  pub kind: OwnedItemKind,
+  pub comments: Vec<Comment>,
+  pub kind: ItemKind,
 }
 
 #[derive(Debug)]
-pub enum OwnedItemKind {
+pub enum ItemKind {
-  Raw(Vec<OwnedTokTree>),
+  Raw(Vec<ParsTokTree>),
-  Member(OwnedMember),
+  Member(Member),
-  Rule(OwnedMacro),
+  Export(Tok<String>),
+  Rule(Macro),
+  Import(CompName),
 }
 
-fn slot_panic(_: &TreeTicket, _: Range<u32>) -> Result<OwnedTokTree, Never> {
+impl Item {
-  panic!("No slots allowed at item stage")
+  pub fn from_api(tree: api::Item, sys: &System) -> Self {
-}
-
-impl OwnedItem {
-  pub fn from_api(tree: Item, sys: &System) -> Self {
     let kind = match tree.kind {
-      ItemKind::Raw(tokv) =>
+      api::ItemKind::Raw(tokv) => ItemKind::Raw(ttv_from_api(tokv, &mut ())),
-        OwnedItemKind::Raw(OwnedTokTree::v_from_api::<Never>(tokv, sys, &mut slot_panic).unwrap()),
+      api::ItemKind::Member(m) => ItemKind::Member(Member::from_api(m, sys)),
-      ItemKind::Member(m) => OwnedItemKind::Member(OwnedMember::from_api(m, sys)),
+      api::ItemKind::Rule(r) => ItemKind::Rule(Macro::from_api(r)),
-      ItemKind::Rule(r) => OwnedItemKind::Rule(OwnedMacro::from_api(r, sys)),
+      api::ItemKind::Import(i) => ItemKind::Import(CompName::from_api(i)),
+      api::ItemKind::Export(e) => ItemKind::Export(deintern(e)),
     };
-    Self { pos: Pos::from_api(&tree.location), kind }
+    let comments = tree
+      .comments
+      .into_iter()
+      .map(|(text, l)| Comment { text, pos: Pos::from_api(&l) })
+      .collect_vec();
+    Self { pos: Pos::from_api(&tree.location), comments, kind }
   }
 }
 
 #[derive(Debug)]
-pub struct OwnedMember {
+pub struct Member {
-  pub public: bool,
+  pub exported: bool,
   pub name: Tok<String>,
-  pub kind: OnceLock<OMemKind>,
+  pub kind: OnceLock<MemberKind>,
   pub lazy: Mutex<Option<LazyMemberHandle>>,
 }
-impl OwnedMember {
+impl Member {
-  pub fn from_api(Member{ public, name, kind }: Member, sys: &System) -> Self {
+  pub fn from_api(api::Member { exported: public, name, kind }: api::Member, sys: &System) -> Self {
     let (kind, lazy) = match kind {
-      MemberKind::Const(c) => (OnceLock::from(OMemKind::Const(RtExpr::from_api(c, sys))), None),
+      api::MemberKind::Const(c) =>
-      MemberKind::Module(m) => (OnceLock::from(OMemKind::Mod(OwnedModule::from_api(m, sys))), None),
+        (OnceLock::from(MemberKind::PreCnst(RtExpr::from_api(c, sys))), None),
-      MemberKind::Lazy(id) => (OnceLock::new(), Some(LazyMemberHandle(id, sys.clone())))
+      api::MemberKind::Module(m) =>
+        (OnceLock::from(MemberKind::Mod(Module::from_api(m, sys))), None),
+      api::MemberKind::Lazy(id) => (OnceLock::new(), Some(LazyMemberHandle(id, sys.clone()))),
     };
-    OwnedMember { public, name: deintern(name), kind, lazy: Mutex::new(lazy) }
+    Member { exported: public, name: deintern(name), kind, lazy: Mutex::new(lazy) }
+  }
+  pub fn new(public: bool, name: Tok<String>, kind: MemberKind) -> Self {
+    Member { exported: public, name, kind: OnceLock::from(kind), lazy: Mutex::default() }
   }
 }
 
 #[derive(Debug)]
-pub enum OMemKind {
+pub enum MemberKind {
-  Const(RtExpr),
+  Const(Vec<ParsTokTree>),
-  Mod(OwnedModule),
+  PreCnst(RtExpr),
+  Mod(Module),
 }
 
 #[derive(Debug)]
-pub struct OwnedModule {
+pub struct Module {
   pub imports: Vec<Sym>,
-  pub items: Vec<OwnedItem>,
+  pub items: Vec<Item>,
 }
-impl OwnedModule {
+impl Module {
-  pub fn from_api(m: Module, sys: &System) -> Self {
+  pub fn from_api(m: api::Module, sys: &System) -> Self {
     Self {
       imports: m.imports.into_iter().map(|m| Sym::from_tok(deintern(m)).unwrap()).collect_vec(),
-      items: m.items.into_iter().map(|i| OwnedItem::from_api(i, sys)).collect_vec(),
+      items: m.items.into_iter().map(|i| Item::from_api(i, sys)).collect_vec(),
     }
   }
 }
 
 #[derive(Debug)]
-pub struct OwnedMacro {
+pub struct Macro {
   pub priority: NotNan<f64>,
-  pub pattern: Vec<OwnedTokTree>,
+  pub pattern: Vec<ParsTokTree>,
-  pub template: Vec<OwnedTokTree>,
+  pub template: Vec<ParsTokTree>,
 }
-impl OwnedMacro {
+impl Macro {
-  pub fn from_api(m: Macro, sys: &System) -> Self {
+  pub fn from_api(m: api::Macro) -> Self {
     Self {
       priority: m.priority,
-      pattern: OwnedTokTree::v_from_api(m.pattern, sys, &mut slot_panic).unwrap(),
+      pattern: ttv_from_api(m.pattern, &mut ()),
-      template: OwnedTokTree::v_from_api(m.template, sys, &mut slot_panic).unwrap(),
+      template: ttv_from_api(m.template, &mut ()),
     }
   }
 }
 
 #[derive(Debug)]
-pub struct LazyMemberHandle(TreeId, System);
+pub struct LazyMemberHandle(api::TreeId, System);
 impl LazyMemberHandle {
-  pub fn run(self) -> OwnedResult<OMemKind> {
+  pub fn run(self) -> OrcRes<MemberKind> {
     match self.1.get_tree(self.0) {
-      MemberKind::Const(c) => Ok(OMemKind::Const(RtExpr::from_api(c, &self.1))),
+      api::MemberKind::Const(c) => Ok(MemberKind::PreCnst(RtExpr::from_api(c, &self.1))),
-      MemberKind::Module(m) => Ok(OMemKind::Mod(OwnedModule::from_api(m, &self.1))),
+      api::MemberKind::Module(m) => Ok(MemberKind::Mod(Module::from_api(m, &self.1))),
-      MemberKind::Lazy(id) => Self(id, self.1).run()
+      api::MemberKind::Lazy(id) => Self(id, self.1).run(),
     }
   }
 }

orchid-std/src/number/num_atom.rs

@@ -1,11 +1,10 @@
 use never::Never;
 use orchid_api_derive::Coding;
-use orchid_extension::atom::{Atomic, ReqPck, TypAtom};
+use orchid_base::error::OrcRes;
+use orchid_extension::atom::{AtomFactory, Atomic, AtomicFeatures, ReqPck, ToAtom, TypAtom};
 use orchid_extension::atom_thin::{ThinAtom, ThinVariant};
-use orchid_extension::conv::{ToExpr, TryFromExpr};
+use orchid_extension::conv::TryFromExpr;
-use orchid_extension::error::{pack_err, ProjectResult};
+use orchid_extension::expr::ExprHandle;
-use orchid_extension::expr::{ExprHandle, GenExpr};
-use orchid_extension::system::SysCtx;
 use ordered_float::NotNan;
 
 #[derive(Clone, Debug, Coding)]
@@ -16,10 +15,10 @@ impl Atomic for Int {
   type Req = Never;
 }
 impl ThinAtom for Int {
-  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+  fn handle_req(&self, pck: impl ReqPck<Self>) { pck.never() }
 }
 impl TryFromExpr for Int {
-  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
     TypAtom::<Int>::try_from_expr(expr).map(|t| t.value)
   }
 }
@@ -32,10 +31,10 @@ impl Atomic for Float {
   type Req = Never;
 }
 impl ThinAtom for Float {
-  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+  fn handle_req(&self, pck: impl ReqPck<Self>) { pck.never() }
 }
 impl TryFromExpr for Float {
-  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
     TypAtom::<Float>::try_from_expr(expr).map(|t| t.value)
   }
 }
@@ -45,17 +44,17 @@ pub enum Numeric {
   Float(NotNan<f64>),
 }
 impl TryFromExpr for Numeric {
-  fn try_from_expr(expr: ExprHandle) -> ProjectResult<Self> {
+  fn try_from_expr(expr: ExprHandle) -> OrcRes<Self> {
     Int::try_from_expr(expr.clone()).map(|t| Numeric::Int(t.0)).or_else(|e| {
-      Float::try_from_expr(expr).map(|t| Numeric::Float(t.0)).map_err(|e2| pack_err([e, e2]))
+      Float::try_from_expr(expr).map(|t| Numeric::Float(t.0)).map_err(|e2| [e, e2].concat())
     })
   }
 }
-impl ToExpr for Numeric {
+impl ToAtom for Numeric {
-  fn to_expr(self) -> GenExpr {
+  fn to_atom_factory(self) -> AtomFactory {
     match self {
-      Self::Float(f) => Float(f).to_expr(),
+      Self::Float(f) => Float(f).factory(),
-      Self::Int(i) => Int(i).to_expr(),
+      Self::Int(i) => Int(i).factory(),
     }
   }
 }

orchid-std/src/number/num_lexer.rs

@@ -1,44 +1,27 @@
 use std::ops::RangeInclusive;
 
-use orchid_base::location::Pos;
+use orchid_base::error::OrcRes;
-use orchid_base::number::{parse_num, NumError, NumErrorKind, Numeric};
+use orchid_base::number::{num_to_err, parse_num, Numeric};
 use orchid_extension::atom::AtomicFeatures;
-use orchid_extension::error::{ProjectError, ProjectResult};
 use orchid_extension::lexer::{LexContext, Lexer};
 use orchid_extension::tree::{GenTok, GenTokTree};
 use ordered_float::NotNan;
 
 use super::num_atom::{Float, Int};
 
-struct NumProjError(u32, NumError);
-impl ProjectError for NumProjError {
-  const DESCRIPTION: &'static str = "Failed to parse number";
-  fn message(&self) -> String {
-    match self.1.kind {
-      NumErrorKind::InvalidDigit => "This character is not meaningful in this base",
-      NumErrorKind::NaN => "Number somehow evaluated to NaN",
-      NumErrorKind::Overflow => "Number literal overflowed its enclosing type",
-    }
-    .to_string()
-  }
-  fn one_position(&self) -> Pos {
-    Pos::Range(self.0 + self.1.range.start as u32..self.0 + self.1.range.end as u32)
-  }
-}
-
 #[derive(Default)]
 pub struct NumLexer;
 impl Lexer for NumLexer {
   const CHAR_FILTER: &'static [RangeInclusive<char>] = &['0'..='9'];
-  fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> ProjectResult<(&'a str, GenTokTree)> {
+  fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> {
     let ends_at = all.find(|c: char| !c.is_ascii_hexdigit() && !"xX._pP".contains(c));
     let (chars, tail) = all.split_at(ends_at.unwrap_or(all.len()));
     let fac = match parse_num(chars) {
       Ok(Numeric::Float(f)) => Float(f).factory(),
       Ok(Numeric::Uint(uint)) => Int(uint.try_into().unwrap()).factory(),
       Ok(Numeric::Decimal(dec)) => Float(NotNan::new(dec.try_into().unwrap()).unwrap()).factory(),
-      Err(e) => return Err(NumProjError(ctx.pos(all), e).pack()),
+      Err(e) => return Err(vec![num_to_err(e, ctx.pos(all))]),
     };
-    Ok((tail, GenTok::Atom(fac).at(ctx.pos(all)..ctx.pos(tail))))
+    Ok((tail, GenTok::X(fac).at(ctx.pos(all)..ctx.pos(tail))))
   }
 }

orchid-std/src/std.rs

@@ -3,10 +3,9 @@ use std::sync::Arc;
 use orchid_base::interner::Tok;
 use orchid_extension::atom::{AtomDynfo, AtomicFeatures};
 use orchid_extension::fs::DeclFs;
-use orchid_extension::fun::Fun;
 use orchid_extension::system::{System, SystemCard};
 use orchid_extension::system_ctor::SystemCtor;
-use orchid_extension::tree::{cnst, module, root_mod, GenMemberKind};
+use orchid_extension::tree::{comments, fun, module, root_mod, GenMemberKind};
 
 use crate::number::num_atom::{Float, Int};
 use crate::string::str_atom::{IntStrAtom, StrAtom};
@@ -29,18 +28,14 @@ impl SystemCard for StdSystem {
 }
 impl System for StdSystem {
   fn lexers() -> Vec<orchid_extension::lexer::LexerObj> { vec![&StringLexer] }
+  fn parsers() -> Vec<orchid_extension::parser::ParserObj> { vec![] }
   fn vfs() -> DeclFs { DeclFs::Mod(&[]) }
   fn env() -> Vec<(Tok<String>, GenMemberKind)> {
-    vec![
+    vec![root_mod("std", [], [module(true, "string", [], [comments(
-      root_mod("std", [], [
+      ["Concatenate two strings"],
-        module(true, "string", [], [
+      fun(true, "concat", |left: OrcString, right: OrcString| {
-          cnst(true, "concat", Fun::new(|left: OrcString| {
-            Fun::new(move |right: OrcString| {
         StrAtom::new(Arc::new(left.get_string().to_string() + &right.get_string()))
-            })
+      }),
-          }))
+    )])])]
-        ]),
-      ])
-    ]
   }
 }

orchid-std/src/string/str_atom.rs

@@ -1,18 +1,18 @@
 use std::borrow::Cow;
+use std::io;
 use std::num::NonZeroU64;
 use std::sync::Arc;
 
 use never::Never;
-use orchid_api::interner::TStr;
 use orchid_api_derive::Coding;
 use orchid_api_traits::{Encode, Request};
+use orchid_base::error::{mk_err, OrcRes};
 use orchid_base::id_store::IdStore;
-use orchid_base::interner::{deintern, Tok};
+use orchid_base::intern;
-use orchid_base::location::Pos;
+use orchid_base::interner::{deintern, intern, Tok};
 use orchid_extension::atom::{Atomic, ReqPck, TypAtom};
-use orchid_extension::atom_owned::{OwnedAtom, OwnedVariant};
+use orchid_extension::atom_owned::{DeserializeCtx, OwnedAtom, OwnedVariant};
 use orchid_extension::conv::TryFromExpr;
-use orchid_extension::error::{ProjectError, ProjectResult};
 use orchid_extension::expr::ExprHandle;
 use orchid_extension::system::SysCtx;
 
@@ -34,17 +34,22 @@ impl StrAtom {
   pub fn new(str: Arc<String>) -> Self { Self(STR_REPO.add(str).id()) }
 }
 impl Clone for StrAtom {
-  fn clone(&self) -> Self { Self(STR_REPO.add(STR_REPO.get(self.0).unwrap().clone()).id()) }
+  fn clone(&self) -> Self { Self::new(self.local_value()) }
 }
 impl StrAtom {
   fn try_local_value(&self) -> Option<Arc<String>> { STR_REPO.get(self.0).map(|r| r.clone()) }
   fn local_value(&self) -> Arc<String> { self.try_local_value().expect("no string found for ID") }
 }
 impl OwnedAtom for StrAtom {
+  type Refs = ();
   fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0) }
-  fn same(&self, _ctx: SysCtx, other: &Self) -> bool { self.local_value() == other.local_value() }
+  fn same(&self, _: SysCtx, other: &Self) -> bool { self.local_value() == other.local_value() }
-  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) {
+  fn handle_req(&self, pck: impl ReqPck<Self>) { self.local_value().encode(pck.unpack().1) }
-    self.local_value().encode(pck.unpack().1)
+  fn serialize(&self, _: SysCtx, sink: &mut (impl io::Write + ?Sized)) -> Self::Refs {
+    self.local_value().encode(sink)
+  }
+  fn deserialize(mut ctx: impl DeserializeCtx, _: Self::Refs) -> Self {
+    Self::new(Arc::new(ctx.read::<String>()))
   }
 }
 
@@ -52,27 +57,30 @@ impl OwnedAtom for StrAtom {
 pub struct IntStrAtom(Tok<String>);
 impl Atomic for IntStrAtom {
   type Variant = OwnedVariant;
-  type Data = TStr;
+  type Data = orchid_api::TStr;
   type Req = Never;
 }
 impl From<Tok<String>> for IntStrAtom {
   fn from(value: Tok<String>) -> Self { Self(value) }
 }
 impl OwnedAtom for IntStrAtom {
+  type Refs = ();
   fn val(&self) -> Cow<'_, Self::Data> { Cow::Owned(self.0.marker()) }
-  fn handle_req(&self, _ctx: SysCtx, pck: impl ReqPck<Self>) { pck.never() }
+  fn handle_req(&self, pck: impl ReqPck<Self>) { pck.never() }
   fn print(&self, _ctx: SysCtx) -> String { format!("{:?}i", self.0.as_str()) }
+  fn serialize(&self, _: SysCtx, write: &mut (impl io::Write + ?Sized)) { self.0.encode(write) }
+  fn deserialize(ctx: impl DeserializeCtx, _: ()) -> Self { Self(intern(&ctx.decode::<String>())) }
 }
 
 #[derive(Clone)]
-pub enum OrcString {
+pub enum OrcString<'a> {
-  Val(TypAtom<StrAtom>),
+  Val(TypAtom<'a, StrAtom>),
-  Int(Tok<String>),
+  Int(TypAtom<'a, IntStrAtom>),
 }
-impl OrcString {
+impl<'a> OrcString<'a> {
   pub fn get_string(&self) -> Arc<String> {
     match &self {
-      Self::Int(tok) => tok.arc(),
+      Self::Int(tok) => deintern(tok.value).arc(),
       Self::Val(atom) => match STR_REPO.get(**atom) {
         Some(rec) => rec.clone(),
         None => Arc::new(atom.request(StringGetVal)),
@@ -80,23 +88,15 @@ impl OrcString {
     }
   }
 }
-impl From<Tok<String>> for OrcString {
-  fn from(value: Tok<String>) -> Self { OrcString::Int(value) }
-}
 
-pub struct NotString(Pos);
+impl TryFromExpr for OrcString<'static> {
-impl ProjectError for NotString {
+  fn try_from_expr(expr: ExprHandle) -> OrcRes<OrcString<'static>> {
-  const DESCRIPTION: &'static str = "A string was expected";
-  fn one_position(&self) -> Pos { self.0.clone() }
-}
-impl TryFromExpr for OrcString {
-  fn try_from_expr(expr: ExprHandle) -> ProjectResult<OrcString> {
     if let Ok(v) = TypAtom::<StrAtom>::downcast(expr.clone()) {
       return Ok(OrcString::Val(v));
     }
     match TypAtom::<IntStrAtom>::downcast(expr) {
-      Ok(t) => Ok(OrcString::Int(deintern(*t))),
+      Ok(t) => Ok(OrcString::Int(t)),
-      Err(e) => Err(NotString(e.0).pack()),
+      Err(e) => Err(vec![mk_err(intern!(str: "A string was expected"), "", [e.0.clone().into()])]),
     }
   }
 }

orchid-std/src/string/str_lexer.rs

@@ -1,12 +1,12 @@
-use orchid_base::intern;
 use itertools::Itertools;
+use orchid_base::error::{mk_err, OrcErr, OrcRes};
 use orchid_base::interner::intern;
 use orchid_base::location::Pos;
-use orchid_base::vname;
+use orchid_base::tree::{vname_tv, wrap_tokv};
+use orchid_base::{intern, vname};
 use orchid_extension::atom::AtomicFeatures;
-use orchid_extension::error::{ErrorSansOrigin, ProjectError, ProjectErrorObj, ProjectResult};
+use orchid_extension::lexer::{err_lexer_na, LexContext, Lexer};
-use orchid_extension::lexer::{LexContext, Lexer, NotApplicableLexerError};
+use orchid_extension::tree::{GenTok, GenTokTree};
-use orchid_extension::tree::{wrap_tokv, GenTok, GenTokTree};
 
 use super::str_atom::IntStrAtom;
 
@@ -30,34 +30,19 @@ struct StringError {
   kind: StringErrorKind,
 }
 
-#[derive(Clone)]
-struct NotHex;
-impl ErrorSansOrigin for NotHex {
-  const DESCRIPTION: &'static str = "Expected a hex digit";
-}
-
-#[derive(Clone)]
-struct BadCodePoint;
-impl ErrorSansOrigin for BadCodePoint {
-  const DESCRIPTION: &'static str = "The specified number is not a Unicode code point";
-}
-
-#[derive(Clone)]
-struct BadEscapeSequence;
-impl ErrorSansOrigin for BadEscapeSequence {
-  const DESCRIPTION: &'static str = "Unrecognized escape sequence";
-}
-
 impl StringError {
   /// Convert into project error for reporting
-  pub fn into_proj(self, pos: u32) -> ProjectErrorObj {
+  pub fn into_proj(self, pos: u32) -> OrcErr {
     let start = pos + self.pos;
-    let pos = Pos::Range(start..start + 1);
+    mk_err(
+      intern!(str: "Failed to parse string"),
       match self.kind {
-      StringErrorKind::NotHex => NotHex.bundle(&pos),
+        StringErrorKind::NotHex => "Expected a hex digit",
-      StringErrorKind::BadCodePoint => BadCodePoint.bundle(&pos),
+        StringErrorKind::BadCodePoint => "The specified number is not a Unicode code point",
-      StringErrorKind::BadEscSeq => BadEscapeSequence.bundle(&pos),
+        StringErrorKind::BadEscSeq => "Unrecognized escape sequence",
-    }
+      },
+      [Pos::Range(start..start + 1).into()],
+    )
   }
 }
 
@@ -106,29 +91,21 @@ fn parse_string(str: &str) -> Result<String, StringError> {
   Ok(target)
 }
 
-#[derive(Clone)]
-pub struct NoStringEnd;
-impl ErrorSansOrigin for NoStringEnd {
-  const DESCRIPTION: &'static str = "String never terminated with \"";
-}
-
 #[derive(Default)]
 pub struct StringLexer;
 impl Lexer for StringLexer {
   const CHAR_FILTER: &'static [std::ops::RangeInclusive<char>] = &['"'..='"'];
-  fn lex<'a>(
+  fn lex<'a>(all: &'a str, ctx: &'a LexContext<'a>) -> OrcRes<(&'a str, GenTokTree<'a>)> {
-    all: &'a str,
+    let mut tail = all.strip_prefix('"').ok_or_else(err_lexer_na)?;
-    ctx: &'a LexContext<'a>,
+    let mut parts = Vec::<GenTokTree<'a>>::new();
-  ) -> ProjectResult<(&'a str, GenTokTree)> {
-    let mut tail = all.strip_prefix('"').ok_or_else(|| NotApplicableLexerError.pack())?;
-    let mut parts = vec![];
     let mut cur = String::new();
     let mut errors = vec![];
-    let commit_str = |str: &mut String, tail: &str, err: &mut Vec<ProjectErrorObj>, parts: &mut Vec<GenTokTree>| {
+    let commit_str =
+      |str: &mut String, tail: &str, err: &mut Vec<OrcErr>, parts: &mut Vec<GenTokTree<'a>>| {
         let str_val = parse_string(str)
           .inspect_err(|e| err.push(e.clone().into_proj(ctx.pos(tail) - str.len() as u32)))
           .unwrap_or_default();
-      let tok = GenTok::Atom(IntStrAtom::from(intern(&*str_val)).factory());
+        let tok = GenTok::X(IntStrAtom::from(intern(&*str_val)).factory());
         parts.push(tok.at(ctx.tok_ran(str.len() as u32, tail)));
         *str = String::new();
       };
@@ -139,8 +116,7 @@ impl Lexer for StringLexer {
       } else if let Some(rest) = tail.strip_prefix('$') {
         commit_str(&mut cur, tail, &mut errors, &mut parts);
         parts.push(GenTok::Name(intern!(str: "++")).at(ctx.tok_ran(1, rest)));
-        parts.extend(GenTok::vname(&vname!(std::string::convert))
+        parts.extend(vname_tv(&vname!(std::string::convert), ctx.tok_ran(1, rest)));
-          .map(|t| t.at(ctx.tok_ran(1, rest))));
         let (new_tail, tree) = ctx.recurse(rest)?;
         tail = new_tail;
         parts.push(tree);
@@ -156,7 +132,11 @@ impl Lexer for StringLexer {
         } else {
           let range = ctx.pos(all)..ctx.pos("");
           commit_str(&mut cur, tail, &mut errors, &mut parts);
-          return Err(NoStringEnd.bundle(&Pos::Range(range.clone())));
+          return Err(vec![mk_err(
+            intern!(str: "No string end"),
+            "String never terminated with \"",
+            [Pos::Range(range.clone()).into()],
+          )]);
         }
       }
     }

orchidlang/src/libs/std/string.rs

@@ -1,4 +1,4 @@
-//! `std::string` String processing
+(//! `std::string` String processing
 
 use std::fmt;
 use std::fmt::Write as _;

orcx/src/main.rs

@@ -1,10 +1,17 @@
-use std::{fs::File, io::Read, process::Command};
+use std::fs::File;
+use std::io::Read;
+use std::process::Command;
+use std::sync::Arc;
 
 use camino::Utf8PathBuf;
 use clap::{Parser, Subcommand};
 use itertools::Itertools;
-use orchid_base::{interner::intern, logging::{LogStrategy, Logger}};
+use orchid_base::interner::intern;
-use orchid_host::{extension::{init_systems, Extension}, lex::lex, tree::fmt_tt_v};
+use orchid_base::logging::{LogStrategy, Logger};
+use orchid_base::tree::ttv_fmt;
+use orchid_host::extension::{init_systems, Extension};
+use orchid_host::lex::lex;
+use orchid_host::subprocess::Subprocess;
 
 #[derive(Parser, Debug)]
 #[command(version, about, long_about)]
@@ -21,23 +28,25 @@ pub struct Args {
 pub enum Commands {
   Lex {
     #[arg(short, long)]
-    file: Utf8PathBuf
+    file: Utf8PathBuf,
   },
 }
 
 fn main() {
   let args = Args::parse();
+  let logger = Logger::new(LogStrategy::StdErr);
   match args.command {
     Commands::Lex { file } => {
       let extensions = (args.extension.iter())
-        .map(|f| Extension::new(Command::new(f.as_os_str()), Logger::new(LogStrategy::StdErr)).unwrap())
+        .map(|f| Subprocess::new(Command::new(f.as_os_str()), logger.clone()).unwrap())
+        .map(|cmd| Extension::new_process(Arc::new(cmd), logger.clone()).unwrap())
         .collect_vec();
       let systems = init_systems(&args.system, &extensions).unwrap();
       let mut file = File::open(file.as_std_path()).unwrap();
       let mut buf = String::new();
       file.read_to_string(&mut buf).unwrap();
       let lexemes = lex(intern(&buf), &systems).unwrap();
-      println!("{}", fmt_tt_v(&lexemes))
+      println!("{}", ttv_fmt(&lexemes))
-    }
+    },
   }
 }

xtask/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "xtask"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+clap = { version = "=4.5.4", features = ["derive"] }

xtask/src/main.rs

@@ -0,0 +1,59 @@
+use std::env;
+use std::ffi::OsStr;
+use std::fs::File;
+use std::io::{self, Read};
+use std::path::PathBuf;
+use std::process::ExitCode;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+use clap::{Parser, Subcommand};
+
+#[derive(Parser)]
+pub struct Args {
+  #[arg(short, long)]
+  verbose: bool,
+  #[command(subcommand)]
+  command: Commands,
+}
+
+#[derive(Subcommand)]
+pub enum Commands {
+  CheckApiRefs,
+}
+
+pub static EXIT_OK: AtomicBool = AtomicBool::new(true);
+
+fn main() -> io::Result<ExitCode> {
+  let args = Args::parse();
+  match args.command {
+    Commands::CheckApiRefs => check_api_refs(&args, env::current_dir()?)?,
+  }
+  Ok(if EXIT_OK.load(Ordering::Relaxed) { ExitCode::SUCCESS } else { ExitCode::FAILURE })
+}
+
+fn check_api_refs(args: &Args, dir: PathBuf) -> io::Result<()> {
+  for file in dir.read_dir()?.collect::<Result<Vec<_>, _>>()? {
+    if args.verbose {
+      eprintln!("Checking {}", file.path().to_string_lossy());
+    }
+    if file.metadata()?.is_dir() {
+      check_api_refs(args, file.path())?
+    }
+    if file.path().extension() != Some(OsStr::new("rs")) || file.file_name() == "lib.rs" {
+      continue;
+    }
+    let mut contents = String::new();
+    File::open(file.path())?.read_to_string(&mut contents)?;
+    for (l, line) in contents.lines().enumerate() {
+      if line.trim().starts_with("use") {
+        if let Some(c) = line.find("orchid_api") {
+          if Some(c) != line.find("orchid_api_") {
+            let dname = file.path().to_string_lossy().to_string();
+            eprintln!("orchid_api imported in {dname} at {};{}", l + 1, c + 1)
+          }
+        }
+      }
+    }
+  }
+  Ok(())
+}